1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2011, 2012 STRATO. All rights reserved. 4 */ 5 6 #include <linux/blkdev.h> 7 #include <linux/ratelimit.h> 8 #include <linux/sched/mm.h> 9 #include <crypto/hash.h> 10 #include "ctree.h" 11 #include "discard.h" 12 #include "volumes.h" 13 #include "disk-io.h" 14 #include "ordered-data.h" 15 #include "transaction.h" 16 #include "backref.h" 17 #include "extent_io.h" 18 #include "dev-replace.h" 19 #include "check-integrity.h" 20 #include "rcu-string.h" 21 #include "raid56.h" 22 #include "block-group.h" 23 #include "zoned.h" 24 25 /* 26 * This is only the first step towards a full-features scrub. It reads all 27 * extent and super block and verifies the checksums. In case a bad checksum 28 * is found or the extent cannot be read, good data will be written back if 29 * any can be found. 30 * 31 * Future enhancements: 32 * - In case an unrepairable extent is encountered, track which files are 33 * affected and report them 34 * - track and record media errors, throw out bad devices 35 * - add a mode to also read unallocated space 36 */ 37 38 struct scrub_block; 39 struct scrub_ctx; 40 41 /* 42 * the following three values only influence the performance. 43 * The last one configures the number of parallel and outstanding I/O 44 * operations. The first two values configure an upper limit for the number 45 * of (dynamically allocated) pages that are added to a bio. 46 */ 47 #define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */ 48 #define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */ 49 #define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */ 50 51 /* 52 * the following value times PAGE_SIZE needs to be large enough to match the 53 * largest node/leaf/sector size that shall be supported. 54 * Values larger than BTRFS_STRIPE_LEN are not supported. 55 */ 56 #define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */ 57 58 struct scrub_recover { 59 refcount_t refs; 60 struct btrfs_bio *bbio; 61 u64 map_length; 62 }; 63 64 struct scrub_page { 65 struct scrub_block *sblock; 66 struct page *page; 67 struct btrfs_device *dev; 68 struct list_head list; 69 u64 flags; /* extent flags */ 70 u64 generation; 71 u64 logical; 72 u64 physical; 73 u64 physical_for_dev_replace; 74 atomic_t refs; 75 u8 mirror_num; 76 int have_csum:1; 77 int io_error:1; 78 u8 csum[BTRFS_CSUM_SIZE]; 79 80 struct scrub_recover *recover; 81 }; 82 83 struct scrub_bio { 84 int index; 85 struct scrub_ctx *sctx; 86 struct btrfs_device *dev; 87 struct bio *bio; 88 blk_status_t status; 89 u64 logical; 90 u64 physical; 91 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO 92 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO]; 93 #else 94 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO]; 95 #endif 96 int page_count; 97 int next_free; 98 struct btrfs_work work; 99 }; 100 101 struct scrub_block { 102 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK]; 103 int page_count; 104 atomic_t outstanding_pages; 105 refcount_t refs; /* free mem on transition to zero */ 106 struct scrub_ctx *sctx; 107 struct scrub_parity *sparity; 108 struct { 109 unsigned int header_error:1; 110 unsigned int checksum_error:1; 111 unsigned int no_io_error_seen:1; 112 unsigned int generation_error:1; /* also sets header_error */ 113 114 /* The following is for the data used to check parity */ 115 /* It is for the data with checksum */ 116 unsigned int data_corrected:1; 117 }; 118 struct btrfs_work work; 119 }; 120 121 /* Used for the chunks with parity stripe such RAID5/6 */ 122 struct scrub_parity { 123 struct scrub_ctx *sctx; 124 125 struct btrfs_device *scrub_dev; 126 127 u64 logic_start; 128 129 u64 logic_end; 130 131 int nsectors; 132 133 u32 stripe_len; 134 135 refcount_t refs; 136 137 struct list_head spages; 138 139 /* Work of parity check and repair */ 140 struct btrfs_work work; 141 142 /* Mark the parity blocks which have data */ 143 unsigned long *dbitmap; 144 145 /* 146 * Mark the parity blocks which have data, but errors happen when 147 * read data or check data 148 */ 149 unsigned long *ebitmap; 150 151 unsigned long bitmap[]; 152 }; 153 154 struct scrub_ctx { 155 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX]; 156 struct btrfs_fs_info *fs_info; 157 int first_free; 158 int curr; 159 atomic_t bios_in_flight; 160 atomic_t workers_pending; 161 spinlock_t list_lock; 162 wait_queue_head_t list_wait; 163 struct list_head csum_list; 164 atomic_t cancel_req; 165 int readonly; 166 int pages_per_rd_bio; 167 168 int is_dev_replace; 169 u64 write_pointer; 170 171 struct scrub_bio *wr_curr_bio; 172 struct mutex wr_lock; 173 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */ 174 struct btrfs_device *wr_tgtdev; 175 bool flush_all_writes; 176 177 /* 178 * statistics 179 */ 180 struct btrfs_scrub_progress stat; 181 spinlock_t stat_lock; 182 183 /* 184 * Use a ref counter to avoid use-after-free issues. Scrub workers 185 * decrement bios_in_flight and workers_pending and then do a wakeup 186 * on the list_wait wait queue. We must ensure the main scrub task 187 * doesn't free the scrub context before or while the workers are 188 * doing the wakeup() call. 189 */ 190 refcount_t refs; 191 }; 192 193 struct scrub_warning { 194 struct btrfs_path *path; 195 u64 extent_item_size; 196 const char *errstr; 197 u64 physical; 198 u64 logical; 199 struct btrfs_device *dev; 200 }; 201 202 struct full_stripe_lock { 203 struct rb_node node; 204 u64 logical; 205 u64 refs; 206 struct mutex mutex; 207 }; 208 209 static void scrub_pending_bio_inc(struct scrub_ctx *sctx); 210 static void scrub_pending_bio_dec(struct scrub_ctx *sctx); 211 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check); 212 static int scrub_setup_recheck_block(struct scrub_block *original_sblock, 213 struct scrub_block *sblocks_for_recheck); 214 static void scrub_recheck_block(struct btrfs_fs_info *fs_info, 215 struct scrub_block *sblock, 216 int retry_failed_mirror); 217 static void scrub_recheck_block_checksum(struct scrub_block *sblock); 218 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad, 219 struct scrub_block *sblock_good); 220 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad, 221 struct scrub_block *sblock_good, 222 int page_num, int force_write); 223 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock); 224 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock, 225 int page_num); 226 static int scrub_checksum_data(struct scrub_block *sblock); 227 static int scrub_checksum_tree_block(struct scrub_block *sblock); 228 static int scrub_checksum_super(struct scrub_block *sblock); 229 static void scrub_block_get(struct scrub_block *sblock); 230 static void scrub_block_put(struct scrub_block *sblock); 231 static void scrub_page_get(struct scrub_page *spage); 232 static void scrub_page_put(struct scrub_page *spage); 233 static void scrub_parity_get(struct scrub_parity *sparity); 234 static void scrub_parity_put(struct scrub_parity *sparity); 235 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, 236 struct scrub_page *spage); 237 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len, 238 u64 physical, struct btrfs_device *dev, u64 flags, 239 u64 gen, int mirror_num, u8 *csum, 240 u64 physical_for_dev_replace); 241 static void scrub_bio_end_io(struct bio *bio); 242 static void scrub_bio_end_io_worker(struct btrfs_work *work); 243 static void scrub_block_complete(struct scrub_block *sblock); 244 static void scrub_remap_extent(struct btrfs_fs_info *fs_info, 245 u64 extent_logical, u32 extent_len, 246 u64 *extent_physical, 247 struct btrfs_device **extent_dev, 248 int *extent_mirror_num); 249 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, 250 struct scrub_page *spage); 251 static void scrub_wr_submit(struct scrub_ctx *sctx); 252 static void scrub_wr_bio_end_io(struct bio *bio); 253 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work); 254 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info); 255 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info); 256 static void scrub_put_ctx(struct scrub_ctx *sctx); 257 258 static inline int scrub_is_page_on_raid56(struct scrub_page *spage) 259 { 260 return spage->recover && 261 (spage->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK); 262 } 263 264 static void scrub_pending_bio_inc(struct scrub_ctx *sctx) 265 { 266 refcount_inc(&sctx->refs); 267 atomic_inc(&sctx->bios_in_flight); 268 } 269 270 static void scrub_pending_bio_dec(struct scrub_ctx *sctx) 271 { 272 atomic_dec(&sctx->bios_in_flight); 273 wake_up(&sctx->list_wait); 274 scrub_put_ctx(sctx); 275 } 276 277 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) 278 { 279 while (atomic_read(&fs_info->scrub_pause_req)) { 280 mutex_unlock(&fs_info->scrub_lock); 281 wait_event(fs_info->scrub_pause_wait, 282 atomic_read(&fs_info->scrub_pause_req) == 0); 283 mutex_lock(&fs_info->scrub_lock); 284 } 285 } 286 287 static void scrub_pause_on(struct btrfs_fs_info *fs_info) 288 { 289 atomic_inc(&fs_info->scrubs_paused); 290 wake_up(&fs_info->scrub_pause_wait); 291 } 292 293 static void scrub_pause_off(struct btrfs_fs_info *fs_info) 294 { 295 mutex_lock(&fs_info->scrub_lock); 296 __scrub_blocked_if_needed(fs_info); 297 atomic_dec(&fs_info->scrubs_paused); 298 mutex_unlock(&fs_info->scrub_lock); 299 300 wake_up(&fs_info->scrub_pause_wait); 301 } 302 303 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) 304 { 305 scrub_pause_on(fs_info); 306 scrub_pause_off(fs_info); 307 } 308 309 /* 310 * Insert new full stripe lock into full stripe locks tree 311 * 312 * Return pointer to existing or newly inserted full_stripe_lock structure if 313 * everything works well. 314 * Return ERR_PTR(-ENOMEM) if we failed to allocate memory 315 * 316 * NOTE: caller must hold full_stripe_locks_root->lock before calling this 317 * function 318 */ 319 static struct full_stripe_lock *insert_full_stripe_lock( 320 struct btrfs_full_stripe_locks_tree *locks_root, 321 u64 fstripe_logical) 322 { 323 struct rb_node **p; 324 struct rb_node *parent = NULL; 325 struct full_stripe_lock *entry; 326 struct full_stripe_lock *ret; 327 328 lockdep_assert_held(&locks_root->lock); 329 330 p = &locks_root->root.rb_node; 331 while (*p) { 332 parent = *p; 333 entry = rb_entry(parent, struct full_stripe_lock, node); 334 if (fstripe_logical < entry->logical) { 335 p = &(*p)->rb_left; 336 } else if (fstripe_logical > entry->logical) { 337 p = &(*p)->rb_right; 338 } else { 339 entry->refs++; 340 return entry; 341 } 342 } 343 344 /* 345 * Insert new lock. 346 */ 347 ret = kmalloc(sizeof(*ret), GFP_KERNEL); 348 if (!ret) 349 return ERR_PTR(-ENOMEM); 350 ret->logical = fstripe_logical; 351 ret->refs = 1; 352 mutex_init(&ret->mutex); 353 354 rb_link_node(&ret->node, parent, p); 355 rb_insert_color(&ret->node, &locks_root->root); 356 return ret; 357 } 358 359 /* 360 * Search for a full stripe lock of a block group 361 * 362 * Return pointer to existing full stripe lock if found 363 * Return NULL if not found 364 */ 365 static struct full_stripe_lock *search_full_stripe_lock( 366 struct btrfs_full_stripe_locks_tree *locks_root, 367 u64 fstripe_logical) 368 { 369 struct rb_node *node; 370 struct full_stripe_lock *entry; 371 372 lockdep_assert_held(&locks_root->lock); 373 374 node = locks_root->root.rb_node; 375 while (node) { 376 entry = rb_entry(node, struct full_stripe_lock, node); 377 if (fstripe_logical < entry->logical) 378 node = node->rb_left; 379 else if (fstripe_logical > entry->logical) 380 node = node->rb_right; 381 else 382 return entry; 383 } 384 return NULL; 385 } 386 387 /* 388 * Helper to get full stripe logical from a normal bytenr. 389 * 390 * Caller must ensure @cache is a RAID56 block group. 391 */ 392 static u64 get_full_stripe_logical(struct btrfs_block_group *cache, u64 bytenr) 393 { 394 u64 ret; 395 396 /* 397 * Due to chunk item size limit, full stripe length should not be 398 * larger than U32_MAX. Just a sanity check here. 399 */ 400 WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX); 401 402 /* 403 * round_down() can only handle power of 2, while RAID56 full 404 * stripe length can be 64KiB * n, so we need to manually round down. 405 */ 406 ret = div64_u64(bytenr - cache->start, cache->full_stripe_len) * 407 cache->full_stripe_len + cache->start; 408 return ret; 409 } 410 411 /* 412 * Lock a full stripe to avoid concurrency of recovery and read 413 * 414 * It's only used for profiles with parities (RAID5/6), for other profiles it 415 * does nothing. 416 * 417 * Return 0 if we locked full stripe covering @bytenr, with a mutex held. 418 * So caller must call unlock_full_stripe() at the same context. 419 * 420 * Return <0 if encounters error. 421 */ 422 static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr, 423 bool *locked_ret) 424 { 425 struct btrfs_block_group *bg_cache; 426 struct btrfs_full_stripe_locks_tree *locks_root; 427 struct full_stripe_lock *existing; 428 u64 fstripe_start; 429 int ret = 0; 430 431 *locked_ret = false; 432 bg_cache = btrfs_lookup_block_group(fs_info, bytenr); 433 if (!bg_cache) { 434 ASSERT(0); 435 return -ENOENT; 436 } 437 438 /* Profiles not based on parity don't need full stripe lock */ 439 if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) 440 goto out; 441 locks_root = &bg_cache->full_stripe_locks_root; 442 443 fstripe_start = get_full_stripe_logical(bg_cache, bytenr); 444 445 /* Now insert the full stripe lock */ 446 mutex_lock(&locks_root->lock); 447 existing = insert_full_stripe_lock(locks_root, fstripe_start); 448 mutex_unlock(&locks_root->lock); 449 if (IS_ERR(existing)) { 450 ret = PTR_ERR(existing); 451 goto out; 452 } 453 mutex_lock(&existing->mutex); 454 *locked_ret = true; 455 out: 456 btrfs_put_block_group(bg_cache); 457 return ret; 458 } 459 460 /* 461 * Unlock a full stripe. 462 * 463 * NOTE: Caller must ensure it's the same context calling corresponding 464 * lock_full_stripe(). 465 * 466 * Return 0 if we unlock full stripe without problem. 467 * Return <0 for error 468 */ 469 static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr, 470 bool locked) 471 { 472 struct btrfs_block_group *bg_cache; 473 struct btrfs_full_stripe_locks_tree *locks_root; 474 struct full_stripe_lock *fstripe_lock; 475 u64 fstripe_start; 476 bool freeit = false; 477 int ret = 0; 478 479 /* If we didn't acquire full stripe lock, no need to continue */ 480 if (!locked) 481 return 0; 482 483 bg_cache = btrfs_lookup_block_group(fs_info, bytenr); 484 if (!bg_cache) { 485 ASSERT(0); 486 return -ENOENT; 487 } 488 if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) 489 goto out; 490 491 locks_root = &bg_cache->full_stripe_locks_root; 492 fstripe_start = get_full_stripe_logical(bg_cache, bytenr); 493 494 mutex_lock(&locks_root->lock); 495 fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start); 496 /* Unpaired unlock_full_stripe() detected */ 497 if (!fstripe_lock) { 498 WARN_ON(1); 499 ret = -ENOENT; 500 mutex_unlock(&locks_root->lock); 501 goto out; 502 } 503 504 if (fstripe_lock->refs == 0) { 505 WARN_ON(1); 506 btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow", 507 fstripe_lock->logical); 508 } else { 509 fstripe_lock->refs--; 510 } 511 512 if (fstripe_lock->refs == 0) { 513 rb_erase(&fstripe_lock->node, &locks_root->root); 514 freeit = true; 515 } 516 mutex_unlock(&locks_root->lock); 517 518 mutex_unlock(&fstripe_lock->mutex); 519 if (freeit) 520 kfree(fstripe_lock); 521 out: 522 btrfs_put_block_group(bg_cache); 523 return ret; 524 } 525 526 static void scrub_free_csums(struct scrub_ctx *sctx) 527 { 528 while (!list_empty(&sctx->csum_list)) { 529 struct btrfs_ordered_sum *sum; 530 sum = list_first_entry(&sctx->csum_list, 531 struct btrfs_ordered_sum, list); 532 list_del(&sum->list); 533 kfree(sum); 534 } 535 } 536 537 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) 538 { 539 int i; 540 541 if (!sctx) 542 return; 543 544 /* this can happen when scrub is cancelled */ 545 if (sctx->curr != -1) { 546 struct scrub_bio *sbio = sctx->bios[sctx->curr]; 547 548 for (i = 0; i < sbio->page_count; i++) { 549 WARN_ON(!sbio->pagev[i]->page); 550 scrub_block_put(sbio->pagev[i]->sblock); 551 } 552 bio_put(sbio->bio); 553 } 554 555 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { 556 struct scrub_bio *sbio = sctx->bios[i]; 557 558 if (!sbio) 559 break; 560 kfree(sbio); 561 } 562 563 kfree(sctx->wr_curr_bio); 564 scrub_free_csums(sctx); 565 kfree(sctx); 566 } 567 568 static void scrub_put_ctx(struct scrub_ctx *sctx) 569 { 570 if (refcount_dec_and_test(&sctx->refs)) 571 scrub_free_ctx(sctx); 572 } 573 574 static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( 575 struct btrfs_fs_info *fs_info, int is_dev_replace) 576 { 577 struct scrub_ctx *sctx; 578 int i; 579 580 sctx = kzalloc(sizeof(*sctx), GFP_KERNEL); 581 if (!sctx) 582 goto nomem; 583 refcount_set(&sctx->refs, 1); 584 sctx->is_dev_replace = is_dev_replace; 585 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; 586 sctx->curr = -1; 587 sctx->fs_info = fs_info; 588 INIT_LIST_HEAD(&sctx->csum_list); 589 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { 590 struct scrub_bio *sbio; 591 592 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL); 593 if (!sbio) 594 goto nomem; 595 sctx->bios[i] = sbio; 596 597 sbio->index = i; 598 sbio->sctx = sctx; 599 sbio->page_count = 0; 600 btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL, 601 NULL); 602 603 if (i != SCRUB_BIOS_PER_SCTX - 1) 604 sctx->bios[i]->next_free = i + 1; 605 else 606 sctx->bios[i]->next_free = -1; 607 } 608 sctx->first_free = 0; 609 atomic_set(&sctx->bios_in_flight, 0); 610 atomic_set(&sctx->workers_pending, 0); 611 atomic_set(&sctx->cancel_req, 0); 612 613 spin_lock_init(&sctx->list_lock); 614 spin_lock_init(&sctx->stat_lock); 615 init_waitqueue_head(&sctx->list_wait); 616 617 WARN_ON(sctx->wr_curr_bio != NULL); 618 mutex_init(&sctx->wr_lock); 619 sctx->wr_curr_bio = NULL; 620 if (is_dev_replace) { 621 WARN_ON(!fs_info->dev_replace.tgtdev); 622 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO; 623 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev; 624 sctx->flush_all_writes = false; 625 } 626 627 return sctx; 628 629 nomem: 630 scrub_free_ctx(sctx); 631 return ERR_PTR(-ENOMEM); 632 } 633 634 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, 635 void *warn_ctx) 636 { 637 u64 isize; 638 u32 nlink; 639 int ret; 640 int i; 641 unsigned nofs_flag; 642 struct extent_buffer *eb; 643 struct btrfs_inode_item *inode_item; 644 struct scrub_warning *swarn = warn_ctx; 645 struct btrfs_fs_info *fs_info = swarn->dev->fs_info; 646 struct inode_fs_paths *ipath = NULL; 647 struct btrfs_root *local_root; 648 struct btrfs_key key; 649 650 local_root = btrfs_get_fs_root(fs_info, root, true); 651 if (IS_ERR(local_root)) { 652 ret = PTR_ERR(local_root); 653 goto err; 654 } 655 656 /* 657 * this makes the path point to (inum INODE_ITEM ioff) 658 */ 659 key.objectid = inum; 660 key.type = BTRFS_INODE_ITEM_KEY; 661 key.offset = 0; 662 663 ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0); 664 if (ret) { 665 btrfs_put_root(local_root); 666 btrfs_release_path(swarn->path); 667 goto err; 668 } 669 670 eb = swarn->path->nodes[0]; 671 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0], 672 struct btrfs_inode_item); 673 isize = btrfs_inode_size(eb, inode_item); 674 nlink = btrfs_inode_nlink(eb, inode_item); 675 btrfs_release_path(swarn->path); 676 677 /* 678 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub 679 * uses GFP_NOFS in this context, so we keep it consistent but it does 680 * not seem to be strictly necessary. 681 */ 682 nofs_flag = memalloc_nofs_save(); 683 ipath = init_ipath(4096, local_root, swarn->path); 684 memalloc_nofs_restore(nofs_flag); 685 if (IS_ERR(ipath)) { 686 btrfs_put_root(local_root); 687 ret = PTR_ERR(ipath); 688 ipath = NULL; 689 goto err; 690 } 691 ret = paths_from_inode(inum, ipath); 692 693 if (ret < 0) 694 goto err; 695 696 /* 697 * we deliberately ignore the bit ipath might have been too small to 698 * hold all of the paths here 699 */ 700 for (i = 0; i < ipath->fspath->elem_cnt; ++i) 701 btrfs_warn_in_rcu(fs_info, 702 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)", 703 swarn->errstr, swarn->logical, 704 rcu_str_deref(swarn->dev->name), 705 swarn->physical, 706 root, inum, offset, 707 min(isize - offset, (u64)PAGE_SIZE), nlink, 708 (char *)(unsigned long)ipath->fspath->val[i]); 709 710 btrfs_put_root(local_root); 711 free_ipath(ipath); 712 return 0; 713 714 err: 715 btrfs_warn_in_rcu(fs_info, 716 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d", 717 swarn->errstr, swarn->logical, 718 rcu_str_deref(swarn->dev->name), 719 swarn->physical, 720 root, inum, offset, ret); 721 722 free_ipath(ipath); 723 return 0; 724 } 725 726 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock) 727 { 728 struct btrfs_device *dev; 729 struct btrfs_fs_info *fs_info; 730 struct btrfs_path *path; 731 struct btrfs_key found_key; 732 struct extent_buffer *eb; 733 struct btrfs_extent_item *ei; 734 struct scrub_warning swarn; 735 unsigned long ptr = 0; 736 u64 extent_item_pos; 737 u64 flags = 0; 738 u64 ref_root; 739 u32 item_size; 740 u8 ref_level = 0; 741 int ret; 742 743 WARN_ON(sblock->page_count < 1); 744 dev = sblock->pagev[0]->dev; 745 fs_info = sblock->sctx->fs_info; 746 747 path = btrfs_alloc_path(); 748 if (!path) 749 return; 750 751 swarn.physical = sblock->pagev[0]->physical; 752 swarn.logical = sblock->pagev[0]->logical; 753 swarn.errstr = errstr; 754 swarn.dev = NULL; 755 756 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key, 757 &flags); 758 if (ret < 0) 759 goto out; 760 761 extent_item_pos = swarn.logical - found_key.objectid; 762 swarn.extent_item_size = found_key.offset; 763 764 eb = path->nodes[0]; 765 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); 766 item_size = btrfs_item_size_nr(eb, path->slots[0]); 767 768 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 769 do { 770 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei, 771 item_size, &ref_root, 772 &ref_level); 773 btrfs_warn_in_rcu(fs_info, 774 "%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu", 775 errstr, swarn.logical, 776 rcu_str_deref(dev->name), 777 swarn.physical, 778 ref_level ? "node" : "leaf", 779 ret < 0 ? -1 : ref_level, 780 ret < 0 ? -1 : ref_root); 781 } while (ret != 1); 782 btrfs_release_path(path); 783 } else { 784 btrfs_release_path(path); 785 swarn.path = path; 786 swarn.dev = dev; 787 iterate_extent_inodes(fs_info, found_key.objectid, 788 extent_item_pos, 1, 789 scrub_print_warning_inode, &swarn, false); 790 } 791 792 out: 793 btrfs_free_path(path); 794 } 795 796 static inline void scrub_get_recover(struct scrub_recover *recover) 797 { 798 refcount_inc(&recover->refs); 799 } 800 801 static inline void scrub_put_recover(struct btrfs_fs_info *fs_info, 802 struct scrub_recover *recover) 803 { 804 if (refcount_dec_and_test(&recover->refs)) { 805 btrfs_bio_counter_dec(fs_info); 806 btrfs_put_bbio(recover->bbio); 807 kfree(recover); 808 } 809 } 810 811 /* 812 * scrub_handle_errored_block gets called when either verification of the 813 * pages failed or the bio failed to read, e.g. with EIO. In the latter 814 * case, this function handles all pages in the bio, even though only one 815 * may be bad. 816 * The goal of this function is to repair the errored block by using the 817 * contents of one of the mirrors. 818 */ 819 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) 820 { 821 struct scrub_ctx *sctx = sblock_to_check->sctx; 822 struct btrfs_device *dev; 823 struct btrfs_fs_info *fs_info; 824 u64 logical; 825 unsigned int failed_mirror_index; 826 unsigned int is_metadata; 827 unsigned int have_csum; 828 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */ 829 struct scrub_block *sblock_bad; 830 int ret; 831 int mirror_index; 832 int page_num; 833 int success; 834 bool full_stripe_locked; 835 unsigned int nofs_flag; 836 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 837 DEFAULT_RATELIMIT_BURST); 838 839 BUG_ON(sblock_to_check->page_count < 1); 840 fs_info = sctx->fs_info; 841 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) { 842 /* 843 * if we find an error in a super block, we just report it. 844 * They will get written with the next transaction commit 845 * anyway 846 */ 847 spin_lock(&sctx->stat_lock); 848 ++sctx->stat.super_errors; 849 spin_unlock(&sctx->stat_lock); 850 return 0; 851 } 852 logical = sblock_to_check->pagev[0]->logical; 853 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1); 854 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1; 855 is_metadata = !(sblock_to_check->pagev[0]->flags & 856 BTRFS_EXTENT_FLAG_DATA); 857 have_csum = sblock_to_check->pagev[0]->have_csum; 858 dev = sblock_to_check->pagev[0]->dev; 859 860 if (btrfs_is_zoned(fs_info) && !sctx->is_dev_replace) 861 return btrfs_repair_one_zone(fs_info, logical); 862 863 /* 864 * We must use GFP_NOFS because the scrub task might be waiting for a 865 * worker task executing this function and in turn a transaction commit 866 * might be waiting the scrub task to pause (which needs to wait for all 867 * the worker tasks to complete before pausing). 868 * We do allocations in the workers through insert_full_stripe_lock() 869 * and scrub_add_page_to_wr_bio(), which happens down the call chain of 870 * this function. 871 */ 872 nofs_flag = memalloc_nofs_save(); 873 /* 874 * For RAID5/6, race can happen for a different device scrub thread. 875 * For data corruption, Parity and Data threads will both try 876 * to recovery the data. 877 * Race can lead to doubly added csum error, or even unrecoverable 878 * error. 879 */ 880 ret = lock_full_stripe(fs_info, logical, &full_stripe_locked); 881 if (ret < 0) { 882 memalloc_nofs_restore(nofs_flag); 883 spin_lock(&sctx->stat_lock); 884 if (ret == -ENOMEM) 885 sctx->stat.malloc_errors++; 886 sctx->stat.read_errors++; 887 sctx->stat.uncorrectable_errors++; 888 spin_unlock(&sctx->stat_lock); 889 return ret; 890 } 891 892 /* 893 * read all mirrors one after the other. This includes to 894 * re-read the extent or metadata block that failed (that was 895 * the cause that this fixup code is called) another time, 896 * page by page this time in order to know which pages 897 * caused I/O errors and which ones are good (for all mirrors). 898 * It is the goal to handle the situation when more than one 899 * mirror contains I/O errors, but the errors do not 900 * overlap, i.e. the data can be repaired by selecting the 901 * pages from those mirrors without I/O error on the 902 * particular pages. One example (with blocks >= 2 * PAGE_SIZE) 903 * would be that mirror #1 has an I/O error on the first page, 904 * the second page is good, and mirror #2 has an I/O error on 905 * the second page, but the first page is good. 906 * Then the first page of the first mirror can be repaired by 907 * taking the first page of the second mirror, and the 908 * second page of the second mirror can be repaired by 909 * copying the contents of the 2nd page of the 1st mirror. 910 * One more note: if the pages of one mirror contain I/O 911 * errors, the checksum cannot be verified. In order to get 912 * the best data for repairing, the first attempt is to find 913 * a mirror without I/O errors and with a validated checksum. 914 * Only if this is not possible, the pages are picked from 915 * mirrors with I/O errors without considering the checksum. 916 * If the latter is the case, at the end, the checksum of the 917 * repaired area is verified in order to correctly maintain 918 * the statistics. 919 */ 920 921 sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS, 922 sizeof(*sblocks_for_recheck), GFP_KERNEL); 923 if (!sblocks_for_recheck) { 924 spin_lock(&sctx->stat_lock); 925 sctx->stat.malloc_errors++; 926 sctx->stat.read_errors++; 927 sctx->stat.uncorrectable_errors++; 928 spin_unlock(&sctx->stat_lock); 929 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 930 goto out; 931 } 932 933 /* setup the context, map the logical blocks and alloc the pages */ 934 ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck); 935 if (ret) { 936 spin_lock(&sctx->stat_lock); 937 sctx->stat.read_errors++; 938 sctx->stat.uncorrectable_errors++; 939 spin_unlock(&sctx->stat_lock); 940 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 941 goto out; 942 } 943 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS); 944 sblock_bad = sblocks_for_recheck + failed_mirror_index; 945 946 /* build and submit the bios for the failed mirror, check checksums */ 947 scrub_recheck_block(fs_info, sblock_bad, 1); 948 949 if (!sblock_bad->header_error && !sblock_bad->checksum_error && 950 sblock_bad->no_io_error_seen) { 951 /* 952 * the error disappeared after reading page by page, or 953 * the area was part of a huge bio and other parts of the 954 * bio caused I/O errors, or the block layer merged several 955 * read requests into one and the error is caused by a 956 * different bio (usually one of the two latter cases is 957 * the cause) 958 */ 959 spin_lock(&sctx->stat_lock); 960 sctx->stat.unverified_errors++; 961 sblock_to_check->data_corrected = 1; 962 spin_unlock(&sctx->stat_lock); 963 964 if (sctx->is_dev_replace) 965 scrub_write_block_to_dev_replace(sblock_bad); 966 goto out; 967 } 968 969 if (!sblock_bad->no_io_error_seen) { 970 spin_lock(&sctx->stat_lock); 971 sctx->stat.read_errors++; 972 spin_unlock(&sctx->stat_lock); 973 if (__ratelimit(&rs)) 974 scrub_print_warning("i/o error", sblock_to_check); 975 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 976 } else if (sblock_bad->checksum_error) { 977 spin_lock(&sctx->stat_lock); 978 sctx->stat.csum_errors++; 979 spin_unlock(&sctx->stat_lock); 980 if (__ratelimit(&rs)) 981 scrub_print_warning("checksum error", sblock_to_check); 982 btrfs_dev_stat_inc_and_print(dev, 983 BTRFS_DEV_STAT_CORRUPTION_ERRS); 984 } else if (sblock_bad->header_error) { 985 spin_lock(&sctx->stat_lock); 986 sctx->stat.verify_errors++; 987 spin_unlock(&sctx->stat_lock); 988 if (__ratelimit(&rs)) 989 scrub_print_warning("checksum/header error", 990 sblock_to_check); 991 if (sblock_bad->generation_error) 992 btrfs_dev_stat_inc_and_print(dev, 993 BTRFS_DEV_STAT_GENERATION_ERRS); 994 else 995 btrfs_dev_stat_inc_and_print(dev, 996 BTRFS_DEV_STAT_CORRUPTION_ERRS); 997 } 998 999 if (sctx->readonly) { 1000 ASSERT(!sctx->is_dev_replace); 1001 goto out; 1002 } 1003 1004 /* 1005 * now build and submit the bios for the other mirrors, check 1006 * checksums. 1007 * First try to pick the mirror which is completely without I/O 1008 * errors and also does not have a checksum error. 1009 * If one is found, and if a checksum is present, the full block 1010 * that is known to contain an error is rewritten. Afterwards 1011 * the block is known to be corrected. 1012 * If a mirror is found which is completely correct, and no 1013 * checksum is present, only those pages are rewritten that had 1014 * an I/O error in the block to be repaired, since it cannot be 1015 * determined, which copy of the other pages is better (and it 1016 * could happen otherwise that a correct page would be 1017 * overwritten by a bad one). 1018 */ 1019 for (mirror_index = 0; ;mirror_index++) { 1020 struct scrub_block *sblock_other; 1021 1022 if (mirror_index == failed_mirror_index) 1023 continue; 1024 1025 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */ 1026 if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) { 1027 if (mirror_index >= BTRFS_MAX_MIRRORS) 1028 break; 1029 if (!sblocks_for_recheck[mirror_index].page_count) 1030 break; 1031 1032 sblock_other = sblocks_for_recheck + mirror_index; 1033 } else { 1034 struct scrub_recover *r = sblock_bad->pagev[0]->recover; 1035 int max_allowed = r->bbio->num_stripes - 1036 r->bbio->num_tgtdevs; 1037 1038 if (mirror_index >= max_allowed) 1039 break; 1040 if (!sblocks_for_recheck[1].page_count) 1041 break; 1042 1043 ASSERT(failed_mirror_index == 0); 1044 sblock_other = sblocks_for_recheck + 1; 1045 sblock_other->pagev[0]->mirror_num = 1 + mirror_index; 1046 } 1047 1048 /* build and submit the bios, check checksums */ 1049 scrub_recheck_block(fs_info, sblock_other, 0); 1050 1051 if (!sblock_other->header_error && 1052 !sblock_other->checksum_error && 1053 sblock_other->no_io_error_seen) { 1054 if (sctx->is_dev_replace) { 1055 scrub_write_block_to_dev_replace(sblock_other); 1056 goto corrected_error; 1057 } else { 1058 ret = scrub_repair_block_from_good_copy( 1059 sblock_bad, sblock_other); 1060 if (!ret) 1061 goto corrected_error; 1062 } 1063 } 1064 } 1065 1066 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace) 1067 goto did_not_correct_error; 1068 1069 /* 1070 * In case of I/O errors in the area that is supposed to be 1071 * repaired, continue by picking good copies of those pages. 1072 * Select the good pages from mirrors to rewrite bad pages from 1073 * the area to fix. Afterwards verify the checksum of the block 1074 * that is supposed to be repaired. This verification step is 1075 * only done for the purpose of statistic counting and for the 1076 * final scrub report, whether errors remain. 1077 * A perfect algorithm could make use of the checksum and try 1078 * all possible combinations of pages from the different mirrors 1079 * until the checksum verification succeeds. For example, when 1080 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page 1081 * of mirror #2 is readable but the final checksum test fails, 1082 * then the 2nd page of mirror #3 could be tried, whether now 1083 * the final checksum succeeds. But this would be a rare 1084 * exception and is therefore not implemented. At least it is 1085 * avoided that the good copy is overwritten. 1086 * A more useful improvement would be to pick the sectors 1087 * without I/O error based on sector sizes (512 bytes on legacy 1088 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one 1089 * mirror could be repaired by taking 512 byte of a different 1090 * mirror, even if other 512 byte sectors in the same PAGE_SIZE 1091 * area are unreadable. 1092 */ 1093 success = 1; 1094 for (page_num = 0; page_num < sblock_bad->page_count; 1095 page_num++) { 1096 struct scrub_page *spage_bad = sblock_bad->pagev[page_num]; 1097 struct scrub_block *sblock_other = NULL; 1098 1099 /* skip no-io-error page in scrub */ 1100 if (!spage_bad->io_error && !sctx->is_dev_replace) 1101 continue; 1102 1103 if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) { 1104 /* 1105 * In case of dev replace, if raid56 rebuild process 1106 * didn't work out correct data, then copy the content 1107 * in sblock_bad to make sure target device is identical 1108 * to source device, instead of writing garbage data in 1109 * sblock_for_recheck array to target device. 1110 */ 1111 sblock_other = NULL; 1112 } else if (spage_bad->io_error) { 1113 /* try to find no-io-error page in mirrors */ 1114 for (mirror_index = 0; 1115 mirror_index < BTRFS_MAX_MIRRORS && 1116 sblocks_for_recheck[mirror_index].page_count > 0; 1117 mirror_index++) { 1118 if (!sblocks_for_recheck[mirror_index]. 1119 pagev[page_num]->io_error) { 1120 sblock_other = sblocks_for_recheck + 1121 mirror_index; 1122 break; 1123 } 1124 } 1125 if (!sblock_other) 1126 success = 0; 1127 } 1128 1129 if (sctx->is_dev_replace) { 1130 /* 1131 * did not find a mirror to fetch the page 1132 * from. scrub_write_page_to_dev_replace() 1133 * handles this case (page->io_error), by 1134 * filling the block with zeros before 1135 * submitting the write request 1136 */ 1137 if (!sblock_other) 1138 sblock_other = sblock_bad; 1139 1140 if (scrub_write_page_to_dev_replace(sblock_other, 1141 page_num) != 0) { 1142 atomic64_inc( 1143 &fs_info->dev_replace.num_write_errors); 1144 success = 0; 1145 } 1146 } else if (sblock_other) { 1147 ret = scrub_repair_page_from_good_copy(sblock_bad, 1148 sblock_other, 1149 page_num, 0); 1150 if (0 == ret) 1151 spage_bad->io_error = 0; 1152 else 1153 success = 0; 1154 } 1155 } 1156 1157 if (success && !sctx->is_dev_replace) { 1158 if (is_metadata || have_csum) { 1159 /* 1160 * need to verify the checksum now that all 1161 * sectors on disk are repaired (the write 1162 * request for data to be repaired is on its way). 1163 * Just be lazy and use scrub_recheck_block() 1164 * which re-reads the data before the checksum 1165 * is verified, but most likely the data comes out 1166 * of the page cache. 1167 */ 1168 scrub_recheck_block(fs_info, sblock_bad, 1); 1169 if (!sblock_bad->header_error && 1170 !sblock_bad->checksum_error && 1171 sblock_bad->no_io_error_seen) 1172 goto corrected_error; 1173 else 1174 goto did_not_correct_error; 1175 } else { 1176 corrected_error: 1177 spin_lock(&sctx->stat_lock); 1178 sctx->stat.corrected_errors++; 1179 sblock_to_check->data_corrected = 1; 1180 spin_unlock(&sctx->stat_lock); 1181 btrfs_err_rl_in_rcu(fs_info, 1182 "fixed up error at logical %llu on dev %s", 1183 logical, rcu_str_deref(dev->name)); 1184 } 1185 } else { 1186 did_not_correct_error: 1187 spin_lock(&sctx->stat_lock); 1188 sctx->stat.uncorrectable_errors++; 1189 spin_unlock(&sctx->stat_lock); 1190 btrfs_err_rl_in_rcu(fs_info, 1191 "unable to fixup (regular) error at logical %llu on dev %s", 1192 logical, rcu_str_deref(dev->name)); 1193 } 1194 1195 out: 1196 if (sblocks_for_recheck) { 1197 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS; 1198 mirror_index++) { 1199 struct scrub_block *sblock = sblocks_for_recheck + 1200 mirror_index; 1201 struct scrub_recover *recover; 1202 int page_index; 1203 1204 for (page_index = 0; page_index < sblock->page_count; 1205 page_index++) { 1206 sblock->pagev[page_index]->sblock = NULL; 1207 recover = sblock->pagev[page_index]->recover; 1208 if (recover) { 1209 scrub_put_recover(fs_info, recover); 1210 sblock->pagev[page_index]->recover = 1211 NULL; 1212 } 1213 scrub_page_put(sblock->pagev[page_index]); 1214 } 1215 } 1216 kfree(sblocks_for_recheck); 1217 } 1218 1219 ret = unlock_full_stripe(fs_info, logical, full_stripe_locked); 1220 memalloc_nofs_restore(nofs_flag); 1221 if (ret < 0) 1222 return ret; 1223 return 0; 1224 } 1225 1226 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio) 1227 { 1228 if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5) 1229 return 2; 1230 else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6) 1231 return 3; 1232 else 1233 return (int)bbio->num_stripes; 1234 } 1235 1236 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type, 1237 u64 *raid_map, 1238 u64 mapped_length, 1239 int nstripes, int mirror, 1240 int *stripe_index, 1241 u64 *stripe_offset) 1242 { 1243 int i; 1244 1245 if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 1246 /* RAID5/6 */ 1247 for (i = 0; i < nstripes; i++) { 1248 if (raid_map[i] == RAID6_Q_STRIPE || 1249 raid_map[i] == RAID5_P_STRIPE) 1250 continue; 1251 1252 if (logical >= raid_map[i] && 1253 logical < raid_map[i] + mapped_length) 1254 break; 1255 } 1256 1257 *stripe_index = i; 1258 *stripe_offset = logical - raid_map[i]; 1259 } else { 1260 /* The other RAID type */ 1261 *stripe_index = mirror; 1262 *stripe_offset = 0; 1263 } 1264 } 1265 1266 static int scrub_setup_recheck_block(struct scrub_block *original_sblock, 1267 struct scrub_block *sblocks_for_recheck) 1268 { 1269 struct scrub_ctx *sctx = original_sblock->sctx; 1270 struct btrfs_fs_info *fs_info = sctx->fs_info; 1271 u64 length = original_sblock->page_count * PAGE_SIZE; 1272 u64 logical = original_sblock->pagev[0]->logical; 1273 u64 generation = original_sblock->pagev[0]->generation; 1274 u64 flags = original_sblock->pagev[0]->flags; 1275 u64 have_csum = original_sblock->pagev[0]->have_csum; 1276 struct scrub_recover *recover; 1277 struct btrfs_bio *bbio; 1278 u64 sublen; 1279 u64 mapped_length; 1280 u64 stripe_offset; 1281 int stripe_index; 1282 int page_index = 0; 1283 int mirror_index; 1284 int nmirrors; 1285 int ret; 1286 1287 /* 1288 * note: the two members refs and outstanding_pages 1289 * are not used (and not set) in the blocks that are used for 1290 * the recheck procedure 1291 */ 1292 1293 while (length > 0) { 1294 sublen = min_t(u64, length, PAGE_SIZE); 1295 mapped_length = sublen; 1296 bbio = NULL; 1297 1298 /* 1299 * with a length of PAGE_SIZE, each returned stripe 1300 * represents one mirror 1301 */ 1302 btrfs_bio_counter_inc_blocked(fs_info); 1303 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, 1304 logical, &mapped_length, &bbio); 1305 if (ret || !bbio || mapped_length < sublen) { 1306 btrfs_put_bbio(bbio); 1307 btrfs_bio_counter_dec(fs_info); 1308 return -EIO; 1309 } 1310 1311 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS); 1312 if (!recover) { 1313 btrfs_put_bbio(bbio); 1314 btrfs_bio_counter_dec(fs_info); 1315 return -ENOMEM; 1316 } 1317 1318 refcount_set(&recover->refs, 1); 1319 recover->bbio = bbio; 1320 recover->map_length = mapped_length; 1321 1322 BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK); 1323 1324 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS); 1325 1326 for (mirror_index = 0; mirror_index < nmirrors; 1327 mirror_index++) { 1328 struct scrub_block *sblock; 1329 struct scrub_page *spage; 1330 1331 sblock = sblocks_for_recheck + mirror_index; 1332 sblock->sctx = sctx; 1333 1334 spage = kzalloc(sizeof(*spage), GFP_NOFS); 1335 if (!spage) { 1336 leave_nomem: 1337 spin_lock(&sctx->stat_lock); 1338 sctx->stat.malloc_errors++; 1339 spin_unlock(&sctx->stat_lock); 1340 scrub_put_recover(fs_info, recover); 1341 return -ENOMEM; 1342 } 1343 scrub_page_get(spage); 1344 sblock->pagev[page_index] = spage; 1345 spage->sblock = sblock; 1346 spage->flags = flags; 1347 spage->generation = generation; 1348 spage->logical = logical; 1349 spage->have_csum = have_csum; 1350 if (have_csum) 1351 memcpy(spage->csum, 1352 original_sblock->pagev[0]->csum, 1353 sctx->fs_info->csum_size); 1354 1355 scrub_stripe_index_and_offset(logical, 1356 bbio->map_type, 1357 bbio->raid_map, 1358 mapped_length, 1359 bbio->num_stripes - 1360 bbio->num_tgtdevs, 1361 mirror_index, 1362 &stripe_index, 1363 &stripe_offset); 1364 spage->physical = bbio->stripes[stripe_index].physical + 1365 stripe_offset; 1366 spage->dev = bbio->stripes[stripe_index].dev; 1367 1368 BUG_ON(page_index >= original_sblock->page_count); 1369 spage->physical_for_dev_replace = 1370 original_sblock->pagev[page_index]-> 1371 physical_for_dev_replace; 1372 /* for missing devices, dev->bdev is NULL */ 1373 spage->mirror_num = mirror_index + 1; 1374 sblock->page_count++; 1375 spage->page = alloc_page(GFP_NOFS); 1376 if (!spage->page) 1377 goto leave_nomem; 1378 1379 scrub_get_recover(recover); 1380 spage->recover = recover; 1381 } 1382 scrub_put_recover(fs_info, recover); 1383 length -= sublen; 1384 logical += sublen; 1385 page_index++; 1386 } 1387 1388 return 0; 1389 } 1390 1391 static void scrub_bio_wait_endio(struct bio *bio) 1392 { 1393 complete(bio->bi_private); 1394 } 1395 1396 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info, 1397 struct bio *bio, 1398 struct scrub_page *spage) 1399 { 1400 DECLARE_COMPLETION_ONSTACK(done); 1401 int ret; 1402 int mirror_num; 1403 1404 bio->bi_iter.bi_sector = spage->logical >> 9; 1405 bio->bi_private = &done; 1406 bio->bi_end_io = scrub_bio_wait_endio; 1407 1408 mirror_num = spage->sblock->pagev[0]->mirror_num; 1409 ret = raid56_parity_recover(fs_info, bio, spage->recover->bbio, 1410 spage->recover->map_length, 1411 mirror_num, 0); 1412 if (ret) 1413 return ret; 1414 1415 wait_for_completion_io(&done); 1416 return blk_status_to_errno(bio->bi_status); 1417 } 1418 1419 static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info, 1420 struct scrub_block *sblock) 1421 { 1422 struct scrub_page *first_page = sblock->pagev[0]; 1423 struct bio *bio; 1424 int page_num; 1425 1426 /* All pages in sblock belong to the same stripe on the same device. */ 1427 ASSERT(first_page->dev); 1428 if (!first_page->dev->bdev) 1429 goto out; 1430 1431 bio = btrfs_io_bio_alloc(BIO_MAX_VECS); 1432 bio_set_dev(bio, first_page->dev->bdev); 1433 1434 for (page_num = 0; page_num < sblock->page_count; page_num++) { 1435 struct scrub_page *spage = sblock->pagev[page_num]; 1436 1437 WARN_ON(!spage->page); 1438 bio_add_page(bio, spage->page, PAGE_SIZE, 0); 1439 } 1440 1441 if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) { 1442 bio_put(bio); 1443 goto out; 1444 } 1445 1446 bio_put(bio); 1447 1448 scrub_recheck_block_checksum(sblock); 1449 1450 return; 1451 out: 1452 for (page_num = 0; page_num < sblock->page_count; page_num++) 1453 sblock->pagev[page_num]->io_error = 1; 1454 1455 sblock->no_io_error_seen = 0; 1456 } 1457 1458 /* 1459 * this function will check the on disk data for checksum errors, header 1460 * errors and read I/O errors. If any I/O errors happen, the exact pages 1461 * which are errored are marked as being bad. The goal is to enable scrub 1462 * to take those pages that are not errored from all the mirrors so that 1463 * the pages that are errored in the just handled mirror can be repaired. 1464 */ 1465 static void scrub_recheck_block(struct btrfs_fs_info *fs_info, 1466 struct scrub_block *sblock, 1467 int retry_failed_mirror) 1468 { 1469 int page_num; 1470 1471 sblock->no_io_error_seen = 1; 1472 1473 /* short cut for raid56 */ 1474 if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0])) 1475 return scrub_recheck_block_on_raid56(fs_info, sblock); 1476 1477 for (page_num = 0; page_num < sblock->page_count; page_num++) { 1478 struct bio *bio; 1479 struct scrub_page *spage = sblock->pagev[page_num]; 1480 1481 if (spage->dev->bdev == NULL) { 1482 spage->io_error = 1; 1483 sblock->no_io_error_seen = 0; 1484 continue; 1485 } 1486 1487 WARN_ON(!spage->page); 1488 bio = btrfs_io_bio_alloc(1); 1489 bio_set_dev(bio, spage->dev->bdev); 1490 1491 bio_add_page(bio, spage->page, PAGE_SIZE, 0); 1492 bio->bi_iter.bi_sector = spage->physical >> 9; 1493 bio->bi_opf = REQ_OP_READ; 1494 1495 if (btrfsic_submit_bio_wait(bio)) { 1496 spage->io_error = 1; 1497 sblock->no_io_error_seen = 0; 1498 } 1499 1500 bio_put(bio); 1501 } 1502 1503 if (sblock->no_io_error_seen) 1504 scrub_recheck_block_checksum(sblock); 1505 } 1506 1507 static inline int scrub_check_fsid(u8 fsid[], 1508 struct scrub_page *spage) 1509 { 1510 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices; 1511 int ret; 1512 1513 ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE); 1514 return !ret; 1515 } 1516 1517 static void scrub_recheck_block_checksum(struct scrub_block *sblock) 1518 { 1519 sblock->header_error = 0; 1520 sblock->checksum_error = 0; 1521 sblock->generation_error = 0; 1522 1523 if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA) 1524 scrub_checksum_data(sblock); 1525 else 1526 scrub_checksum_tree_block(sblock); 1527 } 1528 1529 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad, 1530 struct scrub_block *sblock_good) 1531 { 1532 int page_num; 1533 int ret = 0; 1534 1535 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) { 1536 int ret_sub; 1537 1538 ret_sub = scrub_repair_page_from_good_copy(sblock_bad, 1539 sblock_good, 1540 page_num, 1); 1541 if (ret_sub) 1542 ret = ret_sub; 1543 } 1544 1545 return ret; 1546 } 1547 1548 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad, 1549 struct scrub_block *sblock_good, 1550 int page_num, int force_write) 1551 { 1552 struct scrub_page *spage_bad = sblock_bad->pagev[page_num]; 1553 struct scrub_page *spage_good = sblock_good->pagev[page_num]; 1554 struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; 1555 1556 BUG_ON(spage_bad->page == NULL); 1557 BUG_ON(spage_good->page == NULL); 1558 if (force_write || sblock_bad->header_error || 1559 sblock_bad->checksum_error || spage_bad->io_error) { 1560 struct bio *bio; 1561 int ret; 1562 1563 if (!spage_bad->dev->bdev) { 1564 btrfs_warn_rl(fs_info, 1565 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected"); 1566 return -EIO; 1567 } 1568 1569 bio = btrfs_io_bio_alloc(1); 1570 bio_set_dev(bio, spage_bad->dev->bdev); 1571 bio->bi_iter.bi_sector = spage_bad->physical >> 9; 1572 bio->bi_opf = REQ_OP_WRITE; 1573 1574 ret = bio_add_page(bio, spage_good->page, PAGE_SIZE, 0); 1575 if (PAGE_SIZE != ret) { 1576 bio_put(bio); 1577 return -EIO; 1578 } 1579 1580 if (btrfsic_submit_bio_wait(bio)) { 1581 btrfs_dev_stat_inc_and_print(spage_bad->dev, 1582 BTRFS_DEV_STAT_WRITE_ERRS); 1583 atomic64_inc(&fs_info->dev_replace.num_write_errors); 1584 bio_put(bio); 1585 return -EIO; 1586 } 1587 bio_put(bio); 1588 } 1589 1590 return 0; 1591 } 1592 1593 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock) 1594 { 1595 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; 1596 int page_num; 1597 1598 /* 1599 * This block is used for the check of the parity on the source device, 1600 * so the data needn't be written into the destination device. 1601 */ 1602 if (sblock->sparity) 1603 return; 1604 1605 for (page_num = 0; page_num < sblock->page_count; page_num++) { 1606 int ret; 1607 1608 ret = scrub_write_page_to_dev_replace(sblock, page_num); 1609 if (ret) 1610 atomic64_inc(&fs_info->dev_replace.num_write_errors); 1611 } 1612 } 1613 1614 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock, 1615 int page_num) 1616 { 1617 struct scrub_page *spage = sblock->pagev[page_num]; 1618 1619 BUG_ON(spage->page == NULL); 1620 if (spage->io_error) 1621 clear_page(page_address(spage->page)); 1622 1623 return scrub_add_page_to_wr_bio(sblock->sctx, spage); 1624 } 1625 1626 static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) 1627 { 1628 int ret = 0; 1629 u64 length; 1630 1631 if (!btrfs_is_zoned(sctx->fs_info)) 1632 return 0; 1633 1634 if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) 1635 return 0; 1636 1637 if (sctx->write_pointer < physical) { 1638 length = physical - sctx->write_pointer; 1639 1640 ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev, 1641 sctx->write_pointer, length); 1642 if (!ret) 1643 sctx->write_pointer = physical; 1644 } 1645 return ret; 1646 } 1647 1648 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, 1649 struct scrub_page *spage) 1650 { 1651 struct scrub_bio *sbio; 1652 int ret; 1653 1654 mutex_lock(&sctx->wr_lock); 1655 again: 1656 if (!sctx->wr_curr_bio) { 1657 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio), 1658 GFP_KERNEL); 1659 if (!sctx->wr_curr_bio) { 1660 mutex_unlock(&sctx->wr_lock); 1661 return -ENOMEM; 1662 } 1663 sctx->wr_curr_bio->sctx = sctx; 1664 sctx->wr_curr_bio->page_count = 0; 1665 } 1666 sbio = sctx->wr_curr_bio; 1667 if (sbio->page_count == 0) { 1668 struct bio *bio; 1669 1670 ret = fill_writer_pointer_gap(sctx, 1671 spage->physical_for_dev_replace); 1672 if (ret) { 1673 mutex_unlock(&sctx->wr_lock); 1674 return ret; 1675 } 1676 1677 sbio->physical = spage->physical_for_dev_replace; 1678 sbio->logical = spage->logical; 1679 sbio->dev = sctx->wr_tgtdev; 1680 bio = sbio->bio; 1681 if (!bio) { 1682 bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio); 1683 sbio->bio = bio; 1684 } 1685 1686 bio->bi_private = sbio; 1687 bio->bi_end_io = scrub_wr_bio_end_io; 1688 bio_set_dev(bio, sbio->dev->bdev); 1689 bio->bi_iter.bi_sector = sbio->physical >> 9; 1690 bio->bi_opf = REQ_OP_WRITE; 1691 sbio->status = 0; 1692 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != 1693 spage->physical_for_dev_replace || 1694 sbio->logical + sbio->page_count * PAGE_SIZE != 1695 spage->logical) { 1696 scrub_wr_submit(sctx); 1697 goto again; 1698 } 1699 1700 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); 1701 if (ret != PAGE_SIZE) { 1702 if (sbio->page_count < 1) { 1703 bio_put(sbio->bio); 1704 sbio->bio = NULL; 1705 mutex_unlock(&sctx->wr_lock); 1706 return -EIO; 1707 } 1708 scrub_wr_submit(sctx); 1709 goto again; 1710 } 1711 1712 sbio->pagev[sbio->page_count] = spage; 1713 scrub_page_get(spage); 1714 sbio->page_count++; 1715 if (sbio->page_count == sctx->pages_per_wr_bio) 1716 scrub_wr_submit(sctx); 1717 mutex_unlock(&sctx->wr_lock); 1718 1719 return 0; 1720 } 1721 1722 static void scrub_wr_submit(struct scrub_ctx *sctx) 1723 { 1724 struct scrub_bio *sbio; 1725 1726 if (!sctx->wr_curr_bio) 1727 return; 1728 1729 sbio = sctx->wr_curr_bio; 1730 sctx->wr_curr_bio = NULL; 1731 WARN_ON(!sbio->bio->bi_bdev); 1732 scrub_pending_bio_inc(sctx); 1733 /* process all writes in a single worker thread. Then the block layer 1734 * orders the requests before sending them to the driver which 1735 * doubled the write performance on spinning disks when measured 1736 * with Linux 3.5 */ 1737 btrfsic_submit_bio(sbio->bio); 1738 1739 if (btrfs_is_zoned(sctx->fs_info)) 1740 sctx->write_pointer = sbio->physical + sbio->page_count * PAGE_SIZE; 1741 } 1742 1743 static void scrub_wr_bio_end_io(struct bio *bio) 1744 { 1745 struct scrub_bio *sbio = bio->bi_private; 1746 struct btrfs_fs_info *fs_info = sbio->dev->fs_info; 1747 1748 sbio->status = bio->bi_status; 1749 sbio->bio = bio; 1750 1751 btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL); 1752 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work); 1753 } 1754 1755 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work) 1756 { 1757 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work); 1758 struct scrub_ctx *sctx = sbio->sctx; 1759 int i; 1760 1761 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO); 1762 if (sbio->status) { 1763 struct btrfs_dev_replace *dev_replace = 1764 &sbio->sctx->fs_info->dev_replace; 1765 1766 for (i = 0; i < sbio->page_count; i++) { 1767 struct scrub_page *spage = sbio->pagev[i]; 1768 1769 spage->io_error = 1; 1770 atomic64_inc(&dev_replace->num_write_errors); 1771 } 1772 } 1773 1774 for (i = 0; i < sbio->page_count; i++) 1775 scrub_page_put(sbio->pagev[i]); 1776 1777 bio_put(sbio->bio); 1778 kfree(sbio); 1779 scrub_pending_bio_dec(sctx); 1780 } 1781 1782 static int scrub_checksum(struct scrub_block *sblock) 1783 { 1784 u64 flags; 1785 int ret; 1786 1787 /* 1788 * No need to initialize these stats currently, 1789 * because this function only use return value 1790 * instead of these stats value. 1791 * 1792 * Todo: 1793 * always use stats 1794 */ 1795 sblock->header_error = 0; 1796 sblock->generation_error = 0; 1797 sblock->checksum_error = 0; 1798 1799 WARN_ON(sblock->page_count < 1); 1800 flags = sblock->pagev[0]->flags; 1801 ret = 0; 1802 if (flags & BTRFS_EXTENT_FLAG_DATA) 1803 ret = scrub_checksum_data(sblock); 1804 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) 1805 ret = scrub_checksum_tree_block(sblock); 1806 else if (flags & BTRFS_EXTENT_FLAG_SUPER) 1807 (void)scrub_checksum_super(sblock); 1808 else 1809 WARN_ON(1); 1810 if (ret) 1811 scrub_handle_errored_block(sblock); 1812 1813 return ret; 1814 } 1815 1816 static int scrub_checksum_data(struct scrub_block *sblock) 1817 { 1818 struct scrub_ctx *sctx = sblock->sctx; 1819 struct btrfs_fs_info *fs_info = sctx->fs_info; 1820 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 1821 u8 csum[BTRFS_CSUM_SIZE]; 1822 struct scrub_page *spage; 1823 char *kaddr; 1824 1825 BUG_ON(sblock->page_count < 1); 1826 spage = sblock->pagev[0]; 1827 if (!spage->have_csum) 1828 return 0; 1829 1830 kaddr = page_address(spage->page); 1831 1832 shash->tfm = fs_info->csum_shash; 1833 crypto_shash_init(shash); 1834 1835 /* 1836 * In scrub_pages() and scrub_pages_for_parity() we ensure each spage 1837 * only contains one sector of data. 1838 */ 1839 crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum); 1840 1841 if (memcmp(csum, spage->csum, fs_info->csum_size)) 1842 sblock->checksum_error = 1; 1843 return sblock->checksum_error; 1844 } 1845 1846 static int scrub_checksum_tree_block(struct scrub_block *sblock) 1847 { 1848 struct scrub_ctx *sctx = sblock->sctx; 1849 struct btrfs_header *h; 1850 struct btrfs_fs_info *fs_info = sctx->fs_info; 1851 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 1852 u8 calculated_csum[BTRFS_CSUM_SIZE]; 1853 u8 on_disk_csum[BTRFS_CSUM_SIZE]; 1854 /* 1855 * This is done in sectorsize steps even for metadata as there's a 1856 * constraint for nodesize to be aligned to sectorsize. This will need 1857 * to change so we don't misuse data and metadata units like that. 1858 */ 1859 const u32 sectorsize = sctx->fs_info->sectorsize; 1860 const int num_sectors = fs_info->nodesize >> fs_info->sectorsize_bits; 1861 int i; 1862 struct scrub_page *spage; 1863 char *kaddr; 1864 1865 BUG_ON(sblock->page_count < 1); 1866 1867 /* Each member in pagev is just one block, not a full page */ 1868 ASSERT(sblock->page_count == num_sectors); 1869 1870 spage = sblock->pagev[0]; 1871 kaddr = page_address(spage->page); 1872 h = (struct btrfs_header *)kaddr; 1873 memcpy(on_disk_csum, h->csum, sctx->fs_info->csum_size); 1874 1875 /* 1876 * we don't use the getter functions here, as we 1877 * a) don't have an extent buffer and 1878 * b) the page is already kmapped 1879 */ 1880 if (spage->logical != btrfs_stack_header_bytenr(h)) 1881 sblock->header_error = 1; 1882 1883 if (spage->generation != btrfs_stack_header_generation(h)) { 1884 sblock->header_error = 1; 1885 sblock->generation_error = 1; 1886 } 1887 1888 if (!scrub_check_fsid(h->fsid, spage)) 1889 sblock->header_error = 1; 1890 1891 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid, 1892 BTRFS_UUID_SIZE)) 1893 sblock->header_error = 1; 1894 1895 shash->tfm = fs_info->csum_shash; 1896 crypto_shash_init(shash); 1897 crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE, 1898 sectorsize - BTRFS_CSUM_SIZE); 1899 1900 for (i = 1; i < num_sectors; i++) { 1901 kaddr = page_address(sblock->pagev[i]->page); 1902 crypto_shash_update(shash, kaddr, sectorsize); 1903 } 1904 1905 crypto_shash_final(shash, calculated_csum); 1906 if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size)) 1907 sblock->checksum_error = 1; 1908 1909 return sblock->header_error || sblock->checksum_error; 1910 } 1911 1912 static int scrub_checksum_super(struct scrub_block *sblock) 1913 { 1914 struct btrfs_super_block *s; 1915 struct scrub_ctx *sctx = sblock->sctx; 1916 struct btrfs_fs_info *fs_info = sctx->fs_info; 1917 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 1918 u8 calculated_csum[BTRFS_CSUM_SIZE]; 1919 struct scrub_page *spage; 1920 char *kaddr; 1921 int fail_gen = 0; 1922 int fail_cor = 0; 1923 1924 BUG_ON(sblock->page_count < 1); 1925 spage = sblock->pagev[0]; 1926 kaddr = page_address(spage->page); 1927 s = (struct btrfs_super_block *)kaddr; 1928 1929 if (spage->logical != btrfs_super_bytenr(s)) 1930 ++fail_cor; 1931 1932 if (spage->generation != btrfs_super_generation(s)) 1933 ++fail_gen; 1934 1935 if (!scrub_check_fsid(s->fsid, spage)) 1936 ++fail_cor; 1937 1938 shash->tfm = fs_info->csum_shash; 1939 crypto_shash_init(shash); 1940 crypto_shash_digest(shash, kaddr + BTRFS_CSUM_SIZE, 1941 BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, calculated_csum); 1942 1943 if (memcmp(calculated_csum, s->csum, sctx->fs_info->csum_size)) 1944 ++fail_cor; 1945 1946 if (fail_cor + fail_gen) { 1947 /* 1948 * if we find an error in a super block, we just report it. 1949 * They will get written with the next transaction commit 1950 * anyway 1951 */ 1952 spin_lock(&sctx->stat_lock); 1953 ++sctx->stat.super_errors; 1954 spin_unlock(&sctx->stat_lock); 1955 if (fail_cor) 1956 btrfs_dev_stat_inc_and_print(spage->dev, 1957 BTRFS_DEV_STAT_CORRUPTION_ERRS); 1958 else 1959 btrfs_dev_stat_inc_and_print(spage->dev, 1960 BTRFS_DEV_STAT_GENERATION_ERRS); 1961 } 1962 1963 return fail_cor + fail_gen; 1964 } 1965 1966 static void scrub_block_get(struct scrub_block *sblock) 1967 { 1968 refcount_inc(&sblock->refs); 1969 } 1970 1971 static void scrub_block_put(struct scrub_block *sblock) 1972 { 1973 if (refcount_dec_and_test(&sblock->refs)) { 1974 int i; 1975 1976 if (sblock->sparity) 1977 scrub_parity_put(sblock->sparity); 1978 1979 for (i = 0; i < sblock->page_count; i++) 1980 scrub_page_put(sblock->pagev[i]); 1981 kfree(sblock); 1982 } 1983 } 1984 1985 static void scrub_page_get(struct scrub_page *spage) 1986 { 1987 atomic_inc(&spage->refs); 1988 } 1989 1990 static void scrub_page_put(struct scrub_page *spage) 1991 { 1992 if (atomic_dec_and_test(&spage->refs)) { 1993 if (spage->page) 1994 __free_page(spage->page); 1995 kfree(spage); 1996 } 1997 } 1998 1999 static void scrub_submit(struct scrub_ctx *sctx) 2000 { 2001 struct scrub_bio *sbio; 2002 2003 if (sctx->curr == -1) 2004 return; 2005 2006 sbio = sctx->bios[sctx->curr]; 2007 sctx->curr = -1; 2008 scrub_pending_bio_inc(sctx); 2009 btrfsic_submit_bio(sbio->bio); 2010 } 2011 2012 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, 2013 struct scrub_page *spage) 2014 { 2015 struct scrub_block *sblock = spage->sblock; 2016 struct scrub_bio *sbio; 2017 int ret; 2018 2019 again: 2020 /* 2021 * grab a fresh bio or wait for one to become available 2022 */ 2023 while (sctx->curr == -1) { 2024 spin_lock(&sctx->list_lock); 2025 sctx->curr = sctx->first_free; 2026 if (sctx->curr != -1) { 2027 sctx->first_free = sctx->bios[sctx->curr]->next_free; 2028 sctx->bios[sctx->curr]->next_free = -1; 2029 sctx->bios[sctx->curr]->page_count = 0; 2030 spin_unlock(&sctx->list_lock); 2031 } else { 2032 spin_unlock(&sctx->list_lock); 2033 wait_event(sctx->list_wait, sctx->first_free != -1); 2034 } 2035 } 2036 sbio = sctx->bios[sctx->curr]; 2037 if (sbio->page_count == 0) { 2038 struct bio *bio; 2039 2040 sbio->physical = spage->physical; 2041 sbio->logical = spage->logical; 2042 sbio->dev = spage->dev; 2043 bio = sbio->bio; 2044 if (!bio) { 2045 bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio); 2046 sbio->bio = bio; 2047 } 2048 2049 bio->bi_private = sbio; 2050 bio->bi_end_io = scrub_bio_end_io; 2051 bio_set_dev(bio, sbio->dev->bdev); 2052 bio->bi_iter.bi_sector = sbio->physical >> 9; 2053 bio->bi_opf = REQ_OP_READ; 2054 sbio->status = 0; 2055 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != 2056 spage->physical || 2057 sbio->logical + sbio->page_count * PAGE_SIZE != 2058 spage->logical || 2059 sbio->dev != spage->dev) { 2060 scrub_submit(sctx); 2061 goto again; 2062 } 2063 2064 sbio->pagev[sbio->page_count] = spage; 2065 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); 2066 if (ret != PAGE_SIZE) { 2067 if (sbio->page_count < 1) { 2068 bio_put(sbio->bio); 2069 sbio->bio = NULL; 2070 return -EIO; 2071 } 2072 scrub_submit(sctx); 2073 goto again; 2074 } 2075 2076 scrub_block_get(sblock); /* one for the page added to the bio */ 2077 atomic_inc(&sblock->outstanding_pages); 2078 sbio->page_count++; 2079 if (sbio->page_count == sctx->pages_per_rd_bio) 2080 scrub_submit(sctx); 2081 2082 return 0; 2083 } 2084 2085 static void scrub_missing_raid56_end_io(struct bio *bio) 2086 { 2087 struct scrub_block *sblock = bio->bi_private; 2088 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; 2089 2090 if (bio->bi_status) 2091 sblock->no_io_error_seen = 0; 2092 2093 bio_put(bio); 2094 2095 btrfs_queue_work(fs_info->scrub_workers, &sblock->work); 2096 } 2097 2098 static void scrub_missing_raid56_worker(struct btrfs_work *work) 2099 { 2100 struct scrub_block *sblock = container_of(work, struct scrub_block, work); 2101 struct scrub_ctx *sctx = sblock->sctx; 2102 struct btrfs_fs_info *fs_info = sctx->fs_info; 2103 u64 logical; 2104 struct btrfs_device *dev; 2105 2106 logical = sblock->pagev[0]->logical; 2107 dev = sblock->pagev[0]->dev; 2108 2109 if (sblock->no_io_error_seen) 2110 scrub_recheck_block_checksum(sblock); 2111 2112 if (!sblock->no_io_error_seen) { 2113 spin_lock(&sctx->stat_lock); 2114 sctx->stat.read_errors++; 2115 spin_unlock(&sctx->stat_lock); 2116 btrfs_err_rl_in_rcu(fs_info, 2117 "IO error rebuilding logical %llu for dev %s", 2118 logical, rcu_str_deref(dev->name)); 2119 } else if (sblock->header_error || sblock->checksum_error) { 2120 spin_lock(&sctx->stat_lock); 2121 sctx->stat.uncorrectable_errors++; 2122 spin_unlock(&sctx->stat_lock); 2123 btrfs_err_rl_in_rcu(fs_info, 2124 "failed to rebuild valid logical %llu for dev %s", 2125 logical, rcu_str_deref(dev->name)); 2126 } else { 2127 scrub_write_block_to_dev_replace(sblock); 2128 } 2129 2130 if (sctx->is_dev_replace && sctx->flush_all_writes) { 2131 mutex_lock(&sctx->wr_lock); 2132 scrub_wr_submit(sctx); 2133 mutex_unlock(&sctx->wr_lock); 2134 } 2135 2136 scrub_block_put(sblock); 2137 scrub_pending_bio_dec(sctx); 2138 } 2139 2140 static void scrub_missing_raid56_pages(struct scrub_block *sblock) 2141 { 2142 struct scrub_ctx *sctx = sblock->sctx; 2143 struct btrfs_fs_info *fs_info = sctx->fs_info; 2144 u64 length = sblock->page_count * PAGE_SIZE; 2145 u64 logical = sblock->pagev[0]->logical; 2146 struct btrfs_bio *bbio = NULL; 2147 struct bio *bio; 2148 struct btrfs_raid_bio *rbio; 2149 int ret; 2150 int i; 2151 2152 btrfs_bio_counter_inc_blocked(fs_info); 2153 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, 2154 &length, &bbio); 2155 if (ret || !bbio || !bbio->raid_map) 2156 goto bbio_out; 2157 2158 if (WARN_ON(!sctx->is_dev_replace || 2159 !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) { 2160 /* 2161 * We shouldn't be scrubbing a missing device. Even for dev 2162 * replace, we should only get here for RAID 5/6. We either 2163 * managed to mount something with no mirrors remaining or 2164 * there's a bug in scrub_remap_extent()/btrfs_map_block(). 2165 */ 2166 goto bbio_out; 2167 } 2168 2169 bio = btrfs_io_bio_alloc(0); 2170 bio->bi_iter.bi_sector = logical >> 9; 2171 bio->bi_private = sblock; 2172 bio->bi_end_io = scrub_missing_raid56_end_io; 2173 2174 rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length); 2175 if (!rbio) 2176 goto rbio_out; 2177 2178 for (i = 0; i < sblock->page_count; i++) { 2179 struct scrub_page *spage = sblock->pagev[i]; 2180 2181 raid56_add_scrub_pages(rbio, spage->page, spage->logical); 2182 } 2183 2184 btrfs_init_work(&sblock->work, scrub_missing_raid56_worker, NULL, NULL); 2185 scrub_block_get(sblock); 2186 scrub_pending_bio_inc(sctx); 2187 raid56_submit_missing_rbio(rbio); 2188 return; 2189 2190 rbio_out: 2191 bio_put(bio); 2192 bbio_out: 2193 btrfs_bio_counter_dec(fs_info); 2194 btrfs_put_bbio(bbio); 2195 spin_lock(&sctx->stat_lock); 2196 sctx->stat.malloc_errors++; 2197 spin_unlock(&sctx->stat_lock); 2198 } 2199 2200 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len, 2201 u64 physical, struct btrfs_device *dev, u64 flags, 2202 u64 gen, int mirror_num, u8 *csum, 2203 u64 physical_for_dev_replace) 2204 { 2205 struct scrub_block *sblock; 2206 const u32 sectorsize = sctx->fs_info->sectorsize; 2207 int index; 2208 2209 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); 2210 if (!sblock) { 2211 spin_lock(&sctx->stat_lock); 2212 sctx->stat.malloc_errors++; 2213 spin_unlock(&sctx->stat_lock); 2214 return -ENOMEM; 2215 } 2216 2217 /* one ref inside this function, plus one for each page added to 2218 * a bio later on */ 2219 refcount_set(&sblock->refs, 1); 2220 sblock->sctx = sctx; 2221 sblock->no_io_error_seen = 1; 2222 2223 for (index = 0; len > 0; index++) { 2224 struct scrub_page *spage; 2225 /* 2226 * Here we will allocate one page for one sector to scrub. 2227 * This is fine if PAGE_SIZE == sectorsize, but will cost 2228 * more memory for PAGE_SIZE > sectorsize case. 2229 */ 2230 u32 l = min(sectorsize, len); 2231 2232 spage = kzalloc(sizeof(*spage), GFP_KERNEL); 2233 if (!spage) { 2234 leave_nomem: 2235 spin_lock(&sctx->stat_lock); 2236 sctx->stat.malloc_errors++; 2237 spin_unlock(&sctx->stat_lock); 2238 scrub_block_put(sblock); 2239 return -ENOMEM; 2240 } 2241 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK); 2242 scrub_page_get(spage); 2243 sblock->pagev[index] = spage; 2244 spage->sblock = sblock; 2245 spage->dev = dev; 2246 spage->flags = flags; 2247 spage->generation = gen; 2248 spage->logical = logical; 2249 spage->physical = physical; 2250 spage->physical_for_dev_replace = physical_for_dev_replace; 2251 spage->mirror_num = mirror_num; 2252 if (csum) { 2253 spage->have_csum = 1; 2254 memcpy(spage->csum, csum, sctx->fs_info->csum_size); 2255 } else { 2256 spage->have_csum = 0; 2257 } 2258 sblock->page_count++; 2259 spage->page = alloc_page(GFP_KERNEL); 2260 if (!spage->page) 2261 goto leave_nomem; 2262 len -= l; 2263 logical += l; 2264 physical += l; 2265 physical_for_dev_replace += l; 2266 } 2267 2268 WARN_ON(sblock->page_count == 0); 2269 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) { 2270 /* 2271 * This case should only be hit for RAID 5/6 device replace. See 2272 * the comment in scrub_missing_raid56_pages() for details. 2273 */ 2274 scrub_missing_raid56_pages(sblock); 2275 } else { 2276 for (index = 0; index < sblock->page_count; index++) { 2277 struct scrub_page *spage = sblock->pagev[index]; 2278 int ret; 2279 2280 ret = scrub_add_page_to_rd_bio(sctx, spage); 2281 if (ret) { 2282 scrub_block_put(sblock); 2283 return ret; 2284 } 2285 } 2286 2287 if (flags & BTRFS_EXTENT_FLAG_SUPER) 2288 scrub_submit(sctx); 2289 } 2290 2291 /* last one frees, either here or in bio completion for last page */ 2292 scrub_block_put(sblock); 2293 return 0; 2294 } 2295 2296 static void scrub_bio_end_io(struct bio *bio) 2297 { 2298 struct scrub_bio *sbio = bio->bi_private; 2299 struct btrfs_fs_info *fs_info = sbio->dev->fs_info; 2300 2301 sbio->status = bio->bi_status; 2302 sbio->bio = bio; 2303 2304 btrfs_queue_work(fs_info->scrub_workers, &sbio->work); 2305 } 2306 2307 static void scrub_bio_end_io_worker(struct btrfs_work *work) 2308 { 2309 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work); 2310 struct scrub_ctx *sctx = sbio->sctx; 2311 int i; 2312 2313 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO); 2314 if (sbio->status) { 2315 for (i = 0; i < sbio->page_count; i++) { 2316 struct scrub_page *spage = sbio->pagev[i]; 2317 2318 spage->io_error = 1; 2319 spage->sblock->no_io_error_seen = 0; 2320 } 2321 } 2322 2323 /* now complete the scrub_block items that have all pages completed */ 2324 for (i = 0; i < sbio->page_count; i++) { 2325 struct scrub_page *spage = sbio->pagev[i]; 2326 struct scrub_block *sblock = spage->sblock; 2327 2328 if (atomic_dec_and_test(&sblock->outstanding_pages)) 2329 scrub_block_complete(sblock); 2330 scrub_block_put(sblock); 2331 } 2332 2333 bio_put(sbio->bio); 2334 sbio->bio = NULL; 2335 spin_lock(&sctx->list_lock); 2336 sbio->next_free = sctx->first_free; 2337 sctx->first_free = sbio->index; 2338 spin_unlock(&sctx->list_lock); 2339 2340 if (sctx->is_dev_replace && sctx->flush_all_writes) { 2341 mutex_lock(&sctx->wr_lock); 2342 scrub_wr_submit(sctx); 2343 mutex_unlock(&sctx->wr_lock); 2344 } 2345 2346 scrub_pending_bio_dec(sctx); 2347 } 2348 2349 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity, 2350 unsigned long *bitmap, 2351 u64 start, u32 len) 2352 { 2353 u64 offset; 2354 u32 nsectors; 2355 u32 sectorsize_bits = sparity->sctx->fs_info->sectorsize_bits; 2356 2357 if (len >= sparity->stripe_len) { 2358 bitmap_set(bitmap, 0, sparity->nsectors); 2359 return; 2360 } 2361 2362 start -= sparity->logic_start; 2363 start = div64_u64_rem(start, sparity->stripe_len, &offset); 2364 offset = offset >> sectorsize_bits; 2365 nsectors = len >> sectorsize_bits; 2366 2367 if (offset + nsectors <= sparity->nsectors) { 2368 bitmap_set(bitmap, offset, nsectors); 2369 return; 2370 } 2371 2372 bitmap_set(bitmap, offset, sparity->nsectors - offset); 2373 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset)); 2374 } 2375 2376 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity, 2377 u64 start, u32 len) 2378 { 2379 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len); 2380 } 2381 2382 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity, 2383 u64 start, u32 len) 2384 { 2385 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len); 2386 } 2387 2388 static void scrub_block_complete(struct scrub_block *sblock) 2389 { 2390 int corrupted = 0; 2391 2392 if (!sblock->no_io_error_seen) { 2393 corrupted = 1; 2394 scrub_handle_errored_block(sblock); 2395 } else { 2396 /* 2397 * if has checksum error, write via repair mechanism in 2398 * dev replace case, otherwise write here in dev replace 2399 * case. 2400 */ 2401 corrupted = scrub_checksum(sblock); 2402 if (!corrupted && sblock->sctx->is_dev_replace) 2403 scrub_write_block_to_dev_replace(sblock); 2404 } 2405 2406 if (sblock->sparity && corrupted && !sblock->data_corrected) { 2407 u64 start = sblock->pagev[0]->logical; 2408 u64 end = sblock->pagev[sblock->page_count - 1]->logical + 2409 PAGE_SIZE; 2410 2411 ASSERT(end - start <= U32_MAX); 2412 scrub_parity_mark_sectors_error(sblock->sparity, 2413 start, end - start); 2414 } 2415 } 2416 2417 static void drop_csum_range(struct scrub_ctx *sctx, struct btrfs_ordered_sum *sum) 2418 { 2419 sctx->stat.csum_discards += sum->len >> sctx->fs_info->sectorsize_bits; 2420 list_del(&sum->list); 2421 kfree(sum); 2422 } 2423 2424 /* 2425 * Find the desired csum for range [logical, logical + sectorsize), and store 2426 * the csum into @csum. 2427 * 2428 * The search source is sctx->csum_list, which is a pre-populated list 2429 * storing bytenr ordered csum ranges. We're reponsible to cleanup any range 2430 * that is before @logical. 2431 * 2432 * Return 0 if there is no csum for the range. 2433 * Return 1 if there is csum for the range and copied to @csum. 2434 */ 2435 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum) 2436 { 2437 bool found = false; 2438 2439 while (!list_empty(&sctx->csum_list)) { 2440 struct btrfs_ordered_sum *sum = NULL; 2441 unsigned long index; 2442 unsigned long num_sectors; 2443 2444 sum = list_first_entry(&sctx->csum_list, 2445 struct btrfs_ordered_sum, list); 2446 /* The current csum range is beyond our range, no csum found */ 2447 if (sum->bytenr > logical) 2448 break; 2449 2450 /* 2451 * The current sum is before our bytenr, since scrub is always 2452 * done in bytenr order, the csum will never be used anymore, 2453 * clean it up so that later calls won't bother with the range, 2454 * and continue search the next range. 2455 */ 2456 if (sum->bytenr + sum->len <= logical) { 2457 drop_csum_range(sctx, sum); 2458 continue; 2459 } 2460 2461 /* Now the csum range covers our bytenr, copy the csum */ 2462 found = true; 2463 index = (logical - sum->bytenr) >> sctx->fs_info->sectorsize_bits; 2464 num_sectors = sum->len >> sctx->fs_info->sectorsize_bits; 2465 2466 memcpy(csum, sum->sums + index * sctx->fs_info->csum_size, 2467 sctx->fs_info->csum_size); 2468 2469 /* Cleanup the range if we're at the end of the csum range */ 2470 if (index == num_sectors - 1) 2471 drop_csum_range(sctx, sum); 2472 break; 2473 } 2474 if (!found) 2475 return 0; 2476 return 1; 2477 } 2478 2479 /* scrub extent tries to collect up to 64 kB for each bio */ 2480 static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map, 2481 u64 logical, u32 len, 2482 u64 physical, struct btrfs_device *dev, u64 flags, 2483 u64 gen, int mirror_num, u64 physical_for_dev_replace) 2484 { 2485 int ret; 2486 u8 csum[BTRFS_CSUM_SIZE]; 2487 u32 blocksize; 2488 2489 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2490 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) 2491 blocksize = map->stripe_len; 2492 else 2493 blocksize = sctx->fs_info->sectorsize; 2494 spin_lock(&sctx->stat_lock); 2495 sctx->stat.data_extents_scrubbed++; 2496 sctx->stat.data_bytes_scrubbed += len; 2497 spin_unlock(&sctx->stat_lock); 2498 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 2499 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) 2500 blocksize = map->stripe_len; 2501 else 2502 blocksize = sctx->fs_info->nodesize; 2503 spin_lock(&sctx->stat_lock); 2504 sctx->stat.tree_extents_scrubbed++; 2505 sctx->stat.tree_bytes_scrubbed += len; 2506 spin_unlock(&sctx->stat_lock); 2507 } else { 2508 blocksize = sctx->fs_info->sectorsize; 2509 WARN_ON(1); 2510 } 2511 2512 while (len) { 2513 u32 l = min(len, blocksize); 2514 int have_csum = 0; 2515 2516 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2517 /* push csums to sbio */ 2518 have_csum = scrub_find_csum(sctx, logical, csum); 2519 if (have_csum == 0) 2520 ++sctx->stat.no_csum; 2521 } 2522 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen, 2523 mirror_num, have_csum ? csum : NULL, 2524 physical_for_dev_replace); 2525 if (ret) 2526 return ret; 2527 len -= l; 2528 logical += l; 2529 physical += l; 2530 physical_for_dev_replace += l; 2531 } 2532 return 0; 2533 } 2534 2535 static int scrub_pages_for_parity(struct scrub_parity *sparity, 2536 u64 logical, u32 len, 2537 u64 physical, struct btrfs_device *dev, 2538 u64 flags, u64 gen, int mirror_num, u8 *csum) 2539 { 2540 struct scrub_ctx *sctx = sparity->sctx; 2541 struct scrub_block *sblock; 2542 const u32 sectorsize = sctx->fs_info->sectorsize; 2543 int index; 2544 2545 ASSERT(IS_ALIGNED(len, sectorsize)); 2546 2547 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); 2548 if (!sblock) { 2549 spin_lock(&sctx->stat_lock); 2550 sctx->stat.malloc_errors++; 2551 spin_unlock(&sctx->stat_lock); 2552 return -ENOMEM; 2553 } 2554 2555 /* one ref inside this function, plus one for each page added to 2556 * a bio later on */ 2557 refcount_set(&sblock->refs, 1); 2558 sblock->sctx = sctx; 2559 sblock->no_io_error_seen = 1; 2560 sblock->sparity = sparity; 2561 scrub_parity_get(sparity); 2562 2563 for (index = 0; len > 0; index++) { 2564 struct scrub_page *spage; 2565 2566 spage = kzalloc(sizeof(*spage), GFP_KERNEL); 2567 if (!spage) { 2568 leave_nomem: 2569 spin_lock(&sctx->stat_lock); 2570 sctx->stat.malloc_errors++; 2571 spin_unlock(&sctx->stat_lock); 2572 scrub_block_put(sblock); 2573 return -ENOMEM; 2574 } 2575 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK); 2576 /* For scrub block */ 2577 scrub_page_get(spage); 2578 sblock->pagev[index] = spage; 2579 /* For scrub parity */ 2580 scrub_page_get(spage); 2581 list_add_tail(&spage->list, &sparity->spages); 2582 spage->sblock = sblock; 2583 spage->dev = dev; 2584 spage->flags = flags; 2585 spage->generation = gen; 2586 spage->logical = logical; 2587 spage->physical = physical; 2588 spage->mirror_num = mirror_num; 2589 if (csum) { 2590 spage->have_csum = 1; 2591 memcpy(spage->csum, csum, sctx->fs_info->csum_size); 2592 } else { 2593 spage->have_csum = 0; 2594 } 2595 sblock->page_count++; 2596 spage->page = alloc_page(GFP_KERNEL); 2597 if (!spage->page) 2598 goto leave_nomem; 2599 2600 2601 /* Iterate over the stripe range in sectorsize steps */ 2602 len -= sectorsize; 2603 logical += sectorsize; 2604 physical += sectorsize; 2605 } 2606 2607 WARN_ON(sblock->page_count == 0); 2608 for (index = 0; index < sblock->page_count; index++) { 2609 struct scrub_page *spage = sblock->pagev[index]; 2610 int ret; 2611 2612 ret = scrub_add_page_to_rd_bio(sctx, spage); 2613 if (ret) { 2614 scrub_block_put(sblock); 2615 return ret; 2616 } 2617 } 2618 2619 /* last one frees, either here or in bio completion for last page */ 2620 scrub_block_put(sblock); 2621 return 0; 2622 } 2623 2624 static int scrub_extent_for_parity(struct scrub_parity *sparity, 2625 u64 logical, u32 len, 2626 u64 physical, struct btrfs_device *dev, 2627 u64 flags, u64 gen, int mirror_num) 2628 { 2629 struct scrub_ctx *sctx = sparity->sctx; 2630 int ret; 2631 u8 csum[BTRFS_CSUM_SIZE]; 2632 u32 blocksize; 2633 2634 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) { 2635 scrub_parity_mark_sectors_error(sparity, logical, len); 2636 return 0; 2637 } 2638 2639 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2640 blocksize = sparity->stripe_len; 2641 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 2642 blocksize = sparity->stripe_len; 2643 } else { 2644 blocksize = sctx->fs_info->sectorsize; 2645 WARN_ON(1); 2646 } 2647 2648 while (len) { 2649 u32 l = min(len, blocksize); 2650 int have_csum = 0; 2651 2652 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2653 /* push csums to sbio */ 2654 have_csum = scrub_find_csum(sctx, logical, csum); 2655 if (have_csum == 0) 2656 goto skip; 2657 } 2658 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev, 2659 flags, gen, mirror_num, 2660 have_csum ? csum : NULL); 2661 if (ret) 2662 return ret; 2663 skip: 2664 len -= l; 2665 logical += l; 2666 physical += l; 2667 } 2668 return 0; 2669 } 2670 2671 /* 2672 * Given a physical address, this will calculate it's 2673 * logical offset. if this is a parity stripe, it will return 2674 * the most left data stripe's logical offset. 2675 * 2676 * return 0 if it is a data stripe, 1 means parity stripe. 2677 */ 2678 static int get_raid56_logic_offset(u64 physical, int num, 2679 struct map_lookup *map, u64 *offset, 2680 u64 *stripe_start) 2681 { 2682 int i; 2683 int j = 0; 2684 u64 stripe_nr; 2685 u64 last_offset; 2686 u32 stripe_index; 2687 u32 rot; 2688 const int data_stripes = nr_data_stripes(map); 2689 2690 last_offset = (physical - map->stripes[num].physical) * data_stripes; 2691 if (stripe_start) 2692 *stripe_start = last_offset; 2693 2694 *offset = last_offset; 2695 for (i = 0; i < data_stripes; i++) { 2696 *offset = last_offset + i * map->stripe_len; 2697 2698 stripe_nr = div64_u64(*offset, map->stripe_len); 2699 stripe_nr = div_u64(stripe_nr, data_stripes); 2700 2701 /* Work out the disk rotation on this stripe-set */ 2702 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot); 2703 /* calculate which stripe this data locates */ 2704 rot += i; 2705 stripe_index = rot % map->num_stripes; 2706 if (stripe_index == num) 2707 return 0; 2708 if (stripe_index < num) 2709 j++; 2710 } 2711 *offset = last_offset + j * map->stripe_len; 2712 return 1; 2713 } 2714 2715 static void scrub_free_parity(struct scrub_parity *sparity) 2716 { 2717 struct scrub_ctx *sctx = sparity->sctx; 2718 struct scrub_page *curr, *next; 2719 int nbits; 2720 2721 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors); 2722 if (nbits) { 2723 spin_lock(&sctx->stat_lock); 2724 sctx->stat.read_errors += nbits; 2725 sctx->stat.uncorrectable_errors += nbits; 2726 spin_unlock(&sctx->stat_lock); 2727 } 2728 2729 list_for_each_entry_safe(curr, next, &sparity->spages, list) { 2730 list_del_init(&curr->list); 2731 scrub_page_put(curr); 2732 } 2733 2734 kfree(sparity); 2735 } 2736 2737 static void scrub_parity_bio_endio_worker(struct btrfs_work *work) 2738 { 2739 struct scrub_parity *sparity = container_of(work, struct scrub_parity, 2740 work); 2741 struct scrub_ctx *sctx = sparity->sctx; 2742 2743 scrub_free_parity(sparity); 2744 scrub_pending_bio_dec(sctx); 2745 } 2746 2747 static void scrub_parity_bio_endio(struct bio *bio) 2748 { 2749 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private; 2750 struct btrfs_fs_info *fs_info = sparity->sctx->fs_info; 2751 2752 if (bio->bi_status) 2753 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, 2754 sparity->nsectors); 2755 2756 bio_put(bio); 2757 2758 btrfs_init_work(&sparity->work, scrub_parity_bio_endio_worker, NULL, 2759 NULL); 2760 btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work); 2761 } 2762 2763 static void scrub_parity_check_and_repair(struct scrub_parity *sparity) 2764 { 2765 struct scrub_ctx *sctx = sparity->sctx; 2766 struct btrfs_fs_info *fs_info = sctx->fs_info; 2767 struct bio *bio; 2768 struct btrfs_raid_bio *rbio; 2769 struct btrfs_bio *bbio = NULL; 2770 u64 length; 2771 int ret; 2772 2773 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap, 2774 sparity->nsectors)) 2775 goto out; 2776 2777 length = sparity->logic_end - sparity->logic_start; 2778 2779 btrfs_bio_counter_inc_blocked(fs_info); 2780 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start, 2781 &length, &bbio); 2782 if (ret || !bbio || !bbio->raid_map) 2783 goto bbio_out; 2784 2785 bio = btrfs_io_bio_alloc(0); 2786 bio->bi_iter.bi_sector = sparity->logic_start >> 9; 2787 bio->bi_private = sparity; 2788 bio->bi_end_io = scrub_parity_bio_endio; 2789 2790 rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio, 2791 length, sparity->scrub_dev, 2792 sparity->dbitmap, 2793 sparity->nsectors); 2794 if (!rbio) 2795 goto rbio_out; 2796 2797 scrub_pending_bio_inc(sctx); 2798 raid56_parity_submit_scrub_rbio(rbio); 2799 return; 2800 2801 rbio_out: 2802 bio_put(bio); 2803 bbio_out: 2804 btrfs_bio_counter_dec(fs_info); 2805 btrfs_put_bbio(bbio); 2806 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, 2807 sparity->nsectors); 2808 spin_lock(&sctx->stat_lock); 2809 sctx->stat.malloc_errors++; 2810 spin_unlock(&sctx->stat_lock); 2811 out: 2812 scrub_free_parity(sparity); 2813 } 2814 2815 static inline int scrub_calc_parity_bitmap_len(int nsectors) 2816 { 2817 return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long); 2818 } 2819 2820 static void scrub_parity_get(struct scrub_parity *sparity) 2821 { 2822 refcount_inc(&sparity->refs); 2823 } 2824 2825 static void scrub_parity_put(struct scrub_parity *sparity) 2826 { 2827 if (!refcount_dec_and_test(&sparity->refs)) 2828 return; 2829 2830 scrub_parity_check_and_repair(sparity); 2831 } 2832 2833 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, 2834 struct map_lookup *map, 2835 struct btrfs_device *sdev, 2836 struct btrfs_path *path, 2837 u64 logic_start, 2838 u64 logic_end) 2839 { 2840 struct btrfs_fs_info *fs_info = sctx->fs_info; 2841 struct btrfs_root *root = fs_info->extent_root; 2842 struct btrfs_root *csum_root = fs_info->csum_root; 2843 struct btrfs_extent_item *extent; 2844 struct btrfs_bio *bbio = NULL; 2845 u64 flags; 2846 int ret; 2847 int slot; 2848 struct extent_buffer *l; 2849 struct btrfs_key key; 2850 u64 generation; 2851 u64 extent_logical; 2852 u64 extent_physical; 2853 /* Check the comment in scrub_stripe() for why u32 is enough here */ 2854 u32 extent_len; 2855 u64 mapped_length; 2856 struct btrfs_device *extent_dev; 2857 struct scrub_parity *sparity; 2858 int nsectors; 2859 int bitmap_len; 2860 int extent_mirror_num; 2861 int stop_loop = 0; 2862 2863 ASSERT(map->stripe_len <= U32_MAX); 2864 nsectors = map->stripe_len >> fs_info->sectorsize_bits; 2865 bitmap_len = scrub_calc_parity_bitmap_len(nsectors); 2866 sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len, 2867 GFP_NOFS); 2868 if (!sparity) { 2869 spin_lock(&sctx->stat_lock); 2870 sctx->stat.malloc_errors++; 2871 spin_unlock(&sctx->stat_lock); 2872 return -ENOMEM; 2873 } 2874 2875 ASSERT(map->stripe_len <= U32_MAX); 2876 sparity->stripe_len = map->stripe_len; 2877 sparity->nsectors = nsectors; 2878 sparity->sctx = sctx; 2879 sparity->scrub_dev = sdev; 2880 sparity->logic_start = logic_start; 2881 sparity->logic_end = logic_end; 2882 refcount_set(&sparity->refs, 1); 2883 INIT_LIST_HEAD(&sparity->spages); 2884 sparity->dbitmap = sparity->bitmap; 2885 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len; 2886 2887 ret = 0; 2888 while (logic_start < logic_end) { 2889 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) 2890 key.type = BTRFS_METADATA_ITEM_KEY; 2891 else 2892 key.type = BTRFS_EXTENT_ITEM_KEY; 2893 key.objectid = logic_start; 2894 key.offset = (u64)-1; 2895 2896 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2897 if (ret < 0) 2898 goto out; 2899 2900 if (ret > 0) { 2901 ret = btrfs_previous_extent_item(root, path, 0); 2902 if (ret < 0) 2903 goto out; 2904 if (ret > 0) { 2905 btrfs_release_path(path); 2906 ret = btrfs_search_slot(NULL, root, &key, 2907 path, 0, 0); 2908 if (ret < 0) 2909 goto out; 2910 } 2911 } 2912 2913 stop_loop = 0; 2914 while (1) { 2915 u64 bytes; 2916 2917 l = path->nodes[0]; 2918 slot = path->slots[0]; 2919 if (slot >= btrfs_header_nritems(l)) { 2920 ret = btrfs_next_leaf(root, path); 2921 if (ret == 0) 2922 continue; 2923 if (ret < 0) 2924 goto out; 2925 2926 stop_loop = 1; 2927 break; 2928 } 2929 btrfs_item_key_to_cpu(l, &key, slot); 2930 2931 if (key.type != BTRFS_EXTENT_ITEM_KEY && 2932 key.type != BTRFS_METADATA_ITEM_KEY) 2933 goto next; 2934 2935 if (key.type == BTRFS_METADATA_ITEM_KEY) 2936 bytes = fs_info->nodesize; 2937 else 2938 bytes = key.offset; 2939 2940 if (key.objectid + bytes <= logic_start) 2941 goto next; 2942 2943 if (key.objectid >= logic_end) { 2944 stop_loop = 1; 2945 break; 2946 } 2947 2948 while (key.objectid >= logic_start + map->stripe_len) 2949 logic_start += map->stripe_len; 2950 2951 extent = btrfs_item_ptr(l, slot, 2952 struct btrfs_extent_item); 2953 flags = btrfs_extent_flags(l, extent); 2954 generation = btrfs_extent_generation(l, extent); 2955 2956 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) && 2957 (key.objectid < logic_start || 2958 key.objectid + bytes > 2959 logic_start + map->stripe_len)) { 2960 btrfs_err(fs_info, 2961 "scrub: tree block %llu spanning stripes, ignored. logical=%llu", 2962 key.objectid, logic_start); 2963 spin_lock(&sctx->stat_lock); 2964 sctx->stat.uncorrectable_errors++; 2965 spin_unlock(&sctx->stat_lock); 2966 goto next; 2967 } 2968 again: 2969 extent_logical = key.objectid; 2970 ASSERT(bytes <= U32_MAX); 2971 extent_len = bytes; 2972 2973 if (extent_logical < logic_start) { 2974 extent_len -= logic_start - extent_logical; 2975 extent_logical = logic_start; 2976 } 2977 2978 if (extent_logical + extent_len > 2979 logic_start + map->stripe_len) 2980 extent_len = logic_start + map->stripe_len - 2981 extent_logical; 2982 2983 scrub_parity_mark_sectors_data(sparity, extent_logical, 2984 extent_len); 2985 2986 mapped_length = extent_len; 2987 bbio = NULL; 2988 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, 2989 extent_logical, &mapped_length, &bbio, 2990 0); 2991 if (!ret) { 2992 if (!bbio || mapped_length < extent_len) 2993 ret = -EIO; 2994 } 2995 if (ret) { 2996 btrfs_put_bbio(bbio); 2997 goto out; 2998 } 2999 extent_physical = bbio->stripes[0].physical; 3000 extent_mirror_num = bbio->mirror_num; 3001 extent_dev = bbio->stripes[0].dev; 3002 btrfs_put_bbio(bbio); 3003 3004 ret = btrfs_lookup_csums_range(csum_root, 3005 extent_logical, 3006 extent_logical + extent_len - 1, 3007 &sctx->csum_list, 1); 3008 if (ret) 3009 goto out; 3010 3011 ret = scrub_extent_for_parity(sparity, extent_logical, 3012 extent_len, 3013 extent_physical, 3014 extent_dev, flags, 3015 generation, 3016 extent_mirror_num); 3017 3018 scrub_free_csums(sctx); 3019 3020 if (ret) 3021 goto out; 3022 3023 if (extent_logical + extent_len < 3024 key.objectid + bytes) { 3025 logic_start += map->stripe_len; 3026 3027 if (logic_start >= logic_end) { 3028 stop_loop = 1; 3029 break; 3030 } 3031 3032 if (logic_start < key.objectid + bytes) { 3033 cond_resched(); 3034 goto again; 3035 } 3036 } 3037 next: 3038 path->slots[0]++; 3039 } 3040 3041 btrfs_release_path(path); 3042 3043 if (stop_loop) 3044 break; 3045 3046 logic_start += map->stripe_len; 3047 } 3048 out: 3049 if (ret < 0) { 3050 ASSERT(logic_end - logic_start <= U32_MAX); 3051 scrub_parity_mark_sectors_error(sparity, logic_start, 3052 logic_end - logic_start); 3053 } 3054 scrub_parity_put(sparity); 3055 scrub_submit(sctx); 3056 mutex_lock(&sctx->wr_lock); 3057 scrub_wr_submit(sctx); 3058 mutex_unlock(&sctx->wr_lock); 3059 3060 btrfs_release_path(path); 3061 return ret < 0 ? ret : 0; 3062 } 3063 3064 static void sync_replace_for_zoned(struct scrub_ctx *sctx) 3065 { 3066 if (!btrfs_is_zoned(sctx->fs_info)) 3067 return; 3068 3069 sctx->flush_all_writes = true; 3070 scrub_submit(sctx); 3071 mutex_lock(&sctx->wr_lock); 3072 scrub_wr_submit(sctx); 3073 mutex_unlock(&sctx->wr_lock); 3074 3075 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); 3076 } 3077 3078 static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical, 3079 u64 physical, u64 physical_end) 3080 { 3081 struct btrfs_fs_info *fs_info = sctx->fs_info; 3082 int ret = 0; 3083 3084 if (!btrfs_is_zoned(fs_info)) 3085 return 0; 3086 3087 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); 3088 3089 mutex_lock(&sctx->wr_lock); 3090 if (sctx->write_pointer < physical_end) { 3091 ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical, 3092 physical, 3093 sctx->write_pointer); 3094 if (ret) 3095 btrfs_err(fs_info, 3096 "zoned: failed to recover write pointer"); 3097 } 3098 mutex_unlock(&sctx->wr_lock); 3099 btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical); 3100 3101 return ret; 3102 } 3103 3104 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, 3105 struct map_lookup *map, 3106 struct btrfs_device *scrub_dev, 3107 int num, u64 base, u64 length, 3108 struct btrfs_block_group *cache) 3109 { 3110 struct btrfs_path *path, *ppath; 3111 struct btrfs_fs_info *fs_info = sctx->fs_info; 3112 struct btrfs_root *root = fs_info->extent_root; 3113 struct btrfs_root *csum_root = fs_info->csum_root; 3114 struct btrfs_extent_item *extent; 3115 struct blk_plug plug; 3116 u64 flags; 3117 int ret; 3118 int slot; 3119 u64 nstripes; 3120 struct extent_buffer *l; 3121 u64 physical; 3122 u64 logical; 3123 u64 logic_end; 3124 u64 physical_end; 3125 u64 generation; 3126 int mirror_num; 3127 struct reada_control *reada1; 3128 struct reada_control *reada2; 3129 struct btrfs_key key; 3130 struct btrfs_key key_end; 3131 u64 increment = map->stripe_len; 3132 u64 offset; 3133 u64 extent_logical; 3134 u64 extent_physical; 3135 /* 3136 * Unlike chunk length, extent length should never go beyond 3137 * BTRFS_MAX_EXTENT_SIZE, thus u32 is enough here. 3138 */ 3139 u32 extent_len; 3140 u64 stripe_logical; 3141 u64 stripe_end; 3142 struct btrfs_device *extent_dev; 3143 int extent_mirror_num; 3144 int stop_loop = 0; 3145 3146 physical = map->stripes[num].physical; 3147 offset = 0; 3148 nstripes = div64_u64(length, map->stripe_len); 3149 if (map->type & BTRFS_BLOCK_GROUP_RAID0) { 3150 offset = map->stripe_len * num; 3151 increment = map->stripe_len * map->num_stripes; 3152 mirror_num = 1; 3153 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) { 3154 int factor = map->num_stripes / map->sub_stripes; 3155 offset = map->stripe_len * (num / map->sub_stripes); 3156 increment = map->stripe_len * factor; 3157 mirror_num = num % map->sub_stripes + 1; 3158 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) { 3159 increment = map->stripe_len; 3160 mirror_num = num % map->num_stripes + 1; 3161 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) { 3162 increment = map->stripe_len; 3163 mirror_num = num % map->num_stripes + 1; 3164 } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3165 get_raid56_logic_offset(physical, num, map, &offset, NULL); 3166 increment = map->stripe_len * nr_data_stripes(map); 3167 mirror_num = 1; 3168 } else { 3169 increment = map->stripe_len; 3170 mirror_num = 1; 3171 } 3172 3173 path = btrfs_alloc_path(); 3174 if (!path) 3175 return -ENOMEM; 3176 3177 ppath = btrfs_alloc_path(); 3178 if (!ppath) { 3179 btrfs_free_path(path); 3180 return -ENOMEM; 3181 } 3182 3183 /* 3184 * work on commit root. The related disk blocks are static as 3185 * long as COW is applied. This means, it is save to rewrite 3186 * them to repair disk errors without any race conditions 3187 */ 3188 path->search_commit_root = 1; 3189 path->skip_locking = 1; 3190 3191 ppath->search_commit_root = 1; 3192 ppath->skip_locking = 1; 3193 /* 3194 * trigger the readahead for extent tree csum tree and wait for 3195 * completion. During readahead, the scrub is officially paused 3196 * to not hold off transaction commits 3197 */ 3198 logical = base + offset; 3199 physical_end = physical + nstripes * map->stripe_len; 3200 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3201 get_raid56_logic_offset(physical_end, num, 3202 map, &logic_end, NULL); 3203 logic_end += base; 3204 } else { 3205 logic_end = logical + increment * nstripes; 3206 } 3207 wait_event(sctx->list_wait, 3208 atomic_read(&sctx->bios_in_flight) == 0); 3209 scrub_blocked_if_needed(fs_info); 3210 3211 /* FIXME it might be better to start readahead at commit root */ 3212 key.objectid = logical; 3213 key.type = BTRFS_EXTENT_ITEM_KEY; 3214 key.offset = (u64)0; 3215 key_end.objectid = logic_end; 3216 key_end.type = BTRFS_METADATA_ITEM_KEY; 3217 key_end.offset = (u64)-1; 3218 reada1 = btrfs_reada_add(root, &key, &key_end); 3219 3220 if (cache->flags & BTRFS_BLOCK_GROUP_DATA) { 3221 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 3222 key.type = BTRFS_EXTENT_CSUM_KEY; 3223 key.offset = logical; 3224 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 3225 key_end.type = BTRFS_EXTENT_CSUM_KEY; 3226 key_end.offset = logic_end; 3227 reada2 = btrfs_reada_add(csum_root, &key, &key_end); 3228 } else { 3229 reada2 = NULL; 3230 } 3231 3232 if (!IS_ERR(reada1)) 3233 btrfs_reada_wait(reada1); 3234 if (!IS_ERR_OR_NULL(reada2)) 3235 btrfs_reada_wait(reada2); 3236 3237 3238 /* 3239 * collect all data csums for the stripe to avoid seeking during 3240 * the scrub. This might currently (crc32) end up to be about 1MB 3241 */ 3242 blk_start_plug(&plug); 3243 3244 if (sctx->is_dev_replace && 3245 btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) { 3246 mutex_lock(&sctx->wr_lock); 3247 sctx->write_pointer = physical; 3248 mutex_unlock(&sctx->wr_lock); 3249 sctx->flush_all_writes = true; 3250 } 3251 3252 /* 3253 * now find all extents for each stripe and scrub them 3254 */ 3255 ret = 0; 3256 while (physical < physical_end) { 3257 /* 3258 * canceled? 3259 */ 3260 if (atomic_read(&fs_info->scrub_cancel_req) || 3261 atomic_read(&sctx->cancel_req)) { 3262 ret = -ECANCELED; 3263 goto out; 3264 } 3265 /* 3266 * check to see if we have to pause 3267 */ 3268 if (atomic_read(&fs_info->scrub_pause_req)) { 3269 /* push queued extents */ 3270 sctx->flush_all_writes = true; 3271 scrub_submit(sctx); 3272 mutex_lock(&sctx->wr_lock); 3273 scrub_wr_submit(sctx); 3274 mutex_unlock(&sctx->wr_lock); 3275 wait_event(sctx->list_wait, 3276 atomic_read(&sctx->bios_in_flight) == 0); 3277 sctx->flush_all_writes = false; 3278 scrub_blocked_if_needed(fs_info); 3279 } 3280 3281 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3282 ret = get_raid56_logic_offset(physical, num, map, 3283 &logical, 3284 &stripe_logical); 3285 logical += base; 3286 if (ret) { 3287 /* it is parity strip */ 3288 stripe_logical += base; 3289 stripe_end = stripe_logical + increment; 3290 ret = scrub_raid56_parity(sctx, map, scrub_dev, 3291 ppath, stripe_logical, 3292 stripe_end); 3293 if (ret) 3294 goto out; 3295 goto skip; 3296 } 3297 } 3298 3299 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) 3300 key.type = BTRFS_METADATA_ITEM_KEY; 3301 else 3302 key.type = BTRFS_EXTENT_ITEM_KEY; 3303 key.objectid = logical; 3304 key.offset = (u64)-1; 3305 3306 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3307 if (ret < 0) 3308 goto out; 3309 3310 if (ret > 0) { 3311 ret = btrfs_previous_extent_item(root, path, 0); 3312 if (ret < 0) 3313 goto out; 3314 if (ret > 0) { 3315 /* there's no smaller item, so stick with the 3316 * larger one */ 3317 btrfs_release_path(path); 3318 ret = btrfs_search_slot(NULL, root, &key, 3319 path, 0, 0); 3320 if (ret < 0) 3321 goto out; 3322 } 3323 } 3324 3325 stop_loop = 0; 3326 while (1) { 3327 u64 bytes; 3328 3329 l = path->nodes[0]; 3330 slot = path->slots[0]; 3331 if (slot >= btrfs_header_nritems(l)) { 3332 ret = btrfs_next_leaf(root, path); 3333 if (ret == 0) 3334 continue; 3335 if (ret < 0) 3336 goto out; 3337 3338 stop_loop = 1; 3339 break; 3340 } 3341 btrfs_item_key_to_cpu(l, &key, slot); 3342 3343 if (key.type != BTRFS_EXTENT_ITEM_KEY && 3344 key.type != BTRFS_METADATA_ITEM_KEY) 3345 goto next; 3346 3347 if (key.type == BTRFS_METADATA_ITEM_KEY) 3348 bytes = fs_info->nodesize; 3349 else 3350 bytes = key.offset; 3351 3352 if (key.objectid + bytes <= logical) 3353 goto next; 3354 3355 if (key.objectid >= logical + map->stripe_len) { 3356 /* out of this device extent */ 3357 if (key.objectid >= logic_end) 3358 stop_loop = 1; 3359 break; 3360 } 3361 3362 /* 3363 * If our block group was removed in the meanwhile, just 3364 * stop scrubbing since there is no point in continuing. 3365 * Continuing would prevent reusing its device extents 3366 * for new block groups for a long time. 3367 */ 3368 spin_lock(&cache->lock); 3369 if (cache->removed) { 3370 spin_unlock(&cache->lock); 3371 ret = 0; 3372 goto out; 3373 } 3374 spin_unlock(&cache->lock); 3375 3376 extent = btrfs_item_ptr(l, slot, 3377 struct btrfs_extent_item); 3378 flags = btrfs_extent_flags(l, extent); 3379 generation = btrfs_extent_generation(l, extent); 3380 3381 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) && 3382 (key.objectid < logical || 3383 key.objectid + bytes > 3384 logical + map->stripe_len)) { 3385 btrfs_err(fs_info, 3386 "scrub: tree block %llu spanning stripes, ignored. logical=%llu", 3387 key.objectid, logical); 3388 spin_lock(&sctx->stat_lock); 3389 sctx->stat.uncorrectable_errors++; 3390 spin_unlock(&sctx->stat_lock); 3391 goto next; 3392 } 3393 3394 again: 3395 extent_logical = key.objectid; 3396 ASSERT(bytes <= U32_MAX); 3397 extent_len = bytes; 3398 3399 /* 3400 * trim extent to this stripe 3401 */ 3402 if (extent_logical < logical) { 3403 extent_len -= logical - extent_logical; 3404 extent_logical = logical; 3405 } 3406 if (extent_logical + extent_len > 3407 logical + map->stripe_len) { 3408 extent_len = logical + map->stripe_len - 3409 extent_logical; 3410 } 3411 3412 extent_physical = extent_logical - logical + physical; 3413 extent_dev = scrub_dev; 3414 extent_mirror_num = mirror_num; 3415 if (sctx->is_dev_replace) 3416 scrub_remap_extent(fs_info, extent_logical, 3417 extent_len, &extent_physical, 3418 &extent_dev, 3419 &extent_mirror_num); 3420 3421 if (flags & BTRFS_EXTENT_FLAG_DATA) { 3422 ret = btrfs_lookup_csums_range(csum_root, 3423 extent_logical, 3424 extent_logical + extent_len - 1, 3425 &sctx->csum_list, 1); 3426 if (ret) 3427 goto out; 3428 } 3429 3430 ret = scrub_extent(sctx, map, extent_logical, extent_len, 3431 extent_physical, extent_dev, flags, 3432 generation, extent_mirror_num, 3433 extent_logical - logical + physical); 3434 3435 scrub_free_csums(sctx); 3436 3437 if (ret) 3438 goto out; 3439 3440 if (sctx->is_dev_replace) 3441 sync_replace_for_zoned(sctx); 3442 3443 if (extent_logical + extent_len < 3444 key.objectid + bytes) { 3445 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3446 /* 3447 * loop until we find next data stripe 3448 * or we have finished all stripes. 3449 */ 3450 loop: 3451 physical += map->stripe_len; 3452 ret = get_raid56_logic_offset(physical, 3453 num, map, &logical, 3454 &stripe_logical); 3455 logical += base; 3456 3457 if (ret && physical < physical_end) { 3458 stripe_logical += base; 3459 stripe_end = stripe_logical + 3460 increment; 3461 ret = scrub_raid56_parity(sctx, 3462 map, scrub_dev, ppath, 3463 stripe_logical, 3464 stripe_end); 3465 if (ret) 3466 goto out; 3467 goto loop; 3468 } 3469 } else { 3470 physical += map->stripe_len; 3471 logical += increment; 3472 } 3473 if (logical < key.objectid + bytes) { 3474 cond_resched(); 3475 goto again; 3476 } 3477 3478 if (physical >= physical_end) { 3479 stop_loop = 1; 3480 break; 3481 } 3482 } 3483 next: 3484 path->slots[0]++; 3485 } 3486 btrfs_release_path(path); 3487 skip: 3488 logical += increment; 3489 physical += map->stripe_len; 3490 spin_lock(&sctx->stat_lock); 3491 if (stop_loop) 3492 sctx->stat.last_physical = map->stripes[num].physical + 3493 length; 3494 else 3495 sctx->stat.last_physical = physical; 3496 spin_unlock(&sctx->stat_lock); 3497 if (stop_loop) 3498 break; 3499 } 3500 out: 3501 /* push queued extents */ 3502 scrub_submit(sctx); 3503 mutex_lock(&sctx->wr_lock); 3504 scrub_wr_submit(sctx); 3505 mutex_unlock(&sctx->wr_lock); 3506 3507 blk_finish_plug(&plug); 3508 btrfs_free_path(path); 3509 btrfs_free_path(ppath); 3510 3511 if (sctx->is_dev_replace && ret >= 0) { 3512 int ret2; 3513 3514 ret2 = sync_write_pointer_for_zoned(sctx, base + offset, 3515 map->stripes[num].physical, 3516 physical_end); 3517 if (ret2) 3518 ret = ret2; 3519 } 3520 3521 return ret < 0 ? ret : 0; 3522 } 3523 3524 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, 3525 struct btrfs_device *scrub_dev, 3526 u64 chunk_offset, u64 length, 3527 u64 dev_offset, 3528 struct btrfs_block_group *cache) 3529 { 3530 struct btrfs_fs_info *fs_info = sctx->fs_info; 3531 struct extent_map_tree *map_tree = &fs_info->mapping_tree; 3532 struct map_lookup *map; 3533 struct extent_map *em; 3534 int i; 3535 int ret = 0; 3536 3537 read_lock(&map_tree->lock); 3538 em = lookup_extent_mapping(map_tree, chunk_offset, 1); 3539 read_unlock(&map_tree->lock); 3540 3541 if (!em) { 3542 /* 3543 * Might have been an unused block group deleted by the cleaner 3544 * kthread or relocation. 3545 */ 3546 spin_lock(&cache->lock); 3547 if (!cache->removed) 3548 ret = -EINVAL; 3549 spin_unlock(&cache->lock); 3550 3551 return ret; 3552 } 3553 3554 map = em->map_lookup; 3555 if (em->start != chunk_offset) 3556 goto out; 3557 3558 if (em->len < length) 3559 goto out; 3560 3561 for (i = 0; i < map->num_stripes; ++i) { 3562 if (map->stripes[i].dev->bdev == scrub_dev->bdev && 3563 map->stripes[i].physical == dev_offset) { 3564 ret = scrub_stripe(sctx, map, scrub_dev, i, 3565 chunk_offset, length, cache); 3566 if (ret) 3567 goto out; 3568 } 3569 } 3570 out: 3571 free_extent_map(em); 3572 3573 return ret; 3574 } 3575 3576 static int finish_extent_writes_for_zoned(struct btrfs_root *root, 3577 struct btrfs_block_group *cache) 3578 { 3579 struct btrfs_fs_info *fs_info = cache->fs_info; 3580 struct btrfs_trans_handle *trans; 3581 3582 if (!btrfs_is_zoned(fs_info)) 3583 return 0; 3584 3585 btrfs_wait_block_group_reservations(cache); 3586 btrfs_wait_nocow_writers(cache); 3587 btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start, cache->length); 3588 3589 trans = btrfs_join_transaction(root); 3590 if (IS_ERR(trans)) 3591 return PTR_ERR(trans); 3592 return btrfs_commit_transaction(trans); 3593 } 3594 3595 static noinline_for_stack 3596 int scrub_enumerate_chunks(struct scrub_ctx *sctx, 3597 struct btrfs_device *scrub_dev, u64 start, u64 end) 3598 { 3599 struct btrfs_dev_extent *dev_extent = NULL; 3600 struct btrfs_path *path; 3601 struct btrfs_fs_info *fs_info = sctx->fs_info; 3602 struct btrfs_root *root = fs_info->dev_root; 3603 u64 length; 3604 u64 chunk_offset; 3605 int ret = 0; 3606 int ro_set; 3607 int slot; 3608 struct extent_buffer *l; 3609 struct btrfs_key key; 3610 struct btrfs_key found_key; 3611 struct btrfs_block_group *cache; 3612 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 3613 3614 path = btrfs_alloc_path(); 3615 if (!path) 3616 return -ENOMEM; 3617 3618 path->reada = READA_FORWARD; 3619 path->search_commit_root = 1; 3620 path->skip_locking = 1; 3621 3622 key.objectid = scrub_dev->devid; 3623 key.offset = 0ull; 3624 key.type = BTRFS_DEV_EXTENT_KEY; 3625 3626 while (1) { 3627 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3628 if (ret < 0) 3629 break; 3630 if (ret > 0) { 3631 if (path->slots[0] >= 3632 btrfs_header_nritems(path->nodes[0])) { 3633 ret = btrfs_next_leaf(root, path); 3634 if (ret < 0) 3635 break; 3636 if (ret > 0) { 3637 ret = 0; 3638 break; 3639 } 3640 } else { 3641 ret = 0; 3642 } 3643 } 3644 3645 l = path->nodes[0]; 3646 slot = path->slots[0]; 3647 3648 btrfs_item_key_to_cpu(l, &found_key, slot); 3649 3650 if (found_key.objectid != scrub_dev->devid) 3651 break; 3652 3653 if (found_key.type != BTRFS_DEV_EXTENT_KEY) 3654 break; 3655 3656 if (found_key.offset >= end) 3657 break; 3658 3659 if (found_key.offset < key.offset) 3660 break; 3661 3662 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent); 3663 length = btrfs_dev_extent_length(l, dev_extent); 3664 3665 if (found_key.offset + length <= start) 3666 goto skip; 3667 3668 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent); 3669 3670 /* 3671 * get a reference on the corresponding block group to prevent 3672 * the chunk from going away while we scrub it 3673 */ 3674 cache = btrfs_lookup_block_group(fs_info, chunk_offset); 3675 3676 /* some chunks are removed but not committed to disk yet, 3677 * continue scrubbing */ 3678 if (!cache) 3679 goto skip; 3680 3681 if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { 3682 spin_lock(&cache->lock); 3683 if (!cache->to_copy) { 3684 spin_unlock(&cache->lock); 3685 ro_set = 0; 3686 goto done; 3687 } 3688 spin_unlock(&cache->lock); 3689 } 3690 3691 /* 3692 * Make sure that while we are scrubbing the corresponding block 3693 * group doesn't get its logical address and its device extents 3694 * reused for another block group, which can possibly be of a 3695 * different type and different profile. We do this to prevent 3696 * false error detections and crashes due to bogus attempts to 3697 * repair extents. 3698 */ 3699 spin_lock(&cache->lock); 3700 if (cache->removed) { 3701 spin_unlock(&cache->lock); 3702 btrfs_put_block_group(cache); 3703 goto skip; 3704 } 3705 btrfs_freeze_block_group(cache); 3706 spin_unlock(&cache->lock); 3707 3708 /* 3709 * we need call btrfs_inc_block_group_ro() with scrubs_paused, 3710 * to avoid deadlock caused by: 3711 * btrfs_inc_block_group_ro() 3712 * -> btrfs_wait_for_commit() 3713 * -> btrfs_commit_transaction() 3714 * -> btrfs_scrub_pause() 3715 */ 3716 scrub_pause_on(fs_info); 3717 3718 /* 3719 * Don't do chunk preallocation for scrub. 3720 * 3721 * This is especially important for SYSTEM bgs, or we can hit 3722 * -EFBIG from btrfs_finish_chunk_alloc() like: 3723 * 1. The only SYSTEM bg is marked RO. 3724 * Since SYSTEM bg is small, that's pretty common. 3725 * 2. New SYSTEM bg will be allocated 3726 * Due to regular version will allocate new chunk. 3727 * 3. New SYSTEM bg is empty and will get cleaned up 3728 * Before cleanup really happens, it's marked RO again. 3729 * 4. Empty SYSTEM bg get scrubbed 3730 * We go back to 2. 3731 * 3732 * This can easily boost the amount of SYSTEM chunks if cleaner 3733 * thread can't be triggered fast enough, and use up all space 3734 * of btrfs_super_block::sys_chunk_array 3735 * 3736 * While for dev replace, we need to try our best to mark block 3737 * group RO, to prevent race between: 3738 * - Write duplication 3739 * Contains latest data 3740 * - Scrub copy 3741 * Contains data from commit tree 3742 * 3743 * If target block group is not marked RO, nocow writes can 3744 * be overwritten by scrub copy, causing data corruption. 3745 * So for dev-replace, it's not allowed to continue if a block 3746 * group is not RO. 3747 */ 3748 ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace); 3749 if (!ret && sctx->is_dev_replace) { 3750 ret = finish_extent_writes_for_zoned(root, cache); 3751 if (ret) { 3752 btrfs_dec_block_group_ro(cache); 3753 scrub_pause_off(fs_info); 3754 btrfs_put_block_group(cache); 3755 break; 3756 } 3757 } 3758 3759 if (ret == 0) { 3760 ro_set = 1; 3761 } else if (ret == -ENOSPC && !sctx->is_dev_replace) { 3762 /* 3763 * btrfs_inc_block_group_ro return -ENOSPC when it 3764 * failed in creating new chunk for metadata. 3765 * It is not a problem for scrub, because 3766 * metadata are always cowed, and our scrub paused 3767 * commit_transactions. 3768 */ 3769 ro_set = 0; 3770 } else if (ret == -ETXTBSY) { 3771 btrfs_warn(fs_info, 3772 "skipping scrub of block group %llu due to active swapfile", 3773 cache->start); 3774 scrub_pause_off(fs_info); 3775 ret = 0; 3776 goto skip_unfreeze; 3777 } else { 3778 btrfs_warn(fs_info, 3779 "failed setting block group ro: %d", ret); 3780 btrfs_unfreeze_block_group(cache); 3781 btrfs_put_block_group(cache); 3782 scrub_pause_off(fs_info); 3783 break; 3784 } 3785 3786 /* 3787 * Now the target block is marked RO, wait for nocow writes to 3788 * finish before dev-replace. 3789 * COW is fine, as COW never overwrites extents in commit tree. 3790 */ 3791 if (sctx->is_dev_replace) { 3792 btrfs_wait_nocow_writers(cache); 3793 btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start, 3794 cache->length); 3795 } 3796 3797 scrub_pause_off(fs_info); 3798 down_write(&dev_replace->rwsem); 3799 dev_replace->cursor_right = found_key.offset + length; 3800 dev_replace->cursor_left = found_key.offset; 3801 dev_replace->item_needs_writeback = 1; 3802 up_write(&dev_replace->rwsem); 3803 3804 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length, 3805 found_key.offset, cache); 3806 3807 /* 3808 * flush, submit all pending read and write bios, afterwards 3809 * wait for them. 3810 * Note that in the dev replace case, a read request causes 3811 * write requests that are submitted in the read completion 3812 * worker. Therefore in the current situation, it is required 3813 * that all write requests are flushed, so that all read and 3814 * write requests are really completed when bios_in_flight 3815 * changes to 0. 3816 */ 3817 sctx->flush_all_writes = true; 3818 scrub_submit(sctx); 3819 mutex_lock(&sctx->wr_lock); 3820 scrub_wr_submit(sctx); 3821 mutex_unlock(&sctx->wr_lock); 3822 3823 wait_event(sctx->list_wait, 3824 atomic_read(&sctx->bios_in_flight) == 0); 3825 3826 scrub_pause_on(fs_info); 3827 3828 /* 3829 * must be called before we decrease @scrub_paused. 3830 * make sure we don't block transaction commit while 3831 * we are waiting pending workers finished. 3832 */ 3833 wait_event(sctx->list_wait, 3834 atomic_read(&sctx->workers_pending) == 0); 3835 sctx->flush_all_writes = false; 3836 3837 scrub_pause_off(fs_info); 3838 3839 if (sctx->is_dev_replace && 3840 !btrfs_finish_block_group_to_copy(dev_replace->srcdev, 3841 cache, found_key.offset)) 3842 ro_set = 0; 3843 3844 done: 3845 down_write(&dev_replace->rwsem); 3846 dev_replace->cursor_left = dev_replace->cursor_right; 3847 dev_replace->item_needs_writeback = 1; 3848 up_write(&dev_replace->rwsem); 3849 3850 if (ro_set) 3851 btrfs_dec_block_group_ro(cache); 3852 3853 /* 3854 * We might have prevented the cleaner kthread from deleting 3855 * this block group if it was already unused because we raced 3856 * and set it to RO mode first. So add it back to the unused 3857 * list, otherwise it might not ever be deleted unless a manual 3858 * balance is triggered or it becomes used and unused again. 3859 */ 3860 spin_lock(&cache->lock); 3861 if (!cache->removed && !cache->ro && cache->reserved == 0 && 3862 cache->used == 0) { 3863 spin_unlock(&cache->lock); 3864 if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) 3865 btrfs_discard_queue_work(&fs_info->discard_ctl, 3866 cache); 3867 else 3868 btrfs_mark_bg_unused(cache); 3869 } else { 3870 spin_unlock(&cache->lock); 3871 } 3872 skip_unfreeze: 3873 btrfs_unfreeze_block_group(cache); 3874 btrfs_put_block_group(cache); 3875 if (ret) 3876 break; 3877 if (sctx->is_dev_replace && 3878 atomic64_read(&dev_replace->num_write_errors) > 0) { 3879 ret = -EIO; 3880 break; 3881 } 3882 if (sctx->stat.malloc_errors > 0) { 3883 ret = -ENOMEM; 3884 break; 3885 } 3886 skip: 3887 key.offset = found_key.offset + length; 3888 btrfs_release_path(path); 3889 } 3890 3891 btrfs_free_path(path); 3892 3893 return ret; 3894 } 3895 3896 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx, 3897 struct btrfs_device *scrub_dev) 3898 { 3899 int i; 3900 u64 bytenr; 3901 u64 gen; 3902 int ret; 3903 struct btrfs_fs_info *fs_info = sctx->fs_info; 3904 3905 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 3906 return -EROFS; 3907 3908 /* Seed devices of a new filesystem has their own generation. */ 3909 if (scrub_dev->fs_devices != fs_info->fs_devices) 3910 gen = scrub_dev->generation; 3911 else 3912 gen = fs_info->last_trans_committed; 3913 3914 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 3915 bytenr = btrfs_sb_offset(i); 3916 if (bytenr + BTRFS_SUPER_INFO_SIZE > 3917 scrub_dev->commit_total_bytes) 3918 break; 3919 if (!btrfs_check_super_location(scrub_dev, bytenr)) 3920 continue; 3921 3922 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr, 3923 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i, 3924 NULL, bytenr); 3925 if (ret) 3926 return ret; 3927 } 3928 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); 3929 3930 return 0; 3931 } 3932 3933 static void scrub_workers_put(struct btrfs_fs_info *fs_info) 3934 { 3935 if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt, 3936 &fs_info->scrub_lock)) { 3937 struct btrfs_workqueue *scrub_workers = NULL; 3938 struct btrfs_workqueue *scrub_wr_comp = NULL; 3939 struct btrfs_workqueue *scrub_parity = NULL; 3940 3941 scrub_workers = fs_info->scrub_workers; 3942 scrub_wr_comp = fs_info->scrub_wr_completion_workers; 3943 scrub_parity = fs_info->scrub_parity_workers; 3944 3945 fs_info->scrub_workers = NULL; 3946 fs_info->scrub_wr_completion_workers = NULL; 3947 fs_info->scrub_parity_workers = NULL; 3948 mutex_unlock(&fs_info->scrub_lock); 3949 3950 btrfs_destroy_workqueue(scrub_workers); 3951 btrfs_destroy_workqueue(scrub_wr_comp); 3952 btrfs_destroy_workqueue(scrub_parity); 3953 } 3954 } 3955 3956 /* 3957 * get a reference count on fs_info->scrub_workers. start worker if necessary 3958 */ 3959 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info, 3960 int is_dev_replace) 3961 { 3962 struct btrfs_workqueue *scrub_workers = NULL; 3963 struct btrfs_workqueue *scrub_wr_comp = NULL; 3964 struct btrfs_workqueue *scrub_parity = NULL; 3965 unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND; 3966 int max_active = fs_info->thread_pool_size; 3967 int ret = -ENOMEM; 3968 3969 if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt)) 3970 return 0; 3971 3972 scrub_workers = btrfs_alloc_workqueue(fs_info, "scrub", flags, 3973 is_dev_replace ? 1 : max_active, 4); 3974 if (!scrub_workers) 3975 goto fail_scrub_workers; 3976 3977 scrub_wr_comp = btrfs_alloc_workqueue(fs_info, "scrubwrc", flags, 3978 max_active, 2); 3979 if (!scrub_wr_comp) 3980 goto fail_scrub_wr_completion_workers; 3981 3982 scrub_parity = btrfs_alloc_workqueue(fs_info, "scrubparity", flags, 3983 max_active, 2); 3984 if (!scrub_parity) 3985 goto fail_scrub_parity_workers; 3986 3987 mutex_lock(&fs_info->scrub_lock); 3988 if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) { 3989 ASSERT(fs_info->scrub_workers == NULL && 3990 fs_info->scrub_wr_completion_workers == NULL && 3991 fs_info->scrub_parity_workers == NULL); 3992 fs_info->scrub_workers = scrub_workers; 3993 fs_info->scrub_wr_completion_workers = scrub_wr_comp; 3994 fs_info->scrub_parity_workers = scrub_parity; 3995 refcount_set(&fs_info->scrub_workers_refcnt, 1); 3996 mutex_unlock(&fs_info->scrub_lock); 3997 return 0; 3998 } 3999 /* Other thread raced in and created the workers for us */ 4000 refcount_inc(&fs_info->scrub_workers_refcnt); 4001 mutex_unlock(&fs_info->scrub_lock); 4002 4003 ret = 0; 4004 btrfs_destroy_workqueue(scrub_parity); 4005 fail_scrub_parity_workers: 4006 btrfs_destroy_workqueue(scrub_wr_comp); 4007 fail_scrub_wr_completion_workers: 4008 btrfs_destroy_workqueue(scrub_workers); 4009 fail_scrub_workers: 4010 return ret; 4011 } 4012 4013 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, 4014 u64 end, struct btrfs_scrub_progress *progress, 4015 int readonly, int is_dev_replace) 4016 { 4017 struct scrub_ctx *sctx; 4018 int ret; 4019 struct btrfs_device *dev; 4020 unsigned int nofs_flag; 4021 4022 if (btrfs_fs_closing(fs_info)) 4023 return -EAGAIN; 4024 4025 if (fs_info->nodesize > BTRFS_STRIPE_LEN) { 4026 /* 4027 * in this case scrub is unable to calculate the checksum 4028 * the way scrub is implemented. Do not handle this 4029 * situation at all because it won't ever happen. 4030 */ 4031 btrfs_err(fs_info, 4032 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails", 4033 fs_info->nodesize, 4034 BTRFS_STRIPE_LEN); 4035 return -EINVAL; 4036 } 4037 4038 if (fs_info->nodesize > 4039 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK || 4040 fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) { 4041 /* 4042 * would exhaust the array bounds of pagev member in 4043 * struct scrub_block 4044 */ 4045 btrfs_err(fs_info, 4046 "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails", 4047 fs_info->nodesize, 4048 SCRUB_MAX_PAGES_PER_BLOCK, 4049 fs_info->sectorsize, 4050 SCRUB_MAX_PAGES_PER_BLOCK); 4051 return -EINVAL; 4052 } 4053 4054 /* Allocate outside of device_list_mutex */ 4055 sctx = scrub_setup_ctx(fs_info, is_dev_replace); 4056 if (IS_ERR(sctx)) 4057 return PTR_ERR(sctx); 4058 4059 ret = scrub_workers_get(fs_info, is_dev_replace); 4060 if (ret) 4061 goto out_free_ctx; 4062 4063 mutex_lock(&fs_info->fs_devices->device_list_mutex); 4064 dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL); 4065 if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) && 4066 !is_dev_replace)) { 4067 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4068 ret = -ENODEV; 4069 goto out; 4070 } 4071 4072 if (!is_dev_replace && !readonly && 4073 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) { 4074 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4075 btrfs_err_in_rcu(fs_info, 4076 "scrub on devid %llu: filesystem on %s is not writable", 4077 devid, rcu_str_deref(dev->name)); 4078 ret = -EROFS; 4079 goto out; 4080 } 4081 4082 mutex_lock(&fs_info->scrub_lock); 4083 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || 4084 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) { 4085 mutex_unlock(&fs_info->scrub_lock); 4086 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4087 ret = -EIO; 4088 goto out; 4089 } 4090 4091 down_read(&fs_info->dev_replace.rwsem); 4092 if (dev->scrub_ctx || 4093 (!is_dev_replace && 4094 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) { 4095 up_read(&fs_info->dev_replace.rwsem); 4096 mutex_unlock(&fs_info->scrub_lock); 4097 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4098 ret = -EINPROGRESS; 4099 goto out; 4100 } 4101 up_read(&fs_info->dev_replace.rwsem); 4102 4103 sctx->readonly = readonly; 4104 dev->scrub_ctx = sctx; 4105 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4106 4107 /* 4108 * checking @scrub_pause_req here, we can avoid 4109 * race between committing transaction and scrubbing. 4110 */ 4111 __scrub_blocked_if_needed(fs_info); 4112 atomic_inc(&fs_info->scrubs_running); 4113 mutex_unlock(&fs_info->scrub_lock); 4114 4115 /* 4116 * In order to avoid deadlock with reclaim when there is a transaction 4117 * trying to pause scrub, make sure we use GFP_NOFS for all the 4118 * allocations done at btrfs_scrub_pages() and scrub_pages_for_parity() 4119 * invoked by our callees. The pausing request is done when the 4120 * transaction commit starts, and it blocks the transaction until scrub 4121 * is paused (done at specific points at scrub_stripe() or right above 4122 * before incrementing fs_info->scrubs_running). 4123 */ 4124 nofs_flag = memalloc_nofs_save(); 4125 if (!is_dev_replace) { 4126 btrfs_info(fs_info, "scrub: started on devid %llu", devid); 4127 /* 4128 * by holding device list mutex, we can 4129 * kick off writing super in log tree sync. 4130 */ 4131 mutex_lock(&fs_info->fs_devices->device_list_mutex); 4132 ret = scrub_supers(sctx, dev); 4133 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4134 } 4135 4136 if (!ret) 4137 ret = scrub_enumerate_chunks(sctx, dev, start, end); 4138 memalloc_nofs_restore(nofs_flag); 4139 4140 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); 4141 atomic_dec(&fs_info->scrubs_running); 4142 wake_up(&fs_info->scrub_pause_wait); 4143 4144 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0); 4145 4146 if (progress) 4147 memcpy(progress, &sctx->stat, sizeof(*progress)); 4148 4149 if (!is_dev_replace) 4150 btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d", 4151 ret ? "not finished" : "finished", devid, ret); 4152 4153 mutex_lock(&fs_info->scrub_lock); 4154 dev->scrub_ctx = NULL; 4155 mutex_unlock(&fs_info->scrub_lock); 4156 4157 scrub_workers_put(fs_info); 4158 scrub_put_ctx(sctx); 4159 4160 return ret; 4161 out: 4162 scrub_workers_put(fs_info); 4163 out_free_ctx: 4164 scrub_free_ctx(sctx); 4165 4166 return ret; 4167 } 4168 4169 void btrfs_scrub_pause(struct btrfs_fs_info *fs_info) 4170 { 4171 mutex_lock(&fs_info->scrub_lock); 4172 atomic_inc(&fs_info->scrub_pause_req); 4173 while (atomic_read(&fs_info->scrubs_paused) != 4174 atomic_read(&fs_info->scrubs_running)) { 4175 mutex_unlock(&fs_info->scrub_lock); 4176 wait_event(fs_info->scrub_pause_wait, 4177 atomic_read(&fs_info->scrubs_paused) == 4178 atomic_read(&fs_info->scrubs_running)); 4179 mutex_lock(&fs_info->scrub_lock); 4180 } 4181 mutex_unlock(&fs_info->scrub_lock); 4182 } 4183 4184 void btrfs_scrub_continue(struct btrfs_fs_info *fs_info) 4185 { 4186 atomic_dec(&fs_info->scrub_pause_req); 4187 wake_up(&fs_info->scrub_pause_wait); 4188 } 4189 4190 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info) 4191 { 4192 mutex_lock(&fs_info->scrub_lock); 4193 if (!atomic_read(&fs_info->scrubs_running)) { 4194 mutex_unlock(&fs_info->scrub_lock); 4195 return -ENOTCONN; 4196 } 4197 4198 atomic_inc(&fs_info->scrub_cancel_req); 4199 while (atomic_read(&fs_info->scrubs_running)) { 4200 mutex_unlock(&fs_info->scrub_lock); 4201 wait_event(fs_info->scrub_pause_wait, 4202 atomic_read(&fs_info->scrubs_running) == 0); 4203 mutex_lock(&fs_info->scrub_lock); 4204 } 4205 atomic_dec(&fs_info->scrub_cancel_req); 4206 mutex_unlock(&fs_info->scrub_lock); 4207 4208 return 0; 4209 } 4210 4211 int btrfs_scrub_cancel_dev(struct btrfs_device *dev) 4212 { 4213 struct btrfs_fs_info *fs_info = dev->fs_info; 4214 struct scrub_ctx *sctx; 4215 4216 mutex_lock(&fs_info->scrub_lock); 4217 sctx = dev->scrub_ctx; 4218 if (!sctx) { 4219 mutex_unlock(&fs_info->scrub_lock); 4220 return -ENOTCONN; 4221 } 4222 atomic_inc(&sctx->cancel_req); 4223 while (dev->scrub_ctx) { 4224 mutex_unlock(&fs_info->scrub_lock); 4225 wait_event(fs_info->scrub_pause_wait, 4226 dev->scrub_ctx == NULL); 4227 mutex_lock(&fs_info->scrub_lock); 4228 } 4229 mutex_unlock(&fs_info->scrub_lock); 4230 4231 return 0; 4232 } 4233 4234 int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid, 4235 struct btrfs_scrub_progress *progress) 4236 { 4237 struct btrfs_device *dev; 4238 struct scrub_ctx *sctx = NULL; 4239 4240 mutex_lock(&fs_info->fs_devices->device_list_mutex); 4241 dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL); 4242 if (dev) 4243 sctx = dev->scrub_ctx; 4244 if (sctx) 4245 memcpy(progress, &sctx->stat, sizeof(*progress)); 4246 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4247 4248 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; 4249 } 4250 4251 static void scrub_remap_extent(struct btrfs_fs_info *fs_info, 4252 u64 extent_logical, u32 extent_len, 4253 u64 *extent_physical, 4254 struct btrfs_device **extent_dev, 4255 int *extent_mirror_num) 4256 { 4257 u64 mapped_length; 4258 struct btrfs_bio *bbio = NULL; 4259 int ret; 4260 4261 mapped_length = extent_len; 4262 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical, 4263 &mapped_length, &bbio, 0); 4264 if (ret || !bbio || mapped_length < extent_len || 4265 !bbio->stripes[0].dev->bdev) { 4266 btrfs_put_bbio(bbio); 4267 return; 4268 } 4269 4270 *extent_physical = bbio->stripes[0].physical; 4271 *extent_mirror_num = bbio->mirror_num; 4272 *extent_dev = bbio->stripes[0].dev; 4273 btrfs_put_bbio(bbio); 4274 } 4275