1 /* 2 * Copyright (C) 2011, 2012 STRATO. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public 6 * License v2 as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 * General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public 14 * License along with this program; if not, write to the 15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 16 * Boston, MA 021110-1307, USA. 17 */ 18 19 #include <linux/blkdev.h> 20 #include <linux/ratelimit.h> 21 #include "ctree.h" 22 #include "volumes.h" 23 #include "disk-io.h" 24 #include "ordered-data.h" 25 #include "transaction.h" 26 #include "backref.h" 27 #include "extent_io.h" 28 #include "dev-replace.h" 29 #include "check-integrity.h" 30 #include "rcu-string.h" 31 #include "raid56.h" 32 33 /* 34 * This is only the first step towards a full-features scrub. It reads all 35 * extent and super block and verifies the checksums. In case a bad checksum 36 * is found or the extent cannot be read, good data will be written back if 37 * any can be found. 38 * 39 * Future enhancements: 40 * - In case an unrepairable extent is encountered, track which files are 41 * affected and report them 42 * - track and record media errors, throw out bad devices 43 * - add a mode to also read unallocated space 44 */ 45 46 struct scrub_block; 47 struct scrub_ctx; 48 49 /* 50 * the following three values only influence the performance. 51 * The last one configures the number of parallel and outstanding I/O 52 * operations. The first two values configure an upper limit for the number 53 * of (dynamically allocated) pages that are added to a bio. 54 */ 55 #define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */ 56 #define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */ 57 #define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */ 58 59 /* 60 * the following value times PAGE_SIZE needs to be large enough to match the 61 * largest node/leaf/sector size that shall be supported. 62 * Values larger than BTRFS_STRIPE_LEN are not supported. 63 */ 64 #define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */ 65 66 struct scrub_recover { 67 atomic_t refs; 68 struct btrfs_bio *bbio; 69 u64 map_length; 70 }; 71 72 struct scrub_page { 73 struct scrub_block *sblock; 74 struct page *page; 75 struct btrfs_device *dev; 76 struct list_head list; 77 u64 flags; /* extent flags */ 78 u64 generation; 79 u64 logical; 80 u64 physical; 81 u64 physical_for_dev_replace; 82 atomic_t refs; 83 struct { 84 unsigned int mirror_num:8; 85 unsigned int have_csum:1; 86 unsigned int io_error:1; 87 }; 88 u8 csum[BTRFS_CSUM_SIZE]; 89 90 struct scrub_recover *recover; 91 }; 92 93 struct scrub_bio { 94 int index; 95 struct scrub_ctx *sctx; 96 struct btrfs_device *dev; 97 struct bio *bio; 98 int err; 99 u64 logical; 100 u64 physical; 101 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO 102 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO]; 103 #else 104 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO]; 105 #endif 106 int page_count; 107 int next_free; 108 struct btrfs_work work; 109 }; 110 111 struct scrub_block { 112 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK]; 113 int page_count; 114 atomic_t outstanding_pages; 115 atomic_t refs; /* free mem on transition to zero */ 116 struct scrub_ctx *sctx; 117 struct scrub_parity *sparity; 118 struct { 119 unsigned int header_error:1; 120 unsigned int checksum_error:1; 121 unsigned int no_io_error_seen:1; 122 unsigned int generation_error:1; /* also sets header_error */ 123 124 /* The following is for the data used to check parity */ 125 /* It is for the data with checksum */ 126 unsigned int data_corrected:1; 127 }; 128 struct btrfs_work work; 129 }; 130 131 /* Used for the chunks with parity stripe such RAID5/6 */ 132 struct scrub_parity { 133 struct scrub_ctx *sctx; 134 135 struct btrfs_device *scrub_dev; 136 137 u64 logic_start; 138 139 u64 logic_end; 140 141 int nsectors; 142 143 int stripe_len; 144 145 atomic_t refs; 146 147 struct list_head spages; 148 149 /* Work of parity check and repair */ 150 struct btrfs_work work; 151 152 /* Mark the parity blocks which have data */ 153 unsigned long *dbitmap; 154 155 /* 156 * Mark the parity blocks which have data, but errors happen when 157 * read data or check data 158 */ 159 unsigned long *ebitmap; 160 161 unsigned long bitmap[0]; 162 }; 163 164 struct scrub_wr_ctx { 165 struct scrub_bio *wr_curr_bio; 166 struct btrfs_device *tgtdev; 167 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */ 168 atomic_t flush_all_writes; 169 struct mutex wr_lock; 170 }; 171 172 struct scrub_ctx { 173 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX]; 174 struct btrfs_fs_info *fs_info; 175 int first_free; 176 int curr; 177 atomic_t bios_in_flight; 178 atomic_t workers_pending; 179 spinlock_t list_lock; 180 wait_queue_head_t list_wait; 181 u16 csum_size; 182 struct list_head csum_list; 183 atomic_t cancel_req; 184 int readonly; 185 int pages_per_rd_bio; 186 u32 sectorsize; 187 u32 nodesize; 188 189 int is_dev_replace; 190 struct scrub_wr_ctx wr_ctx; 191 192 /* 193 * statistics 194 */ 195 struct btrfs_scrub_progress stat; 196 spinlock_t stat_lock; 197 198 /* 199 * Use a ref counter to avoid use-after-free issues. Scrub workers 200 * decrement bios_in_flight and workers_pending and then do a wakeup 201 * on the list_wait wait queue. We must ensure the main scrub task 202 * doesn't free the scrub context before or while the workers are 203 * doing the wakeup() call. 204 */ 205 atomic_t refs; 206 }; 207 208 struct scrub_fixup_nodatasum { 209 struct scrub_ctx *sctx; 210 struct btrfs_device *dev; 211 u64 logical; 212 struct btrfs_root *root; 213 struct btrfs_work work; 214 int mirror_num; 215 }; 216 217 struct scrub_nocow_inode { 218 u64 inum; 219 u64 offset; 220 u64 root; 221 struct list_head list; 222 }; 223 224 struct scrub_copy_nocow_ctx { 225 struct scrub_ctx *sctx; 226 u64 logical; 227 u64 len; 228 int mirror_num; 229 u64 physical_for_dev_replace; 230 struct list_head inodes; 231 struct btrfs_work work; 232 }; 233 234 struct scrub_warning { 235 struct btrfs_path *path; 236 u64 extent_item_size; 237 const char *errstr; 238 sector_t sector; 239 u64 logical; 240 struct btrfs_device *dev; 241 }; 242 243 static void scrub_pending_bio_inc(struct scrub_ctx *sctx); 244 static void scrub_pending_bio_dec(struct scrub_ctx *sctx); 245 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx); 246 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx); 247 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check); 248 static int scrub_setup_recheck_block(struct scrub_block *original_sblock, 249 struct scrub_block *sblocks_for_recheck); 250 static void scrub_recheck_block(struct btrfs_fs_info *fs_info, 251 struct scrub_block *sblock, 252 int retry_failed_mirror); 253 static void scrub_recheck_block_checksum(struct scrub_block *sblock); 254 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad, 255 struct scrub_block *sblock_good); 256 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad, 257 struct scrub_block *sblock_good, 258 int page_num, int force_write); 259 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock); 260 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock, 261 int page_num); 262 static int scrub_checksum_data(struct scrub_block *sblock); 263 static int scrub_checksum_tree_block(struct scrub_block *sblock); 264 static int scrub_checksum_super(struct scrub_block *sblock); 265 static void scrub_block_get(struct scrub_block *sblock); 266 static void scrub_block_put(struct scrub_block *sblock); 267 static void scrub_page_get(struct scrub_page *spage); 268 static void scrub_page_put(struct scrub_page *spage); 269 static void scrub_parity_get(struct scrub_parity *sparity); 270 static void scrub_parity_put(struct scrub_parity *sparity); 271 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, 272 struct scrub_page *spage); 273 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len, 274 u64 physical, struct btrfs_device *dev, u64 flags, 275 u64 gen, int mirror_num, u8 *csum, int force, 276 u64 physical_for_dev_replace); 277 static void scrub_bio_end_io(struct bio *bio); 278 static void scrub_bio_end_io_worker(struct btrfs_work *work); 279 static void scrub_block_complete(struct scrub_block *sblock); 280 static void scrub_remap_extent(struct btrfs_fs_info *fs_info, 281 u64 extent_logical, u64 extent_len, 282 u64 *extent_physical, 283 struct btrfs_device **extent_dev, 284 int *extent_mirror_num); 285 static int scrub_setup_wr_ctx(struct scrub_wr_ctx *wr_ctx, 286 struct btrfs_device *dev, 287 int is_dev_replace); 288 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx); 289 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, 290 struct scrub_page *spage); 291 static void scrub_wr_submit(struct scrub_ctx *sctx); 292 static void scrub_wr_bio_end_io(struct bio *bio); 293 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work); 294 static int write_page_nocow(struct scrub_ctx *sctx, 295 u64 physical_for_dev_replace, struct page *page); 296 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root, 297 struct scrub_copy_nocow_ctx *ctx); 298 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len, 299 int mirror_num, u64 physical_for_dev_replace); 300 static void copy_nocow_pages_worker(struct btrfs_work *work); 301 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info); 302 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info); 303 static void scrub_put_ctx(struct scrub_ctx *sctx); 304 305 306 static void scrub_pending_bio_inc(struct scrub_ctx *sctx) 307 { 308 atomic_inc(&sctx->refs); 309 atomic_inc(&sctx->bios_in_flight); 310 } 311 312 static void scrub_pending_bio_dec(struct scrub_ctx *sctx) 313 { 314 atomic_dec(&sctx->bios_in_flight); 315 wake_up(&sctx->list_wait); 316 scrub_put_ctx(sctx); 317 } 318 319 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) 320 { 321 while (atomic_read(&fs_info->scrub_pause_req)) { 322 mutex_unlock(&fs_info->scrub_lock); 323 wait_event(fs_info->scrub_pause_wait, 324 atomic_read(&fs_info->scrub_pause_req) == 0); 325 mutex_lock(&fs_info->scrub_lock); 326 } 327 } 328 329 static void scrub_pause_on(struct btrfs_fs_info *fs_info) 330 { 331 atomic_inc(&fs_info->scrubs_paused); 332 wake_up(&fs_info->scrub_pause_wait); 333 } 334 335 static void scrub_pause_off(struct btrfs_fs_info *fs_info) 336 { 337 mutex_lock(&fs_info->scrub_lock); 338 __scrub_blocked_if_needed(fs_info); 339 atomic_dec(&fs_info->scrubs_paused); 340 mutex_unlock(&fs_info->scrub_lock); 341 342 wake_up(&fs_info->scrub_pause_wait); 343 } 344 345 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) 346 { 347 scrub_pause_on(fs_info); 348 scrub_pause_off(fs_info); 349 } 350 351 /* 352 * used for workers that require transaction commits (i.e., for the 353 * NOCOW case) 354 */ 355 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx) 356 { 357 struct btrfs_fs_info *fs_info = sctx->fs_info; 358 359 atomic_inc(&sctx->refs); 360 /* 361 * increment scrubs_running to prevent cancel requests from 362 * completing as long as a worker is running. we must also 363 * increment scrubs_paused to prevent deadlocking on pause 364 * requests used for transactions commits (as the worker uses a 365 * transaction context). it is safe to regard the worker 366 * as paused for all matters practical. effectively, we only 367 * avoid cancellation requests from completing. 368 */ 369 mutex_lock(&fs_info->scrub_lock); 370 atomic_inc(&fs_info->scrubs_running); 371 atomic_inc(&fs_info->scrubs_paused); 372 mutex_unlock(&fs_info->scrub_lock); 373 374 /* 375 * check if @scrubs_running=@scrubs_paused condition 376 * inside wait_event() is not an atomic operation. 377 * which means we may inc/dec @scrub_running/paused 378 * at any time. Let's wake up @scrub_pause_wait as 379 * much as we can to let commit transaction blocked less. 380 */ 381 wake_up(&fs_info->scrub_pause_wait); 382 383 atomic_inc(&sctx->workers_pending); 384 } 385 386 /* used for workers that require transaction commits */ 387 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx) 388 { 389 struct btrfs_fs_info *fs_info = sctx->fs_info; 390 391 /* 392 * see scrub_pending_trans_workers_inc() why we're pretending 393 * to be paused in the scrub counters 394 */ 395 mutex_lock(&fs_info->scrub_lock); 396 atomic_dec(&fs_info->scrubs_running); 397 atomic_dec(&fs_info->scrubs_paused); 398 mutex_unlock(&fs_info->scrub_lock); 399 atomic_dec(&sctx->workers_pending); 400 wake_up(&fs_info->scrub_pause_wait); 401 wake_up(&sctx->list_wait); 402 scrub_put_ctx(sctx); 403 } 404 405 static void scrub_free_csums(struct scrub_ctx *sctx) 406 { 407 while (!list_empty(&sctx->csum_list)) { 408 struct btrfs_ordered_sum *sum; 409 sum = list_first_entry(&sctx->csum_list, 410 struct btrfs_ordered_sum, list); 411 list_del(&sum->list); 412 kfree(sum); 413 } 414 } 415 416 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) 417 { 418 int i; 419 420 if (!sctx) 421 return; 422 423 scrub_free_wr_ctx(&sctx->wr_ctx); 424 425 /* this can happen when scrub is cancelled */ 426 if (sctx->curr != -1) { 427 struct scrub_bio *sbio = sctx->bios[sctx->curr]; 428 429 for (i = 0; i < sbio->page_count; i++) { 430 WARN_ON(!sbio->pagev[i]->page); 431 scrub_block_put(sbio->pagev[i]->sblock); 432 } 433 bio_put(sbio->bio); 434 } 435 436 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { 437 struct scrub_bio *sbio = sctx->bios[i]; 438 439 if (!sbio) 440 break; 441 kfree(sbio); 442 } 443 444 scrub_free_csums(sctx); 445 kfree(sctx); 446 } 447 448 static void scrub_put_ctx(struct scrub_ctx *sctx) 449 { 450 if (atomic_dec_and_test(&sctx->refs)) 451 scrub_free_ctx(sctx); 452 } 453 454 static noinline_for_stack 455 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace) 456 { 457 struct scrub_ctx *sctx; 458 int i; 459 struct btrfs_fs_info *fs_info = dev->fs_info; 460 int ret; 461 462 sctx = kzalloc(sizeof(*sctx), GFP_KERNEL); 463 if (!sctx) 464 goto nomem; 465 atomic_set(&sctx->refs, 1); 466 sctx->is_dev_replace = is_dev_replace; 467 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; 468 sctx->curr = -1; 469 sctx->fs_info = dev->fs_info; 470 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { 471 struct scrub_bio *sbio; 472 473 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL); 474 if (!sbio) 475 goto nomem; 476 sctx->bios[i] = sbio; 477 478 sbio->index = i; 479 sbio->sctx = sctx; 480 sbio->page_count = 0; 481 btrfs_init_work(&sbio->work, btrfs_scrub_helper, 482 scrub_bio_end_io_worker, NULL, NULL); 483 484 if (i != SCRUB_BIOS_PER_SCTX - 1) 485 sctx->bios[i]->next_free = i + 1; 486 else 487 sctx->bios[i]->next_free = -1; 488 } 489 sctx->first_free = 0; 490 sctx->nodesize = fs_info->nodesize; 491 sctx->sectorsize = fs_info->sectorsize; 492 atomic_set(&sctx->bios_in_flight, 0); 493 atomic_set(&sctx->workers_pending, 0); 494 atomic_set(&sctx->cancel_req, 0); 495 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy); 496 INIT_LIST_HEAD(&sctx->csum_list); 497 498 spin_lock_init(&sctx->list_lock); 499 spin_lock_init(&sctx->stat_lock); 500 init_waitqueue_head(&sctx->list_wait); 501 502 ret = scrub_setup_wr_ctx(&sctx->wr_ctx, 503 fs_info->dev_replace.tgtdev, is_dev_replace); 504 if (ret) { 505 scrub_free_ctx(sctx); 506 return ERR_PTR(ret); 507 } 508 return sctx; 509 510 nomem: 511 scrub_free_ctx(sctx); 512 return ERR_PTR(-ENOMEM); 513 } 514 515 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, 516 void *warn_ctx) 517 { 518 u64 isize; 519 u32 nlink; 520 int ret; 521 int i; 522 struct extent_buffer *eb; 523 struct btrfs_inode_item *inode_item; 524 struct scrub_warning *swarn = warn_ctx; 525 struct btrfs_fs_info *fs_info = swarn->dev->fs_info; 526 struct inode_fs_paths *ipath = NULL; 527 struct btrfs_root *local_root; 528 struct btrfs_key root_key; 529 struct btrfs_key key; 530 531 root_key.objectid = root; 532 root_key.type = BTRFS_ROOT_ITEM_KEY; 533 root_key.offset = (u64)-1; 534 local_root = btrfs_read_fs_root_no_name(fs_info, &root_key); 535 if (IS_ERR(local_root)) { 536 ret = PTR_ERR(local_root); 537 goto err; 538 } 539 540 /* 541 * this makes the path point to (inum INODE_ITEM ioff) 542 */ 543 key.objectid = inum; 544 key.type = BTRFS_INODE_ITEM_KEY; 545 key.offset = 0; 546 547 ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0); 548 if (ret) { 549 btrfs_release_path(swarn->path); 550 goto err; 551 } 552 553 eb = swarn->path->nodes[0]; 554 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0], 555 struct btrfs_inode_item); 556 isize = btrfs_inode_size(eb, inode_item); 557 nlink = btrfs_inode_nlink(eb, inode_item); 558 btrfs_release_path(swarn->path); 559 560 ipath = init_ipath(4096, local_root, swarn->path); 561 if (IS_ERR(ipath)) { 562 ret = PTR_ERR(ipath); 563 ipath = NULL; 564 goto err; 565 } 566 ret = paths_from_inode(inum, ipath); 567 568 if (ret < 0) 569 goto err; 570 571 /* 572 * we deliberately ignore the bit ipath might have been too small to 573 * hold all of the paths here 574 */ 575 for (i = 0; i < ipath->fspath->elem_cnt; ++i) 576 btrfs_warn_in_rcu(fs_info, 577 "%s at logical %llu on dev %s, sector %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)", 578 swarn->errstr, swarn->logical, 579 rcu_str_deref(swarn->dev->name), 580 (unsigned long long)swarn->sector, 581 root, inum, offset, 582 min(isize - offset, (u64)PAGE_SIZE), nlink, 583 (char *)(unsigned long)ipath->fspath->val[i]); 584 585 free_ipath(ipath); 586 return 0; 587 588 err: 589 btrfs_warn_in_rcu(fs_info, 590 "%s at logical %llu on dev %s, sector %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d", 591 swarn->errstr, swarn->logical, 592 rcu_str_deref(swarn->dev->name), 593 (unsigned long long)swarn->sector, 594 root, inum, offset, ret); 595 596 free_ipath(ipath); 597 return 0; 598 } 599 600 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock) 601 { 602 struct btrfs_device *dev; 603 struct btrfs_fs_info *fs_info; 604 struct btrfs_path *path; 605 struct btrfs_key found_key; 606 struct extent_buffer *eb; 607 struct btrfs_extent_item *ei; 608 struct scrub_warning swarn; 609 unsigned long ptr = 0; 610 u64 extent_item_pos; 611 u64 flags = 0; 612 u64 ref_root; 613 u32 item_size; 614 u8 ref_level = 0; 615 int ret; 616 617 WARN_ON(sblock->page_count < 1); 618 dev = sblock->pagev[0]->dev; 619 fs_info = sblock->sctx->fs_info; 620 621 path = btrfs_alloc_path(); 622 if (!path) 623 return; 624 625 swarn.sector = (sblock->pagev[0]->physical) >> 9; 626 swarn.logical = sblock->pagev[0]->logical; 627 swarn.errstr = errstr; 628 swarn.dev = NULL; 629 630 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key, 631 &flags); 632 if (ret < 0) 633 goto out; 634 635 extent_item_pos = swarn.logical - found_key.objectid; 636 swarn.extent_item_size = found_key.offset; 637 638 eb = path->nodes[0]; 639 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); 640 item_size = btrfs_item_size_nr(eb, path->slots[0]); 641 642 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 643 do { 644 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei, 645 item_size, &ref_root, 646 &ref_level); 647 btrfs_warn_in_rcu(fs_info, 648 "%s at logical %llu on dev %s, sector %llu: metadata %s (level %d) in tree %llu", 649 errstr, swarn.logical, 650 rcu_str_deref(dev->name), 651 (unsigned long long)swarn.sector, 652 ref_level ? "node" : "leaf", 653 ret < 0 ? -1 : ref_level, 654 ret < 0 ? -1 : ref_root); 655 } while (ret != 1); 656 btrfs_release_path(path); 657 } else { 658 btrfs_release_path(path); 659 swarn.path = path; 660 swarn.dev = dev; 661 iterate_extent_inodes(fs_info, found_key.objectid, 662 extent_item_pos, 1, 663 scrub_print_warning_inode, &swarn); 664 } 665 666 out: 667 btrfs_free_path(path); 668 } 669 670 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx) 671 { 672 struct page *page = NULL; 673 unsigned long index; 674 struct scrub_fixup_nodatasum *fixup = fixup_ctx; 675 int ret; 676 int corrected = 0; 677 struct btrfs_key key; 678 struct inode *inode = NULL; 679 struct btrfs_fs_info *fs_info; 680 u64 end = offset + PAGE_SIZE - 1; 681 struct btrfs_root *local_root; 682 int srcu_index; 683 684 key.objectid = root; 685 key.type = BTRFS_ROOT_ITEM_KEY; 686 key.offset = (u64)-1; 687 688 fs_info = fixup->root->fs_info; 689 srcu_index = srcu_read_lock(&fs_info->subvol_srcu); 690 691 local_root = btrfs_read_fs_root_no_name(fs_info, &key); 692 if (IS_ERR(local_root)) { 693 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index); 694 return PTR_ERR(local_root); 695 } 696 697 key.type = BTRFS_INODE_ITEM_KEY; 698 key.objectid = inum; 699 key.offset = 0; 700 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL); 701 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index); 702 if (IS_ERR(inode)) 703 return PTR_ERR(inode); 704 705 index = offset >> PAGE_SHIFT; 706 707 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS); 708 if (!page) { 709 ret = -ENOMEM; 710 goto out; 711 } 712 713 if (PageUptodate(page)) { 714 if (PageDirty(page)) { 715 /* 716 * we need to write the data to the defect sector. the 717 * data that was in that sector is not in memory, 718 * because the page was modified. we must not write the 719 * modified page to that sector. 720 * 721 * TODO: what could be done here: wait for the delalloc 722 * runner to write out that page (might involve 723 * COW) and see whether the sector is still 724 * referenced afterwards. 725 * 726 * For the meantime, we'll treat this error 727 * incorrectable, although there is a chance that a 728 * later scrub will find the bad sector again and that 729 * there's no dirty page in memory, then. 730 */ 731 ret = -EIO; 732 goto out; 733 } 734 ret = repair_io_failure(BTRFS_I(inode), offset, PAGE_SIZE, 735 fixup->logical, page, 736 offset - page_offset(page), 737 fixup->mirror_num); 738 unlock_page(page); 739 corrected = !ret; 740 } else { 741 /* 742 * we need to get good data first. the general readpage path 743 * will call repair_io_failure for us, we just have to make 744 * sure we read the bad mirror. 745 */ 746 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end, 747 EXTENT_DAMAGED); 748 if (ret) { 749 /* set_extent_bits should give proper error */ 750 WARN_ON(ret > 0); 751 if (ret > 0) 752 ret = -EFAULT; 753 goto out; 754 } 755 756 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page, 757 btrfs_get_extent, 758 fixup->mirror_num); 759 wait_on_page_locked(page); 760 761 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset, 762 end, EXTENT_DAMAGED, 0, NULL); 763 if (!corrected) 764 clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end, 765 EXTENT_DAMAGED); 766 } 767 768 out: 769 if (page) 770 put_page(page); 771 772 iput(inode); 773 774 if (ret < 0) 775 return ret; 776 777 if (ret == 0 && corrected) { 778 /* 779 * we only need to call readpage for one of the inodes belonging 780 * to this extent. so make iterate_extent_inodes stop 781 */ 782 return 1; 783 } 784 785 return -EIO; 786 } 787 788 static void scrub_fixup_nodatasum(struct btrfs_work *work) 789 { 790 struct btrfs_fs_info *fs_info; 791 int ret; 792 struct scrub_fixup_nodatasum *fixup; 793 struct scrub_ctx *sctx; 794 struct btrfs_trans_handle *trans = NULL; 795 struct btrfs_path *path; 796 int uncorrectable = 0; 797 798 fixup = container_of(work, struct scrub_fixup_nodatasum, work); 799 sctx = fixup->sctx; 800 fs_info = fixup->root->fs_info; 801 802 path = btrfs_alloc_path(); 803 if (!path) { 804 spin_lock(&sctx->stat_lock); 805 ++sctx->stat.malloc_errors; 806 spin_unlock(&sctx->stat_lock); 807 uncorrectable = 1; 808 goto out; 809 } 810 811 trans = btrfs_join_transaction(fixup->root); 812 if (IS_ERR(trans)) { 813 uncorrectable = 1; 814 goto out; 815 } 816 817 /* 818 * the idea is to trigger a regular read through the standard path. we 819 * read a page from the (failed) logical address by specifying the 820 * corresponding copynum of the failed sector. thus, that readpage is 821 * expected to fail. 822 * that is the point where on-the-fly error correction will kick in 823 * (once it's finished) and rewrite the failed sector if a good copy 824 * can be found. 825 */ 826 ret = iterate_inodes_from_logical(fixup->logical, fs_info, path, 827 scrub_fixup_readpage, fixup); 828 if (ret < 0) { 829 uncorrectable = 1; 830 goto out; 831 } 832 WARN_ON(ret != 1); 833 834 spin_lock(&sctx->stat_lock); 835 ++sctx->stat.corrected_errors; 836 spin_unlock(&sctx->stat_lock); 837 838 out: 839 if (trans && !IS_ERR(trans)) 840 btrfs_end_transaction(trans); 841 if (uncorrectable) { 842 spin_lock(&sctx->stat_lock); 843 ++sctx->stat.uncorrectable_errors; 844 spin_unlock(&sctx->stat_lock); 845 btrfs_dev_replace_stats_inc( 846 &fs_info->dev_replace.num_uncorrectable_read_errors); 847 btrfs_err_rl_in_rcu(fs_info, 848 "unable to fixup (nodatasum) error at logical %llu on dev %s", 849 fixup->logical, rcu_str_deref(fixup->dev->name)); 850 } 851 852 btrfs_free_path(path); 853 kfree(fixup); 854 855 scrub_pending_trans_workers_dec(sctx); 856 } 857 858 static inline void scrub_get_recover(struct scrub_recover *recover) 859 { 860 atomic_inc(&recover->refs); 861 } 862 863 static inline void scrub_put_recover(struct scrub_recover *recover) 864 { 865 if (atomic_dec_and_test(&recover->refs)) { 866 btrfs_put_bbio(recover->bbio); 867 kfree(recover); 868 } 869 } 870 871 /* 872 * scrub_handle_errored_block gets called when either verification of the 873 * pages failed or the bio failed to read, e.g. with EIO. In the latter 874 * case, this function handles all pages in the bio, even though only one 875 * may be bad. 876 * The goal of this function is to repair the errored block by using the 877 * contents of one of the mirrors. 878 */ 879 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) 880 { 881 struct scrub_ctx *sctx = sblock_to_check->sctx; 882 struct btrfs_device *dev; 883 struct btrfs_fs_info *fs_info; 884 u64 length; 885 u64 logical; 886 unsigned int failed_mirror_index; 887 unsigned int is_metadata; 888 unsigned int have_csum; 889 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */ 890 struct scrub_block *sblock_bad; 891 int ret; 892 int mirror_index; 893 int page_num; 894 int success; 895 static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL, 896 DEFAULT_RATELIMIT_BURST); 897 898 BUG_ON(sblock_to_check->page_count < 1); 899 fs_info = sctx->fs_info; 900 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) { 901 /* 902 * if we find an error in a super block, we just report it. 903 * They will get written with the next transaction commit 904 * anyway 905 */ 906 spin_lock(&sctx->stat_lock); 907 ++sctx->stat.super_errors; 908 spin_unlock(&sctx->stat_lock); 909 return 0; 910 } 911 length = sblock_to_check->page_count * PAGE_SIZE; 912 logical = sblock_to_check->pagev[0]->logical; 913 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1); 914 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1; 915 is_metadata = !(sblock_to_check->pagev[0]->flags & 916 BTRFS_EXTENT_FLAG_DATA); 917 have_csum = sblock_to_check->pagev[0]->have_csum; 918 dev = sblock_to_check->pagev[0]->dev; 919 920 if (sctx->is_dev_replace && !is_metadata && !have_csum) { 921 sblocks_for_recheck = NULL; 922 goto nodatasum_case; 923 } 924 925 /* 926 * read all mirrors one after the other. This includes to 927 * re-read the extent or metadata block that failed (that was 928 * the cause that this fixup code is called) another time, 929 * page by page this time in order to know which pages 930 * caused I/O errors and which ones are good (for all mirrors). 931 * It is the goal to handle the situation when more than one 932 * mirror contains I/O errors, but the errors do not 933 * overlap, i.e. the data can be repaired by selecting the 934 * pages from those mirrors without I/O error on the 935 * particular pages. One example (with blocks >= 2 * PAGE_SIZE) 936 * would be that mirror #1 has an I/O error on the first page, 937 * the second page is good, and mirror #2 has an I/O error on 938 * the second page, but the first page is good. 939 * Then the first page of the first mirror can be repaired by 940 * taking the first page of the second mirror, and the 941 * second page of the second mirror can be repaired by 942 * copying the contents of the 2nd page of the 1st mirror. 943 * One more note: if the pages of one mirror contain I/O 944 * errors, the checksum cannot be verified. In order to get 945 * the best data for repairing, the first attempt is to find 946 * a mirror without I/O errors and with a validated checksum. 947 * Only if this is not possible, the pages are picked from 948 * mirrors with I/O errors without considering the checksum. 949 * If the latter is the case, at the end, the checksum of the 950 * repaired area is verified in order to correctly maintain 951 * the statistics. 952 */ 953 954 sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS, 955 sizeof(*sblocks_for_recheck), GFP_NOFS); 956 if (!sblocks_for_recheck) { 957 spin_lock(&sctx->stat_lock); 958 sctx->stat.malloc_errors++; 959 sctx->stat.read_errors++; 960 sctx->stat.uncorrectable_errors++; 961 spin_unlock(&sctx->stat_lock); 962 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 963 goto out; 964 } 965 966 /* setup the context, map the logical blocks and alloc the pages */ 967 ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck); 968 if (ret) { 969 spin_lock(&sctx->stat_lock); 970 sctx->stat.read_errors++; 971 sctx->stat.uncorrectable_errors++; 972 spin_unlock(&sctx->stat_lock); 973 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 974 goto out; 975 } 976 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS); 977 sblock_bad = sblocks_for_recheck + failed_mirror_index; 978 979 /* build and submit the bios for the failed mirror, check checksums */ 980 scrub_recheck_block(fs_info, sblock_bad, 1); 981 982 if (!sblock_bad->header_error && !sblock_bad->checksum_error && 983 sblock_bad->no_io_error_seen) { 984 /* 985 * the error disappeared after reading page by page, or 986 * the area was part of a huge bio and other parts of the 987 * bio caused I/O errors, or the block layer merged several 988 * read requests into one and the error is caused by a 989 * different bio (usually one of the two latter cases is 990 * the cause) 991 */ 992 spin_lock(&sctx->stat_lock); 993 sctx->stat.unverified_errors++; 994 sblock_to_check->data_corrected = 1; 995 spin_unlock(&sctx->stat_lock); 996 997 if (sctx->is_dev_replace) 998 scrub_write_block_to_dev_replace(sblock_bad); 999 goto out; 1000 } 1001 1002 if (!sblock_bad->no_io_error_seen) { 1003 spin_lock(&sctx->stat_lock); 1004 sctx->stat.read_errors++; 1005 spin_unlock(&sctx->stat_lock); 1006 if (__ratelimit(&_rs)) 1007 scrub_print_warning("i/o error", sblock_to_check); 1008 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 1009 } else if (sblock_bad->checksum_error) { 1010 spin_lock(&sctx->stat_lock); 1011 sctx->stat.csum_errors++; 1012 spin_unlock(&sctx->stat_lock); 1013 if (__ratelimit(&_rs)) 1014 scrub_print_warning("checksum error", sblock_to_check); 1015 btrfs_dev_stat_inc_and_print(dev, 1016 BTRFS_DEV_STAT_CORRUPTION_ERRS); 1017 } else if (sblock_bad->header_error) { 1018 spin_lock(&sctx->stat_lock); 1019 sctx->stat.verify_errors++; 1020 spin_unlock(&sctx->stat_lock); 1021 if (__ratelimit(&_rs)) 1022 scrub_print_warning("checksum/header error", 1023 sblock_to_check); 1024 if (sblock_bad->generation_error) 1025 btrfs_dev_stat_inc_and_print(dev, 1026 BTRFS_DEV_STAT_GENERATION_ERRS); 1027 else 1028 btrfs_dev_stat_inc_and_print(dev, 1029 BTRFS_DEV_STAT_CORRUPTION_ERRS); 1030 } 1031 1032 if (sctx->readonly) { 1033 ASSERT(!sctx->is_dev_replace); 1034 goto out; 1035 } 1036 1037 if (!is_metadata && !have_csum) { 1038 struct scrub_fixup_nodatasum *fixup_nodatasum; 1039 1040 WARN_ON(sctx->is_dev_replace); 1041 1042 nodatasum_case: 1043 1044 /* 1045 * !is_metadata and !have_csum, this means that the data 1046 * might not be COWed, that it might be modified 1047 * concurrently. The general strategy to work on the 1048 * commit root does not help in the case when COW is not 1049 * used. 1050 */ 1051 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS); 1052 if (!fixup_nodatasum) 1053 goto did_not_correct_error; 1054 fixup_nodatasum->sctx = sctx; 1055 fixup_nodatasum->dev = dev; 1056 fixup_nodatasum->logical = logical; 1057 fixup_nodatasum->root = fs_info->extent_root; 1058 fixup_nodatasum->mirror_num = failed_mirror_index + 1; 1059 scrub_pending_trans_workers_inc(sctx); 1060 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper, 1061 scrub_fixup_nodatasum, NULL, NULL); 1062 btrfs_queue_work(fs_info->scrub_workers, 1063 &fixup_nodatasum->work); 1064 goto out; 1065 } 1066 1067 /* 1068 * now build and submit the bios for the other mirrors, check 1069 * checksums. 1070 * First try to pick the mirror which is completely without I/O 1071 * errors and also does not have a checksum error. 1072 * If one is found, and if a checksum is present, the full block 1073 * that is known to contain an error is rewritten. Afterwards 1074 * the block is known to be corrected. 1075 * If a mirror is found which is completely correct, and no 1076 * checksum is present, only those pages are rewritten that had 1077 * an I/O error in the block to be repaired, since it cannot be 1078 * determined, which copy of the other pages is better (and it 1079 * could happen otherwise that a correct page would be 1080 * overwritten by a bad one). 1081 */ 1082 for (mirror_index = 0; 1083 mirror_index < BTRFS_MAX_MIRRORS && 1084 sblocks_for_recheck[mirror_index].page_count > 0; 1085 mirror_index++) { 1086 struct scrub_block *sblock_other; 1087 1088 if (mirror_index == failed_mirror_index) 1089 continue; 1090 sblock_other = sblocks_for_recheck + mirror_index; 1091 1092 /* build and submit the bios, check checksums */ 1093 scrub_recheck_block(fs_info, sblock_other, 0); 1094 1095 if (!sblock_other->header_error && 1096 !sblock_other->checksum_error && 1097 sblock_other->no_io_error_seen) { 1098 if (sctx->is_dev_replace) { 1099 scrub_write_block_to_dev_replace(sblock_other); 1100 goto corrected_error; 1101 } else { 1102 ret = scrub_repair_block_from_good_copy( 1103 sblock_bad, sblock_other); 1104 if (!ret) 1105 goto corrected_error; 1106 } 1107 } 1108 } 1109 1110 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace) 1111 goto did_not_correct_error; 1112 1113 /* 1114 * In case of I/O errors in the area that is supposed to be 1115 * repaired, continue by picking good copies of those pages. 1116 * Select the good pages from mirrors to rewrite bad pages from 1117 * the area to fix. Afterwards verify the checksum of the block 1118 * that is supposed to be repaired. This verification step is 1119 * only done for the purpose of statistic counting and for the 1120 * final scrub report, whether errors remain. 1121 * A perfect algorithm could make use of the checksum and try 1122 * all possible combinations of pages from the different mirrors 1123 * until the checksum verification succeeds. For example, when 1124 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page 1125 * of mirror #2 is readable but the final checksum test fails, 1126 * then the 2nd page of mirror #3 could be tried, whether now 1127 * the final checksum succeeds. But this would be a rare 1128 * exception and is therefore not implemented. At least it is 1129 * avoided that the good copy is overwritten. 1130 * A more useful improvement would be to pick the sectors 1131 * without I/O error based on sector sizes (512 bytes on legacy 1132 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one 1133 * mirror could be repaired by taking 512 byte of a different 1134 * mirror, even if other 512 byte sectors in the same PAGE_SIZE 1135 * area are unreadable. 1136 */ 1137 success = 1; 1138 for (page_num = 0; page_num < sblock_bad->page_count; 1139 page_num++) { 1140 struct scrub_page *page_bad = sblock_bad->pagev[page_num]; 1141 struct scrub_block *sblock_other = NULL; 1142 1143 /* skip no-io-error page in scrub */ 1144 if (!page_bad->io_error && !sctx->is_dev_replace) 1145 continue; 1146 1147 /* try to find no-io-error page in mirrors */ 1148 if (page_bad->io_error) { 1149 for (mirror_index = 0; 1150 mirror_index < BTRFS_MAX_MIRRORS && 1151 sblocks_for_recheck[mirror_index].page_count > 0; 1152 mirror_index++) { 1153 if (!sblocks_for_recheck[mirror_index]. 1154 pagev[page_num]->io_error) { 1155 sblock_other = sblocks_for_recheck + 1156 mirror_index; 1157 break; 1158 } 1159 } 1160 if (!sblock_other) 1161 success = 0; 1162 } 1163 1164 if (sctx->is_dev_replace) { 1165 /* 1166 * did not find a mirror to fetch the page 1167 * from. scrub_write_page_to_dev_replace() 1168 * handles this case (page->io_error), by 1169 * filling the block with zeros before 1170 * submitting the write request 1171 */ 1172 if (!sblock_other) 1173 sblock_other = sblock_bad; 1174 1175 if (scrub_write_page_to_dev_replace(sblock_other, 1176 page_num) != 0) { 1177 btrfs_dev_replace_stats_inc( 1178 &fs_info->dev_replace.num_write_errors); 1179 success = 0; 1180 } 1181 } else if (sblock_other) { 1182 ret = scrub_repair_page_from_good_copy(sblock_bad, 1183 sblock_other, 1184 page_num, 0); 1185 if (0 == ret) 1186 page_bad->io_error = 0; 1187 else 1188 success = 0; 1189 } 1190 } 1191 1192 if (success && !sctx->is_dev_replace) { 1193 if (is_metadata || have_csum) { 1194 /* 1195 * need to verify the checksum now that all 1196 * sectors on disk are repaired (the write 1197 * request for data to be repaired is on its way). 1198 * Just be lazy and use scrub_recheck_block() 1199 * which re-reads the data before the checksum 1200 * is verified, but most likely the data comes out 1201 * of the page cache. 1202 */ 1203 scrub_recheck_block(fs_info, sblock_bad, 1); 1204 if (!sblock_bad->header_error && 1205 !sblock_bad->checksum_error && 1206 sblock_bad->no_io_error_seen) 1207 goto corrected_error; 1208 else 1209 goto did_not_correct_error; 1210 } else { 1211 corrected_error: 1212 spin_lock(&sctx->stat_lock); 1213 sctx->stat.corrected_errors++; 1214 sblock_to_check->data_corrected = 1; 1215 spin_unlock(&sctx->stat_lock); 1216 btrfs_err_rl_in_rcu(fs_info, 1217 "fixed up error at logical %llu on dev %s", 1218 logical, rcu_str_deref(dev->name)); 1219 } 1220 } else { 1221 did_not_correct_error: 1222 spin_lock(&sctx->stat_lock); 1223 sctx->stat.uncorrectable_errors++; 1224 spin_unlock(&sctx->stat_lock); 1225 btrfs_err_rl_in_rcu(fs_info, 1226 "unable to fixup (regular) error at logical %llu on dev %s", 1227 logical, rcu_str_deref(dev->name)); 1228 } 1229 1230 out: 1231 if (sblocks_for_recheck) { 1232 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS; 1233 mirror_index++) { 1234 struct scrub_block *sblock = sblocks_for_recheck + 1235 mirror_index; 1236 struct scrub_recover *recover; 1237 int page_index; 1238 1239 for (page_index = 0; page_index < sblock->page_count; 1240 page_index++) { 1241 sblock->pagev[page_index]->sblock = NULL; 1242 recover = sblock->pagev[page_index]->recover; 1243 if (recover) { 1244 scrub_put_recover(recover); 1245 sblock->pagev[page_index]->recover = 1246 NULL; 1247 } 1248 scrub_page_put(sblock->pagev[page_index]); 1249 } 1250 } 1251 kfree(sblocks_for_recheck); 1252 } 1253 1254 return 0; 1255 } 1256 1257 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio) 1258 { 1259 if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5) 1260 return 2; 1261 else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6) 1262 return 3; 1263 else 1264 return (int)bbio->num_stripes; 1265 } 1266 1267 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type, 1268 u64 *raid_map, 1269 u64 mapped_length, 1270 int nstripes, int mirror, 1271 int *stripe_index, 1272 u64 *stripe_offset) 1273 { 1274 int i; 1275 1276 if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 1277 /* RAID5/6 */ 1278 for (i = 0; i < nstripes; i++) { 1279 if (raid_map[i] == RAID6_Q_STRIPE || 1280 raid_map[i] == RAID5_P_STRIPE) 1281 continue; 1282 1283 if (logical >= raid_map[i] && 1284 logical < raid_map[i] + mapped_length) 1285 break; 1286 } 1287 1288 *stripe_index = i; 1289 *stripe_offset = logical - raid_map[i]; 1290 } else { 1291 /* The other RAID type */ 1292 *stripe_index = mirror; 1293 *stripe_offset = 0; 1294 } 1295 } 1296 1297 static int scrub_setup_recheck_block(struct scrub_block *original_sblock, 1298 struct scrub_block *sblocks_for_recheck) 1299 { 1300 struct scrub_ctx *sctx = original_sblock->sctx; 1301 struct btrfs_fs_info *fs_info = sctx->fs_info; 1302 u64 length = original_sblock->page_count * PAGE_SIZE; 1303 u64 logical = original_sblock->pagev[0]->logical; 1304 u64 generation = original_sblock->pagev[0]->generation; 1305 u64 flags = original_sblock->pagev[0]->flags; 1306 u64 have_csum = original_sblock->pagev[0]->have_csum; 1307 struct scrub_recover *recover; 1308 struct btrfs_bio *bbio; 1309 u64 sublen; 1310 u64 mapped_length; 1311 u64 stripe_offset; 1312 int stripe_index; 1313 int page_index = 0; 1314 int mirror_index; 1315 int nmirrors; 1316 int ret; 1317 1318 /* 1319 * note: the two members refs and outstanding_pages 1320 * are not used (and not set) in the blocks that are used for 1321 * the recheck procedure 1322 */ 1323 1324 while (length > 0) { 1325 sublen = min_t(u64, length, PAGE_SIZE); 1326 mapped_length = sublen; 1327 bbio = NULL; 1328 1329 /* 1330 * with a length of PAGE_SIZE, each returned stripe 1331 * represents one mirror 1332 */ 1333 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, 1334 logical, &mapped_length, &bbio, 0, 1); 1335 if (ret || !bbio || mapped_length < sublen) { 1336 btrfs_put_bbio(bbio); 1337 return -EIO; 1338 } 1339 1340 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS); 1341 if (!recover) { 1342 btrfs_put_bbio(bbio); 1343 return -ENOMEM; 1344 } 1345 1346 atomic_set(&recover->refs, 1); 1347 recover->bbio = bbio; 1348 recover->map_length = mapped_length; 1349 1350 BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK); 1351 1352 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS); 1353 1354 for (mirror_index = 0; mirror_index < nmirrors; 1355 mirror_index++) { 1356 struct scrub_block *sblock; 1357 struct scrub_page *page; 1358 1359 sblock = sblocks_for_recheck + mirror_index; 1360 sblock->sctx = sctx; 1361 1362 page = kzalloc(sizeof(*page), GFP_NOFS); 1363 if (!page) { 1364 leave_nomem: 1365 spin_lock(&sctx->stat_lock); 1366 sctx->stat.malloc_errors++; 1367 spin_unlock(&sctx->stat_lock); 1368 scrub_put_recover(recover); 1369 return -ENOMEM; 1370 } 1371 scrub_page_get(page); 1372 sblock->pagev[page_index] = page; 1373 page->sblock = sblock; 1374 page->flags = flags; 1375 page->generation = generation; 1376 page->logical = logical; 1377 page->have_csum = have_csum; 1378 if (have_csum) 1379 memcpy(page->csum, 1380 original_sblock->pagev[0]->csum, 1381 sctx->csum_size); 1382 1383 scrub_stripe_index_and_offset(logical, 1384 bbio->map_type, 1385 bbio->raid_map, 1386 mapped_length, 1387 bbio->num_stripes - 1388 bbio->num_tgtdevs, 1389 mirror_index, 1390 &stripe_index, 1391 &stripe_offset); 1392 page->physical = bbio->stripes[stripe_index].physical + 1393 stripe_offset; 1394 page->dev = bbio->stripes[stripe_index].dev; 1395 1396 BUG_ON(page_index >= original_sblock->page_count); 1397 page->physical_for_dev_replace = 1398 original_sblock->pagev[page_index]-> 1399 physical_for_dev_replace; 1400 /* for missing devices, dev->bdev is NULL */ 1401 page->mirror_num = mirror_index + 1; 1402 sblock->page_count++; 1403 page->page = alloc_page(GFP_NOFS); 1404 if (!page->page) 1405 goto leave_nomem; 1406 1407 scrub_get_recover(recover); 1408 page->recover = recover; 1409 } 1410 scrub_put_recover(recover); 1411 length -= sublen; 1412 logical += sublen; 1413 page_index++; 1414 } 1415 1416 return 0; 1417 } 1418 1419 struct scrub_bio_ret { 1420 struct completion event; 1421 int error; 1422 }; 1423 1424 static void scrub_bio_wait_endio(struct bio *bio) 1425 { 1426 struct scrub_bio_ret *ret = bio->bi_private; 1427 1428 ret->error = bio->bi_error; 1429 complete(&ret->event); 1430 } 1431 1432 static inline int scrub_is_page_on_raid56(struct scrub_page *page) 1433 { 1434 return page->recover && 1435 (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK); 1436 } 1437 1438 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info, 1439 struct bio *bio, 1440 struct scrub_page *page) 1441 { 1442 struct scrub_bio_ret done; 1443 int ret; 1444 1445 init_completion(&done.event); 1446 done.error = 0; 1447 bio->bi_iter.bi_sector = page->logical >> 9; 1448 bio->bi_private = &done; 1449 bio->bi_end_io = scrub_bio_wait_endio; 1450 1451 ret = raid56_parity_recover(fs_info, bio, page->recover->bbio, 1452 page->recover->map_length, 1453 page->mirror_num, 0); 1454 if (ret) 1455 return ret; 1456 1457 wait_for_completion(&done.event); 1458 if (done.error) 1459 return -EIO; 1460 1461 return 0; 1462 } 1463 1464 /* 1465 * this function will check the on disk data for checksum errors, header 1466 * errors and read I/O errors. If any I/O errors happen, the exact pages 1467 * which are errored are marked as being bad. The goal is to enable scrub 1468 * to take those pages that are not errored from all the mirrors so that 1469 * the pages that are errored in the just handled mirror can be repaired. 1470 */ 1471 static void scrub_recheck_block(struct btrfs_fs_info *fs_info, 1472 struct scrub_block *sblock, 1473 int retry_failed_mirror) 1474 { 1475 int page_num; 1476 1477 sblock->no_io_error_seen = 1; 1478 1479 for (page_num = 0; page_num < sblock->page_count; page_num++) { 1480 struct bio *bio; 1481 struct scrub_page *page = sblock->pagev[page_num]; 1482 1483 if (page->dev->bdev == NULL) { 1484 page->io_error = 1; 1485 sblock->no_io_error_seen = 0; 1486 continue; 1487 } 1488 1489 WARN_ON(!page->page); 1490 bio = btrfs_io_bio_alloc(GFP_NOFS, 1); 1491 if (!bio) { 1492 page->io_error = 1; 1493 sblock->no_io_error_seen = 0; 1494 continue; 1495 } 1496 bio->bi_bdev = page->dev->bdev; 1497 1498 bio_add_page(bio, page->page, PAGE_SIZE, 0); 1499 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) { 1500 if (scrub_submit_raid56_bio_wait(fs_info, bio, page)) 1501 sblock->no_io_error_seen = 0; 1502 } else { 1503 bio->bi_iter.bi_sector = page->physical >> 9; 1504 bio_set_op_attrs(bio, REQ_OP_READ, 0); 1505 1506 if (btrfsic_submit_bio_wait(bio)) 1507 sblock->no_io_error_seen = 0; 1508 } 1509 1510 bio_put(bio); 1511 } 1512 1513 if (sblock->no_io_error_seen) 1514 scrub_recheck_block_checksum(sblock); 1515 } 1516 1517 static inline int scrub_check_fsid(u8 fsid[], 1518 struct scrub_page *spage) 1519 { 1520 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices; 1521 int ret; 1522 1523 ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE); 1524 return !ret; 1525 } 1526 1527 static void scrub_recheck_block_checksum(struct scrub_block *sblock) 1528 { 1529 sblock->header_error = 0; 1530 sblock->checksum_error = 0; 1531 sblock->generation_error = 0; 1532 1533 if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA) 1534 scrub_checksum_data(sblock); 1535 else 1536 scrub_checksum_tree_block(sblock); 1537 } 1538 1539 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad, 1540 struct scrub_block *sblock_good) 1541 { 1542 int page_num; 1543 int ret = 0; 1544 1545 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) { 1546 int ret_sub; 1547 1548 ret_sub = scrub_repair_page_from_good_copy(sblock_bad, 1549 sblock_good, 1550 page_num, 1); 1551 if (ret_sub) 1552 ret = ret_sub; 1553 } 1554 1555 return ret; 1556 } 1557 1558 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad, 1559 struct scrub_block *sblock_good, 1560 int page_num, int force_write) 1561 { 1562 struct scrub_page *page_bad = sblock_bad->pagev[page_num]; 1563 struct scrub_page *page_good = sblock_good->pagev[page_num]; 1564 struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; 1565 1566 BUG_ON(page_bad->page == NULL); 1567 BUG_ON(page_good->page == NULL); 1568 if (force_write || sblock_bad->header_error || 1569 sblock_bad->checksum_error || page_bad->io_error) { 1570 struct bio *bio; 1571 int ret; 1572 1573 if (!page_bad->dev->bdev) { 1574 btrfs_warn_rl(fs_info, 1575 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected"); 1576 return -EIO; 1577 } 1578 1579 bio = btrfs_io_bio_alloc(GFP_NOFS, 1); 1580 if (!bio) 1581 return -EIO; 1582 bio->bi_bdev = page_bad->dev->bdev; 1583 bio->bi_iter.bi_sector = page_bad->physical >> 9; 1584 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 1585 1586 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0); 1587 if (PAGE_SIZE != ret) { 1588 bio_put(bio); 1589 return -EIO; 1590 } 1591 1592 if (btrfsic_submit_bio_wait(bio)) { 1593 btrfs_dev_stat_inc_and_print(page_bad->dev, 1594 BTRFS_DEV_STAT_WRITE_ERRS); 1595 btrfs_dev_replace_stats_inc( 1596 &fs_info->dev_replace.num_write_errors); 1597 bio_put(bio); 1598 return -EIO; 1599 } 1600 bio_put(bio); 1601 } 1602 1603 return 0; 1604 } 1605 1606 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock) 1607 { 1608 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; 1609 int page_num; 1610 1611 /* 1612 * This block is used for the check of the parity on the source device, 1613 * so the data needn't be written into the destination device. 1614 */ 1615 if (sblock->sparity) 1616 return; 1617 1618 for (page_num = 0; page_num < sblock->page_count; page_num++) { 1619 int ret; 1620 1621 ret = scrub_write_page_to_dev_replace(sblock, page_num); 1622 if (ret) 1623 btrfs_dev_replace_stats_inc( 1624 &fs_info->dev_replace.num_write_errors); 1625 } 1626 } 1627 1628 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock, 1629 int page_num) 1630 { 1631 struct scrub_page *spage = sblock->pagev[page_num]; 1632 1633 BUG_ON(spage->page == NULL); 1634 if (spage->io_error) { 1635 void *mapped_buffer = kmap_atomic(spage->page); 1636 1637 memset(mapped_buffer, 0, PAGE_SIZE); 1638 flush_dcache_page(spage->page); 1639 kunmap_atomic(mapped_buffer); 1640 } 1641 return scrub_add_page_to_wr_bio(sblock->sctx, spage); 1642 } 1643 1644 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, 1645 struct scrub_page *spage) 1646 { 1647 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx; 1648 struct scrub_bio *sbio; 1649 int ret; 1650 1651 mutex_lock(&wr_ctx->wr_lock); 1652 again: 1653 if (!wr_ctx->wr_curr_bio) { 1654 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio), 1655 GFP_KERNEL); 1656 if (!wr_ctx->wr_curr_bio) { 1657 mutex_unlock(&wr_ctx->wr_lock); 1658 return -ENOMEM; 1659 } 1660 wr_ctx->wr_curr_bio->sctx = sctx; 1661 wr_ctx->wr_curr_bio->page_count = 0; 1662 } 1663 sbio = wr_ctx->wr_curr_bio; 1664 if (sbio->page_count == 0) { 1665 struct bio *bio; 1666 1667 sbio->physical = spage->physical_for_dev_replace; 1668 sbio->logical = spage->logical; 1669 sbio->dev = wr_ctx->tgtdev; 1670 bio = sbio->bio; 1671 if (!bio) { 1672 bio = btrfs_io_bio_alloc(GFP_KERNEL, 1673 wr_ctx->pages_per_wr_bio); 1674 if (!bio) { 1675 mutex_unlock(&wr_ctx->wr_lock); 1676 return -ENOMEM; 1677 } 1678 sbio->bio = bio; 1679 } 1680 1681 bio->bi_private = sbio; 1682 bio->bi_end_io = scrub_wr_bio_end_io; 1683 bio->bi_bdev = sbio->dev->bdev; 1684 bio->bi_iter.bi_sector = sbio->physical >> 9; 1685 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 1686 sbio->err = 0; 1687 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != 1688 spage->physical_for_dev_replace || 1689 sbio->logical + sbio->page_count * PAGE_SIZE != 1690 spage->logical) { 1691 scrub_wr_submit(sctx); 1692 goto again; 1693 } 1694 1695 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); 1696 if (ret != PAGE_SIZE) { 1697 if (sbio->page_count < 1) { 1698 bio_put(sbio->bio); 1699 sbio->bio = NULL; 1700 mutex_unlock(&wr_ctx->wr_lock); 1701 return -EIO; 1702 } 1703 scrub_wr_submit(sctx); 1704 goto again; 1705 } 1706 1707 sbio->pagev[sbio->page_count] = spage; 1708 scrub_page_get(spage); 1709 sbio->page_count++; 1710 if (sbio->page_count == wr_ctx->pages_per_wr_bio) 1711 scrub_wr_submit(sctx); 1712 mutex_unlock(&wr_ctx->wr_lock); 1713 1714 return 0; 1715 } 1716 1717 static void scrub_wr_submit(struct scrub_ctx *sctx) 1718 { 1719 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx; 1720 struct scrub_bio *sbio; 1721 1722 if (!wr_ctx->wr_curr_bio) 1723 return; 1724 1725 sbio = wr_ctx->wr_curr_bio; 1726 wr_ctx->wr_curr_bio = NULL; 1727 WARN_ON(!sbio->bio->bi_bdev); 1728 scrub_pending_bio_inc(sctx); 1729 /* process all writes in a single worker thread. Then the block layer 1730 * orders the requests before sending them to the driver which 1731 * doubled the write performance on spinning disks when measured 1732 * with Linux 3.5 */ 1733 btrfsic_submit_bio(sbio->bio); 1734 } 1735 1736 static void scrub_wr_bio_end_io(struct bio *bio) 1737 { 1738 struct scrub_bio *sbio = bio->bi_private; 1739 struct btrfs_fs_info *fs_info = sbio->dev->fs_info; 1740 1741 sbio->err = bio->bi_error; 1742 sbio->bio = bio; 1743 1744 btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper, 1745 scrub_wr_bio_end_io_worker, NULL, NULL); 1746 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work); 1747 } 1748 1749 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work) 1750 { 1751 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work); 1752 struct scrub_ctx *sctx = sbio->sctx; 1753 int i; 1754 1755 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO); 1756 if (sbio->err) { 1757 struct btrfs_dev_replace *dev_replace = 1758 &sbio->sctx->fs_info->dev_replace; 1759 1760 for (i = 0; i < sbio->page_count; i++) { 1761 struct scrub_page *spage = sbio->pagev[i]; 1762 1763 spage->io_error = 1; 1764 btrfs_dev_replace_stats_inc(&dev_replace-> 1765 num_write_errors); 1766 } 1767 } 1768 1769 for (i = 0; i < sbio->page_count; i++) 1770 scrub_page_put(sbio->pagev[i]); 1771 1772 bio_put(sbio->bio); 1773 kfree(sbio); 1774 scrub_pending_bio_dec(sctx); 1775 } 1776 1777 static int scrub_checksum(struct scrub_block *sblock) 1778 { 1779 u64 flags; 1780 int ret; 1781 1782 /* 1783 * No need to initialize these stats currently, 1784 * because this function only use return value 1785 * instead of these stats value. 1786 * 1787 * Todo: 1788 * always use stats 1789 */ 1790 sblock->header_error = 0; 1791 sblock->generation_error = 0; 1792 sblock->checksum_error = 0; 1793 1794 WARN_ON(sblock->page_count < 1); 1795 flags = sblock->pagev[0]->flags; 1796 ret = 0; 1797 if (flags & BTRFS_EXTENT_FLAG_DATA) 1798 ret = scrub_checksum_data(sblock); 1799 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) 1800 ret = scrub_checksum_tree_block(sblock); 1801 else if (flags & BTRFS_EXTENT_FLAG_SUPER) 1802 (void)scrub_checksum_super(sblock); 1803 else 1804 WARN_ON(1); 1805 if (ret) 1806 scrub_handle_errored_block(sblock); 1807 1808 return ret; 1809 } 1810 1811 static int scrub_checksum_data(struct scrub_block *sblock) 1812 { 1813 struct scrub_ctx *sctx = sblock->sctx; 1814 u8 csum[BTRFS_CSUM_SIZE]; 1815 u8 *on_disk_csum; 1816 struct page *page; 1817 void *buffer; 1818 u32 crc = ~(u32)0; 1819 u64 len; 1820 int index; 1821 1822 BUG_ON(sblock->page_count < 1); 1823 if (!sblock->pagev[0]->have_csum) 1824 return 0; 1825 1826 on_disk_csum = sblock->pagev[0]->csum; 1827 page = sblock->pagev[0]->page; 1828 buffer = kmap_atomic(page); 1829 1830 len = sctx->sectorsize; 1831 index = 0; 1832 for (;;) { 1833 u64 l = min_t(u64, len, PAGE_SIZE); 1834 1835 crc = btrfs_csum_data(buffer, crc, l); 1836 kunmap_atomic(buffer); 1837 len -= l; 1838 if (len == 0) 1839 break; 1840 index++; 1841 BUG_ON(index >= sblock->page_count); 1842 BUG_ON(!sblock->pagev[index]->page); 1843 page = sblock->pagev[index]->page; 1844 buffer = kmap_atomic(page); 1845 } 1846 1847 btrfs_csum_final(crc, csum); 1848 if (memcmp(csum, on_disk_csum, sctx->csum_size)) 1849 sblock->checksum_error = 1; 1850 1851 return sblock->checksum_error; 1852 } 1853 1854 static int scrub_checksum_tree_block(struct scrub_block *sblock) 1855 { 1856 struct scrub_ctx *sctx = sblock->sctx; 1857 struct btrfs_header *h; 1858 struct btrfs_fs_info *fs_info = sctx->fs_info; 1859 u8 calculated_csum[BTRFS_CSUM_SIZE]; 1860 u8 on_disk_csum[BTRFS_CSUM_SIZE]; 1861 struct page *page; 1862 void *mapped_buffer; 1863 u64 mapped_size; 1864 void *p; 1865 u32 crc = ~(u32)0; 1866 u64 len; 1867 int index; 1868 1869 BUG_ON(sblock->page_count < 1); 1870 page = sblock->pagev[0]->page; 1871 mapped_buffer = kmap_atomic(page); 1872 h = (struct btrfs_header *)mapped_buffer; 1873 memcpy(on_disk_csum, h->csum, sctx->csum_size); 1874 1875 /* 1876 * we don't use the getter functions here, as we 1877 * a) don't have an extent buffer and 1878 * b) the page is already kmapped 1879 */ 1880 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h)) 1881 sblock->header_error = 1; 1882 1883 if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) { 1884 sblock->header_error = 1; 1885 sblock->generation_error = 1; 1886 } 1887 1888 if (!scrub_check_fsid(h->fsid, sblock->pagev[0])) 1889 sblock->header_error = 1; 1890 1891 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid, 1892 BTRFS_UUID_SIZE)) 1893 sblock->header_error = 1; 1894 1895 len = sctx->nodesize - BTRFS_CSUM_SIZE; 1896 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE; 1897 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE; 1898 index = 0; 1899 for (;;) { 1900 u64 l = min_t(u64, len, mapped_size); 1901 1902 crc = btrfs_csum_data(p, crc, l); 1903 kunmap_atomic(mapped_buffer); 1904 len -= l; 1905 if (len == 0) 1906 break; 1907 index++; 1908 BUG_ON(index >= sblock->page_count); 1909 BUG_ON(!sblock->pagev[index]->page); 1910 page = sblock->pagev[index]->page; 1911 mapped_buffer = kmap_atomic(page); 1912 mapped_size = PAGE_SIZE; 1913 p = mapped_buffer; 1914 } 1915 1916 btrfs_csum_final(crc, calculated_csum); 1917 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) 1918 sblock->checksum_error = 1; 1919 1920 return sblock->header_error || sblock->checksum_error; 1921 } 1922 1923 static int scrub_checksum_super(struct scrub_block *sblock) 1924 { 1925 struct btrfs_super_block *s; 1926 struct scrub_ctx *sctx = sblock->sctx; 1927 u8 calculated_csum[BTRFS_CSUM_SIZE]; 1928 u8 on_disk_csum[BTRFS_CSUM_SIZE]; 1929 struct page *page; 1930 void *mapped_buffer; 1931 u64 mapped_size; 1932 void *p; 1933 u32 crc = ~(u32)0; 1934 int fail_gen = 0; 1935 int fail_cor = 0; 1936 u64 len; 1937 int index; 1938 1939 BUG_ON(sblock->page_count < 1); 1940 page = sblock->pagev[0]->page; 1941 mapped_buffer = kmap_atomic(page); 1942 s = (struct btrfs_super_block *)mapped_buffer; 1943 memcpy(on_disk_csum, s->csum, sctx->csum_size); 1944 1945 if (sblock->pagev[0]->logical != btrfs_super_bytenr(s)) 1946 ++fail_cor; 1947 1948 if (sblock->pagev[0]->generation != btrfs_super_generation(s)) 1949 ++fail_gen; 1950 1951 if (!scrub_check_fsid(s->fsid, sblock->pagev[0])) 1952 ++fail_cor; 1953 1954 len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE; 1955 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE; 1956 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE; 1957 index = 0; 1958 for (;;) { 1959 u64 l = min_t(u64, len, mapped_size); 1960 1961 crc = btrfs_csum_data(p, crc, l); 1962 kunmap_atomic(mapped_buffer); 1963 len -= l; 1964 if (len == 0) 1965 break; 1966 index++; 1967 BUG_ON(index >= sblock->page_count); 1968 BUG_ON(!sblock->pagev[index]->page); 1969 page = sblock->pagev[index]->page; 1970 mapped_buffer = kmap_atomic(page); 1971 mapped_size = PAGE_SIZE; 1972 p = mapped_buffer; 1973 } 1974 1975 btrfs_csum_final(crc, calculated_csum); 1976 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) 1977 ++fail_cor; 1978 1979 if (fail_cor + fail_gen) { 1980 /* 1981 * if we find an error in a super block, we just report it. 1982 * They will get written with the next transaction commit 1983 * anyway 1984 */ 1985 spin_lock(&sctx->stat_lock); 1986 ++sctx->stat.super_errors; 1987 spin_unlock(&sctx->stat_lock); 1988 if (fail_cor) 1989 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev, 1990 BTRFS_DEV_STAT_CORRUPTION_ERRS); 1991 else 1992 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev, 1993 BTRFS_DEV_STAT_GENERATION_ERRS); 1994 } 1995 1996 return fail_cor + fail_gen; 1997 } 1998 1999 static void scrub_block_get(struct scrub_block *sblock) 2000 { 2001 atomic_inc(&sblock->refs); 2002 } 2003 2004 static void scrub_block_put(struct scrub_block *sblock) 2005 { 2006 if (atomic_dec_and_test(&sblock->refs)) { 2007 int i; 2008 2009 if (sblock->sparity) 2010 scrub_parity_put(sblock->sparity); 2011 2012 for (i = 0; i < sblock->page_count; i++) 2013 scrub_page_put(sblock->pagev[i]); 2014 kfree(sblock); 2015 } 2016 } 2017 2018 static void scrub_page_get(struct scrub_page *spage) 2019 { 2020 atomic_inc(&spage->refs); 2021 } 2022 2023 static void scrub_page_put(struct scrub_page *spage) 2024 { 2025 if (atomic_dec_and_test(&spage->refs)) { 2026 if (spage->page) 2027 __free_page(spage->page); 2028 kfree(spage); 2029 } 2030 } 2031 2032 static void scrub_submit(struct scrub_ctx *sctx) 2033 { 2034 struct scrub_bio *sbio; 2035 2036 if (sctx->curr == -1) 2037 return; 2038 2039 sbio = sctx->bios[sctx->curr]; 2040 sctx->curr = -1; 2041 scrub_pending_bio_inc(sctx); 2042 btrfsic_submit_bio(sbio->bio); 2043 } 2044 2045 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, 2046 struct scrub_page *spage) 2047 { 2048 struct scrub_block *sblock = spage->sblock; 2049 struct scrub_bio *sbio; 2050 int ret; 2051 2052 again: 2053 /* 2054 * grab a fresh bio or wait for one to become available 2055 */ 2056 while (sctx->curr == -1) { 2057 spin_lock(&sctx->list_lock); 2058 sctx->curr = sctx->first_free; 2059 if (sctx->curr != -1) { 2060 sctx->first_free = sctx->bios[sctx->curr]->next_free; 2061 sctx->bios[sctx->curr]->next_free = -1; 2062 sctx->bios[sctx->curr]->page_count = 0; 2063 spin_unlock(&sctx->list_lock); 2064 } else { 2065 spin_unlock(&sctx->list_lock); 2066 wait_event(sctx->list_wait, sctx->first_free != -1); 2067 } 2068 } 2069 sbio = sctx->bios[sctx->curr]; 2070 if (sbio->page_count == 0) { 2071 struct bio *bio; 2072 2073 sbio->physical = spage->physical; 2074 sbio->logical = spage->logical; 2075 sbio->dev = spage->dev; 2076 bio = sbio->bio; 2077 if (!bio) { 2078 bio = btrfs_io_bio_alloc(GFP_KERNEL, 2079 sctx->pages_per_rd_bio); 2080 if (!bio) 2081 return -ENOMEM; 2082 sbio->bio = bio; 2083 } 2084 2085 bio->bi_private = sbio; 2086 bio->bi_end_io = scrub_bio_end_io; 2087 bio->bi_bdev = sbio->dev->bdev; 2088 bio->bi_iter.bi_sector = sbio->physical >> 9; 2089 bio_set_op_attrs(bio, REQ_OP_READ, 0); 2090 sbio->err = 0; 2091 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != 2092 spage->physical || 2093 sbio->logical + sbio->page_count * PAGE_SIZE != 2094 spage->logical || 2095 sbio->dev != spage->dev) { 2096 scrub_submit(sctx); 2097 goto again; 2098 } 2099 2100 sbio->pagev[sbio->page_count] = spage; 2101 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); 2102 if (ret != PAGE_SIZE) { 2103 if (sbio->page_count < 1) { 2104 bio_put(sbio->bio); 2105 sbio->bio = NULL; 2106 return -EIO; 2107 } 2108 scrub_submit(sctx); 2109 goto again; 2110 } 2111 2112 scrub_block_get(sblock); /* one for the page added to the bio */ 2113 atomic_inc(&sblock->outstanding_pages); 2114 sbio->page_count++; 2115 if (sbio->page_count == sctx->pages_per_rd_bio) 2116 scrub_submit(sctx); 2117 2118 return 0; 2119 } 2120 2121 static void scrub_missing_raid56_end_io(struct bio *bio) 2122 { 2123 struct scrub_block *sblock = bio->bi_private; 2124 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; 2125 2126 if (bio->bi_error) 2127 sblock->no_io_error_seen = 0; 2128 2129 bio_put(bio); 2130 2131 btrfs_queue_work(fs_info->scrub_workers, &sblock->work); 2132 } 2133 2134 static void scrub_missing_raid56_worker(struct btrfs_work *work) 2135 { 2136 struct scrub_block *sblock = container_of(work, struct scrub_block, work); 2137 struct scrub_ctx *sctx = sblock->sctx; 2138 struct btrfs_fs_info *fs_info = sctx->fs_info; 2139 u64 logical; 2140 struct btrfs_device *dev; 2141 2142 logical = sblock->pagev[0]->logical; 2143 dev = sblock->pagev[0]->dev; 2144 2145 if (sblock->no_io_error_seen) 2146 scrub_recheck_block_checksum(sblock); 2147 2148 if (!sblock->no_io_error_seen) { 2149 spin_lock(&sctx->stat_lock); 2150 sctx->stat.read_errors++; 2151 spin_unlock(&sctx->stat_lock); 2152 btrfs_err_rl_in_rcu(fs_info, 2153 "IO error rebuilding logical %llu for dev %s", 2154 logical, rcu_str_deref(dev->name)); 2155 } else if (sblock->header_error || sblock->checksum_error) { 2156 spin_lock(&sctx->stat_lock); 2157 sctx->stat.uncorrectable_errors++; 2158 spin_unlock(&sctx->stat_lock); 2159 btrfs_err_rl_in_rcu(fs_info, 2160 "failed to rebuild valid logical %llu for dev %s", 2161 logical, rcu_str_deref(dev->name)); 2162 } else { 2163 scrub_write_block_to_dev_replace(sblock); 2164 } 2165 2166 scrub_block_put(sblock); 2167 2168 if (sctx->is_dev_replace && 2169 atomic_read(&sctx->wr_ctx.flush_all_writes)) { 2170 mutex_lock(&sctx->wr_ctx.wr_lock); 2171 scrub_wr_submit(sctx); 2172 mutex_unlock(&sctx->wr_ctx.wr_lock); 2173 } 2174 2175 scrub_pending_bio_dec(sctx); 2176 } 2177 2178 static void scrub_missing_raid56_pages(struct scrub_block *sblock) 2179 { 2180 struct scrub_ctx *sctx = sblock->sctx; 2181 struct btrfs_fs_info *fs_info = sctx->fs_info; 2182 u64 length = sblock->page_count * PAGE_SIZE; 2183 u64 logical = sblock->pagev[0]->logical; 2184 struct btrfs_bio *bbio = NULL; 2185 struct bio *bio; 2186 struct btrfs_raid_bio *rbio; 2187 int ret; 2188 int i; 2189 2190 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, 2191 &length, &bbio, 0, 1); 2192 if (ret || !bbio || !bbio->raid_map) 2193 goto bbio_out; 2194 2195 if (WARN_ON(!sctx->is_dev_replace || 2196 !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) { 2197 /* 2198 * We shouldn't be scrubbing a missing device. Even for dev 2199 * replace, we should only get here for RAID 5/6. We either 2200 * managed to mount something with no mirrors remaining or 2201 * there's a bug in scrub_remap_extent()/btrfs_map_block(). 2202 */ 2203 goto bbio_out; 2204 } 2205 2206 bio = btrfs_io_bio_alloc(GFP_NOFS, 0); 2207 if (!bio) 2208 goto bbio_out; 2209 2210 bio->bi_iter.bi_sector = logical >> 9; 2211 bio->bi_private = sblock; 2212 bio->bi_end_io = scrub_missing_raid56_end_io; 2213 2214 rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length); 2215 if (!rbio) 2216 goto rbio_out; 2217 2218 for (i = 0; i < sblock->page_count; i++) { 2219 struct scrub_page *spage = sblock->pagev[i]; 2220 2221 raid56_add_scrub_pages(rbio, spage->page, spage->logical); 2222 } 2223 2224 btrfs_init_work(&sblock->work, btrfs_scrub_helper, 2225 scrub_missing_raid56_worker, NULL, NULL); 2226 scrub_block_get(sblock); 2227 scrub_pending_bio_inc(sctx); 2228 raid56_submit_missing_rbio(rbio); 2229 return; 2230 2231 rbio_out: 2232 bio_put(bio); 2233 bbio_out: 2234 btrfs_put_bbio(bbio); 2235 spin_lock(&sctx->stat_lock); 2236 sctx->stat.malloc_errors++; 2237 spin_unlock(&sctx->stat_lock); 2238 } 2239 2240 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len, 2241 u64 physical, struct btrfs_device *dev, u64 flags, 2242 u64 gen, int mirror_num, u8 *csum, int force, 2243 u64 physical_for_dev_replace) 2244 { 2245 struct scrub_block *sblock; 2246 int index; 2247 2248 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); 2249 if (!sblock) { 2250 spin_lock(&sctx->stat_lock); 2251 sctx->stat.malloc_errors++; 2252 spin_unlock(&sctx->stat_lock); 2253 return -ENOMEM; 2254 } 2255 2256 /* one ref inside this function, plus one for each page added to 2257 * a bio later on */ 2258 atomic_set(&sblock->refs, 1); 2259 sblock->sctx = sctx; 2260 sblock->no_io_error_seen = 1; 2261 2262 for (index = 0; len > 0; index++) { 2263 struct scrub_page *spage; 2264 u64 l = min_t(u64, len, PAGE_SIZE); 2265 2266 spage = kzalloc(sizeof(*spage), GFP_KERNEL); 2267 if (!spage) { 2268 leave_nomem: 2269 spin_lock(&sctx->stat_lock); 2270 sctx->stat.malloc_errors++; 2271 spin_unlock(&sctx->stat_lock); 2272 scrub_block_put(sblock); 2273 return -ENOMEM; 2274 } 2275 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK); 2276 scrub_page_get(spage); 2277 sblock->pagev[index] = spage; 2278 spage->sblock = sblock; 2279 spage->dev = dev; 2280 spage->flags = flags; 2281 spage->generation = gen; 2282 spage->logical = logical; 2283 spage->physical = physical; 2284 spage->physical_for_dev_replace = physical_for_dev_replace; 2285 spage->mirror_num = mirror_num; 2286 if (csum) { 2287 spage->have_csum = 1; 2288 memcpy(spage->csum, csum, sctx->csum_size); 2289 } else { 2290 spage->have_csum = 0; 2291 } 2292 sblock->page_count++; 2293 spage->page = alloc_page(GFP_KERNEL); 2294 if (!spage->page) 2295 goto leave_nomem; 2296 len -= l; 2297 logical += l; 2298 physical += l; 2299 physical_for_dev_replace += l; 2300 } 2301 2302 WARN_ON(sblock->page_count == 0); 2303 if (dev->missing) { 2304 /* 2305 * This case should only be hit for RAID 5/6 device replace. See 2306 * the comment in scrub_missing_raid56_pages() for details. 2307 */ 2308 scrub_missing_raid56_pages(sblock); 2309 } else { 2310 for (index = 0; index < sblock->page_count; index++) { 2311 struct scrub_page *spage = sblock->pagev[index]; 2312 int ret; 2313 2314 ret = scrub_add_page_to_rd_bio(sctx, spage); 2315 if (ret) { 2316 scrub_block_put(sblock); 2317 return ret; 2318 } 2319 } 2320 2321 if (force) 2322 scrub_submit(sctx); 2323 } 2324 2325 /* last one frees, either here or in bio completion for last page */ 2326 scrub_block_put(sblock); 2327 return 0; 2328 } 2329 2330 static void scrub_bio_end_io(struct bio *bio) 2331 { 2332 struct scrub_bio *sbio = bio->bi_private; 2333 struct btrfs_fs_info *fs_info = sbio->dev->fs_info; 2334 2335 sbio->err = bio->bi_error; 2336 sbio->bio = bio; 2337 2338 btrfs_queue_work(fs_info->scrub_workers, &sbio->work); 2339 } 2340 2341 static void scrub_bio_end_io_worker(struct btrfs_work *work) 2342 { 2343 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work); 2344 struct scrub_ctx *sctx = sbio->sctx; 2345 int i; 2346 2347 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO); 2348 if (sbio->err) { 2349 for (i = 0; i < sbio->page_count; i++) { 2350 struct scrub_page *spage = sbio->pagev[i]; 2351 2352 spage->io_error = 1; 2353 spage->sblock->no_io_error_seen = 0; 2354 } 2355 } 2356 2357 /* now complete the scrub_block items that have all pages completed */ 2358 for (i = 0; i < sbio->page_count; i++) { 2359 struct scrub_page *spage = sbio->pagev[i]; 2360 struct scrub_block *sblock = spage->sblock; 2361 2362 if (atomic_dec_and_test(&sblock->outstanding_pages)) 2363 scrub_block_complete(sblock); 2364 scrub_block_put(sblock); 2365 } 2366 2367 bio_put(sbio->bio); 2368 sbio->bio = NULL; 2369 spin_lock(&sctx->list_lock); 2370 sbio->next_free = sctx->first_free; 2371 sctx->first_free = sbio->index; 2372 spin_unlock(&sctx->list_lock); 2373 2374 if (sctx->is_dev_replace && 2375 atomic_read(&sctx->wr_ctx.flush_all_writes)) { 2376 mutex_lock(&sctx->wr_ctx.wr_lock); 2377 scrub_wr_submit(sctx); 2378 mutex_unlock(&sctx->wr_ctx.wr_lock); 2379 } 2380 2381 scrub_pending_bio_dec(sctx); 2382 } 2383 2384 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity, 2385 unsigned long *bitmap, 2386 u64 start, u64 len) 2387 { 2388 u32 offset; 2389 int nsectors; 2390 int sectorsize = sparity->sctx->fs_info->sectorsize; 2391 2392 if (len >= sparity->stripe_len) { 2393 bitmap_set(bitmap, 0, sparity->nsectors); 2394 return; 2395 } 2396 2397 start -= sparity->logic_start; 2398 start = div_u64_rem(start, sparity->stripe_len, &offset); 2399 offset /= sectorsize; 2400 nsectors = (int)len / sectorsize; 2401 2402 if (offset + nsectors <= sparity->nsectors) { 2403 bitmap_set(bitmap, offset, nsectors); 2404 return; 2405 } 2406 2407 bitmap_set(bitmap, offset, sparity->nsectors - offset); 2408 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset)); 2409 } 2410 2411 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity, 2412 u64 start, u64 len) 2413 { 2414 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len); 2415 } 2416 2417 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity, 2418 u64 start, u64 len) 2419 { 2420 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len); 2421 } 2422 2423 static void scrub_block_complete(struct scrub_block *sblock) 2424 { 2425 int corrupted = 0; 2426 2427 if (!sblock->no_io_error_seen) { 2428 corrupted = 1; 2429 scrub_handle_errored_block(sblock); 2430 } else { 2431 /* 2432 * if has checksum error, write via repair mechanism in 2433 * dev replace case, otherwise write here in dev replace 2434 * case. 2435 */ 2436 corrupted = scrub_checksum(sblock); 2437 if (!corrupted && sblock->sctx->is_dev_replace) 2438 scrub_write_block_to_dev_replace(sblock); 2439 } 2440 2441 if (sblock->sparity && corrupted && !sblock->data_corrected) { 2442 u64 start = sblock->pagev[0]->logical; 2443 u64 end = sblock->pagev[sblock->page_count - 1]->logical + 2444 PAGE_SIZE; 2445 2446 scrub_parity_mark_sectors_error(sblock->sparity, 2447 start, end - start); 2448 } 2449 } 2450 2451 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum) 2452 { 2453 struct btrfs_ordered_sum *sum = NULL; 2454 unsigned long index; 2455 unsigned long num_sectors; 2456 2457 while (!list_empty(&sctx->csum_list)) { 2458 sum = list_first_entry(&sctx->csum_list, 2459 struct btrfs_ordered_sum, list); 2460 if (sum->bytenr > logical) 2461 return 0; 2462 if (sum->bytenr + sum->len > logical) 2463 break; 2464 2465 ++sctx->stat.csum_discards; 2466 list_del(&sum->list); 2467 kfree(sum); 2468 sum = NULL; 2469 } 2470 if (!sum) 2471 return 0; 2472 2473 index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize; 2474 num_sectors = sum->len / sctx->sectorsize; 2475 memcpy(csum, sum->sums + index, sctx->csum_size); 2476 if (index == num_sectors - 1) { 2477 list_del(&sum->list); 2478 kfree(sum); 2479 } 2480 return 1; 2481 } 2482 2483 /* scrub extent tries to collect up to 64 kB for each bio */ 2484 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len, 2485 u64 physical, struct btrfs_device *dev, u64 flags, 2486 u64 gen, int mirror_num, u64 physical_for_dev_replace) 2487 { 2488 int ret; 2489 u8 csum[BTRFS_CSUM_SIZE]; 2490 u32 blocksize; 2491 2492 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2493 blocksize = sctx->sectorsize; 2494 spin_lock(&sctx->stat_lock); 2495 sctx->stat.data_extents_scrubbed++; 2496 sctx->stat.data_bytes_scrubbed += len; 2497 spin_unlock(&sctx->stat_lock); 2498 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 2499 blocksize = sctx->nodesize; 2500 spin_lock(&sctx->stat_lock); 2501 sctx->stat.tree_extents_scrubbed++; 2502 sctx->stat.tree_bytes_scrubbed += len; 2503 spin_unlock(&sctx->stat_lock); 2504 } else { 2505 blocksize = sctx->sectorsize; 2506 WARN_ON(1); 2507 } 2508 2509 while (len) { 2510 u64 l = min_t(u64, len, blocksize); 2511 int have_csum = 0; 2512 2513 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2514 /* push csums to sbio */ 2515 have_csum = scrub_find_csum(sctx, logical, csum); 2516 if (have_csum == 0) 2517 ++sctx->stat.no_csum; 2518 if (sctx->is_dev_replace && !have_csum) { 2519 ret = copy_nocow_pages(sctx, logical, l, 2520 mirror_num, 2521 physical_for_dev_replace); 2522 goto behind_scrub_pages; 2523 } 2524 } 2525 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen, 2526 mirror_num, have_csum ? csum : NULL, 0, 2527 physical_for_dev_replace); 2528 behind_scrub_pages: 2529 if (ret) 2530 return ret; 2531 len -= l; 2532 logical += l; 2533 physical += l; 2534 physical_for_dev_replace += l; 2535 } 2536 return 0; 2537 } 2538 2539 static int scrub_pages_for_parity(struct scrub_parity *sparity, 2540 u64 logical, u64 len, 2541 u64 physical, struct btrfs_device *dev, 2542 u64 flags, u64 gen, int mirror_num, u8 *csum) 2543 { 2544 struct scrub_ctx *sctx = sparity->sctx; 2545 struct scrub_block *sblock; 2546 int index; 2547 2548 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); 2549 if (!sblock) { 2550 spin_lock(&sctx->stat_lock); 2551 sctx->stat.malloc_errors++; 2552 spin_unlock(&sctx->stat_lock); 2553 return -ENOMEM; 2554 } 2555 2556 /* one ref inside this function, plus one for each page added to 2557 * a bio later on */ 2558 atomic_set(&sblock->refs, 1); 2559 sblock->sctx = sctx; 2560 sblock->no_io_error_seen = 1; 2561 sblock->sparity = sparity; 2562 scrub_parity_get(sparity); 2563 2564 for (index = 0; len > 0; index++) { 2565 struct scrub_page *spage; 2566 u64 l = min_t(u64, len, PAGE_SIZE); 2567 2568 spage = kzalloc(sizeof(*spage), GFP_KERNEL); 2569 if (!spage) { 2570 leave_nomem: 2571 spin_lock(&sctx->stat_lock); 2572 sctx->stat.malloc_errors++; 2573 spin_unlock(&sctx->stat_lock); 2574 scrub_block_put(sblock); 2575 return -ENOMEM; 2576 } 2577 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK); 2578 /* For scrub block */ 2579 scrub_page_get(spage); 2580 sblock->pagev[index] = spage; 2581 /* For scrub parity */ 2582 scrub_page_get(spage); 2583 list_add_tail(&spage->list, &sparity->spages); 2584 spage->sblock = sblock; 2585 spage->dev = dev; 2586 spage->flags = flags; 2587 spage->generation = gen; 2588 spage->logical = logical; 2589 spage->physical = physical; 2590 spage->mirror_num = mirror_num; 2591 if (csum) { 2592 spage->have_csum = 1; 2593 memcpy(spage->csum, csum, sctx->csum_size); 2594 } else { 2595 spage->have_csum = 0; 2596 } 2597 sblock->page_count++; 2598 spage->page = alloc_page(GFP_KERNEL); 2599 if (!spage->page) 2600 goto leave_nomem; 2601 len -= l; 2602 logical += l; 2603 physical += l; 2604 } 2605 2606 WARN_ON(sblock->page_count == 0); 2607 for (index = 0; index < sblock->page_count; index++) { 2608 struct scrub_page *spage = sblock->pagev[index]; 2609 int ret; 2610 2611 ret = scrub_add_page_to_rd_bio(sctx, spage); 2612 if (ret) { 2613 scrub_block_put(sblock); 2614 return ret; 2615 } 2616 } 2617 2618 /* last one frees, either here or in bio completion for last page */ 2619 scrub_block_put(sblock); 2620 return 0; 2621 } 2622 2623 static int scrub_extent_for_parity(struct scrub_parity *sparity, 2624 u64 logical, u64 len, 2625 u64 physical, struct btrfs_device *dev, 2626 u64 flags, u64 gen, int mirror_num) 2627 { 2628 struct scrub_ctx *sctx = sparity->sctx; 2629 int ret; 2630 u8 csum[BTRFS_CSUM_SIZE]; 2631 u32 blocksize; 2632 2633 if (dev->missing) { 2634 scrub_parity_mark_sectors_error(sparity, logical, len); 2635 return 0; 2636 } 2637 2638 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2639 blocksize = sctx->sectorsize; 2640 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 2641 blocksize = sctx->nodesize; 2642 } else { 2643 blocksize = sctx->sectorsize; 2644 WARN_ON(1); 2645 } 2646 2647 while (len) { 2648 u64 l = min_t(u64, len, blocksize); 2649 int have_csum = 0; 2650 2651 if (flags & BTRFS_EXTENT_FLAG_DATA) { 2652 /* push csums to sbio */ 2653 have_csum = scrub_find_csum(sctx, logical, csum); 2654 if (have_csum == 0) 2655 goto skip; 2656 } 2657 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev, 2658 flags, gen, mirror_num, 2659 have_csum ? csum : NULL); 2660 if (ret) 2661 return ret; 2662 skip: 2663 len -= l; 2664 logical += l; 2665 physical += l; 2666 } 2667 return 0; 2668 } 2669 2670 /* 2671 * Given a physical address, this will calculate it's 2672 * logical offset. if this is a parity stripe, it will return 2673 * the most left data stripe's logical offset. 2674 * 2675 * return 0 if it is a data stripe, 1 means parity stripe. 2676 */ 2677 static int get_raid56_logic_offset(u64 physical, int num, 2678 struct map_lookup *map, u64 *offset, 2679 u64 *stripe_start) 2680 { 2681 int i; 2682 int j = 0; 2683 u64 stripe_nr; 2684 u64 last_offset; 2685 u32 stripe_index; 2686 u32 rot; 2687 2688 last_offset = (physical - map->stripes[num].physical) * 2689 nr_data_stripes(map); 2690 if (stripe_start) 2691 *stripe_start = last_offset; 2692 2693 *offset = last_offset; 2694 for (i = 0; i < nr_data_stripes(map); i++) { 2695 *offset = last_offset + i * map->stripe_len; 2696 2697 stripe_nr = div_u64(*offset, map->stripe_len); 2698 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map)); 2699 2700 /* Work out the disk rotation on this stripe-set */ 2701 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot); 2702 /* calculate which stripe this data locates */ 2703 rot += i; 2704 stripe_index = rot % map->num_stripes; 2705 if (stripe_index == num) 2706 return 0; 2707 if (stripe_index < num) 2708 j++; 2709 } 2710 *offset = last_offset + j * map->stripe_len; 2711 return 1; 2712 } 2713 2714 static void scrub_free_parity(struct scrub_parity *sparity) 2715 { 2716 struct scrub_ctx *sctx = sparity->sctx; 2717 struct scrub_page *curr, *next; 2718 int nbits; 2719 2720 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors); 2721 if (nbits) { 2722 spin_lock(&sctx->stat_lock); 2723 sctx->stat.read_errors += nbits; 2724 sctx->stat.uncorrectable_errors += nbits; 2725 spin_unlock(&sctx->stat_lock); 2726 } 2727 2728 list_for_each_entry_safe(curr, next, &sparity->spages, list) { 2729 list_del_init(&curr->list); 2730 scrub_page_put(curr); 2731 } 2732 2733 kfree(sparity); 2734 } 2735 2736 static void scrub_parity_bio_endio_worker(struct btrfs_work *work) 2737 { 2738 struct scrub_parity *sparity = container_of(work, struct scrub_parity, 2739 work); 2740 struct scrub_ctx *sctx = sparity->sctx; 2741 2742 scrub_free_parity(sparity); 2743 scrub_pending_bio_dec(sctx); 2744 } 2745 2746 static void scrub_parity_bio_endio(struct bio *bio) 2747 { 2748 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private; 2749 struct btrfs_fs_info *fs_info = sparity->sctx->fs_info; 2750 2751 if (bio->bi_error) 2752 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, 2753 sparity->nsectors); 2754 2755 bio_put(bio); 2756 2757 btrfs_init_work(&sparity->work, btrfs_scrubparity_helper, 2758 scrub_parity_bio_endio_worker, NULL, NULL); 2759 btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work); 2760 } 2761 2762 static void scrub_parity_check_and_repair(struct scrub_parity *sparity) 2763 { 2764 struct scrub_ctx *sctx = sparity->sctx; 2765 struct btrfs_fs_info *fs_info = sctx->fs_info; 2766 struct bio *bio; 2767 struct btrfs_raid_bio *rbio; 2768 struct scrub_page *spage; 2769 struct btrfs_bio *bbio = NULL; 2770 u64 length; 2771 int ret; 2772 2773 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap, 2774 sparity->nsectors)) 2775 goto out; 2776 2777 length = sparity->logic_end - sparity->logic_start; 2778 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start, 2779 &length, &bbio, 0, 1); 2780 if (ret || !bbio || !bbio->raid_map) 2781 goto bbio_out; 2782 2783 bio = btrfs_io_bio_alloc(GFP_NOFS, 0); 2784 if (!bio) 2785 goto bbio_out; 2786 2787 bio->bi_iter.bi_sector = sparity->logic_start >> 9; 2788 bio->bi_private = sparity; 2789 bio->bi_end_io = scrub_parity_bio_endio; 2790 2791 rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio, 2792 length, sparity->scrub_dev, 2793 sparity->dbitmap, 2794 sparity->nsectors); 2795 if (!rbio) 2796 goto rbio_out; 2797 2798 list_for_each_entry(spage, &sparity->spages, list) 2799 raid56_add_scrub_pages(rbio, spage->page, spage->logical); 2800 2801 scrub_pending_bio_inc(sctx); 2802 raid56_parity_submit_scrub_rbio(rbio); 2803 return; 2804 2805 rbio_out: 2806 bio_put(bio); 2807 bbio_out: 2808 btrfs_put_bbio(bbio); 2809 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, 2810 sparity->nsectors); 2811 spin_lock(&sctx->stat_lock); 2812 sctx->stat.malloc_errors++; 2813 spin_unlock(&sctx->stat_lock); 2814 out: 2815 scrub_free_parity(sparity); 2816 } 2817 2818 static inline int scrub_calc_parity_bitmap_len(int nsectors) 2819 { 2820 return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long); 2821 } 2822 2823 static void scrub_parity_get(struct scrub_parity *sparity) 2824 { 2825 atomic_inc(&sparity->refs); 2826 } 2827 2828 static void scrub_parity_put(struct scrub_parity *sparity) 2829 { 2830 if (!atomic_dec_and_test(&sparity->refs)) 2831 return; 2832 2833 scrub_parity_check_and_repair(sparity); 2834 } 2835 2836 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, 2837 struct map_lookup *map, 2838 struct btrfs_device *sdev, 2839 struct btrfs_path *path, 2840 u64 logic_start, 2841 u64 logic_end) 2842 { 2843 struct btrfs_fs_info *fs_info = sctx->fs_info; 2844 struct btrfs_root *root = fs_info->extent_root; 2845 struct btrfs_root *csum_root = fs_info->csum_root; 2846 struct btrfs_extent_item *extent; 2847 struct btrfs_bio *bbio = NULL; 2848 u64 flags; 2849 int ret; 2850 int slot; 2851 struct extent_buffer *l; 2852 struct btrfs_key key; 2853 u64 generation; 2854 u64 extent_logical; 2855 u64 extent_physical; 2856 u64 extent_len; 2857 u64 mapped_length; 2858 struct btrfs_device *extent_dev; 2859 struct scrub_parity *sparity; 2860 int nsectors; 2861 int bitmap_len; 2862 int extent_mirror_num; 2863 int stop_loop = 0; 2864 2865 nsectors = div_u64(map->stripe_len, fs_info->sectorsize); 2866 bitmap_len = scrub_calc_parity_bitmap_len(nsectors); 2867 sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len, 2868 GFP_NOFS); 2869 if (!sparity) { 2870 spin_lock(&sctx->stat_lock); 2871 sctx->stat.malloc_errors++; 2872 spin_unlock(&sctx->stat_lock); 2873 return -ENOMEM; 2874 } 2875 2876 sparity->stripe_len = map->stripe_len; 2877 sparity->nsectors = nsectors; 2878 sparity->sctx = sctx; 2879 sparity->scrub_dev = sdev; 2880 sparity->logic_start = logic_start; 2881 sparity->logic_end = logic_end; 2882 atomic_set(&sparity->refs, 1); 2883 INIT_LIST_HEAD(&sparity->spages); 2884 sparity->dbitmap = sparity->bitmap; 2885 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len; 2886 2887 ret = 0; 2888 while (logic_start < logic_end) { 2889 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) 2890 key.type = BTRFS_METADATA_ITEM_KEY; 2891 else 2892 key.type = BTRFS_EXTENT_ITEM_KEY; 2893 key.objectid = logic_start; 2894 key.offset = (u64)-1; 2895 2896 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 2897 if (ret < 0) 2898 goto out; 2899 2900 if (ret > 0) { 2901 ret = btrfs_previous_extent_item(root, path, 0); 2902 if (ret < 0) 2903 goto out; 2904 if (ret > 0) { 2905 btrfs_release_path(path); 2906 ret = btrfs_search_slot(NULL, root, &key, 2907 path, 0, 0); 2908 if (ret < 0) 2909 goto out; 2910 } 2911 } 2912 2913 stop_loop = 0; 2914 while (1) { 2915 u64 bytes; 2916 2917 l = path->nodes[0]; 2918 slot = path->slots[0]; 2919 if (slot >= btrfs_header_nritems(l)) { 2920 ret = btrfs_next_leaf(root, path); 2921 if (ret == 0) 2922 continue; 2923 if (ret < 0) 2924 goto out; 2925 2926 stop_loop = 1; 2927 break; 2928 } 2929 btrfs_item_key_to_cpu(l, &key, slot); 2930 2931 if (key.type != BTRFS_EXTENT_ITEM_KEY && 2932 key.type != BTRFS_METADATA_ITEM_KEY) 2933 goto next; 2934 2935 if (key.type == BTRFS_METADATA_ITEM_KEY) 2936 bytes = fs_info->nodesize; 2937 else 2938 bytes = key.offset; 2939 2940 if (key.objectid + bytes <= logic_start) 2941 goto next; 2942 2943 if (key.objectid >= logic_end) { 2944 stop_loop = 1; 2945 break; 2946 } 2947 2948 while (key.objectid >= logic_start + map->stripe_len) 2949 logic_start += map->stripe_len; 2950 2951 extent = btrfs_item_ptr(l, slot, 2952 struct btrfs_extent_item); 2953 flags = btrfs_extent_flags(l, extent); 2954 generation = btrfs_extent_generation(l, extent); 2955 2956 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) && 2957 (key.objectid < logic_start || 2958 key.objectid + bytes > 2959 logic_start + map->stripe_len)) { 2960 btrfs_err(fs_info, 2961 "scrub: tree block %llu spanning stripes, ignored. logical=%llu", 2962 key.objectid, logic_start); 2963 spin_lock(&sctx->stat_lock); 2964 sctx->stat.uncorrectable_errors++; 2965 spin_unlock(&sctx->stat_lock); 2966 goto next; 2967 } 2968 again: 2969 extent_logical = key.objectid; 2970 extent_len = bytes; 2971 2972 if (extent_logical < logic_start) { 2973 extent_len -= logic_start - extent_logical; 2974 extent_logical = logic_start; 2975 } 2976 2977 if (extent_logical + extent_len > 2978 logic_start + map->stripe_len) 2979 extent_len = logic_start + map->stripe_len - 2980 extent_logical; 2981 2982 scrub_parity_mark_sectors_data(sparity, extent_logical, 2983 extent_len); 2984 2985 mapped_length = extent_len; 2986 bbio = NULL; 2987 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, 2988 extent_logical, &mapped_length, &bbio, 2989 0); 2990 if (!ret) { 2991 if (!bbio || mapped_length < extent_len) 2992 ret = -EIO; 2993 } 2994 if (ret) { 2995 btrfs_put_bbio(bbio); 2996 goto out; 2997 } 2998 extent_physical = bbio->stripes[0].physical; 2999 extent_mirror_num = bbio->mirror_num; 3000 extent_dev = bbio->stripes[0].dev; 3001 btrfs_put_bbio(bbio); 3002 3003 ret = btrfs_lookup_csums_range(csum_root, 3004 extent_logical, 3005 extent_logical + extent_len - 1, 3006 &sctx->csum_list, 1); 3007 if (ret) 3008 goto out; 3009 3010 ret = scrub_extent_for_parity(sparity, extent_logical, 3011 extent_len, 3012 extent_physical, 3013 extent_dev, flags, 3014 generation, 3015 extent_mirror_num); 3016 3017 scrub_free_csums(sctx); 3018 3019 if (ret) 3020 goto out; 3021 3022 if (extent_logical + extent_len < 3023 key.objectid + bytes) { 3024 logic_start += map->stripe_len; 3025 3026 if (logic_start >= logic_end) { 3027 stop_loop = 1; 3028 break; 3029 } 3030 3031 if (logic_start < key.objectid + bytes) { 3032 cond_resched(); 3033 goto again; 3034 } 3035 } 3036 next: 3037 path->slots[0]++; 3038 } 3039 3040 btrfs_release_path(path); 3041 3042 if (stop_loop) 3043 break; 3044 3045 logic_start += map->stripe_len; 3046 } 3047 out: 3048 if (ret < 0) 3049 scrub_parity_mark_sectors_error(sparity, logic_start, 3050 logic_end - logic_start); 3051 scrub_parity_put(sparity); 3052 scrub_submit(sctx); 3053 mutex_lock(&sctx->wr_ctx.wr_lock); 3054 scrub_wr_submit(sctx); 3055 mutex_unlock(&sctx->wr_ctx.wr_lock); 3056 3057 btrfs_release_path(path); 3058 return ret < 0 ? ret : 0; 3059 } 3060 3061 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, 3062 struct map_lookup *map, 3063 struct btrfs_device *scrub_dev, 3064 int num, u64 base, u64 length, 3065 int is_dev_replace) 3066 { 3067 struct btrfs_path *path, *ppath; 3068 struct btrfs_fs_info *fs_info = sctx->fs_info; 3069 struct btrfs_root *root = fs_info->extent_root; 3070 struct btrfs_root *csum_root = fs_info->csum_root; 3071 struct btrfs_extent_item *extent; 3072 struct blk_plug plug; 3073 u64 flags; 3074 int ret; 3075 int slot; 3076 u64 nstripes; 3077 struct extent_buffer *l; 3078 u64 physical; 3079 u64 logical; 3080 u64 logic_end; 3081 u64 physical_end; 3082 u64 generation; 3083 int mirror_num; 3084 struct reada_control *reada1; 3085 struct reada_control *reada2; 3086 struct btrfs_key key; 3087 struct btrfs_key key_end; 3088 u64 increment = map->stripe_len; 3089 u64 offset; 3090 u64 extent_logical; 3091 u64 extent_physical; 3092 u64 extent_len; 3093 u64 stripe_logical; 3094 u64 stripe_end; 3095 struct btrfs_device *extent_dev; 3096 int extent_mirror_num; 3097 int stop_loop = 0; 3098 3099 physical = map->stripes[num].physical; 3100 offset = 0; 3101 nstripes = div_u64(length, map->stripe_len); 3102 if (map->type & BTRFS_BLOCK_GROUP_RAID0) { 3103 offset = map->stripe_len * num; 3104 increment = map->stripe_len * map->num_stripes; 3105 mirror_num = 1; 3106 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) { 3107 int factor = map->num_stripes / map->sub_stripes; 3108 offset = map->stripe_len * (num / map->sub_stripes); 3109 increment = map->stripe_len * factor; 3110 mirror_num = num % map->sub_stripes + 1; 3111 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) { 3112 increment = map->stripe_len; 3113 mirror_num = num % map->num_stripes + 1; 3114 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) { 3115 increment = map->stripe_len; 3116 mirror_num = num % map->num_stripes + 1; 3117 } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3118 get_raid56_logic_offset(physical, num, map, &offset, NULL); 3119 increment = map->stripe_len * nr_data_stripes(map); 3120 mirror_num = 1; 3121 } else { 3122 increment = map->stripe_len; 3123 mirror_num = 1; 3124 } 3125 3126 path = btrfs_alloc_path(); 3127 if (!path) 3128 return -ENOMEM; 3129 3130 ppath = btrfs_alloc_path(); 3131 if (!ppath) { 3132 btrfs_free_path(path); 3133 return -ENOMEM; 3134 } 3135 3136 /* 3137 * work on commit root. The related disk blocks are static as 3138 * long as COW is applied. This means, it is save to rewrite 3139 * them to repair disk errors without any race conditions 3140 */ 3141 path->search_commit_root = 1; 3142 path->skip_locking = 1; 3143 3144 ppath->search_commit_root = 1; 3145 ppath->skip_locking = 1; 3146 /* 3147 * trigger the readahead for extent tree csum tree and wait for 3148 * completion. During readahead, the scrub is officially paused 3149 * to not hold off transaction commits 3150 */ 3151 logical = base + offset; 3152 physical_end = physical + nstripes * map->stripe_len; 3153 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3154 get_raid56_logic_offset(physical_end, num, 3155 map, &logic_end, NULL); 3156 logic_end += base; 3157 } else { 3158 logic_end = logical + increment * nstripes; 3159 } 3160 wait_event(sctx->list_wait, 3161 atomic_read(&sctx->bios_in_flight) == 0); 3162 scrub_blocked_if_needed(fs_info); 3163 3164 /* FIXME it might be better to start readahead at commit root */ 3165 key.objectid = logical; 3166 key.type = BTRFS_EXTENT_ITEM_KEY; 3167 key.offset = (u64)0; 3168 key_end.objectid = logic_end; 3169 key_end.type = BTRFS_METADATA_ITEM_KEY; 3170 key_end.offset = (u64)-1; 3171 reada1 = btrfs_reada_add(root, &key, &key_end); 3172 3173 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 3174 key.type = BTRFS_EXTENT_CSUM_KEY; 3175 key.offset = logical; 3176 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID; 3177 key_end.type = BTRFS_EXTENT_CSUM_KEY; 3178 key_end.offset = logic_end; 3179 reada2 = btrfs_reada_add(csum_root, &key, &key_end); 3180 3181 if (!IS_ERR(reada1)) 3182 btrfs_reada_wait(reada1); 3183 if (!IS_ERR(reada2)) 3184 btrfs_reada_wait(reada2); 3185 3186 3187 /* 3188 * collect all data csums for the stripe to avoid seeking during 3189 * the scrub. This might currently (crc32) end up to be about 1MB 3190 */ 3191 blk_start_plug(&plug); 3192 3193 /* 3194 * now find all extents for each stripe and scrub them 3195 */ 3196 ret = 0; 3197 while (physical < physical_end) { 3198 /* 3199 * canceled? 3200 */ 3201 if (atomic_read(&fs_info->scrub_cancel_req) || 3202 atomic_read(&sctx->cancel_req)) { 3203 ret = -ECANCELED; 3204 goto out; 3205 } 3206 /* 3207 * check to see if we have to pause 3208 */ 3209 if (atomic_read(&fs_info->scrub_pause_req)) { 3210 /* push queued extents */ 3211 atomic_set(&sctx->wr_ctx.flush_all_writes, 1); 3212 scrub_submit(sctx); 3213 mutex_lock(&sctx->wr_ctx.wr_lock); 3214 scrub_wr_submit(sctx); 3215 mutex_unlock(&sctx->wr_ctx.wr_lock); 3216 wait_event(sctx->list_wait, 3217 atomic_read(&sctx->bios_in_flight) == 0); 3218 atomic_set(&sctx->wr_ctx.flush_all_writes, 0); 3219 scrub_blocked_if_needed(fs_info); 3220 } 3221 3222 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3223 ret = get_raid56_logic_offset(physical, num, map, 3224 &logical, 3225 &stripe_logical); 3226 logical += base; 3227 if (ret) { 3228 /* it is parity strip */ 3229 stripe_logical += base; 3230 stripe_end = stripe_logical + increment; 3231 ret = scrub_raid56_parity(sctx, map, scrub_dev, 3232 ppath, stripe_logical, 3233 stripe_end); 3234 if (ret) 3235 goto out; 3236 goto skip; 3237 } 3238 } 3239 3240 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) 3241 key.type = BTRFS_METADATA_ITEM_KEY; 3242 else 3243 key.type = BTRFS_EXTENT_ITEM_KEY; 3244 key.objectid = logical; 3245 key.offset = (u64)-1; 3246 3247 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3248 if (ret < 0) 3249 goto out; 3250 3251 if (ret > 0) { 3252 ret = btrfs_previous_extent_item(root, path, 0); 3253 if (ret < 0) 3254 goto out; 3255 if (ret > 0) { 3256 /* there's no smaller item, so stick with the 3257 * larger one */ 3258 btrfs_release_path(path); 3259 ret = btrfs_search_slot(NULL, root, &key, 3260 path, 0, 0); 3261 if (ret < 0) 3262 goto out; 3263 } 3264 } 3265 3266 stop_loop = 0; 3267 while (1) { 3268 u64 bytes; 3269 3270 l = path->nodes[0]; 3271 slot = path->slots[0]; 3272 if (slot >= btrfs_header_nritems(l)) { 3273 ret = btrfs_next_leaf(root, path); 3274 if (ret == 0) 3275 continue; 3276 if (ret < 0) 3277 goto out; 3278 3279 stop_loop = 1; 3280 break; 3281 } 3282 btrfs_item_key_to_cpu(l, &key, slot); 3283 3284 if (key.type != BTRFS_EXTENT_ITEM_KEY && 3285 key.type != BTRFS_METADATA_ITEM_KEY) 3286 goto next; 3287 3288 if (key.type == BTRFS_METADATA_ITEM_KEY) 3289 bytes = fs_info->nodesize; 3290 else 3291 bytes = key.offset; 3292 3293 if (key.objectid + bytes <= logical) 3294 goto next; 3295 3296 if (key.objectid >= logical + map->stripe_len) { 3297 /* out of this device extent */ 3298 if (key.objectid >= logic_end) 3299 stop_loop = 1; 3300 break; 3301 } 3302 3303 extent = btrfs_item_ptr(l, slot, 3304 struct btrfs_extent_item); 3305 flags = btrfs_extent_flags(l, extent); 3306 generation = btrfs_extent_generation(l, extent); 3307 3308 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) && 3309 (key.objectid < logical || 3310 key.objectid + bytes > 3311 logical + map->stripe_len)) { 3312 btrfs_err(fs_info, 3313 "scrub: tree block %llu spanning stripes, ignored. logical=%llu", 3314 key.objectid, logical); 3315 spin_lock(&sctx->stat_lock); 3316 sctx->stat.uncorrectable_errors++; 3317 spin_unlock(&sctx->stat_lock); 3318 goto next; 3319 } 3320 3321 again: 3322 extent_logical = key.objectid; 3323 extent_len = bytes; 3324 3325 /* 3326 * trim extent to this stripe 3327 */ 3328 if (extent_logical < logical) { 3329 extent_len -= logical - extent_logical; 3330 extent_logical = logical; 3331 } 3332 if (extent_logical + extent_len > 3333 logical + map->stripe_len) { 3334 extent_len = logical + map->stripe_len - 3335 extent_logical; 3336 } 3337 3338 extent_physical = extent_logical - logical + physical; 3339 extent_dev = scrub_dev; 3340 extent_mirror_num = mirror_num; 3341 if (is_dev_replace) 3342 scrub_remap_extent(fs_info, extent_logical, 3343 extent_len, &extent_physical, 3344 &extent_dev, 3345 &extent_mirror_num); 3346 3347 ret = btrfs_lookup_csums_range(csum_root, 3348 extent_logical, 3349 extent_logical + 3350 extent_len - 1, 3351 &sctx->csum_list, 1); 3352 if (ret) 3353 goto out; 3354 3355 ret = scrub_extent(sctx, extent_logical, extent_len, 3356 extent_physical, extent_dev, flags, 3357 generation, extent_mirror_num, 3358 extent_logical - logical + physical); 3359 3360 scrub_free_csums(sctx); 3361 3362 if (ret) 3363 goto out; 3364 3365 if (extent_logical + extent_len < 3366 key.objectid + bytes) { 3367 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 3368 /* 3369 * loop until we find next data stripe 3370 * or we have finished all stripes. 3371 */ 3372 loop: 3373 physical += map->stripe_len; 3374 ret = get_raid56_logic_offset(physical, 3375 num, map, &logical, 3376 &stripe_logical); 3377 logical += base; 3378 3379 if (ret && physical < physical_end) { 3380 stripe_logical += base; 3381 stripe_end = stripe_logical + 3382 increment; 3383 ret = scrub_raid56_parity(sctx, 3384 map, scrub_dev, ppath, 3385 stripe_logical, 3386 stripe_end); 3387 if (ret) 3388 goto out; 3389 goto loop; 3390 } 3391 } else { 3392 physical += map->stripe_len; 3393 logical += increment; 3394 } 3395 if (logical < key.objectid + bytes) { 3396 cond_resched(); 3397 goto again; 3398 } 3399 3400 if (physical >= physical_end) { 3401 stop_loop = 1; 3402 break; 3403 } 3404 } 3405 next: 3406 path->slots[0]++; 3407 } 3408 btrfs_release_path(path); 3409 skip: 3410 logical += increment; 3411 physical += map->stripe_len; 3412 spin_lock(&sctx->stat_lock); 3413 if (stop_loop) 3414 sctx->stat.last_physical = map->stripes[num].physical + 3415 length; 3416 else 3417 sctx->stat.last_physical = physical; 3418 spin_unlock(&sctx->stat_lock); 3419 if (stop_loop) 3420 break; 3421 } 3422 out: 3423 /* push queued extents */ 3424 scrub_submit(sctx); 3425 mutex_lock(&sctx->wr_ctx.wr_lock); 3426 scrub_wr_submit(sctx); 3427 mutex_unlock(&sctx->wr_ctx.wr_lock); 3428 3429 blk_finish_plug(&plug); 3430 btrfs_free_path(path); 3431 btrfs_free_path(ppath); 3432 return ret < 0 ? ret : 0; 3433 } 3434 3435 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, 3436 struct btrfs_device *scrub_dev, 3437 u64 chunk_offset, u64 length, 3438 u64 dev_offset, 3439 struct btrfs_block_group_cache *cache, 3440 int is_dev_replace) 3441 { 3442 struct btrfs_fs_info *fs_info = sctx->fs_info; 3443 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree; 3444 struct map_lookup *map; 3445 struct extent_map *em; 3446 int i; 3447 int ret = 0; 3448 3449 read_lock(&map_tree->map_tree.lock); 3450 em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1); 3451 read_unlock(&map_tree->map_tree.lock); 3452 3453 if (!em) { 3454 /* 3455 * Might have been an unused block group deleted by the cleaner 3456 * kthread or relocation. 3457 */ 3458 spin_lock(&cache->lock); 3459 if (!cache->removed) 3460 ret = -EINVAL; 3461 spin_unlock(&cache->lock); 3462 3463 return ret; 3464 } 3465 3466 map = em->map_lookup; 3467 if (em->start != chunk_offset) 3468 goto out; 3469 3470 if (em->len < length) 3471 goto out; 3472 3473 for (i = 0; i < map->num_stripes; ++i) { 3474 if (map->stripes[i].dev->bdev == scrub_dev->bdev && 3475 map->stripes[i].physical == dev_offset) { 3476 ret = scrub_stripe(sctx, map, scrub_dev, i, 3477 chunk_offset, length, 3478 is_dev_replace); 3479 if (ret) 3480 goto out; 3481 } 3482 } 3483 out: 3484 free_extent_map(em); 3485 3486 return ret; 3487 } 3488 3489 static noinline_for_stack 3490 int scrub_enumerate_chunks(struct scrub_ctx *sctx, 3491 struct btrfs_device *scrub_dev, u64 start, u64 end, 3492 int is_dev_replace) 3493 { 3494 struct btrfs_dev_extent *dev_extent = NULL; 3495 struct btrfs_path *path; 3496 struct btrfs_fs_info *fs_info = sctx->fs_info; 3497 struct btrfs_root *root = fs_info->dev_root; 3498 u64 length; 3499 u64 chunk_offset; 3500 int ret = 0; 3501 int ro_set; 3502 int slot; 3503 struct extent_buffer *l; 3504 struct btrfs_key key; 3505 struct btrfs_key found_key; 3506 struct btrfs_block_group_cache *cache; 3507 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 3508 3509 path = btrfs_alloc_path(); 3510 if (!path) 3511 return -ENOMEM; 3512 3513 path->reada = READA_FORWARD; 3514 path->search_commit_root = 1; 3515 path->skip_locking = 1; 3516 3517 key.objectid = scrub_dev->devid; 3518 key.offset = 0ull; 3519 key.type = BTRFS_DEV_EXTENT_KEY; 3520 3521 while (1) { 3522 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3523 if (ret < 0) 3524 break; 3525 if (ret > 0) { 3526 if (path->slots[0] >= 3527 btrfs_header_nritems(path->nodes[0])) { 3528 ret = btrfs_next_leaf(root, path); 3529 if (ret < 0) 3530 break; 3531 if (ret > 0) { 3532 ret = 0; 3533 break; 3534 } 3535 } else { 3536 ret = 0; 3537 } 3538 } 3539 3540 l = path->nodes[0]; 3541 slot = path->slots[0]; 3542 3543 btrfs_item_key_to_cpu(l, &found_key, slot); 3544 3545 if (found_key.objectid != scrub_dev->devid) 3546 break; 3547 3548 if (found_key.type != BTRFS_DEV_EXTENT_KEY) 3549 break; 3550 3551 if (found_key.offset >= end) 3552 break; 3553 3554 if (found_key.offset < key.offset) 3555 break; 3556 3557 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent); 3558 length = btrfs_dev_extent_length(l, dev_extent); 3559 3560 if (found_key.offset + length <= start) 3561 goto skip; 3562 3563 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent); 3564 3565 /* 3566 * get a reference on the corresponding block group to prevent 3567 * the chunk from going away while we scrub it 3568 */ 3569 cache = btrfs_lookup_block_group(fs_info, chunk_offset); 3570 3571 /* some chunks are removed but not committed to disk yet, 3572 * continue scrubbing */ 3573 if (!cache) 3574 goto skip; 3575 3576 /* 3577 * we need call btrfs_inc_block_group_ro() with scrubs_paused, 3578 * to avoid deadlock caused by: 3579 * btrfs_inc_block_group_ro() 3580 * -> btrfs_wait_for_commit() 3581 * -> btrfs_commit_transaction() 3582 * -> btrfs_scrub_pause() 3583 */ 3584 scrub_pause_on(fs_info); 3585 ret = btrfs_inc_block_group_ro(fs_info, cache); 3586 if (!ret && is_dev_replace) { 3587 /* 3588 * If we are doing a device replace wait for any tasks 3589 * that started dellaloc right before we set the block 3590 * group to RO mode, as they might have just allocated 3591 * an extent from it or decided they could do a nocow 3592 * write. And if any such tasks did that, wait for their 3593 * ordered extents to complete and then commit the 3594 * current transaction, so that we can later see the new 3595 * extent items in the extent tree - the ordered extents 3596 * create delayed data references (for cow writes) when 3597 * they complete, which will be run and insert the 3598 * corresponding extent items into the extent tree when 3599 * we commit the transaction they used when running 3600 * inode.c:btrfs_finish_ordered_io(). We later use 3601 * the commit root of the extent tree to find extents 3602 * to copy from the srcdev into the tgtdev, and we don't 3603 * want to miss any new extents. 3604 */ 3605 btrfs_wait_block_group_reservations(cache); 3606 btrfs_wait_nocow_writers(cache); 3607 ret = btrfs_wait_ordered_roots(fs_info, -1, 3608 cache->key.objectid, 3609 cache->key.offset); 3610 if (ret > 0) { 3611 struct btrfs_trans_handle *trans; 3612 3613 trans = btrfs_join_transaction(root); 3614 if (IS_ERR(trans)) 3615 ret = PTR_ERR(trans); 3616 else 3617 ret = btrfs_commit_transaction(trans); 3618 if (ret) { 3619 scrub_pause_off(fs_info); 3620 btrfs_put_block_group(cache); 3621 break; 3622 } 3623 } 3624 } 3625 scrub_pause_off(fs_info); 3626 3627 if (ret == 0) { 3628 ro_set = 1; 3629 } else if (ret == -ENOSPC) { 3630 /* 3631 * btrfs_inc_block_group_ro return -ENOSPC when it 3632 * failed in creating new chunk for metadata. 3633 * It is not a problem for scrub/replace, because 3634 * metadata are always cowed, and our scrub paused 3635 * commit_transactions. 3636 */ 3637 ro_set = 0; 3638 } else { 3639 btrfs_warn(fs_info, 3640 "failed setting block group ro, ret=%d\n", 3641 ret); 3642 btrfs_put_block_group(cache); 3643 break; 3644 } 3645 3646 btrfs_dev_replace_lock(&fs_info->dev_replace, 1); 3647 dev_replace->cursor_right = found_key.offset + length; 3648 dev_replace->cursor_left = found_key.offset; 3649 dev_replace->item_needs_writeback = 1; 3650 btrfs_dev_replace_unlock(&fs_info->dev_replace, 1); 3651 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length, 3652 found_key.offset, cache, is_dev_replace); 3653 3654 /* 3655 * flush, submit all pending read and write bios, afterwards 3656 * wait for them. 3657 * Note that in the dev replace case, a read request causes 3658 * write requests that are submitted in the read completion 3659 * worker. Therefore in the current situation, it is required 3660 * that all write requests are flushed, so that all read and 3661 * write requests are really completed when bios_in_flight 3662 * changes to 0. 3663 */ 3664 atomic_set(&sctx->wr_ctx.flush_all_writes, 1); 3665 scrub_submit(sctx); 3666 mutex_lock(&sctx->wr_ctx.wr_lock); 3667 scrub_wr_submit(sctx); 3668 mutex_unlock(&sctx->wr_ctx.wr_lock); 3669 3670 wait_event(sctx->list_wait, 3671 atomic_read(&sctx->bios_in_flight) == 0); 3672 3673 scrub_pause_on(fs_info); 3674 3675 /* 3676 * must be called before we decrease @scrub_paused. 3677 * make sure we don't block transaction commit while 3678 * we are waiting pending workers finished. 3679 */ 3680 wait_event(sctx->list_wait, 3681 atomic_read(&sctx->workers_pending) == 0); 3682 atomic_set(&sctx->wr_ctx.flush_all_writes, 0); 3683 3684 scrub_pause_off(fs_info); 3685 3686 btrfs_dev_replace_lock(&fs_info->dev_replace, 1); 3687 dev_replace->cursor_left = dev_replace->cursor_right; 3688 dev_replace->item_needs_writeback = 1; 3689 btrfs_dev_replace_unlock(&fs_info->dev_replace, 1); 3690 3691 if (ro_set) 3692 btrfs_dec_block_group_ro(cache); 3693 3694 /* 3695 * We might have prevented the cleaner kthread from deleting 3696 * this block group if it was already unused because we raced 3697 * and set it to RO mode first. So add it back to the unused 3698 * list, otherwise it might not ever be deleted unless a manual 3699 * balance is triggered or it becomes used and unused again. 3700 */ 3701 spin_lock(&cache->lock); 3702 if (!cache->removed && !cache->ro && cache->reserved == 0 && 3703 btrfs_block_group_used(&cache->item) == 0) { 3704 spin_unlock(&cache->lock); 3705 spin_lock(&fs_info->unused_bgs_lock); 3706 if (list_empty(&cache->bg_list)) { 3707 btrfs_get_block_group(cache); 3708 list_add_tail(&cache->bg_list, 3709 &fs_info->unused_bgs); 3710 } 3711 spin_unlock(&fs_info->unused_bgs_lock); 3712 } else { 3713 spin_unlock(&cache->lock); 3714 } 3715 3716 btrfs_put_block_group(cache); 3717 if (ret) 3718 break; 3719 if (is_dev_replace && 3720 atomic64_read(&dev_replace->num_write_errors) > 0) { 3721 ret = -EIO; 3722 break; 3723 } 3724 if (sctx->stat.malloc_errors > 0) { 3725 ret = -ENOMEM; 3726 break; 3727 } 3728 skip: 3729 key.offset = found_key.offset + length; 3730 btrfs_release_path(path); 3731 } 3732 3733 btrfs_free_path(path); 3734 3735 return ret; 3736 } 3737 3738 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx, 3739 struct btrfs_device *scrub_dev) 3740 { 3741 int i; 3742 u64 bytenr; 3743 u64 gen; 3744 int ret; 3745 struct btrfs_fs_info *fs_info = sctx->fs_info; 3746 3747 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) 3748 return -EIO; 3749 3750 /* Seed devices of a new filesystem has their own generation. */ 3751 if (scrub_dev->fs_devices != fs_info->fs_devices) 3752 gen = scrub_dev->generation; 3753 else 3754 gen = fs_info->last_trans_committed; 3755 3756 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 3757 bytenr = btrfs_sb_offset(i); 3758 if (bytenr + BTRFS_SUPER_INFO_SIZE > 3759 scrub_dev->commit_total_bytes) 3760 break; 3761 3762 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr, 3763 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i, 3764 NULL, 1, bytenr); 3765 if (ret) 3766 return ret; 3767 } 3768 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); 3769 3770 return 0; 3771 } 3772 3773 /* 3774 * get a reference count on fs_info->scrub_workers. start worker if necessary 3775 */ 3776 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info, 3777 int is_dev_replace) 3778 { 3779 unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND; 3780 int max_active = fs_info->thread_pool_size; 3781 3782 if (fs_info->scrub_workers_refcnt == 0) { 3783 if (is_dev_replace) 3784 fs_info->scrub_workers = 3785 btrfs_alloc_workqueue(fs_info, "scrub", flags, 3786 1, 4); 3787 else 3788 fs_info->scrub_workers = 3789 btrfs_alloc_workqueue(fs_info, "scrub", flags, 3790 max_active, 4); 3791 if (!fs_info->scrub_workers) 3792 goto fail_scrub_workers; 3793 3794 fs_info->scrub_wr_completion_workers = 3795 btrfs_alloc_workqueue(fs_info, "scrubwrc", flags, 3796 max_active, 2); 3797 if (!fs_info->scrub_wr_completion_workers) 3798 goto fail_scrub_wr_completion_workers; 3799 3800 fs_info->scrub_nocow_workers = 3801 btrfs_alloc_workqueue(fs_info, "scrubnc", flags, 1, 0); 3802 if (!fs_info->scrub_nocow_workers) 3803 goto fail_scrub_nocow_workers; 3804 fs_info->scrub_parity_workers = 3805 btrfs_alloc_workqueue(fs_info, "scrubparity", flags, 3806 max_active, 2); 3807 if (!fs_info->scrub_parity_workers) 3808 goto fail_scrub_parity_workers; 3809 } 3810 ++fs_info->scrub_workers_refcnt; 3811 return 0; 3812 3813 fail_scrub_parity_workers: 3814 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers); 3815 fail_scrub_nocow_workers: 3816 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers); 3817 fail_scrub_wr_completion_workers: 3818 btrfs_destroy_workqueue(fs_info->scrub_workers); 3819 fail_scrub_workers: 3820 return -ENOMEM; 3821 } 3822 3823 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info) 3824 { 3825 if (--fs_info->scrub_workers_refcnt == 0) { 3826 btrfs_destroy_workqueue(fs_info->scrub_workers); 3827 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers); 3828 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers); 3829 btrfs_destroy_workqueue(fs_info->scrub_parity_workers); 3830 } 3831 WARN_ON(fs_info->scrub_workers_refcnt < 0); 3832 } 3833 3834 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, 3835 u64 end, struct btrfs_scrub_progress *progress, 3836 int readonly, int is_dev_replace) 3837 { 3838 struct scrub_ctx *sctx; 3839 int ret; 3840 struct btrfs_device *dev; 3841 struct rcu_string *name; 3842 3843 if (btrfs_fs_closing(fs_info)) 3844 return -EINVAL; 3845 3846 if (fs_info->nodesize > BTRFS_STRIPE_LEN) { 3847 /* 3848 * in this case scrub is unable to calculate the checksum 3849 * the way scrub is implemented. Do not handle this 3850 * situation at all because it won't ever happen. 3851 */ 3852 btrfs_err(fs_info, 3853 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails", 3854 fs_info->nodesize, 3855 BTRFS_STRIPE_LEN); 3856 return -EINVAL; 3857 } 3858 3859 if (fs_info->sectorsize != PAGE_SIZE) { 3860 /* not supported for data w/o checksums */ 3861 btrfs_err_rl(fs_info, 3862 "scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails", 3863 fs_info->sectorsize, PAGE_SIZE); 3864 return -EINVAL; 3865 } 3866 3867 if (fs_info->nodesize > 3868 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK || 3869 fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) { 3870 /* 3871 * would exhaust the array bounds of pagev member in 3872 * struct scrub_block 3873 */ 3874 btrfs_err(fs_info, 3875 "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails", 3876 fs_info->nodesize, 3877 SCRUB_MAX_PAGES_PER_BLOCK, 3878 fs_info->sectorsize, 3879 SCRUB_MAX_PAGES_PER_BLOCK); 3880 return -EINVAL; 3881 } 3882 3883 3884 mutex_lock(&fs_info->fs_devices->device_list_mutex); 3885 dev = btrfs_find_device(fs_info, devid, NULL, NULL); 3886 if (!dev || (dev->missing && !is_dev_replace)) { 3887 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3888 return -ENODEV; 3889 } 3890 3891 if (!is_dev_replace && !readonly && !dev->writeable) { 3892 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3893 rcu_read_lock(); 3894 name = rcu_dereference(dev->name); 3895 btrfs_err(fs_info, "scrub: device %s is not writable", 3896 name->str); 3897 rcu_read_unlock(); 3898 return -EROFS; 3899 } 3900 3901 mutex_lock(&fs_info->scrub_lock); 3902 if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) { 3903 mutex_unlock(&fs_info->scrub_lock); 3904 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3905 return -EIO; 3906 } 3907 3908 btrfs_dev_replace_lock(&fs_info->dev_replace, 0); 3909 if (dev->scrub_device || 3910 (!is_dev_replace && 3911 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) { 3912 btrfs_dev_replace_unlock(&fs_info->dev_replace, 0); 3913 mutex_unlock(&fs_info->scrub_lock); 3914 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3915 return -EINPROGRESS; 3916 } 3917 btrfs_dev_replace_unlock(&fs_info->dev_replace, 0); 3918 3919 ret = scrub_workers_get(fs_info, is_dev_replace); 3920 if (ret) { 3921 mutex_unlock(&fs_info->scrub_lock); 3922 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3923 return ret; 3924 } 3925 3926 sctx = scrub_setup_ctx(dev, is_dev_replace); 3927 if (IS_ERR(sctx)) { 3928 mutex_unlock(&fs_info->scrub_lock); 3929 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3930 scrub_workers_put(fs_info); 3931 return PTR_ERR(sctx); 3932 } 3933 sctx->readonly = readonly; 3934 dev->scrub_device = sctx; 3935 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3936 3937 /* 3938 * checking @scrub_pause_req here, we can avoid 3939 * race between committing transaction and scrubbing. 3940 */ 3941 __scrub_blocked_if_needed(fs_info); 3942 atomic_inc(&fs_info->scrubs_running); 3943 mutex_unlock(&fs_info->scrub_lock); 3944 3945 if (!is_dev_replace) { 3946 /* 3947 * by holding device list mutex, we can 3948 * kick off writing super in log tree sync. 3949 */ 3950 mutex_lock(&fs_info->fs_devices->device_list_mutex); 3951 ret = scrub_supers(sctx, dev); 3952 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 3953 } 3954 3955 if (!ret) 3956 ret = scrub_enumerate_chunks(sctx, dev, start, end, 3957 is_dev_replace); 3958 3959 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); 3960 atomic_dec(&fs_info->scrubs_running); 3961 wake_up(&fs_info->scrub_pause_wait); 3962 3963 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0); 3964 3965 if (progress) 3966 memcpy(progress, &sctx->stat, sizeof(*progress)); 3967 3968 mutex_lock(&fs_info->scrub_lock); 3969 dev->scrub_device = NULL; 3970 scrub_workers_put(fs_info); 3971 mutex_unlock(&fs_info->scrub_lock); 3972 3973 scrub_put_ctx(sctx); 3974 3975 return ret; 3976 } 3977 3978 void btrfs_scrub_pause(struct btrfs_fs_info *fs_info) 3979 { 3980 mutex_lock(&fs_info->scrub_lock); 3981 atomic_inc(&fs_info->scrub_pause_req); 3982 while (atomic_read(&fs_info->scrubs_paused) != 3983 atomic_read(&fs_info->scrubs_running)) { 3984 mutex_unlock(&fs_info->scrub_lock); 3985 wait_event(fs_info->scrub_pause_wait, 3986 atomic_read(&fs_info->scrubs_paused) == 3987 atomic_read(&fs_info->scrubs_running)); 3988 mutex_lock(&fs_info->scrub_lock); 3989 } 3990 mutex_unlock(&fs_info->scrub_lock); 3991 } 3992 3993 void btrfs_scrub_continue(struct btrfs_fs_info *fs_info) 3994 { 3995 atomic_dec(&fs_info->scrub_pause_req); 3996 wake_up(&fs_info->scrub_pause_wait); 3997 } 3998 3999 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info) 4000 { 4001 mutex_lock(&fs_info->scrub_lock); 4002 if (!atomic_read(&fs_info->scrubs_running)) { 4003 mutex_unlock(&fs_info->scrub_lock); 4004 return -ENOTCONN; 4005 } 4006 4007 atomic_inc(&fs_info->scrub_cancel_req); 4008 while (atomic_read(&fs_info->scrubs_running)) { 4009 mutex_unlock(&fs_info->scrub_lock); 4010 wait_event(fs_info->scrub_pause_wait, 4011 atomic_read(&fs_info->scrubs_running) == 0); 4012 mutex_lock(&fs_info->scrub_lock); 4013 } 4014 atomic_dec(&fs_info->scrub_cancel_req); 4015 mutex_unlock(&fs_info->scrub_lock); 4016 4017 return 0; 4018 } 4019 4020 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info, 4021 struct btrfs_device *dev) 4022 { 4023 struct scrub_ctx *sctx; 4024 4025 mutex_lock(&fs_info->scrub_lock); 4026 sctx = dev->scrub_device; 4027 if (!sctx) { 4028 mutex_unlock(&fs_info->scrub_lock); 4029 return -ENOTCONN; 4030 } 4031 atomic_inc(&sctx->cancel_req); 4032 while (dev->scrub_device) { 4033 mutex_unlock(&fs_info->scrub_lock); 4034 wait_event(fs_info->scrub_pause_wait, 4035 dev->scrub_device == NULL); 4036 mutex_lock(&fs_info->scrub_lock); 4037 } 4038 mutex_unlock(&fs_info->scrub_lock); 4039 4040 return 0; 4041 } 4042 4043 int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid, 4044 struct btrfs_scrub_progress *progress) 4045 { 4046 struct btrfs_device *dev; 4047 struct scrub_ctx *sctx = NULL; 4048 4049 mutex_lock(&fs_info->fs_devices->device_list_mutex); 4050 dev = btrfs_find_device(fs_info, devid, NULL, NULL); 4051 if (dev) 4052 sctx = dev->scrub_device; 4053 if (sctx) 4054 memcpy(progress, &sctx->stat, sizeof(*progress)); 4055 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 4056 4057 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; 4058 } 4059 4060 static void scrub_remap_extent(struct btrfs_fs_info *fs_info, 4061 u64 extent_logical, u64 extent_len, 4062 u64 *extent_physical, 4063 struct btrfs_device **extent_dev, 4064 int *extent_mirror_num) 4065 { 4066 u64 mapped_length; 4067 struct btrfs_bio *bbio = NULL; 4068 int ret; 4069 4070 mapped_length = extent_len; 4071 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical, 4072 &mapped_length, &bbio, 0); 4073 if (ret || !bbio || mapped_length < extent_len || 4074 !bbio->stripes[0].dev->bdev) { 4075 btrfs_put_bbio(bbio); 4076 return; 4077 } 4078 4079 *extent_physical = bbio->stripes[0].physical; 4080 *extent_mirror_num = bbio->mirror_num; 4081 *extent_dev = bbio->stripes[0].dev; 4082 btrfs_put_bbio(bbio); 4083 } 4084 4085 static int scrub_setup_wr_ctx(struct scrub_wr_ctx *wr_ctx, 4086 struct btrfs_device *dev, 4087 int is_dev_replace) 4088 { 4089 WARN_ON(wr_ctx->wr_curr_bio != NULL); 4090 4091 mutex_init(&wr_ctx->wr_lock); 4092 wr_ctx->wr_curr_bio = NULL; 4093 if (!is_dev_replace) 4094 return 0; 4095 4096 WARN_ON(!dev->bdev); 4097 wr_ctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO; 4098 wr_ctx->tgtdev = dev; 4099 atomic_set(&wr_ctx->flush_all_writes, 0); 4100 return 0; 4101 } 4102 4103 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx) 4104 { 4105 mutex_lock(&wr_ctx->wr_lock); 4106 kfree(wr_ctx->wr_curr_bio); 4107 wr_ctx->wr_curr_bio = NULL; 4108 mutex_unlock(&wr_ctx->wr_lock); 4109 } 4110 4111 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len, 4112 int mirror_num, u64 physical_for_dev_replace) 4113 { 4114 struct scrub_copy_nocow_ctx *nocow_ctx; 4115 struct btrfs_fs_info *fs_info = sctx->fs_info; 4116 4117 nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS); 4118 if (!nocow_ctx) { 4119 spin_lock(&sctx->stat_lock); 4120 sctx->stat.malloc_errors++; 4121 spin_unlock(&sctx->stat_lock); 4122 return -ENOMEM; 4123 } 4124 4125 scrub_pending_trans_workers_inc(sctx); 4126 4127 nocow_ctx->sctx = sctx; 4128 nocow_ctx->logical = logical; 4129 nocow_ctx->len = len; 4130 nocow_ctx->mirror_num = mirror_num; 4131 nocow_ctx->physical_for_dev_replace = physical_for_dev_replace; 4132 btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper, 4133 copy_nocow_pages_worker, NULL, NULL); 4134 INIT_LIST_HEAD(&nocow_ctx->inodes); 4135 btrfs_queue_work(fs_info->scrub_nocow_workers, 4136 &nocow_ctx->work); 4137 4138 return 0; 4139 } 4140 4141 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx) 4142 { 4143 struct scrub_copy_nocow_ctx *nocow_ctx = ctx; 4144 struct scrub_nocow_inode *nocow_inode; 4145 4146 nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS); 4147 if (!nocow_inode) 4148 return -ENOMEM; 4149 nocow_inode->inum = inum; 4150 nocow_inode->offset = offset; 4151 nocow_inode->root = root; 4152 list_add_tail(&nocow_inode->list, &nocow_ctx->inodes); 4153 return 0; 4154 } 4155 4156 #define COPY_COMPLETE 1 4157 4158 static void copy_nocow_pages_worker(struct btrfs_work *work) 4159 { 4160 struct scrub_copy_nocow_ctx *nocow_ctx = 4161 container_of(work, struct scrub_copy_nocow_ctx, work); 4162 struct scrub_ctx *sctx = nocow_ctx->sctx; 4163 struct btrfs_fs_info *fs_info = sctx->fs_info; 4164 struct btrfs_root *root = fs_info->extent_root; 4165 u64 logical = nocow_ctx->logical; 4166 u64 len = nocow_ctx->len; 4167 int mirror_num = nocow_ctx->mirror_num; 4168 u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace; 4169 int ret; 4170 struct btrfs_trans_handle *trans = NULL; 4171 struct btrfs_path *path; 4172 int not_written = 0; 4173 4174 path = btrfs_alloc_path(); 4175 if (!path) { 4176 spin_lock(&sctx->stat_lock); 4177 sctx->stat.malloc_errors++; 4178 spin_unlock(&sctx->stat_lock); 4179 not_written = 1; 4180 goto out; 4181 } 4182 4183 trans = btrfs_join_transaction(root); 4184 if (IS_ERR(trans)) { 4185 not_written = 1; 4186 goto out; 4187 } 4188 4189 ret = iterate_inodes_from_logical(logical, fs_info, path, 4190 record_inode_for_nocow, nocow_ctx); 4191 if (ret != 0 && ret != -ENOENT) { 4192 btrfs_warn(fs_info, 4193 "iterate_inodes_from_logical() failed: log %llu, phys %llu, len %llu, mir %u, ret %d", 4194 logical, physical_for_dev_replace, len, mirror_num, 4195 ret); 4196 not_written = 1; 4197 goto out; 4198 } 4199 4200 btrfs_end_transaction(trans); 4201 trans = NULL; 4202 while (!list_empty(&nocow_ctx->inodes)) { 4203 struct scrub_nocow_inode *entry; 4204 entry = list_first_entry(&nocow_ctx->inodes, 4205 struct scrub_nocow_inode, 4206 list); 4207 list_del_init(&entry->list); 4208 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset, 4209 entry->root, nocow_ctx); 4210 kfree(entry); 4211 if (ret == COPY_COMPLETE) { 4212 ret = 0; 4213 break; 4214 } else if (ret) { 4215 break; 4216 } 4217 } 4218 out: 4219 while (!list_empty(&nocow_ctx->inodes)) { 4220 struct scrub_nocow_inode *entry; 4221 entry = list_first_entry(&nocow_ctx->inodes, 4222 struct scrub_nocow_inode, 4223 list); 4224 list_del_init(&entry->list); 4225 kfree(entry); 4226 } 4227 if (trans && !IS_ERR(trans)) 4228 btrfs_end_transaction(trans); 4229 if (not_written) 4230 btrfs_dev_replace_stats_inc(&fs_info->dev_replace. 4231 num_uncorrectable_read_errors); 4232 4233 btrfs_free_path(path); 4234 kfree(nocow_ctx); 4235 4236 scrub_pending_trans_workers_dec(sctx); 4237 } 4238 4239 static int check_extent_to_block(struct btrfs_inode *inode, u64 start, u64 len, 4240 u64 logical) 4241 { 4242 struct extent_state *cached_state = NULL; 4243 struct btrfs_ordered_extent *ordered; 4244 struct extent_io_tree *io_tree; 4245 struct extent_map *em; 4246 u64 lockstart = start, lockend = start + len - 1; 4247 int ret = 0; 4248 4249 io_tree = &inode->io_tree; 4250 4251 lock_extent_bits(io_tree, lockstart, lockend, &cached_state); 4252 ordered = btrfs_lookup_ordered_range(inode, lockstart, len); 4253 if (ordered) { 4254 btrfs_put_ordered_extent(ordered); 4255 ret = 1; 4256 goto out_unlock; 4257 } 4258 4259 em = btrfs_get_extent(inode, NULL, 0, start, len, 0); 4260 if (IS_ERR(em)) { 4261 ret = PTR_ERR(em); 4262 goto out_unlock; 4263 } 4264 4265 /* 4266 * This extent does not actually cover the logical extent anymore, 4267 * move on to the next inode. 4268 */ 4269 if (em->block_start > logical || 4270 em->block_start + em->block_len < logical + len) { 4271 free_extent_map(em); 4272 ret = 1; 4273 goto out_unlock; 4274 } 4275 free_extent_map(em); 4276 4277 out_unlock: 4278 unlock_extent_cached(io_tree, lockstart, lockend, &cached_state, 4279 GFP_NOFS); 4280 return ret; 4281 } 4282 4283 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root, 4284 struct scrub_copy_nocow_ctx *nocow_ctx) 4285 { 4286 struct btrfs_fs_info *fs_info = nocow_ctx->sctx->fs_info; 4287 struct btrfs_key key; 4288 struct inode *inode; 4289 struct page *page; 4290 struct btrfs_root *local_root; 4291 struct extent_io_tree *io_tree; 4292 u64 physical_for_dev_replace; 4293 u64 nocow_ctx_logical; 4294 u64 len = nocow_ctx->len; 4295 unsigned long index; 4296 int srcu_index; 4297 int ret = 0; 4298 int err = 0; 4299 4300 key.objectid = root; 4301 key.type = BTRFS_ROOT_ITEM_KEY; 4302 key.offset = (u64)-1; 4303 4304 srcu_index = srcu_read_lock(&fs_info->subvol_srcu); 4305 4306 local_root = btrfs_read_fs_root_no_name(fs_info, &key); 4307 if (IS_ERR(local_root)) { 4308 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index); 4309 return PTR_ERR(local_root); 4310 } 4311 4312 key.type = BTRFS_INODE_ITEM_KEY; 4313 key.objectid = inum; 4314 key.offset = 0; 4315 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL); 4316 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index); 4317 if (IS_ERR(inode)) 4318 return PTR_ERR(inode); 4319 4320 /* Avoid truncate/dio/punch hole.. */ 4321 inode_lock(inode); 4322 inode_dio_wait(inode); 4323 4324 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace; 4325 io_tree = &BTRFS_I(inode)->io_tree; 4326 nocow_ctx_logical = nocow_ctx->logical; 4327 4328 ret = check_extent_to_block(BTRFS_I(inode), offset, len, 4329 nocow_ctx_logical); 4330 if (ret) { 4331 ret = ret > 0 ? 0 : ret; 4332 goto out; 4333 } 4334 4335 while (len >= PAGE_SIZE) { 4336 index = offset >> PAGE_SHIFT; 4337 again: 4338 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS); 4339 if (!page) { 4340 btrfs_err(fs_info, "find_or_create_page() failed"); 4341 ret = -ENOMEM; 4342 goto out; 4343 } 4344 4345 if (PageUptodate(page)) { 4346 if (PageDirty(page)) 4347 goto next_page; 4348 } else { 4349 ClearPageError(page); 4350 err = extent_read_full_page(io_tree, page, 4351 btrfs_get_extent, 4352 nocow_ctx->mirror_num); 4353 if (err) { 4354 ret = err; 4355 goto next_page; 4356 } 4357 4358 lock_page(page); 4359 /* 4360 * If the page has been remove from the page cache, 4361 * the data on it is meaningless, because it may be 4362 * old one, the new data may be written into the new 4363 * page in the page cache. 4364 */ 4365 if (page->mapping != inode->i_mapping) { 4366 unlock_page(page); 4367 put_page(page); 4368 goto again; 4369 } 4370 if (!PageUptodate(page)) { 4371 ret = -EIO; 4372 goto next_page; 4373 } 4374 } 4375 4376 ret = check_extent_to_block(BTRFS_I(inode), offset, len, 4377 nocow_ctx_logical); 4378 if (ret) { 4379 ret = ret > 0 ? 0 : ret; 4380 goto next_page; 4381 } 4382 4383 err = write_page_nocow(nocow_ctx->sctx, 4384 physical_for_dev_replace, page); 4385 if (err) 4386 ret = err; 4387 next_page: 4388 unlock_page(page); 4389 put_page(page); 4390 4391 if (ret) 4392 break; 4393 4394 offset += PAGE_SIZE; 4395 physical_for_dev_replace += PAGE_SIZE; 4396 nocow_ctx_logical += PAGE_SIZE; 4397 len -= PAGE_SIZE; 4398 } 4399 ret = COPY_COMPLETE; 4400 out: 4401 inode_unlock(inode); 4402 iput(inode); 4403 return ret; 4404 } 4405 4406 static int write_page_nocow(struct scrub_ctx *sctx, 4407 u64 physical_for_dev_replace, struct page *page) 4408 { 4409 struct bio *bio; 4410 struct btrfs_device *dev; 4411 int ret; 4412 4413 dev = sctx->wr_ctx.tgtdev; 4414 if (!dev) 4415 return -EIO; 4416 if (!dev->bdev) { 4417 btrfs_warn_rl(dev->fs_info, 4418 "scrub write_page_nocow(bdev == NULL) is unexpected"); 4419 return -EIO; 4420 } 4421 bio = btrfs_io_bio_alloc(GFP_NOFS, 1); 4422 if (!bio) { 4423 spin_lock(&sctx->stat_lock); 4424 sctx->stat.malloc_errors++; 4425 spin_unlock(&sctx->stat_lock); 4426 return -ENOMEM; 4427 } 4428 bio->bi_iter.bi_size = 0; 4429 bio->bi_iter.bi_sector = physical_for_dev_replace >> 9; 4430 bio->bi_bdev = dev->bdev; 4431 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC; 4432 ret = bio_add_page(bio, page, PAGE_SIZE, 0); 4433 if (ret != PAGE_SIZE) { 4434 leave_with_eio: 4435 bio_put(bio); 4436 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); 4437 return -EIO; 4438 } 4439 4440 if (btrfsic_submit_bio_wait(bio)) 4441 goto leave_with_eio; 4442 4443 bio_put(bio); 4444 return 0; 4445 } 4446