1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 * Copyright (C) 2022 Christoph Hellwig. 5 */ 6 7 #include <linux/blk_types.h> 8 #include <linux/bio.h> 9 #include "bio.h" 10 #include "ctree.h" 11 #include "volumes.h" 12 #include "raid56.h" 13 #include "async-thread.h" 14 #include "dev-replace.h" 15 #include "zoned.h" 16 #include "file-item.h" 17 #include "raid-stripe-tree.h" 18 19 static struct bio_set btrfs_bioset; 20 static struct bio_set btrfs_clone_bioset; 21 static struct bio_set btrfs_repair_bioset; 22 static mempool_t btrfs_failed_bio_pool; 23 24 struct btrfs_failed_bio { 25 struct btrfs_bio *bbio; 26 int num_copies; 27 atomic_t repair_count; 28 }; 29 30 static bool bbio_has_ordered_extent(const struct btrfs_bio *bbio) 31 { 32 return is_data_inode(bbio->inode) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE; 33 } 34 35 /* 36 * Initialize a btrfs_bio structure. This skips the embedded bio itself as it 37 * is already initialized by the block layer. 38 */ 39 void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, 40 btrfs_bio_end_io_t end_io, void *private) 41 { 42 /* @inode parameter is mandatory. */ 43 ASSERT(inode); 44 45 memset(bbio, 0, offsetof(struct btrfs_bio, bio)); 46 bbio->inode = inode; 47 bbio->end_io = end_io; 48 bbio->private = private; 49 bbio->file_offset = file_offset; 50 atomic_set(&bbio->pending_ios, 1); 51 WRITE_ONCE(bbio->status, BLK_STS_OK); 52 } 53 54 /* 55 * Allocate a btrfs_bio structure. The btrfs_bio is the main I/O container for 56 * btrfs, and is used for all I/O submitted through btrfs_submit_bbio(). 57 * 58 * Just like the underlying bio_alloc_bioset it will not fail as it is backed by 59 * a mempool. 60 */ 61 struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, 62 struct btrfs_inode *inode, u64 file_offset, 63 btrfs_bio_end_io_t end_io, void *private) 64 { 65 struct btrfs_bio *bbio; 66 struct bio *bio; 67 68 bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset); 69 bbio = btrfs_bio(bio); 70 btrfs_bio_init(bbio, inode, file_offset, end_io, private); 71 return bbio; 72 } 73 74 static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, 75 struct btrfs_bio *orig_bbio, 76 u64 map_length) 77 { 78 struct btrfs_bio *bbio; 79 struct bio *bio; 80 81 bio = bio_split(&orig_bbio->bio, map_length >> SECTOR_SHIFT, GFP_NOFS, 82 &btrfs_clone_bioset); 83 if (IS_ERR(bio)) 84 return ERR_CAST(bio); 85 86 bbio = btrfs_bio(bio); 87 btrfs_bio_init(bbio, orig_bbio->inode, orig_bbio->file_offset, NULL, orig_bbio); 88 orig_bbio->file_offset += map_length; 89 if (bbio_has_ordered_extent(bbio)) { 90 refcount_inc(&orig_bbio->ordered->refs); 91 bbio->ordered = orig_bbio->ordered; 92 bbio->orig_logical = orig_bbio->orig_logical; 93 orig_bbio->orig_logical += map_length; 94 } 95 96 bbio->csum_search_commit_root = orig_bbio->csum_search_commit_root; 97 bbio->can_use_append = orig_bbio->can_use_append; 98 bbio->is_scrub = orig_bbio->is_scrub; 99 bbio->is_remap = orig_bbio->is_remap; 100 101 atomic_inc(&orig_bbio->pending_ios); 102 return bbio; 103 } 104 105 void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) 106 { 107 /* Make sure we're already in task context. */ 108 ASSERT(in_task()); 109 110 bbio->bio.bi_status = status; 111 if (bbio->bio.bi_pool == &btrfs_clone_bioset) { 112 struct btrfs_bio *orig_bbio = bbio->private; 113 114 /* Free bio that was never submitted to the underlying device. */ 115 if (bbio_has_ordered_extent(bbio)) 116 btrfs_put_ordered_extent(bbio->ordered); 117 bio_put(&bbio->bio); 118 119 bbio = orig_bbio; 120 } 121 122 /* 123 * At this point, bbio always points to the original btrfs_bio. Save 124 * the first error in it. 125 */ 126 if (status != BLK_STS_OK) 127 cmpxchg(&bbio->status, BLK_STS_OK, status); 128 129 if (atomic_dec_and_test(&bbio->pending_ios)) { 130 /* Load split bio's error which might be set above. */ 131 if (status == BLK_STS_OK) 132 bbio->bio.bi_status = READ_ONCE(bbio->status); 133 134 if (bbio_has_ordered_extent(bbio)) { 135 struct btrfs_ordered_extent *ordered = bbio->ordered; 136 137 bbio->end_io(bbio); 138 btrfs_put_ordered_extent(ordered); 139 } else { 140 bbio->end_io(bbio); 141 } 142 } 143 } 144 145 static int next_repair_mirror(const struct btrfs_failed_bio *fbio, int cur_mirror) 146 { 147 if (cur_mirror == fbio->num_copies) 148 return cur_mirror + 1 - fbio->num_copies; 149 return cur_mirror + 1; 150 } 151 152 static int prev_repair_mirror(const struct btrfs_failed_bio *fbio, int cur_mirror) 153 { 154 if (cur_mirror == 1) 155 return fbio->num_copies; 156 return cur_mirror - 1; 157 } 158 159 static void btrfs_repair_done(struct btrfs_failed_bio *fbio) 160 { 161 if (atomic_dec_and_test(&fbio->repair_count)) { 162 btrfs_bio_end_io(fbio->bbio, fbio->bbio->bio.bi_status); 163 mempool_free(fbio, &btrfs_failed_bio_pool); 164 } 165 } 166 167 static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, 168 struct btrfs_device *dev) 169 { 170 struct btrfs_failed_bio *fbio = repair_bbio->private; 171 struct btrfs_inode *inode = repair_bbio->inode; 172 struct btrfs_fs_info *fs_info = inode->root->fs_info; 173 int mirror = repair_bbio->mirror_num; 174 175 /* Repair bbio should be exactly one block sized. */ 176 ASSERT(repair_bbio->saved_iter.bi_size == fs_info->sectorsize); 177 178 if (repair_bbio->bio.bi_status || 179 !btrfs_bio_data_csum_ok(repair_bbio, &repair_bbio->saved_iter, dev)) { 180 bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ); 181 repair_bbio->bio.bi_iter = repair_bbio->saved_iter; 182 183 mirror = next_repair_mirror(fbio, mirror); 184 if (mirror == fbio->bbio->mirror_num) { 185 btrfs_debug(fs_info, "no mirror left"); 186 fbio->bbio->bio.bi_status = BLK_STS_IOERR; 187 goto done; 188 } 189 190 btrfs_submit_bbio(repair_bbio, mirror); 191 return; 192 } 193 194 do { 195 mirror = prev_repair_mirror(fbio, mirror); 196 btrfs_repair_bbio_failure(repair_bbio, &repair_bbio->saved_iter, 197 fs_info->sectorsize, mirror); 198 } while (mirror != fbio->bbio->mirror_num); 199 200 done: 201 btrfs_repair_done(fbio); 202 bio_put(&repair_bbio->bio); 203 } 204 205 /* 206 * Try to kick off a repair read to the next available mirror for a bad sector. 207 * 208 * This primarily tries to recover good data to serve the actual read request, 209 * but also tries to write the good data back to the bad mirror(s) when a 210 * read succeeded to restore the redundancy. 211 */ 212 static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, 213 const struct bvec_iter *orig_iter, 214 struct btrfs_failed_bio *fbio) 215 { 216 struct btrfs_inode *inode = failed_bbio->inode; 217 struct btrfs_fs_info *fs_info = inode->root->fs_info; 218 struct btrfs_bio *repair_bbio; 219 struct bio *repair_bio; 220 struct bvec_iter iter = *orig_iter; 221 const u32 sectorsize = fs_info->sectorsize; 222 const u32 bio_offset = ((iter.bi_sector - failed_bbio->saved_iter.bi_sector) << 223 SECTOR_SHIFT); 224 const u64 logical = (iter.bi_sector << SECTOR_SHIFT); 225 int num_copies; 226 int mirror; 227 u32 cur = 0; 228 229 btrfs_debug(fs_info, "repair read error: read error at %llu", 230 failed_bbio->file_offset + bio_offset); 231 232 num_copies = btrfs_num_copies(fs_info, logical, sectorsize); 233 if (num_copies == 1) { 234 btrfs_debug(fs_info, "no copy to repair from"); 235 failed_bbio->bio.bi_status = BLK_STS_IOERR; 236 return fbio; 237 } 238 239 if (!fbio) { 240 fbio = mempool_alloc(&btrfs_failed_bio_pool, GFP_NOFS); 241 fbio->bbio = failed_bbio; 242 fbio->num_copies = num_copies; 243 atomic_set(&fbio->repair_count, 1); 244 } 245 246 atomic_inc(&fbio->repair_count); 247 248 repair_bio = bio_alloc_bioset(NULL, max(1, sectorsize >> PAGE_SHIFT), 249 REQ_OP_READ, GFP_NOFS, &btrfs_repair_bioset); 250 repair_bio->bi_iter.bi_sector = logical >> SECTOR_SHIFT; 251 while (cur < sectorsize) { 252 struct page *page = bio_iter_page(&failed_bbio->bio, iter); 253 const u32 pg_off = bio_iter_offset(&failed_bbio->bio, iter); 254 const u32 cur_len = min(bio_iter_len(&failed_bbio->bio, iter), 255 sectorsize - cur); 256 int ret; 257 258 ret = bio_add_page(repair_bio, page, cur_len, pg_off); 259 ASSERT(ret == cur_len); 260 261 bio_advance_iter_single(&failed_bbio->bio, &iter, cur_len); 262 cur += cur_len; 263 } 264 265 repair_bbio = btrfs_bio(repair_bio); 266 btrfs_bio_init(repair_bbio, failed_bbio->inode, failed_bbio->file_offset + bio_offset, 267 NULL, fbio); 268 269 mirror = next_repair_mirror(fbio, failed_bbio->mirror_num); 270 btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror); 271 btrfs_submit_bbio(repair_bbio, mirror); 272 return fbio; 273 } 274 275 static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *dev) 276 { 277 struct btrfs_inode *inode = bbio->inode; 278 struct btrfs_fs_info *fs_info = inode->root->fs_info; 279 const u32 sectorsize = fs_info->sectorsize; 280 struct bvec_iter iter; 281 blk_status_t status = bbio->bio.bi_status; 282 struct btrfs_failed_bio *fbio = NULL; 283 284 /* Read-repair requires the inode field to be set by the submitter. */ 285 ASSERT(inode); 286 287 /* The original bbio should be sectorsize aligned. */ 288 ASSERT(IS_ALIGNED(bbio->saved_iter.bi_size, sectorsize)); 289 290 /* 291 * Hand off repair bios to the repair code as there is no upper level 292 * submitter for them. 293 */ 294 if (bbio->bio.bi_pool == &btrfs_repair_bioset) { 295 btrfs_end_repair_bio(bbio, dev); 296 return; 297 } 298 299 /* Clear the I/O error. A failed repair will reset it. */ 300 bbio->bio.bi_status = BLK_STS_OK; 301 302 for (iter = bbio->saved_iter; iter.bi_size; 303 bio_advance_iter(&bbio->bio, &iter, sectorsize)) { 304 if (status || !btrfs_bio_data_csum_ok(bbio, &iter, dev)) 305 fbio = repair_one_sector(bbio, &iter, fbio); 306 } 307 if (bbio->csum != bbio->csum_inline) 308 kvfree(bbio->csum); 309 310 if (fbio) 311 btrfs_repair_done(fbio); 312 else 313 btrfs_bio_end_io(bbio, bbio->bio.bi_status); 314 } 315 316 static void btrfs_log_dev_io_error(const struct bio *bio, struct btrfs_device *dev) 317 { 318 blk_status_t sts = bio->bi_status; 319 320 if (!dev || !dev->bdev) 321 return; 322 if (unlikely(sts == BLK_STS_OK)) 323 return; 324 if (unlikely(sts != BLK_STS_IOERR && sts != BLK_STS_TARGET && 325 sts != BLK_STS_MEDIUM && sts != BLK_STS_PROTECTION)) { 326 btrfs_warn_rl(dev->fs_info, "bdev %s unexpected block io error: %d", 327 btrfs_dev_name(dev), sts); 328 return; 329 } 330 if (btrfs_op(bio) == BTRFS_MAP_WRITE) 331 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); 332 else if (!(bio->bi_opf & REQ_RAHEAD)) 333 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); 334 if (bio->bi_opf & REQ_PREFLUSH) 335 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_FLUSH_ERRS); 336 } 337 338 static struct workqueue_struct *btrfs_end_io_wq(const struct btrfs_fs_info *fs_info, 339 const struct bio *bio) 340 { 341 if (bio->bi_opf & REQ_META) 342 return fs_info->endio_meta_workers; 343 return fs_info->endio_workers; 344 } 345 346 static void simple_end_io_work(struct work_struct *work) 347 { 348 struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); 349 struct bio *bio = &bbio->bio; 350 351 if (bio_op(bio) == REQ_OP_READ) { 352 /* Metadata reads are checked and repaired by the submitter. */ 353 if (is_data_inode(bbio->inode)) 354 return btrfs_check_read_bio(bbio, bbio->bio.bi_private); 355 return btrfs_bio_end_io(bbio, bbio->bio.bi_status); 356 } 357 if (bio_is_zone_append(bio) && !bio->bi_status) 358 btrfs_record_physical_zoned(bbio); 359 btrfs_bio_end_io(bbio, bbio->bio.bi_status); 360 } 361 362 static void btrfs_simple_end_io(struct bio *bio) 363 { 364 struct btrfs_bio *bbio = btrfs_bio(bio); 365 struct btrfs_device *dev = bio->bi_private; 366 struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; 367 368 btrfs_bio_counter_dec(fs_info); 369 370 if (bio->bi_status) 371 btrfs_log_dev_io_error(bio, dev); 372 373 INIT_WORK(&bbio->end_io_work, simple_end_io_work); 374 queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); 375 } 376 377 static void btrfs_raid56_end_io(struct bio *bio) 378 { 379 struct btrfs_io_context *bioc = bio->bi_private; 380 struct btrfs_bio *bbio = btrfs_bio(bio); 381 382 /* RAID56 endio is always handled in workqueue. */ 383 ASSERT(in_task()); 384 385 btrfs_bio_counter_dec(bioc->fs_info); 386 bbio->mirror_num = bioc->mirror_num; 387 if (bio_op(bio) == REQ_OP_READ && is_data_inode(bbio->inode)) 388 btrfs_check_read_bio(bbio, NULL); 389 else 390 btrfs_bio_end_io(bbio, bbio->bio.bi_status); 391 392 btrfs_put_bioc(bioc); 393 } 394 395 static void orig_write_end_io_work(struct work_struct *work) 396 { 397 struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); 398 struct bio *bio = &bbio->bio; 399 struct btrfs_io_stripe *stripe = bio->bi_private; 400 struct btrfs_io_context *bioc = stripe->bioc; 401 402 btrfs_bio_counter_dec(bioc->fs_info); 403 404 if (bio->bi_status) { 405 atomic_inc(&bioc->error); 406 btrfs_log_dev_io_error(bio, stripe->dev); 407 } 408 409 /* 410 * Only send an error to the higher layers if it is beyond the tolerance 411 * threshold. 412 */ 413 if (atomic_read(&bioc->error) > bioc->max_errors) 414 bio->bi_status = BLK_STS_IOERR; 415 else 416 bio->bi_status = BLK_STS_OK; 417 418 if (bio_is_zone_append(bio) && !bio->bi_status) 419 stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; 420 421 btrfs_bio_end_io(bbio, bbio->bio.bi_status); 422 btrfs_put_bioc(bioc); 423 } 424 425 static void btrfs_orig_write_end_io(struct bio *bio) 426 { 427 struct btrfs_bio *bbio = btrfs_bio(bio); 428 429 INIT_WORK(&bbio->end_io_work, orig_write_end_io_work); 430 queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); 431 } 432 433 static void clone_write_end_io_work(struct work_struct *work) 434 { 435 struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); 436 struct bio *bio = &bbio->bio; 437 struct btrfs_io_stripe *stripe = bio->bi_private; 438 439 if (bio->bi_status) { 440 atomic_inc(&stripe->bioc->error); 441 btrfs_log_dev_io_error(bio, stripe->dev); 442 } else if (bio_is_zone_append(bio)) { 443 stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; 444 } 445 446 /* Pass on control to the original bio this one was cloned from */ 447 bio_endio(stripe->bioc->orig_bio); 448 bio_put(bio); 449 } 450 451 static void btrfs_clone_write_end_io(struct bio *bio) 452 { 453 struct btrfs_bio *bbio = btrfs_bio(bio); 454 455 INIT_WORK(&bbio->end_io_work, clone_write_end_io_work); 456 queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); 457 } 458 459 static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) 460 { 461 u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; 462 463 if (!dev || !dev->bdev || 464 test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || 465 (btrfs_op(bio) == BTRFS_MAP_WRITE && 466 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) { 467 bio_io_error(bio); 468 return; 469 } 470 471 bio_set_dev(bio, dev->bdev); 472 473 /* 474 * For zone append writing, bi_sector must point the beginning of the 475 * zone 476 */ 477 if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) { 478 u64 zone_start = round_down(physical, dev->fs_info->zone_size); 479 480 ASSERT(btrfs_dev_is_sequential(dev, physical)); 481 bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; 482 bio->bi_opf &= ~REQ_OP_WRITE; 483 bio->bi_opf |= REQ_OP_ZONE_APPEND; 484 } 485 btrfs_debug(dev->fs_info, 486 "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", 487 __func__, bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector, 488 (unsigned long)dev->bdev->bd_dev, btrfs_dev_name(dev), 489 dev->devid, bio->bi_iter.bi_size); 490 491 /* 492 * Track reads if tracking is enabled; ignore I/O operations before the 493 * filesystem is fully initialized. 494 */ 495 if (dev->fs_devices->collect_fs_stats && bio_op(bio) == REQ_OP_READ && dev->fs_info) 496 percpu_counter_add(&dev->fs_info->stats_read_blocks, 497 bio->bi_iter.bi_size >> dev->fs_info->sectorsize_bits); 498 499 if (bio->bi_opf & REQ_BTRFS_CGROUP_PUNT) 500 blkcg_punt_bio_submit(bio); 501 else 502 submit_bio(bio); 503 } 504 505 static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) 506 { 507 struct bio *orig_bio = bioc->orig_bio, *bio; 508 struct btrfs_bio *orig_bbio = btrfs_bio(orig_bio); 509 510 ASSERT(bio_op(orig_bio) != REQ_OP_READ); 511 512 /* Reuse the bio embedded into the btrfs_bio for the last mirror */ 513 if (dev_nr == bioc->num_stripes - 1) { 514 bio = orig_bio; 515 bio->bi_end_io = btrfs_orig_write_end_io; 516 } else { 517 /* We need to use endio_work to run end_io in task context. */ 518 bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &btrfs_bioset); 519 bio_inc_remaining(orig_bio); 520 btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, 521 orig_bbio->file_offset, NULL, NULL); 522 bio->bi_end_io = btrfs_clone_write_end_io; 523 } 524 525 bio->bi_private = &bioc->stripes[dev_nr]; 526 bio->bi_iter.bi_sector = bioc->stripes[dev_nr].physical >> SECTOR_SHIFT; 527 bioc->stripes[dev_nr].bioc = bioc; 528 bioc->size = bio->bi_iter.bi_size; 529 btrfs_submit_dev_bio(bioc->stripes[dev_nr].dev, bio); 530 } 531 532 static void btrfs_submit_bio(struct bio *bio, struct btrfs_io_context *bioc, 533 struct btrfs_io_stripe *smap, int mirror_num) 534 { 535 if (!bioc) { 536 /* Single mirror read/write fast path. */ 537 btrfs_bio(bio)->mirror_num = mirror_num; 538 bio->bi_iter.bi_sector = smap->physical >> SECTOR_SHIFT; 539 if (bio_op(bio) != REQ_OP_READ) 540 btrfs_bio(bio)->orig_physical = smap->physical; 541 bio->bi_private = smap->dev; 542 bio->bi_end_io = btrfs_simple_end_io; 543 btrfs_submit_dev_bio(smap->dev, bio); 544 } else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 545 /* Parity RAID write or read recovery. */ 546 bio->bi_private = bioc; 547 bio->bi_end_io = btrfs_raid56_end_io; 548 if (bio_op(bio) == REQ_OP_READ) 549 raid56_parity_recover(bio, bioc, mirror_num); 550 else 551 raid56_parity_write(bio, bioc); 552 } else { 553 /* Write to multiple mirrors. */ 554 int total_devs = bioc->num_stripes; 555 556 bioc->orig_bio = bio; 557 for (int dev_nr = 0; dev_nr < total_devs; dev_nr++) 558 btrfs_submit_mirrored_bio(bioc, dev_nr); 559 } 560 } 561 562 static int btrfs_bio_csum(struct btrfs_bio *bbio) 563 { 564 if (bbio->bio.bi_opf & REQ_META) 565 return btree_csum_one_bio(bbio); 566 #ifdef CONFIG_BTRFS_EXPERIMENTAL 567 return btrfs_csum_one_bio(bbio, true); 568 #else 569 return btrfs_csum_one_bio(bbio, false); 570 #endif 571 } 572 573 /* 574 * Async submit bios are used to offload expensive checksumming onto the worker 575 * threads. 576 */ 577 struct async_submit_bio { 578 struct btrfs_bio *bbio; 579 struct btrfs_io_context *bioc; 580 struct btrfs_io_stripe smap; 581 int mirror_num; 582 struct btrfs_work work; 583 }; 584 585 /* 586 * In order to insert checksums into the metadata in large chunks, we wait 587 * until bio submission time. All the pages in the bio are checksummed and 588 * sums are attached onto the ordered extent record. 589 * 590 * At IO completion time the csums attached on the ordered extent record are 591 * inserted into the btree. 592 */ 593 static void run_one_async_start(struct btrfs_work *work) 594 { 595 struct async_submit_bio *async = 596 container_of(work, struct async_submit_bio, work); 597 int ret; 598 599 ret = btrfs_bio_csum(async->bbio); 600 if (ret) 601 async->bbio->bio.bi_status = errno_to_blk_status(ret); 602 } 603 604 /* 605 * In order to insert checksums into the metadata in large chunks, we wait 606 * until bio submission time. All the pages in the bio are checksummed and 607 * sums are attached onto the ordered extent record. 608 * 609 * At IO completion time the csums attached on the ordered extent record are 610 * inserted into the tree. 611 * 612 * If called with @do_free == true, then it will free the work struct. 613 */ 614 static void run_one_async_done(struct btrfs_work *work, bool do_free) 615 { 616 struct async_submit_bio *async = 617 container_of(work, struct async_submit_bio, work); 618 struct bio *bio = &async->bbio->bio; 619 620 if (do_free) { 621 kfree(container_of(work, struct async_submit_bio, work)); 622 return; 623 } 624 625 /* If an error occurred we just want to clean up the bio and move on. */ 626 if (bio->bi_status) { 627 btrfs_bio_end_io(async->bbio, bio->bi_status); 628 return; 629 } 630 631 /* 632 * All of the bios that pass through here are from async helpers. 633 * Use REQ_BTRFS_CGROUP_PUNT to issue them from the owning cgroup's 634 * context. This changes nothing when cgroups aren't in use. 635 */ 636 bio->bi_opf |= REQ_BTRFS_CGROUP_PUNT; 637 btrfs_submit_bio(bio, async->bioc, &async->smap, async->mirror_num); 638 } 639 640 static bool should_async_write(struct btrfs_bio *bbio) 641 { 642 struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; 643 bool auto_csum_mode = true; 644 645 #ifdef CONFIG_BTRFS_EXPERIMENTAL 646 /* 647 * Write bios will calculate checksum and submit bio at the same time. 648 * Unless explicitly required don't offload serial csum calculate and bio 649 * submit into a workqueue. 650 */ 651 return false; 652 #endif 653 654 /* Submit synchronously if the checksum implementation is fast. */ 655 if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) 656 return false; 657 658 /* 659 * Try to defer the submission to a workqueue to parallelize the 660 * checksum calculation unless the I/O is issued synchronously. 661 */ 662 if (op_is_sync(bbio->bio.bi_opf)) 663 return false; 664 665 /* Zoned devices require I/O to be submitted in order. */ 666 if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(fs_info)) 667 return false; 668 669 return true; 670 } 671 672 /* 673 * Submit bio to an async queue. 674 * 675 * Return true if the work has been successfully submitted, else false. 676 */ 677 static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, 678 struct btrfs_io_context *bioc, 679 struct btrfs_io_stripe *smap, int mirror_num) 680 { 681 struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; 682 struct async_submit_bio *async; 683 684 async = kmalloc_obj(*async, GFP_NOFS); 685 if (!async) 686 return false; 687 688 async->bbio = bbio; 689 async->bioc = bioc; 690 async->smap = *smap; 691 async->mirror_num = mirror_num; 692 693 btrfs_init_work(&async->work, run_one_async_start, run_one_async_done); 694 btrfs_queue_work(fs_info->workers, &async->work); 695 return true; 696 } 697 698 static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) 699 { 700 struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; 701 unsigned int nr_segs; 702 int sector_offset; 703 704 map_length = min(map_length, fs_info->max_zone_append_size); 705 sector_offset = bio_split_rw_at(&bbio->bio, &fs_info->limits, 706 &nr_segs, map_length); 707 if (sector_offset) { 708 /* 709 * bio_split_rw_at() could split at a size smaller than our 710 * sectorsize and thus cause unaligned I/Os. Fix that by 711 * always rounding down to the nearest boundary. 712 */ 713 return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, fs_info->sectorsize); 714 } 715 return map_length; 716 } 717 718 static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) 719 { 720 struct btrfs_inode *inode = bbio->inode; 721 struct btrfs_fs_info *fs_info = inode->root->fs_info; 722 struct bio *bio = &bbio->bio; 723 u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; 724 u64 length = bio->bi_iter.bi_size; 725 u64 map_length = length; 726 struct btrfs_io_context *bioc = NULL; 727 struct btrfs_io_stripe smap; 728 blk_status_t status; 729 int ret; 730 731 if (bbio->is_scrub || btrfs_is_data_reloc_root(inode->root)) 732 smap.rst_search_commit_root = true; 733 else 734 smap.rst_search_commit_root = false; 735 736 btrfs_bio_counter_inc_blocked(fs_info); 737 ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, 738 &bioc, &smap, &mirror_num); 739 if (ret) { 740 status = errno_to_blk_status(ret); 741 btrfs_bio_counter_dec(fs_info); 742 goto end_bbio; 743 } 744 745 /* 746 * For fscrypt writes we will get the encrypted bio after we've remapped 747 * our bio to the physical disk location, so we need to save the 748 * original bytenr so we know what we're checksumming. 749 */ 750 if (bio_op(bio) == REQ_OP_WRITE && is_data_inode(bbio->inode)) 751 bbio->orig_logical = logical; 752 753 bbio->can_use_append = btrfs_use_zone_append(bbio); 754 755 map_length = min(map_length, length); 756 if (bbio->can_use_append) 757 map_length = btrfs_append_map_length(bbio, map_length); 758 759 if (map_length < length) { 760 struct btrfs_bio *split; 761 762 split = btrfs_split_bio(fs_info, bbio, map_length); 763 if (IS_ERR(split)) { 764 status = errno_to_blk_status(PTR_ERR(split)); 765 btrfs_bio_counter_dec(fs_info); 766 goto end_bbio; 767 } 768 bbio = split; 769 bio = &bbio->bio; 770 } 771 772 /* 773 * Save the iter for the end_io handler and preload the checksums for 774 * data reads. 775 */ 776 if (bio_op(bio) == REQ_OP_READ && is_data_inode(bbio->inode)) { 777 bbio->saved_iter = bio->bi_iter; 778 ret = btrfs_lookup_bio_sums(bbio); 779 status = errno_to_blk_status(ret); 780 if (status) 781 goto fail; 782 } 783 784 if (btrfs_op(bio) == BTRFS_MAP_WRITE) { 785 if (is_data_inode(bbio->inode) && bioc && bioc->use_rst) { 786 /* 787 * No locking for the list update, as we only add to 788 * the list in the I/O submission path, and list 789 * iteration only happens in the completion path, which 790 * can't happen until after the last submission. 791 */ 792 btrfs_get_bioc(bioc); 793 list_add_tail(&bioc->rst_ordered_entry, &bbio->ordered->bioc_list); 794 } 795 796 /* 797 * Csum items for reloc roots have already been cloned at this 798 * point, so they are handled as part of the no-checksum case. 799 */ 800 if (!(inode->flags & BTRFS_INODE_NODATASUM) && 801 !test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state) && 802 !btrfs_is_data_reloc_root(inode->root) && !bbio->is_remap) { 803 if (should_async_write(bbio) && 804 btrfs_wq_submit_bio(bbio, bioc, &smap, mirror_num)) 805 goto done; 806 807 ret = btrfs_bio_csum(bbio); 808 status = errno_to_blk_status(ret); 809 if (status) 810 goto fail; 811 } else if (bbio->can_use_append || 812 (btrfs_is_zoned(fs_info) && inode->flags & BTRFS_INODE_NODATASUM)) { 813 ret = btrfs_alloc_dummy_sum(bbio); 814 status = errno_to_blk_status(ret); 815 if (status) 816 goto fail; 817 } 818 } 819 820 btrfs_submit_bio(bio, bioc, &smap, mirror_num); 821 done: 822 return map_length == length; 823 824 fail: 825 btrfs_bio_counter_dec(fs_info); 826 /* 827 * We have split the original bbio, now we have to end both the current 828 * @bbio and remaining one, as the remaining one will never be submitted. 829 */ 830 if (map_length < length) { 831 struct btrfs_bio *remaining = bbio->private; 832 833 ASSERT(bbio->bio.bi_pool == &btrfs_clone_bioset); 834 ASSERT(remaining); 835 836 btrfs_bio_end_io(remaining, status); 837 } 838 end_bbio: 839 btrfs_bio_end_io(bbio, status); 840 /* Do not submit another chunk */ 841 return true; 842 } 843 844 static void assert_bbio_alignment(struct btrfs_bio *bbio) 845 { 846 #ifdef CONFIG_BTRFS_ASSERT 847 struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; 848 struct bio_vec bvec; 849 struct bvec_iter iter; 850 const u32 blocksize = fs_info->sectorsize; 851 const u32 alignment = min(blocksize, PAGE_SIZE); 852 const u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; 853 const u32 length = bbio->bio.bi_iter.bi_size; 854 855 /* The logical and length should still be aligned to blocksize. */ 856 ASSERT(IS_ALIGNED(logical, blocksize) && IS_ALIGNED(length, blocksize) && 857 length != 0, "root=%llu inode=%llu logical=%llu length=%u", 858 btrfs_root_id(bbio->inode->root), 859 btrfs_ino(bbio->inode), logical, length); 860 861 bio_for_each_bvec(bvec, &bbio->bio, iter) 862 ASSERT(IS_ALIGNED(bvec.bv_offset, alignment) && 863 IS_ALIGNED(bvec.bv_len, alignment), 864 "root=%llu inode=%llu logical=%llu length=%u index=%u bv_offset=%u bv_len=%u", 865 btrfs_root_id(bbio->inode->root), 866 btrfs_ino(bbio->inode), logical, length, iter.bi_idx, 867 bvec.bv_offset, bvec.bv_len); 868 #endif 869 } 870 871 void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num) 872 { 873 /* If bbio->inode is not populated, its file_offset must be 0. */ 874 ASSERT(bbio->inode || bbio->file_offset == 0); 875 876 assert_bbio_alignment(bbio); 877 878 while (!btrfs_submit_chunk(bbio, mirror_num)) 879 ; 880 } 881 882 /* 883 * Submit a repair write. 884 * 885 * This bypasses btrfs_submit_bbio() deliberately, as that writes all copies in a 886 * RAID setup. Here we only want to write the one bad copy, so we do the 887 * mapping ourselves and submit the bio directly. 888 * 889 * The I/O is issued synchronously to block the repair read completion from 890 * freeing the bio. 891 * 892 * @bbio: Original bbio where the repair is needed 893 * @orig_iter: Points to where the repair starts 894 * @length: Length of the repair write 895 * @mirror_num: Mirror number to write to. Must not be zero 896 */ 897 int btrfs_repair_bbio_failure(struct btrfs_bio *bbio, const struct bvec_iter *orig_iter, 898 u32 length, int mirror_num) 899 { 900 struct btrfs_inode *inode = bbio->inode; 901 struct btrfs_fs_info *fs_info = inode->root->fs_info; 902 struct btrfs_io_stripe smap = { 0 }; 903 struct bvec_iter iter = *orig_iter; 904 struct bio *repair_bio = NULL; 905 const u64 logical = iter.bi_sector << SECTOR_SHIFT; 906 const u64 fileoff = bbio->file_offset + 907 ((iter.bi_sector - bbio->saved_iter.bi_sector) << SECTOR_SHIFT); 908 u32 cur = 0; 909 int ret = 0; 910 911 BUG_ON(!mirror_num); 912 913 /* Basic alignment checks. */ 914 ASSERT(IS_ALIGNED(logical, fs_info->sectorsize)); 915 ASSERT(IS_ALIGNED(length, fs_info->sectorsize)); 916 ASSERT(IS_ALIGNED(fileoff, fs_info->sectorsize)); 917 /* Either it's a single data or metadata block. */ 918 ASSERT(length <= BTRFS_MAX_BLOCKSIZE); 919 920 /* Our current iter should not be before the original bbio saved_iter. */ 921 ASSERT(iter.bi_sector >= bbio->saved_iter.bi_sector); 922 923 /* 924 * The fs either mounted RO or hit critical errors, no need 925 * to continue repairing. 926 */ 927 if (unlikely(sb_rdonly(fs_info->sb))) 928 return 0; 929 930 if (btrfs_repair_one_zone(fs_info, logical)) 931 return 0; 932 933 /* 934 * Avoid races with device replace and make sure our bioc has devices 935 * associated to its stripes that don't go away while we are doing the 936 * read repair operation. 937 */ 938 btrfs_bio_counter_inc_blocked(fs_info); 939 ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num); 940 if (ret < 0) 941 goto out_counter_dec; 942 943 if (unlikely(!smap.dev->bdev || 944 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &smap.dev->dev_state))) { 945 ret = -EIO; 946 goto out_counter_dec; 947 } 948 949 repair_bio = bio_alloc(smap.dev->bdev, max(1, length >> PAGE_SHIFT), 950 REQ_OP_WRITE | REQ_SYNC, GFP_NOFS); 951 repair_bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; 952 while (cur < length) { 953 struct page *page = bio_iter_page(&bbio->bio, iter); 954 const u32 pg_off = bio_iter_offset(&bbio->bio, iter); 955 const u32 cur_len = min(bio_iter_len(&bbio->bio, iter), length - cur); 956 957 ret = bio_add_page(repair_bio, page, cur_len, pg_off); 958 ASSERT(ret == cur_len); 959 bio_advance_iter_single(&bbio->bio, &iter, cur_len); 960 cur += cur_len; 961 } 962 963 ret = submit_bio_wait(repair_bio); 964 bio_put(repair_bio); 965 if (ret) { 966 /* try to remap that extent elsewhere? */ 967 btrfs_dev_stat_inc_and_print(smap.dev, BTRFS_DEV_STAT_WRITE_ERRS); 968 goto out_counter_dec; 969 } 970 971 btrfs_info_rl(fs_info, 972 "read error corrected: root %llu ino %llu off %llu (dev %s sector %llu)", 973 btrfs_root_id(inode->root), btrfs_ino(inode), fileoff, 974 btrfs_dev_name(smap.dev), 975 smap.physical >> SECTOR_SHIFT); 976 ret = 0; 977 978 out_counter_dec: 979 btrfs_bio_counter_dec(fs_info); 980 return ret; 981 } 982 983 /* 984 * Submit a btrfs_bio based repair write. 985 * 986 * If @dev_replace is true, the write would be submitted to dev-replace target. 987 */ 988 void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace) 989 { 990 struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; 991 u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; 992 u64 length = bbio->bio.bi_iter.bi_size; 993 struct btrfs_io_stripe smap = { 0 }; 994 int ret; 995 996 ASSERT(mirror_num > 0); 997 ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); 998 ASSERT(!is_data_inode(bbio->inode)); 999 ASSERT(bbio->is_scrub); 1000 1001 btrfs_bio_counter_inc_blocked(fs_info); 1002 ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num); 1003 if (ret < 0) 1004 goto fail; 1005 1006 if (dev_replace) { 1007 ASSERT(smap.dev == fs_info->dev_replace.srcdev); 1008 smap.dev = fs_info->dev_replace.tgtdev; 1009 } 1010 btrfs_submit_bio(&bbio->bio, NULL, &smap, mirror_num); 1011 return; 1012 1013 fail: 1014 btrfs_bio_counter_dec(fs_info); 1015 btrfs_bio_end_io(bbio, errno_to_blk_status(ret)); 1016 } 1017 1018 int __init btrfs_bioset_init(void) 1019 { 1020 if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE, 1021 offsetof(struct btrfs_bio, bio), 1022 BIOSET_NEED_BVECS)) 1023 return -ENOMEM; 1024 if (bioset_init(&btrfs_clone_bioset, BIO_POOL_SIZE, 1025 offsetof(struct btrfs_bio, bio), 0)) 1026 goto out; 1027 if (bioset_init(&btrfs_repair_bioset, BIO_POOL_SIZE, 1028 offsetof(struct btrfs_bio, bio), 1029 BIOSET_NEED_BVECS)) 1030 goto out; 1031 if (mempool_init_kmalloc_pool(&btrfs_failed_bio_pool, BIO_POOL_SIZE, 1032 sizeof(struct btrfs_failed_bio))) 1033 goto out; 1034 return 0; 1035 1036 out: 1037 btrfs_bioset_exit(); 1038 return -ENOMEM; 1039 } 1040 1041 void __cold btrfs_bioset_exit(void) 1042 { 1043 mempool_exit(&btrfs_failed_bio_pool); 1044 bioset_exit(&btrfs_repair_bioset); 1045 bioset_exit(&btrfs_clone_bioset); 1046 bioset_exit(&btrfs_bioset); 1047 } 1048