1 // SPDX-License-Identifier: GPL-2.0
12 #include "async-thread.h"
13 #include "dev-replace.h"
15 #include "file-item.h"
16 #include "raid-stripe-tree.h"
29 /* Is this a data path I/O that needs storage layer checksum and repair? */
32 	return bbio->inode && is_data_inode(bbio->inode);  in is_data_bbio()
37 	return is_data_bbio(bbio) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE;  in bbio_has_ordered_extent()
48 	bbio->fs_info = fs_info;  in btrfs_bio_init()
49 	bbio->end_io = end_io;  in btrfs_bio_init()
50 	bbio->private = private;  in btrfs_bio_init()
51 	atomic_set(&bbio->pending_ios, 1);  in btrfs_bio_init()
52 	WRITE_ONCE(bbio->status, BLK_STS_OK);  in btrfs_bio_init()
82 	bio = bio_split(&orig_bbio->bio, map_length >> SECTOR_SHIFT, GFP_NOFS,  in btrfs_split_bio()
89 	bbio->inode = orig_bbio->inode;  in btrfs_split_bio()
90 	bbio->file_offset = orig_bbio->file_offset;  in btrfs_split_bio()
91 	orig_bbio->file_offset += map_length;  in btrfs_split_bio()
93 		refcount_inc(&orig_bbio->ordered->refs);  in btrfs_split_bio()
94 		bbio->ordered = orig_bbio->ordered;  in btrfs_split_bio()
96 	bbio->csum_search_commit_root = orig_bbio->csum_search_commit_root;  in btrfs_split_bio()
97 	atomic_inc(&orig_bbio->pending_ios);  in btrfs_split_bio()
103 	bbio->bio.bi_status = status;  in btrfs_bio_end_io()
104 	if (bbio->bio.bi_pool == &btrfs_clone_bioset) {  in btrfs_bio_end_io()
105 		struct btrfs_bio *orig_bbio = bbio->private;  in btrfs_bio_end_io()
109 			btrfs_put_ordered_extent(bbio->ordered);  in btrfs_bio_end_io()
110 		bio_put(&bbio->bio);  in btrfs_bio_end_io()
120 		cmpxchg(&bbio->status, BLK_STS_OK, status);  in btrfs_bio_end_io()
122 	if (atomic_dec_and_test(&bbio->pending_ios)) {  in btrfs_bio_end_io()
125 			bbio->bio.bi_status = READ_ONCE(bbio->status);  in btrfs_bio_end_io()
128 			struct btrfs_ordered_extent *ordered = bbio->ordered;  in btrfs_bio_end_io()
130 			bbio->end_io(bbio);  in btrfs_bio_end_io()
133 			bbio->end_io(bbio);  in btrfs_bio_end_io()
140 	if (cur_mirror == fbio->num_copies)  in next_repair_mirror()
141 		return cur_mirror + 1 - fbio->num_copies;  in next_repair_mirror()
148 		return fbio->num_copies;  in prev_repair_mirror()
149 	return cur_mirror - 1;  in prev_repair_mirror()
154 	if (atomic_dec_and_test(&fbio->repair_count)) {  in btrfs_repair_done()
155 		btrfs_bio_end_io(fbio->bbio, fbio->bbio->bio.bi_status);  in btrfs_repair_done()
163 	struct btrfs_failed_bio *fbio = repair_bbio->private;  in btrfs_end_repair_bio()
164 	struct btrfs_inode *inode = repair_bbio->inode;  in btrfs_end_repair_bio()
165 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in btrfs_end_repair_bio()
166 	struct bio_vec *bv = bio_first_bvec_all(&repair_bbio->bio);  in btrfs_end_repair_bio()
167 	int mirror = repair_bbio->mirror_num;  in btrfs_end_repair_bio()  local
169 	if (repair_bbio->bio.bi_status ||  in btrfs_end_repair_bio()
171 		bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ);  in btrfs_end_repair_bio()
172 		repair_bbio->bio.bi_iter = repair_bbio->saved_iter;  in btrfs_end_repair_bio()
174 		mirror = next_repair_mirror(fbio, mirror);  in btrfs_end_repair_bio()
175 		if (mirror == fbio->bbio->mirror_num) {  in btrfs_end_repair_bio()
176 			btrfs_debug(fs_info, "no mirror left");  in btrfs_end_repair_bio()
177 			fbio->bbio->bio.bi_status = BLK_STS_IOERR;  in btrfs_end_repair_bio()
181 		btrfs_submit_bbio(repair_bbio, mirror);  in btrfs_end_repair_bio()
186 		mirror = prev_repair_mirror(fbio, mirror);  in btrfs_end_repair_bio()
188 				  repair_bbio->file_offset, fs_info->sectorsize,  in btrfs_end_repair_bio()
189 				  repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT,  in btrfs_end_repair_bio()
190 				  bvec_phys(bv), mirror);  in btrfs_end_repair_bio()
191 	} while (mirror != fbio->bbio->mirror_num);  in btrfs_end_repair_bio()
195 	bio_put(&repair_bbio->bio);  in btrfs_end_repair_bio()
199  * Try to kick off a repair read to the next available mirror for a bad sector.
201  * This primarily tries to recover good data to serve the actual read request,
202  * but also tries to write the good data back to the bad mirror(s) when a
210 	struct btrfs_inode *inode = failed_bbio->inode;  in repair_one_sector()
211 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in repair_one_sector()
213 	const u32 sectorsize = fs_info->sectorsize;  in repair_one_sector()
215 	const u64 logical = (failed_bbio->saved_iter.bi_sector << SECTOR_SHIFT);  in repair_one_sector()
219 	int mirror;  in repair_one_sector()  local
223 		    failed_bbio->file_offset + bio_offset);  in repair_one_sector()
228 		failed_bbio->bio.bi_status = BLK_STS_IOERR;  in repair_one_sector()
234 		fbio->bbio = failed_bbio;  in repair_one_sector()
235 		fbio->num_copies = num_copies;  in repair_one_sector()
236 		atomic_set(&fbio->repair_count, 1);  in repair_one_sector()
239 	atomic_inc(&fbio->repair_count);  in repair_one_sector()
243 	repair_bio->bi_iter.bi_sector = failed_bbio->saved_iter.bi_sector;  in repair_one_sector()
248 	repair_bbio->inode = failed_bbio->inode;  in repair_one_sector()
249 	repair_bbio->file_offset = failed_bbio->file_offset + bio_offset;  in repair_one_sector()
251 	mirror = next_repair_mirror(fbio, failed_bbio->mirror_num);  in repair_one_sector()
252 	btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror);  in repair_one_sector()
253 	btrfs_submit_bbio(repair_bbio, mirror);  in repair_one_sector()
259 	struct btrfs_inode *inode = bbio->inode;  in btrfs_check_read_bio()
260 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in btrfs_check_read_bio()
261 	u32 sectorsize = fs_info->sectorsize;  in btrfs_check_read_bio()
262 	struct bvec_iter *iter = &bbio->saved_iter;  in btrfs_check_read_bio()
263 	blk_status_t status = bbio->bio.bi_status;  in btrfs_check_read_bio()
268 	/* Read-repair requires the inode field to be set by the submitter. */  in btrfs_check_read_bio()
275 	if (bbio->bio.bi_pool == &btrfs_repair_bioset) {  in btrfs_check_read_bio()
281 	bbio->bio.bi_status = BLK_STS_OK;  in btrfs_check_read_bio()
283 	btrfs_bio_for_each_block(paddr, &bbio->bio, iter, fs_info->sectorsize) {  in btrfs_check_read_bio()
288 	if (bbio->csum != bbio->csum_inline)  in btrfs_check_read_bio()
289 		kfree(bbio->csum);  in btrfs_check_read_bio()
294 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_check_read_bio()
299 	if (!dev || !dev->bdev)  in btrfs_log_dev_io_error()
301 	if (bio->bi_status != BLK_STS_IOERR && bio->bi_status != BLK_STS_TARGET)  in btrfs_log_dev_io_error()
306 	else if (!(bio->bi_opf & REQ_RAHEAD))  in btrfs_log_dev_io_error()
308 	if (bio->bi_opf & REQ_PREFLUSH)  in btrfs_log_dev_io_error()
315 	if (bio->bi_opf & REQ_META)  in btrfs_end_io_wq()
316 		return fs_info->endio_meta_workers;  in btrfs_end_io_wq()
317 	return fs_info->endio_workers;  in btrfs_end_io_wq()
326 		btrfs_check_read_bio(bbio, bbio->bio.bi_private);  in btrfs_end_bio_work()
328 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_end_bio_work()
334 	struct btrfs_device *dev = bio->bi_private;  in btrfs_simple_end_io()
335 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_simple_end_io()
339 	if (bio->bi_status)  in btrfs_simple_end_io()
343 		INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work);  in btrfs_simple_end_io()
344 		queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work);  in btrfs_simple_end_io()
346 		if (bio_is_zone_append(bio) && !bio->bi_status)  in btrfs_simple_end_io()
348 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_simple_end_io()
354 	struct btrfs_io_context *bioc = bio->bi_private;  in btrfs_raid56_end_io()
357 	btrfs_bio_counter_dec(bioc->fs_info);  in btrfs_raid56_end_io()
358 	bbio->mirror_num = bioc->mirror_num;  in btrfs_raid56_end_io()
362 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_raid56_end_io()
369 	struct btrfs_io_stripe *stripe = bio->bi_private;  in btrfs_orig_write_end_io()
370 	struct btrfs_io_context *bioc = stripe->bioc;  in btrfs_orig_write_end_io()
373 	btrfs_bio_counter_dec(bioc->fs_info);  in btrfs_orig_write_end_io()
375 	if (bio->bi_status) {  in btrfs_orig_write_end_io()
376 		atomic_inc(&bioc->error);  in btrfs_orig_write_end_io()
377 		btrfs_log_dev_io_error(bio, stripe->dev);  in btrfs_orig_write_end_io()
384 	if (atomic_read(&bioc->error) > bioc->max_errors)  in btrfs_orig_write_end_io()
385 		bio->bi_status = BLK_STS_IOERR;  in btrfs_orig_write_end_io()
387 		bio->bi_status = BLK_STS_OK;  in btrfs_orig_write_end_io()
389 	if (bio_is_zone_append(bio) && !bio->bi_status)  in btrfs_orig_write_end_io()
390 		stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_orig_write_end_io()
392 	btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_orig_write_end_io()
398 	struct btrfs_io_stripe *stripe = bio->bi_private;  in btrfs_clone_write_end_io()
400 	if (bio->bi_status) {  in btrfs_clone_write_end_io()
401 		atomic_inc(&stripe->bioc->error);  in btrfs_clone_write_end_io()
402 		btrfs_log_dev_io_error(bio, stripe->dev);  in btrfs_clone_write_end_io()
404 		stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_clone_write_end_io()
408 	bio_endio(stripe->bioc->orig_bio);  in btrfs_clone_write_end_io()
414 	if (!dev || !dev->bdev ||  in btrfs_submit_dev_bio()
415 	    test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||  in btrfs_submit_dev_bio()
417 	     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {  in btrfs_submit_dev_bio()
422 	bio_set_dev(bio, dev->bdev);  in btrfs_submit_dev_bio()
429 		u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_dev_bio()
430 		u64 zone_start = round_down(physical, dev->fs_info->zone_size);  in btrfs_submit_dev_bio()
433 		bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT;  in btrfs_submit_dev_bio()
435 	btrfs_debug(dev->fs_info,  in btrfs_submit_dev_bio()
437 		__func__, bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector,  in btrfs_submit_dev_bio()
438 		(unsigned long)dev->bdev->bd_dev, btrfs_dev_name(dev),  in btrfs_submit_dev_bio()
439 		dev->devid, bio->bi_iter.bi_size);  in btrfs_submit_dev_bio()
445 	if (dev->fs_devices->collect_fs_stats && bio_op(bio) == REQ_OP_READ && dev->fs_info)  in btrfs_submit_dev_bio()
446 		percpu_counter_add(&dev->fs_info->stats_read_blocks,  in btrfs_submit_dev_bio()
447 				   bio->bi_iter.bi_size >> dev->fs_info->sectorsize_bits);  in btrfs_submit_dev_bio()
449 	if (bio->bi_opf & REQ_BTRFS_CGROUP_PUNT)  in btrfs_submit_dev_bio()
457 	struct bio *orig_bio = bioc->orig_bio, *bio;  in btrfs_submit_mirrored_bio()
461 	/* Reuse the bio embedded into the btrfs_bio for the last mirror */  in btrfs_submit_mirrored_bio()
462 	if (dev_nr == bioc->num_stripes - 1) {  in btrfs_submit_mirrored_bio()
464 		bio->bi_end_io = btrfs_orig_write_end_io;  in btrfs_submit_mirrored_bio()
468 		bio->bi_end_io = btrfs_clone_write_end_io;  in btrfs_submit_mirrored_bio()
471 	bio->bi_private = &bioc->stripes[dev_nr];  in btrfs_submit_mirrored_bio()
472 	bio->bi_iter.bi_sector = bioc->stripes[dev_nr].physical >> SECTOR_SHIFT;  in btrfs_submit_mirrored_bio()
473 	bioc->stripes[dev_nr].bioc = bioc;  in btrfs_submit_mirrored_bio()
474 	bioc->size = bio->bi_iter.bi_size;  in btrfs_submit_mirrored_bio()
475 	btrfs_submit_dev_bio(bioc->stripes[dev_nr].dev, bio);  in btrfs_submit_mirrored_bio()
482 		/* Single mirror read/write fast path. */  in btrfs_submit_bio()
483 		btrfs_bio(bio)->mirror_num = mirror_num;  in btrfs_submit_bio()
484 		bio->bi_iter.bi_sector = smap->physical >> SECTOR_SHIFT;  in btrfs_submit_bio()
486 			btrfs_bio(bio)->orig_physical = smap->physical;  in btrfs_submit_bio()
487 		bio->bi_private = smap->dev;  in btrfs_submit_bio()
488 		bio->bi_end_io = btrfs_simple_end_io;  in btrfs_submit_bio()
489 		btrfs_submit_dev_bio(smap->dev, bio);  in btrfs_submit_bio()
490 	} else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {  in btrfs_submit_bio()
492 		bio->bi_private = bioc;  in btrfs_submit_bio()
493 		bio->bi_end_io = btrfs_raid56_end_io;  in btrfs_submit_bio()
500 		int total_devs = bioc->num_stripes;  in btrfs_submit_bio()
502 		bioc->orig_bio = bio;  in btrfs_submit_bio()
510 	if (bbio->bio.bi_opf & REQ_META)  in btrfs_bio_csum()
541 	ret = btrfs_bio_csum(async->bbio);  in run_one_async_start()
543 		async->bbio->bio.bi_status = errno_to_blk_status(ret);  in run_one_async_start()
560 	struct bio *bio = &async->bbio->bio;  in run_one_async_done()
568 	if (bio->bi_status) {  in run_one_async_done()
569 		btrfs_bio_end_io(async->bbio, bio->bi_status);  in run_one_async_done()
578 	bio->bi_opf |= REQ_BTRFS_CGROUP_PUNT;  in run_one_async_done()
579 	btrfs_submit_bio(bio, async->bioc, &async->smap, async->mirror_num);  in run_one_async_done()
587 	struct btrfs_fs_devices *fs_devices = bbio->fs_info->fs_devices;  in should_async_write()
588 	enum btrfs_offload_csum_mode csum_mode = READ_ONCE(fs_devices->offload_csum_mode);  in should_async_write()
597 	if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &bbio->fs_info->flags))  in should_async_write()
604 	if (op_is_sync(bbio->bio.bi_opf))  in should_async_write()
608 	if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(bbio->fs_info))  in should_async_write()
623 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_wq_submit_bio()
630 	async->bbio = bbio;  in btrfs_wq_submit_bio()
631 	async->bioc = bioc;  in btrfs_wq_submit_bio()
632 	async->smap = *smap;  in btrfs_wq_submit_bio()
633 	async->mirror_num = mirror_num;  in btrfs_wq_submit_bio()
635 	btrfs_init_work(&async->work, run_one_async_start, run_one_async_done);  in btrfs_wq_submit_bio()
636 	btrfs_queue_work(fs_info->workers, &async->work);  in btrfs_wq_submit_bio()
645 	map_length = min(map_length, bbio->fs_info->max_zone_append_size);  in btrfs_append_map_length()
646 	sector_offset = bio_split_rw_at(&bbio->bio, &bbio->fs_info->limits,  in btrfs_append_map_length()
654 		return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, bbio->fs_info->sectorsize);  in btrfs_append_map_length()
661 	struct btrfs_inode *inode = bbio->inode;  in btrfs_submit_chunk()
662 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_submit_chunk()
663 	struct bio *bio = &bbio->bio;  in btrfs_submit_chunk()
664 	u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_chunk()
665 	u64 length = bio->bi_iter.bi_size;  in btrfs_submit_chunk()
673 	if (!bbio->inode || btrfs_is_data_reloc_root(inode->root))  in btrfs_submit_chunk()
701 		bio = &bbio->bio;  in btrfs_submit_chunk()
706 	 * data reads.  in btrfs_submit_chunk()
709 		bbio->saved_iter = bio->bi_iter;  in btrfs_submit_chunk()
718 			bio->bi_opf &= ~REQ_OP_WRITE;  in btrfs_submit_chunk()
719 			bio->bi_opf |= REQ_OP_ZONE_APPEND;  in btrfs_submit_chunk()
722 		if (is_data_bbio(bbio) && bioc && bioc->use_rst) {  in btrfs_submit_chunk()
730 			list_add_tail(&bioc->rst_ordered_entry, &bbio->ordered->bioc_list);  in btrfs_submit_chunk()
735 		 * point, so they are handled as part of the no-checksum case.  in btrfs_submit_chunk()
737 		if (inode && !(inode->flags & BTRFS_INODE_NODATASUM) &&  in btrfs_submit_chunk()
738 		    !test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state) &&  in btrfs_submit_chunk()
739 		    !btrfs_is_data_reloc_root(inode->root)) {  in btrfs_submit_chunk()
750 			    inode->flags & BTRFS_INODE_NODATASUM)) {  in btrfs_submit_chunk()
769 		struct btrfs_bio *remaining = bbio->private;  in btrfs_submit_chunk()
771 		ASSERT(bbio->bio.bi_pool == &btrfs_clone_bioset);  in btrfs_submit_chunk()
785 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in assert_bbio_alignment()
788 	const u32 blocksize = fs_info->sectorsize;  in assert_bbio_alignment()
794 	bio_for_each_bvec(bvec, &bbio->bio, iter)  in assert_bbio_alignment()
798 		btrfs_root_id(bbio->inode->root),  in assert_bbio_alignment()
799 		btrfs_ino(bbio->inode),  in assert_bbio_alignment()
800 		bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT,  in assert_bbio_alignment()
801 		bbio->bio.bi_iter.bi_size, iter.bi_idx,  in assert_bbio_alignment()
809 	/* If bbio->inode is not populated, its file_offset must be 0. */  in btrfs_submit_bbio()
810 	ASSERT(bbio->inode || bbio->file_offset == 0);  in btrfs_submit_bbio()
836 	ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));  in btrfs_repair_io_failure()
852 	if (unlikely(!smap.dev->bdev ||  in btrfs_repair_io_failure()
853 		     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &smap.dev->dev_state))) {  in btrfs_repair_io_failure()
854 		ret = -EIO;  in btrfs_repair_io_failure()
858 	bio_init(&bio, smap.dev->bdev, &bvec, 1, REQ_OP_WRITE | REQ_SYNC);  in btrfs_repair_io_failure()
884  * If @dev_replace is true, the write would be submitted to dev-replace target.
888 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_submit_repair_write()
889 	u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_repair_write()
890 	u64 length = bbio->bio.bi_iter.bi_size;  in btrfs_submit_repair_write()
896 	ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE);  in btrfs_submit_repair_write()
897 	ASSERT(!bbio->inode);  in btrfs_submit_repair_write()
905 		ASSERT(smap.dev == fs_info->dev_replace.srcdev);  in btrfs_submit_repair_write()
906 		smap.dev = fs_info->dev_replace.tgtdev;  in btrfs_submit_repair_write()
908 	btrfs_submit_bio(&bbio->bio, NULL, &smap, mirror_num);  in btrfs_submit_repair_write()
921 		return -ENOMEM;  in btrfs_bioset_init()
936 	return -ENOMEM;  in btrfs_bioset_init()