43 static void dump_bioc(const struct btrfs_fs_info *fs_info, const struct btrfs_io_context *bioc)  in dump_bioc()  argument
45 	if (unlikely(!bioc)) {  in dump_bioc()
51 		bioc->logical, bioc->full_stripe_logical, bioc->size,  in dump_bioc()
52 		bioc->map_type, bioc->mirror_num, bioc->replace_nr_stripes,  in dump_bioc()
53 		bioc->replace_stripe_src, bioc->num_stripes);  in dump_bioc()
54 	for (int i = 0; i < bioc->num_stripes; i++) {  in dump_bioc()
56 			   i, bioc->stripes[i].dev->devid,  in dump_bioc()
57 			   bioc->stripes[i].physical);  in dump_bioc()
67 	dump_bioc(fs_info, rbio->bioc);  in btrfs_dump_rbio()
78 		const struct btrfs_fs_info *__fs_info = (rbio)->bioc ?	\
79 					(rbio)->bioc->fs_info : NULL;	\
89 		const struct btrfs_fs_info *__fs_info = (rbio)->bioc ?	\
90 					(rbio)->bioc->fs_info : NULL;	\
101 		const struct btrfs_fs_info *__fs_info = (rbio)->bioc ?	\
102 					(rbio)->bioc->fs_info : NULL;	\
113 		const struct btrfs_fs_info *__fs_info = (rbio)->bioc ?	\
114 					(rbio)->bioc->fs_info : NULL;	\
182 	btrfs_put_bioc(rbio->bioc);  in free_raid_bio()
190 	queue_work(rbio->bioc->fs_info->rmw_workers, &rbio->work);  in start_async_work()
272 			    rbio->bioc->fs_info->sectorsize);  in cache_rbio_pages()
283 	u64 num = rbio->bioc->full_stripe_logical;  in rbio_bucket()
299 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in full_page_sectors_uptodate()
321 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in index_stripe_sectors()
337 	const u32 sectorsize = src->bioc->fs_info->sectorsize;  in steal_rbio_page()
355 			      rbio->bioc->fs_info->sectorsize_bits;  in is_data_stripe_page()
437 	table = rbio->bioc->fs_info->stripe_hash_table;  in __remove_rbio_from_cache()
491 	table = rbio->bioc->fs_info->stripe_hash_table;  in remove_rbio_from_cache()
549 	table = rbio->bioc->fs_info->stripe_hash_table;  in cache_rbio()
647 	if (last->bioc->full_stripe_logical != cur->bioc->full_stripe_logical)  in rbio_can_merge()
736 	h = rbio->bioc->fs_info->stripe_hash_table->table + rbio_bucket(rbio);  in lock_stripe_add()
740 		if (cur->bioc->full_stripe_logical != rbio->bioc->full_stripe_logical)  in lock_stripe_add()
819 	h = rbio->bioc->fs_info->stripe_hash_table->table + bucket;  in unlock_stripe()
982 					 struct btrfs_io_context *bioc)  in alloc_rbio()  argument
984 	const unsigned int real_stripes = bioc->num_stripes - bioc->replace_nr_stripes;  in alloc_rbio()
1032 	btrfs_get_bioc(bioc);  in alloc_rbio()
1033 	rbio->bioc = bioc;  in alloc_rbio()
1042 	ASSERT(btrfs_nr_parity_stripes(bioc->map_type));  in alloc_rbio()
1043 	rbio->nr_data = real_stripes - btrfs_nr_parity_stripes(bioc->map_type);  in alloc_rbio()
1129 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in rbio_add_io_sector()
1141 	ASSERT_RBIO_STRIPE(stripe_nr >= 0 && stripe_nr < rbio->bioc->num_stripes,  in rbio_add_io_sector()
1147 	stripe = &rbio->bioc->stripes[stripe_nr];  in rbio_add_io_sector()
1160 		if (found_errors > rbio->bioc->max_errors)  in rbio_add_io_sector()
1197 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in index_one_bio()
1201 		     rbio->bioc->full_stripe_logical;  in index_one_bio()
1240 	const struct btrfs_io_context *bioc = rbio->bioc;  in bio_get_trace_info()  local
1243 	ASSERT(bioc);  in bio_get_trace_info()
1249 	for (i = 0; i < bioc->num_stripes; i++) {  in bio_get_trace_info()
1250 		if (bio->bi_bdev != bioc->stripes[i].dev->bdev)  in bio_get_trace_info()
1253 		trace_info->devid = bioc->stripes[i].dev->devid;  in bio_get_trace_info()
1255 				     bioc->stripes[i].physical;  in bio_get_trace_info()
1297 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in generate_pq_vertical()
1300 	const bool has_qstripe = rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6;  in generate_pq_vertical()
1385 	if (likely(!rbio->bioc->replace_nr_stripes))  in rmw_assemble_write_bios()
1393 	ASSERT(rbio->bioc->replace_stripe_src >= 0);  in rmw_assemble_write_bios()
1407 		if (stripe != rbio->bioc->replace_stripe_src) {  in rmw_assemble_write_bios()
1444 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;  in set_rbio_range_error()
1446 		     rbio->bioc->full_stripe_logical;  in set_rbio_range_error()
1465 			if (!rbio->bioc->stripes[stripe_nr].dev->bdev) {  in set_rbio_range_error()
1501 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in set_bio_pages_uptodate()
1557 	     (bio_size >> rbio->bioc->fs_info->sectorsize_bits); i++)  in rbio_update_error_bitmap()
1565 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;  in verify_bio_data_sectors()
1720 	const struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;  in rbio_add_bio()
1722 	const u64 full_stripe_start = rbio->bioc->full_stripe_logical;  in rbio_add_bio()
1748 void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc)  in raid56_parity_write()  argument
1750 	struct btrfs_fs_info *fs_info = bioc->fs_info;  in raid56_parity_write()
1755 	rbio = alloc_rbio(fs_info, bioc);  in raid56_parity_write()
1791 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;  in verify_one_sector()
1831 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;  in recover_vertical()
1857 	if (found_errors > rbio->bioc->max_errors)  in recover_vertical()
1883 	if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6) {  in recover_vertical()
2062 		if (!rbio->bioc->stripes[stripe].dev->bdev ||  in recover_rbio()
2152 void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc,  in raid56_parity_recover()  argument
2155 	struct btrfs_fs_info *fs_info = bioc->fs_info;  in raid56_parity_recover()
2158 	rbio = alloc_rbio(fs_info, bioc);  in raid56_parity_recover()
2183 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;  in fill_data_csums()
2185 						       rbio->bioc->full_stripe_logical);  in fill_data_csums()
2186 	const u64 start = rbio->bioc->full_stripe_logical;  in fill_data_csums()
2205 	if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) ||  in fill_data_csums()
2206 	    rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA)  in fill_data_csums()
2234 			rbio->bioc->full_stripe_logical, ret);  in fill_data_csums()
2412 		if (found_errors > rbio->bioc->max_errors) {  in rmw_rbio()
2446 				struct btrfs_io_context *bioc,  in raid56_parity_alloc_scrub_rbio()  argument
2450 	struct btrfs_fs_info *fs_info = bioc->fs_info;  in raid56_parity_alloc_scrub_rbio()
2454 	rbio = alloc_rbio(fs_info, bioc);  in raid56_parity_alloc_scrub_rbio()
2471 		if (bioc->stripes[i].dev == scrub_dev) {  in raid56_parity_alloc_scrub_rbio()
2488 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in alloc_rbio_essential_pages()
2512 	struct btrfs_io_context *bioc = rbio->bioc;  in finish_parity_scrub()  local
2513 	const u32 sectorsize = bioc->fs_info->sectorsize;  in finish_parity_scrub()
2539 	if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) {  in finish_parity_scrub()
2642 	ASSERT_RBIO(rbio->bioc->replace_stripe_src >= 0, rbio);  in finish_parity_scrub()
2698 		if (found_errors > rbio->bioc->max_errors) {  in recover_scrub_rbio()
2722 		if (dfail > rbio->bioc->max_errors - 1) {  in recover_scrub_rbio()
2830 		if (found_errors > rbio->bioc->max_errors) {  in scrub_rbio()
2861 					  rbio->bioc->full_stripe_logical;  in raid56_parity_cache_data_pages()
2863 	const u32 sectorsize = rbio->bioc->fs_info->sectorsize;  in raid56_parity_cache_data_pages()