1138 raid5_end_read_request(struct bio *bi);
1140 raid5_end_write_request(struct bio *bi);
1162 		struct bio *bi, *rbi;  in ops_run_io()  local
1185 		bi = &dev->req;  in ops_run_io()
1249 			bio_init(bi, rdev->bdev, &dev->vec, 1, op | op_flags);  in ops_run_io()
1250 			bi->bi_end_io = op_is_write(op)  in ops_run_io()
1253 			bi->bi_private = sh;  in ops_run_io()
1257 				bi->bi_opf, i);  in ops_run_io()
1262 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1265 				bi->bi_iter.bi_sector = (sh->sector  in ops_run_io()
1268 				bi->bi_opf |= REQ_NOMERGE;  in ops_run_io()
1283 			bi->bi_vcnt = 1;  in ops_run_io()
1284 			bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);  in ops_run_io()
1285 			bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;  in ops_run_io()
1286 			bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);  in ops_run_io()
1292 				bi->bi_vcnt = 0;  in ops_run_io()
1296 			mddev_trace_remap(conf->mddev, bi, sh->dev[i].sector);  in ops_run_io()
1298 				bio_list_add(&pending_bios, bi);  in ops_run_io()
1300 				submit_bio_noacct(bi);  in ops_run_io()
1344 				bi->bi_opf, i, (unsigned long long)sh->sector);  in ops_run_io()
2710 static void raid5_end_read_request(struct bio * bi)  in raid5_end_read_request()  argument
2712 	struct stripe_head *sh = bi->bi_private;  in raid5_end_read_request()
2719 		if (bi == &sh->dev[i].req)  in raid5_end_read_request()
2724 		bi->bi_status);  in raid5_end_read_request()
2743 	if (!bi->bi_status) {  in raid5_end_read_request()
2775 		if (!(bi->bi_status == BLK_STS_PROTECTION))  in raid5_end_read_request()
2830 	bio_uninit(bi);  in raid5_end_read_request()
2836 static void raid5_end_write_request(struct bio *bi)  in raid5_end_write_request()  argument
2838 	struct stripe_head *sh = bi->bi_private;  in raid5_end_write_request()
2845 		if (bi == &sh->dev[i].req) {  in raid5_end_write_request()
2849 		if (bi == &sh->dev[i].rreq) {  in raid5_end_write_request()
2864 		bi->bi_status);  in raid5_end_write_request()
2871 		if (bi->bi_status)  in raid5_end_write_request()
2877 		if (bi->bi_status) {  in raid5_end_write_request()
2896 	if (sh->batch_head && bi->bi_status && !replacement)  in raid5_end_write_request()
2899 	bio_uninit(bi);  in raid5_end_write_request()
3433 static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi,  in stripe_bio_overlaps()  argument
3440 		 bi->bi_iter.bi_sector, sh->sector);  in stripe_bio_overlaps()
3451 	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {  in stripe_bio_overlaps()
3452 		if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)  in stripe_bio_overlaps()
3457 	if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))  in stripe_bio_overlaps()
3493 static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,  in __add_stripe_bio()  argument
3508 	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector)  in __add_stripe_bio()
3514 	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);  in __add_stripe_bio()
3516 		bi->bi_next = *bip;  in __add_stripe_bio()
3517 	*bip = bi;  in __add_stripe_bio()
3518 	bio_inc_remaining(bi);  in __add_stripe_bio()
3519 	md_write_inc(conf->mddev, bi);  in __add_stripe_bio()
3524 		for (bi=sh->dev[dd_idx].towrite;  in __add_stripe_bio()
3526 			     bi && bi->bi_iter.bi_sector <= sector;  in __add_stripe_bio()
3527 		     bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {  in __add_stripe_bio()
3528 			if (bio_end_sector(bi) >= sector)  in __add_stripe_bio()
3529 				sector = bio_end_sector(bi);  in __add_stripe_bio()
3551 static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi,  in add_stripe_bio()  argument
3556 	if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {  in add_stripe_bio()
3562 	__add_stripe_bio(sh, bi, dd_idx, forwrite, previous);  in add_stripe_bio()
3592 		struct bio *bi;  in handle_failed_stripe()  local
3612 		bi = sh->dev[i].towrite;  in handle_failed_stripe()
3622 		while (bi && bi->bi_iter.bi_sector <  in handle_failed_stripe()
3624 			struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3627 			bio_io_error(bi);  in handle_failed_stripe()
3628 			bi = nextbi;  in handle_failed_stripe()
3631 		bi = sh->dev[i].written;  in handle_failed_stripe()
3638 		while (bi && bi->bi_iter.bi_sector <  in handle_failed_stripe()
3640 			struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3643 			bio_io_error(bi);  in handle_failed_stripe()
3644 			bi = bi2;  in handle_failed_stripe()
3655 			bi = sh->dev[i].toread;  in handle_failed_stripe()
3660 			if (bi)  in handle_failed_stripe()
3662 			while (bi && bi->bi_iter.bi_sector <  in handle_failed_stripe()
3665 					r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3667 				bio_io_error(bi);  in handle_failed_stripe()
3668 				bi = nextbi;  in handle_failed_stripe()
5352 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)  in add_bio_to_retry()  argument
5358 	bi->bi_next = conf->retry_read_aligned_list;  in add_bio_to_retry()
5359 	conf->retry_read_aligned_list = bi;  in add_bio_to_retry()
5368 	struct bio *bi;  in remove_bio_from_retry()  local
5370 	bi = conf->retry_read_aligned;  in remove_bio_from_retry()
5371 	if (bi) {  in remove_bio_from_retry()
5374 		return bi;  in remove_bio_from_retry()
5376 	bi = conf->retry_read_aligned_list;  in remove_bio_from_retry()
5377 	if(bi) {  in remove_bio_from_retry()
5378 		conf->retry_read_aligned_list = bi->bi_next;  in remove_bio_from_retry()
5379 		bi->bi_next = NULL;  in remove_bio_from_retry()
5383 	return bi;  in remove_bio_from_retry()
5392 static void raid5_align_endio(struct bio *bi)  in raid5_align_endio()  argument
5394 	struct bio *raid_bi = bi->bi_private;  in raid5_align_endio()
5398 	blk_status_t error = bi->bi_status;  in raid5_align_endio()
5400 	bio_put(bi);  in raid5_align_endio()
5689 static void make_discard_request(struct mddev *mddev, struct bio *bi)  in make_discard_request()  argument
5697 	if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT))  in make_discard_request()
5704 	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in make_discard_request()
5705 	last_sector = bio_end_sector(bi);  in make_discard_request()
5707 	bi->bi_next = NULL;  in make_discard_request()
5750 			sh->dev[d].towrite = bi;  in make_discard_request()
5752 			bio_inc_remaining(bi);  in make_discard_request()
5753 			md_write_inc(mddev, bi);  in make_discard_request()
5769 	bio_endio(bi);  in make_discard_request()
5815 		struct bio *bi, int forwrite, int previous)  in add_all_stripe_bios()  argument
5831 		if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {  in add_all_stripe_bios()
5856 		__add_stripe_bio(sh, bi, dd_idx, forwrite, previous);  in add_all_stripe_bios()
5951 		sector_t logical_sector, struct bio *bi)  in make_stripe_request()  argument
5953 	const int rw = bio_data_dir(bi);  in make_stripe_request()
5977 	if (bi->bi_opf & REQ_RAHEAD)  in make_stripe_request()
5982 		bi->bi_status = BLK_STS_IOERR;  in make_stripe_request()
6012 	if (!add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {  in make_stripe_request()
6034 	    (bi->bi_opf & REQ_SYNC) &&  in make_stripe_request()
6045 		bi->bi_status = BLK_STS_RESOURCE;  in make_stripe_request()
6057 					      struct bio *bi)  in raid5_bio_lowest_chunk_sector()  argument
6064 	sector_t r_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in raid5_bio_lowest_chunk_sector()
6070 	if (sectors_per_chunk - chunk_offset >= bio_sectors(bi))  in raid5_bio_lowest_chunk_sector()
6084 static bool raid5_make_request(struct mddev *mddev, struct bio * bi)  in raid5_make_request()  argument
6088 	const int rw = bio_data_dir(bi);  in raid5_make_request()
6095 	if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {  in raid5_make_request()
6096 		int ret = log_handle_flush_request(conf, bi);  in raid5_make_request()
6101 			if (md_flush_request(mddev, bi))  in raid5_make_request()
6107 	md_write_start(mddev, bi);  in raid5_make_request()
6115 		bi = chunk_aligned_read(mddev, bi);  in raid5_make_request()
6116 		if (!bi)  in raid5_make_request()
6120 	if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) {  in raid5_make_request()
6121 		make_discard_request(mddev, bi);  in raid5_make_request()
6126 	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in raid5_make_request()
6127 	bi->bi_next = NULL;  in raid5_make_request()
6132 	ctx->last_sector = bio_end_sector(bi);  in raid5_make_request()
6137 	if (unlikely(bi->bi_opf & REQ_PREFLUSH))  in raid5_make_request()
6145 		 bi->bi_iter.bi_sector, ctx->last_sector);  in raid5_make_request()
6148 	if ((bi->bi_opf & REQ_NOWAIT) &&  in raid5_make_request()
6150 		bio_wouldblock_error(bi);  in raid5_make_request()
6156 	md_account_bio(mddev, &bi);  in raid5_make_request()
6166 		logical_sector = raid5_bio_lowest_chunk_sector(conf, bi);  in raid5_make_request()
6176 					  bi);  in raid5_make_request()
6221 		WRITE_ONCE(bi->bi_private, &done);  in raid5_make_request()
6223 		bio_endio(bi);  in raid5_make_request()
6229 	bio_endio(bi);  in raid5_make_request()