xref: /linux/block/blk-merge.c (revision 276f98efb64a2c31c099465ace78d3054c662a0f)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Functions related to segment and merge handling
4  */
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/blk-integrity.h>
10 #include <linux/scatterlist.h>
11 #include <linux/part_stat.h>
12 #include <linux/blk-cgroup.h>
13 
14 #include <trace/events/block.h>
15 
16 #include "blk.h"
17 #include "blk-mq-sched.h"
18 #include "blk-rq-qos.h"
19 #include "blk-throttle.h"
20 
bio_get_first_bvec(struct bio * bio,struct bio_vec * bv)21 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
22 {
23 	*bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
24 }
25 
bio_get_last_bvec(struct bio * bio,struct bio_vec * bv)26 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
27 {
28 	struct bvec_iter iter = bio->bi_iter;
29 	int idx;
30 
31 	bio_get_first_bvec(bio, bv);
32 	if (bv->bv_len == bio->bi_iter.bi_size)
33 		return;		/* this bio only has a single bvec */
34 
35 	bio_advance_iter(bio, &iter, iter.bi_size);
36 
37 	if (!iter.bi_bvec_done)
38 		idx = iter.bi_idx - 1;
39 	else	/* in the middle of bvec */
40 		idx = iter.bi_idx;
41 
42 	*bv = bio->bi_io_vec[idx];
43 
44 	/*
45 	 * iter.bi_bvec_done records actual length of the last bvec
46 	 * if this bio ends in the middle of one io vector
47 	 */
48 	if (iter.bi_bvec_done)
49 		bv->bv_len = iter.bi_bvec_done;
50 }
51 
bio_will_gap(struct request_queue * q,struct request * prev_rq,struct bio * prev,struct bio * next)52 static inline bool bio_will_gap(struct request_queue *q,
53 		struct request *prev_rq, struct bio *prev, struct bio *next)
54 {
55 	struct bio_vec pb, nb;
56 
57 	if (!bio_has_data(prev) || !queue_virt_boundary(q))
58 		return false;
59 
60 	/*
61 	 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
62 	 * is quite difficult to respect the sg gap limit.  We work hard to
63 	 * merge a huge number of small single bios in case of mkfs.
64 	 */
65 	if (prev_rq)
66 		bio_get_first_bvec(prev_rq->bio, &pb);
67 	else
68 		bio_get_first_bvec(prev, &pb);
69 	if (pb.bv_offset & queue_virt_boundary(q))
70 		return true;
71 
72 	/*
73 	 * We don't need to worry about the situation that the merged segment
74 	 * ends in unaligned virt boundary:
75 	 *
76 	 * - if 'pb' ends aligned, the merged segment ends aligned
77 	 * - if 'pb' ends unaligned, the next bio must include
78 	 *   one single bvec of 'nb', otherwise the 'nb' can't
79 	 *   merge with 'pb'
80 	 */
81 	bio_get_last_bvec(prev, &pb);
82 	bio_get_first_bvec(next, &nb);
83 	if (biovec_phys_mergeable(q, &pb, &nb))
84 		return false;
85 	return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
86 }
87 
req_gap_back_merge(struct request * req,struct bio * bio)88 static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
89 {
90 	return bio_will_gap(req->q, req, req->biotail, bio);
91 }
92 
req_gap_front_merge(struct request * req,struct bio * bio)93 static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
94 {
95 	return bio_will_gap(req->q, NULL, bio, req->bio);
96 }
97 
98 /*
99  * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
100  * is defined as 'unsigned int', meantime it has to be aligned to with the
101  * logical block size, which is the minimum accepted unit by hardware.
102  */
bio_allowed_max_sectors(const struct queue_limits * lim)103 static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim)
104 {
105 	return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
106 }
107 
bio_submit_split(struct bio * bio,int split_sectors)108 static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
109 {
110 	if (unlikely(split_sectors < 0))
111 		goto error;
112 
113 	if (split_sectors) {
114 		struct bio *split;
115 
116 		split = bio_split(bio, split_sectors, GFP_NOIO,
117 				&bio->bi_bdev->bd_disk->bio_split);
118 		if (IS_ERR(split)) {
119 			split_sectors = PTR_ERR(split);
120 			goto error;
121 		}
122 		split->bi_opf |= REQ_NOMERGE;
123 		blkcg_bio_issue_init(split);
124 		bio_chain(split, bio);
125 		trace_block_split(split, bio->bi_iter.bi_sector);
126 		WARN_ON_ONCE(bio_zone_write_plugging(bio));
127 		submit_bio_noacct(bio);
128 		return split;
129 	}
130 
131 	return bio;
132 error:
133 	bio->bi_status = errno_to_blk_status(split_sectors);
134 	bio_endio(bio);
135 	return NULL;
136 }
137 
bio_split_discard(struct bio * bio,const struct queue_limits * lim,unsigned * nsegs)138 struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
139 		unsigned *nsegs)
140 {
141 	unsigned int max_discard_sectors, granularity;
142 	sector_t tmp;
143 	unsigned split_sectors;
144 
145 	*nsegs = 1;
146 
147 	granularity = max(lim->discard_granularity >> 9, 1U);
148 
149 	max_discard_sectors =
150 		min(lim->max_discard_sectors, bio_allowed_max_sectors(lim));
151 	max_discard_sectors -= max_discard_sectors % granularity;
152 	if (unlikely(!max_discard_sectors))
153 		return bio;
154 
155 	if (bio_sectors(bio) <= max_discard_sectors)
156 		return bio;
157 
158 	split_sectors = max_discard_sectors;
159 
160 	/*
161 	 * If the next starting sector would be misaligned, stop the discard at
162 	 * the previous aligned sector.
163 	 */
164 	tmp = bio->bi_iter.bi_sector + split_sectors -
165 		((lim->discard_alignment >> 9) % granularity);
166 	tmp = sector_div(tmp, granularity);
167 
168 	if (split_sectors > tmp)
169 		split_sectors -= tmp;
170 
171 	return bio_submit_split(bio, split_sectors);
172 }
173 
blk_boundary_sectors(const struct queue_limits * lim,bool is_atomic)174 static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim,
175 						bool is_atomic)
176 {
177 	/*
178 	 * chunk_sectors must be a multiple of atomic_write_boundary_sectors if
179 	 * both non-zero.
180 	 */
181 	if (is_atomic && lim->atomic_write_boundary_sectors)
182 		return lim->atomic_write_boundary_sectors;
183 
184 	return lim->chunk_sectors;
185 }
186 
187 /*
188  * Return the maximum number of sectors from the start of a bio that may be
189  * submitted as a single request to a block device. If enough sectors remain,
190  * align the end to the physical block size. Otherwise align the end to the
191  * logical block size. This approach minimizes the number of non-aligned
192  * requests that are submitted to a block device if the start of a bio is not
193  * aligned to a physical block boundary.
194  */
get_max_io_size(struct bio * bio,const struct queue_limits * lim)195 static inline unsigned get_max_io_size(struct bio *bio,
196 				       const struct queue_limits *lim)
197 {
198 	unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
199 	unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
200 	bool is_atomic = bio->bi_opf & REQ_ATOMIC;
201 	unsigned boundary_sectors = blk_boundary_sectors(lim, is_atomic);
202 	unsigned max_sectors, start, end;
203 
204 	/*
205 	 * We ignore lim->max_sectors for atomic writes because it may less
206 	 * than the actual bio size, which we cannot tolerate.
207 	 */
208 	if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
209 		max_sectors = lim->max_write_zeroes_sectors;
210 	else if (is_atomic)
211 		max_sectors = lim->atomic_write_max_sectors;
212 	else
213 		max_sectors = lim->max_sectors;
214 
215 	if (boundary_sectors) {
216 		max_sectors = min(max_sectors,
217 			blk_boundary_sectors_left(bio->bi_iter.bi_sector,
218 					      boundary_sectors));
219 	}
220 
221 	start = bio->bi_iter.bi_sector & (pbs - 1);
222 	end = (start + max_sectors) & ~(pbs - 1);
223 	if (end > start)
224 		return end - start;
225 	return max_sectors & ~(lbs - 1);
226 }
227 
228 /**
229  * get_max_segment_size() - maximum number of bytes to add as a single segment
230  * @lim: Request queue limits.
231  * @paddr: address of the range to add
232  * @len: maximum length available to add at @paddr
233  *
234  * Returns the maximum number of bytes of the range starting at @paddr that can
235  * be added to a single segment.
236  */
get_max_segment_size(const struct queue_limits * lim,phys_addr_t paddr,unsigned int len)237 static inline unsigned get_max_segment_size(const struct queue_limits *lim,
238 		phys_addr_t paddr, unsigned int len)
239 {
240 	/*
241 	 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1
242 	 * after having calculated the minimum.
243 	 */
244 	return min_t(unsigned long, len,
245 		min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr),
246 		    (unsigned long)lim->max_segment_size - 1) + 1);
247 }
248 
249 /**
250  * bvec_split_segs - verify whether or not a bvec should be split in the middle
251  * @lim:      [in] queue limits to split based on
252  * @bv:       [in] bvec to examine
253  * @nsegs:    [in,out] Number of segments in the bio being built. Incremented
254  *            by the number of segments from @bv that may be appended to that
255  *            bio without exceeding @max_segs
256  * @bytes:    [in,out] Number of bytes in the bio being built. Incremented
257  *            by the number of bytes from @bv that may be appended to that
258  *            bio without exceeding @max_bytes
259  * @max_segs: [in] upper bound for *@nsegs
260  * @max_bytes: [in] upper bound for *@bytes
261  *
262  * When splitting a bio, it can happen that a bvec is encountered that is too
263  * big to fit in a single segment and hence that it has to be split in the
264  * middle. This function verifies whether or not that should happen. The value
265  * %true is returned if and only if appending the entire @bv to a bio with
266  * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
267  * the block driver.
268  */
bvec_split_segs(const struct queue_limits * lim,const struct bio_vec * bv,unsigned * nsegs,unsigned * bytes,unsigned max_segs,unsigned max_bytes)269 static bool bvec_split_segs(const struct queue_limits *lim,
270 		const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes,
271 		unsigned max_segs, unsigned max_bytes)
272 {
273 	unsigned max_len = max_bytes - *bytes;
274 	unsigned len = min(bv->bv_len, max_len);
275 	unsigned total_len = 0;
276 	unsigned seg_size = 0;
277 
278 	while (len && *nsegs < max_segs) {
279 		seg_size = get_max_segment_size(lim, bvec_phys(bv) + total_len, len);
280 
281 		(*nsegs)++;
282 		total_len += seg_size;
283 		len -= seg_size;
284 
285 		if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
286 			break;
287 	}
288 
289 	*bytes += total_len;
290 
291 	/* tell the caller to split the bvec if it is too big to fit */
292 	return len > 0 || bv->bv_len > max_len;
293 }
294 
bio_split_alignment(struct bio * bio,const struct queue_limits * lim)295 static unsigned int bio_split_alignment(struct bio *bio,
296 		const struct queue_limits *lim)
297 {
298 	if (op_is_write(bio_op(bio)) && lim->zone_write_granularity)
299 		return lim->zone_write_granularity;
300 	return lim->logical_block_size;
301 }
302 
303 /**
304  * bio_split_rw_at - check if and where to split a read/write bio
305  * @bio:  [in] bio to be split
306  * @lim:  [in] queue limits to split based on
307  * @segs: [out] number of segments in the bio with the first half of the sectors
308  * @max_bytes: [in] maximum number of bytes per bio
309  *
310  * Find out if @bio needs to be split to fit the queue limits in @lim and a
311  * maximum size of @max_bytes.  Returns a negative error number if @bio can't be
312  * split, 0 if the bio doesn't have to be split, or a positive sector offset if
313  * @bio needs to be split.
314  */
bio_split_rw_at(struct bio * bio,const struct queue_limits * lim,unsigned * segs,unsigned max_bytes)315 int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim,
316 		unsigned *segs, unsigned max_bytes)
317 {
318 	struct bio_vec bv, bvprv, *bvprvp = NULL;
319 	struct bvec_iter iter;
320 	unsigned nsegs = 0, bytes = 0;
321 
322 	bio_for_each_bvec(bv, bio, iter) {
323 		/*
324 		 * If the queue doesn't support SG gaps and adding this
325 		 * offset would create a gap, disallow it.
326 		 */
327 		if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
328 			goto split;
329 
330 		if (nsegs < lim->max_segments &&
331 		    bytes + bv.bv_len <= max_bytes &&
332 		    bv.bv_offset + bv.bv_len <= lim->min_segment_size) {
333 			nsegs++;
334 			bytes += bv.bv_len;
335 		} else {
336 			if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
337 					lim->max_segments, max_bytes))
338 				goto split;
339 		}
340 
341 		bvprv = bv;
342 		bvprvp = &bvprv;
343 	}
344 
345 	*segs = nsegs;
346 	return 0;
347 split:
348 	if (bio->bi_opf & REQ_ATOMIC)
349 		return -EINVAL;
350 
351 	/*
352 	 * We can't sanely support splitting for a REQ_NOWAIT bio. End it
353 	 * with EAGAIN if splitting is required and return an error pointer.
354 	 */
355 	if (bio->bi_opf & REQ_NOWAIT)
356 		return -EAGAIN;
357 
358 	*segs = nsegs;
359 
360 	/*
361 	 * Individual bvecs might not be logical block aligned. Round down the
362 	 * split size so that each bio is properly block size aligned, even if
363 	 * we do not use the full hardware limits.
364 	 */
365 	bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim));
366 
367 	/*
368 	 * Bio splitting may cause subtle trouble such as hang when doing sync
369 	 * iopoll in direct IO routine. Given performance gain of iopoll for
370 	 * big IO can be trival, disable iopoll when split needed.
371 	 */
372 	bio_clear_polled(bio);
373 	return bytes >> SECTOR_SHIFT;
374 }
375 EXPORT_SYMBOL_GPL(bio_split_rw_at);
376 
bio_split_rw(struct bio * bio,const struct queue_limits * lim,unsigned * nr_segs)377 struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
378 		unsigned *nr_segs)
379 {
380 	return bio_submit_split(bio,
381 		bio_split_rw_at(bio, lim, nr_segs,
382 			get_max_io_size(bio, lim) << SECTOR_SHIFT));
383 }
384 
385 /*
386  * REQ_OP_ZONE_APPEND bios must never be split by the block layer.
387  *
388  * But we want the nr_segs calculation provided by bio_split_rw_at, and having
389  * a good sanity check that the submitter built the bio correctly is nice to
390  * have as well.
391  */
bio_split_zone_append(struct bio * bio,const struct queue_limits * lim,unsigned * nr_segs)392 struct bio *bio_split_zone_append(struct bio *bio,
393 		const struct queue_limits *lim, unsigned *nr_segs)
394 {
395 	int split_sectors;
396 
397 	split_sectors = bio_split_rw_at(bio, lim, nr_segs,
398 			lim->max_zone_append_sectors << SECTOR_SHIFT);
399 	if (WARN_ON_ONCE(split_sectors > 0))
400 		split_sectors = -EINVAL;
401 	return bio_submit_split(bio, split_sectors);
402 }
403 
bio_split_write_zeroes(struct bio * bio,const struct queue_limits * lim,unsigned * nsegs)404 struct bio *bio_split_write_zeroes(struct bio *bio,
405 		const struct queue_limits *lim, unsigned *nsegs)
406 {
407 	unsigned int max_sectors = get_max_io_size(bio, lim);
408 
409 	*nsegs = 0;
410 
411 	/*
412 	 * An unset limit should normally not happen, as bio submission is keyed
413 	 * off having a non-zero limit.  But SCSI can clear the limit in the
414 	 * I/O completion handler, and we can race and see this.  Splitting to a
415 	 * zero limit obviously doesn't make sense, so band-aid it here.
416 	 */
417 	if (!max_sectors)
418 		return bio;
419 	if (bio_sectors(bio) <= max_sectors)
420 		return bio;
421 	return bio_submit_split(bio, max_sectors);
422 }
423 
424 /**
425  * bio_split_to_limits - split a bio to fit the queue limits
426  * @bio:     bio to be split
427  *
428  * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
429  * if so split off a bio fitting the limits from the beginning of @bio and
430  * return it.  @bio is shortened to the remainder and re-submitted.
431  *
432  * The split bio is allocated from @q->bio_split, which is provided by the
433  * block layer.
434  */
bio_split_to_limits(struct bio * bio)435 struct bio *bio_split_to_limits(struct bio *bio)
436 {
437 	unsigned int nr_segs;
438 
439 	return __bio_split_to_limits(bio, bdev_limits(bio->bi_bdev), &nr_segs);
440 }
441 EXPORT_SYMBOL(bio_split_to_limits);
442 
blk_recalc_rq_segments(struct request * rq)443 unsigned int blk_recalc_rq_segments(struct request *rq)
444 {
445 	unsigned int nr_phys_segs = 0;
446 	unsigned int bytes = 0;
447 	struct req_iterator iter;
448 	struct bio_vec bv;
449 
450 	if (!rq->bio)
451 		return 0;
452 
453 	switch (bio_op(rq->bio)) {
454 	case REQ_OP_DISCARD:
455 	case REQ_OP_SECURE_ERASE:
456 		if (queue_max_discard_segments(rq->q) > 1) {
457 			struct bio *bio = rq->bio;
458 
459 			for_each_bio(bio)
460 				nr_phys_segs++;
461 			return nr_phys_segs;
462 		}
463 		return 1;
464 	case REQ_OP_WRITE_ZEROES:
465 		return 0;
466 	default:
467 		break;
468 	}
469 
470 	rq_for_each_bvec(bv, rq, iter)
471 		bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
472 				UINT_MAX, UINT_MAX);
473 	return nr_phys_segs;
474 }
475 
476 struct phys_vec {
477 	phys_addr_t	paddr;
478 	u32		len;
479 };
480 
blk_map_iter_next(struct request * req,struct req_iterator * iter,struct phys_vec * vec)481 static bool blk_map_iter_next(struct request *req,
482 		struct req_iterator *iter, struct phys_vec *vec)
483 {
484 	unsigned int max_size;
485 	struct bio_vec bv;
486 
487 	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
488 		if (!iter->bio)
489 			return false;
490 		vec->paddr = bvec_phys(&req->special_vec);
491 		vec->len = req->special_vec.bv_len;
492 		iter->bio = NULL;
493 		return true;
494 	}
495 
496 	if (!iter->iter.bi_size)
497 		return false;
498 
499 	bv = mp_bvec_iter_bvec(iter->bio->bi_io_vec, iter->iter);
500 	vec->paddr = bvec_phys(&bv);
501 	max_size = get_max_segment_size(&req->q->limits, vec->paddr, UINT_MAX);
502 	bv.bv_len = min(bv.bv_len, max_size);
503 	bio_advance_iter_single(iter->bio, &iter->iter, bv.bv_len);
504 
505 	/*
506 	 * If we are entirely done with this bi_io_vec entry, check if the next
507 	 * one could be merged into it.  This typically happens when moving to
508 	 * the next bio, but some callers also don't pack bvecs tight.
509 	 */
510 	while (!iter->iter.bi_size || !iter->iter.bi_bvec_done) {
511 		struct bio_vec next;
512 
513 		if (!iter->iter.bi_size) {
514 			if (!iter->bio->bi_next)
515 				break;
516 			iter->bio = iter->bio->bi_next;
517 			iter->iter = iter->bio->bi_iter;
518 		}
519 
520 		next = mp_bvec_iter_bvec(iter->bio->bi_io_vec, iter->iter);
521 		if (bv.bv_len + next.bv_len > max_size ||
522 		    !biovec_phys_mergeable(req->q, &bv, &next))
523 			break;
524 
525 		bv.bv_len += next.bv_len;
526 		bio_advance_iter_single(iter->bio, &iter->iter, next.bv_len);
527 	}
528 
529 	vec->len = bv.bv_len;
530 	return true;
531 }
532 
blk_next_sg(struct scatterlist ** sg,struct scatterlist * sglist)533 static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
534 		struct scatterlist *sglist)
535 {
536 	if (!*sg)
537 		return sglist;
538 
539 	/*
540 	 * If the driver previously mapped a shorter list, we could see a
541 	 * termination bit prematurely unless it fully inits the sg table
542 	 * on each mapping. We KNOW that there must be more entries here
543 	 * or the driver would be buggy, so force clear the termination bit
544 	 * to avoid doing a full sg_init_table() in drivers for each command.
545 	 */
546 	sg_unmark_end(*sg);
547 	return sg_next(*sg);
548 }
549 
550 /*
551  * Map a request to scatterlist, return number of sg entries setup. Caller
552  * must make sure sg can hold rq->nr_phys_segments entries.
553  */
__blk_rq_map_sg(struct request_queue * q,struct request * rq,struct scatterlist * sglist,struct scatterlist ** last_sg)554 int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
555 		struct scatterlist *sglist, struct scatterlist **last_sg)
556 {
557 	struct req_iterator iter = {
558 		.bio	= rq->bio,
559 	};
560 	struct phys_vec vec;
561 	int nsegs = 0;
562 
563 	/* the internal flush request may not have bio attached */
564 	if (iter.bio)
565 		iter.iter = iter.bio->bi_iter;
566 
567 	while (blk_map_iter_next(rq, &iter, &vec)) {
568 		*last_sg = blk_next_sg(last_sg, sglist);
569 		sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len,
570 				offset_in_page(vec.paddr));
571 		nsegs++;
572 	}
573 
574 	if (*last_sg)
575 		sg_mark_end(*last_sg);
576 
577 	/*
578 	 * Something must have been wrong if the figured number of
579 	 * segment is bigger than number of req's physical segments
580 	 */
581 	WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
582 
583 	return nsegs;
584 }
585 EXPORT_SYMBOL(__blk_rq_map_sg);
586 
blk_rq_get_max_sectors(struct request * rq,sector_t offset)587 static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
588 						  sector_t offset)
589 {
590 	struct request_queue *q = rq->q;
591 	struct queue_limits *lim = &q->limits;
592 	unsigned int max_sectors, boundary_sectors;
593 	bool is_atomic = rq->cmd_flags & REQ_ATOMIC;
594 
595 	if (blk_rq_is_passthrough(rq))
596 		return q->limits.max_hw_sectors;
597 
598 	boundary_sectors = blk_boundary_sectors(lim, is_atomic);
599 	max_sectors = blk_queue_get_max_sectors(rq);
600 
601 	if (!boundary_sectors ||
602 	    req_op(rq) == REQ_OP_DISCARD ||
603 	    req_op(rq) == REQ_OP_SECURE_ERASE)
604 		return max_sectors;
605 	return min(max_sectors,
606 		   blk_boundary_sectors_left(offset, boundary_sectors));
607 }
608 
ll_new_hw_segment(struct request * req,struct bio * bio,unsigned int nr_phys_segs)609 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
610 		unsigned int nr_phys_segs)
611 {
612 	if (!blk_cgroup_mergeable(req, bio))
613 		goto no_merge;
614 
615 	if (blk_integrity_merge_bio(req->q, req, bio) == false)
616 		goto no_merge;
617 
618 	/* discard request merge won't add new segment */
619 	if (req_op(req) == REQ_OP_DISCARD)
620 		return 1;
621 
622 	if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))
623 		goto no_merge;
624 
625 	/*
626 	 * This will form the start of a new hw segment.  Bump both
627 	 * counters.
628 	 */
629 	req->nr_phys_segments += nr_phys_segs;
630 	if (bio_integrity(bio))
631 		req->nr_integrity_segments += blk_rq_count_integrity_sg(req->q,
632 									bio);
633 	return 1;
634 
635 no_merge:
636 	req_set_nomerge(req->q, req);
637 	return 0;
638 }
639 
ll_back_merge_fn(struct request * req,struct bio * bio,unsigned int nr_segs)640 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
641 {
642 	if (req_gap_back_merge(req, bio))
643 		return 0;
644 	if (blk_integrity_rq(req) &&
645 	    integrity_req_gap_back_merge(req, bio))
646 		return 0;
647 	if (!bio_crypt_ctx_back_mergeable(req, bio))
648 		return 0;
649 	if (blk_rq_sectors(req) + bio_sectors(bio) >
650 	    blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
651 		req_set_nomerge(req->q, req);
652 		return 0;
653 	}
654 
655 	return ll_new_hw_segment(req, bio, nr_segs);
656 }
657 
ll_front_merge_fn(struct request * req,struct bio * bio,unsigned int nr_segs)658 static int ll_front_merge_fn(struct request *req, struct bio *bio,
659 		unsigned int nr_segs)
660 {
661 	if (req_gap_front_merge(req, bio))
662 		return 0;
663 	if (blk_integrity_rq(req) &&
664 	    integrity_req_gap_front_merge(req, bio))
665 		return 0;
666 	if (!bio_crypt_ctx_front_mergeable(req, bio))
667 		return 0;
668 	if (blk_rq_sectors(req) + bio_sectors(bio) >
669 	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
670 		req_set_nomerge(req->q, req);
671 		return 0;
672 	}
673 
674 	return ll_new_hw_segment(req, bio, nr_segs);
675 }
676 
req_attempt_discard_merge(struct request_queue * q,struct request * req,struct request * next)677 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
678 		struct request *next)
679 {
680 	unsigned short segments = blk_rq_nr_discard_segments(req);
681 
682 	if (segments >= queue_max_discard_segments(q))
683 		goto no_merge;
684 	if (blk_rq_sectors(req) + bio_sectors(next->bio) >
685 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
686 		goto no_merge;
687 
688 	req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
689 	return true;
690 no_merge:
691 	req_set_nomerge(q, req);
692 	return false;
693 }
694 
ll_merge_requests_fn(struct request_queue * q,struct request * req,struct request * next)695 static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
696 				struct request *next)
697 {
698 	int total_phys_segments;
699 
700 	if (req_gap_back_merge(req, next->bio))
701 		return 0;
702 
703 	/*
704 	 * Will it become too large?
705 	 */
706 	if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
707 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
708 		return 0;
709 
710 	total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
711 	if (total_phys_segments > blk_rq_get_max_segments(req))
712 		return 0;
713 
714 	if (!blk_cgroup_mergeable(req, next->bio))
715 		return 0;
716 
717 	if (blk_integrity_merge_rq(q, req, next) == false)
718 		return 0;
719 
720 	if (!bio_crypt_ctx_merge_rq(req, next))
721 		return 0;
722 
723 	/* Merge is OK... */
724 	req->nr_phys_segments = total_phys_segments;
725 	req->nr_integrity_segments += next->nr_integrity_segments;
726 	return 1;
727 }
728 
729 /**
730  * blk_rq_set_mixed_merge - mark a request as mixed merge
731  * @rq: request to mark as mixed merge
732  *
733  * Description:
734  *     @rq is about to be mixed merged.  Make sure the attributes
735  *     which can be mixed are set in each bio and mark @rq as mixed
736  *     merged.
737  */
blk_rq_set_mixed_merge(struct request * rq)738 static void blk_rq_set_mixed_merge(struct request *rq)
739 {
740 	blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
741 	struct bio *bio;
742 
743 	if (rq->rq_flags & RQF_MIXED_MERGE)
744 		return;
745 
746 	/*
747 	 * @rq will no longer represent mixable attributes for all the
748 	 * contained bios.  It will just track those of the first one.
749 	 * Distributes the attributs to each bio.
750 	 */
751 	for (bio = rq->bio; bio; bio = bio->bi_next) {
752 		WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
753 			     (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
754 		bio->bi_opf |= ff;
755 	}
756 	rq->rq_flags |= RQF_MIXED_MERGE;
757 }
758 
bio_failfast(const struct bio * bio)759 static inline blk_opf_t bio_failfast(const struct bio *bio)
760 {
761 	if (bio->bi_opf & REQ_RAHEAD)
762 		return REQ_FAILFAST_MASK;
763 
764 	return bio->bi_opf & REQ_FAILFAST_MASK;
765 }
766 
767 /*
768  * After we are marked as MIXED_MERGE, any new RA bio has to be updated
769  * as failfast, and request's failfast has to be updated in case of
770  * front merge.
771  */
blk_update_mixed_merge(struct request * req,struct bio * bio,bool front_merge)772 static inline void blk_update_mixed_merge(struct request *req,
773 		struct bio *bio, bool front_merge)
774 {
775 	if (req->rq_flags & RQF_MIXED_MERGE) {
776 		if (bio->bi_opf & REQ_RAHEAD)
777 			bio->bi_opf |= REQ_FAILFAST_MASK;
778 
779 		if (front_merge) {
780 			req->cmd_flags &= ~REQ_FAILFAST_MASK;
781 			req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK;
782 		}
783 	}
784 }
785 
blk_account_io_merge_request(struct request * req)786 static void blk_account_io_merge_request(struct request *req)
787 {
788 	if (req->rq_flags & RQF_IO_STAT) {
789 		part_stat_lock();
790 		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
791 		part_stat_local_dec(req->part,
792 				    in_flight[op_is_write(req_op(req))]);
793 		part_stat_unlock();
794 	}
795 }
796 
blk_try_req_merge(struct request * req,struct request * next)797 static enum elv_merge blk_try_req_merge(struct request *req,
798 					struct request *next)
799 {
800 	if (blk_discard_mergable(req))
801 		return ELEVATOR_DISCARD_MERGE;
802 	else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
803 		return ELEVATOR_BACK_MERGE;
804 
805 	return ELEVATOR_NO_MERGE;
806 }
807 
blk_atomic_write_mergeable_rq_bio(struct request * rq,struct bio * bio)808 static bool blk_atomic_write_mergeable_rq_bio(struct request *rq,
809 					      struct bio *bio)
810 {
811 	return (rq->cmd_flags & REQ_ATOMIC) == (bio->bi_opf & REQ_ATOMIC);
812 }
813 
blk_atomic_write_mergeable_rqs(struct request * rq,struct request * next)814 static bool blk_atomic_write_mergeable_rqs(struct request *rq,
815 					   struct request *next)
816 {
817 	return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC);
818 }
819 
820 /*
821  * For non-mq, this has to be called with the request spinlock acquired.
822  * For mq with scheduling, the appropriate queue wide lock should be held.
823  */
attempt_merge(struct request_queue * q,struct request * req,struct request * next)824 static struct request *attempt_merge(struct request_queue *q,
825 				     struct request *req, struct request *next)
826 {
827 	if (!rq_mergeable(req) || !rq_mergeable(next))
828 		return NULL;
829 
830 	if (req_op(req) != req_op(next))
831 		return NULL;
832 
833 	if (req->bio->bi_write_hint != next->bio->bi_write_hint)
834 		return NULL;
835 	if (req->bio->bi_ioprio != next->bio->bi_ioprio)
836 		return NULL;
837 	if (!blk_atomic_write_mergeable_rqs(req, next))
838 		return NULL;
839 
840 	/*
841 	 * If we are allowed to merge, then append bio list
842 	 * from next to rq and release next. merge_requests_fn
843 	 * will have updated segment counts, update sector
844 	 * counts here. Handle DISCARDs separately, as they
845 	 * have separate settings.
846 	 */
847 
848 	switch (blk_try_req_merge(req, next)) {
849 	case ELEVATOR_DISCARD_MERGE:
850 		if (!req_attempt_discard_merge(q, req, next))
851 			return NULL;
852 		break;
853 	case ELEVATOR_BACK_MERGE:
854 		if (!ll_merge_requests_fn(q, req, next))
855 			return NULL;
856 		break;
857 	default:
858 		return NULL;
859 	}
860 
861 	/*
862 	 * If failfast settings disagree or any of the two is already
863 	 * a mixed merge, mark both as mixed before proceeding.  This
864 	 * makes sure that all involved bios have mixable attributes
865 	 * set properly.
866 	 */
867 	if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
868 	    (req->cmd_flags & REQ_FAILFAST_MASK) !=
869 	    (next->cmd_flags & REQ_FAILFAST_MASK)) {
870 		blk_rq_set_mixed_merge(req);
871 		blk_rq_set_mixed_merge(next);
872 	}
873 
874 	/*
875 	 * At this point we have either done a back merge or front merge. We
876 	 * need the smaller start_time_ns of the merged requests to be the
877 	 * current request for accounting purposes.
878 	 */
879 	if (next->start_time_ns < req->start_time_ns)
880 		req->start_time_ns = next->start_time_ns;
881 
882 	req->biotail->bi_next = next->bio;
883 	req->biotail = next->biotail;
884 
885 	req->__data_len += blk_rq_bytes(next);
886 
887 	if (!blk_discard_mergable(req))
888 		elv_merge_requests(q, req, next);
889 
890 	blk_crypto_rq_put_keyslot(next);
891 
892 	/*
893 	 * 'next' is going away, so update stats accordingly
894 	 */
895 	blk_account_io_merge_request(next);
896 
897 	trace_block_rq_merge(next);
898 
899 	/*
900 	 * ownership of bio passed from next to req, return 'next' for
901 	 * the caller to free
902 	 */
903 	next->bio = NULL;
904 	return next;
905 }
906 
attempt_back_merge(struct request_queue * q,struct request * rq)907 static struct request *attempt_back_merge(struct request_queue *q,
908 		struct request *rq)
909 {
910 	struct request *next = elv_latter_request(q, rq);
911 
912 	if (next)
913 		return attempt_merge(q, rq, next);
914 
915 	return NULL;
916 }
917 
attempt_front_merge(struct request_queue * q,struct request * rq)918 static struct request *attempt_front_merge(struct request_queue *q,
919 		struct request *rq)
920 {
921 	struct request *prev = elv_former_request(q, rq);
922 
923 	if (prev)
924 		return attempt_merge(q, prev, rq);
925 
926 	return NULL;
927 }
928 
929 /*
930  * Try to merge 'next' into 'rq'. Return true if the merge happened, false
931  * otherwise. The caller is responsible for freeing 'next' if the merge
932  * happened.
933  */
blk_attempt_req_merge(struct request_queue * q,struct request * rq,struct request * next)934 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
935 			   struct request *next)
936 {
937 	return attempt_merge(q, rq, next);
938 }
939 
blk_rq_merge_ok(struct request * rq,struct bio * bio)940 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
941 {
942 	if (!rq_mergeable(rq) || !bio_mergeable(bio))
943 		return false;
944 
945 	if (req_op(rq) != bio_op(bio))
946 		return false;
947 
948 	if (!blk_cgroup_mergeable(rq, bio))
949 		return false;
950 	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
951 		return false;
952 	if (!bio_crypt_rq_ctx_compatible(rq, bio))
953 		return false;
954 	if (rq->bio->bi_write_hint != bio->bi_write_hint)
955 		return false;
956 	if (rq->bio->bi_ioprio != bio->bi_ioprio)
957 		return false;
958 	if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false)
959 		return false;
960 
961 	return true;
962 }
963 
blk_try_merge(struct request * rq,struct bio * bio)964 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
965 {
966 	if (blk_discard_mergable(rq))
967 		return ELEVATOR_DISCARD_MERGE;
968 	else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
969 		return ELEVATOR_BACK_MERGE;
970 	else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
971 		return ELEVATOR_FRONT_MERGE;
972 	return ELEVATOR_NO_MERGE;
973 }
974 
blk_account_io_merge_bio(struct request * req)975 static void blk_account_io_merge_bio(struct request *req)
976 {
977 	if (req->rq_flags & RQF_IO_STAT) {
978 		part_stat_lock();
979 		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
980 		part_stat_unlock();
981 	}
982 }
983 
bio_attempt_back_merge(struct request * req,struct bio * bio,unsigned int nr_segs)984 enum bio_merge_status bio_attempt_back_merge(struct request *req,
985 		struct bio *bio, unsigned int nr_segs)
986 {
987 	const blk_opf_t ff = bio_failfast(bio);
988 
989 	if (!ll_back_merge_fn(req, bio, nr_segs))
990 		return BIO_MERGE_FAILED;
991 
992 	trace_block_bio_backmerge(bio);
993 	rq_qos_merge(req->q, req, bio);
994 
995 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
996 		blk_rq_set_mixed_merge(req);
997 
998 	blk_update_mixed_merge(req, bio, false);
999 
1000 	if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
1001 		blk_zone_write_plug_bio_merged(bio);
1002 
1003 	req->biotail->bi_next = bio;
1004 	req->biotail = bio;
1005 	req->__data_len += bio->bi_iter.bi_size;
1006 
1007 	bio_crypt_free_ctx(bio);
1008 
1009 	blk_account_io_merge_bio(req);
1010 	return BIO_MERGE_OK;
1011 }
1012 
bio_attempt_front_merge(struct request * req,struct bio * bio,unsigned int nr_segs)1013 static enum bio_merge_status bio_attempt_front_merge(struct request *req,
1014 		struct bio *bio, unsigned int nr_segs)
1015 {
1016 	const blk_opf_t ff = bio_failfast(bio);
1017 
1018 	/*
1019 	 * A front merge for writes to sequential zones of a zoned block device
1020 	 * can happen only if the user submitted writes out of order. Do not
1021 	 * merge such write to let it fail.
1022 	 */
1023 	if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
1024 		return BIO_MERGE_FAILED;
1025 
1026 	if (!ll_front_merge_fn(req, bio, nr_segs))
1027 		return BIO_MERGE_FAILED;
1028 
1029 	trace_block_bio_frontmerge(bio);
1030 	rq_qos_merge(req->q, req, bio);
1031 
1032 	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1033 		blk_rq_set_mixed_merge(req);
1034 
1035 	blk_update_mixed_merge(req, bio, true);
1036 
1037 	bio->bi_next = req->bio;
1038 	req->bio = bio;
1039 
1040 	req->__sector = bio->bi_iter.bi_sector;
1041 	req->__data_len += bio->bi_iter.bi_size;
1042 
1043 	bio_crypt_do_front_merge(req, bio);
1044 
1045 	blk_account_io_merge_bio(req);
1046 	return BIO_MERGE_OK;
1047 }
1048 
bio_attempt_discard_merge(struct request_queue * q,struct request * req,struct bio * bio)1049 static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
1050 		struct request *req, struct bio *bio)
1051 {
1052 	unsigned short segments = blk_rq_nr_discard_segments(req);
1053 
1054 	if (segments >= queue_max_discard_segments(q))
1055 		goto no_merge;
1056 	if (blk_rq_sectors(req) + bio_sectors(bio) >
1057 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1058 		goto no_merge;
1059 
1060 	rq_qos_merge(q, req, bio);
1061 
1062 	req->biotail->bi_next = bio;
1063 	req->biotail = bio;
1064 	req->__data_len += bio->bi_iter.bi_size;
1065 	req->nr_phys_segments = segments + 1;
1066 
1067 	blk_account_io_merge_bio(req);
1068 	return BIO_MERGE_OK;
1069 no_merge:
1070 	req_set_nomerge(q, req);
1071 	return BIO_MERGE_FAILED;
1072 }
1073 
blk_attempt_bio_merge(struct request_queue * q,struct request * rq,struct bio * bio,unsigned int nr_segs,bool sched_allow_merge)1074 static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
1075 						   struct request *rq,
1076 						   struct bio *bio,
1077 						   unsigned int nr_segs,
1078 						   bool sched_allow_merge)
1079 {
1080 	if (!blk_rq_merge_ok(rq, bio))
1081 		return BIO_MERGE_NONE;
1082 
1083 	switch (blk_try_merge(rq, bio)) {
1084 	case ELEVATOR_BACK_MERGE:
1085 		if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1086 			return bio_attempt_back_merge(rq, bio, nr_segs);
1087 		break;
1088 	case ELEVATOR_FRONT_MERGE:
1089 		if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1090 			return bio_attempt_front_merge(rq, bio, nr_segs);
1091 		break;
1092 	case ELEVATOR_DISCARD_MERGE:
1093 		return bio_attempt_discard_merge(q, rq, bio);
1094 	default:
1095 		return BIO_MERGE_NONE;
1096 	}
1097 
1098 	return BIO_MERGE_FAILED;
1099 }
1100 
1101 /**
1102  * blk_attempt_plug_merge - try to merge with %current's plugged list
1103  * @q: request_queue new bio is being queued at
1104  * @bio: new bio being queued
1105  * @nr_segs: number of segments in @bio
1106  * from the passed in @q already in the plug list
1107  *
1108  * Determine whether @bio being queued on @q can be merged with the previous
1109  * request on %current's plugged list.  Returns %true if merge was successful,
1110  * otherwise %false.
1111  *
1112  * Plugging coalesces IOs from the same issuer for the same purpose without
1113  * going through @q->queue_lock.  As such it's more of an issuing mechanism
1114  * than scheduling, and the request, while may have elvpriv data, is not
1115  * added on the elevator at this point.  In addition, we don't have
1116  * reliable access to the elevator outside queue lock.  Only check basic
1117  * merging parameters without querying the elevator.
1118  *
1119  * Caller must ensure !blk_queue_nomerges(q) beforehand.
1120  */
blk_attempt_plug_merge(struct request_queue * q,struct bio * bio,unsigned int nr_segs)1121 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1122 		unsigned int nr_segs)
1123 {
1124 	struct blk_plug *plug = current->plug;
1125 	struct request *rq;
1126 
1127 	if (!plug || rq_list_empty(&plug->mq_list))
1128 		return false;
1129 
1130 	rq_list_for_each(&plug->mq_list, rq) {
1131 		if (rq->q == q) {
1132 			if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1133 			    BIO_MERGE_OK)
1134 				return true;
1135 			break;
1136 		}
1137 
1138 		/*
1139 		 * Only keep iterating plug list for merges if we have multiple
1140 		 * queues
1141 		 */
1142 		if (!plug->multiple_queues)
1143 			break;
1144 	}
1145 	return false;
1146 }
1147 
1148 /*
1149  * Iterate list of requests and see if we can merge this bio with any
1150  * of them.
1151  */
blk_bio_list_merge(struct request_queue * q,struct list_head * list,struct bio * bio,unsigned int nr_segs)1152 bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
1153 			struct bio *bio, unsigned int nr_segs)
1154 {
1155 	struct request *rq;
1156 	int checked = 8;
1157 
1158 	list_for_each_entry_reverse(rq, list, queuelist) {
1159 		if (!checked--)
1160 			break;
1161 
1162 		switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
1163 		case BIO_MERGE_NONE:
1164 			continue;
1165 		case BIO_MERGE_OK:
1166 			return true;
1167 		case BIO_MERGE_FAILED:
1168 			return false;
1169 		}
1170 
1171 	}
1172 
1173 	return false;
1174 }
1175 EXPORT_SYMBOL_GPL(blk_bio_list_merge);
1176 
blk_mq_sched_try_merge(struct request_queue * q,struct bio * bio,unsigned int nr_segs,struct request ** merged_request)1177 bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
1178 		unsigned int nr_segs, struct request **merged_request)
1179 {
1180 	struct request *rq;
1181 
1182 	switch (elv_merge(q, &rq, bio)) {
1183 	case ELEVATOR_BACK_MERGE:
1184 		if (!blk_mq_sched_allow_merge(q, rq, bio))
1185 			return false;
1186 		if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1187 			return false;
1188 		*merged_request = attempt_back_merge(q, rq);
1189 		if (!*merged_request)
1190 			elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
1191 		return true;
1192 	case ELEVATOR_FRONT_MERGE:
1193 		if (!blk_mq_sched_allow_merge(q, rq, bio))
1194 			return false;
1195 		if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1196 			return false;
1197 		*merged_request = attempt_front_merge(q, rq);
1198 		if (!*merged_request)
1199 			elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
1200 		return true;
1201 	case ELEVATOR_DISCARD_MERGE:
1202 		return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
1203 	default:
1204 		return false;
1205 	}
1206 }
1207 EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
1208