1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Functions related to segment and merge handling
4 */
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/blk-integrity.h>
10 #include <linux/scatterlist.h>
11 #include <linux/part_stat.h>
12 #include <linux/blk-cgroup.h>
13
14 #include <trace/events/block.h>
15
16 #include "blk.h"
17 #include "blk-mq-sched.h"
18 #include "blk-rq-qos.h"
19 #include "blk-throttle.h"
20
bio_get_first_bvec(struct bio * bio,struct bio_vec * bv)21 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
22 {
23 *bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
24 }
25
bio_get_last_bvec(struct bio * bio,struct bio_vec * bv)26 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
27 {
28 struct bvec_iter iter = bio->bi_iter;
29 int idx;
30
31 bio_get_first_bvec(bio, bv);
32 if (bv->bv_len == bio->bi_iter.bi_size)
33 return; /* this bio only has a single bvec */
34
35 bio_advance_iter(bio, &iter, iter.bi_size);
36
37 if (!iter.bi_bvec_done)
38 idx = iter.bi_idx - 1;
39 else /* in the middle of bvec */
40 idx = iter.bi_idx;
41
42 *bv = bio->bi_io_vec[idx];
43
44 /*
45 * iter.bi_bvec_done records actual length of the last bvec
46 * if this bio ends in the middle of one io vector
47 */
48 if (iter.bi_bvec_done)
49 bv->bv_len = iter.bi_bvec_done;
50 }
51
bio_will_gap(struct request_queue * q,struct request * prev_rq,struct bio * prev,struct bio * next)52 static inline bool bio_will_gap(struct request_queue *q,
53 struct request *prev_rq, struct bio *prev, struct bio *next)
54 {
55 struct bio_vec pb, nb;
56
57 if (!bio_has_data(prev) || !queue_virt_boundary(q))
58 return false;
59
60 /*
61 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
62 * is quite difficult to respect the sg gap limit. We work hard to
63 * merge a huge number of small single bios in case of mkfs.
64 */
65 if (prev_rq)
66 bio_get_first_bvec(prev_rq->bio, &pb);
67 else
68 bio_get_first_bvec(prev, &pb);
69 if (pb.bv_offset & queue_virt_boundary(q))
70 return true;
71
72 /*
73 * We don't need to worry about the situation that the merged segment
74 * ends in unaligned virt boundary:
75 *
76 * - if 'pb' ends aligned, the merged segment ends aligned
77 * - if 'pb' ends unaligned, the next bio must include
78 * one single bvec of 'nb', otherwise the 'nb' can't
79 * merge with 'pb'
80 */
81 bio_get_last_bvec(prev, &pb);
82 bio_get_first_bvec(next, &nb);
83 if (biovec_phys_mergeable(q, &pb, &nb))
84 return false;
85 return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
86 }
87
req_gap_back_merge(struct request * req,struct bio * bio)88 static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
89 {
90 return bio_will_gap(req->q, req, req->biotail, bio);
91 }
92
req_gap_front_merge(struct request * req,struct bio * bio)93 static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
94 {
95 return bio_will_gap(req->q, NULL, bio, req->bio);
96 }
97
98 /*
99 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
100 * is defined as 'unsigned int', meantime it has to be aligned to with the
101 * logical block size, which is the minimum accepted unit by hardware.
102 */
bio_allowed_max_sectors(const struct queue_limits * lim)103 static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim)
104 {
105 return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
106 }
107
bio_submit_split(struct bio * bio,int split_sectors)108 static struct bio *bio_submit_split(struct bio *bio, int split_sectors)
109 {
110 if (unlikely(split_sectors < 0))
111 goto error;
112
113 if (split_sectors) {
114 struct bio *split;
115
116 split = bio_split(bio, split_sectors, GFP_NOIO,
117 &bio->bi_bdev->bd_disk->bio_split);
118 if (IS_ERR(split)) {
119 split_sectors = PTR_ERR(split);
120 goto error;
121 }
122 split->bi_opf |= REQ_NOMERGE;
123 blkcg_bio_issue_init(split);
124 bio_chain(split, bio);
125 trace_block_split(split, bio->bi_iter.bi_sector);
126 WARN_ON_ONCE(bio_zone_write_plugging(bio));
127 submit_bio_noacct(bio);
128 return split;
129 }
130
131 return bio;
132 error:
133 bio->bi_status = errno_to_blk_status(split_sectors);
134 bio_endio(bio);
135 return NULL;
136 }
137
bio_split_discard(struct bio * bio,const struct queue_limits * lim,unsigned * nsegs)138 struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
139 unsigned *nsegs)
140 {
141 unsigned int max_discard_sectors, granularity;
142 sector_t tmp;
143 unsigned split_sectors;
144
145 *nsegs = 1;
146
147 granularity = max(lim->discard_granularity >> 9, 1U);
148
149 max_discard_sectors =
150 min(lim->max_discard_sectors, bio_allowed_max_sectors(lim));
151 max_discard_sectors -= max_discard_sectors % granularity;
152 if (unlikely(!max_discard_sectors))
153 return bio;
154
155 if (bio_sectors(bio) <= max_discard_sectors)
156 return bio;
157
158 split_sectors = max_discard_sectors;
159
160 /*
161 * If the next starting sector would be misaligned, stop the discard at
162 * the previous aligned sector.
163 */
164 tmp = bio->bi_iter.bi_sector + split_sectors -
165 ((lim->discard_alignment >> 9) % granularity);
166 tmp = sector_div(tmp, granularity);
167
168 if (split_sectors > tmp)
169 split_sectors -= tmp;
170
171 return bio_submit_split(bio, split_sectors);
172 }
173
blk_boundary_sectors(const struct queue_limits * lim,bool is_atomic)174 static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim,
175 bool is_atomic)
176 {
177 /*
178 * chunk_sectors must be a multiple of atomic_write_boundary_sectors if
179 * both non-zero.
180 */
181 if (is_atomic && lim->atomic_write_boundary_sectors)
182 return lim->atomic_write_boundary_sectors;
183
184 return lim->chunk_sectors;
185 }
186
187 /*
188 * Return the maximum number of sectors from the start of a bio that may be
189 * submitted as a single request to a block device. If enough sectors remain,
190 * align the end to the physical block size. Otherwise align the end to the
191 * logical block size. This approach minimizes the number of non-aligned
192 * requests that are submitted to a block device if the start of a bio is not
193 * aligned to a physical block boundary.
194 */
get_max_io_size(struct bio * bio,const struct queue_limits * lim)195 static inline unsigned get_max_io_size(struct bio *bio,
196 const struct queue_limits *lim)
197 {
198 unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
199 unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
200 bool is_atomic = bio->bi_opf & REQ_ATOMIC;
201 unsigned boundary_sectors = blk_boundary_sectors(lim, is_atomic);
202 unsigned max_sectors, start, end;
203
204 /*
205 * We ignore lim->max_sectors for atomic writes because it may less
206 * than the actual bio size, which we cannot tolerate.
207 */
208 if (bio_op(bio) == REQ_OP_WRITE_ZEROES)
209 max_sectors = lim->max_write_zeroes_sectors;
210 else if (is_atomic)
211 max_sectors = lim->atomic_write_max_sectors;
212 else
213 max_sectors = lim->max_sectors;
214
215 if (boundary_sectors) {
216 max_sectors = min(max_sectors,
217 blk_boundary_sectors_left(bio->bi_iter.bi_sector,
218 boundary_sectors));
219 }
220
221 start = bio->bi_iter.bi_sector & (pbs - 1);
222 end = (start + max_sectors) & ~(pbs - 1);
223 if (end > start)
224 return end - start;
225 return max_sectors & ~(lbs - 1);
226 }
227
228 /**
229 * get_max_segment_size() - maximum number of bytes to add as a single segment
230 * @lim: Request queue limits.
231 * @paddr: address of the range to add
232 * @len: maximum length available to add at @paddr
233 *
234 * Returns the maximum number of bytes of the range starting at @paddr that can
235 * be added to a single segment.
236 */
get_max_segment_size(const struct queue_limits * lim,phys_addr_t paddr,unsigned int len)237 static inline unsigned get_max_segment_size(const struct queue_limits *lim,
238 phys_addr_t paddr, unsigned int len)
239 {
240 /*
241 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1
242 * after having calculated the minimum.
243 */
244 return min_t(unsigned long, len,
245 min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr),
246 (unsigned long)lim->max_segment_size - 1) + 1);
247 }
248
249 /**
250 * bvec_split_segs - verify whether or not a bvec should be split in the middle
251 * @lim: [in] queue limits to split based on
252 * @bv: [in] bvec to examine
253 * @nsegs: [in,out] Number of segments in the bio being built. Incremented
254 * by the number of segments from @bv that may be appended to that
255 * bio without exceeding @max_segs
256 * @bytes: [in,out] Number of bytes in the bio being built. Incremented
257 * by the number of bytes from @bv that may be appended to that
258 * bio without exceeding @max_bytes
259 * @max_segs: [in] upper bound for *@nsegs
260 * @max_bytes: [in] upper bound for *@bytes
261 *
262 * When splitting a bio, it can happen that a bvec is encountered that is too
263 * big to fit in a single segment and hence that it has to be split in the
264 * middle. This function verifies whether or not that should happen. The value
265 * %true is returned if and only if appending the entire @bv to a bio with
266 * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
267 * the block driver.
268 */
bvec_split_segs(const struct queue_limits * lim,const struct bio_vec * bv,unsigned * nsegs,unsigned * bytes,unsigned max_segs,unsigned max_bytes)269 static bool bvec_split_segs(const struct queue_limits *lim,
270 const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes,
271 unsigned max_segs, unsigned max_bytes)
272 {
273 unsigned max_len = max_bytes - *bytes;
274 unsigned len = min(bv->bv_len, max_len);
275 unsigned total_len = 0;
276 unsigned seg_size = 0;
277
278 while (len && *nsegs < max_segs) {
279 seg_size = get_max_segment_size(lim, bvec_phys(bv) + total_len, len);
280
281 (*nsegs)++;
282 total_len += seg_size;
283 len -= seg_size;
284
285 if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
286 break;
287 }
288
289 *bytes += total_len;
290
291 /* tell the caller to split the bvec if it is too big to fit */
292 return len > 0 || bv->bv_len > max_len;
293 }
294
bio_split_alignment(struct bio * bio,const struct queue_limits * lim)295 static unsigned int bio_split_alignment(struct bio *bio,
296 const struct queue_limits *lim)
297 {
298 if (op_is_write(bio_op(bio)) && lim->zone_write_granularity)
299 return lim->zone_write_granularity;
300 return lim->logical_block_size;
301 }
302
303 /**
304 * bio_split_rw_at - check if and where to split a read/write bio
305 * @bio: [in] bio to be split
306 * @lim: [in] queue limits to split based on
307 * @segs: [out] number of segments in the bio with the first half of the sectors
308 * @max_bytes: [in] maximum number of bytes per bio
309 *
310 * Find out if @bio needs to be split to fit the queue limits in @lim and a
311 * maximum size of @max_bytes. Returns a negative error number if @bio can't be
312 * split, 0 if the bio doesn't have to be split, or a positive sector offset if
313 * @bio needs to be split.
314 */
bio_split_rw_at(struct bio * bio,const struct queue_limits * lim,unsigned * segs,unsigned max_bytes)315 int bio_split_rw_at(struct bio *bio, const struct queue_limits *lim,
316 unsigned *segs, unsigned max_bytes)
317 {
318 struct bio_vec bv, bvprv, *bvprvp = NULL;
319 struct bvec_iter iter;
320 unsigned nsegs = 0, bytes = 0;
321
322 bio_for_each_bvec(bv, bio, iter) {
323 /*
324 * If the queue doesn't support SG gaps and adding this
325 * offset would create a gap, disallow it.
326 */
327 if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
328 goto split;
329
330 if (nsegs < lim->max_segments &&
331 bytes + bv.bv_len <= max_bytes &&
332 bv.bv_offset + bv.bv_len <= lim->min_segment_size) {
333 nsegs++;
334 bytes += bv.bv_len;
335 } else {
336 if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
337 lim->max_segments, max_bytes))
338 goto split;
339 }
340
341 bvprv = bv;
342 bvprvp = &bvprv;
343 }
344
345 *segs = nsegs;
346 return 0;
347 split:
348 if (bio->bi_opf & REQ_ATOMIC)
349 return -EINVAL;
350
351 /*
352 * We can't sanely support splitting for a REQ_NOWAIT bio. End it
353 * with EAGAIN if splitting is required and return an error pointer.
354 */
355 if (bio->bi_opf & REQ_NOWAIT)
356 return -EAGAIN;
357
358 *segs = nsegs;
359
360 /*
361 * Individual bvecs might not be logical block aligned. Round down the
362 * split size so that each bio is properly block size aligned, even if
363 * we do not use the full hardware limits.
364 */
365 bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim));
366
367 /*
368 * Bio splitting may cause subtle trouble such as hang when doing sync
369 * iopoll in direct IO routine. Given performance gain of iopoll for
370 * big IO can be trival, disable iopoll when split needed.
371 */
372 bio_clear_polled(bio);
373 return bytes >> SECTOR_SHIFT;
374 }
375 EXPORT_SYMBOL_GPL(bio_split_rw_at);
376
bio_split_rw(struct bio * bio,const struct queue_limits * lim,unsigned * nr_segs)377 struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
378 unsigned *nr_segs)
379 {
380 return bio_submit_split(bio,
381 bio_split_rw_at(bio, lim, nr_segs,
382 get_max_io_size(bio, lim) << SECTOR_SHIFT));
383 }
384
385 /*
386 * REQ_OP_ZONE_APPEND bios must never be split by the block layer.
387 *
388 * But we want the nr_segs calculation provided by bio_split_rw_at, and having
389 * a good sanity check that the submitter built the bio correctly is nice to
390 * have as well.
391 */
bio_split_zone_append(struct bio * bio,const struct queue_limits * lim,unsigned * nr_segs)392 struct bio *bio_split_zone_append(struct bio *bio,
393 const struct queue_limits *lim, unsigned *nr_segs)
394 {
395 int split_sectors;
396
397 split_sectors = bio_split_rw_at(bio, lim, nr_segs,
398 lim->max_zone_append_sectors << SECTOR_SHIFT);
399 if (WARN_ON_ONCE(split_sectors > 0))
400 split_sectors = -EINVAL;
401 return bio_submit_split(bio, split_sectors);
402 }
403
bio_split_write_zeroes(struct bio * bio,const struct queue_limits * lim,unsigned * nsegs)404 struct bio *bio_split_write_zeroes(struct bio *bio,
405 const struct queue_limits *lim, unsigned *nsegs)
406 {
407 unsigned int max_sectors = get_max_io_size(bio, lim);
408
409 *nsegs = 0;
410
411 /*
412 * An unset limit should normally not happen, as bio submission is keyed
413 * off having a non-zero limit. But SCSI can clear the limit in the
414 * I/O completion handler, and we can race and see this. Splitting to a
415 * zero limit obviously doesn't make sense, so band-aid it here.
416 */
417 if (!max_sectors)
418 return bio;
419 if (bio_sectors(bio) <= max_sectors)
420 return bio;
421 return bio_submit_split(bio, max_sectors);
422 }
423
424 /**
425 * bio_split_to_limits - split a bio to fit the queue limits
426 * @bio: bio to be split
427 *
428 * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and
429 * if so split off a bio fitting the limits from the beginning of @bio and
430 * return it. @bio is shortened to the remainder and re-submitted.
431 *
432 * The split bio is allocated from @q->bio_split, which is provided by the
433 * block layer.
434 */
bio_split_to_limits(struct bio * bio)435 struct bio *bio_split_to_limits(struct bio *bio)
436 {
437 unsigned int nr_segs;
438
439 return __bio_split_to_limits(bio, bdev_limits(bio->bi_bdev), &nr_segs);
440 }
441 EXPORT_SYMBOL(bio_split_to_limits);
442
blk_recalc_rq_segments(struct request * rq)443 unsigned int blk_recalc_rq_segments(struct request *rq)
444 {
445 unsigned int nr_phys_segs = 0;
446 unsigned int bytes = 0;
447 struct req_iterator iter;
448 struct bio_vec bv;
449
450 if (!rq->bio)
451 return 0;
452
453 switch (bio_op(rq->bio)) {
454 case REQ_OP_DISCARD:
455 case REQ_OP_SECURE_ERASE:
456 if (queue_max_discard_segments(rq->q) > 1) {
457 struct bio *bio = rq->bio;
458
459 for_each_bio(bio)
460 nr_phys_segs++;
461 return nr_phys_segs;
462 }
463 return 1;
464 case REQ_OP_WRITE_ZEROES:
465 return 0;
466 default:
467 break;
468 }
469
470 rq_for_each_bvec(bv, rq, iter)
471 bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
472 UINT_MAX, UINT_MAX);
473 return nr_phys_segs;
474 }
475
476 struct phys_vec {
477 phys_addr_t paddr;
478 u32 len;
479 };
480
blk_map_iter_next(struct request * req,struct req_iterator * iter,struct phys_vec * vec)481 static bool blk_map_iter_next(struct request *req,
482 struct req_iterator *iter, struct phys_vec *vec)
483 {
484 unsigned int max_size;
485 struct bio_vec bv;
486
487 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
488 if (!iter->bio)
489 return false;
490 vec->paddr = bvec_phys(&req->special_vec);
491 vec->len = req->special_vec.bv_len;
492 iter->bio = NULL;
493 return true;
494 }
495
496 if (!iter->iter.bi_size)
497 return false;
498
499 bv = mp_bvec_iter_bvec(iter->bio->bi_io_vec, iter->iter);
500 vec->paddr = bvec_phys(&bv);
501 max_size = get_max_segment_size(&req->q->limits, vec->paddr, UINT_MAX);
502 bv.bv_len = min(bv.bv_len, max_size);
503 bio_advance_iter_single(iter->bio, &iter->iter, bv.bv_len);
504
505 /*
506 * If we are entirely done with this bi_io_vec entry, check if the next
507 * one could be merged into it. This typically happens when moving to
508 * the next bio, but some callers also don't pack bvecs tight.
509 */
510 while (!iter->iter.bi_size || !iter->iter.bi_bvec_done) {
511 struct bio_vec next;
512
513 if (!iter->iter.bi_size) {
514 if (!iter->bio->bi_next)
515 break;
516 iter->bio = iter->bio->bi_next;
517 iter->iter = iter->bio->bi_iter;
518 }
519
520 next = mp_bvec_iter_bvec(iter->bio->bi_io_vec, iter->iter);
521 if (bv.bv_len + next.bv_len > max_size ||
522 !biovec_phys_mergeable(req->q, &bv, &next))
523 break;
524
525 bv.bv_len += next.bv_len;
526 bio_advance_iter_single(iter->bio, &iter->iter, next.bv_len);
527 }
528
529 vec->len = bv.bv_len;
530 return true;
531 }
532
blk_next_sg(struct scatterlist ** sg,struct scatterlist * sglist)533 static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
534 struct scatterlist *sglist)
535 {
536 if (!*sg)
537 return sglist;
538
539 /*
540 * If the driver previously mapped a shorter list, we could see a
541 * termination bit prematurely unless it fully inits the sg table
542 * on each mapping. We KNOW that there must be more entries here
543 * or the driver would be buggy, so force clear the termination bit
544 * to avoid doing a full sg_init_table() in drivers for each command.
545 */
546 sg_unmark_end(*sg);
547 return sg_next(*sg);
548 }
549
550 /*
551 * Map a request to scatterlist, return number of sg entries setup. Caller
552 * must make sure sg can hold rq->nr_phys_segments entries.
553 */
__blk_rq_map_sg(struct request_queue * q,struct request * rq,struct scatterlist * sglist,struct scatterlist ** last_sg)554 int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
555 struct scatterlist *sglist, struct scatterlist **last_sg)
556 {
557 struct req_iterator iter = {
558 .bio = rq->bio,
559 };
560 struct phys_vec vec;
561 int nsegs = 0;
562
563 /* the internal flush request may not have bio attached */
564 if (iter.bio)
565 iter.iter = iter.bio->bi_iter;
566
567 while (blk_map_iter_next(rq, &iter, &vec)) {
568 *last_sg = blk_next_sg(last_sg, sglist);
569 sg_set_page(*last_sg, phys_to_page(vec.paddr), vec.len,
570 offset_in_page(vec.paddr));
571 nsegs++;
572 }
573
574 if (*last_sg)
575 sg_mark_end(*last_sg);
576
577 /*
578 * Something must have been wrong if the figured number of
579 * segment is bigger than number of req's physical segments
580 */
581 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
582
583 return nsegs;
584 }
585 EXPORT_SYMBOL(__blk_rq_map_sg);
586
blk_rq_get_max_sectors(struct request * rq,sector_t offset)587 static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
588 sector_t offset)
589 {
590 struct request_queue *q = rq->q;
591 struct queue_limits *lim = &q->limits;
592 unsigned int max_sectors, boundary_sectors;
593 bool is_atomic = rq->cmd_flags & REQ_ATOMIC;
594
595 if (blk_rq_is_passthrough(rq))
596 return q->limits.max_hw_sectors;
597
598 boundary_sectors = blk_boundary_sectors(lim, is_atomic);
599 max_sectors = blk_queue_get_max_sectors(rq);
600
601 if (!boundary_sectors ||
602 req_op(rq) == REQ_OP_DISCARD ||
603 req_op(rq) == REQ_OP_SECURE_ERASE)
604 return max_sectors;
605 return min(max_sectors,
606 blk_boundary_sectors_left(offset, boundary_sectors));
607 }
608
ll_new_hw_segment(struct request * req,struct bio * bio,unsigned int nr_phys_segs)609 static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
610 unsigned int nr_phys_segs)
611 {
612 if (!blk_cgroup_mergeable(req, bio))
613 goto no_merge;
614
615 if (blk_integrity_merge_bio(req->q, req, bio) == false)
616 goto no_merge;
617
618 /* discard request merge won't add new segment */
619 if (req_op(req) == REQ_OP_DISCARD)
620 return 1;
621
622 if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req))
623 goto no_merge;
624
625 /*
626 * This will form the start of a new hw segment. Bump both
627 * counters.
628 */
629 req->nr_phys_segments += nr_phys_segs;
630 if (bio_integrity(bio))
631 req->nr_integrity_segments += blk_rq_count_integrity_sg(req->q,
632 bio);
633 return 1;
634
635 no_merge:
636 req_set_nomerge(req->q, req);
637 return 0;
638 }
639
ll_back_merge_fn(struct request * req,struct bio * bio,unsigned int nr_segs)640 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
641 {
642 if (req_gap_back_merge(req, bio))
643 return 0;
644 if (blk_integrity_rq(req) &&
645 integrity_req_gap_back_merge(req, bio))
646 return 0;
647 if (!bio_crypt_ctx_back_mergeable(req, bio))
648 return 0;
649 if (blk_rq_sectors(req) + bio_sectors(bio) >
650 blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
651 req_set_nomerge(req->q, req);
652 return 0;
653 }
654
655 return ll_new_hw_segment(req, bio, nr_segs);
656 }
657
ll_front_merge_fn(struct request * req,struct bio * bio,unsigned int nr_segs)658 static int ll_front_merge_fn(struct request *req, struct bio *bio,
659 unsigned int nr_segs)
660 {
661 if (req_gap_front_merge(req, bio))
662 return 0;
663 if (blk_integrity_rq(req) &&
664 integrity_req_gap_front_merge(req, bio))
665 return 0;
666 if (!bio_crypt_ctx_front_mergeable(req, bio))
667 return 0;
668 if (blk_rq_sectors(req) + bio_sectors(bio) >
669 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
670 req_set_nomerge(req->q, req);
671 return 0;
672 }
673
674 return ll_new_hw_segment(req, bio, nr_segs);
675 }
676
req_attempt_discard_merge(struct request_queue * q,struct request * req,struct request * next)677 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
678 struct request *next)
679 {
680 unsigned short segments = blk_rq_nr_discard_segments(req);
681
682 if (segments >= queue_max_discard_segments(q))
683 goto no_merge;
684 if (blk_rq_sectors(req) + bio_sectors(next->bio) >
685 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
686 goto no_merge;
687
688 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
689 return true;
690 no_merge:
691 req_set_nomerge(q, req);
692 return false;
693 }
694
ll_merge_requests_fn(struct request_queue * q,struct request * req,struct request * next)695 static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
696 struct request *next)
697 {
698 int total_phys_segments;
699
700 if (req_gap_back_merge(req, next->bio))
701 return 0;
702
703 /*
704 * Will it become too large?
705 */
706 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
707 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
708 return 0;
709
710 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
711 if (total_phys_segments > blk_rq_get_max_segments(req))
712 return 0;
713
714 if (!blk_cgroup_mergeable(req, next->bio))
715 return 0;
716
717 if (blk_integrity_merge_rq(q, req, next) == false)
718 return 0;
719
720 if (!bio_crypt_ctx_merge_rq(req, next))
721 return 0;
722
723 /* Merge is OK... */
724 req->nr_phys_segments = total_phys_segments;
725 req->nr_integrity_segments += next->nr_integrity_segments;
726 return 1;
727 }
728
729 /**
730 * blk_rq_set_mixed_merge - mark a request as mixed merge
731 * @rq: request to mark as mixed merge
732 *
733 * Description:
734 * @rq is about to be mixed merged. Make sure the attributes
735 * which can be mixed are set in each bio and mark @rq as mixed
736 * merged.
737 */
blk_rq_set_mixed_merge(struct request * rq)738 static void blk_rq_set_mixed_merge(struct request *rq)
739 {
740 blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
741 struct bio *bio;
742
743 if (rq->rq_flags & RQF_MIXED_MERGE)
744 return;
745
746 /*
747 * @rq will no longer represent mixable attributes for all the
748 * contained bios. It will just track those of the first one.
749 * Distributes the attributs to each bio.
750 */
751 for (bio = rq->bio; bio; bio = bio->bi_next) {
752 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
753 (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
754 bio->bi_opf |= ff;
755 }
756 rq->rq_flags |= RQF_MIXED_MERGE;
757 }
758
bio_failfast(const struct bio * bio)759 static inline blk_opf_t bio_failfast(const struct bio *bio)
760 {
761 if (bio->bi_opf & REQ_RAHEAD)
762 return REQ_FAILFAST_MASK;
763
764 return bio->bi_opf & REQ_FAILFAST_MASK;
765 }
766
767 /*
768 * After we are marked as MIXED_MERGE, any new RA bio has to be updated
769 * as failfast, and request's failfast has to be updated in case of
770 * front merge.
771 */
blk_update_mixed_merge(struct request * req,struct bio * bio,bool front_merge)772 static inline void blk_update_mixed_merge(struct request *req,
773 struct bio *bio, bool front_merge)
774 {
775 if (req->rq_flags & RQF_MIXED_MERGE) {
776 if (bio->bi_opf & REQ_RAHEAD)
777 bio->bi_opf |= REQ_FAILFAST_MASK;
778
779 if (front_merge) {
780 req->cmd_flags &= ~REQ_FAILFAST_MASK;
781 req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK;
782 }
783 }
784 }
785
blk_account_io_merge_request(struct request * req)786 static void blk_account_io_merge_request(struct request *req)
787 {
788 if (req->rq_flags & RQF_IO_STAT) {
789 part_stat_lock();
790 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
791 part_stat_local_dec(req->part,
792 in_flight[op_is_write(req_op(req))]);
793 part_stat_unlock();
794 }
795 }
796
blk_try_req_merge(struct request * req,struct request * next)797 static enum elv_merge blk_try_req_merge(struct request *req,
798 struct request *next)
799 {
800 if (blk_discard_mergable(req))
801 return ELEVATOR_DISCARD_MERGE;
802 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
803 return ELEVATOR_BACK_MERGE;
804
805 return ELEVATOR_NO_MERGE;
806 }
807
blk_atomic_write_mergeable_rq_bio(struct request * rq,struct bio * bio)808 static bool blk_atomic_write_mergeable_rq_bio(struct request *rq,
809 struct bio *bio)
810 {
811 return (rq->cmd_flags & REQ_ATOMIC) == (bio->bi_opf & REQ_ATOMIC);
812 }
813
blk_atomic_write_mergeable_rqs(struct request * rq,struct request * next)814 static bool blk_atomic_write_mergeable_rqs(struct request *rq,
815 struct request *next)
816 {
817 return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC);
818 }
819
820 /*
821 * For non-mq, this has to be called with the request spinlock acquired.
822 * For mq with scheduling, the appropriate queue wide lock should be held.
823 */
attempt_merge(struct request_queue * q,struct request * req,struct request * next)824 static struct request *attempt_merge(struct request_queue *q,
825 struct request *req, struct request *next)
826 {
827 if (!rq_mergeable(req) || !rq_mergeable(next))
828 return NULL;
829
830 if (req_op(req) != req_op(next))
831 return NULL;
832
833 if (req->bio->bi_write_hint != next->bio->bi_write_hint)
834 return NULL;
835 if (req->bio->bi_ioprio != next->bio->bi_ioprio)
836 return NULL;
837 if (!blk_atomic_write_mergeable_rqs(req, next))
838 return NULL;
839
840 /*
841 * If we are allowed to merge, then append bio list
842 * from next to rq and release next. merge_requests_fn
843 * will have updated segment counts, update sector
844 * counts here. Handle DISCARDs separately, as they
845 * have separate settings.
846 */
847
848 switch (blk_try_req_merge(req, next)) {
849 case ELEVATOR_DISCARD_MERGE:
850 if (!req_attempt_discard_merge(q, req, next))
851 return NULL;
852 break;
853 case ELEVATOR_BACK_MERGE:
854 if (!ll_merge_requests_fn(q, req, next))
855 return NULL;
856 break;
857 default:
858 return NULL;
859 }
860
861 /*
862 * If failfast settings disagree or any of the two is already
863 * a mixed merge, mark both as mixed before proceeding. This
864 * makes sure that all involved bios have mixable attributes
865 * set properly.
866 */
867 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
868 (req->cmd_flags & REQ_FAILFAST_MASK) !=
869 (next->cmd_flags & REQ_FAILFAST_MASK)) {
870 blk_rq_set_mixed_merge(req);
871 blk_rq_set_mixed_merge(next);
872 }
873
874 /*
875 * At this point we have either done a back merge or front merge. We
876 * need the smaller start_time_ns of the merged requests to be the
877 * current request for accounting purposes.
878 */
879 if (next->start_time_ns < req->start_time_ns)
880 req->start_time_ns = next->start_time_ns;
881
882 req->biotail->bi_next = next->bio;
883 req->biotail = next->biotail;
884
885 req->__data_len += blk_rq_bytes(next);
886
887 if (!blk_discard_mergable(req))
888 elv_merge_requests(q, req, next);
889
890 blk_crypto_rq_put_keyslot(next);
891
892 /*
893 * 'next' is going away, so update stats accordingly
894 */
895 blk_account_io_merge_request(next);
896
897 trace_block_rq_merge(next);
898
899 /*
900 * ownership of bio passed from next to req, return 'next' for
901 * the caller to free
902 */
903 next->bio = NULL;
904 return next;
905 }
906
attempt_back_merge(struct request_queue * q,struct request * rq)907 static struct request *attempt_back_merge(struct request_queue *q,
908 struct request *rq)
909 {
910 struct request *next = elv_latter_request(q, rq);
911
912 if (next)
913 return attempt_merge(q, rq, next);
914
915 return NULL;
916 }
917
attempt_front_merge(struct request_queue * q,struct request * rq)918 static struct request *attempt_front_merge(struct request_queue *q,
919 struct request *rq)
920 {
921 struct request *prev = elv_former_request(q, rq);
922
923 if (prev)
924 return attempt_merge(q, prev, rq);
925
926 return NULL;
927 }
928
929 /*
930 * Try to merge 'next' into 'rq'. Return true if the merge happened, false
931 * otherwise. The caller is responsible for freeing 'next' if the merge
932 * happened.
933 */
blk_attempt_req_merge(struct request_queue * q,struct request * rq,struct request * next)934 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
935 struct request *next)
936 {
937 return attempt_merge(q, rq, next);
938 }
939
blk_rq_merge_ok(struct request * rq,struct bio * bio)940 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
941 {
942 if (!rq_mergeable(rq) || !bio_mergeable(bio))
943 return false;
944
945 if (req_op(rq) != bio_op(bio))
946 return false;
947
948 if (!blk_cgroup_mergeable(rq, bio))
949 return false;
950 if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
951 return false;
952 if (!bio_crypt_rq_ctx_compatible(rq, bio))
953 return false;
954 if (rq->bio->bi_write_hint != bio->bi_write_hint)
955 return false;
956 if (rq->bio->bi_ioprio != bio->bi_ioprio)
957 return false;
958 if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false)
959 return false;
960
961 return true;
962 }
963
blk_try_merge(struct request * rq,struct bio * bio)964 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
965 {
966 if (blk_discard_mergable(rq))
967 return ELEVATOR_DISCARD_MERGE;
968 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
969 return ELEVATOR_BACK_MERGE;
970 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
971 return ELEVATOR_FRONT_MERGE;
972 return ELEVATOR_NO_MERGE;
973 }
974
blk_account_io_merge_bio(struct request * req)975 static void blk_account_io_merge_bio(struct request *req)
976 {
977 if (req->rq_flags & RQF_IO_STAT) {
978 part_stat_lock();
979 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);
980 part_stat_unlock();
981 }
982 }
983
bio_attempt_back_merge(struct request * req,struct bio * bio,unsigned int nr_segs)984 enum bio_merge_status bio_attempt_back_merge(struct request *req,
985 struct bio *bio, unsigned int nr_segs)
986 {
987 const blk_opf_t ff = bio_failfast(bio);
988
989 if (!ll_back_merge_fn(req, bio, nr_segs))
990 return BIO_MERGE_FAILED;
991
992 trace_block_bio_backmerge(bio);
993 rq_qos_merge(req->q, req, bio);
994
995 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
996 blk_rq_set_mixed_merge(req);
997
998 blk_update_mixed_merge(req, bio, false);
999
1000 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
1001 blk_zone_write_plug_bio_merged(bio);
1002
1003 req->biotail->bi_next = bio;
1004 req->biotail = bio;
1005 req->__data_len += bio->bi_iter.bi_size;
1006
1007 bio_crypt_free_ctx(bio);
1008
1009 blk_account_io_merge_bio(req);
1010 return BIO_MERGE_OK;
1011 }
1012
bio_attempt_front_merge(struct request * req,struct bio * bio,unsigned int nr_segs)1013 static enum bio_merge_status bio_attempt_front_merge(struct request *req,
1014 struct bio *bio, unsigned int nr_segs)
1015 {
1016 const blk_opf_t ff = bio_failfast(bio);
1017
1018 /*
1019 * A front merge for writes to sequential zones of a zoned block device
1020 * can happen only if the user submitted writes out of order. Do not
1021 * merge such write to let it fail.
1022 */
1023 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING)
1024 return BIO_MERGE_FAILED;
1025
1026 if (!ll_front_merge_fn(req, bio, nr_segs))
1027 return BIO_MERGE_FAILED;
1028
1029 trace_block_bio_frontmerge(bio);
1030 rq_qos_merge(req->q, req, bio);
1031
1032 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1033 blk_rq_set_mixed_merge(req);
1034
1035 blk_update_mixed_merge(req, bio, true);
1036
1037 bio->bi_next = req->bio;
1038 req->bio = bio;
1039
1040 req->__sector = bio->bi_iter.bi_sector;
1041 req->__data_len += bio->bi_iter.bi_size;
1042
1043 bio_crypt_do_front_merge(req, bio);
1044
1045 blk_account_io_merge_bio(req);
1046 return BIO_MERGE_OK;
1047 }
1048
bio_attempt_discard_merge(struct request_queue * q,struct request * req,struct bio * bio)1049 static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
1050 struct request *req, struct bio *bio)
1051 {
1052 unsigned short segments = blk_rq_nr_discard_segments(req);
1053
1054 if (segments >= queue_max_discard_segments(q))
1055 goto no_merge;
1056 if (blk_rq_sectors(req) + bio_sectors(bio) >
1057 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
1058 goto no_merge;
1059
1060 rq_qos_merge(q, req, bio);
1061
1062 req->biotail->bi_next = bio;
1063 req->biotail = bio;
1064 req->__data_len += bio->bi_iter.bi_size;
1065 req->nr_phys_segments = segments + 1;
1066
1067 blk_account_io_merge_bio(req);
1068 return BIO_MERGE_OK;
1069 no_merge:
1070 req_set_nomerge(q, req);
1071 return BIO_MERGE_FAILED;
1072 }
1073
blk_attempt_bio_merge(struct request_queue * q,struct request * rq,struct bio * bio,unsigned int nr_segs,bool sched_allow_merge)1074 static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q,
1075 struct request *rq,
1076 struct bio *bio,
1077 unsigned int nr_segs,
1078 bool sched_allow_merge)
1079 {
1080 if (!blk_rq_merge_ok(rq, bio))
1081 return BIO_MERGE_NONE;
1082
1083 switch (blk_try_merge(rq, bio)) {
1084 case ELEVATOR_BACK_MERGE:
1085 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1086 return bio_attempt_back_merge(rq, bio, nr_segs);
1087 break;
1088 case ELEVATOR_FRONT_MERGE:
1089 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio))
1090 return bio_attempt_front_merge(rq, bio, nr_segs);
1091 break;
1092 case ELEVATOR_DISCARD_MERGE:
1093 return bio_attempt_discard_merge(q, rq, bio);
1094 default:
1095 return BIO_MERGE_NONE;
1096 }
1097
1098 return BIO_MERGE_FAILED;
1099 }
1100
1101 /**
1102 * blk_attempt_plug_merge - try to merge with %current's plugged list
1103 * @q: request_queue new bio is being queued at
1104 * @bio: new bio being queued
1105 * @nr_segs: number of segments in @bio
1106 * from the passed in @q already in the plug list
1107 *
1108 * Determine whether @bio being queued on @q can be merged with the previous
1109 * request on %current's plugged list. Returns %true if merge was successful,
1110 * otherwise %false.
1111 *
1112 * Plugging coalesces IOs from the same issuer for the same purpose without
1113 * going through @q->queue_lock. As such it's more of an issuing mechanism
1114 * than scheduling, and the request, while may have elvpriv data, is not
1115 * added on the elevator at this point. In addition, we don't have
1116 * reliable access to the elevator outside queue lock. Only check basic
1117 * merging parameters without querying the elevator.
1118 *
1119 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1120 */
blk_attempt_plug_merge(struct request_queue * q,struct bio * bio,unsigned int nr_segs)1121 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1122 unsigned int nr_segs)
1123 {
1124 struct blk_plug *plug = current->plug;
1125 struct request *rq;
1126
1127 if (!plug || rq_list_empty(&plug->mq_list))
1128 return false;
1129
1130 rq_list_for_each(&plug->mq_list, rq) {
1131 if (rq->q == q) {
1132 if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) ==
1133 BIO_MERGE_OK)
1134 return true;
1135 break;
1136 }
1137
1138 /*
1139 * Only keep iterating plug list for merges if we have multiple
1140 * queues
1141 */
1142 if (!plug->multiple_queues)
1143 break;
1144 }
1145 return false;
1146 }
1147
1148 /*
1149 * Iterate list of requests and see if we can merge this bio with any
1150 * of them.
1151 */
blk_bio_list_merge(struct request_queue * q,struct list_head * list,struct bio * bio,unsigned int nr_segs)1152 bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
1153 struct bio *bio, unsigned int nr_segs)
1154 {
1155 struct request *rq;
1156 int checked = 8;
1157
1158 list_for_each_entry_reverse(rq, list, queuelist) {
1159 if (!checked--)
1160 break;
1161
1162 switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) {
1163 case BIO_MERGE_NONE:
1164 continue;
1165 case BIO_MERGE_OK:
1166 return true;
1167 case BIO_MERGE_FAILED:
1168 return false;
1169 }
1170
1171 }
1172
1173 return false;
1174 }
1175 EXPORT_SYMBOL_GPL(blk_bio_list_merge);
1176
blk_mq_sched_try_merge(struct request_queue * q,struct bio * bio,unsigned int nr_segs,struct request ** merged_request)1177 bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
1178 unsigned int nr_segs, struct request **merged_request)
1179 {
1180 struct request *rq;
1181
1182 switch (elv_merge(q, &rq, bio)) {
1183 case ELEVATOR_BACK_MERGE:
1184 if (!blk_mq_sched_allow_merge(q, rq, bio))
1185 return false;
1186 if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1187 return false;
1188 *merged_request = attempt_back_merge(q, rq);
1189 if (!*merged_request)
1190 elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
1191 return true;
1192 case ELEVATOR_FRONT_MERGE:
1193 if (!blk_mq_sched_allow_merge(q, rq, bio))
1194 return false;
1195 if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
1196 return false;
1197 *merged_request = attempt_front_merge(q, rq);
1198 if (!*merged_request)
1199 elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
1200 return true;
1201 case ELEVATOR_DISCARD_MERGE:
1202 return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
1203 default:
1204 return false;
1205 }
1206 }
1207 EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
1208