1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to segment and merge handling 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/blk-integrity.h> 10 #include <linux/scatterlist.h> 11 #include <linux/part_stat.h> 12 #include <linux/blk-cgroup.h> 13 14 #include <trace/events/block.h> 15 16 #include "blk.h" 17 #include "blk-mq-sched.h" 18 #include "blk-rq-qos.h" 19 #include "blk-throttle.h" 20 21 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv) 22 { 23 *bv = mp_bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 24 } 25 26 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv) 27 { 28 struct bvec_iter iter = bio->bi_iter; 29 int idx; 30 31 bio_get_first_bvec(bio, bv); 32 if (bv->bv_len == bio->bi_iter.bi_size) 33 return; /* this bio only has a single bvec */ 34 35 bio_advance_iter(bio, &iter, iter.bi_size); 36 37 if (!iter.bi_bvec_done) 38 idx = iter.bi_idx - 1; 39 else /* in the middle of bvec */ 40 idx = iter.bi_idx; 41 42 *bv = bio->bi_io_vec[idx]; 43 44 /* 45 * iter.bi_bvec_done records actual length of the last bvec 46 * if this bio ends in the middle of one io vector 47 */ 48 if (iter.bi_bvec_done) 49 bv->bv_len = iter.bi_bvec_done; 50 } 51 52 static inline bool bio_will_gap(struct request_queue *q, 53 struct request *prev_rq, struct bio *prev, struct bio *next) 54 { 55 struct bio_vec pb, nb; 56 57 if (!bio_has_data(prev) || !queue_virt_boundary(q)) 58 return false; 59 60 /* 61 * Don't merge if the 1st bio starts with non-zero offset, otherwise it 62 * is quite difficult to respect the sg gap limit. We work hard to 63 * merge a huge number of small single bios in case of mkfs. 64 */ 65 if (prev_rq) 66 bio_get_first_bvec(prev_rq->bio, &pb); 67 else 68 bio_get_first_bvec(prev, &pb); 69 if (pb.bv_offset & queue_virt_boundary(q)) 70 return true; 71 72 /* 73 * We don't need to worry about the situation that the merged segment 74 * ends in unaligned virt boundary: 75 * 76 * - if 'pb' ends aligned, the merged segment ends aligned 77 * - if 'pb' ends unaligned, the next bio must include 78 * one single bvec of 'nb', otherwise the 'nb' can't 79 * merge with 'pb' 80 */ 81 bio_get_last_bvec(prev, &pb); 82 bio_get_first_bvec(next, &nb); 83 if (biovec_phys_mergeable(q, &pb, &nb)) 84 return false; 85 return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset); 86 } 87 88 static inline bool req_gap_back_merge(struct request *req, struct bio *bio) 89 { 90 return bio_will_gap(req->q, req, req->biotail, bio); 91 } 92 93 static inline bool req_gap_front_merge(struct request *req, struct bio *bio) 94 { 95 return bio_will_gap(req->q, NULL, bio, req->bio); 96 } 97 98 /* 99 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size 100 * is defined as 'unsigned int', meantime it has to be aligned to with the 101 * logical block size, which is the minimum accepted unit by hardware. 102 */ 103 static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim) 104 { 105 return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT; 106 } 107 108 static struct bio *bio_split_discard(struct bio *bio, 109 const struct queue_limits *lim, 110 unsigned *nsegs, struct bio_set *bs) 111 { 112 unsigned int max_discard_sectors, granularity; 113 sector_t tmp; 114 unsigned split_sectors; 115 116 *nsegs = 1; 117 118 granularity = max(lim->discard_granularity >> 9, 1U); 119 120 max_discard_sectors = 121 min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); 122 max_discard_sectors -= max_discard_sectors % granularity; 123 if (unlikely(!max_discard_sectors)) 124 return NULL; 125 126 if (bio_sectors(bio) <= max_discard_sectors) 127 return NULL; 128 129 split_sectors = max_discard_sectors; 130 131 /* 132 * If the next starting sector would be misaligned, stop the discard at 133 * the previous aligned sector. 134 */ 135 tmp = bio->bi_iter.bi_sector + split_sectors - 136 ((lim->discard_alignment >> 9) % granularity); 137 tmp = sector_div(tmp, granularity); 138 139 if (split_sectors > tmp) 140 split_sectors -= tmp; 141 142 return bio_split(bio, split_sectors, GFP_NOIO, bs); 143 } 144 145 static struct bio *bio_split_write_zeroes(struct bio *bio, 146 const struct queue_limits *lim, 147 unsigned *nsegs, struct bio_set *bs) 148 { 149 *nsegs = 0; 150 if (!lim->max_write_zeroes_sectors) 151 return NULL; 152 if (bio_sectors(bio) <= lim->max_write_zeroes_sectors) 153 return NULL; 154 return bio_split(bio, lim->max_write_zeroes_sectors, GFP_NOIO, bs); 155 } 156 157 /* 158 * Return the maximum number of sectors from the start of a bio that may be 159 * submitted as a single request to a block device. If enough sectors remain, 160 * align the end to the physical block size. Otherwise align the end to the 161 * logical block size. This approach minimizes the number of non-aligned 162 * requests that are submitted to a block device if the start of a bio is not 163 * aligned to a physical block boundary. 164 */ 165 static inline unsigned get_max_io_size(struct bio *bio, 166 const struct queue_limits *lim) 167 { 168 unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT; 169 unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT; 170 unsigned max_sectors = lim->max_sectors, start, end; 171 172 if (lim->chunk_sectors) { 173 max_sectors = min(max_sectors, 174 blk_chunk_sectors_left(bio->bi_iter.bi_sector, 175 lim->chunk_sectors)); 176 } 177 178 start = bio->bi_iter.bi_sector & (pbs - 1); 179 end = (start + max_sectors) & ~(pbs - 1); 180 if (end > start) 181 return end - start; 182 return max_sectors & ~(lbs - 1); 183 } 184 185 /** 186 * get_max_segment_size() - maximum number of bytes to add as a single segment 187 * @lim: Request queue limits. 188 * @start_page: See below. 189 * @offset: Offset from @start_page where to add a segment. 190 * 191 * Returns the maximum number of bytes that can be added as a single segment. 192 */ 193 static inline unsigned get_max_segment_size(const struct queue_limits *lim, 194 struct page *start_page, unsigned long offset) 195 { 196 unsigned long mask = lim->seg_boundary_mask; 197 198 offset = mask & (page_to_phys(start_page) + offset); 199 200 /* 201 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1 202 * after having calculated the minimum. 203 */ 204 return min(mask - offset, (unsigned long)lim->max_segment_size - 1) + 1; 205 } 206 207 /** 208 * bvec_split_segs - verify whether or not a bvec should be split in the middle 209 * @lim: [in] queue limits to split based on 210 * @bv: [in] bvec to examine 211 * @nsegs: [in,out] Number of segments in the bio being built. Incremented 212 * by the number of segments from @bv that may be appended to that 213 * bio without exceeding @max_segs 214 * @bytes: [in,out] Number of bytes in the bio being built. Incremented 215 * by the number of bytes from @bv that may be appended to that 216 * bio without exceeding @max_bytes 217 * @max_segs: [in] upper bound for *@nsegs 218 * @max_bytes: [in] upper bound for *@bytes 219 * 220 * When splitting a bio, it can happen that a bvec is encountered that is too 221 * big to fit in a single segment and hence that it has to be split in the 222 * middle. This function verifies whether or not that should happen. The value 223 * %true is returned if and only if appending the entire @bv to a bio with 224 * *@nsegs segments and *@sectors sectors would make that bio unacceptable for 225 * the block driver. 226 */ 227 static bool bvec_split_segs(const struct queue_limits *lim, 228 const struct bio_vec *bv, unsigned *nsegs, unsigned *bytes, 229 unsigned max_segs, unsigned max_bytes) 230 { 231 unsigned max_len = min(max_bytes, UINT_MAX) - *bytes; 232 unsigned len = min(bv->bv_len, max_len); 233 unsigned total_len = 0; 234 unsigned seg_size = 0; 235 236 while (len && *nsegs < max_segs) { 237 seg_size = get_max_segment_size(lim, bv->bv_page, 238 bv->bv_offset + total_len); 239 seg_size = min(seg_size, len); 240 241 (*nsegs)++; 242 total_len += seg_size; 243 len -= seg_size; 244 245 if ((bv->bv_offset + total_len) & lim->virt_boundary_mask) 246 break; 247 } 248 249 *bytes += total_len; 250 251 /* tell the caller to split the bvec if it is too big to fit */ 252 return len > 0 || bv->bv_len > max_len; 253 } 254 255 /** 256 * bio_split_rw - split a bio in two bios 257 * @bio: [in] bio to be split 258 * @lim: [in] queue limits to split based on 259 * @segs: [out] number of segments in the bio with the first half of the sectors 260 * @bs: [in] bio set to allocate the clone from 261 * @max_bytes: [in] maximum number of bytes per bio 262 * 263 * Clone @bio, update the bi_iter of the clone to represent the first sectors 264 * of @bio and update @bio->bi_iter to represent the remaining sectors. The 265 * following is guaranteed for the cloned bio: 266 * - That it has at most @max_bytes worth of data 267 * - That it has at most queue_max_segments(@q) segments. 268 * 269 * Except for discard requests the cloned bio will point at the bi_io_vec of 270 * the original bio. It is the responsibility of the caller to ensure that the 271 * original bio is not freed before the cloned bio. The caller is also 272 * responsible for ensuring that @bs is only destroyed after processing of the 273 * split bio has finished. 274 */ 275 struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, 276 unsigned *segs, struct bio_set *bs, unsigned max_bytes) 277 { 278 struct bio_vec bv, bvprv, *bvprvp = NULL; 279 struct bvec_iter iter; 280 unsigned nsegs = 0, bytes = 0; 281 282 bio_for_each_bvec(bv, bio, iter) { 283 /* 284 * If the queue doesn't support SG gaps and adding this 285 * offset would create a gap, disallow it. 286 */ 287 if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset)) 288 goto split; 289 290 if (nsegs < lim->max_segments && 291 bytes + bv.bv_len <= max_bytes && 292 bv.bv_offset + bv.bv_len <= PAGE_SIZE) { 293 nsegs++; 294 bytes += bv.bv_len; 295 } else { 296 if (bvec_split_segs(lim, &bv, &nsegs, &bytes, 297 lim->max_segments, max_bytes)) 298 goto split; 299 } 300 301 bvprv = bv; 302 bvprvp = &bvprv; 303 } 304 305 *segs = nsegs; 306 return NULL; 307 split: 308 /* 309 * We can't sanely support splitting for a REQ_NOWAIT bio. End it 310 * with EAGAIN if splitting is required and return an error pointer. 311 */ 312 if (bio->bi_opf & REQ_NOWAIT) { 313 bio->bi_status = BLK_STS_AGAIN; 314 bio_endio(bio); 315 return ERR_PTR(-EAGAIN); 316 } 317 318 *segs = nsegs; 319 320 /* 321 * Individual bvecs might not be logical block aligned. Round down the 322 * split size so that each bio is properly block size aligned, even if 323 * we do not use the full hardware limits. 324 */ 325 bytes = ALIGN_DOWN(bytes, lim->logical_block_size); 326 327 /* 328 * Bio splitting may cause subtle trouble such as hang when doing sync 329 * iopoll in direct IO routine. Given performance gain of iopoll for 330 * big IO can be trival, disable iopoll when split needed. 331 */ 332 bio_clear_polled(bio); 333 return bio_split(bio, bytes >> SECTOR_SHIFT, GFP_NOIO, bs); 334 } 335 EXPORT_SYMBOL_GPL(bio_split_rw); 336 337 /** 338 * __bio_split_to_limits - split a bio to fit the queue limits 339 * @bio: bio to be split 340 * @lim: queue limits to split based on 341 * @nr_segs: returns the number of segments in the returned bio 342 * 343 * Check if @bio needs splitting based on the queue limits, and if so split off 344 * a bio fitting the limits from the beginning of @bio and return it. @bio is 345 * shortened to the remainder and re-submitted. 346 * 347 * The split bio is allocated from @q->bio_split, which is provided by the 348 * block layer. 349 */ 350 struct bio *__bio_split_to_limits(struct bio *bio, 351 const struct queue_limits *lim, 352 unsigned int *nr_segs) 353 { 354 struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split; 355 struct bio *split; 356 357 switch (bio_op(bio)) { 358 case REQ_OP_DISCARD: 359 case REQ_OP_SECURE_ERASE: 360 split = bio_split_discard(bio, lim, nr_segs, bs); 361 break; 362 case REQ_OP_WRITE_ZEROES: 363 split = bio_split_write_zeroes(bio, lim, nr_segs, bs); 364 break; 365 default: 366 split = bio_split_rw(bio, lim, nr_segs, bs, 367 get_max_io_size(bio, lim) << SECTOR_SHIFT); 368 if (IS_ERR(split)) 369 return NULL; 370 break; 371 } 372 373 if (split) { 374 /* there isn't chance to merge the split bio */ 375 split->bi_opf |= REQ_NOMERGE; 376 377 blkcg_bio_issue_init(split); 378 bio_chain(split, bio); 379 trace_block_split(split, bio->bi_iter.bi_sector); 380 WARN_ON_ONCE(bio_zone_write_plugging(bio)); 381 submit_bio_noacct(bio); 382 return split; 383 } 384 return bio; 385 } 386 387 /** 388 * bio_split_to_limits - split a bio to fit the queue limits 389 * @bio: bio to be split 390 * 391 * Check if @bio needs splitting based on the queue limits of @bio->bi_bdev, and 392 * if so split off a bio fitting the limits from the beginning of @bio and 393 * return it. @bio is shortened to the remainder and re-submitted. 394 * 395 * The split bio is allocated from @q->bio_split, which is provided by the 396 * block layer. 397 */ 398 struct bio *bio_split_to_limits(struct bio *bio) 399 { 400 const struct queue_limits *lim = &bdev_get_queue(bio->bi_bdev)->limits; 401 unsigned int nr_segs; 402 403 if (bio_may_exceed_limits(bio, lim)) 404 return __bio_split_to_limits(bio, lim, &nr_segs); 405 return bio; 406 } 407 EXPORT_SYMBOL(bio_split_to_limits); 408 409 unsigned int blk_recalc_rq_segments(struct request *rq) 410 { 411 unsigned int nr_phys_segs = 0; 412 unsigned int bytes = 0; 413 struct req_iterator iter; 414 struct bio_vec bv; 415 416 if (!rq->bio) 417 return 0; 418 419 switch (bio_op(rq->bio)) { 420 case REQ_OP_DISCARD: 421 case REQ_OP_SECURE_ERASE: 422 if (queue_max_discard_segments(rq->q) > 1) { 423 struct bio *bio = rq->bio; 424 425 for_each_bio(bio) 426 nr_phys_segs++; 427 return nr_phys_segs; 428 } 429 return 1; 430 case REQ_OP_WRITE_ZEROES: 431 return 0; 432 default: 433 break; 434 } 435 436 rq_for_each_bvec(bv, rq, iter) 437 bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes, 438 UINT_MAX, UINT_MAX); 439 return nr_phys_segs; 440 } 441 442 static inline struct scatterlist *blk_next_sg(struct scatterlist **sg, 443 struct scatterlist *sglist) 444 { 445 if (!*sg) 446 return sglist; 447 448 /* 449 * If the driver previously mapped a shorter list, we could see a 450 * termination bit prematurely unless it fully inits the sg table 451 * on each mapping. We KNOW that there must be more entries here 452 * or the driver would be buggy, so force clear the termination bit 453 * to avoid doing a full sg_init_table() in drivers for each command. 454 */ 455 sg_unmark_end(*sg); 456 return sg_next(*sg); 457 } 458 459 static unsigned blk_bvec_map_sg(struct request_queue *q, 460 struct bio_vec *bvec, struct scatterlist *sglist, 461 struct scatterlist **sg) 462 { 463 unsigned nbytes = bvec->bv_len; 464 unsigned nsegs = 0, total = 0; 465 466 while (nbytes > 0) { 467 unsigned offset = bvec->bv_offset + total; 468 unsigned len = min(get_max_segment_size(&q->limits, 469 bvec->bv_page, offset), nbytes); 470 struct page *page = bvec->bv_page; 471 472 /* 473 * Unfortunately a fair number of drivers barf on scatterlists 474 * that have an offset larger than PAGE_SIZE, despite other 475 * subsystems dealing with that invariant just fine. For now 476 * stick to the legacy format where we never present those from 477 * the block layer, but the code below should be removed once 478 * these offenders (mostly MMC/SD drivers) are fixed. 479 */ 480 page += (offset >> PAGE_SHIFT); 481 offset &= ~PAGE_MASK; 482 483 *sg = blk_next_sg(sg, sglist); 484 sg_set_page(*sg, page, len, offset); 485 486 total += len; 487 nbytes -= len; 488 nsegs++; 489 } 490 491 return nsegs; 492 } 493 494 static inline int __blk_bvec_map_sg(struct bio_vec bv, 495 struct scatterlist *sglist, struct scatterlist **sg) 496 { 497 *sg = blk_next_sg(sg, sglist); 498 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); 499 return 1; 500 } 501 502 /* only try to merge bvecs into one sg if they are from two bios */ 503 static inline bool 504 __blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec, 505 struct bio_vec *bvprv, struct scatterlist **sg) 506 { 507 508 int nbytes = bvec->bv_len; 509 510 if (!*sg) 511 return false; 512 513 if ((*sg)->length + nbytes > queue_max_segment_size(q)) 514 return false; 515 516 if (!biovec_phys_mergeable(q, bvprv, bvec)) 517 return false; 518 519 (*sg)->length += nbytes; 520 521 return true; 522 } 523 524 static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, 525 struct scatterlist *sglist, 526 struct scatterlist **sg) 527 { 528 struct bio_vec bvec, bvprv = { NULL }; 529 struct bvec_iter iter; 530 int nsegs = 0; 531 bool new_bio = false; 532 533 for_each_bio(bio) { 534 bio_for_each_bvec(bvec, bio, iter) { 535 /* 536 * Only try to merge bvecs from two bios given we 537 * have done bio internal merge when adding pages 538 * to bio 539 */ 540 if (new_bio && 541 __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg)) 542 goto next_bvec; 543 544 if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE) 545 nsegs += __blk_bvec_map_sg(bvec, sglist, sg); 546 else 547 nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg); 548 next_bvec: 549 new_bio = false; 550 } 551 if (likely(bio->bi_iter.bi_size)) { 552 bvprv = bvec; 553 new_bio = true; 554 } 555 } 556 557 return nsegs; 558 } 559 560 /* 561 * map a request to scatterlist, return number of sg entries setup. Caller 562 * must make sure sg can hold rq->nr_phys_segments entries 563 */ 564 int __blk_rq_map_sg(struct request_queue *q, struct request *rq, 565 struct scatterlist *sglist, struct scatterlist **last_sg) 566 { 567 int nsegs = 0; 568 569 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 570 nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, last_sg); 571 else if (rq->bio) 572 nsegs = __blk_bios_map_sg(q, rq->bio, sglist, last_sg); 573 574 if (*last_sg) 575 sg_mark_end(*last_sg); 576 577 /* 578 * Something must have been wrong if the figured number of 579 * segment is bigger than number of req's physical segments 580 */ 581 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq)); 582 583 return nsegs; 584 } 585 EXPORT_SYMBOL(__blk_rq_map_sg); 586 587 static inline unsigned int blk_rq_get_max_sectors(struct request *rq, 588 sector_t offset) 589 { 590 struct request_queue *q = rq->q; 591 unsigned int max_sectors; 592 593 if (blk_rq_is_passthrough(rq)) 594 return q->limits.max_hw_sectors; 595 596 max_sectors = blk_queue_get_max_sectors(q, req_op(rq)); 597 if (!q->limits.chunk_sectors || 598 req_op(rq) == REQ_OP_DISCARD || 599 req_op(rq) == REQ_OP_SECURE_ERASE) 600 return max_sectors; 601 return min(max_sectors, 602 blk_chunk_sectors_left(offset, q->limits.chunk_sectors)); 603 } 604 605 static inline int ll_new_hw_segment(struct request *req, struct bio *bio, 606 unsigned int nr_phys_segs) 607 { 608 if (!blk_cgroup_mergeable(req, bio)) 609 goto no_merge; 610 611 if (blk_integrity_merge_bio(req->q, req, bio) == false) 612 goto no_merge; 613 614 /* discard request merge won't add new segment */ 615 if (req_op(req) == REQ_OP_DISCARD) 616 return 1; 617 618 if (req->nr_phys_segments + nr_phys_segs > blk_rq_get_max_segments(req)) 619 goto no_merge; 620 621 /* 622 * This will form the start of a new hw segment. Bump both 623 * counters. 624 */ 625 req->nr_phys_segments += nr_phys_segs; 626 return 1; 627 628 no_merge: 629 req_set_nomerge(req->q, req); 630 return 0; 631 } 632 633 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) 634 { 635 if (req_gap_back_merge(req, bio)) 636 return 0; 637 if (blk_integrity_rq(req) && 638 integrity_req_gap_back_merge(req, bio)) 639 return 0; 640 if (!bio_crypt_ctx_back_mergeable(req, bio)) 641 return 0; 642 if (blk_rq_sectors(req) + bio_sectors(bio) > 643 blk_rq_get_max_sectors(req, blk_rq_pos(req))) { 644 req_set_nomerge(req->q, req); 645 return 0; 646 } 647 648 return ll_new_hw_segment(req, bio, nr_segs); 649 } 650 651 static int ll_front_merge_fn(struct request *req, struct bio *bio, 652 unsigned int nr_segs) 653 { 654 if (req_gap_front_merge(req, bio)) 655 return 0; 656 if (blk_integrity_rq(req) && 657 integrity_req_gap_front_merge(req, bio)) 658 return 0; 659 if (!bio_crypt_ctx_front_mergeable(req, bio)) 660 return 0; 661 if (blk_rq_sectors(req) + bio_sectors(bio) > 662 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) { 663 req_set_nomerge(req->q, req); 664 return 0; 665 } 666 667 return ll_new_hw_segment(req, bio, nr_segs); 668 } 669 670 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req, 671 struct request *next) 672 { 673 unsigned short segments = blk_rq_nr_discard_segments(req); 674 675 if (segments >= queue_max_discard_segments(q)) 676 goto no_merge; 677 if (blk_rq_sectors(req) + bio_sectors(next->bio) > 678 blk_rq_get_max_sectors(req, blk_rq_pos(req))) 679 goto no_merge; 680 681 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next); 682 return true; 683 no_merge: 684 req_set_nomerge(q, req); 685 return false; 686 } 687 688 static int ll_merge_requests_fn(struct request_queue *q, struct request *req, 689 struct request *next) 690 { 691 int total_phys_segments; 692 693 if (req_gap_back_merge(req, next->bio)) 694 return 0; 695 696 /* 697 * Will it become too large? 698 */ 699 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > 700 blk_rq_get_max_sectors(req, blk_rq_pos(req))) 701 return 0; 702 703 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; 704 if (total_phys_segments > blk_rq_get_max_segments(req)) 705 return 0; 706 707 if (!blk_cgroup_mergeable(req, next->bio)) 708 return 0; 709 710 if (blk_integrity_merge_rq(q, req, next) == false) 711 return 0; 712 713 if (!bio_crypt_ctx_merge_rq(req, next)) 714 return 0; 715 716 /* Merge is OK... */ 717 req->nr_phys_segments = total_phys_segments; 718 return 1; 719 } 720 721 /** 722 * blk_rq_set_mixed_merge - mark a request as mixed merge 723 * @rq: request to mark as mixed merge 724 * 725 * Description: 726 * @rq is about to be mixed merged. Make sure the attributes 727 * which can be mixed are set in each bio and mark @rq as mixed 728 * merged. 729 */ 730 static void blk_rq_set_mixed_merge(struct request *rq) 731 { 732 blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK; 733 struct bio *bio; 734 735 if (rq->rq_flags & RQF_MIXED_MERGE) 736 return; 737 738 /* 739 * @rq will no longer represent mixable attributes for all the 740 * contained bios. It will just track those of the first one. 741 * Distributes the attributs to each bio. 742 */ 743 for (bio = rq->bio; bio; bio = bio->bi_next) { 744 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) && 745 (bio->bi_opf & REQ_FAILFAST_MASK) != ff); 746 bio->bi_opf |= ff; 747 } 748 rq->rq_flags |= RQF_MIXED_MERGE; 749 } 750 751 static inline blk_opf_t bio_failfast(const struct bio *bio) 752 { 753 if (bio->bi_opf & REQ_RAHEAD) 754 return REQ_FAILFAST_MASK; 755 756 return bio->bi_opf & REQ_FAILFAST_MASK; 757 } 758 759 /* 760 * After we are marked as MIXED_MERGE, any new RA bio has to be updated 761 * as failfast, and request's failfast has to be updated in case of 762 * front merge. 763 */ 764 static inline void blk_update_mixed_merge(struct request *req, 765 struct bio *bio, bool front_merge) 766 { 767 if (req->rq_flags & RQF_MIXED_MERGE) { 768 if (bio->bi_opf & REQ_RAHEAD) 769 bio->bi_opf |= REQ_FAILFAST_MASK; 770 771 if (front_merge) { 772 req->cmd_flags &= ~REQ_FAILFAST_MASK; 773 req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK; 774 } 775 } 776 } 777 778 static void blk_account_io_merge_request(struct request *req) 779 { 780 if (blk_do_io_stat(req)) { 781 part_stat_lock(); 782 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); 783 part_stat_local_dec(req->part, 784 in_flight[op_is_write(req_op(req))]); 785 part_stat_unlock(); 786 } 787 } 788 789 static enum elv_merge blk_try_req_merge(struct request *req, 790 struct request *next) 791 { 792 if (blk_discard_mergable(req)) 793 return ELEVATOR_DISCARD_MERGE; 794 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) 795 return ELEVATOR_BACK_MERGE; 796 797 return ELEVATOR_NO_MERGE; 798 } 799 800 /* 801 * For non-mq, this has to be called with the request spinlock acquired. 802 * For mq with scheduling, the appropriate queue wide lock should be held. 803 */ 804 static struct request *attempt_merge(struct request_queue *q, 805 struct request *req, struct request *next) 806 { 807 if (!rq_mergeable(req) || !rq_mergeable(next)) 808 return NULL; 809 810 if (req_op(req) != req_op(next)) 811 return NULL; 812 813 if (rq_data_dir(req) != rq_data_dir(next)) 814 return NULL; 815 816 /* Don't merge requests with different write hints. */ 817 if (req->write_hint != next->write_hint) 818 return NULL; 819 820 if (req->ioprio != next->ioprio) 821 return NULL; 822 823 /* 824 * If we are allowed to merge, then append bio list 825 * from next to rq and release next. merge_requests_fn 826 * will have updated segment counts, update sector 827 * counts here. Handle DISCARDs separately, as they 828 * have separate settings. 829 */ 830 831 switch (blk_try_req_merge(req, next)) { 832 case ELEVATOR_DISCARD_MERGE: 833 if (!req_attempt_discard_merge(q, req, next)) 834 return NULL; 835 break; 836 case ELEVATOR_BACK_MERGE: 837 if (!ll_merge_requests_fn(q, req, next)) 838 return NULL; 839 break; 840 default: 841 return NULL; 842 } 843 844 /* 845 * If failfast settings disagree or any of the two is already 846 * a mixed merge, mark both as mixed before proceeding. This 847 * makes sure that all involved bios have mixable attributes 848 * set properly. 849 */ 850 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) || 851 (req->cmd_flags & REQ_FAILFAST_MASK) != 852 (next->cmd_flags & REQ_FAILFAST_MASK)) { 853 blk_rq_set_mixed_merge(req); 854 blk_rq_set_mixed_merge(next); 855 } 856 857 /* 858 * At this point we have either done a back merge or front merge. We 859 * need the smaller start_time_ns of the merged requests to be the 860 * current request for accounting purposes. 861 */ 862 if (next->start_time_ns < req->start_time_ns) 863 req->start_time_ns = next->start_time_ns; 864 865 req->biotail->bi_next = next->bio; 866 req->biotail = next->biotail; 867 868 req->__data_len += blk_rq_bytes(next); 869 870 if (!blk_discard_mergable(req)) 871 elv_merge_requests(q, req, next); 872 873 blk_crypto_rq_put_keyslot(next); 874 875 /* 876 * 'next' is going away, so update stats accordingly 877 */ 878 blk_account_io_merge_request(next); 879 880 trace_block_rq_merge(next); 881 882 /* 883 * ownership of bio passed from next to req, return 'next' for 884 * the caller to free 885 */ 886 next->bio = NULL; 887 return next; 888 } 889 890 static struct request *attempt_back_merge(struct request_queue *q, 891 struct request *rq) 892 { 893 struct request *next = elv_latter_request(q, rq); 894 895 if (next) 896 return attempt_merge(q, rq, next); 897 898 return NULL; 899 } 900 901 static struct request *attempt_front_merge(struct request_queue *q, 902 struct request *rq) 903 { 904 struct request *prev = elv_former_request(q, rq); 905 906 if (prev) 907 return attempt_merge(q, prev, rq); 908 909 return NULL; 910 } 911 912 /* 913 * Try to merge 'next' into 'rq'. Return true if the merge happened, false 914 * otherwise. The caller is responsible for freeing 'next' if the merge 915 * happened. 916 */ 917 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, 918 struct request *next) 919 { 920 return attempt_merge(q, rq, next); 921 } 922 923 bool blk_rq_merge_ok(struct request *rq, struct bio *bio) 924 { 925 if (!rq_mergeable(rq) || !bio_mergeable(bio)) 926 return false; 927 928 if (req_op(rq) != bio_op(bio)) 929 return false; 930 931 /* different data direction or already started, don't merge */ 932 if (bio_data_dir(bio) != rq_data_dir(rq)) 933 return false; 934 935 /* don't merge across cgroup boundaries */ 936 if (!blk_cgroup_mergeable(rq, bio)) 937 return false; 938 939 /* only merge integrity protected bio into ditto rq */ 940 if (blk_integrity_merge_bio(rq->q, rq, bio) == false) 941 return false; 942 943 /* Only merge if the crypt contexts are compatible */ 944 if (!bio_crypt_rq_ctx_compatible(rq, bio)) 945 return false; 946 947 /* Don't merge requests with different write hints. */ 948 if (rq->write_hint != bio->bi_write_hint) 949 return false; 950 951 if (rq->ioprio != bio_prio(bio)) 952 return false; 953 954 return true; 955 } 956 957 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) 958 { 959 if (blk_discard_mergable(rq)) 960 return ELEVATOR_DISCARD_MERGE; 961 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) 962 return ELEVATOR_BACK_MERGE; 963 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) 964 return ELEVATOR_FRONT_MERGE; 965 return ELEVATOR_NO_MERGE; 966 } 967 968 static void blk_account_io_merge_bio(struct request *req) 969 { 970 if (!blk_do_io_stat(req)) 971 return; 972 973 part_stat_lock(); 974 part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); 975 part_stat_unlock(); 976 } 977 978 enum bio_merge_status bio_attempt_back_merge(struct request *req, 979 struct bio *bio, unsigned int nr_segs) 980 { 981 const blk_opf_t ff = bio_failfast(bio); 982 983 if (!ll_back_merge_fn(req, bio, nr_segs)) 984 return BIO_MERGE_FAILED; 985 986 trace_block_bio_backmerge(bio); 987 rq_qos_merge(req->q, req, bio); 988 989 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) 990 blk_rq_set_mixed_merge(req); 991 992 blk_update_mixed_merge(req, bio, false); 993 994 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING) 995 blk_zone_write_plug_bio_merged(bio); 996 997 req->biotail->bi_next = bio; 998 req->biotail = bio; 999 req->__data_len += bio->bi_iter.bi_size; 1000 1001 bio_crypt_free_ctx(bio); 1002 1003 blk_account_io_merge_bio(req); 1004 return BIO_MERGE_OK; 1005 } 1006 1007 static enum bio_merge_status bio_attempt_front_merge(struct request *req, 1008 struct bio *bio, unsigned int nr_segs) 1009 { 1010 const blk_opf_t ff = bio_failfast(bio); 1011 1012 /* 1013 * A front merge for writes to sequential zones of a zoned block device 1014 * can happen only if the user submitted writes out of order. Do not 1015 * merge such write to let it fail. 1016 */ 1017 if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING) 1018 return BIO_MERGE_FAILED; 1019 1020 if (!ll_front_merge_fn(req, bio, nr_segs)) 1021 return BIO_MERGE_FAILED; 1022 1023 trace_block_bio_frontmerge(bio); 1024 rq_qos_merge(req->q, req, bio); 1025 1026 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) 1027 blk_rq_set_mixed_merge(req); 1028 1029 blk_update_mixed_merge(req, bio, true); 1030 1031 bio->bi_next = req->bio; 1032 req->bio = bio; 1033 1034 req->__sector = bio->bi_iter.bi_sector; 1035 req->__data_len += bio->bi_iter.bi_size; 1036 1037 bio_crypt_do_front_merge(req, bio); 1038 1039 blk_account_io_merge_bio(req); 1040 return BIO_MERGE_OK; 1041 } 1042 1043 static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q, 1044 struct request *req, struct bio *bio) 1045 { 1046 unsigned short segments = blk_rq_nr_discard_segments(req); 1047 1048 if (segments >= queue_max_discard_segments(q)) 1049 goto no_merge; 1050 if (blk_rq_sectors(req) + bio_sectors(bio) > 1051 blk_rq_get_max_sectors(req, blk_rq_pos(req))) 1052 goto no_merge; 1053 1054 rq_qos_merge(q, req, bio); 1055 1056 req->biotail->bi_next = bio; 1057 req->biotail = bio; 1058 req->__data_len += bio->bi_iter.bi_size; 1059 req->nr_phys_segments = segments + 1; 1060 1061 blk_account_io_merge_bio(req); 1062 return BIO_MERGE_OK; 1063 no_merge: 1064 req_set_nomerge(q, req); 1065 return BIO_MERGE_FAILED; 1066 } 1067 1068 static enum bio_merge_status blk_attempt_bio_merge(struct request_queue *q, 1069 struct request *rq, 1070 struct bio *bio, 1071 unsigned int nr_segs, 1072 bool sched_allow_merge) 1073 { 1074 if (!blk_rq_merge_ok(rq, bio)) 1075 return BIO_MERGE_NONE; 1076 1077 switch (blk_try_merge(rq, bio)) { 1078 case ELEVATOR_BACK_MERGE: 1079 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio)) 1080 return bio_attempt_back_merge(rq, bio, nr_segs); 1081 break; 1082 case ELEVATOR_FRONT_MERGE: 1083 if (!sched_allow_merge || blk_mq_sched_allow_merge(q, rq, bio)) 1084 return bio_attempt_front_merge(rq, bio, nr_segs); 1085 break; 1086 case ELEVATOR_DISCARD_MERGE: 1087 return bio_attempt_discard_merge(q, rq, bio); 1088 default: 1089 return BIO_MERGE_NONE; 1090 } 1091 1092 return BIO_MERGE_FAILED; 1093 } 1094 1095 /** 1096 * blk_attempt_plug_merge - try to merge with %current's plugged list 1097 * @q: request_queue new bio is being queued at 1098 * @bio: new bio being queued 1099 * @nr_segs: number of segments in @bio 1100 * from the passed in @q already in the plug list 1101 * 1102 * Determine whether @bio being queued on @q can be merged with the previous 1103 * request on %current's plugged list. Returns %true if merge was successful, 1104 * otherwise %false. 1105 * 1106 * Plugging coalesces IOs from the same issuer for the same purpose without 1107 * going through @q->queue_lock. As such it's more of an issuing mechanism 1108 * than scheduling, and the request, while may have elvpriv data, is not 1109 * added on the elevator at this point. In addition, we don't have 1110 * reliable access to the elevator outside queue lock. Only check basic 1111 * merging parameters without querying the elevator. 1112 * 1113 * Caller must ensure !blk_queue_nomerges(q) beforehand. 1114 */ 1115 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 1116 unsigned int nr_segs) 1117 { 1118 struct blk_plug *plug = current->plug; 1119 struct request *rq; 1120 1121 if (!plug || rq_list_empty(plug->mq_list)) 1122 return false; 1123 1124 rq_list_for_each(&plug->mq_list, rq) { 1125 if (rq->q == q) { 1126 if (blk_attempt_bio_merge(q, rq, bio, nr_segs, false) == 1127 BIO_MERGE_OK) 1128 return true; 1129 break; 1130 } 1131 1132 /* 1133 * Only keep iterating plug list for merges if we have multiple 1134 * queues 1135 */ 1136 if (!plug->multiple_queues) 1137 break; 1138 } 1139 return false; 1140 } 1141 1142 /* 1143 * Iterate list of requests and see if we can merge this bio with any 1144 * of them. 1145 */ 1146 bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, 1147 struct bio *bio, unsigned int nr_segs) 1148 { 1149 struct request *rq; 1150 int checked = 8; 1151 1152 list_for_each_entry_reverse(rq, list, queuelist) { 1153 if (!checked--) 1154 break; 1155 1156 switch (blk_attempt_bio_merge(q, rq, bio, nr_segs, true)) { 1157 case BIO_MERGE_NONE: 1158 continue; 1159 case BIO_MERGE_OK: 1160 return true; 1161 case BIO_MERGE_FAILED: 1162 return false; 1163 } 1164 1165 } 1166 1167 return false; 1168 } 1169 EXPORT_SYMBOL_GPL(blk_bio_list_merge); 1170 1171 bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, 1172 unsigned int nr_segs, struct request **merged_request) 1173 { 1174 struct request *rq; 1175 1176 switch (elv_merge(q, &rq, bio)) { 1177 case ELEVATOR_BACK_MERGE: 1178 if (!blk_mq_sched_allow_merge(q, rq, bio)) 1179 return false; 1180 if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK) 1181 return false; 1182 *merged_request = attempt_back_merge(q, rq); 1183 if (!*merged_request) 1184 elv_merged_request(q, rq, ELEVATOR_BACK_MERGE); 1185 return true; 1186 case ELEVATOR_FRONT_MERGE: 1187 if (!blk_mq_sched_allow_merge(q, rq, bio)) 1188 return false; 1189 if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK) 1190 return false; 1191 *merged_request = attempt_front_merge(q, rq); 1192 if (!*merged_request) 1193 elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE); 1194 return true; 1195 case ELEVATOR_DISCARD_MERGE: 1196 return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK; 1197 default: 1198 return false; 1199 } 1200 } 1201 EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge); 1202