1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1991, 1992 Linus Torvalds 4 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics 5 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE 6 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> 7 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> 8 * - July2000 9 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 10 */ 11 12 /* 13 * This handles all read/write requests to block devices 14 */ 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/bio.h> 18 #include <linux/blkdev.h> 19 #include <linux/blk-mq.h> 20 #include <linux/blk-pm.h> 21 #include <linux/blk-integrity.h> 22 #include <linux/highmem.h> 23 #include <linux/mm.h> 24 #include <linux/pagemap.h> 25 #include <linux/kernel_stat.h> 26 #include <linux/string.h> 27 #include <linux/init.h> 28 #include <linux/completion.h> 29 #include <linux/slab.h> 30 #include <linux/swap.h> 31 #include <linux/writeback.h> 32 #include <linux/task_io_accounting_ops.h> 33 #include <linux/fault-inject.h> 34 #include <linux/list_sort.h> 35 #include <linux/delay.h> 36 #include <linux/ratelimit.h> 37 #include <linux/pm_runtime.h> 38 #include <linux/blk-cgroup.h> 39 #include <linux/t10-pi.h> 40 #include <linux/debugfs.h> 41 #include <linux/bpf.h> 42 #include <linux/psi.h> 43 #include <linux/sched/sysctl.h> 44 #include <linux/blk-crypto.h> 45 46 #define CREATE_TRACE_POINTS 47 #include <trace/events/block.h> 48 49 #include "blk.h" 50 #include "blk-mq.h" 51 #include "blk-mq-sched.h" 52 #include "blk-pm.h" 53 #include "blk-throttle.h" 54 55 struct dentry *blk_debugfs_root; 56 57 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); 58 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); 59 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); 60 EXPORT_TRACEPOINT_SYMBOL_GPL(block_split); 61 EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); 62 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert); 63 64 DEFINE_IDA(blk_queue_ida); 65 66 /* 67 * For queue allocation 68 */ 69 struct kmem_cache *blk_requestq_cachep; 70 71 /* 72 * Controlling structure to kblockd 73 */ 74 static struct workqueue_struct *kblockd_workqueue; 75 76 /** 77 * blk_queue_flag_set - atomically set a queue flag 78 * @flag: flag to be set 79 * @q: request queue 80 */ 81 void blk_queue_flag_set(unsigned int flag, struct request_queue *q) 82 { 83 set_bit(flag, &q->queue_flags); 84 } 85 EXPORT_SYMBOL(blk_queue_flag_set); 86 87 /** 88 * blk_queue_flag_clear - atomically clear a queue flag 89 * @flag: flag to be cleared 90 * @q: request queue 91 */ 92 void blk_queue_flag_clear(unsigned int flag, struct request_queue *q) 93 { 94 clear_bit(flag, &q->queue_flags); 95 } 96 EXPORT_SYMBOL(blk_queue_flag_clear); 97 98 /** 99 * blk_queue_flag_test_and_set - atomically test and set a queue flag 100 * @flag: flag to be set 101 * @q: request queue 102 * 103 * Returns the previous value of @flag - 0 if the flag was not set and 1 if 104 * the flag was already set. 105 */ 106 bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q) 107 { 108 return test_and_set_bit(flag, &q->queue_flags); 109 } 110 EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set); 111 112 void blk_rq_init(struct request_queue *q, struct request *rq) 113 { 114 memset(rq, 0, sizeof(*rq)); 115 116 INIT_LIST_HEAD(&rq->queuelist); 117 rq->q = q; 118 rq->__sector = (sector_t) -1; 119 INIT_HLIST_NODE(&rq->hash); 120 RB_CLEAR_NODE(&rq->rb_node); 121 rq->tag = BLK_MQ_NO_TAG; 122 rq->internal_tag = BLK_MQ_NO_TAG; 123 rq->start_time_ns = ktime_get_ns(); 124 rq->part = NULL; 125 blk_crypto_rq_set_defaults(rq); 126 } 127 EXPORT_SYMBOL(blk_rq_init); 128 129 #define REQ_OP_NAME(name) [REQ_OP_##name] = #name 130 static const char *const blk_op_name[] = { 131 REQ_OP_NAME(READ), 132 REQ_OP_NAME(WRITE), 133 REQ_OP_NAME(FLUSH), 134 REQ_OP_NAME(DISCARD), 135 REQ_OP_NAME(SECURE_ERASE), 136 REQ_OP_NAME(ZONE_RESET), 137 REQ_OP_NAME(ZONE_RESET_ALL), 138 REQ_OP_NAME(ZONE_OPEN), 139 REQ_OP_NAME(ZONE_CLOSE), 140 REQ_OP_NAME(ZONE_FINISH), 141 REQ_OP_NAME(ZONE_APPEND), 142 REQ_OP_NAME(WRITE_SAME), 143 REQ_OP_NAME(WRITE_ZEROES), 144 REQ_OP_NAME(DRV_IN), 145 REQ_OP_NAME(DRV_OUT), 146 }; 147 #undef REQ_OP_NAME 148 149 /** 150 * blk_op_str - Return string XXX in the REQ_OP_XXX. 151 * @op: REQ_OP_XXX. 152 * 153 * Description: Centralize block layer function to convert REQ_OP_XXX into 154 * string format. Useful in the debugging and tracing bio or request. For 155 * invalid REQ_OP_XXX it returns string "UNKNOWN". 156 */ 157 inline const char *blk_op_str(unsigned int op) 158 { 159 const char *op_str = "UNKNOWN"; 160 161 if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op]) 162 op_str = blk_op_name[op]; 163 164 return op_str; 165 } 166 EXPORT_SYMBOL_GPL(blk_op_str); 167 168 static const struct { 169 int errno; 170 const char *name; 171 } blk_errors[] = { 172 [BLK_STS_OK] = { 0, "" }, 173 [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" }, 174 [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" }, 175 [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" }, 176 [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" }, 177 [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" }, 178 [BLK_STS_NEXUS] = { -EBADE, "critical nexus" }, 179 [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" }, 180 [BLK_STS_PROTECTION] = { -EILSEQ, "protection" }, 181 [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" }, 182 [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" }, 183 [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" }, 184 185 /* device mapper special case, should not leak out: */ 186 [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" }, 187 188 /* zone device specific errors */ 189 [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" }, 190 [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" }, 191 192 /* everything else not covered above: */ 193 [BLK_STS_IOERR] = { -EIO, "I/O" }, 194 }; 195 196 blk_status_t errno_to_blk_status(int errno) 197 { 198 int i; 199 200 for (i = 0; i < ARRAY_SIZE(blk_errors); i++) { 201 if (blk_errors[i].errno == errno) 202 return (__force blk_status_t)i; 203 } 204 205 return BLK_STS_IOERR; 206 } 207 EXPORT_SYMBOL_GPL(errno_to_blk_status); 208 209 int blk_status_to_errno(blk_status_t status) 210 { 211 int idx = (__force int)status; 212 213 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 214 return -EIO; 215 return blk_errors[idx].errno; 216 } 217 EXPORT_SYMBOL_GPL(blk_status_to_errno); 218 219 void blk_print_req_error(struct request *req, blk_status_t status) 220 { 221 int idx = (__force int)status; 222 223 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors))) 224 return; 225 226 printk_ratelimited(KERN_ERR 227 "%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x " 228 "phys_seg %u prio class %u\n", 229 blk_errors[idx].name, 230 req->rq_disk ? req->rq_disk->disk_name : "?", 231 blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)), 232 req->cmd_flags & ~REQ_OP_MASK, 233 req->nr_phys_segments, 234 IOPRIO_PRIO_CLASS(req->ioprio)); 235 } 236 237 void blk_dump_rq_flags(struct request *rq, char *msg) 238 { 239 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg, 240 rq->rq_disk ? rq->rq_disk->disk_name : "?", 241 (unsigned long long) rq->cmd_flags); 242 243 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", 244 (unsigned long long)blk_rq_pos(rq), 245 blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); 246 printk(KERN_INFO " bio %p, biotail %p, len %u\n", 247 rq->bio, rq->biotail, blk_rq_bytes(rq)); 248 } 249 EXPORT_SYMBOL(blk_dump_rq_flags); 250 251 /** 252 * blk_sync_queue - cancel any pending callbacks on a queue 253 * @q: the queue 254 * 255 * Description: 256 * The block layer may perform asynchronous callback activity 257 * on a queue, such as calling the unplug function after a timeout. 258 * A block device may call blk_sync_queue to ensure that any 259 * such activity is cancelled, thus allowing it to release resources 260 * that the callbacks might use. The caller must already have made sure 261 * that its ->submit_bio will not re-add plugging prior to calling 262 * this function. 263 * 264 * This function does not cancel any asynchronous activity arising 265 * out of elevator or throttling code. That would require elevator_exit() 266 * and blkcg_exit_queue() to be called with queue lock initialized. 267 * 268 */ 269 void blk_sync_queue(struct request_queue *q) 270 { 271 del_timer_sync(&q->timeout); 272 cancel_work_sync(&q->timeout_work); 273 } 274 EXPORT_SYMBOL(blk_sync_queue); 275 276 /** 277 * blk_set_pm_only - increment pm_only counter 278 * @q: request queue pointer 279 */ 280 void blk_set_pm_only(struct request_queue *q) 281 { 282 atomic_inc(&q->pm_only); 283 } 284 EXPORT_SYMBOL_GPL(blk_set_pm_only); 285 286 void blk_clear_pm_only(struct request_queue *q) 287 { 288 int pm_only; 289 290 pm_only = atomic_dec_return(&q->pm_only); 291 WARN_ON_ONCE(pm_only < 0); 292 if (pm_only == 0) 293 wake_up_all(&q->mq_freeze_wq); 294 } 295 EXPORT_SYMBOL_GPL(blk_clear_pm_only); 296 297 /** 298 * blk_put_queue - decrement the request_queue refcount 299 * @q: the request_queue structure to decrement the refcount for 300 * 301 * Decrements the refcount of the request_queue kobject. When this reaches 0 302 * we'll have blk_release_queue() called. 303 * 304 * Context: Any context, but the last reference must not be dropped from 305 * atomic context. 306 */ 307 void blk_put_queue(struct request_queue *q) 308 { 309 kobject_put(&q->kobj); 310 } 311 EXPORT_SYMBOL(blk_put_queue); 312 313 void blk_queue_start_drain(struct request_queue *q) 314 { 315 /* 316 * When queue DYING flag is set, we need to block new req 317 * entering queue, so we call blk_freeze_queue_start() to 318 * prevent I/O from crossing blk_queue_enter(). 319 */ 320 blk_freeze_queue_start(q); 321 if (queue_is_mq(q)) 322 blk_mq_wake_waiters(q); 323 /* Make blk_queue_enter() reexamine the DYING flag. */ 324 wake_up_all(&q->mq_freeze_wq); 325 } 326 327 void blk_set_queue_dying(struct request_queue *q) 328 { 329 blk_queue_flag_set(QUEUE_FLAG_DYING, q); 330 blk_queue_start_drain(q); 331 } 332 EXPORT_SYMBOL_GPL(blk_set_queue_dying); 333 334 /** 335 * blk_cleanup_queue - shutdown a request queue 336 * @q: request queue to shutdown 337 * 338 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and 339 * put it. All future requests will be failed immediately with -ENODEV. 340 * 341 * Context: can sleep 342 */ 343 void blk_cleanup_queue(struct request_queue *q) 344 { 345 /* cannot be called from atomic context */ 346 might_sleep(); 347 348 WARN_ON_ONCE(blk_queue_registered(q)); 349 350 /* mark @q DYING, no new request or merges will be allowed afterwards */ 351 blk_set_queue_dying(q); 352 353 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q); 354 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q); 355 356 /* 357 * Drain all requests queued before DYING marking. Set DEAD flag to 358 * prevent that blk_mq_run_hw_queues() accesses the hardware queues 359 * after draining finished. 360 */ 361 blk_freeze_queue(q); 362 363 blk_queue_flag_set(QUEUE_FLAG_DEAD, q); 364 365 blk_sync_queue(q); 366 if (queue_is_mq(q)) 367 blk_mq_exit_queue(q); 368 369 /* 370 * In theory, request pool of sched_tags belongs to request queue. 371 * However, the current implementation requires tag_set for freeing 372 * requests, so free the pool now. 373 * 374 * Queue has become frozen, there can't be any in-queue requests, so 375 * it is safe to free requests now. 376 */ 377 mutex_lock(&q->sysfs_lock); 378 if (q->elevator) 379 blk_mq_sched_free_rqs(q); 380 mutex_unlock(&q->sysfs_lock); 381 382 percpu_ref_exit(&q->q_usage_counter); 383 384 /* @q is and will stay empty, shutdown and put */ 385 blk_put_queue(q); 386 } 387 EXPORT_SYMBOL(blk_cleanup_queue); 388 389 static bool blk_try_enter_queue(struct request_queue *q, bool pm) 390 { 391 rcu_read_lock(); 392 if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter)) 393 goto fail; 394 395 /* 396 * The code that increments the pm_only counter must ensure that the 397 * counter is globally visible before the queue is unfrozen. 398 */ 399 if (blk_queue_pm_only(q) && 400 (!pm || queue_rpm_status(q) == RPM_SUSPENDED)) 401 goto fail_put; 402 403 rcu_read_unlock(); 404 return true; 405 406 fail_put: 407 blk_queue_exit(q); 408 fail: 409 rcu_read_unlock(); 410 return false; 411 } 412 413 /** 414 * blk_queue_enter() - try to increase q->q_usage_counter 415 * @q: request queue pointer 416 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM 417 */ 418 int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) 419 { 420 const bool pm = flags & BLK_MQ_REQ_PM; 421 422 while (!blk_try_enter_queue(q, pm)) { 423 if (flags & BLK_MQ_REQ_NOWAIT) 424 return -EBUSY; 425 426 /* 427 * read pair of barrier in blk_freeze_queue_start(), we need to 428 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and 429 * reading .mq_freeze_depth or queue dying flag, otherwise the 430 * following wait may never return if the two reads are 431 * reordered. 432 */ 433 smp_rmb(); 434 wait_event(q->mq_freeze_wq, 435 (!q->mq_freeze_depth && 436 blk_pm_resume_queue(pm, q)) || 437 blk_queue_dying(q)); 438 if (blk_queue_dying(q)) 439 return -ENODEV; 440 } 441 442 return 0; 443 } 444 445 static inline int bio_queue_enter(struct bio *bio) 446 { 447 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 448 449 while (!blk_try_enter_queue(q, false)) { 450 struct gendisk *disk = bio->bi_bdev->bd_disk; 451 452 if (bio->bi_opf & REQ_NOWAIT) { 453 if (test_bit(GD_DEAD, &disk->state)) 454 goto dead; 455 bio_wouldblock_error(bio); 456 return -EBUSY; 457 } 458 459 /* 460 * read pair of barrier in blk_freeze_queue_start(), we need to 461 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and 462 * reading .mq_freeze_depth or queue dying flag, otherwise the 463 * following wait may never return if the two reads are 464 * reordered. 465 */ 466 smp_rmb(); 467 wait_event(q->mq_freeze_wq, 468 (!q->mq_freeze_depth && 469 blk_pm_resume_queue(false, q)) || 470 test_bit(GD_DEAD, &disk->state)); 471 if (test_bit(GD_DEAD, &disk->state)) 472 goto dead; 473 } 474 475 return 0; 476 dead: 477 bio_io_error(bio); 478 return -ENODEV; 479 } 480 481 void blk_queue_exit(struct request_queue *q) 482 { 483 percpu_ref_put(&q->q_usage_counter); 484 } 485 486 static void blk_queue_usage_counter_release(struct percpu_ref *ref) 487 { 488 struct request_queue *q = 489 container_of(ref, struct request_queue, q_usage_counter); 490 491 wake_up_all(&q->mq_freeze_wq); 492 } 493 494 static void blk_rq_timed_out_timer(struct timer_list *t) 495 { 496 struct request_queue *q = from_timer(q, t, timeout); 497 498 kblockd_schedule_work(&q->timeout_work); 499 } 500 501 static void blk_timeout_work(struct work_struct *work) 502 { 503 } 504 505 struct request_queue *blk_alloc_queue(int node_id) 506 { 507 struct request_queue *q; 508 int ret; 509 510 q = kmem_cache_alloc_node(blk_requestq_cachep, 511 GFP_KERNEL | __GFP_ZERO, node_id); 512 if (!q) 513 return NULL; 514 515 q->last_merge = NULL; 516 517 q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL); 518 if (q->id < 0) 519 goto fail_q; 520 521 ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0); 522 if (ret) 523 goto fail_id; 524 525 q->stats = blk_alloc_queue_stats(); 526 if (!q->stats) 527 goto fail_split; 528 529 q->node = node_id; 530 531 atomic_set(&q->nr_active_requests_shared_tags, 0); 532 533 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0); 534 INIT_WORK(&q->timeout_work, blk_timeout_work); 535 INIT_LIST_HEAD(&q->icq_list); 536 #ifdef CONFIG_BLK_CGROUP 537 INIT_LIST_HEAD(&q->blkg_list); 538 #endif 539 540 kobject_init(&q->kobj, &blk_queue_ktype); 541 542 mutex_init(&q->debugfs_mutex); 543 mutex_init(&q->sysfs_lock); 544 mutex_init(&q->sysfs_dir_lock); 545 spin_lock_init(&q->queue_lock); 546 547 init_waitqueue_head(&q->mq_freeze_wq); 548 mutex_init(&q->mq_freeze_lock); 549 550 /* 551 * Init percpu_ref in atomic mode so that it's faster to shutdown. 552 * See blk_register_queue() for details. 553 */ 554 if (percpu_ref_init(&q->q_usage_counter, 555 blk_queue_usage_counter_release, 556 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL)) 557 goto fail_stats; 558 559 if (blkcg_init_queue(q)) 560 goto fail_ref; 561 562 blk_queue_dma_alignment(q, 511); 563 blk_set_default_limits(&q->limits); 564 q->nr_requests = BLKDEV_DEFAULT_RQ; 565 566 return q; 567 568 fail_ref: 569 percpu_ref_exit(&q->q_usage_counter); 570 fail_stats: 571 blk_free_queue_stats(q->stats); 572 fail_split: 573 bioset_exit(&q->bio_split); 574 fail_id: 575 ida_simple_remove(&blk_queue_ida, q->id); 576 fail_q: 577 kmem_cache_free(blk_requestq_cachep, q); 578 return NULL; 579 } 580 581 /** 582 * blk_get_queue - increment the request_queue refcount 583 * @q: the request_queue structure to increment the refcount for 584 * 585 * Increment the refcount of the request_queue kobject. 586 * 587 * Context: Any context. 588 */ 589 bool blk_get_queue(struct request_queue *q) 590 { 591 if (likely(!blk_queue_dying(q))) { 592 __blk_get_queue(q); 593 return true; 594 } 595 596 return false; 597 } 598 EXPORT_SYMBOL(blk_get_queue); 599 600 static void handle_bad_sector(struct bio *bio, sector_t maxsector) 601 { 602 char b[BDEVNAME_SIZE]; 603 604 pr_info_ratelimited("%s: attempt to access beyond end of device\n" 605 "%s: rw=%d, want=%llu, limit=%llu\n", 606 current->comm, 607 bio_devname(bio, b), bio->bi_opf, 608 bio_end_sector(bio), maxsector); 609 } 610 611 #ifdef CONFIG_FAIL_MAKE_REQUEST 612 613 static DECLARE_FAULT_ATTR(fail_make_request); 614 615 static int __init setup_fail_make_request(char *str) 616 { 617 return setup_fault_attr(&fail_make_request, str); 618 } 619 __setup("fail_make_request=", setup_fail_make_request); 620 621 static bool should_fail_request(struct block_device *part, unsigned int bytes) 622 { 623 return part->bd_make_it_fail && should_fail(&fail_make_request, bytes); 624 } 625 626 static int __init fail_make_request_debugfs(void) 627 { 628 struct dentry *dir = fault_create_debugfs_attr("fail_make_request", 629 NULL, &fail_make_request); 630 631 return PTR_ERR_OR_ZERO(dir); 632 } 633 634 late_initcall(fail_make_request_debugfs); 635 636 #else /* CONFIG_FAIL_MAKE_REQUEST */ 637 638 static inline bool should_fail_request(struct block_device *part, 639 unsigned int bytes) 640 { 641 return false; 642 } 643 644 #endif /* CONFIG_FAIL_MAKE_REQUEST */ 645 646 static inline bool bio_check_ro(struct bio *bio) 647 { 648 if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) { 649 char b[BDEVNAME_SIZE]; 650 651 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) 652 return false; 653 654 WARN_ONCE(1, 655 "Trying to write to read-only block-device %s (partno %d)\n", 656 bio_devname(bio, b), bio->bi_bdev->bd_partno); 657 /* Older lvm-tools actually trigger this */ 658 return false; 659 } 660 661 return false; 662 } 663 664 static noinline int should_fail_bio(struct bio *bio) 665 { 666 if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size)) 667 return -EIO; 668 return 0; 669 } 670 ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO); 671 672 /* 673 * Check whether this bio extends beyond the end of the device or partition. 674 * This may well happen - the kernel calls bread() without checking the size of 675 * the device, e.g., when mounting a file system. 676 */ 677 static inline int bio_check_eod(struct bio *bio) 678 { 679 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); 680 unsigned int nr_sectors = bio_sectors(bio); 681 682 if (nr_sectors && maxsector && 683 (nr_sectors > maxsector || 684 bio->bi_iter.bi_sector > maxsector - nr_sectors)) { 685 handle_bad_sector(bio, maxsector); 686 return -EIO; 687 } 688 return 0; 689 } 690 691 /* 692 * Remap block n of partition p to block n+start(p) of the disk. 693 */ 694 static int blk_partition_remap(struct bio *bio) 695 { 696 struct block_device *p = bio->bi_bdev; 697 698 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) 699 return -EIO; 700 if (bio_sectors(bio)) { 701 bio->bi_iter.bi_sector += p->bd_start_sect; 702 trace_block_bio_remap(bio, p->bd_dev, 703 bio->bi_iter.bi_sector - 704 p->bd_start_sect); 705 } 706 bio_set_flag(bio, BIO_REMAPPED); 707 return 0; 708 } 709 710 /* 711 * Check write append to a zoned block device. 712 */ 713 static inline blk_status_t blk_check_zone_append(struct request_queue *q, 714 struct bio *bio) 715 { 716 sector_t pos = bio->bi_iter.bi_sector; 717 int nr_sectors = bio_sectors(bio); 718 719 /* Only applicable to zoned block devices */ 720 if (!blk_queue_is_zoned(q)) 721 return BLK_STS_NOTSUPP; 722 723 /* The bio sector must point to the start of a sequential zone */ 724 if (pos & (blk_queue_zone_sectors(q) - 1) || 725 !blk_queue_zone_is_seq(q, pos)) 726 return BLK_STS_IOERR; 727 728 /* 729 * Not allowed to cross zone boundaries. Otherwise, the BIO will be 730 * split and could result in non-contiguous sectors being written in 731 * different zones. 732 */ 733 if (nr_sectors > q->limits.chunk_sectors) 734 return BLK_STS_IOERR; 735 736 /* Make sure the BIO is small enough and will not get split */ 737 if (nr_sectors > q->limits.max_zone_append_sectors) 738 return BLK_STS_IOERR; 739 740 bio->bi_opf |= REQ_NOMERGE; 741 742 return BLK_STS_OK; 743 } 744 745 static noinline_for_stack bool submit_bio_checks(struct bio *bio) 746 { 747 struct block_device *bdev = bio->bi_bdev; 748 struct request_queue *q = bdev_get_queue(bdev); 749 blk_status_t status = BLK_STS_IOERR; 750 struct blk_plug *plug; 751 752 might_sleep(); 753 754 plug = blk_mq_plug(q, bio); 755 if (plug && plug->nowait) 756 bio->bi_opf |= REQ_NOWAIT; 757 758 /* 759 * For a REQ_NOWAIT based request, return -EOPNOTSUPP 760 * if queue does not support NOWAIT. 761 */ 762 if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q)) 763 goto not_supported; 764 765 if (should_fail_bio(bio)) 766 goto end_io; 767 if (unlikely(bio_check_ro(bio))) 768 goto end_io; 769 if (!bio_flagged(bio, BIO_REMAPPED)) { 770 if (unlikely(bio_check_eod(bio))) 771 goto end_io; 772 if (bdev->bd_partno && unlikely(blk_partition_remap(bio))) 773 goto end_io; 774 } 775 776 /* 777 * Filter flush bio's early so that bio based drivers without flush 778 * support don't have to worry about them. 779 */ 780 if (op_is_flush(bio->bi_opf) && 781 !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) { 782 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA); 783 if (!bio_sectors(bio)) { 784 status = BLK_STS_OK; 785 goto end_io; 786 } 787 } 788 789 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 790 bio_clear_polled(bio); 791 792 switch (bio_op(bio)) { 793 case REQ_OP_DISCARD: 794 if (!blk_queue_discard(q)) 795 goto not_supported; 796 break; 797 case REQ_OP_SECURE_ERASE: 798 if (!blk_queue_secure_erase(q)) 799 goto not_supported; 800 break; 801 case REQ_OP_WRITE_SAME: 802 if (!q->limits.max_write_same_sectors) 803 goto not_supported; 804 break; 805 case REQ_OP_ZONE_APPEND: 806 status = blk_check_zone_append(q, bio); 807 if (status != BLK_STS_OK) 808 goto end_io; 809 break; 810 case REQ_OP_ZONE_RESET: 811 case REQ_OP_ZONE_OPEN: 812 case REQ_OP_ZONE_CLOSE: 813 case REQ_OP_ZONE_FINISH: 814 if (!blk_queue_is_zoned(q)) 815 goto not_supported; 816 break; 817 case REQ_OP_ZONE_RESET_ALL: 818 if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q)) 819 goto not_supported; 820 break; 821 case REQ_OP_WRITE_ZEROES: 822 if (!q->limits.max_write_zeroes_sectors) 823 goto not_supported; 824 break; 825 default: 826 break; 827 } 828 829 /* 830 * Various block parts want %current->io_context, so allocate it up 831 * front rather than dealing with lots of pain to allocate it only 832 * where needed. This may fail and the block layer knows how to live 833 * with it. 834 */ 835 if (unlikely(!current->io_context)) 836 create_task_io_context(current, GFP_ATOMIC, q->node); 837 838 if (blk_throtl_bio(bio)) { 839 blkcg_bio_issue_init(bio); 840 return false; 841 } 842 843 blk_cgroup_bio_start(bio); 844 blkcg_bio_issue_init(bio); 845 846 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) { 847 trace_block_bio_queue(bio); 848 /* Now that enqueuing has been traced, we need to trace 849 * completion as well. 850 */ 851 bio_set_flag(bio, BIO_TRACE_COMPLETION); 852 } 853 return true; 854 855 not_supported: 856 status = BLK_STS_NOTSUPP; 857 end_io: 858 bio->bi_status = status; 859 bio_endio(bio); 860 return false; 861 } 862 863 static void __submit_bio(struct bio *bio) 864 { 865 struct gendisk *disk = bio->bi_bdev->bd_disk; 866 867 if (unlikely(bio_queue_enter(bio) != 0)) 868 return; 869 870 if (!submit_bio_checks(bio) || !blk_crypto_bio_prep(&bio)) 871 goto queue_exit; 872 if (!disk->fops->submit_bio) { 873 blk_mq_submit_bio(bio); 874 return; 875 } 876 disk->fops->submit_bio(bio); 877 queue_exit: 878 blk_queue_exit(disk->queue); 879 } 880 881 /* 882 * The loop in this function may be a bit non-obvious, and so deserves some 883 * explanation: 884 * 885 * - Before entering the loop, bio->bi_next is NULL (as all callers ensure 886 * that), so we have a list with a single bio. 887 * - We pretend that we have just taken it off a longer list, so we assign 888 * bio_list to a pointer to the bio_list_on_stack, thus initialising the 889 * bio_list of new bios to be added. ->submit_bio() may indeed add some more 890 * bios through a recursive call to submit_bio_noacct. If it did, we find a 891 * non-NULL value in bio_list and re-enter the loop from the top. 892 * - In this case we really did just take the bio of the top of the list (no 893 * pretending) and so remove it from bio_list, and call into ->submit_bio() 894 * again. 895 * 896 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio. 897 * bio_list_on_stack[1] contains bios that were submitted before the current 898 * ->submit_bio_bio, but that haven't been processed yet. 899 */ 900 static void __submit_bio_noacct(struct bio *bio) 901 { 902 struct bio_list bio_list_on_stack[2]; 903 904 BUG_ON(bio->bi_next); 905 906 bio_list_init(&bio_list_on_stack[0]); 907 current->bio_list = bio_list_on_stack; 908 909 do { 910 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 911 struct bio_list lower, same; 912 913 /* 914 * Create a fresh bio_list for all subordinate requests. 915 */ 916 bio_list_on_stack[1] = bio_list_on_stack[0]; 917 bio_list_init(&bio_list_on_stack[0]); 918 919 __submit_bio(bio); 920 921 /* 922 * Sort new bios into those for a lower level and those for the 923 * same level. 924 */ 925 bio_list_init(&lower); 926 bio_list_init(&same); 927 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL) 928 if (q == bdev_get_queue(bio->bi_bdev)) 929 bio_list_add(&same, bio); 930 else 931 bio_list_add(&lower, bio); 932 933 /* 934 * Now assemble so we handle the lowest level first. 935 */ 936 bio_list_merge(&bio_list_on_stack[0], &lower); 937 bio_list_merge(&bio_list_on_stack[0], &same); 938 bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]); 939 } while ((bio = bio_list_pop(&bio_list_on_stack[0]))); 940 941 current->bio_list = NULL; 942 } 943 944 static void __submit_bio_noacct_mq(struct bio *bio) 945 { 946 struct bio_list bio_list[2] = { }; 947 948 current->bio_list = bio_list; 949 950 do { 951 __submit_bio(bio); 952 } while ((bio = bio_list_pop(&bio_list[0]))); 953 954 current->bio_list = NULL; 955 } 956 957 /** 958 * submit_bio_noacct - re-submit a bio to the block device layer for I/O 959 * @bio: The bio describing the location in memory and on the device. 960 * 961 * This is a version of submit_bio() that shall only be used for I/O that is 962 * resubmitted to lower level drivers by stacking block drivers. All file 963 * systems and other upper level users of the block layer should use 964 * submit_bio() instead. 965 */ 966 void submit_bio_noacct(struct bio *bio) 967 { 968 /* 969 * We only want one ->submit_bio to be active at a time, else stack 970 * usage with stacked devices could be a problem. Use current->bio_list 971 * to collect a list of requests submited by a ->submit_bio method while 972 * it is active, and then process them after it returned. 973 */ 974 if (current->bio_list) 975 bio_list_add(¤t->bio_list[0], bio); 976 else if (!bio->bi_bdev->bd_disk->fops->submit_bio) 977 __submit_bio_noacct_mq(bio); 978 else 979 __submit_bio_noacct(bio); 980 } 981 EXPORT_SYMBOL(submit_bio_noacct); 982 983 /** 984 * submit_bio - submit a bio to the block device layer for I/O 985 * @bio: The &struct bio which describes the I/O 986 * 987 * submit_bio() is used to submit I/O requests to block devices. It is passed a 988 * fully set up &struct bio that describes the I/O that needs to be done. The 989 * bio will be send to the device described by the bi_bdev field. 990 * 991 * The success/failure status of the request, along with notification of 992 * completion, is delivered asynchronously through the ->bi_end_io() callback 993 * in @bio. The bio must NOT be touched by thecaller until ->bi_end_io() has 994 * been called. 995 */ 996 void submit_bio(struct bio *bio) 997 { 998 if (blkcg_punt_bio_submit(bio)) 999 return; 1000 1001 /* 1002 * If it's a regular read/write or a barrier with data attached, 1003 * go through the normal accounting stuff before submission. 1004 */ 1005 if (bio_has_data(bio)) { 1006 unsigned int count; 1007 1008 if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) 1009 count = queue_logical_block_size( 1010 bdev_get_queue(bio->bi_bdev)) >> 9; 1011 else 1012 count = bio_sectors(bio); 1013 1014 if (op_is_write(bio_op(bio))) { 1015 count_vm_events(PGPGOUT, count); 1016 } else { 1017 task_io_account_read(bio->bi_iter.bi_size); 1018 count_vm_events(PGPGIN, count); 1019 } 1020 } 1021 1022 /* 1023 * If we're reading data that is part of the userspace workingset, count 1024 * submission time as memory stall. When the device is congested, or 1025 * the submitting cgroup IO-throttled, submission can be a significant 1026 * part of overall IO time. 1027 */ 1028 if (unlikely(bio_op(bio) == REQ_OP_READ && 1029 bio_flagged(bio, BIO_WORKINGSET))) { 1030 unsigned long pflags; 1031 1032 psi_memstall_enter(&pflags); 1033 submit_bio_noacct(bio); 1034 psi_memstall_leave(&pflags); 1035 return; 1036 } 1037 1038 submit_bio_noacct(bio); 1039 } 1040 EXPORT_SYMBOL(submit_bio); 1041 1042 /** 1043 * bio_poll - poll for BIO completions 1044 * @bio: bio to poll for 1045 * @flags: BLK_POLL_* flags that control the behavior 1046 * 1047 * Poll for completions on queue associated with the bio. Returns number of 1048 * completed entries found. 1049 * 1050 * Note: the caller must either be the context that submitted @bio, or 1051 * be in a RCU critical section to prevent freeing of @bio. 1052 */ 1053 int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags) 1054 { 1055 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 1056 blk_qc_t cookie = READ_ONCE(bio->bi_cookie); 1057 int ret; 1058 1059 if (cookie == BLK_QC_T_NONE || 1060 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) 1061 return 0; 1062 1063 if (current->plug) 1064 blk_flush_plug(current->plug, false); 1065 1066 if (blk_queue_enter(q, BLK_MQ_REQ_NOWAIT)) 1067 return 0; 1068 if (WARN_ON_ONCE(!queue_is_mq(q))) 1069 ret = 0; /* not yet implemented, should not happen */ 1070 else 1071 ret = blk_mq_poll(q, cookie, iob, flags); 1072 blk_queue_exit(q); 1073 return ret; 1074 } 1075 EXPORT_SYMBOL_GPL(bio_poll); 1076 1077 /* 1078 * Helper to implement file_operations.iopoll. Requires the bio to be stored 1079 * in iocb->private, and cleared before freeing the bio. 1080 */ 1081 int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob, 1082 unsigned int flags) 1083 { 1084 struct bio *bio; 1085 int ret = 0; 1086 1087 /* 1088 * Note: the bio cache only uses SLAB_TYPESAFE_BY_RCU, so bio can 1089 * point to a freshly allocated bio at this point. If that happens 1090 * we have a few cases to consider: 1091 * 1092 * 1) the bio is beeing initialized and bi_bdev is NULL. We can just 1093 * simply nothing in this case 1094 * 2) the bio points to a not poll enabled device. bio_poll will catch 1095 * this and return 0 1096 * 3) the bio points to a poll capable device, including but not 1097 * limited to the one that the original bio pointed to. In this 1098 * case we will call into the actual poll method and poll for I/O, 1099 * even if we don't need to, but it won't cause harm either. 1100 * 1101 * For cases 2) and 3) above the RCU grace period ensures that bi_bdev 1102 * is still allocated. Because partitions hold a reference to the whole 1103 * device bdev and thus disk, the disk is also still valid. Grabbing 1104 * a reference to the queue in bio_poll() ensures the hctxs and requests 1105 * are still valid as well. 1106 */ 1107 rcu_read_lock(); 1108 bio = READ_ONCE(kiocb->private); 1109 if (bio && bio->bi_bdev) 1110 ret = bio_poll(bio, iob, flags); 1111 rcu_read_unlock(); 1112 1113 return ret; 1114 } 1115 EXPORT_SYMBOL_GPL(iocb_bio_iopoll); 1116 1117 /** 1118 * blk_cloned_rq_check_limits - Helper function to check a cloned request 1119 * for the new queue limits 1120 * @q: the queue 1121 * @rq: the request being checked 1122 * 1123 * Description: 1124 * @rq may have been made based on weaker limitations of upper-level queues 1125 * in request stacking drivers, and it may violate the limitation of @q. 1126 * Since the block layer and the underlying device driver trust @rq 1127 * after it is inserted to @q, it should be checked against @q before 1128 * the insertion using this generic function. 1129 * 1130 * Request stacking drivers like request-based dm may change the queue 1131 * limits when retrying requests on other queues. Those requests need 1132 * to be checked against the new queue limits again during dispatch. 1133 */ 1134 static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q, 1135 struct request *rq) 1136 { 1137 unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq)); 1138 1139 if (blk_rq_sectors(rq) > max_sectors) { 1140 /* 1141 * SCSI device does not have a good way to return if 1142 * Write Same/Zero is actually supported. If a device rejects 1143 * a non-read/write command (discard, write same,etc.) the 1144 * low-level device driver will set the relevant queue limit to 1145 * 0 to prevent blk-lib from issuing more of the offending 1146 * operations. Commands queued prior to the queue limit being 1147 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O 1148 * errors being propagated to upper layers. 1149 */ 1150 if (max_sectors == 0) 1151 return BLK_STS_NOTSUPP; 1152 1153 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n", 1154 __func__, blk_rq_sectors(rq), max_sectors); 1155 return BLK_STS_IOERR; 1156 } 1157 1158 /* 1159 * The queue settings related to segment counting may differ from the 1160 * original queue. 1161 */ 1162 rq->nr_phys_segments = blk_recalc_rq_segments(rq); 1163 if (rq->nr_phys_segments > queue_max_segments(q)) { 1164 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n", 1165 __func__, rq->nr_phys_segments, queue_max_segments(q)); 1166 return BLK_STS_IOERR; 1167 } 1168 1169 return BLK_STS_OK; 1170 } 1171 1172 /** 1173 * blk_insert_cloned_request - Helper for stacking drivers to submit a request 1174 * @q: the queue to submit the request 1175 * @rq: the request being queued 1176 */ 1177 blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq) 1178 { 1179 blk_status_t ret; 1180 1181 ret = blk_cloned_rq_check_limits(q, rq); 1182 if (ret != BLK_STS_OK) 1183 return ret; 1184 1185 if (rq->rq_disk && 1186 should_fail_request(rq->rq_disk->part0, blk_rq_bytes(rq))) 1187 return BLK_STS_IOERR; 1188 1189 if (blk_crypto_insert_cloned_request(rq)) 1190 return BLK_STS_IOERR; 1191 1192 blk_account_io_start(rq); 1193 1194 /* 1195 * Since we have a scheduler attached on the top device, 1196 * bypass a potential scheduler on the bottom device for 1197 * insert. 1198 */ 1199 return blk_mq_request_issue_directly(rq, true); 1200 } 1201 EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 1202 1203 /** 1204 * blk_rq_err_bytes - determine number of bytes till the next failure boundary 1205 * @rq: request to examine 1206 * 1207 * Description: 1208 * A request could be merge of IOs which require different failure 1209 * handling. This function determines the number of bytes which 1210 * can be failed from the beginning of the request without 1211 * crossing into area which need to be retried further. 1212 * 1213 * Return: 1214 * The number of bytes to fail. 1215 */ 1216 unsigned int blk_rq_err_bytes(const struct request *rq) 1217 { 1218 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; 1219 unsigned int bytes = 0; 1220 struct bio *bio; 1221 1222 if (!(rq->rq_flags & RQF_MIXED_MERGE)) 1223 return blk_rq_bytes(rq); 1224 1225 /* 1226 * Currently the only 'mixing' which can happen is between 1227 * different fastfail types. We can safely fail portions 1228 * which have all the failfast bits that the first one has - 1229 * the ones which are at least as eager to fail as the first 1230 * one. 1231 */ 1232 for (bio = rq->bio; bio; bio = bio->bi_next) { 1233 if ((bio->bi_opf & ff) != ff) 1234 break; 1235 bytes += bio->bi_iter.bi_size; 1236 } 1237 1238 /* this could lead to infinite loop */ 1239 BUG_ON(blk_rq_bytes(rq) && !bytes); 1240 return bytes; 1241 } 1242 EXPORT_SYMBOL_GPL(blk_rq_err_bytes); 1243 1244 static void update_io_ticks(struct block_device *part, unsigned long now, 1245 bool end) 1246 { 1247 unsigned long stamp; 1248 again: 1249 stamp = READ_ONCE(part->bd_stamp); 1250 if (unlikely(time_after(now, stamp))) { 1251 if (likely(cmpxchg(&part->bd_stamp, stamp, now) == stamp)) 1252 __part_stat_add(part, io_ticks, end ? now - stamp : 1); 1253 } 1254 if (part->bd_partno) { 1255 part = bdev_whole(part); 1256 goto again; 1257 } 1258 } 1259 1260 void __blk_account_io_done(struct request *req, u64 now) 1261 { 1262 const int sgrp = op_stat_group(req_op(req)); 1263 1264 part_stat_lock(); 1265 update_io_ticks(req->part, jiffies, true); 1266 part_stat_inc(req->part, ios[sgrp]); 1267 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns); 1268 part_stat_unlock(); 1269 } 1270 1271 void __blk_account_io_start(struct request *rq) 1272 { 1273 /* passthrough requests can hold bios that do not have ->bi_bdev set */ 1274 if (rq->bio && rq->bio->bi_bdev) 1275 rq->part = rq->bio->bi_bdev; 1276 else 1277 rq->part = rq->rq_disk->part0; 1278 1279 part_stat_lock(); 1280 update_io_ticks(rq->part, jiffies, false); 1281 part_stat_unlock(); 1282 } 1283 1284 static unsigned long __part_start_io_acct(struct block_device *part, 1285 unsigned int sectors, unsigned int op) 1286 { 1287 const int sgrp = op_stat_group(op); 1288 unsigned long now = READ_ONCE(jiffies); 1289 1290 part_stat_lock(); 1291 update_io_ticks(part, now, false); 1292 part_stat_inc(part, ios[sgrp]); 1293 part_stat_add(part, sectors[sgrp], sectors); 1294 part_stat_local_inc(part, in_flight[op_is_write(op)]); 1295 part_stat_unlock(); 1296 1297 return now; 1298 } 1299 1300 /** 1301 * bio_start_io_acct - start I/O accounting for bio based drivers 1302 * @bio: bio to start account for 1303 * 1304 * Returns the start time that should be passed back to bio_end_io_acct(). 1305 */ 1306 unsigned long bio_start_io_acct(struct bio *bio) 1307 { 1308 return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio)); 1309 } 1310 EXPORT_SYMBOL_GPL(bio_start_io_acct); 1311 1312 unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, 1313 unsigned int op) 1314 { 1315 return __part_start_io_acct(disk->part0, sectors, op); 1316 } 1317 EXPORT_SYMBOL(disk_start_io_acct); 1318 1319 static void __part_end_io_acct(struct block_device *part, unsigned int op, 1320 unsigned long start_time) 1321 { 1322 const int sgrp = op_stat_group(op); 1323 unsigned long now = READ_ONCE(jiffies); 1324 unsigned long duration = now - start_time; 1325 1326 part_stat_lock(); 1327 update_io_ticks(part, now, true); 1328 part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration)); 1329 part_stat_local_dec(part, in_flight[op_is_write(op)]); 1330 part_stat_unlock(); 1331 } 1332 1333 void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, 1334 struct block_device *orig_bdev) 1335 { 1336 __part_end_io_acct(orig_bdev, bio_op(bio), start_time); 1337 } 1338 EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped); 1339 1340 void disk_end_io_acct(struct gendisk *disk, unsigned int op, 1341 unsigned long start_time) 1342 { 1343 __part_end_io_acct(disk->part0, op, start_time); 1344 } 1345 EXPORT_SYMBOL(disk_end_io_acct); 1346 1347 /* 1348 * Steal bios from a request and add them to a bio list. 1349 * The request must not have been partially completed before. 1350 */ 1351 void blk_steal_bios(struct bio_list *list, struct request *rq) 1352 { 1353 if (rq->bio) { 1354 if (list->tail) 1355 list->tail->bi_next = rq->bio; 1356 else 1357 list->head = rq->bio; 1358 list->tail = rq->biotail; 1359 1360 rq->bio = NULL; 1361 rq->biotail = NULL; 1362 } 1363 1364 rq->__data_len = 0; 1365 } 1366 EXPORT_SYMBOL_GPL(blk_steal_bios); 1367 1368 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1369 /** 1370 * rq_flush_dcache_pages - Helper function to flush all pages in a request 1371 * @rq: the request to be flushed 1372 * 1373 * Description: 1374 * Flush all pages in @rq. 1375 */ 1376 void rq_flush_dcache_pages(struct request *rq) 1377 { 1378 struct req_iterator iter; 1379 struct bio_vec bvec; 1380 1381 rq_for_each_segment(bvec, rq, iter) 1382 flush_dcache_page(bvec.bv_page); 1383 } 1384 EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); 1385 #endif 1386 1387 /** 1388 * blk_lld_busy - Check if underlying low-level drivers of a device are busy 1389 * @q : the queue of the device being checked 1390 * 1391 * Description: 1392 * Check if underlying low-level drivers of a device are busy. 1393 * If the drivers want to export their busy state, they must set own 1394 * exporting function using blk_queue_lld_busy() first. 1395 * 1396 * Basically, this function is used only by request stacking drivers 1397 * to stop dispatching requests to underlying devices when underlying 1398 * devices are busy. This behavior helps more I/O merging on the queue 1399 * of the request stacking driver and prevents I/O throughput regression 1400 * on burst I/O load. 1401 * 1402 * Return: 1403 * 0 - Not busy (The request stacking driver should dispatch request) 1404 * 1 - Busy (The request stacking driver should stop dispatching request) 1405 */ 1406 int blk_lld_busy(struct request_queue *q) 1407 { 1408 if (queue_is_mq(q) && q->mq_ops->busy) 1409 return q->mq_ops->busy(q); 1410 1411 return 0; 1412 } 1413 EXPORT_SYMBOL_GPL(blk_lld_busy); 1414 1415 /** 1416 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request 1417 * @rq: the clone request to be cleaned up 1418 * 1419 * Description: 1420 * Free all bios in @rq for a cloned request. 1421 */ 1422 void blk_rq_unprep_clone(struct request *rq) 1423 { 1424 struct bio *bio; 1425 1426 while ((bio = rq->bio) != NULL) { 1427 rq->bio = bio->bi_next; 1428 1429 bio_put(bio); 1430 } 1431 } 1432 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 1433 1434 /** 1435 * blk_rq_prep_clone - Helper function to setup clone request 1436 * @rq: the request to be setup 1437 * @rq_src: original request to be cloned 1438 * @bs: bio_set that bios for clone are allocated from 1439 * @gfp_mask: memory allocation mask for bio 1440 * @bio_ctr: setup function to be called for each clone bio. 1441 * Returns %0 for success, non %0 for failure. 1442 * @data: private data to be passed to @bio_ctr 1443 * 1444 * Description: 1445 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. 1446 * Also, pages which the original bios are pointing to are not copied 1447 * and the cloned bios just point same pages. 1448 * So cloned bios must be completed before original bios, which means 1449 * the caller must complete @rq before @rq_src. 1450 */ 1451 int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 1452 struct bio_set *bs, gfp_t gfp_mask, 1453 int (*bio_ctr)(struct bio *, struct bio *, void *), 1454 void *data) 1455 { 1456 struct bio *bio, *bio_src; 1457 1458 if (!bs) 1459 bs = &fs_bio_set; 1460 1461 __rq_for_each_bio(bio_src, rq_src) { 1462 bio = bio_clone_fast(bio_src, gfp_mask, bs); 1463 if (!bio) 1464 goto free_and_out; 1465 1466 if (bio_ctr && bio_ctr(bio, bio_src, data)) 1467 goto free_and_out; 1468 1469 if (rq->bio) { 1470 rq->biotail->bi_next = bio; 1471 rq->biotail = bio; 1472 } else { 1473 rq->bio = rq->biotail = bio; 1474 } 1475 bio = NULL; 1476 } 1477 1478 /* Copy attributes of the original request to the clone request. */ 1479 rq->__sector = blk_rq_pos(rq_src); 1480 rq->__data_len = blk_rq_bytes(rq_src); 1481 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) { 1482 rq->rq_flags |= RQF_SPECIAL_PAYLOAD; 1483 rq->special_vec = rq_src->special_vec; 1484 } 1485 rq->nr_phys_segments = rq_src->nr_phys_segments; 1486 rq->ioprio = rq_src->ioprio; 1487 1488 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) 1489 goto free_and_out; 1490 1491 return 0; 1492 1493 free_and_out: 1494 if (bio) 1495 bio_put(bio); 1496 blk_rq_unprep_clone(rq); 1497 1498 return -ENOMEM; 1499 } 1500 EXPORT_SYMBOL_GPL(blk_rq_prep_clone); 1501 1502 int kblockd_schedule_work(struct work_struct *work) 1503 { 1504 return queue_work(kblockd_workqueue, work); 1505 } 1506 EXPORT_SYMBOL(kblockd_schedule_work); 1507 1508 int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, 1509 unsigned long delay) 1510 { 1511 return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay); 1512 } 1513 EXPORT_SYMBOL(kblockd_mod_delayed_work_on); 1514 1515 void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios) 1516 { 1517 struct task_struct *tsk = current; 1518 1519 /* 1520 * If this is a nested plug, don't actually assign it. 1521 */ 1522 if (tsk->plug) 1523 return; 1524 1525 plug->mq_list = NULL; 1526 plug->cached_rq = NULL; 1527 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT); 1528 plug->rq_count = 0; 1529 plug->multiple_queues = false; 1530 plug->has_elevator = false; 1531 plug->nowait = false; 1532 INIT_LIST_HEAD(&plug->cb_list); 1533 1534 /* 1535 * Store ordering should not be needed here, since a potential 1536 * preempt will imply a full memory barrier 1537 */ 1538 tsk->plug = plug; 1539 } 1540 1541 /** 1542 * blk_start_plug - initialize blk_plug and track it inside the task_struct 1543 * @plug: The &struct blk_plug that needs to be initialized 1544 * 1545 * Description: 1546 * blk_start_plug() indicates to the block layer an intent by the caller 1547 * to submit multiple I/O requests in a batch. The block layer may use 1548 * this hint to defer submitting I/Os from the caller until blk_finish_plug() 1549 * is called. However, the block layer may choose to submit requests 1550 * before a call to blk_finish_plug() if the number of queued I/Os 1551 * exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than 1552 * %BLK_PLUG_FLUSH_SIZE. The queued I/Os may also be submitted early if 1553 * the task schedules (see below). 1554 * 1555 * Tracking blk_plug inside the task_struct will help with auto-flushing the 1556 * pending I/O should the task end up blocking between blk_start_plug() and 1557 * blk_finish_plug(). This is important from a performance perspective, but 1558 * also ensures that we don't deadlock. For instance, if the task is blocking 1559 * for a memory allocation, memory reclaim could end up wanting to free a 1560 * page belonging to that request that is currently residing in our private 1561 * plug. By flushing the pending I/O when the process goes to sleep, we avoid 1562 * this kind of deadlock. 1563 */ 1564 void blk_start_plug(struct blk_plug *plug) 1565 { 1566 blk_start_plug_nr_ios(plug, 1); 1567 } 1568 EXPORT_SYMBOL(blk_start_plug); 1569 1570 static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule) 1571 { 1572 LIST_HEAD(callbacks); 1573 1574 while (!list_empty(&plug->cb_list)) { 1575 list_splice_init(&plug->cb_list, &callbacks); 1576 1577 while (!list_empty(&callbacks)) { 1578 struct blk_plug_cb *cb = list_first_entry(&callbacks, 1579 struct blk_plug_cb, 1580 list); 1581 list_del(&cb->list); 1582 cb->callback(cb, from_schedule); 1583 } 1584 } 1585 } 1586 1587 struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data, 1588 int size) 1589 { 1590 struct blk_plug *plug = current->plug; 1591 struct blk_plug_cb *cb; 1592 1593 if (!plug) 1594 return NULL; 1595 1596 list_for_each_entry(cb, &plug->cb_list, list) 1597 if (cb->callback == unplug && cb->data == data) 1598 return cb; 1599 1600 /* Not currently on the callback list */ 1601 BUG_ON(size < sizeof(*cb)); 1602 cb = kzalloc(size, GFP_ATOMIC); 1603 if (cb) { 1604 cb->data = data; 1605 cb->callback = unplug; 1606 list_add(&cb->list, &plug->cb_list); 1607 } 1608 return cb; 1609 } 1610 EXPORT_SYMBOL(blk_check_plugged); 1611 1612 void blk_flush_plug(struct blk_plug *plug, bool from_schedule) 1613 { 1614 if (!list_empty(&plug->cb_list)) 1615 flush_plug_callbacks(plug, from_schedule); 1616 if (!rq_list_empty(plug->mq_list)) 1617 blk_mq_flush_plug_list(plug, from_schedule); 1618 if (unlikely(!from_schedule && plug->cached_rq)) 1619 blk_mq_free_plug_rqs(plug); 1620 } 1621 1622 /** 1623 * blk_finish_plug - mark the end of a batch of submitted I/O 1624 * @plug: The &struct blk_plug passed to blk_start_plug() 1625 * 1626 * Description: 1627 * Indicate that a batch of I/O submissions is complete. This function 1628 * must be paired with an initial call to blk_start_plug(). The intent 1629 * is to allow the block layer to optimize I/O submission. See the 1630 * documentation for blk_start_plug() for more information. 1631 */ 1632 void blk_finish_plug(struct blk_plug *plug) 1633 { 1634 if (plug == current->plug) { 1635 blk_flush_plug(plug, false); 1636 current->plug = NULL; 1637 } 1638 } 1639 EXPORT_SYMBOL(blk_finish_plug); 1640 1641 void blk_io_schedule(void) 1642 { 1643 /* Prevent hang_check timer from firing at us during very long I/O */ 1644 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2; 1645 1646 if (timeout) 1647 io_schedule_timeout(timeout); 1648 else 1649 io_schedule(); 1650 } 1651 EXPORT_SYMBOL_GPL(blk_io_schedule); 1652 1653 int __init blk_dev_init(void) 1654 { 1655 BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS)); 1656 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * 1657 sizeof_field(struct request, cmd_flags)); 1658 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 * 1659 sizeof_field(struct bio, bi_opf)); 1660 1661 /* used for unplugging and affects IO latency/throughput - HIGHPRI */ 1662 kblockd_workqueue = alloc_workqueue("kblockd", 1663 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1664 if (!kblockd_workqueue) 1665 panic("Failed to create kblockd\n"); 1666 1667 blk_requestq_cachep = kmem_cache_create("request_queue", 1668 sizeof(struct request_queue), 0, SLAB_PANIC, NULL); 1669 1670 blk_debugfs_root = debugfs_create_dir("block", NULL); 1671 1672 return 0; 1673 } 1674