1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block multiqueue core code 4 * 5 * Copyright (C) 2013-2014 Jens Axboe 6 * Copyright (C) 2013-2014 Christoph Hellwig 7 */ 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/backing-dev.h> 11 #include <linux/bio.h> 12 #include <linux/blkdev.h> 13 #include <linux/blk-integrity.h> 14 #include <linux/kmemleak.h> 15 #include <linux/mm.h> 16 #include <linux/init.h> 17 #include <linux/slab.h> 18 #include <linux/workqueue.h> 19 #include <linux/smp.h> 20 #include <linux/interrupt.h> 21 #include <linux/llist.h> 22 #include <linux/cpu.h> 23 #include <linux/cache.h> 24 #include <linux/sched/topology.h> 25 #include <linux/sched/signal.h> 26 #include <linux/suspend.h> 27 #include <linux/delay.h> 28 #include <linux/crash_dump.h> 29 #include <linux/prefetch.h> 30 #include <linux/blk-crypto.h> 31 #include <linux/part_stat.h> 32 #include <linux/sched/isolation.h> 33 34 #include <trace/events/block.h> 35 36 #include <linux/t10-pi.h> 37 #include "blk.h" 38 #include "blk-mq.h" 39 #include "blk-mq-debugfs.h" 40 #include "blk-pm.h" 41 #include "blk-stat.h" 42 #include "blk-mq-sched.h" 43 #include "blk-rq-qos.h" 44 45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); 46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd); 47 static DEFINE_MUTEX(blk_mq_cpuhp_lock); 48 49 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags); 50 static void blk_mq_request_bypass_insert(struct request *rq, 51 blk_insert_t flags); 52 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 53 struct list_head *list); 54 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, 55 struct io_comp_batch *iob, unsigned int flags); 56 57 /* 58 * Check if any of the ctx, dispatch list or elevator 59 * have pending work in this hardware queue. 60 */ 61 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 62 { 63 return !list_empty_careful(&hctx->dispatch) || 64 sbitmap_any_bit_set(&hctx->ctx_map) || 65 blk_mq_sched_has_work(hctx); 66 } 67 68 /* 69 * Mark this ctx as having pending work in this hardware queue 70 */ 71 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 72 struct blk_mq_ctx *ctx) 73 { 74 const int bit = ctx->index_hw[hctx->type]; 75 76 if (!sbitmap_test_bit(&hctx->ctx_map, bit)) 77 sbitmap_set_bit(&hctx->ctx_map, bit); 78 } 79 80 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 81 struct blk_mq_ctx *ctx) 82 { 83 const int bit = ctx->index_hw[hctx->type]; 84 85 sbitmap_clear_bit(&hctx->ctx_map, bit); 86 } 87 88 struct mq_inflight { 89 struct block_device *part; 90 unsigned int inflight[2]; 91 }; 92 93 static bool blk_mq_check_in_driver(struct request *rq, void *priv) 94 { 95 struct mq_inflight *mi = priv; 96 97 if (rq->rq_flags & RQF_IO_STAT && 98 (!bdev_is_partition(mi->part) || rq->part == mi->part) && 99 blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT) 100 mi->inflight[rq_data_dir(rq)]++; 101 102 return true; 103 } 104 105 void blk_mq_in_driver_rw(struct block_device *part, unsigned int inflight[2]) 106 { 107 struct mq_inflight mi = { .part = part }; 108 109 blk_mq_queue_tag_busy_iter(bdev_get_queue(part), blk_mq_check_in_driver, 110 &mi); 111 inflight[READ] = mi.inflight[READ]; 112 inflight[WRITE] = mi.inflight[WRITE]; 113 } 114 115 #ifdef CONFIG_LOCKDEP 116 static bool blk_freeze_set_owner(struct request_queue *q, 117 struct task_struct *owner) 118 { 119 if (!owner) 120 return false; 121 122 if (!q->mq_freeze_depth) { 123 q->mq_freeze_owner = owner; 124 q->mq_freeze_owner_depth = 1; 125 q->mq_freeze_disk_dead = !q->disk || 126 test_bit(GD_DEAD, &q->disk->state) || 127 !blk_queue_registered(q); 128 q->mq_freeze_queue_dying = blk_queue_dying(q); 129 return true; 130 } 131 132 if (owner == q->mq_freeze_owner) 133 q->mq_freeze_owner_depth += 1; 134 return false; 135 } 136 137 /* verify the last unfreeze in owner context */ 138 static bool blk_unfreeze_check_owner(struct request_queue *q) 139 { 140 if (q->mq_freeze_owner != current) 141 return false; 142 if (--q->mq_freeze_owner_depth == 0) { 143 q->mq_freeze_owner = NULL; 144 return true; 145 } 146 return false; 147 } 148 149 #else 150 151 static bool blk_freeze_set_owner(struct request_queue *q, 152 struct task_struct *owner) 153 { 154 return false; 155 } 156 157 static bool blk_unfreeze_check_owner(struct request_queue *q) 158 { 159 return false; 160 } 161 #endif 162 163 bool __blk_freeze_queue_start(struct request_queue *q, 164 struct task_struct *owner) 165 { 166 bool freeze; 167 168 mutex_lock(&q->mq_freeze_lock); 169 freeze = blk_freeze_set_owner(q, owner); 170 if (++q->mq_freeze_depth == 1) { 171 percpu_ref_kill(&q->q_usage_counter); 172 mutex_unlock(&q->mq_freeze_lock); 173 if (queue_is_mq(q)) 174 blk_mq_run_hw_queues(q, false); 175 } else { 176 mutex_unlock(&q->mq_freeze_lock); 177 } 178 179 return freeze; 180 } 181 182 void blk_freeze_queue_start(struct request_queue *q) 183 { 184 if (__blk_freeze_queue_start(q, current)) 185 blk_freeze_acquire_lock(q); 186 } 187 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 188 189 void blk_mq_freeze_queue_wait(struct request_queue *q) 190 { 191 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 192 } 193 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 194 195 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 196 unsigned long timeout) 197 { 198 return wait_event_timeout(q->mq_freeze_wq, 199 percpu_ref_is_zero(&q->q_usage_counter), 200 timeout); 201 } 202 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 203 204 void blk_mq_freeze_queue_nomemsave(struct request_queue *q) 205 { 206 blk_freeze_queue_start(q); 207 blk_mq_freeze_queue_wait(q); 208 } 209 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_nomemsave); 210 211 bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic) 212 { 213 bool unfreeze; 214 215 mutex_lock(&q->mq_freeze_lock); 216 if (force_atomic) 217 q->q_usage_counter.data->force_atomic = true; 218 q->mq_freeze_depth--; 219 WARN_ON_ONCE(q->mq_freeze_depth < 0); 220 if (!q->mq_freeze_depth) { 221 percpu_ref_resurrect(&q->q_usage_counter); 222 wake_up_all(&q->mq_freeze_wq); 223 } 224 unfreeze = blk_unfreeze_check_owner(q); 225 mutex_unlock(&q->mq_freeze_lock); 226 227 return unfreeze; 228 } 229 230 void blk_mq_unfreeze_queue_nomemrestore(struct request_queue *q) 231 { 232 if (__blk_mq_unfreeze_queue(q, false)) 233 blk_unfreeze_release_lock(q); 234 } 235 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_nomemrestore); 236 237 /* 238 * non_owner variant of blk_freeze_queue_start 239 * 240 * Unlike blk_freeze_queue_start, the queue doesn't need to be unfrozen 241 * by the same task. This is fragile and should not be used if at all 242 * possible. 243 */ 244 void blk_freeze_queue_start_non_owner(struct request_queue *q) 245 { 246 __blk_freeze_queue_start(q, NULL); 247 } 248 EXPORT_SYMBOL_GPL(blk_freeze_queue_start_non_owner); 249 250 /* non_owner variant of blk_mq_unfreeze_queue */ 251 void blk_mq_unfreeze_queue_non_owner(struct request_queue *q) 252 { 253 __blk_mq_unfreeze_queue(q, false); 254 } 255 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_non_owner); 256 257 /* 258 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 259 * mpt3sas driver such that this function can be removed. 260 */ 261 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 262 { 263 unsigned long flags; 264 265 spin_lock_irqsave(&q->queue_lock, flags); 266 if (!q->quiesce_depth++) 267 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 268 spin_unlock_irqrestore(&q->queue_lock, flags); 269 } 270 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 271 272 /** 273 * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done 274 * @set: tag_set to wait on 275 * 276 * Note: it is driver's responsibility for making sure that quiesce has 277 * been started on or more of the request_queues of the tag_set. This 278 * function only waits for the quiesce on those request_queues that had 279 * the quiesce flag set using blk_mq_quiesce_queue_nowait. 280 */ 281 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set) 282 { 283 if (set->flags & BLK_MQ_F_BLOCKING) 284 synchronize_srcu(set->srcu); 285 else 286 synchronize_rcu(); 287 } 288 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done); 289 290 /** 291 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 292 * @q: request queue. 293 * 294 * Note: this function does not prevent that the struct request end_io() 295 * callback function is invoked. Once this function is returned, we make 296 * sure no dispatch can happen until the queue is unquiesced via 297 * blk_mq_unquiesce_queue(). 298 */ 299 void blk_mq_quiesce_queue(struct request_queue *q) 300 { 301 blk_mq_quiesce_queue_nowait(q); 302 /* nothing to wait for non-mq queues */ 303 if (queue_is_mq(q)) 304 blk_mq_wait_quiesce_done(q->tag_set); 305 } 306 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 307 308 /* 309 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 310 * @q: request queue. 311 * 312 * This function recovers queue into the state before quiescing 313 * which is done by blk_mq_quiesce_queue. 314 */ 315 void blk_mq_unquiesce_queue(struct request_queue *q) 316 { 317 unsigned long flags; 318 bool run_queue = false; 319 320 spin_lock_irqsave(&q->queue_lock, flags); 321 if (WARN_ON_ONCE(q->quiesce_depth <= 0)) { 322 ; 323 } else if (!--q->quiesce_depth) { 324 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 325 run_queue = true; 326 } 327 spin_unlock_irqrestore(&q->queue_lock, flags); 328 329 /* dispatch requests which are inserted during quiescing */ 330 if (run_queue) 331 blk_mq_run_hw_queues(q, true); 332 } 333 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 334 335 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set) 336 { 337 struct request_queue *q; 338 339 rcu_read_lock(); 340 list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) { 341 if (!blk_queue_skip_tagset_quiesce(q)) 342 blk_mq_quiesce_queue_nowait(q); 343 } 344 rcu_read_unlock(); 345 346 blk_mq_wait_quiesce_done(set); 347 } 348 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset); 349 350 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set) 351 { 352 struct request_queue *q; 353 354 rcu_read_lock(); 355 list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) { 356 if (!blk_queue_skip_tagset_quiesce(q)) 357 blk_mq_unquiesce_queue(q); 358 } 359 rcu_read_unlock(); 360 } 361 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset); 362 363 void blk_mq_wake_waiters(struct request_queue *q) 364 { 365 struct blk_mq_hw_ctx *hctx; 366 unsigned long i; 367 368 queue_for_each_hw_ctx(q, hctx, i) 369 if (blk_mq_hw_queue_mapped(hctx)) 370 blk_mq_tag_wakeup_all(hctx->tags, true); 371 } 372 373 void blk_rq_init(struct request_queue *q, struct request *rq) 374 { 375 memset(rq, 0, sizeof(*rq)); 376 377 INIT_LIST_HEAD(&rq->queuelist); 378 rq->q = q; 379 rq->__sector = (sector_t) -1; 380 rq->phys_gap_bit = 0; 381 INIT_HLIST_NODE(&rq->hash); 382 RB_CLEAR_NODE(&rq->rb_node); 383 rq->tag = BLK_MQ_NO_TAG; 384 rq->internal_tag = BLK_MQ_NO_TAG; 385 rq->start_time_ns = blk_time_get_ns(); 386 blk_crypto_rq_set_defaults(rq); 387 } 388 EXPORT_SYMBOL(blk_rq_init); 389 390 /* Set start and alloc time when the allocated request is actually used */ 391 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns) 392 { 393 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 394 if (blk_queue_rq_alloc_time(rq->q)) 395 rq->alloc_time_ns = alloc_time_ns; 396 else 397 rq->alloc_time_ns = 0; 398 #endif 399 } 400 401 static inline void blk_mq_bio_issue_init(struct request_queue *q, 402 struct bio *bio) 403 { 404 #ifdef CONFIG_BLK_CGROUP 405 if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags)) 406 bio->issue_time_ns = blk_time_get_ns(); 407 #endif 408 } 409 410 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 411 struct blk_mq_tags *tags, unsigned int tag) 412 { 413 struct blk_mq_ctx *ctx = data->ctx; 414 struct blk_mq_hw_ctx *hctx = data->hctx; 415 struct request_queue *q = data->q; 416 struct request *rq = tags->static_rqs[tag]; 417 418 rq->q = q; 419 rq->mq_ctx = ctx; 420 rq->mq_hctx = hctx; 421 rq->cmd_flags = data->cmd_flags; 422 423 if (data->flags & BLK_MQ_REQ_PM) 424 data->rq_flags |= RQF_PM; 425 rq->rq_flags = data->rq_flags; 426 427 if (data->rq_flags & RQF_SCHED_TAGS) { 428 rq->tag = BLK_MQ_NO_TAG; 429 rq->internal_tag = tag; 430 } else { 431 rq->tag = tag; 432 rq->internal_tag = BLK_MQ_NO_TAG; 433 } 434 rq->timeout = 0; 435 436 rq->part = NULL; 437 rq->io_start_time_ns = 0; 438 rq->stats_sectors = 0; 439 rq->nr_phys_segments = 0; 440 rq->nr_integrity_segments = 0; 441 rq->end_io = NULL; 442 rq->end_io_data = NULL; 443 444 blk_crypto_rq_set_defaults(rq); 445 INIT_LIST_HEAD(&rq->queuelist); 446 /* tag was already set */ 447 WRITE_ONCE(rq->deadline, 0); 448 req_ref_set(rq, 1); 449 450 return rq; 451 } 452 453 static inline struct request * 454 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) 455 { 456 unsigned int tag, tag_offset; 457 struct blk_mq_tags *tags; 458 struct request *rq; 459 unsigned long tag_mask; 460 int i, nr = 0; 461 462 do { 463 tag_mask = blk_mq_get_tags(data, data->nr_tags - nr, &tag_offset); 464 if (unlikely(!tag_mask)) { 465 if (nr == 0) 466 return NULL; 467 break; 468 } 469 tags = blk_mq_tags_from_data(data); 470 for (i = 0; tag_mask; i++) { 471 if (!(tag_mask & (1UL << i))) 472 continue; 473 tag = tag_offset + i; 474 prefetch(tags->static_rqs[tag]); 475 tag_mask &= ~(1UL << i); 476 rq = blk_mq_rq_ctx_init(data, tags, tag); 477 rq_list_add_head(data->cached_rqs, rq); 478 nr++; 479 } 480 } while (data->nr_tags > nr); 481 482 if (!(data->rq_flags & RQF_SCHED_TAGS)) 483 blk_mq_add_active_requests(data->hctx, nr); 484 /* caller already holds a reference, add for remainder */ 485 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1); 486 data->nr_tags -= nr; 487 488 return rq_list_pop(data->cached_rqs); 489 } 490 491 static bool blk_op_bypass_sched(blk_opf_t opf) 492 { 493 return (opf & REQ_OP_MASK) == REQ_OP_FLUSH || 494 blk_op_is_passthrough(opf); 495 } 496 497 static void blk_mq_limit_depth(struct blk_mq_alloc_data *data) 498 { 499 struct elevator_mq_ops *ops; 500 501 /* If no I/O scheduler has been configured, don't limit requests */ 502 if (!data->q->elevator) { 503 blk_mq_tag_busy(data->hctx); 504 return; 505 } 506 507 /* 508 * All requests use scheduler tags when an I/O scheduler is 509 * enabled for the queue. 510 */ 511 data->rq_flags |= RQF_SCHED_TAGS; 512 513 /* 514 * Flush/passthrough requests are special and go directly to the 515 * dispatch list, they are not subject to the async_depth limit. 516 */ 517 if (blk_op_bypass_sched(data->cmd_flags)) 518 return; 519 520 WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); 521 522 /* 523 * By default, sync requests have no limit, and async requests are 524 * limited to async_depth. 525 */ 526 ops = &data->q->elevator->type->ops; 527 if (ops->limit_depth) 528 ops->limit_depth(data->cmd_flags, data); 529 } 530 531 /* 532 * Finish initializing a request once it has been claimed for an operation. 533 * Cached requests are allocated before that operation is known. 534 */ 535 static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns) 536 { 537 struct elevator_queue *e; 538 539 blk_mq_rq_time_init(rq, alloc_time_ns); 540 541 if (!(rq->rq_flags & RQF_SCHED_TAGS) || (rq->rq_flags & RQF_RESV) || 542 blk_op_bypass_sched(rq->cmd_flags)) 543 return; 544 545 rq->rq_flags |= RQF_USE_SCHED; 546 INIT_HLIST_NODE(&rq->hash); 547 RB_CLEAR_NODE(&rq->rb_node); 548 549 e = rq->q->elevator; 550 if (e->type->ops.prepare_request) 551 e->type->ops.prepare_request(rq); 552 } 553 554 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) 555 { 556 struct request_queue *q = data->q; 557 u64 alloc_time_ns = 0; 558 struct request *rq; 559 unsigned int tag; 560 561 /* alloc_time includes depth and tag waits */ 562 if (blk_queue_rq_alloc_time(q)) 563 alloc_time_ns = blk_time_get_ns(); 564 565 if (data->cmd_flags & REQ_NOWAIT) 566 data->flags |= BLK_MQ_REQ_NOWAIT; 567 568 retry: 569 data->ctx = blk_mq_get_ctx(q); 570 data->hctx = blk_mq_map_queue(data->cmd_flags, data->ctx); 571 572 blk_mq_limit_depth(data); 573 if (data->flags & BLK_MQ_REQ_RESERVED) 574 data->rq_flags |= RQF_RESV; 575 576 /* 577 * Try batched alloc if we want more than 1 tag. 578 */ 579 if (data->nr_tags > 1) { 580 rq = __blk_mq_alloc_requests_batch(data); 581 if (rq) { 582 blk_mq_rq_late_init(rq, alloc_time_ns); 583 return rq; 584 } 585 data->nr_tags = 1; 586 } 587 588 /* 589 * Waiting allocations only fail because of an inactive hctx. In that 590 * case just retry the hctx assignment and tag allocation as CPU hotplug 591 * should have migrated us to an online CPU by now. 592 */ 593 tag = blk_mq_get_tag(data); 594 if (tag == BLK_MQ_NO_TAG) { 595 if (data->flags & BLK_MQ_REQ_NOWAIT) 596 return NULL; 597 /* 598 * Give up the CPU and sleep for a random short time to 599 * ensure that thread using a realtime scheduling class 600 * are migrated off the CPU, and thus off the hctx that 601 * is going away. 602 */ 603 msleep(3); 604 goto retry; 605 } 606 607 if (!(data->rq_flags & RQF_SCHED_TAGS)) 608 blk_mq_inc_active_requests(data->hctx); 609 rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag); 610 blk_mq_rq_late_init(rq, alloc_time_ns); 611 return rq; 612 } 613 614 static struct request *blk_mq_rq_cache_fill(struct request_queue *q, 615 struct blk_plug *plug, 616 blk_opf_t opf, 617 blk_mq_req_flags_t flags) 618 { 619 struct blk_mq_alloc_data data = { 620 .q = q, 621 .flags = flags, 622 .shallow_depth = 0, 623 .cmd_flags = opf, 624 .rq_flags = 0, 625 .nr_tags = plug->nr_ios, 626 .cached_rqs = &plug->cached_rqs, 627 .ctx = NULL, 628 .hctx = NULL 629 }; 630 struct request *rq; 631 632 if (blk_queue_enter(q, flags)) 633 return NULL; 634 635 plug->nr_ios = 1; 636 637 rq = __blk_mq_alloc_requests(&data); 638 if (unlikely(!rq)) 639 blk_queue_exit(q); 640 return rq; 641 } 642 643 static struct request *blk_mq_alloc_cached_request(struct request_queue *q, 644 blk_opf_t opf, 645 blk_mq_req_flags_t flags) 646 { 647 struct blk_plug *plug = current->plug; 648 struct request *rq; 649 650 if (!plug) 651 return NULL; 652 653 if (rq_list_empty(&plug->cached_rqs)) { 654 if (plug->nr_ios == 1) 655 return NULL; 656 rq = blk_mq_rq_cache_fill(q, plug, opf, flags); 657 } else { 658 rq = rq_list_peek(&plug->cached_rqs); 659 if (!rq || rq->q != q) 660 return NULL; 661 662 if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type) 663 return NULL; 664 665 rq_list_pop(&plug->cached_rqs); 666 rq->cmd_flags = opf; 667 INIT_LIST_HEAD(&rq->queuelist); 668 blk_mq_rq_late_init(rq, blk_time_get_ns()); 669 } 670 671 return rq; 672 } 673 674 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, 675 blk_mq_req_flags_t flags) 676 { 677 struct request *rq; 678 679 rq = blk_mq_alloc_cached_request(q, opf, flags); 680 if (!rq) { 681 struct blk_mq_alloc_data data = { 682 .q = q, 683 .flags = flags, 684 .shallow_depth = 0, 685 .cmd_flags = opf, 686 .rq_flags = 0, 687 .nr_tags = 1, 688 .cached_rqs = NULL, 689 .ctx = NULL, 690 .hctx = NULL 691 }; 692 int ret; 693 694 ret = blk_queue_enter(q, flags); 695 if (ret) 696 return ERR_PTR(ret); 697 698 rq = __blk_mq_alloc_requests(&data); 699 if (!rq) 700 goto out_queue_exit; 701 } 702 rq->__data_len = 0; 703 rq->phys_gap_bit = 0; 704 rq->__sector = (sector_t) -1; 705 rq->bio = rq->biotail = NULL; 706 return rq; 707 out_queue_exit: 708 blk_queue_exit(q); 709 return ERR_PTR(-EWOULDBLOCK); 710 } 711 EXPORT_SYMBOL(blk_mq_alloc_request); 712 713 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 714 blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx) 715 { 716 struct blk_mq_alloc_data data = { 717 .q = q, 718 .flags = flags, 719 .shallow_depth = 0, 720 .cmd_flags = opf, 721 .rq_flags = 0, 722 .nr_tags = 1, 723 .cached_rqs = NULL, 724 .ctx = NULL, 725 .hctx = NULL 726 }; 727 u64 alloc_time_ns = 0; 728 struct request *rq; 729 unsigned int cpu; 730 unsigned int tag; 731 int ret; 732 733 /* alloc_time includes depth and tag waits */ 734 if (blk_queue_rq_alloc_time(q)) 735 alloc_time_ns = blk_time_get_ns(); 736 737 /* 738 * If the tag allocator sleeps we could get an allocation for a 739 * different hardware context. No need to complicate the low level 740 * allocator for this for the rare use case of a command tied to 741 * a specific queue. 742 */ 743 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) || 744 WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED))) 745 return ERR_PTR(-EINVAL); 746 747 if (hctx_idx >= q->nr_hw_queues) 748 return ERR_PTR(-EIO); 749 750 ret = blk_queue_enter(q, flags); 751 if (ret) 752 return ERR_PTR(ret); 753 754 /* 755 * Check if the hardware context is actually mapped to anything. 756 * If not tell the caller that it should skip this queue. 757 */ 758 ret = -EXDEV; 759 data.hctx = q->queue_hw_ctx[hctx_idx]; 760 if (!blk_mq_hw_queue_mapped(data.hctx)) 761 goto out_queue_exit; 762 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask); 763 if (cpu >= nr_cpu_ids) 764 goto out_queue_exit; 765 data.ctx = __blk_mq_get_ctx(q, cpu); 766 767 if (q->elevator) 768 data.rq_flags |= RQF_SCHED_TAGS; 769 else 770 blk_mq_tag_busy(data.hctx); 771 772 if (flags & BLK_MQ_REQ_RESERVED) 773 data.rq_flags |= RQF_RESV; 774 775 ret = -EWOULDBLOCK; 776 tag = blk_mq_get_tag(&data); 777 if (tag == BLK_MQ_NO_TAG) 778 goto out_queue_exit; 779 if (!(data.rq_flags & RQF_SCHED_TAGS)) 780 blk_mq_inc_active_requests(data.hctx); 781 rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag); 782 blk_mq_rq_late_init(rq, alloc_time_ns); 783 rq->__data_len = 0; 784 rq->phys_gap_bit = 0; 785 rq->__sector = (sector_t) -1; 786 rq->bio = rq->biotail = NULL; 787 return rq; 788 789 out_queue_exit: 790 blk_queue_exit(q); 791 return ERR_PTR(ret); 792 } 793 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 794 795 static void blk_mq_finish_request(struct request *rq) 796 { 797 struct request_queue *q = rq->q; 798 799 blk_zone_finish_request(rq); 800 801 if (rq->rq_flags & RQF_USE_SCHED) { 802 q->elevator->type->ops.finish_request(rq); 803 /* 804 * For postflush request that may need to be 805 * completed twice, we should clear this flag 806 * to avoid double finish_request() on the rq. 807 */ 808 rq->rq_flags &= ~RQF_USE_SCHED; 809 } 810 } 811 812 static void __blk_mq_free_request(struct request *rq) 813 { 814 struct request_queue *q = rq->q; 815 struct blk_mq_ctx *ctx = rq->mq_ctx; 816 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 817 const int sched_tag = rq->internal_tag; 818 819 blk_crypto_free_request(rq); 820 blk_pm_mark_last_busy(rq); 821 rq->mq_hctx = NULL; 822 823 if (rq->tag != BLK_MQ_NO_TAG) { 824 blk_mq_dec_active_requests(hctx); 825 blk_mq_put_tag(hctx->tags, ctx, rq->tag); 826 } 827 if (sched_tag != BLK_MQ_NO_TAG) 828 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag); 829 blk_mq_sched_restart(hctx); 830 blk_queue_exit(q); 831 } 832 833 void blk_mq_free_request(struct request *rq) 834 { 835 struct request_queue *q = rq->q; 836 837 blk_mq_finish_request(rq); 838 839 rq_qos_done(q, rq); 840 841 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 842 if (req_ref_put_and_test(rq)) 843 __blk_mq_free_request(rq); 844 } 845 EXPORT_SYMBOL_GPL(blk_mq_free_request); 846 847 void blk_mq_free_plug_rqs(struct blk_plug *plug) 848 { 849 struct request *rq; 850 851 while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL) 852 blk_mq_free_request(rq); 853 } 854 855 void blk_dump_rq_flags(struct request *rq, char *msg) 856 { 857 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg, 858 rq->q->disk ? rq->q->disk->disk_name : "?", 859 (__force unsigned long long) rq->cmd_flags); 860 861 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", 862 (unsigned long long)blk_rq_pos(rq), 863 blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); 864 printk(KERN_INFO " bio %p, biotail %p, len %u\n", 865 rq->bio, rq->biotail, blk_rq_bytes(rq)); 866 } 867 EXPORT_SYMBOL(blk_dump_rq_flags); 868 869 static void blk_account_io_completion(struct request *req, unsigned int bytes) 870 { 871 if (req->rq_flags & RQF_IO_STAT) { 872 const int sgrp = op_stat_group(req_op(req)); 873 874 part_stat_lock(); 875 part_stat_add(req->part, sectors[sgrp], bytes >> 9); 876 part_stat_unlock(); 877 } 878 } 879 880 static void blk_print_req_error(struct request *req, blk_status_t status) 881 { 882 printk_ratelimited(KERN_ERR 883 "%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x " 884 "phys_seg %u prio class %u\n", 885 blk_status_to_str(status), 886 req->q->disk ? req->q->disk->disk_name : "?", 887 blk_rq_pos(req), (__force u32)req_op(req), 888 blk_op_str(req_op(req)), 889 (__force u32)(req->cmd_flags & ~REQ_OP_MASK), 890 req->nr_phys_segments, 891 IOPRIO_PRIO_CLASS(req_get_ioprio(req))); 892 } 893 894 /* 895 * Fully end IO on a request. Does not support partial completions, or 896 * errors. 897 */ 898 static void blk_complete_request(struct request *req) 899 { 900 const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0; 901 int total_bytes = blk_rq_bytes(req); 902 struct bio *bio = req->bio; 903 904 trace_block_rq_complete(req, BLK_STS_OK, total_bytes); 905 906 if (!bio) 907 return; 908 909 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ) 910 blk_integrity_complete(req, total_bytes); 911 912 /* 913 * Upper layers may call blk_crypto_evict_key() anytime after the last 914 * bio_endio(). Therefore, the keyslot must be released before that. 915 */ 916 blk_crypto_rq_put_keyslot(req); 917 918 blk_account_io_completion(req, total_bytes); 919 920 do { 921 struct bio *next = bio->bi_next; 922 923 /* Completion has already been traced */ 924 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 925 926 if (blk_req_bio_is_zone_append(req, bio)) 927 blk_zone_append_update_request_bio(req, bio); 928 929 if (!is_flush) 930 bio_endio(bio); 931 bio = next; 932 } while (bio); 933 934 /* 935 * Reset counters so that the request stacking driver 936 * can find how many bytes remain in the request 937 * later. 938 */ 939 if (!req->end_io) { 940 req->bio = NULL; 941 req->__data_len = 0; 942 } 943 } 944 945 /** 946 * blk_update_request - Complete multiple bytes without completing the request 947 * @req: the request being processed 948 * @error: block status code 949 * @nr_bytes: number of bytes to complete for @req 950 * 951 * Description: 952 * Ends I/O on a number of bytes attached to @req, but doesn't complete 953 * the request structure even if @req doesn't have leftover. 954 * If @req has leftover, sets it up for the next range of segments. 955 * 956 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees 957 * %false return from this function. 958 * 959 * Note: 960 * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function 961 * except in the consistency check at the end of this function. 962 * 963 * Return: 964 * %false - this request doesn't have any more data 965 * %true - this request has more data 966 **/ 967 bool blk_update_request(struct request *req, blk_status_t error, 968 unsigned int nr_bytes) 969 { 970 bool is_flush = req->rq_flags & RQF_FLUSH_SEQ; 971 bool quiet = req->rq_flags & RQF_QUIET; 972 int total_bytes; 973 974 trace_block_rq_complete(req, error, nr_bytes); 975 976 if (!req->bio) 977 return false; 978 979 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ && 980 error == BLK_STS_OK) 981 blk_integrity_complete(req, nr_bytes); 982 983 /* 984 * Upper layers may call blk_crypto_evict_key() anytime after the last 985 * bio_endio(). Therefore, the keyslot must be released before that. 986 */ 987 if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req)) 988 __blk_crypto_rq_put_keyslot(req); 989 990 if (unlikely(error && !blk_rq_is_passthrough(req) && !quiet) && 991 !test_bit(GD_DEAD, &req->q->disk->state)) { 992 blk_print_req_error(req, error); 993 trace_block_rq_error(req, error, nr_bytes); 994 } 995 996 blk_account_io_completion(req, nr_bytes); 997 998 total_bytes = 0; 999 while (req->bio) { 1000 struct bio *bio = req->bio; 1001 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); 1002 1003 if (unlikely(error)) 1004 bio->bi_status = error; 1005 1006 if (bio_bytes == bio->bi_iter.bi_size) { 1007 req->bio = bio->bi_next; 1008 } else if (bio_is_zone_append(bio) && error == BLK_STS_OK) { 1009 /* 1010 * Partial zone append completions cannot be supported 1011 * as the BIO fragments may end up not being written 1012 * sequentially. 1013 */ 1014 bio->bi_status = BLK_STS_IOERR; 1015 } 1016 1017 /* Completion has already been traced */ 1018 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 1019 if (unlikely(quiet)) 1020 bio_set_flag(bio, BIO_QUIET); 1021 1022 bio_advance(bio, bio_bytes); 1023 1024 /* Don't actually finish bio if it's part of flush sequence */ 1025 if (!bio->bi_iter.bi_size) { 1026 if (blk_req_bio_is_zone_append(req, bio)) 1027 blk_zone_append_update_request_bio(req, bio); 1028 if (!is_flush) 1029 bio_endio(bio); 1030 } 1031 1032 total_bytes += bio_bytes; 1033 nr_bytes -= bio_bytes; 1034 1035 if (!nr_bytes) 1036 break; 1037 } 1038 1039 /* 1040 * completely done 1041 */ 1042 if (!req->bio) { 1043 /* 1044 * Reset counters so that the request stacking driver 1045 * can find how many bytes remain in the request 1046 * later. 1047 */ 1048 req->__data_len = 0; 1049 return false; 1050 } 1051 1052 req->__data_len -= total_bytes; 1053 1054 /* update sector only for requests with clear definition of sector */ 1055 if (!blk_rq_is_passthrough(req)) 1056 req->__sector += total_bytes >> 9; 1057 1058 /* mixed attributes always follow the first bio */ 1059 if (req->rq_flags & RQF_MIXED_MERGE) { 1060 req->cmd_flags &= ~REQ_FAILFAST_MASK; 1061 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; 1062 } 1063 1064 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) { 1065 /* 1066 * If total number of sectors is less than the first segment 1067 * size, something has gone terribly wrong. 1068 */ 1069 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { 1070 blk_dump_rq_flags(req, "request botched"); 1071 req->__data_len = blk_rq_cur_bytes(req); 1072 } 1073 1074 /* recalculate the number of segments */ 1075 req->nr_phys_segments = blk_recalc_rq_segments(req); 1076 } 1077 1078 return true; 1079 } 1080 EXPORT_SYMBOL_GPL(blk_update_request); 1081 1082 static inline void blk_account_io_done(struct request *req, u64 now) 1083 { 1084 trace_block_io_done(req); 1085 1086 /* 1087 * Account IO completion. flush_rq isn't accounted as a 1088 * normal IO on queueing nor completion. Accounting the 1089 * containing request is enough. 1090 */ 1091 if ((req->rq_flags & (RQF_IO_STAT|RQF_FLUSH_SEQ)) == RQF_IO_STAT) { 1092 const int sgrp = op_stat_group(req_op(req)); 1093 1094 part_stat_lock(); 1095 update_io_ticks(req->part, jiffies, true); 1096 part_stat_inc(req->part, ios[sgrp]); 1097 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns); 1098 bdev_dec_in_flight(req->part, req_op(req)); 1099 part_stat_unlock(); 1100 } 1101 } 1102 1103 static inline void blk_account_io_start(struct request *req) 1104 { 1105 trace_block_io_start(req); 1106 1107 if (!blk_queue_io_stat(req->q)) 1108 return; 1109 if (blk_rq_is_passthrough(req) && !blk_rq_passthrough_stats(req, req->q)) 1110 return; 1111 1112 req->rq_flags |= RQF_IO_STAT; 1113 req->start_time_ns = blk_time_get_ns(); 1114 1115 /* 1116 * All non-passthrough requests are created from a bio with one 1117 * exception: when a flush command that is part of a flush sequence 1118 * generated by the state machine in blk-flush.c is cloned onto the 1119 * lower device by dm-multipath we can get here without a bio. 1120 */ 1121 if (req->bio) 1122 req->part = req->bio->bi_bdev; 1123 else 1124 req->part = req->q->disk->part0; 1125 1126 part_stat_lock(); 1127 update_io_ticks(req->part, jiffies, false); 1128 bdev_inc_in_flight(req->part, req_op(req)); 1129 part_stat_unlock(); 1130 } 1131 1132 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) 1133 { 1134 if (rq->rq_flags & RQF_STATS) 1135 blk_stat_add(rq, now); 1136 1137 blk_mq_sched_completed_request(rq, now); 1138 blk_account_io_done(rq, now); 1139 } 1140 1141 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 1142 { 1143 if (blk_mq_need_time_stamp(rq)) 1144 __blk_mq_end_request_acct(rq, blk_time_get_ns()); 1145 1146 blk_mq_finish_request(rq); 1147 1148 if (rq->end_io) { 1149 rq_qos_done(rq->q, rq); 1150 if (rq->end_io(rq, error, NULL) == RQ_END_IO_FREE) 1151 blk_mq_free_request(rq); 1152 } else { 1153 blk_mq_free_request(rq); 1154 } 1155 } 1156 EXPORT_SYMBOL(__blk_mq_end_request); 1157 1158 void blk_mq_end_request(struct request *rq, blk_status_t error) 1159 { 1160 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 1161 BUG(); 1162 __blk_mq_end_request(rq, error); 1163 } 1164 EXPORT_SYMBOL(blk_mq_end_request); 1165 1166 #define TAG_COMP_BATCH 32 1167 1168 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx, 1169 int *tag_array, int nr_tags) 1170 { 1171 struct request_queue *q = hctx->queue; 1172 1173 blk_mq_sub_active_requests(hctx, nr_tags); 1174 1175 blk_mq_put_tags(hctx->tags, tag_array, nr_tags); 1176 percpu_ref_put_many(&q->q_usage_counter, nr_tags); 1177 } 1178 1179 void blk_mq_end_request_batch(struct io_comp_batch *iob) 1180 { 1181 int tags[TAG_COMP_BATCH], nr_tags = 0; 1182 struct blk_mq_hw_ctx *cur_hctx = NULL; 1183 struct request *rq; 1184 u64 now = 0; 1185 1186 if (iob->need_ts) 1187 now = blk_time_get_ns(); 1188 1189 while ((rq = rq_list_pop(&iob->req_list)) != NULL) { 1190 prefetch(rq->bio); 1191 prefetch(rq->rq_next); 1192 1193 blk_complete_request(rq); 1194 if (iob->need_ts) 1195 __blk_mq_end_request_acct(rq, now); 1196 1197 blk_mq_finish_request(rq); 1198 1199 rq_qos_done(rq->q, rq); 1200 1201 /* 1202 * If end_io handler returns NONE, then it still has 1203 * ownership of the request. 1204 */ 1205 if (rq->end_io && rq->end_io(rq, 0, iob) == RQ_END_IO_NONE) 1206 continue; 1207 1208 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 1209 if (!req_ref_put_and_test(rq)) 1210 continue; 1211 1212 blk_crypto_free_request(rq); 1213 blk_pm_mark_last_busy(rq); 1214 1215 if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) { 1216 if (cur_hctx) 1217 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags); 1218 nr_tags = 0; 1219 cur_hctx = rq->mq_hctx; 1220 } 1221 tags[nr_tags++] = rq->tag; 1222 } 1223 1224 if (nr_tags) 1225 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags); 1226 } 1227 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch); 1228 1229 static void blk_complete_reqs(struct llist_head *list) 1230 { 1231 struct llist_node *entry = llist_reverse_order(llist_del_all(list)); 1232 struct request *rq, *next; 1233 1234 llist_for_each_entry_safe(rq, next, entry, ipi_list) 1235 rq->q->mq_ops->complete(rq); 1236 } 1237 1238 static __latent_entropy void blk_done_softirq(void) 1239 { 1240 blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); 1241 } 1242 1243 static int blk_softirq_cpu_dead(unsigned int cpu) 1244 { 1245 blk_complete_reqs(&per_cpu(blk_cpu_done, cpu)); 1246 return 0; 1247 } 1248 1249 static void __blk_mq_complete_request_remote(void *data) 1250 { 1251 __raise_softirq_irqoff(BLOCK_SOFTIRQ); 1252 } 1253 1254 static inline bool blk_mq_complete_need_ipi(struct request *rq) 1255 { 1256 int cpu = raw_smp_processor_id(); 1257 1258 if (!IS_ENABLED(CONFIG_SMP) || 1259 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) 1260 return false; 1261 /* 1262 * With force threaded interrupts enabled, raising softirq from an SMP 1263 * function call will always result in waking the ksoftirqd thread. 1264 * This is probably worse than completing the request on a different 1265 * cache domain. 1266 */ 1267 if (force_irqthreads()) 1268 return false; 1269 1270 /* same CPU or cache domain and capacity? Complete locally */ 1271 if (cpu == rq->mq_ctx->cpu || 1272 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) && 1273 cpus_share_cache(cpu, rq->mq_ctx->cpu) && 1274 cpus_equal_capacity(cpu, rq->mq_ctx->cpu))) 1275 return false; 1276 1277 /* don't try to IPI to an offline CPU */ 1278 return cpu_online(rq->mq_ctx->cpu); 1279 } 1280 1281 static void blk_mq_complete_send_ipi(struct request *rq) 1282 { 1283 unsigned int cpu; 1284 1285 cpu = rq->mq_ctx->cpu; 1286 if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu))) 1287 smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu)); 1288 } 1289 1290 static void blk_mq_raise_softirq(struct request *rq) 1291 { 1292 struct llist_head *list; 1293 1294 preempt_disable(); 1295 list = this_cpu_ptr(&blk_cpu_done); 1296 if (llist_add(&rq->ipi_list, list)) 1297 raise_softirq(BLOCK_SOFTIRQ); 1298 preempt_enable(); 1299 } 1300 1301 bool blk_mq_complete_request_remote(struct request *rq) 1302 { 1303 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); 1304 1305 /* 1306 * For request which hctx has only one ctx mapping, 1307 * or a polled request, always complete locally, 1308 * it's pointless to redirect the completion. 1309 */ 1310 if ((rq->mq_hctx->nr_ctx == 1 && 1311 rq->mq_ctx->cpu == raw_smp_processor_id()) || 1312 rq->cmd_flags & REQ_POLLED) 1313 return false; 1314 1315 if (blk_mq_complete_need_ipi(rq)) { 1316 blk_mq_complete_send_ipi(rq); 1317 return true; 1318 } 1319 1320 if (rq->q->nr_hw_queues == 1) { 1321 blk_mq_raise_softirq(rq); 1322 return true; 1323 } 1324 return false; 1325 } 1326 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); 1327 1328 /** 1329 * blk_mq_complete_request - end I/O on a request 1330 * @rq: the request being processed 1331 * 1332 * Description: 1333 * Complete a request by scheduling the ->complete_rq operation. 1334 **/ 1335 void blk_mq_complete_request(struct request *rq) 1336 { 1337 if (!blk_mq_complete_request_remote(rq)) 1338 rq->q->mq_ops->complete(rq); 1339 } 1340 EXPORT_SYMBOL(blk_mq_complete_request); 1341 1342 /** 1343 * blk_mq_start_request - Start processing a request 1344 * @rq: Pointer to request to be started 1345 * 1346 * Function used by device drivers to notify the block layer that a request 1347 * is going to be processed now, so blk layer can do proper initializations 1348 * such as starting the timeout timer. 1349 */ 1350 void blk_mq_start_request(struct request *rq) 1351 { 1352 struct request_queue *q = rq->q; 1353 1354 trace_block_rq_issue(rq); 1355 1356 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags) && 1357 !blk_rq_is_passthrough(rq)) { 1358 rq->io_start_time_ns = blk_time_get_ns(); 1359 rq->stats_sectors = blk_rq_sectors(rq); 1360 rq->rq_flags |= RQF_STATS; 1361 rq_qos_issue(q, rq); 1362 } 1363 1364 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 1365 1366 blk_add_timer(rq); 1367 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 1368 rq->mq_hctx->tags->rqs[rq->tag] = rq; 1369 1370 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE) 1371 blk_integrity_prepare(rq); 1372 1373 if (rq->bio && rq->bio->bi_opf & REQ_POLLED) 1374 WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num); 1375 } 1376 EXPORT_SYMBOL(blk_mq_start_request); 1377 1378 /* 1379 * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple 1380 * queues. This is important for md arrays to benefit from merging 1381 * requests. 1382 */ 1383 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug) 1384 { 1385 if (plug->multiple_queues) 1386 return BLK_MAX_REQUEST_COUNT * 2; 1387 return BLK_MAX_REQUEST_COUNT; 1388 } 1389 1390 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 1391 { 1392 struct request *last = rq_list_peek(&plug->mq_list); 1393 1394 if (!plug->rq_count) { 1395 trace_block_plug(rq->q); 1396 } else if (plug->rq_count >= blk_plug_max_rq_count(plug) || 1397 (!blk_queue_nomerges(rq->q) && 1398 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1399 blk_mq_flush_plug_list(plug, false); 1400 blk_plug_invalidate_ts(); 1401 last = NULL; 1402 trace_block_plug(rq->q); 1403 } 1404 1405 if (!plug->multiple_queues && last && last->q != rq->q) 1406 plug->multiple_queues = true; 1407 /* 1408 * Any request allocated from sched tags can't be issued to 1409 * ->queue_rqs() directly 1410 */ 1411 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS)) 1412 plug->has_elevator = true; 1413 rq_list_add_tail(&plug->mq_list, rq); 1414 plug->rq_count++; 1415 } 1416 1417 /** 1418 * blk_execute_rq_nowait - insert a request to I/O scheduler for execution 1419 * @rq: request to insert 1420 * @at_head: insert request at head or tail of queue 1421 * 1422 * Description: 1423 * Insert a fully prepared request at the back of the I/O scheduler queue 1424 * for execution. Don't wait for completion. 1425 * 1426 * Note: 1427 * This function will invoke @done directly if the queue is dead. 1428 */ 1429 void blk_execute_rq_nowait(struct request *rq, bool at_head) 1430 { 1431 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1432 1433 WARN_ON(irqs_disabled()); 1434 WARN_ON(!blk_rq_is_passthrough(rq)); 1435 1436 blk_account_io_start(rq); 1437 1438 if (current->plug && !at_head) { 1439 blk_add_rq_to_plug(current->plug, rq); 1440 return; 1441 } 1442 1443 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0); 1444 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING); 1445 } 1446 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); 1447 1448 struct blk_rq_wait { 1449 struct completion done; 1450 blk_status_t ret; 1451 }; 1452 1453 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret, 1454 const struct io_comp_batch *iob) 1455 { 1456 struct blk_rq_wait *wait = rq->end_io_data; 1457 1458 wait->ret = ret; 1459 complete(&wait->done); 1460 return RQ_END_IO_NONE; 1461 } 1462 1463 bool blk_rq_is_poll(struct request *rq) 1464 { 1465 if (!rq->mq_hctx) 1466 return false; 1467 if (rq->mq_hctx->type != HCTX_TYPE_POLL) 1468 return false; 1469 return true; 1470 } 1471 EXPORT_SYMBOL_GPL(blk_rq_is_poll); 1472 1473 static void blk_rq_poll_completion(struct request *rq, struct completion *wait) 1474 { 1475 do { 1476 blk_hctx_poll(rq->q, rq->mq_hctx, NULL, BLK_POLL_ONESHOT); 1477 cond_resched(); 1478 } while (!completion_done(wait)); 1479 } 1480 1481 /** 1482 * blk_execute_rq - insert a request into queue for execution 1483 * @rq: request to insert 1484 * @at_head: insert request at head or tail of queue 1485 * 1486 * Description: 1487 * Insert a fully prepared request at the back of the I/O scheduler queue 1488 * for execution and wait for completion. 1489 * Return: The blk_status_t result provided to blk_mq_end_request(). 1490 */ 1491 blk_status_t blk_execute_rq(struct request *rq, bool at_head) 1492 { 1493 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1494 struct blk_rq_wait wait = { 1495 .done = COMPLETION_INITIALIZER_ONSTACK(wait.done), 1496 }; 1497 1498 WARN_ON(irqs_disabled()); 1499 WARN_ON(!blk_rq_is_passthrough(rq)); 1500 1501 rq->end_io_data = &wait; 1502 rq->end_io = blk_end_sync_rq; 1503 1504 blk_account_io_start(rq); 1505 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0); 1506 blk_mq_run_hw_queue(hctx, false); 1507 1508 if (blk_rq_is_poll(rq)) 1509 blk_rq_poll_completion(rq, &wait.done); 1510 else 1511 blk_wait_io(&wait.done); 1512 1513 return wait.ret; 1514 } 1515 EXPORT_SYMBOL(blk_execute_rq); 1516 1517 static void __blk_mq_requeue_request(struct request *rq) 1518 { 1519 struct request_queue *q = rq->q; 1520 1521 blk_mq_put_driver_tag(rq); 1522 1523 trace_block_rq_requeue(rq); 1524 rq_qos_requeue(q, rq); 1525 1526 if (blk_mq_request_started(rq)) { 1527 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 1528 rq->rq_flags &= ~RQF_TIMED_OUT; 1529 } 1530 } 1531 1532 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 1533 { 1534 struct request_queue *q = rq->q; 1535 unsigned long flags; 1536 1537 __blk_mq_requeue_request(rq); 1538 1539 /* this request will be re-inserted to io scheduler queue */ 1540 blk_mq_sched_requeue_request(rq); 1541 1542 spin_lock_irqsave(&q->requeue_lock, flags); 1543 list_add_tail(&rq->queuelist, &q->requeue_list); 1544 spin_unlock_irqrestore(&q->requeue_lock, flags); 1545 1546 if (kick_requeue_list) 1547 blk_mq_kick_requeue_list(q); 1548 } 1549 EXPORT_SYMBOL(blk_mq_requeue_request); 1550 1551 static void blk_mq_requeue_work(struct work_struct *work) 1552 { 1553 struct request_queue *q = 1554 container_of(work, struct request_queue, requeue_work.work); 1555 LIST_HEAD(rq_list); 1556 LIST_HEAD(flush_list); 1557 struct request *rq; 1558 1559 spin_lock_irq(&q->requeue_lock); 1560 list_splice_init(&q->requeue_list, &rq_list); 1561 list_splice_init(&q->flush_list, &flush_list); 1562 spin_unlock_irq(&q->requeue_lock); 1563 1564 while (!list_empty(&rq_list)) { 1565 rq = list_entry(rq_list.next, struct request, queuelist); 1566 list_del_init(&rq->queuelist); 1567 /* 1568 * If RQF_DONTPREP is set, the request has been started by the 1569 * driver already and might have driver-specific data allocated 1570 * already. Insert it into the hctx dispatch list to avoid 1571 * block layer merges for the request. 1572 */ 1573 if (rq->rq_flags & RQF_DONTPREP) 1574 blk_mq_request_bypass_insert(rq, 0); 1575 else 1576 blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD); 1577 } 1578 1579 while (!list_empty(&flush_list)) { 1580 rq = list_entry(flush_list.next, struct request, queuelist); 1581 list_del_init(&rq->queuelist); 1582 blk_mq_insert_request(rq, 0); 1583 } 1584 1585 blk_mq_run_hw_queues(q, false); 1586 } 1587 1588 void blk_mq_kick_requeue_list(struct request_queue *q) 1589 { 1590 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 1591 } 1592 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 1593 1594 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 1595 unsigned long msecs) 1596 { 1597 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 1598 msecs_to_jiffies(msecs)); 1599 } 1600 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 1601 1602 static bool blk_is_flush_data_rq(struct request *rq) 1603 { 1604 return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq); 1605 } 1606 1607 static bool blk_mq_rq_inflight(struct request *rq, void *priv) 1608 { 1609 /* 1610 * If we find a request that isn't idle we know the queue is busy 1611 * as it's checked in the iter. 1612 * Return false to stop the iteration. 1613 * 1614 * In case of queue quiesce, if one flush data request is completed, 1615 * don't count it as inflight given the flush sequence is suspended, 1616 * and the original flush data request is invisible to driver, just 1617 * like other pending requests because of quiesce 1618 */ 1619 if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) && 1620 blk_is_flush_data_rq(rq) && 1621 blk_mq_request_completed(rq))) { 1622 bool *busy = priv; 1623 1624 *busy = true; 1625 return false; 1626 } 1627 1628 return true; 1629 } 1630 1631 bool blk_mq_queue_inflight(struct request_queue *q) 1632 { 1633 bool busy = false; 1634 1635 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy); 1636 return busy; 1637 } 1638 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight); 1639 1640 static void blk_mq_rq_timed_out(struct request *req) 1641 { 1642 req->rq_flags |= RQF_TIMED_OUT; 1643 if (req->q->mq_ops->timeout) { 1644 enum blk_eh_timer_return ret; 1645 1646 ret = req->q->mq_ops->timeout(req); 1647 if (ret == BLK_EH_DONE) 1648 return; 1649 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 1650 } 1651 1652 blk_add_timer(req); 1653 } 1654 1655 struct blk_expired_data { 1656 bool has_timedout_rq; 1657 unsigned long next; 1658 unsigned long timeout_start; 1659 }; 1660 1661 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired) 1662 { 1663 unsigned long deadline; 1664 1665 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 1666 return false; 1667 if (rq->rq_flags & RQF_TIMED_OUT) 1668 return false; 1669 1670 deadline = READ_ONCE(rq->deadline); 1671 if (time_after_eq(expired->timeout_start, deadline)) 1672 return true; 1673 1674 if (expired->next == 0) 1675 expired->next = deadline; 1676 else if (time_after(expired->next, deadline)) 1677 expired->next = deadline; 1678 return false; 1679 } 1680 1681 void blk_mq_put_rq_ref(struct request *rq) 1682 { 1683 if (is_flush_rq(rq)) { 1684 if (rq->end_io(rq, 0, NULL) == RQ_END_IO_FREE) 1685 blk_mq_free_request(rq); 1686 } else if (req_ref_put_and_test(rq)) { 1687 __blk_mq_free_request(rq); 1688 } 1689 } 1690 1691 static bool blk_mq_check_expired(struct request *rq, void *priv) 1692 { 1693 struct blk_expired_data *expired = priv; 1694 1695 /* 1696 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot 1697 * be reallocated underneath the timeout handler's processing, then 1698 * the expire check is reliable. If the request is not expired, then 1699 * it was completed and reallocated as a new request after returning 1700 * from blk_mq_check_expired(). 1701 */ 1702 if (blk_mq_req_expired(rq, expired)) { 1703 expired->has_timedout_rq = true; 1704 return false; 1705 } 1706 return true; 1707 } 1708 1709 static bool blk_mq_handle_expired(struct request *rq, void *priv) 1710 { 1711 struct blk_expired_data *expired = priv; 1712 1713 if (blk_mq_req_expired(rq, expired)) 1714 blk_mq_rq_timed_out(rq); 1715 return true; 1716 } 1717 1718 static void blk_mq_timeout_work(struct work_struct *work) 1719 { 1720 struct request_queue *q = 1721 container_of(work, struct request_queue, timeout_work); 1722 struct blk_expired_data expired = { 1723 .timeout_start = jiffies, 1724 }; 1725 struct blk_mq_hw_ctx *hctx; 1726 unsigned long i; 1727 1728 /* A deadlock might occur if a request is stuck requiring a 1729 * timeout at the same time a queue freeze is waiting 1730 * completion, since the timeout code would not be able to 1731 * acquire the queue reference here. 1732 * 1733 * That's why we don't use blk_queue_enter here; instead, we use 1734 * percpu_ref_tryget directly, because we need to be able to 1735 * obtain a reference even in the short window between the queue 1736 * starting to freeze, by dropping the first reference in 1737 * blk_freeze_queue_start, and the moment the last request is 1738 * consumed, marked by the instant q_usage_counter reaches 1739 * zero. 1740 */ 1741 if (!percpu_ref_tryget(&q->q_usage_counter)) 1742 return; 1743 1744 /* check if there is any timed-out request */ 1745 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired); 1746 if (expired.has_timedout_rq) { 1747 /* 1748 * Before walking tags, we must ensure any submit started 1749 * before the current time has finished. Since the submit 1750 * uses srcu or rcu, wait for a synchronization point to 1751 * ensure all running submits have finished 1752 */ 1753 blk_mq_wait_quiesce_done(q->tag_set); 1754 1755 expired.next = 0; 1756 blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired); 1757 } 1758 1759 if (expired.next != 0) { 1760 mod_timer(&q->timeout, expired.next); 1761 } else { 1762 /* 1763 * Request timeouts are handled as a forward rolling timer. If 1764 * we end up here it means that no requests are pending and 1765 * also that no request has been pending for a while. Mark 1766 * each hctx as idle. 1767 */ 1768 queue_for_each_hw_ctx(q, hctx, i) { 1769 /* the hctx may be unmapped, so check it here */ 1770 if (blk_mq_hw_queue_mapped(hctx)) 1771 blk_mq_tag_idle(hctx); 1772 } 1773 } 1774 blk_queue_exit(q); 1775 } 1776 1777 struct flush_busy_ctx_data { 1778 struct blk_mq_hw_ctx *hctx; 1779 struct list_head *list; 1780 }; 1781 1782 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 1783 { 1784 struct flush_busy_ctx_data *flush_data = data; 1785 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 1786 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1787 enum hctx_type type = hctx->type; 1788 1789 spin_lock(&ctx->lock); 1790 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list); 1791 sbitmap_clear_bit(sb, bitnr); 1792 spin_unlock(&ctx->lock); 1793 return true; 1794 } 1795 1796 /* 1797 * Process software queues that have been marked busy, splicing them 1798 * to the for-dispatch 1799 */ 1800 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 1801 { 1802 struct flush_busy_ctx_data data = { 1803 .hctx = hctx, 1804 .list = list, 1805 }; 1806 1807 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 1808 } 1809 1810 struct dispatch_rq_data { 1811 struct blk_mq_hw_ctx *hctx; 1812 struct request *rq; 1813 }; 1814 1815 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 1816 void *data) 1817 { 1818 struct dispatch_rq_data *dispatch_data = data; 1819 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 1820 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1821 enum hctx_type type = hctx->type; 1822 1823 spin_lock(&ctx->lock); 1824 if (!list_empty(&ctx->rq_lists[type])) { 1825 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next); 1826 list_del_init(&dispatch_data->rq->queuelist); 1827 if (list_empty(&ctx->rq_lists[type])) 1828 sbitmap_clear_bit(sb, bitnr); 1829 } 1830 spin_unlock(&ctx->lock); 1831 1832 return !dispatch_data->rq; 1833 } 1834 1835 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 1836 struct blk_mq_ctx *start) 1837 { 1838 unsigned off = start ? start->index_hw[hctx->type] : 0; 1839 struct dispatch_rq_data data = { 1840 .hctx = hctx, 1841 .rq = NULL, 1842 }; 1843 1844 __sbitmap_for_each_set(&hctx->ctx_map, off, 1845 dispatch_rq_from_ctx, &data); 1846 1847 return data.rq; 1848 } 1849 1850 bool __blk_mq_alloc_driver_tag(struct request *rq) 1851 { 1852 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; 1853 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; 1854 int tag; 1855 1856 blk_mq_tag_busy(rq->mq_hctx); 1857 1858 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) { 1859 bt = &rq->mq_hctx->tags->breserved_tags; 1860 tag_offset = 0; 1861 } else { 1862 if (!hctx_may_queue(rq->mq_hctx, bt)) 1863 return false; 1864 } 1865 1866 tag = __sbitmap_queue_get(bt); 1867 if (tag == BLK_MQ_NO_TAG) 1868 return false; 1869 1870 rq->tag = tag + tag_offset; 1871 blk_mq_inc_active_requests(rq->mq_hctx); 1872 return true; 1873 } 1874 1875 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1876 int flags, void *key) 1877 { 1878 struct blk_mq_hw_ctx *hctx; 1879 1880 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1881 1882 spin_lock(&hctx->dispatch_wait_lock); 1883 if (!list_empty(&wait->entry)) { 1884 struct sbitmap_queue *sbq; 1885 1886 list_del_init(&wait->entry); 1887 sbq = &hctx->tags->bitmap_tags; 1888 atomic_dec(&sbq->ws_active); 1889 } 1890 spin_unlock(&hctx->dispatch_wait_lock); 1891 1892 blk_mq_run_hw_queue(hctx, true); 1893 return 1; 1894 } 1895 1896 /* 1897 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1898 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1899 * restart. For both cases, take care to check the condition again after 1900 * marking us as waiting. 1901 */ 1902 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 1903 struct request *rq) 1904 { 1905 struct sbitmap_queue *sbq; 1906 struct wait_queue_head *wq; 1907 wait_queue_entry_t *wait; 1908 bool ret; 1909 1910 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) && 1911 !(blk_mq_is_shared_tags(hctx->flags))) { 1912 blk_mq_sched_mark_restart_hctx(hctx); 1913 1914 /* 1915 * It's possible that a tag was freed in the window between the 1916 * allocation failure and adding the hardware queue to the wait 1917 * queue. 1918 * 1919 * Don't clear RESTART here, someone else could have set it. 1920 * At most this will cost an extra queue run. 1921 */ 1922 return blk_mq_get_driver_tag(rq); 1923 } 1924 1925 wait = &hctx->dispatch_wait; 1926 if (!list_empty_careful(&wait->entry)) 1927 return false; 1928 1929 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) 1930 sbq = &hctx->tags->breserved_tags; 1931 else 1932 sbq = &hctx->tags->bitmap_tags; 1933 wq = &bt_wait_ptr(sbq, hctx)->wait; 1934 1935 spin_lock_irq(&wq->lock); 1936 spin_lock(&hctx->dispatch_wait_lock); 1937 if (!list_empty(&wait->entry)) { 1938 spin_unlock(&hctx->dispatch_wait_lock); 1939 spin_unlock_irq(&wq->lock); 1940 return false; 1941 } 1942 1943 atomic_inc(&sbq->ws_active); 1944 wait->flags &= ~WQ_FLAG_EXCLUSIVE; 1945 __add_wait_queue(wq, wait); 1946 1947 /* 1948 * Add one explicit barrier since blk_mq_get_driver_tag() may 1949 * not imply barrier in case of failure. 1950 * 1951 * Order adding us to wait queue and allocating driver tag. 1952 * 1953 * The pair is the one implied in sbitmap_queue_wake_up() which 1954 * orders clearing sbitmap tag bits and waitqueue_active() in 1955 * __sbitmap_queue_wake_up(), since waitqueue_active() is lockless 1956 * 1957 * Otherwise, re-order of adding wait queue and getting driver tag 1958 * may cause __sbitmap_queue_wake_up() to wake up nothing because 1959 * the waitqueue_active() may not observe us in wait queue. 1960 */ 1961 smp_mb(); 1962 1963 /* 1964 * It's possible that a tag was freed in the window between the 1965 * allocation failure and adding the hardware queue to the wait 1966 * queue. 1967 */ 1968 ret = blk_mq_get_driver_tag(rq); 1969 if (!ret) { 1970 spin_unlock(&hctx->dispatch_wait_lock); 1971 spin_unlock_irq(&wq->lock); 1972 return false; 1973 } 1974 1975 /* 1976 * We got a tag, remove ourselves from the wait queue to ensure 1977 * someone else gets the wakeup. 1978 */ 1979 list_del_init(&wait->entry); 1980 atomic_dec(&sbq->ws_active); 1981 spin_unlock(&hctx->dispatch_wait_lock); 1982 spin_unlock_irq(&wq->lock); 1983 1984 return true; 1985 } 1986 1987 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8 1988 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4 1989 /* 1990 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA): 1991 * - EWMA is one simple way to compute running average value 1992 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially 1993 * - take 4 as factor for avoiding to get too small(0) result, and this 1994 * factor doesn't matter because EWMA decreases exponentially 1995 */ 1996 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy) 1997 { 1998 unsigned int ewma; 1999 2000 ewma = hctx->dispatch_busy; 2001 2002 if (!ewma && !busy) 2003 return; 2004 2005 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1; 2006 if (busy) 2007 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR; 2008 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT; 2009 2010 hctx->dispatch_busy = ewma; 2011 } 2012 2013 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 2014 2015 static void blk_mq_handle_dev_resource(struct request *rq, 2016 struct list_head *list) 2017 { 2018 list_add(&rq->queuelist, list); 2019 __blk_mq_requeue_request(rq); 2020 } 2021 2022 enum prep_dispatch { 2023 PREP_DISPATCH_OK, 2024 PREP_DISPATCH_NO_TAG, 2025 PREP_DISPATCH_NO_BUDGET, 2026 }; 2027 2028 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq, 2029 bool need_budget) 2030 { 2031 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2032 int budget_token = -1; 2033 2034 if (need_budget) { 2035 budget_token = blk_mq_get_dispatch_budget(rq->q); 2036 if (budget_token < 0) { 2037 blk_mq_put_driver_tag(rq); 2038 return PREP_DISPATCH_NO_BUDGET; 2039 } 2040 blk_mq_set_rq_budget_token(rq, budget_token); 2041 } 2042 2043 if (!blk_mq_get_driver_tag(rq)) { 2044 /* 2045 * The initial allocation attempt failed, so we need to 2046 * rerun the hardware queue when a tag is freed. The 2047 * waitqueue takes care of that. If the queue is run 2048 * before we add this entry back on the dispatch list, 2049 * we'll re-run it below. 2050 */ 2051 if (!blk_mq_mark_tag_wait(hctx, rq)) { 2052 /* 2053 * All budgets not got from this function will be put 2054 * together during handling partial dispatch 2055 */ 2056 if (need_budget) 2057 blk_mq_put_dispatch_budget(rq->q, budget_token); 2058 return PREP_DISPATCH_NO_TAG; 2059 } 2060 } 2061 2062 return PREP_DISPATCH_OK; 2063 } 2064 2065 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */ 2066 static void blk_mq_release_budgets(struct request_queue *q, 2067 struct list_head *list) 2068 { 2069 struct request *rq; 2070 2071 list_for_each_entry(rq, list, queuelist) { 2072 int budget_token = blk_mq_get_rq_budget_token(rq); 2073 2074 if (budget_token >= 0) 2075 blk_mq_put_dispatch_budget(q, budget_token); 2076 } 2077 } 2078 2079 /* 2080 * blk_mq_commit_rqs will notify driver using bd->last that there is no 2081 * more requests. (See comment in struct blk_mq_ops for commit_rqs for 2082 * details) 2083 * Attention, we should explicitly call this in unusual cases: 2084 * 1) did not queue everything initially scheduled to queue 2085 * 2) the last attempt to queue a request failed 2086 */ 2087 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued, 2088 bool from_schedule) 2089 { 2090 if (hctx->queue->mq_ops->commit_rqs && queued) { 2091 trace_block_unplug(hctx->queue, queued, !from_schedule); 2092 hctx->queue->mq_ops->commit_rqs(hctx); 2093 } 2094 } 2095 2096 /* 2097 * Returns true if we did some work AND can potentially do more. 2098 */ 2099 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, 2100 bool get_budget) 2101 { 2102 enum prep_dispatch prep; 2103 struct request_queue *q = hctx->queue; 2104 struct request *rq; 2105 int queued; 2106 blk_status_t ret = BLK_STS_OK; 2107 bool needs_resource = false; 2108 2109 if (list_empty(list)) 2110 return false; 2111 2112 /* 2113 * Now process all the entries, sending them to the driver. 2114 */ 2115 queued = 0; 2116 do { 2117 struct blk_mq_queue_data bd; 2118 2119 rq = list_first_entry(list, struct request, queuelist); 2120 2121 WARN_ON_ONCE(hctx != rq->mq_hctx); 2122 prep = blk_mq_prep_dispatch_rq(rq, get_budget); 2123 if (prep != PREP_DISPATCH_OK) 2124 break; 2125 2126 list_del_init(&rq->queuelist); 2127 2128 bd.rq = rq; 2129 bd.last = list_empty(list); 2130 2131 ret = q->mq_ops->queue_rq(hctx, &bd); 2132 switch (ret) { 2133 case BLK_STS_OK: 2134 queued++; 2135 break; 2136 case BLK_STS_RESOURCE: 2137 needs_resource = true; 2138 fallthrough; 2139 case BLK_STS_DEV_RESOURCE: 2140 blk_mq_handle_dev_resource(rq, list); 2141 goto out; 2142 default: 2143 blk_mq_end_request(rq, ret); 2144 } 2145 } while (!list_empty(list)); 2146 out: 2147 /* If we didn't flush the entire list, we could have told the driver 2148 * there was more coming, but that turned out to be a lie. 2149 */ 2150 if (!list_empty(list) || ret != BLK_STS_OK) 2151 blk_mq_commit_rqs(hctx, queued, false); 2152 2153 /* 2154 * Any items that need requeuing? Stuff them into hctx->dispatch, 2155 * that is where we will continue on next queue run. 2156 */ 2157 if (!list_empty(list)) { 2158 bool needs_restart; 2159 /* For non-shared tags, the RESTART check will suffice */ 2160 bool no_tag = prep == PREP_DISPATCH_NO_TAG && 2161 ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) || 2162 blk_mq_is_shared_tags(hctx->flags)); 2163 2164 /* 2165 * If the caller allocated budgets, free the budgets of the 2166 * requests that have not yet been passed to the block driver. 2167 */ 2168 if (!get_budget) 2169 blk_mq_release_budgets(q, list); 2170 2171 spin_lock(&hctx->lock); 2172 list_splice_tail_init(list, &hctx->dispatch); 2173 spin_unlock(&hctx->lock); 2174 2175 /* 2176 * Order adding requests to hctx->dispatch and checking 2177 * SCHED_RESTART flag. The pair of this smp_mb() is the one 2178 * in blk_mq_sched_restart(). Avoid restart code path to 2179 * miss the new added requests to hctx->dispatch, meantime 2180 * SCHED_RESTART is observed here. 2181 */ 2182 smp_mb(); 2183 2184 /* 2185 * If SCHED_RESTART was set by the caller of this function and 2186 * it is no longer set that means that it was cleared by another 2187 * thread and hence that a queue rerun is needed. 2188 * 2189 * If 'no_tag' is set, that means that we failed getting 2190 * a driver tag with an I/O scheduler attached. If our dispatch 2191 * waitqueue is no longer active, ensure that we run the queue 2192 * AFTER adding our entries back to the list. 2193 * 2194 * If no I/O scheduler has been configured it is possible that 2195 * the hardware queue got stopped and restarted before requests 2196 * were pushed back onto the dispatch list. Rerun the queue to 2197 * avoid starvation. Notes: 2198 * - blk_mq_run_hw_queue() checks whether or not a queue has 2199 * been stopped before rerunning a queue. 2200 * - Some but not all block drivers stop a queue before 2201 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 2202 * and dm-rq. 2203 * 2204 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 2205 * bit is set, run queue after a delay to avoid IO stalls 2206 * that could otherwise occur if the queue is idle. We'll do 2207 * similar if we couldn't get budget or couldn't lock a zone 2208 * and SCHED_RESTART is set. 2209 */ 2210 needs_restart = blk_mq_sched_needs_restart(hctx); 2211 if (prep == PREP_DISPATCH_NO_BUDGET) 2212 needs_resource = true; 2213 if (!needs_restart || 2214 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 2215 blk_mq_run_hw_queue(hctx, true); 2216 else if (needs_resource) 2217 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 2218 2219 blk_mq_update_dispatch_busy(hctx, true); 2220 return false; 2221 } 2222 2223 blk_mq_update_dispatch_busy(hctx, false); 2224 return true; 2225 } 2226 2227 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 2228 { 2229 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 2230 2231 if (cpu >= nr_cpu_ids) 2232 cpu = cpumask_first(hctx->cpumask); 2233 return cpu; 2234 } 2235 2236 /* 2237 * ->next_cpu is always calculated from hctx->cpumask, so simply use 2238 * it for speeding up the check 2239 */ 2240 static bool blk_mq_hctx_empty_cpumask(struct blk_mq_hw_ctx *hctx) 2241 { 2242 return hctx->next_cpu >= nr_cpu_ids; 2243 } 2244 2245 /* 2246 * It'd be great if the workqueue API had a way to pass 2247 * in a mask and had some smarts for more clever placement. 2248 * For now we just round-robin here, switching for every 2249 * BLK_MQ_CPU_WORK_BATCH queued items. 2250 */ 2251 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 2252 { 2253 bool tried = false; 2254 int next_cpu = hctx->next_cpu; 2255 2256 /* Switch to unbound if no allowable CPUs in this hctx */ 2257 if (hctx->queue->nr_hw_queues == 1 || blk_mq_hctx_empty_cpumask(hctx)) 2258 return WORK_CPU_UNBOUND; 2259 2260 if (--hctx->next_cpu_batch <= 0) { 2261 select_cpu: 2262 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 2263 cpu_online_mask); 2264 if (next_cpu >= nr_cpu_ids) 2265 next_cpu = blk_mq_first_mapped_cpu(hctx); 2266 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2267 } 2268 2269 /* 2270 * Do unbound schedule if we can't find a online CPU for this hctx, 2271 * and it should only happen in the path of handling CPU DEAD. 2272 */ 2273 if (!cpu_online(next_cpu)) { 2274 if (!tried) { 2275 tried = true; 2276 goto select_cpu; 2277 } 2278 2279 /* 2280 * Make sure to re-select CPU next time once after CPUs 2281 * in hctx->cpumask become online again. 2282 */ 2283 hctx->next_cpu = next_cpu; 2284 hctx->next_cpu_batch = 1; 2285 return WORK_CPU_UNBOUND; 2286 } 2287 2288 hctx->next_cpu = next_cpu; 2289 return next_cpu; 2290 } 2291 2292 /** 2293 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously. 2294 * @hctx: Pointer to the hardware queue to run. 2295 * @msecs: Milliseconds of delay to wait before running the queue. 2296 * 2297 * Run a hardware queue asynchronously with a delay of @msecs. 2298 */ 2299 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 2300 { 2301 if (unlikely(blk_mq_hctx_stopped(hctx))) 2302 return; 2303 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 2304 msecs_to_jiffies(msecs)); 2305 } 2306 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 2307 2308 static inline bool blk_mq_hw_queue_need_run(struct blk_mq_hw_ctx *hctx) 2309 { 2310 bool need_run; 2311 2312 /* 2313 * When queue is quiesced, we may be switching io scheduler, or 2314 * updating nr_hw_queues, or other things, and we can't run queue 2315 * any more, even blk_mq_hctx_has_pending() can't be called safely. 2316 * 2317 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 2318 * quiesced. 2319 */ 2320 __blk_mq_run_dispatch_ops(hctx->queue, false, 2321 need_run = !blk_queue_quiesced(hctx->queue) && 2322 blk_mq_hctx_has_pending(hctx)); 2323 return need_run; 2324 } 2325 2326 /** 2327 * blk_mq_run_hw_queue - Start to run a hardware queue. 2328 * @hctx: Pointer to the hardware queue to run. 2329 * @async: If we want to run the queue asynchronously. 2330 * 2331 * Check if the request queue is not in a quiesced state and if there are 2332 * pending requests to be sent. If this is true, run the queue to send requests 2333 * to hardware. 2334 */ 2335 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 2336 { 2337 bool need_run; 2338 2339 /* 2340 * We can't run the queue inline with interrupts disabled. 2341 */ 2342 WARN_ON_ONCE(!async && in_interrupt()); 2343 2344 might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING); 2345 2346 need_run = blk_mq_hw_queue_need_run(hctx); 2347 if (!need_run) { 2348 unsigned long flags; 2349 2350 /* 2351 * Synchronize with blk_mq_unquiesce_queue(), because we check 2352 * if hw queue is quiesced locklessly above, we need the use 2353 * ->queue_lock to make sure we see the up-to-date status to 2354 * not miss rerunning the hw queue. 2355 */ 2356 spin_lock_irqsave(&hctx->queue->queue_lock, flags); 2357 need_run = blk_mq_hw_queue_need_run(hctx); 2358 spin_unlock_irqrestore(&hctx->queue->queue_lock, flags); 2359 2360 if (!need_run) 2361 return; 2362 } 2363 2364 if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) { 2365 blk_mq_delay_run_hw_queue(hctx, 0); 2366 return; 2367 } 2368 2369 blk_mq_run_dispatch_ops(hctx->queue, 2370 blk_mq_sched_dispatch_requests(hctx)); 2371 } 2372 EXPORT_SYMBOL(blk_mq_run_hw_queue); 2373 2374 /* 2375 * Return prefered queue to dispatch from (if any) for non-mq aware IO 2376 * scheduler. 2377 */ 2378 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) 2379 { 2380 struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); 2381 /* 2382 * If the IO scheduler does not respect hardware queues when 2383 * dispatching, we just don't bother with multiple HW queues and 2384 * dispatch from hctx for the current CPU since running multiple queues 2385 * just causes lock contention inside the scheduler and pointless cache 2386 * bouncing. 2387 */ 2388 struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT]; 2389 2390 if (!blk_mq_hctx_stopped(hctx)) 2391 return hctx; 2392 return NULL; 2393 } 2394 2395 /** 2396 * blk_mq_run_hw_queues - Run all hardware queues in a request queue. 2397 * @q: Pointer to the request queue to run. 2398 * @async: If we want to run the queue asynchronously. 2399 */ 2400 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 2401 { 2402 struct blk_mq_hw_ctx *hctx, *sq_hctx; 2403 unsigned long i; 2404 2405 sq_hctx = NULL; 2406 if (blk_queue_sq_sched(q)) 2407 sq_hctx = blk_mq_get_sq_hctx(q); 2408 queue_for_each_hw_ctx(q, hctx, i) { 2409 if (blk_mq_hctx_stopped(hctx)) 2410 continue; 2411 /* 2412 * Dispatch from this hctx either if there's no hctx preferred 2413 * by IO scheduler or if it has requests that bypass the 2414 * scheduler. 2415 */ 2416 if (!sq_hctx || sq_hctx == hctx || 2417 !list_empty_careful(&hctx->dispatch)) 2418 blk_mq_run_hw_queue(hctx, async); 2419 } 2420 } 2421 EXPORT_SYMBOL(blk_mq_run_hw_queues); 2422 2423 /** 2424 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously. 2425 * @q: Pointer to the request queue to run. 2426 * @msecs: Milliseconds of delay to wait before running the queues. 2427 */ 2428 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) 2429 { 2430 struct blk_mq_hw_ctx *hctx, *sq_hctx; 2431 unsigned long i; 2432 2433 sq_hctx = NULL; 2434 if (blk_queue_sq_sched(q)) 2435 sq_hctx = blk_mq_get_sq_hctx(q); 2436 queue_for_each_hw_ctx(q, hctx, i) { 2437 if (blk_mq_hctx_stopped(hctx)) 2438 continue; 2439 /* 2440 * If there is already a run_work pending, leave the 2441 * pending delay untouched. Otherwise, a hctx can stall 2442 * if another hctx is re-delaying the other's work 2443 * before the work executes. 2444 */ 2445 if (delayed_work_pending(&hctx->run_work)) 2446 continue; 2447 /* 2448 * Dispatch from this hctx either if there's no hctx preferred 2449 * by IO scheduler or if it has requests that bypass the 2450 * scheduler. 2451 */ 2452 if (!sq_hctx || sq_hctx == hctx || 2453 !list_empty_careful(&hctx->dispatch)) 2454 blk_mq_delay_run_hw_queue(hctx, msecs); 2455 } 2456 } 2457 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues); 2458 2459 /* 2460 * This function is often used for pausing .queue_rq() by driver when 2461 * there isn't enough resource or some conditions aren't satisfied, and 2462 * BLK_STS_RESOURCE is usually returned. 2463 * 2464 * We do not guarantee that dispatch can be drained or blocked 2465 * after blk_mq_stop_hw_queue() returns. Please use 2466 * blk_mq_quiesce_queue() for that requirement. 2467 */ 2468 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 2469 { 2470 cancel_delayed_work(&hctx->run_work); 2471 2472 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 2473 } 2474 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 2475 2476 /* 2477 * This function is often used for pausing .queue_rq() by driver when 2478 * there isn't enough resource or some conditions aren't satisfied, and 2479 * BLK_STS_RESOURCE is usually returned. 2480 * 2481 * We do not guarantee that dispatch can be drained or blocked 2482 * after blk_mq_stop_hw_queues() returns. Please use 2483 * blk_mq_quiesce_queue() for that requirement. 2484 */ 2485 void blk_mq_stop_hw_queues(struct request_queue *q) 2486 { 2487 struct blk_mq_hw_ctx *hctx; 2488 unsigned long i; 2489 2490 queue_for_each_hw_ctx(q, hctx, i) 2491 blk_mq_stop_hw_queue(hctx); 2492 } 2493 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 2494 2495 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 2496 { 2497 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 2498 2499 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING); 2500 } 2501 EXPORT_SYMBOL(blk_mq_start_hw_queue); 2502 2503 void blk_mq_start_hw_queues(struct request_queue *q) 2504 { 2505 struct blk_mq_hw_ctx *hctx; 2506 unsigned long i; 2507 2508 queue_for_each_hw_ctx(q, hctx, i) 2509 blk_mq_start_hw_queue(hctx); 2510 } 2511 EXPORT_SYMBOL(blk_mq_start_hw_queues); 2512 2513 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 2514 { 2515 if (!blk_mq_hctx_stopped(hctx)) 2516 return; 2517 2518 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 2519 /* 2520 * Pairs with the smp_mb() in blk_mq_hctx_stopped() to order the 2521 * clearing of BLK_MQ_S_STOPPED above and the checking of dispatch 2522 * list in the subsequent routine. 2523 */ 2524 smp_mb__after_atomic(); 2525 blk_mq_run_hw_queue(hctx, async); 2526 } 2527 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 2528 2529 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 2530 { 2531 struct blk_mq_hw_ctx *hctx; 2532 unsigned long i; 2533 2534 queue_for_each_hw_ctx(q, hctx, i) 2535 blk_mq_start_stopped_hw_queue(hctx, async || 2536 (hctx->flags & BLK_MQ_F_BLOCKING)); 2537 } 2538 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 2539 2540 static void blk_mq_run_work_fn(struct work_struct *work) 2541 { 2542 struct blk_mq_hw_ctx *hctx = 2543 container_of(work, struct blk_mq_hw_ctx, run_work.work); 2544 2545 blk_mq_run_dispatch_ops(hctx->queue, 2546 blk_mq_sched_dispatch_requests(hctx)); 2547 } 2548 2549 /** 2550 * blk_mq_request_bypass_insert - Insert a request at dispatch list. 2551 * @rq: Pointer to request to be inserted. 2552 * @flags: BLK_MQ_INSERT_* 2553 * 2554 * Should only be used carefully, when the caller knows we want to 2555 * bypass a potential IO scheduler on the target device. 2556 */ 2557 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags) 2558 { 2559 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2560 2561 spin_lock(&hctx->lock); 2562 if (flags & BLK_MQ_INSERT_AT_HEAD) 2563 list_add(&rq->queuelist, &hctx->dispatch); 2564 else 2565 list_add_tail(&rq->queuelist, &hctx->dispatch); 2566 spin_unlock(&hctx->lock); 2567 } 2568 2569 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, 2570 struct blk_mq_ctx *ctx, struct list_head *list, 2571 bool run_queue_async) 2572 { 2573 struct request *rq; 2574 enum hctx_type type = hctx->type; 2575 2576 /* 2577 * Try to issue requests directly if the hw queue isn't busy to save an 2578 * extra enqueue & dequeue to the sw queue. 2579 */ 2580 if (!hctx->dispatch_busy && !run_queue_async) { 2581 blk_mq_run_dispatch_ops(hctx->queue, 2582 blk_mq_try_issue_list_directly(hctx, list)); 2583 if (list_empty(list)) 2584 goto out; 2585 } 2586 2587 /* 2588 * preemption doesn't flush plug list, so it's possible ctx->cpu is 2589 * offline now 2590 */ 2591 list_for_each_entry(rq, list, queuelist) { 2592 BUG_ON(rq->mq_ctx != ctx); 2593 trace_block_rq_insert(rq); 2594 if (rq->cmd_flags & REQ_NOWAIT) 2595 run_queue_async = true; 2596 } 2597 2598 spin_lock(&ctx->lock); 2599 list_splice_tail_init(list, &ctx->rq_lists[type]); 2600 blk_mq_hctx_mark_pending(hctx, ctx); 2601 spin_unlock(&ctx->lock); 2602 out: 2603 blk_mq_run_hw_queue(hctx, run_queue_async); 2604 } 2605 2606 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags) 2607 { 2608 struct request_queue *q = rq->q; 2609 struct blk_mq_ctx *ctx = rq->mq_ctx; 2610 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2611 2612 if (blk_rq_is_passthrough(rq)) { 2613 /* 2614 * Passthrough request have to be added to hctx->dispatch 2615 * directly. The device may be in a situation where it can't 2616 * handle FS request, and always returns BLK_STS_RESOURCE for 2617 * them, which gets them added to hctx->dispatch. 2618 * 2619 * If a passthrough request is required to unblock the queues, 2620 * and it is added to the scheduler queue, there is no chance to 2621 * dispatch it given we prioritize requests in hctx->dispatch. 2622 */ 2623 blk_mq_request_bypass_insert(rq, flags); 2624 } else if (req_op(rq) == REQ_OP_FLUSH) { 2625 /* 2626 * Firstly normal IO request is inserted to scheduler queue or 2627 * sw queue, meantime we add flush request to dispatch queue( 2628 * hctx->dispatch) directly and there is at most one in-flight 2629 * flush request for each hw queue, so it doesn't matter to add 2630 * flush request to tail or front of the dispatch queue. 2631 * 2632 * Secondly in case of NCQ, flush request belongs to non-NCQ 2633 * command, and queueing it will fail when there is any 2634 * in-flight normal IO request(NCQ command). When adding flush 2635 * rq to the front of hctx->dispatch, it is easier to introduce 2636 * extra time to flush rq's latency because of S_SCHED_RESTART 2637 * compared with adding to the tail of dispatch queue, then 2638 * chance of flush merge is increased, and less flush requests 2639 * will be issued to controller. It is observed that ~10% time 2640 * is saved in blktests block/004 on disk attached to AHCI/NCQ 2641 * drive when adding flush rq to the front of hctx->dispatch. 2642 * 2643 * Simply queue flush rq to the front of hctx->dispatch so that 2644 * intensive flush workloads can benefit in case of NCQ HW. 2645 */ 2646 blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD); 2647 } else if (q->elevator) { 2648 LIST_HEAD(list); 2649 2650 WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG); 2651 2652 list_add(&rq->queuelist, &list); 2653 q->elevator->type->ops.insert_requests(hctx, &list, flags); 2654 } else { 2655 trace_block_rq_insert(rq); 2656 2657 spin_lock(&ctx->lock); 2658 if (flags & BLK_MQ_INSERT_AT_HEAD) 2659 list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]); 2660 else 2661 list_add_tail(&rq->queuelist, 2662 &ctx->rq_lists[hctx->type]); 2663 blk_mq_hctx_mark_pending(hctx, ctx); 2664 spin_unlock(&ctx->lock); 2665 } 2666 } 2667 2668 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, 2669 unsigned int nr_segs) 2670 { 2671 int err; 2672 2673 if (bio->bi_opf & REQ_RAHEAD) 2674 rq->cmd_flags |= REQ_FAILFAST_MASK; 2675 2676 rq->bio = rq->biotail = bio; 2677 rq->__sector = bio->bi_iter.bi_sector; 2678 rq->__data_len = bio->bi_iter.bi_size; 2679 rq->phys_gap_bit = bio->bi_bvec_gap_bit; 2680 2681 rq->nr_phys_segments = nr_segs; 2682 if (bio_integrity(bio)) 2683 rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q, 2684 bio); 2685 2686 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */ 2687 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO); 2688 WARN_ON_ONCE(err); 2689 2690 blk_account_io_start(rq); 2691 } 2692 2693 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 2694 struct request *rq, bool last) 2695 { 2696 struct request_queue *q = rq->q; 2697 struct blk_mq_queue_data bd = { 2698 .rq = rq, 2699 .last = last, 2700 }; 2701 blk_status_t ret; 2702 2703 /* 2704 * For OK queue, we are done. For error, caller may kill it. 2705 * Any other error (busy), just add it to our list as we 2706 * previously would have done. 2707 */ 2708 ret = q->mq_ops->queue_rq(hctx, &bd); 2709 switch (ret) { 2710 case BLK_STS_OK: 2711 blk_mq_update_dispatch_busy(hctx, false); 2712 break; 2713 case BLK_STS_RESOURCE: 2714 case BLK_STS_DEV_RESOURCE: 2715 blk_mq_update_dispatch_busy(hctx, true); 2716 __blk_mq_requeue_request(rq); 2717 break; 2718 default: 2719 blk_mq_update_dispatch_busy(hctx, false); 2720 break; 2721 } 2722 2723 return ret; 2724 } 2725 2726 static bool blk_mq_get_budget_and_tag(struct request *rq) 2727 { 2728 int budget_token; 2729 2730 budget_token = blk_mq_get_dispatch_budget(rq->q); 2731 if (budget_token < 0) 2732 return false; 2733 blk_mq_set_rq_budget_token(rq, budget_token); 2734 if (!blk_mq_get_driver_tag(rq)) { 2735 blk_mq_put_dispatch_budget(rq->q, budget_token); 2736 return false; 2737 } 2738 return true; 2739 } 2740 2741 /** 2742 * blk_mq_try_issue_directly - Try to send a request directly to device driver. 2743 * @hctx: Pointer of the associated hardware queue. 2744 * @rq: Pointer to request to be sent. 2745 * 2746 * If the device has enough resources to accept a new request now, send the 2747 * request directly to device driver. Else, insert at hctx->dispatch queue, so 2748 * we can try send it another time in the future. Requests inserted at this 2749 * queue have higher priority. 2750 */ 2751 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 2752 struct request *rq) 2753 { 2754 blk_status_t ret; 2755 2756 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) { 2757 blk_mq_insert_request(rq, 0); 2758 blk_mq_run_hw_queue(hctx, false); 2759 return; 2760 } 2761 2762 if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) { 2763 blk_mq_insert_request(rq, 0); 2764 blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT); 2765 return; 2766 } 2767 2768 ret = __blk_mq_issue_directly(hctx, rq, true); 2769 switch (ret) { 2770 case BLK_STS_OK: 2771 break; 2772 case BLK_STS_RESOURCE: 2773 case BLK_STS_DEV_RESOURCE: 2774 blk_mq_request_bypass_insert(rq, 0); 2775 blk_mq_run_hw_queue(hctx, false); 2776 break; 2777 default: 2778 blk_mq_end_request(rq, ret); 2779 break; 2780 } 2781 } 2782 2783 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) 2784 { 2785 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2786 2787 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) { 2788 blk_mq_insert_request(rq, 0); 2789 blk_mq_run_hw_queue(hctx, false); 2790 return BLK_STS_OK; 2791 } 2792 2793 if (!blk_mq_get_budget_and_tag(rq)) 2794 return BLK_STS_RESOURCE; 2795 return __blk_mq_issue_directly(hctx, rq, last); 2796 } 2797 2798 static void blk_mq_issue_direct(struct rq_list *rqs) 2799 { 2800 struct blk_mq_hw_ctx *hctx = NULL; 2801 struct request *rq; 2802 int queued = 0; 2803 blk_status_t ret = BLK_STS_OK; 2804 2805 while ((rq = rq_list_pop(rqs))) { 2806 bool last = rq_list_empty(rqs); 2807 2808 if (hctx != rq->mq_hctx) { 2809 if (hctx) { 2810 blk_mq_commit_rqs(hctx, queued, false); 2811 queued = 0; 2812 } 2813 hctx = rq->mq_hctx; 2814 } 2815 2816 ret = blk_mq_request_issue_directly(rq, last); 2817 switch (ret) { 2818 case BLK_STS_OK: 2819 queued++; 2820 break; 2821 case BLK_STS_RESOURCE: 2822 case BLK_STS_DEV_RESOURCE: 2823 blk_mq_request_bypass_insert(rq, 0); 2824 blk_mq_run_hw_queue(hctx, false); 2825 goto out; 2826 default: 2827 blk_mq_end_request(rq, ret); 2828 break; 2829 } 2830 } 2831 2832 out: 2833 if (ret != BLK_STS_OK) 2834 blk_mq_commit_rqs(hctx, queued, false); 2835 } 2836 2837 static void __blk_mq_flush_list(struct request_queue *q, struct rq_list *rqs) 2838 { 2839 if (blk_queue_quiesced(q)) 2840 return; 2841 q->mq_ops->queue_rqs(rqs); 2842 } 2843 2844 static unsigned blk_mq_extract_queue_requests(struct rq_list *rqs, 2845 struct rq_list *queue_rqs) 2846 { 2847 struct request *rq = rq_list_pop(rqs); 2848 struct request_queue *this_q = rq->q; 2849 struct request **prev = &rqs->head; 2850 struct rq_list matched_rqs = {}; 2851 struct request *last = NULL; 2852 unsigned depth = 1; 2853 2854 rq_list_add_tail(&matched_rqs, rq); 2855 while ((rq = *prev)) { 2856 if (rq->q == this_q) { 2857 /* move rq from rqs to matched_rqs */ 2858 *prev = rq->rq_next; 2859 rq_list_add_tail(&matched_rqs, rq); 2860 depth++; 2861 } else { 2862 /* leave rq in rqs */ 2863 prev = &rq->rq_next; 2864 last = rq; 2865 } 2866 } 2867 2868 rqs->tail = last; 2869 *queue_rqs = matched_rqs; 2870 return depth; 2871 } 2872 2873 static void blk_mq_dispatch_queue_requests(struct rq_list *rqs, unsigned depth) 2874 { 2875 struct request_queue *q = rq_list_peek(rqs)->q; 2876 2877 trace_block_unplug(q, depth, true); 2878 2879 /* 2880 * Peek first request and see if we have a ->queue_rqs() hook. 2881 * If we do, we can dispatch the whole list in one go. 2882 * We already know at this point that all requests belong to the 2883 * same queue, caller must ensure that's the case. 2884 */ 2885 if (q->mq_ops->queue_rqs) { 2886 blk_mq_run_dispatch_ops(q, __blk_mq_flush_list(q, rqs)); 2887 if (rq_list_empty(rqs)) 2888 return; 2889 } 2890 2891 blk_mq_run_dispatch_ops(q, blk_mq_issue_direct(rqs)); 2892 } 2893 2894 static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched) 2895 { 2896 struct blk_mq_hw_ctx *this_hctx = NULL; 2897 struct blk_mq_ctx *this_ctx = NULL; 2898 struct rq_list requeue_list = {}; 2899 unsigned int depth = 0; 2900 bool is_passthrough = false; 2901 LIST_HEAD(list); 2902 2903 do { 2904 struct request *rq = rq_list_pop(rqs); 2905 2906 if (!this_hctx) { 2907 this_hctx = rq->mq_hctx; 2908 this_ctx = rq->mq_ctx; 2909 is_passthrough = blk_rq_is_passthrough(rq); 2910 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx || 2911 is_passthrough != blk_rq_is_passthrough(rq)) { 2912 rq_list_add_tail(&requeue_list, rq); 2913 continue; 2914 } 2915 list_add_tail(&rq->queuelist, &list); 2916 depth++; 2917 } while (!rq_list_empty(rqs)); 2918 2919 *rqs = requeue_list; 2920 trace_block_unplug(this_hctx->queue, depth, !from_sched); 2921 2922 percpu_ref_get(&this_hctx->queue->q_usage_counter); 2923 /* passthrough requests should never be issued to the I/O scheduler */ 2924 if (is_passthrough) { 2925 spin_lock(&this_hctx->lock); 2926 list_splice_tail_init(&list, &this_hctx->dispatch); 2927 spin_unlock(&this_hctx->lock); 2928 blk_mq_run_hw_queue(this_hctx, from_sched); 2929 } else if (this_hctx->queue->elevator) { 2930 this_hctx->queue->elevator->type->ops.insert_requests(this_hctx, 2931 &list, 0); 2932 blk_mq_run_hw_queue(this_hctx, from_sched); 2933 } else { 2934 blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched); 2935 } 2936 percpu_ref_put(&this_hctx->queue->q_usage_counter); 2937 } 2938 2939 static void blk_mq_dispatch_multiple_queue_requests(struct rq_list *rqs) 2940 { 2941 do { 2942 struct rq_list queue_rqs; 2943 unsigned depth; 2944 2945 depth = blk_mq_extract_queue_requests(rqs, &queue_rqs); 2946 blk_mq_dispatch_queue_requests(&queue_rqs, depth); 2947 while (!rq_list_empty(&queue_rqs)) 2948 blk_mq_dispatch_list(&queue_rqs, false); 2949 } while (!rq_list_empty(rqs)); 2950 } 2951 2952 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 2953 { 2954 unsigned int depth; 2955 2956 /* 2957 * We may have been called recursively midway through handling 2958 * plug->mq_list via a schedule() in the driver's queue_rq() callback. 2959 * To avoid mq_list changing under our feet, clear rq_count early and 2960 * bail out specifically if rq_count is 0 rather than checking 2961 * whether the mq_list is empty. 2962 */ 2963 if (plug->rq_count == 0) 2964 return; 2965 depth = plug->rq_count; 2966 plug->rq_count = 0; 2967 2968 if (!plug->has_elevator && !from_schedule) { 2969 if (plug->multiple_queues) { 2970 blk_mq_dispatch_multiple_queue_requests(&plug->mq_list); 2971 return; 2972 } 2973 2974 blk_mq_dispatch_queue_requests(&plug->mq_list, depth); 2975 if (rq_list_empty(&plug->mq_list)) 2976 return; 2977 } 2978 2979 do { 2980 blk_mq_dispatch_list(&plug->mq_list, from_schedule); 2981 } while (!rq_list_empty(&plug->mq_list)); 2982 } 2983 2984 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 2985 struct list_head *list) 2986 { 2987 int queued = 0; 2988 blk_status_t ret = BLK_STS_OK; 2989 2990 while (!list_empty(list)) { 2991 struct request *rq = list_first_entry(list, struct request, 2992 queuelist); 2993 2994 list_del_init(&rq->queuelist); 2995 ret = blk_mq_request_issue_directly(rq, list_empty(list)); 2996 switch (ret) { 2997 case BLK_STS_OK: 2998 queued++; 2999 break; 3000 case BLK_STS_RESOURCE: 3001 case BLK_STS_DEV_RESOURCE: 3002 blk_mq_request_bypass_insert(rq, 0); 3003 if (list_empty(list)) 3004 blk_mq_run_hw_queue(hctx, false); 3005 goto out; 3006 default: 3007 blk_mq_end_request(rq, ret); 3008 break; 3009 } 3010 } 3011 3012 out: 3013 if (ret != BLK_STS_OK) 3014 blk_mq_commit_rqs(hctx, queued, false); 3015 } 3016 3017 static bool blk_mq_attempt_bio_merge(struct request_queue *q, 3018 struct bio *bio, unsigned int nr_segs) 3019 { 3020 if (!blk_queue_nomerges(q) && bio_mergeable(bio)) { 3021 if (blk_attempt_plug_merge(q, bio, nr_segs)) 3022 return true; 3023 if (blk_mq_sched_bio_merge(q, bio, nr_segs)) 3024 return true; 3025 } 3026 return false; 3027 } 3028 3029 static struct request *blk_mq_get_new_requests(struct request_queue *q, 3030 struct blk_plug *plug, 3031 struct bio *bio) 3032 { 3033 struct blk_mq_alloc_data data = { 3034 .q = q, 3035 .flags = 0, 3036 .shallow_depth = 0, 3037 .cmd_flags = bio->bi_opf, 3038 .rq_flags = 0, 3039 .nr_tags = 1, 3040 .cached_rqs = NULL, 3041 .ctx = NULL, 3042 .hctx = NULL 3043 }; 3044 struct request *rq; 3045 3046 rq_qos_throttle(q, bio); 3047 3048 if (plug) { 3049 data.nr_tags = plug->nr_ios; 3050 plug->nr_ios = 1; 3051 data.cached_rqs = &plug->cached_rqs; 3052 } 3053 3054 rq = __blk_mq_alloc_requests(&data); 3055 if (unlikely(!rq)) 3056 rq_qos_cleanup(q, bio); 3057 return rq; 3058 } 3059 3060 /* 3061 * Check if there is a suitable cached request and return it. 3062 */ 3063 static struct request *blk_mq_get_cached_request(struct blk_plug *plug, 3064 struct request_queue *q, blk_opf_t opf) 3065 { 3066 enum hctx_type type = blk_mq_get_hctx_type(opf); 3067 struct request *rq; 3068 3069 if (!plug) 3070 return NULL; 3071 rq = rq_list_peek(&plug->cached_rqs); 3072 if (!rq || rq->q != q) 3073 return NULL; 3074 if (type != rq->mq_hctx->type && 3075 (type != HCTX_TYPE_READ || rq->mq_hctx->type != HCTX_TYPE_DEFAULT)) 3076 return NULL; 3077 rq_list_pop(&plug->cached_rqs); 3078 return rq; 3079 } 3080 3081 static bool bio_unaligned(const struct bio *bio, struct request_queue *q) 3082 { 3083 unsigned int bs_mask = queue_logical_block_size(q) - 1; 3084 3085 /* .bi_sector of any zero sized bio need to be initialized */ 3086 if ((bio->bi_iter.bi_size & bs_mask) || 3087 ((bio->bi_iter.bi_sector << SECTOR_SHIFT) & bs_mask)) 3088 return true; 3089 return false; 3090 } 3091 3092 /** 3093 * blk_mq_submit_bio - Create and send a request to block device. 3094 * @bio: Bio pointer. 3095 * 3096 * Builds up a request structure from @q and @bio and send to the device. The 3097 * request may not be queued directly to hardware if: 3098 * * This request can be merged with another one 3099 * * We want to place request at plug queue for possible future merging 3100 * * There is an IO scheduler active at this queue 3101 * 3102 * It will not queue the request if there is an error with the bio, or at the 3103 * request creation. 3104 */ 3105 void blk_mq_submit_bio(struct bio *bio) 3106 { 3107 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 3108 struct blk_plug *plug = current->plug; 3109 const int is_sync = op_is_sync(bio->bi_opf); 3110 unsigned int integrity_action; 3111 struct blk_mq_hw_ctx *hctx; 3112 unsigned int nr_segs; 3113 struct request *rq; 3114 blk_status_t ret; 3115 3116 /* 3117 * If the plug has a cached request for this queue, try to use it. 3118 */ 3119 rq = blk_mq_get_cached_request(plug, q, bio->bi_opf); 3120 3121 /* 3122 * A BIO that was released from a zone write plug has already been 3123 * through the preparation in this function, already holds a reference 3124 * on the queue usage counter, and is the only write BIO in-flight for 3125 * the target zone. Go straight to preparing a request for it. 3126 */ 3127 if (bio_zone_write_plugging(bio)) { 3128 nr_segs = bio->__bi_nr_segments; 3129 if (rq) 3130 blk_queue_exit(q); 3131 goto new_request; 3132 } 3133 3134 /* 3135 * The cached request already holds a q_usage_counter reference and we 3136 * don't have to acquire a new one if we use it. 3137 */ 3138 if (!rq) { 3139 if (unlikely(bio_queue_enter(bio))) 3140 return; 3141 } 3142 3143 /* 3144 * Device reconfiguration may change logical block size or reduce the 3145 * number of poll queues, so the checks for alignment and poll support 3146 * have to be done with queue usage counter held. 3147 */ 3148 if (unlikely(bio_unaligned(bio, q))) { 3149 bio_io_error(bio); 3150 goto queue_exit; 3151 } 3152 3153 if (unlikely(op_is_dmabuf(bio->bi_opf) && 3154 bdev_has_integrity_csum(bio->bi_bdev))) { 3155 bio_endio_status(bio, BLK_STS_NOTSUPP); 3156 goto queue_exit; 3157 } 3158 3159 if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) { 3160 bio_endio_status(bio, BLK_STS_NOTSUPP); 3161 goto queue_exit; 3162 } 3163 3164 bio = __bio_split_to_limits(bio, &q->limits, &nr_segs); 3165 if (!bio) 3166 goto queue_exit; 3167 3168 integrity_action = bio_integrity_action(bio); 3169 if (integrity_action) 3170 bio_integrity_prep(bio, integrity_action); 3171 3172 blk_mq_bio_issue_init(q, bio); 3173 if (blk_mq_attempt_bio_merge(q, bio, nr_segs)) 3174 goto queue_exit; 3175 3176 if (bio_needs_zone_write_plugging(bio)) { 3177 if (blk_zone_plug_bio(bio, nr_segs)) 3178 goto queue_exit; 3179 } 3180 3181 new_request: 3182 if (rq) { 3183 rq_qos_throttle(rq->q, bio); 3184 rq->cmd_flags = bio->bi_opf; 3185 INIT_LIST_HEAD(&rq->queuelist); 3186 blk_mq_rq_late_init(rq, blk_time_get_ns()); 3187 } else { 3188 rq = blk_mq_get_new_requests(q, plug, bio); 3189 if (unlikely(!rq)) { 3190 if (bio->bi_opf & REQ_NOWAIT) 3191 bio_wouldblock_error(bio); 3192 goto queue_exit; 3193 } 3194 } 3195 3196 trace_block_getrq(bio); 3197 3198 rq_qos_track(q, rq, bio); 3199 3200 blk_mq_bio_to_request(rq, bio, nr_segs); 3201 3202 ret = blk_crypto_rq_get_keyslot(rq); 3203 if (ret != BLK_STS_OK) { 3204 bio_endio_status(bio, ret); 3205 blk_mq_free_request(rq); 3206 return; 3207 } 3208 3209 if (bio_zone_write_plugging(bio)) 3210 blk_zone_write_plug_init_request(rq); 3211 3212 if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq)) 3213 return; 3214 3215 if (plug) { 3216 blk_add_rq_to_plug(plug, rq); 3217 return; 3218 } 3219 3220 hctx = rq->mq_hctx; 3221 if ((rq->rq_flags & RQF_USE_SCHED) || 3222 (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) { 3223 blk_mq_insert_request(rq, 0); 3224 blk_mq_run_hw_queue(hctx, true); 3225 } else { 3226 blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq)); 3227 } 3228 return; 3229 3230 queue_exit: 3231 if (!rq) 3232 blk_queue_exit(q); 3233 else 3234 rq_list_add_head(&plug->cached_rqs, rq); 3235 } 3236 3237 #ifdef CONFIG_BLK_MQ_STACKING 3238 /** 3239 * blk_insert_cloned_request - Helper for stacking drivers to submit a request 3240 * @rq: the request being queued 3241 */ 3242 blk_status_t blk_insert_cloned_request(struct request *rq) 3243 { 3244 struct request_queue *q = rq->q; 3245 unsigned int max_sectors = blk_queue_get_max_sectors(rq); 3246 unsigned int max_segments = blk_rq_get_max_segments(rq); 3247 blk_status_t ret; 3248 3249 if (blk_rq_sectors(rq) > max_sectors) { 3250 /* 3251 * SCSI device does not have a good way to return if 3252 * Write Same/Zero is actually supported. If a device rejects 3253 * a non-read/write command (discard, write same,etc.) the 3254 * low-level device driver will set the relevant queue limit to 3255 * 0 to prevent blk-lib from issuing more of the offending 3256 * operations. Commands queued prior to the queue limit being 3257 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O 3258 * errors being propagated to upper layers. 3259 */ 3260 if (max_sectors == 0) 3261 return BLK_STS_NOTSUPP; 3262 3263 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n", 3264 __func__, blk_rq_sectors(rq), max_sectors); 3265 return BLK_STS_IOERR; 3266 } 3267 3268 /* 3269 * The queue settings related to segment counting may differ from the 3270 * original queue. 3271 */ 3272 rq->nr_phys_segments = blk_recalc_rq_segments(rq); 3273 if (rq->nr_phys_segments > max_segments) { 3274 printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n", 3275 __func__, rq->nr_phys_segments, max_segments); 3276 return BLK_STS_IOERR; 3277 } 3278 3279 /* 3280 * Integrity segment counting depends on the same queue limits 3281 * (virt_boundary_mask, seg_boundary_mask, max_segment_size) that 3282 * vary across stacked queues, so recompute against the bottom 3283 * queue just like nr_phys_segments above. 3284 */ 3285 if (blk_integrity_rq(rq) && rq->bio) { 3286 unsigned short max_int_segs = queue_max_integrity_segments(q); 3287 3288 rq->nr_integrity_segments = 3289 blk_rq_count_integrity_sg(rq->q, rq->bio); 3290 if (rq->nr_integrity_segments > max_int_segs) { 3291 printk(KERN_ERR "%s: over max integrity segments limit. (%u > %u)\n", 3292 __func__, rq->nr_integrity_segments, 3293 max_int_segs); 3294 return BLK_STS_IOERR; 3295 } 3296 } 3297 3298 if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq))) 3299 return BLK_STS_IOERR; 3300 3301 ret = blk_crypto_rq_get_keyslot(rq); 3302 if (ret != BLK_STS_OK) 3303 return ret; 3304 3305 blk_account_io_start(rq); 3306 3307 /* 3308 * Since we have a scheduler attached on the top device, 3309 * bypass a potential scheduler on the bottom device for 3310 * insert. 3311 */ 3312 blk_mq_run_dispatch_ops(q, 3313 ret = blk_mq_request_issue_directly(rq, true)); 3314 if (ret) 3315 blk_account_io_done(rq, blk_time_get_ns()); 3316 return ret; 3317 } 3318 EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 3319 3320 /** 3321 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request 3322 * @rq: the clone request to be cleaned up 3323 * 3324 * Description: 3325 * Free all bios in @rq for a cloned request. 3326 */ 3327 void blk_rq_unprep_clone(struct request *rq) 3328 { 3329 struct bio *bio; 3330 3331 while ((bio = rq->bio) != NULL) { 3332 rq->bio = bio->bi_next; 3333 3334 bio_put(bio); 3335 } 3336 } 3337 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 3338 3339 /** 3340 * blk_rq_prep_clone - Helper function to setup clone request 3341 * @rq: the request to be setup 3342 * @rq_src: original request to be cloned 3343 * @bs: bio_set that bios for clone are allocated from 3344 * @gfp_mask: memory allocation mask for bio 3345 * @bio_ctr: setup function to be called for each clone bio. 3346 * Returns %0 for success, non %0 for failure. 3347 * @data: private data to be passed to @bio_ctr 3348 * 3349 * Description: 3350 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. 3351 * Also, pages which the original bios are pointing to are not copied 3352 * and the cloned bios just point same pages. 3353 * So cloned bios must be completed before original bios, which means 3354 * the caller must complete @rq before @rq_src. 3355 */ 3356 int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 3357 struct bio_set *bs, gfp_t gfp_mask, 3358 int (*bio_ctr)(struct bio *, struct bio *, void *), 3359 void *data) 3360 { 3361 struct bio *bio_src; 3362 3363 if (!bs) 3364 bs = &fs_bio_set; 3365 3366 __rq_for_each_bio(bio_src, rq_src) { 3367 struct bio *bio = bio_alloc_clone(rq->q->disk->part0, bio_src, 3368 gfp_mask, bs); 3369 if (!bio) 3370 goto free_and_out; 3371 3372 if (bio_ctr && bio_ctr(bio, bio_src, data)) { 3373 bio_put(bio); 3374 goto free_and_out; 3375 } 3376 3377 if (rq->bio) { 3378 rq->biotail->bi_next = bio; 3379 rq->biotail = bio; 3380 } else { 3381 rq->bio = rq->biotail = bio; 3382 } 3383 } 3384 3385 /* Copy attributes of the original request to the clone request. */ 3386 rq->__sector = blk_rq_pos(rq_src); 3387 rq->__data_len = blk_rq_bytes(rq_src); 3388 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) { 3389 rq->rq_flags |= RQF_SPECIAL_PAYLOAD; 3390 rq->special_vec = rq_src->special_vec; 3391 } 3392 rq->nr_phys_segments = rq_src->nr_phys_segments; 3393 rq->nr_integrity_segments = rq_src->nr_integrity_segments; 3394 rq->phys_gap_bit = rq_src->phys_gap_bit; 3395 3396 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) 3397 goto free_and_out; 3398 3399 return 0; 3400 3401 free_and_out: 3402 blk_rq_unprep_clone(rq); 3403 3404 return -ENOMEM; 3405 } 3406 EXPORT_SYMBOL_GPL(blk_rq_prep_clone); 3407 #endif /* CONFIG_BLK_MQ_STACKING */ 3408 3409 /* 3410 * Steal bios from a request and add them to a bio list. 3411 * The request must not have been partially completed before. 3412 */ 3413 void blk_steal_bios(struct bio_list *list, struct request *rq) 3414 { 3415 struct bio *bio; 3416 3417 for (bio = rq->bio; bio; bio = bio->bi_next) { 3418 if (bio->bi_opf & REQ_POLLED) { 3419 bio->bi_opf &= ~REQ_POLLED; 3420 bio->bi_cookie = BLK_QC_T_NONE; 3421 } 3422 /* 3423 * The alternate request queue that we may end up submitting 3424 * the bio to may be frozen temporarily, in this case REQ_NOWAIT 3425 * will fail the I/O immediately with EAGAIN to the issuer. 3426 * We are not in the issuer context which cannot block. Clear 3427 * the flag to avoid spurious EAGAIN I/O failures. 3428 */ 3429 bio->bi_opf &= ~REQ_NOWAIT; 3430 bio_clear_flag(bio, BIO_QOS_THROTTLED); 3431 bio_clear_flag(bio, BIO_QOS_MERGED); 3432 } 3433 3434 if (rq->bio) { 3435 if (list->tail) 3436 list->tail->bi_next = rq->bio; 3437 else 3438 list->head = rq->bio; 3439 list->tail = rq->biotail; 3440 3441 rq->bio = NULL; 3442 rq->biotail = NULL; 3443 } 3444 3445 rq->__data_len = 0; 3446 } 3447 EXPORT_SYMBOL_GPL(blk_steal_bios); 3448 3449 static size_t order_to_size(unsigned int order) 3450 { 3451 return (size_t)PAGE_SIZE << order; 3452 } 3453 3454 /* called before freeing request pool in @tags */ 3455 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, 3456 struct blk_mq_tags *tags) 3457 { 3458 struct page *page; 3459 3460 /* 3461 * There is no need to clear mapping if driver tags is not initialized 3462 * or the mapping belongs to the driver tags. 3463 */ 3464 if (!drv_tags || drv_tags == tags) 3465 return; 3466 3467 list_for_each_entry(page, &tags->page_list, lru) { 3468 unsigned long start = (unsigned long)page_address(page); 3469 unsigned long end = start + order_to_size(page->private); 3470 int i; 3471 3472 for (i = 0; i < drv_tags->nr_tags; i++) { 3473 struct request *rq = drv_tags->rqs[i]; 3474 unsigned long rq_addr = (unsigned long)rq; 3475 3476 if (rq_addr >= start && rq_addr < end) { 3477 WARN_ON_ONCE(req_ref_read(rq) != 0); 3478 cmpxchg(&drv_tags->rqs[i], rq, NULL); 3479 } 3480 } 3481 } 3482 } 3483 3484 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 3485 unsigned int hctx_idx) 3486 { 3487 struct blk_mq_tags *drv_tags; 3488 3489 if (list_empty(&tags->page_list)) 3490 return; 3491 3492 if (blk_mq_is_shared_tags(set->flags)) 3493 drv_tags = set->shared_tags; 3494 else 3495 drv_tags = set->tags[hctx_idx]; 3496 3497 if (tags->static_rqs && set->ops->exit_request) { 3498 int i; 3499 3500 for (i = 0; i < tags->nr_tags; i++) { 3501 struct request *rq = tags->static_rqs[i]; 3502 3503 if (!rq) 3504 continue; 3505 set->ops->exit_request(set, rq, hctx_idx); 3506 tags->static_rqs[i] = NULL; 3507 } 3508 } 3509 3510 blk_mq_clear_rq_mapping(drv_tags, tags); 3511 /* 3512 * Free request pages in SRCU callback, which is called from 3513 * blk_mq_free_tags(). 3514 */ 3515 } 3516 3517 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 3518 { 3519 kfree(tags->rqs); 3520 tags->rqs = NULL; 3521 kfree(tags->static_rqs); 3522 tags->static_rqs = NULL; 3523 3524 blk_mq_free_tags(tags); 3525 } 3526 3527 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set, 3528 unsigned int hctx_idx) 3529 { 3530 int i; 3531 3532 for (i = 0; i < set->nr_maps; i++) { 3533 unsigned int start = set->map[i].queue_offset; 3534 unsigned int end = start + set->map[i].nr_queues; 3535 3536 if (hctx_idx >= start && hctx_idx < end) 3537 break; 3538 } 3539 3540 if (i >= set->nr_maps) 3541 i = HCTX_TYPE_DEFAULT; 3542 3543 return i; 3544 } 3545 3546 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set, 3547 unsigned int hctx_idx) 3548 { 3549 enum hctx_type type = hctx_idx_to_type(set, hctx_idx); 3550 3551 return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx); 3552 } 3553 3554 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 3555 unsigned int hctx_idx, 3556 unsigned int nr_tags, 3557 unsigned int reserved_tags) 3558 { 3559 int node = blk_mq_get_hctx_node(set, hctx_idx); 3560 struct blk_mq_tags *tags; 3561 3562 if (node == NUMA_NO_NODE) 3563 node = set->numa_node; 3564 3565 tags = blk_mq_init_tags(nr_tags, reserved_tags, set->flags, node); 3566 if (!tags) 3567 return NULL; 3568 3569 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 3570 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 3571 node); 3572 if (!tags->rqs) 3573 goto err_free_tags; 3574 3575 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 3576 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 3577 node); 3578 if (!tags->static_rqs) 3579 goto err_free_rqs; 3580 3581 return tags; 3582 3583 err_free_rqs: 3584 kfree(tags->rqs); 3585 err_free_tags: 3586 blk_mq_free_tags(tags); 3587 return NULL; 3588 } 3589 3590 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 3591 unsigned int hctx_idx, int node) 3592 { 3593 int ret; 3594 3595 if (set->ops->init_request) { 3596 ret = set->ops->init_request(set, rq, hctx_idx, node); 3597 if (ret) 3598 return ret; 3599 } 3600 3601 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 3602 return 0; 3603 } 3604 3605 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, 3606 struct blk_mq_tags *tags, 3607 unsigned int hctx_idx, unsigned int depth) 3608 { 3609 unsigned int i, j, entries_per_page, max_order = 4; 3610 int node = blk_mq_get_hctx_node(set, hctx_idx); 3611 size_t rq_size, left; 3612 3613 if (node == NUMA_NO_NODE) 3614 node = set->numa_node; 3615 3616 /* 3617 * rq_size is the size of the request plus driver payload, rounded 3618 * to the cacheline size 3619 */ 3620 rq_size = round_up(sizeof(struct request) + set->cmd_size, 3621 cache_line_size()); 3622 left = rq_size * depth; 3623 3624 for (i = 0; i < depth; ) { 3625 int this_order = max_order; 3626 struct page *page; 3627 int to_do; 3628 void *p; 3629 3630 while (this_order && left < order_to_size(this_order - 1)) 3631 this_order--; 3632 3633 do { 3634 page = alloc_pages_node(node, 3635 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 3636 this_order); 3637 if (page) 3638 break; 3639 if (!this_order--) 3640 break; 3641 if (order_to_size(this_order) < rq_size) 3642 break; 3643 } while (1); 3644 3645 if (!page) 3646 goto fail; 3647 3648 page->private = this_order; 3649 list_add_tail(&page->lru, &tags->page_list); 3650 3651 p = page_address(page); 3652 /* 3653 * Allow kmemleak to scan these pages as they contain pointers 3654 * to additional allocations like via ops->init_request(). 3655 */ 3656 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 3657 entries_per_page = order_to_size(this_order) / rq_size; 3658 to_do = min(entries_per_page, depth - i); 3659 left -= to_do * rq_size; 3660 for (j = 0; j < to_do; j++) { 3661 struct request *rq = p; 3662 3663 tags->static_rqs[i] = rq; 3664 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 3665 tags->static_rqs[i] = NULL; 3666 goto fail; 3667 } 3668 3669 p += rq_size; 3670 i++; 3671 } 3672 } 3673 return 0; 3674 3675 fail: 3676 blk_mq_free_rqs(set, tags, hctx_idx); 3677 return -ENOMEM; 3678 } 3679 3680 struct rq_iter_data { 3681 struct blk_mq_hw_ctx *hctx; 3682 bool has_rq; 3683 }; 3684 3685 static bool blk_mq_has_request(struct request *rq, void *data) 3686 { 3687 struct rq_iter_data *iter_data = data; 3688 3689 if (rq->mq_hctx != iter_data->hctx) 3690 return true; 3691 iter_data->has_rq = true; 3692 return false; 3693 } 3694 3695 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) 3696 { 3697 struct blk_mq_tags *tags = hctx->sched_tags ? 3698 hctx->sched_tags : hctx->tags; 3699 struct rq_iter_data data = { 3700 .hctx = hctx, 3701 }; 3702 int srcu_idx; 3703 3704 srcu_idx = srcu_read_lock(&blk_mq_tags_srcu); 3705 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data); 3706 srcu_read_unlock(&blk_mq_tags_srcu, srcu_idx); 3707 3708 return data.has_rq; 3709 } 3710 3711 static bool blk_mq_hctx_has_online_cpu(struct blk_mq_hw_ctx *hctx, 3712 unsigned int this_cpu) 3713 { 3714 enum hctx_type type = hctx->type; 3715 int cpu; 3716 3717 /* 3718 * hctx->cpumask has to rule out isolated CPUs, but userspace still 3719 * might submit IOs on these isolated CPUs, so use the queue map to 3720 * check if all CPUs mapped to this hctx are offline 3721 */ 3722 for_each_online_cpu(cpu) { 3723 struct blk_mq_hw_ctx *h = blk_mq_map_queue_type(hctx->queue, 3724 type, cpu); 3725 3726 if (h != hctx) 3727 continue; 3728 3729 /* this hctx has at least one online CPU */ 3730 if (this_cpu != cpu) 3731 return true; 3732 } 3733 3734 return false; 3735 } 3736 3737 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) 3738 { 3739 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 3740 struct blk_mq_hw_ctx, cpuhp_online); 3741 int ret = 0; 3742 3743 if (!hctx->nr_ctx || blk_mq_hctx_has_online_cpu(hctx, cpu)) 3744 return 0; 3745 3746 /* 3747 * Prevent new request from being allocated on the current hctx. 3748 * 3749 * The smp_mb__after_atomic() Pairs with the implied barrier in 3750 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is 3751 * seen once we return from the tag allocator. 3752 */ 3753 set_bit(BLK_MQ_S_INACTIVE, &hctx->state); 3754 smp_mb__after_atomic(); 3755 3756 /* 3757 * Try to grab a reference to the queue and wait for any outstanding 3758 * requests. If we could not grab a reference the queue has been 3759 * frozen and there are no requests. 3760 */ 3761 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) { 3762 while (blk_mq_hctx_has_requests(hctx)) { 3763 /* 3764 * The wakeup capable IRQ handler of block device is 3765 * not called during suspend. Skip the loop by checking 3766 * pm_wakeup_pending to prevent the deadlock and improve 3767 * suspend latency. 3768 */ 3769 if (pm_wakeup_pending()) { 3770 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); 3771 ret = -EBUSY; 3772 break; 3773 } 3774 msleep(5); 3775 } 3776 percpu_ref_put(&hctx->queue->q_usage_counter); 3777 } 3778 3779 return ret; 3780 } 3781 3782 /* 3783 * Check if one CPU is mapped to the specified hctx 3784 * 3785 * Isolated CPUs have been ruled out from hctx->cpumask, which is supposed 3786 * to be used for scheduling kworker only. For other usage, please call this 3787 * helper for checking if one CPU belongs to the specified hctx 3788 */ 3789 static bool blk_mq_cpu_mapped_to_hctx(unsigned int cpu, 3790 const struct blk_mq_hw_ctx *hctx) 3791 { 3792 struct blk_mq_hw_ctx *mapped_hctx = blk_mq_map_queue_type(hctx->queue, 3793 hctx->type, cpu); 3794 3795 return mapped_hctx == hctx; 3796 } 3797 3798 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node) 3799 { 3800 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 3801 struct blk_mq_hw_ctx, cpuhp_online); 3802 3803 if (blk_mq_cpu_mapped_to_hctx(cpu, hctx)) 3804 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); 3805 return 0; 3806 } 3807 3808 /* 3809 * 'cpu' is going away. splice any existing rq_list entries from this 3810 * software queue to the hw queue dispatch list, and ensure that it 3811 * gets run. 3812 */ 3813 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 3814 { 3815 struct blk_mq_hw_ctx *hctx; 3816 struct blk_mq_ctx *ctx; 3817 LIST_HEAD(tmp); 3818 enum hctx_type type; 3819 3820 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 3821 if (!blk_mq_cpu_mapped_to_hctx(cpu, hctx)) 3822 return 0; 3823 3824 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 3825 type = hctx->type; 3826 3827 spin_lock(&ctx->lock); 3828 if (!list_empty(&ctx->rq_lists[type])) { 3829 list_splice_init(&ctx->rq_lists[type], &tmp); 3830 blk_mq_hctx_clear_pending(hctx, ctx); 3831 } 3832 spin_unlock(&ctx->lock); 3833 3834 if (list_empty(&tmp)) 3835 return 0; 3836 3837 spin_lock(&hctx->lock); 3838 list_splice_tail_init(&tmp, &hctx->dispatch); 3839 spin_unlock(&hctx->lock); 3840 3841 blk_mq_run_hw_queue(hctx, true); 3842 return 0; 3843 } 3844 3845 static void __blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 3846 { 3847 lockdep_assert_held(&blk_mq_cpuhp_lock); 3848 3849 if (!(hctx->flags & BLK_MQ_F_STACKING) && 3850 !hlist_unhashed(&hctx->cpuhp_online)) { 3851 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 3852 &hctx->cpuhp_online); 3853 INIT_HLIST_NODE(&hctx->cpuhp_online); 3854 } 3855 3856 if (!hlist_unhashed(&hctx->cpuhp_dead)) { 3857 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 3858 &hctx->cpuhp_dead); 3859 INIT_HLIST_NODE(&hctx->cpuhp_dead); 3860 } 3861 } 3862 3863 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 3864 { 3865 mutex_lock(&blk_mq_cpuhp_lock); 3866 __blk_mq_remove_cpuhp(hctx); 3867 mutex_unlock(&blk_mq_cpuhp_lock); 3868 } 3869 3870 static void __blk_mq_add_cpuhp(struct blk_mq_hw_ctx *hctx) 3871 { 3872 lockdep_assert_held(&blk_mq_cpuhp_lock); 3873 3874 if (!(hctx->flags & BLK_MQ_F_STACKING) && 3875 hlist_unhashed(&hctx->cpuhp_online)) 3876 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 3877 &hctx->cpuhp_online); 3878 3879 if (hlist_unhashed(&hctx->cpuhp_dead)) 3880 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, 3881 &hctx->cpuhp_dead); 3882 } 3883 3884 static void __blk_mq_remove_cpuhp_list(struct list_head *head) 3885 { 3886 struct blk_mq_hw_ctx *hctx; 3887 3888 lockdep_assert_held(&blk_mq_cpuhp_lock); 3889 3890 list_for_each_entry(hctx, head, hctx_list) 3891 __blk_mq_remove_cpuhp(hctx); 3892 } 3893 3894 /* 3895 * Unregister cpuhp callbacks from exited hw queues 3896 * 3897 * Safe to call if this `request_queue` is live 3898 */ 3899 static void blk_mq_remove_hw_queues_cpuhp(struct request_queue *q) 3900 { 3901 LIST_HEAD(hctx_list); 3902 3903 spin_lock(&q->unused_hctx_lock); 3904 list_splice_init(&q->unused_hctx_list, &hctx_list); 3905 spin_unlock(&q->unused_hctx_lock); 3906 3907 mutex_lock(&blk_mq_cpuhp_lock); 3908 __blk_mq_remove_cpuhp_list(&hctx_list); 3909 mutex_unlock(&blk_mq_cpuhp_lock); 3910 3911 spin_lock(&q->unused_hctx_lock); 3912 list_splice(&hctx_list, &q->unused_hctx_list); 3913 spin_unlock(&q->unused_hctx_lock); 3914 } 3915 3916 /* 3917 * Register cpuhp callbacks from all hw queues 3918 * 3919 * Safe to call if this `request_queue` is live 3920 */ 3921 static void blk_mq_add_hw_queues_cpuhp(struct request_queue *q) 3922 { 3923 struct blk_mq_hw_ctx *hctx; 3924 unsigned long i; 3925 3926 mutex_lock(&blk_mq_cpuhp_lock); 3927 queue_for_each_hw_ctx(q, hctx, i) 3928 __blk_mq_add_cpuhp(hctx); 3929 mutex_unlock(&blk_mq_cpuhp_lock); 3930 } 3931 3932 /* 3933 * Before freeing hw queue, clearing the flush request reference in 3934 * tags->rqs[] for avoiding potential UAF. 3935 */ 3936 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, 3937 unsigned int queue_depth, struct request *flush_rq) 3938 { 3939 int i; 3940 3941 /* The hw queue may not be mapped yet */ 3942 if (!tags) 3943 return; 3944 3945 WARN_ON_ONCE(req_ref_read(flush_rq) != 0); 3946 3947 for (i = 0; i < queue_depth; i++) 3948 cmpxchg(&tags->rqs[i], flush_rq, NULL); 3949 } 3950 3951 static void blk_free_flush_queue_callback(struct rcu_head *head) 3952 { 3953 struct blk_flush_queue *fq = 3954 container_of(head, struct blk_flush_queue, rcu_head); 3955 3956 blk_free_flush_queue(fq); 3957 } 3958 3959 /* hctx->ctxs will be freed in queue's release handler */ 3960 static void blk_mq_exit_hctx(struct request_queue *q, 3961 struct blk_mq_tag_set *set, 3962 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 3963 { 3964 struct request *flush_rq = hctx->fq->flush_rq; 3965 3966 if (blk_mq_hw_queue_mapped(hctx)) 3967 blk_mq_tag_idle(hctx); 3968 3969 if (blk_queue_init_done(q)) 3970 blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx], 3971 set->queue_depth, flush_rq); 3972 if (set->ops->exit_request) 3973 set->ops->exit_request(set, flush_rq, hctx_idx); 3974 3975 if (set->ops->exit_hctx) 3976 set->ops->exit_hctx(hctx, hctx_idx); 3977 3978 call_srcu(&blk_mq_tags_srcu, &hctx->fq->rcu_head, 3979 blk_free_flush_queue_callback); 3980 hctx->fq = NULL; 3981 3982 spin_lock(&q->unused_hctx_lock); 3983 list_add(&hctx->hctx_list, &q->unused_hctx_list); 3984 spin_unlock(&q->unused_hctx_lock); 3985 } 3986 3987 static void blk_mq_exit_hw_queues(struct request_queue *q, 3988 struct blk_mq_tag_set *set, int nr_queue) 3989 { 3990 struct blk_mq_hw_ctx *hctx; 3991 unsigned long i; 3992 3993 queue_for_each_hw_ctx(q, hctx, i) { 3994 if (i == nr_queue) 3995 break; 3996 blk_mq_remove_cpuhp(hctx); 3997 blk_mq_exit_hctx(q, set, hctx, i); 3998 } 3999 } 4000 4001 static int blk_mq_init_hctx(struct request_queue *q, 4002 struct blk_mq_tag_set *set, 4003 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 4004 { 4005 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 4006 4007 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp); 4008 if (!hctx->fq) 4009 goto fail; 4010 4011 hctx->queue_num = hctx_idx; 4012 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); 4013 4014 hctx->tags = set->tags[hctx_idx]; 4015 4016 if (set->ops->init_hctx && 4017 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 4018 goto fail_free_fq; 4019 4020 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, 4021 hctx->numa_node)) 4022 goto exit_hctx; 4023 4024 return 0; 4025 4026 exit_hctx: 4027 if (set->ops->exit_hctx) 4028 set->ops->exit_hctx(hctx, hctx_idx); 4029 fail_free_fq: 4030 blk_free_flush_queue(hctx->fq); 4031 hctx->fq = NULL; 4032 fail: 4033 return -1; 4034 } 4035 4036 static struct blk_mq_hw_ctx * 4037 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, 4038 int node) 4039 { 4040 struct blk_mq_hw_ctx *hctx; 4041 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 4042 4043 hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node); 4044 if (!hctx) 4045 goto fail_alloc_hctx; 4046 4047 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node)) 4048 goto free_hctx; 4049 4050 atomic_set(&hctx->nr_active, 0); 4051 if (node == NUMA_NO_NODE) 4052 node = set->numa_node; 4053 hctx->numa_node = node; 4054 4055 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 4056 spin_lock_init(&hctx->lock); 4057 INIT_LIST_HEAD(&hctx->dispatch); 4058 INIT_HLIST_NODE(&hctx->cpuhp_dead); 4059 INIT_HLIST_NODE(&hctx->cpuhp_online); 4060 hctx->queue = q; 4061 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED; 4062 4063 INIT_LIST_HEAD(&hctx->hctx_list); 4064 4065 /* 4066 * Allocate space for all possible cpus to avoid allocation at 4067 * runtime 4068 */ 4069 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 4070 gfp, node); 4071 if (!hctx->ctxs) 4072 goto free_cpumask; 4073 4074 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), 4075 gfp, node, false, false)) 4076 goto free_ctxs; 4077 hctx->nr_ctx = 0; 4078 4079 spin_lock_init(&hctx->dispatch_wait_lock); 4080 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 4081 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 4082 4083 blk_mq_hctx_kobj_init(hctx); 4084 4085 return hctx; 4086 4087 free_ctxs: 4088 kfree(hctx->ctxs); 4089 free_cpumask: 4090 free_cpumask_var(hctx->cpumask); 4091 free_hctx: 4092 kfree(hctx); 4093 fail_alloc_hctx: 4094 return NULL; 4095 } 4096 4097 static void blk_mq_init_cpu_queues(struct request_queue *q, 4098 unsigned int nr_hw_queues) 4099 { 4100 struct blk_mq_tag_set *set = q->tag_set; 4101 unsigned int i, j; 4102 4103 for_each_possible_cpu(i) { 4104 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 4105 struct blk_mq_hw_ctx *hctx; 4106 int k; 4107 4108 __ctx->cpu = i; 4109 spin_lock_init(&__ctx->lock); 4110 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++) 4111 INIT_LIST_HEAD(&__ctx->rq_lists[k]); 4112 4113 __ctx->queue = q; 4114 4115 /* 4116 * Set local node, IFF we have more than one hw queue. If 4117 * not, we remain on the home node of the device 4118 */ 4119 for (j = 0; j < set->nr_maps; j++) { 4120 hctx = blk_mq_map_queue_type(q, j, i); 4121 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 4122 hctx->numa_node = cpu_to_node(i); 4123 } 4124 } 4125 } 4126 4127 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, 4128 unsigned int hctx_idx, 4129 unsigned int depth) 4130 { 4131 struct blk_mq_tags *tags; 4132 int ret; 4133 4134 tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags); 4135 if (!tags) 4136 return NULL; 4137 4138 ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth); 4139 if (ret) { 4140 blk_mq_free_rq_map(tags); 4141 return NULL; 4142 } 4143 4144 return tags; 4145 } 4146 4147 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, 4148 int hctx_idx) 4149 { 4150 if (blk_mq_is_shared_tags(set->flags)) { 4151 set->tags[hctx_idx] = set->shared_tags; 4152 4153 return true; 4154 } 4155 4156 set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx, 4157 set->queue_depth); 4158 4159 return set->tags[hctx_idx]; 4160 } 4161 4162 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, 4163 struct blk_mq_tags *tags, 4164 unsigned int hctx_idx) 4165 { 4166 if (tags) { 4167 blk_mq_free_rqs(set, tags, hctx_idx); 4168 blk_mq_free_rq_map(tags); 4169 } 4170 } 4171 4172 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, 4173 unsigned int hctx_idx) 4174 { 4175 if (!blk_mq_is_shared_tags(set->flags)) 4176 blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx); 4177 4178 set->tags[hctx_idx] = NULL; 4179 } 4180 4181 static void blk_mq_map_swqueue(struct request_queue *q) 4182 { 4183 unsigned int j, hctx_idx; 4184 unsigned long i; 4185 struct blk_mq_hw_ctx *hctx; 4186 struct blk_mq_ctx *ctx; 4187 struct blk_mq_tag_set *set = q->tag_set; 4188 4189 queue_for_each_hw_ctx(q, hctx, i) { 4190 cpumask_clear(hctx->cpumask); 4191 hctx->nr_ctx = 0; 4192 hctx->dispatch_from = NULL; 4193 } 4194 4195 /* 4196 * Map software to hardware queues. 4197 * 4198 * If the cpu isn't present, the cpu is mapped to first hctx. 4199 */ 4200 for_each_possible_cpu(i) { 4201 4202 ctx = per_cpu_ptr(q->queue_ctx, i); 4203 for (j = 0; j < set->nr_maps; j++) { 4204 if (!set->map[j].nr_queues) { 4205 ctx->hctxs[j] = blk_mq_map_queue_type(q, 4206 HCTX_TYPE_DEFAULT, i); 4207 continue; 4208 } 4209 hctx_idx = set->map[j].mq_map[i]; 4210 /* unmapped hw queue can be remapped after CPU topo changed */ 4211 if (!set->tags[hctx_idx] && 4212 !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) { 4213 /* 4214 * If tags initialization fail for some hctx, 4215 * that hctx won't be brought online. In this 4216 * case, remap the current ctx to hctx[0] which 4217 * is guaranteed to always have tags allocated 4218 */ 4219 set->map[j].mq_map[i] = 0; 4220 } 4221 4222 hctx = blk_mq_map_queue_type(q, j, i); 4223 ctx->hctxs[j] = hctx; 4224 /* 4225 * If the CPU is already set in the mask, then we've 4226 * mapped this one already. This can happen if 4227 * devices share queues across queue maps. 4228 */ 4229 if (cpumask_test_cpu(i, hctx->cpumask)) 4230 continue; 4231 4232 cpumask_set_cpu(i, hctx->cpumask); 4233 hctx->type = j; 4234 ctx->index_hw[hctx->type] = hctx->nr_ctx; 4235 hctx->ctxs[hctx->nr_ctx++] = ctx; 4236 4237 /* 4238 * If the nr_ctx type overflows, we have exceeded the 4239 * amount of sw queues we can support. 4240 */ 4241 BUG_ON(!hctx->nr_ctx); 4242 } 4243 4244 for (; j < HCTX_MAX_TYPES; j++) 4245 ctx->hctxs[j] = blk_mq_map_queue_type(q, 4246 HCTX_TYPE_DEFAULT, i); 4247 } 4248 4249 queue_for_each_hw_ctx(q, hctx, i) { 4250 int cpu; 4251 4252 /* 4253 * If no software queues are mapped to this hardware queue, 4254 * disable it and free the request entries. 4255 */ 4256 if (!hctx->nr_ctx) { 4257 /* Never unmap queue 0. We need it as a 4258 * fallback in case of a new remap fails 4259 * allocation 4260 */ 4261 if (i) 4262 __blk_mq_free_map_and_rqs(set, i); 4263 4264 hctx->tags = NULL; 4265 continue; 4266 } 4267 4268 hctx->tags = set->tags[i]; 4269 WARN_ON(!hctx->tags); 4270 4271 /* 4272 * Set the map size to the number of mapped software queues. 4273 * This is more accurate and more efficient than looping 4274 * over all possibly mapped software queues. 4275 */ 4276 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 4277 4278 /* 4279 * Rule out isolated CPUs from hctx->cpumask to avoid 4280 * running block kworker on isolated CPUs. 4281 * FIXME: cpuset should propagate further changes to isolated CPUs 4282 * here. 4283 */ 4284 rcu_read_lock(); 4285 for_each_cpu(cpu, hctx->cpumask) { 4286 if (cpu_is_isolated(cpu)) 4287 cpumask_clear_cpu(cpu, hctx->cpumask); 4288 } 4289 rcu_read_unlock(); 4290 4291 /* 4292 * Initialize batch roundrobin counts 4293 */ 4294 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 4295 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 4296 } 4297 } 4298 4299 /* 4300 * Caller needs to ensure that we're either frozen/quiesced, or that 4301 * the queue isn't live yet. 4302 */ 4303 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 4304 { 4305 struct blk_mq_hw_ctx *hctx; 4306 unsigned long i; 4307 4308 queue_for_each_hw_ctx(q, hctx, i) { 4309 if (shared) { 4310 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED; 4311 } else { 4312 blk_mq_tag_idle(hctx); 4313 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; 4314 } 4315 } 4316 } 4317 4318 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set, 4319 bool shared) 4320 { 4321 struct request_queue *q; 4322 unsigned int memflags; 4323 4324 lockdep_assert_held(&set->tag_list_lock); 4325 4326 list_for_each_entry(q, &set->tag_list, tag_set_list) { 4327 memflags = blk_mq_freeze_queue(q); 4328 queue_set_hctx_shared(q, shared); 4329 blk_mq_unfreeze_queue(q, memflags); 4330 } 4331 } 4332 4333 static void blk_mq_del_queue_tag_set(struct request_queue *q) 4334 { 4335 struct blk_mq_tag_set *set = q->tag_set; 4336 4337 mutex_lock(&set->tag_list_lock); 4338 list_del_rcu(&q->tag_set_list); 4339 if (list_is_singular(&set->tag_list)) { 4340 /* just transitioned to unshared */ 4341 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; 4342 /* update existing queue */ 4343 blk_mq_update_tag_set_shared(set, false); 4344 } 4345 mutex_unlock(&set->tag_list_lock); 4346 } 4347 4348 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 4349 struct request_queue *q) 4350 { 4351 mutex_lock(&set->tag_list_lock); 4352 4353 /* 4354 * Check to see if we're transitioning to shared (from 1 to 2 queues). 4355 */ 4356 if (!list_empty(&set->tag_list) && 4357 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) { 4358 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED; 4359 /* update existing queue */ 4360 blk_mq_update_tag_set_shared(set, true); 4361 } 4362 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED) 4363 queue_set_hctx_shared(q, true); 4364 list_add_tail_rcu(&q->tag_set_list, &set->tag_list); 4365 4366 mutex_unlock(&set->tag_list_lock); 4367 } 4368 4369 /* All allocations will be freed in release handler of q->mq_kobj */ 4370 static int blk_mq_alloc_ctxs(struct request_queue *q) 4371 { 4372 struct blk_mq_ctxs *ctxs; 4373 int cpu; 4374 4375 ctxs = kzalloc_obj(*ctxs); 4376 if (!ctxs) 4377 return -ENOMEM; 4378 4379 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx); 4380 if (!ctxs->queue_ctx) 4381 goto fail; 4382 4383 for_each_possible_cpu(cpu) { 4384 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu); 4385 ctx->ctxs = ctxs; 4386 } 4387 4388 q->mq_kobj = &ctxs->kobj; 4389 q->queue_ctx = ctxs->queue_ctx; 4390 4391 return 0; 4392 fail: 4393 kfree(ctxs); 4394 return -ENOMEM; 4395 } 4396 4397 /* 4398 * It is the actual release handler for mq, but we do it from 4399 * request queue's release handler for avoiding use-after-free 4400 * and headache because q->mq_kobj shouldn't have been introduced, 4401 * but we can't group ctx/kctx kobj without it. 4402 */ 4403 void blk_mq_release(struct request_queue *q) 4404 { 4405 struct blk_mq_hw_ctx *hctx, *next; 4406 unsigned long i; 4407 4408 queue_for_each_hw_ctx(q, hctx, i) 4409 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list)); 4410 4411 /* all hctx are in .unused_hctx_list now */ 4412 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) { 4413 list_del_init(&hctx->hctx_list); 4414 kobject_put(&hctx->kobj); 4415 } 4416 4417 kfree(q->queue_hw_ctx); 4418 4419 /* 4420 * release .mq_kobj and sw queue's kobject now because 4421 * both share lifetime with request queue. 4422 */ 4423 blk_mq_sysfs_deinit(q); 4424 } 4425 4426 struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set, 4427 struct queue_limits *lim, void *queuedata) 4428 { 4429 struct queue_limits default_lim = { }; 4430 struct request_queue *q; 4431 int ret; 4432 4433 if (!lim) 4434 lim = &default_lim; 4435 lim->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT; 4436 if (set->nr_maps > HCTX_TYPE_POLL) 4437 lim->features |= BLK_FEAT_POLL; 4438 4439 q = blk_alloc_queue(lim, set->numa_node); 4440 if (IS_ERR(q)) 4441 return q; 4442 q->queuedata = queuedata; 4443 ret = blk_mq_init_allocated_queue(set, q); 4444 if (ret) { 4445 blk_put_queue(q); 4446 return ERR_PTR(ret); 4447 } 4448 return q; 4449 } 4450 EXPORT_SYMBOL(blk_mq_alloc_queue); 4451 4452 /** 4453 * blk_mq_destroy_queue - shutdown a request queue 4454 * @q: request queue to shutdown 4455 * 4456 * This shuts down a request queue allocated by blk_mq_alloc_queue(). All future 4457 * requests will be failed with -ENODEV. The caller is responsible for dropping 4458 * the reference from blk_mq_alloc_queue() by calling blk_put_queue(). 4459 * 4460 * Context: can sleep 4461 */ 4462 void blk_mq_destroy_queue(struct request_queue *q) 4463 { 4464 WARN_ON_ONCE(!queue_is_mq(q)); 4465 WARN_ON_ONCE(blk_queue_registered(q)); 4466 4467 might_sleep(); 4468 4469 blk_queue_flag_set(QUEUE_FLAG_DYING, q); 4470 blk_queue_start_drain(q); 4471 blk_mq_freeze_queue_wait(q); 4472 4473 blk_sync_queue(q); 4474 blk_mq_cancel_work_sync(q); 4475 blk_mq_exit_queue(q); 4476 } 4477 EXPORT_SYMBOL(blk_mq_destroy_queue); 4478 4479 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, 4480 struct queue_limits *lim, void *queuedata, 4481 struct lock_class_key *lkclass) 4482 { 4483 struct request_queue *q; 4484 struct gendisk *disk; 4485 4486 q = blk_mq_alloc_queue(set, lim, queuedata); 4487 if (IS_ERR(q)) 4488 return ERR_CAST(q); 4489 4490 disk = __alloc_disk_node(q, set->numa_node, lkclass); 4491 if (!disk) { 4492 blk_mq_destroy_queue(q); 4493 blk_put_queue(q); 4494 return ERR_PTR(-ENOMEM); 4495 } 4496 set_bit(GD_OWNS_QUEUE, &disk->state); 4497 return disk; 4498 } 4499 EXPORT_SYMBOL(__blk_mq_alloc_disk); 4500 4501 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q, 4502 struct lock_class_key *lkclass) 4503 { 4504 struct gendisk *disk; 4505 4506 if (!blk_get_queue(q)) 4507 return NULL; 4508 disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass); 4509 if (!disk) 4510 blk_put_queue(q); 4511 return disk; 4512 } 4513 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue); 4514 4515 /* 4516 * Only hctx removed from cpuhp list can be reused 4517 */ 4518 static bool blk_mq_hctx_is_reusable(struct blk_mq_hw_ctx *hctx) 4519 { 4520 return hlist_unhashed(&hctx->cpuhp_online) && 4521 hlist_unhashed(&hctx->cpuhp_dead); 4522 } 4523 4524 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( 4525 struct blk_mq_tag_set *set, struct request_queue *q, 4526 int hctx_idx, int node) 4527 { 4528 struct blk_mq_hw_ctx *hctx = NULL, *tmp; 4529 4530 /* reuse dead hctx first */ 4531 spin_lock(&q->unused_hctx_lock); 4532 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) { 4533 if (tmp->numa_node == node && blk_mq_hctx_is_reusable(tmp)) { 4534 hctx = tmp; 4535 break; 4536 } 4537 } 4538 if (hctx) 4539 list_del_init(&hctx->hctx_list); 4540 spin_unlock(&q->unused_hctx_lock); 4541 4542 if (!hctx) 4543 hctx = blk_mq_alloc_hctx(q, set, node); 4544 if (!hctx) 4545 goto fail; 4546 4547 if (blk_mq_init_hctx(q, set, hctx, hctx_idx)) 4548 goto free_hctx; 4549 4550 return hctx; 4551 4552 free_hctx: 4553 kobject_put(&hctx->kobj); 4554 fail: 4555 return NULL; 4556 } 4557 4558 static void __blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 4559 struct request_queue *q) 4560 { 4561 int i, j, end; 4562 struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; 4563 4564 if (q->nr_hw_queues < set->nr_hw_queues) { 4565 struct blk_mq_hw_ctx **new_hctxs; 4566 4567 new_hctxs = kcalloc_node(set->nr_hw_queues, 4568 sizeof(*new_hctxs), GFP_KERNEL, 4569 set->numa_node); 4570 if (!new_hctxs) 4571 return; 4572 if (hctxs) 4573 memcpy(new_hctxs, hctxs, q->nr_hw_queues * 4574 sizeof(*hctxs)); 4575 rcu_assign_pointer(q->queue_hw_ctx, new_hctxs); 4576 /* 4577 * Make sure reading the old queue_hw_ctx from other 4578 * context concurrently won't trigger uaf. 4579 */ 4580 kfree_rcu_mightsleep(hctxs); 4581 hctxs = new_hctxs; 4582 } 4583 4584 for (i = 0; i < set->nr_hw_queues; i++) { 4585 int old_node; 4586 int node = blk_mq_get_hctx_node(set, i); 4587 struct blk_mq_hw_ctx *old_hctx = hctxs[i]; 4588 4589 if (old_hctx) { 4590 old_node = old_hctx->numa_node; 4591 blk_mq_exit_hctx(q, set, old_hctx, i); 4592 } 4593 4594 hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i, node); 4595 if (!hctxs[i]) { 4596 if (!old_hctx) 4597 break; 4598 pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n", 4599 node, old_node); 4600 hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i, 4601 old_node); 4602 WARN_ON_ONCE(!hctxs[i]); 4603 } 4604 } 4605 /* 4606 * Increasing nr_hw_queues fails. Free the newly allocated 4607 * hctxs and keep the previous q->nr_hw_queues. 4608 */ 4609 if (i != set->nr_hw_queues) { 4610 j = q->nr_hw_queues; 4611 end = i; 4612 } else { 4613 j = i; 4614 end = q->nr_hw_queues; 4615 q->nr_hw_queues = set->nr_hw_queues; 4616 } 4617 4618 for (; j < end; j++) { 4619 struct blk_mq_hw_ctx *hctx = hctxs[j]; 4620 4621 if (hctx) { 4622 blk_mq_exit_hctx(q, set, hctx, j); 4623 hctxs[j] = NULL; 4624 } 4625 } 4626 } 4627 4628 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 4629 struct request_queue *q) 4630 { 4631 __blk_mq_realloc_hw_ctxs(set, q); 4632 4633 /* unregister cpuhp callbacks for exited hctxs */ 4634 blk_mq_remove_hw_queues_cpuhp(q); 4635 4636 /* register cpuhp for new initialized hctxs */ 4637 blk_mq_add_hw_queues_cpuhp(q); 4638 } 4639 4640 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 4641 struct request_queue *q) 4642 { 4643 /* mark the queue as mq asap */ 4644 q->mq_ops = set->ops; 4645 4646 /* 4647 * ->tag_set has to be setup before initialize hctx, which cpuphp 4648 * handler needs it for checking queue mapping 4649 */ 4650 q->tag_set = set; 4651 4652 if (blk_mq_alloc_ctxs(q)) 4653 goto err_exit; 4654 4655 /* init q->mq_kobj and sw queues' kobjects */ 4656 blk_mq_sysfs_init(q); 4657 4658 INIT_LIST_HEAD(&q->unused_hctx_list); 4659 spin_lock_init(&q->unused_hctx_lock); 4660 4661 blk_mq_realloc_hw_ctxs(set, q); 4662 if (!q->nr_hw_queues) 4663 goto err_hctxs; 4664 4665 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 4666 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 4667 4668 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 4669 4670 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 4671 INIT_LIST_HEAD(&q->flush_list); 4672 INIT_LIST_HEAD(&q->requeue_list); 4673 spin_lock_init(&q->requeue_lock); 4674 4675 q->nr_requests = set->queue_depth; 4676 q->async_depth = set->queue_depth; 4677 4678 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 4679 blk_mq_map_swqueue(q); 4680 blk_mq_add_queue_tag_set(set, q); 4681 return 0; 4682 4683 err_hctxs: 4684 blk_mq_release(q); 4685 err_exit: 4686 q->mq_ops = NULL; 4687 return -ENOMEM; 4688 } 4689 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 4690 4691 /* tags can _not_ be used after returning from blk_mq_exit_queue */ 4692 void blk_mq_exit_queue(struct request_queue *q) 4693 { 4694 struct blk_mq_tag_set *set = q->tag_set; 4695 4696 /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */ 4697 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 4698 /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */ 4699 blk_mq_del_queue_tag_set(q); 4700 } 4701 4702 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 4703 { 4704 int i; 4705 4706 if (blk_mq_is_shared_tags(set->flags)) { 4707 set->shared_tags = blk_mq_alloc_map_and_rqs(set, 4708 BLK_MQ_NO_HCTX_IDX, 4709 set->queue_depth); 4710 if (!set->shared_tags) 4711 return -ENOMEM; 4712 } 4713 4714 for (i = 0; i < set->nr_hw_queues; i++) { 4715 if (!__blk_mq_alloc_map_and_rqs(set, i)) 4716 goto out_unwind; 4717 cond_resched(); 4718 } 4719 4720 return 0; 4721 4722 out_unwind: 4723 while (--i >= 0) 4724 __blk_mq_free_map_and_rqs(set, i); 4725 4726 if (blk_mq_is_shared_tags(set->flags)) { 4727 blk_mq_free_map_and_rqs(set, set->shared_tags, 4728 BLK_MQ_NO_HCTX_IDX); 4729 } 4730 4731 return -ENOMEM; 4732 } 4733 4734 /* 4735 * Allocate the request maps associated with this tag_set. Note that this 4736 * may reduce the depth asked for, if memory is tight. set->queue_depth 4737 * will be updated to reflect the allocated depth. 4738 */ 4739 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set) 4740 { 4741 unsigned int depth; 4742 int err; 4743 4744 depth = set->queue_depth; 4745 do { 4746 err = __blk_mq_alloc_rq_maps(set); 4747 if (!err) 4748 break; 4749 4750 set->queue_depth >>= 1; 4751 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 4752 err = -ENOMEM; 4753 break; 4754 } 4755 } while (set->queue_depth); 4756 4757 if (!set->queue_depth || err) { 4758 pr_err("blk-mq: failed to allocate request map\n"); 4759 return -ENOMEM; 4760 } 4761 4762 if (depth != set->queue_depth) 4763 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 4764 depth, set->queue_depth); 4765 4766 return 0; 4767 } 4768 4769 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) 4770 { 4771 /* 4772 * blk_mq_map_queues() and multiple .map_queues() implementations 4773 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the 4774 * number of hardware queues. 4775 */ 4776 if (set->nr_maps == 1) 4777 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues; 4778 4779 if (set->ops->map_queues) { 4780 int i; 4781 4782 /* 4783 * transport .map_queues is usually done in the following 4784 * way: 4785 * 4786 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 4787 * mask = get_cpu_mask(queue) 4788 * for_each_cpu(cpu, mask) 4789 * set->map[x].mq_map[cpu] = queue; 4790 * } 4791 * 4792 * When we need to remap, the table has to be cleared for 4793 * killing stale mapping since one CPU may not be mapped 4794 * to any hw queue. 4795 */ 4796 for (i = 0; i < set->nr_maps; i++) 4797 blk_mq_clear_mq_map(&set->map[i]); 4798 4799 set->ops->map_queues(set); 4800 } else { 4801 BUG_ON(set->nr_maps > 1); 4802 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 4803 } 4804 } 4805 4806 static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags( 4807 struct blk_mq_tag_set *set, 4808 int new_nr_hw_queues) 4809 { 4810 struct blk_mq_tags **new_tags; 4811 int i; 4812 4813 if (set->nr_hw_queues >= new_nr_hw_queues) 4814 return NULL; 4815 4816 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), 4817 GFP_KERNEL, set->numa_node); 4818 if (!new_tags) 4819 return ERR_PTR(-ENOMEM); 4820 4821 if (set->tags) 4822 memcpy(new_tags, set->tags, set->nr_hw_queues * 4823 sizeof(*set->tags)); 4824 4825 for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) { 4826 if (blk_mq_is_shared_tags(set->flags)) { 4827 new_tags[i] = set->shared_tags; 4828 } else { 4829 new_tags[i] = blk_mq_alloc_map_and_rqs(set, i, 4830 set->queue_depth); 4831 if (!new_tags[i]) 4832 goto out_unwind; 4833 } 4834 cond_resched(); 4835 } 4836 4837 return new_tags; 4838 out_unwind: 4839 while (--i >= set->nr_hw_queues) { 4840 if (!blk_mq_is_shared_tags(set->flags)) 4841 blk_mq_free_map_and_rqs(set, new_tags[i], i); 4842 } 4843 kfree(new_tags); 4844 return ERR_PTR(-ENOMEM); 4845 } 4846 4847 /* 4848 * Alloc a tag set to be associated with one or more request queues. 4849 * May fail with EINVAL for various error conditions. May adjust the 4850 * requested depth down, if it's too large. In that case, the set 4851 * value will be stored in set->queue_depth. 4852 */ 4853 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 4854 { 4855 int i, ret; 4856 4857 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 4858 4859 if (!set->nr_hw_queues) 4860 return -EINVAL; 4861 if (!set->queue_depth) 4862 return -EINVAL; 4863 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 4864 return -EINVAL; 4865 4866 if (!set->ops->queue_rq) 4867 return -EINVAL; 4868 4869 if (!set->ops->get_budget ^ !set->ops->put_budget) 4870 return -EINVAL; 4871 4872 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 4873 pr_info("blk-mq: reduced tag depth to %u\n", 4874 BLK_MQ_MAX_DEPTH); 4875 set->queue_depth = BLK_MQ_MAX_DEPTH; 4876 } 4877 4878 if (!set->nr_maps) 4879 set->nr_maps = 1; 4880 else if (set->nr_maps > HCTX_MAX_TYPES) 4881 return -EINVAL; 4882 4883 /* 4884 * If a crashdump is active, then we are potentially in a very 4885 * memory constrained environment. Limit us to 64 tags to prevent 4886 * using too much memory. 4887 */ 4888 if (is_kdump_kernel()) 4889 set->queue_depth = min(64U, set->queue_depth); 4890 4891 /* 4892 * There is no use for more h/w queues than cpus if we just have 4893 * a single map 4894 */ 4895 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids) 4896 set->nr_hw_queues = nr_cpu_ids; 4897 4898 if (set->flags & BLK_MQ_F_BLOCKING) { 4899 set->srcu = kmalloc_obj(*set->srcu); 4900 if (!set->srcu) 4901 return -ENOMEM; 4902 ret = init_srcu_struct(set->srcu); 4903 if (ret) 4904 goto out_free_srcu; 4905 } 4906 4907 init_rwsem(&set->update_nr_hwq_lock); 4908 4909 ret = -ENOMEM; 4910 set->tags = kcalloc_node(set->nr_hw_queues, 4911 sizeof(struct blk_mq_tags *), GFP_KERNEL, 4912 set->numa_node); 4913 if (!set->tags) 4914 goto out_cleanup_srcu; 4915 4916 for (i = 0; i < set->nr_maps; i++) { 4917 set->map[i].mq_map = kcalloc_node(nr_cpu_ids, 4918 sizeof(set->map[i].mq_map[0]), 4919 GFP_KERNEL, set->numa_node); 4920 if (!set->map[i].mq_map) 4921 goto out_free_mq_map; 4922 set->map[i].nr_queues = set->nr_hw_queues; 4923 } 4924 4925 blk_mq_update_queue_map(set); 4926 4927 ret = blk_mq_alloc_set_map_and_rqs(set); 4928 if (ret) 4929 goto out_free_mq_map; 4930 4931 mutex_init(&set->tag_list_lock); 4932 INIT_LIST_HEAD(&set->tag_list); 4933 4934 return 0; 4935 4936 out_free_mq_map: 4937 for (i = 0; i < set->nr_maps; i++) { 4938 kfree(set->map[i].mq_map); 4939 set->map[i].mq_map = NULL; 4940 } 4941 kfree(set->tags); 4942 set->tags = NULL; 4943 out_cleanup_srcu: 4944 if (set->flags & BLK_MQ_F_BLOCKING) 4945 cleanup_srcu_struct(set->srcu); 4946 out_free_srcu: 4947 if (set->flags & BLK_MQ_F_BLOCKING) 4948 kfree(set->srcu); 4949 return ret; 4950 } 4951 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 4952 4953 /* allocate and initialize a tagset for a simple single-queue device */ 4954 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set, 4955 const struct blk_mq_ops *ops, unsigned int queue_depth, 4956 unsigned int set_flags) 4957 { 4958 memset(set, 0, sizeof(*set)); 4959 set->ops = ops; 4960 set->nr_hw_queues = 1; 4961 set->nr_maps = 1; 4962 set->queue_depth = queue_depth; 4963 set->numa_node = NUMA_NO_NODE; 4964 set->flags = set_flags; 4965 return blk_mq_alloc_tag_set(set); 4966 } 4967 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set); 4968 4969 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 4970 { 4971 int i, j; 4972 4973 for (i = 0; i < set->nr_hw_queues; i++) 4974 __blk_mq_free_map_and_rqs(set, i); 4975 4976 if (blk_mq_is_shared_tags(set->flags)) { 4977 blk_mq_free_map_and_rqs(set, set->shared_tags, 4978 BLK_MQ_NO_HCTX_IDX); 4979 } 4980 4981 for (j = 0; j < set->nr_maps; j++) { 4982 kfree(set->map[j].mq_map); 4983 set->map[j].mq_map = NULL; 4984 } 4985 4986 kfree(set->tags); 4987 set->tags = NULL; 4988 4989 if (set->flags & BLK_MQ_F_BLOCKING) { 4990 srcu_barrier(set->srcu); 4991 cleanup_srcu_struct(set->srcu); 4992 kfree(set->srcu); 4993 } 4994 } 4995 EXPORT_SYMBOL(blk_mq_free_tag_set); 4996 4997 struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q, 4998 struct elevator_tags *et, 4999 unsigned int nr) 5000 { 5001 struct blk_mq_tag_set *set = q->tag_set; 5002 struct elevator_tags *old_et = NULL; 5003 struct blk_mq_hw_ctx *hctx; 5004 unsigned long i; 5005 5006 blk_mq_quiesce_queue(q); 5007 5008 if (blk_mq_is_shared_tags(set->flags)) { 5009 /* 5010 * Shared tags, for sched tags, we allocate max initially hence 5011 * tags can't grow, see blk_mq_alloc_sched_tags(). 5012 */ 5013 if (q->elevator) 5014 blk_mq_tag_update_sched_shared_tags(q, nr); 5015 else 5016 blk_mq_tag_resize_shared_tags(set, nr); 5017 } else if (!q->elevator) { 5018 /* 5019 * Non-shared hardware tags, nr is already checked from 5020 * queue_requests_store() and tags can't grow. 5021 */ 5022 queue_for_each_hw_ctx(q, hctx, i) { 5023 if (!hctx->tags) 5024 continue; 5025 sbitmap_queue_resize(&hctx->tags->bitmap_tags, 5026 nr - hctx->tags->nr_reserved_tags); 5027 } 5028 } else if (nr <= q->elevator->et->nr_requests) { 5029 /* Non-shared sched tags, and tags don't grow. */ 5030 queue_for_each_hw_ctx(q, hctx, i) { 5031 if (!hctx->sched_tags) 5032 continue; 5033 sbitmap_queue_resize(&hctx->sched_tags->bitmap_tags, 5034 nr - hctx->sched_tags->nr_reserved_tags); 5035 } 5036 } else { 5037 /* Non-shared sched tags, and tags grow */ 5038 queue_for_each_hw_ctx(q, hctx, i) 5039 hctx->sched_tags = et->tags[i]; 5040 old_et = q->elevator->et; 5041 q->elevator->et = et; 5042 } 5043 5044 /* 5045 * Preserve relative value, both nr and async_depth are at most 16 bit 5046 * value, no need to worry about overflow. 5047 */ 5048 q->async_depth = max(q->async_depth * nr / q->nr_requests, 1); 5049 q->nr_requests = nr; 5050 if (q->elevator && q->elevator->type->ops.depth_updated) 5051 q->elevator->type->ops.depth_updated(q); 5052 5053 blk_mq_unquiesce_queue(q); 5054 return old_et; 5055 } 5056 5057 /* 5058 * Switch back to the elevator type stored in the xarray. 5059 */ 5060 static void blk_mq_elv_switch_back(struct request_queue *q, 5061 struct xarray *elv_tbl) 5062 { 5063 struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id); 5064 5065 if (WARN_ON_ONCE(!ctx)) 5066 return; 5067 5068 /* The elv_update_nr_hw_queues unfreezes the queue. */ 5069 elv_update_nr_hw_queues(q, ctx); 5070 5071 /* Drop the reference acquired in blk_mq_elv_switch_none. */ 5072 if (ctx->type) 5073 elevator_put(ctx->type); 5074 } 5075 5076 /* 5077 * Stores elevator name and type in ctx and set current elevator to none. 5078 */ 5079 static int blk_mq_elv_switch_none(struct request_queue *q, 5080 struct xarray *elv_tbl) 5081 { 5082 struct elv_change_ctx *ctx; 5083 5084 lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock); 5085 5086 /* 5087 * Accessing q->elevator without holding q->elevator_lock is safe here 5088 * because we're called from nr_hw_queue update which is protected by 5089 * set->update_nr_hwq_lock in the writer context. So, scheduler update/ 5090 * switch code (which acquires the same lock in the reader context) 5091 * can't run concurrently. 5092 */ 5093 if (q->elevator) { 5094 ctx = xa_load(elv_tbl, q->id); 5095 if (WARN_ON_ONCE(!ctx)) 5096 return -ENOENT; 5097 5098 ctx->name = q->elevator->type->elevator_name; 5099 5100 /* 5101 * Before we switch elevator to 'none', take a reference to 5102 * the elevator module so that while nr_hw_queue update is 5103 * running, no one can remove elevator module. We'd put the 5104 * reference to elevator module later when we switch back 5105 * elevator. 5106 */ 5107 __elevator_get(q->elevator->type); 5108 5109 /* 5110 * Store elevator type so that we can release the reference 5111 * taken above later. 5112 */ 5113 ctx->type = q->elevator->type; 5114 elevator_set_none(q); 5115 } 5116 return 0; 5117 } 5118 5119 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 5120 int nr_hw_queues) 5121 { 5122 struct request_queue *q; 5123 int prev_nr_hw_queues = set->nr_hw_queues; 5124 unsigned int memflags; 5125 int i; 5126 struct xarray elv_tbl; 5127 struct blk_mq_tags **new_tags; 5128 bool queues_frozen = false; 5129 5130 lockdep_assert_held(&set->tag_list_lock); 5131 5132 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids) 5133 nr_hw_queues = nr_cpu_ids; 5134 if (nr_hw_queues < 1) 5135 return; 5136 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues) 5137 return; 5138 5139 memflags = memalloc_noio_save(); 5140 5141 xa_init(&elv_tbl); 5142 if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0) 5143 goto out_free_ctx; 5144 5145 if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0) 5146 goto out_free_ctx; 5147 5148 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5149 blk_mq_debugfs_unregister_hctxs(q); 5150 blk_mq_sysfs_unregister_hctxs(q); 5151 } 5152 5153 /* 5154 * Switch IO scheduler to 'none', cleaning up the data associated 5155 * with the previous scheduler. We will switch back once we are done 5156 * updating the new sw to hw queue mappings. 5157 */ 5158 list_for_each_entry(q, &set->tag_list, tag_set_list) 5159 if (blk_mq_elv_switch_none(q, &elv_tbl)) 5160 goto switch_back; 5161 5162 new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues); 5163 if (IS_ERR(new_tags)) 5164 goto switch_back; 5165 5166 list_for_each_entry(q, &set->tag_list, tag_set_list) 5167 blk_mq_freeze_queue_nomemsave(q); 5168 queues_frozen = true; 5169 if (new_tags) { 5170 kfree(set->tags); 5171 set->tags = new_tags; 5172 } 5173 set->nr_hw_queues = nr_hw_queues; 5174 5175 fallback: 5176 blk_mq_update_queue_map(set); 5177 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5178 __blk_mq_realloc_hw_ctxs(set, q); 5179 5180 if (q->nr_hw_queues != set->nr_hw_queues) { 5181 int i = prev_nr_hw_queues; 5182 5183 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n", 5184 nr_hw_queues, prev_nr_hw_queues); 5185 for (; i < set->nr_hw_queues; i++) 5186 __blk_mq_free_map_and_rqs(set, i); 5187 5188 set->nr_hw_queues = prev_nr_hw_queues; 5189 goto fallback; 5190 } 5191 blk_mq_map_swqueue(q); 5192 } 5193 switch_back: 5194 /* The blk_mq_elv_switch_back unfreezes queue for us. */ 5195 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5196 /* switch_back expects queue to be frozen */ 5197 if (!queues_frozen) 5198 blk_mq_freeze_queue_nomemsave(q); 5199 blk_mq_elv_switch_back(q, &elv_tbl); 5200 } 5201 5202 list_for_each_entry(q, &set->tag_list, tag_set_list) { 5203 blk_mq_sysfs_register_hctxs(q); 5204 blk_mq_debugfs_register_hctxs(q); 5205 5206 blk_mq_remove_hw_queues_cpuhp(q); 5207 blk_mq_add_hw_queues_cpuhp(q); 5208 } 5209 5210 out_free_ctx: 5211 blk_mq_free_sched_ctx_batch(&elv_tbl); 5212 xa_destroy(&elv_tbl); 5213 memalloc_noio_restore(memflags); 5214 5215 /* Free the excess tags when nr_hw_queues shrink. */ 5216 for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++) 5217 __blk_mq_free_map_and_rqs(set, i); 5218 } 5219 5220 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 5221 { 5222 down_write(&set->update_nr_hwq_lock); 5223 mutex_lock(&set->tag_list_lock); 5224 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 5225 mutex_unlock(&set->tag_list_lock); 5226 up_write(&set->update_nr_hwq_lock); 5227 } 5228 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 5229 5230 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, 5231 struct io_comp_batch *iob, unsigned int flags) 5232 { 5233 int ret; 5234 unsigned long timeout = jiffies + 2; 5235 5236 do { 5237 ret = q->mq_ops->poll(hctx, iob); 5238 if (ret > 0) 5239 return ret; 5240 if (task_sigpending(current)) 5241 return 1; 5242 if (ret < 0 || (flags & BLK_POLL_ONESHOT)) 5243 break; 5244 cpu_relax(); 5245 } while (!need_resched() && time_before(jiffies, timeout)); 5246 5247 return 0; 5248 } 5249 5250 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, 5251 struct io_comp_batch *iob, unsigned int flags) 5252 { 5253 if (!blk_mq_can_poll(q)) 5254 return 0; 5255 return blk_hctx_poll(q, q->queue_hw_ctx[cookie], iob, flags); 5256 } 5257 5258 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob, 5259 unsigned int poll_flags) 5260 { 5261 struct request_queue *q = rq->q; 5262 int ret; 5263 5264 if (!blk_rq_is_poll(rq)) 5265 return 0; 5266 if (!percpu_ref_tryget(&q->q_usage_counter)) 5267 return 0; 5268 5269 ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags); 5270 blk_queue_exit(q); 5271 5272 return ret; 5273 } 5274 EXPORT_SYMBOL_GPL(blk_rq_poll); 5275 5276 unsigned int blk_mq_rq_cpu(struct request *rq) 5277 { 5278 return rq->mq_ctx->cpu; 5279 } 5280 EXPORT_SYMBOL(blk_mq_rq_cpu); 5281 5282 void blk_mq_cancel_work_sync(struct request_queue *q) 5283 { 5284 struct blk_mq_hw_ctx *hctx; 5285 unsigned long i; 5286 5287 cancel_delayed_work_sync(&q->requeue_work); 5288 5289 queue_for_each_hw_ctx(q, hctx, i) 5290 cancel_delayed_work_sync(&hctx->run_work); 5291 } 5292 5293 static int __init blk_mq_init(void) 5294 { 5295 int i; 5296 5297 for_each_possible_cpu(i) 5298 init_llist_head(&per_cpu(blk_cpu_done, i)); 5299 for_each_possible_cpu(i) 5300 INIT_CSD(&per_cpu(blk_cpu_csd, i), 5301 __blk_mq_complete_request_remote, NULL); 5302 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); 5303 5304 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, 5305 "block/softirq:dead", NULL, 5306 blk_softirq_cpu_dead); 5307 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 5308 blk_mq_hctx_notify_dead); 5309 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online", 5310 blk_mq_hctx_notify_online, 5311 blk_mq_hctx_notify_offline); 5312 return 0; 5313 } 5314 subsys_initcall(blk_mq_init); 5315