1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block multiqueue core code 4 * 5 * Copyright (C) 2013-2014 Jens Axboe 6 * Copyright (C) 2013-2014 Christoph Hellwig 7 */ 8 #include <linux/kernel.h> 9 #include <linux/module.h> 10 #include <linux/backing-dev.h> 11 #include <linux/bio.h> 12 #include <linux/blkdev.h> 13 #include <linux/blk-integrity.h> 14 #include <linux/kmemleak.h> 15 #include <linux/mm.h> 16 #include <linux/init.h> 17 #include <linux/slab.h> 18 #include <linux/workqueue.h> 19 #include <linux/smp.h> 20 #include <linux/interrupt.h> 21 #include <linux/llist.h> 22 #include <linux/cpu.h> 23 #include <linux/cache.h> 24 #include <linux/sched/sysctl.h> 25 #include <linux/sched/topology.h> 26 #include <linux/sched/signal.h> 27 #include <linux/delay.h> 28 #include <linux/crash_dump.h> 29 #include <linux/prefetch.h> 30 #include <linux/blk-crypto.h> 31 #include <linux/part_stat.h> 32 33 #include <trace/events/block.h> 34 35 #include <linux/t10-pi.h> 36 #include "blk.h" 37 #include "blk-mq.h" 38 #include "blk-mq-debugfs.h" 39 #include "blk-pm.h" 40 #include "blk-stat.h" 41 #include "blk-mq-sched.h" 42 #include "blk-rq-qos.h" 43 #include "blk-ioprio.h" 44 45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); 46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd); 47 48 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags); 49 static void blk_mq_request_bypass_insert(struct request *rq, 50 blk_insert_t flags); 51 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 52 struct list_head *list); 53 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, 54 struct io_comp_batch *iob, unsigned int flags); 55 56 /* 57 * Check if any of the ctx, dispatch list or elevator 58 * have pending work in this hardware queue. 59 */ 60 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 61 { 62 return !list_empty_careful(&hctx->dispatch) || 63 sbitmap_any_bit_set(&hctx->ctx_map) || 64 blk_mq_sched_has_work(hctx); 65 } 66 67 /* 68 * Mark this ctx as having pending work in this hardware queue 69 */ 70 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, 71 struct blk_mq_ctx *ctx) 72 { 73 const int bit = ctx->index_hw[hctx->type]; 74 75 if (!sbitmap_test_bit(&hctx->ctx_map, bit)) 76 sbitmap_set_bit(&hctx->ctx_map, bit); 77 } 78 79 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, 80 struct blk_mq_ctx *ctx) 81 { 82 const int bit = ctx->index_hw[hctx->type]; 83 84 sbitmap_clear_bit(&hctx->ctx_map, bit); 85 } 86 87 struct mq_inflight { 88 struct block_device *part; 89 unsigned int inflight[2]; 90 }; 91 92 static bool blk_mq_check_inflight(struct request *rq, void *priv) 93 { 94 struct mq_inflight *mi = priv; 95 96 if (rq->part && blk_do_io_stat(rq) && 97 (!mi->part->bd_partno || rq->part == mi->part) && 98 blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT) 99 mi->inflight[rq_data_dir(rq)]++; 100 101 return true; 102 } 103 104 unsigned int blk_mq_in_flight(struct request_queue *q, 105 struct block_device *part) 106 { 107 struct mq_inflight mi = { .part = part }; 108 109 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 110 111 return mi.inflight[0] + mi.inflight[1]; 112 } 113 114 void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part, 115 unsigned int inflight[2]) 116 { 117 struct mq_inflight mi = { .part = part }; 118 119 blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); 120 inflight[0] = mi.inflight[0]; 121 inflight[1] = mi.inflight[1]; 122 } 123 124 void blk_freeze_queue_start(struct request_queue *q) 125 { 126 mutex_lock(&q->mq_freeze_lock); 127 if (++q->mq_freeze_depth == 1) { 128 percpu_ref_kill(&q->q_usage_counter); 129 mutex_unlock(&q->mq_freeze_lock); 130 if (queue_is_mq(q)) 131 blk_mq_run_hw_queues(q, false); 132 } else { 133 mutex_unlock(&q->mq_freeze_lock); 134 } 135 } 136 EXPORT_SYMBOL_GPL(blk_freeze_queue_start); 137 138 void blk_mq_freeze_queue_wait(struct request_queue *q) 139 { 140 wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); 141 } 142 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); 143 144 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, 145 unsigned long timeout) 146 { 147 return wait_event_timeout(q->mq_freeze_wq, 148 percpu_ref_is_zero(&q->q_usage_counter), 149 timeout); 150 } 151 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); 152 153 /* 154 * Guarantee no request is in use, so we can change any data structure of 155 * the queue afterward. 156 */ 157 void blk_freeze_queue(struct request_queue *q) 158 { 159 /* 160 * In the !blk_mq case we are only calling this to kill the 161 * q_usage_counter, otherwise this increases the freeze depth 162 * and waits for it to return to zero. For this reason there is 163 * no blk_unfreeze_queue(), and blk_freeze_queue() is not 164 * exported to drivers as the only user for unfreeze is blk_mq. 165 */ 166 blk_freeze_queue_start(q); 167 blk_mq_freeze_queue_wait(q); 168 } 169 170 void blk_mq_freeze_queue(struct request_queue *q) 171 { 172 /* 173 * ...just an alias to keep freeze and unfreeze actions balanced 174 * in the blk_mq_* namespace 175 */ 176 blk_freeze_queue(q); 177 } 178 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); 179 180 void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic) 181 { 182 mutex_lock(&q->mq_freeze_lock); 183 if (force_atomic) 184 q->q_usage_counter.data->force_atomic = true; 185 q->mq_freeze_depth--; 186 WARN_ON_ONCE(q->mq_freeze_depth < 0); 187 if (!q->mq_freeze_depth) { 188 percpu_ref_resurrect(&q->q_usage_counter); 189 wake_up_all(&q->mq_freeze_wq); 190 } 191 mutex_unlock(&q->mq_freeze_lock); 192 } 193 194 void blk_mq_unfreeze_queue(struct request_queue *q) 195 { 196 __blk_mq_unfreeze_queue(q, false); 197 } 198 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); 199 200 /* 201 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the 202 * mpt3sas driver such that this function can be removed. 203 */ 204 void blk_mq_quiesce_queue_nowait(struct request_queue *q) 205 { 206 unsigned long flags; 207 208 spin_lock_irqsave(&q->queue_lock, flags); 209 if (!q->quiesce_depth++) 210 blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q); 211 spin_unlock_irqrestore(&q->queue_lock, flags); 212 } 213 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); 214 215 /** 216 * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done 217 * @set: tag_set to wait on 218 * 219 * Note: it is driver's responsibility for making sure that quiesce has 220 * been started on or more of the request_queues of the tag_set. This 221 * function only waits for the quiesce on those request_queues that had 222 * the quiesce flag set using blk_mq_quiesce_queue_nowait. 223 */ 224 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set) 225 { 226 if (set->flags & BLK_MQ_F_BLOCKING) 227 synchronize_srcu(set->srcu); 228 else 229 synchronize_rcu(); 230 } 231 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done); 232 233 /** 234 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished 235 * @q: request queue. 236 * 237 * Note: this function does not prevent that the struct request end_io() 238 * callback function is invoked. Once this function is returned, we make 239 * sure no dispatch can happen until the queue is unquiesced via 240 * blk_mq_unquiesce_queue(). 241 */ 242 void blk_mq_quiesce_queue(struct request_queue *q) 243 { 244 blk_mq_quiesce_queue_nowait(q); 245 /* nothing to wait for non-mq queues */ 246 if (queue_is_mq(q)) 247 blk_mq_wait_quiesce_done(q->tag_set); 248 } 249 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); 250 251 /* 252 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() 253 * @q: request queue. 254 * 255 * This function recovers queue into the state before quiescing 256 * which is done by blk_mq_quiesce_queue. 257 */ 258 void blk_mq_unquiesce_queue(struct request_queue *q) 259 { 260 unsigned long flags; 261 bool run_queue = false; 262 263 spin_lock_irqsave(&q->queue_lock, flags); 264 if (WARN_ON_ONCE(q->quiesce_depth <= 0)) { 265 ; 266 } else if (!--q->quiesce_depth) { 267 blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q); 268 run_queue = true; 269 } 270 spin_unlock_irqrestore(&q->queue_lock, flags); 271 272 /* dispatch requests which are inserted during quiescing */ 273 if (run_queue) 274 blk_mq_run_hw_queues(q, true); 275 } 276 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); 277 278 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set) 279 { 280 struct request_queue *q; 281 282 mutex_lock(&set->tag_list_lock); 283 list_for_each_entry(q, &set->tag_list, tag_set_list) { 284 if (!blk_queue_skip_tagset_quiesce(q)) 285 blk_mq_quiesce_queue_nowait(q); 286 } 287 blk_mq_wait_quiesce_done(set); 288 mutex_unlock(&set->tag_list_lock); 289 } 290 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset); 291 292 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set) 293 { 294 struct request_queue *q; 295 296 mutex_lock(&set->tag_list_lock); 297 list_for_each_entry(q, &set->tag_list, tag_set_list) { 298 if (!blk_queue_skip_tagset_quiesce(q)) 299 blk_mq_unquiesce_queue(q); 300 } 301 mutex_unlock(&set->tag_list_lock); 302 } 303 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset); 304 305 void blk_mq_wake_waiters(struct request_queue *q) 306 { 307 struct blk_mq_hw_ctx *hctx; 308 unsigned long i; 309 310 queue_for_each_hw_ctx(q, hctx, i) 311 if (blk_mq_hw_queue_mapped(hctx)) 312 blk_mq_tag_wakeup_all(hctx->tags, true); 313 } 314 315 void blk_rq_init(struct request_queue *q, struct request *rq) 316 { 317 memset(rq, 0, sizeof(*rq)); 318 319 INIT_LIST_HEAD(&rq->queuelist); 320 rq->q = q; 321 rq->__sector = (sector_t) -1; 322 INIT_HLIST_NODE(&rq->hash); 323 RB_CLEAR_NODE(&rq->rb_node); 324 rq->tag = BLK_MQ_NO_TAG; 325 rq->internal_tag = BLK_MQ_NO_TAG; 326 rq->start_time_ns = ktime_get_ns(); 327 rq->part = NULL; 328 blk_crypto_rq_set_defaults(rq); 329 } 330 EXPORT_SYMBOL(blk_rq_init); 331 332 /* Set start and alloc time when the allocated request is actually used */ 333 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns) 334 { 335 if (blk_mq_need_time_stamp(rq)) 336 rq->start_time_ns = ktime_get_ns(); 337 else 338 rq->start_time_ns = 0; 339 340 #ifdef CONFIG_BLK_RQ_ALLOC_TIME 341 if (blk_queue_rq_alloc_time(rq->q)) 342 rq->alloc_time_ns = alloc_time_ns ?: rq->start_time_ns; 343 else 344 rq->alloc_time_ns = 0; 345 #endif 346 } 347 348 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, 349 struct blk_mq_tags *tags, unsigned int tag) 350 { 351 struct blk_mq_ctx *ctx = data->ctx; 352 struct blk_mq_hw_ctx *hctx = data->hctx; 353 struct request_queue *q = data->q; 354 struct request *rq = tags->static_rqs[tag]; 355 356 rq->q = q; 357 rq->mq_ctx = ctx; 358 rq->mq_hctx = hctx; 359 rq->cmd_flags = data->cmd_flags; 360 361 if (data->flags & BLK_MQ_REQ_PM) 362 data->rq_flags |= RQF_PM; 363 if (blk_queue_io_stat(q)) 364 data->rq_flags |= RQF_IO_STAT; 365 rq->rq_flags = data->rq_flags; 366 367 if (data->rq_flags & RQF_SCHED_TAGS) { 368 rq->tag = BLK_MQ_NO_TAG; 369 rq->internal_tag = tag; 370 } else { 371 rq->tag = tag; 372 rq->internal_tag = BLK_MQ_NO_TAG; 373 } 374 rq->timeout = 0; 375 376 rq->part = NULL; 377 rq->io_start_time_ns = 0; 378 rq->stats_sectors = 0; 379 rq->nr_phys_segments = 0; 380 #if defined(CONFIG_BLK_DEV_INTEGRITY) 381 rq->nr_integrity_segments = 0; 382 #endif 383 rq->end_io = NULL; 384 rq->end_io_data = NULL; 385 386 blk_crypto_rq_set_defaults(rq); 387 INIT_LIST_HEAD(&rq->queuelist); 388 /* tag was already set */ 389 WRITE_ONCE(rq->deadline, 0); 390 req_ref_set(rq, 1); 391 392 if (rq->rq_flags & RQF_USE_SCHED) { 393 struct elevator_queue *e = data->q->elevator; 394 395 INIT_HLIST_NODE(&rq->hash); 396 RB_CLEAR_NODE(&rq->rb_node); 397 398 if (e->type->ops.prepare_request) 399 e->type->ops.prepare_request(rq); 400 } 401 402 return rq; 403 } 404 405 static inline struct request * 406 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) 407 { 408 unsigned int tag, tag_offset; 409 struct blk_mq_tags *tags; 410 struct request *rq; 411 unsigned long tag_mask; 412 int i, nr = 0; 413 414 tag_mask = blk_mq_get_tags(data, data->nr_tags, &tag_offset); 415 if (unlikely(!tag_mask)) 416 return NULL; 417 418 tags = blk_mq_tags_from_data(data); 419 for (i = 0; tag_mask; i++) { 420 if (!(tag_mask & (1UL << i))) 421 continue; 422 tag = tag_offset + i; 423 prefetch(tags->static_rqs[tag]); 424 tag_mask &= ~(1UL << i); 425 rq = blk_mq_rq_ctx_init(data, tags, tag); 426 rq_list_add(data->cached_rq, rq); 427 nr++; 428 } 429 if (!(data->rq_flags & RQF_SCHED_TAGS)) 430 blk_mq_add_active_requests(data->hctx, nr); 431 /* caller already holds a reference, add for remainder */ 432 percpu_ref_get_many(&data->q->q_usage_counter, nr - 1); 433 data->nr_tags -= nr; 434 435 return rq_list_pop(data->cached_rq); 436 } 437 438 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) 439 { 440 struct request_queue *q = data->q; 441 u64 alloc_time_ns = 0; 442 struct request *rq; 443 unsigned int tag; 444 445 /* alloc_time includes depth and tag waits */ 446 if (blk_queue_rq_alloc_time(q)) 447 alloc_time_ns = ktime_get_ns(); 448 449 if (data->cmd_flags & REQ_NOWAIT) 450 data->flags |= BLK_MQ_REQ_NOWAIT; 451 452 if (q->elevator) { 453 /* 454 * All requests use scheduler tags when an I/O scheduler is 455 * enabled for the queue. 456 */ 457 data->rq_flags |= RQF_SCHED_TAGS; 458 459 /* 460 * Flush/passthrough requests are special and go directly to the 461 * dispatch list. 462 */ 463 if ((data->cmd_flags & REQ_OP_MASK) != REQ_OP_FLUSH && 464 !blk_op_is_passthrough(data->cmd_flags)) { 465 struct elevator_mq_ops *ops = &q->elevator->type->ops; 466 467 WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); 468 469 data->rq_flags |= RQF_USE_SCHED; 470 if (ops->limit_depth) 471 ops->limit_depth(data->cmd_flags, data); 472 } 473 } 474 475 retry: 476 data->ctx = blk_mq_get_ctx(q); 477 data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx); 478 if (!(data->rq_flags & RQF_SCHED_TAGS)) 479 blk_mq_tag_busy(data->hctx); 480 481 if (data->flags & BLK_MQ_REQ_RESERVED) 482 data->rq_flags |= RQF_RESV; 483 484 /* 485 * Try batched alloc if we want more than 1 tag. 486 */ 487 if (data->nr_tags > 1) { 488 rq = __blk_mq_alloc_requests_batch(data); 489 if (rq) { 490 blk_mq_rq_time_init(rq, alloc_time_ns); 491 return rq; 492 } 493 data->nr_tags = 1; 494 } 495 496 /* 497 * Waiting allocations only fail because of an inactive hctx. In that 498 * case just retry the hctx assignment and tag allocation as CPU hotplug 499 * should have migrated us to an online CPU by now. 500 */ 501 tag = blk_mq_get_tag(data); 502 if (tag == BLK_MQ_NO_TAG) { 503 if (data->flags & BLK_MQ_REQ_NOWAIT) 504 return NULL; 505 /* 506 * Give up the CPU and sleep for a random short time to 507 * ensure that thread using a realtime scheduling class 508 * are migrated off the CPU, and thus off the hctx that 509 * is going away. 510 */ 511 msleep(3); 512 goto retry; 513 } 514 515 if (!(data->rq_flags & RQF_SCHED_TAGS)) 516 blk_mq_inc_active_requests(data->hctx); 517 rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag); 518 blk_mq_rq_time_init(rq, alloc_time_ns); 519 return rq; 520 } 521 522 static struct request *blk_mq_rq_cache_fill(struct request_queue *q, 523 struct blk_plug *plug, 524 blk_opf_t opf, 525 blk_mq_req_flags_t flags) 526 { 527 struct blk_mq_alloc_data data = { 528 .q = q, 529 .flags = flags, 530 .cmd_flags = opf, 531 .nr_tags = plug->nr_ios, 532 .cached_rq = &plug->cached_rq, 533 }; 534 struct request *rq; 535 536 if (blk_queue_enter(q, flags)) 537 return NULL; 538 539 plug->nr_ios = 1; 540 541 rq = __blk_mq_alloc_requests(&data); 542 if (unlikely(!rq)) 543 blk_queue_exit(q); 544 return rq; 545 } 546 547 static struct request *blk_mq_alloc_cached_request(struct request_queue *q, 548 blk_opf_t opf, 549 blk_mq_req_flags_t flags) 550 { 551 struct blk_plug *plug = current->plug; 552 struct request *rq; 553 554 if (!plug) 555 return NULL; 556 557 if (rq_list_empty(plug->cached_rq)) { 558 if (plug->nr_ios == 1) 559 return NULL; 560 rq = blk_mq_rq_cache_fill(q, plug, opf, flags); 561 if (!rq) 562 return NULL; 563 } else { 564 rq = rq_list_peek(&plug->cached_rq); 565 if (!rq || rq->q != q) 566 return NULL; 567 568 if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type) 569 return NULL; 570 if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) 571 return NULL; 572 573 plug->cached_rq = rq_list_next(rq); 574 blk_mq_rq_time_init(rq, 0); 575 } 576 577 rq->cmd_flags = opf; 578 INIT_LIST_HEAD(&rq->queuelist); 579 return rq; 580 } 581 582 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, 583 blk_mq_req_flags_t flags) 584 { 585 struct request *rq; 586 587 rq = blk_mq_alloc_cached_request(q, opf, flags); 588 if (!rq) { 589 struct blk_mq_alloc_data data = { 590 .q = q, 591 .flags = flags, 592 .cmd_flags = opf, 593 .nr_tags = 1, 594 }; 595 int ret; 596 597 ret = blk_queue_enter(q, flags); 598 if (ret) 599 return ERR_PTR(ret); 600 601 rq = __blk_mq_alloc_requests(&data); 602 if (!rq) 603 goto out_queue_exit; 604 } 605 rq->__data_len = 0; 606 rq->__sector = (sector_t) -1; 607 rq->bio = rq->biotail = NULL; 608 return rq; 609 out_queue_exit: 610 blk_queue_exit(q); 611 return ERR_PTR(-EWOULDBLOCK); 612 } 613 EXPORT_SYMBOL(blk_mq_alloc_request); 614 615 struct request *blk_mq_alloc_request_hctx(struct request_queue *q, 616 blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx) 617 { 618 struct blk_mq_alloc_data data = { 619 .q = q, 620 .flags = flags, 621 .cmd_flags = opf, 622 .nr_tags = 1, 623 }; 624 u64 alloc_time_ns = 0; 625 struct request *rq; 626 unsigned int cpu; 627 unsigned int tag; 628 int ret; 629 630 /* alloc_time includes depth and tag waits */ 631 if (blk_queue_rq_alloc_time(q)) 632 alloc_time_ns = ktime_get_ns(); 633 634 /* 635 * If the tag allocator sleeps we could get an allocation for a 636 * different hardware context. No need to complicate the low level 637 * allocator for this for the rare use case of a command tied to 638 * a specific queue. 639 */ 640 if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) || 641 WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED))) 642 return ERR_PTR(-EINVAL); 643 644 if (hctx_idx >= q->nr_hw_queues) 645 return ERR_PTR(-EIO); 646 647 ret = blk_queue_enter(q, flags); 648 if (ret) 649 return ERR_PTR(ret); 650 651 /* 652 * Check if the hardware context is actually mapped to anything. 653 * If not tell the caller that it should skip this queue. 654 */ 655 ret = -EXDEV; 656 data.hctx = xa_load(&q->hctx_table, hctx_idx); 657 if (!blk_mq_hw_queue_mapped(data.hctx)) 658 goto out_queue_exit; 659 cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask); 660 if (cpu >= nr_cpu_ids) 661 goto out_queue_exit; 662 data.ctx = __blk_mq_get_ctx(q, cpu); 663 664 if (q->elevator) 665 data.rq_flags |= RQF_SCHED_TAGS; 666 else 667 blk_mq_tag_busy(data.hctx); 668 669 if (flags & BLK_MQ_REQ_RESERVED) 670 data.rq_flags |= RQF_RESV; 671 672 ret = -EWOULDBLOCK; 673 tag = blk_mq_get_tag(&data); 674 if (tag == BLK_MQ_NO_TAG) 675 goto out_queue_exit; 676 if (!(data.rq_flags & RQF_SCHED_TAGS)) 677 blk_mq_inc_active_requests(data.hctx); 678 rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag); 679 blk_mq_rq_time_init(rq, alloc_time_ns); 680 rq->__data_len = 0; 681 rq->__sector = (sector_t) -1; 682 rq->bio = rq->biotail = NULL; 683 return rq; 684 685 out_queue_exit: 686 blk_queue_exit(q); 687 return ERR_PTR(ret); 688 } 689 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); 690 691 static void blk_mq_finish_request(struct request *rq) 692 { 693 struct request_queue *q = rq->q; 694 695 if (rq->rq_flags & RQF_USE_SCHED) { 696 q->elevator->type->ops.finish_request(rq); 697 /* 698 * For postflush request that may need to be 699 * completed twice, we should clear this flag 700 * to avoid double finish_request() on the rq. 701 */ 702 rq->rq_flags &= ~RQF_USE_SCHED; 703 } 704 } 705 706 static void __blk_mq_free_request(struct request *rq) 707 { 708 struct request_queue *q = rq->q; 709 struct blk_mq_ctx *ctx = rq->mq_ctx; 710 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 711 const int sched_tag = rq->internal_tag; 712 713 blk_crypto_free_request(rq); 714 blk_pm_mark_last_busy(rq); 715 rq->mq_hctx = NULL; 716 717 if (rq->tag != BLK_MQ_NO_TAG) { 718 blk_mq_dec_active_requests(hctx); 719 blk_mq_put_tag(hctx->tags, ctx, rq->tag); 720 } 721 if (sched_tag != BLK_MQ_NO_TAG) 722 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag); 723 blk_mq_sched_restart(hctx); 724 blk_queue_exit(q); 725 } 726 727 void blk_mq_free_request(struct request *rq) 728 { 729 struct request_queue *q = rq->q; 730 731 blk_mq_finish_request(rq); 732 733 if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq))) 734 laptop_io_completion(q->disk->bdi); 735 736 rq_qos_done(q, rq); 737 738 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 739 if (req_ref_put_and_test(rq)) 740 __blk_mq_free_request(rq); 741 } 742 EXPORT_SYMBOL_GPL(blk_mq_free_request); 743 744 void blk_mq_free_plug_rqs(struct blk_plug *plug) 745 { 746 struct request *rq; 747 748 while ((rq = rq_list_pop(&plug->cached_rq)) != NULL) 749 blk_mq_free_request(rq); 750 } 751 752 void blk_dump_rq_flags(struct request *rq, char *msg) 753 { 754 printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg, 755 rq->q->disk ? rq->q->disk->disk_name : "?", 756 (__force unsigned long long) rq->cmd_flags); 757 758 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", 759 (unsigned long long)blk_rq_pos(rq), 760 blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); 761 printk(KERN_INFO " bio %p, biotail %p, len %u\n", 762 rq->bio, rq->biotail, blk_rq_bytes(rq)); 763 } 764 EXPORT_SYMBOL(blk_dump_rq_flags); 765 766 static void req_bio_endio(struct request *rq, struct bio *bio, 767 unsigned int nbytes, blk_status_t error) 768 { 769 if (unlikely(error)) { 770 bio->bi_status = error; 771 } else if (req_op(rq) == REQ_OP_ZONE_APPEND) { 772 /* 773 * Partial zone append completions cannot be supported as the 774 * BIO fragments may end up not being written sequentially. 775 */ 776 if (bio->bi_iter.bi_size != nbytes) 777 bio->bi_status = BLK_STS_IOERR; 778 else 779 bio->bi_iter.bi_sector = rq->__sector; 780 } 781 782 bio_advance(bio, nbytes); 783 784 if (unlikely(rq->rq_flags & RQF_QUIET)) 785 bio_set_flag(bio, BIO_QUIET); 786 /* don't actually finish bio if it's part of flush sequence */ 787 if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ)) 788 bio_endio(bio); 789 } 790 791 static void blk_account_io_completion(struct request *req, unsigned int bytes) 792 { 793 if (req->part && blk_do_io_stat(req)) { 794 const int sgrp = op_stat_group(req_op(req)); 795 796 part_stat_lock(); 797 part_stat_add(req->part, sectors[sgrp], bytes >> 9); 798 part_stat_unlock(); 799 } 800 } 801 802 static void blk_print_req_error(struct request *req, blk_status_t status) 803 { 804 printk_ratelimited(KERN_ERR 805 "%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x " 806 "phys_seg %u prio class %u\n", 807 blk_status_to_str(status), 808 req->q->disk ? req->q->disk->disk_name : "?", 809 blk_rq_pos(req), (__force u32)req_op(req), 810 blk_op_str(req_op(req)), 811 (__force u32)(req->cmd_flags & ~REQ_OP_MASK), 812 req->nr_phys_segments, 813 IOPRIO_PRIO_CLASS(req->ioprio)); 814 } 815 816 /* 817 * Fully end IO on a request. Does not support partial completions, or 818 * errors. 819 */ 820 static void blk_complete_request(struct request *req) 821 { 822 const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0; 823 int total_bytes = blk_rq_bytes(req); 824 struct bio *bio = req->bio; 825 826 trace_block_rq_complete(req, BLK_STS_OK, total_bytes); 827 828 if (!bio) 829 return; 830 831 #ifdef CONFIG_BLK_DEV_INTEGRITY 832 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ) 833 req->q->integrity.profile->complete_fn(req, total_bytes); 834 #endif 835 836 /* 837 * Upper layers may call blk_crypto_evict_key() anytime after the last 838 * bio_endio(). Therefore, the keyslot must be released before that. 839 */ 840 blk_crypto_rq_put_keyslot(req); 841 842 blk_account_io_completion(req, total_bytes); 843 844 do { 845 struct bio *next = bio->bi_next; 846 847 /* Completion has already been traced */ 848 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 849 850 if (req_op(req) == REQ_OP_ZONE_APPEND) 851 bio->bi_iter.bi_sector = req->__sector; 852 853 if (!is_flush) 854 bio_endio(bio); 855 bio = next; 856 } while (bio); 857 858 /* 859 * Reset counters so that the request stacking driver 860 * can find how many bytes remain in the request 861 * later. 862 */ 863 if (!req->end_io) { 864 req->bio = NULL; 865 req->__data_len = 0; 866 } 867 } 868 869 /** 870 * blk_update_request - Complete multiple bytes without completing the request 871 * @req: the request being processed 872 * @error: block status code 873 * @nr_bytes: number of bytes to complete for @req 874 * 875 * Description: 876 * Ends I/O on a number of bytes attached to @req, but doesn't complete 877 * the request structure even if @req doesn't have leftover. 878 * If @req has leftover, sets it up for the next range of segments. 879 * 880 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees 881 * %false return from this function. 882 * 883 * Note: 884 * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function 885 * except in the consistency check at the end of this function. 886 * 887 * Return: 888 * %false - this request doesn't have any more data 889 * %true - this request has more data 890 **/ 891 bool blk_update_request(struct request *req, blk_status_t error, 892 unsigned int nr_bytes) 893 { 894 int total_bytes; 895 896 trace_block_rq_complete(req, error, nr_bytes); 897 898 if (!req->bio) 899 return false; 900 901 #ifdef CONFIG_BLK_DEV_INTEGRITY 902 if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ && 903 error == BLK_STS_OK) 904 req->q->integrity.profile->complete_fn(req, nr_bytes); 905 #endif 906 907 /* 908 * Upper layers may call blk_crypto_evict_key() anytime after the last 909 * bio_endio(). Therefore, the keyslot must be released before that. 910 */ 911 if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req)) 912 __blk_crypto_rq_put_keyslot(req); 913 914 if (unlikely(error && !blk_rq_is_passthrough(req) && 915 !(req->rq_flags & RQF_QUIET)) && 916 !test_bit(GD_DEAD, &req->q->disk->state)) { 917 blk_print_req_error(req, error); 918 trace_block_rq_error(req, error, nr_bytes); 919 } 920 921 blk_account_io_completion(req, nr_bytes); 922 923 total_bytes = 0; 924 while (req->bio) { 925 struct bio *bio = req->bio; 926 unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes); 927 928 if (bio_bytes == bio->bi_iter.bi_size) 929 req->bio = bio->bi_next; 930 931 /* Completion has already been traced */ 932 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 933 req_bio_endio(req, bio, bio_bytes, error); 934 935 total_bytes += bio_bytes; 936 nr_bytes -= bio_bytes; 937 938 if (!nr_bytes) 939 break; 940 } 941 942 /* 943 * completely done 944 */ 945 if (!req->bio) { 946 /* 947 * Reset counters so that the request stacking driver 948 * can find how many bytes remain in the request 949 * later. 950 */ 951 req->__data_len = 0; 952 return false; 953 } 954 955 req->__data_len -= total_bytes; 956 957 /* update sector only for requests with clear definition of sector */ 958 if (!blk_rq_is_passthrough(req)) 959 req->__sector += total_bytes >> 9; 960 961 /* mixed attributes always follow the first bio */ 962 if (req->rq_flags & RQF_MIXED_MERGE) { 963 req->cmd_flags &= ~REQ_FAILFAST_MASK; 964 req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK; 965 } 966 967 if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) { 968 /* 969 * If total number of sectors is less than the first segment 970 * size, something has gone terribly wrong. 971 */ 972 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { 973 blk_dump_rq_flags(req, "request botched"); 974 req->__data_len = blk_rq_cur_bytes(req); 975 } 976 977 /* recalculate the number of segments */ 978 req->nr_phys_segments = blk_recalc_rq_segments(req); 979 } 980 981 return true; 982 } 983 EXPORT_SYMBOL_GPL(blk_update_request); 984 985 static inline void blk_account_io_done(struct request *req, u64 now) 986 { 987 trace_block_io_done(req); 988 989 /* 990 * Account IO completion. flush_rq isn't accounted as a 991 * normal IO on queueing nor completion. Accounting the 992 * containing request is enough. 993 */ 994 if (blk_do_io_stat(req) && req->part && 995 !(req->rq_flags & RQF_FLUSH_SEQ)) { 996 const int sgrp = op_stat_group(req_op(req)); 997 998 part_stat_lock(); 999 update_io_ticks(req->part, jiffies, true); 1000 part_stat_inc(req->part, ios[sgrp]); 1001 part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns); 1002 part_stat_unlock(); 1003 } 1004 } 1005 1006 static inline void blk_account_io_start(struct request *req) 1007 { 1008 trace_block_io_start(req); 1009 1010 if (blk_do_io_stat(req)) { 1011 /* 1012 * All non-passthrough requests are created from a bio with one 1013 * exception: when a flush command that is part of a flush sequence 1014 * generated by the state machine in blk-flush.c is cloned onto the 1015 * lower device by dm-multipath we can get here without a bio. 1016 */ 1017 if (req->bio) 1018 req->part = req->bio->bi_bdev; 1019 else 1020 req->part = req->q->disk->part0; 1021 1022 part_stat_lock(); 1023 update_io_ticks(req->part, jiffies, false); 1024 part_stat_unlock(); 1025 } 1026 } 1027 1028 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now) 1029 { 1030 if (rq->rq_flags & RQF_STATS) 1031 blk_stat_add(rq, now); 1032 1033 blk_mq_sched_completed_request(rq, now); 1034 blk_account_io_done(rq, now); 1035 } 1036 1037 inline void __blk_mq_end_request(struct request *rq, blk_status_t error) 1038 { 1039 if (blk_mq_need_time_stamp(rq)) 1040 __blk_mq_end_request_acct(rq, ktime_get_ns()); 1041 1042 blk_mq_finish_request(rq); 1043 1044 if (rq->end_io) { 1045 rq_qos_done(rq->q, rq); 1046 if (rq->end_io(rq, error) == RQ_END_IO_FREE) 1047 blk_mq_free_request(rq); 1048 } else { 1049 blk_mq_free_request(rq); 1050 } 1051 } 1052 EXPORT_SYMBOL(__blk_mq_end_request); 1053 1054 void blk_mq_end_request(struct request *rq, blk_status_t error) 1055 { 1056 if (blk_update_request(rq, error, blk_rq_bytes(rq))) 1057 BUG(); 1058 __blk_mq_end_request(rq, error); 1059 } 1060 EXPORT_SYMBOL(blk_mq_end_request); 1061 1062 #define TAG_COMP_BATCH 32 1063 1064 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx, 1065 int *tag_array, int nr_tags) 1066 { 1067 struct request_queue *q = hctx->queue; 1068 1069 blk_mq_sub_active_requests(hctx, nr_tags); 1070 1071 blk_mq_put_tags(hctx->tags, tag_array, nr_tags); 1072 percpu_ref_put_many(&q->q_usage_counter, nr_tags); 1073 } 1074 1075 void blk_mq_end_request_batch(struct io_comp_batch *iob) 1076 { 1077 int tags[TAG_COMP_BATCH], nr_tags = 0; 1078 struct blk_mq_hw_ctx *cur_hctx = NULL; 1079 struct request *rq; 1080 u64 now = 0; 1081 1082 if (iob->need_ts) 1083 now = ktime_get_ns(); 1084 1085 while ((rq = rq_list_pop(&iob->req_list)) != NULL) { 1086 prefetch(rq->bio); 1087 prefetch(rq->rq_next); 1088 1089 blk_complete_request(rq); 1090 if (iob->need_ts) 1091 __blk_mq_end_request_acct(rq, now); 1092 1093 blk_mq_finish_request(rq); 1094 1095 rq_qos_done(rq->q, rq); 1096 1097 /* 1098 * If end_io handler returns NONE, then it still has 1099 * ownership of the request. 1100 */ 1101 if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE) 1102 continue; 1103 1104 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 1105 if (!req_ref_put_and_test(rq)) 1106 continue; 1107 1108 blk_crypto_free_request(rq); 1109 blk_pm_mark_last_busy(rq); 1110 1111 if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) { 1112 if (cur_hctx) 1113 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags); 1114 nr_tags = 0; 1115 cur_hctx = rq->mq_hctx; 1116 } 1117 tags[nr_tags++] = rq->tag; 1118 } 1119 1120 if (nr_tags) 1121 blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags); 1122 } 1123 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch); 1124 1125 static void blk_complete_reqs(struct llist_head *list) 1126 { 1127 struct llist_node *entry = llist_reverse_order(llist_del_all(list)); 1128 struct request *rq, *next; 1129 1130 llist_for_each_entry_safe(rq, next, entry, ipi_list) 1131 rq->q->mq_ops->complete(rq); 1132 } 1133 1134 static __latent_entropy void blk_done_softirq(struct softirq_action *h) 1135 { 1136 blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); 1137 } 1138 1139 static int blk_softirq_cpu_dead(unsigned int cpu) 1140 { 1141 blk_complete_reqs(&per_cpu(blk_cpu_done, cpu)); 1142 return 0; 1143 } 1144 1145 static void __blk_mq_complete_request_remote(void *data) 1146 { 1147 __raise_softirq_irqoff(BLOCK_SOFTIRQ); 1148 } 1149 1150 static inline bool blk_mq_complete_need_ipi(struct request *rq) 1151 { 1152 int cpu = raw_smp_processor_id(); 1153 1154 if (!IS_ENABLED(CONFIG_SMP) || 1155 !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) 1156 return false; 1157 /* 1158 * With force threaded interrupts enabled, raising softirq from an SMP 1159 * function call will always result in waking the ksoftirqd thread. 1160 * This is probably worse than completing the request on a different 1161 * cache domain. 1162 */ 1163 if (force_irqthreads()) 1164 return false; 1165 1166 /* same CPU or cache domain? Complete locally */ 1167 if (cpu == rq->mq_ctx->cpu || 1168 (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) && 1169 cpus_share_cache(cpu, rq->mq_ctx->cpu))) 1170 return false; 1171 1172 /* don't try to IPI to an offline CPU */ 1173 return cpu_online(rq->mq_ctx->cpu); 1174 } 1175 1176 static void blk_mq_complete_send_ipi(struct request *rq) 1177 { 1178 unsigned int cpu; 1179 1180 cpu = rq->mq_ctx->cpu; 1181 if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu))) 1182 smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu)); 1183 } 1184 1185 static void blk_mq_raise_softirq(struct request *rq) 1186 { 1187 struct llist_head *list; 1188 1189 preempt_disable(); 1190 list = this_cpu_ptr(&blk_cpu_done); 1191 if (llist_add(&rq->ipi_list, list)) 1192 raise_softirq(BLOCK_SOFTIRQ); 1193 preempt_enable(); 1194 } 1195 1196 bool blk_mq_complete_request_remote(struct request *rq) 1197 { 1198 WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); 1199 1200 /* 1201 * For request which hctx has only one ctx mapping, 1202 * or a polled request, always complete locally, 1203 * it's pointless to redirect the completion. 1204 */ 1205 if ((rq->mq_hctx->nr_ctx == 1 && 1206 rq->mq_ctx->cpu == raw_smp_processor_id()) || 1207 rq->cmd_flags & REQ_POLLED) 1208 return false; 1209 1210 if (blk_mq_complete_need_ipi(rq)) { 1211 blk_mq_complete_send_ipi(rq); 1212 return true; 1213 } 1214 1215 if (rq->q->nr_hw_queues == 1) { 1216 blk_mq_raise_softirq(rq); 1217 return true; 1218 } 1219 return false; 1220 } 1221 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); 1222 1223 /** 1224 * blk_mq_complete_request - end I/O on a request 1225 * @rq: the request being processed 1226 * 1227 * Description: 1228 * Complete a request by scheduling the ->complete_rq operation. 1229 **/ 1230 void blk_mq_complete_request(struct request *rq) 1231 { 1232 if (!blk_mq_complete_request_remote(rq)) 1233 rq->q->mq_ops->complete(rq); 1234 } 1235 EXPORT_SYMBOL(blk_mq_complete_request); 1236 1237 /** 1238 * blk_mq_start_request - Start processing a request 1239 * @rq: Pointer to request to be started 1240 * 1241 * Function used by device drivers to notify the block layer that a request 1242 * is going to be processed now, so blk layer can do proper initializations 1243 * such as starting the timeout timer. 1244 */ 1245 void blk_mq_start_request(struct request *rq) 1246 { 1247 struct request_queue *q = rq->q; 1248 1249 trace_block_rq_issue(rq); 1250 1251 if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 1252 rq->io_start_time_ns = ktime_get_ns(); 1253 rq->stats_sectors = blk_rq_sectors(rq); 1254 rq->rq_flags |= RQF_STATS; 1255 rq_qos_issue(q, rq); 1256 } 1257 1258 WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE); 1259 1260 blk_add_timer(rq); 1261 WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT); 1262 rq->mq_hctx->tags->rqs[rq->tag] = rq; 1263 1264 #ifdef CONFIG_BLK_DEV_INTEGRITY 1265 if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE) 1266 q->integrity.profile->prepare_fn(rq); 1267 #endif 1268 if (rq->bio && rq->bio->bi_opf & REQ_POLLED) 1269 WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num); 1270 } 1271 EXPORT_SYMBOL(blk_mq_start_request); 1272 1273 /* 1274 * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple 1275 * queues. This is important for md arrays to benefit from merging 1276 * requests. 1277 */ 1278 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug) 1279 { 1280 if (plug->multiple_queues) 1281 return BLK_MAX_REQUEST_COUNT * 2; 1282 return BLK_MAX_REQUEST_COUNT; 1283 } 1284 1285 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) 1286 { 1287 struct request *last = rq_list_peek(&plug->mq_list); 1288 1289 if (!plug->rq_count) { 1290 trace_block_plug(rq->q); 1291 } else if (plug->rq_count >= blk_plug_max_rq_count(plug) || 1292 (!blk_queue_nomerges(rq->q) && 1293 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { 1294 blk_mq_flush_plug_list(plug, false); 1295 last = NULL; 1296 trace_block_plug(rq->q); 1297 } 1298 1299 if (!plug->multiple_queues && last && last->q != rq->q) 1300 plug->multiple_queues = true; 1301 /* 1302 * Any request allocated from sched tags can't be issued to 1303 * ->queue_rqs() directly 1304 */ 1305 if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS)) 1306 plug->has_elevator = true; 1307 rq->rq_next = NULL; 1308 rq_list_add(&plug->mq_list, rq); 1309 plug->rq_count++; 1310 } 1311 1312 /** 1313 * blk_execute_rq_nowait - insert a request to I/O scheduler for execution 1314 * @rq: request to insert 1315 * @at_head: insert request at head or tail of queue 1316 * 1317 * Description: 1318 * Insert a fully prepared request at the back of the I/O scheduler queue 1319 * for execution. Don't wait for completion. 1320 * 1321 * Note: 1322 * This function will invoke @done directly if the queue is dead. 1323 */ 1324 void blk_execute_rq_nowait(struct request *rq, bool at_head) 1325 { 1326 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1327 1328 WARN_ON(irqs_disabled()); 1329 WARN_ON(!blk_rq_is_passthrough(rq)); 1330 1331 blk_account_io_start(rq); 1332 1333 /* 1334 * As plugging can be enabled for passthrough requests on a zoned 1335 * device, directly accessing the plug instead of using blk_mq_plug() 1336 * should not have any consequences. 1337 */ 1338 if (current->plug && !at_head) { 1339 blk_add_rq_to_plug(current->plug, rq); 1340 return; 1341 } 1342 1343 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0); 1344 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING); 1345 } 1346 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); 1347 1348 struct blk_rq_wait { 1349 struct completion done; 1350 blk_status_t ret; 1351 }; 1352 1353 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret) 1354 { 1355 struct blk_rq_wait *wait = rq->end_io_data; 1356 1357 wait->ret = ret; 1358 complete(&wait->done); 1359 return RQ_END_IO_NONE; 1360 } 1361 1362 bool blk_rq_is_poll(struct request *rq) 1363 { 1364 if (!rq->mq_hctx) 1365 return false; 1366 if (rq->mq_hctx->type != HCTX_TYPE_POLL) 1367 return false; 1368 return true; 1369 } 1370 EXPORT_SYMBOL_GPL(blk_rq_is_poll); 1371 1372 static void blk_rq_poll_completion(struct request *rq, struct completion *wait) 1373 { 1374 do { 1375 blk_hctx_poll(rq->q, rq->mq_hctx, NULL, 0); 1376 cond_resched(); 1377 } while (!completion_done(wait)); 1378 } 1379 1380 /** 1381 * blk_execute_rq - insert a request into queue for execution 1382 * @rq: request to insert 1383 * @at_head: insert request at head or tail of queue 1384 * 1385 * Description: 1386 * Insert a fully prepared request at the back of the I/O scheduler queue 1387 * for execution and wait for completion. 1388 * Return: The blk_status_t result provided to blk_mq_end_request(). 1389 */ 1390 blk_status_t blk_execute_rq(struct request *rq, bool at_head) 1391 { 1392 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1393 struct blk_rq_wait wait = { 1394 .done = COMPLETION_INITIALIZER_ONSTACK(wait.done), 1395 }; 1396 1397 WARN_ON(irqs_disabled()); 1398 WARN_ON(!blk_rq_is_passthrough(rq)); 1399 1400 rq->end_io_data = &wait; 1401 rq->end_io = blk_end_sync_rq; 1402 1403 blk_account_io_start(rq); 1404 blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0); 1405 blk_mq_run_hw_queue(hctx, false); 1406 1407 if (blk_rq_is_poll(rq)) { 1408 blk_rq_poll_completion(rq, &wait.done); 1409 } else { 1410 /* 1411 * Prevent hang_check timer from firing at us during very long 1412 * I/O 1413 */ 1414 unsigned long hang_check = sysctl_hung_task_timeout_secs; 1415 1416 if (hang_check) 1417 while (!wait_for_completion_io_timeout(&wait.done, 1418 hang_check * (HZ/2))) 1419 ; 1420 else 1421 wait_for_completion_io(&wait.done); 1422 } 1423 1424 return wait.ret; 1425 } 1426 EXPORT_SYMBOL(blk_execute_rq); 1427 1428 static void __blk_mq_requeue_request(struct request *rq) 1429 { 1430 struct request_queue *q = rq->q; 1431 1432 blk_mq_put_driver_tag(rq); 1433 1434 trace_block_rq_requeue(rq); 1435 rq_qos_requeue(q, rq); 1436 1437 if (blk_mq_request_started(rq)) { 1438 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 1439 rq->rq_flags &= ~RQF_TIMED_OUT; 1440 } 1441 } 1442 1443 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) 1444 { 1445 struct request_queue *q = rq->q; 1446 unsigned long flags; 1447 1448 __blk_mq_requeue_request(rq); 1449 1450 /* this request will be re-inserted to io scheduler queue */ 1451 blk_mq_sched_requeue_request(rq); 1452 1453 spin_lock_irqsave(&q->requeue_lock, flags); 1454 list_add_tail(&rq->queuelist, &q->requeue_list); 1455 spin_unlock_irqrestore(&q->requeue_lock, flags); 1456 1457 if (kick_requeue_list) 1458 blk_mq_kick_requeue_list(q); 1459 } 1460 EXPORT_SYMBOL(blk_mq_requeue_request); 1461 1462 static void blk_mq_requeue_work(struct work_struct *work) 1463 { 1464 struct request_queue *q = 1465 container_of(work, struct request_queue, requeue_work.work); 1466 LIST_HEAD(rq_list); 1467 LIST_HEAD(flush_list); 1468 struct request *rq; 1469 1470 spin_lock_irq(&q->requeue_lock); 1471 list_splice_init(&q->requeue_list, &rq_list); 1472 list_splice_init(&q->flush_list, &flush_list); 1473 spin_unlock_irq(&q->requeue_lock); 1474 1475 while (!list_empty(&rq_list)) { 1476 rq = list_entry(rq_list.next, struct request, queuelist); 1477 /* 1478 * If RQF_DONTPREP ist set, the request has been started by the 1479 * driver already and might have driver-specific data allocated 1480 * already. Insert it into the hctx dispatch list to avoid 1481 * block layer merges for the request. 1482 */ 1483 if (rq->rq_flags & RQF_DONTPREP) { 1484 list_del_init(&rq->queuelist); 1485 blk_mq_request_bypass_insert(rq, 0); 1486 } else { 1487 list_del_init(&rq->queuelist); 1488 blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD); 1489 } 1490 } 1491 1492 while (!list_empty(&flush_list)) { 1493 rq = list_entry(flush_list.next, struct request, queuelist); 1494 list_del_init(&rq->queuelist); 1495 blk_mq_insert_request(rq, 0); 1496 } 1497 1498 blk_mq_run_hw_queues(q, false); 1499 } 1500 1501 void blk_mq_kick_requeue_list(struct request_queue *q) 1502 { 1503 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0); 1504 } 1505 EXPORT_SYMBOL(blk_mq_kick_requeue_list); 1506 1507 void blk_mq_delay_kick_requeue_list(struct request_queue *q, 1508 unsigned long msecs) 1509 { 1510 kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 1511 msecs_to_jiffies(msecs)); 1512 } 1513 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); 1514 1515 static bool blk_mq_rq_inflight(struct request *rq, void *priv) 1516 { 1517 /* 1518 * If we find a request that isn't idle we know the queue is busy 1519 * as it's checked in the iter. 1520 * Return false to stop the iteration. 1521 */ 1522 if (blk_mq_request_started(rq)) { 1523 bool *busy = priv; 1524 1525 *busy = true; 1526 return false; 1527 } 1528 1529 return true; 1530 } 1531 1532 bool blk_mq_queue_inflight(struct request_queue *q) 1533 { 1534 bool busy = false; 1535 1536 blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy); 1537 return busy; 1538 } 1539 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight); 1540 1541 static void blk_mq_rq_timed_out(struct request *req) 1542 { 1543 req->rq_flags |= RQF_TIMED_OUT; 1544 if (req->q->mq_ops->timeout) { 1545 enum blk_eh_timer_return ret; 1546 1547 ret = req->q->mq_ops->timeout(req); 1548 if (ret == BLK_EH_DONE) 1549 return; 1550 WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER); 1551 } 1552 1553 blk_add_timer(req); 1554 } 1555 1556 struct blk_expired_data { 1557 bool has_timedout_rq; 1558 unsigned long next; 1559 unsigned long timeout_start; 1560 }; 1561 1562 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired) 1563 { 1564 unsigned long deadline; 1565 1566 if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT) 1567 return false; 1568 if (rq->rq_flags & RQF_TIMED_OUT) 1569 return false; 1570 1571 deadline = READ_ONCE(rq->deadline); 1572 if (time_after_eq(expired->timeout_start, deadline)) 1573 return true; 1574 1575 if (expired->next == 0) 1576 expired->next = deadline; 1577 else if (time_after(expired->next, deadline)) 1578 expired->next = deadline; 1579 return false; 1580 } 1581 1582 void blk_mq_put_rq_ref(struct request *rq) 1583 { 1584 if (is_flush_rq(rq)) { 1585 if (rq->end_io(rq, 0) == RQ_END_IO_FREE) 1586 blk_mq_free_request(rq); 1587 } else if (req_ref_put_and_test(rq)) { 1588 __blk_mq_free_request(rq); 1589 } 1590 } 1591 1592 static bool blk_mq_check_expired(struct request *rq, void *priv) 1593 { 1594 struct blk_expired_data *expired = priv; 1595 1596 /* 1597 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot 1598 * be reallocated underneath the timeout handler's processing, then 1599 * the expire check is reliable. If the request is not expired, then 1600 * it was completed and reallocated as a new request after returning 1601 * from blk_mq_check_expired(). 1602 */ 1603 if (blk_mq_req_expired(rq, expired)) { 1604 expired->has_timedout_rq = true; 1605 return false; 1606 } 1607 return true; 1608 } 1609 1610 static bool blk_mq_handle_expired(struct request *rq, void *priv) 1611 { 1612 struct blk_expired_data *expired = priv; 1613 1614 if (blk_mq_req_expired(rq, expired)) 1615 blk_mq_rq_timed_out(rq); 1616 return true; 1617 } 1618 1619 static void blk_mq_timeout_work(struct work_struct *work) 1620 { 1621 struct request_queue *q = 1622 container_of(work, struct request_queue, timeout_work); 1623 struct blk_expired_data expired = { 1624 .timeout_start = jiffies, 1625 }; 1626 struct blk_mq_hw_ctx *hctx; 1627 unsigned long i; 1628 1629 /* A deadlock might occur if a request is stuck requiring a 1630 * timeout at the same time a queue freeze is waiting 1631 * completion, since the timeout code would not be able to 1632 * acquire the queue reference here. 1633 * 1634 * That's why we don't use blk_queue_enter here; instead, we use 1635 * percpu_ref_tryget directly, because we need to be able to 1636 * obtain a reference even in the short window between the queue 1637 * starting to freeze, by dropping the first reference in 1638 * blk_freeze_queue_start, and the moment the last request is 1639 * consumed, marked by the instant q_usage_counter reaches 1640 * zero. 1641 */ 1642 if (!percpu_ref_tryget(&q->q_usage_counter)) 1643 return; 1644 1645 /* check if there is any timed-out request */ 1646 blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired); 1647 if (expired.has_timedout_rq) { 1648 /* 1649 * Before walking tags, we must ensure any submit started 1650 * before the current time has finished. Since the submit 1651 * uses srcu or rcu, wait for a synchronization point to 1652 * ensure all running submits have finished 1653 */ 1654 blk_mq_wait_quiesce_done(q->tag_set); 1655 1656 expired.next = 0; 1657 blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired); 1658 } 1659 1660 if (expired.next != 0) { 1661 mod_timer(&q->timeout, expired.next); 1662 } else { 1663 /* 1664 * Request timeouts are handled as a forward rolling timer. If 1665 * we end up here it means that no requests are pending and 1666 * also that no request has been pending for a while. Mark 1667 * each hctx as idle. 1668 */ 1669 queue_for_each_hw_ctx(q, hctx, i) { 1670 /* the hctx may be unmapped, so check it here */ 1671 if (blk_mq_hw_queue_mapped(hctx)) 1672 blk_mq_tag_idle(hctx); 1673 } 1674 } 1675 blk_queue_exit(q); 1676 } 1677 1678 struct flush_busy_ctx_data { 1679 struct blk_mq_hw_ctx *hctx; 1680 struct list_head *list; 1681 }; 1682 1683 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) 1684 { 1685 struct flush_busy_ctx_data *flush_data = data; 1686 struct blk_mq_hw_ctx *hctx = flush_data->hctx; 1687 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1688 enum hctx_type type = hctx->type; 1689 1690 spin_lock(&ctx->lock); 1691 list_splice_tail_init(&ctx->rq_lists[type], flush_data->list); 1692 sbitmap_clear_bit(sb, bitnr); 1693 spin_unlock(&ctx->lock); 1694 return true; 1695 } 1696 1697 /* 1698 * Process software queues that have been marked busy, splicing them 1699 * to the for-dispatch 1700 */ 1701 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) 1702 { 1703 struct flush_busy_ctx_data data = { 1704 .hctx = hctx, 1705 .list = list, 1706 }; 1707 1708 sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); 1709 } 1710 EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); 1711 1712 struct dispatch_rq_data { 1713 struct blk_mq_hw_ctx *hctx; 1714 struct request *rq; 1715 }; 1716 1717 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr, 1718 void *data) 1719 { 1720 struct dispatch_rq_data *dispatch_data = data; 1721 struct blk_mq_hw_ctx *hctx = dispatch_data->hctx; 1722 struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; 1723 enum hctx_type type = hctx->type; 1724 1725 spin_lock(&ctx->lock); 1726 if (!list_empty(&ctx->rq_lists[type])) { 1727 dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next); 1728 list_del_init(&dispatch_data->rq->queuelist); 1729 if (list_empty(&ctx->rq_lists[type])) 1730 sbitmap_clear_bit(sb, bitnr); 1731 } 1732 spin_unlock(&ctx->lock); 1733 1734 return !dispatch_data->rq; 1735 } 1736 1737 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 1738 struct blk_mq_ctx *start) 1739 { 1740 unsigned off = start ? start->index_hw[hctx->type] : 0; 1741 struct dispatch_rq_data data = { 1742 .hctx = hctx, 1743 .rq = NULL, 1744 }; 1745 1746 __sbitmap_for_each_set(&hctx->ctx_map, off, 1747 dispatch_rq_from_ctx, &data); 1748 1749 return data.rq; 1750 } 1751 1752 bool __blk_mq_alloc_driver_tag(struct request *rq) 1753 { 1754 struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; 1755 unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; 1756 int tag; 1757 1758 blk_mq_tag_busy(rq->mq_hctx); 1759 1760 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) { 1761 bt = &rq->mq_hctx->tags->breserved_tags; 1762 tag_offset = 0; 1763 } else { 1764 if (!hctx_may_queue(rq->mq_hctx, bt)) 1765 return false; 1766 } 1767 1768 tag = __sbitmap_queue_get(bt); 1769 if (tag == BLK_MQ_NO_TAG) 1770 return false; 1771 1772 rq->tag = tag + tag_offset; 1773 blk_mq_inc_active_requests(rq->mq_hctx); 1774 return true; 1775 } 1776 1777 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, 1778 int flags, void *key) 1779 { 1780 struct blk_mq_hw_ctx *hctx; 1781 1782 hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); 1783 1784 spin_lock(&hctx->dispatch_wait_lock); 1785 if (!list_empty(&wait->entry)) { 1786 struct sbitmap_queue *sbq; 1787 1788 list_del_init(&wait->entry); 1789 sbq = &hctx->tags->bitmap_tags; 1790 atomic_dec(&sbq->ws_active); 1791 } 1792 spin_unlock(&hctx->dispatch_wait_lock); 1793 1794 blk_mq_run_hw_queue(hctx, true); 1795 return 1; 1796 } 1797 1798 /* 1799 * Mark us waiting for a tag. For shared tags, this involves hooking us into 1800 * the tag wakeups. For non-shared tags, we can simply mark us needing a 1801 * restart. For both cases, take care to check the condition again after 1802 * marking us as waiting. 1803 */ 1804 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx, 1805 struct request *rq) 1806 { 1807 struct sbitmap_queue *sbq; 1808 struct wait_queue_head *wq; 1809 wait_queue_entry_t *wait; 1810 bool ret; 1811 1812 if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) && 1813 !(blk_mq_is_shared_tags(hctx->flags))) { 1814 blk_mq_sched_mark_restart_hctx(hctx); 1815 1816 /* 1817 * It's possible that a tag was freed in the window between the 1818 * allocation failure and adding the hardware queue to the wait 1819 * queue. 1820 * 1821 * Don't clear RESTART here, someone else could have set it. 1822 * At most this will cost an extra queue run. 1823 */ 1824 return blk_mq_get_driver_tag(rq); 1825 } 1826 1827 wait = &hctx->dispatch_wait; 1828 if (!list_empty_careful(&wait->entry)) 1829 return false; 1830 1831 if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) 1832 sbq = &hctx->tags->breserved_tags; 1833 else 1834 sbq = &hctx->tags->bitmap_tags; 1835 wq = &bt_wait_ptr(sbq, hctx)->wait; 1836 1837 spin_lock_irq(&wq->lock); 1838 spin_lock(&hctx->dispatch_wait_lock); 1839 if (!list_empty(&wait->entry)) { 1840 spin_unlock(&hctx->dispatch_wait_lock); 1841 spin_unlock_irq(&wq->lock); 1842 return false; 1843 } 1844 1845 atomic_inc(&sbq->ws_active); 1846 wait->flags &= ~WQ_FLAG_EXCLUSIVE; 1847 __add_wait_queue(wq, wait); 1848 1849 /* 1850 * It's possible that a tag was freed in the window between the 1851 * allocation failure and adding the hardware queue to the wait 1852 * queue. 1853 */ 1854 ret = blk_mq_get_driver_tag(rq); 1855 if (!ret) { 1856 spin_unlock(&hctx->dispatch_wait_lock); 1857 spin_unlock_irq(&wq->lock); 1858 return false; 1859 } 1860 1861 /* 1862 * We got a tag, remove ourselves from the wait queue to ensure 1863 * someone else gets the wakeup. 1864 */ 1865 list_del_init(&wait->entry); 1866 atomic_dec(&sbq->ws_active); 1867 spin_unlock(&hctx->dispatch_wait_lock); 1868 spin_unlock_irq(&wq->lock); 1869 1870 return true; 1871 } 1872 1873 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8 1874 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4 1875 /* 1876 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA): 1877 * - EWMA is one simple way to compute running average value 1878 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially 1879 * - take 4 as factor for avoiding to get too small(0) result, and this 1880 * factor doesn't matter because EWMA decreases exponentially 1881 */ 1882 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy) 1883 { 1884 unsigned int ewma; 1885 1886 ewma = hctx->dispatch_busy; 1887 1888 if (!ewma && !busy) 1889 return; 1890 1891 ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1; 1892 if (busy) 1893 ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR; 1894 ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT; 1895 1896 hctx->dispatch_busy = ewma; 1897 } 1898 1899 #define BLK_MQ_RESOURCE_DELAY 3 /* ms units */ 1900 1901 static void blk_mq_handle_dev_resource(struct request *rq, 1902 struct list_head *list) 1903 { 1904 list_add(&rq->queuelist, list); 1905 __blk_mq_requeue_request(rq); 1906 } 1907 1908 static void blk_mq_handle_zone_resource(struct request *rq, 1909 struct list_head *zone_list) 1910 { 1911 /* 1912 * If we end up here it is because we cannot dispatch a request to a 1913 * specific zone due to LLD level zone-write locking or other zone 1914 * related resource not being available. In this case, set the request 1915 * aside in zone_list for retrying it later. 1916 */ 1917 list_add(&rq->queuelist, zone_list); 1918 __blk_mq_requeue_request(rq); 1919 } 1920 1921 enum prep_dispatch { 1922 PREP_DISPATCH_OK, 1923 PREP_DISPATCH_NO_TAG, 1924 PREP_DISPATCH_NO_BUDGET, 1925 }; 1926 1927 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq, 1928 bool need_budget) 1929 { 1930 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 1931 int budget_token = -1; 1932 1933 if (need_budget) { 1934 budget_token = blk_mq_get_dispatch_budget(rq->q); 1935 if (budget_token < 0) { 1936 blk_mq_put_driver_tag(rq); 1937 return PREP_DISPATCH_NO_BUDGET; 1938 } 1939 blk_mq_set_rq_budget_token(rq, budget_token); 1940 } 1941 1942 if (!blk_mq_get_driver_tag(rq)) { 1943 /* 1944 * The initial allocation attempt failed, so we need to 1945 * rerun the hardware queue when a tag is freed. The 1946 * waitqueue takes care of that. If the queue is run 1947 * before we add this entry back on the dispatch list, 1948 * we'll re-run it below. 1949 */ 1950 if (!blk_mq_mark_tag_wait(hctx, rq)) { 1951 /* 1952 * All budgets not got from this function will be put 1953 * together during handling partial dispatch 1954 */ 1955 if (need_budget) 1956 blk_mq_put_dispatch_budget(rq->q, budget_token); 1957 return PREP_DISPATCH_NO_TAG; 1958 } 1959 } 1960 1961 return PREP_DISPATCH_OK; 1962 } 1963 1964 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */ 1965 static void blk_mq_release_budgets(struct request_queue *q, 1966 struct list_head *list) 1967 { 1968 struct request *rq; 1969 1970 list_for_each_entry(rq, list, queuelist) { 1971 int budget_token = blk_mq_get_rq_budget_token(rq); 1972 1973 if (budget_token >= 0) 1974 blk_mq_put_dispatch_budget(q, budget_token); 1975 } 1976 } 1977 1978 /* 1979 * blk_mq_commit_rqs will notify driver using bd->last that there is no 1980 * more requests. (See comment in struct blk_mq_ops for commit_rqs for 1981 * details) 1982 * Attention, we should explicitly call this in unusual cases: 1983 * 1) did not queue everything initially scheduled to queue 1984 * 2) the last attempt to queue a request failed 1985 */ 1986 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued, 1987 bool from_schedule) 1988 { 1989 if (hctx->queue->mq_ops->commit_rqs && queued) { 1990 trace_block_unplug(hctx->queue, queued, !from_schedule); 1991 hctx->queue->mq_ops->commit_rqs(hctx); 1992 } 1993 } 1994 1995 /* 1996 * Returns true if we did some work AND can potentially do more. 1997 */ 1998 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list, 1999 unsigned int nr_budgets) 2000 { 2001 enum prep_dispatch prep; 2002 struct request_queue *q = hctx->queue; 2003 struct request *rq; 2004 int queued; 2005 blk_status_t ret = BLK_STS_OK; 2006 LIST_HEAD(zone_list); 2007 bool needs_resource = false; 2008 2009 if (list_empty(list)) 2010 return false; 2011 2012 /* 2013 * Now process all the entries, sending them to the driver. 2014 */ 2015 queued = 0; 2016 do { 2017 struct blk_mq_queue_data bd; 2018 2019 rq = list_first_entry(list, struct request, queuelist); 2020 2021 WARN_ON_ONCE(hctx != rq->mq_hctx); 2022 prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets); 2023 if (prep != PREP_DISPATCH_OK) 2024 break; 2025 2026 list_del_init(&rq->queuelist); 2027 2028 bd.rq = rq; 2029 bd.last = list_empty(list); 2030 2031 /* 2032 * once the request is queued to lld, no need to cover the 2033 * budget any more 2034 */ 2035 if (nr_budgets) 2036 nr_budgets--; 2037 ret = q->mq_ops->queue_rq(hctx, &bd); 2038 switch (ret) { 2039 case BLK_STS_OK: 2040 queued++; 2041 break; 2042 case BLK_STS_RESOURCE: 2043 needs_resource = true; 2044 fallthrough; 2045 case BLK_STS_DEV_RESOURCE: 2046 blk_mq_handle_dev_resource(rq, list); 2047 goto out; 2048 case BLK_STS_ZONE_RESOURCE: 2049 /* 2050 * Move the request to zone_list and keep going through 2051 * the dispatch list to find more requests the drive can 2052 * accept. 2053 */ 2054 blk_mq_handle_zone_resource(rq, &zone_list); 2055 needs_resource = true; 2056 break; 2057 default: 2058 blk_mq_end_request(rq, ret); 2059 } 2060 } while (!list_empty(list)); 2061 out: 2062 if (!list_empty(&zone_list)) 2063 list_splice_tail_init(&zone_list, list); 2064 2065 /* If we didn't flush the entire list, we could have told the driver 2066 * there was more coming, but that turned out to be a lie. 2067 */ 2068 if (!list_empty(list) || ret != BLK_STS_OK) 2069 blk_mq_commit_rqs(hctx, queued, false); 2070 2071 /* 2072 * Any items that need requeuing? Stuff them into hctx->dispatch, 2073 * that is where we will continue on next queue run. 2074 */ 2075 if (!list_empty(list)) { 2076 bool needs_restart; 2077 /* For non-shared tags, the RESTART check will suffice */ 2078 bool no_tag = prep == PREP_DISPATCH_NO_TAG && 2079 ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) || 2080 blk_mq_is_shared_tags(hctx->flags)); 2081 2082 if (nr_budgets) 2083 blk_mq_release_budgets(q, list); 2084 2085 spin_lock(&hctx->lock); 2086 list_splice_tail_init(list, &hctx->dispatch); 2087 spin_unlock(&hctx->lock); 2088 2089 /* 2090 * Order adding requests to hctx->dispatch and checking 2091 * SCHED_RESTART flag. The pair of this smp_mb() is the one 2092 * in blk_mq_sched_restart(). Avoid restart code path to 2093 * miss the new added requests to hctx->dispatch, meantime 2094 * SCHED_RESTART is observed here. 2095 */ 2096 smp_mb(); 2097 2098 /* 2099 * If SCHED_RESTART was set by the caller of this function and 2100 * it is no longer set that means that it was cleared by another 2101 * thread and hence that a queue rerun is needed. 2102 * 2103 * If 'no_tag' is set, that means that we failed getting 2104 * a driver tag with an I/O scheduler attached. If our dispatch 2105 * waitqueue is no longer active, ensure that we run the queue 2106 * AFTER adding our entries back to the list. 2107 * 2108 * If no I/O scheduler has been configured it is possible that 2109 * the hardware queue got stopped and restarted before requests 2110 * were pushed back onto the dispatch list. Rerun the queue to 2111 * avoid starvation. Notes: 2112 * - blk_mq_run_hw_queue() checks whether or not a queue has 2113 * been stopped before rerunning a queue. 2114 * - Some but not all block drivers stop a queue before 2115 * returning BLK_STS_RESOURCE. Two exceptions are scsi-mq 2116 * and dm-rq. 2117 * 2118 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART 2119 * bit is set, run queue after a delay to avoid IO stalls 2120 * that could otherwise occur if the queue is idle. We'll do 2121 * similar if we couldn't get budget or couldn't lock a zone 2122 * and SCHED_RESTART is set. 2123 */ 2124 needs_restart = blk_mq_sched_needs_restart(hctx); 2125 if (prep == PREP_DISPATCH_NO_BUDGET) 2126 needs_resource = true; 2127 if (!needs_restart || 2128 (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) 2129 blk_mq_run_hw_queue(hctx, true); 2130 else if (needs_resource) 2131 blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); 2132 2133 blk_mq_update_dispatch_busy(hctx, true); 2134 return false; 2135 } 2136 2137 blk_mq_update_dispatch_busy(hctx, false); 2138 return true; 2139 } 2140 2141 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx) 2142 { 2143 int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask); 2144 2145 if (cpu >= nr_cpu_ids) 2146 cpu = cpumask_first(hctx->cpumask); 2147 return cpu; 2148 } 2149 2150 /* 2151 * It'd be great if the workqueue API had a way to pass 2152 * in a mask and had some smarts for more clever placement. 2153 * For now we just round-robin here, switching for every 2154 * BLK_MQ_CPU_WORK_BATCH queued items. 2155 */ 2156 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) 2157 { 2158 bool tried = false; 2159 int next_cpu = hctx->next_cpu; 2160 2161 if (hctx->queue->nr_hw_queues == 1) 2162 return WORK_CPU_UNBOUND; 2163 2164 if (--hctx->next_cpu_batch <= 0) { 2165 select_cpu: 2166 next_cpu = cpumask_next_and(next_cpu, hctx->cpumask, 2167 cpu_online_mask); 2168 if (next_cpu >= nr_cpu_ids) 2169 next_cpu = blk_mq_first_mapped_cpu(hctx); 2170 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 2171 } 2172 2173 /* 2174 * Do unbound schedule if we can't find a online CPU for this hctx, 2175 * and it should only happen in the path of handling CPU DEAD. 2176 */ 2177 if (!cpu_online(next_cpu)) { 2178 if (!tried) { 2179 tried = true; 2180 goto select_cpu; 2181 } 2182 2183 /* 2184 * Make sure to re-select CPU next time once after CPUs 2185 * in hctx->cpumask become online again. 2186 */ 2187 hctx->next_cpu = next_cpu; 2188 hctx->next_cpu_batch = 1; 2189 return WORK_CPU_UNBOUND; 2190 } 2191 2192 hctx->next_cpu = next_cpu; 2193 return next_cpu; 2194 } 2195 2196 /** 2197 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously. 2198 * @hctx: Pointer to the hardware queue to run. 2199 * @msecs: Milliseconds of delay to wait before running the queue. 2200 * 2201 * Run a hardware queue asynchronously with a delay of @msecs. 2202 */ 2203 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) 2204 { 2205 if (unlikely(blk_mq_hctx_stopped(hctx))) 2206 return; 2207 kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, 2208 msecs_to_jiffies(msecs)); 2209 } 2210 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 2211 2212 /** 2213 * blk_mq_run_hw_queue - Start to run a hardware queue. 2214 * @hctx: Pointer to the hardware queue to run. 2215 * @async: If we want to run the queue asynchronously. 2216 * 2217 * Check if the request queue is not in a quiesced state and if there are 2218 * pending requests to be sent. If this is true, run the queue to send requests 2219 * to hardware. 2220 */ 2221 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 2222 { 2223 bool need_run; 2224 2225 /* 2226 * We can't run the queue inline with interrupts disabled. 2227 */ 2228 WARN_ON_ONCE(!async && in_interrupt()); 2229 2230 might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING); 2231 2232 /* 2233 * When queue is quiesced, we may be switching io scheduler, or 2234 * updating nr_hw_queues, or other things, and we can't run queue 2235 * any more, even __blk_mq_hctx_has_pending() can't be called safely. 2236 * 2237 * And queue will be rerun in blk_mq_unquiesce_queue() if it is 2238 * quiesced. 2239 */ 2240 __blk_mq_run_dispatch_ops(hctx->queue, false, 2241 need_run = !blk_queue_quiesced(hctx->queue) && 2242 blk_mq_hctx_has_pending(hctx)); 2243 2244 if (!need_run) 2245 return; 2246 2247 if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) { 2248 blk_mq_delay_run_hw_queue(hctx, 0); 2249 return; 2250 } 2251 2252 blk_mq_run_dispatch_ops(hctx->queue, 2253 blk_mq_sched_dispatch_requests(hctx)); 2254 } 2255 EXPORT_SYMBOL(blk_mq_run_hw_queue); 2256 2257 /* 2258 * Return prefered queue to dispatch from (if any) for non-mq aware IO 2259 * scheduler. 2260 */ 2261 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) 2262 { 2263 struct blk_mq_ctx *ctx = blk_mq_get_ctx(q); 2264 /* 2265 * If the IO scheduler does not respect hardware queues when 2266 * dispatching, we just don't bother with multiple HW queues and 2267 * dispatch from hctx for the current CPU since running multiple queues 2268 * just causes lock contention inside the scheduler and pointless cache 2269 * bouncing. 2270 */ 2271 struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT]; 2272 2273 if (!blk_mq_hctx_stopped(hctx)) 2274 return hctx; 2275 return NULL; 2276 } 2277 2278 /** 2279 * blk_mq_run_hw_queues - Run all hardware queues in a request queue. 2280 * @q: Pointer to the request queue to run. 2281 * @async: If we want to run the queue asynchronously. 2282 */ 2283 void blk_mq_run_hw_queues(struct request_queue *q, bool async) 2284 { 2285 struct blk_mq_hw_ctx *hctx, *sq_hctx; 2286 unsigned long i; 2287 2288 sq_hctx = NULL; 2289 if (blk_queue_sq_sched(q)) 2290 sq_hctx = blk_mq_get_sq_hctx(q); 2291 queue_for_each_hw_ctx(q, hctx, i) { 2292 if (blk_mq_hctx_stopped(hctx)) 2293 continue; 2294 /* 2295 * Dispatch from this hctx either if there's no hctx preferred 2296 * by IO scheduler or if it has requests that bypass the 2297 * scheduler. 2298 */ 2299 if (!sq_hctx || sq_hctx == hctx || 2300 !list_empty_careful(&hctx->dispatch)) 2301 blk_mq_run_hw_queue(hctx, async); 2302 } 2303 } 2304 EXPORT_SYMBOL(blk_mq_run_hw_queues); 2305 2306 /** 2307 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously. 2308 * @q: Pointer to the request queue to run. 2309 * @msecs: Milliseconds of delay to wait before running the queues. 2310 */ 2311 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) 2312 { 2313 struct blk_mq_hw_ctx *hctx, *sq_hctx; 2314 unsigned long i; 2315 2316 sq_hctx = NULL; 2317 if (blk_queue_sq_sched(q)) 2318 sq_hctx = blk_mq_get_sq_hctx(q); 2319 queue_for_each_hw_ctx(q, hctx, i) { 2320 if (blk_mq_hctx_stopped(hctx)) 2321 continue; 2322 /* 2323 * If there is already a run_work pending, leave the 2324 * pending delay untouched. Otherwise, a hctx can stall 2325 * if another hctx is re-delaying the other's work 2326 * before the work executes. 2327 */ 2328 if (delayed_work_pending(&hctx->run_work)) 2329 continue; 2330 /* 2331 * Dispatch from this hctx either if there's no hctx preferred 2332 * by IO scheduler or if it has requests that bypass the 2333 * scheduler. 2334 */ 2335 if (!sq_hctx || sq_hctx == hctx || 2336 !list_empty_careful(&hctx->dispatch)) 2337 blk_mq_delay_run_hw_queue(hctx, msecs); 2338 } 2339 } 2340 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues); 2341 2342 /* 2343 * This function is often used for pausing .queue_rq() by driver when 2344 * there isn't enough resource or some conditions aren't satisfied, and 2345 * BLK_STS_RESOURCE is usually returned. 2346 * 2347 * We do not guarantee that dispatch can be drained or blocked 2348 * after blk_mq_stop_hw_queue() returns. Please use 2349 * blk_mq_quiesce_queue() for that requirement. 2350 */ 2351 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) 2352 { 2353 cancel_delayed_work(&hctx->run_work); 2354 2355 set_bit(BLK_MQ_S_STOPPED, &hctx->state); 2356 } 2357 EXPORT_SYMBOL(blk_mq_stop_hw_queue); 2358 2359 /* 2360 * This function is often used for pausing .queue_rq() by driver when 2361 * there isn't enough resource or some conditions aren't satisfied, and 2362 * BLK_STS_RESOURCE is usually returned. 2363 * 2364 * We do not guarantee that dispatch can be drained or blocked 2365 * after blk_mq_stop_hw_queues() returns. Please use 2366 * blk_mq_quiesce_queue() for that requirement. 2367 */ 2368 void blk_mq_stop_hw_queues(struct request_queue *q) 2369 { 2370 struct blk_mq_hw_ctx *hctx; 2371 unsigned long i; 2372 2373 queue_for_each_hw_ctx(q, hctx, i) 2374 blk_mq_stop_hw_queue(hctx); 2375 } 2376 EXPORT_SYMBOL(blk_mq_stop_hw_queues); 2377 2378 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) 2379 { 2380 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 2381 2382 blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING); 2383 } 2384 EXPORT_SYMBOL(blk_mq_start_hw_queue); 2385 2386 void blk_mq_start_hw_queues(struct request_queue *q) 2387 { 2388 struct blk_mq_hw_ctx *hctx; 2389 unsigned long i; 2390 2391 queue_for_each_hw_ctx(q, hctx, i) 2392 blk_mq_start_hw_queue(hctx); 2393 } 2394 EXPORT_SYMBOL(blk_mq_start_hw_queues); 2395 2396 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 2397 { 2398 if (!blk_mq_hctx_stopped(hctx)) 2399 return; 2400 2401 clear_bit(BLK_MQ_S_STOPPED, &hctx->state); 2402 blk_mq_run_hw_queue(hctx, async); 2403 } 2404 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); 2405 2406 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) 2407 { 2408 struct blk_mq_hw_ctx *hctx; 2409 unsigned long i; 2410 2411 queue_for_each_hw_ctx(q, hctx, i) 2412 blk_mq_start_stopped_hw_queue(hctx, async || 2413 (hctx->flags & BLK_MQ_F_BLOCKING)); 2414 } 2415 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); 2416 2417 static void blk_mq_run_work_fn(struct work_struct *work) 2418 { 2419 struct blk_mq_hw_ctx *hctx = 2420 container_of(work, struct blk_mq_hw_ctx, run_work.work); 2421 2422 blk_mq_run_dispatch_ops(hctx->queue, 2423 blk_mq_sched_dispatch_requests(hctx)); 2424 } 2425 2426 /** 2427 * blk_mq_request_bypass_insert - Insert a request at dispatch list. 2428 * @rq: Pointer to request to be inserted. 2429 * @flags: BLK_MQ_INSERT_* 2430 * 2431 * Should only be used carefully, when the caller knows we want to 2432 * bypass a potential IO scheduler on the target device. 2433 */ 2434 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags) 2435 { 2436 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2437 2438 spin_lock(&hctx->lock); 2439 if (flags & BLK_MQ_INSERT_AT_HEAD) 2440 list_add(&rq->queuelist, &hctx->dispatch); 2441 else 2442 list_add_tail(&rq->queuelist, &hctx->dispatch); 2443 spin_unlock(&hctx->lock); 2444 } 2445 2446 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, 2447 struct blk_mq_ctx *ctx, struct list_head *list, 2448 bool run_queue_async) 2449 { 2450 struct request *rq; 2451 enum hctx_type type = hctx->type; 2452 2453 /* 2454 * Try to issue requests directly if the hw queue isn't busy to save an 2455 * extra enqueue & dequeue to the sw queue. 2456 */ 2457 if (!hctx->dispatch_busy && !run_queue_async) { 2458 blk_mq_run_dispatch_ops(hctx->queue, 2459 blk_mq_try_issue_list_directly(hctx, list)); 2460 if (list_empty(list)) 2461 goto out; 2462 } 2463 2464 /* 2465 * preemption doesn't flush plug list, so it's possible ctx->cpu is 2466 * offline now 2467 */ 2468 list_for_each_entry(rq, list, queuelist) { 2469 BUG_ON(rq->mq_ctx != ctx); 2470 trace_block_rq_insert(rq); 2471 if (rq->cmd_flags & REQ_NOWAIT) 2472 run_queue_async = true; 2473 } 2474 2475 spin_lock(&ctx->lock); 2476 list_splice_tail_init(list, &ctx->rq_lists[type]); 2477 blk_mq_hctx_mark_pending(hctx, ctx); 2478 spin_unlock(&ctx->lock); 2479 out: 2480 blk_mq_run_hw_queue(hctx, run_queue_async); 2481 } 2482 2483 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags) 2484 { 2485 struct request_queue *q = rq->q; 2486 struct blk_mq_ctx *ctx = rq->mq_ctx; 2487 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2488 2489 if (blk_rq_is_passthrough(rq)) { 2490 /* 2491 * Passthrough request have to be added to hctx->dispatch 2492 * directly. The device may be in a situation where it can't 2493 * handle FS request, and always returns BLK_STS_RESOURCE for 2494 * them, which gets them added to hctx->dispatch. 2495 * 2496 * If a passthrough request is required to unblock the queues, 2497 * and it is added to the scheduler queue, there is no chance to 2498 * dispatch it given we prioritize requests in hctx->dispatch. 2499 */ 2500 blk_mq_request_bypass_insert(rq, flags); 2501 } else if (req_op(rq) == REQ_OP_FLUSH) { 2502 /* 2503 * Firstly normal IO request is inserted to scheduler queue or 2504 * sw queue, meantime we add flush request to dispatch queue( 2505 * hctx->dispatch) directly and there is at most one in-flight 2506 * flush request for each hw queue, so it doesn't matter to add 2507 * flush request to tail or front of the dispatch queue. 2508 * 2509 * Secondly in case of NCQ, flush request belongs to non-NCQ 2510 * command, and queueing it will fail when there is any 2511 * in-flight normal IO request(NCQ command). When adding flush 2512 * rq to the front of hctx->dispatch, it is easier to introduce 2513 * extra time to flush rq's latency because of S_SCHED_RESTART 2514 * compared with adding to the tail of dispatch queue, then 2515 * chance of flush merge is increased, and less flush requests 2516 * will be issued to controller. It is observed that ~10% time 2517 * is saved in blktests block/004 on disk attached to AHCI/NCQ 2518 * drive when adding flush rq to the front of hctx->dispatch. 2519 * 2520 * Simply queue flush rq to the front of hctx->dispatch so that 2521 * intensive flush workloads can benefit in case of NCQ HW. 2522 */ 2523 blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD); 2524 } else if (q->elevator) { 2525 LIST_HEAD(list); 2526 2527 WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG); 2528 2529 list_add(&rq->queuelist, &list); 2530 q->elevator->type->ops.insert_requests(hctx, &list, flags); 2531 } else { 2532 trace_block_rq_insert(rq); 2533 2534 spin_lock(&ctx->lock); 2535 if (flags & BLK_MQ_INSERT_AT_HEAD) 2536 list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]); 2537 else 2538 list_add_tail(&rq->queuelist, 2539 &ctx->rq_lists[hctx->type]); 2540 blk_mq_hctx_mark_pending(hctx, ctx); 2541 spin_unlock(&ctx->lock); 2542 } 2543 } 2544 2545 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, 2546 unsigned int nr_segs) 2547 { 2548 int err; 2549 2550 if (bio->bi_opf & REQ_RAHEAD) 2551 rq->cmd_flags |= REQ_FAILFAST_MASK; 2552 2553 rq->__sector = bio->bi_iter.bi_sector; 2554 blk_rq_bio_prep(rq, bio, nr_segs); 2555 2556 /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */ 2557 err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO); 2558 WARN_ON_ONCE(err); 2559 2560 blk_account_io_start(rq); 2561 } 2562 2563 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, 2564 struct request *rq, bool last) 2565 { 2566 struct request_queue *q = rq->q; 2567 struct blk_mq_queue_data bd = { 2568 .rq = rq, 2569 .last = last, 2570 }; 2571 blk_status_t ret; 2572 2573 /* 2574 * For OK queue, we are done. For error, caller may kill it. 2575 * Any other error (busy), just add it to our list as we 2576 * previously would have done. 2577 */ 2578 ret = q->mq_ops->queue_rq(hctx, &bd); 2579 switch (ret) { 2580 case BLK_STS_OK: 2581 blk_mq_update_dispatch_busy(hctx, false); 2582 break; 2583 case BLK_STS_RESOURCE: 2584 case BLK_STS_DEV_RESOURCE: 2585 blk_mq_update_dispatch_busy(hctx, true); 2586 __blk_mq_requeue_request(rq); 2587 break; 2588 default: 2589 blk_mq_update_dispatch_busy(hctx, false); 2590 break; 2591 } 2592 2593 return ret; 2594 } 2595 2596 static bool blk_mq_get_budget_and_tag(struct request *rq) 2597 { 2598 int budget_token; 2599 2600 budget_token = blk_mq_get_dispatch_budget(rq->q); 2601 if (budget_token < 0) 2602 return false; 2603 blk_mq_set_rq_budget_token(rq, budget_token); 2604 if (!blk_mq_get_driver_tag(rq)) { 2605 blk_mq_put_dispatch_budget(rq->q, budget_token); 2606 return false; 2607 } 2608 return true; 2609 } 2610 2611 /** 2612 * blk_mq_try_issue_directly - Try to send a request directly to device driver. 2613 * @hctx: Pointer of the associated hardware queue. 2614 * @rq: Pointer to request to be sent. 2615 * 2616 * If the device has enough resources to accept a new request now, send the 2617 * request directly to device driver. Else, insert at hctx->dispatch queue, so 2618 * we can try send it another time in the future. Requests inserted at this 2619 * queue have higher priority. 2620 */ 2621 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, 2622 struct request *rq) 2623 { 2624 blk_status_t ret; 2625 2626 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) { 2627 blk_mq_insert_request(rq, 0); 2628 return; 2629 } 2630 2631 if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) { 2632 blk_mq_insert_request(rq, 0); 2633 blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT); 2634 return; 2635 } 2636 2637 ret = __blk_mq_issue_directly(hctx, rq, true); 2638 switch (ret) { 2639 case BLK_STS_OK: 2640 break; 2641 case BLK_STS_RESOURCE: 2642 case BLK_STS_DEV_RESOURCE: 2643 blk_mq_request_bypass_insert(rq, 0); 2644 blk_mq_run_hw_queue(hctx, false); 2645 break; 2646 default: 2647 blk_mq_end_request(rq, ret); 2648 break; 2649 } 2650 } 2651 2652 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last) 2653 { 2654 struct blk_mq_hw_ctx *hctx = rq->mq_hctx; 2655 2656 if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) { 2657 blk_mq_insert_request(rq, 0); 2658 return BLK_STS_OK; 2659 } 2660 2661 if (!blk_mq_get_budget_and_tag(rq)) 2662 return BLK_STS_RESOURCE; 2663 return __blk_mq_issue_directly(hctx, rq, last); 2664 } 2665 2666 static void blk_mq_plug_issue_direct(struct blk_plug *plug) 2667 { 2668 struct blk_mq_hw_ctx *hctx = NULL; 2669 struct request *rq; 2670 int queued = 0; 2671 blk_status_t ret = BLK_STS_OK; 2672 2673 while ((rq = rq_list_pop(&plug->mq_list))) { 2674 bool last = rq_list_empty(plug->mq_list); 2675 2676 if (hctx != rq->mq_hctx) { 2677 if (hctx) { 2678 blk_mq_commit_rqs(hctx, queued, false); 2679 queued = 0; 2680 } 2681 hctx = rq->mq_hctx; 2682 } 2683 2684 ret = blk_mq_request_issue_directly(rq, last); 2685 switch (ret) { 2686 case BLK_STS_OK: 2687 queued++; 2688 break; 2689 case BLK_STS_RESOURCE: 2690 case BLK_STS_DEV_RESOURCE: 2691 blk_mq_request_bypass_insert(rq, 0); 2692 blk_mq_run_hw_queue(hctx, false); 2693 goto out; 2694 default: 2695 blk_mq_end_request(rq, ret); 2696 break; 2697 } 2698 } 2699 2700 out: 2701 if (ret != BLK_STS_OK) 2702 blk_mq_commit_rqs(hctx, queued, false); 2703 } 2704 2705 static void __blk_mq_flush_plug_list(struct request_queue *q, 2706 struct blk_plug *plug) 2707 { 2708 if (blk_queue_quiesced(q)) 2709 return; 2710 q->mq_ops->queue_rqs(&plug->mq_list); 2711 } 2712 2713 static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched) 2714 { 2715 struct blk_mq_hw_ctx *this_hctx = NULL; 2716 struct blk_mq_ctx *this_ctx = NULL; 2717 struct request *requeue_list = NULL; 2718 struct request **requeue_lastp = &requeue_list; 2719 unsigned int depth = 0; 2720 bool is_passthrough = false; 2721 LIST_HEAD(list); 2722 2723 do { 2724 struct request *rq = rq_list_pop(&plug->mq_list); 2725 2726 if (!this_hctx) { 2727 this_hctx = rq->mq_hctx; 2728 this_ctx = rq->mq_ctx; 2729 is_passthrough = blk_rq_is_passthrough(rq); 2730 } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx || 2731 is_passthrough != blk_rq_is_passthrough(rq)) { 2732 rq_list_add_tail(&requeue_lastp, rq); 2733 continue; 2734 } 2735 list_add(&rq->queuelist, &list); 2736 depth++; 2737 } while (!rq_list_empty(plug->mq_list)); 2738 2739 plug->mq_list = requeue_list; 2740 trace_block_unplug(this_hctx->queue, depth, !from_sched); 2741 2742 percpu_ref_get(&this_hctx->queue->q_usage_counter); 2743 /* passthrough requests should never be issued to the I/O scheduler */ 2744 if (is_passthrough) { 2745 spin_lock(&this_hctx->lock); 2746 list_splice_tail_init(&list, &this_hctx->dispatch); 2747 spin_unlock(&this_hctx->lock); 2748 blk_mq_run_hw_queue(this_hctx, from_sched); 2749 } else if (this_hctx->queue->elevator) { 2750 this_hctx->queue->elevator->type->ops.insert_requests(this_hctx, 2751 &list, 0); 2752 blk_mq_run_hw_queue(this_hctx, from_sched); 2753 } else { 2754 blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched); 2755 } 2756 percpu_ref_put(&this_hctx->queue->q_usage_counter); 2757 } 2758 2759 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) 2760 { 2761 struct request *rq; 2762 2763 /* 2764 * We may have been called recursively midway through handling 2765 * plug->mq_list via a schedule() in the driver's queue_rq() callback. 2766 * To avoid mq_list changing under our feet, clear rq_count early and 2767 * bail out specifically if rq_count is 0 rather than checking 2768 * whether the mq_list is empty. 2769 */ 2770 if (plug->rq_count == 0) 2771 return; 2772 plug->rq_count = 0; 2773 2774 if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) { 2775 struct request_queue *q; 2776 2777 rq = rq_list_peek(&plug->mq_list); 2778 q = rq->q; 2779 2780 /* 2781 * Peek first request and see if we have a ->queue_rqs() hook. 2782 * If we do, we can dispatch the whole plug list in one go. We 2783 * already know at this point that all requests belong to the 2784 * same queue, caller must ensure that's the case. 2785 */ 2786 if (q->mq_ops->queue_rqs) { 2787 blk_mq_run_dispatch_ops(q, 2788 __blk_mq_flush_plug_list(q, plug)); 2789 if (rq_list_empty(plug->mq_list)) 2790 return; 2791 } 2792 2793 blk_mq_run_dispatch_ops(q, 2794 blk_mq_plug_issue_direct(plug)); 2795 if (rq_list_empty(plug->mq_list)) 2796 return; 2797 } 2798 2799 do { 2800 blk_mq_dispatch_plug_list(plug, from_schedule); 2801 } while (!rq_list_empty(plug->mq_list)); 2802 } 2803 2804 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, 2805 struct list_head *list) 2806 { 2807 int queued = 0; 2808 blk_status_t ret = BLK_STS_OK; 2809 2810 while (!list_empty(list)) { 2811 struct request *rq = list_first_entry(list, struct request, 2812 queuelist); 2813 2814 list_del_init(&rq->queuelist); 2815 ret = blk_mq_request_issue_directly(rq, list_empty(list)); 2816 switch (ret) { 2817 case BLK_STS_OK: 2818 queued++; 2819 break; 2820 case BLK_STS_RESOURCE: 2821 case BLK_STS_DEV_RESOURCE: 2822 blk_mq_request_bypass_insert(rq, 0); 2823 if (list_empty(list)) 2824 blk_mq_run_hw_queue(hctx, false); 2825 goto out; 2826 default: 2827 blk_mq_end_request(rq, ret); 2828 break; 2829 } 2830 } 2831 2832 out: 2833 if (ret != BLK_STS_OK) 2834 blk_mq_commit_rqs(hctx, queued, false); 2835 } 2836 2837 static bool blk_mq_attempt_bio_merge(struct request_queue *q, 2838 struct bio *bio, unsigned int nr_segs) 2839 { 2840 if (!blk_queue_nomerges(q) && bio_mergeable(bio)) { 2841 if (blk_attempt_plug_merge(q, bio, nr_segs)) 2842 return true; 2843 if (blk_mq_sched_bio_merge(q, bio, nr_segs)) 2844 return true; 2845 } 2846 return false; 2847 } 2848 2849 static struct request *blk_mq_get_new_requests(struct request_queue *q, 2850 struct blk_plug *plug, 2851 struct bio *bio, 2852 unsigned int nsegs) 2853 { 2854 struct blk_mq_alloc_data data = { 2855 .q = q, 2856 .nr_tags = 1, 2857 .cmd_flags = bio->bi_opf, 2858 }; 2859 struct request *rq; 2860 2861 if (blk_mq_attempt_bio_merge(q, bio, nsegs)) 2862 return NULL; 2863 2864 rq_qos_throttle(q, bio); 2865 2866 if (plug) { 2867 data.nr_tags = plug->nr_ios; 2868 plug->nr_ios = 1; 2869 data.cached_rq = &plug->cached_rq; 2870 } 2871 2872 rq = __blk_mq_alloc_requests(&data); 2873 if (rq) 2874 return rq; 2875 rq_qos_cleanup(q, bio); 2876 if (bio->bi_opf & REQ_NOWAIT) 2877 bio_wouldblock_error(bio); 2878 return NULL; 2879 } 2880 2881 /* return true if this @rq can be used for @bio */ 2882 static bool blk_mq_can_use_cached_rq(struct request *rq, struct blk_plug *plug, 2883 struct bio *bio) 2884 { 2885 enum hctx_type type = blk_mq_get_hctx_type(bio->bi_opf); 2886 enum hctx_type hctx_type = rq->mq_hctx->type; 2887 2888 WARN_ON_ONCE(rq_list_peek(&plug->cached_rq) != rq); 2889 2890 if (type != hctx_type && 2891 !(type == HCTX_TYPE_READ && hctx_type == HCTX_TYPE_DEFAULT)) 2892 return false; 2893 if (op_is_flush(rq->cmd_flags) != op_is_flush(bio->bi_opf)) 2894 return false; 2895 2896 /* 2897 * If any qos ->throttle() end up blocking, we will have flushed the 2898 * plug and hence killed the cached_rq list as well. Pop this entry 2899 * before we throttle. 2900 */ 2901 plug->cached_rq = rq_list_next(rq); 2902 rq_qos_throttle(rq->q, bio); 2903 2904 blk_mq_rq_time_init(rq, 0); 2905 rq->cmd_flags = bio->bi_opf; 2906 INIT_LIST_HEAD(&rq->queuelist); 2907 return true; 2908 } 2909 2910 static void bio_set_ioprio(struct bio *bio) 2911 { 2912 /* Nobody set ioprio so far? Initialize it based on task's nice value */ 2913 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE) 2914 bio->bi_ioprio = get_current_ioprio(); 2915 blkcg_set_ioprio(bio); 2916 } 2917 2918 /** 2919 * blk_mq_submit_bio - Create and send a request to block device. 2920 * @bio: Bio pointer. 2921 * 2922 * Builds up a request structure from @q and @bio and send to the device. The 2923 * request may not be queued directly to hardware if: 2924 * * This request can be merged with another one 2925 * * We want to place request at plug queue for possible future merging 2926 * * There is an IO scheduler active at this queue 2927 * 2928 * It will not queue the request if there is an error with the bio, or at the 2929 * request creation. 2930 */ 2931 void blk_mq_submit_bio(struct bio *bio) 2932 { 2933 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 2934 struct blk_plug *plug = blk_mq_plug(bio); 2935 const int is_sync = op_is_sync(bio->bi_opf); 2936 struct blk_mq_hw_ctx *hctx; 2937 struct request *rq = NULL; 2938 unsigned int nr_segs = 1; 2939 blk_status_t ret; 2940 2941 bio = blk_queue_bounce(bio, q); 2942 if (bio_may_exceed_limits(bio, &q->limits)) { 2943 bio = __bio_split_to_limits(bio, &q->limits, &nr_segs); 2944 if (!bio) 2945 return; 2946 } 2947 2948 bio_set_ioprio(bio); 2949 2950 if (plug) { 2951 rq = rq_list_peek(&plug->cached_rq); 2952 if (rq && rq->q != q) 2953 rq = NULL; 2954 } 2955 if (rq) { 2956 if (!bio_integrity_prep(bio)) 2957 return; 2958 if (blk_mq_attempt_bio_merge(q, bio, nr_segs)) 2959 return; 2960 if (blk_mq_can_use_cached_rq(rq, plug, bio)) 2961 goto done; 2962 percpu_ref_get(&q->q_usage_counter); 2963 } else { 2964 if (unlikely(bio_queue_enter(bio))) 2965 return; 2966 if (!bio_integrity_prep(bio)) 2967 goto fail; 2968 } 2969 2970 rq = blk_mq_get_new_requests(q, plug, bio, nr_segs); 2971 if (unlikely(!rq)) { 2972 fail: 2973 blk_queue_exit(q); 2974 return; 2975 } 2976 2977 done: 2978 trace_block_getrq(bio); 2979 2980 rq_qos_track(q, rq, bio); 2981 2982 blk_mq_bio_to_request(rq, bio, nr_segs); 2983 2984 ret = blk_crypto_rq_get_keyslot(rq); 2985 if (ret != BLK_STS_OK) { 2986 bio->bi_status = ret; 2987 bio_endio(bio); 2988 blk_mq_free_request(rq); 2989 return; 2990 } 2991 2992 if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq)) 2993 return; 2994 2995 if (plug) { 2996 blk_add_rq_to_plug(plug, rq); 2997 return; 2998 } 2999 3000 hctx = rq->mq_hctx; 3001 if ((rq->rq_flags & RQF_USE_SCHED) || 3002 (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) { 3003 blk_mq_insert_request(rq, 0); 3004 blk_mq_run_hw_queue(hctx, true); 3005 } else { 3006 blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq)); 3007 } 3008 } 3009 3010 #ifdef CONFIG_BLK_MQ_STACKING 3011 /** 3012 * blk_insert_cloned_request - Helper for stacking drivers to submit a request 3013 * @rq: the request being queued 3014 */ 3015 blk_status_t blk_insert_cloned_request(struct request *rq) 3016 { 3017 struct request_queue *q = rq->q; 3018 unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq)); 3019 unsigned int max_segments = blk_rq_get_max_segments(rq); 3020 blk_status_t ret; 3021 3022 if (blk_rq_sectors(rq) > max_sectors) { 3023 /* 3024 * SCSI device does not have a good way to return if 3025 * Write Same/Zero is actually supported. If a device rejects 3026 * a non-read/write command (discard, write same,etc.) the 3027 * low-level device driver will set the relevant queue limit to 3028 * 0 to prevent blk-lib from issuing more of the offending 3029 * operations. Commands queued prior to the queue limit being 3030 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O 3031 * errors being propagated to upper layers. 3032 */ 3033 if (max_sectors == 0) 3034 return BLK_STS_NOTSUPP; 3035 3036 printk(KERN_ERR "%s: over max size limit. (%u > %u)\n", 3037 __func__, blk_rq_sectors(rq), max_sectors); 3038 return BLK_STS_IOERR; 3039 } 3040 3041 /* 3042 * The queue settings related to segment counting may differ from the 3043 * original queue. 3044 */ 3045 rq->nr_phys_segments = blk_recalc_rq_segments(rq); 3046 if (rq->nr_phys_segments > max_segments) { 3047 printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n", 3048 __func__, rq->nr_phys_segments, max_segments); 3049 return BLK_STS_IOERR; 3050 } 3051 3052 if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq))) 3053 return BLK_STS_IOERR; 3054 3055 ret = blk_crypto_rq_get_keyslot(rq); 3056 if (ret != BLK_STS_OK) 3057 return ret; 3058 3059 blk_account_io_start(rq); 3060 3061 /* 3062 * Since we have a scheduler attached on the top device, 3063 * bypass a potential scheduler on the bottom device for 3064 * insert. 3065 */ 3066 blk_mq_run_dispatch_ops(q, 3067 ret = blk_mq_request_issue_directly(rq, true)); 3068 if (ret) 3069 blk_account_io_done(rq, ktime_get_ns()); 3070 return ret; 3071 } 3072 EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 3073 3074 /** 3075 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request 3076 * @rq: the clone request to be cleaned up 3077 * 3078 * Description: 3079 * Free all bios in @rq for a cloned request. 3080 */ 3081 void blk_rq_unprep_clone(struct request *rq) 3082 { 3083 struct bio *bio; 3084 3085 while ((bio = rq->bio) != NULL) { 3086 rq->bio = bio->bi_next; 3087 3088 bio_put(bio); 3089 } 3090 } 3091 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); 3092 3093 /** 3094 * blk_rq_prep_clone - Helper function to setup clone request 3095 * @rq: the request to be setup 3096 * @rq_src: original request to be cloned 3097 * @bs: bio_set that bios for clone are allocated from 3098 * @gfp_mask: memory allocation mask for bio 3099 * @bio_ctr: setup function to be called for each clone bio. 3100 * Returns %0 for success, non %0 for failure. 3101 * @data: private data to be passed to @bio_ctr 3102 * 3103 * Description: 3104 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. 3105 * Also, pages which the original bios are pointing to are not copied 3106 * and the cloned bios just point same pages. 3107 * So cloned bios must be completed before original bios, which means 3108 * the caller must complete @rq before @rq_src. 3109 */ 3110 int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 3111 struct bio_set *bs, gfp_t gfp_mask, 3112 int (*bio_ctr)(struct bio *, struct bio *, void *), 3113 void *data) 3114 { 3115 struct bio *bio, *bio_src; 3116 3117 if (!bs) 3118 bs = &fs_bio_set; 3119 3120 __rq_for_each_bio(bio_src, rq_src) { 3121 bio = bio_alloc_clone(rq->q->disk->part0, bio_src, gfp_mask, 3122 bs); 3123 if (!bio) 3124 goto free_and_out; 3125 3126 if (bio_ctr && bio_ctr(bio, bio_src, data)) 3127 goto free_and_out; 3128 3129 if (rq->bio) { 3130 rq->biotail->bi_next = bio; 3131 rq->biotail = bio; 3132 } else { 3133 rq->bio = rq->biotail = bio; 3134 } 3135 bio = NULL; 3136 } 3137 3138 /* Copy attributes of the original request to the clone request. */ 3139 rq->__sector = blk_rq_pos(rq_src); 3140 rq->__data_len = blk_rq_bytes(rq_src); 3141 if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) { 3142 rq->rq_flags |= RQF_SPECIAL_PAYLOAD; 3143 rq->special_vec = rq_src->special_vec; 3144 } 3145 rq->nr_phys_segments = rq_src->nr_phys_segments; 3146 rq->ioprio = rq_src->ioprio; 3147 3148 if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) 3149 goto free_and_out; 3150 3151 return 0; 3152 3153 free_and_out: 3154 if (bio) 3155 bio_put(bio); 3156 blk_rq_unprep_clone(rq); 3157 3158 return -ENOMEM; 3159 } 3160 EXPORT_SYMBOL_GPL(blk_rq_prep_clone); 3161 #endif /* CONFIG_BLK_MQ_STACKING */ 3162 3163 /* 3164 * Steal bios from a request and add them to a bio list. 3165 * The request must not have been partially completed before. 3166 */ 3167 void blk_steal_bios(struct bio_list *list, struct request *rq) 3168 { 3169 if (rq->bio) { 3170 if (list->tail) 3171 list->tail->bi_next = rq->bio; 3172 else 3173 list->head = rq->bio; 3174 list->tail = rq->biotail; 3175 3176 rq->bio = NULL; 3177 rq->biotail = NULL; 3178 } 3179 3180 rq->__data_len = 0; 3181 } 3182 EXPORT_SYMBOL_GPL(blk_steal_bios); 3183 3184 static size_t order_to_size(unsigned int order) 3185 { 3186 return (size_t)PAGE_SIZE << order; 3187 } 3188 3189 /* called before freeing request pool in @tags */ 3190 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags, 3191 struct blk_mq_tags *tags) 3192 { 3193 struct page *page; 3194 unsigned long flags; 3195 3196 /* 3197 * There is no need to clear mapping if driver tags is not initialized 3198 * or the mapping belongs to the driver tags. 3199 */ 3200 if (!drv_tags || drv_tags == tags) 3201 return; 3202 3203 list_for_each_entry(page, &tags->page_list, lru) { 3204 unsigned long start = (unsigned long)page_address(page); 3205 unsigned long end = start + order_to_size(page->private); 3206 int i; 3207 3208 for (i = 0; i < drv_tags->nr_tags; i++) { 3209 struct request *rq = drv_tags->rqs[i]; 3210 unsigned long rq_addr = (unsigned long)rq; 3211 3212 if (rq_addr >= start && rq_addr < end) { 3213 WARN_ON_ONCE(req_ref_read(rq) != 0); 3214 cmpxchg(&drv_tags->rqs[i], rq, NULL); 3215 } 3216 } 3217 } 3218 3219 /* 3220 * Wait until all pending iteration is done. 3221 * 3222 * Request reference is cleared and it is guaranteed to be observed 3223 * after the ->lock is released. 3224 */ 3225 spin_lock_irqsave(&drv_tags->lock, flags); 3226 spin_unlock_irqrestore(&drv_tags->lock, flags); 3227 } 3228 3229 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, 3230 unsigned int hctx_idx) 3231 { 3232 struct blk_mq_tags *drv_tags; 3233 struct page *page; 3234 3235 if (list_empty(&tags->page_list)) 3236 return; 3237 3238 if (blk_mq_is_shared_tags(set->flags)) 3239 drv_tags = set->shared_tags; 3240 else 3241 drv_tags = set->tags[hctx_idx]; 3242 3243 if (tags->static_rqs && set->ops->exit_request) { 3244 int i; 3245 3246 for (i = 0; i < tags->nr_tags; i++) { 3247 struct request *rq = tags->static_rqs[i]; 3248 3249 if (!rq) 3250 continue; 3251 set->ops->exit_request(set, rq, hctx_idx); 3252 tags->static_rqs[i] = NULL; 3253 } 3254 } 3255 3256 blk_mq_clear_rq_mapping(drv_tags, tags); 3257 3258 while (!list_empty(&tags->page_list)) { 3259 page = list_first_entry(&tags->page_list, struct page, lru); 3260 list_del_init(&page->lru); 3261 /* 3262 * Remove kmemleak object previously allocated in 3263 * blk_mq_alloc_rqs(). 3264 */ 3265 kmemleak_free(page_address(page)); 3266 __free_pages(page, page->private); 3267 } 3268 } 3269 3270 void blk_mq_free_rq_map(struct blk_mq_tags *tags) 3271 { 3272 kfree(tags->rqs); 3273 tags->rqs = NULL; 3274 kfree(tags->static_rqs); 3275 tags->static_rqs = NULL; 3276 3277 blk_mq_free_tags(tags); 3278 } 3279 3280 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set, 3281 unsigned int hctx_idx) 3282 { 3283 int i; 3284 3285 for (i = 0; i < set->nr_maps; i++) { 3286 unsigned int start = set->map[i].queue_offset; 3287 unsigned int end = start + set->map[i].nr_queues; 3288 3289 if (hctx_idx >= start && hctx_idx < end) 3290 break; 3291 } 3292 3293 if (i >= set->nr_maps) 3294 i = HCTX_TYPE_DEFAULT; 3295 3296 return i; 3297 } 3298 3299 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set, 3300 unsigned int hctx_idx) 3301 { 3302 enum hctx_type type = hctx_idx_to_type(set, hctx_idx); 3303 3304 return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx); 3305 } 3306 3307 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, 3308 unsigned int hctx_idx, 3309 unsigned int nr_tags, 3310 unsigned int reserved_tags) 3311 { 3312 int node = blk_mq_get_hctx_node(set, hctx_idx); 3313 struct blk_mq_tags *tags; 3314 3315 if (node == NUMA_NO_NODE) 3316 node = set->numa_node; 3317 3318 tags = blk_mq_init_tags(nr_tags, reserved_tags, node, 3319 BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); 3320 if (!tags) 3321 return NULL; 3322 3323 tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *), 3324 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 3325 node); 3326 if (!tags->rqs) 3327 goto err_free_tags; 3328 3329 tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *), 3330 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, 3331 node); 3332 if (!tags->static_rqs) 3333 goto err_free_rqs; 3334 3335 return tags; 3336 3337 err_free_rqs: 3338 kfree(tags->rqs); 3339 err_free_tags: 3340 blk_mq_free_tags(tags); 3341 return NULL; 3342 } 3343 3344 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, 3345 unsigned int hctx_idx, int node) 3346 { 3347 int ret; 3348 3349 if (set->ops->init_request) { 3350 ret = set->ops->init_request(set, rq, hctx_idx, node); 3351 if (ret) 3352 return ret; 3353 } 3354 3355 WRITE_ONCE(rq->state, MQ_RQ_IDLE); 3356 return 0; 3357 } 3358 3359 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, 3360 struct blk_mq_tags *tags, 3361 unsigned int hctx_idx, unsigned int depth) 3362 { 3363 unsigned int i, j, entries_per_page, max_order = 4; 3364 int node = blk_mq_get_hctx_node(set, hctx_idx); 3365 size_t rq_size, left; 3366 3367 if (node == NUMA_NO_NODE) 3368 node = set->numa_node; 3369 3370 INIT_LIST_HEAD(&tags->page_list); 3371 3372 /* 3373 * rq_size is the size of the request plus driver payload, rounded 3374 * to the cacheline size 3375 */ 3376 rq_size = round_up(sizeof(struct request) + set->cmd_size, 3377 cache_line_size()); 3378 left = rq_size * depth; 3379 3380 for (i = 0; i < depth; ) { 3381 int this_order = max_order; 3382 struct page *page; 3383 int to_do; 3384 void *p; 3385 3386 while (this_order && left < order_to_size(this_order - 1)) 3387 this_order--; 3388 3389 do { 3390 page = alloc_pages_node(node, 3391 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, 3392 this_order); 3393 if (page) 3394 break; 3395 if (!this_order--) 3396 break; 3397 if (order_to_size(this_order) < rq_size) 3398 break; 3399 } while (1); 3400 3401 if (!page) 3402 goto fail; 3403 3404 page->private = this_order; 3405 list_add_tail(&page->lru, &tags->page_list); 3406 3407 p = page_address(page); 3408 /* 3409 * Allow kmemleak to scan these pages as they contain pointers 3410 * to additional allocations like via ops->init_request(). 3411 */ 3412 kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); 3413 entries_per_page = order_to_size(this_order) / rq_size; 3414 to_do = min(entries_per_page, depth - i); 3415 left -= to_do * rq_size; 3416 for (j = 0; j < to_do; j++) { 3417 struct request *rq = p; 3418 3419 tags->static_rqs[i] = rq; 3420 if (blk_mq_init_request(set, rq, hctx_idx, node)) { 3421 tags->static_rqs[i] = NULL; 3422 goto fail; 3423 } 3424 3425 p += rq_size; 3426 i++; 3427 } 3428 } 3429 return 0; 3430 3431 fail: 3432 blk_mq_free_rqs(set, tags, hctx_idx); 3433 return -ENOMEM; 3434 } 3435 3436 struct rq_iter_data { 3437 struct blk_mq_hw_ctx *hctx; 3438 bool has_rq; 3439 }; 3440 3441 static bool blk_mq_has_request(struct request *rq, void *data) 3442 { 3443 struct rq_iter_data *iter_data = data; 3444 3445 if (rq->mq_hctx != iter_data->hctx) 3446 return true; 3447 iter_data->has_rq = true; 3448 return false; 3449 } 3450 3451 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx) 3452 { 3453 struct blk_mq_tags *tags = hctx->sched_tags ? 3454 hctx->sched_tags : hctx->tags; 3455 struct rq_iter_data data = { 3456 .hctx = hctx, 3457 }; 3458 3459 blk_mq_all_tag_iter(tags, blk_mq_has_request, &data); 3460 return data.has_rq; 3461 } 3462 3463 static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu, 3464 struct blk_mq_hw_ctx *hctx) 3465 { 3466 if (cpumask_first_and(hctx->cpumask, cpu_online_mask) != cpu) 3467 return false; 3468 if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids) 3469 return false; 3470 return true; 3471 } 3472 3473 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node) 3474 { 3475 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 3476 struct blk_mq_hw_ctx, cpuhp_online); 3477 3478 if (!cpumask_test_cpu(cpu, hctx->cpumask) || 3479 !blk_mq_last_cpu_in_hctx(cpu, hctx)) 3480 return 0; 3481 3482 /* 3483 * Prevent new request from being allocated on the current hctx. 3484 * 3485 * The smp_mb__after_atomic() Pairs with the implied barrier in 3486 * test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is 3487 * seen once we return from the tag allocator. 3488 */ 3489 set_bit(BLK_MQ_S_INACTIVE, &hctx->state); 3490 smp_mb__after_atomic(); 3491 3492 /* 3493 * Try to grab a reference to the queue and wait for any outstanding 3494 * requests. If we could not grab a reference the queue has been 3495 * frozen and there are no requests. 3496 */ 3497 if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) { 3498 while (blk_mq_hctx_has_requests(hctx)) 3499 msleep(5); 3500 percpu_ref_put(&hctx->queue->q_usage_counter); 3501 } 3502 3503 return 0; 3504 } 3505 3506 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node) 3507 { 3508 struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node, 3509 struct blk_mq_hw_ctx, cpuhp_online); 3510 3511 if (cpumask_test_cpu(cpu, hctx->cpumask)) 3512 clear_bit(BLK_MQ_S_INACTIVE, &hctx->state); 3513 return 0; 3514 } 3515 3516 /* 3517 * 'cpu' is going away. splice any existing rq_list entries from this 3518 * software queue to the hw queue dispatch list, and ensure that it 3519 * gets run. 3520 */ 3521 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) 3522 { 3523 struct blk_mq_hw_ctx *hctx; 3524 struct blk_mq_ctx *ctx; 3525 LIST_HEAD(tmp); 3526 enum hctx_type type; 3527 3528 hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); 3529 if (!cpumask_test_cpu(cpu, hctx->cpumask)) 3530 return 0; 3531 3532 ctx = __blk_mq_get_ctx(hctx->queue, cpu); 3533 type = hctx->type; 3534 3535 spin_lock(&ctx->lock); 3536 if (!list_empty(&ctx->rq_lists[type])) { 3537 list_splice_init(&ctx->rq_lists[type], &tmp); 3538 blk_mq_hctx_clear_pending(hctx, ctx); 3539 } 3540 spin_unlock(&ctx->lock); 3541 3542 if (list_empty(&tmp)) 3543 return 0; 3544 3545 spin_lock(&hctx->lock); 3546 list_splice_tail_init(&tmp, &hctx->dispatch); 3547 spin_unlock(&hctx->lock); 3548 3549 blk_mq_run_hw_queue(hctx, true); 3550 return 0; 3551 } 3552 3553 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) 3554 { 3555 if (!(hctx->flags & BLK_MQ_F_STACKING)) 3556 cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 3557 &hctx->cpuhp_online); 3558 cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, 3559 &hctx->cpuhp_dead); 3560 } 3561 3562 /* 3563 * Before freeing hw queue, clearing the flush request reference in 3564 * tags->rqs[] for avoiding potential UAF. 3565 */ 3566 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags, 3567 unsigned int queue_depth, struct request *flush_rq) 3568 { 3569 int i; 3570 unsigned long flags; 3571 3572 /* The hw queue may not be mapped yet */ 3573 if (!tags) 3574 return; 3575 3576 WARN_ON_ONCE(req_ref_read(flush_rq) != 0); 3577 3578 for (i = 0; i < queue_depth; i++) 3579 cmpxchg(&tags->rqs[i], flush_rq, NULL); 3580 3581 /* 3582 * Wait until all pending iteration is done. 3583 * 3584 * Request reference is cleared and it is guaranteed to be observed 3585 * after the ->lock is released. 3586 */ 3587 spin_lock_irqsave(&tags->lock, flags); 3588 spin_unlock_irqrestore(&tags->lock, flags); 3589 } 3590 3591 /* hctx->ctxs will be freed in queue's release handler */ 3592 static void blk_mq_exit_hctx(struct request_queue *q, 3593 struct blk_mq_tag_set *set, 3594 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 3595 { 3596 struct request *flush_rq = hctx->fq->flush_rq; 3597 3598 if (blk_mq_hw_queue_mapped(hctx)) 3599 blk_mq_tag_idle(hctx); 3600 3601 if (blk_queue_init_done(q)) 3602 blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx], 3603 set->queue_depth, flush_rq); 3604 if (set->ops->exit_request) 3605 set->ops->exit_request(set, flush_rq, hctx_idx); 3606 3607 if (set->ops->exit_hctx) 3608 set->ops->exit_hctx(hctx, hctx_idx); 3609 3610 blk_mq_remove_cpuhp(hctx); 3611 3612 xa_erase(&q->hctx_table, hctx_idx); 3613 3614 spin_lock(&q->unused_hctx_lock); 3615 list_add(&hctx->hctx_list, &q->unused_hctx_list); 3616 spin_unlock(&q->unused_hctx_lock); 3617 } 3618 3619 static void blk_mq_exit_hw_queues(struct request_queue *q, 3620 struct blk_mq_tag_set *set, int nr_queue) 3621 { 3622 struct blk_mq_hw_ctx *hctx; 3623 unsigned long i; 3624 3625 queue_for_each_hw_ctx(q, hctx, i) { 3626 if (i == nr_queue) 3627 break; 3628 blk_mq_exit_hctx(q, set, hctx, i); 3629 } 3630 } 3631 3632 static int blk_mq_init_hctx(struct request_queue *q, 3633 struct blk_mq_tag_set *set, 3634 struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) 3635 { 3636 hctx->queue_num = hctx_idx; 3637 3638 if (!(hctx->flags & BLK_MQ_F_STACKING)) 3639 cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE, 3640 &hctx->cpuhp_online); 3641 cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); 3642 3643 hctx->tags = set->tags[hctx_idx]; 3644 3645 if (set->ops->init_hctx && 3646 set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) 3647 goto unregister_cpu_notifier; 3648 3649 if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, 3650 hctx->numa_node)) 3651 goto exit_hctx; 3652 3653 if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL)) 3654 goto exit_flush_rq; 3655 3656 return 0; 3657 3658 exit_flush_rq: 3659 if (set->ops->exit_request) 3660 set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); 3661 exit_hctx: 3662 if (set->ops->exit_hctx) 3663 set->ops->exit_hctx(hctx, hctx_idx); 3664 unregister_cpu_notifier: 3665 blk_mq_remove_cpuhp(hctx); 3666 return -1; 3667 } 3668 3669 static struct blk_mq_hw_ctx * 3670 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, 3671 int node) 3672 { 3673 struct blk_mq_hw_ctx *hctx; 3674 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 3675 3676 hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node); 3677 if (!hctx) 3678 goto fail_alloc_hctx; 3679 3680 if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node)) 3681 goto free_hctx; 3682 3683 atomic_set(&hctx->nr_active, 0); 3684 if (node == NUMA_NO_NODE) 3685 node = set->numa_node; 3686 hctx->numa_node = node; 3687 3688 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); 3689 spin_lock_init(&hctx->lock); 3690 INIT_LIST_HEAD(&hctx->dispatch); 3691 hctx->queue = q; 3692 hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED; 3693 3694 INIT_LIST_HEAD(&hctx->hctx_list); 3695 3696 /* 3697 * Allocate space for all possible cpus to avoid allocation at 3698 * runtime 3699 */ 3700 hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *), 3701 gfp, node); 3702 if (!hctx->ctxs) 3703 goto free_cpumask; 3704 3705 if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), 3706 gfp, node, false, false)) 3707 goto free_ctxs; 3708 hctx->nr_ctx = 0; 3709 3710 spin_lock_init(&hctx->dispatch_wait_lock); 3711 init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); 3712 INIT_LIST_HEAD(&hctx->dispatch_wait.entry); 3713 3714 hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp); 3715 if (!hctx->fq) 3716 goto free_bitmap; 3717 3718 blk_mq_hctx_kobj_init(hctx); 3719 3720 return hctx; 3721 3722 free_bitmap: 3723 sbitmap_free(&hctx->ctx_map); 3724 free_ctxs: 3725 kfree(hctx->ctxs); 3726 free_cpumask: 3727 free_cpumask_var(hctx->cpumask); 3728 free_hctx: 3729 kfree(hctx); 3730 fail_alloc_hctx: 3731 return NULL; 3732 } 3733 3734 static void blk_mq_init_cpu_queues(struct request_queue *q, 3735 unsigned int nr_hw_queues) 3736 { 3737 struct blk_mq_tag_set *set = q->tag_set; 3738 unsigned int i, j; 3739 3740 for_each_possible_cpu(i) { 3741 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); 3742 struct blk_mq_hw_ctx *hctx; 3743 int k; 3744 3745 __ctx->cpu = i; 3746 spin_lock_init(&__ctx->lock); 3747 for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++) 3748 INIT_LIST_HEAD(&__ctx->rq_lists[k]); 3749 3750 __ctx->queue = q; 3751 3752 /* 3753 * Set local node, IFF we have more than one hw queue. If 3754 * not, we remain on the home node of the device 3755 */ 3756 for (j = 0; j < set->nr_maps; j++) { 3757 hctx = blk_mq_map_queue_type(q, j, i); 3758 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) 3759 hctx->numa_node = cpu_to_node(i); 3760 } 3761 } 3762 } 3763 3764 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, 3765 unsigned int hctx_idx, 3766 unsigned int depth) 3767 { 3768 struct blk_mq_tags *tags; 3769 int ret; 3770 3771 tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags); 3772 if (!tags) 3773 return NULL; 3774 3775 ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth); 3776 if (ret) { 3777 blk_mq_free_rq_map(tags); 3778 return NULL; 3779 } 3780 3781 return tags; 3782 } 3783 3784 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set, 3785 int hctx_idx) 3786 { 3787 if (blk_mq_is_shared_tags(set->flags)) { 3788 set->tags[hctx_idx] = set->shared_tags; 3789 3790 return true; 3791 } 3792 3793 set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx, 3794 set->queue_depth); 3795 3796 return set->tags[hctx_idx]; 3797 } 3798 3799 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, 3800 struct blk_mq_tags *tags, 3801 unsigned int hctx_idx) 3802 { 3803 if (tags) { 3804 blk_mq_free_rqs(set, tags, hctx_idx); 3805 blk_mq_free_rq_map(tags); 3806 } 3807 } 3808 3809 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set, 3810 unsigned int hctx_idx) 3811 { 3812 if (!blk_mq_is_shared_tags(set->flags)) 3813 blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx); 3814 3815 set->tags[hctx_idx] = NULL; 3816 } 3817 3818 static void blk_mq_map_swqueue(struct request_queue *q) 3819 { 3820 unsigned int j, hctx_idx; 3821 unsigned long i; 3822 struct blk_mq_hw_ctx *hctx; 3823 struct blk_mq_ctx *ctx; 3824 struct blk_mq_tag_set *set = q->tag_set; 3825 3826 queue_for_each_hw_ctx(q, hctx, i) { 3827 cpumask_clear(hctx->cpumask); 3828 hctx->nr_ctx = 0; 3829 hctx->dispatch_from = NULL; 3830 } 3831 3832 /* 3833 * Map software to hardware queues. 3834 * 3835 * If the cpu isn't present, the cpu is mapped to first hctx. 3836 */ 3837 for_each_possible_cpu(i) { 3838 3839 ctx = per_cpu_ptr(q->queue_ctx, i); 3840 for (j = 0; j < set->nr_maps; j++) { 3841 if (!set->map[j].nr_queues) { 3842 ctx->hctxs[j] = blk_mq_map_queue_type(q, 3843 HCTX_TYPE_DEFAULT, i); 3844 continue; 3845 } 3846 hctx_idx = set->map[j].mq_map[i]; 3847 /* unmapped hw queue can be remapped after CPU topo changed */ 3848 if (!set->tags[hctx_idx] && 3849 !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) { 3850 /* 3851 * If tags initialization fail for some hctx, 3852 * that hctx won't be brought online. In this 3853 * case, remap the current ctx to hctx[0] which 3854 * is guaranteed to always have tags allocated 3855 */ 3856 set->map[j].mq_map[i] = 0; 3857 } 3858 3859 hctx = blk_mq_map_queue_type(q, j, i); 3860 ctx->hctxs[j] = hctx; 3861 /* 3862 * If the CPU is already set in the mask, then we've 3863 * mapped this one already. This can happen if 3864 * devices share queues across queue maps. 3865 */ 3866 if (cpumask_test_cpu(i, hctx->cpumask)) 3867 continue; 3868 3869 cpumask_set_cpu(i, hctx->cpumask); 3870 hctx->type = j; 3871 ctx->index_hw[hctx->type] = hctx->nr_ctx; 3872 hctx->ctxs[hctx->nr_ctx++] = ctx; 3873 3874 /* 3875 * If the nr_ctx type overflows, we have exceeded the 3876 * amount of sw queues we can support. 3877 */ 3878 BUG_ON(!hctx->nr_ctx); 3879 } 3880 3881 for (; j < HCTX_MAX_TYPES; j++) 3882 ctx->hctxs[j] = blk_mq_map_queue_type(q, 3883 HCTX_TYPE_DEFAULT, i); 3884 } 3885 3886 queue_for_each_hw_ctx(q, hctx, i) { 3887 /* 3888 * If no software queues are mapped to this hardware queue, 3889 * disable it and free the request entries. 3890 */ 3891 if (!hctx->nr_ctx) { 3892 /* Never unmap queue 0. We need it as a 3893 * fallback in case of a new remap fails 3894 * allocation 3895 */ 3896 if (i) 3897 __blk_mq_free_map_and_rqs(set, i); 3898 3899 hctx->tags = NULL; 3900 continue; 3901 } 3902 3903 hctx->tags = set->tags[i]; 3904 WARN_ON(!hctx->tags); 3905 3906 /* 3907 * Set the map size to the number of mapped software queues. 3908 * This is more accurate and more efficient than looping 3909 * over all possibly mapped software queues. 3910 */ 3911 sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); 3912 3913 /* 3914 * Initialize batch roundrobin counts 3915 */ 3916 hctx->next_cpu = blk_mq_first_mapped_cpu(hctx); 3917 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; 3918 } 3919 } 3920 3921 /* 3922 * Caller needs to ensure that we're either frozen/quiesced, or that 3923 * the queue isn't live yet. 3924 */ 3925 static void queue_set_hctx_shared(struct request_queue *q, bool shared) 3926 { 3927 struct blk_mq_hw_ctx *hctx; 3928 unsigned long i; 3929 3930 queue_for_each_hw_ctx(q, hctx, i) { 3931 if (shared) { 3932 hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED; 3933 } else { 3934 blk_mq_tag_idle(hctx); 3935 hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; 3936 } 3937 } 3938 } 3939 3940 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set, 3941 bool shared) 3942 { 3943 struct request_queue *q; 3944 3945 lockdep_assert_held(&set->tag_list_lock); 3946 3947 list_for_each_entry(q, &set->tag_list, tag_set_list) { 3948 blk_mq_freeze_queue(q); 3949 queue_set_hctx_shared(q, shared); 3950 blk_mq_unfreeze_queue(q); 3951 } 3952 } 3953 3954 static void blk_mq_del_queue_tag_set(struct request_queue *q) 3955 { 3956 struct blk_mq_tag_set *set = q->tag_set; 3957 3958 mutex_lock(&set->tag_list_lock); 3959 list_del(&q->tag_set_list); 3960 if (list_is_singular(&set->tag_list)) { 3961 /* just transitioned to unshared */ 3962 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED; 3963 /* update existing queue */ 3964 blk_mq_update_tag_set_shared(set, false); 3965 } 3966 mutex_unlock(&set->tag_list_lock); 3967 INIT_LIST_HEAD(&q->tag_set_list); 3968 } 3969 3970 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, 3971 struct request_queue *q) 3972 { 3973 mutex_lock(&set->tag_list_lock); 3974 3975 /* 3976 * Check to see if we're transitioning to shared (from 1 to 2 queues). 3977 */ 3978 if (!list_empty(&set->tag_list) && 3979 !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) { 3980 set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED; 3981 /* update existing queue */ 3982 blk_mq_update_tag_set_shared(set, true); 3983 } 3984 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED) 3985 queue_set_hctx_shared(q, true); 3986 list_add_tail(&q->tag_set_list, &set->tag_list); 3987 3988 mutex_unlock(&set->tag_list_lock); 3989 } 3990 3991 /* All allocations will be freed in release handler of q->mq_kobj */ 3992 static int blk_mq_alloc_ctxs(struct request_queue *q) 3993 { 3994 struct blk_mq_ctxs *ctxs; 3995 int cpu; 3996 3997 ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL); 3998 if (!ctxs) 3999 return -ENOMEM; 4000 4001 ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx); 4002 if (!ctxs->queue_ctx) 4003 goto fail; 4004 4005 for_each_possible_cpu(cpu) { 4006 struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu); 4007 ctx->ctxs = ctxs; 4008 } 4009 4010 q->mq_kobj = &ctxs->kobj; 4011 q->queue_ctx = ctxs->queue_ctx; 4012 4013 return 0; 4014 fail: 4015 kfree(ctxs); 4016 return -ENOMEM; 4017 } 4018 4019 /* 4020 * It is the actual release handler for mq, but we do it from 4021 * request queue's release handler for avoiding use-after-free 4022 * and headache because q->mq_kobj shouldn't have been introduced, 4023 * but we can't group ctx/kctx kobj without it. 4024 */ 4025 void blk_mq_release(struct request_queue *q) 4026 { 4027 struct blk_mq_hw_ctx *hctx, *next; 4028 unsigned long i; 4029 4030 queue_for_each_hw_ctx(q, hctx, i) 4031 WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list)); 4032 4033 /* all hctx are in .unused_hctx_list now */ 4034 list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) { 4035 list_del_init(&hctx->hctx_list); 4036 kobject_put(&hctx->kobj); 4037 } 4038 4039 xa_destroy(&q->hctx_table); 4040 4041 /* 4042 * release .mq_kobj and sw queue's kobject now because 4043 * both share lifetime with request queue. 4044 */ 4045 blk_mq_sysfs_deinit(q); 4046 } 4047 4048 static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set, 4049 void *queuedata) 4050 { 4051 struct request_queue *q; 4052 int ret; 4053 4054 q = blk_alloc_queue(set->numa_node); 4055 if (!q) 4056 return ERR_PTR(-ENOMEM); 4057 q->queuedata = queuedata; 4058 ret = blk_mq_init_allocated_queue(set, q); 4059 if (ret) { 4060 blk_put_queue(q); 4061 return ERR_PTR(ret); 4062 } 4063 return q; 4064 } 4065 4066 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) 4067 { 4068 return blk_mq_init_queue_data(set, NULL); 4069 } 4070 EXPORT_SYMBOL(blk_mq_init_queue); 4071 4072 /** 4073 * blk_mq_destroy_queue - shutdown a request queue 4074 * @q: request queue to shutdown 4075 * 4076 * This shuts down a request queue allocated by blk_mq_init_queue(). All future 4077 * requests will be failed with -ENODEV. The caller is responsible for dropping 4078 * the reference from blk_mq_init_queue() by calling blk_put_queue(). 4079 * 4080 * Context: can sleep 4081 */ 4082 void blk_mq_destroy_queue(struct request_queue *q) 4083 { 4084 WARN_ON_ONCE(!queue_is_mq(q)); 4085 WARN_ON_ONCE(blk_queue_registered(q)); 4086 4087 might_sleep(); 4088 4089 blk_queue_flag_set(QUEUE_FLAG_DYING, q); 4090 blk_queue_start_drain(q); 4091 blk_mq_freeze_queue_wait(q); 4092 4093 blk_sync_queue(q); 4094 blk_mq_cancel_work_sync(q); 4095 blk_mq_exit_queue(q); 4096 } 4097 EXPORT_SYMBOL(blk_mq_destroy_queue); 4098 4099 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata, 4100 struct lock_class_key *lkclass) 4101 { 4102 struct request_queue *q; 4103 struct gendisk *disk; 4104 4105 q = blk_mq_init_queue_data(set, queuedata); 4106 if (IS_ERR(q)) 4107 return ERR_CAST(q); 4108 4109 disk = __alloc_disk_node(q, set->numa_node, lkclass); 4110 if (!disk) { 4111 blk_mq_destroy_queue(q); 4112 blk_put_queue(q); 4113 return ERR_PTR(-ENOMEM); 4114 } 4115 set_bit(GD_OWNS_QUEUE, &disk->state); 4116 return disk; 4117 } 4118 EXPORT_SYMBOL(__blk_mq_alloc_disk); 4119 4120 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q, 4121 struct lock_class_key *lkclass) 4122 { 4123 struct gendisk *disk; 4124 4125 if (!blk_get_queue(q)) 4126 return NULL; 4127 disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass); 4128 if (!disk) 4129 blk_put_queue(q); 4130 return disk; 4131 } 4132 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue); 4133 4134 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx( 4135 struct blk_mq_tag_set *set, struct request_queue *q, 4136 int hctx_idx, int node) 4137 { 4138 struct blk_mq_hw_ctx *hctx = NULL, *tmp; 4139 4140 /* reuse dead hctx first */ 4141 spin_lock(&q->unused_hctx_lock); 4142 list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) { 4143 if (tmp->numa_node == node) { 4144 hctx = tmp; 4145 break; 4146 } 4147 } 4148 if (hctx) 4149 list_del_init(&hctx->hctx_list); 4150 spin_unlock(&q->unused_hctx_lock); 4151 4152 if (!hctx) 4153 hctx = blk_mq_alloc_hctx(q, set, node); 4154 if (!hctx) 4155 goto fail; 4156 4157 if (blk_mq_init_hctx(q, set, hctx, hctx_idx)) 4158 goto free_hctx; 4159 4160 return hctx; 4161 4162 free_hctx: 4163 kobject_put(&hctx->kobj); 4164 fail: 4165 return NULL; 4166 } 4167 4168 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, 4169 struct request_queue *q) 4170 { 4171 struct blk_mq_hw_ctx *hctx; 4172 unsigned long i, j; 4173 4174 /* protect against switching io scheduler */ 4175 mutex_lock(&q->sysfs_lock); 4176 for (i = 0; i < set->nr_hw_queues; i++) { 4177 int old_node; 4178 int node = blk_mq_get_hctx_node(set, i); 4179 struct blk_mq_hw_ctx *old_hctx = xa_load(&q->hctx_table, i); 4180 4181 if (old_hctx) { 4182 old_node = old_hctx->numa_node; 4183 blk_mq_exit_hctx(q, set, old_hctx, i); 4184 } 4185 4186 if (!blk_mq_alloc_and_init_hctx(set, q, i, node)) { 4187 if (!old_hctx) 4188 break; 4189 pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n", 4190 node, old_node); 4191 hctx = blk_mq_alloc_and_init_hctx(set, q, i, old_node); 4192 WARN_ON_ONCE(!hctx); 4193 } 4194 } 4195 /* 4196 * Increasing nr_hw_queues fails. Free the newly allocated 4197 * hctxs and keep the previous q->nr_hw_queues. 4198 */ 4199 if (i != set->nr_hw_queues) { 4200 j = q->nr_hw_queues; 4201 } else { 4202 j = i; 4203 q->nr_hw_queues = set->nr_hw_queues; 4204 } 4205 4206 xa_for_each_start(&q->hctx_table, j, hctx, j) 4207 blk_mq_exit_hctx(q, set, hctx, j); 4208 mutex_unlock(&q->sysfs_lock); 4209 } 4210 4211 static void blk_mq_update_poll_flag(struct request_queue *q) 4212 { 4213 struct blk_mq_tag_set *set = q->tag_set; 4214 4215 if (set->nr_maps > HCTX_TYPE_POLL && 4216 set->map[HCTX_TYPE_POLL].nr_queues) 4217 blk_queue_flag_set(QUEUE_FLAG_POLL, q); 4218 else 4219 blk_queue_flag_clear(QUEUE_FLAG_POLL, q); 4220 } 4221 4222 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 4223 struct request_queue *q) 4224 { 4225 /* mark the queue as mq asap */ 4226 q->mq_ops = set->ops; 4227 4228 if (blk_mq_alloc_ctxs(q)) 4229 goto err_exit; 4230 4231 /* init q->mq_kobj and sw queues' kobjects */ 4232 blk_mq_sysfs_init(q); 4233 4234 INIT_LIST_HEAD(&q->unused_hctx_list); 4235 spin_lock_init(&q->unused_hctx_lock); 4236 4237 xa_init(&q->hctx_table); 4238 4239 blk_mq_realloc_hw_ctxs(set, q); 4240 if (!q->nr_hw_queues) 4241 goto err_hctxs; 4242 4243 INIT_WORK(&q->timeout_work, blk_mq_timeout_work); 4244 blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); 4245 4246 q->tag_set = set; 4247 4248 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; 4249 blk_mq_update_poll_flag(q); 4250 4251 INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); 4252 INIT_LIST_HEAD(&q->flush_list); 4253 INIT_LIST_HEAD(&q->requeue_list); 4254 spin_lock_init(&q->requeue_lock); 4255 4256 q->nr_requests = set->queue_depth; 4257 4258 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 4259 blk_mq_add_queue_tag_set(set, q); 4260 blk_mq_map_swqueue(q); 4261 return 0; 4262 4263 err_hctxs: 4264 blk_mq_release(q); 4265 err_exit: 4266 q->mq_ops = NULL; 4267 return -ENOMEM; 4268 } 4269 EXPORT_SYMBOL(blk_mq_init_allocated_queue); 4270 4271 /* tags can _not_ be used after returning from blk_mq_exit_queue */ 4272 void blk_mq_exit_queue(struct request_queue *q) 4273 { 4274 struct blk_mq_tag_set *set = q->tag_set; 4275 4276 /* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */ 4277 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 4278 /* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */ 4279 blk_mq_del_queue_tag_set(q); 4280 } 4281 4282 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 4283 { 4284 int i; 4285 4286 if (blk_mq_is_shared_tags(set->flags)) { 4287 set->shared_tags = blk_mq_alloc_map_and_rqs(set, 4288 BLK_MQ_NO_HCTX_IDX, 4289 set->queue_depth); 4290 if (!set->shared_tags) 4291 return -ENOMEM; 4292 } 4293 4294 for (i = 0; i < set->nr_hw_queues; i++) { 4295 if (!__blk_mq_alloc_map_and_rqs(set, i)) 4296 goto out_unwind; 4297 cond_resched(); 4298 } 4299 4300 return 0; 4301 4302 out_unwind: 4303 while (--i >= 0) 4304 __blk_mq_free_map_and_rqs(set, i); 4305 4306 if (blk_mq_is_shared_tags(set->flags)) { 4307 blk_mq_free_map_and_rqs(set, set->shared_tags, 4308 BLK_MQ_NO_HCTX_IDX); 4309 } 4310 4311 return -ENOMEM; 4312 } 4313 4314 /* 4315 * Allocate the request maps associated with this tag_set. Note that this 4316 * may reduce the depth asked for, if memory is tight. set->queue_depth 4317 * will be updated to reflect the allocated depth. 4318 */ 4319 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set) 4320 { 4321 unsigned int depth; 4322 int err; 4323 4324 depth = set->queue_depth; 4325 do { 4326 err = __blk_mq_alloc_rq_maps(set); 4327 if (!err) 4328 break; 4329 4330 set->queue_depth >>= 1; 4331 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { 4332 err = -ENOMEM; 4333 break; 4334 } 4335 } while (set->queue_depth); 4336 4337 if (!set->queue_depth || err) { 4338 pr_err("blk-mq: failed to allocate request map\n"); 4339 return -ENOMEM; 4340 } 4341 4342 if (depth != set->queue_depth) 4343 pr_info("blk-mq: reduced tag depth (%u -> %u)\n", 4344 depth, set->queue_depth); 4345 4346 return 0; 4347 } 4348 4349 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) 4350 { 4351 /* 4352 * blk_mq_map_queues() and multiple .map_queues() implementations 4353 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the 4354 * number of hardware queues. 4355 */ 4356 if (set->nr_maps == 1) 4357 set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues; 4358 4359 if (set->ops->map_queues && !is_kdump_kernel()) { 4360 int i; 4361 4362 /* 4363 * transport .map_queues is usually done in the following 4364 * way: 4365 * 4366 * for (queue = 0; queue < set->nr_hw_queues; queue++) { 4367 * mask = get_cpu_mask(queue) 4368 * for_each_cpu(cpu, mask) 4369 * set->map[x].mq_map[cpu] = queue; 4370 * } 4371 * 4372 * When we need to remap, the table has to be cleared for 4373 * killing stale mapping since one CPU may not be mapped 4374 * to any hw queue. 4375 */ 4376 for (i = 0; i < set->nr_maps; i++) 4377 blk_mq_clear_mq_map(&set->map[i]); 4378 4379 set->ops->map_queues(set); 4380 } else { 4381 BUG_ON(set->nr_maps > 1); 4382 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 4383 } 4384 } 4385 4386 static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set, 4387 int new_nr_hw_queues) 4388 { 4389 struct blk_mq_tags **new_tags; 4390 int i; 4391 4392 if (set->nr_hw_queues >= new_nr_hw_queues) 4393 goto done; 4394 4395 new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *), 4396 GFP_KERNEL, set->numa_node); 4397 if (!new_tags) 4398 return -ENOMEM; 4399 4400 if (set->tags) 4401 memcpy(new_tags, set->tags, set->nr_hw_queues * 4402 sizeof(*set->tags)); 4403 kfree(set->tags); 4404 set->tags = new_tags; 4405 4406 for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) { 4407 if (!__blk_mq_alloc_map_and_rqs(set, i)) { 4408 while (--i >= set->nr_hw_queues) 4409 __blk_mq_free_map_and_rqs(set, i); 4410 return -ENOMEM; 4411 } 4412 cond_resched(); 4413 } 4414 4415 done: 4416 set->nr_hw_queues = new_nr_hw_queues; 4417 return 0; 4418 } 4419 4420 /* 4421 * Alloc a tag set to be associated with one or more request queues. 4422 * May fail with EINVAL for various error conditions. May adjust the 4423 * requested depth down, if it's too large. In that case, the set 4424 * value will be stored in set->queue_depth. 4425 */ 4426 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) 4427 { 4428 int i, ret; 4429 4430 BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); 4431 4432 if (!set->nr_hw_queues) 4433 return -EINVAL; 4434 if (!set->queue_depth) 4435 return -EINVAL; 4436 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) 4437 return -EINVAL; 4438 4439 if (!set->ops->queue_rq) 4440 return -EINVAL; 4441 4442 if (!set->ops->get_budget ^ !set->ops->put_budget) 4443 return -EINVAL; 4444 4445 if (set->queue_depth > BLK_MQ_MAX_DEPTH) { 4446 pr_info("blk-mq: reduced tag depth to %u\n", 4447 BLK_MQ_MAX_DEPTH); 4448 set->queue_depth = BLK_MQ_MAX_DEPTH; 4449 } 4450 4451 if (!set->nr_maps) 4452 set->nr_maps = 1; 4453 else if (set->nr_maps > HCTX_MAX_TYPES) 4454 return -EINVAL; 4455 4456 /* 4457 * If a crashdump is active, then we are potentially in a very 4458 * memory constrained environment. Limit us to 1 queue and 4459 * 64 tags to prevent using too much memory. 4460 */ 4461 if (is_kdump_kernel()) { 4462 set->nr_hw_queues = 1; 4463 set->nr_maps = 1; 4464 set->queue_depth = min(64U, set->queue_depth); 4465 } 4466 /* 4467 * There is no use for more h/w queues than cpus if we just have 4468 * a single map 4469 */ 4470 if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids) 4471 set->nr_hw_queues = nr_cpu_ids; 4472 4473 if (set->flags & BLK_MQ_F_BLOCKING) { 4474 set->srcu = kmalloc(sizeof(*set->srcu), GFP_KERNEL); 4475 if (!set->srcu) 4476 return -ENOMEM; 4477 ret = init_srcu_struct(set->srcu); 4478 if (ret) 4479 goto out_free_srcu; 4480 } 4481 4482 ret = -ENOMEM; 4483 set->tags = kcalloc_node(set->nr_hw_queues, 4484 sizeof(struct blk_mq_tags *), GFP_KERNEL, 4485 set->numa_node); 4486 if (!set->tags) 4487 goto out_cleanup_srcu; 4488 4489 for (i = 0; i < set->nr_maps; i++) { 4490 set->map[i].mq_map = kcalloc_node(nr_cpu_ids, 4491 sizeof(set->map[i].mq_map[0]), 4492 GFP_KERNEL, set->numa_node); 4493 if (!set->map[i].mq_map) 4494 goto out_free_mq_map; 4495 set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues; 4496 } 4497 4498 blk_mq_update_queue_map(set); 4499 4500 ret = blk_mq_alloc_set_map_and_rqs(set); 4501 if (ret) 4502 goto out_free_mq_map; 4503 4504 mutex_init(&set->tag_list_lock); 4505 INIT_LIST_HEAD(&set->tag_list); 4506 4507 return 0; 4508 4509 out_free_mq_map: 4510 for (i = 0; i < set->nr_maps; i++) { 4511 kfree(set->map[i].mq_map); 4512 set->map[i].mq_map = NULL; 4513 } 4514 kfree(set->tags); 4515 set->tags = NULL; 4516 out_cleanup_srcu: 4517 if (set->flags & BLK_MQ_F_BLOCKING) 4518 cleanup_srcu_struct(set->srcu); 4519 out_free_srcu: 4520 if (set->flags & BLK_MQ_F_BLOCKING) 4521 kfree(set->srcu); 4522 return ret; 4523 } 4524 EXPORT_SYMBOL(blk_mq_alloc_tag_set); 4525 4526 /* allocate and initialize a tagset for a simple single-queue device */ 4527 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set, 4528 const struct blk_mq_ops *ops, unsigned int queue_depth, 4529 unsigned int set_flags) 4530 { 4531 memset(set, 0, sizeof(*set)); 4532 set->ops = ops; 4533 set->nr_hw_queues = 1; 4534 set->nr_maps = 1; 4535 set->queue_depth = queue_depth; 4536 set->numa_node = NUMA_NO_NODE; 4537 set->flags = set_flags; 4538 return blk_mq_alloc_tag_set(set); 4539 } 4540 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set); 4541 4542 void blk_mq_free_tag_set(struct blk_mq_tag_set *set) 4543 { 4544 int i, j; 4545 4546 for (i = 0; i < set->nr_hw_queues; i++) 4547 __blk_mq_free_map_and_rqs(set, i); 4548 4549 if (blk_mq_is_shared_tags(set->flags)) { 4550 blk_mq_free_map_and_rqs(set, set->shared_tags, 4551 BLK_MQ_NO_HCTX_IDX); 4552 } 4553 4554 for (j = 0; j < set->nr_maps; j++) { 4555 kfree(set->map[j].mq_map); 4556 set->map[j].mq_map = NULL; 4557 } 4558 4559 kfree(set->tags); 4560 set->tags = NULL; 4561 if (set->flags & BLK_MQ_F_BLOCKING) { 4562 cleanup_srcu_struct(set->srcu); 4563 kfree(set->srcu); 4564 } 4565 } 4566 EXPORT_SYMBOL(blk_mq_free_tag_set); 4567 4568 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) 4569 { 4570 struct blk_mq_tag_set *set = q->tag_set; 4571 struct blk_mq_hw_ctx *hctx; 4572 int ret; 4573 unsigned long i; 4574 4575 if (!set) 4576 return -EINVAL; 4577 4578 if (q->nr_requests == nr) 4579 return 0; 4580 4581 blk_mq_freeze_queue(q); 4582 blk_mq_quiesce_queue(q); 4583 4584 ret = 0; 4585 queue_for_each_hw_ctx(q, hctx, i) { 4586 if (!hctx->tags) 4587 continue; 4588 /* 4589 * If we're using an MQ scheduler, just update the scheduler 4590 * queue depth. This is similar to what the old code would do. 4591 */ 4592 if (hctx->sched_tags) { 4593 ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, 4594 nr, true); 4595 } else { 4596 ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr, 4597 false); 4598 } 4599 if (ret) 4600 break; 4601 if (q->elevator && q->elevator->type->ops.depth_updated) 4602 q->elevator->type->ops.depth_updated(hctx); 4603 } 4604 if (!ret) { 4605 q->nr_requests = nr; 4606 if (blk_mq_is_shared_tags(set->flags)) { 4607 if (q->elevator) 4608 blk_mq_tag_update_sched_shared_tags(q); 4609 else 4610 blk_mq_tag_resize_shared_tags(set, nr); 4611 } 4612 } 4613 4614 blk_mq_unquiesce_queue(q); 4615 blk_mq_unfreeze_queue(q); 4616 4617 return ret; 4618 } 4619 4620 /* 4621 * request_queue and elevator_type pair. 4622 * It is just used by __blk_mq_update_nr_hw_queues to cache 4623 * the elevator_type associated with a request_queue. 4624 */ 4625 struct blk_mq_qe_pair { 4626 struct list_head node; 4627 struct request_queue *q; 4628 struct elevator_type *type; 4629 }; 4630 4631 /* 4632 * Cache the elevator_type in qe pair list and switch the 4633 * io scheduler to 'none' 4634 */ 4635 static bool blk_mq_elv_switch_none(struct list_head *head, 4636 struct request_queue *q) 4637 { 4638 struct blk_mq_qe_pair *qe; 4639 4640 qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); 4641 if (!qe) 4642 return false; 4643 4644 /* q->elevator needs protection from ->sysfs_lock */ 4645 mutex_lock(&q->sysfs_lock); 4646 4647 /* the check has to be done with holding sysfs_lock */ 4648 if (!q->elevator) { 4649 kfree(qe); 4650 goto unlock; 4651 } 4652 4653 INIT_LIST_HEAD(&qe->node); 4654 qe->q = q; 4655 qe->type = q->elevator->type; 4656 /* keep a reference to the elevator module as we'll switch back */ 4657 __elevator_get(qe->type); 4658 list_add(&qe->node, head); 4659 elevator_disable(q); 4660 unlock: 4661 mutex_unlock(&q->sysfs_lock); 4662 4663 return true; 4664 } 4665 4666 static struct blk_mq_qe_pair *blk_lookup_qe_pair(struct list_head *head, 4667 struct request_queue *q) 4668 { 4669 struct blk_mq_qe_pair *qe; 4670 4671 list_for_each_entry(qe, head, node) 4672 if (qe->q == q) 4673 return qe; 4674 4675 return NULL; 4676 } 4677 4678 static void blk_mq_elv_switch_back(struct list_head *head, 4679 struct request_queue *q) 4680 { 4681 struct blk_mq_qe_pair *qe; 4682 struct elevator_type *t; 4683 4684 qe = blk_lookup_qe_pair(head, q); 4685 if (!qe) 4686 return; 4687 t = qe->type; 4688 list_del(&qe->node); 4689 kfree(qe); 4690 4691 mutex_lock(&q->sysfs_lock); 4692 elevator_switch(q, t); 4693 /* drop the reference acquired in blk_mq_elv_switch_none */ 4694 elevator_put(t); 4695 mutex_unlock(&q->sysfs_lock); 4696 } 4697 4698 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, 4699 int nr_hw_queues) 4700 { 4701 struct request_queue *q; 4702 LIST_HEAD(head); 4703 int prev_nr_hw_queues = set->nr_hw_queues; 4704 int i; 4705 4706 lockdep_assert_held(&set->tag_list_lock); 4707 4708 if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids) 4709 nr_hw_queues = nr_cpu_ids; 4710 if (nr_hw_queues < 1) 4711 return; 4712 if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues) 4713 return; 4714 4715 list_for_each_entry(q, &set->tag_list, tag_set_list) 4716 blk_mq_freeze_queue(q); 4717 /* 4718 * Switch IO scheduler to 'none', cleaning up the data associated 4719 * with the previous scheduler. We will switch back once we are done 4720 * updating the new sw to hw queue mappings. 4721 */ 4722 list_for_each_entry(q, &set->tag_list, tag_set_list) 4723 if (!blk_mq_elv_switch_none(&head, q)) 4724 goto switch_back; 4725 4726 list_for_each_entry(q, &set->tag_list, tag_set_list) { 4727 blk_mq_debugfs_unregister_hctxs(q); 4728 blk_mq_sysfs_unregister_hctxs(q); 4729 } 4730 4731 if (blk_mq_realloc_tag_set_tags(set, nr_hw_queues) < 0) 4732 goto reregister; 4733 4734 fallback: 4735 blk_mq_update_queue_map(set); 4736 list_for_each_entry(q, &set->tag_list, tag_set_list) { 4737 blk_mq_realloc_hw_ctxs(set, q); 4738 blk_mq_update_poll_flag(q); 4739 if (q->nr_hw_queues != set->nr_hw_queues) { 4740 int i = prev_nr_hw_queues; 4741 4742 pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n", 4743 nr_hw_queues, prev_nr_hw_queues); 4744 for (; i < set->nr_hw_queues; i++) 4745 __blk_mq_free_map_and_rqs(set, i); 4746 4747 set->nr_hw_queues = prev_nr_hw_queues; 4748 goto fallback; 4749 } 4750 blk_mq_map_swqueue(q); 4751 } 4752 4753 reregister: 4754 list_for_each_entry(q, &set->tag_list, tag_set_list) { 4755 blk_mq_sysfs_register_hctxs(q); 4756 blk_mq_debugfs_register_hctxs(q); 4757 } 4758 4759 switch_back: 4760 list_for_each_entry(q, &set->tag_list, tag_set_list) 4761 blk_mq_elv_switch_back(&head, q); 4762 4763 list_for_each_entry(q, &set->tag_list, tag_set_list) 4764 blk_mq_unfreeze_queue(q); 4765 4766 /* Free the excess tags when nr_hw_queues shrink. */ 4767 for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++) 4768 __blk_mq_free_map_and_rqs(set, i); 4769 } 4770 4771 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) 4772 { 4773 mutex_lock(&set->tag_list_lock); 4774 __blk_mq_update_nr_hw_queues(set, nr_hw_queues); 4775 mutex_unlock(&set->tag_list_lock); 4776 } 4777 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); 4778 4779 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx, 4780 struct io_comp_batch *iob, unsigned int flags) 4781 { 4782 long state = get_current_state(); 4783 int ret; 4784 4785 do { 4786 ret = q->mq_ops->poll(hctx, iob); 4787 if (ret > 0) { 4788 __set_current_state(TASK_RUNNING); 4789 return ret; 4790 } 4791 4792 if (signal_pending_state(state, current)) 4793 __set_current_state(TASK_RUNNING); 4794 if (task_is_running(current)) 4795 return 1; 4796 4797 if (ret < 0 || (flags & BLK_POLL_ONESHOT)) 4798 break; 4799 cpu_relax(); 4800 } while (!need_resched()); 4801 4802 __set_current_state(TASK_RUNNING); 4803 return 0; 4804 } 4805 4806 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie, 4807 struct io_comp_batch *iob, unsigned int flags) 4808 { 4809 struct blk_mq_hw_ctx *hctx = xa_load(&q->hctx_table, cookie); 4810 4811 return blk_hctx_poll(q, hctx, iob, flags); 4812 } 4813 4814 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob, 4815 unsigned int poll_flags) 4816 { 4817 struct request_queue *q = rq->q; 4818 int ret; 4819 4820 if (!blk_rq_is_poll(rq)) 4821 return 0; 4822 if (!percpu_ref_tryget(&q->q_usage_counter)) 4823 return 0; 4824 4825 ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags); 4826 blk_queue_exit(q); 4827 4828 return ret; 4829 } 4830 EXPORT_SYMBOL_GPL(blk_rq_poll); 4831 4832 unsigned int blk_mq_rq_cpu(struct request *rq) 4833 { 4834 return rq->mq_ctx->cpu; 4835 } 4836 EXPORT_SYMBOL(blk_mq_rq_cpu); 4837 4838 void blk_mq_cancel_work_sync(struct request_queue *q) 4839 { 4840 struct blk_mq_hw_ctx *hctx; 4841 unsigned long i; 4842 4843 cancel_delayed_work_sync(&q->requeue_work); 4844 4845 queue_for_each_hw_ctx(q, hctx, i) 4846 cancel_delayed_work_sync(&hctx->run_work); 4847 } 4848 4849 static int __init blk_mq_init(void) 4850 { 4851 int i; 4852 4853 for_each_possible_cpu(i) 4854 init_llist_head(&per_cpu(blk_cpu_done, i)); 4855 for_each_possible_cpu(i) 4856 INIT_CSD(&per_cpu(blk_cpu_csd, i), 4857 __blk_mq_complete_request_remote, NULL); 4858 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); 4859 4860 cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, 4861 "block/softirq:dead", NULL, 4862 blk_softirq_cpu_dead); 4863 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 4864 blk_mq_hctx_notify_dead); 4865 cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online", 4866 blk_mq_hctx_notify_online, 4867 blk_mq_hctx_notify_offline); 4868 return 0; 4869 } 4870 subsys_initcall(blk_mq_init); 4871