xref: /linux/block/blk-mq.c (revision 894e4e6084924cfd590968d45fad20908e02ddae)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Block multiqueue core code
4  *
5  * Copyright (C) 2013-2014 Jens Axboe
6  * Copyright (C) 2013-2014 Christoph Hellwig
7  */
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/backing-dev.h>
11 #include <linux/bio.h>
12 #include <linux/blkdev.h>
13 #include <linux/blk-integrity.h>
14 #include <linux/kmemleak.h>
15 #include <linux/mm.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/workqueue.h>
19 #include <linux/smp.h>
20 #include <linux/interrupt.h>
21 #include <linux/llist.h>
22 #include <linux/cpu.h>
23 #include <linux/cache.h>
24 #include <linux/sched/topology.h>
25 #include <linux/sched/signal.h>
26 #include <linux/suspend.h>
27 #include <linux/delay.h>
28 #include <linux/crash_dump.h>
29 #include <linux/prefetch.h>
30 #include <linux/blk-crypto.h>
31 #include <linux/part_stat.h>
32 #include <linux/sched/isolation.h>
33 
34 #include <trace/events/block.h>
35 
36 #include <linux/t10-pi.h>
37 #include "blk.h"
38 #include "blk-mq.h"
39 #include "blk-mq-debugfs.h"
40 #include "blk-pm.h"
41 #include "blk-stat.h"
42 #include "blk-mq-sched.h"
43 #include "blk-rq-qos.h"
44 
45 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
46 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
47 static DEFINE_MUTEX(blk_mq_cpuhp_lock);
48 
49 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
50 static void blk_mq_request_bypass_insert(struct request *rq,
51 		blk_insert_t flags);
52 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
53 		struct list_head *list);
54 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
55 			 struct io_comp_batch *iob, unsigned int flags);
56 
57 /*
58  * Check if any of the ctx, dispatch list or elevator
59  * have pending work in this hardware queue.
60  */
61 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
62 {
63 	return !list_empty_careful(&hctx->dispatch) ||
64 		sbitmap_any_bit_set(&hctx->ctx_map) ||
65 			blk_mq_sched_has_work(hctx);
66 }
67 
68 /*
69  * Mark this ctx as having pending work in this hardware queue
70  */
71 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
72 				     struct blk_mq_ctx *ctx)
73 {
74 	const int bit = ctx->index_hw[hctx->type];
75 
76 	if (!sbitmap_test_bit(&hctx->ctx_map, bit))
77 		sbitmap_set_bit(&hctx->ctx_map, bit);
78 }
79 
80 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
81 				      struct blk_mq_ctx *ctx)
82 {
83 	const int bit = ctx->index_hw[hctx->type];
84 
85 	sbitmap_clear_bit(&hctx->ctx_map, bit);
86 }
87 
88 struct mq_inflight {
89 	struct block_device *part;
90 	unsigned int inflight[2];
91 };
92 
93 static bool blk_mq_check_in_driver(struct request *rq, void *priv)
94 {
95 	struct mq_inflight *mi = priv;
96 
97 	if (rq->rq_flags & RQF_IO_STAT &&
98 	    (!bdev_is_partition(mi->part) || rq->part == mi->part) &&
99 	    blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
100 		mi->inflight[rq_data_dir(rq)]++;
101 
102 	return true;
103 }
104 
105 void blk_mq_in_driver_rw(struct block_device *part, unsigned int inflight[2])
106 {
107 	struct mq_inflight mi = { .part = part };
108 
109 	blk_mq_queue_tag_busy_iter(bdev_get_queue(part), blk_mq_check_in_driver,
110 				   &mi);
111 	inflight[READ] = mi.inflight[READ];
112 	inflight[WRITE] = mi.inflight[WRITE];
113 }
114 
115 #ifdef CONFIG_LOCKDEP
116 static bool blk_freeze_set_owner(struct request_queue *q,
117 				 struct task_struct *owner)
118 {
119 	if (!owner)
120 		return false;
121 
122 	if (!q->mq_freeze_depth) {
123 		q->mq_freeze_owner = owner;
124 		q->mq_freeze_owner_depth = 1;
125 		q->mq_freeze_disk_dead = !q->disk ||
126 			test_bit(GD_DEAD, &q->disk->state) ||
127 			!blk_queue_registered(q);
128 		q->mq_freeze_queue_dying = blk_queue_dying(q);
129 		return true;
130 	}
131 
132 	if (owner == q->mq_freeze_owner)
133 		q->mq_freeze_owner_depth += 1;
134 	return false;
135 }
136 
137 /* verify the last unfreeze in owner context */
138 static bool blk_unfreeze_check_owner(struct request_queue *q)
139 {
140 	if (q->mq_freeze_owner != current)
141 		return false;
142 	if (--q->mq_freeze_owner_depth == 0) {
143 		q->mq_freeze_owner = NULL;
144 		return true;
145 	}
146 	return false;
147 }
148 
149 #else
150 
151 static bool blk_freeze_set_owner(struct request_queue *q,
152 				 struct task_struct *owner)
153 {
154 	return false;
155 }
156 
157 static bool blk_unfreeze_check_owner(struct request_queue *q)
158 {
159 	return false;
160 }
161 #endif
162 
163 bool __blk_freeze_queue_start(struct request_queue *q,
164 			      struct task_struct *owner)
165 {
166 	bool freeze;
167 
168 	mutex_lock(&q->mq_freeze_lock);
169 	freeze = blk_freeze_set_owner(q, owner);
170 	if (++q->mq_freeze_depth == 1) {
171 		percpu_ref_kill(&q->q_usage_counter);
172 		mutex_unlock(&q->mq_freeze_lock);
173 		if (queue_is_mq(q))
174 			blk_mq_run_hw_queues(q, false);
175 	} else {
176 		mutex_unlock(&q->mq_freeze_lock);
177 	}
178 
179 	return freeze;
180 }
181 
182 void blk_freeze_queue_start(struct request_queue *q)
183 {
184 	if (__blk_freeze_queue_start(q, current))
185 		blk_freeze_acquire_lock(q);
186 }
187 EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
188 
189 void blk_mq_freeze_queue_wait(struct request_queue *q)
190 {
191 	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
192 }
193 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
194 
195 int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
196 				     unsigned long timeout)
197 {
198 	return wait_event_timeout(q->mq_freeze_wq,
199 					percpu_ref_is_zero(&q->q_usage_counter),
200 					timeout);
201 }
202 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
203 
204 void blk_mq_freeze_queue_nomemsave(struct request_queue *q)
205 {
206 	blk_freeze_queue_start(q);
207 	blk_mq_freeze_queue_wait(q);
208 }
209 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_nomemsave);
210 
211 bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
212 {
213 	bool unfreeze;
214 
215 	mutex_lock(&q->mq_freeze_lock);
216 	if (force_atomic)
217 		q->q_usage_counter.data->force_atomic = true;
218 	q->mq_freeze_depth--;
219 	WARN_ON_ONCE(q->mq_freeze_depth < 0);
220 	if (!q->mq_freeze_depth) {
221 		percpu_ref_resurrect(&q->q_usage_counter);
222 		wake_up_all(&q->mq_freeze_wq);
223 	}
224 	unfreeze = blk_unfreeze_check_owner(q);
225 	mutex_unlock(&q->mq_freeze_lock);
226 
227 	return unfreeze;
228 }
229 
230 void blk_mq_unfreeze_queue_nomemrestore(struct request_queue *q)
231 {
232 	if (__blk_mq_unfreeze_queue(q, false))
233 		blk_unfreeze_release_lock(q);
234 }
235 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_nomemrestore);
236 
237 /*
238  * non_owner variant of blk_freeze_queue_start
239  *
240  * Unlike blk_freeze_queue_start, the queue doesn't need to be unfrozen
241  * by the same task.  This is fragile and should not be used if at all
242  * possible.
243  */
244 void blk_freeze_queue_start_non_owner(struct request_queue *q)
245 {
246 	__blk_freeze_queue_start(q, NULL);
247 }
248 EXPORT_SYMBOL_GPL(blk_freeze_queue_start_non_owner);
249 
250 /* non_owner variant of blk_mq_unfreeze_queue */
251 void blk_mq_unfreeze_queue_non_owner(struct request_queue *q)
252 {
253 	__blk_mq_unfreeze_queue(q, false);
254 }
255 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_non_owner);
256 
257 /*
258  * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
259  * mpt3sas driver such that this function can be removed.
260  */
261 void blk_mq_quiesce_queue_nowait(struct request_queue *q)
262 {
263 	unsigned long flags;
264 
265 	spin_lock_irqsave(&q->queue_lock, flags);
266 	if (!q->quiesce_depth++)
267 		blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
268 	spin_unlock_irqrestore(&q->queue_lock, flags);
269 }
270 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
271 
272 /**
273  * blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
274  * @set: tag_set to wait on
275  *
276  * Note: it is driver's responsibility for making sure that quiesce has
277  * been started on or more of the request_queues of the tag_set.  This
278  * function only waits for the quiesce on those request_queues that had
279  * the quiesce flag set using blk_mq_quiesce_queue_nowait.
280  */
281 void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
282 {
283 	if (set->flags & BLK_MQ_F_BLOCKING)
284 		synchronize_srcu(set->srcu);
285 	else
286 		synchronize_rcu();
287 }
288 EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
289 
290 /**
291  * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
292  * @q: request queue.
293  *
294  * Note: this function does not prevent that the struct request end_io()
295  * callback function is invoked. Once this function is returned, we make
296  * sure no dispatch can happen until the queue is unquiesced via
297  * blk_mq_unquiesce_queue().
298  */
299 void blk_mq_quiesce_queue(struct request_queue *q)
300 {
301 	blk_mq_quiesce_queue_nowait(q);
302 	/* nothing to wait for non-mq queues */
303 	if (queue_is_mq(q))
304 		blk_mq_wait_quiesce_done(q->tag_set);
305 }
306 EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
307 
308 /*
309  * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
310  * @q: request queue.
311  *
312  * This function recovers queue into the state before quiescing
313  * which is done by blk_mq_quiesce_queue.
314  */
315 void blk_mq_unquiesce_queue(struct request_queue *q)
316 {
317 	unsigned long flags;
318 	bool run_queue = false;
319 
320 	spin_lock_irqsave(&q->queue_lock, flags);
321 	if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
322 		;
323 	} else if (!--q->quiesce_depth) {
324 		blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
325 		run_queue = true;
326 	}
327 	spin_unlock_irqrestore(&q->queue_lock, flags);
328 
329 	/* dispatch requests which are inserted during quiescing */
330 	if (run_queue)
331 		blk_mq_run_hw_queues(q, true);
332 }
333 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
334 
335 void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
336 {
337 	struct request_queue *q;
338 
339 	spin_lock(&set->quiesce_lock);
340 	set->quiesce_depth++;
341 	rcu_read_lock();
342 	list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
343 		if (!blk_queue_skip_tagset_quiesce(q))
344 			blk_mq_quiesce_queue_nowait(q);
345 	}
346 	rcu_read_unlock();
347 	spin_unlock(&set->quiesce_lock);
348 
349 	blk_mq_wait_quiesce_done(set);
350 }
351 EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
352 
353 void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
354 {
355 	struct request_queue *q;
356 
357 	spin_lock(&set->quiesce_lock);
358 	if (set->quiesce_depth)
359 		set->quiesce_depth--;
360 	rcu_read_lock();
361 	list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
362 		if (!blk_queue_skip_tagset_quiesce(q))
363 			blk_mq_unquiesce_queue(q);
364 	}
365 	rcu_read_unlock();
366 	spin_unlock(&set->quiesce_lock);
367 }
368 EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
369 
370 void blk_mq_wake_waiters(struct request_queue *q)
371 {
372 	struct blk_mq_hw_ctx *hctx;
373 	unsigned long i;
374 
375 	queue_for_each_hw_ctx(q, hctx, i)
376 		if (blk_mq_hw_queue_mapped(hctx))
377 			blk_mq_tag_wakeup_all(hctx->tags, true);
378 }
379 
380 void blk_rq_init(struct request_queue *q, struct request *rq)
381 {
382 	memset(rq, 0, sizeof(*rq));
383 
384 	INIT_LIST_HEAD(&rq->queuelist);
385 	rq->q = q;
386 	rq->__sector = (sector_t) -1;
387 	rq->phys_gap_bit = 0;
388 	INIT_HLIST_NODE(&rq->hash);
389 	RB_CLEAR_NODE(&rq->rb_node);
390 	rq->tag = BLK_MQ_NO_TAG;
391 	rq->internal_tag = BLK_MQ_NO_TAG;
392 	rq->start_time_ns = blk_time_get_ns();
393 	blk_crypto_rq_set_defaults(rq);
394 }
395 EXPORT_SYMBOL(blk_rq_init);
396 
397 /* Set start and alloc time when the allocated request is actually used */
398 static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
399 {
400 #ifdef CONFIG_BLK_RQ_ALLOC_TIME
401 	if (blk_queue_rq_alloc_time(rq->q))
402 		rq->alloc_time_ns = alloc_time_ns;
403 	else
404 		rq->alloc_time_ns = 0;
405 #endif
406 }
407 
408 static inline void blk_mq_bio_issue_init(struct request_queue *q,
409 					 struct bio *bio)
410 {
411 #ifdef CONFIG_BLK_CGROUP
412 	if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags))
413 		bio->issue_time_ns = blk_time_get_ns();
414 #endif
415 }
416 
417 static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
418 		struct blk_mq_tags *tags, unsigned int tag)
419 {
420 	struct blk_mq_ctx *ctx = data->ctx;
421 	struct blk_mq_hw_ctx *hctx = data->hctx;
422 	struct request_queue *q = data->q;
423 	struct request *rq = tags->static_rqs[tag];
424 
425 	rq->q = q;
426 	rq->mq_ctx = ctx;
427 	rq->mq_hctx = hctx;
428 	rq->cmd_flags = data->cmd_flags;
429 
430 	if (data->flags & BLK_MQ_REQ_PM)
431 		data->rq_flags |= RQF_PM;
432 	rq->rq_flags = data->rq_flags;
433 
434 	if (data->rq_flags & RQF_SCHED_TAGS) {
435 		rq->tag = BLK_MQ_NO_TAG;
436 		rq->internal_tag = tag;
437 	} else {
438 		rq->tag = tag;
439 		rq->internal_tag = BLK_MQ_NO_TAG;
440 	}
441 	rq->timeout = 0;
442 
443 	rq->part = NULL;
444 	rq->io_start_time_ns = 0;
445 	rq->stats_sectors = 0;
446 	rq->nr_phys_segments = 0;
447 	rq->nr_integrity_segments = 0;
448 	rq->end_io = NULL;
449 	rq->end_io_data = NULL;
450 
451 	blk_crypto_rq_set_defaults(rq);
452 	INIT_LIST_HEAD(&rq->queuelist);
453 	/* tag was already set */
454 	WRITE_ONCE(rq->deadline, 0);
455 	req_ref_set(rq, 1);
456 
457 	return rq;
458 }
459 
460 static inline struct request *
461 __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
462 {
463 	unsigned int tag, tag_offset;
464 	struct blk_mq_tags *tags;
465 	struct request *rq;
466 	unsigned long tag_mask;
467 	int i, nr = 0;
468 
469 	do {
470 		tag_mask = blk_mq_get_tags(data, data->nr_tags - nr, &tag_offset);
471 		if (unlikely(!tag_mask)) {
472 			if (nr == 0)
473 				return NULL;
474 			break;
475 		}
476 		tags = blk_mq_tags_from_data(data);
477 		for (i = 0; tag_mask; i++) {
478 			if (!(tag_mask & (1UL << i)))
479 				continue;
480 			tag = tag_offset + i;
481 			prefetch(tags->static_rqs[tag]);
482 			tag_mask &= ~(1UL << i);
483 			rq = blk_mq_rq_ctx_init(data, tags, tag);
484 			rq_list_add_head(data->cached_rqs, rq);
485 			nr++;
486 		}
487 	} while (data->nr_tags > nr);
488 
489 	if (!(data->rq_flags & RQF_SCHED_TAGS))
490 		blk_mq_add_active_requests(data->hctx, nr);
491 	/* caller already holds a reference, add for remainder */
492 	percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
493 	data->nr_tags -= nr;
494 
495 	return rq_list_pop(data->cached_rqs);
496 }
497 
498 static bool blk_op_bypass_sched(blk_opf_t opf)
499 {
500 	return (opf & REQ_OP_MASK) == REQ_OP_FLUSH ||
501 	       blk_op_is_passthrough(opf);
502 }
503 
504 static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
505 {
506 	struct elevator_mq_ops *ops;
507 
508 	/* If no I/O scheduler has been configured, don't limit requests */
509 	if (!data->q->elevator) {
510 		blk_mq_tag_busy(data->hctx);
511 		return;
512 	}
513 
514 	/*
515 	 * All requests use scheduler tags when an I/O scheduler is
516 	 * enabled for the queue.
517 	 */
518 	data->rq_flags |= RQF_SCHED_TAGS;
519 
520 	/*
521 	 * Flush/passthrough requests are special and go directly to the
522 	 * dispatch list, they are not subject to the async_depth limit.
523 	 */
524 	if (blk_op_bypass_sched(data->cmd_flags))
525 		return;
526 
527 	WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
528 
529 	/*
530 	 * By default, sync requests have no limit, and async requests are
531 	 * limited to async_depth.
532 	 */
533 	ops = &data->q->elevator->type->ops;
534 	if (ops->limit_depth)
535 		ops->limit_depth(data->cmd_flags, data);
536 }
537 
538 /*
539  * Finish initializing a request once it has been claimed for an operation.
540  * Cached requests are allocated before that operation is known.
541  */
542 static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns)
543 {
544 	struct elevator_queue *e;
545 
546 	blk_mq_rq_time_init(rq, alloc_time_ns);
547 
548 	if (!(rq->rq_flags & RQF_SCHED_TAGS) || (rq->rq_flags & RQF_RESV) ||
549 	    blk_op_bypass_sched(rq->cmd_flags))
550 		return;
551 
552 	rq->rq_flags |= RQF_USE_SCHED;
553 	INIT_HLIST_NODE(&rq->hash);
554 	RB_CLEAR_NODE(&rq->rb_node);
555 
556 	e = rq->q->elevator;
557 	if (e->type->ops.prepare_request)
558 		e->type->ops.prepare_request(rq);
559 }
560 
561 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
562 {
563 	struct request_queue *q = data->q;
564 	u64 alloc_time_ns = 0;
565 	struct request *rq;
566 	unsigned int tag;
567 
568 	/* alloc_time includes depth and tag waits */
569 	if (blk_queue_rq_alloc_time(q))
570 		alloc_time_ns = blk_time_get_ns();
571 
572 	if (data->cmd_flags & REQ_NOWAIT)
573 		data->flags |= BLK_MQ_REQ_NOWAIT;
574 
575 retry:
576 	data->ctx = blk_mq_get_ctx(q);
577 	data->hctx = blk_mq_map_queue(data->cmd_flags, data->ctx);
578 
579 	blk_mq_limit_depth(data);
580 	if (data->flags & BLK_MQ_REQ_RESERVED)
581 		data->rq_flags |= RQF_RESV;
582 
583 	/*
584 	 * Try batched alloc if we want more than 1 tag.
585 	 */
586 	if (data->nr_tags > 1) {
587 		rq = __blk_mq_alloc_requests_batch(data);
588 		if (rq) {
589 			blk_mq_rq_late_init(rq, alloc_time_ns);
590 			return rq;
591 		}
592 		data->nr_tags = 1;
593 	}
594 
595 	/*
596 	 * Waiting allocations only fail because of an inactive hctx.  In that
597 	 * case just retry the hctx assignment and tag allocation as CPU hotplug
598 	 * should have migrated us to an online CPU by now.
599 	 */
600 	tag = blk_mq_get_tag(data);
601 	if (tag == BLK_MQ_NO_TAG) {
602 		if (data->flags & BLK_MQ_REQ_NOWAIT)
603 			return NULL;
604 		/*
605 		 * Give up the CPU and sleep for a random short time to
606 		 * ensure that thread using a realtime scheduling class
607 		 * are migrated off the CPU, and thus off the hctx that
608 		 * is going away.
609 		 */
610 		msleep(3);
611 		goto retry;
612 	}
613 
614 	if (!(data->rq_flags & RQF_SCHED_TAGS))
615 		blk_mq_inc_active_requests(data->hctx);
616 	rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
617 	blk_mq_rq_late_init(rq, alloc_time_ns);
618 	return rq;
619 }
620 
621 static struct request *blk_mq_rq_cache_fill(struct request_queue *q,
622 					    struct blk_plug *plug,
623 					    blk_opf_t opf,
624 					    blk_mq_req_flags_t flags)
625 {
626 	struct blk_mq_alloc_data data = {
627 		.q		= q,
628 		.flags		= flags,
629 		.shallow_depth	= 0,
630 		.cmd_flags	= opf,
631 		.rq_flags	= 0,
632 		.nr_tags	= plug->nr_ios,
633 		.cached_rqs	= &plug->cached_rqs,
634 		.ctx		= NULL,
635 		.hctx		= NULL
636 	};
637 	struct request *rq;
638 
639 	if (blk_queue_enter(q, flags))
640 		return NULL;
641 
642 	plug->nr_ios = 1;
643 
644 	rq = __blk_mq_alloc_requests(&data);
645 	if (unlikely(!rq))
646 		blk_queue_exit(q);
647 	return rq;
648 }
649 
650 static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
651 						   blk_opf_t opf,
652 						   blk_mq_req_flags_t flags)
653 {
654 	struct blk_plug *plug = current->plug;
655 	struct request *rq;
656 
657 	if (!plug)
658 		return NULL;
659 
660 	if (rq_list_empty(&plug->cached_rqs)) {
661 		if (plug->nr_ios == 1)
662 			return NULL;
663 		rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
664 	} else {
665 		rq = rq_list_peek(&plug->cached_rqs);
666 		if (!rq || rq->q != q)
667 			return NULL;
668 
669 		if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
670 			return NULL;
671 
672 		rq_list_pop(&plug->cached_rqs);
673 		rq->cmd_flags = opf;
674 		INIT_LIST_HEAD(&rq->queuelist);
675 		blk_mq_rq_late_init(rq, blk_time_get_ns());
676 	}
677 
678 	return rq;
679 }
680 
681 struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf,
682 		blk_mq_req_flags_t flags)
683 {
684 	struct request *rq;
685 
686 	rq = blk_mq_alloc_cached_request(q, opf, flags);
687 	if (!rq) {
688 		struct blk_mq_alloc_data data = {
689 			.q		= q,
690 			.flags		= flags,
691 			.shallow_depth	= 0,
692 			.cmd_flags	= opf,
693 			.rq_flags	= 0,
694 			.nr_tags	= 1,
695 			.cached_rqs	= NULL,
696 			.ctx		= NULL,
697 			.hctx		= NULL
698 		};
699 		int ret;
700 
701 		ret = blk_queue_enter(q, flags);
702 		if (ret)
703 			return ERR_PTR(ret);
704 
705 		rq = __blk_mq_alloc_requests(&data);
706 		if (!rq)
707 			goto out_queue_exit;
708 	}
709 	rq->__data_len = 0;
710 	rq->phys_gap_bit = 0;
711 	rq->__sector = (sector_t) -1;
712 	rq->bio = rq->biotail = NULL;
713 	return rq;
714 out_queue_exit:
715 	blk_queue_exit(q);
716 	return ERR_PTR(-EWOULDBLOCK);
717 }
718 EXPORT_SYMBOL(blk_mq_alloc_request);
719 
720 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
721 	blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
722 {
723 	struct blk_mq_alloc_data data = {
724 		.q		= q,
725 		.flags		= flags,
726 		.shallow_depth	= 0,
727 		.cmd_flags	= opf,
728 		.rq_flags	= 0,
729 		.nr_tags	= 1,
730 		.cached_rqs	= NULL,
731 		.ctx		= NULL,
732 		.hctx		= NULL
733 	};
734 	u64 alloc_time_ns = 0;
735 	struct request *rq;
736 	unsigned int cpu;
737 	unsigned int tag;
738 	int ret;
739 
740 	/* alloc_time includes depth and tag waits */
741 	if (blk_queue_rq_alloc_time(q))
742 		alloc_time_ns = blk_time_get_ns();
743 
744 	/*
745 	 * If the tag allocator sleeps we could get an allocation for a
746 	 * different hardware context.  No need to complicate the low level
747 	 * allocator for this for the rare use case of a command tied to
748 	 * a specific queue.
749 	 */
750 	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) ||
751 	    WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
752 		return ERR_PTR(-EINVAL);
753 
754 	if (hctx_idx >= q->nr_hw_queues)
755 		return ERR_PTR(-EIO);
756 
757 	ret = blk_queue_enter(q, flags);
758 	if (ret)
759 		return ERR_PTR(ret);
760 
761 	/*
762 	 * Check if the hardware context is actually mapped to anything.
763 	 * If not tell the caller that it should skip this queue.
764 	 */
765 	ret = -EXDEV;
766 	data.hctx = q->queue_hw_ctx[hctx_idx];
767 	if (!blk_mq_hw_queue_mapped(data.hctx))
768 		goto out_queue_exit;
769 	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
770 	if (cpu >= nr_cpu_ids)
771 		goto out_queue_exit;
772 	data.ctx = __blk_mq_get_ctx(q, cpu);
773 
774 	if (q->elevator)
775 		data.rq_flags |= RQF_SCHED_TAGS;
776 	else
777 		blk_mq_tag_busy(data.hctx);
778 
779 	if (flags & BLK_MQ_REQ_RESERVED)
780 		data.rq_flags |= RQF_RESV;
781 
782 	ret = -EWOULDBLOCK;
783 	tag = blk_mq_get_tag(&data);
784 	if (tag == BLK_MQ_NO_TAG)
785 		goto out_queue_exit;
786 	if (!(data.rq_flags & RQF_SCHED_TAGS))
787 		blk_mq_inc_active_requests(data.hctx);
788 	rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
789 	blk_mq_rq_late_init(rq, alloc_time_ns);
790 	rq->__data_len = 0;
791 	rq->phys_gap_bit = 0;
792 	rq->__sector = (sector_t) -1;
793 	rq->bio = rq->biotail = NULL;
794 	return rq;
795 
796 out_queue_exit:
797 	blk_queue_exit(q);
798 	return ERR_PTR(ret);
799 }
800 EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
801 
802 static void blk_mq_finish_request(struct request *rq)
803 {
804 	struct request_queue *q = rq->q;
805 
806 	blk_zone_finish_request(rq);
807 
808 	if (rq->rq_flags & RQF_USE_SCHED) {
809 		q->elevator->type->ops.finish_request(rq);
810 		/*
811 		 * For postflush request that may need to be
812 		 * completed twice, we should clear this flag
813 		 * to avoid double finish_request() on the rq.
814 		 */
815 		rq->rq_flags &= ~RQF_USE_SCHED;
816 	}
817 }
818 
819 static void __blk_mq_free_request(struct request *rq)
820 {
821 	struct request_queue *q = rq->q;
822 	struct blk_mq_ctx *ctx = rq->mq_ctx;
823 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
824 	const int sched_tag = rq->internal_tag;
825 
826 	blk_crypto_free_request(rq);
827 	blk_pm_mark_last_busy(rq);
828 	rq->mq_hctx = NULL;
829 
830 	if (rq->tag != BLK_MQ_NO_TAG) {
831 		blk_mq_dec_active_requests(hctx);
832 		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
833 	}
834 	if (sched_tag != BLK_MQ_NO_TAG)
835 		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
836 	blk_mq_sched_restart(hctx);
837 	blk_queue_exit(q);
838 }
839 
840 void blk_mq_free_request(struct request *rq)
841 {
842 	struct request_queue *q = rq->q;
843 
844 	blk_mq_finish_request(rq);
845 
846 	rq_qos_done(q, rq);
847 
848 	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
849 	if (req_ref_put_and_test(rq))
850 		__blk_mq_free_request(rq);
851 }
852 EXPORT_SYMBOL_GPL(blk_mq_free_request);
853 
854 void blk_mq_free_plug_rqs(struct blk_plug *plug)
855 {
856 	struct request *rq;
857 
858 	while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL)
859 		blk_mq_free_request(rq);
860 }
861 
862 void blk_dump_rq_flags(struct request *rq, char *msg)
863 {
864 	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
865 		rq->q->disk ? rq->q->disk->disk_name : "?",
866 		(__force unsigned long long) rq->cmd_flags);
867 
868 	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
869 	       (unsigned long long)blk_rq_pos(rq),
870 	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
871 	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
872 	       rq->bio, rq->biotail, blk_rq_bytes(rq));
873 }
874 EXPORT_SYMBOL(blk_dump_rq_flags);
875 
876 static void blk_account_io_completion(struct request *req, unsigned int bytes)
877 {
878 	if (req->rq_flags & RQF_IO_STAT) {
879 		const int sgrp = op_stat_group(req_op(req));
880 
881 		part_stat_lock();
882 		part_stat_add(req->part, sectors[sgrp], bytes >> 9);
883 		part_stat_unlock();
884 	}
885 }
886 
887 static void blk_print_req_error(struct request *req, blk_status_t status)
888 {
889 	printk_ratelimited(KERN_ERR
890 		"%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
891 		"phys_seg %u prio class %u\n",
892 		blk_status_to_str(status),
893 		req->q->disk ? req->q->disk->disk_name : "?",
894 		blk_rq_pos(req), (__force u32)req_op(req),
895 		blk_op_str(req_op(req)),
896 		(__force u32)(req->cmd_flags & ~REQ_OP_MASK),
897 		req->nr_phys_segments,
898 		IOPRIO_PRIO_CLASS(req_get_ioprio(req)));
899 }
900 
901 /*
902  * Fully end IO on a request. Does not support partial completions, or
903  * errors.
904  */
905 static void blk_complete_request(struct request *req)
906 {
907 	const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
908 	int total_bytes = blk_rq_bytes(req);
909 	struct bio *bio = req->bio;
910 
911 	trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
912 
913 	if (!bio)
914 		return;
915 
916 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
917 		blk_integrity_complete(req, total_bytes);
918 
919 	/*
920 	 * Upper layers may call blk_crypto_evict_key() anytime after the last
921 	 * bio_endio().  Therefore, the keyslot must be released before that.
922 	 */
923 	blk_crypto_rq_put_keyslot(req);
924 
925 	blk_account_io_completion(req, total_bytes);
926 
927 	do {
928 		struct bio *next = bio->bi_next;
929 
930 		/* Completion has already been traced */
931 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
932 
933 		if (blk_req_bio_is_zone_append(req, bio))
934 			blk_zone_append_update_request_bio(req, bio);
935 
936 		if (!is_flush)
937 			bio_endio(bio);
938 		bio = next;
939 	} while (bio);
940 
941 	/*
942 	 * Reset counters so that the request stacking driver
943 	 * can find how many bytes remain in the request
944 	 * later.
945 	 */
946 	if (!req->end_io) {
947 		req->bio = NULL;
948 		req->__data_len = 0;
949 	}
950 }
951 
952 /**
953  * blk_update_request - Complete multiple bytes without completing the request
954  * @req:      the request being processed
955  * @error:    block status code
956  * @nr_bytes: number of bytes to complete for @req
957  *
958  * Description:
959  *     Ends I/O on a number of bytes attached to @req, but doesn't complete
960  *     the request structure even if @req doesn't have leftover.
961  *     If @req has leftover, sets it up for the next range of segments.
962  *
963  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
964  *     %false return from this function.
965  *
966  * Note:
967  *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
968  *      except in the consistency check at the end of this function.
969  *
970  * Return:
971  *     %false - this request doesn't have any more data
972  *     %true  - this request has more data
973  **/
974 bool blk_update_request(struct request *req, blk_status_t error,
975 		unsigned int nr_bytes)
976 {
977 	bool is_flush = req->rq_flags & RQF_FLUSH_SEQ;
978 	bool quiet = req->rq_flags & RQF_QUIET;
979 	int total_bytes;
980 
981 	trace_block_rq_complete(req, error, nr_bytes);
982 
983 	if (!req->bio)
984 		return false;
985 
986 	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
987 	    error == BLK_STS_OK)
988 		blk_integrity_complete(req, nr_bytes);
989 
990 	/*
991 	 * Upper layers may call blk_crypto_evict_key() anytime after the last
992 	 * bio_endio().  Therefore, the keyslot must be released before that.
993 	 */
994 	if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
995 		__blk_crypto_rq_put_keyslot(req);
996 
997 	if (unlikely(error && !blk_rq_is_passthrough(req) && !quiet) &&
998 	    !test_bit(GD_DEAD, &req->q->disk->state)) {
999 		blk_print_req_error(req, error);
1000 		trace_block_rq_error(req, error, nr_bytes);
1001 	}
1002 
1003 	blk_account_io_completion(req, nr_bytes);
1004 
1005 	total_bytes = 0;
1006 	while (req->bio) {
1007 		struct bio *bio = req->bio;
1008 		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
1009 
1010 		if (unlikely(error))
1011 			bio->bi_status = error;
1012 
1013 		if (bio_bytes == bio->bi_iter.bi_size) {
1014 			req->bio = bio->bi_next;
1015 		} else if (bio_is_zone_append(bio) && error == BLK_STS_OK) {
1016 			/*
1017 			 * Partial zone append completions cannot be supported
1018 			 * as the BIO fragments may end up not being written
1019 			 * sequentially.
1020 			 */
1021 			bio->bi_status = BLK_STS_IOERR;
1022 		}
1023 
1024 		/* Completion has already been traced */
1025 		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1026 		if (unlikely(quiet))
1027 			bio_set_flag(bio, BIO_QUIET);
1028 
1029 		bio_advance(bio, bio_bytes);
1030 
1031 		/* Don't actually finish bio if it's part of flush sequence */
1032 		if (!bio->bi_iter.bi_size) {
1033 			if (blk_req_bio_is_zone_append(req, bio))
1034 				blk_zone_append_update_request_bio(req, bio);
1035 			if (!is_flush)
1036 				bio_endio(bio);
1037 		}
1038 
1039 		total_bytes += bio_bytes;
1040 		nr_bytes -= bio_bytes;
1041 
1042 		if (!nr_bytes)
1043 			break;
1044 	}
1045 
1046 	/*
1047 	 * completely done
1048 	 */
1049 	if (!req->bio) {
1050 		/*
1051 		 * Reset counters so that the request stacking driver
1052 		 * can find how many bytes remain in the request
1053 		 * later.
1054 		 */
1055 		req->__data_len = 0;
1056 		return false;
1057 	}
1058 
1059 	req->__data_len -= total_bytes;
1060 
1061 	/* update sector only for requests with clear definition of sector */
1062 	if (!blk_rq_is_passthrough(req))
1063 		req->__sector += total_bytes >> 9;
1064 
1065 	/* mixed attributes always follow the first bio */
1066 	if (req->rq_flags & RQF_MIXED_MERGE) {
1067 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
1068 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
1069 	}
1070 
1071 	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1072 		/*
1073 		 * If total number of sectors is less than the first segment
1074 		 * size, something has gone terribly wrong.
1075 		 */
1076 		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1077 			blk_dump_rq_flags(req, "request botched");
1078 			req->__data_len = blk_rq_cur_bytes(req);
1079 		}
1080 
1081 		/* recalculate the number of segments */
1082 		req->nr_phys_segments = blk_recalc_rq_segments(req);
1083 	}
1084 
1085 	return true;
1086 }
1087 EXPORT_SYMBOL_GPL(blk_update_request);
1088 
1089 static inline void blk_account_io_done(struct request *req, u64 now)
1090 {
1091 	trace_block_io_done(req);
1092 
1093 	/*
1094 	 * Account IO completion.  flush_rq isn't accounted as a
1095 	 * normal IO on queueing nor completion.  Accounting the
1096 	 * containing request is enough.
1097 	 */
1098 	if ((req->rq_flags & (RQF_IO_STAT|RQF_FLUSH_SEQ)) == RQF_IO_STAT) {
1099 		const int sgrp = op_stat_group(req_op(req));
1100 
1101 		part_stat_lock();
1102 		update_io_ticks(req->part, jiffies, true);
1103 		part_stat_inc(req->part, ios[sgrp]);
1104 		part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
1105 		bdev_dec_in_flight(req->part, req_op(req));
1106 		part_stat_unlock();
1107 	}
1108 }
1109 
1110 static inline void blk_account_io_start(struct request *req)
1111 {
1112 	trace_block_io_start(req);
1113 
1114 	if (!blk_queue_io_stat(req->q))
1115 		return;
1116 	if (blk_rq_is_passthrough(req) && !blk_rq_passthrough_stats(req, req->q))
1117 		return;
1118 
1119 	req->rq_flags |= RQF_IO_STAT;
1120 	req->start_time_ns = blk_time_get_ns();
1121 
1122 	/*
1123 	 * All non-passthrough requests are created from a bio with one
1124 	 * exception: when a flush command that is part of a flush sequence
1125 	 * generated by the state machine in blk-flush.c is cloned onto the
1126 	 * lower device by dm-multipath we can get here without a bio.
1127 	 */
1128 	if (req->bio)
1129 		req->part = req->bio->bi_bdev;
1130 	else
1131 		req->part = req->q->disk->part0;
1132 
1133 	part_stat_lock();
1134 	update_io_ticks(req->part, jiffies, false);
1135 	bdev_inc_in_flight(req->part, req_op(req));
1136 	part_stat_unlock();
1137 }
1138 
1139 static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
1140 {
1141 	if (rq->rq_flags & RQF_STATS)
1142 		blk_stat_add(rq, now);
1143 
1144 	blk_mq_sched_completed_request(rq, now);
1145 	blk_account_io_done(rq, now);
1146 }
1147 
1148 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
1149 {
1150 	if (blk_mq_need_time_stamp(rq))
1151 		__blk_mq_end_request_acct(rq, blk_time_get_ns());
1152 
1153 	blk_mq_finish_request(rq);
1154 
1155 	if (rq->end_io) {
1156 		rq_qos_done(rq->q, rq);
1157 		if (rq->end_io(rq, error, NULL) == RQ_END_IO_FREE)
1158 			blk_mq_free_request(rq);
1159 	} else {
1160 		blk_mq_free_request(rq);
1161 	}
1162 }
1163 EXPORT_SYMBOL(__blk_mq_end_request);
1164 
1165 void blk_mq_end_request(struct request *rq, blk_status_t error)
1166 {
1167 	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
1168 		BUG();
1169 	__blk_mq_end_request(rq, error);
1170 }
1171 EXPORT_SYMBOL(blk_mq_end_request);
1172 
1173 #define TAG_COMP_BATCH		32
1174 
1175 static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
1176 					  int *tag_array, int nr_tags)
1177 {
1178 	struct request_queue *q = hctx->queue;
1179 
1180 	blk_mq_sub_active_requests(hctx, nr_tags);
1181 
1182 	blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
1183 	percpu_ref_put_many(&q->q_usage_counter, nr_tags);
1184 }
1185 
1186 void blk_mq_end_request_batch(struct io_comp_batch *iob)
1187 {
1188 	int tags[TAG_COMP_BATCH], nr_tags = 0;
1189 	struct blk_mq_hw_ctx *cur_hctx = NULL;
1190 	struct request *rq;
1191 	u64 now = 0;
1192 
1193 	if (iob->need_ts)
1194 		now = blk_time_get_ns();
1195 
1196 	while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
1197 		prefetch(rq->bio);
1198 		prefetch(rq->rq_next);
1199 
1200 		blk_complete_request(rq);
1201 		if (iob->need_ts)
1202 			__blk_mq_end_request_acct(rq, now);
1203 
1204 		blk_mq_finish_request(rq);
1205 
1206 		rq_qos_done(rq->q, rq);
1207 
1208 		/*
1209 		 * If end_io handler returns NONE, then it still has
1210 		 * ownership of the request.
1211 		 */
1212 		if (rq->end_io && rq->end_io(rq, 0, iob) == RQ_END_IO_NONE)
1213 			continue;
1214 
1215 		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1216 		if (!req_ref_put_and_test(rq))
1217 			continue;
1218 
1219 		blk_crypto_free_request(rq);
1220 		blk_pm_mark_last_busy(rq);
1221 
1222 		if (nr_tags == TAG_COMP_BATCH || cur_hctx != rq->mq_hctx) {
1223 			if (cur_hctx)
1224 				blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1225 			nr_tags = 0;
1226 			cur_hctx = rq->mq_hctx;
1227 		}
1228 		tags[nr_tags++] = rq->tag;
1229 	}
1230 
1231 	if (nr_tags)
1232 		blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1233 }
1234 EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
1235 
1236 static void blk_complete_reqs(struct llist_head *list)
1237 {
1238 	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
1239 	struct request *rq, *next;
1240 
1241 	llist_for_each_entry_safe(rq, next, entry, ipi_list)
1242 		rq->q->mq_ops->complete(rq);
1243 }
1244 
1245 static __latent_entropy void blk_done_softirq(void)
1246 {
1247 	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
1248 }
1249 
1250 static int blk_softirq_cpu_dead(unsigned int cpu)
1251 {
1252 	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
1253 	return 0;
1254 }
1255 
1256 static void __blk_mq_complete_request_remote(void *data)
1257 {
1258 	__raise_softirq_irqoff(BLOCK_SOFTIRQ);
1259 }
1260 
1261 static inline bool blk_mq_complete_need_ipi(struct request *rq)
1262 {
1263 	int cpu = raw_smp_processor_id();
1264 
1265 	if (!IS_ENABLED(CONFIG_SMP) ||
1266 	    !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
1267 		return false;
1268 	/*
1269 	 * With force threaded interrupts enabled, raising softirq from an SMP
1270 	 * function call will always result in waking the ksoftirqd thread.
1271 	 * This is probably worse than completing the request on a different
1272 	 * cache domain.
1273 	 */
1274 	if (force_irqthreads())
1275 		return false;
1276 
1277 	/* same CPU or cache domain and capacity?  Complete locally */
1278 	if (cpu == rq->mq_ctx->cpu ||
1279 	    (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
1280 	     cpus_share_cache(cpu, rq->mq_ctx->cpu) &&
1281 	     cpus_equal_capacity(cpu, rq->mq_ctx->cpu)))
1282 		return false;
1283 
1284 	/* don't try to IPI to an offline CPU */
1285 	return cpu_online(rq->mq_ctx->cpu);
1286 }
1287 
1288 static void blk_mq_complete_send_ipi(struct request *rq)
1289 {
1290 	unsigned int cpu;
1291 
1292 	cpu = rq->mq_ctx->cpu;
1293 	if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
1294 		smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
1295 }
1296 
1297 static void blk_mq_raise_softirq(struct request *rq)
1298 {
1299 	struct llist_head *list;
1300 
1301 	preempt_disable();
1302 	list = this_cpu_ptr(&blk_cpu_done);
1303 	if (llist_add(&rq->ipi_list, list))
1304 		raise_softirq(BLOCK_SOFTIRQ);
1305 	preempt_enable();
1306 }
1307 
1308 bool blk_mq_complete_request_remote(struct request *rq)
1309 {
1310 	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
1311 
1312 	/*
1313 	 * For request which hctx has only one ctx mapping,
1314 	 * or a polled request, always complete locally,
1315 	 * it's pointless to redirect the completion.
1316 	 */
1317 	if ((rq->mq_hctx->nr_ctx == 1 &&
1318 	     rq->mq_ctx->cpu == raw_smp_processor_id()) ||
1319 	     rq->cmd_flags & REQ_POLLED)
1320 		return false;
1321 
1322 	if (blk_mq_complete_need_ipi(rq)) {
1323 		blk_mq_complete_send_ipi(rq);
1324 		return true;
1325 	}
1326 
1327 	if (rq->q->nr_hw_queues == 1) {
1328 		blk_mq_raise_softirq(rq);
1329 		return true;
1330 	}
1331 	return false;
1332 }
1333 EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
1334 
1335 /**
1336  * blk_mq_complete_request - end I/O on a request
1337  * @rq:		the request being processed
1338  *
1339  * Description:
1340  *	Complete a request by scheduling the ->complete_rq operation.
1341  **/
1342 void blk_mq_complete_request(struct request *rq)
1343 {
1344 	if (!blk_mq_complete_request_remote(rq))
1345 		rq->q->mq_ops->complete(rq);
1346 }
1347 EXPORT_SYMBOL(blk_mq_complete_request);
1348 
1349 /**
1350  * blk_mq_start_request - Start processing a request
1351  * @rq: Pointer to request to be started
1352  *
1353  * Function used by device drivers to notify the block layer that a request
1354  * is going to be processed now, so blk layer can do proper initializations
1355  * such as starting the timeout timer.
1356  */
1357 void blk_mq_start_request(struct request *rq)
1358 {
1359 	struct request_queue *q = rq->q;
1360 
1361 	trace_block_rq_issue(rq);
1362 
1363 	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags) &&
1364 	    !blk_rq_is_passthrough(rq)) {
1365 		rq->io_start_time_ns = blk_time_get_ns();
1366 		rq->stats_sectors = blk_rq_sectors(rq);
1367 		rq->rq_flags |= RQF_STATS;
1368 		rq_qos_issue(q, rq);
1369 	}
1370 
1371 	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
1372 
1373 	blk_add_timer(rq);
1374 	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
1375 	rq->mq_hctx->tags->rqs[rq->tag] = rq;
1376 
1377 	if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
1378 		blk_integrity_prepare(rq);
1379 
1380 	if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
1381 	        WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
1382 }
1383 EXPORT_SYMBOL(blk_mq_start_request);
1384 
1385 /*
1386  * Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
1387  * queues. This is important for md arrays to benefit from merging
1388  * requests.
1389  */
1390 static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
1391 {
1392 	if (plug->multiple_queues)
1393 		return BLK_MAX_REQUEST_COUNT * 2;
1394 	return BLK_MAX_REQUEST_COUNT;
1395 }
1396 
1397 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
1398 {
1399 	struct request *last = rq_list_peek(&plug->mq_list);
1400 
1401 	if (!plug->rq_count) {
1402 		trace_block_plug(rq->q);
1403 	} else if (plug->rq_count >= blk_plug_max_rq_count(plug) ||
1404 		   (!blk_queue_nomerges(rq->q) &&
1405 		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1406 		blk_mq_flush_plug_list(plug, false);
1407 		blk_plug_invalidate_ts();
1408 		last = NULL;
1409 		trace_block_plug(rq->q);
1410 	}
1411 
1412 	if (!plug->multiple_queues && last && last->q != rq->q)
1413 		plug->multiple_queues = true;
1414 	/*
1415 	 * Any request allocated from sched tags can't be issued to
1416 	 * ->queue_rqs() directly
1417 	 */
1418 	if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
1419 		plug->has_elevator = true;
1420 	rq_list_add_tail(&plug->mq_list, rq);
1421 	plug->rq_count++;
1422 }
1423 
1424 /**
1425  * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
1426  * @rq:		request to insert
1427  * @at_head:    insert request at head or tail of queue
1428  *
1429  * Description:
1430  *    Insert a fully prepared request at the back of the I/O scheduler queue
1431  *    for execution.  Don't wait for completion.
1432  *
1433  * Note:
1434  *    This function will invoke @done directly if the queue is dead.
1435  */
1436 void blk_execute_rq_nowait(struct request *rq, bool at_head)
1437 {
1438 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1439 
1440 	WARN_ON(irqs_disabled());
1441 	WARN_ON(!blk_rq_is_passthrough(rq));
1442 
1443 	blk_account_io_start(rq);
1444 
1445 	if (current->plug && !at_head) {
1446 		blk_add_rq_to_plug(current->plug, rq);
1447 		return;
1448 	}
1449 
1450 	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1451 	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
1452 }
1453 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
1454 
1455 struct blk_rq_wait {
1456 	struct completion done;
1457 	blk_status_t ret;
1458 };
1459 
1460 static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret,
1461 					  const struct io_comp_batch *iob)
1462 {
1463 	struct blk_rq_wait *wait = rq->end_io_data;
1464 
1465 	wait->ret = ret;
1466 	complete(&wait->done);
1467 	return RQ_END_IO_NONE;
1468 }
1469 
1470 bool blk_rq_is_poll(struct request *rq)
1471 {
1472 	if (!rq->mq_hctx)
1473 		return false;
1474 	if (rq->mq_hctx->type != HCTX_TYPE_POLL)
1475 		return false;
1476 	return true;
1477 }
1478 EXPORT_SYMBOL_GPL(blk_rq_is_poll);
1479 
1480 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
1481 {
1482 	do {
1483 		blk_hctx_poll(rq->q, rq->mq_hctx, NULL, BLK_POLL_ONESHOT);
1484 		cond_resched();
1485 	} while (!completion_done(wait));
1486 }
1487 
1488 /**
1489  * blk_execute_rq - insert a request into queue for execution
1490  * @rq:		request to insert
1491  * @at_head:    insert request at head or tail of queue
1492  *
1493  * Description:
1494  *    Insert a fully prepared request at the back of the I/O scheduler queue
1495  *    for execution and wait for completion.
1496  * Return: The blk_status_t result provided to blk_mq_end_request().
1497  */
1498 blk_status_t blk_execute_rq(struct request *rq, bool at_head)
1499 {
1500 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1501 	struct blk_rq_wait wait = {
1502 		.done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
1503 	};
1504 
1505 	WARN_ON(irqs_disabled());
1506 	WARN_ON(!blk_rq_is_passthrough(rq));
1507 
1508 	rq->end_io_data = &wait;
1509 	rq->end_io = blk_end_sync_rq;
1510 
1511 	blk_account_io_start(rq);
1512 	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1513 	blk_mq_run_hw_queue(hctx, false);
1514 
1515 	if (blk_rq_is_poll(rq))
1516 		blk_rq_poll_completion(rq, &wait.done);
1517 	else
1518 		blk_wait_io(&wait.done);
1519 
1520 	return wait.ret;
1521 }
1522 EXPORT_SYMBOL(blk_execute_rq);
1523 
1524 static void __blk_mq_requeue_request(struct request *rq)
1525 {
1526 	struct request_queue *q = rq->q;
1527 
1528 	blk_mq_put_driver_tag(rq);
1529 
1530 	trace_block_rq_requeue(rq);
1531 	rq_qos_requeue(q, rq);
1532 
1533 	if (blk_mq_request_started(rq)) {
1534 		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1535 		rq->rq_flags &= ~RQF_TIMED_OUT;
1536 	}
1537 }
1538 
1539 void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
1540 {
1541 	struct request_queue *q = rq->q;
1542 	unsigned long flags;
1543 
1544 	__blk_mq_requeue_request(rq);
1545 
1546 	/* this request will be re-inserted to io scheduler queue */
1547 	blk_mq_sched_requeue_request(rq);
1548 
1549 	spin_lock_irqsave(&q->requeue_lock, flags);
1550 	list_add_tail(&rq->queuelist, &q->requeue_list);
1551 	spin_unlock_irqrestore(&q->requeue_lock, flags);
1552 
1553 	if (kick_requeue_list)
1554 		blk_mq_kick_requeue_list(q);
1555 }
1556 EXPORT_SYMBOL(blk_mq_requeue_request);
1557 
1558 static void blk_mq_requeue_work(struct work_struct *work)
1559 {
1560 	struct request_queue *q =
1561 		container_of(work, struct request_queue, requeue_work.work);
1562 	LIST_HEAD(rq_list);
1563 	LIST_HEAD(flush_list);
1564 	struct request *rq;
1565 
1566 	spin_lock_irq(&q->requeue_lock);
1567 	list_splice_init(&q->requeue_list, &rq_list);
1568 	list_splice_init(&q->flush_list, &flush_list);
1569 	spin_unlock_irq(&q->requeue_lock);
1570 
1571 	while (!list_empty(&rq_list)) {
1572 		rq = list_entry(rq_list.next, struct request, queuelist);
1573 		list_del_init(&rq->queuelist);
1574 		/*
1575 		 * If RQF_DONTPREP is set, the request has been started by the
1576 		 * driver already and might have driver-specific data allocated
1577 		 * already.  Insert it into the hctx dispatch list to avoid
1578 		 * block layer merges for the request.
1579 		 */
1580 		if (rq->rq_flags & RQF_DONTPREP)
1581 			blk_mq_request_bypass_insert(rq, 0);
1582 		else
1583 			blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
1584 	}
1585 
1586 	while (!list_empty(&flush_list)) {
1587 		rq = list_entry(flush_list.next, struct request, queuelist);
1588 		list_del_init(&rq->queuelist);
1589 		blk_mq_insert_request(rq, 0);
1590 	}
1591 
1592 	blk_mq_run_hw_queues(q, false);
1593 }
1594 
1595 void blk_mq_kick_requeue_list(struct request_queue *q)
1596 {
1597 	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
1598 }
1599 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
1600 
1601 void blk_mq_delay_kick_requeue_list(struct request_queue *q,
1602 				    unsigned long msecs)
1603 {
1604 	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
1605 				    msecs_to_jiffies(msecs));
1606 }
1607 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
1608 
1609 static bool blk_is_flush_data_rq(struct request *rq)
1610 {
1611 	return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq);
1612 }
1613 
1614 static bool blk_mq_rq_inflight(struct request *rq, void *priv)
1615 {
1616 	/*
1617 	 * If we find a request that isn't idle we know the queue is busy
1618 	 * as it's checked in the iter.
1619 	 * Return false to stop the iteration.
1620 	 *
1621 	 * In case of queue quiesce, if one flush data request is completed,
1622 	 * don't count it as inflight given the flush sequence is suspended,
1623 	 * and the original flush data request is invisible to driver, just
1624 	 * like other pending requests because of quiesce
1625 	 */
1626 	if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) &&
1627 				blk_is_flush_data_rq(rq) &&
1628 				blk_mq_request_completed(rq))) {
1629 		bool *busy = priv;
1630 
1631 		*busy = true;
1632 		return false;
1633 	}
1634 
1635 	return true;
1636 }
1637 
1638 bool blk_mq_queue_inflight(struct request_queue *q)
1639 {
1640 	bool busy = false;
1641 
1642 	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
1643 	return busy;
1644 }
1645 EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
1646 
1647 static void blk_mq_rq_timed_out(struct request *req)
1648 {
1649 	blk_rq_change_timed_out(req, true);
1650 	if (req->q->mq_ops->timeout) {
1651 		enum blk_eh_timer_return ret;
1652 
1653 		ret = req->q->mq_ops->timeout(req);
1654 		if (ret == BLK_EH_DONE)
1655 			return;
1656 		WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
1657 	}
1658 
1659 	blk_add_timer(req);
1660 }
1661 
1662 struct blk_expired_data {
1663 	bool has_timedout_rq;
1664 	unsigned long next;
1665 	unsigned long timeout_start;
1666 };
1667 
1668 static bool blk_mq_req_expired(struct request *rq, struct blk_expired_data *expired)
1669 {
1670 	unsigned long deadline;
1671 
1672 	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
1673 		return false;
1674 	if (rq->rq_flags & RQF_TIMED_OUT)
1675 		return false;
1676 
1677 	deadline = READ_ONCE(rq->deadline);
1678 	if (time_after_eq(expired->timeout_start, deadline))
1679 		return true;
1680 
1681 	if (expired->next == 0)
1682 		expired->next = deadline;
1683 	else if (time_after(expired->next, deadline))
1684 		expired->next = deadline;
1685 	return false;
1686 }
1687 
1688 void blk_mq_put_rq_ref(struct request *rq)
1689 {
1690 	if (is_flush_rq(rq)) {
1691 		if (rq->end_io(rq, 0, NULL) == RQ_END_IO_FREE)
1692 			blk_mq_free_request(rq);
1693 	} else if (req_ref_put_and_test(rq)) {
1694 		__blk_mq_free_request(rq);
1695 	}
1696 }
1697 
1698 static bool blk_mq_check_expired(struct request *rq, void *priv)
1699 {
1700 	struct blk_expired_data *expired = priv;
1701 
1702 	/*
1703 	 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
1704 	 * be reallocated underneath the timeout handler's processing, then
1705 	 * the expire check is reliable. If the request is not expired, then
1706 	 * it was completed and reallocated as a new request after returning
1707 	 * from blk_mq_check_expired().
1708 	 */
1709 	if (blk_mq_req_expired(rq, expired)) {
1710 		expired->has_timedout_rq = true;
1711 		return false;
1712 	}
1713 	return true;
1714 }
1715 
1716 static bool blk_mq_handle_expired(struct request *rq, void *priv)
1717 {
1718 	struct blk_expired_data *expired = priv;
1719 
1720 	if (blk_mq_req_expired(rq, expired)) {
1721 		blk_mq_rq_timed_out(rq);
1722 		/* If its timer has been started again, count the new deadline */
1723 		blk_mq_req_expired(rq, expired);
1724 	}
1725 	return true;
1726 }
1727 
1728 static void blk_mq_timeout_work(struct work_struct *work)
1729 {
1730 	struct request_queue *q =
1731 		container_of(work, struct request_queue, timeout_work);
1732 	struct blk_expired_data expired = {
1733 		.timeout_start = jiffies,
1734 	};
1735 	struct blk_mq_hw_ctx *hctx;
1736 	unsigned long i;
1737 
1738 	/* A deadlock might occur if a request is stuck requiring a
1739 	 * timeout at the same time a queue freeze is waiting
1740 	 * completion, since the timeout code would not be able to
1741 	 * acquire the queue reference here.
1742 	 *
1743 	 * That's why we don't use blk_queue_enter here; instead, we use
1744 	 * percpu_ref_tryget directly, because we need to be able to
1745 	 * obtain a reference even in the short window between the queue
1746 	 * starting to freeze, by dropping the first reference in
1747 	 * blk_freeze_queue_start, and the moment the last request is
1748 	 * consumed, marked by the instant q_usage_counter reaches
1749 	 * zero.
1750 	 */
1751 	if (!percpu_ref_tryget(&q->q_usage_counter))
1752 		return;
1753 
1754 	/* check if there is any timed-out request */
1755 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
1756 	if (expired.has_timedout_rq) {
1757 		/*
1758 		 * Before walking tags, we must ensure any submit started
1759 		 * before the current time has finished. Since the submit
1760 		 * uses srcu or rcu, wait for a synchronization point to
1761 		 * ensure all running submits have finished
1762 		 */
1763 		blk_mq_wait_quiesce_done(q->tag_set);
1764 
1765 		expired.next = 0;
1766 		blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
1767 	}
1768 
1769 	if (expired.next != 0) {
1770 		mod_timer(&q->timeout, expired.next);
1771 	} else {
1772 		/*
1773 		 * Request timeouts are handled as a forward rolling timer. If
1774 		 * we end up here it means that no requests are pending and
1775 		 * also that no request has been pending for a while. Mark
1776 		 * each hctx as idle.
1777 		 */
1778 		queue_for_each_hw_ctx(q, hctx, i) {
1779 			/* the hctx may be unmapped, so check it here */
1780 			if (blk_mq_hw_queue_mapped(hctx))
1781 				blk_mq_tag_idle(hctx);
1782 		}
1783 	}
1784 	blk_queue_exit(q);
1785 }
1786 
1787 struct flush_busy_ctx_data {
1788 	struct blk_mq_hw_ctx *hctx;
1789 	struct list_head *list;
1790 };
1791 
1792 static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
1793 {
1794 	struct flush_busy_ctx_data *flush_data = data;
1795 	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
1796 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1797 	enum hctx_type type = hctx->type;
1798 
1799 	spin_lock(&ctx->lock);
1800 	list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1801 	sbitmap_clear_bit(sb, bitnr);
1802 	spin_unlock(&ctx->lock);
1803 	return true;
1804 }
1805 
1806 /*
1807  * Process software queues that have been marked busy, splicing them
1808  * to the for-dispatch
1809  */
1810 void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1811 {
1812 	struct flush_busy_ctx_data data = {
1813 		.hctx = hctx,
1814 		.list = list,
1815 	};
1816 
1817 	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1818 }
1819 
1820 struct dispatch_rq_data {
1821 	struct blk_mq_hw_ctx *hctx;
1822 	struct request *rq;
1823 };
1824 
1825 static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1826 		void *data)
1827 {
1828 	struct dispatch_rq_data *dispatch_data = data;
1829 	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1830 	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1831 	enum hctx_type type = hctx->type;
1832 
1833 	spin_lock(&ctx->lock);
1834 	if (!list_empty(&ctx->rq_lists[type])) {
1835 		dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1836 		list_del_init(&dispatch_data->rq->queuelist);
1837 		if (list_empty(&ctx->rq_lists[type]))
1838 			sbitmap_clear_bit(sb, bitnr);
1839 	}
1840 	spin_unlock(&ctx->lock);
1841 
1842 	return !dispatch_data->rq;
1843 }
1844 
1845 struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
1846 					struct blk_mq_ctx *start)
1847 {
1848 	unsigned off = start ? start->index_hw[hctx->type] : 0;
1849 	struct dispatch_rq_data data = {
1850 		.hctx = hctx,
1851 		.rq   = NULL,
1852 	};
1853 
1854 	__sbitmap_for_each_set(&hctx->ctx_map, off,
1855 			       dispatch_rq_from_ctx, &data);
1856 
1857 	return data.rq;
1858 }
1859 
1860 bool __blk_mq_alloc_driver_tag(struct request *rq)
1861 {
1862 	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1863 	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1864 	int tag;
1865 
1866 	blk_mq_tag_busy(rq->mq_hctx);
1867 
1868 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1869 		bt = &rq->mq_hctx->tags->breserved_tags;
1870 		tag_offset = 0;
1871 	} else {
1872 		if (!hctx_may_queue(rq->mq_hctx, bt))
1873 			return false;
1874 	}
1875 
1876 	tag = __sbitmap_queue_get(bt);
1877 	if (tag == BLK_MQ_NO_TAG)
1878 		return false;
1879 
1880 	rq->tag = tag + tag_offset;
1881 	blk_mq_inc_active_requests(rq->mq_hctx);
1882 	return true;
1883 }
1884 
1885 static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1886 				int flags, void *key)
1887 {
1888 	struct blk_mq_hw_ctx *hctx;
1889 
1890 	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1891 
1892 	spin_lock(&hctx->dispatch_wait_lock);
1893 	if (!list_empty(&wait->entry)) {
1894 		struct sbitmap_queue *sbq;
1895 
1896 		list_del_init(&wait->entry);
1897 		sbq = &hctx->tags->bitmap_tags;
1898 		atomic_dec(&sbq->ws_active);
1899 	}
1900 	spin_unlock(&hctx->dispatch_wait_lock);
1901 
1902 	blk_mq_run_hw_queue(hctx, true);
1903 	return 1;
1904 }
1905 
1906 /*
1907  * Mark us waiting for a tag. For shared tags, this involves hooking us into
1908  * the tag wakeups. For non-shared tags, we can simply mark us needing a
1909  * restart. For both cases, take care to check the condition again after
1910  * marking us as waiting.
1911  */
1912 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1913 				 struct request *rq)
1914 {
1915 	struct sbitmap_queue *sbq;
1916 	struct wait_queue_head *wq;
1917 	wait_queue_entry_t *wait;
1918 	bool ret;
1919 
1920 	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1921 	    !(blk_mq_is_shared_tags(hctx->flags))) {
1922 		blk_mq_sched_mark_restart_hctx(hctx);
1923 
1924 		/*
1925 		 * It's possible that a tag was freed in the window between the
1926 		 * allocation failure and adding the hardware queue to the wait
1927 		 * queue.
1928 		 *
1929 		 * Don't clear RESTART here, someone else could have set it.
1930 		 * At most this will cost an extra queue run.
1931 		 */
1932 		return blk_mq_get_driver_tag(rq);
1933 	}
1934 
1935 	wait = &hctx->dispatch_wait;
1936 	if (!list_empty_careful(&wait->entry))
1937 		return false;
1938 
1939 	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
1940 		sbq = &hctx->tags->breserved_tags;
1941 	else
1942 		sbq = &hctx->tags->bitmap_tags;
1943 	wq = &bt_wait_ptr(sbq, hctx)->wait;
1944 
1945 	spin_lock_irq(&wq->lock);
1946 	spin_lock(&hctx->dispatch_wait_lock);
1947 	if (!list_empty(&wait->entry)) {
1948 		spin_unlock(&hctx->dispatch_wait_lock);
1949 		spin_unlock_irq(&wq->lock);
1950 		return false;
1951 	}
1952 
1953 	atomic_inc(&sbq->ws_active);
1954 	wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1955 	__add_wait_queue(wq, wait);
1956 
1957 	/*
1958 	 * Add one explicit barrier since blk_mq_get_driver_tag() may
1959 	 * not imply barrier in case of failure.
1960 	 *
1961 	 * Order adding us to wait queue and allocating driver tag.
1962 	 *
1963 	 * The pair is the one implied in sbitmap_queue_wake_up() which
1964 	 * orders clearing sbitmap tag bits and waitqueue_active() in
1965 	 * __sbitmap_queue_wake_up(), since waitqueue_active() is lockless
1966 	 *
1967 	 * Otherwise, re-order of adding wait queue and getting driver tag
1968 	 * may cause __sbitmap_queue_wake_up() to wake up nothing because
1969 	 * the waitqueue_active() may not observe us in wait queue.
1970 	 */
1971 	smp_mb();
1972 
1973 	/*
1974 	 * It's possible that a tag was freed in the window between the
1975 	 * allocation failure and adding the hardware queue to the wait
1976 	 * queue.
1977 	 */
1978 	ret = blk_mq_get_driver_tag(rq);
1979 	if (!ret) {
1980 		spin_unlock(&hctx->dispatch_wait_lock);
1981 		spin_unlock_irq(&wq->lock);
1982 		return false;
1983 	}
1984 
1985 	/*
1986 	 * We got a tag, remove ourselves from the wait queue to ensure
1987 	 * someone else gets the wakeup.
1988 	 */
1989 	list_del_init(&wait->entry);
1990 	atomic_dec(&sbq->ws_active);
1991 	spin_unlock(&hctx->dispatch_wait_lock);
1992 	spin_unlock_irq(&wq->lock);
1993 
1994 	return true;
1995 }
1996 
1997 #define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT  8
1998 #define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR  4
1999 /*
2000  * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
2001  * - EWMA is one simple way to compute running average value
2002  * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
2003  * - take 4 as factor for avoiding to get too small(0) result, and this
2004  *   factor doesn't matter because EWMA decreases exponentially
2005  */
2006 static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
2007 {
2008 	unsigned int ewma;
2009 
2010 	ewma = hctx->dispatch_busy;
2011 
2012 	if (!ewma && !busy)
2013 		return;
2014 
2015 	ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
2016 	if (busy)
2017 		ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
2018 	ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
2019 
2020 	hctx->dispatch_busy = ewma;
2021 }
2022 
2023 #define BLK_MQ_RESOURCE_DELAY	3		/* ms units */
2024 
2025 static void blk_mq_handle_dev_resource(struct request *rq,
2026 				       struct list_head *list)
2027 {
2028 	list_add(&rq->queuelist, list);
2029 	__blk_mq_requeue_request(rq);
2030 }
2031 
2032 enum prep_dispatch {
2033 	PREP_DISPATCH_OK,
2034 	PREP_DISPATCH_NO_TAG,
2035 	PREP_DISPATCH_NO_BUDGET,
2036 };
2037 
2038 static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
2039 						  bool need_budget)
2040 {
2041 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2042 	int budget_token = -1;
2043 
2044 	if (need_budget) {
2045 		budget_token = blk_mq_get_dispatch_budget(rq->q);
2046 		if (budget_token < 0) {
2047 			blk_mq_put_driver_tag(rq);
2048 			return PREP_DISPATCH_NO_BUDGET;
2049 		}
2050 		blk_mq_set_rq_budget_token(rq, budget_token);
2051 	}
2052 
2053 	if (!blk_mq_get_driver_tag(rq)) {
2054 		/*
2055 		 * The initial allocation attempt failed, so we need to
2056 		 * rerun the hardware queue when a tag is freed. The
2057 		 * waitqueue takes care of that. If the queue is run
2058 		 * before we add this entry back on the dispatch list,
2059 		 * we'll re-run it below.
2060 		 */
2061 		if (!blk_mq_mark_tag_wait(hctx, rq)) {
2062 			/*
2063 			 * All budgets not got from this function will be put
2064 			 * together during handling partial dispatch
2065 			 */
2066 			if (need_budget)
2067 				blk_mq_put_dispatch_budget(rq->q, budget_token);
2068 			return PREP_DISPATCH_NO_TAG;
2069 		}
2070 	}
2071 
2072 	return PREP_DISPATCH_OK;
2073 }
2074 
2075 /* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
2076 static void blk_mq_release_budgets(struct request_queue *q,
2077 		struct list_head *list)
2078 {
2079 	struct request *rq;
2080 
2081 	list_for_each_entry(rq, list, queuelist) {
2082 		int budget_token = blk_mq_get_rq_budget_token(rq);
2083 
2084 		if (budget_token >= 0)
2085 			blk_mq_put_dispatch_budget(q, budget_token);
2086 	}
2087 }
2088 
2089 /*
2090  * blk_mq_commit_rqs will notify driver using bd->last that there is no
2091  * more requests. (See comment in struct blk_mq_ops for commit_rqs for
2092  * details)
2093  * Attention, we should explicitly call this in unusual cases:
2094  *  1) did not queue everything initially scheduled to queue
2095  *  2) the last attempt to queue a request failed
2096  */
2097 static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
2098 			      bool from_schedule)
2099 {
2100 	if (hctx->queue->mq_ops->commit_rqs && queued) {
2101 		trace_block_unplug(hctx->queue, queued, !from_schedule);
2102 		hctx->queue->mq_ops->commit_rqs(hctx);
2103 	}
2104 }
2105 
2106 /*
2107  * Returns true if we did some work AND can potentially do more.
2108  */
2109 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
2110 			     bool get_budget)
2111 {
2112 	enum prep_dispatch prep;
2113 	struct request_queue *q = hctx->queue;
2114 	struct request *rq;
2115 	int queued;
2116 	blk_status_t ret = BLK_STS_OK;
2117 	bool needs_resource = false;
2118 
2119 	if (list_empty(list))
2120 		return false;
2121 
2122 	/*
2123 	 * Now process all the entries, sending them to the driver.
2124 	 */
2125 	queued = 0;
2126 	do {
2127 		struct blk_mq_queue_data bd;
2128 
2129 		rq = list_first_entry(list, struct request, queuelist);
2130 
2131 		WARN_ON_ONCE(hctx != rq->mq_hctx);
2132 		prep = blk_mq_prep_dispatch_rq(rq, get_budget);
2133 		if (prep != PREP_DISPATCH_OK)
2134 			break;
2135 
2136 		list_del_init(&rq->queuelist);
2137 
2138 		bd.rq = rq;
2139 		bd.last = list_empty(list);
2140 
2141 		ret = q->mq_ops->queue_rq(hctx, &bd);
2142 		switch (ret) {
2143 		case BLK_STS_OK:
2144 			queued++;
2145 			break;
2146 		case BLK_STS_RESOURCE:
2147 			needs_resource = true;
2148 			fallthrough;
2149 		case BLK_STS_DEV_RESOURCE:
2150 			blk_mq_handle_dev_resource(rq, list);
2151 			goto out;
2152 		default:
2153 			blk_mq_end_request(rq, ret);
2154 		}
2155 	} while (!list_empty(list));
2156 out:
2157 	/* If we didn't flush the entire list, we could have told the driver
2158 	 * there was more coming, but that turned out to be a lie.
2159 	 */
2160 	if (!list_empty(list) || ret != BLK_STS_OK)
2161 		blk_mq_commit_rqs(hctx, queued, false);
2162 
2163 	/*
2164 	 * Any items that need requeuing? Stuff them into hctx->dispatch,
2165 	 * that is where we will continue on next queue run.
2166 	 */
2167 	if (!list_empty(list)) {
2168 		bool needs_restart;
2169 		/* For non-shared tags, the RESTART check will suffice */
2170 		bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
2171 			((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) ||
2172 			blk_mq_is_shared_tags(hctx->flags));
2173 
2174 		/*
2175 		 * If the caller allocated budgets, free the budgets of the
2176 		 * requests that have not yet been passed to the block driver.
2177 		 */
2178 		if (!get_budget)
2179 			blk_mq_release_budgets(q, list);
2180 
2181 		spin_lock(&hctx->lock);
2182 		list_splice_tail_init(list, &hctx->dispatch);
2183 		spin_unlock(&hctx->lock);
2184 
2185 		/*
2186 		 * Order adding requests to hctx->dispatch and checking
2187 		 * SCHED_RESTART flag. The pair of this smp_mb() is the one
2188 		 * in blk_mq_sched_restart(). Avoid restart code path to
2189 		 * miss the new added requests to hctx->dispatch, meantime
2190 		 * SCHED_RESTART is observed here.
2191 		 */
2192 		smp_mb();
2193 
2194 		/*
2195 		 * If SCHED_RESTART was set by the caller of this function and
2196 		 * it is no longer set that means that it was cleared by another
2197 		 * thread and hence that a queue rerun is needed.
2198 		 *
2199 		 * If 'no_tag' is set, that means that we failed getting
2200 		 * a driver tag with an I/O scheduler attached. If our dispatch
2201 		 * waitqueue is no longer active, ensure that we run the queue
2202 		 * AFTER adding our entries back to the list.
2203 		 *
2204 		 * If no I/O scheduler has been configured it is possible that
2205 		 * the hardware queue got stopped and restarted before requests
2206 		 * were pushed back onto the dispatch list. Rerun the queue to
2207 		 * avoid starvation. Notes:
2208 		 * - blk_mq_run_hw_queue() checks whether or not a queue has
2209 		 *   been stopped before rerunning a queue.
2210 		 * - Some but not all block drivers stop a queue before
2211 		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
2212 		 *   and dm-rq.
2213 		 *
2214 		 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
2215 		 * bit is set, run queue after a delay to avoid IO stalls
2216 		 * that could otherwise occur if the queue is idle.  We'll do
2217 		 * similar if we couldn't get budget or couldn't lock a zone
2218 		 * and SCHED_RESTART is set.
2219 		 */
2220 		needs_restart = blk_mq_sched_needs_restart(hctx);
2221 		if (prep == PREP_DISPATCH_NO_BUDGET)
2222 			needs_resource = true;
2223 		if (!needs_restart ||
2224 		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
2225 			blk_mq_run_hw_queue(hctx, true);
2226 		else if (needs_resource)
2227 			blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
2228 
2229 		blk_mq_update_dispatch_busy(hctx, true);
2230 		return false;
2231 	}
2232 
2233 	blk_mq_update_dispatch_busy(hctx, false);
2234 	return true;
2235 }
2236 
2237 static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
2238 {
2239 	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
2240 
2241 	if (cpu >= nr_cpu_ids)
2242 		cpu = cpumask_first(hctx->cpumask);
2243 	return cpu;
2244 }
2245 
2246 /*
2247  * ->next_cpu is always calculated from hctx->cpumask, so simply use
2248  * it for speeding up the check
2249  */
2250 static bool blk_mq_hctx_empty_cpumask(struct blk_mq_hw_ctx *hctx)
2251 {
2252         return hctx->next_cpu >= nr_cpu_ids;
2253 }
2254 
2255 /*
2256  * It'd be great if the workqueue API had a way to pass
2257  * in a mask and had some smarts for more clever placement.
2258  * For now we just round-robin here, switching for every
2259  * BLK_MQ_CPU_WORK_BATCH queued items.
2260  */
2261 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
2262 {
2263 	bool tried = false;
2264 	int next_cpu = hctx->next_cpu;
2265 
2266 	/* Switch to unbound if no allowable CPUs in this hctx */
2267 	if (hctx->queue->nr_hw_queues == 1 || blk_mq_hctx_empty_cpumask(hctx))
2268 		return WORK_CPU_UNBOUND;
2269 
2270 	if (--hctx->next_cpu_batch <= 0) {
2271 select_cpu:
2272 		next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
2273 				cpu_online_mask);
2274 		if (next_cpu >= nr_cpu_ids)
2275 			next_cpu = blk_mq_first_mapped_cpu(hctx);
2276 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2277 	}
2278 
2279 	/*
2280 	 * Do unbound schedule if we can't find a online CPU for this hctx,
2281 	 * and it should only happen in the path of handling CPU DEAD.
2282 	 */
2283 	if (!cpu_online(next_cpu)) {
2284 		if (!tried) {
2285 			tried = true;
2286 			goto select_cpu;
2287 		}
2288 
2289 		/*
2290 		 * Make sure to re-select CPU next time once after CPUs
2291 		 * in hctx->cpumask become online again.
2292 		 */
2293 		hctx->next_cpu = next_cpu;
2294 		hctx->next_cpu_batch = 1;
2295 		return WORK_CPU_UNBOUND;
2296 	}
2297 
2298 	hctx->next_cpu = next_cpu;
2299 	return next_cpu;
2300 }
2301 
2302 /**
2303  * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
2304  * @hctx: Pointer to the hardware queue to run.
2305  * @msecs: Milliseconds of delay to wait before running the queue.
2306  *
2307  * Run a hardware queue asynchronously with a delay of @msecs.
2308  */
2309 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
2310 {
2311 	if (unlikely(blk_mq_hctx_stopped(hctx)))
2312 		return;
2313 	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
2314 				    msecs_to_jiffies(msecs));
2315 }
2316 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
2317 
2318 static inline bool blk_mq_hw_queue_need_run(struct blk_mq_hw_ctx *hctx)
2319 {
2320 	bool need_run;
2321 
2322 	/*
2323 	 * When queue is quiesced, we may be switching io scheduler, or
2324 	 * updating nr_hw_queues, or other things, and we can't run queue
2325 	 * any more, even blk_mq_hctx_has_pending() can't be called safely.
2326 	 *
2327 	 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
2328 	 * quiesced.
2329 	 */
2330 	__blk_mq_run_dispatch_ops(hctx->queue, false,
2331 		need_run = !blk_queue_quiesced(hctx->queue) &&
2332 		blk_mq_hctx_has_pending(hctx));
2333 	return need_run;
2334 }
2335 
2336 /**
2337  * blk_mq_run_hw_queue - Start to run a hardware queue.
2338  * @hctx: Pointer to the hardware queue to run.
2339  * @async: If we want to run the queue asynchronously.
2340  *
2341  * Check if the request queue is not in a quiesced state and if there are
2342  * pending requests to be sent. If this is true, run the queue to send requests
2343  * to hardware.
2344  */
2345 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2346 {
2347 	bool need_run;
2348 
2349 	/*
2350 	 * We can't run the queue inline with interrupts disabled.
2351 	 */
2352 	WARN_ON_ONCE(!async && in_interrupt());
2353 
2354 	might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
2355 
2356 	need_run = blk_mq_hw_queue_need_run(hctx);
2357 	if (!need_run) {
2358 		unsigned long flags;
2359 
2360 		/*
2361 		 * Synchronize with blk_mq_unquiesce_queue(), because we check
2362 		 * if hw queue is quiesced locklessly above, we need the use
2363 		 * ->queue_lock to make sure we see the up-to-date status to
2364 		 * not miss rerunning the hw queue.
2365 		 */
2366 		spin_lock_irqsave(&hctx->queue->queue_lock, flags);
2367 		need_run = blk_mq_hw_queue_need_run(hctx);
2368 		spin_unlock_irqrestore(&hctx->queue->queue_lock, flags);
2369 
2370 		if (!need_run)
2371 			return;
2372 	}
2373 
2374 	if (async || !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
2375 		blk_mq_delay_run_hw_queue(hctx, 0);
2376 		return;
2377 	}
2378 
2379 	blk_mq_run_dispatch_ops(hctx->queue,
2380 				blk_mq_sched_dispatch_requests(hctx));
2381 }
2382 EXPORT_SYMBOL(blk_mq_run_hw_queue);
2383 
2384 /*
2385  * Return prefered queue to dispatch from (if any) for non-mq aware IO
2386  * scheduler.
2387  */
2388 static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
2389 {
2390 	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
2391 	/*
2392 	 * If the IO scheduler does not respect hardware queues when
2393 	 * dispatching, we just don't bother with multiple HW queues and
2394 	 * dispatch from hctx for the current CPU since running multiple queues
2395 	 * just causes lock contention inside the scheduler and pointless cache
2396 	 * bouncing.
2397 	 */
2398 	struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
2399 
2400 	if (!blk_mq_hctx_stopped(hctx))
2401 		return hctx;
2402 	return NULL;
2403 }
2404 
2405 /**
2406  * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
2407  * @q: Pointer to the request queue to run.
2408  * @async: If we want to run the queue asynchronously.
2409  */
2410 void blk_mq_run_hw_queues(struct request_queue *q, bool async)
2411 {
2412 	struct blk_mq_hw_ctx *hctx, *sq_hctx;
2413 	unsigned long i;
2414 
2415 	sq_hctx = NULL;
2416 	if (blk_queue_sq_sched(q))
2417 		sq_hctx = blk_mq_get_sq_hctx(q);
2418 	queue_for_each_hw_ctx(q, hctx, i) {
2419 		if (blk_mq_hctx_stopped(hctx))
2420 			continue;
2421 		/*
2422 		 * Dispatch from this hctx either if there's no hctx preferred
2423 		 * by IO scheduler or if it has requests that bypass the
2424 		 * scheduler.
2425 		 */
2426 		if (!sq_hctx || sq_hctx == hctx ||
2427 		    !list_empty_careful(&hctx->dispatch))
2428 			blk_mq_run_hw_queue(hctx, async);
2429 	}
2430 }
2431 EXPORT_SYMBOL(blk_mq_run_hw_queues);
2432 
2433 /**
2434  * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
2435  * @q: Pointer to the request queue to run.
2436  * @msecs: Milliseconds of delay to wait before running the queues.
2437  */
2438 void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
2439 {
2440 	struct blk_mq_hw_ctx *hctx, *sq_hctx;
2441 	unsigned long i;
2442 
2443 	sq_hctx = NULL;
2444 	if (blk_queue_sq_sched(q))
2445 		sq_hctx = blk_mq_get_sq_hctx(q);
2446 	queue_for_each_hw_ctx(q, hctx, i) {
2447 		if (blk_mq_hctx_stopped(hctx))
2448 			continue;
2449 		/*
2450 		 * If there is already a run_work pending, leave the
2451 		 * pending delay untouched. Otherwise, a hctx can stall
2452 		 * if another hctx is re-delaying the other's work
2453 		 * before the work executes.
2454 		 */
2455 		if (delayed_work_pending(&hctx->run_work))
2456 			continue;
2457 		/*
2458 		 * Dispatch from this hctx either if there's no hctx preferred
2459 		 * by IO scheduler or if it has requests that bypass the
2460 		 * scheduler.
2461 		 */
2462 		if (!sq_hctx || sq_hctx == hctx ||
2463 		    !list_empty_careful(&hctx->dispatch))
2464 			blk_mq_delay_run_hw_queue(hctx, msecs);
2465 	}
2466 }
2467 EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
2468 
2469 /*
2470  * This function is often used for pausing .queue_rq() by driver when
2471  * there isn't enough resource or some conditions aren't satisfied, and
2472  * BLK_STS_RESOURCE is usually returned.
2473  *
2474  * We do not guarantee that dispatch can be drained or blocked
2475  * after blk_mq_stop_hw_queue() returns. Please use
2476  * blk_mq_quiesce_queue() for that requirement.
2477  */
2478 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
2479 {
2480 	cancel_delayed_work(&hctx->run_work);
2481 
2482 	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
2483 }
2484 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
2485 
2486 /*
2487  * This function is often used for pausing .queue_rq() by driver when
2488  * there isn't enough resource or some conditions aren't satisfied, and
2489  * BLK_STS_RESOURCE is usually returned.
2490  *
2491  * We do not guarantee that dispatch can be drained or blocked
2492  * after blk_mq_stop_hw_queues() returns. Please use
2493  * blk_mq_quiesce_queue() for that requirement.
2494  */
2495 void blk_mq_stop_hw_queues(struct request_queue *q)
2496 {
2497 	struct blk_mq_hw_ctx *hctx;
2498 	unsigned long i;
2499 
2500 	queue_for_each_hw_ctx(q, hctx, i)
2501 		blk_mq_stop_hw_queue(hctx);
2502 }
2503 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
2504 
2505 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
2506 {
2507 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2508 
2509 	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
2510 }
2511 EXPORT_SYMBOL(blk_mq_start_hw_queue);
2512 
2513 void blk_mq_start_hw_queues(struct request_queue *q)
2514 {
2515 	struct blk_mq_hw_ctx *hctx;
2516 	unsigned long i;
2517 
2518 	queue_for_each_hw_ctx(q, hctx, i)
2519 		blk_mq_start_hw_queue(hctx);
2520 }
2521 EXPORT_SYMBOL(blk_mq_start_hw_queues);
2522 
2523 void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2524 {
2525 	if (!blk_mq_hctx_stopped(hctx))
2526 		return;
2527 
2528 	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2529 	/*
2530 	 * Pairs with the smp_mb() in blk_mq_hctx_stopped() to order the
2531 	 * clearing of BLK_MQ_S_STOPPED above and the checking of dispatch
2532 	 * list in the subsequent routine.
2533 	 */
2534 	smp_mb__after_atomic();
2535 	blk_mq_run_hw_queue(hctx, async);
2536 }
2537 EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
2538 
2539 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
2540 {
2541 	struct blk_mq_hw_ctx *hctx;
2542 	unsigned long i;
2543 
2544 	queue_for_each_hw_ctx(q, hctx, i)
2545 		blk_mq_start_stopped_hw_queue(hctx, async ||
2546 					(hctx->flags & BLK_MQ_F_BLOCKING));
2547 }
2548 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
2549 
2550 static void blk_mq_run_work_fn(struct work_struct *work)
2551 {
2552 	struct blk_mq_hw_ctx *hctx =
2553 		container_of(work, struct blk_mq_hw_ctx, run_work.work);
2554 
2555 	blk_mq_run_dispatch_ops(hctx->queue,
2556 				blk_mq_sched_dispatch_requests(hctx));
2557 }
2558 
2559 /**
2560  * blk_mq_request_bypass_insert - Insert a request at dispatch list.
2561  * @rq: Pointer to request to be inserted.
2562  * @flags: BLK_MQ_INSERT_*
2563  *
2564  * Should only be used carefully, when the caller knows we want to
2565  * bypass a potential IO scheduler on the target device.
2566  */
2567 static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
2568 {
2569 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2570 
2571 	spin_lock(&hctx->lock);
2572 	if (flags & BLK_MQ_INSERT_AT_HEAD)
2573 		list_add(&rq->queuelist, &hctx->dispatch);
2574 	else
2575 		list_add_tail(&rq->queuelist, &hctx->dispatch);
2576 	spin_unlock(&hctx->lock);
2577 }
2578 
2579 static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
2580 		struct blk_mq_ctx *ctx, struct list_head *list,
2581 		bool run_queue_async)
2582 {
2583 	struct request *rq;
2584 	enum hctx_type type = hctx->type;
2585 
2586 	/*
2587 	 * Try to issue requests directly if the hw queue isn't busy to save an
2588 	 * extra enqueue & dequeue to the sw queue.
2589 	 */
2590 	if (!hctx->dispatch_busy && !run_queue_async) {
2591 		blk_mq_run_dispatch_ops(hctx->queue,
2592 			blk_mq_try_issue_list_directly(hctx, list));
2593 		if (list_empty(list))
2594 			goto out;
2595 	}
2596 
2597 	/*
2598 	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
2599 	 * offline now
2600 	 */
2601 	list_for_each_entry(rq, list, queuelist) {
2602 		BUG_ON(rq->mq_ctx != ctx);
2603 		trace_block_rq_insert(rq);
2604 		if (rq->cmd_flags & REQ_NOWAIT)
2605 			run_queue_async = true;
2606 	}
2607 
2608 	spin_lock(&ctx->lock);
2609 	list_splice_tail_init(list, &ctx->rq_lists[type]);
2610 	blk_mq_hctx_mark_pending(hctx, ctx);
2611 	spin_unlock(&ctx->lock);
2612 out:
2613 	blk_mq_run_hw_queue(hctx, run_queue_async);
2614 }
2615 
2616 static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
2617 {
2618 	struct request_queue *q = rq->q;
2619 	struct blk_mq_ctx *ctx = rq->mq_ctx;
2620 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2621 
2622 	if (blk_rq_is_passthrough(rq)) {
2623 		/*
2624 		 * Passthrough request have to be added to hctx->dispatch
2625 		 * directly.  The device may be in a situation where it can't
2626 		 * handle FS request, and always returns BLK_STS_RESOURCE for
2627 		 * them, which gets them added to hctx->dispatch.
2628 		 *
2629 		 * If a passthrough request is required to unblock the queues,
2630 		 * and it is added to the scheduler queue, there is no chance to
2631 		 * dispatch it given we prioritize requests in hctx->dispatch.
2632 		 */
2633 		blk_mq_request_bypass_insert(rq, flags);
2634 	} else if (req_op(rq) == REQ_OP_FLUSH) {
2635 		/*
2636 		 * Firstly normal IO request is inserted to scheduler queue or
2637 		 * sw queue, meantime we add flush request to dispatch queue(
2638 		 * hctx->dispatch) directly and there is at most one in-flight
2639 		 * flush request for each hw queue, so it doesn't matter to add
2640 		 * flush request to tail or front of the dispatch queue.
2641 		 *
2642 		 * Secondly in case of NCQ, flush request belongs to non-NCQ
2643 		 * command, and queueing it will fail when there is any
2644 		 * in-flight normal IO request(NCQ command). When adding flush
2645 		 * rq to the front of hctx->dispatch, it is easier to introduce
2646 		 * extra time to flush rq's latency because of S_SCHED_RESTART
2647 		 * compared with adding to the tail of dispatch queue, then
2648 		 * chance of flush merge is increased, and less flush requests
2649 		 * will be issued to controller. It is observed that ~10% time
2650 		 * is saved in blktests block/004 on disk attached to AHCI/NCQ
2651 		 * drive when adding flush rq to the front of hctx->dispatch.
2652 		 *
2653 		 * Simply queue flush rq to the front of hctx->dispatch so that
2654 		 * intensive flush workloads can benefit in case of NCQ HW.
2655 		 */
2656 		blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
2657 	} else if (q->elevator) {
2658 		LIST_HEAD(list);
2659 
2660 		WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
2661 
2662 		list_add(&rq->queuelist, &list);
2663 		q->elevator->type->ops.insert_requests(hctx, &list, flags);
2664 	} else {
2665 		trace_block_rq_insert(rq);
2666 
2667 		spin_lock(&ctx->lock);
2668 		if (flags & BLK_MQ_INSERT_AT_HEAD)
2669 			list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
2670 		else
2671 			list_add_tail(&rq->queuelist,
2672 				      &ctx->rq_lists[hctx->type]);
2673 		blk_mq_hctx_mark_pending(hctx, ctx);
2674 		spin_unlock(&ctx->lock);
2675 	}
2676 }
2677 
2678 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
2679 		unsigned int nr_segs)
2680 {
2681 	int err;
2682 
2683 	if (bio->bi_opf & REQ_RAHEAD)
2684 		rq->cmd_flags |= REQ_FAILFAST_MASK;
2685 
2686 	rq->bio = rq->biotail = bio;
2687 	rq->__sector = bio->bi_iter.bi_sector;
2688 	rq->__data_len = bio->bi_iter.bi_size;
2689 	rq->phys_gap_bit = bio->bi_bvec_gap_bit;
2690 
2691 	rq->nr_phys_segments = nr_segs;
2692 	if (bio_integrity(bio))
2693 		rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q,
2694 								      bio);
2695 
2696 	/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
2697 	err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
2698 	WARN_ON_ONCE(err);
2699 
2700 	blk_account_io_start(rq);
2701 }
2702 
2703 static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
2704 					    struct request *rq, bool last)
2705 {
2706 	struct request_queue *q = rq->q;
2707 	struct blk_mq_queue_data bd = {
2708 		.rq = rq,
2709 		.last = last,
2710 	};
2711 	blk_status_t ret;
2712 
2713 	/*
2714 	 * For OK queue, we are done. For error, caller may kill it.
2715 	 * Any other error (busy), just add it to our list as we
2716 	 * previously would have done.
2717 	 */
2718 	ret = q->mq_ops->queue_rq(hctx, &bd);
2719 	switch (ret) {
2720 	case BLK_STS_OK:
2721 		blk_mq_update_dispatch_busy(hctx, false);
2722 		break;
2723 	case BLK_STS_RESOURCE:
2724 	case BLK_STS_DEV_RESOURCE:
2725 		blk_mq_update_dispatch_busy(hctx, true);
2726 		__blk_mq_requeue_request(rq);
2727 		break;
2728 	default:
2729 		blk_mq_update_dispatch_busy(hctx, false);
2730 		break;
2731 	}
2732 
2733 	return ret;
2734 }
2735 
2736 static bool blk_mq_get_budget_and_tag(struct request *rq)
2737 {
2738 	int budget_token;
2739 
2740 	budget_token = blk_mq_get_dispatch_budget(rq->q);
2741 	if (budget_token < 0)
2742 		return false;
2743 	blk_mq_set_rq_budget_token(rq, budget_token);
2744 	if (!blk_mq_get_driver_tag(rq)) {
2745 		blk_mq_put_dispatch_budget(rq->q, budget_token);
2746 		return false;
2747 	}
2748 	return true;
2749 }
2750 
2751 /**
2752  * blk_mq_try_issue_directly - Try to send a request directly to device driver.
2753  * @hctx: Pointer of the associated hardware queue.
2754  * @rq: Pointer to request to be sent.
2755  *
2756  * If the device has enough resources to accept a new request now, send the
2757  * request directly to device driver. Else, insert at hctx->dispatch queue, so
2758  * we can try send it another time in the future. Requests inserted at this
2759  * queue have higher priority.
2760  */
2761 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2762 		struct request *rq)
2763 {
2764 	blk_status_t ret;
2765 
2766 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2767 		blk_mq_insert_request(rq, 0);
2768 		blk_mq_run_hw_queue(hctx, false);
2769 		return;
2770 	}
2771 
2772 	if ((rq->rq_flags & RQF_USE_SCHED) || !blk_mq_get_budget_and_tag(rq)) {
2773 		blk_mq_insert_request(rq, 0);
2774 		blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
2775 		return;
2776 	}
2777 
2778 	ret = __blk_mq_issue_directly(hctx, rq, true);
2779 	switch (ret) {
2780 	case BLK_STS_OK:
2781 		break;
2782 	case BLK_STS_RESOURCE:
2783 	case BLK_STS_DEV_RESOURCE:
2784 		blk_mq_request_bypass_insert(rq, 0);
2785 		blk_mq_run_hw_queue(hctx, false);
2786 		break;
2787 	default:
2788 		blk_mq_end_request(rq, ret);
2789 		break;
2790 	}
2791 }
2792 
2793 static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2794 {
2795 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2796 
2797 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q)) {
2798 		blk_mq_insert_request(rq, 0);
2799 		blk_mq_run_hw_queue(hctx, false);
2800 		return BLK_STS_OK;
2801 	}
2802 
2803 	if (!blk_mq_get_budget_and_tag(rq))
2804 		return BLK_STS_RESOURCE;
2805 	return __blk_mq_issue_directly(hctx, rq, last);
2806 }
2807 
2808 static void blk_mq_issue_direct(struct rq_list *rqs)
2809 {
2810 	struct blk_mq_hw_ctx *hctx = NULL;
2811 	struct request *rq;
2812 	int queued = 0;
2813 	blk_status_t ret = BLK_STS_OK;
2814 
2815 	while ((rq = rq_list_pop(rqs))) {
2816 		bool last = rq_list_empty(rqs);
2817 
2818 		if (hctx != rq->mq_hctx) {
2819 			if (hctx) {
2820 				blk_mq_commit_rqs(hctx, queued, false);
2821 				queued = 0;
2822 			}
2823 			hctx = rq->mq_hctx;
2824 		}
2825 
2826 		ret = blk_mq_request_issue_directly(rq, last);
2827 		switch (ret) {
2828 		case BLK_STS_OK:
2829 			queued++;
2830 			break;
2831 		case BLK_STS_RESOURCE:
2832 		case BLK_STS_DEV_RESOURCE:
2833 			blk_mq_request_bypass_insert(rq, 0);
2834 			blk_mq_run_hw_queue(hctx, false);
2835 			goto out;
2836 		default:
2837 			blk_mq_end_request(rq, ret);
2838 			break;
2839 		}
2840 	}
2841 
2842 out:
2843 	if (ret != BLK_STS_OK)
2844 		blk_mq_commit_rqs(hctx, queued, false);
2845 }
2846 
2847 static void __blk_mq_flush_list(struct request_queue *q, struct rq_list *rqs)
2848 {
2849 	if (blk_queue_quiesced(q))
2850 		return;
2851 	q->mq_ops->queue_rqs(rqs);
2852 }
2853 
2854 static unsigned blk_mq_extract_queue_requests(struct rq_list *rqs,
2855 					      struct rq_list *queue_rqs)
2856 {
2857 	struct request *rq = rq_list_pop(rqs);
2858 	struct request_queue *this_q = rq->q;
2859 	struct request **prev = &rqs->head;
2860 	struct rq_list matched_rqs = {};
2861 	struct request *last = NULL;
2862 	unsigned depth = 1;
2863 
2864 	rq_list_add_tail(&matched_rqs, rq);
2865 	while ((rq = *prev)) {
2866 		if (rq->q == this_q) {
2867 			/* move rq from rqs to matched_rqs */
2868 			*prev = rq->rq_next;
2869 			rq_list_add_tail(&matched_rqs, rq);
2870 			depth++;
2871 		} else {
2872 			/* leave rq in rqs */
2873 			prev = &rq->rq_next;
2874 			last = rq;
2875 		}
2876 	}
2877 
2878 	rqs->tail = last;
2879 	*queue_rqs = matched_rqs;
2880 	return depth;
2881 }
2882 
2883 static void blk_mq_dispatch_queue_requests(struct rq_list *rqs, unsigned depth)
2884 {
2885 	struct request_queue *q = rq_list_peek(rqs)->q;
2886 
2887 	trace_block_unplug(q, depth, true);
2888 
2889 	/*
2890 	 * Peek first request and see if we have a ->queue_rqs() hook.
2891 	 * If we do, we can dispatch the whole list in one go.
2892 	 * We already know at this point that all requests belong to the
2893 	 * same queue, caller must ensure that's the case.
2894 	 */
2895 	if (q->mq_ops->queue_rqs) {
2896 		blk_mq_run_dispatch_ops(q, __blk_mq_flush_list(q, rqs));
2897 		if (rq_list_empty(rqs))
2898 			return;
2899 	}
2900 
2901 	blk_mq_run_dispatch_ops(q, blk_mq_issue_direct(rqs));
2902 }
2903 
2904 static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched)
2905 {
2906 	struct blk_mq_hw_ctx *this_hctx = NULL;
2907 	struct blk_mq_ctx *this_ctx = NULL;
2908 	struct rq_list requeue_list = {};
2909 	unsigned int depth = 0;
2910 	bool is_passthrough = false;
2911 	LIST_HEAD(list);
2912 
2913 	do {
2914 		struct request *rq = rq_list_pop(rqs);
2915 
2916 		if (!this_hctx) {
2917 			this_hctx = rq->mq_hctx;
2918 			this_ctx = rq->mq_ctx;
2919 			is_passthrough = blk_rq_is_passthrough(rq);
2920 		} else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx ||
2921 			   is_passthrough != blk_rq_is_passthrough(rq)) {
2922 			rq_list_add_tail(&requeue_list, rq);
2923 			continue;
2924 		}
2925 		list_add_tail(&rq->queuelist, &list);
2926 		depth++;
2927 	} while (!rq_list_empty(rqs));
2928 
2929 	*rqs = requeue_list;
2930 	trace_block_unplug(this_hctx->queue, depth, !from_sched);
2931 
2932 	percpu_ref_get(&this_hctx->queue->q_usage_counter);
2933 	/* passthrough requests should never be issued to the I/O scheduler */
2934 	if (is_passthrough) {
2935 		spin_lock(&this_hctx->lock);
2936 		list_splice_tail_init(&list, &this_hctx->dispatch);
2937 		spin_unlock(&this_hctx->lock);
2938 		blk_mq_run_hw_queue(this_hctx, from_sched);
2939 	} else if (this_hctx->queue->elevator) {
2940 		this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
2941 				&list, 0);
2942 		blk_mq_run_hw_queue(this_hctx, from_sched);
2943 	} else {
2944 		blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
2945 	}
2946 	percpu_ref_put(&this_hctx->queue->q_usage_counter);
2947 }
2948 
2949 static void blk_mq_dispatch_multiple_queue_requests(struct rq_list *rqs)
2950 {
2951 	do {
2952 		struct rq_list queue_rqs;
2953 		unsigned depth;
2954 
2955 		depth = blk_mq_extract_queue_requests(rqs, &queue_rqs);
2956 		blk_mq_dispatch_queue_requests(&queue_rqs, depth);
2957 		while (!rq_list_empty(&queue_rqs))
2958 			blk_mq_dispatch_list(&queue_rqs, false);
2959 	} while (!rq_list_empty(rqs));
2960 }
2961 
2962 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2963 {
2964 	unsigned int depth;
2965 
2966 	/*
2967 	 * We may have been called recursively midway through handling
2968 	 * plug->mq_list via a schedule() in the driver's queue_rq() callback.
2969 	 * To avoid mq_list changing under our feet, clear rq_count early and
2970 	 * bail out specifically if rq_count is 0 rather than checking
2971 	 * whether the mq_list is empty.
2972 	 */
2973 	if (plug->rq_count == 0)
2974 		return;
2975 	depth = plug->rq_count;
2976 	plug->rq_count = 0;
2977 
2978 	if (!plug->has_elevator && !from_schedule) {
2979 		if (plug->multiple_queues) {
2980 			blk_mq_dispatch_multiple_queue_requests(&plug->mq_list);
2981 			return;
2982 		}
2983 
2984 		blk_mq_dispatch_queue_requests(&plug->mq_list, depth);
2985 		if (rq_list_empty(&plug->mq_list))
2986 			return;
2987 	}
2988 
2989 	do {
2990 		blk_mq_dispatch_list(&plug->mq_list, from_schedule);
2991 	} while (!rq_list_empty(&plug->mq_list));
2992 }
2993 
2994 static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2995 		struct list_head *list)
2996 {
2997 	int queued = 0;
2998 	blk_status_t ret = BLK_STS_OK;
2999 
3000 	while (!list_empty(list)) {
3001 		struct request *rq = list_first_entry(list, struct request,
3002 				queuelist);
3003 
3004 		list_del_init(&rq->queuelist);
3005 		ret = blk_mq_request_issue_directly(rq, list_empty(list));
3006 		switch (ret) {
3007 		case BLK_STS_OK:
3008 			queued++;
3009 			break;
3010 		case BLK_STS_RESOURCE:
3011 		case BLK_STS_DEV_RESOURCE:
3012 			blk_mq_request_bypass_insert(rq, 0);
3013 			if (list_empty(list))
3014 				blk_mq_run_hw_queue(hctx, false);
3015 			goto out;
3016 		default:
3017 			blk_mq_end_request(rq, ret);
3018 			break;
3019 		}
3020 	}
3021 
3022 out:
3023 	if (ret != BLK_STS_OK)
3024 		blk_mq_commit_rqs(hctx, queued, false);
3025 }
3026 
3027 static bool blk_mq_attempt_bio_merge(struct request_queue *q,
3028 				     struct bio *bio, unsigned int nr_segs)
3029 {
3030 	if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
3031 		if (blk_attempt_plug_merge(q, bio, nr_segs))
3032 			return true;
3033 		if (blk_mq_sched_bio_merge(q, bio, nr_segs))
3034 			return true;
3035 	}
3036 	return false;
3037 }
3038 
3039 static struct request *blk_mq_get_new_requests(struct request_queue *q,
3040 					       struct blk_plug *plug,
3041 					       struct bio *bio)
3042 {
3043 	struct blk_mq_alloc_data data = {
3044 		.q		= q,
3045 		.flags		= 0,
3046 		.shallow_depth	= 0,
3047 		.cmd_flags	= bio->bi_opf,
3048 		.rq_flags	= 0,
3049 		.nr_tags	= 1,
3050 		.cached_rqs	= NULL,
3051 		.ctx		= NULL,
3052 		.hctx		= NULL
3053 	};
3054 	struct request *rq;
3055 
3056 	rq_qos_throttle(q, bio);
3057 
3058 	if (plug) {
3059 		data.nr_tags = plug->nr_ios;
3060 		plug->nr_ios = 1;
3061 		data.cached_rqs = &plug->cached_rqs;
3062 	}
3063 
3064 	rq = __blk_mq_alloc_requests(&data);
3065 	if (unlikely(!rq))
3066 		rq_qos_cleanup(q, bio);
3067 	return rq;
3068 }
3069 
3070 /*
3071  * Check if there is a suitable cached request and return it.
3072  */
3073 static struct request *blk_mq_get_cached_request(struct blk_plug *plug,
3074 		struct request_queue *q, blk_opf_t opf)
3075 {
3076 	enum hctx_type type = blk_mq_get_hctx_type(opf);
3077 	struct request *rq;
3078 
3079 	if (!plug)
3080 		return NULL;
3081 	rq = rq_list_peek(&plug->cached_rqs);
3082 	if (!rq || rq->q != q)
3083 		return NULL;
3084 	if (type != rq->mq_hctx->type &&
3085 	    (type != HCTX_TYPE_READ || rq->mq_hctx->type != HCTX_TYPE_DEFAULT))
3086 		return NULL;
3087 	rq_list_pop(&plug->cached_rqs);
3088 	return rq;
3089 }
3090 
3091 static bool bio_unaligned(const struct bio *bio, struct request_queue *q)
3092 {
3093 	unsigned int bs_mask = queue_logical_block_size(q) - 1;
3094 
3095 	/* .bi_sector of any zero sized bio need to be initialized */
3096 	if ((bio->bi_iter.bi_size & bs_mask) ||
3097 	    ((bio->bi_iter.bi_sector << SECTOR_SHIFT) & bs_mask))
3098 		return true;
3099 	return false;
3100 }
3101 
3102 /**
3103  * blk_mq_submit_bio - Create and send a request to block device.
3104  * @bio: Bio pointer.
3105  *
3106  * Builds up a request structure from @q and @bio and send to the device. The
3107  * request may not be queued directly to hardware if:
3108  * * This request can be merged with another one
3109  * * We want to place request at plug queue for possible future merging
3110  * * There is an IO scheduler active at this queue
3111  *
3112  * It will not queue the request if there is an error with the bio, or at the
3113  * request creation.
3114  */
3115 void blk_mq_submit_bio(struct bio *bio)
3116 {
3117 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
3118 	struct blk_plug *plug = current->plug;
3119 	const int is_sync = op_is_sync(bio->bi_opf);
3120 	unsigned int integrity_action;
3121 	struct blk_mq_hw_ctx *hctx;
3122 	unsigned int nr_segs;
3123 	struct request *rq;
3124 	blk_status_t ret;
3125 
3126 	/*
3127 	 * If the plug has a cached request for this queue, try to use it.
3128 	 */
3129 	rq = blk_mq_get_cached_request(plug, q, bio->bi_opf);
3130 
3131 	/*
3132 	 * A BIO that was released from a zone write plug has already been
3133 	 * through the preparation in this function, already holds a reference
3134 	 * on the queue usage counter, and is the only write BIO in-flight for
3135 	 * the target zone. Go straight to preparing a request for it.
3136 	 */
3137 	if (bio_zone_write_plugging(bio)) {
3138 		nr_segs = bio->__bi_nr_segments;
3139 		if (rq)
3140 			blk_queue_exit(q);
3141 		goto new_request;
3142 	}
3143 
3144 	/*
3145 	 * The cached request already holds a q_usage_counter reference and we
3146 	 * don't have to acquire a new one if we use it.
3147 	 */
3148 	if (!rq) {
3149 		if (unlikely(bio_queue_enter(bio)))
3150 			return;
3151 	}
3152 
3153 	/*
3154 	 * Device reconfiguration may change logical block size or reduce the
3155 	 * number of poll queues, so the checks for alignment and poll support
3156 	 * have to be done with queue usage counter held.
3157 	 */
3158 	if (unlikely(bio_unaligned(bio, q))) {
3159 		bio_io_error(bio);
3160 		goto queue_exit;
3161 	}
3162 
3163 	/*
3164 	 * The block layer cannot generate or verify PI for a dma-buf bio, as
3165 	 * that walks the data pages.  Only fail the bio if it needs that, not
3166 	 * if the submitter attached its own integrity payload or if PI
3167 	 * generation or verification is turned off for the device.
3168 	 */
3169 	if (unlikely(op_is_dmabuf(bio->bi_opf) &&
3170 		     bdev_has_integrity_csum(bio->bi_bdev) &&
3171 		     (bio_integrity_action(bio) & BI_ACT_CHECK))) {
3172 		bio_endio_status(bio, BLK_STS_NOTSUPP);
3173 		goto queue_exit;
3174 	}
3175 
3176 	if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) {
3177 		bio_endio_status(bio, BLK_STS_NOTSUPP);
3178 		goto queue_exit;
3179 	}
3180 
3181 	bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3182 	if (!bio)
3183 		goto queue_exit;
3184 
3185 	integrity_action = bio_integrity_action(bio);
3186 	if (integrity_action)
3187 		bio_integrity_prep(bio, integrity_action);
3188 
3189 	blk_mq_bio_issue_init(q, bio);
3190 	if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
3191 		goto queue_exit;
3192 
3193 	if (bio_needs_zone_write_plugging(bio)) {
3194 		if (blk_zone_plug_bio(bio, nr_segs))
3195 			goto queue_exit;
3196 	}
3197 
3198 new_request:
3199 	if (rq) {
3200 		rq_qos_throttle(rq->q, bio);
3201 		rq->cmd_flags = bio->bi_opf;
3202 		INIT_LIST_HEAD(&rq->queuelist);
3203 		blk_mq_rq_late_init(rq, blk_time_get_ns());
3204 	} else {
3205 		rq = blk_mq_get_new_requests(q, plug, bio);
3206 		if (unlikely(!rq)) {
3207 			if (bio->bi_opf & REQ_NOWAIT)
3208 				bio_wouldblock_error(bio);
3209 			goto queue_exit;
3210 		}
3211 	}
3212 
3213 	trace_block_getrq(bio);
3214 
3215 	rq_qos_track(q, rq, bio);
3216 
3217 	blk_mq_bio_to_request(rq, bio, nr_segs);
3218 
3219 	if (bio_zone_write_plugging(bio))
3220 		blk_zone_write_plug_init_request(rq);
3221 
3222 	ret = blk_crypto_rq_get_keyslot(rq);
3223 	if (ret != BLK_STS_OK) {
3224 		blk_mq_end_request(rq, ret);
3225 		return;
3226 	}
3227 
3228 	if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
3229 		return;
3230 
3231 	if (plug) {
3232 		blk_add_rq_to_plug(plug, rq);
3233 		return;
3234 	}
3235 
3236 	hctx = rq->mq_hctx;
3237 	if ((rq->rq_flags & RQF_USE_SCHED) ||
3238 	    (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
3239 		blk_mq_insert_request(rq, 0);
3240 		blk_mq_run_hw_queue(hctx, true);
3241 	} else {
3242 		blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
3243 	}
3244 	return;
3245 
3246 queue_exit:
3247 	if (!rq)
3248 		blk_queue_exit(q);
3249 	else
3250 		rq_list_add_head(&plug->cached_rqs, rq);
3251 }
3252 
3253 #ifdef CONFIG_BLK_MQ_STACKING
3254 /**
3255  * blk_insert_cloned_request - Helper for stacking drivers to submit a request
3256  * @rq: the request being queued
3257  */
3258 blk_status_t blk_insert_cloned_request(struct request *rq)
3259 {
3260 	struct request_queue *q = rq->q;
3261 	unsigned int max_sectors = blk_queue_get_max_sectors(rq);
3262 	unsigned int max_segments = blk_rq_get_max_segments(rq);
3263 	blk_status_t ret;
3264 
3265 	if (blk_rq_sectors(rq) > max_sectors) {
3266 		/*
3267 		 * SCSI device does not have a good way to return if
3268 		 * Write Same/Zero is actually supported. If a device rejects
3269 		 * a non-read/write command (discard, write same,etc.) the
3270 		 * low-level device driver will set the relevant queue limit to
3271 		 * 0 to prevent blk-lib from issuing more of the offending
3272 		 * operations. Commands queued prior to the queue limit being
3273 		 * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
3274 		 * errors being propagated to upper layers.
3275 		 */
3276 		if (max_sectors == 0)
3277 			return BLK_STS_NOTSUPP;
3278 
3279 		printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
3280 			__func__, blk_rq_sectors(rq), max_sectors);
3281 		return BLK_STS_IOERR;
3282 	}
3283 
3284 	/*
3285 	 * The queue settings related to segment counting may differ from the
3286 	 * original queue.
3287 	 */
3288 	rq->nr_phys_segments = blk_recalc_rq_segments(rq);
3289 	if (rq->nr_phys_segments > max_segments) {
3290 		printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
3291 			__func__, rq->nr_phys_segments, max_segments);
3292 		return BLK_STS_IOERR;
3293 	}
3294 
3295 	/*
3296 	 * Integrity segment counting depends on the same queue limits
3297 	 * (virt_boundary_mask, seg_boundary_mask, max_segment_size) that
3298 	 * vary across stacked queues, so recompute against the bottom
3299 	 * queue just like nr_phys_segments above.
3300 	 */
3301 	if (blk_integrity_rq(rq) && rq->bio) {
3302 		unsigned short max_int_segs = queue_max_integrity_segments(q);
3303 
3304 		rq->nr_integrity_segments =
3305 			blk_rq_count_integrity_sg(rq->q, rq->bio);
3306 		if (rq->nr_integrity_segments > max_int_segs) {
3307 			printk(KERN_ERR "%s: over max integrity segments limit. (%u > %u)\n",
3308 				__func__, rq->nr_integrity_segments,
3309 				max_int_segs);
3310 			return BLK_STS_IOERR;
3311 		}
3312 	}
3313 
3314 	if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
3315 		return BLK_STS_IOERR;
3316 
3317 	ret = blk_crypto_rq_get_keyslot(rq);
3318 	if (ret != BLK_STS_OK)
3319 		return ret;
3320 
3321 	blk_account_io_start(rq);
3322 
3323 	/*
3324 	 * Since we have a scheduler attached on the top device,
3325 	 * bypass a potential scheduler on the bottom device for
3326 	 * insert.
3327 	 */
3328 	blk_mq_run_dispatch_ops(q,
3329 			ret = blk_mq_request_issue_directly(rq, true));
3330 	if (ret)
3331 		blk_account_io_done(rq, blk_time_get_ns());
3332 	return ret;
3333 }
3334 EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
3335 
3336 /**
3337  * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3338  * @rq: the clone request to be cleaned up
3339  *
3340  * Description:
3341  *     Free all bios in @rq for a cloned request.
3342  */
3343 void blk_rq_unprep_clone(struct request *rq)
3344 {
3345 	struct bio *bio;
3346 
3347 	while ((bio = rq->bio) != NULL) {
3348 		rq->bio = bio->bi_next;
3349 
3350 		bio_put(bio);
3351 	}
3352 }
3353 EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3354 
3355 /**
3356  * blk_rq_prep_clone - Helper function to setup clone request
3357  * @rq: the request to be setup
3358  * @rq_src: original request to be cloned
3359  * @bs: bio_set that bios for clone are allocated from
3360  * @gfp_mask: memory allocation mask for bio
3361  * @bio_ctr: setup function to be called for each clone bio.
3362  *           Returns %0 for success, non %0 for failure.
3363  * @data: private data to be passed to @bio_ctr
3364  *
3365  * Description:
3366  *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3367  *     Also, pages which the original bios are pointing to are not copied
3368  *     and the cloned bios just point same pages.
3369  *     So cloned bios must be completed before original bios, which means
3370  *     the caller must complete @rq before @rq_src.
3371  */
3372 int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
3373 		      struct bio_set *bs, gfp_t gfp_mask,
3374 		      int (*bio_ctr)(struct bio *, struct bio *, void *),
3375 		      void *data)
3376 {
3377 	struct bio *bio_src;
3378 
3379 	if (!bs)
3380 		bs = &fs_bio_set;
3381 
3382 	__rq_for_each_bio(bio_src, rq_src) {
3383 		struct bio *bio	 = bio_alloc_clone(rq->q->disk->part0, bio_src,
3384 					gfp_mask, bs);
3385 		if (!bio)
3386 			goto free_and_out;
3387 
3388 		if (bio_ctr && bio_ctr(bio, bio_src, data)) {
3389 			bio_put(bio);
3390 			goto free_and_out;
3391 		}
3392 
3393 		if (rq->bio) {
3394 			rq->biotail->bi_next = bio;
3395 			rq->biotail = bio;
3396 		} else {
3397 			rq->bio = rq->biotail = bio;
3398 		}
3399 	}
3400 
3401 	/* Copy attributes of the original request to the clone request. */
3402 	rq->__sector = blk_rq_pos(rq_src);
3403 	rq->__data_len = blk_rq_bytes(rq_src);
3404 	if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
3405 		rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
3406 		rq->special_vec = rq_src->special_vec;
3407 	}
3408 	rq->nr_phys_segments = rq_src->nr_phys_segments;
3409 	rq->nr_integrity_segments = rq_src->nr_integrity_segments;
3410 	rq->phys_gap_bit = rq_src->phys_gap_bit;
3411 
3412 	if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
3413 		goto free_and_out;
3414 
3415 	return 0;
3416 
3417 free_and_out:
3418 	blk_rq_unprep_clone(rq);
3419 
3420 	return -ENOMEM;
3421 }
3422 EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3423 #endif /* CONFIG_BLK_MQ_STACKING */
3424 
3425 /*
3426  * Steal bios from a request and add them to a bio list.
3427  * The request must not have been partially completed before.
3428  */
3429 void blk_steal_bios(struct bio_list *list, struct request *rq)
3430 {
3431 	struct bio *bio;
3432 
3433 	for (bio = rq->bio; bio; bio = bio->bi_next) {
3434 		if (bio->bi_opf & REQ_POLLED) {
3435 			bio->bi_opf &= ~REQ_POLLED;
3436 			bio->bi_cookie = BLK_QC_T_NONE;
3437 		}
3438 		/*
3439 		 * The alternate request queue that we may end up submitting
3440 		 * the bio to may be frozen temporarily, in this case REQ_NOWAIT
3441 		 * will fail the I/O immediately with EAGAIN to the issuer.
3442 		 * We are not in the issuer context which cannot block. Clear
3443 		 * the flag to avoid spurious EAGAIN I/O failures.
3444 		 */
3445 		bio->bi_opf &= ~REQ_NOWAIT;
3446 		bio_clear_flag(bio, BIO_QOS_THROTTLED);
3447 		bio_clear_flag(bio, BIO_QOS_MERGED);
3448 	}
3449 
3450 	if (rq->bio) {
3451 		if (list->tail)
3452 			list->tail->bi_next = rq->bio;
3453 		else
3454 			list->head = rq->bio;
3455 		list->tail = rq->biotail;
3456 
3457 		rq->bio = NULL;
3458 		rq->biotail = NULL;
3459 	}
3460 
3461 	rq->__data_len = 0;
3462 }
3463 EXPORT_SYMBOL_GPL(blk_steal_bios);
3464 
3465 static size_t order_to_size(unsigned int order)
3466 {
3467 	return (size_t)PAGE_SIZE << order;
3468 }
3469 
3470 /* called before freeing request pool in @tags */
3471 static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
3472 				    struct blk_mq_tags *tags)
3473 {
3474 	struct page *page;
3475 
3476 	/*
3477 	 * There is no need to clear mapping if driver tags is not initialized
3478 	 * or the mapping belongs to the driver tags.
3479 	 */
3480 	if (!drv_tags || drv_tags == tags)
3481 		return;
3482 
3483 	list_for_each_entry(page, &tags->page_list, lru) {
3484 		unsigned long start = (unsigned long)page_address(page);
3485 		unsigned long end = start + order_to_size(page->private);
3486 		int i;
3487 
3488 		for (i = 0; i < drv_tags->nr_tags; i++) {
3489 			struct request *rq = drv_tags->rqs[i];
3490 			unsigned long rq_addr = (unsigned long)rq;
3491 
3492 			if (rq_addr >= start && rq_addr < end) {
3493 				WARN_ON_ONCE(req_ref_read(rq) != 0);
3494 				cmpxchg(&drv_tags->rqs[i], rq, NULL);
3495 			}
3496 		}
3497 	}
3498 }
3499 
3500 void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
3501 		     unsigned int hctx_idx)
3502 {
3503 	struct blk_mq_tags *drv_tags;
3504 
3505 	if (list_empty(&tags->page_list))
3506 		return;
3507 
3508 	if (blk_mq_is_shared_tags(set->flags))
3509 		drv_tags = set->shared_tags;
3510 	else if (hctx_idx < set->nr_hw_queues)
3511 		drv_tags = set->tags[hctx_idx];
3512 	else
3513 		drv_tags = NULL;
3514 
3515 	if (tags->static_rqs && set->ops->exit_request) {
3516 		int i;
3517 
3518 		for (i = 0; i < tags->nr_tags; i++) {
3519 			struct request *rq = tags->static_rqs[i];
3520 
3521 			if (!rq)
3522 				continue;
3523 			set->ops->exit_request(set, rq, hctx_idx);
3524 			tags->static_rqs[i] = NULL;
3525 		}
3526 	}
3527 
3528 	blk_mq_clear_rq_mapping(drv_tags, tags);
3529 	/*
3530 	 * Free request pages in SRCU callback, which is called from
3531 	 * blk_mq_free_tags().
3532 	 */
3533 }
3534 
3535 void blk_mq_free_rq_map(struct blk_mq_tags *tags)
3536 {
3537 	kfree(tags->rqs);
3538 	tags->rqs = NULL;
3539 	kfree(tags->static_rqs);
3540 	tags->static_rqs = NULL;
3541 
3542 	blk_mq_free_tags(tags);
3543 }
3544 
3545 static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
3546 		unsigned int hctx_idx)
3547 {
3548 	int i;
3549 
3550 	for (i = 0; i < set->nr_maps; i++) {
3551 		unsigned int start = set->map[i].queue_offset;
3552 		unsigned int end = start + set->map[i].nr_queues;
3553 
3554 		if (hctx_idx >= start && hctx_idx < end)
3555 			break;
3556 	}
3557 
3558 	if (i >= set->nr_maps)
3559 		i = HCTX_TYPE_DEFAULT;
3560 
3561 	return i;
3562 }
3563 
3564 static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
3565 		unsigned int hctx_idx)
3566 {
3567 	enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
3568 
3569 	return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
3570 }
3571 
3572 static struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
3573 					       unsigned int hctx_idx,
3574 					       unsigned int nr_tags,
3575 					       unsigned int reserved_tags)
3576 {
3577 	int node = blk_mq_get_hctx_node(set, hctx_idx);
3578 	struct blk_mq_tags *tags;
3579 
3580 	if (node == NUMA_NO_NODE)
3581 		node = set->numa_node;
3582 
3583 	tags = blk_mq_init_tags(nr_tags, reserved_tags, set->flags, node);
3584 	if (!tags)
3585 		return NULL;
3586 
3587 	tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3588 				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3589 				 node);
3590 	if (!tags->rqs)
3591 		goto err_free_tags;
3592 
3593 	tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3594 					GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
3595 					node);
3596 	if (!tags->static_rqs)
3597 		goto err_free_rqs;
3598 
3599 	return tags;
3600 
3601 err_free_rqs:
3602 	kfree(tags->rqs);
3603 err_free_tags:
3604 	blk_mq_free_tags(tags);
3605 	return NULL;
3606 }
3607 
3608 static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
3609 			       unsigned int hctx_idx, int node)
3610 {
3611 	int ret;
3612 
3613 	if (set->ops->init_request) {
3614 		ret = set->ops->init_request(set, rq, hctx_idx, node);
3615 		if (ret)
3616 			return ret;
3617 	}
3618 
3619 	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
3620 	return 0;
3621 }
3622 
3623 static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
3624 			    struct blk_mq_tags *tags,
3625 			    unsigned int hctx_idx, unsigned int depth)
3626 {
3627 	unsigned int i, j, entries_per_page, max_order = 4;
3628 	int node = blk_mq_get_hctx_node(set, hctx_idx);
3629 	size_t rq_size, left;
3630 
3631 	if (node == NUMA_NO_NODE)
3632 		node = set->numa_node;
3633 
3634 	/*
3635 	 * rq_size is the size of the request plus driver payload, rounded
3636 	 * to the cacheline size
3637 	 */
3638 	rq_size = round_up(sizeof(struct request) + set->cmd_size,
3639 				cache_line_size());
3640 	left = rq_size * depth;
3641 
3642 	for (i = 0; i < depth; ) {
3643 		int this_order = max_order;
3644 		struct page *page;
3645 		int to_do;
3646 		void *p;
3647 
3648 		while (this_order && left < order_to_size(this_order - 1))
3649 			this_order--;
3650 
3651 		do {
3652 			page = alloc_pages_node(node,
3653 				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
3654 				this_order);
3655 			if (page)
3656 				break;
3657 			if (!this_order--)
3658 				break;
3659 			if (order_to_size(this_order) < rq_size)
3660 				break;
3661 		} while (1);
3662 
3663 		if (!page)
3664 			goto fail;
3665 
3666 		page->private = this_order;
3667 		list_add_tail(&page->lru, &tags->page_list);
3668 
3669 		p = page_address(page);
3670 		/*
3671 		 * Allow kmemleak to scan these pages as they contain pointers
3672 		 * to additional allocations like via ops->init_request().
3673 		 */
3674 		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
3675 		entries_per_page = order_to_size(this_order) / rq_size;
3676 		to_do = min(entries_per_page, depth - i);
3677 		left -= to_do * rq_size;
3678 		for (j = 0; j < to_do; j++) {
3679 			struct request *rq = p;
3680 
3681 			tags->static_rqs[i] = rq;
3682 			if (blk_mq_init_request(set, rq, hctx_idx, node)) {
3683 				tags->static_rqs[i] = NULL;
3684 				goto fail;
3685 			}
3686 
3687 			p += rq_size;
3688 			i++;
3689 		}
3690 	}
3691 	return 0;
3692 
3693 fail:
3694 	blk_mq_free_rqs(set, tags, hctx_idx);
3695 	return -ENOMEM;
3696 }
3697 
3698 struct rq_iter_data {
3699 	struct blk_mq_hw_ctx *hctx;
3700 	bool has_rq;
3701 };
3702 
3703 static bool blk_mq_has_request(struct request *rq, void *data)
3704 {
3705 	struct rq_iter_data *iter_data = data;
3706 
3707 	if (rq->mq_hctx != iter_data->hctx)
3708 		return true;
3709 	iter_data->has_rq = true;
3710 	return false;
3711 }
3712 
3713 static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
3714 {
3715 	struct blk_mq_tags *tags = hctx->sched_tags ?
3716 			hctx->sched_tags : hctx->tags;
3717 	struct rq_iter_data data = {
3718 		.hctx	= hctx,
3719 	};
3720 	int srcu_idx;
3721 
3722 	srcu_idx = srcu_read_lock(&blk_mq_tags_srcu);
3723 	blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
3724 	srcu_read_unlock(&blk_mq_tags_srcu, srcu_idx);
3725 
3726 	return data.has_rq;
3727 }
3728 
3729 static bool blk_mq_hctx_has_online_cpu(struct blk_mq_hw_ctx *hctx,
3730 		unsigned int this_cpu)
3731 {
3732 	enum hctx_type type = hctx->type;
3733 	int cpu;
3734 
3735 	/*
3736 	 * hctx->cpumask has to rule out isolated CPUs, but userspace still
3737 	 * might submit IOs on these isolated CPUs, so use the queue map to
3738 	 * check if all CPUs mapped to this hctx are offline
3739 	 */
3740 	for_each_online_cpu(cpu) {
3741 		struct blk_mq_hw_ctx *h = blk_mq_map_queue_type(hctx->queue,
3742 				type, cpu);
3743 
3744 		if (h != hctx)
3745 			continue;
3746 
3747 		/* this hctx has at least one online CPU */
3748 		if (this_cpu != cpu)
3749 			return true;
3750 	}
3751 
3752 	return false;
3753 }
3754 
3755 static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
3756 {
3757 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3758 			struct blk_mq_hw_ctx, cpuhp_online);
3759 	int ret = 0;
3760 
3761 	if (!hctx->nr_ctx || blk_mq_hctx_has_online_cpu(hctx, cpu))
3762 		return 0;
3763 
3764 	/*
3765 	 * Prevent new request from being allocated on the current hctx.
3766 	 *
3767 	 * The smp_mb__after_atomic() Pairs with the implied barrier in
3768 	 * test_and_set_bit_lock in sbitmap_get().  Ensures the inactive flag is
3769 	 * seen once we return from the tag allocator.
3770 	 */
3771 	set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3772 	smp_mb__after_atomic();
3773 
3774 	/*
3775 	 * Try to grab a reference to the queue and wait for any outstanding
3776 	 * requests.  If we could not grab a reference the queue has been
3777 	 * frozen and there are no requests.
3778 	 */
3779 	if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
3780 		while (blk_mq_hctx_has_requests(hctx)) {
3781 			/*
3782 			 * The wakeup capable IRQ handler of block device is
3783 			 * not called during suspend. Skip the loop by checking
3784 			 * pm_wakeup_pending to prevent the deadlock and improve
3785 			 * suspend latency.
3786 			 */
3787 			if (pm_wakeup_pending()) {
3788 				clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3789 				ret = -EBUSY;
3790 				break;
3791 			}
3792 			msleep(5);
3793 		}
3794 		percpu_ref_put(&hctx->queue->q_usage_counter);
3795 	}
3796 
3797 	return ret;
3798 }
3799 
3800 /*
3801  * Check if one CPU is mapped to the specified hctx
3802  *
3803  * Isolated CPUs have been ruled out from hctx->cpumask, which is supposed
3804  * to be used for scheduling kworker only. For other usage, please call this
3805  * helper for checking if one CPU belongs to the specified hctx
3806  */
3807 static bool blk_mq_cpu_mapped_to_hctx(unsigned int cpu,
3808 		const struct blk_mq_hw_ctx *hctx)
3809 {
3810 	struct blk_mq_hw_ctx *mapped_hctx = blk_mq_map_queue_type(hctx->queue,
3811 			hctx->type, cpu);
3812 
3813 	return mapped_hctx == hctx;
3814 }
3815 
3816 static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
3817 {
3818 	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3819 			struct blk_mq_hw_ctx, cpuhp_online);
3820 
3821 	if (blk_mq_cpu_mapped_to_hctx(cpu, hctx))
3822 		clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3823 	return 0;
3824 }
3825 
3826 /*
3827  * 'cpu' is going away. splice any existing rq_list entries from this
3828  * software queue to the hw queue dispatch list, and ensure that it
3829  * gets run.
3830  */
3831 static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
3832 {
3833 	struct blk_mq_hw_ctx *hctx;
3834 	struct blk_mq_ctx *ctx;
3835 	LIST_HEAD(tmp);
3836 	enum hctx_type type;
3837 
3838 	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
3839 	if (!blk_mq_cpu_mapped_to_hctx(cpu, hctx))
3840 		return 0;
3841 
3842 	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
3843 	type = hctx->type;
3844 
3845 	spin_lock(&ctx->lock);
3846 	if (!list_empty(&ctx->rq_lists[type])) {
3847 		list_splice_init(&ctx->rq_lists[type], &tmp);
3848 		blk_mq_hctx_clear_pending(hctx, ctx);
3849 	}
3850 	spin_unlock(&ctx->lock);
3851 
3852 	if (list_empty(&tmp))
3853 		return 0;
3854 
3855 	spin_lock(&hctx->lock);
3856 	list_splice_tail_init(&tmp, &hctx->dispatch);
3857 	spin_unlock(&hctx->lock);
3858 
3859 	blk_mq_run_hw_queue(hctx, true);
3860 	return 0;
3861 }
3862 
3863 static void __blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3864 {
3865 	lockdep_assert_held(&blk_mq_cpuhp_lock);
3866 
3867 	if (!(hctx->flags & BLK_MQ_F_STACKING) &&
3868 	    !hlist_unhashed(&hctx->cpuhp_online)) {
3869 		cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3870 						    &hctx->cpuhp_online);
3871 		INIT_HLIST_NODE(&hctx->cpuhp_online);
3872 	}
3873 
3874 	if (!hlist_unhashed(&hctx->cpuhp_dead)) {
3875 		cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3876 						    &hctx->cpuhp_dead);
3877 		INIT_HLIST_NODE(&hctx->cpuhp_dead);
3878 	}
3879 }
3880 
3881 static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3882 {
3883 	mutex_lock(&blk_mq_cpuhp_lock);
3884 	__blk_mq_remove_cpuhp(hctx);
3885 	mutex_unlock(&blk_mq_cpuhp_lock);
3886 }
3887 
3888 static void __blk_mq_add_cpuhp(struct blk_mq_hw_ctx *hctx)
3889 {
3890 	lockdep_assert_held(&blk_mq_cpuhp_lock);
3891 
3892 	if (!(hctx->flags & BLK_MQ_F_STACKING) &&
3893 	    hlist_unhashed(&hctx->cpuhp_online))
3894 		cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3895 				&hctx->cpuhp_online);
3896 
3897 	if (hlist_unhashed(&hctx->cpuhp_dead))
3898 		cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3899 				&hctx->cpuhp_dead);
3900 }
3901 
3902 static void __blk_mq_remove_cpuhp_list(struct list_head *head)
3903 {
3904 	struct blk_mq_hw_ctx *hctx;
3905 
3906 	lockdep_assert_held(&blk_mq_cpuhp_lock);
3907 
3908 	list_for_each_entry(hctx, head, hctx_list)
3909 		__blk_mq_remove_cpuhp(hctx);
3910 }
3911 
3912 /*
3913  * Unregister cpuhp callbacks from exited hw queues
3914  *
3915  * Safe to call if this `request_queue` is live
3916  */
3917 static void blk_mq_remove_hw_queues_cpuhp(struct request_queue *q)
3918 {
3919 	LIST_HEAD(hctx_list);
3920 
3921 	spin_lock(&q->unused_hctx_lock);
3922 	list_splice_init(&q->unused_hctx_list, &hctx_list);
3923 	spin_unlock(&q->unused_hctx_lock);
3924 
3925 	mutex_lock(&blk_mq_cpuhp_lock);
3926 	__blk_mq_remove_cpuhp_list(&hctx_list);
3927 	mutex_unlock(&blk_mq_cpuhp_lock);
3928 
3929 	spin_lock(&q->unused_hctx_lock);
3930 	list_splice(&hctx_list, &q->unused_hctx_list);
3931 	spin_unlock(&q->unused_hctx_lock);
3932 }
3933 
3934 /*
3935  * Register cpuhp callbacks from all hw queues
3936  *
3937  * Safe to call if this `request_queue` is live
3938  */
3939 static void blk_mq_add_hw_queues_cpuhp(struct request_queue *q)
3940 {
3941 	struct blk_mq_hw_ctx *hctx;
3942 	unsigned long i;
3943 
3944 	mutex_lock(&blk_mq_cpuhp_lock);
3945 	queue_for_each_hw_ctx(q, hctx, i)
3946 		__blk_mq_add_cpuhp(hctx);
3947 	mutex_unlock(&blk_mq_cpuhp_lock);
3948 }
3949 
3950 /*
3951  * Before freeing hw queue, clearing the flush request reference in
3952  * tags->rqs[] for avoiding potential UAF.
3953  */
3954 static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
3955 		unsigned int queue_depth, struct request *flush_rq)
3956 {
3957 	int i;
3958 
3959 	/* The hw queue may not be mapped yet */
3960 	if (!tags)
3961 		return;
3962 
3963 	WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
3964 
3965 	for (i = 0; i < queue_depth; i++)
3966 		cmpxchg(&tags->rqs[i], flush_rq, NULL);
3967 }
3968 
3969 static void blk_free_flush_queue_callback(struct rcu_head *head)
3970 {
3971 	struct blk_flush_queue *fq =
3972 		container_of(head, struct blk_flush_queue, rcu_head);
3973 
3974 	blk_free_flush_queue(fq);
3975 }
3976 
3977 /* hctx->ctxs will be freed in queue's release handler */
3978 static void blk_mq_exit_hctx(struct request_queue *q,
3979 		struct blk_mq_tag_set *set,
3980 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
3981 {
3982 	struct request *flush_rq = hctx->fq->flush_rq;
3983 
3984 	if (blk_mq_hw_queue_mapped(hctx))
3985 		blk_mq_tag_idle(hctx);
3986 
3987 	if (blk_queue_init_done(q))
3988 		blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
3989 				set->queue_depth, flush_rq);
3990 	if (set->ops->exit_request)
3991 		set->ops->exit_request(set, flush_rq, hctx_idx);
3992 
3993 	if (set->ops->exit_hctx)
3994 		set->ops->exit_hctx(hctx, hctx_idx);
3995 
3996 	call_srcu(&blk_mq_tags_srcu, &hctx->fq->rcu_head,
3997 			blk_free_flush_queue_callback);
3998 	hctx->fq = NULL;
3999 
4000 	spin_lock(&q->unused_hctx_lock);
4001 	list_add(&hctx->hctx_list, &q->unused_hctx_list);
4002 	spin_unlock(&q->unused_hctx_lock);
4003 }
4004 
4005 static void blk_mq_exit_hw_queues(struct request_queue *q,
4006 		struct blk_mq_tag_set *set, int nr_queue)
4007 {
4008 	struct blk_mq_hw_ctx *hctx;
4009 	unsigned long i;
4010 
4011 	queue_for_each_hw_ctx(q, hctx, i) {
4012 		if (i == nr_queue)
4013 			break;
4014 		blk_mq_remove_cpuhp(hctx);
4015 		blk_mq_exit_hctx(q, set, hctx, i);
4016 	}
4017 }
4018 
4019 static int blk_mq_init_hctx(struct request_queue *q,
4020 		struct blk_mq_tag_set *set,
4021 		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
4022 {
4023 	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
4024 
4025 	hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
4026 	if (!hctx->fq)
4027 		goto fail;
4028 
4029 	hctx->queue_num = hctx_idx;
4030 	clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
4031 
4032 	hctx->tags = set->tags[hctx_idx];
4033 
4034 	if (set->ops->init_hctx &&
4035 	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
4036 		goto fail_free_fq;
4037 
4038 	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
4039 				hctx->numa_node))
4040 		goto exit_hctx;
4041 
4042 	return 0;
4043 
4044  exit_hctx:
4045 	if (set->ops->exit_hctx)
4046 		set->ops->exit_hctx(hctx, hctx_idx);
4047  fail_free_fq:
4048 	blk_free_flush_queue(hctx->fq);
4049 	hctx->fq = NULL;
4050  fail:
4051 	return -1;
4052 }
4053 
4054 static struct blk_mq_hw_ctx *
4055 blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
4056 		int node)
4057 {
4058 	struct blk_mq_hw_ctx *hctx;
4059 	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
4060 
4061 	hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
4062 	if (!hctx)
4063 		goto fail_alloc_hctx;
4064 
4065 	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
4066 		goto free_hctx;
4067 
4068 	atomic_set(&hctx->nr_active, 0);
4069 	if (node == NUMA_NO_NODE)
4070 		node = set->numa_node;
4071 	hctx->numa_node = node;
4072 
4073 	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
4074 	spin_lock_init(&hctx->lock);
4075 	INIT_LIST_HEAD(&hctx->dispatch);
4076 	INIT_HLIST_NODE(&hctx->cpuhp_dead);
4077 	INIT_HLIST_NODE(&hctx->cpuhp_online);
4078 	hctx->queue = q;
4079 	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
4080 
4081 	INIT_LIST_HEAD(&hctx->hctx_list);
4082 
4083 	/*
4084 	 * Allocate space for all possible cpus to avoid allocation at
4085 	 * runtime
4086 	 */
4087 	hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
4088 			gfp, node);
4089 	if (!hctx->ctxs)
4090 		goto free_cpumask;
4091 
4092 	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
4093 				gfp, node, false, false))
4094 		goto free_ctxs;
4095 	hctx->nr_ctx = 0;
4096 
4097 	spin_lock_init(&hctx->dispatch_wait_lock);
4098 	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
4099 	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
4100 
4101 	blk_mq_hctx_kobj_init(hctx);
4102 
4103 	return hctx;
4104 
4105  free_ctxs:
4106 	kfree(hctx->ctxs);
4107  free_cpumask:
4108 	free_cpumask_var(hctx->cpumask);
4109  free_hctx:
4110 	kfree(hctx);
4111  fail_alloc_hctx:
4112 	return NULL;
4113 }
4114 
4115 static void blk_mq_init_cpu_queues(struct request_queue *q,
4116 				   unsigned int nr_hw_queues)
4117 {
4118 	struct blk_mq_tag_set *set = q->tag_set;
4119 	unsigned int i, j;
4120 
4121 	for_each_possible_cpu(i) {
4122 		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
4123 		struct blk_mq_hw_ctx *hctx;
4124 		int k;
4125 
4126 		__ctx->cpu = i;
4127 		spin_lock_init(&__ctx->lock);
4128 		for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
4129 			INIT_LIST_HEAD(&__ctx->rq_lists[k]);
4130 
4131 		__ctx->queue = q;
4132 
4133 		/*
4134 		 * Set local node, IFF we have more than one hw queue. If
4135 		 * not, we remain on the home node of the device
4136 		 */
4137 		for (j = 0; j < set->nr_maps; j++) {
4138 			hctx = blk_mq_map_queue_type(q, j, i);
4139 			if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
4140 				hctx->numa_node = cpu_to_node(i);
4141 		}
4142 	}
4143 }
4144 
4145 struct blk_mq_tags *blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
4146 					     unsigned int hctx_idx,
4147 					     unsigned int depth)
4148 {
4149 	struct blk_mq_tags *tags;
4150 	int ret;
4151 
4152 	tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
4153 	if (!tags)
4154 		return NULL;
4155 
4156 	ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
4157 	if (ret) {
4158 		blk_mq_free_rq_map(tags);
4159 		return NULL;
4160 	}
4161 
4162 	return tags;
4163 }
4164 
4165 static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
4166 				       int hctx_idx)
4167 {
4168 	if (blk_mq_is_shared_tags(set->flags)) {
4169 		set->tags[hctx_idx] = set->shared_tags;
4170 
4171 		return true;
4172 	}
4173 
4174 	set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
4175 						       set->queue_depth);
4176 
4177 	return set->tags[hctx_idx];
4178 }
4179 
4180 void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
4181 			     struct blk_mq_tags *tags,
4182 			     unsigned int hctx_idx)
4183 {
4184 	if (tags) {
4185 		blk_mq_free_rqs(set, tags, hctx_idx);
4186 		blk_mq_free_rq_map(tags);
4187 	}
4188 }
4189 
4190 static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
4191 				      unsigned int hctx_idx)
4192 {
4193 	if (!blk_mq_is_shared_tags(set->flags))
4194 		blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
4195 
4196 	set->tags[hctx_idx] = NULL;
4197 }
4198 
4199 static void blk_mq_map_swqueue(struct request_queue *q)
4200 {
4201 	unsigned int j, hctx_idx;
4202 	unsigned long i;
4203 	struct blk_mq_hw_ctx *hctx;
4204 	struct blk_mq_ctx *ctx;
4205 	struct blk_mq_tag_set *set = q->tag_set;
4206 
4207 	queue_for_each_hw_ctx(q, hctx, i) {
4208 		cpumask_clear(hctx->cpumask);
4209 		hctx->nr_ctx = 0;
4210 		hctx->dispatch_from = NULL;
4211 	}
4212 
4213 	/*
4214 	 * Map software to hardware queues.
4215 	 *
4216 	 * If the cpu isn't present, the cpu is mapped to first hctx.
4217 	 */
4218 	for_each_possible_cpu(i) {
4219 
4220 		ctx = per_cpu_ptr(q->queue_ctx, i);
4221 		for (j = 0; j < set->nr_maps; j++) {
4222 			if (!set->map[j].nr_queues) {
4223 				ctx->hctxs[j] = blk_mq_map_queue_type(q,
4224 						HCTX_TYPE_DEFAULT, i);
4225 				continue;
4226 			}
4227 			hctx_idx = set->map[j].mq_map[i];
4228 			/* unmapped hw queue can be remapped after CPU topo changed */
4229 			if (!set->tags[hctx_idx] &&
4230 			    !__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
4231 				/*
4232 				 * If tags initialization fail for some hctx,
4233 				 * that hctx won't be brought online.  In this
4234 				 * case, remap the current ctx to hctx[0] which
4235 				 * is guaranteed to always have tags allocated
4236 				 */
4237 				set->map[j].mq_map[i] = 0;
4238 			}
4239 
4240 			hctx = blk_mq_map_queue_type(q, j, i);
4241 			ctx->hctxs[j] = hctx;
4242 			/*
4243 			 * If the CPU is already set in the mask, then we've
4244 			 * mapped this one already. This can happen if
4245 			 * devices share queues across queue maps.
4246 			 */
4247 			if (cpumask_test_cpu(i, hctx->cpumask))
4248 				continue;
4249 
4250 			cpumask_set_cpu(i, hctx->cpumask);
4251 			hctx->type = j;
4252 			ctx->index_hw[hctx->type] = hctx->nr_ctx;
4253 			hctx->ctxs[hctx->nr_ctx++] = ctx;
4254 
4255 			/*
4256 			 * If the nr_ctx type overflows, we have exceeded the
4257 			 * amount of sw queues we can support.
4258 			 */
4259 			BUG_ON(!hctx->nr_ctx);
4260 		}
4261 
4262 		for (; j < HCTX_MAX_TYPES; j++)
4263 			ctx->hctxs[j] = blk_mq_map_queue_type(q,
4264 					HCTX_TYPE_DEFAULT, i);
4265 	}
4266 
4267 	queue_for_each_hw_ctx(q, hctx, i) {
4268 		int cpu;
4269 
4270 		/*
4271 		 * If no software queues are mapped to this hardware queue,
4272 		 * disable it and free the request entries.
4273 		 */
4274 		if (!hctx->nr_ctx) {
4275 			/* Never unmap queue 0.  We need it as a
4276 			 * fallback in case of a new remap fails
4277 			 * allocation
4278 			 */
4279 			if (i)
4280 				__blk_mq_free_map_and_rqs(set, i);
4281 
4282 			hctx->tags = NULL;
4283 			continue;
4284 		}
4285 
4286 		hctx->tags = set->tags[i];
4287 		WARN_ON(!hctx->tags);
4288 
4289 		/*
4290 		 * Set the map size to the number of mapped software queues.
4291 		 * This is more accurate and more efficient than looping
4292 		 * over all possibly mapped software queues.
4293 		 */
4294 		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
4295 
4296 		/*
4297 		 * Rule out isolated CPUs from hctx->cpumask to avoid
4298 		 * running block kworker on isolated CPUs.
4299 		 * FIXME: cpuset should propagate further changes to isolated CPUs
4300 		 * here.
4301 		 */
4302 		rcu_read_lock();
4303 		for_each_cpu(cpu, hctx->cpumask) {
4304 			if (cpu_is_isolated(cpu))
4305 				cpumask_clear_cpu(cpu, hctx->cpumask);
4306 		}
4307 		rcu_read_unlock();
4308 
4309 		/*
4310 		 * Initialize batch roundrobin counts
4311 		 */
4312 		hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
4313 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
4314 	}
4315 }
4316 
4317 /*
4318  * Caller needs to ensure that we're either frozen/quiesced, or that
4319  * the queue isn't live yet.
4320  */
4321 static void queue_set_hctx_shared(struct request_queue *q, bool shared)
4322 {
4323 	struct blk_mq_hw_ctx *hctx;
4324 	unsigned long i;
4325 
4326 	queue_for_each_hw_ctx(q, hctx, i) {
4327 		if (shared) {
4328 			hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
4329 		} else {
4330 			blk_mq_tag_idle(hctx);
4331 			hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4332 		}
4333 	}
4334 }
4335 
4336 static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
4337 					 bool shared)
4338 {
4339 	struct request_queue *q;
4340 	unsigned int memflags;
4341 
4342 	lockdep_assert_held(&set->tag_list_lock);
4343 
4344 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
4345 		memflags = blk_mq_freeze_queue(q);
4346 		queue_set_hctx_shared(q, shared);
4347 		blk_mq_unfreeze_queue(q, memflags);
4348 	}
4349 }
4350 
4351 static void blk_mq_del_queue_tag_set(struct request_queue *q)
4352 {
4353 	struct blk_mq_tag_set *set = q->tag_set;
4354 
4355 	lockdep_assert_held(&set->tag_list_lock);
4356 
4357 	list_del_rcu(&q->tag_set_list);
4358 	if (list_is_singular(&set->tag_list)) {
4359 		/* just transitioned to unshared */
4360 		set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4361 		/* update existing queue */
4362 		blk_mq_update_tag_set_shared(set, false);
4363 	}
4364 }
4365 
4366 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
4367 				     struct request_queue *q)
4368 {
4369 	unsigned int i;
4370 
4371 	lockdep_assert_held(&set->tag_list_lock);
4372 
4373 	/*
4374 	 * Check to see if we're transitioning to shared (from 1 to 2 queues).
4375 	 */
4376 	if (!list_empty(&set->tag_list) &&
4377 	    !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
4378 		set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
4379 		/* update existing queue */
4380 		blk_mq_update_tag_set_shared(set, true);
4381 	}
4382 	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
4383 		queue_set_hctx_shared(q, true);
4384 
4385 	/*
4386 	 * A queue that joins a quiesced tag set is quiesced like the others,
4387 	 * and blk_mq_unquiesce_tagset() finds what it is going to undo.
4388 	 */
4389 	spin_lock(&set->quiesce_lock);
4390 	list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
4391 	if (!blk_queue_skip_tagset_quiesce(q)) {
4392 		for (i = 0; i < set->quiesce_depth; i++)
4393 			blk_mq_quiesce_queue_nowait(q);
4394 	}
4395 	spin_unlock(&set->quiesce_lock);
4396 }
4397 
4398 /* All allocations will be freed in release handler of q->mq_kobj */
4399 static int blk_mq_alloc_ctxs(struct request_queue *q)
4400 {
4401 	struct blk_mq_ctxs *ctxs;
4402 	int cpu;
4403 
4404 	ctxs = kzalloc_obj(*ctxs);
4405 	if (!ctxs)
4406 		return -ENOMEM;
4407 
4408 	ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
4409 	if (!ctxs->queue_ctx)
4410 		goto fail;
4411 
4412 	for_each_possible_cpu(cpu) {
4413 		struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
4414 		ctx->ctxs = ctxs;
4415 	}
4416 
4417 	q->mq_kobj = &ctxs->kobj;
4418 	q->queue_ctx = ctxs->queue_ctx;
4419 
4420 	return 0;
4421  fail:
4422 	kfree(ctxs);
4423 	return -ENOMEM;
4424 }
4425 
4426 /*
4427  * It is the actual release handler for mq, but we do it from
4428  * request queue's release handler for avoiding use-after-free
4429  * and headache because q->mq_kobj shouldn't have been introduced,
4430  * but we can't group ctx/kctx kobj without it.
4431  */
4432 void blk_mq_release(struct request_queue *q)
4433 {
4434 	struct blk_mq_hw_ctx *hctx, *next;
4435 	unsigned long i;
4436 
4437 	queue_for_each_hw_ctx(q, hctx, i)
4438 		WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
4439 
4440 	/* all hctx are in .unused_hctx_list now */
4441 	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
4442 		list_del_init(&hctx->hctx_list);
4443 		kobject_put(&hctx->kobj);
4444 	}
4445 
4446 	kfree(q->queue_hw_ctx);
4447 
4448 	/*
4449 	 * release .mq_kobj and sw queue's kobject now because
4450 	 * both share lifetime with request queue.
4451 	 */
4452 	blk_mq_sysfs_deinit(q);
4453 }
4454 
4455 struct request_queue *blk_mq_alloc_queue(struct blk_mq_tag_set *set,
4456 		struct queue_limits *lim, void *queuedata)
4457 {
4458 	struct queue_limits default_lim = { };
4459 	struct request_queue *q;
4460 	int ret;
4461 
4462 	if (!lim)
4463 		lim = &default_lim;
4464 	lim->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT;
4465 	if (set->nr_maps > HCTX_TYPE_POLL)
4466 		lim->features |= BLK_FEAT_POLL;
4467 
4468 	q = blk_alloc_queue(lim, set->numa_node);
4469 	if (IS_ERR(q))
4470 		return q;
4471 	q->queuedata = queuedata;
4472 	ret = blk_mq_init_allocated_queue(set, q);
4473 	if (ret) {
4474 		blk_put_queue(q);
4475 		return ERR_PTR(ret);
4476 	}
4477 	return q;
4478 }
4479 EXPORT_SYMBOL(blk_mq_alloc_queue);
4480 
4481 /**
4482  * blk_mq_destroy_queue - shutdown a request queue
4483  * @q: request queue to shutdown
4484  *
4485  * This shuts down a request queue allocated by blk_mq_alloc_queue(). All future
4486  * requests will be failed with -ENODEV. The caller is responsible for dropping
4487  * the reference from blk_mq_alloc_queue() by calling blk_put_queue().
4488  *
4489  * Context: can sleep
4490  */
4491 void blk_mq_destroy_queue(struct request_queue *q)
4492 {
4493 	WARN_ON_ONCE(!queue_is_mq(q));
4494 	WARN_ON_ONCE(blk_queue_registered(q));
4495 
4496 	might_sleep();
4497 
4498 	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
4499 	blk_queue_start_drain(q);
4500 	blk_mq_freeze_queue_wait(q);
4501 
4502 	blk_sync_queue(q);
4503 	blk_mq_cancel_work_sync(q);
4504 	blk_mq_exit_queue(q);
4505 }
4506 EXPORT_SYMBOL(blk_mq_destroy_queue);
4507 
4508 struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,
4509 		struct queue_limits *lim, void *queuedata,
4510 		struct lock_class_key *lkclass)
4511 {
4512 	struct request_queue *q;
4513 	struct gendisk *disk;
4514 
4515 	q = blk_mq_alloc_queue(set, lim, queuedata);
4516 	if (IS_ERR(q))
4517 		return ERR_CAST(q);
4518 
4519 	disk = __alloc_disk_node(q, set->numa_node, lkclass);
4520 	if (!disk) {
4521 		blk_mq_destroy_queue(q);
4522 		blk_put_queue(q);
4523 		return ERR_PTR(-ENOMEM);
4524 	}
4525 	set_bit(GD_OWNS_QUEUE, &disk->state);
4526 	return disk;
4527 }
4528 EXPORT_SYMBOL(__blk_mq_alloc_disk);
4529 
4530 struct gendisk *blk_mq_alloc_disk_for_queue(struct request_queue *q,
4531 		struct lock_class_key *lkclass)
4532 {
4533 	struct gendisk *disk;
4534 
4535 	if (!blk_get_queue(q))
4536 		return NULL;
4537 	disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
4538 	if (!disk)
4539 		blk_put_queue(q);
4540 	return disk;
4541 }
4542 EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
4543 
4544 /*
4545  * Only hctx removed from cpuhp list can be reused
4546  */
4547 static bool blk_mq_hctx_is_reusable(struct blk_mq_hw_ctx *hctx)
4548 {
4549 	return hlist_unhashed(&hctx->cpuhp_online) &&
4550 		hlist_unhashed(&hctx->cpuhp_dead);
4551 }
4552 
4553 static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
4554 		struct blk_mq_tag_set *set, struct request_queue *q,
4555 		int hctx_idx, int node)
4556 {
4557 	struct blk_mq_hw_ctx *hctx = NULL, *tmp;
4558 
4559 	/* reuse dead hctx first */
4560 	spin_lock(&q->unused_hctx_lock);
4561 	list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
4562 		if (tmp->numa_node == node && blk_mq_hctx_is_reusable(tmp)) {
4563 			hctx = tmp;
4564 			break;
4565 		}
4566 	}
4567 	if (hctx)
4568 		list_del_init(&hctx->hctx_list);
4569 	spin_unlock(&q->unused_hctx_lock);
4570 
4571 	if (!hctx)
4572 		hctx = blk_mq_alloc_hctx(q, set, node);
4573 	if (!hctx)
4574 		goto fail;
4575 
4576 	if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
4577 		goto free_hctx;
4578 
4579 	return hctx;
4580 
4581  free_hctx:
4582 	kobject_put(&hctx->kobj);
4583  fail:
4584 	return NULL;
4585 }
4586 
4587 static void __blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4588 				     struct request_queue *q)
4589 {
4590 	int i, j, end;
4591 	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
4592 
4593 	if (q->nr_hw_queues < set->nr_hw_queues) {
4594 		struct blk_mq_hw_ctx **new_hctxs;
4595 
4596 		new_hctxs = kcalloc_node(set->nr_hw_queues,
4597 				       sizeof(*new_hctxs), GFP_KERNEL,
4598 				       set->numa_node);
4599 		if (!new_hctxs)
4600 			return;
4601 		if (hctxs)
4602 			memcpy(new_hctxs, hctxs, q->nr_hw_queues *
4603 			       sizeof(*hctxs));
4604 		rcu_assign_pointer(q->queue_hw_ctx, new_hctxs);
4605 		/*
4606 		 * Make sure reading the old queue_hw_ctx from other
4607 		 * context concurrently won't trigger uaf.
4608 		 */
4609 		kfree_rcu_mightsleep(hctxs);
4610 		hctxs = new_hctxs;
4611 	}
4612 
4613 	for (i = 0; i < set->nr_hw_queues; i++) {
4614 		int old_node;
4615 		int node = blk_mq_get_hctx_node(set, i);
4616 		struct blk_mq_hw_ctx *old_hctx = hctxs[i];
4617 
4618 		if (old_hctx) {
4619 			old_node = old_hctx->numa_node;
4620 			blk_mq_exit_hctx(q, set, old_hctx, i);
4621 		}
4622 
4623 		hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i, node);
4624 		if (!hctxs[i]) {
4625 			if (!old_hctx)
4626 				break;
4627 			pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
4628 					node, old_node);
4629 			hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i,
4630 					old_node);
4631 			WARN_ON_ONCE(!hctxs[i]);
4632 		}
4633 	}
4634 	/*
4635 	 * Increasing nr_hw_queues fails. Free the newly allocated
4636 	 * hctxs and keep the previous q->nr_hw_queues.
4637 	 */
4638 	if (i != set->nr_hw_queues) {
4639 		j = q->nr_hw_queues;
4640 		end = i;
4641 	} else {
4642 		j = i;
4643 		end = q->nr_hw_queues;
4644 		q->nr_hw_queues = set->nr_hw_queues;
4645 	}
4646 
4647 	for (; j < end; j++) {
4648 		struct blk_mq_hw_ctx *hctx = hctxs[j];
4649 
4650 		if (hctx) {
4651 			blk_mq_exit_hctx(q, set, hctx, j);
4652 			hctxs[j] = NULL;
4653 		}
4654 	}
4655 }
4656 
4657 static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4658 				   struct request_queue *q)
4659 {
4660 	__blk_mq_realloc_hw_ctxs(set, q);
4661 
4662 	/* unregister cpuhp callbacks for exited hctxs */
4663 	blk_mq_remove_hw_queues_cpuhp(q);
4664 
4665 	/* register cpuhp for new initialized hctxs */
4666 	blk_mq_add_hw_queues_cpuhp(q);
4667 }
4668 
4669 int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
4670 		struct request_queue *q)
4671 {
4672 	unsigned int memflags;
4673 
4674 	/* mark the queue as mq asap */
4675 	q->mq_ops = set->ops;
4676 
4677 	/*
4678 	 * ->tag_set has to be setup before initialize hctx, which cpuphp
4679 	 * handler needs it for checking queue mapping
4680 	 */
4681 	q->tag_set = set;
4682 
4683 	if (blk_mq_alloc_ctxs(q))
4684 		goto err_exit;
4685 
4686 	/* init q->mq_kobj and sw queues' kobjects */
4687 	blk_mq_sysfs_init(q);
4688 
4689 	INIT_LIST_HEAD(&q->unused_hctx_list);
4690 	spin_lock_init(&q->unused_hctx_lock);
4691 
4692 	/*
4693 	 * Keep blk_mq_update_nr_hw_queues() away until the queue is on
4694 	 * set->tag_list: it changes the number of hardware queues, the tags and
4695 	 * the maps, and blk_mq_map_swqueue() allocates and frees tags of the set.
4696 	 * An update can be part of the error handling of a driver, so it must
4697 	 * not have to wait for somebody who waits for I/O.
4698 	 */
4699 	memflags = memalloc_noio_save();
4700 	mutex_lock(&set->tag_list_lock);
4701 
4702 	blk_mq_realloc_hw_ctxs(set, q);
4703 	if (!q->nr_hw_queues)
4704 		goto err_hctxs;
4705 
4706 	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
4707 	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
4708 
4709 	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
4710 
4711 	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
4712 	INIT_LIST_HEAD(&q->flush_list);
4713 	INIT_LIST_HEAD(&q->requeue_list);
4714 	spin_lock_init(&q->requeue_lock);
4715 
4716 	q->nr_requests = set->queue_depth;
4717 	q->async_depth = set->queue_depth;
4718 
4719 	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
4720 	blk_mq_map_swqueue(q);
4721 	blk_mq_add_queue_tag_set(set, q);
4722 
4723 	mutex_unlock(&set->tag_list_lock);
4724 	memalloc_noio_restore(memflags);
4725 	return 0;
4726 
4727 err_hctxs:
4728 	mutex_unlock(&set->tag_list_lock);
4729 	memalloc_noio_restore(memflags);
4730 	blk_mq_release(q);
4731 err_exit:
4732 	q->mq_ops = NULL;
4733 	return -ENOMEM;
4734 }
4735 EXPORT_SYMBOL(blk_mq_init_allocated_queue);
4736 
4737 /* tags can _not_ be used after returning from blk_mq_exit_queue */
4738 void blk_mq_exit_queue(struct request_queue *q)
4739 {
4740 	struct blk_mq_tag_set *set = q->tag_set;
4741 
4742 	/*
4743 	 * As long as the queue is on set->tag_list, its hctxs belong to
4744 	 * blk_mq_update_nr_hw_queues() as well, which exits and replaces them.
4745 	 */
4746 	mutex_lock(&set->tag_list_lock);
4747 	/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
4748 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
4749 	/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
4750 	blk_mq_del_queue_tag_set(q);
4751 	mutex_unlock(&set->tag_list_lock);
4752 }
4753 
4754 static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
4755 {
4756 	int i;
4757 
4758 	if (blk_mq_is_shared_tags(set->flags)) {
4759 		set->shared_tags = blk_mq_alloc_map_and_rqs(set,
4760 						BLK_MQ_NO_HCTX_IDX,
4761 						set->queue_depth);
4762 		if (!set->shared_tags)
4763 			return -ENOMEM;
4764 	}
4765 
4766 	for (i = 0; i < set->nr_hw_queues; i++) {
4767 		if (!__blk_mq_alloc_map_and_rqs(set, i))
4768 			goto out_unwind;
4769 		cond_resched();
4770 	}
4771 
4772 	return 0;
4773 
4774 out_unwind:
4775 	while (--i >= 0)
4776 		__blk_mq_free_map_and_rqs(set, i);
4777 
4778 	if (blk_mq_is_shared_tags(set->flags)) {
4779 		blk_mq_free_map_and_rqs(set, set->shared_tags,
4780 					BLK_MQ_NO_HCTX_IDX);
4781 	}
4782 
4783 	return -ENOMEM;
4784 }
4785 
4786 /*
4787  * Allocate the request maps associated with this tag_set. Note that this
4788  * may reduce the depth asked for, if memory is tight. set->queue_depth
4789  * will be updated to reflect the allocated depth.
4790  */
4791 static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
4792 {
4793 	unsigned int depth;
4794 	int err;
4795 
4796 	depth = set->queue_depth;
4797 	do {
4798 		err = __blk_mq_alloc_rq_maps(set);
4799 		if (!err)
4800 			break;
4801 
4802 		set->queue_depth >>= 1;
4803 		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
4804 			err = -ENOMEM;
4805 			break;
4806 		}
4807 	} while (set->queue_depth);
4808 
4809 	if (!set->queue_depth || err) {
4810 		pr_err("blk-mq: failed to allocate request map\n");
4811 		return -ENOMEM;
4812 	}
4813 
4814 	if (depth != set->queue_depth)
4815 		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
4816 						depth, set->queue_depth);
4817 
4818 	return 0;
4819 }
4820 
4821 static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
4822 {
4823 	/*
4824 	 * blk_mq_map_queues() and multiple .map_queues() implementations
4825 	 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
4826 	 * number of hardware queues.
4827 	 */
4828 	if (set->nr_maps == 1)
4829 		set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
4830 
4831 	if (set->ops->map_queues) {
4832 		int i;
4833 
4834 		/*
4835 		 * transport .map_queues is usually done in the following
4836 		 * way:
4837 		 *
4838 		 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
4839 		 * 	mask = get_cpu_mask(queue)
4840 		 * 	for_each_cpu(cpu, mask)
4841 		 * 		set->map[x].mq_map[cpu] = queue;
4842 		 * }
4843 		 *
4844 		 * When we need to remap, the table has to be cleared for
4845 		 * killing stale mapping since one CPU may not be mapped
4846 		 * to any hw queue.
4847 		 */
4848 		for (i = 0; i < set->nr_maps; i++)
4849 			blk_mq_clear_mq_map(&set->map[i]);
4850 
4851 		set->ops->map_queues(set);
4852 	} else {
4853 		BUG_ON(set->nr_maps > 1);
4854 		blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
4855 	}
4856 }
4857 
4858 static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags(
4859 				struct blk_mq_tag_set *set,
4860 				int new_nr_hw_queues)
4861 {
4862 	struct blk_mq_tags **new_tags;
4863 	int i;
4864 
4865 	if (set->nr_hw_queues >= new_nr_hw_queues)
4866 		return NULL;
4867 
4868 	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
4869 				GFP_KERNEL, set->numa_node);
4870 	if (!new_tags)
4871 		return ERR_PTR(-ENOMEM);
4872 
4873 	if (set->tags)
4874 		memcpy(new_tags, set->tags, set->nr_hw_queues *
4875 		       sizeof(*set->tags));
4876 
4877 	for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
4878 		if (blk_mq_is_shared_tags(set->flags)) {
4879 			new_tags[i] = set->shared_tags;
4880 		} else {
4881 			new_tags[i] = blk_mq_alloc_map_and_rqs(set, i,
4882 					set->queue_depth);
4883 			if (!new_tags[i])
4884 				goto out_unwind;
4885 		}
4886 		cond_resched();
4887 	}
4888 
4889 	return new_tags;
4890 out_unwind:
4891 	while (--i >= set->nr_hw_queues) {
4892 		if (!blk_mq_is_shared_tags(set->flags))
4893 			blk_mq_free_map_and_rqs(set, new_tags[i], i);
4894 	}
4895 	kfree(new_tags);
4896 	return ERR_PTR(-ENOMEM);
4897 }
4898 
4899 /*
4900  * Alloc a tag set to be associated with one or more request queues.
4901  * May fail with EINVAL for various error conditions. May adjust the
4902  * requested depth down, if it's too large. In that case, the set
4903  * value will be stored in set->queue_depth.
4904  */
4905 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
4906 {
4907 	int i, ret;
4908 
4909 	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
4910 
4911 	if (!set->nr_hw_queues)
4912 		return -EINVAL;
4913 	if (!set->queue_depth)
4914 		return -EINVAL;
4915 	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
4916 		return -EINVAL;
4917 
4918 	if (!set->ops->queue_rq)
4919 		return -EINVAL;
4920 
4921 	if (!set->ops->get_budget ^ !set->ops->put_budget)
4922 		return -EINVAL;
4923 
4924 	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
4925 		pr_info("blk-mq: reduced tag depth to %u\n",
4926 			BLK_MQ_MAX_DEPTH);
4927 		set->queue_depth = BLK_MQ_MAX_DEPTH;
4928 	}
4929 
4930 	if (!set->nr_maps)
4931 		set->nr_maps = 1;
4932 	else if (set->nr_maps > HCTX_MAX_TYPES)
4933 		return -EINVAL;
4934 
4935 	/*
4936 	 * If a crashdump is active, then we are potentially in a very
4937 	 * memory constrained environment. Limit us to  64 tags to prevent
4938 	 * using too much memory.
4939 	 */
4940 	if (is_kdump_kernel())
4941 		set->queue_depth = min(64U, set->queue_depth);
4942 
4943 	/*
4944 	 * There is no use for more h/w queues than cpus if we just have
4945 	 * a single map
4946 	 */
4947 	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
4948 		set->nr_hw_queues = nr_cpu_ids;
4949 
4950 	if (set->flags & BLK_MQ_F_BLOCKING) {
4951 		set->srcu = kmalloc_obj(*set->srcu);
4952 		if (!set->srcu)
4953 			return -ENOMEM;
4954 		ret = init_srcu_struct(set->srcu);
4955 		if (ret)
4956 			goto out_free_srcu;
4957 	}
4958 
4959 	init_rwsem(&set->update_nr_hwq_lock);
4960 
4961 	ret = -ENOMEM;
4962 	set->tags = kcalloc_node(set->nr_hw_queues,
4963 				 sizeof(struct blk_mq_tags *), GFP_KERNEL,
4964 				 set->numa_node);
4965 	if (!set->tags)
4966 		goto out_cleanup_srcu;
4967 
4968 	for (i = 0; i < set->nr_maps; i++) {
4969 		set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
4970 						  sizeof(set->map[i].mq_map[0]),
4971 						  GFP_KERNEL, set->numa_node);
4972 		if (!set->map[i].mq_map)
4973 			goto out_free_mq_map;
4974 		set->map[i].nr_queues = set->nr_hw_queues;
4975 	}
4976 
4977 	blk_mq_update_queue_map(set);
4978 
4979 	ret = blk_mq_alloc_set_map_and_rqs(set);
4980 	if (ret)
4981 		goto out_free_mq_map;
4982 
4983 	mutex_init(&set->tag_list_lock);
4984 	INIT_LIST_HEAD(&set->tag_list);
4985 	spin_lock_init(&set->quiesce_lock);
4986 	set->quiesce_depth = 0;
4987 
4988 	return 0;
4989 
4990 out_free_mq_map:
4991 	for (i = 0; i < set->nr_maps; i++) {
4992 		kfree(set->map[i].mq_map);
4993 		set->map[i].mq_map = NULL;
4994 	}
4995 	kfree(set->tags);
4996 	set->tags = NULL;
4997 out_cleanup_srcu:
4998 	if (set->flags & BLK_MQ_F_BLOCKING)
4999 		cleanup_srcu_struct(set->srcu);
5000 out_free_srcu:
5001 	if (set->flags & BLK_MQ_F_BLOCKING)
5002 		kfree(set->srcu);
5003 	return ret;
5004 }
5005 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
5006 
5007 /* allocate and initialize a tagset for a simple single-queue device */
5008 int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
5009 		const struct blk_mq_ops *ops, unsigned int queue_depth,
5010 		unsigned int set_flags)
5011 {
5012 	memset(set, 0, sizeof(*set));
5013 	set->ops = ops;
5014 	set->nr_hw_queues = 1;
5015 	set->nr_maps = 1;
5016 	set->queue_depth = queue_depth;
5017 	set->numa_node = NUMA_NO_NODE;
5018 	set->flags = set_flags;
5019 	return blk_mq_alloc_tag_set(set);
5020 }
5021 EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
5022 
5023 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
5024 {
5025 	int i, j;
5026 
5027 	for (i = 0; i < set->nr_hw_queues; i++)
5028 		__blk_mq_free_map_and_rqs(set, i);
5029 
5030 	if (blk_mq_is_shared_tags(set->flags)) {
5031 		blk_mq_free_map_and_rqs(set, set->shared_tags,
5032 					BLK_MQ_NO_HCTX_IDX);
5033 	}
5034 
5035 	for (j = 0; j < set->nr_maps; j++) {
5036 		kfree(set->map[j].mq_map);
5037 		set->map[j].mq_map = NULL;
5038 	}
5039 
5040 	kfree(set->tags);
5041 	set->tags = NULL;
5042 
5043 	if (set->flags & BLK_MQ_F_BLOCKING) {
5044 		srcu_barrier(set->srcu);
5045 		cleanup_srcu_struct(set->srcu);
5046 		kfree(set->srcu);
5047 	}
5048 }
5049 EXPORT_SYMBOL(blk_mq_free_tag_set);
5050 
5051 struct elevator_tags *blk_mq_update_nr_requests(struct request_queue *q,
5052 						struct elevator_tags *et,
5053 						unsigned int nr)
5054 {
5055 	struct blk_mq_tag_set *set = q->tag_set;
5056 	struct elevator_tags *old_et = NULL;
5057 	struct blk_mq_hw_ctx *hctx;
5058 	unsigned long i;
5059 
5060 	blk_mq_quiesce_queue(q);
5061 
5062 	if (blk_mq_is_shared_tags(set->flags)) {
5063 		/*
5064 		 * Shared tags, for sched tags, we allocate max initially hence
5065 		 * tags can't grow, see blk_mq_alloc_sched_tags().
5066 		 */
5067 		if (q->elevator)
5068 			blk_mq_tag_update_sched_shared_tags(q, nr);
5069 		else
5070 			blk_mq_tag_resize_shared_tags(set, nr);
5071 	} else if (!q->elevator) {
5072 		/*
5073 		 * Non-shared hardware tags, nr is already checked from
5074 		 * queue_requests_store() and tags can't grow.
5075 		 */
5076 		queue_for_each_hw_ctx(q, hctx, i) {
5077 			if (!hctx->tags)
5078 				continue;
5079 			sbitmap_queue_resize(&hctx->tags->bitmap_tags,
5080 				nr - hctx->tags->nr_reserved_tags);
5081 		}
5082 	} else if (nr <= q->elevator->et->nr_requests) {
5083 		/* Non-shared sched tags, and tags don't grow. */
5084 		queue_for_each_hw_ctx(q, hctx, i) {
5085 			if (!hctx->sched_tags)
5086 				continue;
5087 			sbitmap_queue_resize(&hctx->sched_tags->bitmap_tags,
5088 				nr - hctx->sched_tags->nr_reserved_tags);
5089 		}
5090 	} else {
5091 		/* Non-shared sched tags, and tags grow */
5092 		queue_for_each_hw_ctx(q, hctx, i)
5093 			hctx->sched_tags = et->tags[i];
5094 		old_et =  q->elevator->et;
5095 		q->elevator->et = et;
5096 	}
5097 
5098 	/*
5099 	 * Preserve relative value, both nr and async_depth are at most 16 bit
5100 	 * value, no need to worry about overflow.
5101 	 */
5102 	q->async_depth = max(q->async_depth * nr / q->nr_requests, 1);
5103 	q->nr_requests = nr;
5104 	if (q->elevator && q->elevator->type->ops.depth_updated)
5105 		q->elevator->type->ops.depth_updated(q);
5106 
5107 	blk_mq_unquiesce_queue(q);
5108 	return old_et;
5109 }
5110 
5111 /*
5112  * Switch back to the elevator type stored in the xarray.
5113  */
5114 static void blk_mq_elv_switch_back(struct request_queue *q,
5115 		struct xarray *elv_tbl)
5116 {
5117 	struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id);
5118 
5119 	if (WARN_ON_ONCE(!ctx))
5120 		return;
5121 
5122 	/* The elv_update_nr_hw_queues unfreezes the queue. */
5123 	elv_update_nr_hw_queues(q, ctx);
5124 
5125 	/* Drop the reference acquired in blk_mq_elv_switch_none. */
5126 	if (ctx->type)
5127 		elevator_put(ctx->type);
5128 }
5129 
5130 /*
5131  * Stores elevator name and type in ctx and set current elevator to none.
5132  */
5133 static int blk_mq_elv_switch_none(struct request_queue *q,
5134 		struct xarray *elv_tbl)
5135 {
5136 	struct elv_change_ctx *ctx;
5137 
5138 	lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock);
5139 
5140 	/*
5141 	 * Accessing q->elevator without holding q->elevator_lock is safe here
5142 	 * because we're called from nr_hw_queue update which is protected by
5143 	 * set->update_nr_hwq_lock in the writer context. So, scheduler update/
5144 	 * switch code (which acquires the same lock in the reader context)
5145 	 * can't run concurrently.
5146 	 */
5147 	if (q->elevator) {
5148 		ctx = xa_load(elv_tbl, q->id);
5149 		if (WARN_ON_ONCE(!ctx))
5150 			return -ENOENT;
5151 
5152 		ctx->name = q->elevator->type->elevator_name;
5153 
5154 		/*
5155 		 * Before we switch elevator to 'none', take a reference to
5156 		 * the elevator module so that while nr_hw_queue update is
5157 		 * running, no one can remove elevator module. We'd put the
5158 		 * reference to elevator module later when we switch back
5159 		 * elevator.
5160 		 */
5161 		__elevator_get(q->elevator->type);
5162 
5163 		/*
5164 		 * Store elevator type so that we can release the reference
5165 		 * taken above later.
5166 		 */
5167 		ctx->type = q->elevator->type;
5168 		elevator_set_none(q);
5169 	}
5170 	return 0;
5171 }
5172 
5173 static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
5174 							int nr_hw_queues)
5175 {
5176 	struct request_queue *q;
5177 	int prev_nr_hw_queues = set->nr_hw_queues;
5178 	unsigned int memflags;
5179 	int i;
5180 	struct xarray elv_tbl;
5181 	struct blk_mq_tags **new_tags;
5182 	bool queues_frozen = false;
5183 
5184 	lockdep_assert_held(&set->tag_list_lock);
5185 
5186 	if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
5187 		nr_hw_queues = nr_cpu_ids;
5188 	if (nr_hw_queues < 1)
5189 		return;
5190 	if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
5191 		return;
5192 
5193 	memflags = memalloc_noio_save();
5194 
5195 	xa_init(&elv_tbl);
5196 	if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0)
5197 		goto out_free_ctx;
5198 
5199 	if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0)
5200 		goto out_free_ctx;
5201 
5202 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5203 		blk_mq_debugfs_unregister_hctxs(q);
5204 		blk_mq_sysfs_unregister_hctxs(q);
5205 	}
5206 
5207 	/*
5208 	 * Switch IO scheduler to 'none', cleaning up the data associated
5209 	 * with the previous scheduler. We will switch back once we are done
5210 	 * updating the new sw to hw queue mappings.
5211 	 */
5212 	list_for_each_entry(q, &set->tag_list, tag_set_list)
5213 		if (blk_mq_elv_switch_none(q, &elv_tbl))
5214 			goto switch_back;
5215 
5216 	new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues);
5217 	if (IS_ERR(new_tags))
5218 		goto switch_back;
5219 
5220 	list_for_each_entry(q, &set->tag_list, tag_set_list)
5221 		blk_mq_freeze_queue_nomemsave(q);
5222 	queues_frozen = true;
5223 	if (new_tags) {
5224 		kfree(set->tags);
5225 		set->tags = new_tags;
5226 	}
5227 	set->nr_hw_queues = nr_hw_queues;
5228 
5229 fallback:
5230 	blk_mq_update_queue_map(set);
5231 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5232 		__blk_mq_realloc_hw_ctxs(set, q);
5233 
5234 		if (q->nr_hw_queues != set->nr_hw_queues) {
5235 			int i = prev_nr_hw_queues;
5236 
5237 			pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
5238 					nr_hw_queues, prev_nr_hw_queues);
5239 			for (; i < set->nr_hw_queues; i++)
5240 				__blk_mq_free_map_and_rqs(set, i);
5241 
5242 			set->nr_hw_queues = prev_nr_hw_queues;
5243 			goto fallback;
5244 		}
5245 		blk_mq_map_swqueue(q);
5246 		/* The new hctxs don't know yet that they share their tags */
5247 		if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
5248 			queue_set_hctx_shared(q, true);
5249 	}
5250 switch_back:
5251 	/* The blk_mq_elv_switch_back unfreezes queue for us. */
5252 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5253 		/* switch_back expects queue to be frozen */
5254 		if (!queues_frozen)
5255 			blk_mq_freeze_queue_nomemsave(q);
5256 		blk_mq_elv_switch_back(q, &elv_tbl);
5257 	}
5258 
5259 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5260 		blk_mq_sysfs_register_hctxs(q);
5261 		blk_mq_debugfs_register_hctxs(q);
5262 
5263 		blk_mq_remove_hw_queues_cpuhp(q);
5264 		blk_mq_add_hw_queues_cpuhp(q);
5265 	}
5266 
5267 out_free_ctx:
5268 	blk_mq_free_sched_ctx_batch(&elv_tbl);
5269 	xa_destroy(&elv_tbl);
5270 	memalloc_noio_restore(memflags);
5271 
5272 	/* Free the excess tags when nr_hw_queues shrink. */
5273 	for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
5274 		__blk_mq_free_map_and_rqs(set, i);
5275 }
5276 
5277 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
5278 {
5279 	down_write(&set->update_nr_hwq_lock);
5280 	mutex_lock(&set->tag_list_lock);
5281 	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
5282 	mutex_unlock(&set->tag_list_lock);
5283 	up_write(&set->update_nr_hwq_lock);
5284 }
5285 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
5286 
5287 static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
5288 			 struct io_comp_batch *iob, unsigned int flags)
5289 {
5290 	int ret;
5291 	unsigned long timeout = jiffies + 2;
5292 
5293 	do {
5294 		ret = q->mq_ops->poll(hctx, iob);
5295 		if (ret > 0)
5296 			return ret;
5297 		if (task_sigpending(current))
5298 			return 1;
5299 		if (ret < 0 || (flags & BLK_POLL_ONESHOT))
5300 			break;
5301 		cpu_relax();
5302 	} while (!need_resched() && time_before(jiffies, timeout));
5303 
5304 	return 0;
5305 }
5306 
5307 int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
5308 		struct io_comp_batch *iob, unsigned int flags)
5309 {
5310 	struct blk_mq_hw_ctx *hctx;
5311 
5312 	if (!blk_mq_can_poll(q))
5313 		return 0;
5314 
5315 	/*
5316 	 * The cookie is no more than a hint. iocb_bio_iopoll() may have read it
5317 	 * from a bio that was reused for another queue right afterwards, and
5318 	 * zone write plugging keeps the number of segments in the same field.
5319 	 */
5320 	if (cookie >= q->nr_hw_queues)
5321 		return 0;
5322 	hctx = q->queue_hw_ctx[cookie];
5323 	if (hctx->type != HCTX_TYPE_POLL)
5324 		return 0;
5325 	return blk_hctx_poll(q, hctx, iob, flags);
5326 }
5327 
5328 int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
5329 		unsigned int poll_flags)
5330 {
5331 	struct request_queue *q = rq->q;
5332 	int ret;
5333 
5334 	if (!blk_rq_is_poll(rq))
5335 		return 0;
5336 	if (!percpu_ref_tryget(&q->q_usage_counter))
5337 		return 0;
5338 
5339 	ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
5340 	blk_queue_exit(q);
5341 
5342 	return ret;
5343 }
5344 EXPORT_SYMBOL_GPL(blk_rq_poll);
5345 
5346 unsigned int blk_mq_rq_cpu(struct request *rq)
5347 {
5348 	return rq->mq_ctx->cpu;
5349 }
5350 EXPORT_SYMBOL(blk_mq_rq_cpu);
5351 
5352 void blk_mq_cancel_work_sync(struct request_queue *q)
5353 {
5354 	struct blk_mq_hw_ctx *hctx;
5355 	unsigned long i;
5356 
5357 	cancel_delayed_work_sync(&q->requeue_work);
5358 
5359 	queue_for_each_hw_ctx(q, hctx, i)
5360 		cancel_delayed_work_sync(&hctx->run_work);
5361 }
5362 
5363 static int __init blk_mq_init(void)
5364 {
5365 	int i;
5366 
5367 	for_each_possible_cpu(i)
5368 		init_llist_head(&per_cpu(blk_cpu_done, i));
5369 	for_each_possible_cpu(i)
5370 		INIT_CSD(&per_cpu(blk_cpu_csd, i),
5371 			 __blk_mq_complete_request_remote, NULL);
5372 	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
5373 
5374 	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
5375 				  "block/softirq:dead", NULL,
5376 				  blk_softirq_cpu_dead);
5377 	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
5378 				blk_mq_hctx_notify_dead);
5379 	cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
5380 				blk_mq_hctx_notify_online,
5381 				blk_mq_hctx_notify_offline);
5382 	return 0;
5383 }
5384 subsys_initcall(blk_mq_init);
5385