1 // SPDX-License-Identifier: GPL-2.0
9  * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request
15  * REQ_FUA means that the data must be on non-volatile media on request
28  * The actual execution of flush is double buffered.  Whenever a request
30  * fq->flush_queue[fq->flush_pending_idx].  Once certain criteria are met, a
40  *     double buffering sufficient.
74 #include "blk-mq.h"
75 #include "blk-mq-sched.h"
79 	REQ_FSEQ_PREFLUSH	= (1 << 0), /* pre-flushing in progress */
81 	REQ_FSEQ_POSTFLUSH	= (1 << 2), /* post-flushing in progress */
100 	return blk_mq_map_queue(REQ_OP_FLUSH, ctx)->fq;  in blk_get_flush_queue()
105 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
111 	 * After flush data completion, @rq->bio is %NULL but we need to  in blk_flush_restore_request()
112 	 * complete the bio again.  @rq->biotail is guaranteed to equal the  in blk_flush_restore_request()
113 	 * original @rq->bio.  Restore it.  in blk_flush_restore_request()
115 	rq->bio = rq->biotail;  in blk_flush_restore_request()
116 	if (rq->bio)  in blk_flush_restore_request()
117 		rq->__sector = rq->bio->bi_iter.bi_sector;  in blk_flush_restore_request()
120 	rq->rq_flags &= ~RQF_FLUSH_SEQ;  in blk_flush_restore_request()
121 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
126 	struct block_device *part = rq->q->disk->part0;  in blk_account_io_flush()
131 		      blk_time_get_ns() - rq->start_time_ns);  in blk_account_io_flush()
136  * blk_flush_complete_seq - complete flush sequence
146  * spin_lock_irq(fq->mq_flush_lock)
152 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
153 	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];  in blk_flush_complete_seq()
156 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
157 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
158 	cmd_flags = rq->cmd_flags;  in blk_flush_complete_seq()
170 			fq->flush_pending_since = jiffies;  in blk_flush_complete_seq()
171 		list_add_tail(&rq->queuelist, pending);  in blk_flush_complete_seq()
175 		fq->flush_data_in_flight++;  in blk_flush_complete_seq()
176 		spin_lock(&q->requeue_lock);  in blk_flush_complete_seq()
177 		list_move(&rq->queuelist, &q->requeue_list);  in blk_flush_complete_seq()
178 		spin_unlock(&q->requeue_lock);  in blk_flush_complete_seq()
189 		list_del_init(&rq->queuelist);  in blk_flush_complete_seq()
204 	struct request_queue *q = flush_rq->q;  in flush_end_io()
208 	struct blk_flush_queue *fq = blk_get_flush_queue(flush_rq->mq_ctx);  in flush_end_io()
211 	spin_lock_irqsave(&fq->mq_flush_lock, flags);  in flush_end_io()
214 		fq->rq_status = error;  in flush_end_io()
215 		spin_unlock_irqrestore(&fq->mq_flush_lock, flags);  in flush_end_io()
223 	 * avoiding use-after-free.  in flush_end_io()
225 	WRITE_ONCE(flush_rq->state, MQ_RQ_IDLE);  in flush_end_io()
226 	if (fq->rq_status != BLK_STS_OK) {  in flush_end_io()
227 		error = fq->rq_status;  in flush_end_io()
228 		fq->rq_status = BLK_STS_OK;  in flush_end_io()
231 	if (!q->elevator) {  in flush_end_io()
232 		flush_rq->tag = BLK_MQ_NO_TAG;  in flush_end_io()
235 		flush_rq->internal_tag = BLK_MQ_NO_TAG;  in flush_end_io()
238 	running = &fq->flush_queue[fq->flush_running_idx];  in flush_end_io()
239 	BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);  in flush_end_io()
242 	fq->flush_running_idx ^= 1;  in flush_end_io()
249 		list_del_init(&rq->queuelist);  in flush_end_io()
253 	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);  in flush_end_io()
259 	return rq->end_io == flush_end_io;  in is_flush_rq()
263  * blk_kick_flush - consider issuing flush request
272  * spin_lock_irq(fq->mq_flush_lock)
278 	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];  in blk_kick_flush()
281 	struct request *flush_rq = fq->flush_rq;  in blk_kick_flush()
284 	if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))  in blk_kick_flush()
288 	if (fq->flush_data_in_flight &&  in blk_kick_flush()
290 			fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))  in blk_kick_flush()
297 	fq->flush_pending_idx ^= 1;  in blk_kick_flush()
309 	flush_rq->mq_ctx = first_rq->mq_ctx;  in blk_kick_flush()
310 	flush_rq->mq_hctx = first_rq->mq_hctx;  in blk_kick_flush()
312 	if (!q->elevator)  in blk_kick_flush()
313 		flush_rq->tag = first_rq->tag;  in blk_kick_flush()
315 		flush_rq->internal_tag = first_rq->internal_tag;  in blk_kick_flush()
317 	flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH;  in blk_kick_flush()
318 	flush_rq->cmd_flags |= (flags & REQ_DRV) | (flags & REQ_FAILFAST_MASK);  in blk_kick_flush()
319 	flush_rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_kick_flush()
320 	flush_rq->end_io = flush_end_io;  in blk_kick_flush()
322 	 * Order WRITE ->end_io and WRITE rq->ref, and its pair is the one  in blk_kick_flush()
324 	 * blk_mq_find_and_get_req(), which orders WRITE/READ flush_rq->ref  in blk_kick_flush()
325 	 * and READ flush_rq->end_io  in blk_kick_flush()
330 	spin_lock(&q->requeue_lock);  in blk_kick_flush()
331 	list_add_tail(&flush_rq->queuelist, &q->flush_list);  in blk_kick_flush()
332 	spin_unlock(&q->requeue_lock);  in blk_kick_flush()
340 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
341 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in mq_flush_data_end_io()
342 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
346 	if (q->elevator) {  in mq_flush_data_end_io()
347 		WARN_ON(rq->tag < 0);  in mq_flush_data_end_io()
355 	spin_lock_irqsave(&fq->mq_flush_lock, flags);  in mq_flush_data_end_io()
356 	fq->flush_data_in_flight--;  in mq_flush_data_end_io()
358 	 * May have been corrupted by rq->rq_next reuse, we need to  in mq_flush_data_end_io()
359 	 * re-initialize rq->queuelist before reusing it here.  in mq_flush_data_end_io()
361 	INIT_LIST_HEAD(&rq->queuelist);  in mq_flush_data_end_io()
363 	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);  in mq_flush_data_end_io()
371 	rq->flush.seq = 0;  in blk_rq_init_flush()
372 	rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_rq_init_flush()
373 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_rq_init_flush()
374 	rq->end_io = mq_flush_data_end_io;  in blk_rq_init_flush()
384 	struct request_queue *q = rq->q;  in blk_insert_flush()
385 	struct blk_flush_queue *fq = blk_get_flush_queue(rq->mq_ctx);  in blk_insert_flush()
386 	bool supports_fua = q->limits.features & BLK_FEAT_FUA;  in blk_insert_flush()
390 	WARN_ON_ONCE(rq->bio != rq->biotail);  in blk_insert_flush()
399 		if (rq->cmd_flags & REQ_PREFLUSH)  in blk_insert_flush()
401 		if ((rq->cmd_flags & REQ_FUA) && !supports_fua)  in blk_insert_flush()
409 	rq->cmd_flags &= ~REQ_PREFLUSH;  in blk_insert_flush()
411 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
418 	rq->cmd_flags |= REQ_SYNC;  in blk_insert_flush()
425 		 * advertise a write-back cache.  In this case, simply  in blk_insert_flush()
443 		rq->flush.seq |= REQ_FSEQ_PREFLUSH;  in blk_insert_flush()
444 		spin_lock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
445 		fq->flush_data_in_flight++;  in blk_insert_flush()
446 		spin_unlock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
454 		spin_lock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
456 		spin_unlock_irq(&fq->mq_flush_lock);  in blk_insert_flush()
462  * blkdev_issue_flush - queue a flush
487 	spin_lock_init(&fq->mq_flush_lock);  in blk_alloc_flush_queue()
490 	fq->flush_rq = kzalloc_node(rq_sz, flags, node);  in blk_alloc_flush_queue()
491 	if (!fq->flush_rq)  in blk_alloc_flush_queue()
494 	INIT_LIST_HEAD(&fq->flush_queue[0]);  in blk_alloc_flush_queue()
495 	INIT_LIST_HEAD(&fq->flush_queue[1]);  in blk_alloc_flush_queue()
511 	kfree(fq->flush_rq);  in blk_free_flush_queue()
516  * Allow driver to set its own lock class to fq->mq_flush_lock for
520  * nvme-loop, so lockdep may complain 'possible recursive locking' because
523  * fq->mq_flush_lock for avoiding the lockdep warning.
526  * instance is over-kill, and more worse it introduces horrible boot delay
529  * destroy lots of MQ request_queues for non-existent devices, and some robot
531  * an hour is taken during SCSI MQ probe with per-fq lock class.
536 	lockdep_set_class(&hctx->fq->mq_flush_lock, key);  in blk_mq_hctx_set_fq_lock_class()