blk-mq.c (01e99aeca3979600302913cef3f89076786f32c8) blk-mq.c (cae740a04b4d6d5166f19ee5faf04ea2a1f34b3d)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
8#include <linux/kernel.h>

--- 463 unchanged lines hidden (view full) ---

472 struct request_queue *q = rq->q;
473 struct blk_mq_ctx *ctx = rq->mq_ctx;
474 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
475 const int sched_tag = rq->internal_tag;
476
477 blk_pm_mark_last_busy(rq);
478 rq->mq_hctx = NULL;
479 if (rq->tag != -1)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Block multiqueue core code
4 *
5 * Copyright (C) 2013-2014 Jens Axboe
6 * Copyright (C) 2013-2014 Christoph Hellwig
7 */
8#include <linux/kernel.h>

--- 463 unchanged lines hidden (view full) ---

472 struct request_queue *q = rq->q;
473 struct blk_mq_ctx *ctx = rq->mq_ctx;
474 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
475 const int sched_tag = rq->internal_tag;
476
477 blk_pm_mark_last_busy(rq);
478 rq->mq_hctx = NULL;
479 if (rq->tag != -1)
480 blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
480 blk_mq_put_tag(hctx->tags, ctx, rq->tag);
481 if (sched_tag != -1)
481 if (sched_tag != -1)
482 blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
482 blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
483 blk_mq_sched_restart(hctx);
484 blk_queue_exit(q);
485}
486
487void blk_mq_free_request(struct request *rq)
488{
489 struct request_queue *q = rq->q;
490 struct elevator_queue *e = q->elevator;

--- 2906 unchanged lines hidden (view full) ---

3397
3398 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
3399 if (cb->stat[bucket].nr_samples)
3400 q->poll_stat[bucket] = cb->stat[bucket];
3401 }
3402}
3403
3404static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
483 blk_mq_sched_restart(hctx);
484 blk_queue_exit(q);
485}
486
487void blk_mq_free_request(struct request *rq)
488{
489 struct request_queue *q = rq->q;
490 struct elevator_queue *e = q->elevator;

--- 2906 unchanged lines hidden (view full) ---

3397
3398 for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
3399 if (cb->stat[bucket].nr_samples)
3400 q->poll_stat[bucket] = cb->stat[bucket];
3401 }
3402}
3403
3404static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
3405 struct blk_mq_hw_ctx *hctx,
3406 struct request *rq)
3407{
3408 unsigned long ret = 0;
3409 int bucket;
3410
3411 /*
3412 * If stats collection isn't on, don't sleep but turn it on for
3413 * future users

--- 16 unchanged lines hidden (view full) ---

3430
3431 if (q->poll_stat[bucket].nr_samples)
3432 ret = (q->poll_stat[bucket].mean + 1) / 2;
3433
3434 return ret;
3435}
3436
3437static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
3405 struct request *rq)
3406{
3407 unsigned long ret = 0;
3408 int bucket;
3409
3410 /*
3411 * If stats collection isn't on, don't sleep but turn it on for
3412 * future users

--- 16 unchanged lines hidden (view full) ---

3429
3430 if (q->poll_stat[bucket].nr_samples)
3431 ret = (q->poll_stat[bucket].mean + 1) / 2;
3432
3433 return ret;
3434}
3435
3436static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
3438 struct blk_mq_hw_ctx *hctx,
3439 struct request *rq)
3440{
3441 struct hrtimer_sleeper hs;
3442 enum hrtimer_mode mode;
3443 unsigned int nsecs;
3444 ktime_t kt;
3445
3446 if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
3447 return false;
3448
3449 /*
3450 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
3451 *
3452 * 0: use half of prev avg
3453 * >0: use this specific value
3454 */
3455 if (q->poll_nsec > 0)
3456 nsecs = q->poll_nsec;
3457 else
3437 struct request *rq)
3438{
3439 struct hrtimer_sleeper hs;
3440 enum hrtimer_mode mode;
3441 unsigned int nsecs;
3442 ktime_t kt;
3443
3444 if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
3445 return false;
3446
3447 /*
3448 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
3449 *
3450 * 0: use half of prev avg
3451 * >0: use this specific value
3452 */
3453 if (q->poll_nsec > 0)
3454 nsecs = q->poll_nsec;
3455 else
3458 nsecs = blk_mq_poll_nsecs(q, hctx, rq);
3456 nsecs = blk_mq_poll_nsecs(q, rq);
3459
3460 if (!nsecs)
3461 return false;
3462
3463 rq->rq_flags |= RQF_MQ_POLL_SLEPT;
3464
3465 /*
3466 * This will be replaced with the stats tracking code, using

--- 38 unchanged lines hidden (view full) ---

3505 * get a NULL return here, as we clear the sched tag when
3506 * that happens. The request still remains valid, like always,
3507 * so we should be safe with just the NULL check.
3508 */
3509 if (!rq)
3510 return false;
3511 }
3512
3457
3458 if (!nsecs)
3459 return false;
3460
3461 rq->rq_flags |= RQF_MQ_POLL_SLEPT;
3462
3463 /*
3464 * This will be replaced with the stats tracking code, using

--- 38 unchanged lines hidden (view full) ---

3503 * get a NULL return here, as we clear the sched tag when
3504 * that happens. The request still remains valid, like always,
3505 * so we should be safe with just the NULL check.
3506 */
3507 if (!rq)
3508 return false;
3509 }
3510
3513 return blk_mq_poll_hybrid_sleep(q, hctx, rq);
3511 return blk_mq_poll_hybrid_sleep(q, rq);
3514}
3515
3516/**
3517 * blk_poll - poll for IO completions
3518 * @q: the queue
3519 * @cookie: cookie passed back at IO submission time
3520 * @spin: whether to spin for completions
3521 *

--- 73 unchanged lines hidden --- 		3512}
3513
3514/**
3515 * blk_poll - poll for IO completions
3516 * @q: the queue
3517 * @cookie: cookie passed back at IO submission time
3518 * @spin: whether to spin for completions
3519 *

--- 73 unchanged lines hidden ---