blk-mq.c (1aecfe4887713838c79bc52f774609a57db4f988) blk-mq.c (cb96a42cc1f50ba1c7b1e9b2343bec80b926107f)
1/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
7#include <linux/kernel.h>
8#include <linux/module.h>

--- 196 unchanged lines hidden (view full) ---

205 rq->end_io = NULL;
206 rq->end_io_data = NULL;
207 rq->next_rq = NULL;
208
209 ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
210}
211
212static struct request *
1/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
7#include <linux/kernel.h>
8#include <linux/module.h>

--- 196 unchanged lines hidden (view full) ---

205 rq->end_io = NULL;
206 rq->end_io_data = NULL;
207 rq->next_rq = NULL;
208
209 ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
210}
211
212static struct request *
213__blk_mq_alloc_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
214 struct blk_mq_ctx *ctx, int rw, gfp_t gfp, bool reserved)
213__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
215{
216 struct request *rq;
217 unsigned int tag;
218
214{
215 struct request *rq;
216 unsigned int tag;
217
219 tag = blk_mq_get_tag(hctx, &ctx->last_tag, gfp, reserved);
218 tag = blk_mq_get_tag(data);
220 if (tag != BLK_MQ_TAG_FAIL) {
219 if (tag != BLK_MQ_TAG_FAIL) {
221 rq = hctx->tags->rqs[tag];
220 rq = data->hctx->tags->rqs[tag];
222
223 rq->cmd_flags = 0;
221
222 rq->cmd_flags = 0;
224 if (blk_mq_tag_busy(hctx)) {
223 if (blk_mq_tag_busy(data->hctx)) {
225 rq->cmd_flags = REQ_MQ_INFLIGHT;
224 rq->cmd_flags = REQ_MQ_INFLIGHT;
226 atomic_inc(&hctx->nr_active);
225 atomic_inc(&data->hctx->nr_active);
227 }
228
229 rq->tag = tag;
226 }
227
228 rq->tag = tag;
230 blk_mq_rq_ctx_init(q, ctx, rq, rw);
229 blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
231 return rq;
232 }
233
234 return NULL;
235}
236
237struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
238 bool reserved)
239{
240 struct blk_mq_ctx *ctx;
241 struct blk_mq_hw_ctx *hctx;
242 struct request *rq;
230 return rq;
231 }
232
233 return NULL;
234}
235
236struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
237 bool reserved)
238{
239 struct blk_mq_ctx *ctx;
240 struct blk_mq_hw_ctx *hctx;
241 struct request *rq;
242 struct blk_mq_alloc_data alloc_data;
243
244 if (blk_mq_queue_enter(q))
245 return NULL;
246
247 ctx = blk_mq_get_ctx(q);
248 hctx = q->mq_ops->map_queue(q, ctx->cpu);
243
244 if (blk_mq_queue_enter(q))
245 return NULL;
246
247 ctx = blk_mq_get_ctx(q);
248 hctx = q->mq_ops->map_queue(q, ctx->cpu);
249 blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
250 reserved, ctx, hctx);
249
251
250 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, gfp & ~__GFP_WAIT,
251 reserved);
252 rq = __blk_mq_alloc_request(&alloc_data, rw);
252 if (!rq && (gfp & __GFP_WAIT)) {
253 __blk_mq_run_hw_queue(hctx);
254 blk_mq_put_ctx(ctx);
255
256 ctx = blk_mq_get_ctx(q);
257 hctx = q->mq_ops->map_queue(q, ctx->cpu);
253 if (!rq && (gfp & __GFP_WAIT)) {
254 __blk_mq_run_hw_queue(hctx);
255 blk_mq_put_ctx(ctx);
256
257 ctx = blk_mq_get_ctx(q);
258 hctx = q->mq_ops->map_queue(q, ctx->cpu);
258 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, gfp, reserved);
259 blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
260 hctx);
261 rq = __blk_mq_alloc_request(&alloc_data, rw);
262 ctx = alloc_data.ctx;
259 }
260 blk_mq_put_ctx(ctx);
261 return rq;
262}
263EXPORT_SYMBOL(blk_mq_alloc_request);
264
265static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
266 struct blk_mq_ctx *ctx, struct request *rq)

--- 864 unchanged lines hidden (view full) ---

1131static struct request *blk_mq_map_request(struct request_queue *q,
1132 struct bio *bio,
1133 struct blk_map_ctx *data)
1134{
1135 struct blk_mq_hw_ctx *hctx;
1136 struct blk_mq_ctx *ctx;
1137 struct request *rq;
1138 int rw = bio_data_dir(bio);
263 }
264 blk_mq_put_ctx(ctx);
265 return rq;
266}
267EXPORT_SYMBOL(blk_mq_alloc_request);
268
269static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
270 struct blk_mq_ctx *ctx, struct request *rq)

--- 864 unchanged lines hidden (view full) ---

1135static struct request *blk_mq_map_request(struct request_queue *q,
1136 struct bio *bio,
1137 struct blk_map_ctx *data)
1138{
1139 struct blk_mq_hw_ctx *hctx;
1140 struct blk_mq_ctx *ctx;
1141 struct request *rq;
1142 int rw = bio_data_dir(bio);
1143 struct blk_mq_alloc_data alloc_data;
1139
1140 if (unlikely(blk_mq_queue_enter(q))) {
1141 bio_endio(bio, -EIO);
1142 return NULL;
1143 }
1144
1145 ctx = blk_mq_get_ctx(q);
1146 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1147
1148 if (rw_is_sync(bio->bi_rw))
1149 rw |= REQ_SYNC;
1150
1151 trace_block_getrq(q, bio, rw);
1144
1145 if (unlikely(blk_mq_queue_enter(q))) {
1146 bio_endio(bio, -EIO);
1147 return NULL;
1148 }
1149
1150 ctx = blk_mq_get_ctx(q);
1151 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1152
1153 if (rw_is_sync(bio->bi_rw))
1154 rw |= REQ_SYNC;
1155
1156 trace_block_getrq(q, bio, rw);
1152 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, GFP_ATOMIC, false);
1157 blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
1158 hctx);
1159 rq = __blk_mq_alloc_request(&alloc_data, rw);
1153 if (unlikely(!rq)) {
1154 __blk_mq_run_hw_queue(hctx);
1155 blk_mq_put_ctx(ctx);
1156 trace_block_sleeprq(q, bio, rw);
1157
1158 ctx = blk_mq_get_ctx(q);
1159 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1160 if (unlikely(!rq)) {
1161 __blk_mq_run_hw_queue(hctx);
1162 blk_mq_put_ctx(ctx);
1163 trace_block_sleeprq(q, bio, rw);
1164
1165 ctx = blk_mq_get_ctx(q);
1166 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1160 rq = __blk_mq_alloc_request(q, hctx, ctx, rw,
1161 __GFP_WAIT|GFP_ATOMIC, false);
1167 blk_mq_set_alloc_data(&alloc_data, q,
1168 __GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
1169 rq = __blk_mq_alloc_request(&alloc_data, rw);
1170 ctx = alloc_data.ctx;
1171 hctx = alloc_data.hctx;
1162 }
1163
1164 hctx->queued++;
1165 data->hctx = hctx;
1166 data->ctx = ctx;
1167 return rq;
1168}
1169

--- 876 unchanged lines hidden --- 		1172 }
1173
1174 hctx->queued++;
1175 data->hctx = hctx;
1176 data->ctx = ctx;
1177 return rq;
1178}
1179

--- 876 unchanged lines hidden ---