xref: /linux/drivers/nvme/host/rdma.c (revision bcb63314e2c23f1ed622418b65f9409512659c73)
1 /*
2  * NVMe over Fabrics RDMA host code.
3  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  */
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/module.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/err.h>
19 #include <linux/string.h>
20 #include <linux/atomic.h>
21 #include <linux/blk-mq.h>
22 #include <linux/types.h>
23 #include <linux/list.h>
24 #include <linux/mutex.h>
25 #include <linux/scatterlist.h>
26 #include <linux/nvme.h>
27 #include <asm/unaligned.h>
28 
29 #include <rdma/ib_verbs.h>
30 #include <rdma/rdma_cm.h>
31 #include <linux/nvme-rdma.h>
32 
33 #include "nvme.h"
34 #include "fabrics.h"
35 
36 
37 #define NVME_RDMA_CONNECT_TIMEOUT_MS	1000		/* 1 second */
38 
39 #define NVME_RDMA_MAX_SEGMENT_SIZE	0xffffff	/* 24-bit SGL field */
40 
41 #define NVME_RDMA_MAX_SEGMENTS		256
42 
43 #define NVME_RDMA_MAX_INLINE_SEGMENTS	1
44 
45 static const char *const nvme_rdma_cm_status_strs[] = {
46 	[NVME_RDMA_CM_INVALID_LEN]	= "invalid length",
47 	[NVME_RDMA_CM_INVALID_RECFMT]	= "invalid record format",
48 	[NVME_RDMA_CM_INVALID_QID]	= "invalid queue ID",
49 	[NVME_RDMA_CM_INVALID_HSQSIZE]	= "invalid host SQ size",
50 	[NVME_RDMA_CM_INVALID_HRQSIZE]	= "invalid host RQ size",
51 	[NVME_RDMA_CM_NO_RSC]		= "resource not found",
52 	[NVME_RDMA_CM_INVALID_IRD]	= "invalid IRD",
53 	[NVME_RDMA_CM_INVALID_ORD]	= "Invalid ORD",
54 };
55 
56 static const char *nvme_rdma_cm_msg(enum nvme_rdma_cm_status status)
57 {
58 	size_t index = status;
59 
60 	if (index < ARRAY_SIZE(nvme_rdma_cm_status_strs) &&
61 	    nvme_rdma_cm_status_strs[index])
62 		return nvme_rdma_cm_status_strs[index];
63 	else
64 		return "unrecognized reason";
65 };
66 
67 /*
68  * We handle AEN commands ourselves and don't even let the
69  * block layer know about them.
70  */
71 #define NVME_RDMA_NR_AEN_COMMANDS      1
72 #define NVME_RDMA_AQ_BLKMQ_DEPTH       \
73 	(NVMF_AQ_DEPTH - NVME_RDMA_NR_AEN_COMMANDS)
74 
75 struct nvme_rdma_device {
76 	struct ib_device       *dev;
77 	struct ib_pd	       *pd;
78 	struct kref		ref;
79 	struct list_head	entry;
80 };
81 
82 struct nvme_rdma_qe {
83 	struct ib_cqe		cqe;
84 	void			*data;
85 	u64			dma;
86 };
87 
88 struct nvme_rdma_queue;
89 struct nvme_rdma_request {
90 	struct nvme_request	req;
91 	struct ib_mr		*mr;
92 	struct nvme_rdma_qe	sqe;
93 	struct ib_sge		sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
94 	u32			num_sge;
95 	int			nents;
96 	bool			inline_data;
97 	struct ib_reg_wr	reg_wr;
98 	struct ib_cqe		reg_cqe;
99 	struct nvme_rdma_queue  *queue;
100 	struct sg_table		sg_table;
101 	struct scatterlist	first_sgl[];
102 };
103 
104 enum nvme_rdma_queue_flags {
105 	NVME_RDMA_Q_CONNECTED = (1 << 0),
106 	NVME_RDMA_IB_QUEUE_ALLOCATED = (1 << 1),
107 	NVME_RDMA_Q_DELETING = (1 << 2),
108 	NVME_RDMA_Q_LIVE = (1 << 3),
109 };
110 
111 struct nvme_rdma_queue {
112 	struct nvme_rdma_qe	*rsp_ring;
113 	u8			sig_count;
114 	int			queue_size;
115 	size_t			cmnd_capsule_len;
116 	struct nvme_rdma_ctrl	*ctrl;
117 	struct nvme_rdma_device	*device;
118 	struct ib_cq		*ib_cq;
119 	struct ib_qp		*qp;
120 
121 	unsigned long		flags;
122 	struct rdma_cm_id	*cm_id;
123 	int			cm_error;
124 	struct completion	cm_done;
125 };
126 
127 struct nvme_rdma_ctrl {
128 	/* read and written in the hot path */
129 	spinlock_t		lock;
130 
131 	/* read only in the hot path */
132 	struct nvme_rdma_queue	*queues;
133 	u32			queue_count;
134 
135 	/* other member variables */
136 	struct blk_mq_tag_set	tag_set;
137 	struct work_struct	delete_work;
138 	struct work_struct	reset_work;
139 	struct work_struct	err_work;
140 
141 	struct nvme_rdma_qe	async_event_sqe;
142 
143 	int			reconnect_delay;
144 	struct delayed_work	reconnect_work;
145 
146 	struct list_head	list;
147 
148 	struct blk_mq_tag_set	admin_tag_set;
149 	struct nvme_rdma_device	*device;
150 
151 	u64			cap;
152 	u32			max_fr_pages;
153 
154 	union {
155 		struct sockaddr addr;
156 		struct sockaddr_in addr_in;
157 	};
158 
159 	struct nvme_ctrl	ctrl;
160 };
161 
162 static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
163 {
164 	return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
165 }
166 
167 static LIST_HEAD(device_list);
168 static DEFINE_MUTEX(device_list_mutex);
169 
170 static LIST_HEAD(nvme_rdma_ctrl_list);
171 static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
172 
173 static struct workqueue_struct *nvme_rdma_wq;
174 
175 /*
176  * Disabling this option makes small I/O goes faster, but is fundamentally
177  * unsafe.  With it turned off we will have to register a global rkey that
178  * allows read and write access to all physical memory.
179  */
180 static bool register_always = true;
181 module_param(register_always, bool, 0444);
182 MODULE_PARM_DESC(register_always,
183 	 "Use memory registration even for contiguous memory regions");
184 
185 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
186 		struct rdma_cm_event *event);
187 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
188 
189 /* XXX: really should move to a generic header sooner or later.. */
190 static inline void put_unaligned_le24(u32 val, u8 *p)
191 {
192 	*p++ = val;
193 	*p++ = val >> 8;
194 	*p++ = val >> 16;
195 }
196 
197 static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
198 {
199 	return queue - queue->ctrl->queues;
200 }
201 
202 static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
203 {
204 	return queue->cmnd_capsule_len - sizeof(struct nvme_command);
205 }
206 
207 static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
208 		size_t capsule_size, enum dma_data_direction dir)
209 {
210 	ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
211 	kfree(qe->data);
212 }
213 
214 static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
215 		size_t capsule_size, enum dma_data_direction dir)
216 {
217 	qe->data = kzalloc(capsule_size, GFP_KERNEL);
218 	if (!qe->data)
219 		return -ENOMEM;
220 
221 	qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
222 	if (ib_dma_mapping_error(ibdev, qe->dma)) {
223 		kfree(qe->data);
224 		return -ENOMEM;
225 	}
226 
227 	return 0;
228 }
229 
230 static void nvme_rdma_free_ring(struct ib_device *ibdev,
231 		struct nvme_rdma_qe *ring, size_t ib_queue_size,
232 		size_t capsule_size, enum dma_data_direction dir)
233 {
234 	int i;
235 
236 	for (i = 0; i < ib_queue_size; i++)
237 		nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
238 	kfree(ring);
239 }
240 
241 static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
242 		size_t ib_queue_size, size_t capsule_size,
243 		enum dma_data_direction dir)
244 {
245 	struct nvme_rdma_qe *ring;
246 	int i;
247 
248 	ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
249 	if (!ring)
250 		return NULL;
251 
252 	for (i = 0; i < ib_queue_size; i++) {
253 		if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
254 			goto out_free_ring;
255 	}
256 
257 	return ring;
258 
259 out_free_ring:
260 	nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
261 	return NULL;
262 }
263 
264 static void nvme_rdma_qp_event(struct ib_event *event, void *context)
265 {
266 	pr_debug("QP event %s (%d)\n",
267 		 ib_event_msg(event->event), event->event);
268 
269 }
270 
271 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
272 {
273 	wait_for_completion_interruptible_timeout(&queue->cm_done,
274 			msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
275 	return queue->cm_error;
276 }
277 
278 static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
279 {
280 	struct nvme_rdma_device *dev = queue->device;
281 	struct ib_qp_init_attr init_attr;
282 	int ret;
283 
284 	memset(&init_attr, 0, sizeof(init_attr));
285 	init_attr.event_handler = nvme_rdma_qp_event;
286 	/* +1 for drain */
287 	init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
288 	/* +1 for drain */
289 	init_attr.cap.max_recv_wr = queue->queue_size + 1;
290 	init_attr.cap.max_recv_sge = 1;
291 	init_attr.cap.max_send_sge = 1 + NVME_RDMA_MAX_INLINE_SEGMENTS;
292 	init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
293 	init_attr.qp_type = IB_QPT_RC;
294 	init_attr.send_cq = queue->ib_cq;
295 	init_attr.recv_cq = queue->ib_cq;
296 
297 	ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
298 
299 	queue->qp = queue->cm_id->qp;
300 	return ret;
301 }
302 
303 static int nvme_rdma_reinit_request(void *data, struct request *rq)
304 {
305 	struct nvme_rdma_ctrl *ctrl = data;
306 	struct nvme_rdma_device *dev = ctrl->device;
307 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
308 	int ret = 0;
309 
310 	if (!req->mr->need_inval)
311 		goto out;
312 
313 	ib_dereg_mr(req->mr);
314 
315 	req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
316 			ctrl->max_fr_pages);
317 	if (IS_ERR(req->mr)) {
318 		ret = PTR_ERR(req->mr);
319 		req->mr = NULL;
320 		goto out;
321 	}
322 
323 	req->mr->need_inval = false;
324 
325 out:
326 	return ret;
327 }
328 
329 static void __nvme_rdma_exit_request(struct nvme_rdma_ctrl *ctrl,
330 		struct request *rq, unsigned int queue_idx)
331 {
332 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
333 	struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
334 	struct nvme_rdma_device *dev = queue->device;
335 
336 	if (req->mr)
337 		ib_dereg_mr(req->mr);
338 
339 	nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
340 			DMA_TO_DEVICE);
341 }
342 
343 static void nvme_rdma_exit_request(void *data, struct request *rq,
344 				unsigned int hctx_idx, unsigned int rq_idx)
345 {
346 	return __nvme_rdma_exit_request(data, rq, hctx_idx + 1);
347 }
348 
349 static void nvme_rdma_exit_admin_request(void *data, struct request *rq,
350 				unsigned int hctx_idx, unsigned int rq_idx)
351 {
352 	return __nvme_rdma_exit_request(data, rq, 0);
353 }
354 
355 static int __nvme_rdma_init_request(struct nvme_rdma_ctrl *ctrl,
356 		struct request *rq, unsigned int queue_idx)
357 {
358 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
359 	struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
360 	struct nvme_rdma_device *dev = queue->device;
361 	struct ib_device *ibdev = dev->dev;
362 	int ret;
363 
364 	BUG_ON(queue_idx >= ctrl->queue_count);
365 
366 	ret = nvme_rdma_alloc_qe(ibdev, &req->sqe, sizeof(struct nvme_command),
367 			DMA_TO_DEVICE);
368 	if (ret)
369 		return ret;
370 
371 	req->mr = ib_alloc_mr(dev->pd, IB_MR_TYPE_MEM_REG,
372 			ctrl->max_fr_pages);
373 	if (IS_ERR(req->mr)) {
374 		ret = PTR_ERR(req->mr);
375 		goto out_free_qe;
376 	}
377 
378 	req->queue = queue;
379 
380 	return 0;
381 
382 out_free_qe:
383 	nvme_rdma_free_qe(dev->dev, &req->sqe, sizeof(struct nvme_command),
384 			DMA_TO_DEVICE);
385 	return -ENOMEM;
386 }
387 
388 static int nvme_rdma_init_request(void *data, struct request *rq,
389 				unsigned int hctx_idx, unsigned int rq_idx,
390 				unsigned int numa_node)
391 {
392 	return __nvme_rdma_init_request(data, rq, hctx_idx + 1);
393 }
394 
395 static int nvme_rdma_init_admin_request(void *data, struct request *rq,
396 				unsigned int hctx_idx, unsigned int rq_idx,
397 				unsigned int numa_node)
398 {
399 	return __nvme_rdma_init_request(data, rq, 0);
400 }
401 
402 static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
403 		unsigned int hctx_idx)
404 {
405 	struct nvme_rdma_ctrl *ctrl = data;
406 	struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
407 
408 	BUG_ON(hctx_idx >= ctrl->queue_count);
409 
410 	hctx->driver_data = queue;
411 	return 0;
412 }
413 
414 static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
415 		unsigned int hctx_idx)
416 {
417 	struct nvme_rdma_ctrl *ctrl = data;
418 	struct nvme_rdma_queue *queue = &ctrl->queues[0];
419 
420 	BUG_ON(hctx_idx != 0);
421 
422 	hctx->driver_data = queue;
423 	return 0;
424 }
425 
426 static void nvme_rdma_free_dev(struct kref *ref)
427 {
428 	struct nvme_rdma_device *ndev =
429 		container_of(ref, struct nvme_rdma_device, ref);
430 
431 	mutex_lock(&device_list_mutex);
432 	list_del(&ndev->entry);
433 	mutex_unlock(&device_list_mutex);
434 
435 	ib_dealloc_pd(ndev->pd);
436 	kfree(ndev);
437 }
438 
439 static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
440 {
441 	kref_put(&dev->ref, nvme_rdma_free_dev);
442 }
443 
444 static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
445 {
446 	return kref_get_unless_zero(&dev->ref);
447 }
448 
449 static struct nvme_rdma_device *
450 nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
451 {
452 	struct nvme_rdma_device *ndev;
453 
454 	mutex_lock(&device_list_mutex);
455 	list_for_each_entry(ndev, &device_list, entry) {
456 		if (ndev->dev->node_guid == cm_id->device->node_guid &&
457 		    nvme_rdma_dev_get(ndev))
458 			goto out_unlock;
459 	}
460 
461 	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
462 	if (!ndev)
463 		goto out_err;
464 
465 	ndev->dev = cm_id->device;
466 	kref_init(&ndev->ref);
467 
468 	ndev->pd = ib_alloc_pd(ndev->dev,
469 		register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
470 	if (IS_ERR(ndev->pd))
471 		goto out_free_dev;
472 
473 	if (!(ndev->dev->attrs.device_cap_flags &
474 	      IB_DEVICE_MEM_MGT_EXTENSIONS)) {
475 		dev_err(&ndev->dev->dev,
476 			"Memory registrations not supported.\n");
477 		goto out_free_pd;
478 	}
479 
480 	list_add(&ndev->entry, &device_list);
481 out_unlock:
482 	mutex_unlock(&device_list_mutex);
483 	return ndev;
484 
485 out_free_pd:
486 	ib_dealloc_pd(ndev->pd);
487 out_free_dev:
488 	kfree(ndev);
489 out_err:
490 	mutex_unlock(&device_list_mutex);
491 	return NULL;
492 }
493 
494 static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
495 {
496 	struct nvme_rdma_device *dev;
497 	struct ib_device *ibdev;
498 
499 	if (!test_and_clear_bit(NVME_RDMA_IB_QUEUE_ALLOCATED, &queue->flags))
500 		return;
501 
502 	dev = queue->device;
503 	ibdev = dev->dev;
504 	rdma_destroy_qp(queue->cm_id);
505 	ib_free_cq(queue->ib_cq);
506 
507 	nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
508 			sizeof(struct nvme_completion), DMA_FROM_DEVICE);
509 
510 	nvme_rdma_dev_put(dev);
511 }
512 
513 static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue,
514 		struct nvme_rdma_device *dev)
515 {
516 	struct ib_device *ibdev = dev->dev;
517 	const int send_wr_factor = 3;			/* MR, SEND, INV */
518 	const int cq_factor = send_wr_factor + 1;	/* + RECV */
519 	int comp_vector, idx = nvme_rdma_queue_idx(queue);
520 
521 	int ret;
522 
523 	queue->device = dev;
524 
525 	/*
526 	 * The admin queue is barely used once the controller is live, so don't
527 	 * bother to spread it out.
528 	 */
529 	if (idx == 0)
530 		comp_vector = 0;
531 	else
532 		comp_vector = idx % ibdev->num_comp_vectors;
533 
534 
535 	/* +1 for ib_stop_cq */
536 	queue->ib_cq = ib_alloc_cq(dev->dev, queue,
537 				cq_factor * queue->queue_size + 1, comp_vector,
538 				IB_POLL_SOFTIRQ);
539 	if (IS_ERR(queue->ib_cq)) {
540 		ret = PTR_ERR(queue->ib_cq);
541 		goto out;
542 	}
543 
544 	ret = nvme_rdma_create_qp(queue, send_wr_factor);
545 	if (ret)
546 		goto out_destroy_ib_cq;
547 
548 	queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
549 			sizeof(struct nvme_completion), DMA_FROM_DEVICE);
550 	if (!queue->rsp_ring) {
551 		ret = -ENOMEM;
552 		goto out_destroy_qp;
553 	}
554 	set_bit(NVME_RDMA_IB_QUEUE_ALLOCATED, &queue->flags);
555 
556 	return 0;
557 
558 out_destroy_qp:
559 	ib_destroy_qp(queue->qp);
560 out_destroy_ib_cq:
561 	ib_free_cq(queue->ib_cq);
562 out:
563 	return ret;
564 }
565 
566 static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
567 		int idx, size_t queue_size)
568 {
569 	struct nvme_rdma_queue *queue;
570 	int ret;
571 
572 	queue = &ctrl->queues[idx];
573 	queue->ctrl = ctrl;
574 	init_completion(&queue->cm_done);
575 
576 	if (idx > 0)
577 		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
578 	else
579 		queue->cmnd_capsule_len = sizeof(struct nvme_command);
580 
581 	queue->queue_size = queue_size;
582 
583 	queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
584 			RDMA_PS_TCP, IB_QPT_RC);
585 	if (IS_ERR(queue->cm_id)) {
586 		dev_info(ctrl->ctrl.device,
587 			"failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
588 		return PTR_ERR(queue->cm_id);
589 	}
590 
591 	queue->cm_error = -ETIMEDOUT;
592 	ret = rdma_resolve_addr(queue->cm_id, NULL, &ctrl->addr,
593 			NVME_RDMA_CONNECT_TIMEOUT_MS);
594 	if (ret) {
595 		dev_info(ctrl->ctrl.device,
596 			"rdma_resolve_addr failed (%d).\n", ret);
597 		goto out_destroy_cm_id;
598 	}
599 
600 	ret = nvme_rdma_wait_for_cm(queue);
601 	if (ret) {
602 		dev_info(ctrl->ctrl.device,
603 			"rdma_resolve_addr wait failed (%d).\n", ret);
604 		goto out_destroy_cm_id;
605 	}
606 
607 	clear_bit(NVME_RDMA_Q_DELETING, &queue->flags);
608 	set_bit(NVME_RDMA_Q_CONNECTED, &queue->flags);
609 
610 	return 0;
611 
612 out_destroy_cm_id:
613 	nvme_rdma_destroy_queue_ib(queue);
614 	rdma_destroy_id(queue->cm_id);
615 	return ret;
616 }
617 
618 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
619 {
620 	rdma_disconnect(queue->cm_id);
621 	ib_drain_qp(queue->qp);
622 }
623 
624 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
625 {
626 	nvme_rdma_destroy_queue_ib(queue);
627 	rdma_destroy_id(queue->cm_id);
628 }
629 
630 static void nvme_rdma_stop_and_free_queue(struct nvme_rdma_queue *queue)
631 {
632 	if (test_and_set_bit(NVME_RDMA_Q_DELETING, &queue->flags))
633 		return;
634 	nvme_rdma_stop_queue(queue);
635 	nvme_rdma_free_queue(queue);
636 }
637 
638 static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
639 {
640 	int i;
641 
642 	for (i = 1; i < ctrl->queue_count; i++)
643 		nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
644 }
645 
646 static int nvme_rdma_connect_io_queues(struct nvme_rdma_ctrl *ctrl)
647 {
648 	int i, ret = 0;
649 
650 	for (i = 1; i < ctrl->queue_count; i++) {
651 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
652 		if (ret) {
653 			dev_info(ctrl->ctrl.device,
654 				"failed to connect i/o queue: %d\n", ret);
655 			goto out_free_queues;
656 		}
657 		set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
658 	}
659 
660 	return 0;
661 
662 out_free_queues:
663 	nvme_rdma_free_io_queues(ctrl);
664 	return ret;
665 }
666 
667 static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
668 {
669 	int i, ret;
670 
671 	for (i = 1; i < ctrl->queue_count; i++) {
672 		ret = nvme_rdma_init_queue(ctrl, i,
673 					   ctrl->ctrl.opts->queue_size);
674 		if (ret) {
675 			dev_info(ctrl->ctrl.device,
676 				"failed to initialize i/o queue: %d\n", ret);
677 			goto out_free_queues;
678 		}
679 	}
680 
681 	return 0;
682 
683 out_free_queues:
684 	for (i--; i >= 1; i--)
685 		nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
686 
687 	return ret;
688 }
689 
690 static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
691 {
692 	nvme_rdma_free_qe(ctrl->queues[0].device->dev, &ctrl->async_event_sqe,
693 			sizeof(struct nvme_command), DMA_TO_DEVICE);
694 	nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
695 	blk_cleanup_queue(ctrl->ctrl.admin_q);
696 	blk_mq_free_tag_set(&ctrl->admin_tag_set);
697 	nvme_rdma_dev_put(ctrl->device);
698 }
699 
700 static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
701 {
702 	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
703 
704 	if (list_empty(&ctrl->list))
705 		goto free_ctrl;
706 
707 	mutex_lock(&nvme_rdma_ctrl_mutex);
708 	list_del(&ctrl->list);
709 	mutex_unlock(&nvme_rdma_ctrl_mutex);
710 
711 	kfree(ctrl->queues);
712 	nvmf_free_options(nctrl->opts);
713 free_ctrl:
714 	kfree(ctrl);
715 }
716 
717 static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
718 {
719 	struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
720 			struct nvme_rdma_ctrl, reconnect_work);
721 	bool changed;
722 	int ret;
723 
724 	if (ctrl->queue_count > 1) {
725 		nvme_rdma_free_io_queues(ctrl);
726 
727 		ret = blk_mq_reinit_tagset(&ctrl->tag_set);
728 		if (ret)
729 			goto requeue;
730 	}
731 
732 	nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
733 
734 	ret = blk_mq_reinit_tagset(&ctrl->admin_tag_set);
735 	if (ret)
736 		goto requeue;
737 
738 	ret = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
739 	if (ret)
740 		goto requeue;
741 
742 	blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true);
743 
744 	ret = nvmf_connect_admin_queue(&ctrl->ctrl);
745 	if (ret)
746 		goto stop_admin_q;
747 
748 	set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
749 
750 	ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
751 	if (ret)
752 		goto stop_admin_q;
753 
754 	nvme_start_keep_alive(&ctrl->ctrl);
755 
756 	if (ctrl->queue_count > 1) {
757 		ret = nvme_rdma_init_io_queues(ctrl);
758 		if (ret)
759 			goto stop_admin_q;
760 
761 		ret = nvme_rdma_connect_io_queues(ctrl);
762 		if (ret)
763 			goto stop_admin_q;
764 	}
765 
766 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
767 	WARN_ON_ONCE(!changed);
768 
769 	if (ctrl->queue_count > 1) {
770 		nvme_start_queues(&ctrl->ctrl);
771 		nvme_queue_scan(&ctrl->ctrl);
772 		nvme_queue_async_events(&ctrl->ctrl);
773 	}
774 
775 	dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
776 
777 	return;
778 
779 stop_admin_q:
780 	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
781 requeue:
782 	/* Make sure we are not resetting/deleting */
783 	if (ctrl->ctrl.state == NVME_CTRL_RECONNECTING) {
784 		dev_info(ctrl->ctrl.device,
785 			"Failed reconnect attempt, requeueing...\n");
786 		queue_delayed_work(nvme_rdma_wq, &ctrl->reconnect_work,
787 					ctrl->reconnect_delay * HZ);
788 	}
789 }
790 
791 static void nvme_rdma_error_recovery_work(struct work_struct *work)
792 {
793 	struct nvme_rdma_ctrl *ctrl = container_of(work,
794 			struct nvme_rdma_ctrl, err_work);
795 	int i;
796 
797 	nvme_stop_keep_alive(&ctrl->ctrl);
798 
799 	for (i = 0; i < ctrl->queue_count; i++) {
800 		clear_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[i].flags);
801 		clear_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
802 	}
803 
804 	if (ctrl->queue_count > 1)
805 		nvme_stop_queues(&ctrl->ctrl);
806 	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
807 
808 	/* We must take care of fastfail/requeue all our inflight requests */
809 	if (ctrl->queue_count > 1)
810 		blk_mq_tagset_busy_iter(&ctrl->tag_set,
811 					nvme_cancel_request, &ctrl->ctrl);
812 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
813 				nvme_cancel_request, &ctrl->ctrl);
814 
815 	dev_info(ctrl->ctrl.device, "reconnecting in %d seconds\n",
816 		ctrl->reconnect_delay);
817 
818 	queue_delayed_work(nvme_rdma_wq, &ctrl->reconnect_work,
819 				ctrl->reconnect_delay * HZ);
820 }
821 
822 static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
823 {
824 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RECONNECTING))
825 		return;
826 
827 	queue_work(nvme_rdma_wq, &ctrl->err_work);
828 }
829 
830 static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
831 		const char *op)
832 {
833 	struct nvme_rdma_queue *queue = cq->cq_context;
834 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
835 
836 	if (ctrl->ctrl.state == NVME_CTRL_LIVE)
837 		dev_info(ctrl->ctrl.device,
838 			     "%s for CQE 0x%p failed with status %s (%d)\n",
839 			     op, wc->wr_cqe,
840 			     ib_wc_status_msg(wc->status), wc->status);
841 	nvme_rdma_error_recovery(ctrl);
842 }
843 
844 static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
845 {
846 	if (unlikely(wc->status != IB_WC_SUCCESS))
847 		nvme_rdma_wr_error(cq, wc, "MEMREG");
848 }
849 
850 static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
851 {
852 	if (unlikely(wc->status != IB_WC_SUCCESS))
853 		nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
854 }
855 
856 static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
857 		struct nvme_rdma_request *req)
858 {
859 	struct ib_send_wr *bad_wr;
860 	struct ib_send_wr wr = {
861 		.opcode		    = IB_WR_LOCAL_INV,
862 		.next		    = NULL,
863 		.num_sge	    = 0,
864 		.send_flags	    = 0,
865 		.ex.invalidate_rkey = req->mr->rkey,
866 	};
867 
868 	req->reg_cqe.done = nvme_rdma_inv_rkey_done;
869 	wr.wr_cqe = &req->reg_cqe;
870 
871 	return ib_post_send(queue->qp, &wr, &bad_wr);
872 }
873 
874 static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
875 		struct request *rq)
876 {
877 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
878 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
879 	struct nvme_rdma_device *dev = queue->device;
880 	struct ib_device *ibdev = dev->dev;
881 	int res;
882 
883 	if (!blk_rq_bytes(rq))
884 		return;
885 
886 	if (req->mr->need_inval) {
887 		res = nvme_rdma_inv_rkey(queue, req);
888 		if (res < 0) {
889 			dev_err(ctrl->ctrl.device,
890 				"Queueing INV WR for rkey %#x failed (%d)\n",
891 				req->mr->rkey, res);
892 			nvme_rdma_error_recovery(queue->ctrl);
893 		}
894 	}
895 
896 	ib_dma_unmap_sg(ibdev, req->sg_table.sgl,
897 			req->nents, rq_data_dir(rq) ==
898 				    WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
899 
900 	nvme_cleanup_cmd(rq);
901 	sg_free_table_chained(&req->sg_table, true);
902 }
903 
904 static int nvme_rdma_set_sg_null(struct nvme_command *c)
905 {
906 	struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
907 
908 	sg->addr = 0;
909 	put_unaligned_le24(0, sg->length);
910 	put_unaligned_le32(0, sg->key);
911 	sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
912 	return 0;
913 }
914 
915 static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
916 		struct nvme_rdma_request *req, struct nvme_command *c)
917 {
918 	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
919 
920 	req->sge[1].addr = sg_dma_address(req->sg_table.sgl);
921 	req->sge[1].length = sg_dma_len(req->sg_table.sgl);
922 	req->sge[1].lkey = queue->device->pd->local_dma_lkey;
923 
924 	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
925 	sg->length = cpu_to_le32(sg_dma_len(req->sg_table.sgl));
926 	sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
927 
928 	req->inline_data = true;
929 	req->num_sge++;
930 	return 0;
931 }
932 
933 static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
934 		struct nvme_rdma_request *req, struct nvme_command *c)
935 {
936 	struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
937 
938 	sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl));
939 	put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length);
940 	put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
941 	sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
942 	return 0;
943 }
944 
945 static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
946 		struct nvme_rdma_request *req, struct nvme_command *c,
947 		int count)
948 {
949 	struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
950 	int nr;
951 
952 	nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, PAGE_SIZE);
953 	if (nr < count) {
954 		if (nr < 0)
955 			return nr;
956 		return -EINVAL;
957 	}
958 
959 	ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
960 
961 	req->reg_cqe.done = nvme_rdma_memreg_done;
962 	memset(&req->reg_wr, 0, sizeof(req->reg_wr));
963 	req->reg_wr.wr.opcode = IB_WR_REG_MR;
964 	req->reg_wr.wr.wr_cqe = &req->reg_cqe;
965 	req->reg_wr.wr.num_sge = 0;
966 	req->reg_wr.mr = req->mr;
967 	req->reg_wr.key = req->mr->rkey;
968 	req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
969 			     IB_ACCESS_REMOTE_READ |
970 			     IB_ACCESS_REMOTE_WRITE;
971 
972 	req->mr->need_inval = true;
973 
974 	sg->addr = cpu_to_le64(req->mr->iova);
975 	put_unaligned_le24(req->mr->length, sg->length);
976 	put_unaligned_le32(req->mr->rkey, sg->key);
977 	sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
978 			NVME_SGL_FMT_INVALIDATE;
979 
980 	return 0;
981 }
982 
983 static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
984 		struct request *rq, unsigned int map_len,
985 		struct nvme_command *c)
986 {
987 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
988 	struct nvme_rdma_device *dev = queue->device;
989 	struct ib_device *ibdev = dev->dev;
990 	int count, ret;
991 
992 	req->num_sge = 1;
993 	req->inline_data = false;
994 	req->mr->need_inval = false;
995 
996 	c->common.flags |= NVME_CMD_SGL_METABUF;
997 
998 	if (!blk_rq_bytes(rq))
999 		return nvme_rdma_set_sg_null(c);
1000 
1001 	req->sg_table.sgl = req->first_sgl;
1002 	ret = sg_alloc_table_chained(&req->sg_table,
1003 			blk_rq_nr_phys_segments(rq), req->sg_table.sgl);
1004 	if (ret)
1005 		return -ENOMEM;
1006 
1007 	req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
1008 
1009 	count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents,
1010 		    rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1011 	if (unlikely(count <= 0)) {
1012 		sg_free_table_chained(&req->sg_table, true);
1013 		return -EIO;
1014 	}
1015 
1016 	if (count == 1) {
1017 		if (rq_data_dir(rq) == WRITE &&
1018 		    map_len <= nvme_rdma_inline_data_size(queue) &&
1019 		    nvme_rdma_queue_idx(queue))
1020 			return nvme_rdma_map_sg_inline(queue, req, c);
1021 
1022 		if (dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)
1023 			return nvme_rdma_map_sg_single(queue, req, c);
1024 	}
1025 
1026 	return nvme_rdma_map_sg_fr(queue, req, c, count);
1027 }
1028 
1029 static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1030 {
1031 	if (unlikely(wc->status != IB_WC_SUCCESS))
1032 		nvme_rdma_wr_error(cq, wc, "SEND");
1033 }
1034 
1035 static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1036 		struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
1037 		struct ib_send_wr *first, bool flush)
1038 {
1039 	struct ib_send_wr wr, *bad_wr;
1040 	int ret;
1041 
1042 	sge->addr   = qe->dma;
1043 	sge->length = sizeof(struct nvme_command),
1044 	sge->lkey   = queue->device->pd->local_dma_lkey;
1045 
1046 	qe->cqe.done = nvme_rdma_send_done;
1047 
1048 	wr.next       = NULL;
1049 	wr.wr_cqe     = &qe->cqe;
1050 	wr.sg_list    = sge;
1051 	wr.num_sge    = num_sge;
1052 	wr.opcode     = IB_WR_SEND;
1053 	wr.send_flags = 0;
1054 
1055 	/*
1056 	 * Unsignalled send completions are another giant desaster in the
1057 	 * IB Verbs spec:  If we don't regularly post signalled sends
1058 	 * the send queue will fill up and only a QP reset will rescue us.
1059 	 * Would have been way to obvious to handle this in hardware or
1060 	 * at least the RDMA stack..
1061 	 *
1062 	 * This messy and racy code sniplet is copy and pasted from the iSER
1063 	 * initiator, and the magic '32' comes from there as well.
1064 	 *
1065 	 * Always signal the flushes. The magic request used for the flush
1066 	 * sequencer is not allocated in our driver's tagset and it's
1067 	 * triggered to be freed by blk_cleanup_queue(). So we need to
1068 	 * always mark it as signaled to ensure that the "wr_cqe", which is
1069 	 * embeded in request's payload, is not freed when __ib_process_cq()
1070 	 * calls wr_cqe->done().
1071 	 */
1072 	if ((++queue->sig_count % 32) == 0 || flush)
1073 		wr.send_flags |= IB_SEND_SIGNALED;
1074 
1075 	if (first)
1076 		first->next = &wr;
1077 	else
1078 		first = &wr;
1079 
1080 	ret = ib_post_send(queue->qp, first, &bad_wr);
1081 	if (ret) {
1082 		dev_err(queue->ctrl->ctrl.device,
1083 			     "%s failed with error code %d\n", __func__, ret);
1084 	}
1085 	return ret;
1086 }
1087 
1088 static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1089 		struct nvme_rdma_qe *qe)
1090 {
1091 	struct ib_recv_wr wr, *bad_wr;
1092 	struct ib_sge list;
1093 	int ret;
1094 
1095 	list.addr   = qe->dma;
1096 	list.length = sizeof(struct nvme_completion);
1097 	list.lkey   = queue->device->pd->local_dma_lkey;
1098 
1099 	qe->cqe.done = nvme_rdma_recv_done;
1100 
1101 	wr.next     = NULL;
1102 	wr.wr_cqe   = &qe->cqe;
1103 	wr.sg_list  = &list;
1104 	wr.num_sge  = 1;
1105 
1106 	ret = ib_post_recv(queue->qp, &wr, &bad_wr);
1107 	if (ret) {
1108 		dev_err(queue->ctrl->ctrl.device,
1109 			"%s failed with error code %d\n", __func__, ret);
1110 	}
1111 	return ret;
1112 }
1113 
1114 static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1115 {
1116 	u32 queue_idx = nvme_rdma_queue_idx(queue);
1117 
1118 	if (queue_idx == 0)
1119 		return queue->ctrl->admin_tag_set.tags[queue_idx];
1120 	return queue->ctrl->tag_set.tags[queue_idx - 1];
1121 }
1122 
1123 static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
1124 {
1125 	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1126 	struct nvme_rdma_queue *queue = &ctrl->queues[0];
1127 	struct ib_device *dev = queue->device->dev;
1128 	struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1129 	struct nvme_command *cmd = sqe->data;
1130 	struct ib_sge sge;
1131 	int ret;
1132 
1133 	if (WARN_ON_ONCE(aer_idx != 0))
1134 		return;
1135 
1136 	ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1137 
1138 	memset(cmd, 0, sizeof(*cmd));
1139 	cmd->common.opcode = nvme_admin_async_event;
1140 	cmd->common.command_id = NVME_RDMA_AQ_BLKMQ_DEPTH;
1141 	cmd->common.flags |= NVME_CMD_SGL_METABUF;
1142 	nvme_rdma_set_sg_null(cmd);
1143 
1144 	ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1145 			DMA_TO_DEVICE);
1146 
1147 	ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL, false);
1148 	WARN_ON_ONCE(ret);
1149 }
1150 
1151 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1152 		struct nvme_completion *cqe, struct ib_wc *wc, int tag)
1153 {
1154 	struct request *rq;
1155 	struct nvme_rdma_request *req;
1156 	int ret = 0;
1157 
1158 	rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
1159 	if (!rq) {
1160 		dev_err(queue->ctrl->ctrl.device,
1161 			"tag 0x%x on QP %#x not found\n",
1162 			cqe->command_id, queue->qp->qp_num);
1163 		nvme_rdma_error_recovery(queue->ctrl);
1164 		return ret;
1165 	}
1166 	req = blk_mq_rq_to_pdu(rq);
1167 
1168 	if (rq->tag == tag)
1169 		ret = 1;
1170 
1171 	if ((wc->wc_flags & IB_WC_WITH_INVALIDATE) &&
1172 	    wc->ex.invalidate_rkey == req->mr->rkey)
1173 		req->mr->need_inval = false;
1174 
1175 	req->req.result = cqe->result;
1176 	blk_mq_complete_request(rq, le16_to_cpu(cqe->status) >> 1);
1177 	return ret;
1178 }
1179 
1180 static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
1181 {
1182 	struct nvme_rdma_qe *qe =
1183 		container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1184 	struct nvme_rdma_queue *queue = cq->cq_context;
1185 	struct ib_device *ibdev = queue->device->dev;
1186 	struct nvme_completion *cqe = qe->data;
1187 	const size_t len = sizeof(struct nvme_completion);
1188 	int ret = 0;
1189 
1190 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1191 		nvme_rdma_wr_error(cq, wc, "RECV");
1192 		return 0;
1193 	}
1194 
1195 	ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1196 	/*
1197 	 * AEN requests are special as they don't time out and can
1198 	 * survive any kind of queue freeze and often don't respond to
1199 	 * aborts.  We don't even bother to allocate a struct request
1200 	 * for them but rather special case them here.
1201 	 */
1202 	if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
1203 			cqe->command_id >= NVME_RDMA_AQ_BLKMQ_DEPTH))
1204 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1205 				&cqe->result);
1206 	else
1207 		ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
1208 	ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1209 
1210 	nvme_rdma_post_recv(queue, qe);
1211 	return ret;
1212 }
1213 
1214 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1215 {
1216 	__nvme_rdma_recv_done(cq, wc, -1);
1217 }
1218 
1219 static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1220 {
1221 	int ret, i;
1222 
1223 	for (i = 0; i < queue->queue_size; i++) {
1224 		ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1225 		if (ret)
1226 			goto out_destroy_queue_ib;
1227 	}
1228 
1229 	return 0;
1230 
1231 out_destroy_queue_ib:
1232 	nvme_rdma_destroy_queue_ib(queue);
1233 	return ret;
1234 }
1235 
1236 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1237 		struct rdma_cm_event *ev)
1238 {
1239 	struct rdma_cm_id *cm_id = queue->cm_id;
1240 	int status = ev->status;
1241 	const char *rej_msg;
1242 	const struct nvme_rdma_cm_rej *rej_data;
1243 	u8 rej_data_len;
1244 
1245 	rej_msg = rdma_reject_msg(cm_id, status);
1246 	rej_data = rdma_consumer_reject_data(cm_id, ev, &rej_data_len);
1247 
1248 	if (rej_data && rej_data_len >= sizeof(u16)) {
1249 		u16 sts = le16_to_cpu(rej_data->sts);
1250 
1251 		dev_err(queue->ctrl->ctrl.device,
1252 		      "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1253 		      status, rej_msg, sts, nvme_rdma_cm_msg(sts));
1254 	} else {
1255 		dev_err(queue->ctrl->ctrl.device,
1256 			"Connect rejected: status %d (%s).\n", status, rej_msg);
1257 	}
1258 
1259 	return -ECONNRESET;
1260 }
1261 
1262 static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1263 {
1264 	struct nvme_rdma_device *dev;
1265 	int ret;
1266 
1267 	dev = nvme_rdma_find_get_device(queue->cm_id);
1268 	if (!dev) {
1269 		dev_err(queue->cm_id->device->dma_device,
1270 			"no client data found!\n");
1271 		return -ECONNREFUSED;
1272 	}
1273 
1274 	ret = nvme_rdma_create_queue_ib(queue, dev);
1275 	if (ret) {
1276 		nvme_rdma_dev_put(dev);
1277 		goto out;
1278 	}
1279 
1280 	ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1281 	if (ret) {
1282 		dev_err(queue->ctrl->ctrl.device,
1283 			"rdma_resolve_route failed (%d).\n",
1284 			queue->cm_error);
1285 		goto out_destroy_queue;
1286 	}
1287 
1288 	return 0;
1289 
1290 out_destroy_queue:
1291 	nvme_rdma_destroy_queue_ib(queue);
1292 out:
1293 	return ret;
1294 }
1295 
1296 static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1297 {
1298 	struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1299 	struct rdma_conn_param param = { };
1300 	struct nvme_rdma_cm_req priv = { };
1301 	int ret;
1302 
1303 	param.qp_num = queue->qp->qp_num;
1304 	param.flow_control = 1;
1305 
1306 	param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
1307 	/* maximum retry count */
1308 	param.retry_count = 7;
1309 	param.rnr_retry_count = 7;
1310 	param.private_data = &priv;
1311 	param.private_data_len = sizeof(priv);
1312 
1313 	priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1314 	priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
1315 	/*
1316 	 * set the admin queue depth to the minimum size
1317 	 * specified by the Fabrics standard.
1318 	 */
1319 	if (priv.qid == 0) {
1320 		priv.hrqsize = cpu_to_le16(NVMF_AQ_DEPTH);
1321 		priv.hsqsize = cpu_to_le16(NVMF_AQ_DEPTH - 1);
1322 	} else {
1323 		/*
1324 		 * current interpretation of the fabrics spec
1325 		 * is at minimum you make hrqsize sqsize+1, or a
1326 		 * 1's based representation of sqsize.
1327 		 */
1328 		priv.hrqsize = cpu_to_le16(queue->queue_size);
1329 		priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
1330 	}
1331 
1332 	ret = rdma_connect(queue->cm_id, &param);
1333 	if (ret) {
1334 		dev_err(ctrl->ctrl.device,
1335 			"rdma_connect failed (%d).\n", ret);
1336 		goto out_destroy_queue_ib;
1337 	}
1338 
1339 	return 0;
1340 
1341 out_destroy_queue_ib:
1342 	nvme_rdma_destroy_queue_ib(queue);
1343 	return ret;
1344 }
1345 
1346 static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1347 		struct rdma_cm_event *ev)
1348 {
1349 	struct nvme_rdma_queue *queue = cm_id->context;
1350 	int cm_error = 0;
1351 
1352 	dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1353 		rdma_event_msg(ev->event), ev->event,
1354 		ev->status, cm_id);
1355 
1356 	switch (ev->event) {
1357 	case RDMA_CM_EVENT_ADDR_RESOLVED:
1358 		cm_error = nvme_rdma_addr_resolved(queue);
1359 		break;
1360 	case RDMA_CM_EVENT_ROUTE_RESOLVED:
1361 		cm_error = nvme_rdma_route_resolved(queue);
1362 		break;
1363 	case RDMA_CM_EVENT_ESTABLISHED:
1364 		queue->cm_error = nvme_rdma_conn_established(queue);
1365 		/* complete cm_done regardless of success/failure */
1366 		complete(&queue->cm_done);
1367 		return 0;
1368 	case RDMA_CM_EVENT_REJECTED:
1369 		cm_error = nvme_rdma_conn_rejected(queue, ev);
1370 		break;
1371 	case RDMA_CM_EVENT_ADDR_ERROR:
1372 	case RDMA_CM_EVENT_ROUTE_ERROR:
1373 	case RDMA_CM_EVENT_CONNECT_ERROR:
1374 	case RDMA_CM_EVENT_UNREACHABLE:
1375 		dev_dbg(queue->ctrl->ctrl.device,
1376 			"CM error event %d\n", ev->event);
1377 		cm_error = -ECONNRESET;
1378 		break;
1379 	case RDMA_CM_EVENT_DISCONNECTED:
1380 	case RDMA_CM_EVENT_ADDR_CHANGE:
1381 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1382 		dev_dbg(queue->ctrl->ctrl.device,
1383 			"disconnect received - connection closed\n");
1384 		nvme_rdma_error_recovery(queue->ctrl);
1385 		break;
1386 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
1387 		/* device removal is handled via the ib_client API */
1388 		break;
1389 	default:
1390 		dev_err(queue->ctrl->ctrl.device,
1391 			"Unexpected RDMA CM event (%d)\n", ev->event);
1392 		nvme_rdma_error_recovery(queue->ctrl);
1393 		break;
1394 	}
1395 
1396 	if (cm_error) {
1397 		queue->cm_error = cm_error;
1398 		complete(&queue->cm_done);
1399 	}
1400 
1401 	return 0;
1402 }
1403 
1404 static enum blk_eh_timer_return
1405 nvme_rdma_timeout(struct request *rq, bool reserved)
1406 {
1407 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1408 
1409 	/* queue error recovery */
1410 	nvme_rdma_error_recovery(req->queue->ctrl);
1411 
1412 	/* fail with DNR on cmd timeout */
1413 	rq->errors = NVME_SC_ABORT_REQ | NVME_SC_DNR;
1414 
1415 	return BLK_EH_HANDLED;
1416 }
1417 
1418 /*
1419  * We cannot accept any other command until the Connect command has completed.
1420  */
1421 static inline bool nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue,
1422 		struct request *rq)
1423 {
1424 	if (unlikely(!test_bit(NVME_RDMA_Q_LIVE, &queue->flags))) {
1425 		struct nvme_command *cmd = (struct nvme_command *)rq->cmd;
1426 
1427 		if (rq->cmd_type != REQ_TYPE_DRV_PRIV ||
1428 		    cmd->common.opcode != nvme_fabrics_command ||
1429 		    cmd->fabrics.fctype != nvme_fabrics_type_connect)
1430 			return false;
1431 	}
1432 
1433 	return true;
1434 }
1435 
1436 static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
1437 		const struct blk_mq_queue_data *bd)
1438 {
1439 	struct nvme_ns *ns = hctx->queue->queuedata;
1440 	struct nvme_rdma_queue *queue = hctx->driver_data;
1441 	struct request *rq = bd->rq;
1442 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1443 	struct nvme_rdma_qe *sqe = &req->sqe;
1444 	struct nvme_command *c = sqe->data;
1445 	bool flush = false;
1446 	struct ib_device *dev;
1447 	unsigned int map_len;
1448 	int ret;
1449 
1450 	WARN_ON_ONCE(rq->tag < 0);
1451 
1452 	if (!nvme_rdma_queue_is_ready(queue, rq))
1453 		return BLK_MQ_RQ_QUEUE_BUSY;
1454 
1455 	dev = queue->device->dev;
1456 	ib_dma_sync_single_for_cpu(dev, sqe->dma,
1457 			sizeof(struct nvme_command), DMA_TO_DEVICE);
1458 
1459 	ret = nvme_setup_cmd(ns, rq, c);
1460 	if (ret != BLK_MQ_RQ_QUEUE_OK)
1461 		return ret;
1462 
1463 	blk_mq_start_request(rq);
1464 
1465 	map_len = nvme_map_len(rq);
1466 	ret = nvme_rdma_map_data(queue, rq, map_len, c);
1467 	if (ret < 0) {
1468 		dev_err(queue->ctrl->ctrl.device,
1469 			     "Failed to map data (%d)\n", ret);
1470 		nvme_cleanup_cmd(rq);
1471 		goto err;
1472 	}
1473 
1474 	ib_dma_sync_single_for_device(dev, sqe->dma,
1475 			sizeof(struct nvme_command), DMA_TO_DEVICE);
1476 
1477 	if (rq->cmd_type == REQ_TYPE_FS && req_op(rq) == REQ_OP_FLUSH)
1478 		flush = true;
1479 	ret = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
1480 			req->mr->need_inval ? &req->reg_wr.wr : NULL, flush);
1481 	if (ret) {
1482 		nvme_rdma_unmap_data(queue, rq);
1483 		goto err;
1484 	}
1485 
1486 	return BLK_MQ_RQ_QUEUE_OK;
1487 err:
1488 	return (ret == -ENOMEM || ret == -EAGAIN) ?
1489 		BLK_MQ_RQ_QUEUE_BUSY : BLK_MQ_RQ_QUEUE_ERROR;
1490 }
1491 
1492 static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
1493 {
1494 	struct nvme_rdma_queue *queue = hctx->driver_data;
1495 	struct ib_cq *cq = queue->ib_cq;
1496 	struct ib_wc wc;
1497 	int found = 0;
1498 
1499 	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1500 	while (ib_poll_cq(cq, 1, &wc) > 0) {
1501 		struct ib_cqe *cqe = wc.wr_cqe;
1502 
1503 		if (cqe) {
1504 			if (cqe->done == nvme_rdma_recv_done)
1505 				found |= __nvme_rdma_recv_done(cq, &wc, tag);
1506 			else
1507 				cqe->done(cq, &wc);
1508 		}
1509 	}
1510 
1511 	return found;
1512 }
1513 
1514 static void nvme_rdma_complete_rq(struct request *rq)
1515 {
1516 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1517 	struct nvme_rdma_queue *queue = req->queue;
1518 	int error = 0;
1519 
1520 	nvme_rdma_unmap_data(queue, rq);
1521 
1522 	if (unlikely(rq->errors)) {
1523 		if (nvme_req_needs_retry(rq, rq->errors)) {
1524 			nvme_requeue_req(rq);
1525 			return;
1526 		}
1527 
1528 		if (rq->cmd_type == REQ_TYPE_DRV_PRIV)
1529 			error = rq->errors;
1530 		else
1531 			error = nvme_error_status(rq->errors);
1532 	}
1533 
1534 	blk_mq_end_request(rq, error);
1535 }
1536 
1537 static struct blk_mq_ops nvme_rdma_mq_ops = {
1538 	.queue_rq	= nvme_rdma_queue_rq,
1539 	.complete	= nvme_rdma_complete_rq,
1540 	.init_request	= nvme_rdma_init_request,
1541 	.exit_request	= nvme_rdma_exit_request,
1542 	.reinit_request	= nvme_rdma_reinit_request,
1543 	.init_hctx	= nvme_rdma_init_hctx,
1544 	.poll		= nvme_rdma_poll,
1545 	.timeout	= nvme_rdma_timeout,
1546 };
1547 
1548 static struct blk_mq_ops nvme_rdma_admin_mq_ops = {
1549 	.queue_rq	= nvme_rdma_queue_rq,
1550 	.complete	= nvme_rdma_complete_rq,
1551 	.init_request	= nvme_rdma_init_admin_request,
1552 	.exit_request	= nvme_rdma_exit_admin_request,
1553 	.reinit_request	= nvme_rdma_reinit_request,
1554 	.init_hctx	= nvme_rdma_init_admin_hctx,
1555 	.timeout	= nvme_rdma_timeout,
1556 };
1557 
1558 static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl)
1559 {
1560 	int error;
1561 
1562 	error = nvme_rdma_init_queue(ctrl, 0, NVMF_AQ_DEPTH);
1563 	if (error)
1564 		return error;
1565 
1566 	ctrl->device = ctrl->queues[0].device;
1567 
1568 	/*
1569 	 * We need a reference on the device as long as the tag_set is alive,
1570 	 * as the MRs in the request structures need a valid ib_device.
1571 	 */
1572 	error = -EINVAL;
1573 	if (!nvme_rdma_dev_get(ctrl->device))
1574 		goto out_free_queue;
1575 
1576 	ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
1577 		ctrl->device->dev->attrs.max_fast_reg_page_list_len);
1578 
1579 	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
1580 	ctrl->admin_tag_set.ops = &nvme_rdma_admin_mq_ops;
1581 	ctrl->admin_tag_set.queue_depth = NVME_RDMA_AQ_BLKMQ_DEPTH;
1582 	ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
1583 	ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
1584 	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1585 		SG_CHUNK_SIZE * sizeof(struct scatterlist);
1586 	ctrl->admin_tag_set.driver_data = ctrl;
1587 	ctrl->admin_tag_set.nr_hw_queues = 1;
1588 	ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
1589 
1590 	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
1591 	if (error)
1592 		goto out_put_dev;
1593 
1594 	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
1595 	if (IS_ERR(ctrl->ctrl.admin_q)) {
1596 		error = PTR_ERR(ctrl->ctrl.admin_q);
1597 		goto out_free_tagset;
1598 	}
1599 
1600 	error = nvmf_connect_admin_queue(&ctrl->ctrl);
1601 	if (error)
1602 		goto out_cleanup_queue;
1603 
1604 	set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
1605 
1606 	error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);
1607 	if (error) {
1608 		dev_err(ctrl->ctrl.device,
1609 			"prop_get NVME_REG_CAP failed\n");
1610 		goto out_cleanup_queue;
1611 	}
1612 
1613 	ctrl->ctrl.sqsize =
1614 		min_t(int, NVME_CAP_MQES(ctrl->cap) + 1, ctrl->ctrl.sqsize);
1615 
1616 	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
1617 	if (error)
1618 		goto out_cleanup_queue;
1619 
1620 	ctrl->ctrl.max_hw_sectors =
1621 		(ctrl->max_fr_pages - 1) << (PAGE_SHIFT - 9);
1622 
1623 	error = nvme_init_identify(&ctrl->ctrl);
1624 	if (error)
1625 		goto out_cleanup_queue;
1626 
1627 	error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
1628 			&ctrl->async_event_sqe, sizeof(struct nvme_command),
1629 			DMA_TO_DEVICE);
1630 	if (error)
1631 		goto out_cleanup_queue;
1632 
1633 	nvme_start_keep_alive(&ctrl->ctrl);
1634 
1635 	return 0;
1636 
1637 out_cleanup_queue:
1638 	blk_cleanup_queue(ctrl->ctrl.admin_q);
1639 out_free_tagset:
1640 	/* disconnect and drain the queue before freeing the tagset */
1641 	nvme_rdma_stop_queue(&ctrl->queues[0]);
1642 	blk_mq_free_tag_set(&ctrl->admin_tag_set);
1643 out_put_dev:
1644 	nvme_rdma_dev_put(ctrl->device);
1645 out_free_queue:
1646 	nvme_rdma_free_queue(&ctrl->queues[0]);
1647 	return error;
1648 }
1649 
1650 static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
1651 {
1652 	nvme_stop_keep_alive(&ctrl->ctrl);
1653 	cancel_work_sync(&ctrl->err_work);
1654 	cancel_delayed_work_sync(&ctrl->reconnect_work);
1655 
1656 	if (ctrl->queue_count > 1) {
1657 		nvme_stop_queues(&ctrl->ctrl);
1658 		blk_mq_tagset_busy_iter(&ctrl->tag_set,
1659 					nvme_cancel_request, &ctrl->ctrl);
1660 		nvme_rdma_free_io_queues(ctrl);
1661 	}
1662 
1663 	if (test_bit(NVME_RDMA_Q_CONNECTED, &ctrl->queues[0].flags))
1664 		nvme_shutdown_ctrl(&ctrl->ctrl);
1665 
1666 	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
1667 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1668 				nvme_cancel_request, &ctrl->ctrl);
1669 	nvme_rdma_destroy_admin_queue(ctrl);
1670 }
1671 
1672 static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1673 {
1674 	nvme_uninit_ctrl(&ctrl->ctrl);
1675 	if (shutdown)
1676 		nvme_rdma_shutdown_ctrl(ctrl);
1677 
1678 	if (ctrl->ctrl.tagset) {
1679 		blk_cleanup_queue(ctrl->ctrl.connect_q);
1680 		blk_mq_free_tag_set(&ctrl->tag_set);
1681 		nvme_rdma_dev_put(ctrl->device);
1682 	}
1683 
1684 	nvme_put_ctrl(&ctrl->ctrl);
1685 }
1686 
1687 static void nvme_rdma_del_ctrl_work(struct work_struct *work)
1688 {
1689 	struct nvme_rdma_ctrl *ctrl = container_of(work,
1690 				struct nvme_rdma_ctrl, delete_work);
1691 
1692 	__nvme_rdma_remove_ctrl(ctrl, true);
1693 }
1694 
1695 static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
1696 {
1697 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
1698 		return -EBUSY;
1699 
1700 	if (!queue_work(nvme_rdma_wq, &ctrl->delete_work))
1701 		return -EBUSY;
1702 
1703 	return 0;
1704 }
1705 
1706 static int nvme_rdma_del_ctrl(struct nvme_ctrl *nctrl)
1707 {
1708 	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1709 	int ret = 0;
1710 
1711 	/*
1712 	 * Keep a reference until all work is flushed since
1713 	 * __nvme_rdma_del_ctrl can free the ctrl mem
1714 	 */
1715 	if (!kref_get_unless_zero(&ctrl->ctrl.kref))
1716 		return -EBUSY;
1717 	ret = __nvme_rdma_del_ctrl(ctrl);
1718 	if (!ret)
1719 		flush_work(&ctrl->delete_work);
1720 	nvme_put_ctrl(&ctrl->ctrl);
1721 	return ret;
1722 }
1723 
1724 static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
1725 {
1726 	struct nvme_rdma_ctrl *ctrl = container_of(work,
1727 				struct nvme_rdma_ctrl, delete_work);
1728 
1729 	__nvme_rdma_remove_ctrl(ctrl, false);
1730 }
1731 
1732 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1733 {
1734 	struct nvme_rdma_ctrl *ctrl = container_of(work,
1735 					struct nvme_rdma_ctrl, reset_work);
1736 	int ret;
1737 	bool changed;
1738 
1739 	nvme_rdma_shutdown_ctrl(ctrl);
1740 
1741 	ret = nvme_rdma_configure_admin_queue(ctrl);
1742 	if (ret) {
1743 		/* ctrl is already shutdown, just remove the ctrl */
1744 		INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
1745 		goto del_dead_ctrl;
1746 	}
1747 
1748 	if (ctrl->queue_count > 1) {
1749 		ret = blk_mq_reinit_tagset(&ctrl->tag_set);
1750 		if (ret)
1751 			goto del_dead_ctrl;
1752 
1753 		ret = nvme_rdma_init_io_queues(ctrl);
1754 		if (ret)
1755 			goto del_dead_ctrl;
1756 
1757 		ret = nvme_rdma_connect_io_queues(ctrl);
1758 		if (ret)
1759 			goto del_dead_ctrl;
1760 	}
1761 
1762 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1763 	WARN_ON_ONCE(!changed);
1764 
1765 	if (ctrl->queue_count > 1) {
1766 		nvme_start_queues(&ctrl->ctrl);
1767 		nvme_queue_scan(&ctrl->ctrl);
1768 		nvme_queue_async_events(&ctrl->ctrl);
1769 	}
1770 
1771 	return;
1772 
1773 del_dead_ctrl:
1774 	/* Deleting this dead controller... */
1775 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1776 	WARN_ON(!queue_work(nvme_rdma_wq, &ctrl->delete_work));
1777 }
1778 
1779 static int nvme_rdma_reset_ctrl(struct nvme_ctrl *nctrl)
1780 {
1781 	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1782 
1783 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
1784 		return -EBUSY;
1785 
1786 	if (!queue_work(nvme_rdma_wq, &ctrl->reset_work))
1787 		return -EBUSY;
1788 
1789 	flush_work(&ctrl->reset_work);
1790 
1791 	return 0;
1792 }
1793 
1794 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
1795 	.name			= "rdma",
1796 	.module			= THIS_MODULE,
1797 	.is_fabrics		= true,
1798 	.reg_read32		= nvmf_reg_read32,
1799 	.reg_read64		= nvmf_reg_read64,
1800 	.reg_write32		= nvmf_reg_write32,
1801 	.reset_ctrl		= nvme_rdma_reset_ctrl,
1802 	.free_ctrl		= nvme_rdma_free_ctrl,
1803 	.submit_async_event	= nvme_rdma_submit_async_event,
1804 	.delete_ctrl		= nvme_rdma_del_ctrl,
1805 	.get_subsysnqn		= nvmf_get_subsysnqn,
1806 	.get_address		= nvmf_get_address,
1807 };
1808 
1809 static int nvme_rdma_create_io_queues(struct nvme_rdma_ctrl *ctrl)
1810 {
1811 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
1812 	int ret;
1813 
1814 	ret = nvme_set_queue_count(&ctrl->ctrl, &opts->nr_io_queues);
1815 	if (ret)
1816 		return ret;
1817 
1818 	ctrl->queue_count = opts->nr_io_queues + 1;
1819 	if (ctrl->queue_count < 2)
1820 		return 0;
1821 
1822 	dev_info(ctrl->ctrl.device,
1823 		"creating %d I/O queues.\n", opts->nr_io_queues);
1824 
1825 	ret = nvme_rdma_init_io_queues(ctrl);
1826 	if (ret)
1827 		return ret;
1828 
1829 	/*
1830 	 * We need a reference on the device as long as the tag_set is alive,
1831 	 * as the MRs in the request structures need a valid ib_device.
1832 	 */
1833 	ret = -EINVAL;
1834 	if (!nvme_rdma_dev_get(ctrl->device))
1835 		goto out_free_io_queues;
1836 
1837 	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
1838 	ctrl->tag_set.ops = &nvme_rdma_mq_ops;
1839 	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
1840 	ctrl->tag_set.reserved_tags = 1; /* fabric connect */
1841 	ctrl->tag_set.numa_node = NUMA_NO_NODE;
1842 	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1843 	ctrl->tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
1844 		SG_CHUNK_SIZE * sizeof(struct scatterlist);
1845 	ctrl->tag_set.driver_data = ctrl;
1846 	ctrl->tag_set.nr_hw_queues = ctrl->queue_count - 1;
1847 	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
1848 
1849 	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
1850 	if (ret)
1851 		goto out_put_dev;
1852 	ctrl->ctrl.tagset = &ctrl->tag_set;
1853 
1854 	ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
1855 	if (IS_ERR(ctrl->ctrl.connect_q)) {
1856 		ret = PTR_ERR(ctrl->ctrl.connect_q);
1857 		goto out_free_tag_set;
1858 	}
1859 
1860 	ret = nvme_rdma_connect_io_queues(ctrl);
1861 	if (ret)
1862 		goto out_cleanup_connect_q;
1863 
1864 	return 0;
1865 
1866 out_cleanup_connect_q:
1867 	blk_cleanup_queue(ctrl->ctrl.connect_q);
1868 out_free_tag_set:
1869 	blk_mq_free_tag_set(&ctrl->tag_set);
1870 out_put_dev:
1871 	nvme_rdma_dev_put(ctrl->device);
1872 out_free_io_queues:
1873 	nvme_rdma_free_io_queues(ctrl);
1874 	return ret;
1875 }
1876 
1877 static int nvme_rdma_parse_ipaddr(struct sockaddr_in *in_addr, char *p)
1878 {
1879 	u8 *addr = (u8 *)&in_addr->sin_addr.s_addr;
1880 	size_t buflen = strlen(p);
1881 
1882 	/* XXX: handle IPv6 addresses */
1883 
1884 	if (buflen > INET_ADDRSTRLEN)
1885 		return -EINVAL;
1886 	if (in4_pton(p, buflen, addr, '\0', NULL) == 0)
1887 		return -EINVAL;
1888 	in_addr->sin_family = AF_INET;
1889 	return 0;
1890 }
1891 
1892 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
1893 		struct nvmf_ctrl_options *opts)
1894 {
1895 	struct nvme_rdma_ctrl *ctrl;
1896 	int ret;
1897 	bool changed;
1898 
1899 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
1900 	if (!ctrl)
1901 		return ERR_PTR(-ENOMEM);
1902 	ctrl->ctrl.opts = opts;
1903 	INIT_LIST_HEAD(&ctrl->list);
1904 
1905 	ret = nvme_rdma_parse_ipaddr(&ctrl->addr_in, opts->traddr);
1906 	if (ret) {
1907 		pr_err("malformed IP address passed: %s\n", opts->traddr);
1908 		goto out_free_ctrl;
1909 	}
1910 
1911 	if (opts->mask & NVMF_OPT_TRSVCID) {
1912 		u16 port;
1913 
1914 		ret = kstrtou16(opts->trsvcid, 0, &port);
1915 		if (ret)
1916 			goto out_free_ctrl;
1917 
1918 		ctrl->addr_in.sin_port = cpu_to_be16(port);
1919 	} else {
1920 		ctrl->addr_in.sin_port = cpu_to_be16(NVME_RDMA_IP_PORT);
1921 	}
1922 
1923 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
1924 				0 /* no quirks, we're perfect! */);
1925 	if (ret)
1926 		goto out_free_ctrl;
1927 
1928 	ctrl->reconnect_delay = opts->reconnect_delay;
1929 	INIT_DELAYED_WORK(&ctrl->reconnect_work,
1930 			nvme_rdma_reconnect_ctrl_work);
1931 	INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
1932 	INIT_WORK(&ctrl->delete_work, nvme_rdma_del_ctrl_work);
1933 	INIT_WORK(&ctrl->reset_work, nvme_rdma_reset_ctrl_work);
1934 	spin_lock_init(&ctrl->lock);
1935 
1936 	ctrl->queue_count = opts->nr_io_queues + 1; /* +1 for admin queue */
1937 	ctrl->ctrl.sqsize = opts->queue_size - 1;
1938 	ctrl->ctrl.kato = opts->kato;
1939 
1940 	ret = -ENOMEM;
1941 	ctrl->queues = kcalloc(ctrl->queue_count, sizeof(*ctrl->queues),
1942 				GFP_KERNEL);
1943 	if (!ctrl->queues)
1944 		goto out_uninit_ctrl;
1945 
1946 	ret = nvme_rdma_configure_admin_queue(ctrl);
1947 	if (ret)
1948 		goto out_kfree_queues;
1949 
1950 	/* sanity check icdoff */
1951 	if (ctrl->ctrl.icdoff) {
1952 		dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
1953 		goto out_remove_admin_queue;
1954 	}
1955 
1956 	/* sanity check keyed sgls */
1957 	if (!(ctrl->ctrl.sgls & (1 << 20))) {
1958 		dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not support\n");
1959 		goto out_remove_admin_queue;
1960 	}
1961 
1962 	if (opts->queue_size > ctrl->ctrl.maxcmd) {
1963 		/* warn if maxcmd is lower than queue_size */
1964 		dev_warn(ctrl->ctrl.device,
1965 			"queue_size %zu > ctrl maxcmd %u, clamping down\n",
1966 			opts->queue_size, ctrl->ctrl.maxcmd);
1967 		opts->queue_size = ctrl->ctrl.maxcmd;
1968 	}
1969 
1970 	if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
1971 		/* warn if sqsize is lower than queue_size */
1972 		dev_warn(ctrl->ctrl.device,
1973 			"queue_size %zu > ctrl sqsize %u, clamping down\n",
1974 			opts->queue_size, ctrl->ctrl.sqsize + 1);
1975 		opts->queue_size = ctrl->ctrl.sqsize + 1;
1976 	}
1977 
1978 	if (opts->nr_io_queues) {
1979 		ret = nvme_rdma_create_io_queues(ctrl);
1980 		if (ret)
1981 			goto out_remove_admin_queue;
1982 	}
1983 
1984 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1985 	WARN_ON_ONCE(!changed);
1986 
1987 	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
1988 		ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1989 
1990 	kref_get(&ctrl->ctrl.kref);
1991 
1992 	mutex_lock(&nvme_rdma_ctrl_mutex);
1993 	list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
1994 	mutex_unlock(&nvme_rdma_ctrl_mutex);
1995 
1996 	if (opts->nr_io_queues) {
1997 		nvme_queue_scan(&ctrl->ctrl);
1998 		nvme_queue_async_events(&ctrl->ctrl);
1999 	}
2000 
2001 	return &ctrl->ctrl;
2002 
2003 out_remove_admin_queue:
2004 	nvme_stop_keep_alive(&ctrl->ctrl);
2005 	nvme_rdma_destroy_admin_queue(ctrl);
2006 out_kfree_queues:
2007 	kfree(ctrl->queues);
2008 out_uninit_ctrl:
2009 	nvme_uninit_ctrl(&ctrl->ctrl);
2010 	nvme_put_ctrl(&ctrl->ctrl);
2011 	if (ret > 0)
2012 		ret = -EIO;
2013 	return ERR_PTR(ret);
2014 out_free_ctrl:
2015 	kfree(ctrl);
2016 	return ERR_PTR(ret);
2017 }
2018 
2019 static struct nvmf_transport_ops nvme_rdma_transport = {
2020 	.name		= "rdma",
2021 	.required_opts	= NVMF_OPT_TRADDR,
2022 	.allowed_opts	= NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY,
2023 	.create_ctrl	= nvme_rdma_create_ctrl,
2024 };
2025 
2026 static void nvme_rdma_add_one(struct ib_device *ib_device)
2027 {
2028 }
2029 
2030 static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
2031 {
2032 	struct nvme_rdma_ctrl *ctrl;
2033 
2034 	/* Delete all controllers using this device */
2035 	mutex_lock(&nvme_rdma_ctrl_mutex);
2036 	list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
2037 		if (ctrl->device->dev != ib_device)
2038 			continue;
2039 		dev_info(ctrl->ctrl.device,
2040 			"Removing ctrl: NQN \"%s\", addr %pISp\n",
2041 			ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2042 		__nvme_rdma_del_ctrl(ctrl);
2043 	}
2044 	mutex_unlock(&nvme_rdma_ctrl_mutex);
2045 
2046 	flush_workqueue(nvme_rdma_wq);
2047 }
2048 
2049 static struct ib_client nvme_rdma_ib_client = {
2050 	.name   = "nvme_rdma",
2051 	.add = nvme_rdma_add_one,
2052 	.remove = nvme_rdma_remove_one
2053 };
2054 
2055 static int __init nvme_rdma_init_module(void)
2056 {
2057 	int ret;
2058 
2059 	nvme_rdma_wq = create_workqueue("nvme_rdma_wq");
2060 	if (!nvme_rdma_wq)
2061 		return -ENOMEM;
2062 
2063 	ret = ib_register_client(&nvme_rdma_ib_client);
2064 	if (ret) {
2065 		destroy_workqueue(nvme_rdma_wq);
2066 		return ret;
2067 	}
2068 
2069 	nvmf_register_transport(&nvme_rdma_transport);
2070 	return 0;
2071 }
2072 
2073 static void __exit nvme_rdma_cleanup_module(void)
2074 {
2075 	nvmf_unregister_transport(&nvme_rdma_transport);
2076 	ib_unregister_client(&nvme_rdma_ib_client);
2077 	destroy_workqueue(nvme_rdma_wq);
2078 }
2079 
2080 module_init(nvme_rdma_init_module);
2081 module_exit(nvme_rdma_cleanup_module);
2082 
2083 MODULE_LICENSE("GPL v2");
2084