xref: /linux/drivers/nvme/host/fc.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
4  */
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/parser.h>
8 #include <uapi/scsi/fc/fc_fs.h>
9 #include <uapi/scsi/fc/fc_els.h>
10 #include <linux/delay.h>
11 #include <linux/overflow.h>
12 #include <linux/blk-cgroup.h>
13 #include "nvme.h"
14 #include "fabrics.h"
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "fc.h"
18 #include <scsi/scsi_transport_fc.h>
19 #include <linux/blk-mq-pci.h>
20 
21 /* *************************** Data Structures/Defines ****************** */
22 
23 
24 enum nvme_fc_queue_flags {
25 	NVME_FC_Q_CONNECTED = 0,
26 	NVME_FC_Q_LIVE,
27 };
28 
29 #define NVME_FC_DEFAULT_DEV_LOSS_TMO	60	/* seconds */
30 #define NVME_FC_DEFAULT_RECONNECT_TMO	2	/* delay between reconnects
31 						 * when connected and a
32 						 * connection failure.
33 						 */
34 
35 struct nvme_fc_queue {
36 	struct nvme_fc_ctrl	*ctrl;
37 	struct device		*dev;
38 	struct blk_mq_hw_ctx	*hctx;
39 	void			*lldd_handle;
40 	size_t			cmnd_capsule_len;
41 	u32			qnum;
42 	u32			rqcnt;
43 	u32			seqno;
44 
45 	u64			connection_id;
46 	atomic_t		csn;
47 
48 	unsigned long		flags;
49 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
50 
51 enum nvme_fcop_flags {
52 	FCOP_FLAGS_TERMIO	= (1 << 0),
53 	FCOP_FLAGS_AEN		= (1 << 1),
54 };
55 
56 struct nvmefc_ls_req_op {
57 	struct nvmefc_ls_req	ls_req;
58 
59 	struct nvme_fc_rport	*rport;
60 	struct nvme_fc_queue	*queue;
61 	struct request		*rq;
62 	u32			flags;
63 
64 	int			ls_error;
65 	struct completion	ls_done;
66 	struct list_head	lsreq_list;	/* rport->ls_req_list */
67 	bool			req_queued;
68 };
69 
70 struct nvmefc_ls_rcv_op {
71 	struct nvme_fc_rport		*rport;
72 	struct nvmefc_ls_rsp		*lsrsp;
73 	union nvmefc_ls_requests	*rqstbuf;
74 	union nvmefc_ls_responses	*rspbuf;
75 	u16				rqstdatalen;
76 	bool				handled;
77 	dma_addr_t			rspdma;
78 	struct list_head		lsrcv_list;	/* rport->ls_rcv_list */
79 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
80 
81 enum nvme_fcpop_state {
82 	FCPOP_STATE_UNINIT	= 0,
83 	FCPOP_STATE_IDLE	= 1,
84 	FCPOP_STATE_ACTIVE	= 2,
85 	FCPOP_STATE_ABORTED	= 3,
86 	FCPOP_STATE_COMPLETE	= 4,
87 };
88 
89 struct nvme_fc_fcp_op {
90 	struct nvme_request	nreq;		/*
91 						 * nvme/host/core.c
92 						 * requires this to be
93 						 * the 1st element in the
94 						 * private structure
95 						 * associated with the
96 						 * request.
97 						 */
98 	struct nvmefc_fcp_req	fcp_req;
99 
100 	struct nvme_fc_ctrl	*ctrl;
101 	struct nvme_fc_queue	*queue;
102 	struct request		*rq;
103 
104 	atomic_t		state;
105 	u32			flags;
106 	u32			rqno;
107 	u32			nents;
108 
109 	struct nvme_fc_cmd_iu	cmd_iu;
110 	struct nvme_fc_ersp_iu	rsp_iu;
111 };
112 
113 struct nvme_fcp_op_w_sgl {
114 	struct nvme_fc_fcp_op	op;
115 	struct scatterlist	sgl[NVME_INLINE_SG_CNT];
116 	uint8_t			priv[];
117 };
118 
119 struct nvme_fc_lport {
120 	struct nvme_fc_local_port	localport;
121 
122 	struct ida			endp_cnt;
123 	struct list_head		port_list;	/* nvme_fc_port_list */
124 	struct list_head		endp_list;
125 	struct device			*dev;	/* physical device for dma */
126 	struct nvme_fc_port_template	*ops;
127 	struct kref			ref;
128 	atomic_t                        act_rport_cnt;
129 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
130 
131 struct nvme_fc_rport {
132 	struct nvme_fc_remote_port	remoteport;
133 
134 	struct list_head		endp_list; /* for lport->endp_list */
135 	struct list_head		ctrl_list;
136 	struct list_head		ls_req_list;
137 	struct list_head		ls_rcv_list;
138 	struct list_head		disc_list;
139 	struct device			*dev;	/* physical device for dma */
140 	struct nvme_fc_lport		*lport;
141 	spinlock_t			lock;
142 	struct kref			ref;
143 	atomic_t                        act_ctrl_cnt;
144 	unsigned long			dev_loss_end;
145 	struct work_struct		lsrcv_work;
146 } __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
147 
148 /* fc_ctrl flags values - specified as bit positions */
149 #define ASSOC_ACTIVE		0
150 #define ASSOC_FAILED		1
151 #define FCCTRL_TERMIO		2
152 
153 struct nvme_fc_ctrl {
154 	spinlock_t		lock;
155 	struct nvme_fc_queue	*queues;
156 	struct device		*dev;
157 	struct nvme_fc_lport	*lport;
158 	struct nvme_fc_rport	*rport;
159 	u32			cnum;
160 
161 	bool			ioq_live;
162 	u64			association_id;
163 	struct nvmefc_ls_rcv_op	*rcv_disconn;
164 
165 	struct list_head	ctrl_list;	/* rport->ctrl_list */
166 
167 	struct blk_mq_tag_set	admin_tag_set;
168 	struct blk_mq_tag_set	tag_set;
169 
170 	struct work_struct	ioerr_work;
171 	struct delayed_work	connect_work;
172 
173 	struct kref		ref;
174 	unsigned long		flags;
175 	u32			iocnt;
176 	wait_queue_head_t	ioabort_wait;
177 
178 	struct nvme_fc_fcp_op	aen_ops[NVME_NR_AEN_COMMANDS];
179 
180 	struct nvme_ctrl	ctrl;
181 };
182 
183 static inline struct nvme_fc_ctrl *
to_fc_ctrl(struct nvme_ctrl * ctrl)184 to_fc_ctrl(struct nvme_ctrl *ctrl)
185 {
186 	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
187 }
188 
189 static inline struct nvme_fc_lport *
localport_to_lport(struct nvme_fc_local_port * portptr)190 localport_to_lport(struct nvme_fc_local_port *portptr)
191 {
192 	return container_of(portptr, struct nvme_fc_lport, localport);
193 }
194 
195 static inline struct nvme_fc_rport *
remoteport_to_rport(struct nvme_fc_remote_port * portptr)196 remoteport_to_rport(struct nvme_fc_remote_port *portptr)
197 {
198 	return container_of(portptr, struct nvme_fc_rport, remoteport);
199 }
200 
201 static inline struct nvmefc_ls_req_op *
ls_req_to_lsop(struct nvmefc_ls_req * lsreq)202 ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
203 {
204 	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
205 }
206 
207 static inline struct nvme_fc_fcp_op *
fcp_req_to_fcp_op(struct nvmefc_fcp_req * fcpreq)208 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
209 {
210 	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
211 }
212 
213 
214 
215 /* *************************** Globals **************************** */
216 
217 
218 static DEFINE_SPINLOCK(nvme_fc_lock);
219 
220 static LIST_HEAD(nvme_fc_lport_list);
221 static DEFINE_IDA(nvme_fc_local_port_cnt);
222 static DEFINE_IDA(nvme_fc_ctrl_cnt);
223 
224 /*
225  * These items are short-term. They will eventually be moved into
226  * a generic FC class. See comments in module init.
227  */
228 static struct device *fc_udev_device;
229 
230 static void nvme_fc_complete_rq(struct request *rq);
231 
232 /* *********************** FC-NVME Port Management ************************ */
233 
234 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
235 			struct nvme_fc_queue *, unsigned int);
236 
237 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
238 
239 
240 static void
nvme_fc_free_lport(struct kref * ref)241 nvme_fc_free_lport(struct kref *ref)
242 {
243 	struct nvme_fc_lport *lport =
244 		container_of(ref, struct nvme_fc_lport, ref);
245 	unsigned long flags;
246 
247 	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
248 	WARN_ON(!list_empty(&lport->endp_list));
249 
250 	/* remove from transport list */
251 	spin_lock_irqsave(&nvme_fc_lock, flags);
252 	list_del(&lport->port_list);
253 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
254 
255 	ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
256 	ida_destroy(&lport->endp_cnt);
257 
258 	put_device(lport->dev);
259 
260 	kfree(lport);
261 }
262 
263 static void
nvme_fc_lport_put(struct nvme_fc_lport * lport)264 nvme_fc_lport_put(struct nvme_fc_lport *lport)
265 {
266 	kref_put(&lport->ref, nvme_fc_free_lport);
267 }
268 
269 static int
nvme_fc_lport_get(struct nvme_fc_lport * lport)270 nvme_fc_lport_get(struct nvme_fc_lport *lport)
271 {
272 	return kref_get_unless_zero(&lport->ref);
273 }
274 
275 
276 static struct nvme_fc_lport *
nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info * pinfo,struct nvme_fc_port_template * ops,struct device * dev)277 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
278 			struct nvme_fc_port_template *ops,
279 			struct device *dev)
280 {
281 	struct nvme_fc_lport *lport;
282 	unsigned long flags;
283 
284 	spin_lock_irqsave(&nvme_fc_lock, flags);
285 
286 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
287 		if (lport->localport.node_name != pinfo->node_name ||
288 		    lport->localport.port_name != pinfo->port_name)
289 			continue;
290 
291 		if (lport->dev != dev) {
292 			lport = ERR_PTR(-EXDEV);
293 			goto out_done;
294 		}
295 
296 		if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
297 			lport = ERR_PTR(-EEXIST);
298 			goto out_done;
299 		}
300 
301 		if (!nvme_fc_lport_get(lport)) {
302 			/*
303 			 * fails if ref cnt already 0. If so,
304 			 * act as if lport already deleted
305 			 */
306 			lport = NULL;
307 			goto out_done;
308 		}
309 
310 		/* resume the lport */
311 
312 		lport->ops = ops;
313 		lport->localport.port_role = pinfo->port_role;
314 		lport->localport.port_id = pinfo->port_id;
315 		lport->localport.port_state = FC_OBJSTATE_ONLINE;
316 
317 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
318 
319 		return lport;
320 	}
321 
322 	lport = NULL;
323 
324 out_done:
325 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
326 
327 	return lport;
328 }
329 
330 /**
331  * nvme_fc_register_localport - transport entry point called by an
332  *                              LLDD to register the existence of a NVME
333  *                              host FC port.
334  * @pinfo:     pointer to information about the port to be registered
335  * @template:  LLDD entrypoints and operational parameters for the port
336  * @dev:       physical hardware device node port corresponds to. Will be
337  *             used for DMA mappings
338  * @portptr:   pointer to a local port pointer. Upon success, the routine
339  *             will allocate a nvme_fc_local_port structure and place its
340  *             address in the local port pointer. Upon failure, local port
341  *             pointer will be set to 0.
342  *
343  * Returns:
344  * a completion status. Must be 0 upon success; a negative errno
345  * (ex: -ENXIO) upon failure.
346  */
347 int
nvme_fc_register_localport(struct nvme_fc_port_info * pinfo,struct nvme_fc_port_template * template,struct device * dev,struct nvme_fc_local_port ** portptr)348 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
349 			struct nvme_fc_port_template *template,
350 			struct device *dev,
351 			struct nvme_fc_local_port **portptr)
352 {
353 	struct nvme_fc_lport *newrec;
354 	unsigned long flags;
355 	int ret, idx;
356 
357 	if (!template->localport_delete || !template->remoteport_delete ||
358 	    !template->ls_req || !template->fcp_io ||
359 	    !template->ls_abort || !template->fcp_abort ||
360 	    !template->max_hw_queues || !template->max_sgl_segments ||
361 	    !template->max_dif_sgl_segments || !template->dma_boundary) {
362 		ret = -EINVAL;
363 		goto out_reghost_failed;
364 	}
365 
366 	/*
367 	 * look to see if there is already a localport that had been
368 	 * deregistered and in the process of waiting for all the
369 	 * references to fully be removed.  If the references haven't
370 	 * expired, we can simply re-enable the localport. Remoteports
371 	 * and controller reconnections should resume naturally.
372 	 */
373 	newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
374 
375 	/* found an lport, but something about its state is bad */
376 	if (IS_ERR(newrec)) {
377 		ret = PTR_ERR(newrec);
378 		goto out_reghost_failed;
379 
380 	/* found existing lport, which was resumed */
381 	} else if (newrec) {
382 		*portptr = &newrec->localport;
383 		return 0;
384 	}
385 
386 	/* nothing found - allocate a new localport struct */
387 
388 	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
389 			 GFP_KERNEL);
390 	if (!newrec) {
391 		ret = -ENOMEM;
392 		goto out_reghost_failed;
393 	}
394 
395 	idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
396 	if (idx < 0) {
397 		ret = -ENOSPC;
398 		goto out_fail_kfree;
399 	}
400 
401 	if (!get_device(dev) && dev) {
402 		ret = -ENODEV;
403 		goto out_ida_put;
404 	}
405 
406 	INIT_LIST_HEAD(&newrec->port_list);
407 	INIT_LIST_HEAD(&newrec->endp_list);
408 	kref_init(&newrec->ref);
409 	atomic_set(&newrec->act_rport_cnt, 0);
410 	newrec->ops = template;
411 	newrec->dev = dev;
412 	ida_init(&newrec->endp_cnt);
413 	if (template->local_priv_sz)
414 		newrec->localport.private = &newrec[1];
415 	else
416 		newrec->localport.private = NULL;
417 	newrec->localport.node_name = pinfo->node_name;
418 	newrec->localport.port_name = pinfo->port_name;
419 	newrec->localport.port_role = pinfo->port_role;
420 	newrec->localport.port_id = pinfo->port_id;
421 	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
422 	newrec->localport.port_num = idx;
423 
424 	spin_lock_irqsave(&nvme_fc_lock, flags);
425 	list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
426 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
427 
428 	if (dev)
429 		dma_set_seg_boundary(dev, template->dma_boundary);
430 
431 	*portptr = &newrec->localport;
432 	return 0;
433 
434 out_ida_put:
435 	ida_free(&nvme_fc_local_port_cnt, idx);
436 out_fail_kfree:
437 	kfree(newrec);
438 out_reghost_failed:
439 	*portptr = NULL;
440 
441 	return ret;
442 }
443 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
444 
445 /**
446  * nvme_fc_unregister_localport - transport entry point called by an
447  *                              LLDD to deregister/remove a previously
448  *                              registered a NVME host FC port.
449  * @portptr: pointer to the (registered) local port that is to be deregistered.
450  *
451  * Returns:
452  * a completion status. Must be 0 upon success; a negative errno
453  * (ex: -ENXIO) upon failure.
454  */
455 int
nvme_fc_unregister_localport(struct nvme_fc_local_port * portptr)456 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
457 {
458 	struct nvme_fc_lport *lport = localport_to_lport(portptr);
459 	unsigned long flags;
460 
461 	if (!portptr)
462 		return -EINVAL;
463 
464 	spin_lock_irqsave(&nvme_fc_lock, flags);
465 
466 	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
467 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
468 		return -EINVAL;
469 	}
470 	portptr->port_state = FC_OBJSTATE_DELETED;
471 
472 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
473 
474 	if (atomic_read(&lport->act_rport_cnt) == 0)
475 		lport->ops->localport_delete(&lport->localport);
476 
477 	nvme_fc_lport_put(lport);
478 
479 	return 0;
480 }
481 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
482 
483 /*
484  * TRADDR strings, per FC-NVME are fixed format:
485  *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
486  * udev event will only differ by prefix of what field is
487  * being specified:
488  *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
489  *  19 + 43 + null_fudge = 64 characters
490  */
491 #define FCNVME_TRADDR_LENGTH		64
492 
493 static void
nvme_fc_signal_discovery_scan(struct nvme_fc_lport * lport,struct nvme_fc_rport * rport)494 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
495 		struct nvme_fc_rport *rport)
496 {
497 	char hostaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_HOST_TRADDR=...*/
498 	char tgtaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_TRADDR=...*/
499 	char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
500 
501 	if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
502 		return;
503 
504 	snprintf(hostaddr, sizeof(hostaddr),
505 		"NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
506 		lport->localport.node_name, lport->localport.port_name);
507 	snprintf(tgtaddr, sizeof(tgtaddr),
508 		"NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
509 		rport->remoteport.node_name, rport->remoteport.port_name);
510 	kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
511 }
512 
513 static void
nvme_fc_free_rport(struct kref * ref)514 nvme_fc_free_rport(struct kref *ref)
515 {
516 	struct nvme_fc_rport *rport =
517 		container_of(ref, struct nvme_fc_rport, ref);
518 	struct nvme_fc_lport *lport =
519 			localport_to_lport(rport->remoteport.localport);
520 	unsigned long flags;
521 
522 	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
523 	WARN_ON(!list_empty(&rport->ctrl_list));
524 
525 	/* remove from lport list */
526 	spin_lock_irqsave(&nvme_fc_lock, flags);
527 	list_del(&rport->endp_list);
528 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
529 
530 	WARN_ON(!list_empty(&rport->disc_list));
531 	ida_free(&lport->endp_cnt, rport->remoteport.port_num);
532 
533 	kfree(rport);
534 
535 	nvme_fc_lport_put(lport);
536 }
537 
538 static void
nvme_fc_rport_put(struct nvme_fc_rport * rport)539 nvme_fc_rport_put(struct nvme_fc_rport *rport)
540 {
541 	kref_put(&rport->ref, nvme_fc_free_rport);
542 }
543 
544 static int
nvme_fc_rport_get(struct nvme_fc_rport * rport)545 nvme_fc_rport_get(struct nvme_fc_rport *rport)
546 {
547 	return kref_get_unless_zero(&rport->ref);
548 }
549 
550 static void
nvme_fc_resume_controller(struct nvme_fc_ctrl * ctrl)551 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
552 {
553 	switch (nvme_ctrl_state(&ctrl->ctrl)) {
554 	case NVME_CTRL_NEW:
555 	case NVME_CTRL_CONNECTING:
556 		/*
557 		 * As all reconnects were suppressed, schedule a
558 		 * connect.
559 		 */
560 		dev_info(ctrl->ctrl.device,
561 			"NVME-FC{%d}: connectivity re-established. "
562 			"Attempting reconnect\n", ctrl->cnum);
563 
564 		queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
565 		break;
566 
567 	case NVME_CTRL_RESETTING:
568 		/*
569 		 * Controller is already in the process of terminating the
570 		 * association. No need to do anything further. The reconnect
571 		 * step will naturally occur after the reset completes.
572 		 */
573 		break;
574 
575 	default:
576 		/* no action to take - let it delete */
577 		break;
578 	}
579 }
580 
581 static struct nvme_fc_rport *
nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport * lport,struct nvme_fc_port_info * pinfo)582 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
583 				struct nvme_fc_port_info *pinfo)
584 {
585 	struct nvme_fc_rport *rport;
586 	struct nvme_fc_ctrl *ctrl;
587 	unsigned long flags;
588 
589 	spin_lock_irqsave(&nvme_fc_lock, flags);
590 
591 	list_for_each_entry(rport, &lport->endp_list, endp_list) {
592 		if (rport->remoteport.node_name != pinfo->node_name ||
593 		    rport->remoteport.port_name != pinfo->port_name)
594 			continue;
595 
596 		if (!nvme_fc_rport_get(rport)) {
597 			rport = ERR_PTR(-ENOLCK);
598 			goto out_done;
599 		}
600 
601 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
602 
603 		spin_lock_irqsave(&rport->lock, flags);
604 
605 		/* has it been unregistered */
606 		if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
607 			/* means lldd called us twice */
608 			spin_unlock_irqrestore(&rport->lock, flags);
609 			nvme_fc_rport_put(rport);
610 			return ERR_PTR(-ESTALE);
611 		}
612 
613 		rport->remoteport.port_role = pinfo->port_role;
614 		rport->remoteport.port_id = pinfo->port_id;
615 		rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
616 		rport->dev_loss_end = 0;
617 
618 		/*
619 		 * kick off a reconnect attempt on all associations to the
620 		 * remote port. A successful reconnects will resume i/o.
621 		 */
622 		list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
623 			nvme_fc_resume_controller(ctrl);
624 
625 		spin_unlock_irqrestore(&rport->lock, flags);
626 
627 		return rport;
628 	}
629 
630 	rport = NULL;
631 
632 out_done:
633 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
634 
635 	return rport;
636 }
637 
638 static inline void
__nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport * rport,struct nvme_fc_port_info * pinfo)639 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
640 			struct nvme_fc_port_info *pinfo)
641 {
642 	if (pinfo->dev_loss_tmo)
643 		rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
644 	else
645 		rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
646 }
647 
648 /**
649  * nvme_fc_register_remoteport - transport entry point called by an
650  *                              LLDD to register the existence of a NVME
651  *                              subsystem FC port on its fabric.
652  * @localport: pointer to the (registered) local port that the remote
653  *             subsystem port is connected to.
654  * @pinfo:     pointer to information about the port to be registered
655  * @portptr:   pointer to a remote port pointer. Upon success, the routine
656  *             will allocate a nvme_fc_remote_port structure and place its
657  *             address in the remote port pointer. Upon failure, remote port
658  *             pointer will be set to 0.
659  *
660  * Returns:
661  * a completion status. Must be 0 upon success; a negative errno
662  * (ex: -ENXIO) upon failure.
663  */
664 int
nvme_fc_register_remoteport(struct nvme_fc_local_port * localport,struct nvme_fc_port_info * pinfo,struct nvme_fc_remote_port ** portptr)665 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
666 				struct nvme_fc_port_info *pinfo,
667 				struct nvme_fc_remote_port **portptr)
668 {
669 	struct nvme_fc_lport *lport = localport_to_lport(localport);
670 	struct nvme_fc_rport *newrec;
671 	unsigned long flags;
672 	int ret, idx;
673 
674 	if (!nvme_fc_lport_get(lport)) {
675 		ret = -ESHUTDOWN;
676 		goto out_reghost_failed;
677 	}
678 
679 	/*
680 	 * look to see if there is already a remoteport that is waiting
681 	 * for a reconnect (within dev_loss_tmo) with the same WWN's.
682 	 * If so, transition to it and reconnect.
683 	 */
684 	newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
685 
686 	/* found an rport, but something about its state is bad */
687 	if (IS_ERR(newrec)) {
688 		ret = PTR_ERR(newrec);
689 		goto out_lport_put;
690 
691 	/* found existing rport, which was resumed */
692 	} else if (newrec) {
693 		nvme_fc_lport_put(lport);
694 		__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
695 		nvme_fc_signal_discovery_scan(lport, newrec);
696 		*portptr = &newrec->remoteport;
697 		return 0;
698 	}
699 
700 	/* nothing found - allocate a new remoteport struct */
701 
702 	newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
703 			 GFP_KERNEL);
704 	if (!newrec) {
705 		ret = -ENOMEM;
706 		goto out_lport_put;
707 	}
708 
709 	idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
710 	if (idx < 0) {
711 		ret = -ENOSPC;
712 		goto out_kfree_rport;
713 	}
714 
715 	INIT_LIST_HEAD(&newrec->endp_list);
716 	INIT_LIST_HEAD(&newrec->ctrl_list);
717 	INIT_LIST_HEAD(&newrec->ls_req_list);
718 	INIT_LIST_HEAD(&newrec->disc_list);
719 	kref_init(&newrec->ref);
720 	atomic_set(&newrec->act_ctrl_cnt, 0);
721 	spin_lock_init(&newrec->lock);
722 	newrec->remoteport.localport = &lport->localport;
723 	INIT_LIST_HEAD(&newrec->ls_rcv_list);
724 	newrec->dev = lport->dev;
725 	newrec->lport = lport;
726 	if (lport->ops->remote_priv_sz)
727 		newrec->remoteport.private = &newrec[1];
728 	else
729 		newrec->remoteport.private = NULL;
730 	newrec->remoteport.port_role = pinfo->port_role;
731 	newrec->remoteport.node_name = pinfo->node_name;
732 	newrec->remoteport.port_name = pinfo->port_name;
733 	newrec->remoteport.port_id = pinfo->port_id;
734 	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
735 	newrec->remoteport.port_num = idx;
736 	__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
737 	INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
738 
739 	spin_lock_irqsave(&nvme_fc_lock, flags);
740 	list_add_tail(&newrec->endp_list, &lport->endp_list);
741 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
742 
743 	nvme_fc_signal_discovery_scan(lport, newrec);
744 
745 	*portptr = &newrec->remoteport;
746 	return 0;
747 
748 out_kfree_rport:
749 	kfree(newrec);
750 out_lport_put:
751 	nvme_fc_lport_put(lport);
752 out_reghost_failed:
753 	*portptr = NULL;
754 	return ret;
755 }
756 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
757 
758 static int
nvme_fc_abort_lsops(struct nvme_fc_rport * rport)759 nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
760 {
761 	struct nvmefc_ls_req_op *lsop;
762 	unsigned long flags;
763 
764 restart:
765 	spin_lock_irqsave(&rport->lock, flags);
766 
767 	list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
768 		if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
769 			lsop->flags |= FCOP_FLAGS_TERMIO;
770 			spin_unlock_irqrestore(&rport->lock, flags);
771 			rport->lport->ops->ls_abort(&rport->lport->localport,
772 						&rport->remoteport,
773 						&lsop->ls_req);
774 			goto restart;
775 		}
776 	}
777 	spin_unlock_irqrestore(&rport->lock, flags);
778 
779 	return 0;
780 }
781 
782 static void
nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl * ctrl)783 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
784 {
785 	dev_info(ctrl->ctrl.device,
786 		"NVME-FC{%d}: controller connectivity lost. Awaiting "
787 		"Reconnect", ctrl->cnum);
788 
789 	switch (nvme_ctrl_state(&ctrl->ctrl)) {
790 	case NVME_CTRL_NEW:
791 	case NVME_CTRL_LIVE:
792 		/*
793 		 * Schedule a controller reset. The reset will terminate the
794 		 * association and schedule the reconnect timer.  Reconnects
795 		 * will be attempted until either the ctlr_loss_tmo
796 		 * (max_retries * connect_delay) expires or the remoteport's
797 		 * dev_loss_tmo expires.
798 		 */
799 		if (nvme_reset_ctrl(&ctrl->ctrl)) {
800 			dev_warn(ctrl->ctrl.device,
801 				"NVME-FC{%d}: Couldn't schedule reset.\n",
802 				ctrl->cnum);
803 			nvme_delete_ctrl(&ctrl->ctrl);
804 		}
805 		break;
806 
807 	case NVME_CTRL_CONNECTING:
808 		/*
809 		 * The association has already been terminated and the
810 		 * controller is attempting reconnects.  No need to do anything
811 		 * futher.  Reconnects will be attempted until either the
812 		 * ctlr_loss_tmo (max_retries * connect_delay) expires or the
813 		 * remoteport's dev_loss_tmo expires.
814 		 */
815 		break;
816 
817 	case NVME_CTRL_RESETTING:
818 		/*
819 		 * Controller is already in the process of terminating the
820 		 * association.  No need to do anything further. The reconnect
821 		 * step will kick in naturally after the association is
822 		 * terminated.
823 		 */
824 		break;
825 
826 	case NVME_CTRL_DELETING:
827 	case NVME_CTRL_DELETING_NOIO:
828 	default:
829 		/* no action to take - let it delete */
830 		break;
831 	}
832 }
833 
834 /**
835  * nvme_fc_unregister_remoteport - transport entry point called by an
836  *                              LLDD to deregister/remove a previously
837  *                              registered a NVME subsystem FC port.
838  * @portptr: pointer to the (registered) remote port that is to be
839  *           deregistered.
840  *
841  * Returns:
842  * a completion status. Must be 0 upon success; a negative errno
843  * (ex: -ENXIO) upon failure.
844  */
845 int
nvme_fc_unregister_remoteport(struct nvme_fc_remote_port * portptr)846 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
847 {
848 	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
849 	struct nvme_fc_ctrl *ctrl;
850 	unsigned long flags;
851 
852 	if (!portptr)
853 		return -EINVAL;
854 
855 	spin_lock_irqsave(&rport->lock, flags);
856 
857 	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
858 		spin_unlock_irqrestore(&rport->lock, flags);
859 		return -EINVAL;
860 	}
861 	portptr->port_state = FC_OBJSTATE_DELETED;
862 
863 	rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
864 
865 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
866 		/* if dev_loss_tmo==0, dev loss is immediate */
867 		if (!portptr->dev_loss_tmo) {
868 			dev_warn(ctrl->ctrl.device,
869 				"NVME-FC{%d}: controller connectivity lost.\n",
870 				ctrl->cnum);
871 			nvme_delete_ctrl(&ctrl->ctrl);
872 		} else
873 			nvme_fc_ctrl_connectivity_loss(ctrl);
874 	}
875 
876 	spin_unlock_irqrestore(&rport->lock, flags);
877 
878 	nvme_fc_abort_lsops(rport);
879 
880 	if (atomic_read(&rport->act_ctrl_cnt) == 0)
881 		rport->lport->ops->remoteport_delete(portptr);
882 
883 	/*
884 	 * release the reference, which will allow, if all controllers
885 	 * go away, which should only occur after dev_loss_tmo occurs,
886 	 * for the rport to be torn down.
887 	 */
888 	nvme_fc_rport_put(rport);
889 
890 	return 0;
891 }
892 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
893 
894 /**
895  * nvme_fc_rescan_remoteport - transport entry point called by an
896  *                              LLDD to request a nvme device rescan.
897  * @remoteport: pointer to the (registered) remote port that is to be
898  *              rescanned.
899  *
900  * Returns: N/A
901  */
902 void
nvme_fc_rescan_remoteport(struct nvme_fc_remote_port * remoteport)903 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
904 {
905 	struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
906 
907 	nvme_fc_signal_discovery_scan(rport->lport, rport);
908 }
909 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
910 
911 int
nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port * portptr,u32 dev_loss_tmo)912 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
913 			u32 dev_loss_tmo)
914 {
915 	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
916 	unsigned long flags;
917 
918 	spin_lock_irqsave(&rport->lock, flags);
919 
920 	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
921 		spin_unlock_irqrestore(&rport->lock, flags);
922 		return -EINVAL;
923 	}
924 
925 	/* a dev_loss_tmo of 0 (immediate) is allowed to be set */
926 	rport->remoteport.dev_loss_tmo = dev_loss_tmo;
927 
928 	spin_unlock_irqrestore(&rport->lock, flags);
929 
930 	return 0;
931 }
932 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
933 
934 
935 /* *********************** FC-NVME DMA Handling **************************** */
936 
937 /*
938  * The fcloop device passes in a NULL device pointer. Real LLD's will
939  * pass in a valid device pointer. If NULL is passed to the dma mapping
940  * routines, depending on the platform, it may or may not succeed, and
941  * may crash.
942  *
943  * As such:
944  * Wrapper all the dma routines and check the dev pointer.
945  *
946  * If simple mappings (return just a dma address, we'll noop them,
947  * returning a dma address of 0.
948  *
949  * On more complex mappings (dma_map_sg), a pseudo routine fills
950  * in the scatter list, setting all dma addresses to 0.
951  */
952 
953 static inline dma_addr_t
fc_dma_map_single(struct device * dev,void * ptr,size_t size,enum dma_data_direction dir)954 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
955 		enum dma_data_direction dir)
956 {
957 	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
958 }
959 
960 static inline int
fc_dma_mapping_error(struct device * dev,dma_addr_t dma_addr)961 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
962 {
963 	return dev ? dma_mapping_error(dev, dma_addr) : 0;
964 }
965 
966 static inline void
fc_dma_unmap_single(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)967 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
968 	enum dma_data_direction dir)
969 {
970 	if (dev)
971 		dma_unmap_single(dev, addr, size, dir);
972 }
973 
974 static inline void
fc_dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)975 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
976 		enum dma_data_direction dir)
977 {
978 	if (dev)
979 		dma_sync_single_for_cpu(dev, addr, size, dir);
980 }
981 
982 static inline void
fc_dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)983 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
984 		enum dma_data_direction dir)
985 {
986 	if (dev)
987 		dma_sync_single_for_device(dev, addr, size, dir);
988 }
989 
990 /* pseudo dma_map_sg call */
991 static int
fc_map_sg(struct scatterlist * sg,int nents)992 fc_map_sg(struct scatterlist *sg, int nents)
993 {
994 	struct scatterlist *s;
995 	int i;
996 
997 	WARN_ON(nents == 0 || sg[0].length == 0);
998 
999 	for_each_sg(sg, s, nents, i) {
1000 		s->dma_address = 0L;
1001 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1002 		s->dma_length = s->length;
1003 #endif
1004 	}
1005 	return nents;
1006 }
1007 
1008 static inline int
fc_dma_map_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)1009 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1010 		enum dma_data_direction dir)
1011 {
1012 	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1013 }
1014 
1015 static inline void
fc_dma_unmap_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)1016 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1017 		enum dma_data_direction dir)
1018 {
1019 	if (dev)
1020 		dma_unmap_sg(dev, sg, nents, dir);
1021 }
1022 
1023 /* *********************** FC-NVME LS Handling **************************** */
1024 
1025 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1026 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1027 
1028 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1029 
1030 static void
__nvme_fc_finish_ls_req(struct nvmefc_ls_req_op * lsop)1031 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1032 {
1033 	struct nvme_fc_rport *rport = lsop->rport;
1034 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1035 	unsigned long flags;
1036 
1037 	spin_lock_irqsave(&rport->lock, flags);
1038 
1039 	if (!lsop->req_queued) {
1040 		spin_unlock_irqrestore(&rport->lock, flags);
1041 		return;
1042 	}
1043 
1044 	list_del(&lsop->lsreq_list);
1045 
1046 	lsop->req_queued = false;
1047 
1048 	spin_unlock_irqrestore(&rport->lock, flags);
1049 
1050 	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1051 				  (lsreq->rqstlen + lsreq->rsplen),
1052 				  DMA_BIDIRECTIONAL);
1053 
1054 	nvme_fc_rport_put(rport);
1055 }
1056 
1057 static int
__nvme_fc_send_ls_req(struct nvme_fc_rport * rport,struct nvmefc_ls_req_op * lsop,void (* done)(struct nvmefc_ls_req * req,int status))1058 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1059 		struct nvmefc_ls_req_op *lsop,
1060 		void (*done)(struct nvmefc_ls_req *req, int status))
1061 {
1062 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1063 	unsigned long flags;
1064 	int ret = 0;
1065 
1066 	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1067 		return -ECONNREFUSED;
1068 
1069 	if (!nvme_fc_rport_get(rport))
1070 		return -ESHUTDOWN;
1071 
1072 	lsreq->done = done;
1073 	lsop->rport = rport;
1074 	lsop->req_queued = false;
1075 	INIT_LIST_HEAD(&lsop->lsreq_list);
1076 	init_completion(&lsop->ls_done);
1077 
1078 	lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1079 				  lsreq->rqstlen + lsreq->rsplen,
1080 				  DMA_BIDIRECTIONAL);
1081 	if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1082 		ret = -EFAULT;
1083 		goto out_putrport;
1084 	}
1085 	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1086 
1087 	spin_lock_irqsave(&rport->lock, flags);
1088 
1089 	list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1090 
1091 	lsop->req_queued = true;
1092 
1093 	spin_unlock_irqrestore(&rport->lock, flags);
1094 
1095 	ret = rport->lport->ops->ls_req(&rport->lport->localport,
1096 					&rport->remoteport, lsreq);
1097 	if (ret)
1098 		goto out_unlink;
1099 
1100 	return 0;
1101 
1102 out_unlink:
1103 	lsop->ls_error = ret;
1104 	spin_lock_irqsave(&rport->lock, flags);
1105 	lsop->req_queued = false;
1106 	list_del(&lsop->lsreq_list);
1107 	spin_unlock_irqrestore(&rport->lock, flags);
1108 	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1109 				  (lsreq->rqstlen + lsreq->rsplen),
1110 				  DMA_BIDIRECTIONAL);
1111 out_putrport:
1112 	nvme_fc_rport_put(rport);
1113 
1114 	return ret;
1115 }
1116 
1117 static void
nvme_fc_send_ls_req_done(struct nvmefc_ls_req * lsreq,int status)1118 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1119 {
1120 	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1121 
1122 	lsop->ls_error = status;
1123 	complete(&lsop->ls_done);
1124 }
1125 
1126 static int
nvme_fc_send_ls_req(struct nvme_fc_rport * rport,struct nvmefc_ls_req_op * lsop)1127 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1128 {
1129 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1130 	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1131 	int ret;
1132 
1133 	ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1134 
1135 	if (!ret) {
1136 		/*
1137 		 * No timeout/not interruptible as we need the struct
1138 		 * to exist until the lldd calls us back. Thus mandate
1139 		 * wait until driver calls back. lldd responsible for
1140 		 * the timeout action
1141 		 */
1142 		wait_for_completion(&lsop->ls_done);
1143 
1144 		__nvme_fc_finish_ls_req(lsop);
1145 
1146 		ret = lsop->ls_error;
1147 	}
1148 
1149 	if (ret)
1150 		return ret;
1151 
1152 	/* ACC or RJT payload ? */
1153 	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1154 		return -ENXIO;
1155 
1156 	return 0;
1157 }
1158 
1159 static int
nvme_fc_send_ls_req_async(struct nvme_fc_rport * rport,struct nvmefc_ls_req_op * lsop,void (* done)(struct nvmefc_ls_req * req,int status))1160 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1161 		struct nvmefc_ls_req_op *lsop,
1162 		void (*done)(struct nvmefc_ls_req *req, int status))
1163 {
1164 	/* don't wait for completion */
1165 
1166 	return __nvme_fc_send_ls_req(rport, lsop, done);
1167 }
1168 
1169 static int
nvme_fc_connect_admin_queue(struct nvme_fc_ctrl * ctrl,struct nvme_fc_queue * queue,u16 qsize,u16 ersp_ratio)1170 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1171 	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1172 {
1173 	struct nvmefc_ls_req_op *lsop;
1174 	struct nvmefc_ls_req *lsreq;
1175 	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1176 	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1177 	unsigned long flags;
1178 	int ret, fcret = 0;
1179 
1180 	lsop = kzalloc((sizeof(*lsop) +
1181 			 sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1182 			 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1183 	if (!lsop) {
1184 		dev_info(ctrl->ctrl.device,
1185 			"NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1186 			ctrl->cnum);
1187 		ret = -ENOMEM;
1188 		goto out_no_memory;
1189 	}
1190 
1191 	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1192 	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1193 	lsreq = &lsop->ls_req;
1194 	if (ctrl->lport->ops->lsrqst_priv_sz)
1195 		lsreq->private = &assoc_acc[1];
1196 	else
1197 		lsreq->private = NULL;
1198 
1199 	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1200 	assoc_rqst->desc_list_len =
1201 			cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1202 
1203 	assoc_rqst->assoc_cmd.desc_tag =
1204 			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1205 	assoc_rqst->assoc_cmd.desc_len =
1206 			fcnvme_lsdesc_len(
1207 				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1208 
1209 	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1210 	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1211 	/* Linux supports only Dynamic controllers */
1212 	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1213 	uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1214 	strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1215 		sizeof(assoc_rqst->assoc_cmd.hostnqn));
1216 	strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1217 		sizeof(assoc_rqst->assoc_cmd.subnqn));
1218 
1219 	lsop->queue = queue;
1220 	lsreq->rqstaddr = assoc_rqst;
1221 	lsreq->rqstlen = sizeof(*assoc_rqst);
1222 	lsreq->rspaddr = assoc_acc;
1223 	lsreq->rsplen = sizeof(*assoc_acc);
1224 	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1225 
1226 	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1227 	if (ret)
1228 		goto out_free_buffer;
1229 
1230 	/* process connect LS completion */
1231 
1232 	/* validate the ACC response */
1233 	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1234 		fcret = VERR_LSACC;
1235 	else if (assoc_acc->hdr.desc_list_len !=
1236 			fcnvme_lsdesc_len(
1237 				sizeof(struct fcnvme_ls_cr_assoc_acc)))
1238 		fcret = VERR_CR_ASSOC_ACC_LEN;
1239 	else if (assoc_acc->hdr.rqst.desc_tag !=
1240 			cpu_to_be32(FCNVME_LSDESC_RQST))
1241 		fcret = VERR_LSDESC_RQST;
1242 	else if (assoc_acc->hdr.rqst.desc_len !=
1243 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1244 		fcret = VERR_LSDESC_RQST_LEN;
1245 	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1246 		fcret = VERR_CR_ASSOC;
1247 	else if (assoc_acc->associd.desc_tag !=
1248 			cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1249 		fcret = VERR_ASSOC_ID;
1250 	else if (assoc_acc->associd.desc_len !=
1251 			fcnvme_lsdesc_len(
1252 				sizeof(struct fcnvme_lsdesc_assoc_id)))
1253 		fcret = VERR_ASSOC_ID_LEN;
1254 	else if (assoc_acc->connectid.desc_tag !=
1255 			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1256 		fcret = VERR_CONN_ID;
1257 	else if (assoc_acc->connectid.desc_len !=
1258 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1259 		fcret = VERR_CONN_ID_LEN;
1260 
1261 	if (fcret) {
1262 		ret = -EBADF;
1263 		dev_err(ctrl->dev,
1264 			"q %d Create Association LS failed: %s\n",
1265 			queue->qnum, validation_errors[fcret]);
1266 	} else {
1267 		spin_lock_irqsave(&ctrl->lock, flags);
1268 		ctrl->association_id =
1269 			be64_to_cpu(assoc_acc->associd.association_id);
1270 		queue->connection_id =
1271 			be64_to_cpu(assoc_acc->connectid.connection_id);
1272 		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1273 		spin_unlock_irqrestore(&ctrl->lock, flags);
1274 	}
1275 
1276 out_free_buffer:
1277 	kfree(lsop);
1278 out_no_memory:
1279 	if (ret)
1280 		dev_err(ctrl->dev,
1281 			"queue %d connect admin queue failed (%d).\n",
1282 			queue->qnum, ret);
1283 	return ret;
1284 }
1285 
1286 static int
nvme_fc_connect_queue(struct nvme_fc_ctrl * ctrl,struct nvme_fc_queue * queue,u16 qsize,u16 ersp_ratio)1287 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1288 			u16 qsize, u16 ersp_ratio)
1289 {
1290 	struct nvmefc_ls_req_op *lsop;
1291 	struct nvmefc_ls_req *lsreq;
1292 	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1293 	struct fcnvme_ls_cr_conn_acc *conn_acc;
1294 	int ret, fcret = 0;
1295 
1296 	lsop = kzalloc((sizeof(*lsop) +
1297 			 sizeof(*conn_rqst) + sizeof(*conn_acc) +
1298 			 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1299 	if (!lsop) {
1300 		dev_info(ctrl->ctrl.device,
1301 			"NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1302 			ctrl->cnum);
1303 		ret = -ENOMEM;
1304 		goto out_no_memory;
1305 	}
1306 
1307 	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1308 	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1309 	lsreq = &lsop->ls_req;
1310 	if (ctrl->lport->ops->lsrqst_priv_sz)
1311 		lsreq->private = (void *)&conn_acc[1];
1312 	else
1313 		lsreq->private = NULL;
1314 
1315 	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1316 	conn_rqst->desc_list_len = cpu_to_be32(
1317 				sizeof(struct fcnvme_lsdesc_assoc_id) +
1318 				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1319 
1320 	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1321 	conn_rqst->associd.desc_len =
1322 			fcnvme_lsdesc_len(
1323 				sizeof(struct fcnvme_lsdesc_assoc_id));
1324 	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1325 	conn_rqst->connect_cmd.desc_tag =
1326 			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1327 	conn_rqst->connect_cmd.desc_len =
1328 			fcnvme_lsdesc_len(
1329 				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1330 	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1331 	conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1332 	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1333 
1334 	lsop->queue = queue;
1335 	lsreq->rqstaddr = conn_rqst;
1336 	lsreq->rqstlen = sizeof(*conn_rqst);
1337 	lsreq->rspaddr = conn_acc;
1338 	lsreq->rsplen = sizeof(*conn_acc);
1339 	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1340 
1341 	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1342 	if (ret)
1343 		goto out_free_buffer;
1344 
1345 	/* process connect LS completion */
1346 
1347 	/* validate the ACC response */
1348 	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1349 		fcret = VERR_LSACC;
1350 	else if (conn_acc->hdr.desc_list_len !=
1351 			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1352 		fcret = VERR_CR_CONN_ACC_LEN;
1353 	else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1354 		fcret = VERR_LSDESC_RQST;
1355 	else if (conn_acc->hdr.rqst.desc_len !=
1356 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1357 		fcret = VERR_LSDESC_RQST_LEN;
1358 	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1359 		fcret = VERR_CR_CONN;
1360 	else if (conn_acc->connectid.desc_tag !=
1361 			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1362 		fcret = VERR_CONN_ID;
1363 	else if (conn_acc->connectid.desc_len !=
1364 			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1365 		fcret = VERR_CONN_ID_LEN;
1366 
1367 	if (fcret) {
1368 		ret = -EBADF;
1369 		dev_err(ctrl->dev,
1370 			"q %d Create I/O Connection LS failed: %s\n",
1371 			queue->qnum, validation_errors[fcret]);
1372 	} else {
1373 		queue->connection_id =
1374 			be64_to_cpu(conn_acc->connectid.connection_id);
1375 		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1376 	}
1377 
1378 out_free_buffer:
1379 	kfree(lsop);
1380 out_no_memory:
1381 	if (ret)
1382 		dev_err(ctrl->dev,
1383 			"queue %d connect I/O queue failed (%d).\n",
1384 			queue->qnum, ret);
1385 	return ret;
1386 }
1387 
1388 static void
nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req * lsreq,int status)1389 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1390 {
1391 	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1392 
1393 	__nvme_fc_finish_ls_req(lsop);
1394 
1395 	/* fc-nvme initiator doesn't care about success or failure of cmd */
1396 
1397 	kfree(lsop);
1398 }
1399 
1400 /*
1401  * This routine sends a FC-NVME LS to disconnect (aka terminate)
1402  * the FC-NVME Association.  Terminating the association also
1403  * terminates the FC-NVME connections (per queue, both admin and io
1404  * queues) that are part of the association. E.g. things are torn
1405  * down, and the related FC-NVME Association ID and Connection IDs
1406  * become invalid.
1407  *
1408  * The behavior of the fc-nvme initiator is such that it's
1409  * understanding of the association and connections will implicitly
1410  * be torn down. The action is implicit as it may be due to a loss of
1411  * connectivity with the fc-nvme target, so you may never get a
1412  * response even if you tried.  As such, the action of this routine
1413  * is to asynchronously send the LS, ignore any results of the LS, and
1414  * continue on with terminating the association. If the fc-nvme target
1415  * is present and receives the LS, it too can tear down.
1416  */
1417 static void
nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl * ctrl)1418 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1419 {
1420 	struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1421 	struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1422 	struct nvmefc_ls_req_op *lsop;
1423 	struct nvmefc_ls_req *lsreq;
1424 	int ret;
1425 
1426 	lsop = kzalloc((sizeof(*lsop) +
1427 			sizeof(*discon_rqst) + sizeof(*discon_acc) +
1428 			ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1429 	if (!lsop) {
1430 		dev_info(ctrl->ctrl.device,
1431 			"NVME-FC{%d}: send Disconnect Association "
1432 			"failed: ENOMEM\n",
1433 			ctrl->cnum);
1434 		return;
1435 	}
1436 
1437 	discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1438 	discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1439 	lsreq = &lsop->ls_req;
1440 	if (ctrl->lport->ops->lsrqst_priv_sz)
1441 		lsreq->private = (void *)&discon_acc[1];
1442 	else
1443 		lsreq->private = NULL;
1444 
1445 	nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1446 				ctrl->association_id);
1447 
1448 	ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1449 				nvme_fc_disconnect_assoc_done);
1450 	if (ret)
1451 		kfree(lsop);
1452 }
1453 
1454 static void
nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp * lsrsp)1455 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1456 {
1457 	struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1458 	struct nvme_fc_rport *rport = lsop->rport;
1459 	struct nvme_fc_lport *lport = rport->lport;
1460 	unsigned long flags;
1461 
1462 	spin_lock_irqsave(&rport->lock, flags);
1463 	list_del(&lsop->lsrcv_list);
1464 	spin_unlock_irqrestore(&rport->lock, flags);
1465 
1466 	fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1467 				sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1468 	fc_dma_unmap_single(lport->dev, lsop->rspdma,
1469 			sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1470 
1471 	kfree(lsop->rspbuf);
1472 	kfree(lsop->rqstbuf);
1473 	kfree(lsop);
1474 
1475 	nvme_fc_rport_put(rport);
1476 }
1477 
1478 static void
nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op * lsop)1479 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1480 {
1481 	struct nvme_fc_rport *rport = lsop->rport;
1482 	struct nvme_fc_lport *lport = rport->lport;
1483 	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1484 	int ret;
1485 
1486 	fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1487 				  sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1488 
1489 	ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1490 				     lsop->lsrsp);
1491 	if (ret) {
1492 		dev_warn(lport->dev,
1493 			"LLDD rejected LS RSP xmt: LS %d status %d\n",
1494 			w0->ls_cmd, ret);
1495 		nvme_fc_xmt_ls_rsp_done(lsop->lsrsp);
1496 		return;
1497 	}
1498 }
1499 
1500 static struct nvme_fc_ctrl *
nvme_fc_match_disconn_ls(struct nvme_fc_rport * rport,struct nvmefc_ls_rcv_op * lsop)1501 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1502 		      struct nvmefc_ls_rcv_op *lsop)
1503 {
1504 	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1505 					&lsop->rqstbuf->rq_dis_assoc;
1506 	struct nvme_fc_ctrl *ctrl, *ret = NULL;
1507 	struct nvmefc_ls_rcv_op *oldls = NULL;
1508 	u64 association_id = be64_to_cpu(rqst->associd.association_id);
1509 	unsigned long flags;
1510 
1511 	spin_lock_irqsave(&rport->lock, flags);
1512 
1513 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1514 		if (!nvme_fc_ctrl_get(ctrl))
1515 			continue;
1516 		spin_lock(&ctrl->lock);
1517 		if (association_id == ctrl->association_id) {
1518 			oldls = ctrl->rcv_disconn;
1519 			ctrl->rcv_disconn = lsop;
1520 			ret = ctrl;
1521 		}
1522 		spin_unlock(&ctrl->lock);
1523 		if (ret)
1524 			/* leave the ctrl get reference */
1525 			break;
1526 		nvme_fc_ctrl_put(ctrl);
1527 	}
1528 
1529 	spin_unlock_irqrestore(&rport->lock, flags);
1530 
1531 	/* transmit a response for anything that was pending */
1532 	if (oldls) {
1533 		dev_info(rport->lport->dev,
1534 			"NVME-FC{%d}: Multiple Disconnect Association "
1535 			"LS's received\n", ctrl->cnum);
1536 		/* overwrite good response with bogus failure */
1537 		oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1538 						sizeof(*oldls->rspbuf),
1539 						rqst->w0.ls_cmd,
1540 						FCNVME_RJT_RC_UNAB,
1541 						FCNVME_RJT_EXP_NONE, 0);
1542 		nvme_fc_xmt_ls_rsp(oldls);
1543 	}
1544 
1545 	return ret;
1546 }
1547 
1548 /*
1549  * returns true to mean LS handled and ls_rsp can be sent
1550  * returns false to defer ls_rsp xmt (will be done as part of
1551  *     association termination)
1552  */
1553 static bool
nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op * lsop)1554 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1555 {
1556 	struct nvme_fc_rport *rport = lsop->rport;
1557 	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1558 					&lsop->rqstbuf->rq_dis_assoc;
1559 	struct fcnvme_ls_disconnect_assoc_acc *acc =
1560 					&lsop->rspbuf->rsp_dis_assoc;
1561 	struct nvme_fc_ctrl *ctrl = NULL;
1562 	int ret = 0;
1563 
1564 	memset(acc, 0, sizeof(*acc));
1565 
1566 	ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1567 	if (!ret) {
1568 		/* match an active association */
1569 		ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1570 		if (!ctrl)
1571 			ret = VERR_NO_ASSOC;
1572 	}
1573 
1574 	if (ret) {
1575 		dev_info(rport->lport->dev,
1576 			"Disconnect LS failed: %s\n",
1577 			validation_errors[ret]);
1578 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1579 					sizeof(*acc), rqst->w0.ls_cmd,
1580 					(ret == VERR_NO_ASSOC) ?
1581 						FCNVME_RJT_RC_INV_ASSOC :
1582 						FCNVME_RJT_RC_LOGIC,
1583 					FCNVME_RJT_EXP_NONE, 0);
1584 		return true;
1585 	}
1586 
1587 	/* format an ACCept response */
1588 
1589 	lsop->lsrsp->rsplen = sizeof(*acc);
1590 
1591 	nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1592 			fcnvme_lsdesc_len(
1593 				sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1594 			FCNVME_LS_DISCONNECT_ASSOC);
1595 
1596 	/*
1597 	 * the transmit of the response will occur after the exchanges
1598 	 * for the association have been ABTS'd by
1599 	 * nvme_fc_delete_association().
1600 	 */
1601 
1602 	/* fail the association */
1603 	nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1604 
1605 	/* release the reference taken by nvme_fc_match_disconn_ls() */
1606 	nvme_fc_ctrl_put(ctrl);
1607 
1608 	return false;
1609 }
1610 
1611 /*
1612  * Actual Processing routine for received FC-NVME LS Requests from the LLD
1613  * returns true if a response should be sent afterward, false if rsp will
1614  * be sent asynchronously.
1615  */
1616 static bool
nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op * lsop)1617 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1618 {
1619 	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1620 	bool ret = true;
1621 
1622 	lsop->lsrsp->nvme_fc_private = lsop;
1623 	lsop->lsrsp->rspbuf = lsop->rspbuf;
1624 	lsop->lsrsp->rspdma = lsop->rspdma;
1625 	lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1626 	/* Be preventative. handlers will later set to valid length */
1627 	lsop->lsrsp->rsplen = 0;
1628 
1629 	/*
1630 	 * handlers:
1631 	 *   parse request input, execute the request, and format the
1632 	 *   LS response
1633 	 */
1634 	switch (w0->ls_cmd) {
1635 	case FCNVME_LS_DISCONNECT_ASSOC:
1636 		ret = nvme_fc_ls_disconnect_assoc(lsop);
1637 		break;
1638 	case FCNVME_LS_DISCONNECT_CONN:
1639 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1640 				sizeof(*lsop->rspbuf), w0->ls_cmd,
1641 				FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1642 		break;
1643 	case FCNVME_LS_CREATE_ASSOCIATION:
1644 	case FCNVME_LS_CREATE_CONNECTION:
1645 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1646 				sizeof(*lsop->rspbuf), w0->ls_cmd,
1647 				FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1648 		break;
1649 	default:
1650 		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1651 				sizeof(*lsop->rspbuf), w0->ls_cmd,
1652 				FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1653 		break;
1654 	}
1655 
1656 	return(ret);
1657 }
1658 
1659 static void
nvme_fc_handle_ls_rqst_work(struct work_struct * work)1660 nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1661 {
1662 	struct nvme_fc_rport *rport =
1663 		container_of(work, struct nvme_fc_rport, lsrcv_work);
1664 	struct fcnvme_ls_rqst_w0 *w0;
1665 	struct nvmefc_ls_rcv_op *lsop;
1666 	unsigned long flags;
1667 	bool sendrsp;
1668 
1669 restart:
1670 	sendrsp = true;
1671 	spin_lock_irqsave(&rport->lock, flags);
1672 	list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1673 		if (lsop->handled)
1674 			continue;
1675 
1676 		lsop->handled = true;
1677 		if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1678 			spin_unlock_irqrestore(&rport->lock, flags);
1679 			sendrsp = nvme_fc_handle_ls_rqst(lsop);
1680 		} else {
1681 			spin_unlock_irqrestore(&rport->lock, flags);
1682 			w0 = &lsop->rqstbuf->w0;
1683 			lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1684 						lsop->rspbuf,
1685 						sizeof(*lsop->rspbuf),
1686 						w0->ls_cmd,
1687 						FCNVME_RJT_RC_UNAB,
1688 						FCNVME_RJT_EXP_NONE, 0);
1689 		}
1690 		if (sendrsp)
1691 			nvme_fc_xmt_ls_rsp(lsop);
1692 		goto restart;
1693 	}
1694 	spin_unlock_irqrestore(&rport->lock, flags);
1695 }
1696 
1697 static
nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport * lport,struct fcnvme_ls_rqst_w0 * w0)1698 void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1699 				struct fcnvme_ls_rqst_w0 *w0)
1700 {
1701 	dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1702 		(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1703 			nvmefc_ls_names[w0->ls_cmd] : "");
1704 }
1705 
1706 /**
1707  * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1708  *                       upon the reception of a NVME LS request.
1709  *
1710  * The nvme-fc layer will copy payload to an internal structure for
1711  * processing.  As such, upon completion of the routine, the LLDD may
1712  * immediately free/reuse the LS request buffer passed in the call.
1713  *
1714  * If this routine returns error, the LLDD should abort the exchange.
1715  *
1716  * @portptr:    pointer to the (registered) remote port that the LS
1717  *              was received from. The remoteport is associated with
1718  *              a specific localport.
1719  * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1720  *              used to reference the exchange corresponding to the LS
1721  *              when issuing an ls response.
1722  * @lsreqbuf:   pointer to the buffer containing the LS Request
1723  * @lsreqbuf_len: length, in bytes, of the received LS request
1724  */
1725 int
nvme_fc_rcv_ls_req(struct nvme_fc_remote_port * portptr,struct nvmefc_ls_rsp * lsrsp,void * lsreqbuf,u32 lsreqbuf_len)1726 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1727 			struct nvmefc_ls_rsp *lsrsp,
1728 			void *lsreqbuf, u32 lsreqbuf_len)
1729 {
1730 	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1731 	struct nvme_fc_lport *lport = rport->lport;
1732 	struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1733 	struct nvmefc_ls_rcv_op *lsop;
1734 	unsigned long flags;
1735 	int ret;
1736 
1737 	nvme_fc_rport_get(rport);
1738 
1739 	/* validate there's a routine to transmit a response */
1740 	if (!lport->ops->xmt_ls_rsp) {
1741 		dev_info(lport->dev,
1742 			"RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1743 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1744 				nvmefc_ls_names[w0->ls_cmd] : "");
1745 		ret = -EINVAL;
1746 		goto out_put;
1747 	}
1748 
1749 	if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1750 		dev_info(lport->dev,
1751 			"RCV %s LS failed: payload too large\n",
1752 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1753 				nvmefc_ls_names[w0->ls_cmd] : "");
1754 		ret = -E2BIG;
1755 		goto out_put;
1756 	}
1757 
1758 	lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1759 	if (!lsop) {
1760 		nvme_fc_rcv_ls_req_err_msg(lport, w0);
1761 		ret = -ENOMEM;
1762 		goto out_put;
1763 	}
1764 
1765 	lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1766 	lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1767 	if (!lsop->rqstbuf || !lsop->rspbuf) {
1768 		nvme_fc_rcv_ls_req_err_msg(lport, w0);
1769 		ret = -ENOMEM;
1770 		goto out_free;
1771 	}
1772 
1773 	lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1774 					sizeof(*lsop->rspbuf),
1775 					DMA_TO_DEVICE);
1776 	if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1777 		dev_info(lport->dev,
1778 			"RCV %s LS failed: DMA mapping failure\n",
1779 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1780 				nvmefc_ls_names[w0->ls_cmd] : "");
1781 		ret = -EFAULT;
1782 		goto out_free;
1783 	}
1784 
1785 	lsop->rport = rport;
1786 	lsop->lsrsp = lsrsp;
1787 
1788 	memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1789 	lsop->rqstdatalen = lsreqbuf_len;
1790 
1791 	spin_lock_irqsave(&rport->lock, flags);
1792 	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1793 		spin_unlock_irqrestore(&rport->lock, flags);
1794 		ret = -ENOTCONN;
1795 		goto out_unmap;
1796 	}
1797 	list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1798 	spin_unlock_irqrestore(&rport->lock, flags);
1799 
1800 	schedule_work(&rport->lsrcv_work);
1801 
1802 	return 0;
1803 
1804 out_unmap:
1805 	fc_dma_unmap_single(lport->dev, lsop->rspdma,
1806 			sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1807 out_free:
1808 	kfree(lsop->rspbuf);
1809 	kfree(lsop->rqstbuf);
1810 	kfree(lsop);
1811 out_put:
1812 	nvme_fc_rport_put(rport);
1813 	return ret;
1814 }
1815 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1816 
1817 
1818 /* *********************** NVME Ctrl Routines **************************** */
1819 
1820 static void
__nvme_fc_exit_request(struct nvme_fc_ctrl * ctrl,struct nvme_fc_fcp_op * op)1821 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1822 		struct nvme_fc_fcp_op *op)
1823 {
1824 	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1825 				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1826 	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1827 				sizeof(op->cmd_iu), DMA_TO_DEVICE);
1828 
1829 	atomic_set(&op->state, FCPOP_STATE_UNINIT);
1830 }
1831 
1832 static void
nvme_fc_exit_request(struct blk_mq_tag_set * set,struct request * rq,unsigned int hctx_idx)1833 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1834 		unsigned int hctx_idx)
1835 {
1836 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1837 
1838 	return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op);
1839 }
1840 
1841 static int
__nvme_fc_abort_op(struct nvme_fc_ctrl * ctrl,struct nvme_fc_fcp_op * op)1842 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1843 {
1844 	unsigned long flags;
1845 	int opstate;
1846 
1847 	spin_lock_irqsave(&ctrl->lock, flags);
1848 	opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1849 	if (opstate != FCPOP_STATE_ACTIVE)
1850 		atomic_set(&op->state, opstate);
1851 	else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1852 		op->flags |= FCOP_FLAGS_TERMIO;
1853 		ctrl->iocnt++;
1854 	}
1855 	spin_unlock_irqrestore(&ctrl->lock, flags);
1856 
1857 	if (opstate != FCPOP_STATE_ACTIVE)
1858 		return -ECANCELED;
1859 
1860 	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1861 					&ctrl->rport->remoteport,
1862 					op->queue->lldd_handle,
1863 					&op->fcp_req);
1864 
1865 	return 0;
1866 }
1867 
1868 static void
nvme_fc_abort_aen_ops(struct nvme_fc_ctrl * ctrl)1869 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1870 {
1871 	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1872 	int i;
1873 
1874 	/* ensure we've initialized the ops once */
1875 	if (!(aen_op->flags & FCOP_FLAGS_AEN))
1876 		return;
1877 
1878 	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1879 		__nvme_fc_abort_op(ctrl, aen_op);
1880 }
1881 
1882 static inline void
__nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl * ctrl,struct nvme_fc_fcp_op * op,int opstate)1883 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1884 		struct nvme_fc_fcp_op *op, int opstate)
1885 {
1886 	unsigned long flags;
1887 
1888 	if (opstate == FCPOP_STATE_ABORTED) {
1889 		spin_lock_irqsave(&ctrl->lock, flags);
1890 		if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1891 		    op->flags & FCOP_FLAGS_TERMIO) {
1892 			if (!--ctrl->iocnt)
1893 				wake_up(&ctrl->ioabort_wait);
1894 		}
1895 		spin_unlock_irqrestore(&ctrl->lock, flags);
1896 	}
1897 }
1898 
1899 static void
nvme_fc_ctrl_ioerr_work(struct work_struct * work)1900 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1901 {
1902 	struct nvme_fc_ctrl *ctrl =
1903 			container_of(work, struct nvme_fc_ctrl, ioerr_work);
1904 
1905 	nvme_fc_error_recovery(ctrl, "transport detected io error");
1906 }
1907 
1908 /*
1909  * nvme_fc_io_getuuid - Routine called to get the appid field
1910  * associated with request by the lldd
1911  * @req:IO request from nvme fc to driver
1912  * Returns: UUID if there is an appid associated with VM or
1913  * NULL if the user/libvirt has not set the appid to VM
1914  */
nvme_fc_io_getuuid(struct nvmefc_fcp_req * req)1915 char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1916 {
1917 	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1918 	struct request *rq = op->rq;
1919 
1920 	if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1921 		return NULL;
1922 	return blkcg_get_fc_appid(rq->bio);
1923 }
1924 EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1925 
1926 static void
nvme_fc_fcpio_done(struct nvmefc_fcp_req * req)1927 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1928 {
1929 	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1930 	struct request *rq = op->rq;
1931 	struct nvmefc_fcp_req *freq = &op->fcp_req;
1932 	struct nvme_fc_ctrl *ctrl = op->ctrl;
1933 	struct nvme_fc_queue *queue = op->queue;
1934 	struct nvme_completion *cqe = &op->rsp_iu.cqe;
1935 	struct nvme_command *sqe = &op->cmd_iu.sqe;
1936 	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1937 	union nvme_result result;
1938 	bool terminate_assoc = true;
1939 	int opstate;
1940 
1941 	/*
1942 	 * WARNING:
1943 	 * The current linux implementation of a nvme controller
1944 	 * allocates a single tag set for all io queues and sizes
1945 	 * the io queues to fully hold all possible tags. Thus, the
1946 	 * implementation does not reference or care about the sqhd
1947 	 * value as it never needs to use the sqhd/sqtail pointers
1948 	 * for submission pacing.
1949 	 *
1950 	 * This affects the FC-NVME implementation in two ways:
1951 	 * 1) As the value doesn't matter, we don't need to waste
1952 	 *    cycles extracting it from ERSPs and stamping it in the
1953 	 *    cases where the transport fabricates CQEs on successful
1954 	 *    completions.
1955 	 * 2) The FC-NVME implementation requires that delivery of
1956 	 *    ERSP completions are to go back to the nvme layer in order
1957 	 *    relative to the rsn, such that the sqhd value will always
1958 	 *    be "in order" for the nvme layer. As the nvme layer in
1959 	 *    linux doesn't care about sqhd, there's no need to return
1960 	 *    them in order.
1961 	 *
1962 	 * Additionally:
1963 	 * As the core nvme layer in linux currently does not look at
1964 	 * every field in the cqe - in cases where the FC transport must
1965 	 * fabricate a CQE, the following fields will not be set as they
1966 	 * are not referenced:
1967 	 *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1968 	 *
1969 	 * Failure or error of an individual i/o, in a transport
1970 	 * detected fashion unrelated to the nvme completion status,
1971 	 * potentially cause the initiator and target sides to get out
1972 	 * of sync on SQ head/tail (aka outstanding io count allowed).
1973 	 * Per FC-NVME spec, failure of an individual command requires
1974 	 * the connection to be terminated, which in turn requires the
1975 	 * association to be terminated.
1976 	 */
1977 
1978 	opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1979 
1980 	fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1981 				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1982 
1983 	if (opstate == FCPOP_STATE_ABORTED)
1984 		status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1985 	else if (freq->status) {
1986 		status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1987 		dev_info(ctrl->ctrl.device,
1988 			"NVME-FC{%d}: io failed due to lldd error %d\n",
1989 			ctrl->cnum, freq->status);
1990 	}
1991 
1992 	/*
1993 	 * For the linux implementation, if we have an unsuccesful
1994 	 * status, they blk-mq layer can typically be called with the
1995 	 * non-zero status and the content of the cqe isn't important.
1996 	 */
1997 	if (status)
1998 		goto done;
1999 
2000 	/*
2001 	 * command completed successfully relative to the wire
2002 	 * protocol. However, validate anything received and
2003 	 * extract the status and result from the cqe (create it
2004 	 * where necessary).
2005 	 */
2006 
2007 	switch (freq->rcv_rsplen) {
2008 
2009 	case 0:
2010 	case NVME_FC_SIZEOF_ZEROS_RSP:
2011 		/*
2012 		 * No response payload or 12 bytes of payload (which
2013 		 * should all be zeros) are considered successful and
2014 		 * no payload in the CQE by the transport.
2015 		 */
2016 		if (freq->transferred_length !=
2017 		    be32_to_cpu(op->cmd_iu.data_len)) {
2018 			status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2019 			dev_info(ctrl->ctrl.device,
2020 				"NVME-FC{%d}: io failed due to bad transfer "
2021 				"length: %d vs expected %d\n",
2022 				ctrl->cnum, freq->transferred_length,
2023 				be32_to_cpu(op->cmd_iu.data_len));
2024 			goto done;
2025 		}
2026 		result.u64 = 0;
2027 		break;
2028 
2029 	case sizeof(struct nvme_fc_ersp_iu):
2030 		/*
2031 		 * The ERSP IU contains a full completion with CQE.
2032 		 * Validate ERSP IU and look at cqe.
2033 		 */
2034 		if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2035 					(freq->rcv_rsplen / 4) ||
2036 			     be32_to_cpu(op->rsp_iu.xfrd_len) !=
2037 					freq->transferred_length ||
2038 			     op->rsp_iu.ersp_result ||
2039 			     sqe->common.command_id != cqe->command_id)) {
2040 			status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2041 			dev_info(ctrl->ctrl.device,
2042 				"NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2043 				"iu len %d, xfr len %d vs %d, status code "
2044 				"%d, cmdid %d vs %d\n",
2045 				ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2046 				be32_to_cpu(op->rsp_iu.xfrd_len),
2047 				freq->transferred_length,
2048 				op->rsp_iu.ersp_result,
2049 				sqe->common.command_id,
2050 				cqe->command_id);
2051 			goto done;
2052 		}
2053 		result = cqe->result;
2054 		status = cqe->status;
2055 		break;
2056 
2057 	default:
2058 		status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2059 		dev_info(ctrl->ctrl.device,
2060 			"NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2061 			"len %d\n",
2062 			ctrl->cnum, freq->rcv_rsplen);
2063 		goto done;
2064 	}
2065 
2066 	terminate_assoc = false;
2067 
2068 done:
2069 	if (op->flags & FCOP_FLAGS_AEN) {
2070 		nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2071 		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2072 		atomic_set(&op->state, FCPOP_STATE_IDLE);
2073 		op->flags = FCOP_FLAGS_AEN;	/* clear other flags */
2074 		nvme_fc_ctrl_put(ctrl);
2075 		goto check_error;
2076 	}
2077 
2078 	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2079 	if (!nvme_try_complete_req(rq, status, result))
2080 		nvme_fc_complete_rq(rq);
2081 
2082 check_error:
2083 	if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2084 		queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2085 }
2086 
2087 static int
__nvme_fc_init_request(struct nvme_fc_ctrl * ctrl,struct nvme_fc_queue * queue,struct nvme_fc_fcp_op * op,struct request * rq,u32 rqno)2088 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2089 		struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2090 		struct request *rq, u32 rqno)
2091 {
2092 	struct nvme_fcp_op_w_sgl *op_w_sgl =
2093 		container_of(op, typeof(*op_w_sgl), op);
2094 	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2095 	int ret = 0;
2096 
2097 	memset(op, 0, sizeof(*op));
2098 	op->fcp_req.cmdaddr = &op->cmd_iu;
2099 	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2100 	op->fcp_req.rspaddr = &op->rsp_iu;
2101 	op->fcp_req.rsplen = sizeof(op->rsp_iu);
2102 	op->fcp_req.done = nvme_fc_fcpio_done;
2103 	op->ctrl = ctrl;
2104 	op->queue = queue;
2105 	op->rq = rq;
2106 	op->rqno = rqno;
2107 
2108 	cmdiu->format_id = NVME_CMD_FORMAT_ID;
2109 	cmdiu->fc_id = NVME_CMD_FC_ID;
2110 	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2111 	if (queue->qnum)
2112 		cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2113 					(NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2114 	else
2115 		cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2116 
2117 	op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2118 				&op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2119 	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2120 		dev_err(ctrl->dev,
2121 			"FCP Op failed - cmdiu dma mapping failed.\n");
2122 		ret = -EFAULT;
2123 		goto out_on_error;
2124 	}
2125 
2126 	op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2127 				&op->rsp_iu, sizeof(op->rsp_iu),
2128 				DMA_FROM_DEVICE);
2129 	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2130 		dev_err(ctrl->dev,
2131 			"FCP Op failed - rspiu dma mapping failed.\n");
2132 		ret = -EFAULT;
2133 	}
2134 
2135 	atomic_set(&op->state, FCPOP_STATE_IDLE);
2136 out_on_error:
2137 	return ret;
2138 }
2139 
2140 static int
nvme_fc_init_request(struct blk_mq_tag_set * set,struct request * rq,unsigned int hctx_idx,unsigned int numa_node)2141 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2142 		unsigned int hctx_idx, unsigned int numa_node)
2143 {
2144 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2145 	struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2146 	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2147 	struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2148 	int res;
2149 
2150 	res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2151 	if (res)
2152 		return res;
2153 	op->op.fcp_req.first_sgl = op->sgl;
2154 	op->op.fcp_req.private = &op->priv[0];
2155 	nvme_req(rq)->ctrl = &ctrl->ctrl;
2156 	nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2157 	return res;
2158 }
2159 
2160 static int
nvme_fc_init_aen_ops(struct nvme_fc_ctrl * ctrl)2161 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2162 {
2163 	struct nvme_fc_fcp_op *aen_op;
2164 	struct nvme_fc_cmd_iu *cmdiu;
2165 	struct nvme_command *sqe;
2166 	void *private = NULL;
2167 	int i, ret;
2168 
2169 	aen_op = ctrl->aen_ops;
2170 	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2171 		if (ctrl->lport->ops->fcprqst_priv_sz) {
2172 			private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2173 						GFP_KERNEL);
2174 			if (!private)
2175 				return -ENOMEM;
2176 		}
2177 
2178 		cmdiu = &aen_op->cmd_iu;
2179 		sqe = &cmdiu->sqe;
2180 		ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2181 				aen_op, (struct request *)NULL,
2182 				(NVME_AQ_BLK_MQ_DEPTH + i));
2183 		if (ret) {
2184 			kfree(private);
2185 			return ret;
2186 		}
2187 
2188 		aen_op->flags = FCOP_FLAGS_AEN;
2189 		aen_op->fcp_req.private = private;
2190 
2191 		memset(sqe, 0, sizeof(*sqe));
2192 		sqe->common.opcode = nvme_admin_async_event;
2193 		/* Note: core layer may overwrite the sqe.command_id value */
2194 		sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2195 	}
2196 	return 0;
2197 }
2198 
2199 static void
nvme_fc_term_aen_ops(struct nvme_fc_ctrl * ctrl)2200 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2201 {
2202 	struct nvme_fc_fcp_op *aen_op;
2203 	int i;
2204 
2205 	cancel_work_sync(&ctrl->ctrl.async_event_work);
2206 	aen_op = ctrl->aen_ops;
2207 	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2208 		__nvme_fc_exit_request(ctrl, aen_op);
2209 
2210 		kfree(aen_op->fcp_req.private);
2211 		aen_op->fcp_req.private = NULL;
2212 	}
2213 }
2214 
2215 static inline int
__nvme_fc_init_hctx(struct blk_mq_hw_ctx * hctx,void * data,unsigned int qidx)2216 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2217 {
2218 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data);
2219 	struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2220 
2221 	hctx->driver_data = queue;
2222 	queue->hctx = hctx;
2223 	return 0;
2224 }
2225 
2226 static int
nvme_fc_init_hctx(struct blk_mq_hw_ctx * hctx,void * data,unsigned int hctx_idx)2227 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2228 {
2229 	return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1);
2230 }
2231 
2232 static int
nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx * hctx,void * data,unsigned int hctx_idx)2233 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2234 		unsigned int hctx_idx)
2235 {
2236 	return __nvme_fc_init_hctx(hctx, data, hctx_idx);
2237 }
2238 
2239 static void
nvme_fc_init_queue(struct nvme_fc_ctrl * ctrl,int idx)2240 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2241 {
2242 	struct nvme_fc_queue *queue;
2243 
2244 	queue = &ctrl->queues[idx];
2245 	memset(queue, 0, sizeof(*queue));
2246 	queue->ctrl = ctrl;
2247 	queue->qnum = idx;
2248 	atomic_set(&queue->csn, 0);
2249 	queue->dev = ctrl->dev;
2250 
2251 	if (idx > 0)
2252 		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2253 	else
2254 		queue->cmnd_capsule_len = sizeof(struct nvme_command);
2255 
2256 	/*
2257 	 * Considered whether we should allocate buffers for all SQEs
2258 	 * and CQEs and dma map them - mapping their respective entries
2259 	 * into the request structures (kernel vm addr and dma address)
2260 	 * thus the driver could use the buffers/mappings directly.
2261 	 * It only makes sense if the LLDD would use them for its
2262 	 * messaging api. It's very unlikely most adapter api's would use
2263 	 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2264 	 * structures were used instead.
2265 	 */
2266 }
2267 
2268 /*
2269  * This routine terminates a queue at the transport level.
2270  * The transport has already ensured that all outstanding ios on
2271  * the queue have been terminated.
2272  * The transport will send a Disconnect LS request to terminate
2273  * the queue's connection. Termination of the admin queue will also
2274  * terminate the association at the target.
2275  */
2276 static void
nvme_fc_free_queue(struct nvme_fc_queue * queue)2277 nvme_fc_free_queue(struct nvme_fc_queue *queue)
2278 {
2279 	if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2280 		return;
2281 
2282 	clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2283 	/*
2284 	 * Current implementation never disconnects a single queue.
2285 	 * It always terminates a whole association. So there is never
2286 	 * a disconnect(queue) LS sent to the target.
2287 	 */
2288 
2289 	queue->connection_id = 0;
2290 	atomic_set(&queue->csn, 0);
2291 }
2292 
2293 static void
__nvme_fc_delete_hw_queue(struct nvme_fc_ctrl * ctrl,struct nvme_fc_queue * queue,unsigned int qidx)2294 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2295 	struct nvme_fc_queue *queue, unsigned int qidx)
2296 {
2297 	if (ctrl->lport->ops->delete_queue)
2298 		ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2299 				queue->lldd_handle);
2300 	queue->lldd_handle = NULL;
2301 }
2302 
2303 static void
nvme_fc_free_io_queues(struct nvme_fc_ctrl * ctrl)2304 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2305 {
2306 	int i;
2307 
2308 	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2309 		nvme_fc_free_queue(&ctrl->queues[i]);
2310 }
2311 
2312 static int
__nvme_fc_create_hw_queue(struct nvme_fc_ctrl * ctrl,struct nvme_fc_queue * queue,unsigned int qidx,u16 qsize)2313 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2314 	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2315 {
2316 	int ret = 0;
2317 
2318 	queue->lldd_handle = NULL;
2319 	if (ctrl->lport->ops->create_queue)
2320 		ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2321 				qidx, qsize, &queue->lldd_handle);
2322 
2323 	return ret;
2324 }
2325 
2326 static void
nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl * ctrl)2327 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2328 {
2329 	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2330 	int i;
2331 
2332 	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2333 		__nvme_fc_delete_hw_queue(ctrl, queue, i);
2334 }
2335 
2336 static int
nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl * ctrl,u16 qsize)2337 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2338 {
2339 	struct nvme_fc_queue *queue = &ctrl->queues[1];
2340 	int i, ret;
2341 
2342 	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2343 		ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2344 		if (ret)
2345 			goto delete_queues;
2346 	}
2347 
2348 	return 0;
2349 
2350 delete_queues:
2351 	for (; i > 0; i--)
2352 		__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2353 	return ret;
2354 }
2355 
2356 static int
nvme_fc_connect_io_queues(struct nvme_fc_ctrl * ctrl,u16 qsize)2357 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2358 {
2359 	int i, ret = 0;
2360 
2361 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2362 		ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2363 					(qsize / 5));
2364 		if (ret)
2365 			break;
2366 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2367 		if (ret)
2368 			break;
2369 
2370 		set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2371 	}
2372 
2373 	return ret;
2374 }
2375 
2376 static void
nvme_fc_init_io_queues(struct nvme_fc_ctrl * ctrl)2377 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2378 {
2379 	int i;
2380 
2381 	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2382 		nvme_fc_init_queue(ctrl, i);
2383 }
2384 
2385 static void
nvme_fc_ctrl_free(struct kref * ref)2386 nvme_fc_ctrl_free(struct kref *ref)
2387 {
2388 	struct nvme_fc_ctrl *ctrl =
2389 		container_of(ref, struct nvme_fc_ctrl, ref);
2390 	unsigned long flags;
2391 
2392 	if (ctrl->ctrl.tagset)
2393 		nvme_remove_io_tag_set(&ctrl->ctrl);
2394 
2395 	/* remove from rport list */
2396 	spin_lock_irqsave(&ctrl->rport->lock, flags);
2397 	list_del(&ctrl->ctrl_list);
2398 	spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2399 
2400 	nvme_unquiesce_admin_queue(&ctrl->ctrl);
2401 	nvme_remove_admin_tag_set(&ctrl->ctrl);
2402 
2403 	kfree(ctrl->queues);
2404 
2405 	put_device(ctrl->dev);
2406 	nvme_fc_rport_put(ctrl->rport);
2407 
2408 	ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2409 	if (ctrl->ctrl.opts)
2410 		nvmf_free_options(ctrl->ctrl.opts);
2411 	kfree(ctrl);
2412 }
2413 
2414 static void
nvme_fc_ctrl_put(struct nvme_fc_ctrl * ctrl)2415 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2416 {
2417 	kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2418 }
2419 
2420 static int
nvme_fc_ctrl_get(struct nvme_fc_ctrl * ctrl)2421 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2422 {
2423 	return kref_get_unless_zero(&ctrl->ref);
2424 }
2425 
2426 /*
2427  * All accesses from nvme core layer done - can now free the
2428  * controller. Called after last nvme_put_ctrl() call
2429  */
2430 static void
nvme_fc_free_ctrl(struct nvme_ctrl * nctrl)2431 nvme_fc_free_ctrl(struct nvme_ctrl *nctrl)
2432 {
2433 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2434 
2435 	WARN_ON(nctrl != &ctrl->ctrl);
2436 
2437 	nvme_fc_ctrl_put(ctrl);
2438 }
2439 
2440 /*
2441  * This routine is used by the transport when it needs to find active
2442  * io on a queue that is to be terminated. The transport uses
2443  * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2444  * this routine to kill them on a 1 by 1 basis.
2445  *
2446  * As FC allocates FC exchange for each io, the transport must contact
2447  * the LLDD to terminate the exchange, thus releasing the FC exchange.
2448  * After terminating the exchange the LLDD will call the transport's
2449  * normal io done path for the request, but it will have an aborted
2450  * status. The done path will return the io request back to the block
2451  * layer with an error status.
2452  */
nvme_fc_terminate_exchange(struct request * req,void * data)2453 static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2454 {
2455 	struct nvme_ctrl *nctrl = data;
2456 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2457 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2458 
2459 	op->nreq.flags |= NVME_REQ_CANCELLED;
2460 	__nvme_fc_abort_op(ctrl, op);
2461 	return true;
2462 }
2463 
2464 /*
2465  * This routine runs through all outstanding commands on the association
2466  * and aborts them.  This routine is typically be called by the
2467  * delete_association routine. It is also called due to an error during
2468  * reconnect. In that scenario, it is most likely a command that initializes
2469  * the controller, including fabric Connect commands on io queues, that
2470  * may have timed out or failed thus the io must be killed for the connect
2471  * thread to see the error.
2472  */
2473 static void
__nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl * ctrl,bool start_queues)2474 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2475 {
2476 	int q;
2477 
2478 	/*
2479 	 * if aborting io, the queues are no longer good, mark them
2480 	 * all as not live.
2481 	 */
2482 	if (ctrl->ctrl.queue_count > 1) {
2483 		for (q = 1; q < ctrl->ctrl.queue_count; q++)
2484 			clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2485 	}
2486 	clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2487 
2488 	/*
2489 	 * If io queues are present, stop them and terminate all outstanding
2490 	 * ios on them. As FC allocates FC exchange for each io, the
2491 	 * transport must contact the LLDD to terminate the exchange,
2492 	 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2493 	 * to tell us what io's are busy and invoke a transport routine
2494 	 * to kill them with the LLDD.  After terminating the exchange
2495 	 * the LLDD will call the transport's normal io done path, but it
2496 	 * will have an aborted status. The done path will return the
2497 	 * io requests back to the block layer as part of normal completions
2498 	 * (but with error status).
2499 	 */
2500 	if (ctrl->ctrl.queue_count > 1) {
2501 		nvme_quiesce_io_queues(&ctrl->ctrl);
2502 		nvme_sync_io_queues(&ctrl->ctrl);
2503 		blk_mq_tagset_busy_iter(&ctrl->tag_set,
2504 				nvme_fc_terminate_exchange, &ctrl->ctrl);
2505 		blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2506 		if (start_queues)
2507 			nvme_unquiesce_io_queues(&ctrl->ctrl);
2508 	}
2509 
2510 	/*
2511 	 * Other transports, which don't have link-level contexts bound
2512 	 * to sqe's, would try to gracefully shutdown the controller by
2513 	 * writing the registers for shutdown and polling (call
2514 	 * nvme_disable_ctrl()). Given a bunch of i/o was potentially
2515 	 * just aborted and we will wait on those contexts, and given
2516 	 * there was no indication of how live the controlelr is on the
2517 	 * link, don't send more io to create more contexts for the
2518 	 * shutdown. Let the controller fail via keepalive failure if
2519 	 * its still present.
2520 	 */
2521 
2522 	/*
2523 	 * clean up the admin queue. Same thing as above.
2524 	 */
2525 	nvme_quiesce_admin_queue(&ctrl->ctrl);
2526 	blk_sync_queue(ctrl->ctrl.admin_q);
2527 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2528 				nvme_fc_terminate_exchange, &ctrl->ctrl);
2529 	blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2530 	if (start_queues)
2531 		nvme_unquiesce_admin_queue(&ctrl->ctrl);
2532 }
2533 
2534 static void
nvme_fc_error_recovery(struct nvme_fc_ctrl * ctrl,char * errmsg)2535 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2536 {
2537 	/*
2538 	 * if an error (io timeout, etc) while (re)connecting, the remote
2539 	 * port requested terminating of the association (disconnect_ls)
2540 	 * or an error (timeout or abort) occurred on an io while creating
2541 	 * the controller.  Abort any ios on the association and let the
2542 	 * create_association error path resolve things.
2543 	 */
2544 	if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
2545 		__nvme_fc_abort_outstanding_ios(ctrl, true);
2546 		set_bit(ASSOC_FAILED, &ctrl->flags);
2547 		dev_warn(ctrl->ctrl.device,
2548 			"NVME-FC{%d}: transport error during (re)connect\n",
2549 			ctrl->cnum);
2550 		return;
2551 	}
2552 
2553 	/* Otherwise, only proceed if in LIVE state - e.g. on first error */
2554 	if (ctrl->ctrl.state != NVME_CTRL_LIVE)
2555 		return;
2556 
2557 	dev_warn(ctrl->ctrl.device,
2558 		"NVME-FC{%d}: transport association event: %s\n",
2559 		ctrl->cnum, errmsg);
2560 	dev_warn(ctrl->ctrl.device,
2561 		"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2562 
2563 	nvme_reset_ctrl(&ctrl->ctrl);
2564 }
2565 
nvme_fc_timeout(struct request * rq)2566 static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2567 {
2568 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2569 	struct nvme_fc_ctrl *ctrl = op->ctrl;
2570 	u16 qnum = op->queue->qnum;
2571 	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2572 	struct nvme_command *sqe = &cmdiu->sqe;
2573 
2574 	/*
2575 	 * Attempt to abort the offending command. Command completion
2576 	 * will detect the aborted io and will fail the connection.
2577 	 */
2578 	dev_info(ctrl->ctrl.device,
2579 		"NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: "
2580 		"x%08x/x%08x\n",
2581 		ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype,
2582 		nvme_fabrics_opcode_str(qnum, sqe),
2583 		sqe->common.cdw10, sqe->common.cdw11);
2584 	if (__nvme_fc_abort_op(ctrl, op))
2585 		nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2586 
2587 	/*
2588 	 * the io abort has been initiated. Have the reset timer
2589 	 * restarted and the abort completion will complete the io
2590 	 * shortly. Avoids a synchronous wait while the abort finishes.
2591 	 */
2592 	return BLK_EH_RESET_TIMER;
2593 }
2594 
2595 static int
nvme_fc_map_data(struct nvme_fc_ctrl * ctrl,struct request * rq,struct nvme_fc_fcp_op * op)2596 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2597 		struct nvme_fc_fcp_op *op)
2598 {
2599 	struct nvmefc_fcp_req *freq = &op->fcp_req;
2600 	int ret;
2601 
2602 	freq->sg_cnt = 0;
2603 
2604 	if (!blk_rq_nr_phys_segments(rq))
2605 		return 0;
2606 
2607 	freq->sg_table.sgl = freq->first_sgl;
2608 	ret = sg_alloc_table_chained(&freq->sg_table,
2609 			blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2610 			NVME_INLINE_SG_CNT);
2611 	if (ret)
2612 		return -ENOMEM;
2613 
2614 	op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
2615 	WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2616 	freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2617 				op->nents, rq_dma_dir(rq));
2618 	if (unlikely(freq->sg_cnt <= 0)) {
2619 		sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2620 		freq->sg_cnt = 0;
2621 		return -EFAULT;
2622 	}
2623 
2624 	/*
2625 	 * TODO: blk_integrity_rq(rq)  for DIF
2626 	 */
2627 	return 0;
2628 }
2629 
2630 static void
nvme_fc_unmap_data(struct nvme_fc_ctrl * ctrl,struct request * rq,struct nvme_fc_fcp_op * op)2631 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2632 		struct nvme_fc_fcp_op *op)
2633 {
2634 	struct nvmefc_fcp_req *freq = &op->fcp_req;
2635 
2636 	if (!freq->sg_cnt)
2637 		return;
2638 
2639 	fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2640 			rq_dma_dir(rq));
2641 
2642 	sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2643 
2644 	freq->sg_cnt = 0;
2645 }
2646 
2647 /*
2648  * In FC, the queue is a logical thing. At transport connect, the target
2649  * creates its "queue" and returns a handle that is to be given to the
2650  * target whenever it posts something to the corresponding SQ.  When an
2651  * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2652  * command contained within the SQE, an io, and assigns a FC exchange
2653  * to it. The SQE and the associated SQ handle are sent in the initial
2654  * CMD IU sents on the exchange. All transfers relative to the io occur
2655  * as part of the exchange.  The CQE is the last thing for the io,
2656  * which is transferred (explicitly or implicitly) with the RSP IU
2657  * sent on the exchange. After the CQE is received, the FC exchange is
2658  * terminaed and the Exchange may be used on a different io.
2659  *
2660  * The transport to LLDD api has the transport making a request for a
2661  * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2662  * resource and transfers the command. The LLDD will then process all
2663  * steps to complete the io. Upon completion, the transport done routine
2664  * is called.
2665  *
2666  * So - while the operation is outstanding to the LLDD, there is a link
2667  * level FC exchange resource that is also outstanding. This must be
2668  * considered in all cleanup operations.
2669  */
2670 static blk_status_t
nvme_fc_start_fcp_op(struct nvme_fc_ctrl * ctrl,struct nvme_fc_queue * queue,struct nvme_fc_fcp_op * op,u32 data_len,enum nvmefc_fcp_datadir io_dir)2671 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2672 	struct nvme_fc_fcp_op *op, u32 data_len,
2673 	enum nvmefc_fcp_datadir	io_dir)
2674 {
2675 	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2676 	struct nvme_command *sqe = &cmdiu->sqe;
2677 	int ret, opstate;
2678 
2679 	/*
2680 	 * before attempting to send the io, check to see if we believe
2681 	 * the target device is present
2682 	 */
2683 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2684 		return BLK_STS_RESOURCE;
2685 
2686 	if (!nvme_fc_ctrl_get(ctrl))
2687 		return BLK_STS_IOERR;
2688 
2689 	/* format the FC-NVME CMD IU and fcp_req */
2690 	cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2691 	cmdiu->data_len = cpu_to_be32(data_len);
2692 	switch (io_dir) {
2693 	case NVMEFC_FCP_WRITE:
2694 		cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2695 		break;
2696 	case NVMEFC_FCP_READ:
2697 		cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2698 		break;
2699 	case NVMEFC_FCP_NODATA:
2700 		cmdiu->flags = 0;
2701 		break;
2702 	}
2703 	op->fcp_req.payload_length = data_len;
2704 	op->fcp_req.io_dir = io_dir;
2705 	op->fcp_req.transferred_length = 0;
2706 	op->fcp_req.rcv_rsplen = 0;
2707 	op->fcp_req.status = NVME_SC_SUCCESS;
2708 	op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2709 
2710 	/*
2711 	 * validate per fabric rules, set fields mandated by fabric spec
2712 	 * as well as those by FC-NVME spec.
2713 	 */
2714 	WARN_ON_ONCE(sqe->common.metadata);
2715 	sqe->common.flags |= NVME_CMD_SGL_METABUF;
2716 
2717 	/*
2718 	 * format SQE DPTR field per FC-NVME rules:
2719 	 *    type=0x5     Transport SGL Data Block Descriptor
2720 	 *    subtype=0xA  Transport-specific value
2721 	 *    address=0
2722 	 *    length=length of the data series
2723 	 */
2724 	sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2725 					NVME_SGL_FMT_TRANSPORT_A;
2726 	sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2727 	sqe->rw.dptr.sgl.addr = 0;
2728 
2729 	if (!(op->flags & FCOP_FLAGS_AEN)) {
2730 		ret = nvme_fc_map_data(ctrl, op->rq, op);
2731 		if (ret < 0) {
2732 			nvme_cleanup_cmd(op->rq);
2733 			nvme_fc_ctrl_put(ctrl);
2734 			if (ret == -ENOMEM || ret == -EAGAIN)
2735 				return BLK_STS_RESOURCE;
2736 			return BLK_STS_IOERR;
2737 		}
2738 	}
2739 
2740 	fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2741 				  sizeof(op->cmd_iu), DMA_TO_DEVICE);
2742 
2743 	atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2744 
2745 	if (!(op->flags & FCOP_FLAGS_AEN))
2746 		nvme_start_request(op->rq);
2747 
2748 	cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2749 	ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2750 					&ctrl->rport->remoteport,
2751 					queue->lldd_handle, &op->fcp_req);
2752 
2753 	if (ret) {
2754 		/*
2755 		 * If the lld fails to send the command is there an issue with
2756 		 * the csn value?  If the command that fails is the Connect,
2757 		 * no - as the connection won't be live.  If it is a command
2758 		 * post-connect, it's possible a gap in csn may be created.
2759 		 * Does this matter?  As Linux initiators don't send fused
2760 		 * commands, no.  The gap would exist, but as there's nothing
2761 		 * that depends on csn order to be delivered on the target
2762 		 * side, it shouldn't hurt.  It would be difficult for a
2763 		 * target to even detect the csn gap as it has no idea when the
2764 		 * cmd with the csn was supposed to arrive.
2765 		 */
2766 		opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2767 		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2768 
2769 		if (!(op->flags & FCOP_FLAGS_AEN)) {
2770 			nvme_fc_unmap_data(ctrl, op->rq, op);
2771 			nvme_cleanup_cmd(op->rq);
2772 		}
2773 
2774 		nvme_fc_ctrl_put(ctrl);
2775 
2776 		if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2777 				ret != -EBUSY)
2778 			return BLK_STS_IOERR;
2779 
2780 		return BLK_STS_RESOURCE;
2781 	}
2782 
2783 	return BLK_STS_OK;
2784 }
2785 
2786 static blk_status_t
nvme_fc_queue_rq(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * bd)2787 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2788 			const struct blk_mq_queue_data *bd)
2789 {
2790 	struct nvme_ns *ns = hctx->queue->queuedata;
2791 	struct nvme_fc_queue *queue = hctx->driver_data;
2792 	struct nvme_fc_ctrl *ctrl = queue->ctrl;
2793 	struct request *rq = bd->rq;
2794 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2795 	enum nvmefc_fcp_datadir	io_dir;
2796 	bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2797 	u32 data_len;
2798 	blk_status_t ret;
2799 
2800 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2801 	    !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2802 		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2803 
2804 	ret = nvme_setup_cmd(ns, rq);
2805 	if (ret)
2806 		return ret;
2807 
2808 	/*
2809 	 * nvme core doesn't quite treat the rq opaquely. Commands such
2810 	 * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2811 	 * there is no actual payload to be transferred.
2812 	 * To get it right, key data transmission on there being 1 or
2813 	 * more physical segments in the sg list. If there is no
2814 	 * physical segments, there is no payload.
2815 	 */
2816 	if (blk_rq_nr_phys_segments(rq)) {
2817 		data_len = blk_rq_payload_bytes(rq);
2818 		io_dir = ((rq_data_dir(rq) == WRITE) ?
2819 					NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2820 	} else {
2821 		data_len = 0;
2822 		io_dir = NVMEFC_FCP_NODATA;
2823 	}
2824 
2825 
2826 	return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2827 }
2828 
2829 static void
nvme_fc_submit_async_event(struct nvme_ctrl * arg)2830 nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2831 {
2832 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2833 	struct nvme_fc_fcp_op *aen_op;
2834 	blk_status_t ret;
2835 
2836 	if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2837 		return;
2838 
2839 	aen_op = &ctrl->aen_ops[0];
2840 
2841 	ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2842 					NVMEFC_FCP_NODATA);
2843 	if (ret)
2844 		dev_err(ctrl->ctrl.device,
2845 			"failed async event work\n");
2846 }
2847 
2848 static void
nvme_fc_complete_rq(struct request * rq)2849 nvme_fc_complete_rq(struct request *rq)
2850 {
2851 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2852 	struct nvme_fc_ctrl *ctrl = op->ctrl;
2853 
2854 	atomic_set(&op->state, FCPOP_STATE_IDLE);
2855 	op->flags &= ~FCOP_FLAGS_TERMIO;
2856 
2857 	nvme_fc_unmap_data(ctrl, rq, op);
2858 	nvme_complete_rq(rq);
2859 	nvme_fc_ctrl_put(ctrl);
2860 }
2861 
nvme_fc_map_queues(struct blk_mq_tag_set * set)2862 static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2863 {
2864 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2865 	int i;
2866 
2867 	for (i = 0; i < set->nr_maps; i++) {
2868 		struct blk_mq_queue_map *map = &set->map[i];
2869 
2870 		if (!map->nr_queues) {
2871 			WARN_ON(i == HCTX_TYPE_DEFAULT);
2872 			continue;
2873 		}
2874 
2875 		/* Call LLDD map queue functionality if defined */
2876 		if (ctrl->lport->ops->map_queues)
2877 			ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2878 						     map);
2879 		else
2880 			blk_mq_map_queues(map);
2881 	}
2882 }
2883 
2884 static const struct blk_mq_ops nvme_fc_mq_ops = {
2885 	.queue_rq	= nvme_fc_queue_rq,
2886 	.complete	= nvme_fc_complete_rq,
2887 	.init_request	= nvme_fc_init_request,
2888 	.exit_request	= nvme_fc_exit_request,
2889 	.init_hctx	= nvme_fc_init_hctx,
2890 	.timeout	= nvme_fc_timeout,
2891 	.map_queues	= nvme_fc_map_queues,
2892 };
2893 
2894 static int
nvme_fc_create_io_queues(struct nvme_fc_ctrl * ctrl)2895 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2896 {
2897 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2898 	unsigned int nr_io_queues;
2899 	int ret;
2900 
2901 	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2902 				ctrl->lport->ops->max_hw_queues);
2903 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2904 	if (ret) {
2905 		dev_info(ctrl->ctrl.device,
2906 			"set_queue_count failed: %d\n", ret);
2907 		return ret;
2908 	}
2909 
2910 	ctrl->ctrl.queue_count = nr_io_queues + 1;
2911 	if (!nr_io_queues)
2912 		return 0;
2913 
2914 	nvme_fc_init_io_queues(ctrl);
2915 
2916 	ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
2917 			&nvme_fc_mq_ops, 1,
2918 			struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2919 				      ctrl->lport->ops->fcprqst_priv_sz));
2920 	if (ret)
2921 		return ret;
2922 
2923 	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2924 	if (ret)
2925 		goto out_cleanup_tagset;
2926 
2927 	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2928 	if (ret)
2929 		goto out_delete_hw_queues;
2930 
2931 	ctrl->ioq_live = true;
2932 
2933 	return 0;
2934 
2935 out_delete_hw_queues:
2936 	nvme_fc_delete_hw_io_queues(ctrl);
2937 out_cleanup_tagset:
2938 	nvme_remove_io_tag_set(&ctrl->ctrl);
2939 	nvme_fc_free_io_queues(ctrl);
2940 
2941 	/* force put free routine to ignore io queues */
2942 	ctrl->ctrl.tagset = NULL;
2943 
2944 	return ret;
2945 }
2946 
2947 static int
nvme_fc_recreate_io_queues(struct nvme_fc_ctrl * ctrl)2948 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2949 {
2950 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2951 	u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2952 	unsigned int nr_io_queues;
2953 	int ret;
2954 
2955 	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2956 				ctrl->lport->ops->max_hw_queues);
2957 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2958 	if (ret) {
2959 		dev_info(ctrl->ctrl.device,
2960 			"set_queue_count failed: %d\n", ret);
2961 		return ret;
2962 	}
2963 
2964 	if (!nr_io_queues && prior_ioq_cnt) {
2965 		dev_info(ctrl->ctrl.device,
2966 			"Fail Reconnect: At least 1 io queue "
2967 			"required (was %d)\n", prior_ioq_cnt);
2968 		return -ENOSPC;
2969 	}
2970 
2971 	ctrl->ctrl.queue_count = nr_io_queues + 1;
2972 	/* check for io queues existing */
2973 	if (ctrl->ctrl.queue_count == 1)
2974 		return 0;
2975 
2976 	if (prior_ioq_cnt != nr_io_queues) {
2977 		dev_info(ctrl->ctrl.device,
2978 			"reconnect: revising io queue count from %d to %d\n",
2979 			prior_ioq_cnt, nr_io_queues);
2980 		blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2981 	}
2982 
2983 	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2984 	if (ret)
2985 		goto out_free_io_queues;
2986 
2987 	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2988 	if (ret)
2989 		goto out_delete_hw_queues;
2990 
2991 	return 0;
2992 
2993 out_delete_hw_queues:
2994 	nvme_fc_delete_hw_io_queues(ctrl);
2995 out_free_io_queues:
2996 	nvme_fc_free_io_queues(ctrl);
2997 	return ret;
2998 }
2999 
3000 static void
nvme_fc_rport_active_on_lport(struct nvme_fc_rport * rport)3001 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3002 {
3003 	struct nvme_fc_lport *lport = rport->lport;
3004 
3005 	atomic_inc(&lport->act_rport_cnt);
3006 }
3007 
3008 static void
nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport * rport)3009 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3010 {
3011 	struct nvme_fc_lport *lport = rport->lport;
3012 	u32 cnt;
3013 
3014 	cnt = atomic_dec_return(&lport->act_rport_cnt);
3015 	if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3016 		lport->ops->localport_delete(&lport->localport);
3017 }
3018 
3019 static int
nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl * ctrl)3020 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3021 {
3022 	struct nvme_fc_rport *rport = ctrl->rport;
3023 	u32 cnt;
3024 
3025 	if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
3026 		return 1;
3027 
3028 	cnt = atomic_inc_return(&rport->act_ctrl_cnt);
3029 	if (cnt == 1)
3030 		nvme_fc_rport_active_on_lport(rport);
3031 
3032 	return 0;
3033 }
3034 
3035 static int
nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl * ctrl)3036 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3037 {
3038 	struct nvme_fc_rport *rport = ctrl->rport;
3039 	struct nvme_fc_lport *lport = rport->lport;
3040 	u32 cnt;
3041 
3042 	/* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3043 
3044 	cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3045 	if (cnt == 0) {
3046 		if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3047 			lport->ops->remoteport_delete(&rport->remoteport);
3048 		nvme_fc_rport_inactive_on_lport(rport);
3049 	}
3050 
3051 	return 0;
3052 }
3053 
3054 /*
3055  * This routine restarts the controller on the host side, and
3056  * on the link side, recreates the controller association.
3057  */
3058 static int
nvme_fc_create_association(struct nvme_fc_ctrl * ctrl)3059 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3060 {
3061 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3062 	struct nvmefc_ls_rcv_op *disls = NULL;
3063 	unsigned long flags;
3064 	int ret;
3065 	bool changed;
3066 
3067 	++ctrl->ctrl.nr_reconnects;
3068 
3069 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3070 		return -ENODEV;
3071 
3072 	if (nvme_fc_ctlr_active_on_rport(ctrl))
3073 		return -ENOTUNIQ;
3074 
3075 	dev_info(ctrl->ctrl.device,
3076 		"NVME-FC{%d}: create association : host wwpn 0x%016llx "
3077 		" rport wwpn 0x%016llx: NQN \"%s\"\n",
3078 		ctrl->cnum, ctrl->lport->localport.port_name,
3079 		ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3080 
3081 	clear_bit(ASSOC_FAILED, &ctrl->flags);
3082 
3083 	/*
3084 	 * Create the admin queue
3085 	 */
3086 
3087 	ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3088 				NVME_AQ_DEPTH);
3089 	if (ret)
3090 		goto out_free_queue;
3091 
3092 	ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3093 				NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3094 	if (ret)
3095 		goto out_delete_hw_queue;
3096 
3097 	ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3098 	if (ret)
3099 		goto out_disconnect_admin_queue;
3100 
3101 	set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3102 
3103 	/*
3104 	 * Check controller capabilities
3105 	 *
3106 	 * todo:- add code to check if ctrl attributes changed from
3107 	 * prior connection values
3108 	 */
3109 
3110 	ret = nvme_enable_ctrl(&ctrl->ctrl);
3111 	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3112 		ret = -EIO;
3113 	if (ret)
3114 		goto out_disconnect_admin_queue;
3115 
3116 	ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3117 	ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3118 						(ilog2(SZ_4K) - 9);
3119 
3120 	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3121 
3122 	ret = nvme_init_ctrl_finish(&ctrl->ctrl, false);
3123 	if (ret)
3124 		goto out_disconnect_admin_queue;
3125 	if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3126 		ret = -EIO;
3127 		goto out_stop_keep_alive;
3128 	}
3129 	/* sanity checks */
3130 
3131 	/* FC-NVME does not have other data in the capsule */
3132 	if (ctrl->ctrl.icdoff) {
3133 		dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3134 				ctrl->ctrl.icdoff);
3135 		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3136 		goto out_stop_keep_alive;
3137 	}
3138 
3139 	/* FC-NVME supports normal SGL Data Block Descriptors */
3140 	if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3141 		dev_err(ctrl->ctrl.device,
3142 			"Mandatory sgls are not supported!\n");
3143 		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3144 		goto out_stop_keep_alive;
3145 	}
3146 
3147 	if (opts->queue_size > ctrl->ctrl.maxcmd) {
3148 		/* warn if maxcmd is lower than queue_size */
3149 		dev_warn(ctrl->ctrl.device,
3150 			"queue_size %zu > ctrl maxcmd %u, reducing "
3151 			"to maxcmd\n",
3152 			opts->queue_size, ctrl->ctrl.maxcmd);
3153 		opts->queue_size = ctrl->ctrl.maxcmd;
3154 		ctrl->ctrl.sqsize = opts->queue_size - 1;
3155 	}
3156 
3157 	ret = nvme_fc_init_aen_ops(ctrl);
3158 	if (ret)
3159 		goto out_term_aen_ops;
3160 
3161 	/*
3162 	 * Create the io queues
3163 	 */
3164 
3165 	if (ctrl->ctrl.queue_count > 1) {
3166 		if (!ctrl->ioq_live)
3167 			ret = nvme_fc_create_io_queues(ctrl);
3168 		else
3169 			ret = nvme_fc_recreate_io_queues(ctrl);
3170 	}
3171 	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3172 		ret = -EIO;
3173 	if (ret)
3174 		goto out_term_aen_ops;
3175 
3176 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
3177 
3178 	ctrl->ctrl.nr_reconnects = 0;
3179 
3180 	if (changed)
3181 		nvme_start_ctrl(&ctrl->ctrl);
3182 
3183 	return 0;	/* Success */
3184 
3185 out_term_aen_ops:
3186 	nvme_fc_term_aen_ops(ctrl);
3187 out_stop_keep_alive:
3188 	nvme_stop_keep_alive(&ctrl->ctrl);
3189 out_disconnect_admin_queue:
3190 	dev_warn(ctrl->ctrl.device,
3191 		"NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3192 		ctrl->cnum, ctrl->association_id, ret);
3193 	/* send a Disconnect(association) LS to fc-nvme target */
3194 	nvme_fc_xmt_disconnect_assoc(ctrl);
3195 	spin_lock_irqsave(&ctrl->lock, flags);
3196 	ctrl->association_id = 0;
3197 	disls = ctrl->rcv_disconn;
3198 	ctrl->rcv_disconn = NULL;
3199 	spin_unlock_irqrestore(&ctrl->lock, flags);
3200 	if (disls)
3201 		nvme_fc_xmt_ls_rsp(disls);
3202 out_delete_hw_queue:
3203 	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3204 out_free_queue:
3205 	nvme_fc_free_queue(&ctrl->queues[0]);
3206 	clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3207 	nvme_fc_ctlr_inactive_on_rport(ctrl);
3208 
3209 	return ret;
3210 }
3211 
3212 
3213 /*
3214  * This routine stops operation of the controller on the host side.
3215  * On the host os stack side: Admin and IO queues are stopped,
3216  *   outstanding ios on them terminated via FC ABTS.
3217  * On the link side: the association is terminated.
3218  */
3219 static void
nvme_fc_delete_association(struct nvme_fc_ctrl * ctrl)3220 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3221 {
3222 	struct nvmefc_ls_rcv_op *disls = NULL;
3223 	unsigned long flags;
3224 
3225 	if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3226 		return;
3227 
3228 	spin_lock_irqsave(&ctrl->lock, flags);
3229 	set_bit(FCCTRL_TERMIO, &ctrl->flags);
3230 	ctrl->iocnt = 0;
3231 	spin_unlock_irqrestore(&ctrl->lock, flags);
3232 
3233 	__nvme_fc_abort_outstanding_ios(ctrl, false);
3234 
3235 	/* kill the aens as they are a separate path */
3236 	nvme_fc_abort_aen_ops(ctrl);
3237 
3238 	/* wait for all io that had to be aborted */
3239 	spin_lock_irq(&ctrl->lock);
3240 	wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3241 	clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3242 	spin_unlock_irq(&ctrl->lock);
3243 
3244 	nvme_fc_term_aen_ops(ctrl);
3245 
3246 	/*
3247 	 * send a Disconnect(association) LS to fc-nvme target
3248 	 * Note: could have been sent at top of process, but
3249 	 * cleaner on link traffic if after the aborts complete.
3250 	 * Note: if association doesn't exist, association_id will be 0
3251 	 */
3252 	if (ctrl->association_id)
3253 		nvme_fc_xmt_disconnect_assoc(ctrl);
3254 
3255 	spin_lock_irqsave(&ctrl->lock, flags);
3256 	ctrl->association_id = 0;
3257 	disls = ctrl->rcv_disconn;
3258 	ctrl->rcv_disconn = NULL;
3259 	spin_unlock_irqrestore(&ctrl->lock, flags);
3260 	if (disls)
3261 		/*
3262 		 * if a Disconnect Request was waiting for a response, send
3263 		 * now that all ABTS's have been issued (and are complete).
3264 		 */
3265 		nvme_fc_xmt_ls_rsp(disls);
3266 
3267 	if (ctrl->ctrl.tagset) {
3268 		nvme_fc_delete_hw_io_queues(ctrl);
3269 		nvme_fc_free_io_queues(ctrl);
3270 	}
3271 
3272 	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3273 	nvme_fc_free_queue(&ctrl->queues[0]);
3274 
3275 	/* re-enable the admin_q so anything new can fast fail */
3276 	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3277 
3278 	/* resume the io queues so that things will fast fail */
3279 	nvme_unquiesce_io_queues(&ctrl->ctrl);
3280 
3281 	nvme_fc_ctlr_inactive_on_rport(ctrl);
3282 }
3283 
3284 static void
nvme_fc_delete_ctrl(struct nvme_ctrl * nctrl)3285 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3286 {
3287 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3288 
3289 	cancel_work_sync(&ctrl->ioerr_work);
3290 	cancel_delayed_work_sync(&ctrl->connect_work);
3291 	/*
3292 	 * kill the association on the link side.  this will block
3293 	 * waiting for io to terminate
3294 	 */
3295 	nvme_fc_delete_association(ctrl);
3296 }
3297 
3298 static void
nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl * ctrl,int status)3299 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3300 {
3301 	struct nvme_fc_rport *rport = ctrl->rport;
3302 	struct nvme_fc_remote_port *portptr = &rport->remoteport;
3303 	unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3304 	bool recon = true;
3305 
3306 	if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_CONNECTING)
3307 		return;
3308 
3309 	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3310 		dev_info(ctrl->ctrl.device,
3311 			"NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3312 			ctrl->cnum, status);
3313 	} else if (time_after_eq(jiffies, rport->dev_loss_end))
3314 		recon = false;
3315 
3316 	if (recon && nvmf_should_reconnect(&ctrl->ctrl, status)) {
3317 		if (portptr->port_state == FC_OBJSTATE_ONLINE)
3318 			dev_info(ctrl->ctrl.device,
3319 				"NVME-FC{%d}: Reconnect attempt in %ld "
3320 				"seconds\n",
3321 				ctrl->cnum, recon_delay / HZ);
3322 		else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3323 			recon_delay = rport->dev_loss_end - jiffies;
3324 
3325 		queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3326 	} else {
3327 		if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3328 			if (status > 0 && (status & NVME_STATUS_DNR))
3329 				dev_warn(ctrl->ctrl.device,
3330 					 "NVME-FC{%d}: reconnect failure\n",
3331 					 ctrl->cnum);
3332 			else
3333 				dev_warn(ctrl->ctrl.device,
3334 					 "NVME-FC{%d}: Max reconnect attempts "
3335 					 "(%d) reached.\n",
3336 					 ctrl->cnum, ctrl->ctrl.nr_reconnects);
3337 		} else
3338 			dev_warn(ctrl->ctrl.device,
3339 				"NVME-FC{%d}: dev_loss_tmo (%d) expired "
3340 				"while waiting for remoteport connectivity.\n",
3341 				ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3342 					(ctrl->ctrl.opts->max_reconnects *
3343 					 ctrl->ctrl.opts->reconnect_delay)));
3344 		WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3345 	}
3346 }
3347 
3348 static void
nvme_fc_reset_ctrl_work(struct work_struct * work)3349 nvme_fc_reset_ctrl_work(struct work_struct *work)
3350 {
3351 	struct nvme_fc_ctrl *ctrl =
3352 		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3353 
3354 	nvme_stop_ctrl(&ctrl->ctrl);
3355 
3356 	/* will block will waiting for io to terminate */
3357 	nvme_fc_delete_association(ctrl);
3358 
3359 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3360 		dev_err(ctrl->ctrl.device,
3361 			"NVME-FC{%d}: error_recovery: Couldn't change state "
3362 			"to CONNECTING\n", ctrl->cnum);
3363 
3364 	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3365 		if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3366 			dev_err(ctrl->ctrl.device,
3367 				"NVME-FC{%d}: failed to schedule connect "
3368 				"after reset\n", ctrl->cnum);
3369 		} else {
3370 			flush_delayed_work(&ctrl->connect_work);
3371 		}
3372 	} else {
3373 		nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3374 	}
3375 }
3376 
3377 
3378 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3379 	.name			= "fc",
3380 	.module			= THIS_MODULE,
3381 	.flags			= NVME_F_FABRICS,
3382 	.reg_read32		= nvmf_reg_read32,
3383 	.reg_read64		= nvmf_reg_read64,
3384 	.reg_write32		= nvmf_reg_write32,
3385 	.subsystem_reset	= nvmf_subsystem_reset,
3386 	.free_ctrl		= nvme_fc_free_ctrl,
3387 	.submit_async_event	= nvme_fc_submit_async_event,
3388 	.delete_ctrl		= nvme_fc_delete_ctrl,
3389 	.get_address		= nvmf_get_address,
3390 };
3391 
3392 static void
nvme_fc_connect_ctrl_work(struct work_struct * work)3393 nvme_fc_connect_ctrl_work(struct work_struct *work)
3394 {
3395 	int ret;
3396 
3397 	struct nvme_fc_ctrl *ctrl =
3398 			container_of(to_delayed_work(work),
3399 				struct nvme_fc_ctrl, connect_work);
3400 
3401 	ret = nvme_fc_create_association(ctrl);
3402 	if (ret)
3403 		nvme_fc_reconnect_or_delete(ctrl, ret);
3404 	else
3405 		dev_info(ctrl->ctrl.device,
3406 			"NVME-FC{%d}: controller connect complete\n",
3407 			ctrl->cnum);
3408 }
3409 
3410 
3411 static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3412 	.queue_rq	= nvme_fc_queue_rq,
3413 	.complete	= nvme_fc_complete_rq,
3414 	.init_request	= nvme_fc_init_request,
3415 	.exit_request	= nvme_fc_exit_request,
3416 	.init_hctx	= nvme_fc_init_admin_hctx,
3417 	.timeout	= nvme_fc_timeout,
3418 };
3419 
3420 
3421 /*
3422  * Fails a controller request if it matches an existing controller
3423  * (association) with the same tuple:
3424  * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3425  *
3426  * The ports don't need to be compared as they are intrinsically
3427  * already matched by the port pointers supplied.
3428  */
3429 static bool
nvme_fc_existing_controller(struct nvme_fc_rport * rport,struct nvmf_ctrl_options * opts)3430 nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3431 		struct nvmf_ctrl_options *opts)
3432 {
3433 	struct nvme_fc_ctrl *ctrl;
3434 	unsigned long flags;
3435 	bool found = false;
3436 
3437 	spin_lock_irqsave(&rport->lock, flags);
3438 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3439 		found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3440 		if (found)
3441 			break;
3442 	}
3443 	spin_unlock_irqrestore(&rport->lock, flags);
3444 
3445 	return found;
3446 }
3447 
3448 static struct nvme_fc_ctrl *
nvme_fc_alloc_ctrl(struct device * dev,struct nvmf_ctrl_options * opts,struct nvme_fc_lport * lport,struct nvme_fc_rport * rport)3449 nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3450 	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3451 {
3452 	struct nvme_fc_ctrl *ctrl;
3453 	int ret, idx, ctrl_loss_tmo;
3454 
3455 	if (!(rport->remoteport.port_role &
3456 	    (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3457 		ret = -EBADR;
3458 		goto out_fail;
3459 	}
3460 
3461 	if (!opts->duplicate_connect &&
3462 	    nvme_fc_existing_controller(rport, opts)) {
3463 		ret = -EALREADY;
3464 		goto out_fail;
3465 	}
3466 
3467 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3468 	if (!ctrl) {
3469 		ret = -ENOMEM;
3470 		goto out_fail;
3471 	}
3472 
3473 	idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3474 	if (idx < 0) {
3475 		ret = -ENOSPC;
3476 		goto out_free_ctrl;
3477 	}
3478 
3479 	/*
3480 	 * if ctrl_loss_tmo is being enforced and the default reconnect delay
3481 	 * is being used, change to a shorter reconnect delay for FC.
3482 	 */
3483 	if (opts->max_reconnects != -1 &&
3484 	    opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3485 	    opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3486 		ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3487 		opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3488 		opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3489 						opts->reconnect_delay);
3490 	}
3491 
3492 	ctrl->ctrl.opts = opts;
3493 	ctrl->ctrl.nr_reconnects = 0;
3494 	INIT_LIST_HEAD(&ctrl->ctrl_list);
3495 	ctrl->lport = lport;
3496 	ctrl->rport = rport;
3497 	ctrl->dev = lport->dev;
3498 	ctrl->cnum = idx;
3499 	ctrl->ioq_live = false;
3500 	init_waitqueue_head(&ctrl->ioabort_wait);
3501 
3502 	get_device(ctrl->dev);
3503 	kref_init(&ctrl->ref);
3504 
3505 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3506 	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3507 	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3508 	spin_lock_init(&ctrl->lock);
3509 
3510 	/* io queue count */
3511 	ctrl->ctrl.queue_count = min_t(unsigned int,
3512 				opts->nr_io_queues,
3513 				lport->ops->max_hw_queues);
3514 	ctrl->ctrl.queue_count++;	/* +1 for admin queue */
3515 
3516 	ctrl->ctrl.sqsize = opts->queue_size - 1;
3517 	ctrl->ctrl.kato = opts->kato;
3518 	ctrl->ctrl.cntlid = 0xffff;
3519 
3520 	ret = -ENOMEM;
3521 	ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3522 				sizeof(struct nvme_fc_queue), GFP_KERNEL);
3523 	if (!ctrl->queues)
3524 		goto out_free_ida;
3525 
3526 	nvme_fc_init_queue(ctrl, 0);
3527 
3528 	/*
3529 	 * Would have been nice to init io queues tag set as well.
3530 	 * However, we require interaction from the controller
3531 	 * for max io queue count before we can do so.
3532 	 * Defer this to the connect path.
3533 	 */
3534 
3535 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3536 	if (ret)
3537 		goto out_free_queues;
3538 	if (lport->dev)
3539 		ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3540 
3541 	return ctrl;
3542 
3543 out_free_queues:
3544 	kfree(ctrl->queues);
3545 out_free_ida:
3546 	put_device(ctrl->dev);
3547 	ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3548 out_free_ctrl:
3549 	kfree(ctrl);
3550 out_fail:
3551 	/* exit via here doesn't follow ctlr ref points */
3552 	return ERR_PTR(ret);
3553 }
3554 
3555 static struct nvme_ctrl *
nvme_fc_init_ctrl(struct device * dev,struct nvmf_ctrl_options * opts,struct nvme_fc_lport * lport,struct nvme_fc_rport * rport)3556 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3557 	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3558 {
3559 	struct nvme_fc_ctrl *ctrl;
3560 	unsigned long flags;
3561 	int ret;
3562 
3563 	ctrl = nvme_fc_alloc_ctrl(dev, opts, lport, rport);
3564 	if (IS_ERR(ctrl))
3565 		return ERR_CAST(ctrl);
3566 
3567 	ret = nvme_add_ctrl(&ctrl->ctrl);
3568 	if (ret)
3569 		goto out_put_ctrl;
3570 
3571 	ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
3572 			&nvme_fc_admin_mq_ops,
3573 			struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3574 				      ctrl->lport->ops->fcprqst_priv_sz));
3575 	if (ret)
3576 		goto fail_ctrl;
3577 
3578 	spin_lock_irqsave(&rport->lock, flags);
3579 	list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3580 	spin_unlock_irqrestore(&rport->lock, flags);
3581 
3582 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING) ||
3583 	    !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3584 		dev_err(ctrl->ctrl.device,
3585 			"NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3586 		goto fail_ctrl;
3587 	}
3588 
3589 	if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3590 		dev_err(ctrl->ctrl.device,
3591 			"NVME-FC{%d}: failed to schedule initial connect\n",
3592 			ctrl->cnum);
3593 		goto fail_ctrl;
3594 	}
3595 
3596 	flush_delayed_work(&ctrl->connect_work);
3597 
3598 	dev_info(ctrl->ctrl.device,
3599 		"NVME-FC{%d}: new ctrl: NQN \"%s\", hostnqn: %s\n",
3600 		ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl), opts->host->nqn);
3601 
3602 	return &ctrl->ctrl;
3603 
3604 fail_ctrl:
3605 	nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3606 	cancel_work_sync(&ctrl->ioerr_work);
3607 	cancel_work_sync(&ctrl->ctrl.reset_work);
3608 	cancel_delayed_work_sync(&ctrl->connect_work);
3609 
3610 	ctrl->ctrl.opts = NULL;
3611 
3612 	/* initiate nvme ctrl ref counting teardown */
3613 	nvme_uninit_ctrl(&ctrl->ctrl);
3614 
3615 out_put_ctrl:
3616 	/* Remove core ctrl ref. */
3617 	nvme_put_ctrl(&ctrl->ctrl);
3618 
3619 	/* as we're past the point where we transition to the ref
3620 	 * counting teardown path, if we return a bad pointer here,
3621 	 * the calling routine, thinking it's prior to the
3622 	 * transition, will do an rport put. Since the teardown
3623 	 * path also does a rport put, we do an extra get here to
3624 	 * so proper order/teardown happens.
3625 	 */
3626 	nvme_fc_rport_get(rport);
3627 
3628 	return ERR_PTR(-EIO);
3629 }
3630 
3631 struct nvmet_fc_traddr {
3632 	u64	nn;
3633 	u64	pn;
3634 };
3635 
3636 static int
__nvme_fc_parse_u64(substring_t * sstr,u64 * val)3637 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3638 {
3639 	u64 token64;
3640 
3641 	if (match_u64(sstr, &token64))
3642 		return -EINVAL;
3643 	*val = token64;
3644 
3645 	return 0;
3646 }
3647 
3648 /*
3649  * This routine validates and extracts the WWN's from the TRADDR string.
3650  * As kernel parsers need the 0x to determine number base, universally
3651  * build string to parse with 0x prefix before parsing name strings.
3652  */
3653 static int
nvme_fc_parse_traddr(struct nvmet_fc_traddr * traddr,char * buf,size_t blen)3654 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3655 {
3656 	char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3657 	substring_t wwn = { name, &name[sizeof(name)-1] };
3658 	int nnoffset, pnoffset;
3659 
3660 	/* validate if string is one of the 2 allowed formats */
3661 	if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3662 			!strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3663 			!strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3664 				"pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3665 		nnoffset = NVME_FC_TRADDR_OXNNLEN;
3666 		pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3667 						NVME_FC_TRADDR_OXNNLEN;
3668 	} else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3669 			!strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3670 			!strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3671 				"pn-", NVME_FC_TRADDR_NNLEN))) {
3672 		nnoffset = NVME_FC_TRADDR_NNLEN;
3673 		pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3674 	} else
3675 		goto out_einval;
3676 
3677 	name[0] = '0';
3678 	name[1] = 'x';
3679 	name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3680 
3681 	memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3682 	if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3683 		goto out_einval;
3684 
3685 	memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3686 	if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3687 		goto out_einval;
3688 
3689 	return 0;
3690 
3691 out_einval:
3692 	pr_warn("%s: bad traddr string\n", __func__);
3693 	return -EINVAL;
3694 }
3695 
3696 static struct nvme_ctrl *
nvme_fc_create_ctrl(struct device * dev,struct nvmf_ctrl_options * opts)3697 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3698 {
3699 	struct nvme_fc_lport *lport;
3700 	struct nvme_fc_rport *rport;
3701 	struct nvme_ctrl *ctrl;
3702 	struct nvmet_fc_traddr laddr = { 0L, 0L };
3703 	struct nvmet_fc_traddr raddr = { 0L, 0L };
3704 	unsigned long flags;
3705 	int ret;
3706 
3707 	ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3708 	if (ret || !raddr.nn || !raddr.pn)
3709 		return ERR_PTR(-EINVAL);
3710 
3711 	ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3712 	if (ret || !laddr.nn || !laddr.pn)
3713 		return ERR_PTR(-EINVAL);
3714 
3715 	/* find the host and remote ports to connect together */
3716 	spin_lock_irqsave(&nvme_fc_lock, flags);
3717 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3718 		if (lport->localport.node_name != laddr.nn ||
3719 		    lport->localport.port_name != laddr.pn ||
3720 		    lport->localport.port_state != FC_OBJSTATE_ONLINE)
3721 			continue;
3722 
3723 		list_for_each_entry(rport, &lport->endp_list, endp_list) {
3724 			if (rport->remoteport.node_name != raddr.nn ||
3725 			    rport->remoteport.port_name != raddr.pn ||
3726 			    rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3727 				continue;
3728 
3729 			/* if fail to get reference fall through. Will error */
3730 			if (!nvme_fc_rport_get(rport))
3731 				break;
3732 
3733 			spin_unlock_irqrestore(&nvme_fc_lock, flags);
3734 
3735 			ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3736 			if (IS_ERR(ctrl))
3737 				nvme_fc_rport_put(rport);
3738 			return ctrl;
3739 		}
3740 	}
3741 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3742 
3743 	pr_warn("%s: %s - %s combination not found\n",
3744 		__func__, opts->traddr, opts->host_traddr);
3745 	return ERR_PTR(-ENOENT);
3746 }
3747 
3748 
3749 static struct nvmf_transport_ops nvme_fc_transport = {
3750 	.name		= "fc",
3751 	.module		= THIS_MODULE,
3752 	.required_opts	= NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3753 	.allowed_opts	= NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3754 	.create_ctrl	= nvme_fc_create_ctrl,
3755 };
3756 
3757 /* Arbitrary successive failures max. With lots of subsystems could be high */
3758 #define DISCOVERY_MAX_FAIL	20
3759 
nvme_fc_nvme_discovery_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3760 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3761 		struct device_attribute *attr, const char *buf, size_t count)
3762 {
3763 	unsigned long flags;
3764 	LIST_HEAD(local_disc_list);
3765 	struct nvme_fc_lport *lport;
3766 	struct nvme_fc_rport *rport;
3767 	int failcnt = 0;
3768 
3769 	spin_lock_irqsave(&nvme_fc_lock, flags);
3770 restart:
3771 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3772 		list_for_each_entry(rport, &lport->endp_list, endp_list) {
3773 			if (!nvme_fc_lport_get(lport))
3774 				continue;
3775 			if (!nvme_fc_rport_get(rport)) {
3776 				/*
3777 				 * This is a temporary condition. Upon restart
3778 				 * this rport will be gone from the list.
3779 				 *
3780 				 * Revert the lport put and retry.  Anything
3781 				 * added to the list already will be skipped (as
3782 				 * they are no longer list_empty).  Loops should
3783 				 * resume at rports that were not yet seen.
3784 				 */
3785 				nvme_fc_lport_put(lport);
3786 
3787 				if (failcnt++ < DISCOVERY_MAX_FAIL)
3788 					goto restart;
3789 
3790 				pr_err("nvme_discovery: too many reference "
3791 				       "failures\n");
3792 				goto process_local_list;
3793 			}
3794 			if (list_empty(&rport->disc_list))
3795 				list_add_tail(&rport->disc_list,
3796 					      &local_disc_list);
3797 		}
3798 	}
3799 
3800 process_local_list:
3801 	while (!list_empty(&local_disc_list)) {
3802 		rport = list_first_entry(&local_disc_list,
3803 					 struct nvme_fc_rport, disc_list);
3804 		list_del_init(&rport->disc_list);
3805 		spin_unlock_irqrestore(&nvme_fc_lock, flags);
3806 
3807 		lport = rport->lport;
3808 		/* signal discovery. Won't hurt if it repeats */
3809 		nvme_fc_signal_discovery_scan(lport, rport);
3810 		nvme_fc_rport_put(rport);
3811 		nvme_fc_lport_put(lport);
3812 
3813 		spin_lock_irqsave(&nvme_fc_lock, flags);
3814 	}
3815 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3816 
3817 	return count;
3818 }
3819 
3820 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3821 
3822 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3823 /* Parse the cgroup id from a buf and return the length of cgrpid */
fc_parse_cgrpid(const char * buf,u64 * id)3824 static int fc_parse_cgrpid(const char *buf, u64 *id)
3825 {
3826 	char cgrp_id[16+1];
3827 	int cgrpid_len, j;
3828 
3829 	memset(cgrp_id, 0x0, sizeof(cgrp_id));
3830 	for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3831 		if (buf[cgrpid_len] != ':')
3832 			cgrp_id[cgrpid_len] = buf[cgrpid_len];
3833 		else {
3834 			j = 1;
3835 			break;
3836 		}
3837 	}
3838 	if (!j)
3839 		return -EINVAL;
3840 	if (kstrtou64(cgrp_id, 16, id) < 0)
3841 		return -EINVAL;
3842 	return cgrpid_len;
3843 }
3844 
3845 /*
3846  * Parse and update the appid in the blkcg associated with the cgroupid.
3847  */
fc_appid_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)3848 static ssize_t fc_appid_store(struct device *dev,
3849 		struct device_attribute *attr, const char *buf, size_t count)
3850 {
3851 	size_t orig_count = count;
3852 	u64 cgrp_id;
3853 	int appid_len = 0;
3854 	int cgrpid_len = 0;
3855 	char app_id[FC_APPID_LEN];
3856 	int ret = 0;
3857 
3858 	if (buf[count-1] == '\n')
3859 		count--;
3860 
3861 	if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3862 		return -EINVAL;
3863 
3864 	cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3865 	if (cgrpid_len < 0)
3866 		return -EINVAL;
3867 	appid_len = count - cgrpid_len - 1;
3868 	if (appid_len > FC_APPID_LEN)
3869 		return -EINVAL;
3870 
3871 	memset(app_id, 0x0, sizeof(app_id));
3872 	memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3873 	ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3874 	if (ret < 0)
3875 		return ret;
3876 	return orig_count;
3877 }
3878 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3879 #endif /* CONFIG_BLK_CGROUP_FC_APPID */
3880 
3881 static struct attribute *nvme_fc_attrs[] = {
3882 	&dev_attr_nvme_discovery.attr,
3883 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3884 	&dev_attr_appid_store.attr,
3885 #endif
3886 	NULL
3887 };
3888 
3889 static const struct attribute_group nvme_fc_attr_group = {
3890 	.attrs = nvme_fc_attrs,
3891 };
3892 
3893 static const struct attribute_group *nvme_fc_attr_groups[] = {
3894 	&nvme_fc_attr_group,
3895 	NULL
3896 };
3897 
3898 static struct class fc_class = {
3899 	.name = "fc",
3900 	.dev_groups = nvme_fc_attr_groups,
3901 };
3902 
nvme_fc_init_module(void)3903 static int __init nvme_fc_init_module(void)
3904 {
3905 	int ret;
3906 
3907 	/*
3908 	 * NOTE:
3909 	 * It is expected that in the future the kernel will combine
3910 	 * the FC-isms that are currently under scsi and now being
3911 	 * added to by NVME into a new standalone FC class. The SCSI
3912 	 * and NVME protocols and their devices would be under this
3913 	 * new FC class.
3914 	 *
3915 	 * As we need something to post FC-specific udev events to,
3916 	 * specifically for nvme probe events, start by creating the
3917 	 * new device class.  When the new standalone FC class is
3918 	 * put in place, this code will move to a more generic
3919 	 * location for the class.
3920 	 */
3921 	ret = class_register(&fc_class);
3922 	if (ret) {
3923 		pr_err("couldn't register class fc\n");
3924 		return ret;
3925 	}
3926 
3927 	/*
3928 	 * Create a device for the FC-centric udev events
3929 	 */
3930 	fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3931 				"fc_udev_device");
3932 	if (IS_ERR(fc_udev_device)) {
3933 		pr_err("couldn't create fc_udev device!\n");
3934 		ret = PTR_ERR(fc_udev_device);
3935 		goto out_destroy_class;
3936 	}
3937 
3938 	ret = nvmf_register_transport(&nvme_fc_transport);
3939 	if (ret)
3940 		goto out_destroy_device;
3941 
3942 	return 0;
3943 
3944 out_destroy_device:
3945 	device_destroy(&fc_class, MKDEV(0, 0));
3946 out_destroy_class:
3947 	class_unregister(&fc_class);
3948 
3949 	return ret;
3950 }
3951 
3952 static void
nvme_fc_delete_controllers(struct nvme_fc_rport * rport)3953 nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3954 {
3955 	struct nvme_fc_ctrl *ctrl;
3956 
3957 	spin_lock(&rport->lock);
3958 	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3959 		dev_warn(ctrl->ctrl.device,
3960 			"NVME-FC{%d}: transport unloading: deleting ctrl\n",
3961 			ctrl->cnum);
3962 		nvme_delete_ctrl(&ctrl->ctrl);
3963 	}
3964 	spin_unlock(&rport->lock);
3965 }
3966 
nvme_fc_exit_module(void)3967 static void __exit nvme_fc_exit_module(void)
3968 {
3969 	struct nvme_fc_lport *lport;
3970 	struct nvme_fc_rport *rport;
3971 	unsigned long flags;
3972 
3973 	spin_lock_irqsave(&nvme_fc_lock, flags);
3974 	list_for_each_entry(lport, &nvme_fc_lport_list, port_list)
3975 		list_for_each_entry(rport, &lport->endp_list, endp_list)
3976 			nvme_fc_delete_controllers(rport);
3977 	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3978 	flush_workqueue(nvme_delete_wq);
3979 
3980 	nvmf_unregister_transport(&nvme_fc_transport);
3981 
3982 	device_destroy(&fc_class, MKDEV(0, 0));
3983 	class_unregister(&fc_class);
3984 }
3985 
3986 module_init(nvme_fc_init_module);
3987 module_exit(nvme_fc_exit_module);
3988 
3989 MODULE_DESCRIPTION("NVMe host FC transport driver");
3990 MODULE_LICENSE("GPL v2");
3991