xref: /linux/drivers/nvme/target/fc.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
4  */
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/blk-mq.h>
9 #include <linux/parser.h>
10 #include <linux/random.h>
11 #include <uapi/scsi/fc/fc_fs.h>
12 #include <uapi/scsi/fc/fc_els.h>
13 
14 #include "nvmet.h"
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "../host/fc.h"
18 
19 
20 /* *************************** Data Structures/Defines ****************** */
21 
22 
23 #define NVMET_LS_CTX_COUNT		256
24 
25 struct nvmet_fc_tgtport;
26 struct nvmet_fc_tgt_assoc;
27 
28 struct nvmet_fc_ls_iod {		/* for an LS RQST RCV */
29 	struct nvmefc_ls_rsp		*lsrsp;
30 	struct nvmefc_tgt_fcp_req	*fcpreq;	/* only if RS */
31 
32 	struct list_head		ls_rcv_list; /* tgtport->ls_rcv_list */
33 
34 	struct nvmet_fc_tgtport		*tgtport;
35 	struct nvmet_fc_tgt_assoc	*assoc;
36 	void				*hosthandle;
37 
38 	union nvmefc_ls_requests	*rqstbuf;
39 	union nvmefc_ls_responses	*rspbuf;
40 	u16				rqstdatalen;
41 	dma_addr_t			rspdma;
42 
43 	struct scatterlist		sg[2];
44 
45 	struct work_struct		work;
46 } __aligned(sizeof(unsigned long long));
47 
48 struct nvmet_fc_ls_req_op {		/* for an LS RQST XMT */
49 	struct nvmefc_ls_req		ls_req;
50 
51 	struct nvmet_fc_tgtport		*tgtport;
52 	void				*hosthandle;
53 
54 	int				ls_error;
55 	struct list_head		lsreq_list; /* tgtport->ls_req_list */
56 	bool				req_queued;
57 };
58 
59 
60 /* desired maximum for a single sequence - if sg list allows it */
61 #define NVMET_FC_MAX_SEQ_LENGTH		(256 * 1024)
62 
63 enum nvmet_fcp_datadir {
64 	NVMET_FCP_NODATA,
65 	NVMET_FCP_WRITE,
66 	NVMET_FCP_READ,
67 	NVMET_FCP_ABORTED,
68 };
69 
70 struct nvmet_fc_fcp_iod {
71 	struct nvmefc_tgt_fcp_req	*fcpreq;
72 
73 	struct nvme_fc_cmd_iu		cmdiubuf;
74 	struct nvme_fc_ersp_iu		rspiubuf;
75 	dma_addr_t			rspdma;
76 	struct scatterlist		*next_sg;
77 	struct scatterlist		*data_sg;
78 	int				data_sg_cnt;
79 	u32				offset;
80 	enum nvmet_fcp_datadir		io_dir;
81 	bool				active;
82 	bool				abort;
83 	bool				aborted;
84 	bool				writedataactive;
85 	spinlock_t			flock;
86 
87 	struct nvmet_req		req;
88 	struct work_struct		defer_work;
89 
90 	struct nvmet_fc_tgtport		*tgtport;
91 	struct nvmet_fc_tgt_queue	*queue;
92 
93 	struct list_head		fcp_list;	/* tgtport->fcp_list */
94 };
95 
96 struct nvmet_fc_tgtport {
97 	struct nvmet_fc_target_port	fc_target_port;
98 
99 	struct list_head		tgt_list; /* nvmet_fc_target_list */
100 	struct device			*dev;	/* dev for dma mapping */
101 	struct nvmet_fc_target_template	*ops;
102 
103 	struct nvmet_fc_ls_iod		*iod;
104 	spinlock_t			lock;
105 	struct list_head		ls_rcv_list;
106 	struct list_head		ls_req_list;
107 	struct list_head		ls_busylist;
108 	struct list_head		assoc_list;
109 	struct list_head		host_list;
110 	struct ida			assoc_cnt;
111 	struct nvmet_fc_port_entry	*pe;
112 	struct kref			ref;
113 	u32				max_sg_cnt;
114 
115 	struct work_struct		put_work;
116 };
117 
118 struct nvmet_fc_port_entry {
119 	struct nvmet_fc_tgtport		*tgtport;
120 	struct nvmet_port		*port;
121 	u64				node_name;
122 	u64				port_name;
123 	struct list_head		pe_list;
124 };
125 
126 struct nvmet_fc_defer_fcp_req {
127 	struct list_head		req_list;
128 	struct nvmefc_tgt_fcp_req	*fcp_req;
129 };
130 
131 struct nvmet_fc_tgt_queue {
132 	bool				ninetypercent;
133 	u16				qid;
134 	u16				sqsize;
135 	u16				ersp_ratio;
136 	__le16				sqhd;
137 	atomic_t			connected;
138 	atomic_t			sqtail;
139 	atomic_t			zrspcnt;
140 	atomic_t			rsn;
141 	spinlock_t			qlock;
142 	struct nvmet_cq			nvme_cq;
143 	struct nvmet_sq			nvme_sq;
144 	struct nvmet_fc_tgt_assoc	*assoc;
145 	struct list_head		fod_list;
146 	struct list_head		pending_cmd_list;
147 	struct list_head		avail_defer_list;
148 	struct workqueue_struct		*work_q;
149 	struct kref			ref;
150 	/* array of fcp_iods */
151 	struct nvmet_fc_fcp_iod		fod[] /* __counted_by(sqsize) */;
152 } __aligned(sizeof(unsigned long long));
153 
154 struct nvmet_fc_hostport {
155 	struct nvmet_fc_tgtport		*tgtport;
156 	void				*hosthandle;
157 	struct list_head		host_list;
158 	struct kref			ref;
159 	u8				invalid;
160 };
161 
162 struct nvmet_fc_tgt_assoc {
163 	u64				association_id;
164 	u32				a_id;
165 	atomic_t			terminating;
166 	struct nvmet_fc_tgtport		*tgtport;
167 	struct nvmet_fc_hostport	*hostport;
168 	struct nvmet_fc_ls_iod		*rcv_disconn;
169 	struct list_head		a_list;
170 	struct nvmet_fc_tgt_queue 	*queues[NVMET_NR_QUEUES + 1];
171 	struct kref			ref;
172 	struct work_struct		del_work;
173 };
174 
175 
176 static inline int
nvmet_fc_iodnum(struct nvmet_fc_ls_iod * iodptr)177 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
178 {
179 	return (iodptr - iodptr->tgtport->iod);
180 }
181 
182 static inline int
nvmet_fc_fodnum(struct nvmet_fc_fcp_iod * fodptr)183 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
184 {
185 	return (fodptr - fodptr->queue->fod);
186 }
187 
188 
189 /*
190  * Association and Connection IDs:
191  *
192  * Association ID will have random number in upper 6 bytes and zero
193  *   in lower 2 bytes
194  *
195  * Connection IDs will be Association ID with QID or'd in lower 2 bytes
196  *
197  * note: Association ID = Connection ID for queue 0
198  */
199 #define BYTES_FOR_QID			sizeof(u16)
200 #define BYTES_FOR_QID_SHIFT		(BYTES_FOR_QID * 8)
201 #define NVMET_FC_QUEUEID_MASK		((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
202 
203 static inline u64
nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc * assoc,u16 qid)204 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
205 {
206 	return (assoc->association_id | qid);
207 }
208 
209 static inline u64
nvmet_fc_getassociationid(u64 connectionid)210 nvmet_fc_getassociationid(u64 connectionid)
211 {
212 	return connectionid & ~NVMET_FC_QUEUEID_MASK;
213 }
214 
215 static inline u16
nvmet_fc_getqueueid(u64 connectionid)216 nvmet_fc_getqueueid(u64 connectionid)
217 {
218 	return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
219 }
220 
221 static inline struct nvmet_fc_tgtport *
targetport_to_tgtport(struct nvmet_fc_target_port * targetport)222 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
223 {
224 	return container_of(targetport, struct nvmet_fc_tgtport,
225 				 fc_target_port);
226 }
227 
228 static inline struct nvmet_fc_fcp_iod *
nvmet_req_to_fod(struct nvmet_req * nvme_req)229 nvmet_req_to_fod(struct nvmet_req *nvme_req)
230 {
231 	return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
232 }
233 
234 
235 /* *************************** Globals **************************** */
236 
237 
238 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
239 
240 static LIST_HEAD(nvmet_fc_target_list);
241 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
242 static LIST_HEAD(nvmet_fc_portentry_list);
243 
244 
245 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
246 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
247 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
248 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
249 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
250 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
251 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
nvmet_fc_put_tgtport_work(struct work_struct * work)252 static void nvmet_fc_put_tgtport_work(struct work_struct *work)
253 {
254 	struct nvmet_fc_tgtport *tgtport =
255 		container_of(work, struct nvmet_fc_tgtport, put_work);
256 
257 	nvmet_fc_tgtport_put(tgtport);
258 }
259 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
260 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
261 					struct nvmet_fc_fcp_iod *fod);
262 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
263 static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
264 				struct nvmet_fc_ls_iod *iod);
265 
266 
267 /* *********************** FC-NVME DMA Handling **************************** */
268 
269 /*
270  * The fcloop device passes in a NULL device pointer. Real LLD's will
271  * pass in a valid device pointer. If NULL is passed to the dma mapping
272  * routines, depending on the platform, it may or may not succeed, and
273  * may crash.
274  *
275  * As such:
276  * Wrapper all the dma routines and check the dev pointer.
277  *
278  * If simple mappings (return just a dma address, we'll noop them,
279  * returning a dma address of 0.
280  *
281  * On more complex mappings (dma_map_sg), a pseudo routine fills
282  * in the scatter list, setting all dma addresses to 0.
283  */
284 
285 static inline dma_addr_t
fc_dma_map_single(struct device * dev,void * ptr,size_t size,enum dma_data_direction dir)286 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
287 		enum dma_data_direction dir)
288 {
289 	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
290 }
291 
292 static inline int
fc_dma_mapping_error(struct device * dev,dma_addr_t dma_addr)293 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
294 {
295 	return dev ? dma_mapping_error(dev, dma_addr) : 0;
296 }
297 
298 static inline void
fc_dma_unmap_single(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)299 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
300 	enum dma_data_direction dir)
301 {
302 	if (dev)
303 		dma_unmap_single(dev, addr, size, dir);
304 }
305 
306 static inline void
fc_dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)307 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
308 		enum dma_data_direction dir)
309 {
310 	if (dev)
311 		dma_sync_single_for_cpu(dev, addr, size, dir);
312 }
313 
314 static inline void
fc_dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)315 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
316 		enum dma_data_direction dir)
317 {
318 	if (dev)
319 		dma_sync_single_for_device(dev, addr, size, dir);
320 }
321 
322 /* pseudo dma_map_sg call */
323 static int
fc_map_sg(struct scatterlist * sg,int nents)324 fc_map_sg(struct scatterlist *sg, int nents)
325 {
326 	struct scatterlist *s;
327 	int i;
328 
329 	WARN_ON(nents == 0 || sg[0].length == 0);
330 
331 	for_each_sg(sg, s, nents, i) {
332 		s->dma_address = 0L;
333 #ifdef CONFIG_NEED_SG_DMA_LENGTH
334 		s->dma_length = s->length;
335 #endif
336 	}
337 	return nents;
338 }
339 
340 static inline int
fc_dma_map_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)341 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
342 		enum dma_data_direction dir)
343 {
344 	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
345 }
346 
347 static inline void
fc_dma_unmap_sg(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir)348 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
349 		enum dma_data_direction dir)
350 {
351 	if (dev)
352 		dma_unmap_sg(dev, sg, nents, dir);
353 }
354 
355 
356 /* ********************** FC-NVME LS XMT Handling ************************* */
357 
358 
359 static void
__nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op * lsop)360 __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)
361 {
362 	struct nvmet_fc_tgtport *tgtport = lsop->tgtport;
363 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
364 	unsigned long flags;
365 
366 	spin_lock_irqsave(&tgtport->lock, flags);
367 
368 	if (!lsop->req_queued) {
369 		spin_unlock_irqrestore(&tgtport->lock, flags);
370 		goto out_putwork;
371 	}
372 
373 	list_del(&lsop->lsreq_list);
374 
375 	lsop->req_queued = false;
376 
377 	spin_unlock_irqrestore(&tgtport->lock, flags);
378 
379 	fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
380 				  (lsreq->rqstlen + lsreq->rsplen),
381 				  DMA_BIDIRECTIONAL);
382 
383 out_putwork:
384 	queue_work(nvmet_wq, &tgtport->put_work);
385 }
386 
387 static int
__nvmet_fc_send_ls_req(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_req_op * lsop,void (* done)(struct nvmefc_ls_req * req,int status))388 __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,
389 		struct nvmet_fc_ls_req_op *lsop,
390 		void (*done)(struct nvmefc_ls_req *req, int status))
391 {
392 	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
393 	unsigned long flags;
394 	int ret = 0;
395 
396 	if (!tgtport->ops->ls_req)
397 		return -EOPNOTSUPP;
398 
399 	if (!nvmet_fc_tgtport_get(tgtport))
400 		return -ESHUTDOWN;
401 
402 	lsreq->done = done;
403 	lsop->req_queued = false;
404 	INIT_LIST_HEAD(&lsop->lsreq_list);
405 
406 	lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr,
407 				  lsreq->rqstlen + lsreq->rsplen,
408 				  DMA_BIDIRECTIONAL);
409 	if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) {
410 		ret = -EFAULT;
411 		goto out_puttgtport;
412 	}
413 	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
414 
415 	spin_lock_irqsave(&tgtport->lock, flags);
416 
417 	list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list);
418 
419 	lsop->req_queued = true;
420 
421 	spin_unlock_irqrestore(&tgtport->lock, flags);
422 
423 	ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle,
424 				   lsreq);
425 	if (ret)
426 		goto out_unlink;
427 
428 	return 0;
429 
430 out_unlink:
431 	lsop->ls_error = ret;
432 	spin_lock_irqsave(&tgtport->lock, flags);
433 	lsop->req_queued = false;
434 	list_del(&lsop->lsreq_list);
435 	spin_unlock_irqrestore(&tgtport->lock, flags);
436 	fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma,
437 				  (lsreq->rqstlen + lsreq->rsplen),
438 				  DMA_BIDIRECTIONAL);
439 out_puttgtport:
440 	nvmet_fc_tgtport_put(tgtport);
441 
442 	return ret;
443 }
444 
445 static int
nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_req_op * lsop,void (* done)(struct nvmefc_ls_req * req,int status))446 nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport,
447 		struct nvmet_fc_ls_req_op *lsop,
448 		void (*done)(struct nvmefc_ls_req *req, int status))
449 {
450 	/* don't wait for completion */
451 
452 	return __nvmet_fc_send_ls_req(tgtport, lsop, done);
453 }
454 
455 static void
nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req * lsreq,int status)456 nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
457 {
458 	struct nvmet_fc_ls_req_op *lsop =
459 		container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req);
460 
461 	__nvmet_fc_finish_ls_req(lsop);
462 
463 	/* fc-nvme target doesn't care about success or failure of cmd */
464 
465 	kfree(lsop);
466 }
467 
468 /*
469  * This routine sends a FC-NVME LS to disconnect (aka terminate)
470  * the FC-NVME Association.  Terminating the association also
471  * terminates the FC-NVME connections (per queue, both admin and io
472  * queues) that are part of the association. E.g. things are torn
473  * down, and the related FC-NVME Association ID and Connection IDs
474  * become invalid.
475  *
476  * The behavior of the fc-nvme target is such that it's
477  * understanding of the association and connections will implicitly
478  * be torn down. The action is implicit as it may be due to a loss of
479  * connectivity with the fc-nvme host, so the target may never get a
480  * response even if it tried.  As such, the action of this routine
481  * is to asynchronously send the LS, ignore any results of the LS, and
482  * continue on with terminating the association. If the fc-nvme host
483  * is present and receives the LS, it too can tear down.
484  */
485 static void
nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc * assoc)486 nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)
487 {
488 	struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
489 	struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
490 	struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
491 	struct nvmet_fc_ls_req_op *lsop;
492 	struct nvmefc_ls_req *lsreq;
493 	int ret;
494 
495 	/*
496 	 * If ls_req is NULL or no hosthandle, it's an older lldd and no
497 	 * message is normal. Otherwise, send unless the hostport has
498 	 * already been invalidated by the lldd.
499 	 */
500 	if (!tgtport->ops->ls_req || assoc->hostport->invalid)
501 		return;
502 
503 	lsop = kzalloc((sizeof(*lsop) +
504 			sizeof(*discon_rqst) + sizeof(*discon_acc) +
505 			tgtport->ops->lsrqst_priv_sz), GFP_KERNEL);
506 	if (!lsop) {
507 		dev_info(tgtport->dev,
508 			"{%d:%d} send Disconnect Association failed: ENOMEM\n",
509 			tgtport->fc_target_port.port_num, assoc->a_id);
510 		return;
511 	}
512 
513 	discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
514 	discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
515 	lsreq = &lsop->ls_req;
516 	if (tgtport->ops->lsrqst_priv_sz)
517 		lsreq->private = (void *)&discon_acc[1];
518 	else
519 		lsreq->private = NULL;
520 
521 	lsop->tgtport = tgtport;
522 	lsop->hosthandle = assoc->hostport->hosthandle;
523 
524 	nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
525 				assoc->association_id);
526 
527 	ret = nvmet_fc_send_ls_req_async(tgtport, lsop,
528 				nvmet_fc_disconnect_assoc_done);
529 	if (ret) {
530 		dev_info(tgtport->dev,
531 			"{%d:%d} XMT Disconnect Association failed: %d\n",
532 			tgtport->fc_target_port.port_num, assoc->a_id, ret);
533 		kfree(lsop);
534 	}
535 }
536 
537 
538 /* *********************** FC-NVME Port Management ************************ */
539 
540 
541 static int
nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport * tgtport)542 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
543 {
544 	struct nvmet_fc_ls_iod *iod;
545 	int i;
546 
547 	iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
548 			GFP_KERNEL);
549 	if (!iod)
550 		return -ENOMEM;
551 
552 	tgtport->iod = iod;
553 
554 	for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
555 		INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
556 		iod->tgtport = tgtport;
557 		list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
558 
559 		iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) +
560 				       sizeof(union nvmefc_ls_responses),
561 				       GFP_KERNEL);
562 		if (!iod->rqstbuf)
563 			goto out_fail;
564 
565 		iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1];
566 
567 		iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
568 						sizeof(*iod->rspbuf),
569 						DMA_TO_DEVICE);
570 		if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
571 			goto out_fail;
572 	}
573 
574 	return 0;
575 
576 out_fail:
577 	kfree(iod->rqstbuf);
578 	list_del(&iod->ls_rcv_list);
579 	for (iod--, i--; i >= 0; iod--, i--) {
580 		fc_dma_unmap_single(tgtport->dev, iod->rspdma,
581 				sizeof(*iod->rspbuf), DMA_TO_DEVICE);
582 		kfree(iod->rqstbuf);
583 		list_del(&iod->ls_rcv_list);
584 	}
585 
586 	kfree(iod);
587 
588 	return -EFAULT;
589 }
590 
591 static void
nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport * tgtport)592 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
593 {
594 	struct nvmet_fc_ls_iod *iod = tgtport->iod;
595 	int i;
596 
597 	for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
598 		fc_dma_unmap_single(tgtport->dev,
599 				iod->rspdma, sizeof(*iod->rspbuf),
600 				DMA_TO_DEVICE);
601 		kfree(iod->rqstbuf);
602 		list_del(&iod->ls_rcv_list);
603 	}
604 	kfree(tgtport->iod);
605 }
606 
607 static struct nvmet_fc_ls_iod *
nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport * tgtport)608 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
609 {
610 	struct nvmet_fc_ls_iod *iod;
611 	unsigned long flags;
612 
613 	spin_lock_irqsave(&tgtport->lock, flags);
614 	iod = list_first_entry_or_null(&tgtport->ls_rcv_list,
615 					struct nvmet_fc_ls_iod, ls_rcv_list);
616 	if (iod)
617 		list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist);
618 	spin_unlock_irqrestore(&tgtport->lock, flags);
619 	return iod;
620 }
621 
622 
623 static void
nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)624 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
625 			struct nvmet_fc_ls_iod *iod)
626 {
627 	unsigned long flags;
628 
629 	spin_lock_irqsave(&tgtport->lock, flags);
630 	list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list);
631 	spin_unlock_irqrestore(&tgtport->lock, flags);
632 }
633 
634 static void
nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_tgt_queue * queue)635 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
636 				struct nvmet_fc_tgt_queue *queue)
637 {
638 	struct nvmet_fc_fcp_iod *fod = queue->fod;
639 	int i;
640 
641 	for (i = 0; i < queue->sqsize; fod++, i++) {
642 		INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
643 		fod->tgtport = tgtport;
644 		fod->queue = queue;
645 		fod->active = false;
646 		fod->abort = false;
647 		fod->aborted = false;
648 		fod->fcpreq = NULL;
649 		list_add_tail(&fod->fcp_list, &queue->fod_list);
650 		spin_lock_init(&fod->flock);
651 
652 		fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
653 					sizeof(fod->rspiubuf), DMA_TO_DEVICE);
654 		if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
655 			list_del(&fod->fcp_list);
656 			for (fod--, i--; i >= 0; fod--, i--) {
657 				fc_dma_unmap_single(tgtport->dev, fod->rspdma,
658 						sizeof(fod->rspiubuf),
659 						DMA_TO_DEVICE);
660 				fod->rspdma = 0L;
661 				list_del(&fod->fcp_list);
662 			}
663 
664 			return;
665 		}
666 	}
667 }
668 
669 static void
nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_tgt_queue * queue)670 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
671 				struct nvmet_fc_tgt_queue *queue)
672 {
673 	struct nvmet_fc_fcp_iod *fod = queue->fod;
674 	int i;
675 
676 	for (i = 0; i < queue->sqsize; fod++, i++) {
677 		if (fod->rspdma)
678 			fc_dma_unmap_single(tgtport->dev, fod->rspdma,
679 				sizeof(fod->rspiubuf), DMA_TO_DEVICE);
680 	}
681 }
682 
683 static struct nvmet_fc_fcp_iod *
nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue * queue)684 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
685 {
686 	struct nvmet_fc_fcp_iod *fod;
687 
688 	lockdep_assert_held(&queue->qlock);
689 
690 	fod = list_first_entry_or_null(&queue->fod_list,
691 					struct nvmet_fc_fcp_iod, fcp_list);
692 	if (fod) {
693 		list_del(&fod->fcp_list);
694 		fod->active = true;
695 		/*
696 		 * no queue reference is taken, as it was taken by the
697 		 * queue lookup just prior to the allocation. The iod
698 		 * will "inherit" that reference.
699 		 */
700 	}
701 	return fod;
702 }
703 
704 
705 static void
nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_tgt_queue * queue,struct nvmefc_tgt_fcp_req * fcpreq)706 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
707 		       struct nvmet_fc_tgt_queue *queue,
708 		       struct nvmefc_tgt_fcp_req *fcpreq)
709 {
710 	struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
711 
712 	/*
713 	 * put all admin cmds on hw queue id 0. All io commands go to
714 	 * the respective hw queue based on a modulo basis
715 	 */
716 	fcpreq->hwqid = queue->qid ?
717 			((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
718 
719 	nvmet_fc_handle_fcp_rqst(tgtport, fod);
720 }
721 
722 static void
nvmet_fc_fcp_rqst_op_defer_work(struct work_struct * work)723 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
724 {
725 	struct nvmet_fc_fcp_iod *fod =
726 		container_of(work, struct nvmet_fc_fcp_iod, defer_work);
727 
728 	/* Submit deferred IO for processing */
729 	nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
730 
731 }
732 
733 static void
nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue * queue,struct nvmet_fc_fcp_iod * fod)734 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
735 			struct nvmet_fc_fcp_iod *fod)
736 {
737 	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
738 	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
739 	struct nvmet_fc_defer_fcp_req *deferfcp;
740 	unsigned long flags;
741 
742 	fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
743 				sizeof(fod->rspiubuf), DMA_TO_DEVICE);
744 
745 	fcpreq->nvmet_fc_private = NULL;
746 
747 	fod->active = false;
748 	fod->abort = false;
749 	fod->aborted = false;
750 	fod->writedataactive = false;
751 	fod->fcpreq = NULL;
752 
753 	tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
754 
755 	/* release the queue lookup reference on the completed IO */
756 	nvmet_fc_tgt_q_put(queue);
757 
758 	spin_lock_irqsave(&queue->qlock, flags);
759 	deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
760 				struct nvmet_fc_defer_fcp_req, req_list);
761 	if (!deferfcp) {
762 		list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
763 		spin_unlock_irqrestore(&queue->qlock, flags);
764 		return;
765 	}
766 
767 	/* Re-use the fod for the next pending cmd that was deferred */
768 	list_del(&deferfcp->req_list);
769 
770 	fcpreq = deferfcp->fcp_req;
771 
772 	/* deferfcp can be reused for another IO at a later date */
773 	list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
774 
775 	spin_unlock_irqrestore(&queue->qlock, flags);
776 
777 	/* Save NVME CMD IO in fod */
778 	memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
779 
780 	/* Setup new fcpreq to be processed */
781 	fcpreq->rspaddr = NULL;
782 	fcpreq->rsplen  = 0;
783 	fcpreq->nvmet_fc_private = fod;
784 	fod->fcpreq = fcpreq;
785 	fod->active = true;
786 
787 	/* inform LLDD IO is now being processed */
788 	tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
789 
790 	/*
791 	 * Leave the queue lookup get reference taken when
792 	 * fod was originally allocated.
793 	 */
794 
795 	queue_work(queue->work_q, &fod->defer_work);
796 }
797 
798 static struct nvmet_fc_tgt_queue *
nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc * assoc,u16 qid,u16 sqsize)799 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
800 			u16 qid, u16 sqsize)
801 {
802 	struct nvmet_fc_tgt_queue *queue;
803 	int ret;
804 
805 	if (qid > NVMET_NR_QUEUES)
806 		return NULL;
807 
808 	queue = kzalloc(struct_size(queue, fod, sqsize), GFP_KERNEL);
809 	if (!queue)
810 		return NULL;
811 
812 	queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
813 				assoc->tgtport->fc_target_port.port_num,
814 				assoc->a_id, qid);
815 	if (!queue->work_q)
816 		goto out_free_queue;
817 
818 	queue->qid = qid;
819 	queue->sqsize = sqsize;
820 	queue->assoc = assoc;
821 	INIT_LIST_HEAD(&queue->fod_list);
822 	INIT_LIST_HEAD(&queue->avail_defer_list);
823 	INIT_LIST_HEAD(&queue->pending_cmd_list);
824 	atomic_set(&queue->connected, 0);
825 	atomic_set(&queue->sqtail, 0);
826 	atomic_set(&queue->rsn, 1);
827 	atomic_set(&queue->zrspcnt, 0);
828 	spin_lock_init(&queue->qlock);
829 	kref_init(&queue->ref);
830 
831 	nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
832 
833 	ret = nvmet_sq_init(&queue->nvme_sq);
834 	if (ret)
835 		goto out_fail_iodlist;
836 
837 	WARN_ON(assoc->queues[qid]);
838 	assoc->queues[qid] = queue;
839 
840 	return queue;
841 
842 out_fail_iodlist:
843 	nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
844 	destroy_workqueue(queue->work_q);
845 out_free_queue:
846 	kfree(queue);
847 	return NULL;
848 }
849 
850 
851 static void
nvmet_fc_tgt_queue_free(struct kref * ref)852 nvmet_fc_tgt_queue_free(struct kref *ref)
853 {
854 	struct nvmet_fc_tgt_queue *queue =
855 		container_of(ref, struct nvmet_fc_tgt_queue, ref);
856 
857 	nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
858 
859 	destroy_workqueue(queue->work_q);
860 
861 	kfree(queue);
862 }
863 
864 static void
nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue * queue)865 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
866 {
867 	kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
868 }
869 
870 static int
nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue * queue)871 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
872 {
873 	return kref_get_unless_zero(&queue->ref);
874 }
875 
876 
877 static void
nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue * queue)878 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
879 {
880 	struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
881 	struct nvmet_fc_fcp_iod *fod = queue->fod;
882 	struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
883 	unsigned long flags;
884 	int i;
885 	bool disconnect;
886 
887 	disconnect = atomic_xchg(&queue->connected, 0);
888 
889 	/* if not connected, nothing to do */
890 	if (!disconnect)
891 		return;
892 
893 	spin_lock_irqsave(&queue->qlock, flags);
894 	/* abort outstanding io's */
895 	for (i = 0; i < queue->sqsize; fod++, i++) {
896 		if (fod->active) {
897 			spin_lock(&fod->flock);
898 			fod->abort = true;
899 			/*
900 			 * only call lldd abort routine if waiting for
901 			 * writedata. other outstanding ops should finish
902 			 * on their own.
903 			 */
904 			if (fod->writedataactive) {
905 				fod->aborted = true;
906 				spin_unlock(&fod->flock);
907 				tgtport->ops->fcp_abort(
908 					&tgtport->fc_target_port, fod->fcpreq);
909 			} else
910 				spin_unlock(&fod->flock);
911 		}
912 	}
913 
914 	/* Cleanup defer'ed IOs in queue */
915 	list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
916 				req_list) {
917 		list_del(&deferfcp->req_list);
918 		kfree(deferfcp);
919 	}
920 
921 	for (;;) {
922 		deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
923 				struct nvmet_fc_defer_fcp_req, req_list);
924 		if (!deferfcp)
925 			break;
926 
927 		list_del(&deferfcp->req_list);
928 		spin_unlock_irqrestore(&queue->qlock, flags);
929 
930 		tgtport->ops->defer_rcv(&tgtport->fc_target_port,
931 				deferfcp->fcp_req);
932 
933 		tgtport->ops->fcp_abort(&tgtport->fc_target_port,
934 				deferfcp->fcp_req);
935 
936 		tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
937 				deferfcp->fcp_req);
938 
939 		/* release the queue lookup reference */
940 		nvmet_fc_tgt_q_put(queue);
941 
942 		kfree(deferfcp);
943 
944 		spin_lock_irqsave(&queue->qlock, flags);
945 	}
946 	spin_unlock_irqrestore(&queue->qlock, flags);
947 
948 	flush_workqueue(queue->work_q);
949 
950 	nvmet_sq_destroy(&queue->nvme_sq);
951 
952 	nvmet_fc_tgt_q_put(queue);
953 }
954 
955 static struct nvmet_fc_tgt_queue *
nvmet_fc_find_target_queue(struct nvmet_fc_tgtport * tgtport,u64 connection_id)956 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
957 				u64 connection_id)
958 {
959 	struct nvmet_fc_tgt_assoc *assoc;
960 	struct nvmet_fc_tgt_queue *queue;
961 	u64 association_id = nvmet_fc_getassociationid(connection_id);
962 	u16 qid = nvmet_fc_getqueueid(connection_id);
963 
964 	if (qid > NVMET_NR_QUEUES)
965 		return NULL;
966 
967 	rcu_read_lock();
968 	list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
969 		if (association_id == assoc->association_id) {
970 			queue = assoc->queues[qid];
971 			if (queue &&
972 			    (!atomic_read(&queue->connected) ||
973 			     !nvmet_fc_tgt_q_get(queue)))
974 				queue = NULL;
975 			rcu_read_unlock();
976 			return queue;
977 		}
978 	}
979 	rcu_read_unlock();
980 	return NULL;
981 }
982 
983 static void
nvmet_fc_hostport_free(struct kref * ref)984 nvmet_fc_hostport_free(struct kref *ref)
985 {
986 	struct nvmet_fc_hostport *hostport =
987 		container_of(ref, struct nvmet_fc_hostport, ref);
988 	struct nvmet_fc_tgtport *tgtport = hostport->tgtport;
989 	unsigned long flags;
990 
991 	spin_lock_irqsave(&tgtport->lock, flags);
992 	list_del(&hostport->host_list);
993 	spin_unlock_irqrestore(&tgtport->lock, flags);
994 	if (tgtport->ops->host_release && hostport->invalid)
995 		tgtport->ops->host_release(hostport->hosthandle);
996 	kfree(hostport);
997 	nvmet_fc_tgtport_put(tgtport);
998 }
999 
1000 static void
nvmet_fc_hostport_put(struct nvmet_fc_hostport * hostport)1001 nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport)
1002 {
1003 	kref_put(&hostport->ref, nvmet_fc_hostport_free);
1004 }
1005 
1006 static int
nvmet_fc_hostport_get(struct nvmet_fc_hostport * hostport)1007 nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport)
1008 {
1009 	return kref_get_unless_zero(&hostport->ref);
1010 }
1011 
1012 static void
nvmet_fc_free_hostport(struct nvmet_fc_hostport * hostport)1013 nvmet_fc_free_hostport(struct nvmet_fc_hostport *hostport)
1014 {
1015 	/* if LLDD not implemented, leave as NULL */
1016 	if (!hostport || !hostport->hosthandle)
1017 		return;
1018 
1019 	nvmet_fc_hostport_put(hostport);
1020 }
1021 
1022 static struct nvmet_fc_hostport *
nvmet_fc_match_hostport(struct nvmet_fc_tgtport * tgtport,void * hosthandle)1023 nvmet_fc_match_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1024 {
1025 	struct nvmet_fc_hostport *host;
1026 
1027 	lockdep_assert_held(&tgtport->lock);
1028 
1029 	list_for_each_entry(host, &tgtport->host_list, host_list) {
1030 		if (host->hosthandle == hosthandle && !host->invalid) {
1031 			if (nvmet_fc_hostport_get(host))
1032 				return host;
1033 		}
1034 	}
1035 
1036 	return NULL;
1037 }
1038 
1039 static struct nvmet_fc_hostport *
nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport * tgtport,void * hosthandle)1040 nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1041 {
1042 	struct nvmet_fc_hostport *newhost, *match = NULL;
1043 	unsigned long flags;
1044 
1045 	/* if LLDD not implemented, leave as NULL */
1046 	if (!hosthandle)
1047 		return NULL;
1048 
1049 	/*
1050 	 * take reference for what will be the newly allocated hostport if
1051 	 * we end up using a new allocation
1052 	 */
1053 	if (!nvmet_fc_tgtport_get(tgtport))
1054 		return ERR_PTR(-EINVAL);
1055 
1056 	spin_lock_irqsave(&tgtport->lock, flags);
1057 	match = nvmet_fc_match_hostport(tgtport, hosthandle);
1058 	spin_unlock_irqrestore(&tgtport->lock, flags);
1059 
1060 	if (match) {
1061 		/* no new allocation - release reference */
1062 		nvmet_fc_tgtport_put(tgtport);
1063 		return match;
1064 	}
1065 
1066 	newhost = kzalloc(sizeof(*newhost), GFP_KERNEL);
1067 	if (!newhost) {
1068 		/* no new allocation - release reference */
1069 		nvmet_fc_tgtport_put(tgtport);
1070 		return ERR_PTR(-ENOMEM);
1071 	}
1072 
1073 	spin_lock_irqsave(&tgtport->lock, flags);
1074 	match = nvmet_fc_match_hostport(tgtport, hosthandle);
1075 	if (match) {
1076 		/* new allocation not needed */
1077 		kfree(newhost);
1078 		newhost = match;
1079 	} else {
1080 		newhost->tgtport = tgtport;
1081 		newhost->hosthandle = hosthandle;
1082 		INIT_LIST_HEAD(&newhost->host_list);
1083 		kref_init(&newhost->ref);
1084 
1085 		list_add_tail(&newhost->host_list, &tgtport->host_list);
1086 	}
1087 	spin_unlock_irqrestore(&tgtport->lock, flags);
1088 
1089 	return newhost;
1090 }
1091 
1092 static void
nvmet_fc_delete_assoc(struct nvmet_fc_tgt_assoc * assoc)1093 nvmet_fc_delete_assoc(struct nvmet_fc_tgt_assoc *assoc)
1094 {
1095 	nvmet_fc_delete_target_assoc(assoc);
1096 	nvmet_fc_tgt_a_put(assoc);
1097 }
1098 
1099 static void
nvmet_fc_delete_assoc_work(struct work_struct * work)1100 nvmet_fc_delete_assoc_work(struct work_struct *work)
1101 {
1102 	struct nvmet_fc_tgt_assoc *assoc =
1103 		container_of(work, struct nvmet_fc_tgt_assoc, del_work);
1104 	struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1105 
1106 	nvmet_fc_delete_assoc(assoc);
1107 	nvmet_fc_tgtport_put(tgtport);
1108 }
1109 
1110 static void
nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc * assoc)1111 nvmet_fc_schedule_delete_assoc(struct nvmet_fc_tgt_assoc *assoc)
1112 {
1113 	nvmet_fc_tgtport_get(assoc->tgtport);
1114 	queue_work(nvmet_wq, &assoc->del_work);
1115 }
1116 
1117 static bool
nvmet_fc_assoc_exists(struct nvmet_fc_tgtport * tgtport,u64 association_id)1118 nvmet_fc_assoc_exists(struct nvmet_fc_tgtport *tgtport, u64 association_id)
1119 {
1120 	struct nvmet_fc_tgt_assoc *a;
1121 	bool found = false;
1122 
1123 	rcu_read_lock();
1124 	list_for_each_entry_rcu(a, &tgtport->assoc_list, a_list) {
1125 		if (association_id == a->association_id) {
1126 			found = true;
1127 			break;
1128 		}
1129 	}
1130 	rcu_read_unlock();
1131 
1132 	return found;
1133 }
1134 
1135 static struct nvmet_fc_tgt_assoc *
nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport * tgtport,void * hosthandle)1136 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle)
1137 {
1138 	struct nvmet_fc_tgt_assoc *assoc;
1139 	unsigned long flags;
1140 	bool done;
1141 	u64 ran;
1142 	int idx;
1143 
1144 	if (!tgtport->pe)
1145 		return NULL;
1146 
1147 	assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
1148 	if (!assoc)
1149 		return NULL;
1150 
1151 	idx = ida_alloc(&tgtport->assoc_cnt, GFP_KERNEL);
1152 	if (idx < 0)
1153 		goto out_free_assoc;
1154 
1155 	assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle);
1156 	if (IS_ERR(assoc->hostport))
1157 		goto out_ida;
1158 
1159 	assoc->tgtport = tgtport;
1160 	assoc->a_id = idx;
1161 	INIT_LIST_HEAD(&assoc->a_list);
1162 	kref_init(&assoc->ref);
1163 	INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc_work);
1164 	atomic_set(&assoc->terminating, 0);
1165 
1166 	done = false;
1167 	do {
1168 		get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
1169 		ran = ran << BYTES_FOR_QID_SHIFT;
1170 
1171 		spin_lock_irqsave(&tgtport->lock, flags);
1172 		if (!nvmet_fc_assoc_exists(tgtport, ran)) {
1173 			assoc->association_id = ran;
1174 			list_add_tail_rcu(&assoc->a_list, &tgtport->assoc_list);
1175 			done = true;
1176 		}
1177 		spin_unlock_irqrestore(&tgtport->lock, flags);
1178 	} while (!done);
1179 
1180 	return assoc;
1181 
1182 out_ida:
1183 	ida_free(&tgtport->assoc_cnt, idx);
1184 out_free_assoc:
1185 	kfree(assoc);
1186 	return NULL;
1187 }
1188 
1189 static void
nvmet_fc_target_assoc_free(struct kref * ref)1190 nvmet_fc_target_assoc_free(struct kref *ref)
1191 {
1192 	struct nvmet_fc_tgt_assoc *assoc =
1193 		container_of(ref, struct nvmet_fc_tgt_assoc, ref);
1194 	struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1195 	struct nvmet_fc_ls_iod	*oldls;
1196 	unsigned long flags;
1197 	int i;
1198 
1199 	for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1200 		if (assoc->queues[i])
1201 			nvmet_fc_delete_target_queue(assoc->queues[i]);
1202 	}
1203 
1204 	/* Send Disconnect now that all i/o has completed */
1205 	nvmet_fc_xmt_disconnect_assoc(assoc);
1206 
1207 	nvmet_fc_free_hostport(assoc->hostport);
1208 	spin_lock_irqsave(&tgtport->lock, flags);
1209 	oldls = assoc->rcv_disconn;
1210 	spin_unlock_irqrestore(&tgtport->lock, flags);
1211 	/* if pending Rcv Disconnect Association LS, send rsp now */
1212 	if (oldls)
1213 		nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1214 	ida_free(&tgtport->assoc_cnt, assoc->a_id);
1215 	dev_info(tgtport->dev,
1216 		"{%d:%d} Association freed\n",
1217 		tgtport->fc_target_port.port_num, assoc->a_id);
1218 	kfree(assoc);
1219 }
1220 
1221 static void
nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc * assoc)1222 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
1223 {
1224 	kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
1225 }
1226 
1227 static int
nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc * assoc)1228 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
1229 {
1230 	return kref_get_unless_zero(&assoc->ref);
1231 }
1232 
1233 static void
nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc * assoc)1234 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
1235 {
1236 	struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
1237 	unsigned long flags;
1238 	int i, terminating;
1239 
1240 	terminating = atomic_xchg(&assoc->terminating, 1);
1241 
1242 	/* if already terminating, do nothing */
1243 	if (terminating)
1244 		return;
1245 
1246 	spin_lock_irqsave(&tgtport->lock, flags);
1247 	list_del_rcu(&assoc->a_list);
1248 	spin_unlock_irqrestore(&tgtport->lock, flags);
1249 
1250 	synchronize_rcu();
1251 
1252 	/* ensure all in-flight I/Os have been processed */
1253 	for (i = NVMET_NR_QUEUES; i >= 0; i--) {
1254 		if (assoc->queues[i])
1255 			flush_workqueue(assoc->queues[i]->work_q);
1256 	}
1257 
1258 	dev_info(tgtport->dev,
1259 		"{%d:%d} Association deleted\n",
1260 		tgtport->fc_target_port.port_num, assoc->a_id);
1261 }
1262 
1263 static struct nvmet_fc_tgt_assoc *
nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport * tgtport,u64 association_id)1264 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
1265 				u64 association_id)
1266 {
1267 	struct nvmet_fc_tgt_assoc *assoc;
1268 	struct nvmet_fc_tgt_assoc *ret = NULL;
1269 
1270 	rcu_read_lock();
1271 	list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1272 		if (association_id == assoc->association_id) {
1273 			ret = assoc;
1274 			if (!nvmet_fc_tgt_a_get(assoc))
1275 				ret = NULL;
1276 			break;
1277 		}
1278 	}
1279 	rcu_read_unlock();
1280 
1281 	return ret;
1282 }
1283 
1284 static void
nvmet_fc_portentry_bind(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_port_entry * pe,struct nvmet_port * port)1285 nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport,
1286 			struct nvmet_fc_port_entry *pe,
1287 			struct nvmet_port *port)
1288 {
1289 	lockdep_assert_held(&nvmet_fc_tgtlock);
1290 
1291 	pe->tgtport = tgtport;
1292 	tgtport->pe = pe;
1293 
1294 	pe->port = port;
1295 	port->priv = pe;
1296 
1297 	pe->node_name = tgtport->fc_target_port.node_name;
1298 	pe->port_name = tgtport->fc_target_port.port_name;
1299 	INIT_LIST_HEAD(&pe->pe_list);
1300 
1301 	list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list);
1302 }
1303 
1304 static void
nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry * pe)1305 nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe)
1306 {
1307 	unsigned long flags;
1308 
1309 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1310 	if (pe->tgtport)
1311 		pe->tgtport->pe = NULL;
1312 	list_del(&pe->pe_list);
1313 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1314 }
1315 
1316 /*
1317  * called when a targetport deregisters. Breaks the relationship
1318  * with the nvmet port, but leaves the port_entry in place so that
1319  * re-registration can resume operation.
1320  */
1321 static void
nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport * tgtport)1322 nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport)
1323 {
1324 	struct nvmet_fc_port_entry *pe;
1325 	unsigned long flags;
1326 
1327 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1328 	pe = tgtport->pe;
1329 	if (pe)
1330 		pe->tgtport = NULL;
1331 	tgtport->pe = NULL;
1332 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1333 }
1334 
1335 /*
1336  * called when a new targetport is registered. Looks in the
1337  * existing nvmet port_entries to see if the nvmet layer is
1338  * configured for the targetport's wwn's. (the targetport existed,
1339  * nvmet configured, the lldd unregistered the tgtport, and is now
1340  * reregistering the same targetport).  If so, set the nvmet port
1341  * port entry on the targetport.
1342  */
1343 static void
nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport * tgtport)1344 nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport)
1345 {
1346 	struct nvmet_fc_port_entry *pe;
1347 	unsigned long flags;
1348 
1349 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1350 	list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) {
1351 		if (tgtport->fc_target_port.node_name == pe->node_name &&
1352 		    tgtport->fc_target_port.port_name == pe->port_name) {
1353 			WARN_ON(pe->tgtport);
1354 			tgtport->pe = pe;
1355 			pe->tgtport = tgtport;
1356 			break;
1357 		}
1358 	}
1359 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1360 }
1361 
1362 /**
1363  * nvmet_fc_register_targetport - transport entry point called by an
1364  *                              LLDD to register the existence of a local
1365  *                              NVME subystem FC port.
1366  * @pinfo:     pointer to information about the port to be registered
1367  * @template:  LLDD entrypoints and operational parameters for the port
1368  * @dev:       physical hardware device node port corresponds to. Will be
1369  *             used for DMA mappings
1370  * @portptr:   pointer to a local port pointer. Upon success, the routine
1371  *             will allocate a nvme_fc_local_port structure and place its
1372  *             address in the local port pointer. Upon failure, local port
1373  *             pointer will be set to NULL.
1374  *
1375  * Returns:
1376  * a completion status. Must be 0 upon success; a negative errno
1377  * (ex: -ENXIO) upon failure.
1378  */
1379 int
nvmet_fc_register_targetport(struct nvmet_fc_port_info * pinfo,struct nvmet_fc_target_template * template,struct device * dev,struct nvmet_fc_target_port ** portptr)1380 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
1381 			struct nvmet_fc_target_template *template,
1382 			struct device *dev,
1383 			struct nvmet_fc_target_port **portptr)
1384 {
1385 	struct nvmet_fc_tgtport *newrec;
1386 	unsigned long flags;
1387 	int ret, idx;
1388 
1389 	if (!template->xmt_ls_rsp || !template->fcp_op ||
1390 	    !template->fcp_abort ||
1391 	    !template->fcp_req_release || !template->targetport_delete ||
1392 	    !template->max_hw_queues || !template->max_sgl_segments ||
1393 	    !template->max_dif_sgl_segments || !template->dma_boundary) {
1394 		ret = -EINVAL;
1395 		goto out_regtgt_failed;
1396 	}
1397 
1398 	newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
1399 			 GFP_KERNEL);
1400 	if (!newrec) {
1401 		ret = -ENOMEM;
1402 		goto out_regtgt_failed;
1403 	}
1404 
1405 	idx = ida_alloc(&nvmet_fc_tgtport_cnt, GFP_KERNEL);
1406 	if (idx < 0) {
1407 		ret = -ENOSPC;
1408 		goto out_fail_kfree;
1409 	}
1410 
1411 	if (!get_device(dev) && dev) {
1412 		ret = -ENODEV;
1413 		goto out_ida_put;
1414 	}
1415 
1416 	newrec->fc_target_port.node_name = pinfo->node_name;
1417 	newrec->fc_target_port.port_name = pinfo->port_name;
1418 	if (template->target_priv_sz)
1419 		newrec->fc_target_port.private = &newrec[1];
1420 	else
1421 		newrec->fc_target_port.private = NULL;
1422 	newrec->fc_target_port.port_id = pinfo->port_id;
1423 	newrec->fc_target_port.port_num = idx;
1424 	INIT_LIST_HEAD(&newrec->tgt_list);
1425 	newrec->dev = dev;
1426 	newrec->ops = template;
1427 	spin_lock_init(&newrec->lock);
1428 	INIT_LIST_HEAD(&newrec->ls_rcv_list);
1429 	INIT_LIST_HEAD(&newrec->ls_req_list);
1430 	INIT_LIST_HEAD(&newrec->ls_busylist);
1431 	INIT_LIST_HEAD(&newrec->assoc_list);
1432 	INIT_LIST_HEAD(&newrec->host_list);
1433 	kref_init(&newrec->ref);
1434 	ida_init(&newrec->assoc_cnt);
1435 	newrec->max_sg_cnt = template->max_sgl_segments;
1436 	INIT_WORK(&newrec->put_work, nvmet_fc_put_tgtport_work);
1437 
1438 	ret = nvmet_fc_alloc_ls_iodlist(newrec);
1439 	if (ret) {
1440 		ret = -ENOMEM;
1441 		goto out_free_newrec;
1442 	}
1443 
1444 	nvmet_fc_portentry_rebind_tgt(newrec);
1445 
1446 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1447 	list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1448 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1449 
1450 	*portptr = &newrec->fc_target_port;
1451 	return 0;
1452 
1453 out_free_newrec:
1454 	put_device(dev);
1455 out_ida_put:
1456 	ida_free(&nvmet_fc_tgtport_cnt, idx);
1457 out_fail_kfree:
1458 	kfree(newrec);
1459 out_regtgt_failed:
1460 	*portptr = NULL;
1461 	return ret;
1462 }
1463 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1464 
1465 
1466 static void
nvmet_fc_free_tgtport(struct kref * ref)1467 nvmet_fc_free_tgtport(struct kref *ref)
1468 {
1469 	struct nvmet_fc_tgtport *tgtport =
1470 		container_of(ref, struct nvmet_fc_tgtport, ref);
1471 	struct device *dev = tgtport->dev;
1472 	unsigned long flags;
1473 
1474 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1475 	list_del(&tgtport->tgt_list);
1476 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1477 
1478 	nvmet_fc_free_ls_iodlist(tgtport);
1479 
1480 	/* let the LLDD know we've finished tearing it down */
1481 	tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1482 
1483 	ida_free(&nvmet_fc_tgtport_cnt,
1484 			tgtport->fc_target_port.port_num);
1485 
1486 	ida_destroy(&tgtport->assoc_cnt);
1487 
1488 	kfree(tgtport);
1489 
1490 	put_device(dev);
1491 }
1492 
1493 static void
nvmet_fc_tgtport_put(struct nvmet_fc_tgtport * tgtport)1494 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1495 {
1496 	kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1497 }
1498 
1499 static int
nvmet_fc_tgtport_get(struct nvmet_fc_tgtport * tgtport)1500 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1501 {
1502 	return kref_get_unless_zero(&tgtport->ref);
1503 }
1504 
1505 static void
__nvmet_fc_free_assocs(struct nvmet_fc_tgtport * tgtport)1506 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1507 {
1508 	struct nvmet_fc_tgt_assoc *assoc;
1509 
1510 	rcu_read_lock();
1511 	list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1512 		if (!nvmet_fc_tgt_a_get(assoc))
1513 			continue;
1514 		nvmet_fc_schedule_delete_assoc(assoc);
1515 		nvmet_fc_tgt_a_put(assoc);
1516 	}
1517 	rcu_read_unlock();
1518 }
1519 
1520 /**
1521  * nvmet_fc_invalidate_host - transport entry point called by an LLDD
1522  *                       to remove references to a hosthandle for LS's.
1523  *
1524  * The nvmet-fc layer ensures that any references to the hosthandle
1525  * on the targetport are forgotten (set to NULL).  The LLDD will
1526  * typically call this when a login with a remote host port has been
1527  * lost, thus LS's for the remote host port are no longer possible.
1528  *
1529  * If an LS request is outstanding to the targetport/hosthandle (or
1530  * issued concurrently with the call to invalidate the host), the
1531  * LLDD is responsible for terminating/aborting the LS and completing
1532  * the LS request. It is recommended that these terminations/aborts
1533  * occur after calling to invalidate the host handle to avoid additional
1534  * retries by the nvmet-fc transport. The nvmet-fc transport may
1535  * continue to reference host handle while it cleans up outstanding
1536  * NVME associations. The nvmet-fc transport will call the
1537  * ops->host_release() callback to notify the LLDD that all references
1538  * are complete and the related host handle can be recovered.
1539  * Note: if there are no references, the callback may be called before
1540  * the invalidate host call returns.
1541  *
1542  * @target_port: pointer to the (registered) target port that a prior
1543  *              LS was received on and which supplied the transport the
1544  *              hosthandle.
1545  * @hosthandle: the handle (pointer) that represents the host port
1546  *              that no longer has connectivity and that LS's should
1547  *              no longer be directed to.
1548  */
1549 void
nvmet_fc_invalidate_host(struct nvmet_fc_target_port * target_port,void * hosthandle)1550 nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port,
1551 			void *hosthandle)
1552 {
1553 	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1554 	struct nvmet_fc_tgt_assoc *assoc, *next;
1555 	unsigned long flags;
1556 	bool noassoc = true;
1557 
1558 	spin_lock_irqsave(&tgtport->lock, flags);
1559 	list_for_each_entry_safe(assoc, next,
1560 				&tgtport->assoc_list, a_list) {
1561 		if (assoc->hostport->hosthandle != hosthandle)
1562 			continue;
1563 		if (!nvmet_fc_tgt_a_get(assoc))
1564 			continue;
1565 		assoc->hostport->invalid = 1;
1566 		noassoc = false;
1567 		nvmet_fc_schedule_delete_assoc(assoc);
1568 		nvmet_fc_tgt_a_put(assoc);
1569 	}
1570 	spin_unlock_irqrestore(&tgtport->lock, flags);
1571 
1572 	/* if there's nothing to wait for - call the callback */
1573 	if (noassoc && tgtport->ops->host_release)
1574 		tgtport->ops->host_release(hosthandle);
1575 }
1576 EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host);
1577 
1578 /*
1579  * nvmet layer has called to terminate an association
1580  */
1581 static void
nvmet_fc_delete_ctrl(struct nvmet_ctrl * ctrl)1582 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1583 {
1584 	struct nvmet_fc_tgtport *tgtport, *next;
1585 	struct nvmet_fc_tgt_assoc *assoc;
1586 	struct nvmet_fc_tgt_queue *queue;
1587 	unsigned long flags;
1588 	bool found_ctrl = false;
1589 
1590 	/* this is a bit ugly, but don't want to make locks layered */
1591 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1592 	list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1593 			tgt_list) {
1594 		if (!nvmet_fc_tgtport_get(tgtport))
1595 			continue;
1596 		spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1597 
1598 		rcu_read_lock();
1599 		list_for_each_entry_rcu(assoc, &tgtport->assoc_list, a_list) {
1600 			queue = assoc->queues[0];
1601 			if (queue && queue->nvme_sq.ctrl == ctrl) {
1602 				if (nvmet_fc_tgt_a_get(assoc))
1603 					found_ctrl = true;
1604 				break;
1605 			}
1606 		}
1607 		rcu_read_unlock();
1608 
1609 		nvmet_fc_tgtport_put(tgtport);
1610 
1611 		if (found_ctrl) {
1612 			nvmet_fc_schedule_delete_assoc(assoc);
1613 			nvmet_fc_tgt_a_put(assoc);
1614 			return;
1615 		}
1616 
1617 		spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1618 	}
1619 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1620 }
1621 
1622 /**
1623  * nvmet_fc_unregister_targetport - transport entry point called by an
1624  *                              LLDD to deregister/remove a previously
1625  *                              registered a local NVME subsystem FC port.
1626  * @target_port: pointer to the (registered) target port that is to be
1627  *               deregistered.
1628  *
1629  * Returns:
1630  * a completion status. Must be 0 upon success; a negative errno
1631  * (ex: -ENXIO) upon failure.
1632  */
1633 int
nvmet_fc_unregister_targetport(struct nvmet_fc_target_port * target_port)1634 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1635 {
1636 	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1637 
1638 	nvmet_fc_portentry_unbind_tgt(tgtport);
1639 
1640 	/* terminate any outstanding associations */
1641 	__nvmet_fc_free_assocs(tgtport);
1642 
1643 	flush_workqueue(nvmet_wq);
1644 
1645 	/*
1646 	 * should terminate LS's as well. However, LS's will be generated
1647 	 * at the tail end of association termination, so they likely don't
1648 	 * exist yet. And even if they did, it's worthwhile to just let
1649 	 * them finish and targetport ref counting will clean things up.
1650 	 */
1651 
1652 	nvmet_fc_tgtport_put(tgtport);
1653 
1654 	return 0;
1655 }
1656 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1657 
1658 
1659 /* ********************** FC-NVME LS RCV Handling ************************* */
1660 
1661 
1662 static void
nvmet_fc_ls_create_association(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1663 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1664 			struct nvmet_fc_ls_iod *iod)
1665 {
1666 	struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc;
1667 	struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc;
1668 	struct nvmet_fc_tgt_queue *queue;
1669 	int ret = 0;
1670 
1671 	memset(acc, 0, sizeof(*acc));
1672 
1673 	/*
1674 	 * FC-NVME spec changes. There are initiators sending different
1675 	 * lengths as padding sizes for Create Association Cmd descriptor
1676 	 * was incorrect.
1677 	 * Accept anything of "minimum" length. Assume format per 1.15
1678 	 * spec (with HOSTID reduced to 16 bytes), ignore how long the
1679 	 * trailing pad length is.
1680 	 */
1681 	if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1682 		ret = VERR_CR_ASSOC_LEN;
1683 	else if (be32_to_cpu(rqst->desc_list_len) <
1684 			FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1685 		ret = VERR_CR_ASSOC_RQST_LEN;
1686 	else if (rqst->assoc_cmd.desc_tag !=
1687 			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1688 		ret = VERR_CR_ASSOC_CMD;
1689 	else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1690 			FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1691 		ret = VERR_CR_ASSOC_CMD_LEN;
1692 	else if (!rqst->assoc_cmd.ersp_ratio ||
1693 		 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1694 				be16_to_cpu(rqst->assoc_cmd.sqsize)))
1695 		ret = VERR_ERSP_RATIO;
1696 
1697 	else {
1698 		/* new association w/ admin queue */
1699 		iod->assoc = nvmet_fc_alloc_target_assoc(
1700 						tgtport, iod->hosthandle);
1701 		if (!iod->assoc)
1702 			ret = VERR_ASSOC_ALLOC_FAIL;
1703 		else {
1704 			queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1705 					be16_to_cpu(rqst->assoc_cmd.sqsize));
1706 			if (!queue) {
1707 				ret = VERR_QUEUE_ALLOC_FAIL;
1708 				nvmet_fc_tgt_a_put(iod->assoc);
1709 			}
1710 		}
1711 	}
1712 
1713 	if (ret) {
1714 		dev_err(tgtport->dev,
1715 			"Create Association LS failed: %s\n",
1716 			validation_errors[ret]);
1717 		iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1718 				sizeof(*acc), rqst->w0.ls_cmd,
1719 				FCNVME_RJT_RC_LOGIC,
1720 				FCNVME_RJT_EXP_NONE, 0);
1721 		return;
1722 	}
1723 
1724 	queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1725 	atomic_set(&queue->connected, 1);
1726 	queue->sqhd = 0;	/* best place to init value */
1727 
1728 	dev_info(tgtport->dev,
1729 		"{%d:%d} Association created\n",
1730 		tgtport->fc_target_port.port_num, iod->assoc->a_id);
1731 
1732 	/* format a response */
1733 
1734 	iod->lsrsp->rsplen = sizeof(*acc);
1735 
1736 	nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1737 			fcnvme_lsdesc_len(
1738 				sizeof(struct fcnvme_ls_cr_assoc_acc)),
1739 			FCNVME_LS_CREATE_ASSOCIATION);
1740 	acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1741 	acc->associd.desc_len =
1742 			fcnvme_lsdesc_len(
1743 				sizeof(struct fcnvme_lsdesc_assoc_id));
1744 	acc->associd.association_id =
1745 			cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1746 	acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1747 	acc->connectid.desc_len =
1748 			fcnvme_lsdesc_len(
1749 				sizeof(struct fcnvme_lsdesc_conn_id));
1750 	acc->connectid.connection_id = acc->associd.association_id;
1751 }
1752 
1753 static void
nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1754 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1755 			struct nvmet_fc_ls_iod *iod)
1756 {
1757 	struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn;
1758 	struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn;
1759 	struct nvmet_fc_tgt_queue *queue;
1760 	int ret = 0;
1761 
1762 	memset(acc, 0, sizeof(*acc));
1763 
1764 	if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1765 		ret = VERR_CR_CONN_LEN;
1766 	else if (rqst->desc_list_len !=
1767 			fcnvme_lsdesc_len(
1768 				sizeof(struct fcnvme_ls_cr_conn_rqst)))
1769 		ret = VERR_CR_CONN_RQST_LEN;
1770 	else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1771 		ret = VERR_ASSOC_ID;
1772 	else if (rqst->associd.desc_len !=
1773 			fcnvme_lsdesc_len(
1774 				sizeof(struct fcnvme_lsdesc_assoc_id)))
1775 		ret = VERR_ASSOC_ID_LEN;
1776 	else if (rqst->connect_cmd.desc_tag !=
1777 			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1778 		ret = VERR_CR_CONN_CMD;
1779 	else if (rqst->connect_cmd.desc_len !=
1780 			fcnvme_lsdesc_len(
1781 				sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1782 		ret = VERR_CR_CONN_CMD_LEN;
1783 	else if (!rqst->connect_cmd.ersp_ratio ||
1784 		 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1785 				be16_to_cpu(rqst->connect_cmd.sqsize)))
1786 		ret = VERR_ERSP_RATIO;
1787 
1788 	else {
1789 		/* new io queue */
1790 		iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1791 				be64_to_cpu(rqst->associd.association_id));
1792 		if (!iod->assoc)
1793 			ret = VERR_NO_ASSOC;
1794 		else {
1795 			queue = nvmet_fc_alloc_target_queue(iod->assoc,
1796 					be16_to_cpu(rqst->connect_cmd.qid),
1797 					be16_to_cpu(rqst->connect_cmd.sqsize));
1798 			if (!queue)
1799 				ret = VERR_QUEUE_ALLOC_FAIL;
1800 
1801 			/* release get taken in nvmet_fc_find_target_assoc */
1802 			nvmet_fc_tgt_a_put(iod->assoc);
1803 		}
1804 	}
1805 
1806 	if (ret) {
1807 		dev_err(tgtport->dev,
1808 			"Create Connection LS failed: %s\n",
1809 			validation_errors[ret]);
1810 		iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1811 				sizeof(*acc), rqst->w0.ls_cmd,
1812 				(ret == VERR_NO_ASSOC) ?
1813 					FCNVME_RJT_RC_INV_ASSOC :
1814 					FCNVME_RJT_RC_LOGIC,
1815 				FCNVME_RJT_EXP_NONE, 0);
1816 		return;
1817 	}
1818 
1819 	queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1820 	atomic_set(&queue->connected, 1);
1821 	queue->sqhd = 0;	/* best place to init value */
1822 
1823 	/* format a response */
1824 
1825 	iod->lsrsp->rsplen = sizeof(*acc);
1826 
1827 	nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1828 			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1829 			FCNVME_LS_CREATE_CONNECTION);
1830 	acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1831 	acc->connectid.desc_len =
1832 			fcnvme_lsdesc_len(
1833 				sizeof(struct fcnvme_lsdesc_conn_id));
1834 	acc->connectid.connection_id =
1835 			cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1836 				be16_to_cpu(rqst->connect_cmd.qid)));
1837 }
1838 
1839 /*
1840  * Returns true if the LS response is to be transmit
1841  * Returns false if the LS response is to be delayed
1842  */
1843 static int
nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1844 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1845 			struct nvmet_fc_ls_iod *iod)
1846 {
1847 	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1848 						&iod->rqstbuf->rq_dis_assoc;
1849 	struct fcnvme_ls_disconnect_assoc_acc *acc =
1850 						&iod->rspbuf->rsp_dis_assoc;
1851 	struct nvmet_fc_tgt_assoc *assoc = NULL;
1852 	struct nvmet_fc_ls_iod *oldls = NULL;
1853 	unsigned long flags;
1854 	int ret = 0;
1855 
1856 	memset(acc, 0, sizeof(*acc));
1857 
1858 	ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst);
1859 	if (!ret) {
1860 		/* match an active association - takes an assoc ref if !NULL */
1861 		assoc = nvmet_fc_find_target_assoc(tgtport,
1862 				be64_to_cpu(rqst->associd.association_id));
1863 		iod->assoc = assoc;
1864 		if (!assoc)
1865 			ret = VERR_NO_ASSOC;
1866 	}
1867 
1868 	if (ret || !assoc) {
1869 		dev_err(tgtport->dev,
1870 			"Disconnect LS failed: %s\n",
1871 			validation_errors[ret]);
1872 		iod->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1873 				sizeof(*acc), rqst->w0.ls_cmd,
1874 				(ret == VERR_NO_ASSOC) ?
1875 					FCNVME_RJT_RC_INV_ASSOC :
1876 					FCNVME_RJT_RC_LOGIC,
1877 				FCNVME_RJT_EXP_NONE, 0);
1878 		return true;
1879 	}
1880 
1881 	/* format a response */
1882 
1883 	iod->lsrsp->rsplen = sizeof(*acc);
1884 
1885 	nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1886 			fcnvme_lsdesc_len(
1887 				sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1888 			FCNVME_LS_DISCONNECT_ASSOC);
1889 
1890 	/*
1891 	 * The rules for LS response says the response cannot
1892 	 * go back until ABTS's have been sent for all outstanding
1893 	 * I/O and a Disconnect Association LS has been sent.
1894 	 * So... save off the Disconnect LS to send the response
1895 	 * later. If there was a prior LS already saved, replace
1896 	 * it with the newer one and send a can't perform reject
1897 	 * on the older one.
1898 	 */
1899 	spin_lock_irqsave(&tgtport->lock, flags);
1900 	oldls = assoc->rcv_disconn;
1901 	assoc->rcv_disconn = iod;
1902 	spin_unlock_irqrestore(&tgtport->lock, flags);
1903 
1904 	if (oldls) {
1905 		dev_info(tgtport->dev,
1906 			"{%d:%d} Multiple Disconnect Association LS's "
1907 			"received\n",
1908 			tgtport->fc_target_port.port_num, assoc->a_id);
1909 		/* overwrite good response with bogus failure */
1910 		oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1911 						sizeof(*iod->rspbuf),
1912 						/* ok to use rqst, LS is same */
1913 						rqst->w0.ls_cmd,
1914 						FCNVME_RJT_RC_UNAB,
1915 						FCNVME_RJT_EXP_NONE, 0);
1916 		nvmet_fc_xmt_ls_rsp(tgtport, oldls);
1917 	}
1918 
1919 	nvmet_fc_schedule_delete_assoc(assoc);
1920 	nvmet_fc_tgt_a_put(assoc);
1921 
1922 	return false;
1923 }
1924 
1925 
1926 /* *********************** NVME Ctrl Routines **************************** */
1927 
1928 
1929 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1930 
1931 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1932 
1933 static void
nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp * lsrsp)1934 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1935 {
1936 	struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private;
1937 	struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1938 
1939 	fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1940 				sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1941 	nvmet_fc_free_ls_iod(tgtport, iod);
1942 	nvmet_fc_tgtport_put(tgtport);
1943 }
1944 
1945 static void
nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1946 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1947 				struct nvmet_fc_ls_iod *iod)
1948 {
1949 	int ret;
1950 
1951 	fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1952 				  sizeof(*iod->rspbuf), DMA_TO_DEVICE);
1953 
1954 	ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp);
1955 	if (ret)
1956 		nvmet_fc_xmt_ls_rsp_done(iod->lsrsp);
1957 }
1958 
1959 /*
1960  * Actual processing routine for received FC-NVME LS Requests from the LLD
1961  */
1962 static void
nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_ls_iod * iod)1963 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1964 			struct nvmet_fc_ls_iod *iod)
1965 {
1966 	struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0;
1967 	bool sendrsp = true;
1968 
1969 	iod->lsrsp->nvme_fc_private = iod;
1970 	iod->lsrsp->rspbuf = iod->rspbuf;
1971 	iod->lsrsp->rspdma = iod->rspdma;
1972 	iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done;
1973 	/* Be preventative. handlers will later set to valid length */
1974 	iod->lsrsp->rsplen = 0;
1975 
1976 	iod->assoc = NULL;
1977 
1978 	/*
1979 	 * handlers:
1980 	 *   parse request input, execute the request, and format the
1981 	 *   LS response
1982 	 */
1983 	switch (w0->ls_cmd) {
1984 	case FCNVME_LS_CREATE_ASSOCIATION:
1985 		/* Creates Association and initial Admin Queue/Connection */
1986 		nvmet_fc_ls_create_association(tgtport, iod);
1987 		break;
1988 	case FCNVME_LS_CREATE_CONNECTION:
1989 		/* Creates an IO Queue/Connection */
1990 		nvmet_fc_ls_create_connection(tgtport, iod);
1991 		break;
1992 	case FCNVME_LS_DISCONNECT_ASSOC:
1993 		/* Terminate a Queue/Connection or the Association */
1994 		sendrsp = nvmet_fc_ls_disconnect(tgtport, iod);
1995 		break;
1996 	default:
1997 		iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf,
1998 				sizeof(*iod->rspbuf), w0->ls_cmd,
1999 				FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
2000 	}
2001 
2002 	if (sendrsp)
2003 		nvmet_fc_xmt_ls_rsp(tgtport, iod);
2004 }
2005 
2006 /*
2007  * Actual processing routine for received FC-NVME LS Requests from the LLD
2008  */
2009 static void
nvmet_fc_handle_ls_rqst_work(struct work_struct * work)2010 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
2011 {
2012 	struct nvmet_fc_ls_iod *iod =
2013 		container_of(work, struct nvmet_fc_ls_iod, work);
2014 	struct nvmet_fc_tgtport *tgtport = iod->tgtport;
2015 
2016 	nvmet_fc_handle_ls_rqst(tgtport, iod);
2017 }
2018 
2019 
2020 /**
2021  * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
2022  *                       upon the reception of a NVME LS request.
2023  *
2024  * The nvmet-fc layer will copy payload to an internal structure for
2025  * processing.  As such, upon completion of the routine, the LLDD may
2026  * immediately free/reuse the LS request buffer passed in the call.
2027  *
2028  * If this routine returns error, the LLDD should abort the exchange.
2029  *
2030  * @target_port: pointer to the (registered) target port the LS was
2031  *              received on.
2032  * @hosthandle: pointer to the host specific data, gets stored in iod.
2033  * @lsrsp:      pointer to a lsrsp structure to be used to reference
2034  *              the exchange corresponding to the LS.
2035  * @lsreqbuf:   pointer to the buffer containing the LS Request
2036  * @lsreqbuf_len: length, in bytes, of the received LS request
2037  */
2038 int
nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port * target_port,void * hosthandle,struct nvmefc_ls_rsp * lsrsp,void * lsreqbuf,u32 lsreqbuf_len)2039 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
2040 			void *hosthandle,
2041 			struct nvmefc_ls_rsp *lsrsp,
2042 			void *lsreqbuf, u32 lsreqbuf_len)
2043 {
2044 	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2045 	struct nvmet_fc_ls_iod *iod;
2046 	struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
2047 
2048 	if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
2049 		dev_info(tgtport->dev,
2050 			"RCV %s LS failed: payload too large (%d)\n",
2051 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2052 				nvmefc_ls_names[w0->ls_cmd] : "",
2053 			lsreqbuf_len);
2054 		return -E2BIG;
2055 	}
2056 
2057 	if (!nvmet_fc_tgtport_get(tgtport)) {
2058 		dev_info(tgtport->dev,
2059 			"RCV %s LS failed: target deleting\n",
2060 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2061 				nvmefc_ls_names[w0->ls_cmd] : "");
2062 		return -ESHUTDOWN;
2063 	}
2064 
2065 	iod = nvmet_fc_alloc_ls_iod(tgtport);
2066 	if (!iod) {
2067 		dev_info(tgtport->dev,
2068 			"RCV %s LS failed: context allocation failed\n",
2069 			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
2070 				nvmefc_ls_names[w0->ls_cmd] : "");
2071 		nvmet_fc_tgtport_put(tgtport);
2072 		return -ENOENT;
2073 	}
2074 
2075 	iod->lsrsp = lsrsp;
2076 	iod->fcpreq = NULL;
2077 	memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
2078 	iod->rqstdatalen = lsreqbuf_len;
2079 	iod->hosthandle = hosthandle;
2080 
2081 	queue_work(nvmet_wq, &iod->work);
2082 
2083 	return 0;
2084 }
2085 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
2086 
2087 
2088 /*
2089  * **********************
2090  * Start of FCP handling
2091  * **********************
2092  */
2093 
2094 static int
nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod * fod)2095 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2096 {
2097 	struct scatterlist *sg;
2098 	unsigned int nent;
2099 
2100 	sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
2101 	if (!sg)
2102 		goto out;
2103 
2104 	fod->data_sg = sg;
2105 	fod->data_sg_cnt = nent;
2106 	fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
2107 				((fod->io_dir == NVMET_FCP_WRITE) ?
2108 					DMA_FROM_DEVICE : DMA_TO_DEVICE));
2109 				/* note: write from initiator perspective */
2110 	fod->next_sg = fod->data_sg;
2111 
2112 	return 0;
2113 
2114 out:
2115 	return NVME_SC_INTERNAL;
2116 }
2117 
2118 static void
nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod * fod)2119 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
2120 {
2121 	if (!fod->data_sg || !fod->data_sg_cnt)
2122 		return;
2123 
2124 	fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
2125 				((fod->io_dir == NVMET_FCP_WRITE) ?
2126 					DMA_FROM_DEVICE : DMA_TO_DEVICE));
2127 	sgl_free(fod->data_sg);
2128 	fod->data_sg = NULL;
2129 	fod->data_sg_cnt = 0;
2130 }
2131 
2132 
2133 static bool
queue_90percent_full(struct nvmet_fc_tgt_queue * q,u32 sqhd)2134 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
2135 {
2136 	u32 sqtail, used;
2137 
2138 	/* egad, this is ugly. And sqtail is just a best guess */
2139 	sqtail = atomic_read(&q->sqtail) % q->sqsize;
2140 
2141 	used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
2142 	return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
2143 }
2144 
2145 /*
2146  * Prep RSP payload.
2147  * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
2148  */
2149 static void
nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2150 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2151 				struct nvmet_fc_fcp_iod *fod)
2152 {
2153 	struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
2154 	struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2155 	struct nvme_completion *cqe = &ersp->cqe;
2156 	u32 *cqewd = (u32 *)cqe;
2157 	bool send_ersp = false;
2158 	u32 rsn, rspcnt, xfr_length;
2159 
2160 	if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
2161 		xfr_length = fod->req.transfer_len;
2162 	else
2163 		xfr_length = fod->offset;
2164 
2165 	/*
2166 	 * check to see if we can send a 0's rsp.
2167 	 *   Note: to send a 0's response, the NVME-FC host transport will
2168 	 *   recreate the CQE. The host transport knows: sq id, SQHD (last
2169 	 *   seen in an ersp), and command_id. Thus it will create a
2170 	 *   zero-filled CQE with those known fields filled in. Transport
2171 	 *   must send an ersp for any condition where the cqe won't match
2172 	 *   this.
2173 	 *
2174 	 * Here are the FC-NVME mandated cases where we must send an ersp:
2175 	 *  every N responses, where N=ersp_ratio
2176 	 *  force fabric commands to send ersp's (not in FC-NVME but good
2177 	 *    practice)
2178 	 *  normal cmds: any time status is non-zero, or status is zero
2179 	 *     but words 0 or 1 are non-zero.
2180 	 *  the SQ is 90% or more full
2181 	 *  the cmd is a fused command
2182 	 *  transferred data length not equal to cmd iu length
2183 	 */
2184 	rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
2185 	if (!(rspcnt % fod->queue->ersp_ratio) ||
2186 	    nvme_is_fabrics((struct nvme_command *) sqe) ||
2187 	    xfr_length != fod->req.transfer_len ||
2188 	    (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
2189 	    (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
2190 	    queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
2191 		send_ersp = true;
2192 
2193 	/* re-set the fields */
2194 	fod->fcpreq->rspaddr = ersp;
2195 	fod->fcpreq->rspdma = fod->rspdma;
2196 
2197 	if (!send_ersp) {
2198 		memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
2199 		fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
2200 	} else {
2201 		ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
2202 		rsn = atomic_inc_return(&fod->queue->rsn);
2203 		ersp->rsn = cpu_to_be32(rsn);
2204 		ersp->xfrd_len = cpu_to_be32(xfr_length);
2205 		fod->fcpreq->rsplen = sizeof(*ersp);
2206 	}
2207 
2208 	fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
2209 				  sizeof(fod->rspiubuf), DMA_TO_DEVICE);
2210 }
2211 
2212 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
2213 
2214 static void
nvmet_fc_abort_op(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2215 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
2216 				struct nvmet_fc_fcp_iod *fod)
2217 {
2218 	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2219 
2220 	/* data no longer needed */
2221 	nvmet_fc_free_tgt_pgs(fod);
2222 
2223 	/*
2224 	 * if an ABTS was received or we issued the fcp_abort early
2225 	 * don't call abort routine again.
2226 	 */
2227 	/* no need to take lock - lock was taken earlier to get here */
2228 	if (!fod->aborted)
2229 		tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
2230 
2231 	nvmet_fc_free_fcp_iod(fod->queue, fod);
2232 }
2233 
2234 static void
nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2235 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
2236 				struct nvmet_fc_fcp_iod *fod)
2237 {
2238 	int ret;
2239 
2240 	fod->fcpreq->op = NVMET_FCOP_RSP;
2241 	fod->fcpreq->timeout = 0;
2242 
2243 	nvmet_fc_prep_fcp_rsp(tgtport, fod);
2244 
2245 	ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2246 	if (ret)
2247 		nvmet_fc_abort_op(tgtport, fod);
2248 }
2249 
2250 static void
nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod,u8 op)2251 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
2252 				struct nvmet_fc_fcp_iod *fod, u8 op)
2253 {
2254 	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2255 	struct scatterlist *sg = fod->next_sg;
2256 	unsigned long flags;
2257 	u32 remaininglen = fod->req.transfer_len - fod->offset;
2258 	u32 tlen = 0;
2259 	int ret;
2260 
2261 	fcpreq->op = op;
2262 	fcpreq->offset = fod->offset;
2263 	fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
2264 
2265 	/*
2266 	 * for next sequence:
2267 	 *  break at a sg element boundary
2268 	 *  attempt to keep sequence length capped at
2269 	 *    NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
2270 	 *    be longer if a single sg element is larger
2271 	 *    than that amount. This is done to avoid creating
2272 	 *    a new sg list to use for the tgtport api.
2273 	 */
2274 	fcpreq->sg = sg;
2275 	fcpreq->sg_cnt = 0;
2276 	while (tlen < remaininglen &&
2277 	       fcpreq->sg_cnt < tgtport->max_sg_cnt &&
2278 	       tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
2279 		fcpreq->sg_cnt++;
2280 		tlen += sg_dma_len(sg);
2281 		sg = sg_next(sg);
2282 	}
2283 	if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
2284 		fcpreq->sg_cnt++;
2285 		tlen += min_t(u32, sg_dma_len(sg), remaininglen);
2286 		sg = sg_next(sg);
2287 	}
2288 	if (tlen < remaininglen)
2289 		fod->next_sg = sg;
2290 	else
2291 		fod->next_sg = NULL;
2292 
2293 	fcpreq->transfer_length = tlen;
2294 	fcpreq->transferred_length = 0;
2295 	fcpreq->fcp_error = 0;
2296 	fcpreq->rsplen = 0;
2297 
2298 	/*
2299 	 * If the last READDATA request: check if LLDD supports
2300 	 * combined xfr with response.
2301 	 */
2302 	if ((op == NVMET_FCOP_READDATA) &&
2303 	    ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
2304 	    (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
2305 		fcpreq->op = NVMET_FCOP_READDATA_RSP;
2306 		nvmet_fc_prep_fcp_rsp(tgtport, fod);
2307 	}
2308 
2309 	ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
2310 	if (ret) {
2311 		/*
2312 		 * should be ok to set w/o lock as its in the thread of
2313 		 * execution (not an async timer routine) and doesn't
2314 		 * contend with any clearing action
2315 		 */
2316 		fod->abort = true;
2317 
2318 		if (op == NVMET_FCOP_WRITEDATA) {
2319 			spin_lock_irqsave(&fod->flock, flags);
2320 			fod->writedataactive = false;
2321 			spin_unlock_irqrestore(&fod->flock, flags);
2322 			nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2323 		} else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
2324 			fcpreq->fcp_error = ret;
2325 			fcpreq->transferred_length = 0;
2326 			nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
2327 		}
2328 	}
2329 }
2330 
2331 static inline bool
__nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod * fod,bool abort)2332 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
2333 {
2334 	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2335 	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2336 
2337 	/* if in the middle of an io and we need to tear down */
2338 	if (abort) {
2339 		if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
2340 			nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2341 			return true;
2342 		}
2343 
2344 		nvmet_fc_abort_op(tgtport, fod);
2345 		return true;
2346 	}
2347 
2348 	return false;
2349 }
2350 
2351 /*
2352  * actual done handler for FCP operations when completed by the lldd
2353  */
2354 static void
nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod * fod)2355 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
2356 {
2357 	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2358 	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2359 	unsigned long flags;
2360 	bool abort;
2361 
2362 	spin_lock_irqsave(&fod->flock, flags);
2363 	abort = fod->abort;
2364 	fod->writedataactive = false;
2365 	spin_unlock_irqrestore(&fod->flock, flags);
2366 
2367 	switch (fcpreq->op) {
2368 
2369 	case NVMET_FCOP_WRITEDATA:
2370 		if (__nvmet_fc_fod_op_abort(fod, abort))
2371 			return;
2372 		if (fcpreq->fcp_error ||
2373 		    fcpreq->transferred_length != fcpreq->transfer_length) {
2374 			spin_lock_irqsave(&fod->flock, flags);
2375 			fod->abort = true;
2376 			spin_unlock_irqrestore(&fod->flock, flags);
2377 
2378 			nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2379 			return;
2380 		}
2381 
2382 		fod->offset += fcpreq->transferred_length;
2383 		if (fod->offset != fod->req.transfer_len) {
2384 			spin_lock_irqsave(&fod->flock, flags);
2385 			fod->writedataactive = true;
2386 			spin_unlock_irqrestore(&fod->flock, flags);
2387 
2388 			/* transfer the next chunk */
2389 			nvmet_fc_transfer_fcp_data(tgtport, fod,
2390 						NVMET_FCOP_WRITEDATA);
2391 			return;
2392 		}
2393 
2394 		/* data transfer complete, resume with nvmet layer */
2395 		fod->req.execute(&fod->req);
2396 		break;
2397 
2398 	case NVMET_FCOP_READDATA:
2399 	case NVMET_FCOP_READDATA_RSP:
2400 		if (__nvmet_fc_fod_op_abort(fod, abort))
2401 			return;
2402 		if (fcpreq->fcp_error ||
2403 		    fcpreq->transferred_length != fcpreq->transfer_length) {
2404 			nvmet_fc_abort_op(tgtport, fod);
2405 			return;
2406 		}
2407 
2408 		/* success */
2409 
2410 		if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2411 			/* data no longer needed */
2412 			nvmet_fc_free_tgt_pgs(fod);
2413 			nvmet_fc_free_fcp_iod(fod->queue, fod);
2414 			return;
2415 		}
2416 
2417 		fod->offset += fcpreq->transferred_length;
2418 		if (fod->offset != fod->req.transfer_len) {
2419 			/* transfer the next chunk */
2420 			nvmet_fc_transfer_fcp_data(tgtport, fod,
2421 						NVMET_FCOP_READDATA);
2422 			return;
2423 		}
2424 
2425 		/* data transfer complete, send response */
2426 
2427 		/* data no longer needed */
2428 		nvmet_fc_free_tgt_pgs(fod);
2429 
2430 		nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2431 
2432 		break;
2433 
2434 	case NVMET_FCOP_RSP:
2435 		if (__nvmet_fc_fod_op_abort(fod, abort))
2436 			return;
2437 		nvmet_fc_free_fcp_iod(fod->queue, fod);
2438 		break;
2439 
2440 	default:
2441 		break;
2442 	}
2443 }
2444 
2445 static void
nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req * fcpreq)2446 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2447 {
2448 	struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2449 
2450 	nvmet_fc_fod_op_done(fod);
2451 }
2452 
2453 /*
2454  * actual completion handler after execution by the nvmet layer
2455  */
2456 static void
__nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod,int status)2457 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2458 			struct nvmet_fc_fcp_iod *fod, int status)
2459 {
2460 	struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2461 	struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2462 	unsigned long flags;
2463 	bool abort;
2464 
2465 	spin_lock_irqsave(&fod->flock, flags);
2466 	abort = fod->abort;
2467 	spin_unlock_irqrestore(&fod->flock, flags);
2468 
2469 	/* if we have a CQE, snoop the last sq_head value */
2470 	if (!status)
2471 		fod->queue->sqhd = cqe->sq_head;
2472 
2473 	if (abort) {
2474 		nvmet_fc_abort_op(tgtport, fod);
2475 		return;
2476 	}
2477 
2478 	/* if an error handling the cmd post initial parsing */
2479 	if (status) {
2480 		/* fudge up a failed CQE status for our transport error */
2481 		memset(cqe, 0, sizeof(*cqe));
2482 		cqe->sq_head = fod->queue->sqhd;	/* echo last cqe sqhd */
2483 		cqe->sq_id = cpu_to_le16(fod->queue->qid);
2484 		cqe->command_id = sqe->command_id;
2485 		cqe->status = cpu_to_le16(status);
2486 	} else {
2487 
2488 		/*
2489 		 * try to push the data even if the SQE status is non-zero.
2490 		 * There may be a status where data still was intended to
2491 		 * be moved
2492 		 */
2493 		if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2494 			/* push the data over before sending rsp */
2495 			nvmet_fc_transfer_fcp_data(tgtport, fod,
2496 						NVMET_FCOP_READDATA);
2497 			return;
2498 		}
2499 
2500 		/* writes & no data - fall thru */
2501 	}
2502 
2503 	/* data no longer needed */
2504 	nvmet_fc_free_tgt_pgs(fod);
2505 
2506 	nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2507 }
2508 
2509 
2510 static void
nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req * nvme_req)2511 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2512 {
2513 	struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2514 	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2515 
2516 	__nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2517 }
2518 
2519 
2520 /*
2521  * Actual processing routine for received FC-NVME I/O Requests from the LLD
2522  */
2523 static void
nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport * tgtport,struct nvmet_fc_fcp_iod * fod)2524 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2525 			struct nvmet_fc_fcp_iod *fod)
2526 {
2527 	struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2528 	u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2529 	int ret;
2530 
2531 	/*
2532 	 * Fused commands are currently not supported in the linux
2533 	 * implementation.
2534 	 *
2535 	 * As such, the implementation of the FC transport does not
2536 	 * look at the fused commands and order delivery to the upper
2537 	 * layer until we have both based on csn.
2538 	 */
2539 
2540 	fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2541 
2542 	if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2543 		fod->io_dir = NVMET_FCP_WRITE;
2544 		if (!nvme_is_write(&cmdiu->sqe))
2545 			goto transport_error;
2546 	} else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2547 		fod->io_dir = NVMET_FCP_READ;
2548 		if (nvme_is_write(&cmdiu->sqe))
2549 			goto transport_error;
2550 	} else {
2551 		fod->io_dir = NVMET_FCP_NODATA;
2552 		if (xfrlen)
2553 			goto transport_error;
2554 	}
2555 
2556 	fod->req.cmd = &fod->cmdiubuf.sqe;
2557 	fod->req.cqe = &fod->rspiubuf.cqe;
2558 	if (!tgtport->pe)
2559 		goto transport_error;
2560 	fod->req.port = tgtport->pe->port;
2561 
2562 	/* clear any response payload */
2563 	memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2564 
2565 	fod->data_sg = NULL;
2566 	fod->data_sg_cnt = 0;
2567 
2568 	ret = nvmet_req_init(&fod->req,
2569 				&fod->queue->nvme_cq,
2570 				&fod->queue->nvme_sq,
2571 				&nvmet_fc_tgt_fcp_ops);
2572 	if (!ret) {
2573 		/* bad SQE content or invalid ctrl state */
2574 		/* nvmet layer has already called op done to send rsp. */
2575 		return;
2576 	}
2577 
2578 	fod->req.transfer_len = xfrlen;
2579 
2580 	/* keep a running counter of tail position */
2581 	atomic_inc(&fod->queue->sqtail);
2582 
2583 	if (fod->req.transfer_len) {
2584 		ret = nvmet_fc_alloc_tgt_pgs(fod);
2585 		if (ret) {
2586 			nvmet_req_complete(&fod->req, ret);
2587 			return;
2588 		}
2589 	}
2590 	fod->req.sg = fod->data_sg;
2591 	fod->req.sg_cnt = fod->data_sg_cnt;
2592 	fod->offset = 0;
2593 
2594 	if (fod->io_dir == NVMET_FCP_WRITE) {
2595 		/* pull the data over before invoking nvmet layer */
2596 		nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2597 		return;
2598 	}
2599 
2600 	/*
2601 	 * Reads or no data:
2602 	 *
2603 	 * can invoke the nvmet_layer now. If read data, cmd completion will
2604 	 * push the data
2605 	 */
2606 	fod->req.execute(&fod->req);
2607 	return;
2608 
2609 transport_error:
2610 	nvmet_fc_abort_op(tgtport, fod);
2611 }
2612 
2613 /**
2614  * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2615  *                       upon the reception of a NVME FCP CMD IU.
2616  *
2617  * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2618  * layer for processing.
2619  *
2620  * The nvmet_fc layer allocates a local job structure (struct
2621  * nvmet_fc_fcp_iod) from the queue for the io and copies the
2622  * CMD IU buffer to the job structure. As such, on a successful
2623  * completion (returns 0), the LLDD may immediately free/reuse
2624  * the CMD IU buffer passed in the call.
2625  *
2626  * However, in some circumstances, due to the packetized nature of FC
2627  * and the api of the FC LLDD which may issue a hw command to send the
2628  * response, but the LLDD may not get the hw completion for that command
2629  * and upcall the nvmet_fc layer before a new command may be
2630  * asynchronously received - its possible for a command to be received
2631  * before the LLDD and nvmet_fc have recycled the job structure. It gives
2632  * the appearance of more commands received than fits in the sq.
2633  * To alleviate this scenario, a temporary queue is maintained in the
2634  * transport for pending LLDD requests waiting for a queue job structure.
2635  * In these "overrun" cases, a temporary queue element is allocated
2636  * the LLDD request and CMD iu buffer information remembered, and the
2637  * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2638  * structure is freed, it is immediately reallocated for anything on the
2639  * pending request list. The LLDDs defer_rcv() callback is called,
2640  * informing the LLDD that it may reuse the CMD IU buffer, and the io
2641  * is then started normally with the transport.
2642  *
2643  * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2644  * the completion as successful but must not reuse the CMD IU buffer
2645  * until the LLDD's defer_rcv() callback has been called for the
2646  * corresponding struct nvmefc_tgt_fcp_req pointer.
2647  *
2648  * If there is any other condition in which an error occurs, the
2649  * transport will return a non-zero status indicating the error.
2650  * In all cases other than -EOVERFLOW, the transport has not accepted the
2651  * request and the LLDD should abort the exchange.
2652  *
2653  * @target_port: pointer to the (registered) target port the FCP CMD IU
2654  *              was received on.
2655  * @fcpreq:     pointer to a fcpreq request structure to be used to reference
2656  *              the exchange corresponding to the FCP Exchange.
2657  * @cmdiubuf:   pointer to the buffer containing the FCP CMD IU
2658  * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2659  */
2660 int
nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port * target_port,struct nvmefc_tgt_fcp_req * fcpreq,void * cmdiubuf,u32 cmdiubuf_len)2661 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2662 			struct nvmefc_tgt_fcp_req *fcpreq,
2663 			void *cmdiubuf, u32 cmdiubuf_len)
2664 {
2665 	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2666 	struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2667 	struct nvmet_fc_tgt_queue *queue;
2668 	struct nvmet_fc_fcp_iod *fod;
2669 	struct nvmet_fc_defer_fcp_req *deferfcp;
2670 	unsigned long flags;
2671 
2672 	/* validate iu, so the connection id can be used to find the queue */
2673 	if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2674 			(cmdiu->format_id != NVME_CMD_FORMAT_ID) ||
2675 			(cmdiu->fc_id != NVME_CMD_FC_ID) ||
2676 			(be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2677 		return -EIO;
2678 
2679 	queue = nvmet_fc_find_target_queue(tgtport,
2680 				be64_to_cpu(cmdiu->connection_id));
2681 	if (!queue)
2682 		return -ENOTCONN;
2683 
2684 	/*
2685 	 * note: reference taken by find_target_queue
2686 	 * After successful fod allocation, the fod will inherit the
2687 	 * ownership of that reference and will remove the reference
2688 	 * when the fod is freed.
2689 	 */
2690 
2691 	spin_lock_irqsave(&queue->qlock, flags);
2692 
2693 	fod = nvmet_fc_alloc_fcp_iod(queue);
2694 	if (fod) {
2695 		spin_unlock_irqrestore(&queue->qlock, flags);
2696 
2697 		fcpreq->nvmet_fc_private = fod;
2698 		fod->fcpreq = fcpreq;
2699 
2700 		memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2701 
2702 		nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2703 
2704 		return 0;
2705 	}
2706 
2707 	if (!tgtport->ops->defer_rcv) {
2708 		spin_unlock_irqrestore(&queue->qlock, flags);
2709 		/* release the queue lookup reference */
2710 		nvmet_fc_tgt_q_put(queue);
2711 		return -ENOENT;
2712 	}
2713 
2714 	deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2715 			struct nvmet_fc_defer_fcp_req, req_list);
2716 	if (deferfcp) {
2717 		/* Just re-use one that was previously allocated */
2718 		list_del(&deferfcp->req_list);
2719 	} else {
2720 		spin_unlock_irqrestore(&queue->qlock, flags);
2721 
2722 		/* Now we need to dynamically allocate one */
2723 		deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2724 		if (!deferfcp) {
2725 			/* release the queue lookup reference */
2726 			nvmet_fc_tgt_q_put(queue);
2727 			return -ENOMEM;
2728 		}
2729 		spin_lock_irqsave(&queue->qlock, flags);
2730 	}
2731 
2732 	/* For now, use rspaddr / rsplen to save payload information */
2733 	fcpreq->rspaddr = cmdiubuf;
2734 	fcpreq->rsplen  = cmdiubuf_len;
2735 	deferfcp->fcp_req = fcpreq;
2736 
2737 	/* defer processing till a fod becomes available */
2738 	list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2739 
2740 	/* NOTE: the queue lookup reference is still valid */
2741 
2742 	spin_unlock_irqrestore(&queue->qlock, flags);
2743 
2744 	return -EOVERFLOW;
2745 }
2746 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2747 
2748 /**
2749  * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2750  *                       upon the reception of an ABTS for a FCP command
2751  *
2752  * Notify the transport that an ABTS has been received for a FCP command
2753  * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2754  * LLDD believes the command is still being worked on
2755  * (template_ops->fcp_req_release() has not been called).
2756  *
2757  * The transport will wait for any outstanding work (an op to the LLDD,
2758  * which the lldd should complete with error due to the ABTS; or the
2759  * completion from the nvmet layer of the nvme command), then will
2760  * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2761  * return the i/o context to the LLDD.  The LLDD may send the BA_ACC
2762  * to the ABTS either after return from this function (assuming any
2763  * outstanding op work has been terminated) or upon the callback being
2764  * called.
2765  *
2766  * @target_port: pointer to the (registered) target port the FCP CMD IU
2767  *              was received on.
2768  * @fcpreq:     pointer to the fcpreq request structure that corresponds
2769  *              to the exchange that received the ABTS.
2770  */
2771 void
nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port * target_port,struct nvmefc_tgt_fcp_req * fcpreq)2772 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2773 			struct nvmefc_tgt_fcp_req *fcpreq)
2774 {
2775 	struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2776 	struct nvmet_fc_tgt_queue *queue;
2777 	unsigned long flags;
2778 
2779 	if (!fod || fod->fcpreq != fcpreq)
2780 		/* job appears to have already completed, ignore abort */
2781 		return;
2782 
2783 	queue = fod->queue;
2784 
2785 	spin_lock_irqsave(&queue->qlock, flags);
2786 	if (fod->active) {
2787 		/*
2788 		 * mark as abort. The abort handler, invoked upon completion
2789 		 * of any work, will detect the aborted status and do the
2790 		 * callback.
2791 		 */
2792 		spin_lock(&fod->flock);
2793 		fod->abort = true;
2794 		fod->aborted = true;
2795 		spin_unlock(&fod->flock);
2796 	}
2797 	spin_unlock_irqrestore(&queue->qlock, flags);
2798 }
2799 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2800 
2801 
2802 struct nvmet_fc_traddr {
2803 	u64	nn;
2804 	u64	pn;
2805 };
2806 
2807 static int
__nvme_fc_parse_u64(substring_t * sstr,u64 * val)2808 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2809 {
2810 	u64 token64;
2811 
2812 	if (match_u64(sstr, &token64))
2813 		return -EINVAL;
2814 	*val = token64;
2815 
2816 	return 0;
2817 }
2818 
2819 /*
2820  * This routine validates and extracts the WWN's from the TRADDR string.
2821  * As kernel parsers need the 0x to determine number base, universally
2822  * build string to parse with 0x prefix before parsing name strings.
2823  */
2824 static int
nvme_fc_parse_traddr(struct nvmet_fc_traddr * traddr,char * buf,size_t blen)2825 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2826 {
2827 	char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2828 	substring_t wwn = { name, &name[sizeof(name)-1] };
2829 	int nnoffset, pnoffset;
2830 
2831 	/* validate if string is one of the 2 allowed formats */
2832 	if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2833 			!strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2834 			!strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2835 				"pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2836 		nnoffset = NVME_FC_TRADDR_OXNNLEN;
2837 		pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2838 						NVME_FC_TRADDR_OXNNLEN;
2839 	} else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2840 			!strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2841 			!strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2842 				"pn-", NVME_FC_TRADDR_NNLEN))) {
2843 		nnoffset = NVME_FC_TRADDR_NNLEN;
2844 		pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2845 	} else
2846 		goto out_einval;
2847 
2848 	name[0] = '0';
2849 	name[1] = 'x';
2850 	name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2851 
2852 	memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2853 	if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2854 		goto out_einval;
2855 
2856 	memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2857 	if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2858 		goto out_einval;
2859 
2860 	return 0;
2861 
2862 out_einval:
2863 	pr_warn("%s: bad traddr string\n", __func__);
2864 	return -EINVAL;
2865 }
2866 
2867 static int
nvmet_fc_add_port(struct nvmet_port * port)2868 nvmet_fc_add_port(struct nvmet_port *port)
2869 {
2870 	struct nvmet_fc_tgtport *tgtport;
2871 	struct nvmet_fc_port_entry *pe;
2872 	struct nvmet_fc_traddr traddr = { 0L, 0L };
2873 	unsigned long flags;
2874 	int ret;
2875 
2876 	/* validate the address info */
2877 	if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2878 	    (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2879 		return -EINVAL;
2880 
2881 	/* map the traddr address info to a target port */
2882 
2883 	ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2884 			sizeof(port->disc_addr.traddr));
2885 	if (ret)
2886 		return ret;
2887 
2888 	pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2889 	if (!pe)
2890 		return -ENOMEM;
2891 
2892 	ret = -ENXIO;
2893 	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2894 	list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2895 		if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2896 		    (tgtport->fc_target_port.port_name == traddr.pn)) {
2897 			/* a FC port can only be 1 nvmet port id */
2898 			if (!tgtport->pe) {
2899 				nvmet_fc_portentry_bind(tgtport, pe, port);
2900 				ret = 0;
2901 			} else
2902 				ret = -EALREADY;
2903 			break;
2904 		}
2905 	}
2906 	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2907 
2908 	if (ret)
2909 		kfree(pe);
2910 
2911 	return ret;
2912 }
2913 
2914 static void
nvmet_fc_remove_port(struct nvmet_port * port)2915 nvmet_fc_remove_port(struct nvmet_port *port)
2916 {
2917 	struct nvmet_fc_port_entry *pe = port->priv;
2918 
2919 	nvmet_fc_portentry_unbind(pe);
2920 
2921 	/* terminate any outstanding associations */
2922 	__nvmet_fc_free_assocs(pe->tgtport);
2923 
2924 	kfree(pe);
2925 }
2926 
2927 static void
nvmet_fc_discovery_chg(struct nvmet_port * port)2928 nvmet_fc_discovery_chg(struct nvmet_port *port)
2929 {
2930 	struct nvmet_fc_port_entry *pe = port->priv;
2931 	struct nvmet_fc_tgtport *tgtport = pe->tgtport;
2932 
2933 	if (tgtport && tgtport->ops->discovery_event)
2934 		tgtport->ops->discovery_event(&tgtport->fc_target_port);
2935 }
2936 
2937 static ssize_t
nvmet_fc_host_traddr(struct nvmet_ctrl * ctrl,char * traddr,size_t traddr_size)2938 nvmet_fc_host_traddr(struct nvmet_ctrl *ctrl,
2939 		char *traddr, size_t traddr_size)
2940 {
2941 	struct nvmet_sq *sq = ctrl->sqs[0];
2942 	struct nvmet_fc_tgt_queue *queue =
2943 		container_of(sq, struct nvmet_fc_tgt_queue, nvme_sq);
2944 	struct nvmet_fc_tgtport *tgtport = queue->assoc ? queue->assoc->tgtport : NULL;
2945 	struct nvmet_fc_hostport *hostport = queue->assoc ? queue->assoc->hostport : NULL;
2946 	u64 wwnn, wwpn;
2947 	ssize_t ret = 0;
2948 
2949 	if (!tgtport || !nvmet_fc_tgtport_get(tgtport))
2950 		return -ENODEV;
2951 	if (!hostport || !nvmet_fc_hostport_get(hostport)) {
2952 		ret = -ENODEV;
2953 		goto out_put;
2954 	}
2955 
2956 	if (tgtport->ops->host_traddr) {
2957 		ret = tgtport->ops->host_traddr(hostport->hosthandle, &wwnn, &wwpn);
2958 		if (ret)
2959 			goto out_put_host;
2960 		ret = snprintf(traddr, traddr_size, "nn-0x%llx:pn-0x%llx", wwnn, wwpn);
2961 	}
2962 out_put_host:
2963 	nvmet_fc_hostport_put(hostport);
2964 out_put:
2965 	nvmet_fc_tgtport_put(tgtport);
2966 	return ret;
2967 }
2968 
2969 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2970 	.owner			= THIS_MODULE,
2971 	.type			= NVMF_TRTYPE_FC,
2972 	.msdbd			= 1,
2973 	.add_port		= nvmet_fc_add_port,
2974 	.remove_port		= nvmet_fc_remove_port,
2975 	.queue_response		= nvmet_fc_fcp_nvme_cmd_done,
2976 	.delete_ctrl		= nvmet_fc_delete_ctrl,
2977 	.discovery_chg		= nvmet_fc_discovery_chg,
2978 	.host_traddr		= nvmet_fc_host_traddr,
2979 };
2980 
nvmet_fc_init_module(void)2981 static int __init nvmet_fc_init_module(void)
2982 {
2983 	return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2984 }
2985 
nvmet_fc_exit_module(void)2986 static void __exit nvmet_fc_exit_module(void)
2987 {
2988 	/* ensure any shutdown operation, e.g. delete ctrls have finished */
2989 	flush_workqueue(nvmet_wq);
2990 
2991 	/* sanity check - all lports should be removed */
2992 	if (!list_empty(&nvmet_fc_target_list))
2993 		pr_warn("%s: targetport list not empty\n", __func__);
2994 
2995 	nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2996 
2997 	ida_destroy(&nvmet_fc_tgtport_cnt);
2998 }
2999 
3000 module_init(nvmet_fc_init_module);
3001 module_exit(nvmet_fc_exit_module);
3002 
3003 MODULE_DESCRIPTION("NVMe target FC transport driver");
3004 MODULE_LICENSE("GPL v2");
3005