xref: /linux/drivers/scsi/lpfc/lpfc_nvme.c (revision 621cde16e49b3ecf7d59a8106a20aaebfb4a59a9)
1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channel Host Bus Adapters.                                *
4  * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *
5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30 
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38 
39 #include "lpfc_version.h"
40 #include "lpfc_hw4.h"
41 #include "lpfc_hw.h"
42 #include "lpfc_sli.h"
43 #include "lpfc_sli4.h"
44 #include "lpfc_nl.h"
45 #include "lpfc_disc.h"
46 #include "lpfc.h"
47 #include "lpfc_nvme.h"
48 #include "lpfc_scsi.h"
49 #include "lpfc_logmsg.h"
50 #include "lpfc_crtn.h"
51 #include "lpfc_vport.h"
52 #include "lpfc_debugfs.h"
53 
54 /* NVME initiator-based functions */
55 
56 static struct lpfc_io_buf *
57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 		  int idx, int expedite);
59 
60 static void
61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62 
63 static struct nvme_fc_port_template lpfc_nvme_template;
64 
65 /**
66  * lpfc_nvme_create_queue -
67  * @pnvme_lport: Transport localport that LS is to be issued from
68  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
69  * @qsize: Size of the queue in bytes
70  * @handle: An opaque driver handle used in follow-up calls.
71  *
72  * Driver registers this routine to preallocate and initialize any
73  * internal data structures to bind the @qidx to its internal IO queues.
74  * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
75  *
76  * Return value :
77  *   0 - Success
78  *   -EINVAL - Unsupported input value.
79  *   -ENOMEM - Could not alloc necessary memory
80  **/
81 static int
lpfc_nvme_create_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,u16 qsize,void ** handle)82 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
83 		       unsigned int qidx, u16 qsize,
84 		       void **handle)
85 {
86 	struct lpfc_nvme_lport *lport;
87 	struct lpfc_vport *vport;
88 	struct lpfc_nvme_qhandle *qhandle;
89 	char *str;
90 
91 	if (!pnvme_lport->private)
92 		return -ENOMEM;
93 
94 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
95 	vport = lport->vport;
96 
97 	if (!vport || test_bit(FC_UNLOADING, &vport->load_flag) ||
98 	    test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
99 		return -ENODEV;
100 
101 	qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
102 	if (qhandle == NULL)
103 		return -ENOMEM;
104 
105 	qhandle->cpu_id = raw_smp_processor_id();
106 	qhandle->qidx = qidx;
107 	/*
108 	 * NVME qidx == 0 is the admin queue, so both admin queue
109 	 * and first IO queue will use MSI-X vector and associated
110 	 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
111 	 */
112 	if (qidx) {
113 		str = "IO ";  /* IO queue */
114 		qhandle->index = ((qidx - 1) %
115 			lpfc_nvme_template.max_hw_queues);
116 	} else {
117 		str = "ADM";  /* Admin queue */
118 		qhandle->index = qidx;
119 	}
120 
121 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
122 			 "6073 Binding %s HdwQueue %d  (cpu %d) to "
123 			 "hdw_queue %d qhandle x%px\n", str,
124 			 qidx, qhandle->cpu_id, qhandle->index, qhandle);
125 	*handle = (void *)qhandle;
126 	return 0;
127 }
128 
129 /**
130  * lpfc_nvme_delete_queue -
131  * @pnvme_lport: Transport localport that LS is to be issued from
132  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
133  * @handle: An opaque driver handle from lpfc_nvme_create_queue
134  *
135  * Driver registers this routine to free
136  * any internal data structures to bind the @qidx to its internal
137  * IO queues.
138  *
139  * Return value :
140  *   0 - Success
141  *   TODO:  What are the failure codes.
142  **/
143 static void
lpfc_nvme_delete_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,void * handle)144 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
145 		       unsigned int qidx,
146 		       void *handle)
147 {
148 	struct lpfc_nvme_lport *lport;
149 	struct lpfc_vport *vport;
150 
151 	if (!pnvme_lport->private)
152 		return;
153 
154 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
155 	vport = lport->vport;
156 
157 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
158 			"6001 ENTER.  lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
159 			lport, qidx, handle);
160 	kfree(handle);
161 }
162 
163 static void
lpfc_nvme_localport_delete(struct nvme_fc_local_port * localport)164 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
165 {
166 	struct lpfc_nvme_lport *lport = localport->private;
167 
168 	lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
169 			 "6173 localport x%px delete complete\n",
170 			 lport);
171 
172 	/* release any threads waiting for the unreg to complete */
173 	if (lport->vport->localport)
174 		complete(lport->lport_unreg_cmp);
175 }
176 
177 /* lpfc_nvme_remoteport_delete
178  *
179  * @remoteport: Pointer to an nvme transport remoteport instance.
180  *
181  * This is a template downcall.  NVME transport calls this function
182  * when it has completed the unregistration of a previously
183  * registered remoteport.
184  *
185  * Return value :
186  * None
187  */
188 static void
lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port * remoteport)189 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
190 {
191 	struct lpfc_nvme_rport *rport = remoteport->private;
192 	struct lpfc_vport *vport;
193 	struct lpfc_nodelist *ndlp;
194 	u32 fc4_xpt_flags;
195 
196 	ndlp = rport->ndlp;
197 	if (!ndlp) {
198 		pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",
199 		       __func__, rport, remoteport);
200 		goto rport_err;
201 	}
202 
203 	vport = ndlp->vport;
204 	if (!vport) {
205 		pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",
206 		       __func__, ndlp, ndlp->nlp_state, rport);
207 		goto rport_err;
208 	}
209 
210 	fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;
211 
212 	/* Remove this rport from the lport's list - memory is owned by the
213 	 * transport. Remove the ndlp reference for the NVME transport before
214 	 * calling state machine to remove the node.
215 	 */
216 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
217 			 "6146 remoteport delete of remoteport x%px, ndlp x%px "
218 			 "DID x%x xflags x%x\n",
219 			 remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);
220 	spin_lock_irq(&ndlp->lock);
221 
222 	/* The register rebind might have occurred before the delete
223 	 * downcall.  Guard against this race.
224 	 */
225 	if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)
226 		ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);
227 
228 	spin_unlock_irq(&ndlp->lock);
229 
230 	/* On a devloss timeout event, one more put is executed provided the
231 	 * NVME and SCSI rport unregister requests are complete.
232 	 */
233 	if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))
234 		lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
235 
236  rport_err:
237 	return;
238 }
239 
240 /**
241  * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
242  * @phba: pointer to lpfc hba data structure.
243  * @axchg: pointer to exchange context for the NVME LS request
244  *
245  * This routine is used for processing an asychronously received NVME LS
246  * request. Any remaining validation is done and the LS is then forwarded
247  * to the nvme-fc transport via nvme_fc_rcv_ls_req().
248  *
249  * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
250  * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
251  * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
252  *
253  * Returns 0 if LS was handled and delivered to the transport
254  * Returns 1 if LS failed to be handled and should be dropped
255  */
256 int
lpfc_nvme_handle_lsreq(struct lpfc_hba * phba,struct lpfc_async_xchg_ctx * axchg)257 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
258 			struct lpfc_async_xchg_ctx *axchg)
259 {
260 #if (IS_ENABLED(CONFIG_NVME_FC))
261 	struct lpfc_vport *vport;
262 	struct lpfc_nvme_rport *lpfc_rport;
263 	struct nvme_fc_remote_port *remoteport;
264 	struct lpfc_nvme_lport *lport;
265 	uint32_t *payload = axchg->payload;
266 	int rc;
267 
268 	vport = axchg->ndlp->vport;
269 	lpfc_rport = axchg->ndlp->nrport;
270 	if (!lpfc_rport)
271 		return -EINVAL;
272 
273 	remoteport = lpfc_rport->remoteport;
274 	if (!vport->localport ||
275 	    test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
276 		return -EINVAL;
277 
278 	lport = vport->localport->private;
279 	if (!lport)
280 		return -EINVAL;
281 
282 	rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
283 				axchg->size);
284 
285 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
286 			"6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
287 			"%08x %08x %08x\n",
288 			axchg->size, rc,
289 			*payload, *(payload+1), *(payload+2),
290 			*(payload+3), *(payload+4), *(payload+5));
291 
292 	if (!rc)
293 		return 0;
294 #endif
295 	return 1;
296 }
297 
298 /**
299  * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
300  *        LS request.
301  * @phba: Pointer to HBA context object
302  * @vport: The local port that issued the LS
303  * @cmdwqe: Pointer to driver command WQE object.
304  * @wcqe: Pointer to driver response CQE object.
305  *
306  * This function is the generic completion handler for NVME LS requests.
307  * The function updates any states and statistics, calls the transport
308  * ls_req done() routine, then tears down the command and buffers used
309  * for the LS request.
310  **/
311 void
__lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_vport * vport,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe)312 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
313 			struct lpfc_iocbq *cmdwqe,
314 			struct lpfc_wcqe_complete *wcqe)
315 {
316 	struct nvmefc_ls_req *pnvme_lsreq;
317 	struct lpfc_dmabuf *buf_ptr;
318 	struct lpfc_nodelist *ndlp;
319 	int status;
320 
321 	pnvme_lsreq = cmdwqe->context_un.nvme_lsreq;
322 	ndlp = cmdwqe->ndlp;
323 	buf_ptr = cmdwqe->bpl_dmabuf;
324 
325 	status = bf_get(lpfc_wcqe_c_status, wcqe);
326 
327 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
328 			 "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "
329 			 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
330 			 "ndlp:x%px\n",
331 			 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
332 			 cmdwqe->sli4_xritag, status,
333 			 (wcqe->parameter & 0xffff),
334 			 cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf,
335 			 ndlp);
336 
337 	lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
338 			 cmdwqe->sli4_xritag, status, wcqe->parameter);
339 
340 	if (buf_ptr) {
341 		lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
342 		kfree(buf_ptr);
343 		cmdwqe->bpl_dmabuf = NULL;
344 	}
345 	if (pnvme_lsreq->done) {
346 		if (status != CQE_STATUS_SUCCESS)
347 			status = -ENXIO;
348 		pnvme_lsreq->done(pnvme_lsreq, status);
349 	} else {
350 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
351 				 "6046 NVMEx cmpl without done call back? "
352 				 "Data x%px DID %x Xri: %x status %x\n",
353 				pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
354 				cmdwqe->sli4_xritag, status);
355 	}
356 	if (ndlp) {
357 		lpfc_nlp_put(ndlp);
358 		cmdwqe->ndlp = NULL;
359 	}
360 	lpfc_sli_release_iocbq(phba, cmdwqe);
361 }
362 
363 static void
lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe)364 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
365 		     struct lpfc_iocbq *rspwqe)
366 {
367 	struct lpfc_vport *vport = cmdwqe->vport;
368 	struct lpfc_nvme_lport *lport;
369 	uint32_t status;
370 	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
371 
372 	status = bf_get(lpfc_wcqe_c_status, wcqe);
373 
374 	if (vport->localport) {
375 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
376 		if (lport) {
377 			atomic_inc(&lport->fc4NvmeLsCmpls);
378 			if (status) {
379 				if (bf_get(lpfc_wcqe_c_xb, wcqe))
380 					atomic_inc(&lport->cmpl_ls_xb);
381 				atomic_inc(&lport->cmpl_ls_err);
382 			}
383 		}
384 	}
385 
386 	__lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
387 }
388 
389 static int
lpfc_nvme_gen_req(struct lpfc_vport * vport,struct lpfc_dmabuf * bmp,struct lpfc_dmabuf * inp,struct nvmefc_ls_req * pnvme_lsreq,void (* cmpl)(struct lpfc_hba *,struct lpfc_iocbq *,struct lpfc_iocbq *),struct lpfc_nodelist * ndlp,uint32_t num_entry,uint32_t tmo,uint8_t retry)390 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
391 		  struct lpfc_dmabuf *inp,
392 		  struct nvmefc_ls_req *pnvme_lsreq,
393 		  void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
394 			       struct lpfc_iocbq *),
395 		  struct lpfc_nodelist *ndlp, uint32_t num_entry,
396 		  uint32_t tmo, uint8_t retry)
397 {
398 	struct lpfc_hba *phba = vport->phba;
399 	union lpfc_wqe128 *wqe;
400 	struct lpfc_iocbq *genwqe;
401 	struct ulp_bde64 *bpl;
402 	struct ulp_bde64 bde;
403 	int i, rc, xmit_len, first_len;
404 
405 	/* Allocate buffer for  command WQE */
406 	genwqe = lpfc_sli_get_iocbq(phba);
407 	if (genwqe == NULL)
408 		return 1;
409 
410 	wqe = &genwqe->wqe;
411 	/* Initialize only 64 bytes */
412 	memset(wqe, 0, sizeof(union lpfc_wqe));
413 
414 	genwqe->bpl_dmabuf = bmp;
415 	genwqe->cmd_flag |= LPFC_IO_NVME_LS;
416 
417 	/* Save for completion so we can release these resources */
418 	genwqe->ndlp = lpfc_nlp_get(ndlp);
419 	if (!genwqe->ndlp) {
420 		dev_warn(&phba->pcidev->dev,
421 			 "Warning: Failed node ref, not sending LS_REQ\n");
422 		lpfc_sli_release_iocbq(phba, genwqe);
423 		return 1;
424 	}
425 
426 	genwqe->context_un.nvme_lsreq = pnvme_lsreq;
427 	/* Fill in payload, bp points to frame payload */
428 
429 	if (!tmo)
430 		/* FC spec states we need 3 * ratov for CT requests */
431 		tmo = (3 * phba->fc_ratov);
432 
433 	/* For this command calculate the xmit length of the request bde. */
434 	xmit_len = 0;
435 	first_len = 0;
436 	bpl = (struct ulp_bde64 *)bmp->virt;
437 	for (i = 0; i < num_entry; i++) {
438 		bde.tus.w = bpl[i].tus.w;
439 		if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
440 			break;
441 		xmit_len += bde.tus.f.bdeSize;
442 		if (i == 0)
443 			first_len = xmit_len;
444 	}
445 
446 	genwqe->num_bdes = num_entry;
447 	genwqe->hba_wqidx = 0;
448 
449 	/* Words 0 - 2 */
450 	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
451 	wqe->generic.bde.tus.f.bdeSize = first_len;
452 	wqe->generic.bde.addrLow = bpl[0].addrLow;
453 	wqe->generic.bde.addrHigh = bpl[0].addrHigh;
454 
455 	/* Word 3 */
456 	wqe->gen_req.request_payload_len = first_len;
457 
458 	/* Word 4 */
459 
460 	/* Word 5 */
461 	bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
462 	bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
463 	bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
464 	bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
465 	bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
466 
467 	/* Word 6 */
468 	bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
469 	       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
470 	bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
471 
472 	/* Word 7 */
473 	bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);
474 	bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
475 	bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
476 	bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
477 
478 	/* Word 8 */
479 	wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
480 
481 	/* Word 9 */
482 	bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
483 
484 	/* Word 10 */
485 	bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
486 	bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
487 	bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
488 	bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
489 	bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
490 
491 	/* Word 11 */
492 	bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
493 	bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
494 
495 
496 	/* Issue GEN REQ WQE for NPORT <did> */
497 	genwqe->cmd_cmpl = cmpl;
498 	genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
499 	genwqe->vport = vport;
500 	genwqe->retry = retry;
501 
502 	lpfc_nvmeio_data(phba, "NVME LS  XMIT: xri x%x iotag x%x to x%06x\n",
503 			 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
504 
505 	rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
506 	if (rc) {
507 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
508 				 "6045 Issue GEN REQ WQE to NPORT x%x "
509 				 "Data: x%x x%x  rc x%x\n",
510 				 ndlp->nlp_DID, genwqe->iotag,
511 				 vport->port_state, rc);
512 		lpfc_nlp_put(ndlp);
513 		lpfc_sli_release_iocbq(phba, genwqe);
514 		return 1;
515 	}
516 
517 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
518 			 "6050 Issue GEN REQ WQE to NPORT x%x "
519 			 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
520 			 "bmp:x%px xmit:%d 1st:%d\n",
521 			 ndlp->nlp_DID, genwqe->sli4_xritag,
522 			 vport->port_state,
523 			 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
524 	return 0;
525 }
526 
527 
528 /**
529  * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
530  * @vport: The local port issuing the LS
531  * @ndlp: The remote port to send the LS to
532  * @pnvme_lsreq: Pointer to LS request structure from the transport
533  * @gen_req_cmp: Completion call-back
534  *
535  * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
536  * WQE to perform the LS operation.
537  *
538  * Return value :
539  *   0 - Success
540  *   non-zero: various error codes, in form of -Exxx
541  **/
542 int
__lpfc_nvme_ls_req(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq,void (* gen_req_cmp)(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe))543 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
544 		      struct nvmefc_ls_req *pnvme_lsreq,
545 		      void (*gen_req_cmp)(struct lpfc_hba *phba,
546 				struct lpfc_iocbq *cmdwqe,
547 				struct lpfc_iocbq *rspwqe))
548 {
549 	struct lpfc_dmabuf *bmp;
550 	struct ulp_bde64 *bpl;
551 	int ret;
552 	uint16_t ntype, nstate;
553 
554 	if (!ndlp) {
555 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
556 				 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
557 				 "LS Req\n",
558 				 ndlp);
559 		return -ENODEV;
560 	}
561 
562 	ntype = ndlp->nlp_type;
563 	nstate = ndlp->nlp_state;
564 	if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
565 	    (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
566 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
567 				 "6088 NVMEx LS REQ: Fail DID x%06x not "
568 				 "ready for IO. Type x%x, State x%x\n",
569 				 ndlp->nlp_DID, ntype, nstate);
570 		return -ENODEV;
571 	}
572 	if (test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
573 		return -ENODEV;
574 
575 	if (!vport->phba->sli4_hba.nvmels_wq)
576 		return -ENOMEM;
577 
578 	/*
579 	 * there are two dma buf in the request, actually there is one and
580 	 * the second one is just the start address + cmd size.
581 	 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
582 	 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
583 	 * because the nvem layer owns the data bufs.
584 	 * We do not have to break these packets open, we don't care what is
585 	 * in them. And we do not have to look at the resonse data, we only
586 	 * care that we got a response. All of the caring is going to happen
587 	 * in the nvme-fc layer.
588 	 */
589 
590 	bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
591 	if (!bmp) {
592 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
593 				 "6044 NVMEx LS REQ: Could not alloc LS buf "
594 				 "for DID %x\n",
595 				 ndlp->nlp_DID);
596 		return -ENOMEM;
597 	}
598 
599 	bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
600 	if (!bmp->virt) {
601 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
602 				 "6042 NVMEx LS REQ: Could not alloc mbuf "
603 				 "for DID %x\n",
604 				 ndlp->nlp_DID);
605 		kfree(bmp);
606 		return -ENOMEM;
607 	}
608 
609 	INIT_LIST_HEAD(&bmp->list);
610 
611 	bpl = (struct ulp_bde64 *)bmp->virt;
612 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
613 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
614 	bpl->tus.f.bdeFlags = 0;
615 	bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
616 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
617 	bpl++;
618 
619 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
620 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
621 	bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
622 	bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
623 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
624 
625 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
626 			"6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
627 			"rqstlen:%d rsplen:%d %pad %pad\n",
628 			ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
629 			pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
630 			&pnvme_lsreq->rspdma);
631 
632 	ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
633 				pnvme_lsreq, gen_req_cmp, ndlp, 2,
634 				pnvme_lsreq->timeout, 0);
635 	if (ret != WQE_SUCCESS) {
636 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
637 				 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
638 				 "lsreq x%px Status %x DID %x\n",
639 				 pnvme_lsreq, ret, ndlp->nlp_DID);
640 		lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
641 		kfree(bmp);
642 		return -EIO;
643 	}
644 
645 	return 0;
646 }
647 
648 /**
649  * lpfc_nvme_ls_req - Issue an NVME Link Service request
650  * @pnvme_lport: Transport localport that LS is to be issued from.
651  * @pnvme_rport: Transport remoteport that LS is to be sent to.
652  * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
653  *
654  * Driver registers this routine to handle any link service request
655  * from the nvme_fc transport to a remote nvme-aware port.
656  *
657  * Return value :
658  *   0 - Success
659  *   non-zero: various error codes, in form of -Exxx
660  **/
661 static int
lpfc_nvme_ls_req(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)662 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
663 		 struct nvme_fc_remote_port *pnvme_rport,
664 		 struct nvmefc_ls_req *pnvme_lsreq)
665 {
666 	struct lpfc_nvme_lport *lport;
667 	struct lpfc_nvme_rport *rport;
668 	struct lpfc_vport *vport;
669 	int ret;
670 
671 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
672 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
673 	if (unlikely(!lport) || unlikely(!rport))
674 		return -EINVAL;
675 
676 	vport = lport->vport;
677 	if (test_bit(FC_UNLOADING, &vport->load_flag) ||
678 	    test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
679 		return -ENODEV;
680 
681 	atomic_inc(&lport->fc4NvmeLsRequests);
682 
683 	ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
684 				 lpfc_nvme_ls_req_cmp);
685 	if (ret)
686 		atomic_inc(&lport->xmt_ls_err);
687 
688 	return ret;
689 }
690 
691 /**
692  * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
693  *         NVME LS request
694  * @vport: The local port that issued the LS
695  * @ndlp: The remote port the LS was sent to
696  * @pnvme_lsreq: Pointer to LS request structure from the transport
697  *
698  * The driver validates the ndlp, looks for the LS, and aborts the
699  * LS if found.
700  *
701  * Returns:
702  * 0 : if LS found and aborted
703  * non-zero: various error conditions in form -Exxx
704  **/
705 int
__lpfc_nvme_ls_abort(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq)706 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
707 			struct nvmefc_ls_req *pnvme_lsreq)
708 {
709 	struct lpfc_hba *phba = vport->phba;
710 	struct lpfc_sli_ring *pring;
711 	struct lpfc_iocbq *wqe, *next_wqe;
712 	bool foundit = false;
713 
714 	if (!ndlp) {
715 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
716 				"6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
717 				"x%06x, Failing LS Req\n",
718 				ndlp, ndlp ? ndlp->nlp_DID : 0);
719 		return -EINVAL;
720 	}
721 
722 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
723 			 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
724 			 "x%px rqstlen:%d rsplen:%d %pad %pad\n",
725 			 pnvme_lsreq, pnvme_lsreq->rqstlen,
726 			 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
727 			 &pnvme_lsreq->rspdma);
728 
729 	/*
730 	 * Lock the ELS ring txcmplq and look for the wqe that matches
731 	 * this ELS. If found, issue an abort on the wqe.
732 	 */
733 	pring = phba->sli4_hba.nvmels_wq->pring;
734 	spin_lock_irq(&phba->hbalock);
735 	spin_lock(&pring->ring_lock);
736 	list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
737 		if (wqe->context_un.nvme_lsreq == pnvme_lsreq) {
738 			wqe->cmd_flag |= LPFC_DRIVER_ABORTED;
739 			foundit = true;
740 			break;
741 		}
742 	}
743 	spin_unlock(&pring->ring_lock);
744 
745 	if (foundit)
746 		lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);
747 	spin_unlock_irq(&phba->hbalock);
748 
749 	if (foundit)
750 		return 0;
751 
752 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
753 			 "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",
754 			 pnvme_lsreq);
755 	return -EINVAL;
756 }
757 
758 static int
lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port * localport,struct nvme_fc_remote_port * remoteport,struct nvmefc_ls_rsp * ls_rsp)759 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
760 		     struct nvme_fc_remote_port *remoteport,
761 		     struct nvmefc_ls_rsp *ls_rsp)
762 {
763 	struct lpfc_async_xchg_ctx *axchg =
764 		container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
765 	struct lpfc_nvme_lport *lport;
766 	int rc;
767 
768 	if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag))
769 		return -ENODEV;
770 
771 	lport = (struct lpfc_nvme_lport *)localport->private;
772 
773 	rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
774 
775 	if (rc) {
776 		/*
777 		 * unless the failure is due to having already sent
778 		 * the response, an abort will be generated for the
779 		 * exchange if the rsp can't be sent.
780 		 */
781 		if (rc != -EALREADY)
782 			atomic_inc(&lport->xmt_ls_abort);
783 		return rc;
784 	}
785 
786 	return 0;
787 }
788 
789 /**
790  * lpfc_nvme_ls_abort - Abort a prior NVME LS request
791  * @pnvme_lport: Transport localport that LS is to be issued from.
792  * @pnvme_rport: Transport remoteport that LS is to be sent to.
793  * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
794  *
795  * Driver registers this routine to abort a NVME LS request that is
796  * in progress (from the transports perspective).
797  **/
798 static void
lpfc_nvme_ls_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)799 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
800 		   struct nvme_fc_remote_port *pnvme_rport,
801 		   struct nvmefc_ls_req *pnvme_lsreq)
802 {
803 	struct lpfc_nvme_lport *lport;
804 	struct lpfc_vport *vport;
805 	struct lpfc_nodelist *ndlp;
806 	int ret;
807 
808 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
809 	if (unlikely(!lport))
810 		return;
811 	vport = lport->vport;
812 
813 	if (test_bit(FC_UNLOADING, &vport->load_flag))
814 		return;
815 
816 	ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
817 
818 	ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
819 	if (!ret)
820 		atomic_inc(&lport->xmt_ls_abort);
821 }
822 
823 /* Fix up the existing sgls for NVME IO. */
824 static inline void
lpfc_nvme_adj_fcp_sgls(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct nvmefc_fcp_req * nCmd)825 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
826 		       struct lpfc_io_buf *lpfc_ncmd,
827 		       struct nvmefc_fcp_req *nCmd)
828 {
829 	struct lpfc_hba  *phba = vport->phba;
830 	struct sli4_sge *sgl;
831 	union lpfc_wqe128 *wqe;
832 	uint32_t *wptr, *dptr;
833 
834 	/*
835 	 * Get a local pointer to the built-in wqe and correct
836 	 * the cmd size to match NVME's 96 bytes and fix
837 	 * the dma address.
838 	 */
839 
840 	wqe = &lpfc_ncmd->cur_iocbq.wqe;
841 
842 	/*
843 	 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
844 	 * match NVME.  NVME sends 96 bytes. Also, use the
845 	 * nvme commands command and response dma addresses
846 	 * rather than the virtual memory to ease the restore
847 	 * operation.
848 	 */
849 	sgl = lpfc_ncmd->dma_sgl;
850 	sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
851 	if (phba->cfg_nvme_embed_cmd) {
852 		sgl->addr_hi = 0;
853 		sgl->addr_lo = 0;
854 
855 		/* Word 0-2 - NVME CMND IU (embedded payload) */
856 		wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
857 		wqe->generic.bde.tus.f.bdeSize = 56;
858 		wqe->generic.bde.addrHigh = 0;
859 		wqe->generic.bde.addrLow =  64;  /* Word 16 */
860 
861 		/* Word 10  - dbde is 0, wqes is 1 in template */
862 
863 		/*
864 		 * Embed the payload in the last half of the WQE
865 		 * WQE words 16-30 get the NVME CMD IU payload
866 		 *
867 		 * WQE words 16-19 get payload Words 1-4
868 		 * WQE words 20-21 get payload Words 6-7
869 		 * WQE words 22-29 get payload Words 16-23
870 		 */
871 		wptr = &wqe->words[16];  /* WQE ptr */
872 		dptr = (uint32_t *)nCmd->cmdaddr;  /* payload ptr */
873 		dptr++;			/* Skip Word 0 in payload */
874 
875 		*wptr++ = *dptr++;	/* Word 1 */
876 		*wptr++ = *dptr++;	/* Word 2 */
877 		*wptr++ = *dptr++;	/* Word 3 */
878 		*wptr++ = *dptr++;	/* Word 4 */
879 		dptr++;			/* Skip Word 5 in payload */
880 		*wptr++ = *dptr++;	/* Word 6 */
881 		*wptr++ = *dptr++;	/* Word 7 */
882 		dptr += 8;		/* Skip Words 8-15 in payload */
883 		*wptr++ = *dptr++;	/* Word 16 */
884 		*wptr++ = *dptr++;	/* Word 17 */
885 		*wptr++ = *dptr++;	/* Word 18 */
886 		*wptr++ = *dptr++;	/* Word 19 */
887 		*wptr++ = *dptr++;	/* Word 20 */
888 		*wptr++ = *dptr++;	/* Word 21 */
889 		*wptr++ = *dptr++;	/* Word 22 */
890 		*wptr   = *dptr;	/* Word 23 */
891 	} else {
892 		sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
893 		sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
894 
895 		/* Word 0-2 - NVME CMND IU Inline BDE */
896 		wqe->generic.bde.tus.f.bdeFlags =  BUFF_TYPE_BDE_64;
897 		wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
898 		wqe->generic.bde.addrHigh = sgl->addr_hi;
899 		wqe->generic.bde.addrLow =  sgl->addr_lo;
900 
901 		/* Word 10 */
902 		bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
903 		bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
904 	}
905 
906 	sgl++;
907 
908 	/* Setup the physical region for the FCP RSP */
909 	sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
910 	sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
911 	sgl->word2 = le32_to_cpu(sgl->word2);
912 	if (nCmd->sg_cnt)
913 		bf_set(lpfc_sli4_sge_last, sgl, 0);
914 	else
915 		bf_set(lpfc_sli4_sge_last, sgl, 1);
916 	sgl->word2 = cpu_to_le32(sgl->word2);
917 	sgl->sge_len = cpu_to_le32(nCmd->rsplen);
918 }
919 
920 
921 /*
922  * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO
923  *
924  * Driver registers this routine as it io request handler.  This
925  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
926  * data structure to the rport indicated in @lpfc_nvme_rport.
927  *
928  * Return value :
929  *   0 - Success
930  *   TODO: What are the failure codes.
931  **/
932 static void
lpfc_nvme_io_cmd_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,struct lpfc_iocbq * pwqeOut)933 lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
934 		      struct lpfc_iocbq *pwqeOut)
935 {
936 	struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;
937 	struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
938 	struct lpfc_vport *vport = pwqeIn->vport;
939 	struct nvmefc_fcp_req *nCmd;
940 	struct nvme_fc_ersp_iu *ep;
941 	struct nvme_fc_cmd_iu *cp;
942 	struct lpfc_nodelist *ndlp;
943 	struct lpfc_nvme_fcpreq_priv *freqpriv;
944 	struct lpfc_nvme_lport *lport;
945 	uint32_t code, status, idx;
946 	uint16_t cid, sqhd, data;
947 	uint32_t *ptr;
948 	uint32_t lat;
949 	bool call_done = false;
950 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
951 	int cpu;
952 #endif
953 	bool offline = false;
954 
955 	/* Sanity check on return of outstanding command */
956 	if (!lpfc_ncmd) {
957 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
958 				 "6071 Null lpfc_ncmd pointer. No "
959 				 "release, skip completion\n");
960 		return;
961 	}
962 
963 	/* Guard against abort handler being called at same time */
964 	spin_lock(&lpfc_ncmd->buf_lock);
965 
966 	if (!lpfc_ncmd->nvmeCmd) {
967 		spin_unlock(&lpfc_ncmd->buf_lock);
968 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
969 				 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
970 				 "nvmeCmd x%px\n",
971 				 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
972 
973 		/* Release the lpfc_ncmd regardless of the missing elements. */
974 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
975 		return;
976 	}
977 	nCmd = lpfc_ncmd->nvmeCmd;
978 	status = bf_get(lpfc_wcqe_c_status, wcqe);
979 
980 	idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
981 	phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
982 
983 	if (unlikely(status && vport->localport)) {
984 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
985 		if (lport) {
986 			if (bf_get(lpfc_wcqe_c_xb, wcqe))
987 				atomic_inc(&lport->cmpl_fcp_xb);
988 			atomic_inc(&lport->cmpl_fcp_err);
989 		}
990 	}
991 
992 	lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
993 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
994 			 status, wcqe->parameter);
995 	/*
996 	 * Catch race where our node has transitioned, but the
997 	 * transport is still transitioning.
998 	 */
999 	ndlp = lpfc_ncmd->ndlp;
1000 	if (!ndlp) {
1001 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1002 				 "6062 Ignoring NVME cmpl.  No ndlp\n");
1003 		goto out_err;
1004 	}
1005 
1006 	code = bf_get(lpfc_wcqe_c_code, wcqe);
1007 	if (code == CQE_CODE_NVME_ERSP) {
1008 		/* For this type of CQE, we need to rebuild the rsp */
1009 		ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1010 
1011 		/*
1012 		 * Get Command Id from cmd to plug into response. This
1013 		 * code is not needed in the next NVME Transport drop.
1014 		 */
1015 		cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1016 		cid = cp->sqe.common.command_id;
1017 
1018 		/*
1019 		 * RSN is in CQE word 2
1020 		 * SQHD is in CQE Word 3 bits 15:0
1021 		 * Cmd Specific info is in CQE Word 1
1022 		 * and in CQE Word 0 bits 15:0
1023 		 */
1024 		sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1025 
1026 		/* Now lets build the NVME ERSP IU */
1027 		ep->iu_len = cpu_to_be16(8);
1028 		ep->rsn = wcqe->parameter;
1029 		ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1030 		ep->rsvd12 = 0;
1031 		ptr = (uint32_t *)&ep->cqe.result.u64;
1032 		*ptr++ = wcqe->total_data_placed;
1033 		data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1034 		*ptr = (uint32_t)data;
1035 		ep->cqe.sq_head = sqhd;
1036 		ep->cqe.sq_id =  nCmd->sqid;
1037 		ep->cqe.command_id = cid;
1038 		ep->cqe.status = 0;
1039 
1040 		lpfc_ncmd->status = IOSTAT_SUCCESS;
1041 		lpfc_ncmd->result = 0;
1042 		nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1043 		nCmd->transferred_length = nCmd->payload_length;
1044 	} else {
1045 		lpfc_ncmd->status = status;
1046 		lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1047 
1048 		/* For NVME, the only failure path that results in an
1049 		 * IO error is when the adapter rejects it.  All other
1050 		 * conditions are a success case and resolved by the
1051 		 * transport.
1052 		 * IOSTAT_FCP_RSP_ERROR means:
1053 		 * 1. Length of data received doesn't match total
1054 		 *    transfer length in WQE
1055 		 * 2. If the RSP payload does NOT match these cases:
1056 		 *    a. RSP length 12/24 bytes and all zeros
1057 		 *    b. NVME ERSP
1058 		 */
1059 		switch (lpfc_ncmd->status) {
1060 		case IOSTAT_SUCCESS:
1061 			nCmd->transferred_length = wcqe->total_data_placed;
1062 			nCmd->rcv_rsplen = 0;
1063 			nCmd->status = 0;
1064 			break;
1065 		case IOSTAT_FCP_RSP_ERROR:
1066 			nCmd->transferred_length = wcqe->total_data_placed;
1067 			nCmd->rcv_rsplen = wcqe->parameter;
1068 			nCmd->status = 0;
1069 
1070 			/* Get the NVME cmd details for this unique error. */
1071 			cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1072 			ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1073 
1074 			/* Check if this is really an ERSP */
1075 			if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {
1076 				lpfc_ncmd->status = IOSTAT_SUCCESS;
1077 				lpfc_ncmd->result = 0;
1078 
1079 				lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1080 					"6084 NVME FCP_ERR ERSP: "
1081 					"xri %x placed x%x opcode x%x cmd_id "
1082 					"x%x cqe_status x%x\n",
1083 					lpfc_ncmd->cur_iocbq.sli4_xritag,
1084 					wcqe->total_data_placed,
1085 					cp->sqe.common.opcode,
1086 					cp->sqe.common.command_id,
1087 					ep->cqe.status);
1088 				break;
1089 			}
1090 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1091 					 "6081 NVME Completion Protocol Error: "
1092 					 "xri %x status x%x result x%x "
1093 					 "placed x%x opcode x%x cmd_id x%x, "
1094 					 "cqe_status x%x\n",
1095 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1096 					 lpfc_ncmd->status, lpfc_ncmd->result,
1097 					 wcqe->total_data_placed,
1098 					 cp->sqe.common.opcode,
1099 					 cp->sqe.common.command_id,
1100 					 ep->cqe.status);
1101 			break;
1102 		case IOSTAT_LOCAL_REJECT:
1103 			/* Let fall through to set command final state. */
1104 			if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1105 				lpfc_printf_vlog(vport, KERN_INFO,
1106 					 LOG_NVME_IOERR,
1107 					 "6032 Delay Aborted cmd x%px "
1108 					 "nvme cmd x%px, xri x%x, "
1109 					 "xb %d\n",
1110 					 lpfc_ncmd, nCmd,
1111 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1112 					 bf_get(lpfc_wcqe_c_xb, wcqe));
1113 			fallthrough;
1114 		default:
1115 out_err:
1116 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1117 					 "6072 NVME Completion Error: xri %x "
1118 					 "status x%x result x%x [x%x] "
1119 					 "placed x%x\n",
1120 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1121 					 lpfc_ncmd->status, lpfc_ncmd->result,
1122 					 wcqe->parameter,
1123 					 wcqe->total_data_placed);
1124 			nCmd->transferred_length = 0;
1125 			nCmd->rcv_rsplen = 0;
1126 			nCmd->status = NVME_SC_INTERNAL;
1127 			if (pci_channel_offline(vport->phba->pcidev) ||
1128 			    lpfc_ncmd->result == IOERR_SLI_DOWN)
1129 				offline = true;
1130 		}
1131 	}
1132 
1133 	/* pick up SLI4 exhange busy condition */
1134 	if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline)
1135 		lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1136 	else
1137 		lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1138 
1139 	/* Update stats and complete the IO.  There is
1140 	 * no need for dma unprep because the nvme_transport
1141 	 * owns the dma address.
1142 	 */
1143 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1144 	if (lpfc_ncmd->ts_cmd_start) {
1145 		lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1146 		lpfc_ncmd->ts_data_io = ktime_get_ns();
1147 		phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1148 		lpfc_io_ktime(phba, lpfc_ncmd);
1149 	}
1150 	if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1151 		cpu = raw_smp_processor_id();
1152 		this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1153 		if (lpfc_ncmd->cpu != cpu)
1154 			lpfc_printf_vlog(vport,
1155 					 KERN_INFO, LOG_NVME_IOERR,
1156 					 "6701 CPU Check cmpl: "
1157 					 "cpu %d expect %d\n",
1158 					 cpu, lpfc_ncmd->cpu);
1159 	}
1160 #endif
1161 
1162 	/* NVME targets need completion held off until the abort exchange
1163 	 * completes unless the NVME Rport is getting unregistered.
1164 	 */
1165 
1166 	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1167 		freqpriv = nCmd->private;
1168 		freqpriv->nvme_buf = NULL;
1169 		lpfc_ncmd->nvmeCmd = NULL;
1170 		call_done = true;
1171 	}
1172 	spin_unlock(&lpfc_ncmd->buf_lock);
1173 
1174 	/* Check if IO qualified for CMF */
1175 	if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1176 	    nCmd->io_dir == NVMEFC_FCP_READ &&
1177 	    nCmd->payload_length) {
1178 		/* Used when calculating average latency */
1179 		lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;
1180 		lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL);
1181 	}
1182 
1183 	if (call_done)
1184 		nCmd->done(nCmd);
1185 
1186 	/* Call release with XB=1 to queue the IO into the abort list. */
1187 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1188 }
1189 
1190 
1191 /**
1192  * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1193  * @vport: pointer to a host virtual N_Port data structure
1194  * @lpfc_ncmd: Pointer to lpfc scsi command
1195  * @pnode: pointer to a node-list data structure
1196  * @cstat: pointer to the control status structure
1197  *
1198  * Driver registers this routine as it io request handler.  This
1199  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1200  * data structure to the rport indicated in @lpfc_nvme_rport.
1201  *
1202  * Return value :
1203  *   0 - Success
1204  *   TODO: What are the failure codes.
1205  **/
1206 static int
lpfc_nvme_prep_io_cmd(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct lpfc_nodelist * pnode,struct lpfc_fc4_ctrl_stat * cstat)1207 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1208 		      struct lpfc_io_buf *lpfc_ncmd,
1209 		      struct lpfc_nodelist *pnode,
1210 		      struct lpfc_fc4_ctrl_stat *cstat)
1211 {
1212 	struct lpfc_hba *phba = vport->phba;
1213 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1214 	struct nvme_common_command *sqe;
1215 	struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;
1216 	union lpfc_wqe128 *wqe = &pwqeq->wqe;
1217 	uint32_t req_len;
1218 
1219 	/*
1220 	 * There are three possibilities here - use scatter-gather segment, use
1221 	 * the single mapping, or neither.
1222 	 */
1223 	if (nCmd->sg_cnt) {
1224 		if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1225 			/* From the iwrite template, initialize words 7 - 11 */
1226 			memcpy(&wqe->words[7],
1227 			       &lpfc_iwrite_cmd_template.words[7],
1228 			       sizeof(uint32_t) * 5);
1229 
1230 			/* Word 4 */
1231 			wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1232 
1233 			/* Word 5 */
1234 			if ((phba->cfg_nvme_enable_fb) &&
1235 			    (pnode->nlp_flag & NLP_FIRSTBURST)) {
1236 				req_len = lpfc_ncmd->nvmeCmd->payload_length;
1237 				if (req_len < pnode->nvme_fb_size)
1238 					wqe->fcp_iwrite.initial_xfer_len =
1239 						req_len;
1240 				else
1241 					wqe->fcp_iwrite.initial_xfer_len =
1242 						pnode->nvme_fb_size;
1243 			} else {
1244 				wqe->fcp_iwrite.initial_xfer_len = 0;
1245 			}
1246 			cstat->output_requests++;
1247 		} else {
1248 			/* From the iread template, initialize words 7 - 11 */
1249 			memcpy(&wqe->words[7],
1250 			       &lpfc_iread_cmd_template.words[7],
1251 			       sizeof(uint32_t) * 5);
1252 
1253 			/* Word 4 */
1254 			wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1255 
1256 			/* Word 5 */
1257 			wqe->fcp_iread.rsrvd5 = 0;
1258 
1259 			/* For a CMF Managed port, iod must be zero'ed */
1260 			if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
1261 				bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
1262 				       LPFC_WQE_IOD_NONE);
1263 			cstat->input_requests++;
1264 		}
1265 	} else {
1266 		/* From the icmnd template, initialize words 4 - 11 */
1267 		memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1268 		       sizeof(uint32_t) * 8);
1269 		cstat->control_requests++;
1270 	}
1271 
1272 	if (pnode->nlp_nvme_info & NLP_NVME_NSLER) {
1273 		bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1274 		sqe = &((struct nvme_fc_cmd_iu *)
1275 			nCmd->cmdaddr)->sqe.common;
1276 		if (sqe->opcode == nvme_admin_async_event)
1277 			bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1);
1278 	}
1279 
1280 	/*
1281 	 * Finish initializing those WQE fields that are independent
1282 	 * of the nvme_cmnd request_buffer
1283 	 */
1284 
1285 	/* Word 3 */
1286 	bf_set(payload_offset_len, &wqe->fcp_icmd,
1287 	       (nCmd->rsplen + nCmd->cmdlen));
1288 
1289 	/* Word 6 */
1290 	bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1291 	       phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1292 	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1293 
1294 	/* Word 8 */
1295 	wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1296 
1297 	/* Word 9 */
1298 	bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1299 
1300 	/* Word 10 */
1301 	bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);
1302 
1303 	/* Words 13 14 15 are for PBDE support */
1304 
1305 	/* add the VMID tags as per switch response */
1306 	if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {
1307 		if (phba->pport->vmid_priority_tagging) {
1308 			bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
1309 			bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
1310 			       lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);
1311 		} else {
1312 			bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
1313 			bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
1314 			wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;
1315 		}
1316 	}
1317 
1318 	pwqeq->vport = vport;
1319 	return 0;
1320 }
1321 
1322 
1323 /**
1324  * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1325  * @vport: pointer to a host virtual N_Port data structure
1326  * @lpfc_ncmd: Pointer to lpfc scsi command
1327  *
1328  * Driver registers this routine as it io request handler.  This
1329  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1330  * data structure to the rport indicated in @lpfc_nvme_rport.
1331  *
1332  * Return value :
1333  *   0 - Success
1334  *   TODO: What are the failure codes.
1335  **/
1336 static int
lpfc_nvme_prep_io_dma(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd)1337 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1338 		      struct lpfc_io_buf *lpfc_ncmd)
1339 {
1340 	struct lpfc_hba *phba = vport->phba;
1341 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1342 	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1343 	struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1344 	struct sli4_hybrid_sgl *sgl_xtra = NULL;
1345 	struct scatterlist *data_sg;
1346 	struct sli4_sge *first_data_sgl;
1347 	struct ulp_bde64 *bde;
1348 	dma_addr_t physaddr = 0;
1349 	uint32_t dma_len = 0;
1350 	uint32_t dma_offset = 0;
1351 	int nseg, i, j;
1352 	bool lsp_just_set = false;
1353 
1354 	/* Fix up the command and response DMA stuff. */
1355 	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1356 
1357 	/*
1358 	 * There are three possibilities here - use scatter-gather segment, use
1359 	 * the single mapping, or neither.
1360 	 */
1361 	if (nCmd->sg_cnt) {
1362 		/*
1363 		 * Jump over the cmd and rsp SGEs.  The fix routine
1364 		 * has already adjusted for this.
1365 		 */
1366 		sgl += 2;
1367 
1368 		first_data_sgl = sgl;
1369 		lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1370 		if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1371 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1372 					"6058 Too many sg segments from "
1373 					"NVME Transport.  Max %d, "
1374 					"nvmeIO sg_cnt %d\n",
1375 					phba->cfg_nvme_seg_cnt + 1,
1376 					lpfc_ncmd->seg_cnt);
1377 			lpfc_ncmd->seg_cnt = 0;
1378 			return 1;
1379 		}
1380 
1381 		/*
1382 		 * The driver established a maximum scatter-gather segment count
1383 		 * during probe that limits the number of sg elements in any
1384 		 * single nvme command.  Just run through the seg_cnt and format
1385 		 * the sge's.
1386 		 */
1387 		nseg = nCmd->sg_cnt;
1388 		data_sg = nCmd->first_sgl;
1389 
1390 		/* for tracking the segment boundaries */
1391 		j = 2;
1392 		for (i = 0; i < nseg; i++) {
1393 			if (data_sg == NULL) {
1394 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1395 						"6059 dptr err %d, nseg %d\n",
1396 						i, nseg);
1397 				lpfc_ncmd->seg_cnt = 0;
1398 				return 1;
1399 			}
1400 
1401 			sgl->word2 = 0;
1402 			if (nseg == 1) {
1403 				bf_set(lpfc_sli4_sge_last, sgl, 1);
1404 				bf_set(lpfc_sli4_sge_type, sgl,
1405 				       LPFC_SGE_TYPE_DATA);
1406 			} else {
1407 				bf_set(lpfc_sli4_sge_last, sgl, 0);
1408 
1409 				/* expand the segment */
1410 				if (!lsp_just_set &&
1411 				    !((j + 1) % phba->border_sge_num) &&
1412 				    ((nseg - 1) != i)) {
1413 					/* set LSP type */
1414 					bf_set(lpfc_sli4_sge_type, sgl,
1415 					       LPFC_SGE_TYPE_LSP);
1416 
1417 					sgl_xtra = lpfc_get_sgl_per_hdwq(
1418 							phba, lpfc_ncmd);
1419 
1420 					if (unlikely(!sgl_xtra)) {
1421 						lpfc_ncmd->seg_cnt = 0;
1422 						return 1;
1423 					}
1424 					sgl->addr_lo = cpu_to_le32(putPaddrLow(
1425 						       sgl_xtra->dma_phys_sgl));
1426 					sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1427 						       sgl_xtra->dma_phys_sgl));
1428 
1429 				} else {
1430 					bf_set(lpfc_sli4_sge_type, sgl,
1431 					       LPFC_SGE_TYPE_DATA);
1432 				}
1433 			}
1434 
1435 			if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1436 				     LPFC_SGE_TYPE_LSP)) {
1437 				if ((nseg - 1) == i)
1438 					bf_set(lpfc_sli4_sge_last, sgl, 1);
1439 
1440 				physaddr = sg_dma_address(data_sg);
1441 				dma_len = sg_dma_len(data_sg);
1442 				sgl->addr_lo = cpu_to_le32(
1443 							 putPaddrLow(physaddr));
1444 				sgl->addr_hi = cpu_to_le32(
1445 							putPaddrHigh(physaddr));
1446 
1447 				bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1448 				sgl->word2 = cpu_to_le32(sgl->word2);
1449 				sgl->sge_len = cpu_to_le32(dma_len);
1450 
1451 				dma_offset += dma_len;
1452 				data_sg = sg_next(data_sg);
1453 
1454 				sgl++;
1455 
1456 				lsp_just_set = false;
1457 			} else {
1458 				sgl->word2 = cpu_to_le32(sgl->word2);
1459 
1460 				sgl->sge_len = cpu_to_le32(
1461 						     phba->cfg_sg_dma_buf_size);
1462 
1463 				sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1464 				i = i - 1;
1465 
1466 				lsp_just_set = true;
1467 			}
1468 
1469 			j++;
1470 		}
1471 
1472 		/* PBDE support for first data SGE only */
1473 		if (nseg == 1 && phba->cfg_enable_pbde) {
1474 			/* Words 13-15 */
1475 			bde = (struct ulp_bde64 *)
1476 				&wqe->words[13];
1477 			bde->addrLow = first_data_sgl->addr_lo;
1478 			bde->addrHigh = first_data_sgl->addr_hi;
1479 			bde->tus.f.bdeSize =
1480 				le32_to_cpu(first_data_sgl->sge_len);
1481 			bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1482 			bde->tus.w = cpu_to_le32(bde->tus.w);
1483 
1484 			/* Word 11 - set PBDE bit */
1485 			bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
1486 		} else {
1487 			memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1488 			/* Word 11 - PBDE bit disabled by default template */
1489 		}
1490 
1491 	} else {
1492 		lpfc_ncmd->seg_cnt = 0;
1493 
1494 		/* For this clause to be valid, the payload_length
1495 		 * and sg_cnt must zero.
1496 		 */
1497 		if (nCmd->payload_length != 0) {
1498 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1499 					"6063 NVME DMA Prep Err: sg_cnt %d "
1500 					"payload_length x%x\n",
1501 					nCmd->sg_cnt, nCmd->payload_length);
1502 			return 1;
1503 		}
1504 	}
1505 	return 0;
1506 }
1507 
1508 /**
1509  * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1510  * @pnvme_lport: Pointer to the driver's local port data
1511  * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1512  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1513  * @pnvme_fcreq: IO request from nvme fc to driver.
1514  *
1515  * Driver registers this routine as it io request handler.  This
1516  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1517  * data structure to the rport indicated in @lpfc_nvme_rport.
1518  *
1519  * Return value :
1520  *   0 - Success
1521  *   TODO: What are the failure codes.
1522  **/
1523 static int
lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1524 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1525 			struct nvme_fc_remote_port *pnvme_rport,
1526 			void *hw_queue_handle,
1527 			struct nvmefc_fcp_req *pnvme_fcreq)
1528 {
1529 	int ret = 0;
1530 	int expedite = 0;
1531 	int idx, cpu;
1532 	struct lpfc_nvme_lport *lport;
1533 	struct lpfc_fc4_ctrl_stat *cstat;
1534 	struct lpfc_vport *vport;
1535 	struct lpfc_hba *phba;
1536 	struct lpfc_nodelist *ndlp;
1537 	struct lpfc_io_buf *lpfc_ncmd;
1538 	struct lpfc_nvme_rport *rport;
1539 	struct lpfc_nvme_qhandle *lpfc_queue_info;
1540 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1541 	struct nvme_common_command *sqe;
1542 	uint64_t start = 0;
1543 #if (IS_ENABLED(CONFIG_NVME_FC))
1544 	u8 *uuid = NULL;
1545 	int err;
1546 	enum dma_data_direction iodir;
1547 #endif
1548 
1549 	/* Validate pointers. LLDD fault handling with transport does
1550 	 * have timing races.
1551 	 */
1552 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1553 	if (unlikely(!lport)) {
1554 		ret = -EINVAL;
1555 		goto out_fail;
1556 	}
1557 
1558 	vport = lport->vport;
1559 
1560 	if (unlikely(!hw_queue_handle)) {
1561 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1562 				 "6117 Fail IO, NULL hw_queue_handle\n");
1563 		atomic_inc(&lport->xmt_fcp_err);
1564 		ret = -EBUSY;
1565 		goto out_fail;
1566 	}
1567 
1568 	phba = vport->phba;
1569 
1570 	if ((unlikely(test_bit(FC_UNLOADING, &vport->load_flag))) ||
1571 	    test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
1572 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1573 				 "6124 Fail IO, Driver unload\n");
1574 		atomic_inc(&lport->xmt_fcp_err);
1575 		ret = -ENODEV;
1576 		goto out_fail;
1577 	}
1578 
1579 	freqpriv = pnvme_fcreq->private;
1580 	if (unlikely(!freqpriv)) {
1581 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1582 				 "6158 Fail IO, NULL request data\n");
1583 		atomic_inc(&lport->xmt_fcp_err);
1584 		ret = -EINVAL;
1585 		goto out_fail;
1586 	}
1587 
1588 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1589 	if (phba->ktime_on)
1590 		start = ktime_get_ns();
1591 #endif
1592 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1593 	lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1594 
1595 	/*
1596 	 * Catch race where our node has transitioned, but the
1597 	 * transport is still transitioning.
1598 	 */
1599 	ndlp = rport->ndlp;
1600 	if (!ndlp) {
1601 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1602 				 "6053 Busy IO, ndlp not ready: rport x%px "
1603 				  "ndlp x%px, DID x%06x\n",
1604 				 rport, ndlp, pnvme_rport->port_id);
1605 		atomic_inc(&lport->xmt_fcp_err);
1606 		ret = -EBUSY;
1607 		goto out_fail;
1608 	}
1609 
1610 	/* The remote node has to be a mapped target or it's an error. */
1611 	if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1612 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1613 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1614 				 "6036 Fail IO, DID x%06x not ready for "
1615 				 "IO. State x%x, Type x%x Flg x%x\n",
1616 				 pnvme_rport->port_id,
1617 				 ndlp->nlp_state, ndlp->nlp_type,
1618 				 ndlp->fc4_xpt_flags);
1619 		atomic_inc(&lport->xmt_fcp_bad_ndlp);
1620 		ret = -EBUSY;
1621 		goto out_fail;
1622 
1623 	}
1624 
1625 	/* Currently only NVME Keep alive commands should be expedited
1626 	 * if the driver runs out of a resource. These should only be
1627 	 * issued on the admin queue, qidx 0
1628 	 */
1629 	if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1630 		sqe = &((struct nvme_fc_cmd_iu *)
1631 			pnvme_fcreq->cmdaddr)->sqe.common;
1632 		if (sqe->opcode == nvme_admin_keep_alive)
1633 			expedite = 1;
1634 	}
1635 
1636 	/* Check if IO qualifies for CMF */
1637 	if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1638 	    pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&
1639 	    pnvme_fcreq->payload_length) {
1640 		ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length);
1641 		if (ret) {
1642 			ret = -EBUSY;
1643 			goto out_fail;
1644 		}
1645 		/* Get start time for IO latency */
1646 		start = ktime_get_ns();
1647 	}
1648 
1649 	/* The node is shared with FCP IO, make sure the IO pending count does
1650 	 * not exceed the programmed depth.
1651 	 */
1652 	if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1653 		if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1654 		    !expedite) {
1655 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1656 					 "6174 Fail IO, ndlp qdepth exceeded: "
1657 					 "idx %d DID %x pend %d qdepth %d\n",
1658 					 lpfc_queue_info->index, ndlp->nlp_DID,
1659 					 atomic_read(&ndlp->cmd_pending),
1660 					 ndlp->cmd_qdepth);
1661 			atomic_inc(&lport->xmt_fcp_qdepth);
1662 			ret = -EBUSY;
1663 			goto out_fail1;
1664 		}
1665 	}
1666 
1667 	/* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1668 	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1669 		idx = lpfc_queue_info->index;
1670 	} else {
1671 		cpu = raw_smp_processor_id();
1672 		idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1673 	}
1674 
1675 	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1676 	if (lpfc_ncmd == NULL) {
1677 		atomic_inc(&lport->xmt_fcp_noxri);
1678 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1679 				 "6065 Fail IO, driver buffer pool is empty: "
1680 				 "idx %d DID %x\n",
1681 				 lpfc_queue_info->index, ndlp->nlp_DID);
1682 		ret = -EBUSY;
1683 		goto out_fail1;
1684 	}
1685 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1686 	if (start) {
1687 		lpfc_ncmd->ts_cmd_start = start;
1688 		lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1689 	} else {
1690 		lpfc_ncmd->ts_cmd_start = 0;
1691 	}
1692 #endif
1693 	lpfc_ncmd->rx_cmd_start = start;
1694 
1695 	/*
1696 	 * Store the data needed by the driver to issue, abort, and complete
1697 	 * an IO.
1698 	 * Do not let the IO hang out forever.  There is no midlayer issuing
1699 	 * an abort so inform the FW of the maximum IO pending time.
1700 	 */
1701 	freqpriv->nvme_buf = lpfc_ncmd;
1702 	lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1703 	lpfc_ncmd->ndlp = ndlp;
1704 	lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1705 
1706 #if (IS_ENABLED(CONFIG_NVME_FC))
1707 	/* check the necessary and sufficient condition to support VMID */
1708 	if (lpfc_is_vmid_enabled(phba) &&
1709 	    (ndlp->vmid_support ||
1710 	     phba->pport->vmid_priority_tagging ==
1711 	     LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
1712 		/* is the I/O generated by a VM, get the associated virtual */
1713 		/* entity id */
1714 		uuid = nvme_fc_io_getuuid(pnvme_fcreq);
1715 
1716 		if (uuid) {
1717 			if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE)
1718 				iodir = DMA_TO_DEVICE;
1719 			else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ)
1720 				iodir = DMA_FROM_DEVICE;
1721 			else
1722 				iodir = DMA_NONE;
1723 
1724 			err = lpfc_vmid_get_appid(vport, uuid, iodir,
1725 					(union lpfc_vmid_io_tag *)
1726 						&lpfc_ncmd->cur_iocbq.vmid_tag);
1727 			if (!err)
1728 				lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;
1729 		}
1730 	}
1731 #endif
1732 
1733 	/*
1734 	 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1735 	 * This identfier was create in our hardware queue create callback
1736 	 * routine. The driver now is dependent on the IO queue steering from
1737 	 * the transport.  We are trusting the upper NVME layers know which
1738 	 * index to use and that they have affinitized a CPU to this hardware
1739 	 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1740 	 */
1741 	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1742 	cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1743 
1744 	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1745 	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1746 	if (ret) {
1747 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1748 				 "6175 Fail IO, Prep DMA: "
1749 				 "idx %d DID %x\n",
1750 				 lpfc_queue_info->index, ndlp->nlp_DID);
1751 		atomic_inc(&lport->xmt_fcp_err);
1752 		ret = -ENOMEM;
1753 		goto out_free_nvme_buf;
1754 	}
1755 
1756 	lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1757 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
1758 			 lpfc_queue_info->index, ndlp->nlp_DID);
1759 
1760 	ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1761 	if (ret) {
1762 		atomic_inc(&lport->xmt_fcp_wqerr);
1763 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1764 				 "6113 Fail IO, Could not issue WQE err %x "
1765 				 "sid: x%x did: x%x oxid: x%x\n",
1766 				 ret, vport->fc_myDID, ndlp->nlp_DID,
1767 				 lpfc_ncmd->cur_iocbq.sli4_xritag);
1768 		goto out_free_nvme_buf;
1769 	}
1770 
1771 	if (phba->cfg_xri_rebalancing)
1772 		lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1773 
1774 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1775 	if (lpfc_ncmd->ts_cmd_start)
1776 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1777 
1778 	if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1779 		cpu = raw_smp_processor_id();
1780 		this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1781 		lpfc_ncmd->cpu = cpu;
1782 		if (idx != cpu)
1783 			lpfc_printf_vlog(vport,
1784 					 KERN_INFO, LOG_NVME_IOERR,
1785 					"6702 CPU Check cmd: "
1786 					"cpu %d wq %d\n",
1787 					lpfc_ncmd->cpu,
1788 					lpfc_queue_info->index);
1789 	}
1790 #endif
1791 	return 0;
1792 
1793  out_free_nvme_buf:
1794 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1795 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1796 			cstat->output_requests--;
1797 		else
1798 			cstat->input_requests--;
1799 	} else
1800 		cstat->control_requests--;
1801 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1802  out_fail1:
1803 	lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,
1804 			     pnvme_fcreq->payload_length, NULL);
1805  out_fail:
1806 	return ret;
1807 }
1808 
1809 /**
1810  * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1811  * @phba: Pointer to HBA context object
1812  * @cmdiocb: Pointer to command iocb object.
1813  * @rspiocb: Pointer to response iocb object.
1814  *
1815  * This is the callback function for any NVME FCP IO that was aborted.
1816  *
1817  * Return value:
1818  *   None
1819  **/
1820 void
lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * cmdiocb,struct lpfc_iocbq * rspiocb)1821 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1822 			   struct lpfc_iocbq *rspiocb)
1823 {
1824 	struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl;
1825 
1826 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1827 			"6145 ABORT_XRI_CN completing on rpi x%x "
1828 			"original iotag x%x, abort cmd iotag x%x "
1829 			"req_tag x%x, status x%x, hwstatus x%x\n",
1830 			bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com),
1831 			get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag,
1832 			bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1833 			bf_get(lpfc_wcqe_c_status, abts_cmpl),
1834 			bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1835 	lpfc_sli_release_iocbq(phba, cmdiocb);
1836 }
1837 
1838 /**
1839  * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1840  * @pnvme_lport: Pointer to the driver's local port data
1841  * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1842  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1843  * @pnvme_fcreq: IO request from nvme fc to driver.
1844  *
1845  * Driver registers this routine as its nvme request io abort handler.  This
1846  * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1847  * data structure to the rport indicated in @lpfc_nvme_rport.  This routine
1848  * is executed asynchronously - one the target is validated as "MAPPED" and
1849  * ready for IO, the driver issues the abort request and returns.
1850  *
1851  * Return value:
1852  *   None
1853  **/
1854 static void
lpfc_nvme_fcp_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1855 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1856 		    struct nvme_fc_remote_port *pnvme_rport,
1857 		    void *hw_queue_handle,
1858 		    struct nvmefc_fcp_req *pnvme_fcreq)
1859 {
1860 	struct lpfc_nvme_lport *lport;
1861 	struct lpfc_vport *vport;
1862 	struct lpfc_hba *phba;
1863 	struct lpfc_io_buf *lpfc_nbuf;
1864 	struct lpfc_iocbq *nvmereq_wqe;
1865 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1866 	unsigned long flags;
1867 	int ret_val;
1868 
1869 	/* Validate pointers. LLDD fault handling with transport does
1870 	 * have timing races.
1871 	 */
1872 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1873 	if (unlikely(!lport))
1874 		return;
1875 
1876 	vport = lport->vport;
1877 
1878 	if (unlikely(!hw_queue_handle)) {
1879 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1880 				 "6129 Fail Abort, HW Queue Handle NULL.\n");
1881 		return;
1882 	}
1883 
1884 	phba = vport->phba;
1885 	freqpriv = pnvme_fcreq->private;
1886 
1887 	if (unlikely(!freqpriv))
1888 		return;
1889 	if (test_bit(FC_UNLOADING, &vport->load_flag))
1890 		return;
1891 
1892 	/* Announce entry to new IO submit field. */
1893 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1894 			 "6002 Abort Request to rport DID x%06x "
1895 			 "for nvme_fc_req x%px\n",
1896 			 pnvme_rport->port_id,
1897 			 pnvme_fcreq);
1898 
1899 	lpfc_nbuf = freqpriv->nvme_buf;
1900 	if (!lpfc_nbuf) {
1901 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1902 				 "6140 NVME IO req has no matching lpfc nvme "
1903 				 "io buffer.  Skipping abort req.\n");
1904 		return;
1905 	} else if (!lpfc_nbuf->nvmeCmd) {
1906 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1907 				 "6141 lpfc NVME IO req has no nvme_fcreq "
1908 				 "io buffer.  Skipping abort req.\n");
1909 		return;
1910 	}
1911 
1912 	/* driver queued commands are in process of being flushed */
1913 	if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
1914 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1915 				 "6139 Driver in reset cleanup - flushing "
1916 				 "NVME Req now.  hba_flag x%lx\n",
1917 				 phba->hba_flag);
1918 		return;
1919 	}
1920 
1921 	/* Guard against IO completion being called at same time */
1922 	spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags);
1923 	spin_lock(&phba->hbalock);
1924 
1925 	nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1926 
1927 	/*
1928 	 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1929 	 * state must match the nvme_fcreq passed by the nvme
1930 	 * transport.  If they don't match, it is likely the driver
1931 	 * has already completed the NVME IO and the nvme transport
1932 	 * has not seen it yet.
1933 	 */
1934 	if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1935 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1936 				 "6143 NVME req mismatch: "
1937 				 "lpfc_nbuf x%px nvmeCmd x%px, "
1938 				 "pnvme_fcreq x%px.  Skipping Abort xri x%x\n",
1939 				 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1940 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1941 		goto out_unlock;
1942 	}
1943 
1944 	/* Don't abort IOs no longer on the pending queue. */
1945 	if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
1946 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1947 				 "6142 NVME IO req x%px not queued - skipping "
1948 				 "abort req xri x%x\n",
1949 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1950 		goto out_unlock;
1951 	}
1952 
1953 	atomic_inc(&lport->xmt_fcp_abort);
1954 	lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1955 			 nvmereq_wqe->sli4_xritag,
1956 			 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1957 
1958 	/* Outstanding abort is in progress */
1959 	if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) {
1960 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1961 				 "6144 Outstanding NVME I/O Abort Request "
1962 				 "still pending on nvme_fcreq x%px, "
1963 				 "lpfc_ncmd x%px xri x%x\n",
1964 				 pnvme_fcreq, lpfc_nbuf,
1965 				 nvmereq_wqe->sli4_xritag);
1966 		goto out_unlock;
1967 	}
1968 
1969 	ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe,
1970 					      lpfc_nvme_abort_fcreq_cmpl);
1971 
1972 	spin_unlock(&phba->hbalock);
1973 	spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1974 
1975 	/* Make sure HBA is alive */
1976 	lpfc_issue_hb_tmo(phba);
1977 
1978 	if (ret_val != WQE_SUCCESS) {
1979 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1980 				 "6137 Failed abts issue_wqe with status x%x "
1981 				 "for nvme_fcreq x%px.\n",
1982 				 ret_val, pnvme_fcreq);
1983 		return;
1984 	}
1985 
1986 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1987 			 "6138 Transport Abort NVME Request Issued for "
1988 			 "ox_id x%x\n",
1989 			 nvmereq_wqe->sli4_xritag);
1990 	return;
1991 
1992 out_unlock:
1993 	spin_unlock(&phba->hbalock);
1994 	spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1995 	return;
1996 }
1997 
1998 /* Declare and initialization an instance of the FC NVME template. */
1999 static struct nvme_fc_port_template lpfc_nvme_template = {
2000 	/* initiator-based functions */
2001 	.localport_delete  = lpfc_nvme_localport_delete,
2002 	.remoteport_delete = lpfc_nvme_remoteport_delete,
2003 	.create_queue = lpfc_nvme_create_queue,
2004 	.delete_queue = lpfc_nvme_delete_queue,
2005 	.ls_req       = lpfc_nvme_ls_req,
2006 	.fcp_io       = lpfc_nvme_fcp_io_submit,
2007 	.ls_abort     = lpfc_nvme_ls_abort,
2008 	.fcp_abort    = lpfc_nvme_fcp_abort,
2009 	.xmt_ls_rsp   = lpfc_nvme_xmt_ls_rsp,
2010 
2011 	.max_hw_queues = 1,
2012 	.max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2013 	.max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2014 	.dma_boundary = 0xFFFFFFFF,
2015 
2016 	/* Sizes of additional private data for data structures.
2017 	 * No use for the last two sizes at this time.
2018 	 */
2019 	.local_priv_sz = sizeof(struct lpfc_nvme_lport),
2020 	.remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2021 	.lsrqst_priv_sz = 0,
2022 	.fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2023 };
2024 
2025 /*
2026  * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2027  *
2028  * This routine removes a nvme buffer from head of @hdwq io_buf_list
2029  * and returns to caller.
2030  *
2031  * Return codes:
2032  *   NULL - Error
2033  *   Pointer to lpfc_nvme_buf - Success
2034  **/
2035 static struct lpfc_io_buf *
lpfc_get_nvme_buf(struct lpfc_hba * phba,struct lpfc_nodelist * ndlp,int idx,int expedite)2036 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2037 		  int idx, int expedite)
2038 {
2039 	struct lpfc_io_buf *lpfc_ncmd;
2040 	struct lpfc_sli4_hdw_queue *qp;
2041 	struct sli4_sge *sgl;
2042 	struct lpfc_iocbq *pwqeq;
2043 	union lpfc_wqe128 *wqe;
2044 
2045 	lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2046 
2047 	if (lpfc_ncmd) {
2048 		pwqeq = &(lpfc_ncmd->cur_iocbq);
2049 		wqe = &pwqeq->wqe;
2050 
2051 		/* Setup key fields in buffer that may have been changed
2052 		 * if other protocols used this buffer.
2053 		 */
2054 		pwqeq->cmd_flag = LPFC_IO_NVME;
2055 		pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl;
2056 		lpfc_ncmd->start_time = jiffies;
2057 		lpfc_ncmd->flags = 0;
2058 
2059 		/* Rsp SGE will be filled in when we rcv an IO
2060 		 * from the NVME Layer to be sent.
2061 		 * The cmd is going to be embedded so we need a SKIP SGE.
2062 		 */
2063 		sgl = lpfc_ncmd->dma_sgl;
2064 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2065 		bf_set(lpfc_sli4_sge_last, sgl, 0);
2066 		sgl->word2 = cpu_to_le32(sgl->word2);
2067 		/* Fill in word 3 / sgl_len during cmd submission */
2068 
2069 		/* Initialize 64 bytes only */
2070 		memset(wqe, 0, sizeof(union lpfc_wqe));
2071 
2072 		if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2073 			atomic_inc(&ndlp->cmd_pending);
2074 			lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2075 		}
2076 
2077 	} else {
2078 		qp = &phba->sli4_hba.hdwq[idx];
2079 		qp->empty_io_bufs++;
2080 	}
2081 
2082 	return  lpfc_ncmd;
2083 }
2084 
2085 /**
2086  * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2087  * @phba: The Hba for which this call is being executed.
2088  * @lpfc_ncmd: The nvme buffer which is being released.
2089  *
2090  * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2091  * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2092  * and cannot be reused for at least RA_TOV amount of time if it was
2093  * aborted.
2094  **/
2095 static void
lpfc_release_nvme_buf(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2096 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2097 {
2098 	struct lpfc_sli4_hdw_queue *qp;
2099 	unsigned long iflag = 0;
2100 
2101 	if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2102 		atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2103 
2104 	lpfc_ncmd->ndlp = NULL;
2105 	lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2106 
2107 	qp = lpfc_ncmd->hdwq;
2108 	if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2109 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2110 				"6310 XB release deferred for "
2111 				"ox_id x%x on reqtag x%x\n",
2112 				lpfc_ncmd->cur_iocbq.sli4_xritag,
2113 				lpfc_ncmd->cur_iocbq.iotag);
2114 
2115 		spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2116 		list_add_tail(&lpfc_ncmd->list,
2117 			&qp->lpfc_abts_io_buf_list);
2118 		qp->abts_nvme_io_bufs++;
2119 		spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2120 	} else
2121 		lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2122 }
2123 
2124 /**
2125  * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2126  * @vport: the lpfc_vport instance requesting a localport.
2127  *
2128  * This routine is invoked to create an nvme localport instance to bind
2129  * to the nvme_fc_transport.  It is called once during driver load
2130  * like lpfc_create_shost after all other services are initialized.
2131  * It requires a vport, vpi, and wwns at call time.  Other localport
2132  * parameters are modified as the driver's FCID and the Fabric WWN
2133  * are established.
2134  *
2135  * Return codes
2136  *      0 - successful
2137  *      -ENOMEM - no heap memory available
2138  *      other values - from nvme registration upcall
2139  **/
2140 int
lpfc_nvme_create_localport(struct lpfc_vport * vport)2141 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2142 {
2143 	int ret = 0;
2144 	struct lpfc_hba  *phba = vport->phba;
2145 	struct nvme_fc_port_info nfcp_info;
2146 	struct nvme_fc_local_port *localport;
2147 	struct lpfc_nvme_lport *lport;
2148 
2149 	/* Initialize this localport instance.  The vport wwn usage ensures
2150 	 * that NPIV is accounted for.
2151 	 */
2152 	memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2153 	nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2154 	nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2155 	nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2156 
2157 	/* We need to tell the transport layer + 1 because it takes page
2158 	 * alignment into account. When space for the SGL is allocated we
2159 	 * allocate + 3, one for cmd, one for rsp and one for this alignment
2160 	 */
2161 	lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2162 
2163 	/* Advertise how many hw queues we support based on cfg_hdw_queue,
2164 	 * which will not exceed cpu count.
2165 	 */
2166 	lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2167 
2168 	if (!IS_ENABLED(CONFIG_NVME_FC))
2169 		return ret;
2170 
2171 	/* localport is allocated from the stack, but the registration
2172 	 * call allocates heap memory as well as the private area.
2173 	 */
2174 
2175 	ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2176 					 &vport->phba->pcidev->dev, &localport);
2177 	if (!ret) {
2178 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2179 				 "6005 Successfully registered local "
2180 				 "NVME port num %d, localP x%px, private "
2181 				 "x%px, sg_seg %d\n",
2182 				 localport->port_num, localport,
2183 				 localport->private,
2184 				 lpfc_nvme_template.max_sgl_segments);
2185 
2186 		/* Private is our lport size declared in the template. */
2187 		lport = (struct lpfc_nvme_lport *)localport->private;
2188 		vport->localport = localport;
2189 		lport->vport = vport;
2190 		vport->nvmei_support = 1;
2191 
2192 		atomic_set(&lport->xmt_fcp_noxri, 0);
2193 		atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2194 		atomic_set(&lport->xmt_fcp_qdepth, 0);
2195 		atomic_set(&lport->xmt_fcp_err, 0);
2196 		atomic_set(&lport->xmt_fcp_wqerr, 0);
2197 		atomic_set(&lport->xmt_fcp_abort, 0);
2198 		atomic_set(&lport->xmt_ls_abort, 0);
2199 		atomic_set(&lport->xmt_ls_err, 0);
2200 		atomic_set(&lport->cmpl_fcp_xb, 0);
2201 		atomic_set(&lport->cmpl_fcp_err, 0);
2202 		atomic_set(&lport->cmpl_ls_xb, 0);
2203 		atomic_set(&lport->cmpl_ls_err, 0);
2204 
2205 		atomic_set(&lport->fc4NvmeLsRequests, 0);
2206 		atomic_set(&lport->fc4NvmeLsCmpls, 0);
2207 	}
2208 
2209 	return ret;
2210 }
2211 
2212 #if (IS_ENABLED(CONFIG_NVME_FC))
2213 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2214  *
2215  * The driver has to wait for the host nvme transport to callback
2216  * indicating the localport has successfully unregistered all
2217  * resources.  Since this is an uninterruptible wait, loop every ten
2218  * seconds and print a message indicating no progress.
2219  *
2220  * An uninterruptible wait is used because of the risk of transport-to-
2221  * driver state mismatch.
2222  */
2223 static void
lpfc_nvme_lport_unreg_wait(struct lpfc_vport * vport,struct lpfc_nvme_lport * lport,struct completion * lport_unreg_cmp)2224 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2225 			   struct lpfc_nvme_lport *lport,
2226 			   struct completion *lport_unreg_cmp)
2227 {
2228 	u32 wait_tmo;
2229 	int ret, i, pending = 0;
2230 	struct lpfc_sli_ring  *pring;
2231 	struct lpfc_hba  *phba = vport->phba;
2232 	struct lpfc_sli4_hdw_queue *qp;
2233 	int abts_scsi, abts_nvme;
2234 
2235 	/* Host transport has to clean up and confirm requiring an indefinite
2236 	 * wait. Print a message if a 10 second wait expires and renew the
2237 	 * wait. This is unexpected.
2238 	 */
2239 	wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2240 	while (true) {
2241 		ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2242 		if (unlikely(!ret)) {
2243 			pending = 0;
2244 			abts_scsi = 0;
2245 			abts_nvme = 0;
2246 			for (i = 0; i < phba->cfg_hdw_queue; i++) {
2247 				qp = &phba->sli4_hba.hdwq[i];
2248 				if (!vport->localport || !qp || !qp->io_wq)
2249 					return;
2250 
2251 				pring = qp->io_wq->pring;
2252 				if (!pring)
2253 					continue;
2254 				pending += pring->txcmplq_cnt;
2255 				abts_scsi += qp->abts_scsi_io_bufs;
2256 				abts_nvme += qp->abts_nvme_io_bufs;
2257 			}
2258 			if (!vport->localport ||
2259 			    test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
2260 			    phba->link_state == LPFC_HBA_ERROR ||
2261 			    test_bit(FC_UNLOADING, &vport->load_flag))
2262 				return;
2263 
2264 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2265 					 "6176 Lport x%px Localport x%px wait "
2266 					 "timed out. Pending %d [%d:%d]. "
2267 					 "Renewing.\n",
2268 					 lport, vport->localport, pending,
2269 					 abts_scsi, abts_nvme);
2270 			continue;
2271 		}
2272 		break;
2273 	}
2274 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2275 			 "6177 Lport x%px Localport x%px Complete Success\n",
2276 			 lport, vport->localport);
2277 }
2278 #endif
2279 
2280 /**
2281  * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2282  * @vport: pointer to a host virtual N_Port data structure
2283  *
2284  * This routine is invoked to destroy all lports bound to the phba.
2285  * The lport memory was allocated by the nvme fc transport and is
2286  * released there.  This routine ensures all rports bound to the
2287  * lport have been disconnected.
2288  *
2289  **/
2290 void
lpfc_nvme_destroy_localport(struct lpfc_vport * vport)2291 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2292 {
2293 #if (IS_ENABLED(CONFIG_NVME_FC))
2294 	struct nvme_fc_local_port *localport;
2295 	struct lpfc_nvme_lport *lport;
2296 	int ret;
2297 	DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2298 
2299 	if (vport->nvmei_support == 0)
2300 		return;
2301 
2302 	localport = vport->localport;
2303 	if (!localport)
2304 		return;
2305 	lport = (struct lpfc_nvme_lport *)localport->private;
2306 
2307 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2308 			 "6011 Destroying NVME localport x%px\n",
2309 			 localport);
2310 
2311 	/* lport's rport list is clear.  Unregister
2312 	 * lport and release resources.
2313 	 */
2314 	lport->lport_unreg_cmp = &lport_unreg_cmp;
2315 	ret = nvme_fc_unregister_localport(localport);
2316 
2317 	/* Wait for completion.  This either blocks
2318 	 * indefinitely or succeeds
2319 	 */
2320 	lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2321 	vport->localport = NULL;
2322 
2323 	/* Regardless of the unregister upcall response, clear
2324 	 * nvmei_support.  All rports are unregistered and the
2325 	 * driver will clean up.
2326 	 */
2327 	vport->nvmei_support = 0;
2328 	if (ret == 0) {
2329 		lpfc_printf_vlog(vport,
2330 				 KERN_INFO, LOG_NVME_DISC,
2331 				 "6009 Unregistered lport Success\n");
2332 	} else {
2333 		lpfc_printf_vlog(vport,
2334 				 KERN_INFO, LOG_NVME_DISC,
2335 				 "6010 Unregistered lport "
2336 				 "Failed, status x%x\n",
2337 				 ret);
2338 	}
2339 #endif
2340 }
2341 
2342 void
lpfc_nvme_update_localport(struct lpfc_vport * vport)2343 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2344 {
2345 #if (IS_ENABLED(CONFIG_NVME_FC))
2346 	struct nvme_fc_local_port *localport;
2347 	struct lpfc_nvme_lport *lport;
2348 
2349 	localport = vport->localport;
2350 	if (!localport) {
2351 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2352 				 "6710 Update NVME fail. No localport\n");
2353 		return;
2354 	}
2355 	lport = (struct lpfc_nvme_lport *)localport->private;
2356 	if (!lport) {
2357 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2358 				 "6171 Update NVME fail. localP x%px, No lport\n",
2359 				 localport);
2360 		return;
2361 	}
2362 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2363 			 "6012 Update NVME lport x%px did x%x\n",
2364 			 localport, vport->fc_myDID);
2365 
2366 	localport->port_id = vport->fc_myDID;
2367 	if (localport->port_id == 0)
2368 		localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2369 	else
2370 		localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2371 
2372 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2373 			 "6030 bound lport x%px to DID x%06x\n",
2374 			 lport, localport->port_id);
2375 #endif
2376 }
2377 
2378 int
lpfc_nvme_register_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2379 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2380 {
2381 #if (IS_ENABLED(CONFIG_NVME_FC))
2382 	int ret = 0;
2383 	struct nvme_fc_local_port *localport;
2384 	struct lpfc_nvme_lport *lport;
2385 	struct lpfc_nvme_rport *rport;
2386 	struct lpfc_nvme_rport *oldrport;
2387 	struct nvme_fc_remote_port *remote_port;
2388 	struct nvme_fc_port_info rpinfo;
2389 	struct lpfc_nodelist *prev_ndlp = NULL;
2390 	struct fc_rport *srport = ndlp->rport;
2391 
2392 	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2393 			 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2394 			 ndlp->nlp_DID, ndlp->nlp_type);
2395 
2396 	localport = vport->localport;
2397 	if (!localport)
2398 		return 0;
2399 
2400 	lport = (struct lpfc_nvme_lport *)localport->private;
2401 
2402 	/* NVME rports are not preserved across devloss.
2403 	 * Just register this instance.  Note, rpinfo->dev_loss_tmo
2404 	 * is left 0 to indicate accept transport defaults.  The
2405 	 * driver communicates port role capabilities consistent
2406 	 * with the PRLI response data.
2407 	 */
2408 	memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2409 	rpinfo.port_id = ndlp->nlp_DID;
2410 	if (ndlp->nlp_type & NLP_NVME_TARGET)
2411 		rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2412 	if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2413 		rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2414 
2415 	if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2416 		rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2417 
2418 	rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2419 	rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2420 	if (srport)
2421 		rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2422 	else
2423 		rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2424 
2425 	spin_lock_irq(&ndlp->lock);
2426 
2427 	/* If an oldrport exists, so does the ndlp reference.  If not
2428 	 * a new reference is needed because either the node has never
2429 	 * been registered or it's been unregistered and getting deleted.
2430 	 */
2431 	oldrport = lpfc_ndlp_get_nrport(ndlp);
2432 	if (oldrport) {
2433 		prev_ndlp = oldrport->ndlp;
2434 		spin_unlock_irq(&ndlp->lock);
2435 	} else {
2436 		spin_unlock_irq(&ndlp->lock);
2437 		if (!lpfc_nlp_get(ndlp)) {
2438 			dev_warn(&vport->phba->pcidev->dev,
2439 				 "Warning - No node ref - exit register\n");
2440 			return 0;
2441 		}
2442 	}
2443 
2444 	ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2445 	if (!ret) {
2446 		/* If the ndlp already has an nrport, this is just
2447 		 * a resume of the existing rport.  Else this is a
2448 		 * new rport.
2449 		 */
2450 		/* Guard against an unregister/reregister
2451 		 * race that leaves the WAIT flag set.
2452 		 */
2453 		spin_lock_irq(&ndlp->lock);
2454 		ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2455 		ndlp->fc4_xpt_flags |= NVME_XPT_REGD;
2456 		spin_unlock_irq(&ndlp->lock);
2457 		rport = remote_port->private;
2458 		if (oldrport) {
2459 
2460 			/* Sever the ndlp<->rport association
2461 			 * before dropping the ndlp ref from
2462 			 * register.
2463 			 */
2464 			spin_lock_irq(&ndlp->lock);
2465 			ndlp->nrport = NULL;
2466 			ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2467 			spin_unlock_irq(&ndlp->lock);
2468 			rport->ndlp = NULL;
2469 			rport->remoteport = NULL;
2470 
2471 			/* Reference only removed if previous NDLP is no longer
2472 			 * active. It might be just a swap and removing the
2473 			 * reference would cause a premature cleanup.
2474 			 */
2475 			if (prev_ndlp && prev_ndlp != ndlp) {
2476 				if (!prev_ndlp->nrport)
2477 					lpfc_nlp_put(prev_ndlp);
2478 			}
2479 		}
2480 
2481 		/* Clean bind the rport to the ndlp. */
2482 		rport->remoteport = remote_port;
2483 		rport->lport = lport;
2484 		rport->ndlp = ndlp;
2485 		spin_lock_irq(&ndlp->lock);
2486 		ndlp->nrport = rport;
2487 		spin_unlock_irq(&ndlp->lock);
2488 		lpfc_printf_vlog(vport, KERN_INFO,
2489 				 LOG_NVME_DISC | LOG_NODE,
2490 				 "6022 Bind lport x%px to remoteport x%px "
2491 				 "rport x%px WWNN 0x%llx, "
2492 				 "Rport WWPN 0x%llx DID "
2493 				 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2494 				 lport, remote_port, rport,
2495 				 rpinfo.node_name, rpinfo.port_name,
2496 				 rpinfo.port_id, rpinfo.port_role,
2497 				 ndlp, prev_ndlp);
2498 	} else {
2499 		lpfc_printf_vlog(vport, KERN_ERR,
2500 				 LOG_TRACE_EVENT,
2501 				 "6031 RemotePort Registration failed "
2502 				 "err: %d, DID x%06x ref %u\n",
2503 				 ret, ndlp->nlp_DID, kref_read(&ndlp->kref));
2504 		lpfc_nlp_put(ndlp);
2505 	}
2506 
2507 	return ret;
2508 #else
2509 	return 0;
2510 #endif
2511 }
2512 
2513 /*
2514  * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2515  *
2516  * If the ndlp represents an NVME Target, that we are logged into,
2517  * ping the NVME FC Transport layer to initiate a device rescan
2518  * on this remote NPort.
2519  */
2520 void
lpfc_nvme_rescan_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2521 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2522 {
2523 #if (IS_ENABLED(CONFIG_NVME_FC))
2524 	struct lpfc_nvme_rport *nrport;
2525 	struct nvme_fc_remote_port *remoteport = NULL;
2526 
2527 	spin_lock_irq(&ndlp->lock);
2528 	nrport = lpfc_ndlp_get_nrport(ndlp);
2529 	if (nrport)
2530 		remoteport = nrport->remoteport;
2531 	spin_unlock_irq(&ndlp->lock);
2532 
2533 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2534 			 "6170 Rescan NPort DID x%06x type x%x "
2535 			 "state x%x nrport x%px remoteport x%px\n",
2536 			 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2537 			 nrport, remoteport);
2538 
2539 	if (!nrport || !remoteport)
2540 		goto rescan_exit;
2541 
2542 	/* Rescan an NVME target in MAPPED state with DISCOVERY role set */
2543 	if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2544 	    ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2545 		nvme_fc_rescan_remoteport(remoteport);
2546 
2547 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2548 				 "6172 NVME rescanned DID x%06x "
2549 				 "port_state x%x\n",
2550 				 ndlp->nlp_DID, remoteport->port_state);
2551 	}
2552 	return;
2553  rescan_exit:
2554 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2555 			 "6169 Skip NVME Rport Rescan, NVME remoteport "
2556 			 "unregistered\n");
2557 #endif
2558 }
2559 
2560 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2561  *
2562  * There is no notion of Devloss or rport recovery from the current
2563  * nvme_transport perspective.  Loss of an rport just means IO cannot
2564  * be sent and recovery is completely up to the initator.
2565  * For now, the driver just unbinds the DID and port_role so that
2566  * no further IO can be issued.
2567  */
2568 void
lpfc_nvme_unregister_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2569 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2570 {
2571 #if (IS_ENABLED(CONFIG_NVME_FC))
2572 	int ret;
2573 	struct nvme_fc_local_port *localport;
2574 	struct lpfc_nvme_lport *lport;
2575 	struct lpfc_nvme_rport *rport;
2576 	struct nvme_fc_remote_port *remoteport = NULL;
2577 
2578 	localport = vport->localport;
2579 
2580 	/* This is fundamental error.  The localport is always
2581 	 * available until driver unload.  Just exit.
2582 	 */
2583 	if (!localport)
2584 		return;
2585 
2586 	lport = (struct lpfc_nvme_lport *)localport->private;
2587 	if (!lport)
2588 		goto input_err;
2589 
2590 	spin_lock_irq(&ndlp->lock);
2591 	rport = lpfc_ndlp_get_nrport(ndlp);
2592 	if (rport)
2593 		remoteport = rport->remoteport;
2594 	spin_unlock_irq(&ndlp->lock);
2595 	if (!remoteport)
2596 		goto input_err;
2597 
2598 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2599 			 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2600 			 "port_id x%06x, portstate x%x port type x%x "
2601 			 "refcnt %d\n",
2602 			 remoteport, remoteport->port_name,
2603 			 remoteport->port_id, remoteport->port_state,
2604 			 ndlp->nlp_type, kref_read(&ndlp->kref));
2605 
2606 	/* Sanity check ndlp type.  Only call for NVME ports. Don't
2607 	 * clear any rport state until the transport calls back.
2608 	 */
2609 
2610 	if (ndlp->nlp_type & NLP_NVME_TARGET) {
2611 		/* No concern about the role change on the nvme remoteport.
2612 		 * The transport will update it.
2613 		 */
2614 		spin_lock_irq(&ndlp->lock);
2615 		ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;
2616 		spin_unlock_irq(&ndlp->lock);
2617 
2618 		/* Don't let the host nvme transport keep sending keep-alives
2619 		 * on this remoteport. Vport is unloading, no recovery. The
2620 		 * return values is ignored.  The upcall is a courtesy to the
2621 		 * transport.
2622 		 */
2623 		if (test_bit(FC_UNLOADING, &vport->load_flag) ||
2624 		    unlikely(vport->phba->link_state == LPFC_HBA_ERROR))
2625 			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2626 
2627 		ret = nvme_fc_unregister_remoteport(remoteport);
2628 
2629 		/* The driver no longer knows if the nrport memory is valid.
2630 		 * because the controller teardown process has begun and
2631 		 * is asynchronous.  Break the binding in the ndlp. Also
2632 		 * remove the register ndlp reference to setup node release.
2633 		 */
2634 		ndlp->nrport = NULL;
2635 		lpfc_nlp_put(ndlp);
2636 		if (ret != 0) {
2637 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2638 					 "6167 NVME unregister failed %d "
2639 					 "port_state x%x\n",
2640 					 ret, remoteport->port_state);
2641 
2642 			if (test_bit(FC_UNLOADING, &vport->load_flag)) {
2643 				/* Only 1 thread can drop the initial node
2644 				 * reference. Check if another thread has set
2645 				 * NLP_DROPPED.
2646 				 */
2647 				spin_lock_irq(&ndlp->lock);
2648 				if (!(ndlp->nlp_flag & NLP_DROPPED)) {
2649 					ndlp->nlp_flag |= NLP_DROPPED;
2650 					spin_unlock_irq(&ndlp->lock);
2651 					lpfc_nlp_put(ndlp);
2652 					return;
2653 				}
2654 				spin_unlock_irq(&ndlp->lock);
2655 			}
2656 		}
2657 	}
2658 	return;
2659 
2660  input_err:
2661 #endif
2662 	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2663 			 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2664 			 vport->localport, ndlp->rport, ndlp->nlp_DID);
2665 }
2666 
2667 /**
2668  * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort
2669  * @phba: pointer to lpfc hba data structure.
2670  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2671  *
2672  * This routine is invoked by the worker thread to process a SLI4 fast-path
2673  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2674  * here.
2675  **/
2676 void
lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2677 lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba,
2678 				   struct lpfc_io_buf *lpfc_ncmd)
2679 {
2680 	struct nvmefc_fcp_req *nvme_cmd = NULL;
2681 
2682 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2683 			"6533 %s nvme_cmd %p tag x%x abort complete and "
2684 			"xri released\n", __func__,
2685 			lpfc_ncmd->nvmeCmd,
2686 			lpfc_ncmd->cur_iocbq.iotag);
2687 
2688 	/* Aborted NVME commands are required to not complete
2689 	 * before the abort exchange command fully completes.
2690 	 * Once completed, it is available via the put list.
2691 	 */
2692 	if (lpfc_ncmd->nvmeCmd) {
2693 		nvme_cmd = lpfc_ncmd->nvmeCmd;
2694 		nvme_cmd->transferred_length = 0;
2695 		nvme_cmd->rcv_rsplen = 0;
2696 		nvme_cmd->status = NVME_SC_INTERNAL;
2697 		nvme_cmd->done(nvme_cmd);
2698 		lpfc_ncmd->nvmeCmd = NULL;
2699 	}
2700 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2701 }
2702 
2703 /**
2704  * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2705  * @phba: pointer to lpfc hba data structure.
2706  * @axri: pointer to the fcp xri abort wcqe structure.
2707  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2708  *
2709  * This routine is invoked by the worker thread to process a SLI4 fast-path
2710  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2711  * here.
2712  **/
2713 void
lpfc_sli4_nvme_xri_aborted(struct lpfc_hba * phba,struct sli4_wcqe_xri_aborted * axri,struct lpfc_io_buf * lpfc_ncmd)2714 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2715 			   struct sli4_wcqe_xri_aborted *axri,
2716 			   struct lpfc_io_buf *lpfc_ncmd)
2717 {
2718 	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2719 	struct nvmefc_fcp_req *nvme_cmd = NULL;
2720 	struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2721 
2722 
2723 	if (ndlp)
2724 		lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2725 
2726 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2727 			"6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2728 			"xri released\n",
2729 			lpfc_ncmd->nvmeCmd, xri,
2730 			lpfc_ncmd->cur_iocbq.iotag);
2731 
2732 	/* Aborted NVME commands are required to not complete
2733 	 * before the abort exchange command fully completes.
2734 	 * Once completed, it is available via the put list.
2735 	 */
2736 	if (lpfc_ncmd->nvmeCmd) {
2737 		nvme_cmd = lpfc_ncmd->nvmeCmd;
2738 		nvme_cmd->done(nvme_cmd);
2739 		lpfc_ncmd->nvmeCmd = NULL;
2740 	}
2741 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2742 }
2743 
2744 /**
2745  * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2746  * @phba: Pointer to HBA context object.
2747  *
2748  * This function flushes all wqes in the nvme rings and frees all resources
2749  * in the txcmplq. This function does not issue abort wqes for the IO
2750  * commands in txcmplq, they will just be returned with
2751  * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2752  * slot has been permanently disabled.
2753  **/
2754 void
lpfc_nvme_wait_for_io_drain(struct lpfc_hba * phba)2755 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2756 {
2757 	struct lpfc_sli_ring  *pring;
2758 	u32 i, wait_cnt = 0;
2759 
2760 	if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2761 		return;
2762 
2763 	/* Cycle through all IO rings and make sure all outstanding
2764 	 * WQEs have been removed from the txcmplqs.
2765 	 */
2766 	for (i = 0; i < phba->cfg_hdw_queue; i++) {
2767 		if (!phba->sli4_hba.hdwq[i].io_wq)
2768 			continue;
2769 		pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2770 
2771 		if (!pring)
2772 			continue;
2773 
2774 		/* Retrieve everything on the txcmplq */
2775 		while (!list_empty(&pring->txcmplq)) {
2776 			msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2777 			wait_cnt++;
2778 
2779 			/* The sleep is 10mS.  Every ten seconds,
2780 			 * dump a message.  Something is wrong.
2781 			 */
2782 			if ((wait_cnt % 1000) == 0) {
2783 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2784 						"6178 NVME IO not empty, "
2785 						"cnt %d\n", wait_cnt);
2786 			}
2787 		}
2788 	}
2789 
2790 	/* Make sure HBA is alive */
2791 	lpfc_issue_hb_tmo(phba);
2792 
2793 }
2794 
2795 void
lpfc_nvme_cancel_iocb(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,uint32_t stat,uint32_t param)2796 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
2797 		      uint32_t stat, uint32_t param)
2798 {
2799 #if (IS_ENABLED(CONFIG_NVME_FC))
2800 	struct lpfc_io_buf *lpfc_ncmd;
2801 	struct nvmefc_fcp_req *nCmd;
2802 	struct lpfc_wcqe_complete wcqe;
2803 	struct lpfc_wcqe_complete *wcqep = &wcqe;
2804 
2805 	lpfc_ncmd = pwqeIn->io_buf;
2806 	if (!lpfc_ncmd) {
2807 		lpfc_sli_release_iocbq(phba, pwqeIn);
2808 		return;
2809 	}
2810 	/* For abort iocb just return, IO iocb will do a done call */
2811 	if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2812 	    CMD_ABORT_XRI_CX) {
2813 		lpfc_sli_release_iocbq(phba, pwqeIn);
2814 		return;
2815 	}
2816 
2817 	spin_lock(&lpfc_ncmd->buf_lock);
2818 	nCmd = lpfc_ncmd->nvmeCmd;
2819 	if (!nCmd) {
2820 		spin_unlock(&lpfc_ncmd->buf_lock);
2821 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
2822 		return;
2823 	}
2824 	spin_unlock(&lpfc_ncmd->buf_lock);
2825 
2826 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2827 			"6194 NVME Cancel xri %x\n",
2828 			lpfc_ncmd->cur_iocbq.sli4_xritag);
2829 
2830 	wcqep->word0 = 0;
2831 	bf_set(lpfc_wcqe_c_status, wcqep, stat);
2832 	wcqep->parameter = param;
2833 	wcqep->total_data_placed = 0;
2834 	wcqep->word3 = 0; /* xb is 0 */
2835 
2836 	/* Call release with XB=1 to queue the IO into the abort list. */
2837 	if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
2838 		bf_set(lpfc_wcqe_c_xb, wcqep, 1);
2839 
2840 	memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep));
2841 	(pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn);
2842 #endif
2843 }
2844