1 /*******************************************************************
2 * This file is part of the Emulex Linux Device Driver for *
3 * Fibre Channel Host Bus Adapters. *
4 * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *
5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *
7 * EMULEX and SLI are trademarks of Emulex. *
8 * www.broadcom.com *
9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *
10 * *
11 * This program is free software; you can redistribute it and/or *
12 * modify it under the terms of version 2 of the GNU General *
13 * Public License as published by the Free Software Foundation. *
14 * This program is distributed in the hope that it will be useful. *
15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19 * TO BE LEGALLY INVALID. See the GNU General Public License for *
20 * more details, a copy of which can be found in the file COPYING *
21 * included with this package. *
22 ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <linux/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38
39 #include "lpfc_version.h"
40 #include "lpfc_hw4.h"
41 #include "lpfc_hw.h"
42 #include "lpfc_sli.h"
43 #include "lpfc_sli4.h"
44 #include "lpfc_nl.h"
45 #include "lpfc_disc.h"
46 #include "lpfc.h"
47 #include "lpfc_nvme.h"
48 #include "lpfc_scsi.h"
49 #include "lpfc_logmsg.h"
50 #include "lpfc_crtn.h"
51 #include "lpfc_vport.h"
52 #include "lpfc_debugfs.h"
53
54 /* NVME initiator-based functions */
55
56 static struct lpfc_io_buf *
57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 int idx, int expedite);
59
60 static void
61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62
63 static struct nvme_fc_port_template lpfc_nvme_template;
64
65 /**
66 * lpfc_nvme_create_queue -
67 * @pnvme_lport: Transport localport that LS is to be issued from
68 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
69 * @qsize: Size of the queue in bytes
70 * @handle: An opaque driver handle used in follow-up calls.
71 *
72 * Driver registers this routine to preallocate and initialize any
73 * internal data structures to bind the @qidx to its internal IO queues.
74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
75 *
76 * Return value :
77 * 0 - Success
78 * -EINVAL - Unsupported input value.
79 * -ENOMEM - Could not alloc necessary memory
80 **/
81 static int
lpfc_nvme_create_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,u16 qsize,void ** handle)82 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
83 unsigned int qidx, u16 qsize,
84 void **handle)
85 {
86 struct lpfc_nvme_lport *lport;
87 struct lpfc_vport *vport;
88 struct lpfc_nvme_qhandle *qhandle;
89 char *str;
90
91 if (!pnvme_lport->private)
92 return -ENOMEM;
93
94 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
95 vport = lport->vport;
96
97 if (!vport || test_bit(FC_UNLOADING, &vport->load_flag) ||
98 test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
99 return -ENODEV;
100
101 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
102 if (qhandle == NULL)
103 return -ENOMEM;
104
105 qhandle->cpu_id = raw_smp_processor_id();
106 qhandle->qidx = qidx;
107 /*
108 * NVME qidx == 0 is the admin queue, so both admin queue
109 * and first IO queue will use MSI-X vector and associated
110 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
111 */
112 if (qidx) {
113 str = "IO "; /* IO queue */
114 qhandle->index = ((qidx - 1) %
115 lpfc_nvme_template.max_hw_queues);
116 } else {
117 str = "ADM"; /* Admin queue */
118 qhandle->index = qidx;
119 }
120
121 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
122 "6073 Binding %s HdwQueue %d (cpu %d) to "
123 "hdw_queue %d qhandle x%px\n", str,
124 qidx, qhandle->cpu_id, qhandle->index, qhandle);
125 *handle = (void *)qhandle;
126 return 0;
127 }
128
129 /**
130 * lpfc_nvme_delete_queue -
131 * @pnvme_lport: Transport localport that LS is to be issued from
132 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
133 * @handle: An opaque driver handle from lpfc_nvme_create_queue
134 *
135 * Driver registers this routine to free
136 * any internal data structures to bind the @qidx to its internal
137 * IO queues.
138 *
139 * Return value :
140 * 0 - Success
141 * TODO: What are the failure codes.
142 **/
143 static void
lpfc_nvme_delete_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,void * handle)144 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
145 unsigned int qidx,
146 void *handle)
147 {
148 struct lpfc_nvme_lport *lport;
149 struct lpfc_vport *vport;
150
151 if (!pnvme_lport->private)
152 return;
153
154 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
155 vport = lport->vport;
156
157 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
158 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
159 lport, qidx, handle);
160 kfree(handle);
161 }
162
163 static void
lpfc_nvme_localport_delete(struct nvme_fc_local_port * localport)164 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
165 {
166 struct lpfc_nvme_lport *lport = localport->private;
167
168 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
169 "6173 localport x%px delete complete\n",
170 lport);
171
172 /* release any threads waiting for the unreg to complete */
173 if (lport->vport->localport)
174 complete(lport->lport_unreg_cmp);
175 }
176
177 /* lpfc_nvme_remoteport_delete
178 *
179 * @remoteport: Pointer to an nvme transport remoteport instance.
180 *
181 * This is a template downcall. NVME transport calls this function
182 * when it has completed the unregistration of a previously
183 * registered remoteport.
184 *
185 * Return value :
186 * None
187 */
188 static void
lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port * remoteport)189 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
190 {
191 struct lpfc_nvme_rport *rport = remoteport->private;
192 struct lpfc_vport *vport;
193 struct lpfc_nodelist *ndlp;
194 u32 fc4_xpt_flags;
195
196 ndlp = rport->ndlp;
197 if (!ndlp) {
198 pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",
199 __func__, rport, remoteport);
200 goto rport_err;
201 }
202
203 vport = ndlp->vport;
204 if (!vport) {
205 pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",
206 __func__, ndlp, ndlp->nlp_state, rport);
207 goto rport_err;
208 }
209
210 fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;
211
212 /* Remove this rport from the lport's list - memory is owned by the
213 * transport. Remove the ndlp reference for the NVME transport before
214 * calling state machine to remove the node.
215 */
216 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
217 "6146 remoteport delete of remoteport x%px, ndlp x%px "
218 "DID x%x xflags x%x\n",
219 remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);
220 spin_lock_irq(&ndlp->lock);
221
222 /* The register rebind might have occurred before the delete
223 * downcall. Guard against this race.
224 */
225 if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)
226 ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);
227
228 spin_unlock_irq(&ndlp->lock);
229
230 /* On a devloss timeout event, one more put is executed provided the
231 * NVME and SCSI rport unregister requests are complete.
232 */
233 if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))
234 lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
235
236 rport_err:
237 return;
238 }
239
240 /**
241 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
242 * @phba: pointer to lpfc hba data structure.
243 * @axchg: pointer to exchange context for the NVME LS request
244 *
245 * This routine is used for processing an asynchronously received NVME LS
246 * request. Any remaining validation is done and the LS is then forwarded
247 * to the nvme-fc transport via nvme_fc_rcv_ls_req().
248 *
249 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
250 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
251 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
252 *
253 * Returns 0 if LS was handled and delivered to the transport
254 * Returns 1 if LS failed to be handled and should be dropped
255 */
256 int
lpfc_nvme_handle_lsreq(struct lpfc_hba * phba,struct lpfc_async_xchg_ctx * axchg)257 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
258 struct lpfc_async_xchg_ctx *axchg)
259 {
260 #if (IS_ENABLED(CONFIG_NVME_FC))
261 struct lpfc_vport *vport;
262 struct lpfc_nvme_rport *lpfc_rport;
263 struct nvme_fc_remote_port *remoteport;
264 struct lpfc_nvme_lport *lport;
265 uint32_t *payload = axchg->payload;
266 int rc;
267
268 vport = axchg->ndlp->vport;
269 lpfc_rport = axchg->ndlp->nrport;
270 if (!lpfc_rport)
271 return -EINVAL;
272
273 remoteport = lpfc_rport->remoteport;
274 if (!vport->localport ||
275 test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
276 return -EINVAL;
277
278 lport = vport->localport->private;
279 if (!lport)
280 return -EINVAL;
281
282 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
283 axchg->size);
284
285 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
286 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
287 "%08x %08x %08x\n",
288 axchg->size, rc,
289 *payload, *(payload+1), *(payload+2),
290 *(payload+3), *(payload+4), *(payload+5));
291
292 if (!rc)
293 return 0;
294 #endif
295 return 1;
296 }
297
298 /**
299 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
300 * LS request.
301 * @phba: Pointer to HBA context object
302 * @vport: The local port that issued the LS
303 * @cmdwqe: Pointer to driver command WQE object.
304 * @wcqe: Pointer to driver response CQE object.
305 *
306 * This function is the generic completion handler for NVME LS requests.
307 * The function updates any states and statistics, calls the transport
308 * ls_req done() routine, then tears down the command and buffers used
309 * for the LS request.
310 **/
311 void
__lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_vport * vport,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe)312 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
313 struct lpfc_iocbq *cmdwqe,
314 struct lpfc_wcqe_complete *wcqe)
315 {
316 struct nvmefc_ls_req *pnvme_lsreq;
317 struct lpfc_dmabuf *buf_ptr;
318 struct lpfc_nodelist *ndlp;
319 int status;
320
321 pnvme_lsreq = cmdwqe->context_un.nvme_lsreq;
322 ndlp = cmdwqe->ndlp;
323 buf_ptr = cmdwqe->bpl_dmabuf;
324
325 status = bf_get(lpfc_wcqe_c_status, wcqe);
326
327 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
328 "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "
329 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
330 "ndlp:x%px\n",
331 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
332 cmdwqe->sli4_xritag, status,
333 (wcqe->parameter & 0xffff),
334 cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf,
335 ndlp);
336
337 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
338 cmdwqe->sli4_xritag, status, wcqe->parameter);
339
340 if (buf_ptr) {
341 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
342 kfree(buf_ptr);
343 cmdwqe->bpl_dmabuf = NULL;
344 }
345 if (pnvme_lsreq->done) {
346 if (status != CQE_STATUS_SUCCESS)
347 status = -ENXIO;
348 pnvme_lsreq->done(pnvme_lsreq, status);
349 } else {
350 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
351 "6046 NVMEx cmpl without done call back? "
352 "Data x%px DID %x Xri: %x status %x\n",
353 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
354 cmdwqe->sli4_xritag, status);
355 }
356 if (ndlp) {
357 lpfc_nlp_put(ndlp);
358 cmdwqe->ndlp = NULL;
359 }
360 lpfc_sli_release_iocbq(phba, cmdwqe);
361 }
362
363 static void
lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe)364 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
365 struct lpfc_iocbq *rspwqe)
366 {
367 struct lpfc_vport *vport = cmdwqe->vport;
368 struct lpfc_nvme_lport *lport;
369 uint32_t status;
370 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
371
372 status = bf_get(lpfc_wcqe_c_status, wcqe);
373
374 if (vport->localport) {
375 lport = (struct lpfc_nvme_lport *)vport->localport->private;
376 if (lport) {
377 atomic_inc(&lport->fc4NvmeLsCmpls);
378 if (status) {
379 if (bf_get(lpfc_wcqe_c_xb, wcqe))
380 atomic_inc(&lport->cmpl_ls_xb);
381 atomic_inc(&lport->cmpl_ls_err);
382 }
383 }
384 }
385
386 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
387 }
388
389 static int
lpfc_nvme_gen_req(struct lpfc_vport * vport,struct lpfc_dmabuf * bmp,struct lpfc_dmabuf * inp,struct nvmefc_ls_req * pnvme_lsreq,void (* cmpl)(struct lpfc_hba *,struct lpfc_iocbq *,struct lpfc_iocbq *),struct lpfc_nodelist * ndlp,uint32_t num_entry,uint32_t tmo,uint8_t retry)390 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
391 struct lpfc_dmabuf *inp,
392 struct nvmefc_ls_req *pnvme_lsreq,
393 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
394 struct lpfc_iocbq *),
395 struct lpfc_nodelist *ndlp, uint32_t num_entry,
396 uint32_t tmo, uint8_t retry)
397 {
398 struct lpfc_hba *phba = vport->phba;
399 union lpfc_wqe128 *wqe;
400 struct lpfc_iocbq *genwqe;
401 struct ulp_bde64 *bpl;
402 struct ulp_bde64 bde;
403 int i, rc, xmit_len, first_len;
404
405 /* Allocate buffer for command WQE */
406 genwqe = lpfc_sli_get_iocbq(phba);
407 if (genwqe == NULL)
408 return 1;
409
410 wqe = &genwqe->wqe;
411 /* Initialize only 64 bytes */
412 memset(wqe, 0, sizeof(union lpfc_wqe));
413
414 genwqe->bpl_dmabuf = bmp;
415 genwqe->cmd_flag |= LPFC_IO_NVME_LS;
416
417 /* Save for completion so we can release these resources */
418 genwqe->ndlp = lpfc_nlp_get(ndlp);
419 if (!genwqe->ndlp) {
420 dev_warn(&phba->pcidev->dev,
421 "Warning: Failed node ref, not sending LS_REQ\n");
422 lpfc_sli_release_iocbq(phba, genwqe);
423 return 1;
424 }
425
426 genwqe->context_un.nvme_lsreq = pnvme_lsreq;
427 /* Fill in payload, bp points to frame payload */
428
429 if (!tmo)
430 /* FC spec states we need 3 * ratov for CT requests */
431 tmo = (3 * phba->fc_ratov);
432
433 /* For this command calculate the xmit length of the request bde. */
434 xmit_len = 0;
435 first_len = 0;
436 bpl = (struct ulp_bde64 *)bmp->virt;
437 for (i = 0; i < num_entry; i++) {
438 bde.tus.w = bpl[i].tus.w;
439 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
440 break;
441 xmit_len += bde.tus.f.bdeSize;
442 if (i == 0)
443 first_len = xmit_len;
444 }
445
446 genwqe->num_bdes = num_entry;
447 genwqe->hba_wqidx = 0;
448
449 /* Words 0 - 2 */
450 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
451 wqe->generic.bde.tus.f.bdeSize = first_len;
452 wqe->generic.bde.addrLow = bpl[0].addrLow;
453 wqe->generic.bde.addrHigh = bpl[0].addrHigh;
454
455 /* Word 3 */
456 wqe->gen_req.request_payload_len = first_len;
457
458 /* Word 4 */
459
460 /* Word 5 */
461 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
462 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
463 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
464 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
465 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
466
467 /* Word 6 */
468 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
469 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
470 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
471
472 /* Word 7 */
473 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);
474 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
475 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
476 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
477
478 /* Word 8 */
479 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
480
481 /* Word 9 */
482 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
483
484 /* Word 10 */
485 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
486 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
487 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
488 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
489 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
490
491 /* Word 11 */
492 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
493 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
494
495
496 /* Issue GEN REQ WQE for NPORT <did> */
497 genwqe->cmd_cmpl = cmpl;
498 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
499 genwqe->vport = vport;
500 genwqe->retry = retry;
501
502 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
503 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
504
505 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
506 if (rc) {
507 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
508 "6045 Issue GEN REQ WQE to NPORT x%x "
509 "Data: x%x x%x rc x%x\n",
510 ndlp->nlp_DID, genwqe->iotag,
511 vport->port_state, rc);
512 lpfc_nlp_put(ndlp);
513 lpfc_sli_release_iocbq(phba, genwqe);
514 return 1;
515 }
516
517 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
518 "6050 Issue GEN REQ WQE to NPORT x%x "
519 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
520 "bmp:x%px xmit:%d 1st:%d\n",
521 ndlp->nlp_DID, genwqe->sli4_xritag,
522 vport->port_state,
523 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
524 return 0;
525 }
526
527
528 /**
529 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
530 * @vport: The local port issuing the LS
531 * @ndlp: The remote port to send the LS to
532 * @pnvme_lsreq: Pointer to LS request structure from the transport
533 * @gen_req_cmp: Completion call-back
534 *
535 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
536 * WQE to perform the LS operation.
537 *
538 * Return value :
539 * 0 - Success
540 * non-zero: various error codes, in form of -Exxx
541 **/
542 int
__lpfc_nvme_ls_req(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq,void (* gen_req_cmp)(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe))543 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
544 struct nvmefc_ls_req *pnvme_lsreq,
545 void (*gen_req_cmp)(struct lpfc_hba *phba,
546 struct lpfc_iocbq *cmdwqe,
547 struct lpfc_iocbq *rspwqe))
548 {
549 struct lpfc_dmabuf *bmp;
550 struct ulp_bde64 *bpl;
551 int ret;
552 uint16_t ntype, nstate;
553
554 if (!ndlp) {
555 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
556 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
557 "LS Req\n",
558 ndlp);
559 return -ENODEV;
560 }
561
562 ntype = ndlp->nlp_type;
563 nstate = ndlp->nlp_state;
564 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
565 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
566 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
567 "6088 NVMEx LS REQ: Fail DID x%06x not "
568 "ready for IO. Type x%x, State x%x\n",
569 ndlp->nlp_DID, ntype, nstate);
570 return -ENODEV;
571 }
572 if (test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
573 return -ENODEV;
574
575 if (!vport->phba->sli4_hba.nvmels_wq)
576 return -ENOMEM;
577
578 /*
579 * there are two dma buf in the request, actually there is one and
580 * the second one is just the start address + cmd size.
581 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
582 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
583 * because the nvem layer owns the data bufs.
584 * We do not have to break these packets open, we don't care what is
585 * in them. And we do not have to look at the resonse data, we only
586 * care that we got a response. All of the caring is going to happen
587 * in the nvme-fc layer.
588 */
589
590 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
591 if (!bmp) {
592 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
593 "6044 NVMEx LS REQ: Could not alloc LS buf "
594 "for DID %x\n",
595 ndlp->nlp_DID);
596 return -ENOMEM;
597 }
598
599 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
600 if (!bmp->virt) {
601 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
602 "6042 NVMEx LS REQ: Could not alloc mbuf "
603 "for DID %x\n",
604 ndlp->nlp_DID);
605 kfree(bmp);
606 return -ENOMEM;
607 }
608
609 INIT_LIST_HEAD(&bmp->list);
610
611 bpl = (struct ulp_bde64 *)bmp->virt;
612 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
613 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
614 bpl->tus.f.bdeFlags = 0;
615 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
616 bpl->tus.w = le32_to_cpu(bpl->tus.w);
617 bpl++;
618
619 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
620 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
621 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
622 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
623 bpl->tus.w = le32_to_cpu(bpl->tus.w);
624
625 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
626 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
627 "rqstlen:%d rsplen:%d %pad %pad\n",
628 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
629 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
630 &pnvme_lsreq->rspdma);
631
632 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
633 pnvme_lsreq, gen_req_cmp, ndlp, 2,
634 pnvme_lsreq->timeout, 0);
635 if (ret != WQE_SUCCESS) {
636 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
637 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
638 "lsreq x%px Status %x DID %x\n",
639 pnvme_lsreq, ret, ndlp->nlp_DID);
640 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
641 kfree(bmp);
642 return -EIO;
643 }
644
645 return 0;
646 }
647
648 /**
649 * lpfc_nvme_ls_req - Issue an NVME Link Service request
650 * @pnvme_lport: Transport localport that LS is to be issued from.
651 * @pnvme_rport: Transport remoteport that LS is to be sent to.
652 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
653 *
654 * Driver registers this routine to handle any link service request
655 * from the nvme_fc transport to a remote nvme-aware port.
656 *
657 * Return value :
658 * 0 - Success
659 * non-zero: various error codes, in form of -Exxx
660 **/
661 static int
lpfc_nvme_ls_req(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)662 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
663 struct nvme_fc_remote_port *pnvme_rport,
664 struct nvmefc_ls_req *pnvme_lsreq)
665 {
666 struct lpfc_nvme_lport *lport;
667 struct lpfc_nvme_rport *rport;
668 struct lpfc_vport *vport;
669 int ret;
670
671 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
672 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
673 if (unlikely(!lport) || unlikely(!rport))
674 return -EINVAL;
675
676 vport = lport->vport;
677 if (test_bit(FC_UNLOADING, &vport->load_flag) ||
678 test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
679 return -ENODEV;
680
681 atomic_inc(&lport->fc4NvmeLsRequests);
682
683 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
684 lpfc_nvme_ls_req_cmp);
685 if (ret)
686 atomic_inc(&lport->xmt_ls_err);
687
688 return ret;
689 }
690
691 /**
692 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
693 * NVME LS request
694 * @vport: The local port that issued the LS
695 * @ndlp: The remote port the LS was sent to
696 * @pnvme_lsreq: Pointer to LS request structure from the transport
697 *
698 * The driver validates the ndlp, looks for the LS, and aborts the
699 * LS if found.
700 *
701 * Returns:
702 * 0 : if LS found and aborted
703 * non-zero: various error conditions in form -Exxx
704 **/
705 int
__lpfc_nvme_ls_abort(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq)706 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
707 struct nvmefc_ls_req *pnvme_lsreq)
708 {
709 struct lpfc_hba *phba = vport->phba;
710 struct lpfc_sli_ring *pring;
711 struct lpfc_iocbq *wqe, *next_wqe;
712 bool foundit = false;
713
714 if (!ndlp) {
715 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
716 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
717 "x%06x, Failing LS Req\n",
718 ndlp, ndlp ? ndlp->nlp_DID : 0);
719 return -EINVAL;
720 }
721
722 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
723 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
724 "x%px rqstlen:%d rsplen:%d %pad %pad\n",
725 pnvme_lsreq, pnvme_lsreq->rqstlen,
726 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
727 &pnvme_lsreq->rspdma);
728
729 /*
730 * Lock the ELS ring txcmplq and look for the wqe that matches
731 * this ELS. If found, issue an abort on the wqe.
732 */
733 pring = phba->sli4_hba.nvmels_wq->pring;
734 spin_lock_irq(&phba->hbalock);
735 spin_lock(&pring->ring_lock);
736 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
737 if (wqe->context_un.nvme_lsreq == pnvme_lsreq) {
738 wqe->cmd_flag |= LPFC_DRIVER_ABORTED;
739 foundit = true;
740 break;
741 }
742 }
743 spin_unlock(&pring->ring_lock);
744
745 if (foundit)
746 lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);
747 spin_unlock_irq(&phba->hbalock);
748
749 if (foundit)
750 return 0;
751
752 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
753 "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",
754 pnvme_lsreq);
755 return -EINVAL;
756 }
757
758 static int
lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port * localport,struct nvme_fc_remote_port * remoteport,struct nvmefc_ls_rsp * ls_rsp)759 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
760 struct nvme_fc_remote_port *remoteport,
761 struct nvmefc_ls_rsp *ls_rsp)
762 {
763 struct lpfc_async_xchg_ctx *axchg =
764 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
765 struct lpfc_nvme_lport *lport;
766 int rc;
767
768 if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag))
769 return -ENODEV;
770
771 lport = (struct lpfc_nvme_lport *)localport->private;
772
773 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
774
775 if (rc) {
776 /*
777 * unless the failure is due to having already sent
778 * the response, an abort will be generated for the
779 * exchange if the rsp can't be sent.
780 */
781 if (rc != -EALREADY)
782 atomic_inc(&lport->xmt_ls_abort);
783 return rc;
784 }
785
786 return 0;
787 }
788
789 /**
790 * lpfc_nvme_ls_abort - Abort a prior NVME LS request
791 * @pnvme_lport: Transport localport that LS is to be issued from.
792 * @pnvme_rport: Transport remoteport that LS is to be sent to.
793 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
794 *
795 * Driver registers this routine to abort a NVME LS request that is
796 * in progress (from the transports perspective).
797 **/
798 static void
lpfc_nvme_ls_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)799 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
800 struct nvme_fc_remote_port *pnvme_rport,
801 struct nvmefc_ls_req *pnvme_lsreq)
802 {
803 struct lpfc_nvme_lport *lport;
804 struct lpfc_vport *vport;
805 struct lpfc_nodelist *ndlp;
806 int ret;
807
808 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
809 if (unlikely(!lport))
810 return;
811 vport = lport->vport;
812
813 if (test_bit(FC_UNLOADING, &vport->load_flag))
814 return;
815
816 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
817
818 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
819 if (!ret)
820 atomic_inc(&lport->xmt_ls_abort);
821 }
822
823 /* Fix up the existing sgls for NVME IO. */
824 static inline void
lpfc_nvme_adj_fcp_sgls(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct nvmefc_fcp_req * nCmd)825 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
826 struct lpfc_io_buf *lpfc_ncmd,
827 struct nvmefc_fcp_req *nCmd)
828 {
829 struct lpfc_hba *phba = vport->phba;
830 struct sli4_sge *sgl;
831 union lpfc_wqe128 *wqe;
832 uint32_t *wptr, *dptr;
833
834 /*
835 * Get a local pointer to the built-in wqe and correct
836 * the cmd size to match NVME's 96 bytes and fix
837 * the dma address.
838 */
839
840 wqe = &lpfc_ncmd->cur_iocbq.wqe;
841
842 /*
843 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
844 * match NVME. NVME sends 96 bytes. Also, use the
845 * nvme commands command and response dma addresses
846 * rather than the virtual memory to ease the restore
847 * operation.
848 */
849 sgl = lpfc_ncmd->dma_sgl;
850 sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
851 if (phba->cfg_nvme_embed_cmd) {
852 sgl->addr_hi = 0;
853 sgl->addr_lo = 0;
854
855 /* Word 0-2 - NVME CMND IU (embedded payload) */
856 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
857 wqe->generic.bde.tus.f.bdeSize = 56;
858 wqe->generic.bde.addrHigh = 0;
859 wqe->generic.bde.addrLow = 64; /* Word 16 */
860
861 /* Word 10 - dbde is 0, wqes is 1 in template */
862
863 /*
864 * Embed the payload in the last half of the WQE
865 * WQE words 16-30 get the NVME CMD IU payload
866 *
867 * WQE words 16-19 get payload Words 1-4
868 * WQE words 20-21 get payload Words 6-7
869 * WQE words 22-29 get payload Words 16-23
870 */
871 wptr = &wqe->words[16]; /* WQE ptr */
872 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */
873 dptr++; /* Skip Word 0 in payload */
874
875 *wptr++ = *dptr++; /* Word 1 */
876 *wptr++ = *dptr++; /* Word 2 */
877 *wptr++ = *dptr++; /* Word 3 */
878 *wptr++ = *dptr++; /* Word 4 */
879 dptr++; /* Skip Word 5 in payload */
880 *wptr++ = *dptr++; /* Word 6 */
881 *wptr++ = *dptr++; /* Word 7 */
882 dptr += 8; /* Skip Words 8-15 in payload */
883 *wptr++ = *dptr++; /* Word 16 */
884 *wptr++ = *dptr++; /* Word 17 */
885 *wptr++ = *dptr++; /* Word 18 */
886 *wptr++ = *dptr++; /* Word 19 */
887 *wptr++ = *dptr++; /* Word 20 */
888 *wptr++ = *dptr++; /* Word 21 */
889 *wptr++ = *dptr++; /* Word 22 */
890 *wptr = *dptr; /* Word 23 */
891 } else {
892 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
893 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
894
895 /* Word 0-2 - NVME CMND IU Inline BDE */
896 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
897 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
898 wqe->generic.bde.addrHigh = sgl->addr_hi;
899 wqe->generic.bde.addrLow = sgl->addr_lo;
900
901 /* Word 10 */
902 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
903 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
904 }
905
906 sgl++;
907
908 /* Setup the physical region for the FCP RSP */
909 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
910 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
911 sgl->word2 = le32_to_cpu(sgl->word2);
912 if (nCmd->sg_cnt)
913 bf_set(lpfc_sli4_sge_last, sgl, 0);
914 else
915 bf_set(lpfc_sli4_sge_last, sgl, 1);
916 sgl->word2 = cpu_to_le32(sgl->word2);
917 sgl->sge_len = cpu_to_le32(nCmd->rsplen);
918 }
919
920
921 /*
922 * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO
923 *
924 * Driver registers this routine as it io request handler. This
925 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
926 * data structure to the rport indicated in @lpfc_nvme_rport.
927 *
928 * Return value :
929 * 0 - Success
930 * TODO: What are the failure codes.
931 **/
932 static void
lpfc_nvme_io_cmd_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,struct lpfc_iocbq * pwqeOut)933 lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
934 struct lpfc_iocbq *pwqeOut)
935 {
936 struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;
937 struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
938 struct lpfc_vport *vport = pwqeIn->vport;
939 struct nvmefc_fcp_req *nCmd;
940 struct nvme_fc_ersp_iu *ep;
941 struct nvme_fc_cmd_iu *cp;
942 struct lpfc_nodelist *ndlp;
943 struct lpfc_nvme_fcpreq_priv *freqpriv;
944 struct lpfc_nvme_lport *lport;
945 uint32_t code, status, idx;
946 uint16_t cid, sqhd, data;
947 uint32_t *ptr;
948 uint32_t lat;
949 bool call_done = false;
950 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
951 int cpu;
952 #endif
953 bool offline = false;
954
955 /* Sanity check on return of outstanding command */
956 if (!lpfc_ncmd) {
957 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
958 "6071 Null lpfc_ncmd pointer. No "
959 "release, skip completion\n");
960 return;
961 }
962
963 /* Guard against abort handler being called at same time */
964 spin_lock(&lpfc_ncmd->buf_lock);
965
966 if (!lpfc_ncmd->nvmeCmd) {
967 spin_unlock(&lpfc_ncmd->buf_lock);
968 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
969 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
970 "nvmeCmd x%px\n",
971 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
972
973 /* Release the lpfc_ncmd regardless of the missing elements. */
974 lpfc_release_nvme_buf(phba, lpfc_ncmd);
975 return;
976 }
977 nCmd = lpfc_ncmd->nvmeCmd;
978 status = bf_get(lpfc_wcqe_c_status, wcqe);
979
980 idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
981 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
982
983 if (unlikely(status && vport->localport)) {
984 lport = (struct lpfc_nvme_lport *)vport->localport->private;
985 if (lport) {
986 if (bf_get(lpfc_wcqe_c_xb, wcqe))
987 atomic_inc(&lport->cmpl_fcp_xb);
988 atomic_inc(&lport->cmpl_fcp_err);
989 }
990 }
991
992 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
993 lpfc_ncmd->cur_iocbq.sli4_xritag,
994 status, wcqe->parameter);
995 /*
996 * Catch race where our node has transitioned, but the
997 * transport is still transitioning.
998 */
999 ndlp = lpfc_ncmd->ndlp;
1000 if (!ndlp) {
1001 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1002 "6062 Ignoring NVME cmpl. No ndlp\n");
1003 goto out_err;
1004 }
1005
1006 code = bf_get(lpfc_wcqe_c_code, wcqe);
1007 if (code == CQE_CODE_NVME_ERSP) {
1008 /* For this type of CQE, we need to rebuild the rsp */
1009 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1010
1011 /*
1012 * Get Command Id from cmd to plug into response. This
1013 * code is not needed in the next NVME Transport drop.
1014 */
1015 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1016 cid = cp->sqe.common.command_id;
1017
1018 /*
1019 * RSN is in CQE word 2
1020 * SQHD is in CQE Word 3 bits 15:0
1021 * Cmd Specific info is in CQE Word 1
1022 * and in CQE Word 0 bits 15:0
1023 */
1024 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1025
1026 /* Now lets build the NVME ERSP IU */
1027 ep->iu_len = cpu_to_be16(8);
1028 ep->rsn = wcqe->parameter;
1029 ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1030 ep->rsvd12 = 0;
1031 ptr = (uint32_t *)&ep->cqe.result.u64;
1032 *ptr++ = wcqe->total_data_placed;
1033 data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1034 *ptr = (uint32_t)data;
1035 ep->cqe.sq_head = sqhd;
1036 ep->cqe.sq_id = nCmd->sqid;
1037 ep->cqe.command_id = cid;
1038 ep->cqe.status = 0;
1039
1040 lpfc_ncmd->status = IOSTAT_SUCCESS;
1041 lpfc_ncmd->result = 0;
1042 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1043 nCmd->transferred_length = nCmd->payload_length;
1044 } else {
1045 lpfc_ncmd->status = status;
1046 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1047
1048 /* For NVME, the only failure path that results in an
1049 * IO error is when the adapter rejects it. All other
1050 * conditions are a success case and resolved by the
1051 * transport.
1052 * IOSTAT_FCP_RSP_ERROR means:
1053 * 1. Length of data received doesn't match total
1054 * transfer length in WQE
1055 * 2. If the RSP payload does NOT match these cases:
1056 * a. RSP length 12/24 bytes and all zeros
1057 * b. NVME ERSP
1058 */
1059 switch (lpfc_ncmd->status) {
1060 case IOSTAT_SUCCESS:
1061 nCmd->transferred_length = wcqe->total_data_placed;
1062 nCmd->rcv_rsplen = 0;
1063 nCmd->status = 0;
1064 break;
1065 case IOSTAT_FCP_RSP_ERROR:
1066 nCmd->transferred_length = wcqe->total_data_placed;
1067 nCmd->rcv_rsplen = wcqe->parameter;
1068 nCmd->status = 0;
1069
1070 /* Get the NVME cmd details for this unique error. */
1071 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1072 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1073
1074 /* Check if this is really an ERSP */
1075 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {
1076 lpfc_ncmd->status = IOSTAT_SUCCESS;
1077 lpfc_ncmd->result = 0;
1078
1079 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1080 "6084 NVME FCP_ERR ERSP: "
1081 "xri %x placed x%x opcode x%x cmd_id "
1082 "x%x cqe_status x%x\n",
1083 lpfc_ncmd->cur_iocbq.sli4_xritag,
1084 wcqe->total_data_placed,
1085 cp->sqe.common.opcode,
1086 cp->sqe.common.command_id,
1087 ep->cqe.status);
1088 break;
1089 }
1090 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1091 "6081 NVME Completion Protocol Error: "
1092 "xri %x status x%x result x%x "
1093 "placed x%x opcode x%x cmd_id x%x, "
1094 "cqe_status x%x\n",
1095 lpfc_ncmd->cur_iocbq.sli4_xritag,
1096 lpfc_ncmd->status, lpfc_ncmd->result,
1097 wcqe->total_data_placed,
1098 cp->sqe.common.opcode,
1099 cp->sqe.common.command_id,
1100 ep->cqe.status);
1101 break;
1102 case IOSTAT_LOCAL_REJECT:
1103 /* Let fall through to set command final state. */
1104 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1105 lpfc_printf_vlog(vport, KERN_INFO,
1106 LOG_NVME_IOERR,
1107 "6032 Delay Aborted cmd x%px "
1108 "nvme cmd x%px, xri x%x, "
1109 "xb %d\n",
1110 lpfc_ncmd, nCmd,
1111 lpfc_ncmd->cur_iocbq.sli4_xritag,
1112 bf_get(lpfc_wcqe_c_xb, wcqe));
1113 fallthrough;
1114 default:
1115 out_err:
1116 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1117 "6072 NVME Completion Error: xri %x "
1118 "status x%x result x%x [x%x] "
1119 "placed x%x\n",
1120 lpfc_ncmd->cur_iocbq.sli4_xritag,
1121 lpfc_ncmd->status, lpfc_ncmd->result,
1122 wcqe->parameter,
1123 wcqe->total_data_placed);
1124 nCmd->transferred_length = 0;
1125 nCmd->rcv_rsplen = 0;
1126 nCmd->status = NVME_SC_INTERNAL;
1127 if (pci_channel_offline(vport->phba->pcidev) ||
1128 lpfc_ncmd->result == IOERR_SLI_DOWN)
1129 offline = true;
1130 }
1131 }
1132
1133 /* pick up SLI4 exhange busy condition */
1134 if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline)
1135 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1136 else
1137 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1138
1139 /* Update stats and complete the IO. There is
1140 * no need for dma unprep because the nvme_transport
1141 * owns the dma address.
1142 */
1143 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1144 if (lpfc_ncmd->ts_cmd_start) {
1145 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1146 lpfc_ncmd->ts_data_io = ktime_get_ns();
1147 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1148 lpfc_io_ktime(phba, lpfc_ncmd);
1149 }
1150 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1151 cpu = raw_smp_processor_id();
1152 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1153 if (lpfc_ncmd->cpu != cpu)
1154 lpfc_printf_vlog(vport,
1155 KERN_INFO, LOG_NVME_IOERR,
1156 "6701 CPU Check cmpl: "
1157 "cpu %d expect %d\n",
1158 cpu, lpfc_ncmd->cpu);
1159 }
1160 #endif
1161
1162 /* NVME targets need completion held off until the abort exchange
1163 * completes unless the NVME Rport is getting unregistered.
1164 */
1165
1166 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1167 freqpriv = nCmd->private;
1168 freqpriv->nvme_buf = NULL;
1169 lpfc_ncmd->nvmeCmd = NULL;
1170 call_done = true;
1171 }
1172 spin_unlock(&lpfc_ncmd->buf_lock);
1173
1174 /* Check if IO qualified for CMF */
1175 if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1176 nCmd->io_dir == NVMEFC_FCP_READ &&
1177 nCmd->payload_length) {
1178 /* Used when calculating average latency */
1179 lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;
1180 lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL);
1181 }
1182
1183 if (call_done)
1184 nCmd->done(nCmd);
1185
1186 /* Call release with XB=1 to queue the IO into the abort list. */
1187 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1188 }
1189
1190
1191 /**
1192 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1193 * @vport: pointer to a host virtual N_Port data structure
1194 * @lpfc_ncmd: Pointer to lpfc scsi command
1195 * @pnode: pointer to a node-list data structure
1196 * @cstat: pointer to the control status structure
1197 *
1198 * Driver registers this routine as it io request handler. This
1199 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1200 * data structure to the rport indicated in @lpfc_nvme_rport.
1201 *
1202 * Return value :
1203 * 0 - Success
1204 * TODO: What are the failure codes.
1205 **/
1206 static int
lpfc_nvme_prep_io_cmd(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct lpfc_nodelist * pnode,struct lpfc_fc4_ctrl_stat * cstat)1207 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1208 struct lpfc_io_buf *lpfc_ncmd,
1209 struct lpfc_nodelist *pnode,
1210 struct lpfc_fc4_ctrl_stat *cstat)
1211 {
1212 struct lpfc_hba *phba = vport->phba;
1213 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1214 struct nvme_common_command *sqe;
1215 struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;
1216 union lpfc_wqe128 *wqe = &pwqeq->wqe;
1217 uint32_t req_len;
1218
1219 /*
1220 * There are three possibilities here - use scatter-gather segment, use
1221 * the single mapping, or neither.
1222 */
1223 if (nCmd->sg_cnt) {
1224 if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1225 /* From the iwrite template, initialize words 7 - 11 */
1226 memcpy(&wqe->words[7],
1227 &lpfc_iwrite_cmd_template.words[7],
1228 sizeof(uint32_t) * 5);
1229
1230 /* Word 4 */
1231 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1232
1233 /* Word 5 */
1234 if ((phba->cfg_nvme_enable_fb) &&
1235 test_bit(NLP_FIRSTBURST, &pnode->nlp_flag)) {
1236 req_len = lpfc_ncmd->nvmeCmd->payload_length;
1237 if (req_len < pnode->nvme_fb_size)
1238 wqe->fcp_iwrite.initial_xfer_len =
1239 req_len;
1240 else
1241 wqe->fcp_iwrite.initial_xfer_len =
1242 pnode->nvme_fb_size;
1243 } else {
1244 wqe->fcp_iwrite.initial_xfer_len = 0;
1245 }
1246 cstat->output_requests++;
1247 } else {
1248 /* From the iread template, initialize words 7 - 11 */
1249 memcpy(&wqe->words[7],
1250 &lpfc_iread_cmd_template.words[7],
1251 sizeof(uint32_t) * 5);
1252
1253 /* Word 4 */
1254 wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1255
1256 /* Word 5 */
1257 wqe->fcp_iread.rsrvd5 = 0;
1258
1259 /* For a CMF Managed port, iod must be zero'ed */
1260 if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
1261 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
1262 LPFC_WQE_IOD_NONE);
1263 cstat->input_requests++;
1264 }
1265 } else {
1266 /* From the icmnd template, initialize words 4 - 11 */
1267 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1268 sizeof(uint32_t) * 8);
1269 cstat->control_requests++;
1270 }
1271
1272 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) {
1273 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1274 sqe = &((struct nvme_fc_cmd_iu *)
1275 nCmd->cmdaddr)->sqe.common;
1276 if (sqe->opcode == nvme_admin_async_event)
1277 bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1);
1278 }
1279
1280 /*
1281 * Finish initializing those WQE fields that are independent
1282 * of the nvme_cmnd request_buffer
1283 */
1284
1285 /* Word 3 */
1286 bf_set(payload_offset_len, &wqe->fcp_icmd,
1287 (nCmd->rsplen + nCmd->cmdlen));
1288
1289 /* Word 6 */
1290 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1291 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1292 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1293
1294 /* Word 8 */
1295 wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1296
1297 /* Word 9 */
1298 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1299
1300 /* Word 10 */
1301 bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);
1302
1303 /* Words 13 14 15 are for PBDE support */
1304
1305 /* add the VMID tags as per switch response */
1306 if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {
1307 if (phba->pport->vmid_priority_tagging) {
1308 bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
1309 bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
1310 lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);
1311 } else {
1312 bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
1313 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
1314 wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;
1315 }
1316 }
1317
1318 pwqeq->vport = vport;
1319 return 0;
1320 }
1321
1322
1323 /**
1324 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1325 * @vport: pointer to a host virtual N_Port data structure
1326 * @lpfc_ncmd: Pointer to lpfc scsi command
1327 *
1328 * Driver registers this routine as it io request handler. This
1329 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1330 * data structure to the rport indicated in @lpfc_nvme_rport.
1331 *
1332 * Return value :
1333 * 0 - Success
1334 * TODO: What are the failure codes.
1335 **/
1336 static int
lpfc_nvme_prep_io_dma(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd)1337 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1338 struct lpfc_io_buf *lpfc_ncmd)
1339 {
1340 struct lpfc_hba *phba = vport->phba;
1341 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1342 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1343 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1344 struct sli4_hybrid_sgl *sgl_xtra = NULL;
1345 struct scatterlist *data_sg;
1346 struct sli4_sge *first_data_sgl;
1347 struct ulp_bde64 *bde;
1348 dma_addr_t physaddr = 0;
1349 uint32_t dma_len = 0;
1350 uint32_t dma_offset = 0;
1351 int nseg, i, j;
1352 bool lsp_just_set = false;
1353
1354 /* Fix up the command and response DMA stuff. */
1355 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1356
1357 /*
1358 * There are three possibilities here - use scatter-gather segment, use
1359 * the single mapping, or neither.
1360 */
1361 if (nCmd->sg_cnt) {
1362 /*
1363 * Jump over the cmd and rsp SGEs. The fix routine
1364 * has already adjusted for this.
1365 */
1366 sgl += 2;
1367
1368 first_data_sgl = sgl;
1369 lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1370 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1371 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1372 "6058 Too many sg segments from "
1373 "NVME Transport. Max %d, "
1374 "nvmeIO sg_cnt %d\n",
1375 phba->cfg_nvme_seg_cnt + 1,
1376 lpfc_ncmd->seg_cnt);
1377 lpfc_ncmd->seg_cnt = 0;
1378 return 1;
1379 }
1380
1381 /*
1382 * The driver established a maximum scatter-gather segment count
1383 * during probe that limits the number of sg elements in any
1384 * single nvme command. Just run through the seg_cnt and format
1385 * the sge's.
1386 */
1387 nseg = nCmd->sg_cnt;
1388 data_sg = nCmd->first_sgl;
1389
1390 /* for tracking the segment boundaries */
1391 j = 2;
1392 for (i = 0; i < nseg; i++) {
1393 if (data_sg == NULL) {
1394 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1395 "6059 dptr err %d, nseg %d\n",
1396 i, nseg);
1397 lpfc_ncmd->seg_cnt = 0;
1398 return 1;
1399 }
1400
1401 sgl->word2 = 0;
1402 if (nseg == 1) {
1403 bf_set(lpfc_sli4_sge_last, sgl, 1);
1404 bf_set(lpfc_sli4_sge_type, sgl,
1405 LPFC_SGE_TYPE_DATA);
1406 } else {
1407 bf_set(lpfc_sli4_sge_last, sgl, 0);
1408
1409 /* expand the segment */
1410 if (!lsp_just_set &&
1411 !((j + 1) % phba->border_sge_num) &&
1412 ((nseg - 1) != i)) {
1413 /* set LSP type */
1414 bf_set(lpfc_sli4_sge_type, sgl,
1415 LPFC_SGE_TYPE_LSP);
1416
1417 sgl_xtra = lpfc_get_sgl_per_hdwq(
1418 phba, lpfc_ncmd);
1419
1420 if (unlikely(!sgl_xtra)) {
1421 lpfc_ncmd->seg_cnt = 0;
1422 return 1;
1423 }
1424 sgl->addr_lo = cpu_to_le32(putPaddrLow(
1425 sgl_xtra->dma_phys_sgl));
1426 sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1427 sgl_xtra->dma_phys_sgl));
1428
1429 } else {
1430 bf_set(lpfc_sli4_sge_type, sgl,
1431 LPFC_SGE_TYPE_DATA);
1432 }
1433 }
1434
1435 if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1436 LPFC_SGE_TYPE_LSP)) {
1437 if ((nseg - 1) == i)
1438 bf_set(lpfc_sli4_sge_last, sgl, 1);
1439
1440 physaddr = sg_dma_address(data_sg);
1441 dma_len = sg_dma_len(data_sg);
1442 sgl->addr_lo = cpu_to_le32(
1443 putPaddrLow(physaddr));
1444 sgl->addr_hi = cpu_to_le32(
1445 putPaddrHigh(physaddr));
1446
1447 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1448 sgl->word2 = cpu_to_le32(sgl->word2);
1449 sgl->sge_len = cpu_to_le32(dma_len);
1450
1451 dma_offset += dma_len;
1452 data_sg = sg_next(data_sg);
1453
1454 sgl++;
1455
1456 lsp_just_set = false;
1457 } else {
1458 sgl->word2 = cpu_to_le32(sgl->word2);
1459
1460 sgl->sge_len = cpu_to_le32(
1461 phba->cfg_sg_dma_buf_size);
1462
1463 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1464 i = i - 1;
1465
1466 lsp_just_set = true;
1467 }
1468
1469 j++;
1470 }
1471
1472 /* PBDE support for first data SGE only */
1473 if (nseg == 1 && phba->cfg_enable_pbde) {
1474 /* Words 13-15 */
1475 bde = (struct ulp_bde64 *)
1476 &wqe->words[13];
1477 bde->addrLow = first_data_sgl->addr_lo;
1478 bde->addrHigh = first_data_sgl->addr_hi;
1479 bde->tus.f.bdeSize =
1480 le32_to_cpu(first_data_sgl->sge_len);
1481 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1482 bde->tus.w = cpu_to_le32(bde->tus.w);
1483
1484 /* Word 11 - set PBDE bit */
1485 bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
1486 } else {
1487 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1488 /* Word 11 - PBDE bit disabled by default template */
1489 }
1490
1491 } else {
1492 lpfc_ncmd->seg_cnt = 0;
1493
1494 /* For this clause to be valid, the payload_length
1495 * and sg_cnt must zero.
1496 */
1497 if (nCmd->payload_length != 0) {
1498 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1499 "6063 NVME DMA Prep Err: sg_cnt %d "
1500 "payload_length x%x\n",
1501 nCmd->sg_cnt, nCmd->payload_length);
1502 return 1;
1503 }
1504 }
1505 return 0;
1506 }
1507
1508 /**
1509 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1510 * @pnvme_lport: Pointer to the driver's local port data
1511 * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1512 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1513 * @pnvme_fcreq: IO request from nvme fc to driver.
1514 *
1515 * Driver registers this routine as it io request handler. This
1516 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1517 * data structure to the rport indicated in @lpfc_nvme_rport.
1518 *
1519 * Return value :
1520 * 0 - Success
1521 * TODO: What are the failure codes.
1522 **/
1523 static int
lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1524 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1525 struct nvme_fc_remote_port *pnvme_rport,
1526 void *hw_queue_handle,
1527 struct nvmefc_fcp_req *pnvme_fcreq)
1528 {
1529 int ret = 0;
1530 int expedite = 0;
1531 int idx, cpu;
1532 struct lpfc_nvme_lport *lport;
1533 struct lpfc_fc4_ctrl_stat *cstat;
1534 struct lpfc_vport *vport;
1535 struct lpfc_hba *phba;
1536 struct lpfc_nodelist *ndlp;
1537 struct lpfc_io_buf *lpfc_ncmd;
1538 struct lpfc_nvme_rport *rport;
1539 struct lpfc_nvme_qhandle *lpfc_queue_info;
1540 struct lpfc_nvme_fcpreq_priv *freqpriv;
1541 struct nvme_common_command *sqe;
1542 uint64_t start = 0;
1543 #if (IS_ENABLED(CONFIG_NVME_FC))
1544 u8 *uuid = NULL;
1545 int err;
1546 enum dma_data_direction iodir;
1547 #endif
1548
1549 /* Validate pointers. LLDD fault handling with transport does
1550 * have timing races.
1551 */
1552 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1553 if (unlikely(!lport)) {
1554 ret = -EINVAL;
1555 goto out_fail;
1556 }
1557
1558 vport = lport->vport;
1559
1560 if (unlikely(!hw_queue_handle)) {
1561 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1562 "6117 Fail IO, NULL hw_queue_handle\n");
1563 atomic_inc(&lport->xmt_fcp_err);
1564 ret = -EBUSY;
1565 goto out_fail;
1566 }
1567
1568 phba = vport->phba;
1569
1570 if ((unlikely(test_bit(FC_UNLOADING, &vport->load_flag))) ||
1571 test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
1572 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1573 "6124 Fail IO, Driver unload\n");
1574 atomic_inc(&lport->xmt_fcp_err);
1575 ret = -ENODEV;
1576 goto out_fail;
1577 }
1578
1579 freqpriv = pnvme_fcreq->private;
1580 if (unlikely(!freqpriv)) {
1581 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1582 "6158 Fail IO, NULL request data\n");
1583 atomic_inc(&lport->xmt_fcp_err);
1584 ret = -EINVAL;
1585 goto out_fail;
1586 }
1587
1588 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1589 if (phba->ktime_on)
1590 start = ktime_get_ns();
1591 #endif
1592 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1593 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1594
1595 /*
1596 * Catch race where our node has transitioned, but the
1597 * transport is still transitioning.
1598 */
1599 ndlp = rport->ndlp;
1600 if (!ndlp) {
1601 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1602 "6053 Busy IO, ndlp not ready: rport x%px "
1603 "ndlp x%px, DID x%06x\n",
1604 rport, ndlp, pnvme_rport->port_id);
1605 atomic_inc(&lport->xmt_fcp_err);
1606 ret = -EBUSY;
1607 goto out_fail;
1608 }
1609
1610 /* The remote node has to be a mapped target or it's an error. */
1611 if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1612 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1613 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1614 "6036 Fail IO, DID x%06x not ready for "
1615 "IO. State x%x, Type x%x Flg x%x\n",
1616 pnvme_rport->port_id,
1617 ndlp->nlp_state, ndlp->nlp_type,
1618 ndlp->fc4_xpt_flags);
1619 atomic_inc(&lport->xmt_fcp_bad_ndlp);
1620 ret = -EBUSY;
1621 goto out_fail;
1622
1623 }
1624
1625 /* Currently only NVME Keep alive commands should be expedited
1626 * if the driver runs out of a resource. These should only be
1627 * issued on the admin queue, qidx 0
1628 */
1629 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1630 sqe = &((struct nvme_fc_cmd_iu *)
1631 pnvme_fcreq->cmdaddr)->sqe.common;
1632 if (sqe->opcode == nvme_admin_keep_alive)
1633 expedite = 1;
1634 }
1635
1636 /* Check if IO qualifies for CMF */
1637 if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1638 pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&
1639 pnvme_fcreq->payload_length) {
1640 ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length);
1641 if (ret) {
1642 ret = -EBUSY;
1643 goto out_fail;
1644 }
1645 /* Get start time for IO latency */
1646 start = ktime_get_ns();
1647 }
1648
1649 /* The node is shared with FCP IO, make sure the IO pending count does
1650 * not exceed the programmed depth.
1651 */
1652 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1653 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1654 !expedite) {
1655 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1656 "6174 Fail IO, ndlp qdepth exceeded: "
1657 "idx %d DID %x pend %d qdepth %d\n",
1658 lpfc_queue_info->index, ndlp->nlp_DID,
1659 atomic_read(&ndlp->cmd_pending),
1660 ndlp->cmd_qdepth);
1661 atomic_inc(&lport->xmt_fcp_qdepth);
1662 ret = -EBUSY;
1663 goto out_fail1;
1664 }
1665 }
1666
1667 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1668 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1669 idx = lpfc_queue_info->index;
1670 } else {
1671 cpu = raw_smp_processor_id();
1672 idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1673 }
1674
1675 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1676 if (lpfc_ncmd == NULL) {
1677 atomic_inc(&lport->xmt_fcp_noxri);
1678 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1679 "6065 Fail IO, driver buffer pool is empty: "
1680 "idx %d DID %x\n",
1681 lpfc_queue_info->index, ndlp->nlp_DID);
1682 ret = -EBUSY;
1683 goto out_fail1;
1684 }
1685 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1686 if (start) {
1687 lpfc_ncmd->ts_cmd_start = start;
1688 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1689 } else {
1690 lpfc_ncmd->ts_cmd_start = 0;
1691 }
1692 #endif
1693 lpfc_ncmd->rx_cmd_start = start;
1694
1695 /*
1696 * Store the data needed by the driver to issue, abort, and complete
1697 * an IO.
1698 * Do not let the IO hang out forever. There is no midlayer issuing
1699 * an abort so inform the FW of the maximum IO pending time.
1700 */
1701 freqpriv->nvme_buf = lpfc_ncmd;
1702 lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1703 lpfc_ncmd->ndlp = ndlp;
1704 lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1705
1706 #if (IS_ENABLED(CONFIG_NVME_FC))
1707 /* check the necessary and sufficient condition to support VMID */
1708 if (lpfc_is_vmid_enabled(phba) &&
1709 (ndlp->vmid_support ||
1710 phba->pport->vmid_priority_tagging ==
1711 LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
1712 /* is the I/O generated by a VM, get the associated virtual */
1713 /* entity id */
1714 uuid = nvme_fc_io_getuuid(pnvme_fcreq);
1715
1716 if (uuid) {
1717 if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE)
1718 iodir = DMA_TO_DEVICE;
1719 else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ)
1720 iodir = DMA_FROM_DEVICE;
1721 else
1722 iodir = DMA_NONE;
1723
1724 err = lpfc_vmid_get_appid(vport, uuid, iodir,
1725 (union lpfc_vmid_io_tag *)
1726 &lpfc_ncmd->cur_iocbq.vmid_tag);
1727 if (!err)
1728 lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;
1729 }
1730 }
1731 #endif
1732
1733 /*
1734 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1735 * This identfier was create in our hardware queue create callback
1736 * routine. The driver now is dependent on the IO queue steering from
1737 * the transport. We are trusting the upper NVME layers know which
1738 * index to use and that they have affinitized a CPU to this hardware
1739 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1740 */
1741 lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1742 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1743
1744 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1745 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1746 if (ret) {
1747 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1748 "6175 Fail IO, Prep DMA: "
1749 "idx %d DID %x\n",
1750 lpfc_queue_info->index, ndlp->nlp_DID);
1751 atomic_inc(&lport->xmt_fcp_err);
1752 ret = -ENOMEM;
1753 goto out_free_nvme_buf;
1754 }
1755
1756 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1757 lpfc_ncmd->cur_iocbq.sli4_xritag,
1758 lpfc_queue_info->index, ndlp->nlp_DID);
1759
1760 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1761 if (ret) {
1762 atomic_inc(&lport->xmt_fcp_wqerr);
1763 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1764 "6113 Fail IO, Could not issue WQE err %x "
1765 "sid: x%x did: x%x oxid: x%x\n",
1766 ret, vport->fc_myDID, ndlp->nlp_DID,
1767 lpfc_ncmd->cur_iocbq.sli4_xritag);
1768 goto out_free_nvme_buf;
1769 }
1770
1771 if (phba->cfg_xri_rebalancing)
1772 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1773
1774 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1775 if (lpfc_ncmd->ts_cmd_start)
1776 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1777
1778 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1779 cpu = raw_smp_processor_id();
1780 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1781 lpfc_ncmd->cpu = cpu;
1782 if (idx != cpu)
1783 lpfc_printf_vlog(vport,
1784 KERN_INFO, LOG_NVME_IOERR,
1785 "6702 CPU Check cmd: "
1786 "cpu %d wq %d\n",
1787 lpfc_ncmd->cpu,
1788 lpfc_queue_info->index);
1789 }
1790 #endif
1791 return 0;
1792
1793 out_free_nvme_buf:
1794 if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1795 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1796 cstat->output_requests--;
1797 else
1798 cstat->input_requests--;
1799 } else
1800 cstat->control_requests--;
1801 lpfc_release_nvme_buf(phba, lpfc_ncmd);
1802 out_fail1:
1803 lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,
1804 pnvme_fcreq->payload_length, NULL);
1805 out_fail:
1806 return ret;
1807 }
1808
1809 /**
1810 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1811 * @phba: Pointer to HBA context object
1812 * @cmdiocb: Pointer to command iocb object.
1813 * @rspiocb: Pointer to response iocb object.
1814 *
1815 * This is the callback function for any NVME FCP IO that was aborted.
1816 *
1817 * Return value:
1818 * None
1819 **/
1820 void
lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * cmdiocb,struct lpfc_iocbq * rspiocb)1821 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1822 struct lpfc_iocbq *rspiocb)
1823 {
1824 struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl;
1825
1826 lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1827 "6145 ABORT_XRI_CN completing on rpi x%x "
1828 "original iotag x%x, abort cmd iotag x%x "
1829 "req_tag x%x, status x%x, hwstatus x%x\n",
1830 bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com),
1831 get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag,
1832 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1833 bf_get(lpfc_wcqe_c_status, abts_cmpl),
1834 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1835 lpfc_sli_release_iocbq(phba, cmdiocb);
1836 }
1837
1838 /**
1839 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1840 * @pnvme_lport: Pointer to the driver's local port data
1841 * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1842 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1843 * @pnvme_fcreq: IO request from nvme fc to driver.
1844 *
1845 * Driver registers this routine as its nvme request io abort handler. This
1846 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1847 * data structure to the rport indicated in @lpfc_nvme_rport. This routine
1848 * is executed asynchronously - one the target is validated as "MAPPED" and
1849 * ready for IO, the driver issues the abort request and returns.
1850 *
1851 * Return value:
1852 * None
1853 **/
1854 static void
lpfc_nvme_fcp_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1855 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1856 struct nvme_fc_remote_port *pnvme_rport,
1857 void *hw_queue_handle,
1858 struct nvmefc_fcp_req *pnvme_fcreq)
1859 {
1860 struct lpfc_nvme_lport *lport;
1861 struct lpfc_vport *vport;
1862 struct lpfc_hba *phba;
1863 struct lpfc_io_buf *lpfc_nbuf;
1864 struct lpfc_iocbq *nvmereq_wqe;
1865 struct lpfc_nvme_fcpreq_priv *freqpriv;
1866 unsigned long flags;
1867 int ret_val;
1868
1869 /* Validate pointers. LLDD fault handling with transport does
1870 * have timing races.
1871 */
1872 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1873 if (unlikely(!lport))
1874 return;
1875
1876 vport = lport->vport;
1877
1878 if (unlikely(!hw_queue_handle)) {
1879 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1880 "6129 Fail Abort, HW Queue Handle NULL.\n");
1881 return;
1882 }
1883
1884 phba = vport->phba;
1885 freqpriv = pnvme_fcreq->private;
1886
1887 if (unlikely(!freqpriv))
1888 return;
1889 if (test_bit(FC_UNLOADING, &vport->load_flag))
1890 return;
1891
1892 /* Announce entry to new IO submit field. */
1893 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1894 "6002 Abort Request to rport DID x%06x "
1895 "for nvme_fc_req x%px\n",
1896 pnvme_rport->port_id,
1897 pnvme_fcreq);
1898
1899 lpfc_nbuf = freqpriv->nvme_buf;
1900 if (!lpfc_nbuf) {
1901 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1902 "6140 NVME IO req has no matching lpfc nvme "
1903 "io buffer. Skipping abort req.\n");
1904 return;
1905 } else if (!lpfc_nbuf->nvmeCmd) {
1906 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1907 "6141 lpfc NVME IO req has no nvme_fcreq "
1908 "io buffer. Skipping abort req.\n");
1909 return;
1910 }
1911
1912 /* driver queued commands are in process of being flushed */
1913 if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
1914 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1915 "6139 Driver in reset cleanup - flushing "
1916 "NVME Req now. hba_flag x%lx\n",
1917 phba->hba_flag);
1918 return;
1919 }
1920
1921 /* Guard against IO completion being called at same time */
1922 spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags);
1923 spin_lock(&phba->hbalock);
1924
1925 nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1926
1927 /*
1928 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1929 * state must match the nvme_fcreq passed by the nvme
1930 * transport. If they don't match, it is likely the driver
1931 * has already completed the NVME IO and the nvme transport
1932 * has not seen it yet.
1933 */
1934 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1935 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1936 "6143 NVME req mismatch: "
1937 "lpfc_nbuf x%px nvmeCmd x%px, "
1938 "pnvme_fcreq x%px. Skipping Abort xri x%x\n",
1939 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1940 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1941 goto out_unlock;
1942 }
1943
1944 /* Don't abort IOs no longer on the pending queue. */
1945 if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
1946 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1947 "6142 NVME IO req x%px not queued - skipping "
1948 "abort req xri x%x\n",
1949 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1950 goto out_unlock;
1951 }
1952
1953 atomic_inc(&lport->xmt_fcp_abort);
1954 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1955 nvmereq_wqe->sli4_xritag,
1956 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1957
1958 /* Outstanding abort is in progress */
1959 if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) {
1960 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1961 "6144 Outstanding NVME I/O Abort Request "
1962 "still pending on nvme_fcreq x%px, "
1963 "lpfc_ncmd x%px xri x%x\n",
1964 pnvme_fcreq, lpfc_nbuf,
1965 nvmereq_wqe->sli4_xritag);
1966 goto out_unlock;
1967 }
1968
1969 ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe,
1970 lpfc_nvme_abort_fcreq_cmpl);
1971
1972 spin_unlock(&phba->hbalock);
1973 spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1974
1975 /* Make sure HBA is alive */
1976 lpfc_issue_hb_tmo(phba);
1977
1978 if (ret_val != WQE_SUCCESS) {
1979 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1980 "6137 Failed abts issue_wqe with status x%x "
1981 "for nvme_fcreq x%px.\n",
1982 ret_val, pnvme_fcreq);
1983 return;
1984 }
1985
1986 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1987 "6138 Transport Abort NVME Request Issued for "
1988 "ox_id x%x\n",
1989 nvmereq_wqe->sli4_xritag);
1990 return;
1991
1992 out_unlock:
1993 spin_unlock(&phba->hbalock);
1994 spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1995 return;
1996 }
1997
1998 /* Declare and initialization an instance of the FC NVME template. */
1999 static struct nvme_fc_port_template lpfc_nvme_template = {
2000 /* initiator-based functions */
2001 .localport_delete = lpfc_nvme_localport_delete,
2002 .remoteport_delete = lpfc_nvme_remoteport_delete,
2003 .create_queue = lpfc_nvme_create_queue,
2004 .delete_queue = lpfc_nvme_delete_queue,
2005 .ls_req = lpfc_nvme_ls_req,
2006 .fcp_io = lpfc_nvme_fcp_io_submit,
2007 .ls_abort = lpfc_nvme_ls_abort,
2008 .fcp_abort = lpfc_nvme_fcp_abort,
2009 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,
2010
2011 .max_hw_queues = 1,
2012 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2013 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2014 .dma_boundary = 0xFFFFFFFF,
2015
2016 /* Sizes of additional private data for data structures.
2017 * No use for the last two sizes at this time.
2018 */
2019 .local_priv_sz = sizeof(struct lpfc_nvme_lport),
2020 .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2021 .lsrqst_priv_sz = 0,
2022 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2023 };
2024
2025 /*
2026 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2027 *
2028 * This routine removes a nvme buffer from head of @hdwq io_buf_list
2029 * and returns to caller.
2030 *
2031 * Return codes:
2032 * NULL - Error
2033 * Pointer to lpfc_nvme_buf - Success
2034 **/
2035 static struct lpfc_io_buf *
lpfc_get_nvme_buf(struct lpfc_hba * phba,struct lpfc_nodelist * ndlp,int idx,int expedite)2036 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2037 int idx, int expedite)
2038 {
2039 struct lpfc_io_buf *lpfc_ncmd;
2040 struct lpfc_sli4_hdw_queue *qp;
2041 struct sli4_sge *sgl;
2042 struct lpfc_iocbq *pwqeq;
2043 union lpfc_wqe128 *wqe;
2044
2045 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2046
2047 if (lpfc_ncmd) {
2048 pwqeq = &(lpfc_ncmd->cur_iocbq);
2049 wqe = &pwqeq->wqe;
2050
2051 /* Setup key fields in buffer that may have been changed
2052 * if other protocols used this buffer.
2053 */
2054 pwqeq->cmd_flag = LPFC_IO_NVME;
2055 pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl;
2056 lpfc_ncmd->start_time = jiffies;
2057 lpfc_ncmd->flags = 0;
2058
2059 /* Rsp SGE will be filled in when we rcv an IO
2060 * from the NVME Layer to be sent.
2061 * The cmd is going to be embedded so we need a SKIP SGE.
2062 */
2063 sgl = lpfc_ncmd->dma_sgl;
2064 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2065 bf_set(lpfc_sli4_sge_last, sgl, 0);
2066 sgl->word2 = cpu_to_le32(sgl->word2);
2067 /* Fill in word 3 / sgl_len during cmd submission */
2068
2069 /* Initialize 64 bytes only */
2070 memset(wqe, 0, sizeof(union lpfc_wqe));
2071
2072 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2073 atomic_inc(&ndlp->cmd_pending);
2074 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2075 }
2076
2077 } else {
2078 qp = &phba->sli4_hba.hdwq[idx];
2079 qp->empty_io_bufs++;
2080 }
2081
2082 return lpfc_ncmd;
2083 }
2084
2085 /**
2086 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2087 * @phba: The Hba for which this call is being executed.
2088 * @lpfc_ncmd: The nvme buffer which is being released.
2089 *
2090 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2091 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2092 * and cannot be reused for at least RA_TOV amount of time if it was
2093 * aborted.
2094 **/
2095 static void
lpfc_release_nvme_buf(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2096 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2097 {
2098 struct lpfc_sli4_hdw_queue *qp;
2099 unsigned long iflag = 0;
2100
2101 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2102 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2103
2104 lpfc_ncmd->ndlp = NULL;
2105 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2106
2107 qp = lpfc_ncmd->hdwq;
2108 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2109 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2110 "6310 XB release deferred for "
2111 "ox_id x%x on reqtag x%x\n",
2112 lpfc_ncmd->cur_iocbq.sli4_xritag,
2113 lpfc_ncmd->cur_iocbq.iotag);
2114
2115 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2116 list_add_tail(&lpfc_ncmd->list,
2117 &qp->lpfc_abts_io_buf_list);
2118 qp->abts_nvme_io_bufs++;
2119 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2120 } else
2121 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2122 }
2123
2124 /**
2125 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2126 * @vport: the lpfc_vport instance requesting a localport.
2127 *
2128 * This routine is invoked to create an nvme localport instance to bind
2129 * to the nvme_fc_transport. It is called once during driver load
2130 * like lpfc_create_shost after all other services are initialized.
2131 * It requires a vport, vpi, and wwns at call time. Other localport
2132 * parameters are modified as the driver's FCID and the Fabric WWN
2133 * are established.
2134 *
2135 * Return codes
2136 * 0 - successful
2137 * -ENOMEM - no heap memory available
2138 * other values - from nvme registration upcall
2139 **/
2140 int
lpfc_nvme_create_localport(struct lpfc_vport * vport)2141 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2142 {
2143 int ret = 0;
2144 struct lpfc_hba *phba = vport->phba;
2145 struct nvme_fc_port_info nfcp_info;
2146 struct nvme_fc_local_port *localport;
2147 struct lpfc_nvme_lport *lport;
2148
2149 /* Initialize this localport instance. The vport wwn usage ensures
2150 * that NPIV is accounted for.
2151 */
2152 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2153 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2154 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2155 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2156
2157 /* We need to tell the transport layer + 1 because it takes page
2158 * alignment into account. When space for the SGL is allocated we
2159 * allocate + 3, one for cmd, one for rsp and one for this alignment
2160 */
2161 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2162
2163 /* Advertise how many hw queues we support based on cfg_hdw_queue,
2164 * which will not exceed cpu count.
2165 */
2166 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2167
2168 if (!IS_ENABLED(CONFIG_NVME_FC))
2169 return ret;
2170
2171 /* localport is allocated from the stack, but the registration
2172 * call allocates heap memory as well as the private area.
2173 */
2174
2175 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2176 &vport->phba->pcidev->dev, &localport);
2177 if (!ret) {
2178 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2179 "6005 Successfully registered local "
2180 "NVME port num %d, localP x%px, private "
2181 "x%px, sg_seg %d\n",
2182 localport->port_num, localport,
2183 localport->private,
2184 lpfc_nvme_template.max_sgl_segments);
2185
2186 /* Private is our lport size declared in the template. */
2187 lport = (struct lpfc_nvme_lport *)localport->private;
2188 vport->localport = localport;
2189 lport->vport = vport;
2190 vport->nvmei_support = 1;
2191
2192 atomic_set(&lport->xmt_fcp_noxri, 0);
2193 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2194 atomic_set(&lport->xmt_fcp_qdepth, 0);
2195 atomic_set(&lport->xmt_fcp_err, 0);
2196 atomic_set(&lport->xmt_fcp_wqerr, 0);
2197 atomic_set(&lport->xmt_fcp_abort, 0);
2198 atomic_set(&lport->xmt_ls_abort, 0);
2199 atomic_set(&lport->xmt_ls_err, 0);
2200 atomic_set(&lport->cmpl_fcp_xb, 0);
2201 atomic_set(&lport->cmpl_fcp_err, 0);
2202 atomic_set(&lport->cmpl_ls_xb, 0);
2203 atomic_set(&lport->cmpl_ls_err, 0);
2204
2205 atomic_set(&lport->fc4NvmeLsRequests, 0);
2206 atomic_set(&lport->fc4NvmeLsCmpls, 0);
2207 }
2208
2209 return ret;
2210 }
2211
2212 #if (IS_ENABLED(CONFIG_NVME_FC))
2213 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2214 *
2215 * The driver has to wait for the host nvme transport to callback
2216 * indicating the localport has successfully unregistered all
2217 * resources. Since this is an uninterruptible wait, loop every ten
2218 * seconds and print a message indicating no progress.
2219 *
2220 * An uninterruptible wait is used because of the risk of transport-to-
2221 * driver state mismatch.
2222 */
2223 static void
lpfc_nvme_lport_unreg_wait(struct lpfc_vport * vport,struct lpfc_nvme_lport * lport,struct completion * lport_unreg_cmp)2224 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2225 struct lpfc_nvme_lport *lport,
2226 struct completion *lport_unreg_cmp)
2227 {
2228 u32 wait_tmo;
2229 int ret, i, pending = 0;
2230 struct lpfc_sli_ring *pring;
2231 struct lpfc_hba *phba = vport->phba;
2232 struct lpfc_sli4_hdw_queue *qp;
2233 int abts_scsi, abts_nvme;
2234 u16 nvmels_cnt;
2235
2236 /* Host transport has to clean up and confirm requiring an indefinite
2237 * wait. Print a message if a 10 second wait expires and renew the
2238 * wait. This is unexpected.
2239 */
2240 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2241 while (true) {
2242 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2243 if (unlikely(!ret)) {
2244 pending = 0;
2245 abts_scsi = 0;
2246 abts_nvme = 0;
2247 nvmels_cnt = 0;
2248 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2249 qp = &phba->sli4_hba.hdwq[i];
2250 if (!vport->localport || !qp || !qp->io_wq)
2251 return;
2252
2253 pring = qp->io_wq->pring;
2254 if (!pring)
2255 continue;
2256 pending += pring->txcmplq_cnt;
2257 abts_scsi += qp->abts_scsi_io_bufs;
2258 abts_nvme += qp->abts_nvme_io_bufs;
2259 }
2260 if (phba->sli4_hba.nvmels_wq) {
2261 pring = phba->sli4_hba.nvmels_wq->pring;
2262 if (pring)
2263 nvmels_cnt = pring->txcmplq_cnt;
2264 }
2265 if (!vport->localport ||
2266 test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
2267 phba->link_state == LPFC_HBA_ERROR ||
2268 test_bit(FC_UNLOADING, &vport->load_flag))
2269 return;
2270
2271 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2272 "6176 Lport x%px Localport x%px wait "
2273 "timed out. Pending %d [%d:%d:%d]. "
2274 "Renewing.\n",
2275 lport, vport->localport, pending,
2276 abts_scsi, abts_nvme, nvmels_cnt);
2277 continue;
2278 }
2279 break;
2280 }
2281 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2282 "6177 Lport x%px Localport x%px Complete Success\n",
2283 lport, vport->localport);
2284 }
2285 #endif
2286
2287 /**
2288 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2289 * @vport: pointer to a host virtual N_Port data structure
2290 *
2291 * This routine is invoked to destroy all lports bound to the phba.
2292 * The lport memory was allocated by the nvme fc transport and is
2293 * released there. This routine ensures all rports bound to the
2294 * lport have been disconnected.
2295 *
2296 **/
2297 void
lpfc_nvme_destroy_localport(struct lpfc_vport * vport)2298 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2299 {
2300 #if (IS_ENABLED(CONFIG_NVME_FC))
2301 struct nvme_fc_local_port *localport;
2302 struct lpfc_nvme_lport *lport;
2303 int ret;
2304 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2305
2306 if (vport->nvmei_support == 0)
2307 return;
2308
2309 localport = vport->localport;
2310 if (!localport)
2311 return;
2312 lport = (struct lpfc_nvme_lport *)localport->private;
2313
2314 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2315 "6011 Destroying NVME localport x%px\n",
2316 localport);
2317
2318 /* lport's rport list is clear. Unregister
2319 * lport and release resources.
2320 */
2321 lport->lport_unreg_cmp = &lport_unreg_cmp;
2322 ret = nvme_fc_unregister_localport(localport);
2323
2324 /* Wait for completion. This either blocks
2325 * indefinitely or succeeds
2326 */
2327 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2328 vport->localport = NULL;
2329
2330 /* Regardless of the unregister upcall response, clear
2331 * nvmei_support. All rports are unregistered and the
2332 * driver will clean up.
2333 */
2334 vport->nvmei_support = 0;
2335 if (ret == 0) {
2336 lpfc_printf_vlog(vport,
2337 KERN_INFO, LOG_NVME_DISC,
2338 "6009 Unregistered lport Success\n");
2339 } else {
2340 lpfc_printf_vlog(vport,
2341 KERN_INFO, LOG_NVME_DISC,
2342 "6010 Unregistered lport "
2343 "Failed, status x%x\n",
2344 ret);
2345 }
2346 #endif
2347 }
2348
2349 void
lpfc_nvme_update_localport(struct lpfc_vport * vport)2350 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2351 {
2352 #if (IS_ENABLED(CONFIG_NVME_FC))
2353 struct nvme_fc_local_port *localport;
2354 struct lpfc_nvme_lport *lport;
2355
2356 localport = vport->localport;
2357 if (!localport) {
2358 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2359 "6710 Update NVME fail. No localport\n");
2360 return;
2361 }
2362 lport = (struct lpfc_nvme_lport *)localport->private;
2363 if (!lport) {
2364 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2365 "6171 Update NVME fail. localP x%px, No lport\n",
2366 localport);
2367 return;
2368 }
2369 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2370 "6012 Update NVME lport x%px did x%x\n",
2371 localport, vport->fc_myDID);
2372
2373 localport->port_id = vport->fc_myDID;
2374 if (localport->port_id == 0)
2375 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2376 else
2377 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2378
2379 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2380 "6030 bound lport x%px to DID x%06x\n",
2381 lport, localport->port_id);
2382 #endif
2383 }
2384
2385 int
lpfc_nvme_register_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2386 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2387 {
2388 #if (IS_ENABLED(CONFIG_NVME_FC))
2389 int ret = 0;
2390 struct nvme_fc_local_port *localport;
2391 struct lpfc_nvme_lport *lport;
2392 struct lpfc_nvme_rport *rport;
2393 struct lpfc_nvme_rport *oldrport;
2394 struct nvme_fc_remote_port *remote_port;
2395 struct nvme_fc_port_info rpinfo;
2396 struct lpfc_nodelist *prev_ndlp = NULL;
2397 struct fc_rport *srport = ndlp->rport;
2398
2399 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2400 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2401 ndlp->nlp_DID, ndlp->nlp_type);
2402
2403 localport = vport->localport;
2404 if (!localport)
2405 return 0;
2406
2407 lport = (struct lpfc_nvme_lport *)localport->private;
2408
2409 /* NVME rports are not preserved across devloss.
2410 * Just register this instance. Note, rpinfo->dev_loss_tmo
2411 * is left 0 to indicate accept transport defaults. The
2412 * driver communicates port role capabilities consistent
2413 * with the PRLI response data.
2414 */
2415 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2416 rpinfo.port_id = ndlp->nlp_DID;
2417 if (ndlp->nlp_type & NLP_NVME_TARGET)
2418 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2419 if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2420 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2421
2422 if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2423 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2424
2425 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2426 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2427 if (srport)
2428 rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2429 else
2430 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2431
2432 spin_lock_irq(&ndlp->lock);
2433
2434 /* If an oldrport exists, so does the ndlp reference. If not
2435 * a new reference is needed because either the node has never
2436 * been registered or it's been unregistered and getting deleted.
2437 */
2438 oldrport = lpfc_ndlp_get_nrport(ndlp);
2439 if (oldrport) {
2440 prev_ndlp = oldrport->ndlp;
2441 spin_unlock_irq(&ndlp->lock);
2442 } else {
2443 spin_unlock_irq(&ndlp->lock);
2444 if (!lpfc_nlp_get(ndlp)) {
2445 dev_warn(&vport->phba->pcidev->dev,
2446 "Warning - No node ref - exit register\n");
2447 return 0;
2448 }
2449 }
2450
2451 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2452 if (!ret) {
2453 /* If the ndlp already has an nrport, this is just
2454 * a resume of the existing rport. Else this is a
2455 * new rport.
2456 */
2457 /* Guard against an unregister/reregister
2458 * race that leaves the WAIT flag set.
2459 */
2460 spin_lock_irq(&ndlp->lock);
2461 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2462 ndlp->fc4_xpt_flags |= NVME_XPT_REGD;
2463 spin_unlock_irq(&ndlp->lock);
2464 rport = remote_port->private;
2465 if (oldrport) {
2466
2467 /* Sever the ndlp<->rport association
2468 * before dropping the ndlp ref from
2469 * register.
2470 */
2471 spin_lock_irq(&ndlp->lock);
2472 ndlp->nrport = NULL;
2473 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2474 spin_unlock_irq(&ndlp->lock);
2475 rport->ndlp = NULL;
2476 rport->remoteport = NULL;
2477
2478 /* Reference only removed if previous NDLP is no longer
2479 * active. It might be just a swap and removing the
2480 * reference would cause a premature cleanup.
2481 */
2482 if (prev_ndlp && prev_ndlp != ndlp) {
2483 if (!prev_ndlp->nrport)
2484 lpfc_nlp_put(prev_ndlp);
2485 }
2486 }
2487
2488 /* Clean bind the rport to the ndlp. */
2489 rport->remoteport = remote_port;
2490 rport->lport = lport;
2491 rport->ndlp = ndlp;
2492 spin_lock_irq(&ndlp->lock);
2493 ndlp->nrport = rport;
2494 spin_unlock_irq(&ndlp->lock);
2495 lpfc_printf_vlog(vport, KERN_INFO,
2496 LOG_NVME_DISC | LOG_NODE,
2497 "6022 Bind lport x%px to remoteport x%px "
2498 "rport x%px WWNN 0x%llx, "
2499 "Rport WWPN 0x%llx DID "
2500 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2501 lport, remote_port, rport,
2502 rpinfo.node_name, rpinfo.port_name,
2503 rpinfo.port_id, rpinfo.port_role,
2504 ndlp, prev_ndlp);
2505 } else {
2506 lpfc_printf_vlog(vport, KERN_ERR,
2507 LOG_TRACE_EVENT,
2508 "6031 RemotePort Registration failed "
2509 "err: %d, DID x%06x ref %u\n",
2510 ret, ndlp->nlp_DID, kref_read(&ndlp->kref));
2511 lpfc_nlp_put(ndlp);
2512 }
2513
2514 return ret;
2515 #else
2516 return 0;
2517 #endif
2518 }
2519
2520 /*
2521 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2522 *
2523 * If the ndlp represents an NVME Target, that we are logged into,
2524 * ping the NVME FC Transport layer to initiate a device rescan
2525 * on this remote NPort.
2526 */
2527 void
lpfc_nvme_rescan_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2528 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2529 {
2530 #if (IS_ENABLED(CONFIG_NVME_FC))
2531 struct lpfc_nvme_rport *nrport;
2532 struct nvme_fc_remote_port *remoteport = NULL;
2533
2534 spin_lock_irq(&ndlp->lock);
2535 nrport = lpfc_ndlp_get_nrport(ndlp);
2536 if (nrport)
2537 remoteport = nrport->remoteport;
2538 spin_unlock_irq(&ndlp->lock);
2539
2540 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2541 "6170 Rescan NPort DID x%06x type x%x "
2542 "state x%x nrport x%px remoteport x%px\n",
2543 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2544 nrport, remoteport);
2545
2546 if (!nrport || !remoteport)
2547 goto rescan_exit;
2548
2549 /* Rescan an NVME target in MAPPED state with DISCOVERY role set */
2550 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2551 ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2552 nvme_fc_rescan_remoteport(remoteport);
2553
2554 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2555 "6172 NVME rescanned DID x%06x "
2556 "port_state x%x\n",
2557 ndlp->nlp_DID, remoteport->port_state);
2558 }
2559 return;
2560 rescan_exit:
2561 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2562 "6169 Skip NVME Rport Rescan, NVME remoteport "
2563 "unregistered\n");
2564 #endif
2565 }
2566
2567 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2568 *
2569 * There is no notion of Devloss or rport recovery from the current
2570 * nvme_transport perspective. Loss of an rport just means IO cannot
2571 * be sent and recovery is completely up to the initator.
2572 * For now, the driver just unbinds the DID and port_role so that
2573 * no further IO can be issued.
2574 */
2575 void
lpfc_nvme_unregister_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2576 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2577 {
2578 #if (IS_ENABLED(CONFIG_NVME_FC))
2579 int ret;
2580 struct nvme_fc_local_port *localport;
2581 struct lpfc_nvme_lport *lport;
2582 struct lpfc_nvme_rport *rport;
2583 struct nvme_fc_remote_port *remoteport = NULL;
2584
2585 localport = vport->localport;
2586
2587 /* This is fundamental error. The localport is always
2588 * available until driver unload. Just exit.
2589 */
2590 if (!localport)
2591 return;
2592
2593 lport = (struct lpfc_nvme_lport *)localport->private;
2594 if (!lport)
2595 goto input_err;
2596
2597 spin_lock_irq(&ndlp->lock);
2598 rport = lpfc_ndlp_get_nrport(ndlp);
2599 if (rport)
2600 remoteport = rport->remoteport;
2601 spin_unlock_irq(&ndlp->lock);
2602 if (!remoteport)
2603 goto input_err;
2604
2605 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2606 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2607 "port_id x%06x, portstate x%x port type x%x "
2608 "refcnt %d\n",
2609 remoteport, remoteport->port_name,
2610 remoteport->port_id, remoteport->port_state,
2611 ndlp->nlp_type, kref_read(&ndlp->kref));
2612
2613 /* Sanity check ndlp type. Only call for NVME ports. Don't
2614 * clear any rport state until the transport calls back.
2615 */
2616
2617 if (ndlp->nlp_type & NLP_NVME_TARGET) {
2618 /* No concern about the role change on the nvme remoteport.
2619 * The transport will update it.
2620 */
2621 spin_lock_irq(&ndlp->lock);
2622 ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;
2623 spin_unlock_irq(&ndlp->lock);
2624
2625 /* Don't let the host nvme transport keep sending keep-alives
2626 * on this remoteport. Vport is unloading, no recovery. The
2627 * return values is ignored. The upcall is a courtesy to the
2628 * transport.
2629 */
2630 if (test_bit(FC_UNLOADING, &vport->load_flag) ||
2631 unlikely(vport->phba->link_state == LPFC_HBA_ERROR))
2632 (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2633
2634 ret = nvme_fc_unregister_remoteport(remoteport);
2635
2636 /* The driver no longer knows if the nrport memory is valid.
2637 * because the controller teardown process has begun and
2638 * is asynchronous. Break the binding in the ndlp. Also
2639 * remove the register ndlp reference to setup node release.
2640 */
2641 ndlp->nrport = NULL;
2642 lpfc_nlp_put(ndlp);
2643 if (ret != 0) {
2644 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2645 "6167 NVME unregister failed %d "
2646 "port_state x%x\n",
2647 ret, remoteport->port_state);
2648
2649 if (test_bit(FC_UNLOADING, &vport->load_flag)) {
2650 /* Only 1 thread can drop the initial node
2651 * reference. Check if another thread has set
2652 * NLP_DROPPED.
2653 */
2654 if (!test_and_set_bit(NLP_DROPPED,
2655 &ndlp->nlp_flag)) {
2656 lpfc_nlp_put(ndlp);
2657 return;
2658 }
2659 }
2660 }
2661 }
2662 return;
2663
2664 input_err:
2665 #endif
2666 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2667 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2668 vport->localport, ndlp->rport, ndlp->nlp_DID);
2669 }
2670
2671 /**
2672 * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort
2673 * @phba: pointer to lpfc hba data structure.
2674 * @lpfc_ncmd: The nvme job structure for the request being aborted.
2675 *
2676 * This routine is invoked by the worker thread to process a SLI4 fast-path
2677 * NVME aborted xri. Aborted NVME IO commands are completed to the transport
2678 * here.
2679 **/
2680 void
lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2681 lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba,
2682 struct lpfc_io_buf *lpfc_ncmd)
2683 {
2684 struct nvmefc_fcp_req *nvme_cmd = NULL;
2685
2686 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2687 "6533 %s nvme_cmd %p tag x%x abort complete and "
2688 "xri released\n", __func__,
2689 lpfc_ncmd->nvmeCmd,
2690 lpfc_ncmd->cur_iocbq.iotag);
2691
2692 /* Aborted NVME commands are required to not complete
2693 * before the abort exchange command fully completes.
2694 * Once completed, it is available via the put list.
2695 */
2696 if (lpfc_ncmd->nvmeCmd) {
2697 nvme_cmd = lpfc_ncmd->nvmeCmd;
2698 nvme_cmd->transferred_length = 0;
2699 nvme_cmd->rcv_rsplen = 0;
2700 nvme_cmd->status = NVME_SC_INTERNAL;
2701 nvme_cmd->done(nvme_cmd);
2702 lpfc_ncmd->nvmeCmd = NULL;
2703 }
2704 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2705 }
2706
2707 /**
2708 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2709 * @phba: pointer to lpfc hba data structure.
2710 * @axri: pointer to the fcp xri abort wcqe structure.
2711 * @lpfc_ncmd: The nvme job structure for the request being aborted.
2712 *
2713 * This routine is invoked by the worker thread to process a SLI4 fast-path
2714 * NVME aborted xri. Aborted NVME IO commands are completed to the transport
2715 * here.
2716 **/
2717 void
lpfc_sli4_nvme_xri_aborted(struct lpfc_hba * phba,struct sli4_wcqe_xri_aborted * axri,struct lpfc_io_buf * lpfc_ncmd)2718 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2719 struct sli4_wcqe_xri_aborted *axri,
2720 struct lpfc_io_buf *lpfc_ncmd)
2721 {
2722 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2723 struct nvmefc_fcp_req *nvme_cmd = NULL;
2724 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2725
2726
2727 if (ndlp)
2728 lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2729
2730 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2731 "6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2732 "xri released\n",
2733 lpfc_ncmd->nvmeCmd, xri,
2734 lpfc_ncmd->cur_iocbq.iotag);
2735
2736 /* Aborted NVME commands are required to not complete
2737 * before the abort exchange command fully completes.
2738 * Once completed, it is available via the put list.
2739 */
2740 if (lpfc_ncmd->nvmeCmd) {
2741 nvme_cmd = lpfc_ncmd->nvmeCmd;
2742 nvme_cmd->done(nvme_cmd);
2743 lpfc_ncmd->nvmeCmd = NULL;
2744 }
2745 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2746 }
2747
2748 /**
2749 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2750 * @phba: Pointer to HBA context object.
2751 *
2752 * This function flushes all wqes in the nvme rings and frees all resources
2753 * in the txcmplq. This function does not issue abort wqes for the IO
2754 * commands in txcmplq, they will just be returned with
2755 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2756 * slot has been permanently disabled.
2757 **/
2758 void
lpfc_nvme_wait_for_io_drain(struct lpfc_hba * phba)2759 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2760 {
2761 struct lpfc_sli_ring *pring;
2762 u32 i, wait_cnt = 0;
2763
2764 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2765 return;
2766
2767 /* Cycle through all IO rings and make sure all outstanding
2768 * WQEs have been removed from the txcmplqs.
2769 */
2770 for (i = 0; i < phba->cfg_hdw_queue; i++) {
2771 if (!phba->sli4_hba.hdwq[i].io_wq)
2772 continue;
2773 pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2774
2775 if (!pring)
2776 continue;
2777
2778 /* Retrieve everything on the txcmplq */
2779 while (!list_empty(&pring->txcmplq)) {
2780 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2781 wait_cnt++;
2782
2783 /* The sleep is 10mS. Every ten seconds,
2784 * dump a message. Something is wrong.
2785 */
2786 if ((wait_cnt % 1000) == 0) {
2787 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2788 "6178 NVME IO not empty, "
2789 "cnt %d\n", wait_cnt);
2790 }
2791 }
2792 }
2793
2794 /* Make sure HBA is alive */
2795 lpfc_issue_hb_tmo(phba);
2796
2797 }
2798
2799 void
lpfc_nvme_cancel_iocb(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,uint32_t stat,uint32_t param)2800 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
2801 uint32_t stat, uint32_t param)
2802 {
2803 #if (IS_ENABLED(CONFIG_NVME_FC))
2804 struct lpfc_io_buf *lpfc_ncmd;
2805 struct nvmefc_fcp_req *nCmd;
2806 struct lpfc_wcqe_complete wcqe;
2807 struct lpfc_wcqe_complete *wcqep = &wcqe;
2808
2809 lpfc_ncmd = pwqeIn->io_buf;
2810 if (!lpfc_ncmd) {
2811 lpfc_sli_release_iocbq(phba, pwqeIn);
2812 return;
2813 }
2814 /* For abort iocb just return, IO iocb will do a done call */
2815 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2816 CMD_ABORT_XRI_CX) {
2817 lpfc_sli_release_iocbq(phba, pwqeIn);
2818 return;
2819 }
2820
2821 spin_lock(&lpfc_ncmd->buf_lock);
2822 nCmd = lpfc_ncmd->nvmeCmd;
2823 if (!nCmd) {
2824 spin_unlock(&lpfc_ncmd->buf_lock);
2825 lpfc_release_nvme_buf(phba, lpfc_ncmd);
2826 return;
2827 }
2828 spin_unlock(&lpfc_ncmd->buf_lock);
2829
2830 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2831 "6194 NVME Cancel xri %x\n",
2832 lpfc_ncmd->cur_iocbq.sli4_xritag);
2833
2834 wcqep->word0 = 0;
2835 bf_set(lpfc_wcqe_c_status, wcqep, stat);
2836 wcqep->parameter = param;
2837 wcqep->total_data_placed = 0;
2838 wcqep->word3 = 0; /* xb is 0 */
2839
2840 /* Call release with XB=1 to queue the IO into the abort list. */
2841 if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
2842 bf_set(lpfc_wcqe_c_xb, wcqep, 1);
2843
2844 memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep));
2845 (pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn);
2846 #endif
2847 }
2848
2849 /**
2850 * lpfc_nvmels_flush_cmd - Clean up outstanding nvmels commands for a port
2851 * @phba: Pointer to HBA context object.
2852 *
2853 **/
2854 void
lpfc_nvmels_flush_cmd(struct lpfc_hba * phba)2855 lpfc_nvmels_flush_cmd(struct lpfc_hba *phba)
2856 {
2857 #if (IS_ENABLED(CONFIG_NVME_FC))
2858 LIST_HEAD(cancel_list);
2859 struct lpfc_sli_ring *pring = NULL;
2860 struct lpfc_iocbq *piocb, *tmp_iocb;
2861 unsigned long iflags;
2862
2863 if (phba->sli4_hba.nvmels_wq)
2864 pring = phba->sli4_hba.nvmels_wq->pring;
2865
2866 if (unlikely(!pring))
2867 return;
2868
2869 spin_lock_irqsave(&phba->hbalock, iflags);
2870 spin_lock(&pring->ring_lock);
2871 list_splice_init(&pring->txq, &cancel_list);
2872 pring->txq_cnt = 0;
2873 list_for_each_entry_safe(piocb, tmp_iocb, &pring->txcmplq, list) {
2874 if (piocb->cmd_flag & LPFC_IO_NVME_LS) {
2875 list_move_tail(&piocb->list, &cancel_list);
2876 pring->txcmplq_cnt--;
2877 piocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ;
2878 }
2879 }
2880 spin_unlock(&pring->ring_lock);
2881 spin_unlock_irqrestore(&phba->hbalock, iflags);
2882
2883 if (!list_empty(&cancel_list))
2884 lpfc_sli_cancel_iocbs(phba, &cancel_list, IOSTAT_LOCAL_REJECT,
2885 IOERR_SLI_DOWN);
2886 #endif
2887 }
2888