101649561SJames Smart /*******************************************************************
201649561SJames Smart * This file is part of the Emulex Linux Device Driver for *
301649561SJames Smart * Fibre Channel Host Bus Adapters. *
4ea4044e4SJustin Tee * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *
53e21d1cbSJames Smart * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
601649561SJames Smart * Copyright (C) 2004-2016 Emulex. All rights reserved. *
701649561SJames Smart * EMULEX and SLI are trademarks of Emulex. *
8d080abe0SJames Smart * www.broadcom.com *
901649561SJames Smart * Portions Copyright (C) 2004-2005 Christoph Hellwig *
1001649561SJames Smart * *
1101649561SJames Smart * This program is free software; you can redistribute it and/or *
1201649561SJames Smart * modify it under the terms of version 2 of the GNU General *
1301649561SJames Smart * Public License as published by the Free Software Foundation. *
1401649561SJames Smart * This program is distributed in the hope that it will be useful. *
1501649561SJames Smart * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
1601649561SJames Smart * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
1701649561SJames Smart * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
1801649561SJames Smart * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
1901649561SJames Smart * TO BE LEGALLY INVALID. See the GNU General Public License for *
2001649561SJames Smart * more details, a copy of which can be found in the file COPYING *
2101649561SJames Smart * included with this package. *
2201649561SJames Smart ********************************************************************/
2301649561SJames Smart #include <linux/pci.h>
2401649561SJames Smart #include <linux/slab.h>
2501649561SJames Smart #include <linux/interrupt.h>
2601649561SJames Smart #include <linux/delay.h>
2701649561SJames Smart #include <asm/unaligned.h>
2801649561SJames Smart #include <linux/crc-t10dif.h>
2901649561SJames Smart #include <net/checksum.h>
3001649561SJames Smart
3101649561SJames Smart #include <scsi/scsi.h>
3201649561SJames Smart #include <scsi/scsi_device.h>
3301649561SJames Smart #include <scsi/scsi_eh.h>
3401649561SJames Smart #include <scsi/scsi_host.h>
3501649561SJames Smart #include <scsi/scsi_tcq.h>
3601649561SJames Smart #include <scsi/scsi_transport_fc.h>
3701649561SJames Smart #include <scsi/fc/fc_fs.h>
3801649561SJames Smart
3901649561SJames Smart #include "lpfc_version.h"
4001649561SJames Smart #include "lpfc_hw4.h"
4101649561SJames Smart #include "lpfc_hw.h"
4201649561SJames Smart #include "lpfc_sli.h"
4301649561SJames Smart #include "lpfc_sli4.h"
4401649561SJames Smart #include "lpfc_nl.h"
4501649561SJames Smart #include "lpfc_disc.h"
4601649561SJames Smart #include "lpfc.h"
4701649561SJames Smart #include "lpfc_nvme.h"
4801649561SJames Smart #include "lpfc_scsi.h"
4901649561SJames Smart #include "lpfc_logmsg.h"
5001649561SJames Smart #include "lpfc_crtn.h"
5101649561SJames Smart #include "lpfc_vport.h"
52bd2cdd5eSJames Smart #include "lpfc_debugfs.h"
5301649561SJames Smart
5401649561SJames Smart /* NVME initiator-based functions */
5501649561SJames Smart
56c490850aSJames Smart static struct lpfc_io_buf *
57cf1a1d3eSJames Smart lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
585e5b511dSJames Smart int idx, int expedite);
5901649561SJames Smart
6001649561SJames Smart static void
61c490850aSJames Smart lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
6201649561SJames Smart
6381e6a637SJames Smart static struct nvme_fc_port_template lpfc_nvme_template;
6401649561SJames Smart
6501649561SJames Smart /**
6601649561SJames Smart * lpfc_nvme_create_queue -
6709d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from
6801649561SJames Smart * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
6909d99705SLee Jones * @qsize: Size of the queue in bytes
7001649561SJames Smart * @handle: An opaque driver handle used in follow-up calls.
7101649561SJames Smart *
7201649561SJames Smart * Driver registers this routine to preallocate and initialize any
7301649561SJames Smart * internal data structures to bind the @qidx to its internal IO queues.
7401649561SJames Smart * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
7501649561SJames Smart *
7601649561SJames Smart * Return value :
7701649561SJames Smart * 0 - Success
7801649561SJames Smart * -EINVAL - Unsupported input value.
7901649561SJames Smart * -ENOMEM - Could not alloc necessary memory
8001649561SJames Smart **/
8101649561SJames Smart static int
lpfc_nvme_create_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,u16 qsize,void ** handle)8201649561SJames Smart lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
8301649561SJames Smart unsigned int qidx, u16 qsize,
8401649561SJames Smart void **handle)
8501649561SJames Smart {
8601649561SJames Smart struct lpfc_nvme_lport *lport;
8701649561SJames Smart struct lpfc_vport *vport;
8801649561SJames Smart struct lpfc_nvme_qhandle *qhandle;
8901649561SJames Smart char *str;
9001649561SJames Smart
91c3725bdcSJames Smart if (!pnvme_lport->private)
92c3725bdcSJames Smart return -ENOMEM;
93c3725bdcSJames Smart
9401649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
9501649561SJames Smart vport = lport->vport;
96a4691038SJames Smart
97e39811beSJustin Tee if (!vport || test_bit(FC_UNLOADING, &vport->load_flag) ||
98*e780c942SJustin Tee test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
99a4691038SJames Smart return -ENODEV;
100a4691038SJames Smart
10101649561SJames Smart qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
10201649561SJames Smart if (qhandle == NULL)
10301649561SJames Smart return -ENOMEM;
10401649561SJames Smart
105d6d189ceSBart Van Assche qhandle->cpu_id = raw_smp_processor_id();
10601649561SJames Smart qhandle->qidx = qidx;
10701649561SJames Smart /*
10801649561SJames Smart * NVME qidx == 0 is the admin queue, so both admin queue
10901649561SJames Smart * and first IO queue will use MSI-X vector and associated
11001649561SJames Smart * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
11101649561SJames Smart */
11201649561SJames Smart if (qidx) {
11301649561SJames Smart str = "IO "; /* IO queue */
11401649561SJames Smart qhandle->index = ((qidx - 1) %
1156a828b0fSJames Smart lpfc_nvme_template.max_hw_queues);
11601649561SJames Smart } else {
11701649561SJames Smart str = "ADM"; /* Admin queue */
11801649561SJames Smart qhandle->index = qidx;
11901649561SJames Smart }
12001649561SJames Smart
121d58734f0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
12201649561SJames Smart "6073 Binding %s HdwQueue %d (cpu %d) to "
12332350664SJames Smart "hdw_queue %d qhandle x%px\n", str,
12401649561SJames Smart qidx, qhandle->cpu_id, qhandle->index, qhandle);
12501649561SJames Smart *handle = (void *)qhandle;
12601649561SJames Smart return 0;
12701649561SJames Smart }
12801649561SJames Smart
12901649561SJames Smart /**
13001649561SJames Smart * lpfc_nvme_delete_queue -
13109d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from
13201649561SJames Smart * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
13301649561SJames Smart * @handle: An opaque driver handle from lpfc_nvme_create_queue
13401649561SJames Smart *
13501649561SJames Smart * Driver registers this routine to free
13601649561SJames Smart * any internal data structures to bind the @qidx to its internal
13701649561SJames Smart * IO queues.
13801649561SJames Smart *
13901649561SJames Smart * Return value :
14001649561SJames Smart * 0 - Success
14101649561SJames Smart * TODO: What are the failure codes.
14201649561SJames Smart **/
14301649561SJames Smart static void
lpfc_nvme_delete_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,void * handle)14401649561SJames Smart lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
14501649561SJames Smart unsigned int qidx,
14601649561SJames Smart void *handle)
14701649561SJames Smart {
14801649561SJames Smart struct lpfc_nvme_lport *lport;
14901649561SJames Smart struct lpfc_vport *vport;
15001649561SJames Smart
151c3725bdcSJames Smart if (!pnvme_lport->private)
152c3725bdcSJames Smart return;
153c3725bdcSJames Smart
15401649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
15501649561SJames Smart vport = lport->vport;
15601649561SJames Smart
15701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
15832350664SJames Smart "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
15901649561SJames Smart lport, qidx, handle);
16001649561SJames Smart kfree(handle);
16101649561SJames Smart }
16201649561SJames Smart
16301649561SJames Smart static void
lpfc_nvme_localport_delete(struct nvme_fc_local_port * localport)16401649561SJames Smart lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
16501649561SJames Smart {
16601649561SJames Smart struct lpfc_nvme_lport *lport = localport->private;
16701649561SJames Smart
168add9d6beSJames Smart lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
16932350664SJames Smart "6173 localport x%px delete complete\n",
170add9d6beSJames Smart lport);
171add9d6beSJames Smart
17201649561SJames Smart /* release any threads waiting for the unreg to complete */
1737961cba6SEwan D. Milne if (lport->vport->localport)
1747961cba6SEwan D. Milne complete(lport->lport_unreg_cmp);
17501649561SJames Smart }
17601649561SJames Smart
17701649561SJames Smart /* lpfc_nvme_remoteport_delete
17801649561SJames Smart *
17901649561SJames Smart * @remoteport: Pointer to an nvme transport remoteport instance.
18001649561SJames Smart *
18101649561SJames Smart * This is a template downcall. NVME transport calls this function
18201649561SJames Smart * when it has completed the unregistration of a previously
18301649561SJames Smart * registered remoteport.
18401649561SJames Smart *
18501649561SJames Smart * Return value :
18601649561SJames Smart * None
18701649561SJames Smart */
1883999df75SBart Van Assche static void
lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port * remoteport)18901649561SJames Smart lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
19001649561SJames Smart {
19101649561SJames Smart struct lpfc_nvme_rport *rport = remoteport->private;
19201649561SJames Smart struct lpfc_vport *vport;
19301649561SJames Smart struct lpfc_nodelist *ndlp;
194e9b11083SJames Smart u32 fc4_xpt_flags;
19501649561SJames Smart
19601649561SJames Smart ndlp = rport->ndlp;
197e9b11083SJames Smart if (!ndlp) {
198f1156125SJames Smart pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",
199e9b11083SJames Smart __func__, rport, remoteport);
20001649561SJames Smart goto rport_err;
201e9b11083SJames Smart }
20201649561SJames Smart
20301649561SJames Smart vport = ndlp->vport;
204e9b11083SJames Smart if (!vport) {
205f1156125SJames Smart pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",
206e9b11083SJames Smart __func__, ndlp, ndlp->nlp_state, rport);
20701649561SJames Smart goto rport_err;
208e9b11083SJames Smart }
209e9b11083SJames Smart
210e9b11083SJames Smart fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;
21101649561SJames Smart
21201649561SJames Smart /* Remove this rport from the lport's list - memory is owned by the
21301649561SJames Smart * transport. Remove the ndlp reference for the NVME transport before
2147a06dcd3SJames Smart * calling state machine to remove the node.
21501649561SJames Smart */
21601649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
217982fc396SJames Smart "6146 remoteport delete of remoteport x%px, ndlp x%px "
218982fc396SJames Smart "DID x%x xflags x%x\n",
219982fc396SJames Smart remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);
220c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
221b15bd3e6SJames Smart
222b15bd3e6SJames Smart /* The register rebind might have occurred before the delete
223b15bd3e6SJames Smart * downcall. Guard against this race.
224b15bd3e6SJames Smart */
22506145683SJames Smart if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)
22606145683SJames Smart ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);
227e9b11083SJames Smart
228c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
2293fd78355SJames Smart
230e9b11083SJames Smart /* On a devloss timeout event, one more put is executed provided the
231dae40be7SJustin Tee * NVME and SCSI rport unregister requests are complete.
2323fd78355SJames Smart */
233e9b11083SJames Smart if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))
234e9b11083SJames Smart lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
23501649561SJames Smart
23601649561SJames Smart rport_err:
2373fd78355SJames Smart return;
23801649561SJames Smart }
23901649561SJames Smart
2403a8070c5SJames Smart /**
2413a8070c5SJames Smart * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
2423a8070c5SJames Smart * @phba: pointer to lpfc hba data structure.
2433a8070c5SJames Smart * @axchg: pointer to exchange context for the NVME LS request
2443a8070c5SJames Smart *
2453a8070c5SJames Smart * This routine is used for processing an asychronously received NVME LS
2463a8070c5SJames Smart * request. Any remaining validation is done and the LS is then forwarded
2473a8070c5SJames Smart * to the nvme-fc transport via nvme_fc_rcv_ls_req().
2483a8070c5SJames Smart *
2499aa09e98SJames Smart * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
2509aa09e98SJames Smart * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
2519aa09e98SJames Smart * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
2529aa09e98SJames Smart *
2533a8070c5SJames Smart * Returns 0 if LS was handled and delivered to the transport
2543a8070c5SJames Smart * Returns 1 if LS failed to be handled and should be dropped
2553a8070c5SJames Smart */
2563a8070c5SJames Smart int
lpfc_nvme_handle_lsreq(struct lpfc_hba * phba,struct lpfc_async_xchg_ctx * axchg)2573a8070c5SJames Smart lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
2583a8070c5SJames Smart struct lpfc_async_xchg_ctx *axchg)
2593a8070c5SJames Smart {
2609aa09e98SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
2619aa09e98SJames Smart struct lpfc_vport *vport;
2629aa09e98SJames Smart struct lpfc_nvme_rport *lpfc_rport;
2639aa09e98SJames Smart struct nvme_fc_remote_port *remoteport;
2649aa09e98SJames Smart struct lpfc_nvme_lport *lport;
2659aa09e98SJames Smart uint32_t *payload = axchg->payload;
2669aa09e98SJames Smart int rc;
2679aa09e98SJames Smart
2689aa09e98SJames Smart vport = axchg->ndlp->vport;
2699aa09e98SJames Smart lpfc_rport = axchg->ndlp->nrport;
2709aa09e98SJames Smart if (!lpfc_rport)
2719aa09e98SJames Smart return -EINVAL;
2729aa09e98SJames Smart
2739aa09e98SJames Smart remoteport = lpfc_rport->remoteport;
274a4691038SJames Smart if (!vport->localport ||
275*e780c942SJustin Tee test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
2769aa09e98SJames Smart return -EINVAL;
2779aa09e98SJames Smart
2789aa09e98SJames Smart lport = vport->localport->private;
2799aa09e98SJames Smart if (!lport)
2809aa09e98SJames Smart return -EINVAL;
2819aa09e98SJames Smart
2829aa09e98SJames Smart rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
2839aa09e98SJames Smart axchg->size);
2849aa09e98SJames Smart
2859aa09e98SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
2869aa09e98SJames Smart "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
2879aa09e98SJames Smart "%08x %08x %08x\n",
2889aa09e98SJames Smart axchg->size, rc,
2899aa09e98SJames Smart *payload, *(payload+1), *(payload+2),
2909aa09e98SJames Smart *(payload+3), *(payload+4), *(payload+5));
2919aa09e98SJames Smart
2929aa09e98SJames Smart if (!rc)
2939aa09e98SJames Smart return 0;
2949aa09e98SJames Smart #endif
2953a8070c5SJames Smart return 1;
2963a8070c5SJames Smart }
2973a8070c5SJames Smart
2986514b25dSJames Smart /**
2996514b25dSJames Smart * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
3006514b25dSJames Smart * LS request.
3016514b25dSJames Smart * @phba: Pointer to HBA context object
3026514b25dSJames Smart * @vport: The local port that issued the LS
3036514b25dSJames Smart * @cmdwqe: Pointer to driver command WQE object.
3046514b25dSJames Smart * @wcqe: Pointer to driver response CQE object.
3056514b25dSJames Smart *
3066514b25dSJames Smart * This function is the generic completion handler for NVME LS requests.
3076514b25dSJames Smart * The function updates any states and statistics, calls the transport
3086514b25dSJames Smart * ls_req done() routine, then tears down the command and buffers used
3096514b25dSJames Smart * for the LS request.
3106514b25dSJames Smart **/
3116514b25dSJames Smart void
__lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_vport * vport,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe)3126514b25dSJames Smart __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
3136514b25dSJames Smart struct lpfc_iocbq *cmdwqe,
3146514b25dSJames Smart struct lpfc_wcqe_complete *wcqe)
3156514b25dSJames Smart {
3166514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq;
3176514b25dSJames Smart struct lpfc_dmabuf *buf_ptr;
3186514b25dSJames Smart struct lpfc_nodelist *ndlp;
3196a84d015SJustin Tee int status;
3206514b25dSJames Smart
321d51cf5bdSJames Smart pnvme_lsreq = cmdwqe->context_un.nvme_lsreq;
322d51cf5bdSJames Smart ndlp = cmdwqe->ndlp;
323d51cf5bdSJames Smart buf_ptr = cmdwqe->bpl_dmabuf;
324d51cf5bdSJames Smart
3256a84d015SJustin Tee status = bf_get(lpfc_wcqe_c_status, wcqe);
3266514b25dSJames Smart
3276514b25dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
328f1156125SJames Smart "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "
3296514b25dSJames Smart "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
3306514b25dSJames Smart "ndlp:x%px\n",
3316514b25dSJames Smart pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
3326514b25dSJames Smart cmdwqe->sli4_xritag, status,
3336514b25dSJames Smart (wcqe->parameter & 0xffff),
334d51cf5bdSJames Smart cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf,
335d51cf5bdSJames Smart ndlp);
3366514b25dSJames Smart
3376514b25dSJames Smart lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
3386514b25dSJames Smart cmdwqe->sli4_xritag, status, wcqe->parameter);
3396514b25dSJames Smart
340d51cf5bdSJames Smart if (buf_ptr) {
3416514b25dSJames Smart lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
3426514b25dSJames Smart kfree(buf_ptr);
343d51cf5bdSJames Smart cmdwqe->bpl_dmabuf = NULL;
3446514b25dSJames Smart }
3456a84d015SJustin Tee if (pnvme_lsreq->done) {
3466a84d015SJustin Tee if (status != CQE_STATUS_SUCCESS)
3476a84d015SJustin Tee status = -ENXIO;
3486514b25dSJames Smart pnvme_lsreq->done(pnvme_lsreq, status);
3496a84d015SJustin Tee } else {
350372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
3516514b25dSJames Smart "6046 NVMEx cmpl without done call back? "
352f1156125SJames Smart "Data x%px DID %x Xri: %x status %x\n",
3536514b25dSJames Smart pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
3546514b25dSJames Smart cmdwqe->sli4_xritag, status);
3556a84d015SJustin Tee }
3566514b25dSJames Smart if (ndlp) {
3576514b25dSJames Smart lpfc_nlp_put(ndlp);
358d51cf5bdSJames Smart cmdwqe->ndlp = NULL;
3596514b25dSJames Smart }
3606514b25dSJames Smart lpfc_sli_release_iocbq(phba, cmdwqe);
3616514b25dSJames Smart }
3626514b25dSJames Smart
36301649561SJames Smart static void
lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe)3646514b25dSJames Smart lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
365a680a929SJames Smart struct lpfc_iocbq *rspwqe)
36601649561SJames Smart {
36701649561SJames Smart struct lpfc_vport *vport = cmdwqe->vport;
3684b056682SJames Smart struct lpfc_nvme_lport *lport;
36901649561SJames Smart uint32_t status;
370a680a929SJames Smart struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
37101649561SJames Smart
3726a84d015SJustin Tee status = bf_get(lpfc_wcqe_c_status, wcqe);
37366a210ffSJames Smart
37466a85155SJames Smart if (vport->localport) {
37566a85155SJames Smart lport = (struct lpfc_nvme_lport *)vport->localport->private;
37666a210ffSJames Smart if (lport) {
37766a210ffSJames Smart atomic_inc(&lport->fc4NvmeLsCmpls);
3784b056682SJames Smart if (status) {
3794b056682SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe))
3804b056682SJames Smart atomic_inc(&lport->cmpl_ls_xb);
3814b056682SJames Smart atomic_inc(&lport->cmpl_ls_err);
3824b056682SJames Smart }
38366a210ffSJames Smart }
38466a85155SJames Smart }
3854b056682SJames Smart
3866514b25dSJames Smart __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
38701649561SJames Smart }
38801649561SJames Smart
38901649561SJames Smart static int
lpfc_nvme_gen_req(struct lpfc_vport * vport,struct lpfc_dmabuf * bmp,struct lpfc_dmabuf * inp,struct nvmefc_ls_req * pnvme_lsreq,void (* cmpl)(struct lpfc_hba *,struct lpfc_iocbq *,struct lpfc_iocbq *),struct lpfc_nodelist * ndlp,uint32_t num_entry,uint32_t tmo,uint8_t retry)39001649561SJames Smart lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
39101649561SJames Smart struct lpfc_dmabuf *inp,
39201649561SJames Smart struct nvmefc_ls_req *pnvme_lsreq,
39301649561SJames Smart void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
394a680a929SJames Smart struct lpfc_iocbq *),
39501649561SJames Smart struct lpfc_nodelist *ndlp, uint32_t num_entry,
39601649561SJames Smart uint32_t tmo, uint8_t retry)
39701649561SJames Smart {
39801649561SJames Smart struct lpfc_hba *phba = vport->phba;
399205e8240SJames Smart union lpfc_wqe128 *wqe;
40001649561SJames Smart struct lpfc_iocbq *genwqe;
40101649561SJames Smart struct ulp_bde64 *bpl;
40201649561SJames Smart struct ulp_bde64 bde;
40301649561SJames Smart int i, rc, xmit_len, first_len;
40401649561SJames Smart
40501649561SJames Smart /* Allocate buffer for command WQE */
40601649561SJames Smart genwqe = lpfc_sli_get_iocbq(phba);
40701649561SJames Smart if (genwqe == NULL)
40801649561SJames Smart return 1;
40901649561SJames Smart
41001649561SJames Smart wqe = &genwqe->wqe;
411d9f492a1SJames Smart /* Initialize only 64 bytes */
41201649561SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe));
41301649561SJames Smart
414d51cf5bdSJames Smart genwqe->bpl_dmabuf = bmp;
415a680a929SJames Smart genwqe->cmd_flag |= LPFC_IO_NVME_LS;
41601649561SJames Smart
41701649561SJames Smart /* Save for completion so we can release these resources */
418d51cf5bdSJames Smart genwqe->ndlp = lpfc_nlp_get(ndlp);
419d51cf5bdSJames Smart if (!genwqe->ndlp) {
4204430f7fdSJames Smart dev_warn(&phba->pcidev->dev,
4214430f7fdSJames Smart "Warning: Failed node ref, not sending LS_REQ\n");
4224430f7fdSJames Smart lpfc_sli_release_iocbq(phba, genwqe);
4234430f7fdSJames Smart return 1;
4244430f7fdSJames Smart }
4254430f7fdSJames Smart
426d51cf5bdSJames Smart genwqe->context_un.nvme_lsreq = pnvme_lsreq;
42701649561SJames Smart /* Fill in payload, bp points to frame payload */
42801649561SJames Smart
42901649561SJames Smart if (!tmo)
43001649561SJames Smart /* FC spec states we need 3 * ratov for CT requests */
43101649561SJames Smart tmo = (3 * phba->fc_ratov);
43201649561SJames Smart
43301649561SJames Smart /* For this command calculate the xmit length of the request bde. */
43401649561SJames Smart xmit_len = 0;
43501649561SJames Smart first_len = 0;
43601649561SJames Smart bpl = (struct ulp_bde64 *)bmp->virt;
43701649561SJames Smart for (i = 0; i < num_entry; i++) {
43801649561SJames Smart bde.tus.w = bpl[i].tus.w;
43901649561SJames Smart if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
44001649561SJames Smart break;
44101649561SJames Smart xmit_len += bde.tus.f.bdeSize;
44201649561SJames Smart if (i == 0)
44301649561SJames Smart first_len = xmit_len;
44401649561SJames Smart }
44501649561SJames Smart
446a680a929SJames Smart genwqe->num_bdes = num_entry;
44701649561SJames Smart genwqe->hba_wqidx = 0;
44801649561SJames Smart
44901649561SJames Smart /* Words 0 - 2 */
45001649561SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
45101649561SJames Smart wqe->generic.bde.tus.f.bdeSize = first_len;
45201649561SJames Smart wqe->generic.bde.addrLow = bpl[0].addrLow;
45301649561SJames Smart wqe->generic.bde.addrHigh = bpl[0].addrHigh;
45401649561SJames Smart
45501649561SJames Smart /* Word 3 */
45601649561SJames Smart wqe->gen_req.request_payload_len = first_len;
45701649561SJames Smart
45801649561SJames Smart /* Word 4 */
45901649561SJames Smart
46001649561SJames Smart /* Word 5 */
46101649561SJames Smart bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
46201649561SJames Smart bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
46301649561SJames Smart bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
4648b361639SJames Smart bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
46501649561SJames Smart bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
46601649561SJames Smart
46701649561SJames Smart /* Word 6 */
46801649561SJames Smart bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
46901649561SJames Smart phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
47001649561SJames Smart bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
47101649561SJames Smart
47201649561SJames Smart /* Word 7 */
473c33b1609SJames Smart bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);
47401649561SJames Smart bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
47501649561SJames Smart bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
47601649561SJames Smart bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
47701649561SJames Smart
47801649561SJames Smart /* Word 8 */
47901649561SJames Smart wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
48001649561SJames Smart
48101649561SJames Smart /* Word 9 */
48201649561SJames Smart bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
48301649561SJames Smart
48401649561SJames Smart /* Word 10 */
48501649561SJames Smart bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
48601649561SJames Smart bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
48701649561SJames Smart bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
48801649561SJames Smart bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
48901649561SJames Smart bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
49001649561SJames Smart
49101649561SJames Smart /* Word 11 */
49201649561SJames Smart bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
49301649561SJames Smart bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
49401649561SJames Smart
49501649561SJames Smart
49601649561SJames Smart /* Issue GEN REQ WQE for NPORT <did> */
497a680a929SJames Smart genwqe->cmd_cmpl = cmpl;
49801649561SJames Smart genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
49901649561SJames Smart genwqe->vport = vport;
50001649561SJames Smart genwqe->retry = retry;
50101649561SJames Smart
502bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
503bd2cdd5eSJames Smart genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
504bd2cdd5eSJames Smart
5051fbf9742SJames Smart rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
506cd22d605SDick Kennedy if (rc) {
507372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
50801649561SJames Smart "6045 Issue GEN REQ WQE to NPORT x%x "
5096514b25dSJames Smart "Data: x%x x%x rc x%x\n",
51001649561SJames Smart ndlp->nlp_DID, genwqe->iotag,
5116514b25dSJames Smart vport->port_state, rc);
512e9b11083SJames Smart lpfc_nlp_put(ndlp);
51301649561SJames Smart lpfc_sli_release_iocbq(phba, genwqe);
51401649561SJames Smart return 1;
51501649561SJames Smart }
5166514b25dSJames Smart
5176514b25dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
5186514b25dSJames Smart "6050 Issue GEN REQ WQE to NPORT x%x "
5196514b25dSJames Smart "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
5206514b25dSJames Smart "bmp:x%px xmit:%d 1st:%d\n",
5216514b25dSJames Smart ndlp->nlp_DID, genwqe->sli4_xritag,
5226514b25dSJames Smart vport->port_state,
5236514b25dSJames Smart genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
52401649561SJames Smart return 0;
52501649561SJames Smart }
52601649561SJames Smart
5276514b25dSJames Smart
52801649561SJames Smart /**
5296514b25dSJames Smart * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
5306514b25dSJames Smart * @vport: The local port issuing the LS
5316514b25dSJames Smart * @ndlp: The remote port to send the LS to
5326514b25dSJames Smart * @pnvme_lsreq: Pointer to LS request structure from the transport
53309d99705SLee Jones * @gen_req_cmp: Completion call-back
53401649561SJames Smart *
5356514b25dSJames Smart * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
5366514b25dSJames Smart * WQE to perform the LS operation.
53701649561SJames Smart *
53801649561SJames Smart * Return value :
53901649561SJames Smart * 0 - Success
5406514b25dSJames Smart * non-zero: various error codes, in form of -Exxx
54101649561SJames Smart **/
5426514b25dSJames Smart int
__lpfc_nvme_ls_req(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq,void (* gen_req_cmp)(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe))5436514b25dSJames Smart __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
5446514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq,
5456514b25dSJames Smart void (*gen_req_cmp)(struct lpfc_hba *phba,
5466514b25dSJames Smart struct lpfc_iocbq *cmdwqe,
547a680a929SJames Smart struct lpfc_iocbq *rspwqe))
54801649561SJames Smart {
54901649561SJames Smart struct lpfc_dmabuf *bmp;
5506514b25dSJames Smart struct ulp_bde64 *bpl;
5516514b25dSJames Smart int ret;
552ba43c4d0SJames Smart uint16_t ntype, nstate;
55301649561SJames Smart
554307e3380SJames Smart if (!ndlp) {
555372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5566514b25dSJames Smart "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
5576514b25dSJames Smart "LS Req\n",
5586514b25dSJames Smart ndlp);
559ba43c4d0SJames Smart return -ENODEV;
560ba43c4d0SJames Smart }
561ba43c4d0SJames Smart
562ba43c4d0SJames Smart ntype = ndlp->nlp_type;
563ba43c4d0SJames Smart nstate = ndlp->nlp_state;
564ba43c4d0SJames Smart if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
565ba43c4d0SJames Smart (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
566372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5676514b25dSJames Smart "6088 NVMEx LS REQ: Fail DID x%06x not "
5686514b25dSJames Smart "ready for IO. Type x%x, State x%x\n",
5696514b25dSJames Smart ndlp->nlp_DID, ntype, nstate);
570ba43c4d0SJames Smart return -ENODEV;
57101649561SJames Smart }
572*e780c942SJustin Tee if (test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
573a4691038SJames Smart return -ENODEV;
5746514b25dSJames Smart
5758c65830aSJames Smart if (!vport->phba->sli4_hba.nvmels_wq)
5768c65830aSJames Smart return -ENOMEM;
5778c65830aSJames Smart
5786514b25dSJames Smart /*
5796514b25dSJames Smart * there are two dma buf in the request, actually there is one and
5806514b25dSJames Smart * the second one is just the start address + cmd size.
5816514b25dSJames Smart * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
5826514b25dSJames Smart * in a lpfc_dmabuf struct. When freeing we just free the wrapper
5836514b25dSJames Smart * because the nvem layer owns the data bufs.
5846514b25dSJames Smart * We do not have to break these packets open, we don't care what is
5856514b25dSJames Smart * in them. And we do not have to look at the resonse data, we only
5866514b25dSJames Smart * care that we got a response. All of the caring is going to happen
5876514b25dSJames Smart * in the nvme-fc layer.
5886514b25dSJames Smart */
5896514b25dSJames Smart
5906514b25dSJames Smart bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
59101649561SJames Smart if (!bmp) {
592372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
5936514b25dSJames Smart "6044 NVMEx LS REQ: Could not alloc LS buf "
5946514b25dSJames Smart "for DID %x\n",
5956514b25dSJames Smart ndlp->nlp_DID);
5966514b25dSJames Smart return -ENOMEM;
59701649561SJames Smart }
5986514b25dSJames Smart
59901649561SJames Smart bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
60001649561SJames Smart if (!bmp->virt) {
601372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6026514b25dSJames Smart "6042 NVMEx LS REQ: Could not alloc mbuf "
6036514b25dSJames Smart "for DID %x\n",
6046514b25dSJames Smart ndlp->nlp_DID);
60501649561SJames Smart kfree(bmp);
6066514b25dSJames Smart return -ENOMEM;
60701649561SJames Smart }
6086514b25dSJames Smart
6096514b25dSJames Smart INIT_LIST_HEAD(&bmp->list);
6106514b25dSJames Smart
61101649561SJames Smart bpl = (struct ulp_bde64 *)bmp->virt;
61201649561SJames Smart bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
61301649561SJames Smart bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
61401649561SJames Smart bpl->tus.f.bdeFlags = 0;
61501649561SJames Smart bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
61601649561SJames Smart bpl->tus.w = le32_to_cpu(bpl->tus.w);
61701649561SJames Smart bpl++;
61801649561SJames Smart
61901649561SJames Smart bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
62001649561SJames Smart bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
62101649561SJames Smart bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
62201649561SJames Smart bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
62301649561SJames Smart bpl->tus.w = le32_to_cpu(bpl->tus.w);
62401649561SJames Smart
62501649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
6266514b25dSJames Smart "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
6276514b25dSJames Smart "rqstlen:%d rsplen:%d %pad %pad\n",
6286514b25dSJames Smart ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
629825c6abbSArnd Bergmann pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
630825c6abbSArnd Bergmann &pnvme_lsreq->rspdma);
63101649561SJames Smart
63201649561SJames Smart ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
6336514b25dSJames Smart pnvme_lsreq, gen_req_cmp, ndlp, 2,
634c33b1609SJames Smart pnvme_lsreq->timeout, 0);
63501649561SJames Smart if (ret != WQE_SUCCESS) {
636372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
6376514b25dSJames Smart "6052 NVMEx REQ: EXIT. issue ls wqe failed "
6386514b25dSJames Smart "lsreq x%px Status %x DID %x\n",
6396514b25dSJames Smart pnvme_lsreq, ret, ndlp->nlp_DID);
64001649561SJames Smart lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
64101649561SJames Smart kfree(bmp);
6426514b25dSJames Smart return -EIO;
64301649561SJames Smart }
64401649561SJames Smart
6456514b25dSJames Smart return 0;
6466514b25dSJames Smart }
6476514b25dSJames Smart
6486514b25dSJames Smart /**
6496514b25dSJames Smart * lpfc_nvme_ls_req - Issue an NVME Link Service request
65009d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from.
6513e749170SLee Jones * @pnvme_rport: Transport remoteport that LS is to be sent to.
65209d99705SLee Jones * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
6536514b25dSJames Smart *
6546514b25dSJames Smart * Driver registers this routine to handle any link service request
6556514b25dSJames Smart * from the nvme_fc transport to a remote nvme-aware port.
6566514b25dSJames Smart *
6576514b25dSJames Smart * Return value :
6586514b25dSJames Smart * 0 - Success
6596514b25dSJames Smart * non-zero: various error codes, in form of -Exxx
6606514b25dSJames Smart **/
6616514b25dSJames Smart static int
lpfc_nvme_ls_req(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)6626514b25dSJames Smart lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
6636514b25dSJames Smart struct nvme_fc_remote_port *pnvme_rport,
6646514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq)
6656514b25dSJames Smart {
6666514b25dSJames Smart struct lpfc_nvme_lport *lport;
6676514b25dSJames Smart struct lpfc_nvme_rport *rport;
6686514b25dSJames Smart struct lpfc_vport *vport;
6696514b25dSJames Smart int ret;
6706514b25dSJames Smart
6716514b25dSJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
6726514b25dSJames Smart rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
6736514b25dSJames Smart if (unlikely(!lport) || unlikely(!rport))
6746514b25dSJames Smart return -EINVAL;
6756514b25dSJames Smart
6766514b25dSJames Smart vport = lport->vport;
677e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag) ||
678*e780c942SJustin Tee test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))
6796514b25dSJames Smart return -ENODEV;
6806514b25dSJames Smart
6816514b25dSJames Smart atomic_inc(&lport->fc4NvmeLsRequests);
6826514b25dSJames Smart
6836514b25dSJames Smart ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
6846514b25dSJames Smart lpfc_nvme_ls_req_cmp);
6856514b25dSJames Smart if (ret)
6866514b25dSJames Smart atomic_inc(&lport->xmt_ls_err);
6876514b25dSJames Smart
68801649561SJames Smart return ret;
68901649561SJames Smart }
69001649561SJames Smart
69101649561SJames Smart /**
692e96a22b0SJames Smart * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
693e96a22b0SJames Smart * NVME LS request
694e96a22b0SJames Smart * @vport: The local port that issued the LS
695e96a22b0SJames Smart * @ndlp: The remote port the LS was sent to
696e96a22b0SJames Smart * @pnvme_lsreq: Pointer to LS request structure from the transport
69701649561SJames Smart *
698e96a22b0SJames Smart * The driver validates the ndlp, looks for the LS, and aborts the
699e96a22b0SJames Smart * LS if found.
70001649561SJames Smart *
701e96a22b0SJames Smart * Returns:
702e96a22b0SJames Smart * 0 : if LS found and aborted
703e96a22b0SJames Smart * non-zero: various error conditions in form -Exxx
704e96a22b0SJames Smart **/
705e96a22b0SJames Smart int
__lpfc_nvme_ls_abort(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq)706e96a22b0SJames Smart __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
707e96a22b0SJames Smart struct nvmefc_ls_req *pnvme_lsreq)
708e96a22b0SJames Smart {
709e96a22b0SJames Smart struct lpfc_hba *phba = vport->phba;
710e96a22b0SJames Smart struct lpfc_sli_ring *pring;
711e96a22b0SJames Smart struct lpfc_iocbq *wqe, *next_wqe;
712e96a22b0SJames Smart bool foundit = false;
713e96a22b0SJames Smart
714e96a22b0SJames Smart if (!ndlp) {
715372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
716e96a22b0SJames Smart "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
717e96a22b0SJames Smart "x%06x, Failing LS Req\n",
718e96a22b0SJames Smart ndlp, ndlp ? ndlp->nlp_DID : 0);
719e96a22b0SJames Smart return -EINVAL;
720e96a22b0SJames Smart }
721e96a22b0SJames Smart
722e96a22b0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
723e96a22b0SJames Smart "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
724f1156125SJames Smart "x%px rqstlen:%d rsplen:%d %pad %pad\n",
725e96a22b0SJames Smart pnvme_lsreq, pnvme_lsreq->rqstlen,
726e96a22b0SJames Smart pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
727e96a22b0SJames Smart &pnvme_lsreq->rspdma);
728e96a22b0SJames Smart
729e96a22b0SJames Smart /*
730e96a22b0SJames Smart * Lock the ELS ring txcmplq and look for the wqe that matches
731e96a22b0SJames Smart * this ELS. If found, issue an abort on the wqe.
732e96a22b0SJames Smart */
733e96a22b0SJames Smart pring = phba->sli4_hba.nvmels_wq->pring;
734e96a22b0SJames Smart spin_lock_irq(&phba->hbalock);
735e96a22b0SJames Smart spin_lock(&pring->ring_lock);
736e96a22b0SJames Smart list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
737d51cf5bdSJames Smart if (wqe->context_un.nvme_lsreq == pnvme_lsreq) {
738a680a929SJames Smart wqe->cmd_flag |= LPFC_DRIVER_ABORTED;
739e96a22b0SJames Smart foundit = true;
740e96a22b0SJames Smart break;
741e96a22b0SJames Smart }
742e96a22b0SJames Smart }
743e96a22b0SJames Smart spin_unlock(&pring->ring_lock);
744e96a22b0SJames Smart
745e96a22b0SJames Smart if (foundit)
746db7531d2SJames Smart lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);
747e96a22b0SJames Smart spin_unlock_irq(&phba->hbalock);
748e96a22b0SJames Smart
749e96a22b0SJames Smart if (foundit)
750e96a22b0SJames Smart return 0;
751e96a22b0SJames Smart
752e96a22b0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
753f1156125SJames Smart "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",
754e96a22b0SJames Smart pnvme_lsreq);
7556b6e8963SJames Smart return -EINVAL;
756e96a22b0SJames Smart }
757e96a22b0SJames Smart
7589aa09e98SJames Smart static int
lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port * localport,struct nvme_fc_remote_port * remoteport,struct nvmefc_ls_rsp * ls_rsp)7599aa09e98SJames Smart lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
7609aa09e98SJames Smart struct nvme_fc_remote_port *remoteport,
7619aa09e98SJames Smart struct nvmefc_ls_rsp *ls_rsp)
7629aa09e98SJames Smart {
7639aa09e98SJames Smart struct lpfc_async_xchg_ctx *axchg =
7649aa09e98SJames Smart container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
7659aa09e98SJames Smart struct lpfc_nvme_lport *lport;
7669aa09e98SJames Smart int rc;
7679aa09e98SJames Smart
768e39811beSJustin Tee if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag))
7699aa09e98SJames Smart return -ENODEV;
7709aa09e98SJames Smart
7719aa09e98SJames Smart lport = (struct lpfc_nvme_lport *)localport->private;
7729aa09e98SJames Smart
7739aa09e98SJames Smart rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
7749aa09e98SJames Smart
7759aa09e98SJames Smart if (rc) {
7769aa09e98SJames Smart /*
7779aa09e98SJames Smart * unless the failure is due to having already sent
7789aa09e98SJames Smart * the response, an abort will be generated for the
7799aa09e98SJames Smart * exchange if the rsp can't be sent.
7809aa09e98SJames Smart */
7819aa09e98SJames Smart if (rc != -EALREADY)
7829aa09e98SJames Smart atomic_inc(&lport->xmt_ls_abort);
7839aa09e98SJames Smart return rc;
7849aa09e98SJames Smart }
7859aa09e98SJames Smart
7869aa09e98SJames Smart return 0;
7879aa09e98SJames Smart }
7889aa09e98SJames Smart
789e96a22b0SJames Smart /**
790e96a22b0SJames Smart * lpfc_nvme_ls_abort - Abort a prior NVME LS request
79109d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from.
79209d99705SLee Jones * @pnvme_rport: Transport remoteport that LS is to be sent to.
79309d99705SLee Jones * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
794e96a22b0SJames Smart *
795e96a22b0SJames Smart * Driver registers this routine to abort a NVME LS request that is
796e96a22b0SJames Smart * in progress (from the transports perspective).
79701649561SJames Smart **/
79801649561SJames Smart static void
lpfc_nvme_ls_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)79901649561SJames Smart lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
80001649561SJames Smart struct nvme_fc_remote_port *pnvme_rport,
80101649561SJames Smart struct nvmefc_ls_req *pnvme_lsreq)
80201649561SJames Smart {
80301649561SJames Smart struct lpfc_nvme_lport *lport;
80401649561SJames Smart struct lpfc_vport *vport;
80501649561SJames Smart struct lpfc_nodelist *ndlp;
806e96a22b0SJames Smart int ret;
80701649561SJames Smart
80801649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
80966a210ffSJames Smart if (unlikely(!lport))
81066a210ffSJames Smart return;
81101649561SJames Smart vport = lport->vport;
81201649561SJames Smart
813e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag))
8143386f4bdSJames Smart return;
8153386f4bdSJames Smart
81601649561SJames Smart ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
81701649561SJames Smart
818e96a22b0SJames Smart ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
819e96a22b0SJames Smart if (!ret)
8204b056682SJames Smart atomic_inc(&lport->xmt_ls_abort);
82101649561SJames Smart }
82201649561SJames Smart
82301649561SJames Smart /* Fix up the existing sgls for NVME IO. */
8245fd11085SJames Smart static inline void
lpfc_nvme_adj_fcp_sgls(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct nvmefc_fcp_req * nCmd)82501649561SJames Smart lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
826c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd,
82701649561SJames Smart struct nvmefc_fcp_req *nCmd)
82801649561SJames Smart {
8294e565cf0SJames Smart struct lpfc_hba *phba = vport->phba;
83001649561SJames Smart struct sli4_sge *sgl;
83101649561SJames Smart union lpfc_wqe128 *wqe;
83201649561SJames Smart uint32_t *wptr, *dptr;
83301649561SJames Smart
83401649561SJames Smart /*
8354e565cf0SJames Smart * Get a local pointer to the built-in wqe and correct
8364e565cf0SJames Smart * the cmd size to match NVME's 96 bytes and fix
8374e565cf0SJames Smart * the dma address.
8384e565cf0SJames Smart */
8394e565cf0SJames Smart
840205e8240SJames Smart wqe = &lpfc_ncmd->cur_iocbq.wqe;
8414e565cf0SJames Smart
8424e565cf0SJames Smart /*
84301649561SJames Smart * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
84401649561SJames Smart * match NVME. NVME sends 96 bytes. Also, use the
84501649561SJames Smart * nvme commands command and response dma addresses
84601649561SJames Smart * rather than the virtual memory to ease the restore
84701649561SJames Smart * operation.
84801649561SJames Smart */
8490794d601SJames Smart sgl = lpfc_ncmd->dma_sgl;
85001649561SJames Smart sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
8514e565cf0SJames Smart if (phba->cfg_nvme_embed_cmd) {
8524e565cf0SJames Smart sgl->addr_hi = 0;
8534e565cf0SJames Smart sgl->addr_lo = 0;
8544e565cf0SJames Smart
8554e565cf0SJames Smart /* Word 0-2 - NVME CMND IU (embedded payload) */
8564e565cf0SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
8574e565cf0SJames Smart wqe->generic.bde.tus.f.bdeSize = 56;
8584e565cf0SJames Smart wqe->generic.bde.addrHigh = 0;
8594e565cf0SJames Smart wqe->generic.bde.addrLow = 64; /* Word 16 */
8604e565cf0SJames Smart
8615fd11085SJames Smart /* Word 10 - dbde is 0, wqes is 1 in template */
86201649561SJames Smart
86301649561SJames Smart /*
86401649561SJames Smart * Embed the payload in the last half of the WQE
86501649561SJames Smart * WQE words 16-30 get the NVME CMD IU payload
86601649561SJames Smart *
867b06a622fSJames Smart * WQE words 16-19 get payload Words 1-4
86801649561SJames Smart * WQE words 20-21 get payload Words 6-7
86901649561SJames Smart * WQE words 22-29 get payload Words 16-23
87001649561SJames Smart */
871b06a622fSJames Smart wptr = &wqe->words[16]; /* WQE ptr */
87201649561SJames Smart dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */
873b06a622fSJames Smart dptr++; /* Skip Word 0 in payload */
87401649561SJames Smart
875b06a622fSJames Smart *wptr++ = *dptr++; /* Word 1 */
87601649561SJames Smart *wptr++ = *dptr++; /* Word 2 */
87701649561SJames Smart *wptr++ = *dptr++; /* Word 3 */
87801649561SJames Smart *wptr++ = *dptr++; /* Word 4 */
87901649561SJames Smart dptr++; /* Skip Word 5 in payload */
88001649561SJames Smart *wptr++ = *dptr++; /* Word 6 */
88101649561SJames Smart *wptr++ = *dptr++; /* Word 7 */
88201649561SJames Smart dptr += 8; /* Skip Words 8-15 in payload */
88301649561SJames Smart *wptr++ = *dptr++; /* Word 16 */
88401649561SJames Smart *wptr++ = *dptr++; /* Word 17 */
88501649561SJames Smart *wptr++ = *dptr++; /* Word 18 */
88601649561SJames Smart *wptr++ = *dptr++; /* Word 19 */
88701649561SJames Smart *wptr++ = *dptr++; /* Word 20 */
88801649561SJames Smart *wptr++ = *dptr++; /* Word 21 */
88901649561SJames Smart *wptr++ = *dptr++; /* Word 22 */
89001649561SJames Smart *wptr = *dptr; /* Word 23 */
8915fd11085SJames Smart } else {
8925fd11085SJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
8935fd11085SJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
8945fd11085SJames Smart
8955fd11085SJames Smart /* Word 0-2 - NVME CMND IU Inline BDE */
8965fd11085SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
8975fd11085SJames Smart wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
8985fd11085SJames Smart wqe->generic.bde.addrHigh = sgl->addr_hi;
8995fd11085SJames Smart wqe->generic.bde.addrLow = sgl->addr_lo;
9005fd11085SJames Smart
9015fd11085SJames Smart /* Word 10 */
9025fd11085SJames Smart bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
9035fd11085SJames Smart bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
9045fd11085SJames Smart }
9055fd11085SJames Smart
9065fd11085SJames Smart sgl++;
9075fd11085SJames Smart
9085fd11085SJames Smart /* Setup the physical region for the FCP RSP */
9095fd11085SJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
9105fd11085SJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
9115fd11085SJames Smart sgl->word2 = le32_to_cpu(sgl->word2);
9125fd11085SJames Smart if (nCmd->sg_cnt)
9135fd11085SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0);
9145fd11085SJames Smart else
9155fd11085SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1);
9165fd11085SJames Smart sgl->word2 = cpu_to_le32(sgl->word2);
9175fd11085SJames Smart sgl->sge_len = cpu_to_le32(nCmd->rsplen);
91801649561SJames Smart }
91901649561SJames Smart
920bd2cdd5eSJames Smart
92109d99705SLee Jones /*
922a680a929SJames Smart * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO
92301649561SJames Smart *
92401649561SJames Smart * Driver registers this routine as it io request handler. This
92501649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
92601649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport.
92701649561SJames Smart *
92801649561SJames Smart * Return value :
92901649561SJames Smart * 0 - Success
93001649561SJames Smart * TODO: What are the failure codes.
93101649561SJames Smart **/
93201649561SJames Smart static void
lpfc_nvme_io_cmd_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,struct lpfc_iocbq * pwqeOut)933a680a929SJames Smart lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
934a680a929SJames Smart struct lpfc_iocbq *pwqeOut)
93501649561SJames Smart {
936d51cf5bdSJames Smart struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;
937a680a929SJames Smart struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
93801649561SJames Smart struct lpfc_vport *vport = pwqeIn->vport;
93901649561SJames Smart struct nvmefc_fcp_req *nCmd;
94001649561SJames Smart struct nvme_fc_ersp_iu *ep;
94101649561SJames Smart struct nvme_fc_cmd_iu *cp;
94201649561SJames Smart struct lpfc_nodelist *ndlp;
943bbe3012bSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv;
9444b056682SJames Smart struct lpfc_nvme_lport *lport;
945352b205aSArnd Bergmann uint32_t code, status, idx;
94601649561SJames Smart uint16_t cid, sqhd, data;
94701649561SJames Smart uint32_t *ptr;
94802243836SJames Smart uint32_t lat;
94902243836SJames Smart bool call_done = false;
950840eda96SJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
951840eda96SJames Smart int cpu;
952840eda96SJames Smart #endif
953d472a766SJustin Tee bool offline = false;
95401649561SJames Smart
95501649561SJames Smart /* Sanity check on return of outstanding command */
9564d5e789aSJames Smart if (!lpfc_ncmd) {
957372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
9584d5e789aSJames Smart "6071 Null lpfc_ncmd pointer. No "
9594d5e789aSJames Smart "release, skip completion\n");
9604d5e789aSJames Smart return;
9614d5e789aSJames Smart }
9624d5e789aSJames Smart
963c2017260SJames Smart /* Guard against abort handler being called at same time */
964c2017260SJames Smart spin_lock(&lpfc_ncmd->buf_lock);
965c2017260SJames Smart
966c2017260SJames Smart if (!lpfc_ncmd->nvmeCmd) {
967c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock);
968372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
96932350664SJames Smart "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
97032350664SJames Smart "nvmeCmd x%px\n",
9710b05e9feSJames Smart lpfc_ncmd, lpfc_ncmd->nvmeCmd);
9724d5e789aSJames Smart
9734d5e789aSJames Smart /* Release the lpfc_ncmd regardless of the missing elements. */
9744d5e789aSJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd);
97501649561SJames Smart return;
97601649561SJames Smart }
97701649561SJames Smart nCmd = lpfc_ncmd->nvmeCmd;
9784b056682SJames Smart status = bf_get(lpfc_wcqe_c_status, wcqe);
97966a210ffSJames Smart
9804c47efc1SJames Smart idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
9814c47efc1SJames Smart phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
9824c47efc1SJames Smart
9831df09449SJames Smart if (unlikely(status && vport->localport)) {
9844b056682SJames Smart lport = (struct lpfc_nvme_lport *)vport->localport->private;
9851df09449SJames Smart if (lport) {
9864b056682SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe))
9874b056682SJames Smart atomic_inc(&lport->cmpl_fcp_xb);
9884b056682SJames Smart atomic_inc(&lport->cmpl_fcp_err);
9894b056682SJames Smart }
99066a210ffSJames Smart }
99101649561SJames Smart
992bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
993bd2cdd5eSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag,
9944b056682SJames Smart status, wcqe->parameter);
99501649561SJames Smart /*
99601649561SJames Smart * Catch race where our node has transitioned, but the
99701649561SJames Smart * transport is still transitioning.
99801649561SJames Smart */
9990b05e9feSJames Smart ndlp = lpfc_ncmd->ndlp;
1000307e3380SJames Smart if (!ndlp) {
1001372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
100201649561SJames Smart "6062 Ignoring NVME cmpl. No ndlp\n");
100301649561SJames Smart goto out_err;
100401649561SJames Smart }
100501649561SJames Smart
100601649561SJames Smart code = bf_get(lpfc_wcqe_c_code, wcqe);
100701649561SJames Smart if (code == CQE_CODE_NVME_ERSP) {
100801649561SJames Smart /* For this type of CQE, we need to rebuild the rsp */
100901649561SJames Smart ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
101001649561SJames Smart
101101649561SJames Smart /*
101201649561SJames Smart * Get Command Id from cmd to plug into response. This
101301649561SJames Smart * code is not needed in the next NVME Transport drop.
101401649561SJames Smart */
101501649561SJames Smart cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
101601649561SJames Smart cid = cp->sqe.common.command_id;
101701649561SJames Smart
101801649561SJames Smart /*
101901649561SJames Smart * RSN is in CQE word 2
102001649561SJames Smart * SQHD is in CQE Word 3 bits 15:0
102101649561SJames Smart * Cmd Specific info is in CQE Word 1
102201649561SJames Smart * and in CQE Word 0 bits 15:0
102301649561SJames Smart */
102401649561SJames Smart sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
102501649561SJames Smart
102601649561SJames Smart /* Now lets build the NVME ERSP IU */
102701649561SJames Smart ep->iu_len = cpu_to_be16(8);
102801649561SJames Smart ep->rsn = wcqe->parameter;
102901649561SJames Smart ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
103001649561SJames Smart ep->rsvd12 = 0;
103101649561SJames Smart ptr = (uint32_t *)&ep->cqe.result.u64;
103201649561SJames Smart *ptr++ = wcqe->total_data_placed;
103301649561SJames Smart data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
103401649561SJames Smart *ptr = (uint32_t)data;
103501649561SJames Smart ep->cqe.sq_head = sqhd;
103601649561SJames Smart ep->cqe.sq_id = nCmd->sqid;
103701649561SJames Smart ep->cqe.command_id = cid;
103801649561SJames Smart ep->cqe.status = 0;
103901649561SJames Smart
104001649561SJames Smart lpfc_ncmd->status = IOSTAT_SUCCESS;
104101649561SJames Smart lpfc_ncmd->result = 0;
104201649561SJames Smart nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
104301649561SJames Smart nCmd->transferred_length = nCmd->payload_length;
104401649561SJames Smart } else {
10456a84d015SJustin Tee lpfc_ncmd->status = status;
1046952c303bSDick Kennedy lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
104701649561SJames Smart
104801649561SJames Smart /* For NVME, the only failure path that results in an
104901649561SJames Smart * IO error is when the adapter rejects it. All other
105001649561SJames Smart * conditions are a success case and resolved by the
105101649561SJames Smart * transport.
105201649561SJames Smart * IOSTAT_FCP_RSP_ERROR means:
105301649561SJames Smart * 1. Length of data received doesn't match total
105401649561SJames Smart * transfer length in WQE
105501649561SJames Smart * 2. If the RSP payload does NOT match these cases:
105601649561SJames Smart * a. RSP length 12/24 bytes and all zeros
105701649561SJames Smart * b. NVME ERSP
105801649561SJames Smart */
105901649561SJames Smart switch (lpfc_ncmd->status) {
106001649561SJames Smart case IOSTAT_SUCCESS:
106101649561SJames Smart nCmd->transferred_length = wcqe->total_data_placed;
106201649561SJames Smart nCmd->rcv_rsplen = 0;
106301649561SJames Smart nCmd->status = 0;
106401649561SJames Smart break;
106501649561SJames Smart case IOSTAT_FCP_RSP_ERROR:
106601649561SJames Smart nCmd->transferred_length = wcqe->total_data_placed;
106701649561SJames Smart nCmd->rcv_rsplen = wcqe->parameter;
106801649561SJames Smart nCmd->status = 0;
1069fa21189dSJames Smart
1070ea7bd1f3SJames Smart /* Get the NVME cmd details for this unique error. */
1071ea7bd1f3SJames Smart cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1072ea7bd1f3SJames Smart ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1073ea7bd1f3SJames Smart
1074fa21189dSJames Smart /* Check if this is really an ERSP */
1075fa21189dSJames Smart if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {
1076fa21189dSJames Smart lpfc_ncmd->status = IOSTAT_SUCCESS;
1077fa21189dSJames Smart lpfc_ncmd->result = 0;
1078fa21189dSJames Smart
1079fa21189dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1080ea7bd1f3SJames Smart "6084 NVME FCP_ERR ERSP: "
1081ea7bd1f3SJames Smart "xri %x placed x%x opcode x%x cmd_id "
1082ea7bd1f3SJames Smart "x%x cqe_status x%x\n",
1083fa21189dSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag,
1084ea7bd1f3SJames Smart wcqe->total_data_placed,
1085ea7bd1f3SJames Smart cp->sqe.common.opcode,
1086ea7bd1f3SJames Smart cp->sqe.common.command_id,
1087ea7bd1f3SJames Smart ep->cqe.status);
108801649561SJames Smart break;
1089fa21189dSJames Smart }
1090372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
109101649561SJames Smart "6081 NVME Completion Protocol Error: "
109286c67379SJames Smart "xri %x status x%x result x%x "
1093ea7bd1f3SJames Smart "placed x%x opcode x%x cmd_id x%x, "
1094ea7bd1f3SJames Smart "cqe_status x%x\n",
109586c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag,
109601649561SJames Smart lpfc_ncmd->status, lpfc_ncmd->result,
1097ea7bd1f3SJames Smart wcqe->total_data_placed,
1098ea7bd1f3SJames Smart cp->sqe.common.opcode,
1099ea7bd1f3SJames Smart cp->sqe.common.command_id,
1100ea7bd1f3SJames Smart ep->cqe.status);
110101649561SJames Smart break;
1102952c303bSDick Kennedy case IOSTAT_LOCAL_REJECT:
1103952c303bSDick Kennedy /* Let fall through to set command final state. */
1104952c303bSDick Kennedy if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1105952c303bSDick Kennedy lpfc_printf_vlog(vport, KERN_INFO,
1106952c303bSDick Kennedy LOG_NVME_IOERR,
110732350664SJames Smart "6032 Delay Aborted cmd x%px "
110832350664SJames Smart "nvme cmd x%px, xri x%x, "
1109952c303bSDick Kennedy "xb %d\n",
1110952c303bSDick Kennedy lpfc_ncmd, nCmd,
1111952c303bSDick Kennedy lpfc_ncmd->cur_iocbq.sli4_xritag,
1112952c303bSDick Kennedy bf_get(lpfc_wcqe_c_xb, wcqe));
1113df561f66SGustavo A. R. Silva fallthrough;
111401649561SJames Smart default:
111501649561SJames Smart out_err:
1116e3246a12SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
111786c67379SJames Smart "6072 NVME Completion Error: xri %x "
11182c013a3aSJames Smart "status x%x result x%x [x%x] "
11192c013a3aSJames Smart "placed x%x\n",
112086c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag,
112101649561SJames Smart lpfc_ncmd->status, lpfc_ncmd->result,
11222c013a3aSJames Smart wcqe->parameter,
112301649561SJames Smart wcqe->total_data_placed);
112401649561SJames Smart nCmd->transferred_length = 0;
112501649561SJames Smart nCmd->rcv_rsplen = 0;
11268e009ce8SJames Smart nCmd->status = NVME_SC_INTERNAL;
1127d472a766SJustin Tee if (pci_channel_offline(vport->phba->pcidev) ||
1128d472a766SJustin Tee lpfc_ncmd->result == IOERR_SLI_DOWN)
1129d472a766SJustin Tee offline = true;
113001649561SJames Smart }
113101649561SJames Smart }
113201649561SJames Smart
113301649561SJames Smart /* pick up SLI4 exhange busy condition */
113425ac2c97SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline)
113501649561SJames Smart lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
113601649561SJames Smart else
113701649561SJames Smart lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
113801649561SJames Smart
113901649561SJames Smart /* Update stats and complete the IO. There is
114001649561SJames Smart * no need for dma unprep because the nvme_transport
114101649561SJames Smart * owns the dma address.
114201649561SJames Smart */
1143bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1144c8a4ce0bSDick Kennedy if (lpfc_ncmd->ts_cmd_start) {
1145bd2cdd5eSJames Smart lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
11462fcbc569SJames Smart lpfc_ncmd->ts_data_io = ktime_get_ns();
11472fcbc569SJames Smart phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
11482fcbc569SJames Smart lpfc_io_ktime(phba, lpfc_ncmd);
1149bd2cdd5eSJames Smart }
1150840eda96SJames Smart if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1151d6d189ceSBart Van Assche cpu = raw_smp_processor_id();
1152840eda96SJames Smart this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
115363df6d63SJames Smart if (lpfc_ncmd->cpu != cpu)
115463df6d63SJames Smart lpfc_printf_vlog(vport,
115563df6d63SJames Smart KERN_INFO, LOG_NVME_IOERR,
1156bd2cdd5eSJames Smart "6701 CPU Check cmpl: "
1157bd2cdd5eSJames Smart "cpu %d expect %d\n",
115863df6d63SJames Smart cpu, lpfc_ncmd->cpu);
1159bd2cdd5eSJames Smart }
1160bd2cdd5eSJames Smart #endif
1161952c303bSDick Kennedy
1162952c303bSDick Kennedy /* NVME targets need completion held off until the abort exchange
1163add9d6beSJames Smart * completes unless the NVME Rport is getting unregistered.
1164952c303bSDick Kennedy */
1165add9d6beSJames Smart
11663fd78355SJames Smart if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
116791455b85SJames Smart freqpriv = nCmd->private;
116891455b85SJames Smart freqpriv->nvme_buf = NULL;
11693fd78355SJames Smart lpfc_ncmd->nvmeCmd = NULL;
117002243836SJames Smart call_done = true;
117102243836SJames Smart }
1172c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock);
117302243836SJames Smart
117402243836SJames Smart /* Check if IO qualified for CMF */
117502243836SJames Smart if (phba->cmf_active_mode != LPFC_CFG_OFF &&
117602243836SJames Smart nCmd->io_dir == NVMEFC_FCP_READ &&
117702243836SJames Smart nCmd->payload_length) {
117802243836SJames Smart /* Used when calculating average latency */
117902243836SJames Smart lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;
118002243836SJames Smart lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL);
118102243836SJames Smart }
118202243836SJames Smart
118302243836SJames Smart if (call_done)
1184c2017260SJames Smart nCmd->done(nCmd);
118501649561SJames Smart
1186952c303bSDick Kennedy /* Call release with XB=1 to queue the IO into the abort list. */
118701649561SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd);
118801649561SJames Smart }
118901649561SJames Smart
119001649561SJames Smart
119101649561SJames Smart /**
119201649561SJames Smart * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
119309d99705SLee Jones * @vport: pointer to a host virtual N_Port data structure
11943e749170SLee Jones * @lpfc_ncmd: Pointer to lpfc scsi command
119509d99705SLee Jones * @pnode: pointer to a node-list data structure
119609d99705SLee Jones * @cstat: pointer to the control status structure
119701649561SJames Smart *
119801649561SJames Smart * Driver registers this routine as it io request handler. This
119901649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
120001649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport.
120101649561SJames Smart *
120201649561SJames Smart * Return value :
120301649561SJames Smart * 0 - Success
120401649561SJames Smart * TODO: What are the failure codes.
120501649561SJames Smart **/
120601649561SJames Smart static int
lpfc_nvme_prep_io_cmd(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct lpfc_nodelist * pnode,struct lpfc_fc4_ctrl_stat * cstat)120701649561SJames Smart lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1208c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd,
120966a210ffSJames Smart struct lpfc_nodelist *pnode,
12104c47efc1SJames Smart struct lpfc_fc4_ctrl_stat *cstat)
121101649561SJames Smart {
121201649561SJames Smart struct lpfc_hba *phba = vport->phba;
121301649561SJames Smart struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
12142e7e9c0cSJames Smart struct nvme_common_command *sqe;
12152e7e9c0cSJames Smart struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;
1216205e8240SJames Smart union lpfc_wqe128 *wqe = &pwqeq->wqe;
121701649561SJames Smart uint32_t req_len;
121801649561SJames Smart
121901649561SJames Smart /*
122001649561SJames Smart * There are three possibilities here - use scatter-gather segment, use
122101649561SJames Smart * the single mapping, or neither.
122201649561SJames Smart */
122301649561SJames Smart if (nCmd->sg_cnt) {
122401649561SJames Smart if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
12255fd11085SJames Smart /* From the iwrite template, initialize words 7 - 11 */
12265fd11085SJames Smart memcpy(&wqe->words[7],
12275fd11085SJames Smart &lpfc_iwrite_cmd_template.words[7],
12285fd11085SJames Smart sizeof(uint32_t) * 5);
12295fd11085SJames Smart
12305fd11085SJames Smart /* Word 4 */
12315fd11085SJames Smart wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
12325fd11085SJames Smart
123301649561SJames Smart /* Word 5 */
123401649561SJames Smart if ((phba->cfg_nvme_enable_fb) &&
123501649561SJames Smart (pnode->nlp_flag & NLP_FIRSTBURST)) {
123601649561SJames Smart req_len = lpfc_ncmd->nvmeCmd->payload_length;
123701649561SJames Smart if (req_len < pnode->nvme_fb_size)
123801649561SJames Smart wqe->fcp_iwrite.initial_xfer_len =
123901649561SJames Smart req_len;
124001649561SJames Smart else
124101649561SJames Smart wqe->fcp_iwrite.initial_xfer_len =
124201649561SJames Smart pnode->nvme_fb_size;
12435fd11085SJames Smart } else {
12445fd11085SJames Smart wqe->fcp_iwrite.initial_xfer_len = 0;
124501649561SJames Smart }
12464c47efc1SJames Smart cstat->output_requests++;
124701649561SJames Smart } else {
12485fd11085SJames Smart /* From the iread template, initialize words 7 - 11 */
12495fd11085SJames Smart memcpy(&wqe->words[7],
12505fd11085SJames Smart &lpfc_iread_cmd_template.words[7],
12515fd11085SJames Smart sizeof(uint32_t) * 5);
125201649561SJames Smart
12535fd11085SJames Smart /* Word 4 */
12545fd11085SJames Smart wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
125501649561SJames Smart
12565fd11085SJames Smart /* Word 5 */
12575fd11085SJames Smart wqe->fcp_iread.rsrvd5 = 0;
125801649561SJames Smart
125902243836SJames Smart /* For a CMF Managed port, iod must be zero'ed */
126002243836SJames Smart if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
126102243836SJames Smart bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
126202243836SJames Smart LPFC_WQE_IOD_NONE);
12634c47efc1SJames Smart cstat->input_requests++;
126401649561SJames Smart }
126501649561SJames Smart } else {
12665fd11085SJames Smart /* From the icmnd template, initialize words 4 - 11 */
12675fd11085SJames Smart memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
12685fd11085SJames Smart sizeof(uint32_t) * 8);
12694c47efc1SJames Smart cstat->control_requests++;
127001649561SJames Smart }
12710d8af096SJames Smart
12722e7e9c0cSJames Smart if (pnode->nlp_nvme_info & NLP_NVME_NSLER) {
12730d8af096SJames Smart bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
12742e7e9c0cSJames Smart sqe = &((struct nvme_fc_cmd_iu *)
12752e7e9c0cSJames Smart nCmd->cmdaddr)->sqe.common;
12762e7e9c0cSJames Smart if (sqe->opcode == nvme_admin_async_event)
12772e7e9c0cSJames Smart bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1);
12782e7e9c0cSJames Smart }
12792e7e9c0cSJames Smart
128001649561SJames Smart /*
128101649561SJames Smart * Finish initializing those WQE fields that are independent
128201649561SJames Smart * of the nvme_cmnd request_buffer
128301649561SJames Smart */
128401649561SJames Smart
12855fd11085SJames Smart /* Word 3 */
12865fd11085SJames Smart bf_set(payload_offset_len, &wqe->fcp_icmd,
12875fd11085SJames Smart (nCmd->rsplen + nCmd->cmdlen));
12885fd11085SJames Smart
128901649561SJames Smart /* Word 6 */
129001649561SJames Smart bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
129101649561SJames Smart phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
129201649561SJames Smart bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
129301649561SJames Smart
129401649561SJames Smart /* Word 8 */
129501649561SJames Smart wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
129601649561SJames Smart
129701649561SJames Smart /* Word 9 */
129801649561SJames Smart bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
129901649561SJames Smart
1300840a4701SJames Smart /* Word 10 */
1301840a4701SJames Smart bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);
1302840a4701SJames Smart
1303414abe0aSJames Smart /* Words 13 14 15 are for PBDE support */
1304414abe0aSJames Smart
1305896325a8SJames Smart /* add the VMID tags as per switch response */
1306896325a8SJames Smart if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {
1307896325a8SJames Smart if (phba->pport->vmid_priority_tagging) {
1308896325a8SJames Smart bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
1309896325a8SJames Smart bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
1310896325a8SJames Smart lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);
1311896325a8SJames Smart } else {
1312896325a8SJames Smart bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
1313896325a8SJames Smart bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
1314896325a8SJames Smart wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;
1315896325a8SJames Smart }
1316896325a8SJames Smart }
1317896325a8SJames Smart
131801649561SJames Smart pwqeq->vport = vport;
131901649561SJames Smart return 0;
132001649561SJames Smart }
132101649561SJames Smart
132201649561SJames Smart
132301649561SJames Smart /**
132401649561SJames Smart * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
132509d99705SLee Jones * @vport: pointer to a host virtual N_Port data structure
13263e749170SLee Jones * @lpfc_ncmd: Pointer to lpfc scsi command
132701649561SJames Smart *
132801649561SJames Smart * Driver registers this routine as it io request handler. This
132901649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
133001649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport.
133101649561SJames Smart *
133201649561SJames Smart * Return value :
133301649561SJames Smart * 0 - Success
133401649561SJames Smart * TODO: What are the failure codes.
133501649561SJames Smart **/
133601649561SJames Smart static int
lpfc_nvme_prep_io_dma(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd)133701649561SJames Smart lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1338c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd)
133901649561SJames Smart {
134001649561SJames Smart struct lpfc_hba *phba = vport->phba;
134101649561SJames Smart struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1342205e8240SJames Smart union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
13430794d601SJames Smart struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1344d79c9e9dSJames Smart struct sli4_hybrid_sgl *sgl_xtra = NULL;
134501649561SJames Smart struct scatterlist *data_sg;
134601649561SJames Smart struct sli4_sge *first_data_sgl;
13470bc2b7c5SJames Smart struct ulp_bde64 *bde;
1348d79c9e9dSJames Smart dma_addr_t physaddr = 0;
1349d79c9e9dSJames Smart uint32_t dma_len = 0;
135001649561SJames Smart uint32_t dma_offset = 0;
1351d79c9e9dSJames Smart int nseg, i, j;
1352d79c9e9dSJames Smart bool lsp_just_set = false;
135301649561SJames Smart
135401649561SJames Smart /* Fix up the command and response DMA stuff. */
135501649561SJames Smart lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
135601649561SJames Smart
135701649561SJames Smart /*
135801649561SJames Smart * There are three possibilities here - use scatter-gather segment, use
135901649561SJames Smart * the single mapping, or neither.
136001649561SJames Smart */
136101649561SJames Smart if (nCmd->sg_cnt) {
136201649561SJames Smart /*
136301649561SJames Smart * Jump over the cmd and rsp SGEs. The fix routine
136401649561SJames Smart * has already adjusted for this.
136501649561SJames Smart */
136601649561SJames Smart sgl += 2;
136701649561SJames Smart
136801649561SJames Smart first_data_sgl = sgl;
136901649561SJames Smart lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
137081e6a637SJames Smart if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1371372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
137201649561SJames Smart "6058 Too many sg segments from "
137301649561SJames Smart "NVME Transport. Max %d, "
137401649561SJames Smart "nvmeIO sg_cnt %d\n",
137529bfd55aSJames Smart phba->cfg_nvme_seg_cnt + 1,
137601649561SJames Smart lpfc_ncmd->seg_cnt);
137701649561SJames Smart lpfc_ncmd->seg_cnt = 0;
137801649561SJames Smart return 1;
137901649561SJames Smart }
138001649561SJames Smart
138101649561SJames Smart /*
138201649561SJames Smart * The driver established a maximum scatter-gather segment count
138301649561SJames Smart * during probe that limits the number of sg elements in any
138401649561SJames Smart * single nvme command. Just run through the seg_cnt and format
138501649561SJames Smart * the sge's.
138601649561SJames Smart */
138701649561SJames Smart nseg = nCmd->sg_cnt;
138801649561SJames Smart data_sg = nCmd->first_sgl;
1389d79c9e9dSJames Smart
1390d79c9e9dSJames Smart /* for tracking the segment boundaries */
1391d79c9e9dSJames Smart j = 2;
139201649561SJames Smart for (i = 0; i < nseg; i++) {
139301649561SJames Smart if (data_sg == NULL) {
1394372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
139501649561SJames Smart "6059 dptr err %d, nseg %d\n",
139601649561SJames Smart i, nseg);
139701649561SJames Smart lpfc_ncmd->seg_cnt = 0;
139801649561SJames Smart return 1;
139901649561SJames Smart }
1400d79c9e9dSJames Smart
1401d79c9e9dSJames Smart sgl->word2 = 0;
1402315b3fd1SJames Smart if (nseg == 1) {
1403d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1);
1404d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl,
1405d79c9e9dSJames Smart LPFC_SGE_TYPE_DATA);
1406d79c9e9dSJames Smart } else {
1407d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0);
1408d79c9e9dSJames Smart
1409d79c9e9dSJames Smart /* expand the segment */
1410d79c9e9dSJames Smart if (!lsp_just_set &&
1411d79c9e9dSJames Smart !((j + 1) % phba->border_sge_num) &&
1412d79c9e9dSJames Smart ((nseg - 1) != i)) {
1413d79c9e9dSJames Smart /* set LSP type */
1414d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl,
1415d79c9e9dSJames Smart LPFC_SGE_TYPE_LSP);
1416d79c9e9dSJames Smart
1417d79c9e9dSJames Smart sgl_xtra = lpfc_get_sgl_per_hdwq(
1418d79c9e9dSJames Smart phba, lpfc_ncmd);
1419d79c9e9dSJames Smart
1420d79c9e9dSJames Smart if (unlikely(!sgl_xtra)) {
1421d79c9e9dSJames Smart lpfc_ncmd->seg_cnt = 0;
1422d79c9e9dSJames Smart return 1;
1423d79c9e9dSJames Smart }
1424d79c9e9dSJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(
1425d79c9e9dSJames Smart sgl_xtra->dma_phys_sgl));
1426d79c9e9dSJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1427d79c9e9dSJames Smart sgl_xtra->dma_phys_sgl));
1428d79c9e9dSJames Smart
1429d79c9e9dSJames Smart } else {
1430d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl,
1431d79c9e9dSJames Smart LPFC_SGE_TYPE_DATA);
1432d79c9e9dSJames Smart }
1433d79c9e9dSJames Smart }
1434d79c9e9dSJames Smart
1435d79c9e9dSJames Smart if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1436d79c9e9dSJames Smart LPFC_SGE_TYPE_LSP)) {
1437d79c9e9dSJames Smart if ((nseg - 1) == i)
1438d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1);
1439d79c9e9dSJames Smart
1440a14396b6SJames Smart physaddr = sg_dma_address(data_sg);
1441a14396b6SJames Smart dma_len = sg_dma_len(data_sg);
1442d79c9e9dSJames Smart sgl->addr_lo = cpu_to_le32(
1443d79c9e9dSJames Smart putPaddrLow(physaddr));
1444d79c9e9dSJames Smart sgl->addr_hi = cpu_to_le32(
1445d79c9e9dSJames Smart putPaddrHigh(physaddr));
1446d79c9e9dSJames Smart
144701649561SJames Smart bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
144801649561SJames Smart sgl->word2 = cpu_to_le32(sgl->word2);
144901649561SJames Smart sgl->sge_len = cpu_to_le32(dma_len);
145001649561SJames Smart
145101649561SJames Smart dma_offset += dma_len;
145201649561SJames Smart data_sg = sg_next(data_sg);
1453d79c9e9dSJames Smart
145401649561SJames Smart sgl++;
1455d79c9e9dSJames Smart
1456d79c9e9dSJames Smart lsp_just_set = false;
1457d79c9e9dSJames Smart } else {
1458d79c9e9dSJames Smart sgl->word2 = cpu_to_le32(sgl->word2);
1459d79c9e9dSJames Smart
1460d79c9e9dSJames Smart sgl->sge_len = cpu_to_le32(
1461d79c9e9dSJames Smart phba->cfg_sg_dma_buf_size);
1462d79c9e9dSJames Smart
1463d79c9e9dSJames Smart sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1464d79c9e9dSJames Smart i = i - 1;
1465d79c9e9dSJames Smart
1466d79c9e9dSJames Smart lsp_just_set = true;
1467d79c9e9dSJames Smart }
1468d79c9e9dSJames Smart
1469d79c9e9dSJames Smart j++;
147001649561SJames Smart }
1471315b3fd1SJames Smart
1472315b3fd1SJames Smart /* PBDE support for first data SGE only */
1473315b3fd1SJames Smart if (nseg == 1 && phba->cfg_enable_pbde) {
14740bc2b7c5SJames Smart /* Words 13-15 */
14750bc2b7c5SJames Smart bde = (struct ulp_bde64 *)
14760bc2b7c5SJames Smart &wqe->words[13];
14770bc2b7c5SJames Smart bde->addrLow = first_data_sgl->addr_lo;
14780bc2b7c5SJames Smart bde->addrHigh = first_data_sgl->addr_hi;
14790bc2b7c5SJames Smart bde->tus.f.bdeSize =
14800bc2b7c5SJames Smart le32_to_cpu(first_data_sgl->sge_len);
14810bc2b7c5SJames Smart bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
14820bc2b7c5SJames Smart bde->tus.w = cpu_to_le32(bde->tus.w);
1483b101eb27SJames Smart
1484315b3fd1SJames Smart /* Word 11 - set PBDE bit */
1485b101eb27SJames Smart bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
14865fd11085SJames Smart } else {
14875fd11085SJames Smart memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1488315b3fd1SJames Smart /* Word 11 - PBDE bit disabled by default template */
14895fd11085SJames Smart }
14900bc2b7c5SJames Smart
1491414abe0aSJames Smart } else {
14920794d601SJames Smart lpfc_ncmd->seg_cnt = 0;
14930794d601SJames Smart
149401649561SJames Smart /* For this clause to be valid, the payload_length
149501649561SJames Smart * and sg_cnt must zero.
149601649561SJames Smart */
149701649561SJames Smart if (nCmd->payload_length != 0) {
1498372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
149901649561SJames Smart "6063 NVME DMA Prep Err: sg_cnt %d "
150001649561SJames Smart "payload_length x%x\n",
150101649561SJames Smart nCmd->sg_cnt, nCmd->payload_length);
150201649561SJames Smart return 1;
150301649561SJames Smart }
150401649561SJames Smart }
150501649561SJames Smart return 0;
150601649561SJames Smart }
150701649561SJames Smart
150801649561SJames Smart /**
150901649561SJames Smart * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
15103e749170SLee Jones * @pnvme_lport: Pointer to the driver's local port data
15113e749170SLee Jones * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
151201649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
15133e749170SLee Jones * @pnvme_fcreq: IO request from nvme fc to driver.
151401649561SJames Smart *
151501649561SJames Smart * Driver registers this routine as it io request handler. This
151601649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
15173e749170SLee Jones * data structure to the rport indicated in @lpfc_nvme_rport.
151801649561SJames Smart *
151901649561SJames Smart * Return value :
152001649561SJames Smart * 0 - Success
152101649561SJames Smart * TODO: What are the failure codes.
152201649561SJames Smart **/
152301649561SJames Smart static int
lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)152401649561SJames Smart lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
152501649561SJames Smart struct nvme_fc_remote_port *pnvme_rport,
152601649561SJames Smart void *hw_queue_handle,
152701649561SJames Smart struct nvmefc_fcp_req *pnvme_fcreq)
152801649561SJames Smart {
152901649561SJames Smart int ret = 0;
1530cf1a1d3eSJames Smart int expedite = 0;
153163df6d63SJames Smart int idx, cpu;
153201649561SJames Smart struct lpfc_nvme_lport *lport;
15334c47efc1SJames Smart struct lpfc_fc4_ctrl_stat *cstat;
153401649561SJames Smart struct lpfc_vport *vport;
153501649561SJames Smart struct lpfc_hba *phba;
153601649561SJames Smart struct lpfc_nodelist *ndlp;
1537c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd;
153801649561SJames Smart struct lpfc_nvme_rport *rport;
153901649561SJames Smart struct lpfc_nvme_qhandle *lpfc_queue_info;
1540c3725bdcSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv;
1541cf1a1d3eSJames Smart struct nvme_common_command *sqe;
1542bd2cdd5eSJames Smart uint64_t start = 0;
1543896325a8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
1544896325a8SJames Smart u8 *uuid = NULL;
1545896325a8SJames Smart int err;
1546896325a8SJames Smart enum dma_data_direction iodir;
1547896325a8SJames Smart #endif
154801649561SJames Smart
1549c3725bdcSJames Smart /* Validate pointers. LLDD fault handling with transport does
1550c3725bdcSJames Smart * have timing races.
1551c3725bdcSJames Smart */
155201649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1553c3725bdcSJames Smart if (unlikely(!lport)) {
1554c3725bdcSJames Smart ret = -EINVAL;
1555c3725bdcSJames Smart goto out_fail;
1556c3725bdcSJames Smart }
1557c3725bdcSJames Smart
155801649561SJames Smart vport = lport->vport;
1559c3725bdcSJames Smart
1560c3725bdcSJames Smart if (unlikely(!hw_queue_handle)) {
156144c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
156244c2757bSJames Smart "6117 Fail IO, NULL hw_queue_handle\n");
156344c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err);
1564cd240071SJames Smart ret = -EBUSY;
1565c3725bdcSJames Smart goto out_fail;
1566c3725bdcSJames Smart }
1567c3725bdcSJames Smart
156801649561SJames Smart phba = vport->phba;
156901649561SJames Smart
1570e39811beSJustin Tee if ((unlikely(test_bit(FC_UNLOADING, &vport->load_flag))) ||
1571*e780c942SJustin Tee test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
157244c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
157344c2757bSJames Smart "6124 Fail IO, Driver unload\n");
157444c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err);
15753386f4bdSJames Smart ret = -ENODEV;
15763386f4bdSJames Smart goto out_fail;
15773386f4bdSJames Smart }
15783386f4bdSJames Smart
1579c3725bdcSJames Smart freqpriv = pnvme_fcreq->private;
1580c3725bdcSJames Smart if (unlikely(!freqpriv)) {
158144c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
158244c2757bSJames Smart "6158 Fail IO, NULL request data\n");
158344c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err);
1584c3725bdcSJames Smart ret = -EINVAL;
1585b7672ae6SDick Kennedy goto out_fail;
1586b7672ae6SDick Kennedy }
1587b7672ae6SDick Kennedy
1588bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1589bd2cdd5eSJames Smart if (phba->ktime_on)
1590bd2cdd5eSJames Smart start = ktime_get_ns();
1591bd2cdd5eSJames Smart #endif
159201649561SJames Smart rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
159301649561SJames Smart lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
159401649561SJames Smart
159501649561SJames Smart /*
159601649561SJames Smart * Catch race where our node has transitioned, but the
159701649561SJames Smart * transport is still transitioning.
159801649561SJames Smart */
159901649561SJames Smart ndlp = rport->ndlp;
1600307e3380SJames Smart if (!ndlp) {
160144c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
160232350664SJames Smart "6053 Busy IO, ndlp not ready: rport x%px "
160332350664SJames Smart "ndlp x%px, DID x%06x\n",
160401649561SJames Smart rport, ndlp, pnvme_rport->port_id);
160544c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err);
1606cd240071SJames Smart ret = -EBUSY;
160701649561SJames Smart goto out_fail;
160801649561SJames Smart }
160901649561SJames Smart
161001649561SJames Smart /* The remote node has to be a mapped target or it's an error. */
161101649561SJames Smart if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
161201649561SJames Smart (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
161344c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
161444c2757bSJames Smart "6036 Fail IO, DID x%06x not ready for "
1615cd240071SJames Smart "IO. State x%x, Type x%x Flg x%x\n",
161644c2757bSJames Smart pnvme_rport->port_id,
1617cd240071SJames Smart ndlp->nlp_state, ndlp->nlp_type,
1618e9b11083SJames Smart ndlp->fc4_xpt_flags);
16194b056682SJames Smart atomic_inc(&lport->xmt_fcp_bad_ndlp);
1620cd240071SJames Smart ret = -EBUSY;
162101649561SJames Smart goto out_fail;
162201649561SJames Smart
162301649561SJames Smart }
162401649561SJames Smart
1625cf1a1d3eSJames Smart /* Currently only NVME Keep alive commands should be expedited
1626cf1a1d3eSJames Smart * if the driver runs out of a resource. These should only be
1627cf1a1d3eSJames Smart * issued on the admin queue, qidx 0
1628cf1a1d3eSJames Smart */
1629cf1a1d3eSJames Smart if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1630cf1a1d3eSJames Smart sqe = &((struct nvme_fc_cmd_iu *)
1631cf1a1d3eSJames Smart pnvme_fcreq->cmdaddr)->sqe.common;
1632cf1a1d3eSJames Smart if (sqe->opcode == nvme_admin_keep_alive)
1633cf1a1d3eSJames Smart expedite = 1;
1634cf1a1d3eSJames Smart }
1635cf1a1d3eSJames Smart
163602243836SJames Smart /* Check if IO qualifies for CMF */
163702243836SJames Smart if (phba->cmf_active_mode != LPFC_CFG_OFF &&
163802243836SJames Smart pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&
163902243836SJames Smart pnvme_fcreq->payload_length) {
164002243836SJames Smart ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length);
164102243836SJames Smart if (ret) {
164202243836SJames Smart ret = -EBUSY;
164302243836SJames Smart goto out_fail;
164402243836SJames Smart }
164502243836SJames Smart /* Get start time for IO latency */
164602243836SJames Smart start = ktime_get_ns();
164702243836SJames Smart }
164802243836SJames Smart
164901649561SJames Smart /* The node is shared with FCP IO, make sure the IO pending count does
165001649561SJames Smart * not exceed the programmed depth.
165101649561SJames Smart */
16522a5b7d62SJames Smart if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1653cf1a1d3eSJames Smart if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1654cf1a1d3eSJames Smart !expedite) {
165544c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
165644c2757bSJames Smart "6174 Fail IO, ndlp qdepth exceeded: "
16574d5e789aSJames Smart "idx %d DID %x pend %d qdepth %d\n",
16584d5e789aSJames Smart lpfc_queue_info->index, ndlp->nlp_DID,
16594d5e789aSJames Smart atomic_read(&ndlp->cmd_pending),
16604d5e789aSJames Smart ndlp->cmd_qdepth);
16614b056682SJames Smart atomic_inc(&lport->xmt_fcp_qdepth);
1662cd22d605SDick Kennedy ret = -EBUSY;
166302243836SJames Smart goto out_fail1;
166401649561SJames Smart }
16652a5b7d62SJames Smart }
166601649561SJames Smart
16676a828b0fSJames Smart /* Lookup Hardware Queue index based on fcp_io_sched module parameter */
166845aa312eSJames Smart if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
166945aa312eSJames Smart idx = lpfc_queue_info->index;
167045aa312eSJames Smart } else {
1671d6d189ceSBart Van Assche cpu = raw_smp_processor_id();
16726a828b0fSJames Smart idx = phba->sli4_hba.cpu_map[cpu].hdwq;
167345aa312eSJames Smart }
167445aa312eSJames Smart
167545aa312eSJames Smart lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
167601649561SJames Smart if (lpfc_ncmd == NULL) {
16774b056682SJames Smart atomic_inc(&lport->xmt_fcp_noxri);
167801649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
167944c2757bSJames Smart "6065 Fail IO, driver buffer pool is empty: "
168044c2757bSJames Smart "idx %d DID %x\n",
168144c2757bSJames Smart lpfc_queue_info->index, ndlp->nlp_DID);
1682cd22d605SDick Kennedy ret = -EBUSY;
168302243836SJames Smart goto out_fail1;
168401649561SJames Smart }
1685bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1686c8a4ce0bSDick Kennedy if (start) {
1687bd2cdd5eSJames Smart lpfc_ncmd->ts_cmd_start = start;
1688bd2cdd5eSJames Smart lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1689c8a4ce0bSDick Kennedy } else {
1690c8a4ce0bSDick Kennedy lpfc_ncmd->ts_cmd_start = 0;
1691bd2cdd5eSJames Smart }
1692bd2cdd5eSJames Smart #endif
169302243836SJames Smart lpfc_ncmd->rx_cmd_start = start;
169401649561SJames Smart
169501649561SJames Smart /*
169601649561SJames Smart * Store the data needed by the driver to issue, abort, and complete
169701649561SJames Smart * an IO.
169801649561SJames Smart * Do not let the IO hang out forever. There is no midlayer issuing
169901649561SJames Smart * an abort so inform the FW of the maximum IO pending time.
170001649561SJames Smart */
1701bbe3012bSJames Smart freqpriv->nvme_buf = lpfc_ncmd;
170201649561SJames Smart lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1703318083adSJames Smart lpfc_ncmd->ndlp = ndlp;
17040794d601SJames Smart lpfc_ncmd->qidx = lpfc_queue_info->qidx;
170501649561SJames Smart
1706896325a8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
1707896325a8SJames Smart /* check the necessary and sufficient condition to support VMID */
1708896325a8SJames Smart if (lpfc_is_vmid_enabled(phba) &&
1709896325a8SJames Smart (ndlp->vmid_support ||
1710896325a8SJames Smart phba->pport->vmid_priority_tagging ==
1711896325a8SJames Smart LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
1712896325a8SJames Smart /* is the I/O generated by a VM, get the associated virtual */
1713896325a8SJames Smart /* entity id */
1714896325a8SJames Smart uuid = nvme_fc_io_getuuid(pnvme_fcreq);
1715896325a8SJames Smart
1716896325a8SJames Smart if (uuid) {
1717896325a8SJames Smart if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE)
1718896325a8SJames Smart iodir = DMA_TO_DEVICE;
1719896325a8SJames Smart else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ)
1720896325a8SJames Smart iodir = DMA_FROM_DEVICE;
1721896325a8SJames Smart else
1722896325a8SJames Smart iodir = DMA_NONE;
1723896325a8SJames Smart
1724896325a8SJames Smart err = lpfc_vmid_get_appid(vport, uuid, iodir,
1725896325a8SJames Smart (union lpfc_vmid_io_tag *)
1726896325a8SJames Smart &lpfc_ncmd->cur_iocbq.vmid_tag);
1727896325a8SJames Smart if (!err)
1728896325a8SJames Smart lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;
1729896325a8SJames Smart }
1730896325a8SJames Smart }
1731896325a8SJames Smart #endif
1732896325a8SJames Smart
173301649561SJames Smart /*
173401649561SJames Smart * Issue the IO on the WQ indicated by index in the hw_queue_handle.
173501649561SJames Smart * This identfier was create in our hardware queue create callback
173601649561SJames Smart * routine. The driver now is dependent on the IO queue steering from
173701649561SJames Smart * the transport. We are trusting the upper NVME layers know which
173801649561SJames Smart * index to use and that they have affinitized a CPU to this hardware
173901649561SJames Smart * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
174001649561SJames Smart */
174166a210ffSJames Smart lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
17424c47efc1SJames Smart cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
174366a210ffSJames Smart
174466a210ffSJames Smart lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
174566a210ffSJames Smart ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
174666a210ffSJames Smart if (ret) {
174744c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
174844c2757bSJames Smart "6175 Fail IO, Prep DMA: "
174944c2757bSJames Smart "idx %d DID %x\n",
175044c2757bSJames Smart lpfc_queue_info->index, ndlp->nlp_DID);
175144c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err);
175266a210ffSJames Smart ret = -ENOMEM;
175366a210ffSJames Smart goto out_free_nvme_buf;
175466a210ffSJames Smart }
175566a210ffSJames Smart
1756bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1757bd2cdd5eSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag,
1758bd2cdd5eSJames Smart lpfc_queue_info->index, ndlp->nlp_DID);
1759bd2cdd5eSJames Smart
17601fbf9742SJames Smart ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
176101649561SJames Smart if (ret) {
17624b056682SJames Smart atomic_inc(&lport->xmt_fcp_wqerr);
1763e3246a12SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
176444c2757bSJames Smart "6113 Fail IO, Could not issue WQE err %x "
176501649561SJames Smart "sid: x%x did: x%x oxid: x%x\n",
176601649561SJames Smart ret, vport->fc_myDID, ndlp->nlp_DID,
176701649561SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag);
176801649561SJames Smart goto out_free_nvme_buf;
176901649561SJames Smart }
177001649561SJames Smart
1771c490850aSJames Smart if (phba->cfg_xri_rebalancing)
1772c490850aSJames Smart lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1773c490850aSJames Smart
1774bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1775c8a4ce0bSDick Kennedy if (lpfc_ncmd->ts_cmd_start)
1776bd2cdd5eSJames Smart lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1777bd2cdd5eSJames Smart
1778840eda96SJames Smart if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1779d6d189ceSBart Van Assche cpu = raw_smp_processor_id();
1780840eda96SJames Smart this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
178163df6d63SJames Smart lpfc_ncmd->cpu = cpu;
178263df6d63SJames Smart if (idx != cpu)
1783bd2cdd5eSJames Smart lpfc_printf_vlog(vport,
178463df6d63SJames Smart KERN_INFO, LOG_NVME_IOERR,
1785bd2cdd5eSJames Smart "6702 CPU Check cmd: "
1786bd2cdd5eSJames Smart "cpu %d wq %d\n",
1787bd2cdd5eSJames Smart lpfc_ncmd->cpu,
1788bd2cdd5eSJames Smart lpfc_queue_info->index);
1789bd2cdd5eSJames Smart }
1790bd2cdd5eSJames Smart #endif
179101649561SJames Smart return 0;
179201649561SJames Smart
179301649561SJames Smart out_free_nvme_buf:
17942cee7808SJames Smart if (lpfc_ncmd->nvmeCmd->sg_cnt) {
17952cee7808SJames Smart if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
17964c47efc1SJames Smart cstat->output_requests--;
17972cee7808SJames Smart else
17984c47efc1SJames Smart cstat->input_requests--;
17992cee7808SJames Smart } else
18004c47efc1SJames Smart cstat->control_requests--;
180101649561SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd);
180202243836SJames Smart out_fail1:
180302243836SJames Smart lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,
180402243836SJames Smart pnvme_fcreq->payload_length, NULL);
180501649561SJames Smart out_fail:
180601649561SJames Smart return ret;
180701649561SJames Smart }
180801649561SJames Smart
180901649561SJames Smart /**
181001649561SJames Smart * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
181101649561SJames Smart * @phba: Pointer to HBA context object
181201649561SJames Smart * @cmdiocb: Pointer to command iocb object.
181324e1f056SJames Smart * @rspiocb: Pointer to response iocb object.
181401649561SJames Smart *
181501649561SJames Smart * This is the callback function for any NVME FCP IO that was aborted.
181601649561SJames Smart *
181701649561SJames Smart * Return value:
181801649561SJames Smart * None
181901649561SJames Smart **/
182001649561SJames Smart void
lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * cmdiocb,struct lpfc_iocbq * rspiocb)182101649561SJames Smart lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
182224e1f056SJames Smart struct lpfc_iocbq *rspiocb)
182301649561SJames Smart {
182424e1f056SJames Smart struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl;
182524e1f056SJames Smart
1826e3246a12SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
182701649561SJames Smart "6145 ABORT_XRI_CN completing on rpi x%x "
182801649561SJames Smart "original iotag x%x, abort cmd iotag x%x "
182901649561SJames Smart "req_tag x%x, status x%x, hwstatus x%x\n",
183031a59f75SJames Smart bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com),
183131a59f75SJames Smart get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag,
183201649561SJames Smart bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
183301649561SJames Smart bf_get(lpfc_wcqe_c_status, abts_cmpl),
183401649561SJames Smart bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
183501649561SJames Smart lpfc_sli_release_iocbq(phba, cmdiocb);
183601649561SJames Smart }
183701649561SJames Smart
183801649561SJames Smart /**
183901649561SJames Smart * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
18403e749170SLee Jones * @pnvme_lport: Pointer to the driver's local port data
18413e749170SLee Jones * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
184201649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
18433e749170SLee Jones * @pnvme_fcreq: IO request from nvme fc to driver.
184401649561SJames Smart *
184501649561SJames Smart * Driver registers this routine as its nvme request io abort handler. This
184601649561SJames Smart * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
184701649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. This routine
184801649561SJames Smart * is executed asynchronously - one the target is validated as "MAPPED" and
184901649561SJames Smart * ready for IO, the driver issues the abort request and returns.
185001649561SJames Smart *
185101649561SJames Smart * Return value:
185201649561SJames Smart * None
185301649561SJames Smart **/
185401649561SJames Smart static void
lpfc_nvme_fcp_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)185501649561SJames Smart lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
185601649561SJames Smart struct nvme_fc_remote_port *pnvme_rport,
185701649561SJames Smart void *hw_queue_handle,
185801649561SJames Smart struct nvmefc_fcp_req *pnvme_fcreq)
185901649561SJames Smart {
186001649561SJames Smart struct lpfc_nvme_lport *lport;
186101649561SJames Smart struct lpfc_vport *vport;
186201649561SJames Smart struct lpfc_hba *phba;
1863c490850aSJames Smart struct lpfc_io_buf *lpfc_nbuf;
186401649561SJames Smart struct lpfc_iocbq *nvmereq_wqe;
1865c3725bdcSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv;
186601649561SJames Smart unsigned long flags;
186701649561SJames Smart int ret_val;
186801649561SJames Smart
1869c3725bdcSJames Smart /* Validate pointers. LLDD fault handling with transport does
1870c3725bdcSJames Smart * have timing races.
1871c3725bdcSJames Smart */
187201649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1873c3725bdcSJames Smart if (unlikely(!lport))
1874c3725bdcSJames Smart return;
187501649561SJames Smart
1876c3725bdcSJames Smart vport = lport->vport;
1877c3725bdcSJames Smart
1878c3725bdcSJames Smart if (unlikely(!hw_queue_handle)) {
1879c3725bdcSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1880c3725bdcSJames Smart "6129 Fail Abort, HW Queue Handle NULL.\n");
1881c3725bdcSJames Smart return;
1882c3725bdcSJames Smart }
1883c3725bdcSJames Smart
1884c3725bdcSJames Smart phba = vport->phba;
1885c3725bdcSJames Smart freqpriv = pnvme_fcreq->private;
1886c3725bdcSJames Smart
1887c3725bdcSJames Smart if (unlikely(!freqpriv))
1888c3725bdcSJames Smart return;
1889e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag))
18903386f4bdSJames Smart return;
18913386f4bdSJames Smart
189201649561SJames Smart /* Announce entry to new IO submit field. */
189386c67379SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
189401649561SJames Smart "6002 Abort Request to rport DID x%06x "
189532350664SJames Smart "for nvme_fc_req x%px\n",
189601649561SJames Smart pnvme_rport->port_id,
189701649561SJames Smart pnvme_fcreq);
189801649561SJames Smart
189978e9e350SJustin Tee lpfc_nbuf = freqpriv->nvme_buf;
190078e9e350SJustin Tee if (!lpfc_nbuf) {
190178e9e350SJustin Tee lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
190278e9e350SJustin Tee "6140 NVME IO req has no matching lpfc nvme "
190378e9e350SJustin Tee "io buffer. Skipping abort req.\n");
190478e9e350SJustin Tee return;
190578e9e350SJustin Tee } else if (!lpfc_nbuf->nvmeCmd) {
190678e9e350SJustin Tee lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
190778e9e350SJustin Tee "6141 lpfc NVME IO req has no nvme_fcreq "
190878e9e350SJustin Tee "io buffer. Skipping abort req.\n");
190978e9e350SJustin Tee return;
191078e9e350SJustin Tee }
191178e9e350SJustin Tee
191201649561SJames Smart /* driver queued commands are in process of being flushed */
1913*e780c942SJustin Tee if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {
1914372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
191501649561SJames Smart "6139 Driver in reset cleanup - flushing "
1916*e780c942SJustin Tee "NVME Req now. hba_flag x%lx\n",
191701649561SJames Smart phba->hba_flag);
191801649561SJames Smart return;
191901649561SJames Smart }
192001649561SJames Smart
1921*e780c942SJustin Tee /* Guard against IO completion being called at same time */
1922*e780c942SJustin Tee spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags);
1923*e780c942SJustin Tee spin_lock(&phba->hbalock);
1924*e780c942SJustin Tee
19252b7824d0SJames Smart nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
192601649561SJames Smart
192701649561SJames Smart /*
192801649561SJames Smart * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
192901649561SJames Smart * state must match the nvme_fcreq passed by the nvme
193001649561SJames Smart * transport. If they don't match, it is likely the driver
193101649561SJames Smart * has already completed the NVME IO and the nvme transport
193201649561SJames Smart * has not seen it yet.
193301649561SJames Smart */
193401649561SJames Smart if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1935372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
193601649561SJames Smart "6143 NVME req mismatch: "
193732350664SJames Smart "lpfc_nbuf x%px nvmeCmd x%px, "
193832350664SJames Smart "pnvme_fcreq x%px. Skipping Abort xri x%x\n",
193901649561SJames Smart lpfc_nbuf, lpfc_nbuf->nvmeCmd,
19402b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1941c2017260SJames Smart goto out_unlock;
194201649561SJames Smart }
194301649561SJames Smart
194401649561SJames Smart /* Don't abort IOs no longer on the pending queue. */
1945a680a929SJames Smart if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
1946372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
194732350664SJames Smart "6142 NVME IO req x%px not queued - skipping "
19482b7824d0SJames Smart "abort req xri x%x\n",
19492b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1950c2017260SJames Smart goto out_unlock;
195101649561SJames Smart }
195201649561SJames Smart
19534b056682SJames Smart atomic_inc(&lport->xmt_fcp_abort);
1954bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1955bd2cdd5eSJames Smart nvmereq_wqe->sli4_xritag,
195600cefeb9SJames Smart nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1957bd2cdd5eSJames Smart
195801649561SJames Smart /* Outstanding abort is in progress */
1959a680a929SJames Smart if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) {
1960372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
196101649561SJames Smart "6144 Outstanding NVME I/O Abort Request "
196232350664SJames Smart "still pending on nvme_fcreq x%px, "
1963f1156125SJames Smart "lpfc_ncmd x%px xri x%x\n",
19642b7824d0SJames Smart pnvme_fcreq, lpfc_nbuf,
19652b7824d0SJames Smart nvmereq_wqe->sli4_xritag);
1966c2017260SJames Smart goto out_unlock;
196701649561SJames Smart }
196801649561SJames Smart
1969db7531d2SJames Smart ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe,
1970db7531d2SJames Smart lpfc_nvme_abort_fcreq_cmpl);
197101649561SJames Smart
197278e9e350SJustin Tee spin_unlock(&phba->hbalock);
197378e9e350SJustin Tee spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1974a22d73b6SJames Smart
1975a22d73b6SJames Smart /* Make sure HBA is alive */
1976a22d73b6SJames Smart lpfc_issue_hb_tmo(phba);
1977a22d73b6SJames Smart
1978db7531d2SJames Smart if (ret_val != WQE_SUCCESS) {
1979372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
198001649561SJames Smart "6137 Failed abts issue_wqe with status x%x "
198132350664SJames Smart "for nvme_fcreq x%px.\n",
198201649561SJames Smart ret_val, pnvme_fcreq);
198301649561SJames Smart return;
198401649561SJames Smart }
198501649561SJames Smart
198686c67379SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
19872b7824d0SJames Smart "6138 Transport Abort NVME Request Issued for "
19881a5cd3d0SJustin Tee "ox_id x%x\n",
19891a5cd3d0SJustin Tee nvmereq_wqe->sli4_xritag);
1990c2017260SJames Smart return;
1991c2017260SJames Smart
1992c2017260SJames Smart out_unlock:
199378e9e350SJustin Tee spin_unlock(&phba->hbalock);
199478e9e350SJustin Tee spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1995c2017260SJames Smart return;
199601649561SJames Smart }
199701649561SJames Smart
199801649561SJames Smart /* Declare and initialization an instance of the FC NVME template. */
199901649561SJames Smart static struct nvme_fc_port_template lpfc_nvme_template = {
200001649561SJames Smart /* initiator-based functions */
200101649561SJames Smart .localport_delete = lpfc_nvme_localport_delete,
200201649561SJames Smart .remoteport_delete = lpfc_nvme_remoteport_delete,
200301649561SJames Smart .create_queue = lpfc_nvme_create_queue,
200401649561SJames Smart .delete_queue = lpfc_nvme_delete_queue,
200501649561SJames Smart .ls_req = lpfc_nvme_ls_req,
200601649561SJames Smart .fcp_io = lpfc_nvme_fcp_io_submit,
200701649561SJames Smart .ls_abort = lpfc_nvme_ls_abort,
200801649561SJames Smart .fcp_abort = lpfc_nvme_fcp_abort,
20099aa09e98SJames Smart .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,
201001649561SJames Smart
201101649561SJames Smart .max_hw_queues = 1,
201201649561SJames Smart .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
201301649561SJames Smart .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
201401649561SJames Smart .dma_boundary = 0xFFFFFFFF,
201501649561SJames Smart
201601649561SJames Smart /* Sizes of additional private data for data structures.
201701649561SJames Smart * No use for the last two sizes at this time.
201801649561SJames Smart */
201901649561SJames Smart .local_priv_sz = sizeof(struct lpfc_nvme_lport),
202001649561SJames Smart .remote_priv_sz = sizeof(struct lpfc_nvme_rport),
202101649561SJames Smart .lsrqst_priv_sz = 0,
2022bbe3012bSJames Smart .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
202301649561SJames Smart };
202401649561SJames Smart
20253e749170SLee Jones /*
20265e5b511dSJames Smart * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
202701649561SJames Smart *
20285e5b511dSJames Smart * This routine removes a nvme buffer from head of @hdwq io_buf_list
202901649561SJames Smart * and returns to caller.
203001649561SJames Smart *
203101649561SJames Smart * Return codes:
203201649561SJames Smart * NULL - Error
203301649561SJames Smart * Pointer to lpfc_nvme_buf - Success
203401649561SJames Smart **/
2035c490850aSJames Smart static struct lpfc_io_buf *
lpfc_get_nvme_buf(struct lpfc_hba * phba,struct lpfc_nodelist * ndlp,int idx,int expedite)2036cf1a1d3eSJames Smart lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
20375e5b511dSJames Smart int idx, int expedite)
203801649561SJames Smart {
2039c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd;
20405e5b511dSJames Smart struct lpfc_sli4_hdw_queue *qp;
20410794d601SJames Smart struct sli4_sge *sgl;
20420794d601SJames Smart struct lpfc_iocbq *pwqeq;
20430794d601SJames Smart union lpfc_wqe128 *wqe;
204401649561SJames Smart
2045c490850aSJames Smart lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
20462a5b7d62SJames Smart
20470794d601SJames Smart if (lpfc_ncmd) {
20480794d601SJames Smart pwqeq = &(lpfc_ncmd->cur_iocbq);
20490794d601SJames Smart wqe = &pwqeq->wqe;
20500794d601SJames Smart
20510794d601SJames Smart /* Setup key fields in buffer that may have been changed
20520794d601SJames Smart * if other protocols used this buffer.
20530794d601SJames Smart */
2054a680a929SJames Smart pwqeq->cmd_flag = LPFC_IO_NVME;
2055a680a929SJames Smart pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl;
20560794d601SJames Smart lpfc_ncmd->start_time = jiffies;
20570794d601SJames Smart lpfc_ncmd->flags = 0;
20580794d601SJames Smart
20590794d601SJames Smart /* Rsp SGE will be filled in when we rcv an IO
20600794d601SJames Smart * from the NVME Layer to be sent.
20610794d601SJames Smart * The cmd is going to be embedded so we need a SKIP SGE.
20620794d601SJames Smart */
20630794d601SJames Smart sgl = lpfc_ncmd->dma_sgl;
20640794d601SJames Smart bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
20650794d601SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0);
20660794d601SJames Smart sgl->word2 = cpu_to_le32(sgl->word2);
20670794d601SJames Smart /* Fill in word 3 / sgl_len during cmd submission */
20680794d601SJames Smart
2069d9f492a1SJames Smart /* Initialize 64 bytes only */
20700794d601SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe));
20710794d601SJames Smart
20720794d601SJames Smart if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
20732a5b7d62SJames Smart atomic_inc(&ndlp->cmd_pending);
2074c490850aSJames Smart lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
20752a5b7d62SJames Smart }
20765e5b511dSJames Smart
2077c490850aSJames Smart } else {
2078c490850aSJames Smart qp = &phba->sli4_hba.hdwq[idx];
20795e5b511dSJames Smart qp->empty_io_bufs++;
2080c490850aSJames Smart }
20815e5b511dSJames Smart
208201649561SJames Smart return lpfc_ncmd;
208301649561SJames Smart }
208401649561SJames Smart
208501649561SJames Smart /**
208601649561SJames Smart * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
208701649561SJames Smart * @phba: The Hba for which this call is being executed.
208801649561SJames Smart * @lpfc_ncmd: The nvme buffer which is being released.
208901649561SJames Smart *
209001649561SJames Smart * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
20915e5b511dSJames Smart * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
209201649561SJames Smart * and cannot be reused for at least RA_TOV amount of time if it was
209301649561SJames Smart * aborted.
209401649561SJames Smart **/
209501649561SJames Smart static void
lpfc_release_nvme_buf(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2096c490850aSJames Smart lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
209701649561SJames Smart {
20985e5b511dSJames Smart struct lpfc_sli4_hdw_queue *qp;
209901649561SJames Smart unsigned long iflag = 0;
210001649561SJames Smart
2101c490850aSJames Smart if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
21022a5b7d62SJames Smart atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
21032a5b7d62SJames Smart
21042a5b7d62SJames Smart lpfc_ncmd->ndlp = NULL;
2105c490850aSJames Smart lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
21062a5b7d62SJames Smart
21071fbf9742SJames Smart qp = lpfc_ncmd->hdwq;
210822770cbaSJames Smart if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
210986c67379SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
211086c67379SJames Smart "6310 XB release deferred for "
211186c67379SJames Smart "ox_id x%x on reqtag x%x\n",
211286c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag,
211386c67379SJames Smart lpfc_ncmd->cur_iocbq.iotag);
211486c67379SJames Smart
2115c00f62e6SJames Smart spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
211601649561SJames Smart list_add_tail(&lpfc_ncmd->list,
2117c00f62e6SJames Smart &qp->lpfc_abts_io_buf_list);
21185e5b511dSJames Smart qp->abts_nvme_io_bufs++;
2119c00f62e6SJames Smart spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2120c490850aSJames Smart } else
2121c490850aSJames Smart lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
212201649561SJames Smart }
212301649561SJames Smart
212401649561SJames Smart /**
212501649561SJames Smart * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2126f6b35a75SLee Jones * @vport: the lpfc_vport instance requesting a localport.
212701649561SJames Smart *
212801649561SJames Smart * This routine is invoked to create an nvme localport instance to bind
212901649561SJames Smart * to the nvme_fc_transport. It is called once during driver load
213001649561SJames Smart * like lpfc_create_shost after all other services are initialized.
213101649561SJames Smart * It requires a vport, vpi, and wwns at call time. Other localport
213201649561SJames Smart * parameters are modified as the driver's FCID and the Fabric WWN
213301649561SJames Smart * are established.
213401649561SJames Smart *
213501649561SJames Smart * Return codes
213601649561SJames Smart * 0 - successful
213701649561SJames Smart * -ENOMEM - no heap memory available
213801649561SJames Smart * other values - from nvme registration upcall
213901649561SJames Smart **/
214001649561SJames Smart int
lpfc_nvme_create_localport(struct lpfc_vport * vport)214101649561SJames Smart lpfc_nvme_create_localport(struct lpfc_vport *vport)
214201649561SJames Smart {
2143166d7211SJames Smart int ret = 0;
214401649561SJames Smart struct lpfc_hba *phba = vport->phba;
214501649561SJames Smart struct nvme_fc_port_info nfcp_info;
214601649561SJames Smart struct nvme_fc_local_port *localport;
214701649561SJames Smart struct lpfc_nvme_lport *lport;
214801649561SJames Smart
214901649561SJames Smart /* Initialize this localport instance. The vport wwn usage ensures
215001649561SJames Smart * that NPIV is accounted for.
215101649561SJames Smart */
215201649561SJames Smart memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
215301649561SJames Smart nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
215401649561SJames Smart nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
215501649561SJames Smart nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
215601649561SJames Smart
21575b9e70b2SJames Smart /* We need to tell the transport layer + 1 because it takes page
21585b9e70b2SJames Smart * alignment into account. When space for the SGL is allocated we
21595b9e70b2SJames Smart * allocate + 3, one for cmd, one for rsp and one for this alignment
21604d4c4a4aSJames Smart */
21614d4c4a4aSJames Smart lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
216201649561SJames Smart
2163542ddc9bSJames Smart /* Advertise how many hw queues we support based on cfg_hdw_queue,
2164542ddc9bSJames Smart * which will not exceed cpu count.
2165542ddc9bSJames Smart */
2166cdb42becSJames Smart lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
216766a210ffSJames Smart
2168faf5a744SArnd Bergmann if (!IS_ENABLED(CONFIG_NVME_FC))
2169faf5a744SArnd Bergmann return ret;
2170faf5a744SArnd Bergmann
217101649561SJames Smart /* localport is allocated from the stack, but the registration
217201649561SJames Smart * call allocates heap memory as well as the private area.
217301649561SJames Smart */
2174faf5a744SArnd Bergmann
217501649561SJames Smart ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
217601649561SJames Smart &vport->phba->pcidev->dev, &localport);
217701649561SJames Smart if (!ret) {
217801649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
217901649561SJames Smart "6005 Successfully registered local "
218032350664SJames Smart "NVME port num %d, localP x%px, private "
218132350664SJames Smart "x%px, sg_seg %d\n",
218201649561SJames Smart localport->port_num, localport,
218301649561SJames Smart localport->private,
218401649561SJames Smart lpfc_nvme_template.max_sgl_segments);
218501649561SJames Smart
218601649561SJames Smart /* Private is our lport size declared in the template. */
218701649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private;
218801649561SJames Smart vport->localport = localport;
218901649561SJames Smart lport->vport = vport;
219001649561SJames Smart vport->nvmei_support = 1;
21916b486ce9SDick Kennedy
21924b056682SJames Smart atomic_set(&lport->xmt_fcp_noxri, 0);
21934b056682SJames Smart atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
21944b056682SJames Smart atomic_set(&lport->xmt_fcp_qdepth, 0);
219544c2757bSJames Smart atomic_set(&lport->xmt_fcp_err, 0);
21964b056682SJames Smart atomic_set(&lport->xmt_fcp_wqerr, 0);
21974b056682SJames Smart atomic_set(&lport->xmt_fcp_abort, 0);
21984b056682SJames Smart atomic_set(&lport->xmt_ls_abort, 0);
21994b056682SJames Smart atomic_set(&lport->xmt_ls_err, 0);
22004b056682SJames Smart atomic_set(&lport->cmpl_fcp_xb, 0);
22014b056682SJames Smart atomic_set(&lport->cmpl_fcp_err, 0);
22024b056682SJames Smart atomic_set(&lport->cmpl_ls_xb, 0);
22034b056682SJames Smart atomic_set(&lport->cmpl_ls_err, 0);
22049aa09e98SJames Smart
220566a210ffSJames Smart atomic_set(&lport->fc4NvmeLsRequests, 0);
220666a210ffSJames Smart atomic_set(&lport->fc4NvmeLsCmpls, 0);
22076b486ce9SDick Kennedy }
22083ebd9b47SJames Smart
220901649561SJames Smart return ret;
221001649561SJames Smart }
221101649561SJames Smart
2212d964b3e5SBart Van Assche #if (IS_ENABLED(CONFIG_NVME_FC))
2213add9d6beSJames Smart /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2214add9d6beSJames Smart *
2215add9d6beSJames Smart * The driver has to wait for the host nvme transport to callback
2216add9d6beSJames Smart * indicating the localport has successfully unregistered all
2217add9d6beSJames Smart * resources. Since this is an uninterruptible wait, loop every ten
2218add9d6beSJames Smart * seconds and print a message indicating no progress.
2219add9d6beSJames Smart *
2220add9d6beSJames Smart * An uninterruptible wait is used because of the risk of transport-to-
2221add9d6beSJames Smart * driver state mismatch.
2222add9d6beSJames Smart */
22233999df75SBart Van Assche static void
lpfc_nvme_lport_unreg_wait(struct lpfc_vport * vport,struct lpfc_nvme_lport * lport,struct completion * lport_unreg_cmp)2224add9d6beSJames Smart lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
22257961cba6SEwan D. Milne struct lpfc_nvme_lport *lport,
22267961cba6SEwan D. Milne struct completion *lport_unreg_cmp)
2227add9d6beSJames Smart {
2228add9d6beSJames Smart u32 wait_tmo;
22292ab70c21SJames Smart int ret, i, pending = 0;
22302ab70c21SJames Smart struct lpfc_sli_ring *pring;
22312ab70c21SJames Smart struct lpfc_hba *phba = vport->phba;
2232e7dab164SJames Smart struct lpfc_sli4_hdw_queue *qp;
2233e7dab164SJames Smart int abts_scsi, abts_nvme;
2234add9d6beSJames Smart
2235add9d6beSJames Smart /* Host transport has to clean up and confirm requiring an indefinite
2236add9d6beSJames Smart * wait. Print a message if a 10 second wait expires and renew the
2237add9d6beSJames Smart * wait. This is unexpected.
2238add9d6beSJames Smart */
2239add9d6beSJames Smart wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2240add9d6beSJames Smart while (true) {
22417961cba6SEwan D. Milne ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2242add9d6beSJames Smart if (unlikely(!ret)) {
22432ab70c21SJames Smart pending = 0;
2244e7dab164SJames Smart abts_scsi = 0;
2245e7dab164SJames Smart abts_nvme = 0;
22462ab70c21SJames Smart for (i = 0; i < phba->cfg_hdw_queue; i++) {
2247e7dab164SJames Smart qp = &phba->sli4_hba.hdwq[i];
224835ed9613SJames Smart if (!vport->localport || !qp || !qp->io_wq)
224925ac2c97SJames Smart return;
225025ac2c97SJames Smart
2251e7dab164SJames Smart pring = qp->io_wq->pring;
22522ab70c21SJames Smart if (!pring)
22532ab70c21SJames Smart continue;
22542ab70c21SJames Smart pending += pring->txcmplq_cnt;
2255e7dab164SJames Smart abts_scsi += qp->abts_scsi_io_bufs;
2256e7dab164SJames Smart abts_nvme += qp->abts_nvme_io_bufs;
22572ab70c21SJames Smart }
225835ed9613SJames Smart if (!vport->localport ||
225935ed9613SJames Smart test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
226027c2bcf0SJustin Tee phba->link_state == LPFC_HBA_ERROR ||
2261e39811beSJustin Tee test_bit(FC_UNLOADING, &vport->load_flag))
226225ac2c97SJames Smart return;
226325ac2c97SJames Smart
2264372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
226532350664SJames Smart "6176 Lport x%px Localport x%px wait "
2266e7dab164SJames Smart "timed out. Pending %d [%d:%d]. "
2267e7dab164SJames Smart "Renewing.\n",
2268e7dab164SJames Smart lport, vport->localport, pending,
2269e7dab164SJames Smart abts_scsi, abts_nvme);
2270add9d6beSJames Smart continue;
2271add9d6beSJames Smart }
2272add9d6beSJames Smart break;
2273add9d6beSJames Smart }
2274add9d6beSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
227532350664SJames Smart "6177 Lport x%px Localport x%px Complete Success\n",
2276add9d6beSJames Smart lport, vport->localport);
2277add9d6beSJames Smart }
2278d964b3e5SBart Van Assche #endif
2279add9d6beSJames Smart
228001649561SJames Smart /**
228101649561SJames Smart * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
22823e749170SLee Jones * @vport: pointer to a host virtual N_Port data structure
228301649561SJames Smart *
228401649561SJames Smart * This routine is invoked to destroy all lports bound to the phba.
228501649561SJames Smart * The lport memory was allocated by the nvme fc transport and is
228601649561SJames Smart * released there. This routine ensures all rports bound to the
228701649561SJames Smart * lport have been disconnected.
228801649561SJames Smart *
228901649561SJames Smart **/
229001649561SJames Smart void
lpfc_nvme_destroy_localport(struct lpfc_vport * vport)229101649561SJames Smart lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
229201649561SJames Smart {
22937d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
229401649561SJames Smart struct nvme_fc_local_port *localport;
229501649561SJames Smart struct lpfc_nvme_lport *lport;
229601649561SJames Smart int ret;
22977961cba6SEwan D. Milne DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
229801649561SJames Smart
229901649561SJames Smart if (vport->nvmei_support == 0)
230001649561SJames Smart return;
230101649561SJames Smart
230201649561SJames Smart localport = vport->localport;
230325ac2c97SJames Smart if (!localport)
230425ac2c97SJames Smart return;
230501649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private;
230601649561SJames Smart
230701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
230832350664SJames Smart "6011 Destroying NVME localport x%px\n",
230901649561SJames Smart localport);
2310166d7211SJames Smart
231101649561SJames Smart /* lport's rport list is clear. Unregister
231201649561SJames Smart * lport and release resources.
231301649561SJames Smart */
23147961cba6SEwan D. Milne lport->lport_unreg_cmp = &lport_unreg_cmp;
231501649561SJames Smart ret = nvme_fc_unregister_localport(localport);
2316add9d6beSJames Smart
2317add9d6beSJames Smart /* Wait for completion. This either blocks
2318add9d6beSJames Smart * indefinitely or succeeds
2319add9d6beSJames Smart */
23207961cba6SEwan D. Milne lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
23217961cba6SEwan D. Milne vport->localport = NULL;
232201649561SJames Smart
232301649561SJames Smart /* Regardless of the unregister upcall response, clear
232401649561SJames Smart * nvmei_support. All rports are unregistered and the
232501649561SJames Smart * driver will clean up.
232601649561SJames Smart */
232701649561SJames Smart vport->nvmei_support = 0;
232801649561SJames Smart if (ret == 0) {
232901649561SJames Smart lpfc_printf_vlog(vport,
233001649561SJames Smart KERN_INFO, LOG_NVME_DISC,
233101649561SJames Smart "6009 Unregistered lport Success\n");
233201649561SJames Smart } else {
233301649561SJames Smart lpfc_printf_vlog(vport,
233401649561SJames Smart KERN_INFO, LOG_NVME_DISC,
233501649561SJames Smart "6010 Unregistered lport "
233601649561SJames Smart "Failed, status x%x\n",
233701649561SJames Smart ret);
233801649561SJames Smart }
2339166d7211SJames Smart #endif
234001649561SJames Smart }
234101649561SJames Smart
234201649561SJames Smart void
lpfc_nvme_update_localport(struct lpfc_vport * vport)234301649561SJames Smart lpfc_nvme_update_localport(struct lpfc_vport *vport)
234401649561SJames Smart {
23454410a67aSJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
234601649561SJames Smart struct nvme_fc_local_port *localport;
234701649561SJames Smart struct lpfc_nvme_lport *lport;
234801649561SJames Smart
234901649561SJames Smart localport = vport->localport;
23504410a67aSJames Smart if (!localport) {
23514410a67aSJames Smart lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
23524410a67aSJames Smart "6710 Update NVME fail. No localport\n");
23534410a67aSJames Smart return;
23544410a67aSJames Smart }
235501649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private;
23564410a67aSJames Smart if (!lport) {
23574410a67aSJames Smart lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
235832350664SJames Smart "6171 Update NVME fail. localP x%px, No lport\n",
23594410a67aSJames Smart localport);
23604410a67aSJames Smart return;
23614410a67aSJames Smart }
236201649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
236332350664SJames Smart "6012 Update NVME lport x%px did x%x\n",
236401649561SJames Smart localport, vport->fc_myDID);
236501649561SJames Smart
236601649561SJames Smart localport->port_id = vport->fc_myDID;
236701649561SJames Smart if (localport->port_id == 0)
236801649561SJames Smart localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
236901649561SJames Smart else
237001649561SJames Smart localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
237101649561SJames Smart
237201649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
237332350664SJames Smart "6030 bound lport x%px to DID x%06x\n",
237401649561SJames Smart lport, localport->port_id);
23754410a67aSJames Smart #endif
237601649561SJames Smart }
237701649561SJames Smart
237801649561SJames Smart int
lpfc_nvme_register_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)237901649561SJames Smart lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
238001649561SJames Smart {
23817d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
238201649561SJames Smart int ret = 0;
238301649561SJames Smart struct nvme_fc_local_port *localport;
238401649561SJames Smart struct lpfc_nvme_lport *lport;
238501649561SJames Smart struct lpfc_nvme_rport *rport;
238601466024SJames Smart struct lpfc_nvme_rport *oldrport;
238701649561SJames Smart struct nvme_fc_remote_port *remote_port;
238801649561SJames Smart struct nvme_fc_port_info rpinfo;
238993a3922dSJames Smart struct lpfc_nodelist *prev_ndlp = NULL;
23901f546823SJames Smart struct fc_rport *srport = ndlp->rport;
239101649561SJames Smart
239201649561SJames Smart lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
239301649561SJames Smart "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
239401649561SJames Smart ndlp->nlp_DID, ndlp->nlp_type);
239501649561SJames Smart
239601649561SJames Smart localport = vport->localport;
2397bb6a8a2cSDick Kennedy if (!localport)
2398bb6a8a2cSDick Kennedy return 0;
2399bb6a8a2cSDick Kennedy
240001649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private;
240101649561SJames Smart
24027a06dcd3SJames Smart /* NVME rports are not preserved across devloss.
24037a06dcd3SJames Smart * Just register this instance. Note, rpinfo->dev_loss_tmo
24047a06dcd3SJames Smart * is left 0 to indicate accept transport defaults. The
24057a06dcd3SJames Smart * driver communicates port role capabilities consistent
24067a06dcd3SJames Smart * with the PRLI response data.
240701649561SJames Smart */
24087a06dcd3SJames Smart memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
24097a06dcd3SJames Smart rpinfo.port_id = ndlp->nlp_DID;
241001649561SJames Smart if (ndlp->nlp_type & NLP_NVME_TARGET)
24117a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
241201649561SJames Smart if (ndlp->nlp_type & NLP_NVME_INITIATOR)
24137a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
241401649561SJames Smart
24157a06dcd3SJames Smart if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
24167a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
24177a06dcd3SJames Smart
24187a06dcd3SJames Smart rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
24197a06dcd3SJames Smart rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
24201f546823SJames Smart if (srport)
24211f546823SJames Smart rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
24221f546823SJames Smart else
24231f546823SJames Smart rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
242401466024SJames Smart
2425c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
2426ba3d58a1SJames Smart
2427ba3d58a1SJames Smart /* If an oldrport exists, so does the ndlp reference. If not
2428ba3d58a1SJames Smart * a new reference is needed because either the node has never
2429ba3d58a1SJames Smart * been registered or it's been unregistered and getting deleted.
2430ba3d58a1SJames Smart */
243101466024SJames Smart oldrport = lpfc_ndlp_get_nrport(ndlp);
243261184f17SJames Smart if (oldrport) {
243361184f17SJames Smart prev_ndlp = oldrport->ndlp;
2434c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
243561184f17SJames Smart } else {
2436c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
24374430f7fdSJames Smart if (!lpfc_nlp_get(ndlp)) {
24384430f7fdSJames Smart dev_warn(&vport->phba->pcidev->dev,
24394430f7fdSJames Smart "Warning - No node ref - exit register\n");
24404430f7fdSJames Smart return 0;
24414430f7fdSJames Smart }
244261184f17SJames Smart }
24433fd78355SJames Smart
24447a06dcd3SJames Smart ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
24457a06dcd3SJames Smart if (!ret) {
24467a06dcd3SJames Smart /* If the ndlp already has an nrport, this is just
24477a06dcd3SJames Smart * a resume of the existing rport. Else this is a
24487a06dcd3SJames Smart * new rport.
24497a06dcd3SJames Smart */
2450b15bd3e6SJames Smart /* Guard against an unregister/reregister
2451b15bd3e6SJames Smart * race that leaves the WAIT flag set.
2452b15bd3e6SJames Smart */
2453c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
245406145683SJames Smart ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2455e9b11083SJames Smart ndlp->fc4_xpt_flags |= NVME_XPT_REGD;
2456c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
24577a06dcd3SJames Smart rport = remote_port->private;
245801466024SJames Smart if (oldrport) {
24592b75d0f9SDick Kennedy
24603fd78355SJames Smart /* Sever the ndlp<->rport association
24613fd78355SJames Smart * before dropping the ndlp ref from
24623fd78355SJames Smart * register.
24632b75d0f9SDick Kennedy */
2464c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
24652b75d0f9SDick Kennedy ndlp->nrport = NULL;
246606145683SJames Smart ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2467c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
24682b75d0f9SDick Kennedy rport->ndlp = NULL;
24693fd78355SJames Smart rport->remoteport = NULL;
2470b04744ceSJames Smart
2471b04744ceSJames Smart /* Reference only removed if previous NDLP is no longer
2472b04744ceSJames Smart * active. It might be just a swap and removing the
2473b04744ceSJames Smart * reference would cause a premature cleanup.
2474b04744ceSJames Smart */
2475b04744ceSJames Smart if (prev_ndlp && prev_ndlp != ndlp) {
2476307e3380SJames Smart if (!prev_ndlp->nrport)
2477b04744ceSJames Smart lpfc_nlp_put(prev_ndlp);
2478b04744ceSJames Smart }
24792b75d0f9SDick Kennedy }
24802b75d0f9SDick Kennedy
24812b75d0f9SDick Kennedy /* Clean bind the rport to the ndlp. */
248201649561SJames Smart rport->remoteport = remote_port;
248301649561SJames Smart rport->lport = lport;
24843fd78355SJames Smart rport->ndlp = ndlp;
2485c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
248601649561SJames Smart ndlp->nrport = rport;
2487c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
248801649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO,
248901649561SJames Smart LOG_NVME_DISC | LOG_NODE,
24906825b7bdSJames Smart "6022 Bind lport x%px to remoteport x%px "
24916825b7bdSJames Smart "rport x%px WWNN 0x%llx, "
24927a06dcd3SJames Smart "Rport WWPN 0x%llx DID "
249332350664SJames Smart "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
249493a3922dSJames Smart lport, remote_port, rport,
249501649561SJames Smart rpinfo.node_name, rpinfo.port_name,
24963fd78355SJames Smart rpinfo.port_id, rpinfo.port_role,
249793a3922dSJames Smart ndlp, prev_ndlp);
249801649561SJames Smart } else {
249901649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR,
2500372c187bSDick Kennedy LOG_TRACE_EVENT,
250101649561SJames Smart "6031 RemotePort Registration failed "
250290cec07fSJustin Tee "err: %d, DID x%06x ref %u\n",
250390cec07fSJustin Tee ret, ndlp->nlp_DID, kref_read(&ndlp->kref));
250490cec07fSJustin Tee lpfc_nlp_put(ndlp);
250501649561SJames Smart }
25067a06dcd3SJames Smart
250701649561SJames Smart return ret;
2508166d7211SJames Smart #else
2509166d7211SJames Smart return 0;
2510166d7211SJames Smart #endif
251101649561SJames Smart }
251201649561SJames Smart
25133e749170SLee Jones /*
25146f2589f4SJames Smart * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
25156f2589f4SJames Smart *
25166f2589f4SJames Smart * If the ndlp represents an NVME Target, that we are logged into,
25176f2589f4SJames Smart * ping the NVME FC Transport layer to initiate a device rescan
25186f2589f4SJames Smart * on this remote NPort.
25196f2589f4SJames Smart */
25206f2589f4SJames Smart void
lpfc_nvme_rescan_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)25216f2589f4SJames Smart lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
25226f2589f4SJames Smart {
25236f2589f4SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
2524a6d10f24SJames Smart struct lpfc_nvme_rport *nrport;
2525a6d10f24SJames Smart struct nvme_fc_remote_port *remoteport = NULL;
25266f2589f4SJames Smart
2527c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
2528a6d10f24SJames Smart nrport = lpfc_ndlp_get_nrport(ndlp);
2529a6d10f24SJames Smart if (nrport)
2530a6d10f24SJames Smart remoteport = nrport->remoteport;
2531c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
25326f2589f4SJames Smart
25336f2589f4SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
25346f2589f4SJames Smart "6170 Rescan NPort DID x%06x type x%x "
2535a6d10f24SJames Smart "state x%x nrport x%px remoteport x%px\n",
2536a6d10f24SJames Smart ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2537a6d10f24SJames Smart nrport, remoteport);
2538a6d10f24SJames Smart
2539a6d10f24SJames Smart if (!nrport || !remoteport)
2540a6d10f24SJames Smart goto rescan_exit;
25416f2589f4SJames Smart
2542b83a8c21SJames Smart /* Rescan an NVME target in MAPPED state with DISCOVERY role set */
25436f2589f4SJames Smart if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
25446f2589f4SJames Smart ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
25456f2589f4SJames Smart nvme_fc_rescan_remoteport(remoteport);
25466f2589f4SJames Smart
2547b83a8c21SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
25486f2589f4SJames Smart "6172 NVME rescanned DID x%06x "
25496f2589f4SJames Smart "port_state x%x\n",
25506f2589f4SJames Smart ndlp->nlp_DID, remoteport->port_state);
25516f2589f4SJames Smart }
25526f2589f4SJames Smart return;
2553a6d10f24SJames Smart rescan_exit:
2554a6d10f24SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2555a6d10f24SJames Smart "6169 Skip NVME Rport Rescan, NVME remoteport "
2556a6d10f24SJames Smart "unregistered\n");
25576f2589f4SJames Smart #endif
25586f2589f4SJames Smart }
25596f2589f4SJames Smart
256001649561SJames Smart /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
256101649561SJames Smart *
256201649561SJames Smart * There is no notion of Devloss or rport recovery from the current
256301649561SJames Smart * nvme_transport perspective. Loss of an rport just means IO cannot
256401649561SJames Smart * be sent and recovery is completely up to the initator.
256501649561SJames Smart * For now, the driver just unbinds the DID and port_role so that
2566dae40be7SJustin Tee * no further IO can be issued.
256701649561SJames Smart */
256801649561SJames Smart void
lpfc_nvme_unregister_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)256901649561SJames Smart lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
257001649561SJames Smart {
25717d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
257201649561SJames Smart int ret;
257301649561SJames Smart struct nvme_fc_local_port *localport;
257401649561SJames Smart struct lpfc_nvme_lport *lport;
257501649561SJames Smart struct lpfc_nvme_rport *rport;
25769e210178SJames Smart struct nvme_fc_remote_port *remoteport = NULL;
257701649561SJames Smart
257801649561SJames Smart localport = vport->localport;
257901649561SJames Smart
258001649561SJames Smart /* This is fundamental error. The localport is always
258101649561SJames Smart * available until driver unload. Just exit.
258201649561SJames Smart */
258301649561SJames Smart if (!localport)
258401649561SJames Smart return;
258501649561SJames Smart
258601649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private;
258701649561SJames Smart if (!lport)
258801649561SJames Smart goto input_err;
258901649561SJames Smart
2590c6adba15SJames Smart spin_lock_irq(&ndlp->lock);
259101466024SJames Smart rport = lpfc_ndlp_get_nrport(ndlp);
25929e210178SJames Smart if (rport)
25939e210178SJames Smart remoteport = rport->remoteport;
2594c6adba15SJames Smart spin_unlock_irq(&ndlp->lock);
25959e210178SJames Smart if (!remoteport)
259601649561SJames Smart goto input_err;
259701649561SJames Smart
259801649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
259932350664SJames Smart "6033 Unreg nvme remoteport x%px, portname x%llx, "
2600e9b11083SJames Smart "port_id x%06x, portstate x%x port type x%x "
2601e9b11083SJames Smart "refcnt %d\n",
260201649561SJames Smart remoteport, remoteport->port_name,
260301649561SJames Smart remoteport->port_id, remoteport->port_state,
2604e9b11083SJames Smart ndlp->nlp_type, kref_read(&ndlp->kref));
260501649561SJames Smart
260601649561SJames Smart /* Sanity check ndlp type. Only call for NVME ports. Don't
260701649561SJames Smart * clear any rport state until the transport calls back.
260801649561SJames Smart */
26093b5bde69SJames Smart
26103b5bde69SJames Smart if (ndlp->nlp_type & NLP_NVME_TARGET) {
26117a06dcd3SJames Smart /* No concern about the role change on the nvme remoteport.
26127a06dcd3SJames Smart * The transport will update it.
26137a06dcd3SJames Smart */
2614d11272beSJustin Tee spin_lock_irq(&ndlp->lock);
261506145683SJames Smart ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;
2616d11272beSJustin Tee spin_unlock_irq(&ndlp->lock);
26177438273fSJames Smart
26187438273fSJames Smart /* Don't let the host nvme transport keep sending keep-alives
26197438273fSJames Smart * on this remoteport. Vport is unloading, no recovery. The
26207438273fSJames Smart * return values is ignored. The upcall is a courtesy to the
26217438273fSJames Smart * transport.
26227438273fSJames Smart */
2623e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag) ||
262427c2bcf0SJustin Tee unlikely(vport->phba->link_state == LPFC_HBA_ERROR))
26257438273fSJames Smart (void)nvme_fc_set_remoteport_devloss(remoteport, 0);
26267438273fSJames Smart
262701649561SJames Smart ret = nvme_fc_unregister_remoteport(remoteport);
2628e9b11083SJames Smart
2629e9b11083SJames Smart /* The driver no longer knows if the nrport memory is valid.
2630e9b11083SJames Smart * because the controller teardown process has begun and
2631e9b11083SJames Smart * is asynchronous. Break the binding in the ndlp. Also
2632e9b11083SJames Smart * remove the register ndlp reference to setup node release.
2633e9b11083SJames Smart */
2634e9b11083SJames Smart ndlp->nrport = NULL;
26353fd78355SJames Smart lpfc_nlp_put(ndlp);
2636e9b11083SJames Smart if (ret != 0) {
2637372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
263801649561SJames Smart "6167 NVME unregister failed %d "
263901649561SJames Smart "port_state x%x\n",
264001649561SJames Smart ret, remoteport->port_state);
2641dae40be7SJustin Tee
2642e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag)) {
2643dae40be7SJustin Tee /* Only 1 thread can drop the initial node
2644dae40be7SJustin Tee * reference. Check if another thread has set
2645dae40be7SJustin Tee * NLP_DROPPED.
2646dae40be7SJustin Tee */
2647dae40be7SJustin Tee spin_lock_irq(&ndlp->lock);
2648dae40be7SJustin Tee if (!(ndlp->nlp_flag & NLP_DROPPED)) {
2649dae40be7SJustin Tee ndlp->nlp_flag |= NLP_DROPPED;
2650dae40be7SJustin Tee spin_unlock_irq(&ndlp->lock);
2651dae40be7SJustin Tee lpfc_nlp_put(ndlp);
2652dae40be7SJustin Tee return;
2653dae40be7SJustin Tee }
2654dae40be7SJustin Tee spin_unlock_irq(&ndlp->lock);
2655dae40be7SJustin Tee }
26563fd78355SJames Smart }
265701649561SJames Smart }
265801649561SJames Smart return;
265901649561SJames Smart
266001649561SJames Smart input_err:
2661166d7211SJames Smart #endif
2662372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
266332350664SJames Smart "6168 State error: lport x%px, rport x%px FCID x%06x\n",
266401649561SJames Smart vport->localport, ndlp->rport, ndlp->nlp_DID);
266501649561SJames Smart }
2666318083adSJames Smart
2667318083adSJames Smart /**
266825ac2c97SJames Smart * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort
266925ac2c97SJames Smart * @phba: pointer to lpfc hba data structure.
267025ac2c97SJames Smart * @lpfc_ncmd: The nvme job structure for the request being aborted.
267125ac2c97SJames Smart *
267225ac2c97SJames Smart * This routine is invoked by the worker thread to process a SLI4 fast-path
267325ac2c97SJames Smart * NVME aborted xri. Aborted NVME IO commands are completed to the transport
267425ac2c97SJames Smart * here.
267525ac2c97SJames Smart **/
267625ac2c97SJames Smart void
lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)267725ac2c97SJames Smart lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba,
267825ac2c97SJames Smart struct lpfc_io_buf *lpfc_ncmd)
267925ac2c97SJames Smart {
268025ac2c97SJames Smart struct nvmefc_fcp_req *nvme_cmd = NULL;
268125ac2c97SJames Smart
268225ac2c97SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
268325ac2c97SJames Smart "6533 %s nvme_cmd %p tag x%x abort complete and "
268425ac2c97SJames Smart "xri released\n", __func__,
268525ac2c97SJames Smart lpfc_ncmd->nvmeCmd,
268625ac2c97SJames Smart lpfc_ncmd->cur_iocbq.iotag);
268725ac2c97SJames Smart
268825ac2c97SJames Smart /* Aborted NVME commands are required to not complete
268925ac2c97SJames Smart * before the abort exchange command fully completes.
269025ac2c97SJames Smart * Once completed, it is available via the put list.
269125ac2c97SJames Smart */
269225ac2c97SJames Smart if (lpfc_ncmd->nvmeCmd) {
269325ac2c97SJames Smart nvme_cmd = lpfc_ncmd->nvmeCmd;
269425ac2c97SJames Smart nvme_cmd->transferred_length = 0;
269525ac2c97SJames Smart nvme_cmd->rcv_rsplen = 0;
269625ac2c97SJames Smart nvme_cmd->status = NVME_SC_INTERNAL;
269725ac2c97SJames Smart nvme_cmd->done(nvme_cmd);
269825ac2c97SJames Smart lpfc_ncmd->nvmeCmd = NULL;
269925ac2c97SJames Smart }
270025ac2c97SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd);
270125ac2c97SJames Smart }
270225ac2c97SJames Smart
270325ac2c97SJames Smart /**
2704318083adSJames Smart * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2705318083adSJames Smart * @phba: pointer to lpfc hba data structure.
2706318083adSJames Smart * @axri: pointer to the fcp xri abort wcqe structure.
2707c00f62e6SJames Smart * @lpfc_ncmd: The nvme job structure for the request being aborted.
2708318083adSJames Smart *
2709318083adSJames Smart * This routine is invoked by the worker thread to process a SLI4 fast-path
2710952c303bSDick Kennedy * NVME aborted xri. Aborted NVME IO commands are completed to the transport
2711952c303bSDick Kennedy * here.
2712318083adSJames Smart **/
2713318083adSJames Smart void
lpfc_sli4_nvme_xri_aborted(struct lpfc_hba * phba,struct sli4_wcqe_xri_aborted * axri,struct lpfc_io_buf * lpfc_ncmd)2714318083adSJames Smart lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2715c00f62e6SJames Smart struct sli4_wcqe_xri_aborted *axri,
2716c00f62e6SJames Smart struct lpfc_io_buf *lpfc_ncmd)
2717318083adSJames Smart {
2718318083adSJames Smart uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2719952c303bSDick Kennedy struct nvmefc_fcp_req *nvme_cmd = NULL;
2720c00f62e6SJames Smart struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2721318083adSJames Smart
2722318083adSJames Smart
2723952c303bSDick Kennedy if (ndlp)
2724318083adSJames Smart lpfc_sli4_abts_err_handler(phba, ndlp, axri);
272586c67379SJames Smart
272686c67379SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2727c00f62e6SJames Smart "6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2728c00f62e6SJames Smart "xri released\n",
2729952c303bSDick Kennedy lpfc_ncmd->nvmeCmd, xri,
2730952c303bSDick Kennedy lpfc_ncmd->cur_iocbq.iotag);
273186c67379SJames Smart
2732952c303bSDick Kennedy /* Aborted NVME commands are required to not complete
2733952c303bSDick Kennedy * before the abort exchange command fully completes.
2734952c303bSDick Kennedy * Once completed, it is available via the put list.
2735952c303bSDick Kennedy */
27363fd78355SJames Smart if (lpfc_ncmd->nvmeCmd) {
2737952c303bSDick Kennedy nvme_cmd = lpfc_ncmd->nvmeCmd;
2738952c303bSDick Kennedy nvme_cmd->done(nvme_cmd);
27393fd78355SJames Smart lpfc_ncmd->nvmeCmd = NULL;
27403fd78355SJames Smart }
2741318083adSJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd);
2742318083adSJames Smart }
2743c3725bdcSJames Smart
2744c3725bdcSJames Smart /**
2745c3725bdcSJames Smart * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2746c3725bdcSJames Smart * @phba: Pointer to HBA context object.
2747c3725bdcSJames Smart *
2748c3725bdcSJames Smart * This function flushes all wqes in the nvme rings and frees all resources
2749c3725bdcSJames Smart * in the txcmplq. This function does not issue abort wqes for the IO
2750c3725bdcSJames Smart * commands in txcmplq, they will just be returned with
2751c3725bdcSJames Smart * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2752c3725bdcSJames Smart * slot has been permanently disabled.
2753c3725bdcSJames Smart **/
2754c3725bdcSJames Smart void
lpfc_nvme_wait_for_io_drain(struct lpfc_hba * phba)2755c3725bdcSJames Smart lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2756c3725bdcSJames Smart {
2757c3725bdcSJames Smart struct lpfc_sli_ring *pring;
2758c3725bdcSJames Smart u32 i, wait_cnt = 0;
2759c3725bdcSJames Smart
2760cdb42becSJames Smart if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2761c3725bdcSJames Smart return;
2762c3725bdcSJames Smart
2763c00f62e6SJames Smart /* Cycle through all IO rings and make sure all outstanding
2764c3725bdcSJames Smart * WQEs have been removed from the txcmplqs.
2765c3725bdcSJames Smart */
2766cdb42becSJames Smart for (i = 0; i < phba->cfg_hdw_queue; i++) {
2767c00f62e6SJames Smart if (!phba->sli4_hba.hdwq[i].io_wq)
27686a828b0fSJames Smart continue;
2769c00f62e6SJames Smart pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2770c3725bdcSJames Smart
2771d580c613SJames Smart if (!pring)
2772d580c613SJames Smart continue;
2773d580c613SJames Smart
2774c3725bdcSJames Smart /* Retrieve everything on the txcmplq */
2775c3725bdcSJames Smart while (!list_empty(&pring->txcmplq)) {
2776c3725bdcSJames Smart msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2777c3725bdcSJames Smart wait_cnt++;
2778c3725bdcSJames Smart
2779c3725bdcSJames Smart /* The sleep is 10mS. Every ten seconds,
2780c3725bdcSJames Smart * dump a message. Something is wrong.
2781c3725bdcSJames Smart */
2782c3725bdcSJames Smart if ((wait_cnt % 1000) == 0) {
2783372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2784c3725bdcSJames Smart "6178 NVME IO not empty, "
2785c3725bdcSJames Smart "cnt %d\n", wait_cnt);
2786c3725bdcSJames Smart }
2787c3725bdcSJames Smart }
2788c3725bdcSJames Smart }
2789a22d73b6SJames Smart
2790a22d73b6SJames Smart /* Make sure HBA is alive */
2791a22d73b6SJames Smart lpfc_issue_hb_tmo(phba);
2792a22d73b6SJames Smart
2793c3725bdcSJames Smart }
279484f2ddf8SJames Smart
279584f2ddf8SJames Smart void
lpfc_nvme_cancel_iocb(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,uint32_t stat,uint32_t param)27969ec58ec7SJames Smart lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
27979ec58ec7SJames Smart uint32_t stat, uint32_t param)
279884f2ddf8SJames Smart {
279984f2ddf8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC))
280084f2ddf8SJames Smart struct lpfc_io_buf *lpfc_ncmd;
280184f2ddf8SJames Smart struct nvmefc_fcp_req *nCmd;
28029ec58ec7SJames Smart struct lpfc_wcqe_complete wcqe;
28039ec58ec7SJames Smart struct lpfc_wcqe_complete *wcqep = &wcqe;
280484f2ddf8SJames Smart
2805d51cf5bdSJames Smart lpfc_ncmd = pwqeIn->io_buf;
28069ec58ec7SJames Smart if (!lpfc_ncmd) {
280784f2ddf8SJames Smart lpfc_sli_release_iocbq(phba, pwqeIn);
280884f2ddf8SJames Smart return;
280984f2ddf8SJames Smart }
281084f2ddf8SJames Smart /* For abort iocb just return, IO iocb will do a done call */
281184f2ddf8SJames Smart if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
281284f2ddf8SJames Smart CMD_ABORT_XRI_CX) {
281384f2ddf8SJames Smart lpfc_sli_release_iocbq(phba, pwqeIn);
281484f2ddf8SJames Smart return;
281584f2ddf8SJames Smart }
281684f2ddf8SJames Smart
281784f2ddf8SJames Smart spin_lock(&lpfc_ncmd->buf_lock);
28189ec58ec7SJames Smart nCmd = lpfc_ncmd->nvmeCmd;
28199ec58ec7SJames Smart if (!nCmd) {
282084f2ddf8SJames Smart spin_unlock(&lpfc_ncmd->buf_lock);
282184f2ddf8SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd);
282284f2ddf8SJames Smart return;
282384f2ddf8SJames Smart }
28249ec58ec7SJames Smart spin_unlock(&lpfc_ncmd->buf_lock);
282584f2ddf8SJames Smart
282684f2ddf8SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
282784f2ddf8SJames Smart "6194 NVME Cancel xri %x\n",
282884f2ddf8SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag);
282984f2ddf8SJames Smart
28309ec58ec7SJames Smart wcqep->word0 = 0;
28319ec58ec7SJames Smart bf_set(lpfc_wcqe_c_status, wcqep, stat);
28329ec58ec7SJames Smart wcqep->parameter = param;
28334ecc9b02SJames Smart wcqep->total_data_placed = 0;
28349ec58ec7SJames Smart wcqep->word3 = 0; /* xb is 0 */
283584f2ddf8SJames Smart
283684f2ddf8SJames Smart /* Call release with XB=1 to queue the IO into the abort list. */
28379ec58ec7SJames Smart if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
28389ec58ec7SJames Smart bf_set(lpfc_wcqe_c_xb, wcqep, 1);
28399ec58ec7SJames Smart
2840a680a929SJames Smart memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep));
2841a680a929SJames Smart (pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn);
284284f2ddf8SJames Smart #endif
284384f2ddf8SJames Smart }
2844