101649561SJames Smart /******************************************************************* 201649561SJames Smart * This file is part of the Emulex Linux Device Driver for * 301649561SJames Smart * Fibre Channel Host Bus Adapters. * 422871fe3SJustin Tee * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term * 53e21d1cbSJames Smart * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 601649561SJames Smart * Copyright (C) 2004-2016 Emulex. All rights reserved. * 701649561SJames Smart * EMULEX and SLI are trademarks of Emulex. * 8d080abe0SJames Smart * www.broadcom.com * 901649561SJames Smart * Portions Copyright (C) 2004-2005 Christoph Hellwig * 1001649561SJames Smart * * 1101649561SJames Smart * This program is free software; you can redistribute it and/or * 1201649561SJames Smart * modify it under the terms of version 2 of the GNU General * 1301649561SJames Smart * Public License as published by the Free Software Foundation. * 1401649561SJames Smart * This program is distributed in the hope that it will be useful. * 1501649561SJames Smart * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 1601649561SJames Smart * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 1701649561SJames Smart * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 1801649561SJames Smart * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 1901649561SJames Smart * TO BE LEGALLY INVALID. See the GNU General Public License for * 2001649561SJames Smart * more details, a copy of which can be found in the file COPYING * 2101649561SJames Smart * included with this package. * 2201649561SJames Smart ********************************************************************/ 2301649561SJames Smart #include <linux/pci.h> 2401649561SJames Smart #include <linux/slab.h> 2501649561SJames Smart #include <linux/interrupt.h> 2601649561SJames Smart #include <linux/delay.h> 2701649561SJames Smart #include <asm/unaligned.h> 2801649561SJames Smart #include <linux/crc-t10dif.h> 2901649561SJames Smart #include <net/checksum.h> 3001649561SJames Smart 3101649561SJames Smart #include <scsi/scsi.h> 3201649561SJames Smart #include <scsi/scsi_device.h> 3301649561SJames Smart #include <scsi/scsi_eh.h> 3401649561SJames Smart #include <scsi/scsi_host.h> 3501649561SJames Smart #include <scsi/scsi_tcq.h> 3601649561SJames Smart #include <scsi/scsi_transport_fc.h> 3701649561SJames Smart #include <scsi/fc/fc_fs.h> 3801649561SJames Smart 3901649561SJames Smart #include "lpfc_version.h" 4001649561SJames Smart #include "lpfc_hw4.h" 4101649561SJames Smart #include "lpfc_hw.h" 4201649561SJames Smart #include "lpfc_sli.h" 4301649561SJames Smart #include "lpfc_sli4.h" 4401649561SJames Smart #include "lpfc_nl.h" 4501649561SJames Smart #include "lpfc_disc.h" 4601649561SJames Smart #include "lpfc.h" 4701649561SJames Smart #include "lpfc_nvme.h" 4801649561SJames Smart #include "lpfc_scsi.h" 4901649561SJames Smart #include "lpfc_logmsg.h" 5001649561SJames Smart #include "lpfc_crtn.h" 5101649561SJames Smart #include "lpfc_vport.h" 52bd2cdd5eSJames Smart #include "lpfc_debugfs.h" 5301649561SJames Smart 5401649561SJames Smart /* NVME initiator-based functions */ 5501649561SJames Smart 56c490850aSJames Smart static struct lpfc_io_buf * 57cf1a1d3eSJames Smart lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 585e5b511dSJames Smart int idx, int expedite); 5901649561SJames Smart 6001649561SJames Smart static void 61c490850aSJames Smart lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 6201649561SJames Smart 6381e6a637SJames Smart static struct nvme_fc_port_template lpfc_nvme_template; 6401649561SJames Smart 6501649561SJames Smart /** 6601649561SJames Smart * lpfc_nvme_create_queue - 6709d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from 6801649561SJames Smart * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 6909d99705SLee Jones * @qsize: Size of the queue in bytes 7001649561SJames Smart * @handle: An opaque driver handle used in follow-up calls. 7101649561SJames Smart * 7201649561SJames Smart * Driver registers this routine to preallocate and initialize any 7301649561SJames Smart * internal data structures to bind the @qidx to its internal IO queues. 7401649561SJames Smart * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 7501649561SJames Smart * 7601649561SJames Smart * Return value : 7701649561SJames Smart * 0 - Success 7801649561SJames Smart * -EINVAL - Unsupported input value. 7901649561SJames Smart * -ENOMEM - Could not alloc necessary memory 8001649561SJames Smart **/ 8101649561SJames Smart static int 8201649561SJames Smart lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 8301649561SJames Smart unsigned int qidx, u16 qsize, 8401649561SJames Smart void **handle) 8501649561SJames Smart { 8601649561SJames Smart struct lpfc_nvme_lport *lport; 8701649561SJames Smart struct lpfc_vport *vport; 8801649561SJames Smart struct lpfc_nvme_qhandle *qhandle; 8901649561SJames Smart char *str; 9001649561SJames Smart 91c3725bdcSJames Smart if (!pnvme_lport->private) 92c3725bdcSJames Smart return -ENOMEM; 93c3725bdcSJames Smart 9401649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 9501649561SJames Smart vport = lport->vport; 96a4691038SJames Smart 97*e39811beSJustin Tee if (!vport || test_bit(FC_UNLOADING, &vport->load_flag) || 98a4691038SJames Smart vport->phba->hba_flag & HBA_IOQ_FLUSH) 99a4691038SJames Smart return -ENODEV; 100a4691038SJames Smart 10101649561SJames Smart qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 10201649561SJames Smart if (qhandle == NULL) 10301649561SJames Smart return -ENOMEM; 10401649561SJames Smart 105d6d189ceSBart Van Assche qhandle->cpu_id = raw_smp_processor_id(); 10601649561SJames Smart qhandle->qidx = qidx; 10701649561SJames Smart /* 10801649561SJames Smart * NVME qidx == 0 is the admin queue, so both admin queue 10901649561SJames Smart * and first IO queue will use MSI-X vector and associated 11001649561SJames Smart * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 11101649561SJames Smart */ 11201649561SJames Smart if (qidx) { 11301649561SJames Smart str = "IO "; /* IO queue */ 11401649561SJames Smart qhandle->index = ((qidx - 1) % 1156a828b0fSJames Smart lpfc_nvme_template.max_hw_queues); 11601649561SJames Smart } else { 11701649561SJames Smart str = "ADM"; /* Admin queue */ 11801649561SJames Smart qhandle->index = qidx; 11901649561SJames Smart } 12001649561SJames Smart 121d58734f0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 12201649561SJames Smart "6073 Binding %s HdwQueue %d (cpu %d) to " 12332350664SJames Smart "hdw_queue %d qhandle x%px\n", str, 12401649561SJames Smart qidx, qhandle->cpu_id, qhandle->index, qhandle); 12501649561SJames Smart *handle = (void *)qhandle; 12601649561SJames Smart return 0; 12701649561SJames Smart } 12801649561SJames Smart 12901649561SJames Smart /** 13001649561SJames Smart * lpfc_nvme_delete_queue - 13109d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from 13201649561SJames Smart * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 13301649561SJames Smart * @handle: An opaque driver handle from lpfc_nvme_create_queue 13401649561SJames Smart * 13501649561SJames Smart * Driver registers this routine to free 13601649561SJames Smart * any internal data structures to bind the @qidx to its internal 13701649561SJames Smart * IO queues. 13801649561SJames Smart * 13901649561SJames Smart * Return value : 14001649561SJames Smart * 0 - Success 14101649561SJames Smart * TODO: What are the failure codes. 14201649561SJames Smart **/ 14301649561SJames Smart static void 14401649561SJames Smart lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 14501649561SJames Smart unsigned int qidx, 14601649561SJames Smart void *handle) 14701649561SJames Smart { 14801649561SJames Smart struct lpfc_nvme_lport *lport; 14901649561SJames Smart struct lpfc_vport *vport; 15001649561SJames Smart 151c3725bdcSJames Smart if (!pnvme_lport->private) 152c3725bdcSJames Smart return; 153c3725bdcSJames Smart 15401649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 15501649561SJames Smart vport = lport->vport; 15601649561SJames Smart 15701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 15832350664SJames Smart "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 15901649561SJames Smart lport, qidx, handle); 16001649561SJames Smart kfree(handle); 16101649561SJames Smart } 16201649561SJames Smart 16301649561SJames Smart static void 16401649561SJames Smart lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 16501649561SJames Smart { 16601649561SJames Smart struct lpfc_nvme_lport *lport = localport->private; 16701649561SJames Smart 168add9d6beSJames Smart lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 16932350664SJames Smart "6173 localport x%px delete complete\n", 170add9d6beSJames Smart lport); 171add9d6beSJames Smart 17201649561SJames Smart /* release any threads waiting for the unreg to complete */ 1737961cba6SEwan D. Milne if (lport->vport->localport) 1747961cba6SEwan D. Milne complete(lport->lport_unreg_cmp); 17501649561SJames Smart } 17601649561SJames Smart 17701649561SJames Smart /* lpfc_nvme_remoteport_delete 17801649561SJames Smart * 17901649561SJames Smart * @remoteport: Pointer to an nvme transport remoteport instance. 18001649561SJames Smart * 18101649561SJames Smart * This is a template downcall. NVME transport calls this function 18201649561SJames Smart * when it has completed the unregistration of a previously 18301649561SJames Smart * registered remoteport. 18401649561SJames Smart * 18501649561SJames Smart * Return value : 18601649561SJames Smart * None 18701649561SJames Smart */ 1883999df75SBart Van Assche static void 18901649561SJames Smart lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 19001649561SJames Smart { 19101649561SJames Smart struct lpfc_nvme_rport *rport = remoteport->private; 19201649561SJames Smart struct lpfc_vport *vport; 19301649561SJames Smart struct lpfc_nodelist *ndlp; 194e9b11083SJames Smart u32 fc4_xpt_flags; 19501649561SJames Smart 19601649561SJames Smart ndlp = rport->ndlp; 197e9b11083SJames Smart if (!ndlp) { 198f1156125SJames Smart pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n", 199e9b11083SJames Smart __func__, rport, remoteport); 20001649561SJames Smart goto rport_err; 201e9b11083SJames Smart } 20201649561SJames Smart 20301649561SJames Smart vport = ndlp->vport; 204e9b11083SJames Smart if (!vport) { 205f1156125SJames Smart pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n", 206e9b11083SJames Smart __func__, ndlp, ndlp->nlp_state, rport); 20701649561SJames Smart goto rport_err; 208e9b11083SJames Smart } 209e9b11083SJames Smart 210e9b11083SJames Smart fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD; 21101649561SJames Smart 21201649561SJames Smart /* Remove this rport from the lport's list - memory is owned by the 21301649561SJames Smart * transport. Remove the ndlp reference for the NVME transport before 2147a06dcd3SJames Smart * calling state machine to remove the node. 21501649561SJames Smart */ 21601649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 217982fc396SJames Smart "6146 remoteport delete of remoteport x%px, ndlp x%px " 218982fc396SJames Smart "DID x%x xflags x%x\n", 219982fc396SJames Smart remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags); 220c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 221b15bd3e6SJames Smart 222b15bd3e6SJames Smart /* The register rebind might have occurred before the delete 223b15bd3e6SJames Smart * downcall. Guard against this race. 224b15bd3e6SJames Smart */ 22506145683SJames Smart if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT) 22606145683SJames Smart ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD); 227e9b11083SJames Smart 228c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 2293fd78355SJames Smart 230e9b11083SJames Smart /* On a devloss timeout event, one more put is executed provided the 231dae40be7SJustin Tee * NVME and SCSI rport unregister requests are complete. 2323fd78355SJames Smart */ 233e9b11083SJames Smart if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags)) 234e9b11083SJames Smart lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM); 23501649561SJames Smart 23601649561SJames Smart rport_err: 2373fd78355SJames Smart return; 23801649561SJames Smart } 23901649561SJames Smart 2403a8070c5SJames Smart /** 2413a8070c5SJames Smart * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request 2423a8070c5SJames Smart * @phba: pointer to lpfc hba data structure. 2433a8070c5SJames Smart * @axchg: pointer to exchange context for the NVME LS request 2443a8070c5SJames Smart * 2453a8070c5SJames Smart * This routine is used for processing an asychronously received NVME LS 2463a8070c5SJames Smart * request. Any remaining validation is done and the LS is then forwarded 2473a8070c5SJames Smart * to the nvme-fc transport via nvme_fc_rcv_ls_req(). 2483a8070c5SJames Smart * 2499aa09e98SJames Smart * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) 2509aa09e98SJames Smart * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. 2519aa09e98SJames Smart * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 2529aa09e98SJames Smart * 2533a8070c5SJames Smart * Returns 0 if LS was handled and delivered to the transport 2543a8070c5SJames Smart * Returns 1 if LS failed to be handled and should be dropped 2553a8070c5SJames Smart */ 2563a8070c5SJames Smart int 2573a8070c5SJames Smart lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, 2583a8070c5SJames Smart struct lpfc_async_xchg_ctx *axchg) 2593a8070c5SJames Smart { 2609aa09e98SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 2619aa09e98SJames Smart struct lpfc_vport *vport; 2629aa09e98SJames Smart struct lpfc_nvme_rport *lpfc_rport; 2639aa09e98SJames Smart struct nvme_fc_remote_port *remoteport; 2649aa09e98SJames Smart struct lpfc_nvme_lport *lport; 2659aa09e98SJames Smart uint32_t *payload = axchg->payload; 2669aa09e98SJames Smart int rc; 2679aa09e98SJames Smart 2689aa09e98SJames Smart vport = axchg->ndlp->vport; 2699aa09e98SJames Smart lpfc_rport = axchg->ndlp->nrport; 2709aa09e98SJames Smart if (!lpfc_rport) 2719aa09e98SJames Smart return -EINVAL; 2729aa09e98SJames Smart 2739aa09e98SJames Smart remoteport = lpfc_rport->remoteport; 274a4691038SJames Smart if (!vport->localport || 275a4691038SJames Smart vport->phba->hba_flag & HBA_IOQ_FLUSH) 2769aa09e98SJames Smart return -EINVAL; 2779aa09e98SJames Smart 2789aa09e98SJames Smart lport = vport->localport->private; 2799aa09e98SJames Smart if (!lport) 2809aa09e98SJames Smart return -EINVAL; 2819aa09e98SJames Smart 2829aa09e98SJames Smart rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, 2839aa09e98SJames Smart axchg->size); 2849aa09e98SJames Smart 2859aa09e98SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 2869aa09e98SJames Smart "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " 2879aa09e98SJames Smart "%08x %08x %08x\n", 2889aa09e98SJames Smart axchg->size, rc, 2899aa09e98SJames Smart *payload, *(payload+1), *(payload+2), 2909aa09e98SJames Smart *(payload+3), *(payload+4), *(payload+5)); 2919aa09e98SJames Smart 2929aa09e98SJames Smart if (!rc) 2939aa09e98SJames Smart return 0; 2949aa09e98SJames Smart #endif 2953a8070c5SJames Smart return 1; 2963a8070c5SJames Smart } 2973a8070c5SJames Smart 2986514b25dSJames Smart /** 2996514b25dSJames Smart * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME 3006514b25dSJames Smart * LS request. 3016514b25dSJames Smart * @phba: Pointer to HBA context object 3026514b25dSJames Smart * @vport: The local port that issued the LS 3036514b25dSJames Smart * @cmdwqe: Pointer to driver command WQE object. 3046514b25dSJames Smart * @wcqe: Pointer to driver response CQE object. 3056514b25dSJames Smart * 3066514b25dSJames Smart * This function is the generic completion handler for NVME LS requests. 3076514b25dSJames Smart * The function updates any states and statistics, calls the transport 3086514b25dSJames Smart * ls_req done() routine, then tears down the command and buffers used 3096514b25dSJames Smart * for the LS request. 3106514b25dSJames Smart **/ 3116514b25dSJames Smart void 3126514b25dSJames Smart __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 3136514b25dSJames Smart struct lpfc_iocbq *cmdwqe, 3146514b25dSJames Smart struct lpfc_wcqe_complete *wcqe) 3156514b25dSJames Smart { 3166514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq; 3176514b25dSJames Smart struct lpfc_dmabuf *buf_ptr; 3186514b25dSJames Smart struct lpfc_nodelist *ndlp; 3196a84d015SJustin Tee int status; 3206514b25dSJames Smart 321d51cf5bdSJames Smart pnvme_lsreq = cmdwqe->context_un.nvme_lsreq; 322d51cf5bdSJames Smart ndlp = cmdwqe->ndlp; 323d51cf5bdSJames Smart buf_ptr = cmdwqe->bpl_dmabuf; 324d51cf5bdSJames Smart 3256a84d015SJustin Tee status = bf_get(lpfc_wcqe_c_status, wcqe); 3266514b25dSJames Smart 3276514b25dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 328f1156125SJames Smart "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x " 3296514b25dSJames Smart "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " 3306514b25dSJames Smart "ndlp:x%px\n", 3316514b25dSJames Smart pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 3326514b25dSJames Smart cmdwqe->sli4_xritag, status, 3336514b25dSJames Smart (wcqe->parameter & 0xffff), 334d51cf5bdSJames Smart cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf, 335d51cf5bdSJames Smart ndlp); 3366514b25dSJames Smart 3376514b25dSJames Smart lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", 3386514b25dSJames Smart cmdwqe->sli4_xritag, status, wcqe->parameter); 3396514b25dSJames Smart 340d51cf5bdSJames Smart if (buf_ptr) { 3416514b25dSJames Smart lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 3426514b25dSJames Smart kfree(buf_ptr); 343d51cf5bdSJames Smart cmdwqe->bpl_dmabuf = NULL; 3446514b25dSJames Smart } 3456a84d015SJustin Tee if (pnvme_lsreq->done) { 3466a84d015SJustin Tee if (status != CQE_STATUS_SUCCESS) 3476a84d015SJustin Tee status = -ENXIO; 3486514b25dSJames Smart pnvme_lsreq->done(pnvme_lsreq, status); 3496a84d015SJustin Tee } else { 350372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 3516514b25dSJames Smart "6046 NVMEx cmpl without done call back? " 352f1156125SJames Smart "Data x%px DID %x Xri: %x status %x\n", 3536514b25dSJames Smart pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 3546514b25dSJames Smart cmdwqe->sli4_xritag, status); 3556a84d015SJustin Tee } 3566514b25dSJames Smart if (ndlp) { 3576514b25dSJames Smart lpfc_nlp_put(ndlp); 358d51cf5bdSJames Smart cmdwqe->ndlp = NULL; 3596514b25dSJames Smart } 3606514b25dSJames Smart lpfc_sli_release_iocbq(phba, cmdwqe); 3616514b25dSJames Smart } 3626514b25dSJames Smart 36301649561SJames Smart static void 3646514b25dSJames Smart lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 365a680a929SJames Smart struct lpfc_iocbq *rspwqe) 36601649561SJames Smart { 36701649561SJames Smart struct lpfc_vport *vport = cmdwqe->vport; 3684b056682SJames Smart struct lpfc_nvme_lport *lport; 36901649561SJames Smart uint32_t status; 370a680a929SJames Smart struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl; 37101649561SJames Smart 3726a84d015SJustin Tee status = bf_get(lpfc_wcqe_c_status, wcqe); 37366a210ffSJames Smart 37466a85155SJames Smart if (vport->localport) { 37566a85155SJames Smart lport = (struct lpfc_nvme_lport *)vport->localport->private; 37666a210ffSJames Smart if (lport) { 37766a210ffSJames Smart atomic_inc(&lport->fc4NvmeLsCmpls); 3784b056682SJames Smart if (status) { 3794b056682SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe)) 3804b056682SJames Smart atomic_inc(&lport->cmpl_ls_xb); 3814b056682SJames Smart atomic_inc(&lport->cmpl_ls_err); 3824b056682SJames Smart } 38366a210ffSJames Smart } 38466a85155SJames Smart } 3854b056682SJames Smart 3866514b25dSJames Smart __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); 38701649561SJames Smart } 38801649561SJames Smart 38901649561SJames Smart static int 39001649561SJames Smart lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 39101649561SJames Smart struct lpfc_dmabuf *inp, 39201649561SJames Smart struct nvmefc_ls_req *pnvme_lsreq, 39301649561SJames Smart void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 394a680a929SJames Smart struct lpfc_iocbq *), 39501649561SJames Smart struct lpfc_nodelist *ndlp, uint32_t num_entry, 39601649561SJames Smart uint32_t tmo, uint8_t retry) 39701649561SJames Smart { 39801649561SJames Smart struct lpfc_hba *phba = vport->phba; 399205e8240SJames Smart union lpfc_wqe128 *wqe; 40001649561SJames Smart struct lpfc_iocbq *genwqe; 40101649561SJames Smart struct ulp_bde64 *bpl; 40201649561SJames Smart struct ulp_bde64 bde; 40301649561SJames Smart int i, rc, xmit_len, first_len; 40401649561SJames Smart 40501649561SJames Smart /* Allocate buffer for command WQE */ 40601649561SJames Smart genwqe = lpfc_sli_get_iocbq(phba); 40701649561SJames Smart if (genwqe == NULL) 40801649561SJames Smart return 1; 40901649561SJames Smart 41001649561SJames Smart wqe = &genwqe->wqe; 411d9f492a1SJames Smart /* Initialize only 64 bytes */ 41201649561SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe)); 41301649561SJames Smart 414d51cf5bdSJames Smart genwqe->bpl_dmabuf = bmp; 415a680a929SJames Smart genwqe->cmd_flag |= LPFC_IO_NVME_LS; 41601649561SJames Smart 41701649561SJames Smart /* Save for completion so we can release these resources */ 418d51cf5bdSJames Smart genwqe->ndlp = lpfc_nlp_get(ndlp); 419d51cf5bdSJames Smart if (!genwqe->ndlp) { 4204430f7fdSJames Smart dev_warn(&phba->pcidev->dev, 4214430f7fdSJames Smart "Warning: Failed node ref, not sending LS_REQ\n"); 4224430f7fdSJames Smart lpfc_sli_release_iocbq(phba, genwqe); 4234430f7fdSJames Smart return 1; 4244430f7fdSJames Smart } 4254430f7fdSJames Smart 426d51cf5bdSJames Smart genwqe->context_un.nvme_lsreq = pnvme_lsreq; 42701649561SJames Smart /* Fill in payload, bp points to frame payload */ 42801649561SJames Smart 42901649561SJames Smart if (!tmo) 43001649561SJames Smart /* FC spec states we need 3 * ratov for CT requests */ 43101649561SJames Smart tmo = (3 * phba->fc_ratov); 43201649561SJames Smart 43301649561SJames Smart /* For this command calculate the xmit length of the request bde. */ 43401649561SJames Smart xmit_len = 0; 43501649561SJames Smart first_len = 0; 43601649561SJames Smart bpl = (struct ulp_bde64 *)bmp->virt; 43701649561SJames Smart for (i = 0; i < num_entry; i++) { 43801649561SJames Smart bde.tus.w = bpl[i].tus.w; 43901649561SJames Smart if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 44001649561SJames Smart break; 44101649561SJames Smart xmit_len += bde.tus.f.bdeSize; 44201649561SJames Smart if (i == 0) 44301649561SJames Smart first_len = xmit_len; 44401649561SJames Smart } 44501649561SJames Smart 446a680a929SJames Smart genwqe->num_bdes = num_entry; 44701649561SJames Smart genwqe->hba_wqidx = 0; 44801649561SJames Smart 44901649561SJames Smart /* Words 0 - 2 */ 45001649561SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 45101649561SJames Smart wqe->generic.bde.tus.f.bdeSize = first_len; 45201649561SJames Smart wqe->generic.bde.addrLow = bpl[0].addrLow; 45301649561SJames Smart wqe->generic.bde.addrHigh = bpl[0].addrHigh; 45401649561SJames Smart 45501649561SJames Smart /* Word 3 */ 45601649561SJames Smart wqe->gen_req.request_payload_len = first_len; 45701649561SJames Smart 45801649561SJames Smart /* Word 4 */ 45901649561SJames Smart 46001649561SJames Smart /* Word 5 */ 46101649561SJames Smart bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 46201649561SJames Smart bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 46301649561SJames Smart bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 4648b361639SJames Smart bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 46501649561SJames Smart bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 46601649561SJames Smart 46701649561SJames Smart /* Word 6 */ 46801649561SJames Smart bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 46901649561SJames Smart phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 47001649561SJames Smart bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 47101649561SJames Smart 47201649561SJames Smart /* Word 7 */ 473c33b1609SJames Smart bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo); 47401649561SJames Smart bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 47501649561SJames Smart bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 47601649561SJames Smart bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 47701649561SJames Smart 47801649561SJames Smart /* Word 8 */ 47901649561SJames Smart wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 48001649561SJames Smart 48101649561SJames Smart /* Word 9 */ 48201649561SJames Smart bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 48301649561SJames Smart 48401649561SJames Smart /* Word 10 */ 48501649561SJames Smart bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 48601649561SJames Smart bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 48701649561SJames Smart bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 48801649561SJames Smart bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 48901649561SJames Smart bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 49001649561SJames Smart 49101649561SJames Smart /* Word 11 */ 49201649561SJames Smart bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 49301649561SJames Smart bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 49401649561SJames Smart 49501649561SJames Smart 49601649561SJames Smart /* Issue GEN REQ WQE for NPORT <did> */ 497a680a929SJames Smart genwqe->cmd_cmpl = cmpl; 49801649561SJames Smart genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 49901649561SJames Smart genwqe->vport = vport; 50001649561SJames Smart genwqe->retry = retry; 50101649561SJames Smart 502bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 503bd2cdd5eSJames Smart genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 504bd2cdd5eSJames Smart 5051fbf9742SJames Smart rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 506cd22d605SDick Kennedy if (rc) { 507372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 50801649561SJames Smart "6045 Issue GEN REQ WQE to NPORT x%x " 5096514b25dSJames Smart "Data: x%x x%x rc x%x\n", 51001649561SJames Smart ndlp->nlp_DID, genwqe->iotag, 5116514b25dSJames Smart vport->port_state, rc); 512e9b11083SJames Smart lpfc_nlp_put(ndlp); 51301649561SJames Smart lpfc_sli_release_iocbq(phba, genwqe); 51401649561SJames Smart return 1; 51501649561SJames Smart } 5166514b25dSJames Smart 5176514b25dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, 5186514b25dSJames Smart "6050 Issue GEN REQ WQE to NPORT x%x " 5196514b25dSJames Smart "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " 5206514b25dSJames Smart "bmp:x%px xmit:%d 1st:%d\n", 5216514b25dSJames Smart ndlp->nlp_DID, genwqe->sli4_xritag, 5226514b25dSJames Smart vport->port_state, 5236514b25dSJames Smart genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 52401649561SJames Smart return 0; 52501649561SJames Smart } 52601649561SJames Smart 5276514b25dSJames Smart 52801649561SJames Smart /** 5296514b25dSJames Smart * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request 5306514b25dSJames Smart * @vport: The local port issuing the LS 5316514b25dSJames Smart * @ndlp: The remote port to send the LS to 5326514b25dSJames Smart * @pnvme_lsreq: Pointer to LS request structure from the transport 53309d99705SLee Jones * @gen_req_cmp: Completion call-back 53401649561SJames Smart * 5356514b25dSJames Smart * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST 5366514b25dSJames Smart * WQE to perform the LS operation. 53701649561SJames Smart * 53801649561SJames Smart * Return value : 53901649561SJames Smart * 0 - Success 5406514b25dSJames Smart * non-zero: various error codes, in form of -Exxx 54101649561SJames Smart **/ 5426514b25dSJames Smart int 5436514b25dSJames Smart __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 5446514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq, 5456514b25dSJames Smart void (*gen_req_cmp)(struct lpfc_hba *phba, 5466514b25dSJames Smart struct lpfc_iocbq *cmdwqe, 547a680a929SJames Smart struct lpfc_iocbq *rspwqe)) 54801649561SJames Smart { 54901649561SJames Smart struct lpfc_dmabuf *bmp; 5506514b25dSJames Smart struct ulp_bde64 *bpl; 5516514b25dSJames Smart int ret; 552ba43c4d0SJames Smart uint16_t ntype, nstate; 55301649561SJames Smart 554307e3380SJames Smart if (!ndlp) { 555372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 5566514b25dSJames Smart "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " 5576514b25dSJames Smart "LS Req\n", 5586514b25dSJames Smart ndlp); 559ba43c4d0SJames Smart return -ENODEV; 560ba43c4d0SJames Smart } 561ba43c4d0SJames Smart 562ba43c4d0SJames Smart ntype = ndlp->nlp_type; 563ba43c4d0SJames Smart nstate = ndlp->nlp_state; 564ba43c4d0SJames Smart if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 565ba43c4d0SJames Smart (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 566372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 5676514b25dSJames Smart "6088 NVMEx LS REQ: Fail DID x%06x not " 5686514b25dSJames Smart "ready for IO. Type x%x, State x%x\n", 5696514b25dSJames Smart ndlp->nlp_DID, ntype, nstate); 570ba43c4d0SJames Smart return -ENODEV; 57101649561SJames Smart } 572a4691038SJames Smart if (vport->phba->hba_flag & HBA_IOQ_FLUSH) 573a4691038SJames Smart return -ENODEV; 5746514b25dSJames Smart 5758c65830aSJames Smart if (!vport->phba->sli4_hba.nvmels_wq) 5768c65830aSJames Smart return -ENOMEM; 5778c65830aSJames Smart 5786514b25dSJames Smart /* 5796514b25dSJames Smart * there are two dma buf in the request, actually there is one and 5806514b25dSJames Smart * the second one is just the start address + cmd size. 5816514b25dSJames Smart * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 5826514b25dSJames Smart * in a lpfc_dmabuf struct. When freeing we just free the wrapper 5836514b25dSJames Smart * because the nvem layer owns the data bufs. 5846514b25dSJames Smart * We do not have to break these packets open, we don't care what is 5856514b25dSJames Smart * in them. And we do not have to look at the resonse data, we only 5866514b25dSJames Smart * care that we got a response. All of the caring is going to happen 5876514b25dSJames Smart * in the nvme-fc layer. 5886514b25dSJames Smart */ 5896514b25dSJames Smart 5906514b25dSJames Smart bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); 59101649561SJames Smart if (!bmp) { 592372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 5936514b25dSJames Smart "6044 NVMEx LS REQ: Could not alloc LS buf " 5946514b25dSJames Smart "for DID %x\n", 5956514b25dSJames Smart ndlp->nlp_DID); 5966514b25dSJames Smart return -ENOMEM; 59701649561SJames Smart } 5986514b25dSJames Smart 59901649561SJames Smart bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 60001649561SJames Smart if (!bmp->virt) { 601372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 6026514b25dSJames Smart "6042 NVMEx LS REQ: Could not alloc mbuf " 6036514b25dSJames Smart "for DID %x\n", 6046514b25dSJames Smart ndlp->nlp_DID); 60501649561SJames Smart kfree(bmp); 6066514b25dSJames Smart return -ENOMEM; 60701649561SJames Smart } 6086514b25dSJames Smart 6096514b25dSJames Smart INIT_LIST_HEAD(&bmp->list); 6106514b25dSJames Smart 61101649561SJames Smart bpl = (struct ulp_bde64 *)bmp->virt; 61201649561SJames Smart bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 61301649561SJames Smart bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 61401649561SJames Smart bpl->tus.f.bdeFlags = 0; 61501649561SJames Smart bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 61601649561SJames Smart bpl->tus.w = le32_to_cpu(bpl->tus.w); 61701649561SJames Smart bpl++; 61801649561SJames Smart 61901649561SJames Smart bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 62001649561SJames Smart bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 62101649561SJames Smart bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 62201649561SJames Smart bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 62301649561SJames Smart bpl->tus.w = le32_to_cpu(bpl->tus.w); 62401649561SJames Smart 62501649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 6266514b25dSJames Smart "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " 6276514b25dSJames Smart "rqstlen:%d rsplen:%d %pad %pad\n", 6286514b25dSJames Smart ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, 629825c6abbSArnd Bergmann pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 630825c6abbSArnd Bergmann &pnvme_lsreq->rspdma); 63101649561SJames Smart 63201649561SJames Smart ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 6336514b25dSJames Smart pnvme_lsreq, gen_req_cmp, ndlp, 2, 634c33b1609SJames Smart pnvme_lsreq->timeout, 0); 63501649561SJames Smart if (ret != WQE_SUCCESS) { 636372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 6376514b25dSJames Smart "6052 NVMEx REQ: EXIT. issue ls wqe failed " 6386514b25dSJames Smart "lsreq x%px Status %x DID %x\n", 6396514b25dSJames Smart pnvme_lsreq, ret, ndlp->nlp_DID); 64001649561SJames Smart lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 64101649561SJames Smart kfree(bmp); 6426514b25dSJames Smart return -EIO; 64301649561SJames Smart } 64401649561SJames Smart 6456514b25dSJames Smart return 0; 6466514b25dSJames Smart } 6476514b25dSJames Smart 6486514b25dSJames Smart /** 6496514b25dSJames Smart * lpfc_nvme_ls_req - Issue an NVME Link Service request 65009d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from. 6513e749170SLee Jones * @pnvme_rport: Transport remoteport that LS is to be sent to. 65209d99705SLee Jones * @pnvme_lsreq: the transport nvme_ls_req structure for the LS 6536514b25dSJames Smart * 6546514b25dSJames Smart * Driver registers this routine to handle any link service request 6556514b25dSJames Smart * from the nvme_fc transport to a remote nvme-aware port. 6566514b25dSJames Smart * 6576514b25dSJames Smart * Return value : 6586514b25dSJames Smart * 0 - Success 6596514b25dSJames Smart * non-zero: various error codes, in form of -Exxx 6606514b25dSJames Smart **/ 6616514b25dSJames Smart static int 6626514b25dSJames Smart lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 6636514b25dSJames Smart struct nvme_fc_remote_port *pnvme_rport, 6646514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq) 6656514b25dSJames Smart { 6666514b25dSJames Smart struct lpfc_nvme_lport *lport; 6676514b25dSJames Smart struct lpfc_nvme_rport *rport; 6686514b25dSJames Smart struct lpfc_vport *vport; 6696514b25dSJames Smart int ret; 6706514b25dSJames Smart 6716514b25dSJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 6726514b25dSJames Smart rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 6736514b25dSJames Smart if (unlikely(!lport) || unlikely(!rport)) 6746514b25dSJames Smart return -EINVAL; 6756514b25dSJames Smart 6766514b25dSJames Smart vport = lport->vport; 677*e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag) || 678a4691038SJames Smart vport->phba->hba_flag & HBA_IOQ_FLUSH) 6796514b25dSJames Smart return -ENODEV; 6806514b25dSJames Smart 6816514b25dSJames Smart atomic_inc(&lport->fc4NvmeLsRequests); 6826514b25dSJames Smart 6836514b25dSJames Smart ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, 6846514b25dSJames Smart lpfc_nvme_ls_req_cmp); 6856514b25dSJames Smart if (ret) 6866514b25dSJames Smart atomic_inc(&lport->xmt_ls_err); 6876514b25dSJames Smart 68801649561SJames Smart return ret; 68901649561SJames Smart } 69001649561SJames Smart 69101649561SJames Smart /** 692e96a22b0SJames Smart * __lpfc_nvme_ls_abort - Generic service routine to abort a prior 693e96a22b0SJames Smart * NVME LS request 694e96a22b0SJames Smart * @vport: The local port that issued the LS 695e96a22b0SJames Smart * @ndlp: The remote port the LS was sent to 696e96a22b0SJames Smart * @pnvme_lsreq: Pointer to LS request structure from the transport 69701649561SJames Smart * 698e96a22b0SJames Smart * The driver validates the ndlp, looks for the LS, and aborts the 699e96a22b0SJames Smart * LS if found. 70001649561SJames Smart * 701e96a22b0SJames Smart * Returns: 702e96a22b0SJames Smart * 0 : if LS found and aborted 703e96a22b0SJames Smart * non-zero: various error conditions in form -Exxx 704e96a22b0SJames Smart **/ 705e96a22b0SJames Smart int 706e96a22b0SJames Smart __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 707e96a22b0SJames Smart struct nvmefc_ls_req *pnvme_lsreq) 708e96a22b0SJames Smart { 709e96a22b0SJames Smart struct lpfc_hba *phba = vport->phba; 710e96a22b0SJames Smart struct lpfc_sli_ring *pring; 711e96a22b0SJames Smart struct lpfc_iocbq *wqe, *next_wqe; 712e96a22b0SJames Smart bool foundit = false; 713e96a22b0SJames Smart 714e96a22b0SJames Smart if (!ndlp) { 715372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 716e96a22b0SJames Smart "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " 717e96a22b0SJames Smart "x%06x, Failing LS Req\n", 718e96a22b0SJames Smart ndlp, ndlp ? ndlp->nlp_DID : 0); 719e96a22b0SJames Smart return -EINVAL; 720e96a22b0SJames Smart } 721e96a22b0SJames Smart 722e96a22b0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 723e96a22b0SJames Smart "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " 724f1156125SJames Smart "x%px rqstlen:%d rsplen:%d %pad %pad\n", 725e96a22b0SJames Smart pnvme_lsreq, pnvme_lsreq->rqstlen, 726e96a22b0SJames Smart pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 727e96a22b0SJames Smart &pnvme_lsreq->rspdma); 728e96a22b0SJames Smart 729e96a22b0SJames Smart /* 730e96a22b0SJames Smart * Lock the ELS ring txcmplq and look for the wqe that matches 731e96a22b0SJames Smart * this ELS. If found, issue an abort on the wqe. 732e96a22b0SJames Smart */ 733e96a22b0SJames Smart pring = phba->sli4_hba.nvmels_wq->pring; 734e96a22b0SJames Smart spin_lock_irq(&phba->hbalock); 735e96a22b0SJames Smart spin_lock(&pring->ring_lock); 736e96a22b0SJames Smart list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 737d51cf5bdSJames Smart if (wqe->context_un.nvme_lsreq == pnvme_lsreq) { 738a680a929SJames Smart wqe->cmd_flag |= LPFC_DRIVER_ABORTED; 739e96a22b0SJames Smart foundit = true; 740e96a22b0SJames Smart break; 741e96a22b0SJames Smart } 742e96a22b0SJames Smart } 743e96a22b0SJames Smart spin_unlock(&pring->ring_lock); 744e96a22b0SJames Smart 745e96a22b0SJames Smart if (foundit) 746db7531d2SJames Smart lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL); 747e96a22b0SJames Smart spin_unlock_irq(&phba->hbalock); 748e96a22b0SJames Smart 749e96a22b0SJames Smart if (foundit) 750e96a22b0SJames Smart return 0; 751e96a22b0SJames Smart 752e96a22b0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 753f1156125SJames Smart "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n", 754e96a22b0SJames Smart pnvme_lsreq); 7556b6e8963SJames Smart return -EINVAL; 756e96a22b0SJames Smart } 757e96a22b0SJames Smart 7589aa09e98SJames Smart static int 7599aa09e98SJames Smart lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, 7609aa09e98SJames Smart struct nvme_fc_remote_port *remoteport, 7619aa09e98SJames Smart struct nvmefc_ls_rsp *ls_rsp) 7629aa09e98SJames Smart { 7639aa09e98SJames Smart struct lpfc_async_xchg_ctx *axchg = 7649aa09e98SJames Smart container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 7659aa09e98SJames Smart struct lpfc_nvme_lport *lport; 7669aa09e98SJames Smart int rc; 7679aa09e98SJames Smart 768*e39811beSJustin Tee if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag)) 7699aa09e98SJames Smart return -ENODEV; 7709aa09e98SJames Smart 7719aa09e98SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 7729aa09e98SJames Smart 7739aa09e98SJames Smart rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); 7749aa09e98SJames Smart 7759aa09e98SJames Smart if (rc) { 7769aa09e98SJames Smart /* 7779aa09e98SJames Smart * unless the failure is due to having already sent 7789aa09e98SJames Smart * the response, an abort will be generated for the 7799aa09e98SJames Smart * exchange if the rsp can't be sent. 7809aa09e98SJames Smart */ 7819aa09e98SJames Smart if (rc != -EALREADY) 7829aa09e98SJames Smart atomic_inc(&lport->xmt_ls_abort); 7839aa09e98SJames Smart return rc; 7849aa09e98SJames Smart } 7859aa09e98SJames Smart 7869aa09e98SJames Smart return 0; 7879aa09e98SJames Smart } 7889aa09e98SJames Smart 789e96a22b0SJames Smart /** 790e96a22b0SJames Smart * lpfc_nvme_ls_abort - Abort a prior NVME LS request 79109d99705SLee Jones * @pnvme_lport: Transport localport that LS is to be issued from. 79209d99705SLee Jones * @pnvme_rport: Transport remoteport that LS is to be sent to. 79309d99705SLee Jones * @pnvme_lsreq: the transport nvme_ls_req structure for the LS 794e96a22b0SJames Smart * 795e96a22b0SJames Smart * Driver registers this routine to abort a NVME LS request that is 796e96a22b0SJames Smart * in progress (from the transports perspective). 79701649561SJames Smart **/ 79801649561SJames Smart static void 79901649561SJames Smart lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 80001649561SJames Smart struct nvme_fc_remote_port *pnvme_rport, 80101649561SJames Smart struct nvmefc_ls_req *pnvme_lsreq) 80201649561SJames Smart { 80301649561SJames Smart struct lpfc_nvme_lport *lport; 80401649561SJames Smart struct lpfc_vport *vport; 80501649561SJames Smart struct lpfc_nodelist *ndlp; 806e96a22b0SJames Smart int ret; 80701649561SJames Smart 80801649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 80966a210ffSJames Smart if (unlikely(!lport)) 81066a210ffSJames Smart return; 81101649561SJames Smart vport = lport->vport; 81201649561SJames Smart 813*e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag)) 8143386f4bdSJames Smart return; 8153386f4bdSJames Smart 81601649561SJames Smart ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 81701649561SJames Smart 818e96a22b0SJames Smart ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); 819e96a22b0SJames Smart if (!ret) 8204b056682SJames Smart atomic_inc(&lport->xmt_ls_abort); 82101649561SJames Smart } 82201649561SJames Smart 82301649561SJames Smart /* Fix up the existing sgls for NVME IO. */ 8245fd11085SJames Smart static inline void 82501649561SJames Smart lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 826c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd, 82701649561SJames Smart struct nvmefc_fcp_req *nCmd) 82801649561SJames Smart { 8294e565cf0SJames Smart struct lpfc_hba *phba = vport->phba; 83001649561SJames Smart struct sli4_sge *sgl; 83101649561SJames Smart union lpfc_wqe128 *wqe; 83201649561SJames Smart uint32_t *wptr, *dptr; 83301649561SJames Smart 83401649561SJames Smart /* 8354e565cf0SJames Smart * Get a local pointer to the built-in wqe and correct 8364e565cf0SJames Smart * the cmd size to match NVME's 96 bytes and fix 8374e565cf0SJames Smart * the dma address. 8384e565cf0SJames Smart */ 8394e565cf0SJames Smart 840205e8240SJames Smart wqe = &lpfc_ncmd->cur_iocbq.wqe; 8414e565cf0SJames Smart 8424e565cf0SJames Smart /* 84301649561SJames Smart * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 84401649561SJames Smart * match NVME. NVME sends 96 bytes. Also, use the 84501649561SJames Smart * nvme commands command and response dma addresses 84601649561SJames Smart * rather than the virtual memory to ease the restore 84701649561SJames Smart * operation. 84801649561SJames Smart */ 8490794d601SJames Smart sgl = lpfc_ncmd->dma_sgl; 85001649561SJames Smart sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 8514e565cf0SJames Smart if (phba->cfg_nvme_embed_cmd) { 8524e565cf0SJames Smart sgl->addr_hi = 0; 8534e565cf0SJames Smart sgl->addr_lo = 0; 8544e565cf0SJames Smart 8554e565cf0SJames Smart /* Word 0-2 - NVME CMND IU (embedded payload) */ 8564e565cf0SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 8574e565cf0SJames Smart wqe->generic.bde.tus.f.bdeSize = 56; 8584e565cf0SJames Smart wqe->generic.bde.addrHigh = 0; 8594e565cf0SJames Smart wqe->generic.bde.addrLow = 64; /* Word 16 */ 8604e565cf0SJames Smart 8615fd11085SJames Smart /* Word 10 - dbde is 0, wqes is 1 in template */ 86201649561SJames Smart 86301649561SJames Smart /* 86401649561SJames Smart * Embed the payload in the last half of the WQE 86501649561SJames Smart * WQE words 16-30 get the NVME CMD IU payload 86601649561SJames Smart * 867b06a622fSJames Smart * WQE words 16-19 get payload Words 1-4 86801649561SJames Smart * WQE words 20-21 get payload Words 6-7 86901649561SJames Smart * WQE words 22-29 get payload Words 16-23 87001649561SJames Smart */ 871b06a622fSJames Smart wptr = &wqe->words[16]; /* WQE ptr */ 87201649561SJames Smart dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 873b06a622fSJames Smart dptr++; /* Skip Word 0 in payload */ 87401649561SJames Smart 875b06a622fSJames Smart *wptr++ = *dptr++; /* Word 1 */ 87601649561SJames Smart *wptr++ = *dptr++; /* Word 2 */ 87701649561SJames Smart *wptr++ = *dptr++; /* Word 3 */ 87801649561SJames Smart *wptr++ = *dptr++; /* Word 4 */ 87901649561SJames Smart dptr++; /* Skip Word 5 in payload */ 88001649561SJames Smart *wptr++ = *dptr++; /* Word 6 */ 88101649561SJames Smart *wptr++ = *dptr++; /* Word 7 */ 88201649561SJames Smart dptr += 8; /* Skip Words 8-15 in payload */ 88301649561SJames Smart *wptr++ = *dptr++; /* Word 16 */ 88401649561SJames Smart *wptr++ = *dptr++; /* Word 17 */ 88501649561SJames Smart *wptr++ = *dptr++; /* Word 18 */ 88601649561SJames Smart *wptr++ = *dptr++; /* Word 19 */ 88701649561SJames Smart *wptr++ = *dptr++; /* Word 20 */ 88801649561SJames Smart *wptr++ = *dptr++; /* Word 21 */ 88901649561SJames Smart *wptr++ = *dptr++; /* Word 22 */ 89001649561SJames Smart *wptr = *dptr; /* Word 23 */ 8915fd11085SJames Smart } else { 8925fd11085SJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 8935fd11085SJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 8945fd11085SJames Smart 8955fd11085SJames Smart /* Word 0-2 - NVME CMND IU Inline BDE */ 8965fd11085SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 8975fd11085SJames Smart wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 8985fd11085SJames Smart wqe->generic.bde.addrHigh = sgl->addr_hi; 8995fd11085SJames Smart wqe->generic.bde.addrLow = sgl->addr_lo; 9005fd11085SJames Smart 9015fd11085SJames Smart /* Word 10 */ 9025fd11085SJames Smart bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 9035fd11085SJames Smart bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 9045fd11085SJames Smart } 9055fd11085SJames Smart 9065fd11085SJames Smart sgl++; 9075fd11085SJames Smart 9085fd11085SJames Smart /* Setup the physical region for the FCP RSP */ 9095fd11085SJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 9105fd11085SJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 9115fd11085SJames Smart sgl->word2 = le32_to_cpu(sgl->word2); 9125fd11085SJames Smart if (nCmd->sg_cnt) 9135fd11085SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0); 9145fd11085SJames Smart else 9155fd11085SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1); 9165fd11085SJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 9175fd11085SJames Smart sgl->sge_len = cpu_to_le32(nCmd->rsplen); 91801649561SJames Smart } 91901649561SJames Smart 920bd2cdd5eSJames Smart 92109d99705SLee Jones /* 922a680a929SJames Smart * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO 92301649561SJames Smart * 92401649561SJames Smart * Driver registers this routine as it io request handler. This 92501649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 92601649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. 92701649561SJames Smart * 92801649561SJames Smart * Return value : 92901649561SJames Smart * 0 - Success 93001649561SJames Smart * TODO: What are the failure codes. 93101649561SJames Smart **/ 93201649561SJames Smart static void 933a680a929SJames Smart lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 934a680a929SJames Smart struct lpfc_iocbq *pwqeOut) 93501649561SJames Smart { 936d51cf5bdSJames Smart struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf; 937a680a929SJames Smart struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl; 93801649561SJames Smart struct lpfc_vport *vport = pwqeIn->vport; 93901649561SJames Smart struct nvmefc_fcp_req *nCmd; 94001649561SJames Smart struct nvme_fc_ersp_iu *ep; 94101649561SJames Smart struct nvme_fc_cmd_iu *cp; 94201649561SJames Smart struct lpfc_nodelist *ndlp; 943bbe3012bSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 9444b056682SJames Smart struct lpfc_nvme_lport *lport; 945352b205aSArnd Bergmann uint32_t code, status, idx; 94601649561SJames Smart uint16_t cid, sqhd, data; 94701649561SJames Smart uint32_t *ptr; 94802243836SJames Smart uint32_t lat; 94902243836SJames Smart bool call_done = false; 950840eda96SJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 951840eda96SJames Smart int cpu; 952840eda96SJames Smart #endif 953d472a766SJustin Tee bool offline = false; 95401649561SJames Smart 95501649561SJames Smart /* Sanity check on return of outstanding command */ 9564d5e789aSJames Smart if (!lpfc_ncmd) { 957372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 9584d5e789aSJames Smart "6071 Null lpfc_ncmd pointer. No " 9594d5e789aSJames Smart "release, skip completion\n"); 9604d5e789aSJames Smart return; 9614d5e789aSJames Smart } 9624d5e789aSJames Smart 963c2017260SJames Smart /* Guard against abort handler being called at same time */ 964c2017260SJames Smart spin_lock(&lpfc_ncmd->buf_lock); 965c2017260SJames Smart 966c2017260SJames Smart if (!lpfc_ncmd->nvmeCmd) { 967c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 968372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 96932350664SJames Smart "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 97032350664SJames Smart "nvmeCmd x%px\n", 9710b05e9feSJames Smart lpfc_ncmd, lpfc_ncmd->nvmeCmd); 9724d5e789aSJames Smart 9734d5e789aSJames Smart /* Release the lpfc_ncmd regardless of the missing elements. */ 9744d5e789aSJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 97501649561SJames Smart return; 97601649561SJames Smart } 97701649561SJames Smart nCmd = lpfc_ncmd->nvmeCmd; 9784b056682SJames Smart status = bf_get(lpfc_wcqe_c_status, wcqe); 97966a210ffSJames Smart 9804c47efc1SJames Smart idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 9814c47efc1SJames Smart phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 9824c47efc1SJames Smart 9831df09449SJames Smart if (unlikely(status && vport->localport)) { 9844b056682SJames Smart lport = (struct lpfc_nvme_lport *)vport->localport->private; 9851df09449SJames Smart if (lport) { 9864b056682SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe)) 9874b056682SJames Smart atomic_inc(&lport->cmpl_fcp_xb); 9884b056682SJames Smart atomic_inc(&lport->cmpl_fcp_err); 9894b056682SJames Smart } 99066a210ffSJames Smart } 99101649561SJames Smart 992bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 993bd2cdd5eSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 9944b056682SJames Smart status, wcqe->parameter); 99501649561SJames Smart /* 99601649561SJames Smart * Catch race where our node has transitioned, but the 99701649561SJames Smart * transport is still transitioning. 99801649561SJames Smart */ 9990b05e9feSJames Smart ndlp = lpfc_ncmd->ndlp; 1000307e3380SJames Smart if (!ndlp) { 1001372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 100201649561SJames Smart "6062 Ignoring NVME cmpl. No ndlp\n"); 100301649561SJames Smart goto out_err; 100401649561SJames Smart } 100501649561SJames Smart 100601649561SJames Smart code = bf_get(lpfc_wcqe_c_code, wcqe); 100701649561SJames Smart if (code == CQE_CODE_NVME_ERSP) { 100801649561SJames Smart /* For this type of CQE, we need to rebuild the rsp */ 100901649561SJames Smart ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 101001649561SJames Smart 101101649561SJames Smart /* 101201649561SJames Smart * Get Command Id from cmd to plug into response. This 101301649561SJames Smart * code is not needed in the next NVME Transport drop. 101401649561SJames Smart */ 101501649561SJames Smart cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 101601649561SJames Smart cid = cp->sqe.common.command_id; 101701649561SJames Smart 101801649561SJames Smart /* 101901649561SJames Smart * RSN is in CQE word 2 102001649561SJames Smart * SQHD is in CQE Word 3 bits 15:0 102101649561SJames Smart * Cmd Specific info is in CQE Word 1 102201649561SJames Smart * and in CQE Word 0 bits 15:0 102301649561SJames Smart */ 102401649561SJames Smart sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 102501649561SJames Smart 102601649561SJames Smart /* Now lets build the NVME ERSP IU */ 102701649561SJames Smart ep->iu_len = cpu_to_be16(8); 102801649561SJames Smart ep->rsn = wcqe->parameter; 102901649561SJames Smart ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 103001649561SJames Smart ep->rsvd12 = 0; 103101649561SJames Smart ptr = (uint32_t *)&ep->cqe.result.u64; 103201649561SJames Smart *ptr++ = wcqe->total_data_placed; 103301649561SJames Smart data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 103401649561SJames Smart *ptr = (uint32_t)data; 103501649561SJames Smart ep->cqe.sq_head = sqhd; 103601649561SJames Smart ep->cqe.sq_id = nCmd->sqid; 103701649561SJames Smart ep->cqe.command_id = cid; 103801649561SJames Smart ep->cqe.status = 0; 103901649561SJames Smart 104001649561SJames Smart lpfc_ncmd->status = IOSTAT_SUCCESS; 104101649561SJames Smart lpfc_ncmd->result = 0; 104201649561SJames Smart nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 104301649561SJames Smart nCmd->transferred_length = nCmd->payload_length; 104401649561SJames Smart } else { 10456a84d015SJustin Tee lpfc_ncmd->status = status; 1046952c303bSDick Kennedy lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 104701649561SJames Smart 104801649561SJames Smart /* For NVME, the only failure path that results in an 104901649561SJames Smart * IO error is when the adapter rejects it. All other 105001649561SJames Smart * conditions are a success case and resolved by the 105101649561SJames Smart * transport. 105201649561SJames Smart * IOSTAT_FCP_RSP_ERROR means: 105301649561SJames Smart * 1. Length of data received doesn't match total 105401649561SJames Smart * transfer length in WQE 105501649561SJames Smart * 2. If the RSP payload does NOT match these cases: 105601649561SJames Smart * a. RSP length 12/24 bytes and all zeros 105701649561SJames Smart * b. NVME ERSP 105801649561SJames Smart */ 105901649561SJames Smart switch (lpfc_ncmd->status) { 106001649561SJames Smart case IOSTAT_SUCCESS: 106101649561SJames Smart nCmd->transferred_length = wcqe->total_data_placed; 106201649561SJames Smart nCmd->rcv_rsplen = 0; 106301649561SJames Smart nCmd->status = 0; 106401649561SJames Smart break; 106501649561SJames Smart case IOSTAT_FCP_RSP_ERROR: 106601649561SJames Smart nCmd->transferred_length = wcqe->total_data_placed; 106701649561SJames Smart nCmd->rcv_rsplen = wcqe->parameter; 106801649561SJames Smart nCmd->status = 0; 1069fa21189dSJames Smart 1070ea7bd1f3SJames Smart /* Get the NVME cmd details for this unique error. */ 1071ea7bd1f3SJames Smart cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1072ea7bd1f3SJames Smart ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 1073ea7bd1f3SJames Smart 1074fa21189dSJames Smart /* Check if this is really an ERSP */ 1075fa21189dSJames Smart if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) { 1076fa21189dSJames Smart lpfc_ncmd->status = IOSTAT_SUCCESS; 1077fa21189dSJames Smart lpfc_ncmd->result = 0; 1078fa21189dSJames Smart 1079fa21189dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 1080ea7bd1f3SJames Smart "6084 NVME FCP_ERR ERSP: " 1081ea7bd1f3SJames Smart "xri %x placed x%x opcode x%x cmd_id " 1082ea7bd1f3SJames Smart "x%x cqe_status x%x\n", 1083fa21189dSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 1084ea7bd1f3SJames Smart wcqe->total_data_placed, 1085ea7bd1f3SJames Smart cp->sqe.common.opcode, 1086ea7bd1f3SJames Smart cp->sqe.common.command_id, 1087ea7bd1f3SJames Smart ep->cqe.status); 108801649561SJames Smart break; 1089fa21189dSJames Smart } 1090372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 109101649561SJames Smart "6081 NVME Completion Protocol Error: " 109286c67379SJames Smart "xri %x status x%x result x%x " 1093ea7bd1f3SJames Smart "placed x%x opcode x%x cmd_id x%x, " 1094ea7bd1f3SJames Smart "cqe_status x%x\n", 109586c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 109601649561SJames Smart lpfc_ncmd->status, lpfc_ncmd->result, 1097ea7bd1f3SJames Smart wcqe->total_data_placed, 1098ea7bd1f3SJames Smart cp->sqe.common.opcode, 1099ea7bd1f3SJames Smart cp->sqe.common.command_id, 1100ea7bd1f3SJames Smart ep->cqe.status); 110101649561SJames Smart break; 1102952c303bSDick Kennedy case IOSTAT_LOCAL_REJECT: 1103952c303bSDick Kennedy /* Let fall through to set command final state. */ 1104952c303bSDick Kennedy if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1105952c303bSDick Kennedy lpfc_printf_vlog(vport, KERN_INFO, 1106952c303bSDick Kennedy LOG_NVME_IOERR, 110732350664SJames Smart "6032 Delay Aborted cmd x%px " 110832350664SJames Smart "nvme cmd x%px, xri x%x, " 1109952c303bSDick Kennedy "xb %d\n", 1110952c303bSDick Kennedy lpfc_ncmd, nCmd, 1111952c303bSDick Kennedy lpfc_ncmd->cur_iocbq.sli4_xritag, 1112952c303bSDick Kennedy bf_get(lpfc_wcqe_c_xb, wcqe)); 1113df561f66SGustavo A. R. Silva fallthrough; 111401649561SJames Smart default: 111501649561SJames Smart out_err: 1116e3246a12SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 111786c67379SJames Smart "6072 NVME Completion Error: xri %x " 11182c013a3aSJames Smart "status x%x result x%x [x%x] " 11192c013a3aSJames Smart "placed x%x\n", 112086c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 112101649561SJames Smart lpfc_ncmd->status, lpfc_ncmd->result, 11222c013a3aSJames Smart wcqe->parameter, 112301649561SJames Smart wcqe->total_data_placed); 112401649561SJames Smart nCmd->transferred_length = 0; 112501649561SJames Smart nCmd->rcv_rsplen = 0; 11268e009ce8SJames Smart nCmd->status = NVME_SC_INTERNAL; 1127d472a766SJustin Tee if (pci_channel_offline(vport->phba->pcidev) || 1128d472a766SJustin Tee lpfc_ncmd->result == IOERR_SLI_DOWN) 1129d472a766SJustin Tee offline = true; 113001649561SJames Smart } 113101649561SJames Smart } 113201649561SJames Smart 113301649561SJames Smart /* pick up SLI4 exhange busy condition */ 113425ac2c97SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline) 113501649561SJames Smart lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 113601649561SJames Smart else 113701649561SJames Smart lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 113801649561SJames Smart 113901649561SJames Smart /* Update stats and complete the IO. There is 114001649561SJames Smart * no need for dma unprep because the nvme_transport 114101649561SJames Smart * owns the dma address. 114201649561SJames Smart */ 1143bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1144c8a4ce0bSDick Kennedy if (lpfc_ncmd->ts_cmd_start) { 1145bd2cdd5eSJames Smart lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 11462fcbc569SJames Smart lpfc_ncmd->ts_data_io = ktime_get_ns(); 11472fcbc569SJames Smart phba->ktime_last_cmd = lpfc_ncmd->ts_data_io; 11482fcbc569SJames Smart lpfc_io_ktime(phba, lpfc_ncmd); 1149bd2cdd5eSJames Smart } 1150840eda96SJames Smart if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) { 1151d6d189ceSBart Van Assche cpu = raw_smp_processor_id(); 1152840eda96SJames Smart this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 115363df6d63SJames Smart if (lpfc_ncmd->cpu != cpu) 115463df6d63SJames Smart lpfc_printf_vlog(vport, 115563df6d63SJames Smart KERN_INFO, LOG_NVME_IOERR, 1156bd2cdd5eSJames Smart "6701 CPU Check cmpl: " 1157bd2cdd5eSJames Smart "cpu %d expect %d\n", 115863df6d63SJames Smart cpu, lpfc_ncmd->cpu); 1159bd2cdd5eSJames Smart } 1160bd2cdd5eSJames Smart #endif 1161952c303bSDick Kennedy 1162952c303bSDick Kennedy /* NVME targets need completion held off until the abort exchange 1163add9d6beSJames Smart * completes unless the NVME Rport is getting unregistered. 1164952c303bSDick Kennedy */ 1165add9d6beSJames Smart 11663fd78355SJames Smart if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 116791455b85SJames Smart freqpriv = nCmd->private; 116891455b85SJames Smart freqpriv->nvme_buf = NULL; 11693fd78355SJames Smart lpfc_ncmd->nvmeCmd = NULL; 117002243836SJames Smart call_done = true; 117102243836SJames Smart } 1172c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 117302243836SJames Smart 117402243836SJames Smart /* Check if IO qualified for CMF */ 117502243836SJames Smart if (phba->cmf_active_mode != LPFC_CFG_OFF && 117602243836SJames Smart nCmd->io_dir == NVMEFC_FCP_READ && 117702243836SJames Smart nCmd->payload_length) { 117802243836SJames Smart /* Used when calculating average latency */ 117902243836SJames Smart lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start; 118002243836SJames Smart lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL); 118102243836SJames Smart } 118202243836SJames Smart 118302243836SJames Smart if (call_done) 1184c2017260SJames Smart nCmd->done(nCmd); 118501649561SJames Smart 1186952c303bSDick Kennedy /* Call release with XB=1 to queue the IO into the abort list. */ 118701649561SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 118801649561SJames Smart } 118901649561SJames Smart 119001649561SJames Smart 119101649561SJames Smart /** 119201649561SJames Smart * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 119309d99705SLee Jones * @vport: pointer to a host virtual N_Port data structure 11943e749170SLee Jones * @lpfc_ncmd: Pointer to lpfc scsi command 119509d99705SLee Jones * @pnode: pointer to a node-list data structure 119609d99705SLee Jones * @cstat: pointer to the control status structure 119701649561SJames Smart * 119801649561SJames Smart * Driver registers this routine as it io request handler. This 119901649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 120001649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. 120101649561SJames Smart * 120201649561SJames Smart * Return value : 120301649561SJames Smart * 0 - Success 120401649561SJames Smart * TODO: What are the failure codes. 120501649561SJames Smart **/ 120601649561SJames Smart static int 120701649561SJames Smart lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1208c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd, 120966a210ffSJames Smart struct lpfc_nodelist *pnode, 12104c47efc1SJames Smart struct lpfc_fc4_ctrl_stat *cstat) 121101649561SJames Smart { 121201649561SJames Smart struct lpfc_hba *phba = vport->phba; 121301649561SJames Smart struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 12142e7e9c0cSJames Smart struct nvme_common_command *sqe; 12152e7e9c0cSJames Smart struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq; 1216205e8240SJames Smart union lpfc_wqe128 *wqe = &pwqeq->wqe; 121701649561SJames Smart uint32_t req_len; 121801649561SJames Smart 121901649561SJames Smart /* 122001649561SJames Smart * There are three possibilities here - use scatter-gather segment, use 122101649561SJames Smart * the single mapping, or neither. 122201649561SJames Smart */ 122301649561SJames Smart if (nCmd->sg_cnt) { 122401649561SJames Smart if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 12255fd11085SJames Smart /* From the iwrite template, initialize words 7 - 11 */ 12265fd11085SJames Smart memcpy(&wqe->words[7], 12275fd11085SJames Smart &lpfc_iwrite_cmd_template.words[7], 12285fd11085SJames Smart sizeof(uint32_t) * 5); 12295fd11085SJames Smart 12305fd11085SJames Smart /* Word 4 */ 12315fd11085SJames Smart wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 12325fd11085SJames Smart 123301649561SJames Smart /* Word 5 */ 123401649561SJames Smart if ((phba->cfg_nvme_enable_fb) && 123501649561SJames Smart (pnode->nlp_flag & NLP_FIRSTBURST)) { 123601649561SJames Smart req_len = lpfc_ncmd->nvmeCmd->payload_length; 123701649561SJames Smart if (req_len < pnode->nvme_fb_size) 123801649561SJames Smart wqe->fcp_iwrite.initial_xfer_len = 123901649561SJames Smart req_len; 124001649561SJames Smart else 124101649561SJames Smart wqe->fcp_iwrite.initial_xfer_len = 124201649561SJames Smart pnode->nvme_fb_size; 12435fd11085SJames Smart } else { 12445fd11085SJames Smart wqe->fcp_iwrite.initial_xfer_len = 0; 124501649561SJames Smart } 12464c47efc1SJames Smart cstat->output_requests++; 124701649561SJames Smart } else { 12485fd11085SJames Smart /* From the iread template, initialize words 7 - 11 */ 12495fd11085SJames Smart memcpy(&wqe->words[7], 12505fd11085SJames Smart &lpfc_iread_cmd_template.words[7], 12515fd11085SJames Smart sizeof(uint32_t) * 5); 125201649561SJames Smart 12535fd11085SJames Smart /* Word 4 */ 12545fd11085SJames Smart wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 125501649561SJames Smart 12565fd11085SJames Smart /* Word 5 */ 12575fd11085SJames Smart wqe->fcp_iread.rsrvd5 = 0; 125801649561SJames Smart 125902243836SJames Smart /* For a CMF Managed port, iod must be zero'ed */ 126002243836SJames Smart if (phba->cmf_active_mode == LPFC_CFG_MANAGED) 126102243836SJames Smart bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, 126202243836SJames Smart LPFC_WQE_IOD_NONE); 12634c47efc1SJames Smart cstat->input_requests++; 126401649561SJames Smart } 126501649561SJames Smart } else { 12665fd11085SJames Smart /* From the icmnd template, initialize words 4 - 11 */ 12675fd11085SJames Smart memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 12685fd11085SJames Smart sizeof(uint32_t) * 8); 12694c47efc1SJames Smart cstat->control_requests++; 127001649561SJames Smart } 12710d8af096SJames Smart 12722e7e9c0cSJames Smart if (pnode->nlp_nvme_info & NLP_NVME_NSLER) { 12730d8af096SJames Smart bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 12742e7e9c0cSJames Smart sqe = &((struct nvme_fc_cmd_iu *) 12752e7e9c0cSJames Smart nCmd->cmdaddr)->sqe.common; 12762e7e9c0cSJames Smart if (sqe->opcode == nvme_admin_async_event) 12772e7e9c0cSJames Smart bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1); 12782e7e9c0cSJames Smart } 12792e7e9c0cSJames Smart 128001649561SJames Smart /* 128101649561SJames Smart * Finish initializing those WQE fields that are independent 128201649561SJames Smart * of the nvme_cmnd request_buffer 128301649561SJames Smart */ 128401649561SJames Smart 12855fd11085SJames Smart /* Word 3 */ 12865fd11085SJames Smart bf_set(payload_offset_len, &wqe->fcp_icmd, 12875fd11085SJames Smart (nCmd->rsplen + nCmd->cmdlen)); 12885fd11085SJames Smart 128901649561SJames Smart /* Word 6 */ 129001649561SJames Smart bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 129101649561SJames Smart phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 129201649561SJames Smart bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 129301649561SJames Smart 129401649561SJames Smart /* Word 8 */ 129501649561SJames Smart wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 129601649561SJames Smart 129701649561SJames Smart /* Word 9 */ 129801649561SJames Smart bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 129901649561SJames Smart 1300840a4701SJames Smart /* Word 10 */ 1301840a4701SJames Smart bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG); 1302840a4701SJames Smart 1303414abe0aSJames Smart /* Words 13 14 15 are for PBDE support */ 1304414abe0aSJames Smart 1305896325a8SJames Smart /* add the VMID tags as per switch response */ 1306896325a8SJames Smart if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) { 1307896325a8SJames Smart if (phba->pport->vmid_priority_tagging) { 1308896325a8SJames Smart bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1); 1309896325a8SJames Smart bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com, 1310896325a8SJames Smart lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid); 1311896325a8SJames Smart } else { 1312896325a8SJames Smart bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1); 1313896325a8SJames Smart bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 1314896325a8SJames Smart wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id; 1315896325a8SJames Smart } 1316896325a8SJames Smart } 1317896325a8SJames Smart 131801649561SJames Smart pwqeq->vport = vport; 131901649561SJames Smart return 0; 132001649561SJames Smart } 132101649561SJames Smart 132201649561SJames Smart 132301649561SJames Smart /** 132401649561SJames Smart * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 132509d99705SLee Jones * @vport: pointer to a host virtual N_Port data structure 13263e749170SLee Jones * @lpfc_ncmd: Pointer to lpfc scsi command 132701649561SJames Smart * 132801649561SJames Smart * Driver registers this routine as it io request handler. This 132901649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 133001649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. 133101649561SJames Smart * 133201649561SJames Smart * Return value : 133301649561SJames Smart * 0 - Success 133401649561SJames Smart * TODO: What are the failure codes. 133501649561SJames Smart **/ 133601649561SJames Smart static int 133701649561SJames Smart lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1338c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd) 133901649561SJames Smart { 134001649561SJames Smart struct lpfc_hba *phba = vport->phba; 134101649561SJames Smart struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1342205e8240SJames Smart union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 13430794d601SJames Smart struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1344d79c9e9dSJames Smart struct sli4_hybrid_sgl *sgl_xtra = NULL; 134501649561SJames Smart struct scatterlist *data_sg; 134601649561SJames Smart struct sli4_sge *first_data_sgl; 13470bc2b7c5SJames Smart struct ulp_bde64 *bde; 1348d79c9e9dSJames Smart dma_addr_t physaddr = 0; 1349d79c9e9dSJames Smart uint32_t dma_len = 0; 135001649561SJames Smart uint32_t dma_offset = 0; 1351d79c9e9dSJames Smart int nseg, i, j; 1352d79c9e9dSJames Smart bool lsp_just_set = false; 135301649561SJames Smart 135401649561SJames Smart /* Fix up the command and response DMA stuff. */ 135501649561SJames Smart lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 135601649561SJames Smart 135701649561SJames Smart /* 135801649561SJames Smart * There are three possibilities here - use scatter-gather segment, use 135901649561SJames Smart * the single mapping, or neither. 136001649561SJames Smart */ 136101649561SJames Smart if (nCmd->sg_cnt) { 136201649561SJames Smart /* 136301649561SJames Smart * Jump over the cmd and rsp SGEs. The fix routine 136401649561SJames Smart * has already adjusted for this. 136501649561SJames Smart */ 136601649561SJames Smart sgl += 2; 136701649561SJames Smart 136801649561SJames Smart first_data_sgl = sgl; 136901649561SJames Smart lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 137081e6a637SJames Smart if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1371372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 137201649561SJames Smart "6058 Too many sg segments from " 137301649561SJames Smart "NVME Transport. Max %d, " 137401649561SJames Smart "nvmeIO sg_cnt %d\n", 137529bfd55aSJames Smart phba->cfg_nvme_seg_cnt + 1, 137601649561SJames Smart lpfc_ncmd->seg_cnt); 137701649561SJames Smart lpfc_ncmd->seg_cnt = 0; 137801649561SJames Smart return 1; 137901649561SJames Smart } 138001649561SJames Smart 138101649561SJames Smart /* 138201649561SJames Smart * The driver established a maximum scatter-gather segment count 138301649561SJames Smart * during probe that limits the number of sg elements in any 138401649561SJames Smart * single nvme command. Just run through the seg_cnt and format 138501649561SJames Smart * the sge's. 138601649561SJames Smart */ 138701649561SJames Smart nseg = nCmd->sg_cnt; 138801649561SJames Smart data_sg = nCmd->first_sgl; 1389d79c9e9dSJames Smart 1390d79c9e9dSJames Smart /* for tracking the segment boundaries */ 1391d79c9e9dSJames Smart j = 2; 139201649561SJames Smart for (i = 0; i < nseg; i++) { 139301649561SJames Smart if (data_sg == NULL) { 1394372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 139501649561SJames Smart "6059 dptr err %d, nseg %d\n", 139601649561SJames Smart i, nseg); 139701649561SJames Smart lpfc_ncmd->seg_cnt = 0; 139801649561SJames Smart return 1; 139901649561SJames Smart } 1400d79c9e9dSJames Smart 1401d79c9e9dSJames Smart sgl->word2 = 0; 1402315b3fd1SJames Smart if (nseg == 1) { 1403d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1); 1404d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl, 1405d79c9e9dSJames Smart LPFC_SGE_TYPE_DATA); 1406d79c9e9dSJames Smart } else { 1407d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0); 1408d79c9e9dSJames Smart 1409d79c9e9dSJames Smart /* expand the segment */ 1410d79c9e9dSJames Smart if (!lsp_just_set && 1411d79c9e9dSJames Smart !((j + 1) % phba->border_sge_num) && 1412d79c9e9dSJames Smart ((nseg - 1) != i)) { 1413d79c9e9dSJames Smart /* set LSP type */ 1414d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl, 1415d79c9e9dSJames Smart LPFC_SGE_TYPE_LSP); 1416d79c9e9dSJames Smart 1417d79c9e9dSJames Smart sgl_xtra = lpfc_get_sgl_per_hdwq( 1418d79c9e9dSJames Smart phba, lpfc_ncmd); 1419d79c9e9dSJames Smart 1420d79c9e9dSJames Smart if (unlikely(!sgl_xtra)) { 1421d79c9e9dSJames Smart lpfc_ncmd->seg_cnt = 0; 1422d79c9e9dSJames Smart return 1; 1423d79c9e9dSJames Smart } 1424d79c9e9dSJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow( 1425d79c9e9dSJames Smart sgl_xtra->dma_phys_sgl)); 1426d79c9e9dSJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1427d79c9e9dSJames Smart sgl_xtra->dma_phys_sgl)); 1428d79c9e9dSJames Smart 1429d79c9e9dSJames Smart } else { 1430d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl, 1431d79c9e9dSJames Smart LPFC_SGE_TYPE_DATA); 1432d79c9e9dSJames Smart } 1433d79c9e9dSJames Smart } 1434d79c9e9dSJames Smart 1435d79c9e9dSJames Smart if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1436d79c9e9dSJames Smart LPFC_SGE_TYPE_LSP)) { 1437d79c9e9dSJames Smart if ((nseg - 1) == i) 1438d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1); 1439d79c9e9dSJames Smart 1440a14396b6SJames Smart physaddr = sg_dma_address(data_sg); 1441a14396b6SJames Smart dma_len = sg_dma_len(data_sg); 1442d79c9e9dSJames Smart sgl->addr_lo = cpu_to_le32( 1443d79c9e9dSJames Smart putPaddrLow(physaddr)); 1444d79c9e9dSJames Smart sgl->addr_hi = cpu_to_le32( 1445d79c9e9dSJames Smart putPaddrHigh(physaddr)); 1446d79c9e9dSJames Smart 144701649561SJames Smart bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 144801649561SJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 144901649561SJames Smart sgl->sge_len = cpu_to_le32(dma_len); 145001649561SJames Smart 145101649561SJames Smart dma_offset += dma_len; 145201649561SJames Smart data_sg = sg_next(data_sg); 1453d79c9e9dSJames Smart 145401649561SJames Smart sgl++; 1455d79c9e9dSJames Smart 1456d79c9e9dSJames Smart lsp_just_set = false; 1457d79c9e9dSJames Smart } else { 1458d79c9e9dSJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 1459d79c9e9dSJames Smart 1460d79c9e9dSJames Smart sgl->sge_len = cpu_to_le32( 1461d79c9e9dSJames Smart phba->cfg_sg_dma_buf_size); 1462d79c9e9dSJames Smart 1463d79c9e9dSJames Smart sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1464d79c9e9dSJames Smart i = i - 1; 1465d79c9e9dSJames Smart 1466d79c9e9dSJames Smart lsp_just_set = true; 1467d79c9e9dSJames Smart } 1468d79c9e9dSJames Smart 1469d79c9e9dSJames Smart j++; 147001649561SJames Smart } 1471315b3fd1SJames Smart 1472315b3fd1SJames Smart /* PBDE support for first data SGE only */ 1473315b3fd1SJames Smart if (nseg == 1 && phba->cfg_enable_pbde) { 14740bc2b7c5SJames Smart /* Words 13-15 */ 14750bc2b7c5SJames Smart bde = (struct ulp_bde64 *) 14760bc2b7c5SJames Smart &wqe->words[13]; 14770bc2b7c5SJames Smart bde->addrLow = first_data_sgl->addr_lo; 14780bc2b7c5SJames Smart bde->addrHigh = first_data_sgl->addr_hi; 14790bc2b7c5SJames Smart bde->tus.f.bdeSize = 14800bc2b7c5SJames Smart le32_to_cpu(first_data_sgl->sge_len); 14810bc2b7c5SJames Smart bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 14820bc2b7c5SJames Smart bde->tus.w = cpu_to_le32(bde->tus.w); 1483b101eb27SJames Smart 1484315b3fd1SJames Smart /* Word 11 - set PBDE bit */ 1485b101eb27SJames Smart bf_set(wqe_pbde, &wqe->generic.wqe_com, 1); 14865fd11085SJames Smart } else { 14875fd11085SJames Smart memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1488315b3fd1SJames Smart /* Word 11 - PBDE bit disabled by default template */ 14895fd11085SJames Smart } 14900bc2b7c5SJames Smart 1491414abe0aSJames Smart } else { 14920794d601SJames Smart lpfc_ncmd->seg_cnt = 0; 14930794d601SJames Smart 149401649561SJames Smart /* For this clause to be valid, the payload_length 149501649561SJames Smart * and sg_cnt must zero. 149601649561SJames Smart */ 149701649561SJames Smart if (nCmd->payload_length != 0) { 1498372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 149901649561SJames Smart "6063 NVME DMA Prep Err: sg_cnt %d " 150001649561SJames Smart "payload_length x%x\n", 150101649561SJames Smart nCmd->sg_cnt, nCmd->payload_length); 150201649561SJames Smart return 1; 150301649561SJames Smart } 150401649561SJames Smart } 150501649561SJames Smart return 0; 150601649561SJames Smart } 150701649561SJames Smart 150801649561SJames Smart /** 150901649561SJames Smart * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 15103e749170SLee Jones * @pnvme_lport: Pointer to the driver's local port data 15113e749170SLee Jones * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 151201649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 15133e749170SLee Jones * @pnvme_fcreq: IO request from nvme fc to driver. 151401649561SJames Smart * 151501649561SJames Smart * Driver registers this routine as it io request handler. This 151601649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 15173e749170SLee Jones * data structure to the rport indicated in @lpfc_nvme_rport. 151801649561SJames Smart * 151901649561SJames Smart * Return value : 152001649561SJames Smart * 0 - Success 152101649561SJames Smart * TODO: What are the failure codes. 152201649561SJames Smart **/ 152301649561SJames Smart static int 152401649561SJames Smart lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 152501649561SJames Smart struct nvme_fc_remote_port *pnvme_rport, 152601649561SJames Smart void *hw_queue_handle, 152701649561SJames Smart struct nvmefc_fcp_req *pnvme_fcreq) 152801649561SJames Smart { 152901649561SJames Smart int ret = 0; 1530cf1a1d3eSJames Smart int expedite = 0; 153163df6d63SJames Smart int idx, cpu; 153201649561SJames Smart struct lpfc_nvme_lport *lport; 15334c47efc1SJames Smart struct lpfc_fc4_ctrl_stat *cstat; 153401649561SJames Smart struct lpfc_vport *vport; 153501649561SJames Smart struct lpfc_hba *phba; 153601649561SJames Smart struct lpfc_nodelist *ndlp; 1537c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd; 153801649561SJames Smart struct lpfc_nvme_rport *rport; 153901649561SJames Smart struct lpfc_nvme_qhandle *lpfc_queue_info; 1540c3725bdcSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 1541cf1a1d3eSJames Smart struct nvme_common_command *sqe; 1542bd2cdd5eSJames Smart uint64_t start = 0; 1543896325a8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 1544896325a8SJames Smart u8 *uuid = NULL; 1545896325a8SJames Smart int err; 1546896325a8SJames Smart enum dma_data_direction iodir; 1547896325a8SJames Smart #endif 154801649561SJames Smart 1549c3725bdcSJames Smart /* Validate pointers. LLDD fault handling with transport does 1550c3725bdcSJames Smart * have timing races. 1551c3725bdcSJames Smart */ 155201649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1553c3725bdcSJames Smart if (unlikely(!lport)) { 1554c3725bdcSJames Smart ret = -EINVAL; 1555c3725bdcSJames Smart goto out_fail; 1556c3725bdcSJames Smart } 1557c3725bdcSJames Smart 155801649561SJames Smart vport = lport->vport; 1559c3725bdcSJames Smart 1560c3725bdcSJames Smart if (unlikely(!hw_queue_handle)) { 156144c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 156244c2757bSJames Smart "6117 Fail IO, NULL hw_queue_handle\n"); 156344c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 1564cd240071SJames Smart ret = -EBUSY; 1565c3725bdcSJames Smart goto out_fail; 1566c3725bdcSJames Smart } 1567c3725bdcSJames Smart 156801649561SJames Smart phba = vport->phba; 156901649561SJames Smart 1570*e39811beSJustin Tee if ((unlikely(test_bit(FC_UNLOADING, &vport->load_flag))) || 1571a4691038SJames Smart phba->hba_flag & HBA_IOQ_FLUSH) { 157244c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 157344c2757bSJames Smart "6124 Fail IO, Driver unload\n"); 157444c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 15753386f4bdSJames Smart ret = -ENODEV; 15763386f4bdSJames Smart goto out_fail; 15773386f4bdSJames Smart } 15783386f4bdSJames Smart 1579c3725bdcSJames Smart freqpriv = pnvme_fcreq->private; 1580c3725bdcSJames Smart if (unlikely(!freqpriv)) { 158144c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 158244c2757bSJames Smart "6158 Fail IO, NULL request data\n"); 158344c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 1584c3725bdcSJames Smart ret = -EINVAL; 1585b7672ae6SDick Kennedy goto out_fail; 1586b7672ae6SDick Kennedy } 1587b7672ae6SDick Kennedy 1588bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1589bd2cdd5eSJames Smart if (phba->ktime_on) 1590bd2cdd5eSJames Smart start = ktime_get_ns(); 1591bd2cdd5eSJames Smart #endif 159201649561SJames Smart rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 159301649561SJames Smart lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 159401649561SJames Smart 159501649561SJames Smart /* 159601649561SJames Smart * Catch race where our node has transitioned, but the 159701649561SJames Smart * transport is still transitioning. 159801649561SJames Smart */ 159901649561SJames Smart ndlp = rport->ndlp; 1600307e3380SJames Smart if (!ndlp) { 160144c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 160232350664SJames Smart "6053 Busy IO, ndlp not ready: rport x%px " 160332350664SJames Smart "ndlp x%px, DID x%06x\n", 160401649561SJames Smart rport, ndlp, pnvme_rport->port_id); 160544c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 1606cd240071SJames Smart ret = -EBUSY; 160701649561SJames Smart goto out_fail; 160801649561SJames Smart } 160901649561SJames Smart 161001649561SJames Smart /* The remote node has to be a mapped target or it's an error. */ 161101649561SJames Smart if ((ndlp->nlp_type & NLP_NVME_TARGET) && 161201649561SJames Smart (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 161344c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 161444c2757bSJames Smart "6036 Fail IO, DID x%06x not ready for " 1615cd240071SJames Smart "IO. State x%x, Type x%x Flg x%x\n", 161644c2757bSJames Smart pnvme_rport->port_id, 1617cd240071SJames Smart ndlp->nlp_state, ndlp->nlp_type, 1618e9b11083SJames Smart ndlp->fc4_xpt_flags); 16194b056682SJames Smart atomic_inc(&lport->xmt_fcp_bad_ndlp); 1620cd240071SJames Smart ret = -EBUSY; 162101649561SJames Smart goto out_fail; 162201649561SJames Smart 162301649561SJames Smart } 162401649561SJames Smart 1625cf1a1d3eSJames Smart /* Currently only NVME Keep alive commands should be expedited 1626cf1a1d3eSJames Smart * if the driver runs out of a resource. These should only be 1627cf1a1d3eSJames Smart * issued on the admin queue, qidx 0 1628cf1a1d3eSJames Smart */ 1629cf1a1d3eSJames Smart if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1630cf1a1d3eSJames Smart sqe = &((struct nvme_fc_cmd_iu *) 1631cf1a1d3eSJames Smart pnvme_fcreq->cmdaddr)->sqe.common; 1632cf1a1d3eSJames Smart if (sqe->opcode == nvme_admin_keep_alive) 1633cf1a1d3eSJames Smart expedite = 1; 1634cf1a1d3eSJames Smart } 1635cf1a1d3eSJames Smart 163602243836SJames Smart /* Check if IO qualifies for CMF */ 163702243836SJames Smart if (phba->cmf_active_mode != LPFC_CFG_OFF && 163802243836SJames Smart pnvme_fcreq->io_dir == NVMEFC_FCP_READ && 163902243836SJames Smart pnvme_fcreq->payload_length) { 164002243836SJames Smart ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length); 164102243836SJames Smart if (ret) { 164202243836SJames Smart ret = -EBUSY; 164302243836SJames Smart goto out_fail; 164402243836SJames Smart } 164502243836SJames Smart /* Get start time for IO latency */ 164602243836SJames Smart start = ktime_get_ns(); 164702243836SJames Smart } 164802243836SJames Smart 164901649561SJames Smart /* The node is shared with FCP IO, make sure the IO pending count does 165001649561SJames Smart * not exceed the programmed depth. 165101649561SJames Smart */ 16522a5b7d62SJames Smart if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1653cf1a1d3eSJames Smart if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1654cf1a1d3eSJames Smart !expedite) { 165544c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 165644c2757bSJames Smart "6174 Fail IO, ndlp qdepth exceeded: " 16574d5e789aSJames Smart "idx %d DID %x pend %d qdepth %d\n", 16584d5e789aSJames Smart lpfc_queue_info->index, ndlp->nlp_DID, 16594d5e789aSJames Smart atomic_read(&ndlp->cmd_pending), 16604d5e789aSJames Smart ndlp->cmd_qdepth); 16614b056682SJames Smart atomic_inc(&lport->xmt_fcp_qdepth); 1662cd22d605SDick Kennedy ret = -EBUSY; 166302243836SJames Smart goto out_fail1; 166401649561SJames Smart } 16652a5b7d62SJames Smart } 166601649561SJames Smart 16676a828b0fSJames Smart /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 166845aa312eSJames Smart if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 166945aa312eSJames Smart idx = lpfc_queue_info->index; 167045aa312eSJames Smart } else { 1671d6d189ceSBart Van Assche cpu = raw_smp_processor_id(); 16726a828b0fSJames Smart idx = phba->sli4_hba.cpu_map[cpu].hdwq; 167345aa312eSJames Smart } 167445aa312eSJames Smart 167545aa312eSJames Smart lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 167601649561SJames Smart if (lpfc_ncmd == NULL) { 16774b056682SJames Smart atomic_inc(&lport->xmt_fcp_noxri); 167801649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 167944c2757bSJames Smart "6065 Fail IO, driver buffer pool is empty: " 168044c2757bSJames Smart "idx %d DID %x\n", 168144c2757bSJames Smart lpfc_queue_info->index, ndlp->nlp_DID); 1682cd22d605SDick Kennedy ret = -EBUSY; 168302243836SJames Smart goto out_fail1; 168401649561SJames Smart } 1685bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1686c8a4ce0bSDick Kennedy if (start) { 1687bd2cdd5eSJames Smart lpfc_ncmd->ts_cmd_start = start; 1688bd2cdd5eSJames Smart lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1689c8a4ce0bSDick Kennedy } else { 1690c8a4ce0bSDick Kennedy lpfc_ncmd->ts_cmd_start = 0; 1691bd2cdd5eSJames Smart } 1692bd2cdd5eSJames Smart #endif 169302243836SJames Smart lpfc_ncmd->rx_cmd_start = start; 169401649561SJames Smart 169501649561SJames Smart /* 169601649561SJames Smart * Store the data needed by the driver to issue, abort, and complete 169701649561SJames Smart * an IO. 169801649561SJames Smart * Do not let the IO hang out forever. There is no midlayer issuing 169901649561SJames Smart * an abort so inform the FW of the maximum IO pending time. 170001649561SJames Smart */ 1701bbe3012bSJames Smart freqpriv->nvme_buf = lpfc_ncmd; 170201649561SJames Smart lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1703318083adSJames Smart lpfc_ncmd->ndlp = ndlp; 17040794d601SJames Smart lpfc_ncmd->qidx = lpfc_queue_info->qidx; 170501649561SJames Smart 1706896325a8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 1707896325a8SJames Smart /* check the necessary and sufficient condition to support VMID */ 1708896325a8SJames Smart if (lpfc_is_vmid_enabled(phba) && 1709896325a8SJames Smart (ndlp->vmid_support || 1710896325a8SJames Smart phba->pport->vmid_priority_tagging == 1711896325a8SJames Smart LPFC_VMID_PRIO_TAG_ALL_TARGETS)) { 1712896325a8SJames Smart /* is the I/O generated by a VM, get the associated virtual */ 1713896325a8SJames Smart /* entity id */ 1714896325a8SJames Smart uuid = nvme_fc_io_getuuid(pnvme_fcreq); 1715896325a8SJames Smart 1716896325a8SJames Smart if (uuid) { 1717896325a8SJames Smart if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE) 1718896325a8SJames Smart iodir = DMA_TO_DEVICE; 1719896325a8SJames Smart else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ) 1720896325a8SJames Smart iodir = DMA_FROM_DEVICE; 1721896325a8SJames Smart else 1722896325a8SJames Smart iodir = DMA_NONE; 1723896325a8SJames Smart 1724896325a8SJames Smart err = lpfc_vmid_get_appid(vport, uuid, iodir, 1725896325a8SJames Smart (union lpfc_vmid_io_tag *) 1726896325a8SJames Smart &lpfc_ncmd->cur_iocbq.vmid_tag); 1727896325a8SJames Smart if (!err) 1728896325a8SJames Smart lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID; 1729896325a8SJames Smart } 1730896325a8SJames Smart } 1731896325a8SJames Smart #endif 1732896325a8SJames Smart 173301649561SJames Smart /* 173401649561SJames Smart * Issue the IO on the WQ indicated by index in the hw_queue_handle. 173501649561SJames Smart * This identfier was create in our hardware queue create callback 173601649561SJames Smart * routine. The driver now is dependent on the IO queue steering from 173701649561SJames Smart * the transport. We are trusting the upper NVME layers know which 173801649561SJames Smart * index to use and that they have affinitized a CPU to this hardware 173901649561SJames Smart * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 174001649561SJames Smart */ 174166a210ffSJames Smart lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 17424c47efc1SJames Smart cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 174366a210ffSJames Smart 174466a210ffSJames Smart lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 174566a210ffSJames Smart ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 174666a210ffSJames Smart if (ret) { 174744c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 174844c2757bSJames Smart "6175 Fail IO, Prep DMA: " 174944c2757bSJames Smart "idx %d DID %x\n", 175044c2757bSJames Smart lpfc_queue_info->index, ndlp->nlp_DID); 175144c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 175266a210ffSJames Smart ret = -ENOMEM; 175366a210ffSJames Smart goto out_free_nvme_buf; 175466a210ffSJames Smart } 175566a210ffSJames Smart 1756bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1757bd2cdd5eSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 1758bd2cdd5eSJames Smart lpfc_queue_info->index, ndlp->nlp_DID); 1759bd2cdd5eSJames Smart 17601fbf9742SJames Smart ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 176101649561SJames Smart if (ret) { 17624b056682SJames Smart atomic_inc(&lport->xmt_fcp_wqerr); 1763e3246a12SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 176444c2757bSJames Smart "6113 Fail IO, Could not issue WQE err %x " 176501649561SJames Smart "sid: x%x did: x%x oxid: x%x\n", 176601649561SJames Smart ret, vport->fc_myDID, ndlp->nlp_DID, 176701649561SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag); 176801649561SJames Smart goto out_free_nvme_buf; 176901649561SJames Smart } 177001649561SJames Smart 1771c490850aSJames Smart if (phba->cfg_xri_rebalancing) 1772c490850aSJames Smart lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1773c490850aSJames Smart 1774bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1775c8a4ce0bSDick Kennedy if (lpfc_ncmd->ts_cmd_start) 1776bd2cdd5eSJames Smart lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1777bd2cdd5eSJames Smart 1778840eda96SJames Smart if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) { 1779d6d189ceSBart Van Assche cpu = raw_smp_processor_id(); 1780840eda96SJames Smart this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 178163df6d63SJames Smart lpfc_ncmd->cpu = cpu; 178263df6d63SJames Smart if (idx != cpu) 1783bd2cdd5eSJames Smart lpfc_printf_vlog(vport, 178463df6d63SJames Smart KERN_INFO, LOG_NVME_IOERR, 1785bd2cdd5eSJames Smart "6702 CPU Check cmd: " 1786bd2cdd5eSJames Smart "cpu %d wq %d\n", 1787bd2cdd5eSJames Smart lpfc_ncmd->cpu, 1788bd2cdd5eSJames Smart lpfc_queue_info->index); 1789bd2cdd5eSJames Smart } 1790bd2cdd5eSJames Smart #endif 179101649561SJames Smart return 0; 179201649561SJames Smart 179301649561SJames Smart out_free_nvme_buf: 17942cee7808SJames Smart if (lpfc_ncmd->nvmeCmd->sg_cnt) { 17952cee7808SJames Smart if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 17964c47efc1SJames Smart cstat->output_requests--; 17972cee7808SJames Smart else 17984c47efc1SJames Smart cstat->input_requests--; 17992cee7808SJames Smart } else 18004c47efc1SJames Smart cstat->control_requests--; 180101649561SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 180202243836SJames Smart out_fail1: 180302243836SJames Smart lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT, 180402243836SJames Smart pnvme_fcreq->payload_length, NULL); 180501649561SJames Smart out_fail: 180601649561SJames Smart return ret; 180701649561SJames Smart } 180801649561SJames Smart 180901649561SJames Smart /** 181001649561SJames Smart * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 181101649561SJames Smart * @phba: Pointer to HBA context object 181201649561SJames Smart * @cmdiocb: Pointer to command iocb object. 181324e1f056SJames Smart * @rspiocb: Pointer to response iocb object. 181401649561SJames Smart * 181501649561SJames Smart * This is the callback function for any NVME FCP IO that was aborted. 181601649561SJames Smart * 181701649561SJames Smart * Return value: 181801649561SJames Smart * None 181901649561SJames Smart **/ 182001649561SJames Smart void 182101649561SJames Smart lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 182224e1f056SJames Smart struct lpfc_iocbq *rspiocb) 182301649561SJames Smart { 182424e1f056SJames Smart struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl; 182524e1f056SJames Smart 1826e3246a12SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 182701649561SJames Smart "6145 ABORT_XRI_CN completing on rpi x%x " 182801649561SJames Smart "original iotag x%x, abort cmd iotag x%x " 182901649561SJames Smart "req_tag x%x, status x%x, hwstatus x%x\n", 183031a59f75SJames Smart bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com), 183131a59f75SJames Smart get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag, 183201649561SJames Smart bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 183301649561SJames Smart bf_get(lpfc_wcqe_c_status, abts_cmpl), 183401649561SJames Smart bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 183501649561SJames Smart lpfc_sli_release_iocbq(phba, cmdiocb); 183601649561SJames Smart } 183701649561SJames Smart 183801649561SJames Smart /** 183901649561SJames Smart * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 18403e749170SLee Jones * @pnvme_lport: Pointer to the driver's local port data 18413e749170SLee Jones * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 184201649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 18433e749170SLee Jones * @pnvme_fcreq: IO request from nvme fc to driver. 184401649561SJames Smart * 184501649561SJames Smart * Driver registers this routine as its nvme request io abort handler. This 184601649561SJames Smart * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 184701649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. This routine 184801649561SJames Smart * is executed asynchronously - one the target is validated as "MAPPED" and 184901649561SJames Smart * ready for IO, the driver issues the abort request and returns. 185001649561SJames Smart * 185101649561SJames Smart * Return value: 185201649561SJames Smart * None 185301649561SJames Smart **/ 185401649561SJames Smart static void 185501649561SJames Smart lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 185601649561SJames Smart struct nvme_fc_remote_port *pnvme_rport, 185701649561SJames Smart void *hw_queue_handle, 185801649561SJames Smart struct nvmefc_fcp_req *pnvme_fcreq) 185901649561SJames Smart { 186001649561SJames Smart struct lpfc_nvme_lport *lport; 186101649561SJames Smart struct lpfc_vport *vport; 186201649561SJames Smart struct lpfc_hba *phba; 1863c490850aSJames Smart struct lpfc_io_buf *lpfc_nbuf; 186401649561SJames Smart struct lpfc_iocbq *nvmereq_wqe; 1865c3725bdcSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 186601649561SJames Smart unsigned long flags; 186701649561SJames Smart int ret_val; 186801649561SJames Smart 1869c3725bdcSJames Smart /* Validate pointers. LLDD fault handling with transport does 1870c3725bdcSJames Smart * have timing races. 1871c3725bdcSJames Smart */ 187201649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1873c3725bdcSJames Smart if (unlikely(!lport)) 1874c3725bdcSJames Smart return; 187501649561SJames Smart 1876c3725bdcSJames Smart vport = lport->vport; 1877c3725bdcSJames Smart 1878c3725bdcSJames Smart if (unlikely(!hw_queue_handle)) { 1879c3725bdcSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1880c3725bdcSJames Smart "6129 Fail Abort, HW Queue Handle NULL.\n"); 1881c3725bdcSJames Smart return; 1882c3725bdcSJames Smart } 1883c3725bdcSJames Smart 1884c3725bdcSJames Smart phba = vport->phba; 1885c3725bdcSJames Smart freqpriv = pnvme_fcreq->private; 1886c3725bdcSJames Smart 1887c3725bdcSJames Smart if (unlikely(!freqpriv)) 1888c3725bdcSJames Smart return; 1889*e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag)) 18903386f4bdSJames Smart return; 18913386f4bdSJames Smart 189201649561SJames Smart /* Announce entry to new IO submit field. */ 189386c67379SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 189401649561SJames Smart "6002 Abort Request to rport DID x%06x " 189532350664SJames Smart "for nvme_fc_req x%px\n", 189601649561SJames Smart pnvme_rport->port_id, 189701649561SJames Smart pnvme_fcreq); 189801649561SJames Smart 189978e9e350SJustin Tee lpfc_nbuf = freqpriv->nvme_buf; 190078e9e350SJustin Tee if (!lpfc_nbuf) { 190178e9e350SJustin Tee lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 190278e9e350SJustin Tee "6140 NVME IO req has no matching lpfc nvme " 190378e9e350SJustin Tee "io buffer. Skipping abort req.\n"); 190478e9e350SJustin Tee return; 190578e9e350SJustin Tee } else if (!lpfc_nbuf->nvmeCmd) { 190678e9e350SJustin Tee lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 190778e9e350SJustin Tee "6141 lpfc NVME IO req has no nvme_fcreq " 190878e9e350SJustin Tee "io buffer. Skipping abort req.\n"); 190978e9e350SJustin Tee return; 191078e9e350SJustin Tee } 191178e9e350SJustin Tee 191278e9e350SJustin Tee /* Guard against IO completion being called at same time */ 191378e9e350SJustin Tee spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags); 191478e9e350SJustin Tee 191501649561SJames Smart /* If the hba is getting reset, this flag is set. It is 191601649561SJames Smart * cleared when the reset is complete and rings reestablished. 191701649561SJames Smart */ 191878e9e350SJustin Tee spin_lock(&phba->hbalock); 191901649561SJames Smart /* driver queued commands are in process of being flushed */ 1920c00f62e6SJames Smart if (phba->hba_flag & HBA_IOQ_FLUSH) { 192178e9e350SJustin Tee spin_unlock(&phba->hbalock); 192278e9e350SJustin Tee spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags); 1923372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 192401649561SJames Smart "6139 Driver in reset cleanup - flushing " 192501649561SJames Smart "NVME Req now. hba_flag x%x\n", 192601649561SJames Smart phba->hba_flag); 192701649561SJames Smart return; 192801649561SJames Smart } 192901649561SJames Smart 19302b7824d0SJames Smart nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 193101649561SJames Smart 193201649561SJames Smart /* 193301649561SJames Smart * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 193401649561SJames Smart * state must match the nvme_fcreq passed by the nvme 193501649561SJames Smart * transport. If they don't match, it is likely the driver 193601649561SJames Smart * has already completed the NVME IO and the nvme transport 193701649561SJames Smart * has not seen it yet. 193801649561SJames Smart */ 193901649561SJames Smart if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1940372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 194101649561SJames Smart "6143 NVME req mismatch: " 194232350664SJames Smart "lpfc_nbuf x%px nvmeCmd x%px, " 194332350664SJames Smart "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 194401649561SJames Smart lpfc_nbuf, lpfc_nbuf->nvmeCmd, 19452b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1946c2017260SJames Smart goto out_unlock; 194701649561SJames Smart } 194801649561SJames Smart 194901649561SJames Smart /* Don't abort IOs no longer on the pending queue. */ 1950a680a929SJames Smart if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) { 1951372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 195232350664SJames Smart "6142 NVME IO req x%px not queued - skipping " 19532b7824d0SJames Smart "abort req xri x%x\n", 19542b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1955c2017260SJames Smart goto out_unlock; 195601649561SJames Smart } 195701649561SJames Smart 19584b056682SJames Smart atomic_inc(&lport->xmt_fcp_abort); 1959bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 1960bd2cdd5eSJames Smart nvmereq_wqe->sli4_xritag, 196100cefeb9SJames Smart nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 1962bd2cdd5eSJames Smart 196301649561SJames Smart /* Outstanding abort is in progress */ 1964a680a929SJames Smart if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) { 1965372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 196601649561SJames Smart "6144 Outstanding NVME I/O Abort Request " 196732350664SJames Smart "still pending on nvme_fcreq x%px, " 1968f1156125SJames Smart "lpfc_ncmd x%px xri x%x\n", 19692b7824d0SJames Smart pnvme_fcreq, lpfc_nbuf, 19702b7824d0SJames Smart nvmereq_wqe->sli4_xritag); 1971c2017260SJames Smart goto out_unlock; 197201649561SJames Smart } 197301649561SJames Smart 1974db7531d2SJames Smart ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe, 1975db7531d2SJames Smart lpfc_nvme_abort_fcreq_cmpl); 197601649561SJames Smart 197778e9e350SJustin Tee spin_unlock(&phba->hbalock); 197878e9e350SJustin Tee spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags); 1979a22d73b6SJames Smart 1980a22d73b6SJames Smart /* Make sure HBA is alive */ 1981a22d73b6SJames Smart lpfc_issue_hb_tmo(phba); 1982a22d73b6SJames Smart 1983db7531d2SJames Smart if (ret_val != WQE_SUCCESS) { 1984372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 198501649561SJames Smart "6137 Failed abts issue_wqe with status x%x " 198632350664SJames Smart "for nvme_fcreq x%px.\n", 198701649561SJames Smart ret_val, pnvme_fcreq); 198801649561SJames Smart return; 198901649561SJames Smart } 199001649561SJames Smart 199186c67379SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 19922b7824d0SJames Smart "6138 Transport Abort NVME Request Issued for " 19931a5cd3d0SJustin Tee "ox_id x%x\n", 19941a5cd3d0SJustin Tee nvmereq_wqe->sli4_xritag); 1995c2017260SJames Smart return; 1996c2017260SJames Smart 1997c2017260SJames Smart out_unlock: 199878e9e350SJustin Tee spin_unlock(&phba->hbalock); 199978e9e350SJustin Tee spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags); 2000c2017260SJames Smart return; 200101649561SJames Smart } 200201649561SJames Smart 200301649561SJames Smart /* Declare and initialization an instance of the FC NVME template. */ 200401649561SJames Smart static struct nvme_fc_port_template lpfc_nvme_template = { 200501649561SJames Smart /* initiator-based functions */ 200601649561SJames Smart .localport_delete = lpfc_nvme_localport_delete, 200701649561SJames Smart .remoteport_delete = lpfc_nvme_remoteport_delete, 200801649561SJames Smart .create_queue = lpfc_nvme_create_queue, 200901649561SJames Smart .delete_queue = lpfc_nvme_delete_queue, 201001649561SJames Smart .ls_req = lpfc_nvme_ls_req, 201101649561SJames Smart .fcp_io = lpfc_nvme_fcp_io_submit, 201201649561SJames Smart .ls_abort = lpfc_nvme_ls_abort, 201301649561SJames Smart .fcp_abort = lpfc_nvme_fcp_abort, 20149aa09e98SJames Smart .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, 201501649561SJames Smart 201601649561SJames Smart .max_hw_queues = 1, 201701649561SJames Smart .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 201801649561SJames Smart .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 201901649561SJames Smart .dma_boundary = 0xFFFFFFFF, 202001649561SJames Smart 202101649561SJames Smart /* Sizes of additional private data for data structures. 202201649561SJames Smart * No use for the last two sizes at this time. 202301649561SJames Smart */ 202401649561SJames Smart .local_priv_sz = sizeof(struct lpfc_nvme_lport), 202501649561SJames Smart .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 202601649561SJames Smart .lsrqst_priv_sz = 0, 2027bbe3012bSJames Smart .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 202801649561SJames Smart }; 202901649561SJames Smart 20303e749170SLee Jones /* 20315e5b511dSJames Smart * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 203201649561SJames Smart * 20335e5b511dSJames Smart * This routine removes a nvme buffer from head of @hdwq io_buf_list 203401649561SJames Smart * and returns to caller. 203501649561SJames Smart * 203601649561SJames Smart * Return codes: 203701649561SJames Smart * NULL - Error 203801649561SJames Smart * Pointer to lpfc_nvme_buf - Success 203901649561SJames Smart **/ 2040c490850aSJames Smart static struct lpfc_io_buf * 2041cf1a1d3eSJames Smart lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 20425e5b511dSJames Smart int idx, int expedite) 204301649561SJames Smart { 2044c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd; 20455e5b511dSJames Smart struct lpfc_sli4_hdw_queue *qp; 20460794d601SJames Smart struct sli4_sge *sgl; 20470794d601SJames Smart struct lpfc_iocbq *pwqeq; 20480794d601SJames Smart union lpfc_wqe128 *wqe; 204901649561SJames Smart 2050c490850aSJames Smart lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 20512a5b7d62SJames Smart 20520794d601SJames Smart if (lpfc_ncmd) { 20530794d601SJames Smart pwqeq = &(lpfc_ncmd->cur_iocbq); 20540794d601SJames Smart wqe = &pwqeq->wqe; 20550794d601SJames Smart 20560794d601SJames Smart /* Setup key fields in buffer that may have been changed 20570794d601SJames Smart * if other protocols used this buffer. 20580794d601SJames Smart */ 2059a680a929SJames Smart pwqeq->cmd_flag = LPFC_IO_NVME; 2060a680a929SJames Smart pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl; 20610794d601SJames Smart lpfc_ncmd->start_time = jiffies; 20620794d601SJames Smart lpfc_ncmd->flags = 0; 20630794d601SJames Smart 20640794d601SJames Smart /* Rsp SGE will be filled in when we rcv an IO 20650794d601SJames Smart * from the NVME Layer to be sent. 20660794d601SJames Smart * The cmd is going to be embedded so we need a SKIP SGE. 20670794d601SJames Smart */ 20680794d601SJames Smart sgl = lpfc_ncmd->dma_sgl; 20690794d601SJames Smart bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 20700794d601SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0); 20710794d601SJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 20720794d601SJames Smart /* Fill in word 3 / sgl_len during cmd submission */ 20730794d601SJames Smart 2074d9f492a1SJames Smart /* Initialize 64 bytes only */ 20750794d601SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe)); 20760794d601SJames Smart 20770794d601SJames Smart if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 20782a5b7d62SJames Smart atomic_inc(&ndlp->cmd_pending); 2079c490850aSJames Smart lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 20802a5b7d62SJames Smart } 20815e5b511dSJames Smart 2082c490850aSJames Smart } else { 2083c490850aSJames Smart qp = &phba->sli4_hba.hdwq[idx]; 20845e5b511dSJames Smart qp->empty_io_bufs++; 2085c490850aSJames Smart } 20865e5b511dSJames Smart 208701649561SJames Smart return lpfc_ncmd; 208801649561SJames Smart } 208901649561SJames Smart 209001649561SJames Smart /** 209101649561SJames Smart * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 209201649561SJames Smart * @phba: The Hba for which this call is being executed. 209301649561SJames Smart * @lpfc_ncmd: The nvme buffer which is being released. 209401649561SJames Smart * 209501649561SJames Smart * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 20965e5b511dSJames Smart * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 209701649561SJames Smart * and cannot be reused for at least RA_TOV amount of time if it was 209801649561SJames Smart * aborted. 209901649561SJames Smart **/ 210001649561SJames Smart static void 2101c490850aSJames Smart lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 210201649561SJames Smart { 21035e5b511dSJames Smart struct lpfc_sli4_hdw_queue *qp; 210401649561SJames Smart unsigned long iflag = 0; 210501649561SJames Smart 2106c490850aSJames Smart if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 21072a5b7d62SJames Smart atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 21082a5b7d62SJames Smart 21092a5b7d62SJames Smart lpfc_ncmd->ndlp = NULL; 2110c490850aSJames Smart lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 21112a5b7d62SJames Smart 21121fbf9742SJames Smart qp = lpfc_ncmd->hdwq; 211322770cbaSJames Smart if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 211486c67379SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 211586c67379SJames Smart "6310 XB release deferred for " 211686c67379SJames Smart "ox_id x%x on reqtag x%x\n", 211786c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 211886c67379SJames Smart lpfc_ncmd->cur_iocbq.iotag); 211986c67379SJames Smart 2120c00f62e6SJames Smart spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 212101649561SJames Smart list_add_tail(&lpfc_ncmd->list, 2122c00f62e6SJames Smart &qp->lpfc_abts_io_buf_list); 21235e5b511dSJames Smart qp->abts_nvme_io_bufs++; 2124c00f62e6SJames Smart spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2125c490850aSJames Smart } else 2126c490850aSJames Smart lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 212701649561SJames Smart } 212801649561SJames Smart 212901649561SJames Smart /** 213001649561SJames Smart * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2131f6b35a75SLee Jones * @vport: the lpfc_vport instance requesting a localport. 213201649561SJames Smart * 213301649561SJames Smart * This routine is invoked to create an nvme localport instance to bind 213401649561SJames Smart * to the nvme_fc_transport. It is called once during driver load 213501649561SJames Smart * like lpfc_create_shost after all other services are initialized. 213601649561SJames Smart * It requires a vport, vpi, and wwns at call time. Other localport 213701649561SJames Smart * parameters are modified as the driver's FCID and the Fabric WWN 213801649561SJames Smart * are established. 213901649561SJames Smart * 214001649561SJames Smart * Return codes 214101649561SJames Smart * 0 - successful 214201649561SJames Smart * -ENOMEM - no heap memory available 214301649561SJames Smart * other values - from nvme registration upcall 214401649561SJames Smart **/ 214501649561SJames Smart int 214601649561SJames Smart lpfc_nvme_create_localport(struct lpfc_vport *vport) 214701649561SJames Smart { 2148166d7211SJames Smart int ret = 0; 214901649561SJames Smart struct lpfc_hba *phba = vport->phba; 215001649561SJames Smart struct nvme_fc_port_info nfcp_info; 215101649561SJames Smart struct nvme_fc_local_port *localport; 215201649561SJames Smart struct lpfc_nvme_lport *lport; 215301649561SJames Smart 215401649561SJames Smart /* Initialize this localport instance. The vport wwn usage ensures 215501649561SJames Smart * that NPIV is accounted for. 215601649561SJames Smart */ 215701649561SJames Smart memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 215801649561SJames Smart nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 215901649561SJames Smart nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 216001649561SJames Smart nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 216101649561SJames Smart 21625b9e70b2SJames Smart /* We need to tell the transport layer + 1 because it takes page 21635b9e70b2SJames Smart * alignment into account. When space for the SGL is allocated we 21645b9e70b2SJames Smart * allocate + 3, one for cmd, one for rsp and one for this alignment 21654d4c4a4aSJames Smart */ 21664d4c4a4aSJames Smart lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 216701649561SJames Smart 2168542ddc9bSJames Smart /* Advertise how many hw queues we support based on cfg_hdw_queue, 2169542ddc9bSJames Smart * which will not exceed cpu count. 2170542ddc9bSJames Smart */ 2171cdb42becSJames Smart lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 217266a210ffSJames Smart 2173faf5a744SArnd Bergmann if (!IS_ENABLED(CONFIG_NVME_FC)) 2174faf5a744SArnd Bergmann return ret; 2175faf5a744SArnd Bergmann 217601649561SJames Smart /* localport is allocated from the stack, but the registration 217701649561SJames Smart * call allocates heap memory as well as the private area. 217801649561SJames Smart */ 2179faf5a744SArnd Bergmann 218001649561SJames Smart ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 218101649561SJames Smart &vport->phba->pcidev->dev, &localport); 218201649561SJames Smart if (!ret) { 218301649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 218401649561SJames Smart "6005 Successfully registered local " 218532350664SJames Smart "NVME port num %d, localP x%px, private " 218632350664SJames Smart "x%px, sg_seg %d\n", 218701649561SJames Smart localport->port_num, localport, 218801649561SJames Smart localport->private, 218901649561SJames Smart lpfc_nvme_template.max_sgl_segments); 219001649561SJames Smart 219101649561SJames Smart /* Private is our lport size declared in the template. */ 219201649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 219301649561SJames Smart vport->localport = localport; 219401649561SJames Smart lport->vport = vport; 219501649561SJames Smart vport->nvmei_support = 1; 21966b486ce9SDick Kennedy 21974b056682SJames Smart atomic_set(&lport->xmt_fcp_noxri, 0); 21984b056682SJames Smart atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 21994b056682SJames Smart atomic_set(&lport->xmt_fcp_qdepth, 0); 220044c2757bSJames Smart atomic_set(&lport->xmt_fcp_err, 0); 22014b056682SJames Smart atomic_set(&lport->xmt_fcp_wqerr, 0); 22024b056682SJames Smart atomic_set(&lport->xmt_fcp_abort, 0); 22034b056682SJames Smart atomic_set(&lport->xmt_ls_abort, 0); 22044b056682SJames Smart atomic_set(&lport->xmt_ls_err, 0); 22054b056682SJames Smart atomic_set(&lport->cmpl_fcp_xb, 0); 22064b056682SJames Smart atomic_set(&lport->cmpl_fcp_err, 0); 22074b056682SJames Smart atomic_set(&lport->cmpl_ls_xb, 0); 22084b056682SJames Smart atomic_set(&lport->cmpl_ls_err, 0); 22099aa09e98SJames Smart 221066a210ffSJames Smart atomic_set(&lport->fc4NvmeLsRequests, 0); 221166a210ffSJames Smart atomic_set(&lport->fc4NvmeLsCmpls, 0); 22126b486ce9SDick Kennedy } 22133ebd9b47SJames Smart 221401649561SJames Smart return ret; 221501649561SJames Smart } 221601649561SJames Smart 2217d964b3e5SBart Van Assche #if (IS_ENABLED(CONFIG_NVME_FC)) 2218add9d6beSJames Smart /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2219add9d6beSJames Smart * 2220add9d6beSJames Smart * The driver has to wait for the host nvme transport to callback 2221add9d6beSJames Smart * indicating the localport has successfully unregistered all 2222add9d6beSJames Smart * resources. Since this is an uninterruptible wait, loop every ten 2223add9d6beSJames Smart * seconds and print a message indicating no progress. 2224add9d6beSJames Smart * 2225add9d6beSJames Smart * An uninterruptible wait is used because of the risk of transport-to- 2226add9d6beSJames Smart * driver state mismatch. 2227add9d6beSJames Smart */ 22283999df75SBart Van Assche static void 2229add9d6beSJames Smart lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 22307961cba6SEwan D. Milne struct lpfc_nvme_lport *lport, 22317961cba6SEwan D. Milne struct completion *lport_unreg_cmp) 2232add9d6beSJames Smart { 2233add9d6beSJames Smart u32 wait_tmo; 22342ab70c21SJames Smart int ret, i, pending = 0; 22352ab70c21SJames Smart struct lpfc_sli_ring *pring; 22362ab70c21SJames Smart struct lpfc_hba *phba = vport->phba; 2237e7dab164SJames Smart struct lpfc_sli4_hdw_queue *qp; 2238e7dab164SJames Smart int abts_scsi, abts_nvme; 2239add9d6beSJames Smart 2240add9d6beSJames Smart /* Host transport has to clean up and confirm requiring an indefinite 2241add9d6beSJames Smart * wait. Print a message if a 10 second wait expires and renew the 2242add9d6beSJames Smart * wait. This is unexpected. 2243add9d6beSJames Smart */ 2244add9d6beSJames Smart wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2245add9d6beSJames Smart while (true) { 22467961cba6SEwan D. Milne ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2247add9d6beSJames Smart if (unlikely(!ret)) { 22482ab70c21SJames Smart pending = 0; 2249e7dab164SJames Smart abts_scsi = 0; 2250e7dab164SJames Smart abts_nvme = 0; 22512ab70c21SJames Smart for (i = 0; i < phba->cfg_hdw_queue; i++) { 2252e7dab164SJames Smart qp = &phba->sli4_hba.hdwq[i]; 225335ed9613SJames Smart if (!vport->localport || !qp || !qp->io_wq) 225425ac2c97SJames Smart return; 225525ac2c97SJames Smart 2256e7dab164SJames Smart pring = qp->io_wq->pring; 22572ab70c21SJames Smart if (!pring) 22582ab70c21SJames Smart continue; 22592ab70c21SJames Smart pending += pring->txcmplq_cnt; 2260e7dab164SJames Smart abts_scsi += qp->abts_scsi_io_bufs; 2261e7dab164SJames Smart abts_nvme += qp->abts_nvme_io_bufs; 22622ab70c21SJames Smart } 226335ed9613SJames Smart if (!vport->localport || 226435ed9613SJames Smart test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) || 226527c2bcf0SJustin Tee phba->link_state == LPFC_HBA_ERROR || 2266*e39811beSJustin Tee test_bit(FC_UNLOADING, &vport->load_flag)) 226725ac2c97SJames Smart return; 226825ac2c97SJames Smart 2269372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 227032350664SJames Smart "6176 Lport x%px Localport x%px wait " 2271e7dab164SJames Smart "timed out. Pending %d [%d:%d]. " 2272e7dab164SJames Smart "Renewing.\n", 2273e7dab164SJames Smart lport, vport->localport, pending, 2274e7dab164SJames Smart abts_scsi, abts_nvme); 2275add9d6beSJames Smart continue; 2276add9d6beSJames Smart } 2277add9d6beSJames Smart break; 2278add9d6beSJames Smart } 2279add9d6beSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 228032350664SJames Smart "6177 Lport x%px Localport x%px Complete Success\n", 2281add9d6beSJames Smart lport, vport->localport); 2282add9d6beSJames Smart } 2283d964b3e5SBart Van Assche #endif 2284add9d6beSJames Smart 228501649561SJames Smart /** 228601649561SJames Smart * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 22873e749170SLee Jones * @vport: pointer to a host virtual N_Port data structure 228801649561SJames Smart * 228901649561SJames Smart * This routine is invoked to destroy all lports bound to the phba. 229001649561SJames Smart * The lport memory was allocated by the nvme fc transport and is 229101649561SJames Smart * released there. This routine ensures all rports bound to the 229201649561SJames Smart * lport have been disconnected. 229301649561SJames Smart * 229401649561SJames Smart **/ 229501649561SJames Smart void 229601649561SJames Smart lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 229701649561SJames Smart { 22987d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 229901649561SJames Smart struct nvme_fc_local_port *localport; 230001649561SJames Smart struct lpfc_nvme_lport *lport; 230101649561SJames Smart int ret; 23027961cba6SEwan D. Milne DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 230301649561SJames Smart 230401649561SJames Smart if (vport->nvmei_support == 0) 230501649561SJames Smart return; 230601649561SJames Smart 230701649561SJames Smart localport = vport->localport; 230825ac2c97SJames Smart if (!localport) 230925ac2c97SJames Smart return; 231001649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 231101649561SJames Smart 231201649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 231332350664SJames Smart "6011 Destroying NVME localport x%px\n", 231401649561SJames Smart localport); 2315166d7211SJames Smart 231601649561SJames Smart /* lport's rport list is clear. Unregister 231701649561SJames Smart * lport and release resources. 231801649561SJames Smart */ 23197961cba6SEwan D. Milne lport->lport_unreg_cmp = &lport_unreg_cmp; 232001649561SJames Smart ret = nvme_fc_unregister_localport(localport); 2321add9d6beSJames Smart 2322add9d6beSJames Smart /* Wait for completion. This either blocks 2323add9d6beSJames Smart * indefinitely or succeeds 2324add9d6beSJames Smart */ 23257961cba6SEwan D. Milne lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 23267961cba6SEwan D. Milne vport->localport = NULL; 232701649561SJames Smart 232801649561SJames Smart /* Regardless of the unregister upcall response, clear 232901649561SJames Smart * nvmei_support. All rports are unregistered and the 233001649561SJames Smart * driver will clean up. 233101649561SJames Smart */ 233201649561SJames Smart vport->nvmei_support = 0; 233301649561SJames Smart if (ret == 0) { 233401649561SJames Smart lpfc_printf_vlog(vport, 233501649561SJames Smart KERN_INFO, LOG_NVME_DISC, 233601649561SJames Smart "6009 Unregistered lport Success\n"); 233701649561SJames Smart } else { 233801649561SJames Smart lpfc_printf_vlog(vport, 233901649561SJames Smart KERN_INFO, LOG_NVME_DISC, 234001649561SJames Smart "6010 Unregistered lport " 234101649561SJames Smart "Failed, status x%x\n", 234201649561SJames Smart ret); 234301649561SJames Smart } 2344166d7211SJames Smart #endif 234501649561SJames Smart } 234601649561SJames Smart 234701649561SJames Smart void 234801649561SJames Smart lpfc_nvme_update_localport(struct lpfc_vport *vport) 234901649561SJames Smart { 23504410a67aSJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 235101649561SJames Smart struct nvme_fc_local_port *localport; 235201649561SJames Smart struct lpfc_nvme_lport *lport; 235301649561SJames Smart 235401649561SJames Smart localport = vport->localport; 23554410a67aSJames Smart if (!localport) { 23564410a67aSJames Smart lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 23574410a67aSJames Smart "6710 Update NVME fail. No localport\n"); 23584410a67aSJames Smart return; 23594410a67aSJames Smart } 236001649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 23614410a67aSJames Smart if (!lport) { 23624410a67aSJames Smart lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 236332350664SJames Smart "6171 Update NVME fail. localP x%px, No lport\n", 23644410a67aSJames Smart localport); 23654410a67aSJames Smart return; 23664410a67aSJames Smart } 236701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 236832350664SJames Smart "6012 Update NVME lport x%px did x%x\n", 236901649561SJames Smart localport, vport->fc_myDID); 237001649561SJames Smart 237101649561SJames Smart localport->port_id = vport->fc_myDID; 237201649561SJames Smart if (localport->port_id == 0) 237301649561SJames Smart localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 237401649561SJames Smart else 237501649561SJames Smart localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 237601649561SJames Smart 237701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 237832350664SJames Smart "6030 bound lport x%px to DID x%06x\n", 237901649561SJames Smart lport, localport->port_id); 23804410a67aSJames Smart #endif 238101649561SJames Smart } 238201649561SJames Smart 238301649561SJames Smart int 238401649561SJames Smart lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 238501649561SJames Smart { 23867d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 238701649561SJames Smart int ret = 0; 238801649561SJames Smart struct nvme_fc_local_port *localport; 238901649561SJames Smart struct lpfc_nvme_lport *lport; 239001649561SJames Smart struct lpfc_nvme_rport *rport; 239101466024SJames Smart struct lpfc_nvme_rport *oldrport; 239201649561SJames Smart struct nvme_fc_remote_port *remote_port; 239301649561SJames Smart struct nvme_fc_port_info rpinfo; 239493a3922dSJames Smart struct lpfc_nodelist *prev_ndlp = NULL; 23951f546823SJames Smart struct fc_rport *srport = ndlp->rport; 239601649561SJames Smart 239701649561SJames Smart lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 239801649561SJames Smart "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 239901649561SJames Smart ndlp->nlp_DID, ndlp->nlp_type); 240001649561SJames Smart 240101649561SJames Smart localport = vport->localport; 2402bb6a8a2cSDick Kennedy if (!localport) 2403bb6a8a2cSDick Kennedy return 0; 2404bb6a8a2cSDick Kennedy 240501649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 240601649561SJames Smart 24077a06dcd3SJames Smart /* NVME rports are not preserved across devloss. 24087a06dcd3SJames Smart * Just register this instance. Note, rpinfo->dev_loss_tmo 24097a06dcd3SJames Smart * is left 0 to indicate accept transport defaults. The 24107a06dcd3SJames Smart * driver communicates port role capabilities consistent 24117a06dcd3SJames Smart * with the PRLI response data. 241201649561SJames Smart */ 24137a06dcd3SJames Smart memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 24147a06dcd3SJames Smart rpinfo.port_id = ndlp->nlp_DID; 241501649561SJames Smart if (ndlp->nlp_type & NLP_NVME_TARGET) 24167a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 241701649561SJames Smart if (ndlp->nlp_type & NLP_NVME_INITIATOR) 24187a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 241901649561SJames Smart 24207a06dcd3SJames Smart if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 24217a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 24227a06dcd3SJames Smart 24237a06dcd3SJames Smart rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 24247a06dcd3SJames Smart rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 24251f546823SJames Smart if (srport) 24261f546823SJames Smart rpinfo.dev_loss_tmo = srport->dev_loss_tmo; 24271f546823SJames Smart else 24281f546823SJames Smart rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo; 242901466024SJames Smart 2430c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 2431ba3d58a1SJames Smart 2432ba3d58a1SJames Smart /* If an oldrport exists, so does the ndlp reference. If not 2433ba3d58a1SJames Smart * a new reference is needed because either the node has never 2434ba3d58a1SJames Smart * been registered or it's been unregistered and getting deleted. 2435ba3d58a1SJames Smart */ 243601466024SJames Smart oldrport = lpfc_ndlp_get_nrport(ndlp); 243761184f17SJames Smart if (oldrport) { 243861184f17SJames Smart prev_ndlp = oldrport->ndlp; 2439c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 244061184f17SJames Smart } else { 2441c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 24424430f7fdSJames Smart if (!lpfc_nlp_get(ndlp)) { 24434430f7fdSJames Smart dev_warn(&vport->phba->pcidev->dev, 24444430f7fdSJames Smart "Warning - No node ref - exit register\n"); 24454430f7fdSJames Smart return 0; 24464430f7fdSJames Smart } 244761184f17SJames Smart } 24483fd78355SJames Smart 24497a06dcd3SJames Smart ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 24507a06dcd3SJames Smart if (!ret) { 24517a06dcd3SJames Smart /* If the ndlp already has an nrport, this is just 24527a06dcd3SJames Smart * a resume of the existing rport. Else this is a 24537a06dcd3SJames Smart * new rport. 24547a06dcd3SJames Smart */ 2455b15bd3e6SJames Smart /* Guard against an unregister/reregister 2456b15bd3e6SJames Smart * race that leaves the WAIT flag set. 2457b15bd3e6SJames Smart */ 2458c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 245906145683SJames Smart ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT; 2460e9b11083SJames Smart ndlp->fc4_xpt_flags |= NVME_XPT_REGD; 2461c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 24627a06dcd3SJames Smart rport = remote_port->private; 246301466024SJames Smart if (oldrport) { 24642b75d0f9SDick Kennedy 24653fd78355SJames Smart /* Sever the ndlp<->rport association 24663fd78355SJames Smart * before dropping the ndlp ref from 24673fd78355SJames Smart * register. 24682b75d0f9SDick Kennedy */ 2469c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 24702b75d0f9SDick Kennedy ndlp->nrport = NULL; 247106145683SJames Smart ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT; 2472c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 24732b75d0f9SDick Kennedy rport->ndlp = NULL; 24743fd78355SJames Smart rport->remoteport = NULL; 2475b04744ceSJames Smart 2476b04744ceSJames Smart /* Reference only removed if previous NDLP is no longer 2477b04744ceSJames Smart * active. It might be just a swap and removing the 2478b04744ceSJames Smart * reference would cause a premature cleanup. 2479b04744ceSJames Smart */ 2480b04744ceSJames Smart if (prev_ndlp && prev_ndlp != ndlp) { 2481307e3380SJames Smart if (!prev_ndlp->nrport) 2482b04744ceSJames Smart lpfc_nlp_put(prev_ndlp); 2483b04744ceSJames Smart } 24842b75d0f9SDick Kennedy } 24852b75d0f9SDick Kennedy 24862b75d0f9SDick Kennedy /* Clean bind the rport to the ndlp. */ 248701649561SJames Smart rport->remoteport = remote_port; 248801649561SJames Smart rport->lport = lport; 24893fd78355SJames Smart rport->ndlp = ndlp; 2490c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 249101649561SJames Smart ndlp->nrport = rport; 2492c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 249301649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, 249401649561SJames Smart LOG_NVME_DISC | LOG_NODE, 24956825b7bdSJames Smart "6022 Bind lport x%px to remoteport x%px " 24966825b7bdSJames Smart "rport x%px WWNN 0x%llx, " 24977a06dcd3SJames Smart "Rport WWPN 0x%llx DID " 249832350664SJames Smart "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 249993a3922dSJames Smart lport, remote_port, rport, 250001649561SJames Smart rpinfo.node_name, rpinfo.port_name, 25013fd78355SJames Smart rpinfo.port_id, rpinfo.port_role, 250293a3922dSJames Smart ndlp, prev_ndlp); 250301649561SJames Smart } else { 250401649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, 2505372c187bSDick Kennedy LOG_TRACE_EVENT, 250601649561SJames Smart "6031 RemotePort Registration failed " 250790cec07fSJustin Tee "err: %d, DID x%06x ref %u\n", 250890cec07fSJustin Tee ret, ndlp->nlp_DID, kref_read(&ndlp->kref)); 250990cec07fSJustin Tee lpfc_nlp_put(ndlp); 251001649561SJames Smart } 25117a06dcd3SJames Smart 251201649561SJames Smart return ret; 2513166d7211SJames Smart #else 2514166d7211SJames Smart return 0; 2515166d7211SJames Smart #endif 251601649561SJames Smart } 251701649561SJames Smart 25183e749170SLee Jones /* 25196f2589f4SJames Smart * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 25206f2589f4SJames Smart * 25216f2589f4SJames Smart * If the ndlp represents an NVME Target, that we are logged into, 25226f2589f4SJames Smart * ping the NVME FC Transport layer to initiate a device rescan 25236f2589f4SJames Smart * on this remote NPort. 25246f2589f4SJames Smart */ 25256f2589f4SJames Smart void 25266f2589f4SJames Smart lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 25276f2589f4SJames Smart { 25286f2589f4SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 2529a6d10f24SJames Smart struct lpfc_nvme_rport *nrport; 2530a6d10f24SJames Smart struct nvme_fc_remote_port *remoteport = NULL; 25316f2589f4SJames Smart 2532c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 2533a6d10f24SJames Smart nrport = lpfc_ndlp_get_nrport(ndlp); 2534a6d10f24SJames Smart if (nrport) 2535a6d10f24SJames Smart remoteport = nrport->remoteport; 2536c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 25376f2589f4SJames Smart 25386f2589f4SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 25396f2589f4SJames Smart "6170 Rescan NPort DID x%06x type x%x " 2540a6d10f24SJames Smart "state x%x nrport x%px remoteport x%px\n", 2541a6d10f24SJames Smart ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2542a6d10f24SJames Smart nrport, remoteport); 2543a6d10f24SJames Smart 2544a6d10f24SJames Smart if (!nrport || !remoteport) 2545a6d10f24SJames Smart goto rescan_exit; 25466f2589f4SJames Smart 2547b83a8c21SJames Smart /* Rescan an NVME target in MAPPED state with DISCOVERY role set */ 25486f2589f4SJames Smart if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 25496f2589f4SJames Smart ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 25506f2589f4SJames Smart nvme_fc_rescan_remoteport(remoteport); 25516f2589f4SJames Smart 2552b83a8c21SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 25536f2589f4SJames Smart "6172 NVME rescanned DID x%06x " 25546f2589f4SJames Smart "port_state x%x\n", 25556f2589f4SJames Smart ndlp->nlp_DID, remoteport->port_state); 25566f2589f4SJames Smart } 25576f2589f4SJames Smart return; 2558a6d10f24SJames Smart rescan_exit: 2559a6d10f24SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2560a6d10f24SJames Smart "6169 Skip NVME Rport Rescan, NVME remoteport " 2561a6d10f24SJames Smart "unregistered\n"); 25626f2589f4SJames Smart #endif 25636f2589f4SJames Smart } 25646f2589f4SJames Smart 256501649561SJames Smart /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 256601649561SJames Smart * 256701649561SJames Smart * There is no notion of Devloss or rport recovery from the current 256801649561SJames Smart * nvme_transport perspective. Loss of an rport just means IO cannot 256901649561SJames Smart * be sent and recovery is completely up to the initator. 257001649561SJames Smart * For now, the driver just unbinds the DID and port_role so that 2571dae40be7SJustin Tee * no further IO can be issued. 257201649561SJames Smart */ 257301649561SJames Smart void 257401649561SJames Smart lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 257501649561SJames Smart { 25767d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 257701649561SJames Smart int ret; 257801649561SJames Smart struct nvme_fc_local_port *localport; 257901649561SJames Smart struct lpfc_nvme_lport *lport; 258001649561SJames Smart struct lpfc_nvme_rport *rport; 25819e210178SJames Smart struct nvme_fc_remote_port *remoteport = NULL; 258201649561SJames Smart 258301649561SJames Smart localport = vport->localport; 258401649561SJames Smart 258501649561SJames Smart /* This is fundamental error. The localport is always 258601649561SJames Smart * available until driver unload. Just exit. 258701649561SJames Smart */ 258801649561SJames Smart if (!localport) 258901649561SJames Smart return; 259001649561SJames Smart 259101649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 259201649561SJames Smart if (!lport) 259301649561SJames Smart goto input_err; 259401649561SJames Smart 2595c6adba15SJames Smart spin_lock_irq(&ndlp->lock); 259601466024SJames Smart rport = lpfc_ndlp_get_nrport(ndlp); 25979e210178SJames Smart if (rport) 25989e210178SJames Smart remoteport = rport->remoteport; 2599c6adba15SJames Smart spin_unlock_irq(&ndlp->lock); 26009e210178SJames Smart if (!remoteport) 260101649561SJames Smart goto input_err; 260201649561SJames Smart 260301649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 260432350664SJames Smart "6033 Unreg nvme remoteport x%px, portname x%llx, " 2605e9b11083SJames Smart "port_id x%06x, portstate x%x port type x%x " 2606e9b11083SJames Smart "refcnt %d\n", 260701649561SJames Smart remoteport, remoteport->port_name, 260801649561SJames Smart remoteport->port_id, remoteport->port_state, 2609e9b11083SJames Smart ndlp->nlp_type, kref_read(&ndlp->kref)); 261001649561SJames Smart 261101649561SJames Smart /* Sanity check ndlp type. Only call for NVME ports. Don't 261201649561SJames Smart * clear any rport state until the transport calls back. 261301649561SJames Smart */ 26143b5bde69SJames Smart 26153b5bde69SJames Smart if (ndlp->nlp_type & NLP_NVME_TARGET) { 26167a06dcd3SJames Smart /* No concern about the role change on the nvme remoteport. 26177a06dcd3SJames Smart * The transport will update it. 26187a06dcd3SJames Smart */ 2619e9b11083SJames Smart spin_lock_irq(&vport->phba->hbalock); 262006145683SJames Smart ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT; 2621e9b11083SJames Smart spin_unlock_irq(&vport->phba->hbalock); 26227438273fSJames Smart 26237438273fSJames Smart /* Don't let the host nvme transport keep sending keep-alives 26247438273fSJames Smart * on this remoteport. Vport is unloading, no recovery. The 26257438273fSJames Smart * return values is ignored. The upcall is a courtesy to the 26267438273fSJames Smart * transport. 26277438273fSJames Smart */ 2628*e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag) || 262927c2bcf0SJustin Tee unlikely(vport->phba->link_state == LPFC_HBA_ERROR)) 26307438273fSJames Smart (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 26317438273fSJames Smart 263201649561SJames Smart ret = nvme_fc_unregister_remoteport(remoteport); 2633e9b11083SJames Smart 2634e9b11083SJames Smart /* The driver no longer knows if the nrport memory is valid. 2635e9b11083SJames Smart * because the controller teardown process has begun and 2636e9b11083SJames Smart * is asynchronous. Break the binding in the ndlp. Also 2637e9b11083SJames Smart * remove the register ndlp reference to setup node release. 2638e9b11083SJames Smart */ 2639e9b11083SJames Smart ndlp->nrport = NULL; 26403fd78355SJames Smart lpfc_nlp_put(ndlp); 2641e9b11083SJames Smart if (ret != 0) { 2642372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 264301649561SJames Smart "6167 NVME unregister failed %d " 264401649561SJames Smart "port_state x%x\n", 264501649561SJames Smart ret, remoteport->port_state); 2646dae40be7SJustin Tee 2647*e39811beSJustin Tee if (test_bit(FC_UNLOADING, &vport->load_flag)) { 2648dae40be7SJustin Tee /* Only 1 thread can drop the initial node 2649dae40be7SJustin Tee * reference. Check if another thread has set 2650dae40be7SJustin Tee * NLP_DROPPED. 2651dae40be7SJustin Tee */ 2652dae40be7SJustin Tee spin_lock_irq(&ndlp->lock); 2653dae40be7SJustin Tee if (!(ndlp->nlp_flag & NLP_DROPPED)) { 2654dae40be7SJustin Tee ndlp->nlp_flag |= NLP_DROPPED; 2655dae40be7SJustin Tee spin_unlock_irq(&ndlp->lock); 2656dae40be7SJustin Tee lpfc_nlp_put(ndlp); 2657dae40be7SJustin Tee return; 2658dae40be7SJustin Tee } 2659dae40be7SJustin Tee spin_unlock_irq(&ndlp->lock); 2660dae40be7SJustin Tee } 26613fd78355SJames Smart } 266201649561SJames Smart } 266301649561SJames Smart return; 266401649561SJames Smart 266501649561SJames Smart input_err: 2666166d7211SJames Smart #endif 2667372c187bSDick Kennedy lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, 266832350664SJames Smart "6168 State error: lport x%px, rport x%px FCID x%06x\n", 266901649561SJames Smart vport->localport, ndlp->rport, ndlp->nlp_DID); 267001649561SJames Smart } 2671318083adSJames Smart 2672318083adSJames Smart /** 267325ac2c97SJames Smart * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort 267425ac2c97SJames Smart * @phba: pointer to lpfc hba data structure. 267525ac2c97SJames Smart * @lpfc_ncmd: The nvme job structure for the request being aborted. 267625ac2c97SJames Smart * 267725ac2c97SJames Smart * This routine is invoked by the worker thread to process a SLI4 fast-path 267825ac2c97SJames Smart * NVME aborted xri. Aborted NVME IO commands are completed to the transport 267925ac2c97SJames Smart * here. 268025ac2c97SJames Smart **/ 268125ac2c97SJames Smart void 268225ac2c97SJames Smart lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba, 268325ac2c97SJames Smart struct lpfc_io_buf *lpfc_ncmd) 268425ac2c97SJames Smart { 268525ac2c97SJames Smart struct nvmefc_fcp_req *nvme_cmd = NULL; 268625ac2c97SJames Smart 268725ac2c97SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 268825ac2c97SJames Smart "6533 %s nvme_cmd %p tag x%x abort complete and " 268925ac2c97SJames Smart "xri released\n", __func__, 269025ac2c97SJames Smart lpfc_ncmd->nvmeCmd, 269125ac2c97SJames Smart lpfc_ncmd->cur_iocbq.iotag); 269225ac2c97SJames Smart 269325ac2c97SJames Smart /* Aborted NVME commands are required to not complete 269425ac2c97SJames Smart * before the abort exchange command fully completes. 269525ac2c97SJames Smart * Once completed, it is available via the put list. 269625ac2c97SJames Smart */ 269725ac2c97SJames Smart if (lpfc_ncmd->nvmeCmd) { 269825ac2c97SJames Smart nvme_cmd = lpfc_ncmd->nvmeCmd; 269925ac2c97SJames Smart nvme_cmd->transferred_length = 0; 270025ac2c97SJames Smart nvme_cmd->rcv_rsplen = 0; 270125ac2c97SJames Smart nvme_cmd->status = NVME_SC_INTERNAL; 270225ac2c97SJames Smart nvme_cmd->done(nvme_cmd); 270325ac2c97SJames Smart lpfc_ncmd->nvmeCmd = NULL; 270425ac2c97SJames Smart } 270525ac2c97SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 270625ac2c97SJames Smart } 270725ac2c97SJames Smart 270825ac2c97SJames Smart /** 2709318083adSJames Smart * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2710318083adSJames Smart * @phba: pointer to lpfc hba data structure. 2711318083adSJames Smart * @axri: pointer to the fcp xri abort wcqe structure. 2712c00f62e6SJames Smart * @lpfc_ncmd: The nvme job structure for the request being aborted. 2713318083adSJames Smart * 2714318083adSJames Smart * This routine is invoked by the worker thread to process a SLI4 fast-path 2715952c303bSDick Kennedy * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2716952c303bSDick Kennedy * here. 2717318083adSJames Smart **/ 2718318083adSJames Smart void 2719318083adSJames Smart lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2720c00f62e6SJames Smart struct sli4_wcqe_xri_aborted *axri, 2721c00f62e6SJames Smart struct lpfc_io_buf *lpfc_ncmd) 2722318083adSJames Smart { 2723318083adSJames Smart uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2724952c303bSDick Kennedy struct nvmefc_fcp_req *nvme_cmd = NULL; 2725c00f62e6SJames Smart struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2726318083adSJames Smart 2727318083adSJames Smart 2728952c303bSDick Kennedy if (ndlp) 2729318083adSJames Smart lpfc_sli4_abts_err_handler(phba, ndlp, axri); 273086c67379SJames Smart 273186c67379SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2732c00f62e6SJames Smart "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2733c00f62e6SJames Smart "xri released\n", 2734952c303bSDick Kennedy lpfc_ncmd->nvmeCmd, xri, 2735952c303bSDick Kennedy lpfc_ncmd->cur_iocbq.iotag); 273686c67379SJames Smart 2737952c303bSDick Kennedy /* Aborted NVME commands are required to not complete 2738952c303bSDick Kennedy * before the abort exchange command fully completes. 2739952c303bSDick Kennedy * Once completed, it is available via the put list. 2740952c303bSDick Kennedy */ 27413fd78355SJames Smart if (lpfc_ncmd->nvmeCmd) { 2742952c303bSDick Kennedy nvme_cmd = lpfc_ncmd->nvmeCmd; 2743952c303bSDick Kennedy nvme_cmd->done(nvme_cmd); 27443fd78355SJames Smart lpfc_ncmd->nvmeCmd = NULL; 27453fd78355SJames Smart } 2746318083adSJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 2747318083adSJames Smart } 2748c3725bdcSJames Smart 2749c3725bdcSJames Smart /** 2750c3725bdcSJames Smart * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2751c3725bdcSJames Smart * @phba: Pointer to HBA context object. 2752c3725bdcSJames Smart * 2753c3725bdcSJames Smart * This function flushes all wqes in the nvme rings and frees all resources 2754c3725bdcSJames Smart * in the txcmplq. This function does not issue abort wqes for the IO 2755c3725bdcSJames Smart * commands in txcmplq, they will just be returned with 2756c3725bdcSJames Smart * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2757c3725bdcSJames Smart * slot has been permanently disabled. 2758c3725bdcSJames Smart **/ 2759c3725bdcSJames Smart void 2760c3725bdcSJames Smart lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2761c3725bdcSJames Smart { 2762c3725bdcSJames Smart struct lpfc_sli_ring *pring; 2763c3725bdcSJames Smart u32 i, wait_cnt = 0; 2764c3725bdcSJames Smart 2765cdb42becSJames Smart if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2766c3725bdcSJames Smart return; 2767c3725bdcSJames Smart 2768c00f62e6SJames Smart /* Cycle through all IO rings and make sure all outstanding 2769c3725bdcSJames Smart * WQEs have been removed from the txcmplqs. 2770c3725bdcSJames Smart */ 2771cdb42becSJames Smart for (i = 0; i < phba->cfg_hdw_queue; i++) { 2772c00f62e6SJames Smart if (!phba->sli4_hba.hdwq[i].io_wq) 27736a828b0fSJames Smart continue; 2774c00f62e6SJames Smart pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2775c3725bdcSJames Smart 2776d580c613SJames Smart if (!pring) 2777d580c613SJames Smart continue; 2778d580c613SJames Smart 2779c3725bdcSJames Smart /* Retrieve everything on the txcmplq */ 2780c3725bdcSJames Smart while (!list_empty(&pring->txcmplq)) { 2781c3725bdcSJames Smart msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2782c3725bdcSJames Smart wait_cnt++; 2783c3725bdcSJames Smart 2784c3725bdcSJames Smart /* The sleep is 10mS. Every ten seconds, 2785c3725bdcSJames Smart * dump a message. Something is wrong. 2786c3725bdcSJames Smart */ 2787c3725bdcSJames Smart if ((wait_cnt % 1000) == 0) { 2788372c187bSDick Kennedy lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, 2789c3725bdcSJames Smart "6178 NVME IO not empty, " 2790c3725bdcSJames Smart "cnt %d\n", wait_cnt); 2791c3725bdcSJames Smart } 2792c3725bdcSJames Smart } 2793c3725bdcSJames Smart } 2794a22d73b6SJames Smart 2795a22d73b6SJames Smart /* Make sure HBA is alive */ 2796a22d73b6SJames Smart lpfc_issue_hb_tmo(phba); 2797a22d73b6SJames Smart 2798c3725bdcSJames Smart } 279984f2ddf8SJames Smart 280084f2ddf8SJames Smart void 28019ec58ec7SJames Smart lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 28029ec58ec7SJames Smart uint32_t stat, uint32_t param) 280384f2ddf8SJames Smart { 280484f2ddf8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 280584f2ddf8SJames Smart struct lpfc_io_buf *lpfc_ncmd; 280684f2ddf8SJames Smart struct nvmefc_fcp_req *nCmd; 28079ec58ec7SJames Smart struct lpfc_wcqe_complete wcqe; 28089ec58ec7SJames Smart struct lpfc_wcqe_complete *wcqep = &wcqe; 280984f2ddf8SJames Smart 2810d51cf5bdSJames Smart lpfc_ncmd = pwqeIn->io_buf; 28119ec58ec7SJames Smart if (!lpfc_ncmd) { 281284f2ddf8SJames Smart lpfc_sli_release_iocbq(phba, pwqeIn); 281384f2ddf8SJames Smart return; 281484f2ddf8SJames Smart } 281584f2ddf8SJames Smart /* For abort iocb just return, IO iocb will do a done call */ 281684f2ddf8SJames Smart if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 281784f2ddf8SJames Smart CMD_ABORT_XRI_CX) { 281884f2ddf8SJames Smart lpfc_sli_release_iocbq(phba, pwqeIn); 281984f2ddf8SJames Smart return; 282084f2ddf8SJames Smart } 282184f2ddf8SJames Smart 282284f2ddf8SJames Smart spin_lock(&lpfc_ncmd->buf_lock); 28239ec58ec7SJames Smart nCmd = lpfc_ncmd->nvmeCmd; 28249ec58ec7SJames Smart if (!nCmd) { 282584f2ddf8SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 282684f2ddf8SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 282784f2ddf8SJames Smart return; 282884f2ddf8SJames Smart } 28299ec58ec7SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 283084f2ddf8SJames Smart 283184f2ddf8SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 283284f2ddf8SJames Smart "6194 NVME Cancel xri %x\n", 283384f2ddf8SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag); 283484f2ddf8SJames Smart 28359ec58ec7SJames Smart wcqep->word0 = 0; 28369ec58ec7SJames Smart bf_set(lpfc_wcqe_c_status, wcqep, stat); 28379ec58ec7SJames Smart wcqep->parameter = param; 28384ecc9b02SJames Smart wcqep->total_data_placed = 0; 28399ec58ec7SJames Smart wcqep->word3 = 0; /* xb is 0 */ 284084f2ddf8SJames Smart 284184f2ddf8SJames Smart /* Call release with XB=1 to queue the IO into the abort list. */ 28429ec58ec7SJames Smart if (phba->sli.sli_flag & LPFC_SLI_ACTIVE) 28439ec58ec7SJames Smart bf_set(lpfc_wcqe_c_xb, wcqep, 1); 28449ec58ec7SJames Smart 2845a680a929SJames Smart memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep)); 2846a680a929SJames Smart (pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn); 284784f2ddf8SJames Smart #endif 284884f2ddf8SJames Smart } 2849