101649561SJames Smart /******************************************************************* 201649561SJames Smart * This file is part of the Emulex Linux Device Driver for * 301649561SJames Smart * Fibre Channel Host Bus Adapters. * 40d041215SJames Smart * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 53e21d1cbSJames Smart * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 601649561SJames Smart * Copyright (C) 2004-2016 Emulex. All rights reserved. * 701649561SJames Smart * EMULEX and SLI are trademarks of Emulex. * 8d080abe0SJames Smart * www.broadcom.com * 901649561SJames Smart * Portions Copyright (C) 2004-2005 Christoph Hellwig * 1001649561SJames Smart * * 1101649561SJames Smart * This program is free software; you can redistribute it and/or * 1201649561SJames Smart * modify it under the terms of version 2 of the GNU General * 1301649561SJames Smart * Public License as published by the Free Software Foundation. * 1401649561SJames Smart * This program is distributed in the hope that it will be useful. * 1501649561SJames Smart * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 1601649561SJames Smart * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 1701649561SJames Smart * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 1801649561SJames Smart * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 1901649561SJames Smart * TO BE LEGALLY INVALID. See the GNU General Public License for * 2001649561SJames Smart * more details, a copy of which can be found in the file COPYING * 2101649561SJames Smart * included with this package. * 2201649561SJames Smart ********************************************************************/ 2301649561SJames Smart #include <linux/pci.h> 2401649561SJames Smart #include <linux/slab.h> 2501649561SJames Smart #include <linux/interrupt.h> 2601649561SJames Smart #include <linux/delay.h> 2701649561SJames Smart #include <asm/unaligned.h> 2801649561SJames Smart #include <linux/crc-t10dif.h> 2901649561SJames Smart #include <net/checksum.h> 3001649561SJames Smart 3101649561SJames Smart #include <scsi/scsi.h> 3201649561SJames Smart #include <scsi/scsi_device.h> 3301649561SJames Smart #include <scsi/scsi_eh.h> 3401649561SJames Smart #include <scsi/scsi_host.h> 3501649561SJames Smart #include <scsi/scsi_tcq.h> 3601649561SJames Smart #include <scsi/scsi_transport_fc.h> 3701649561SJames Smart #include <scsi/fc/fc_fs.h> 3801649561SJames Smart 3901649561SJames Smart #include "lpfc_version.h" 4001649561SJames Smart #include "lpfc_hw4.h" 4101649561SJames Smart #include "lpfc_hw.h" 4201649561SJames Smart #include "lpfc_sli.h" 4301649561SJames Smart #include "lpfc_sli4.h" 4401649561SJames Smart #include "lpfc_nl.h" 4501649561SJames Smart #include "lpfc_disc.h" 4601649561SJames Smart #include "lpfc.h" 4701649561SJames Smart #include "lpfc_nvme.h" 4801649561SJames Smart #include "lpfc_scsi.h" 4901649561SJames Smart #include "lpfc_logmsg.h" 5001649561SJames Smart #include "lpfc_crtn.h" 5101649561SJames Smart #include "lpfc_vport.h" 52bd2cdd5eSJames Smart #include "lpfc_debugfs.h" 5301649561SJames Smart 5401649561SJames Smart /* NVME initiator-based functions */ 5501649561SJames Smart 56c490850aSJames Smart static struct lpfc_io_buf * 57cf1a1d3eSJames Smart lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 585e5b511dSJames Smart int idx, int expedite); 5901649561SJames Smart 6001649561SJames Smart static void 61c490850aSJames Smart lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 6201649561SJames Smart 6381e6a637SJames Smart static struct nvme_fc_port_template lpfc_nvme_template; 6401649561SJames Smart 65fab2e466SColin Ian King static union lpfc_wqe128 lpfc_iread_cmd_template; 66fab2e466SColin Ian King static union lpfc_wqe128 lpfc_iwrite_cmd_template; 67fab2e466SColin Ian King static union lpfc_wqe128 lpfc_icmnd_cmd_template; 685fd11085SJames Smart 695fd11085SJames Smart /* Setup WQE templates for NVME IOs */ 705fd11085SJames Smart void 71fab2e466SColin Ian King lpfc_nvme_cmd_template(void) 725fd11085SJames Smart { 735fd11085SJames Smart union lpfc_wqe128 *wqe; 745fd11085SJames Smart 755fd11085SJames Smart /* IREAD template */ 765fd11085SJames Smart wqe = &lpfc_iread_cmd_template; 775fd11085SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe128)); 785fd11085SJames Smart 795fd11085SJames Smart /* Word 0, 1, 2 - BDE is variable */ 805fd11085SJames Smart 815fd11085SJames Smart /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 825fd11085SJames Smart 835fd11085SJames Smart /* Word 4 - total_xfer_len is variable */ 845fd11085SJames Smart 855fd11085SJames Smart /* Word 5 - is zero */ 865fd11085SJames Smart 875fd11085SJames Smart /* Word 6 - ctxt_tag, xri_tag is variable */ 885fd11085SJames Smart 895fd11085SJames Smart /* Word 7 */ 905fd11085SJames Smart bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); 915fd11085SJames Smart bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); 925fd11085SJames Smart bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); 935fd11085SJames Smart bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); 945fd11085SJames Smart 955fd11085SJames Smart /* Word 8 - abort_tag is variable */ 965fd11085SJames Smart 975fd11085SJames Smart /* Word 9 - reqtag is variable */ 985fd11085SJames Smart 995fd11085SJames Smart /* Word 10 - dbde, wqes is variable */ 1005fd11085SJames Smart bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 1015fd11085SJames Smart bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); 1025fd11085SJames Smart bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); 1035fd11085SJames Smart bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); 1045fd11085SJames Smart bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); 1055fd11085SJames Smart bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); 1065fd11085SJames Smart 1075fd11085SJames Smart /* Word 11 - pbde is variable */ 1085fd11085SJames Smart bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); 1095fd11085SJames Smart bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 1105fd11085SJames Smart bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); 1115fd11085SJames Smart 1125fd11085SJames Smart /* Word 12 - is zero */ 1135fd11085SJames Smart 1145fd11085SJames Smart /* Word 13, 14, 15 - PBDE is variable */ 1155fd11085SJames Smart 1165fd11085SJames Smart /* IWRITE template */ 1175fd11085SJames Smart wqe = &lpfc_iwrite_cmd_template; 1185fd11085SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe128)); 1195fd11085SJames Smart 1205fd11085SJames Smart /* Word 0, 1, 2 - BDE is variable */ 1215fd11085SJames Smart 1225fd11085SJames Smart /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 1235fd11085SJames Smart 1245fd11085SJames Smart /* Word 4 - total_xfer_len is variable */ 1255fd11085SJames Smart 1265fd11085SJames Smart /* Word 5 - initial_xfer_len is variable */ 1275fd11085SJames Smart 1285fd11085SJames Smart /* Word 6 - ctxt_tag, xri_tag is variable */ 1295fd11085SJames Smart 1305fd11085SJames Smart /* Word 7 */ 1315fd11085SJames Smart bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); 1325fd11085SJames Smart bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); 1335fd11085SJames Smart bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); 1345fd11085SJames Smart bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); 1355fd11085SJames Smart 1365fd11085SJames Smart /* Word 8 - abort_tag is variable */ 1375fd11085SJames Smart 1385fd11085SJames Smart /* Word 9 - reqtag is variable */ 1395fd11085SJames Smart 1405fd11085SJames Smart /* Word 10 - dbde, wqes is variable */ 1415fd11085SJames Smart bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 1425fd11085SJames Smart bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); 1435fd11085SJames Smart bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); 1445fd11085SJames Smart bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); 1455fd11085SJames Smart bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); 1465fd11085SJames Smart bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 1475fd11085SJames Smart 1485fd11085SJames Smart /* Word 11 - pbde is variable */ 1495fd11085SJames Smart bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); 1505fd11085SJames Smart bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 1515fd11085SJames Smart bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); 1525fd11085SJames Smart 1535fd11085SJames Smart /* Word 12 - is zero */ 1545fd11085SJames Smart 1555fd11085SJames Smart /* Word 13, 14, 15 - PBDE is variable */ 1565fd11085SJames Smart 1575fd11085SJames Smart /* ICMND template */ 1585fd11085SJames Smart wqe = &lpfc_icmnd_cmd_template; 1595fd11085SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe128)); 1605fd11085SJames Smart 1615fd11085SJames Smart /* Word 0, 1, 2 - BDE is variable */ 1625fd11085SJames Smart 1635fd11085SJames Smart /* Word 3 - payload_offset_len is variable */ 1645fd11085SJames Smart 1655fd11085SJames Smart /* Word 4, 5 - is zero */ 1665fd11085SJames Smart 1675fd11085SJames Smart /* Word 6 - ctxt_tag, xri_tag is variable */ 1685fd11085SJames Smart 1695fd11085SJames Smart /* Word 7 */ 1705fd11085SJames Smart bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); 1715fd11085SJames Smart bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); 1725fd11085SJames Smart bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); 1735fd11085SJames Smart bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); 1745fd11085SJames Smart 1755fd11085SJames Smart /* Word 8 - abort_tag is variable */ 1765fd11085SJames Smart 1775fd11085SJames Smart /* Word 9 - reqtag is variable */ 1785fd11085SJames Smart 1795fd11085SJames Smart /* Word 10 - dbde, wqes is variable */ 1805fd11085SJames Smart bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 1815fd11085SJames Smart bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 1825fd11085SJames Smart bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); 1835fd11085SJames Smart bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); 1845fd11085SJames Smart bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); 1855fd11085SJames Smart bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 1865fd11085SJames Smart 1875fd11085SJames Smart /* Word 11 */ 1885fd11085SJames Smart bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); 1895fd11085SJames Smart bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 1905fd11085SJames Smart bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); 1915fd11085SJames Smart 1925fd11085SJames Smart /* Word 12, 13, 14, 15 - is zero */ 1935fd11085SJames Smart } 1945fd11085SJames Smart 19501649561SJames Smart /** 19651f8e43eSJames Smart * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry. 19751f8e43eSJames Smart * @pwqeq: Pointer to command iocb. 19851f8e43eSJames Smart * @xritag: Tag that uniqely identifies the local exchange resource. 19951f8e43eSJames Smart * @opt: Option bits - 20051f8e43eSJames Smart * bit 0 = inhibit sending abts on the link 20151f8e43eSJames Smart * 20251f8e43eSJames Smart * This function is called with hbalock held. 20351f8e43eSJames Smart **/ 20451f8e43eSJames Smart void 20551f8e43eSJames Smart lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt) 20651f8e43eSJames Smart { 20751f8e43eSJames Smart union lpfc_wqe128 *wqe = &pwqeq->wqe; 20851f8e43eSJames Smart 20951f8e43eSJames Smart /* WQEs are reused. Clear stale data and set key fields to 21051f8e43eSJames Smart * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 21151f8e43eSJames Smart */ 21251f8e43eSJames Smart memset(wqe, 0, sizeof(*wqe)); 21351f8e43eSJames Smart 21451f8e43eSJames Smart if (opt & INHIBIT_ABORT) 21551f8e43eSJames Smart bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); 21651f8e43eSJames Smart /* Abort specified xri tag, with the mask deliberately zeroed */ 21751f8e43eSJames Smart bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); 21851f8e43eSJames Smart 21951f8e43eSJames Smart bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 22051f8e43eSJames Smart 22151f8e43eSJames Smart /* Abort the IO associated with this outstanding exchange ID. */ 22251f8e43eSJames Smart wqe->abort_cmd.wqe_com.abort_tag = xritag; 22351f8e43eSJames Smart 22451f8e43eSJames Smart /* iotag for the wqe completion. */ 22551f8e43eSJames Smart bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag); 22651f8e43eSJames Smart 22751f8e43eSJames Smart bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); 22851f8e43eSJames Smart bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 22951f8e43eSJames Smart 23051f8e43eSJames Smart bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND); 23151f8e43eSJames Smart bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1); 23251f8e43eSJames Smart bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 23351f8e43eSJames Smart } 23451f8e43eSJames Smart 23551f8e43eSJames Smart /** 23601649561SJames Smart * lpfc_nvme_create_queue - 23701649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 23801649561SJames Smart * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 23901649561SJames Smart * @handle: An opaque driver handle used in follow-up calls. 24001649561SJames Smart * 24101649561SJames Smart * Driver registers this routine to preallocate and initialize any 24201649561SJames Smart * internal data structures to bind the @qidx to its internal IO queues. 24301649561SJames Smart * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 24401649561SJames Smart * 24501649561SJames Smart * Return value : 24601649561SJames Smart * 0 - Success 24701649561SJames Smart * -EINVAL - Unsupported input value. 24801649561SJames Smart * -ENOMEM - Could not alloc necessary memory 24901649561SJames Smart **/ 25001649561SJames Smart static int 25101649561SJames Smart lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 25201649561SJames Smart unsigned int qidx, u16 qsize, 25301649561SJames Smart void **handle) 25401649561SJames Smart { 25501649561SJames Smart struct lpfc_nvme_lport *lport; 25601649561SJames Smart struct lpfc_vport *vport; 25701649561SJames Smart struct lpfc_nvme_qhandle *qhandle; 25801649561SJames Smart char *str; 25901649561SJames Smart 260c3725bdcSJames Smart if (!pnvme_lport->private) 261c3725bdcSJames Smart return -ENOMEM; 262c3725bdcSJames Smart 26301649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 26401649561SJames Smart vport = lport->vport; 26501649561SJames Smart qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 26601649561SJames Smart if (qhandle == NULL) 26701649561SJames Smart return -ENOMEM; 26801649561SJames Smart 269d6d189ceSBart Van Assche qhandle->cpu_id = raw_smp_processor_id(); 27001649561SJames Smart qhandle->qidx = qidx; 27101649561SJames Smart /* 27201649561SJames Smart * NVME qidx == 0 is the admin queue, so both admin queue 27301649561SJames Smart * and first IO queue will use MSI-X vector and associated 27401649561SJames Smart * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 27501649561SJames Smart */ 27601649561SJames Smart if (qidx) { 27701649561SJames Smart str = "IO "; /* IO queue */ 27801649561SJames Smart qhandle->index = ((qidx - 1) % 2796a828b0fSJames Smart lpfc_nvme_template.max_hw_queues); 28001649561SJames Smart } else { 28101649561SJames Smart str = "ADM"; /* Admin queue */ 28201649561SJames Smart qhandle->index = qidx; 28301649561SJames Smart } 28401649561SJames Smart 285d58734f0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 28601649561SJames Smart "6073 Binding %s HdwQueue %d (cpu %d) to " 28732350664SJames Smart "hdw_queue %d qhandle x%px\n", str, 28801649561SJames Smart qidx, qhandle->cpu_id, qhandle->index, qhandle); 28901649561SJames Smart *handle = (void *)qhandle; 29001649561SJames Smart return 0; 29101649561SJames Smart } 29201649561SJames Smart 29301649561SJames Smart /** 29401649561SJames Smart * lpfc_nvme_delete_queue - 29501649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 29601649561SJames Smart * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 29701649561SJames Smart * @handle: An opaque driver handle from lpfc_nvme_create_queue 29801649561SJames Smart * 29901649561SJames Smart * Driver registers this routine to free 30001649561SJames Smart * any internal data structures to bind the @qidx to its internal 30101649561SJames Smart * IO queues. 30201649561SJames Smart * 30301649561SJames Smart * Return value : 30401649561SJames Smart * 0 - Success 30501649561SJames Smart * TODO: What are the failure codes. 30601649561SJames Smart **/ 30701649561SJames Smart static void 30801649561SJames Smart lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 30901649561SJames Smart unsigned int qidx, 31001649561SJames Smart void *handle) 31101649561SJames Smart { 31201649561SJames Smart struct lpfc_nvme_lport *lport; 31301649561SJames Smart struct lpfc_vport *vport; 31401649561SJames Smart 315c3725bdcSJames Smart if (!pnvme_lport->private) 316c3725bdcSJames Smart return; 317c3725bdcSJames Smart 31801649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 31901649561SJames Smart vport = lport->vport; 32001649561SJames Smart 32101649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 32232350664SJames Smart "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 32301649561SJames Smart lport, qidx, handle); 32401649561SJames Smart kfree(handle); 32501649561SJames Smart } 32601649561SJames Smart 32701649561SJames Smart static void 32801649561SJames Smart lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 32901649561SJames Smart { 33001649561SJames Smart struct lpfc_nvme_lport *lport = localport->private; 33101649561SJames Smart 332add9d6beSJames Smart lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 33332350664SJames Smart "6173 localport x%px delete complete\n", 334add9d6beSJames Smart lport); 335add9d6beSJames Smart 33601649561SJames Smart /* release any threads waiting for the unreg to complete */ 3377961cba6SEwan D. Milne if (lport->vport->localport) 3387961cba6SEwan D. Milne complete(lport->lport_unreg_cmp); 33901649561SJames Smart } 34001649561SJames Smart 34101649561SJames Smart /* lpfc_nvme_remoteport_delete 34201649561SJames Smart * 34301649561SJames Smart * @remoteport: Pointer to an nvme transport remoteport instance. 34401649561SJames Smart * 34501649561SJames Smart * This is a template downcall. NVME transport calls this function 34601649561SJames Smart * when it has completed the unregistration of a previously 34701649561SJames Smart * registered remoteport. 34801649561SJames Smart * 34901649561SJames Smart * Return value : 35001649561SJames Smart * None 35101649561SJames Smart */ 3523999df75SBart Van Assche static void 35301649561SJames Smart lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 35401649561SJames Smart { 35501649561SJames Smart struct lpfc_nvme_rport *rport = remoteport->private; 35601649561SJames Smart struct lpfc_vport *vport; 35701649561SJames Smart struct lpfc_nodelist *ndlp; 35801649561SJames Smart 35901649561SJames Smart ndlp = rport->ndlp; 36001649561SJames Smart if (!ndlp) 36101649561SJames Smart goto rport_err; 36201649561SJames Smart 36301649561SJames Smart vport = ndlp->vport; 36401649561SJames Smart if (!vport) 36501649561SJames Smart goto rport_err; 36601649561SJames Smart 36701649561SJames Smart /* Remove this rport from the lport's list - memory is owned by the 36801649561SJames Smart * transport. Remove the ndlp reference for the NVME transport before 3697a06dcd3SJames Smart * calling state machine to remove the node. 37001649561SJames Smart */ 37101649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 37232350664SJames Smart "6146 remoteport delete of remoteport x%px\n", 37301649561SJames Smart remoteport); 3743fd78355SJames Smart spin_lock_irq(&vport->phba->hbalock); 375b15bd3e6SJames Smart 376b15bd3e6SJames Smart /* The register rebind might have occurred before the delete 377b15bd3e6SJames Smart * downcall. Guard against this race. 378b15bd3e6SJames Smart */ 379b15bd3e6SJames Smart if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) { 3807a06dcd3SJames Smart ndlp->nrport = NULL; 38101466024SJames Smart ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 3823fd78355SJames Smart spin_unlock_irq(&vport->phba->hbalock); 3833fd78355SJames Smart 3843fd78355SJames Smart /* Remove original register reference. The host transport 3853fd78355SJames Smart * won't reference this rport/remoteport any further. 3863fd78355SJames Smart */ 38701649561SJames Smart lpfc_nlp_put(ndlp); 388807e7353SJames Smart } else { 389807e7353SJames Smart spin_unlock_irq(&vport->phba->hbalock); 390807e7353SJames Smart } 39101649561SJames Smart 39201649561SJames Smart rport_err: 3933fd78355SJames Smart return; 39401649561SJames Smart } 39501649561SJames Smart 3963a8070c5SJames Smart /** 3973a8070c5SJames Smart * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request 3983a8070c5SJames Smart * @phba: pointer to lpfc hba data structure. 3993a8070c5SJames Smart * @axchg: pointer to exchange context for the NVME LS request 4003a8070c5SJames Smart * 4013a8070c5SJames Smart * This routine is used for processing an asychronously received NVME LS 4023a8070c5SJames Smart * request. Any remaining validation is done and the LS is then forwarded 4033a8070c5SJames Smart * to the nvme-fc transport via nvme_fc_rcv_ls_req(). 4043a8070c5SJames Smart * 4059aa09e98SJames Smart * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) 4069aa09e98SJames Smart * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. 4079aa09e98SJames Smart * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 4089aa09e98SJames Smart * 4093a8070c5SJames Smart * Returns 0 if LS was handled and delivered to the transport 4103a8070c5SJames Smart * Returns 1 if LS failed to be handled and should be dropped 4113a8070c5SJames Smart */ 4123a8070c5SJames Smart int 4133a8070c5SJames Smart lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, 4143a8070c5SJames Smart struct lpfc_async_xchg_ctx *axchg) 4153a8070c5SJames Smart { 4169aa09e98SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 4179aa09e98SJames Smart struct lpfc_vport *vport; 4189aa09e98SJames Smart struct lpfc_nvme_rport *lpfc_rport; 4199aa09e98SJames Smart struct nvme_fc_remote_port *remoteport; 4209aa09e98SJames Smart struct lpfc_nvme_lport *lport; 4219aa09e98SJames Smart uint32_t *payload = axchg->payload; 4229aa09e98SJames Smart int rc; 4239aa09e98SJames Smart 4249aa09e98SJames Smart vport = axchg->ndlp->vport; 4259aa09e98SJames Smart lpfc_rport = axchg->ndlp->nrport; 4269aa09e98SJames Smart if (!lpfc_rport) 4279aa09e98SJames Smart return -EINVAL; 4289aa09e98SJames Smart 4299aa09e98SJames Smart remoteport = lpfc_rport->remoteport; 4309aa09e98SJames Smart if (!vport->localport) 4319aa09e98SJames Smart return -EINVAL; 4329aa09e98SJames Smart 4339aa09e98SJames Smart lport = vport->localport->private; 4349aa09e98SJames Smart if (!lport) 4359aa09e98SJames Smart return -EINVAL; 4369aa09e98SJames Smart 4379aa09e98SJames Smart rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, 4389aa09e98SJames Smart axchg->size); 4399aa09e98SJames Smart 4409aa09e98SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 4419aa09e98SJames Smart "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " 4429aa09e98SJames Smart "%08x %08x %08x\n", 4439aa09e98SJames Smart axchg->size, rc, 4449aa09e98SJames Smart *payload, *(payload+1), *(payload+2), 4459aa09e98SJames Smart *(payload+3), *(payload+4), *(payload+5)); 4469aa09e98SJames Smart 4479aa09e98SJames Smart if (!rc) 4489aa09e98SJames Smart return 0; 4499aa09e98SJames Smart #endif 4503a8070c5SJames Smart return 1; 4513a8070c5SJames Smart } 4523a8070c5SJames Smart 4536514b25dSJames Smart /** 4546514b25dSJames Smart * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME 4556514b25dSJames Smart * LS request. 4566514b25dSJames Smart * @phba: Pointer to HBA context object 4576514b25dSJames Smart * @vport: The local port that issued the LS 4586514b25dSJames Smart * @cmdwqe: Pointer to driver command WQE object. 4596514b25dSJames Smart * @wcqe: Pointer to driver response CQE object. 4606514b25dSJames Smart * 4616514b25dSJames Smart * This function is the generic completion handler for NVME LS requests. 4626514b25dSJames Smart * The function updates any states and statistics, calls the transport 4636514b25dSJames Smart * ls_req done() routine, then tears down the command and buffers used 4646514b25dSJames Smart * for the LS request. 4656514b25dSJames Smart **/ 4666514b25dSJames Smart void 4676514b25dSJames Smart __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 4686514b25dSJames Smart struct lpfc_iocbq *cmdwqe, 4696514b25dSJames Smart struct lpfc_wcqe_complete *wcqe) 4706514b25dSJames Smart { 4716514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq; 4726514b25dSJames Smart struct lpfc_dmabuf *buf_ptr; 4736514b25dSJames Smart struct lpfc_nodelist *ndlp; 4746514b25dSJames Smart uint32_t status; 4756514b25dSJames Smart 4766514b25dSJames Smart pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 4776514b25dSJames Smart ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 4786514b25dSJames Smart status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 4796514b25dSJames Smart 4806514b25dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 4816514b25dSJames Smart "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x " 4826514b25dSJames Smart "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " 4836514b25dSJames Smart "ndlp:x%px\n", 4846514b25dSJames Smart pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 4856514b25dSJames Smart cmdwqe->sli4_xritag, status, 4866514b25dSJames Smart (wcqe->parameter & 0xffff), 4876514b25dSJames Smart cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 4886514b25dSJames Smart 4896514b25dSJames Smart lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", 4906514b25dSJames Smart cmdwqe->sli4_xritag, status, wcqe->parameter); 4916514b25dSJames Smart 4926514b25dSJames Smart if (cmdwqe->context3) { 4936514b25dSJames Smart buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 4946514b25dSJames Smart lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 4956514b25dSJames Smart kfree(buf_ptr); 4966514b25dSJames Smart cmdwqe->context3 = NULL; 4976514b25dSJames Smart } 4986514b25dSJames Smart if (pnvme_lsreq->done) 4996514b25dSJames Smart pnvme_lsreq->done(pnvme_lsreq, status); 5006514b25dSJames Smart else 5016514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 5026514b25dSJames Smart "6046 NVMEx cmpl without done call back? " 5036514b25dSJames Smart "Data %px DID %x Xri: %x status %x\n", 5046514b25dSJames Smart pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 5056514b25dSJames Smart cmdwqe->sli4_xritag, status); 5066514b25dSJames Smart if (ndlp) { 5076514b25dSJames Smart lpfc_nlp_put(ndlp); 5086514b25dSJames Smart cmdwqe->context1 = NULL; 5096514b25dSJames Smart } 5106514b25dSJames Smart lpfc_sli_release_iocbq(phba, cmdwqe); 5116514b25dSJames Smart } 5126514b25dSJames Smart 51301649561SJames Smart static void 5146514b25dSJames Smart lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 51501649561SJames Smart struct lpfc_wcqe_complete *wcqe) 51601649561SJames Smart { 51701649561SJames Smart struct lpfc_vport *vport = cmdwqe->vport; 5184b056682SJames Smart struct lpfc_nvme_lport *lport; 51901649561SJames Smart uint32_t status; 52001649561SJames Smart 52101649561SJames Smart status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 52266a210ffSJames Smart 52366a85155SJames Smart if (vport->localport) { 52466a85155SJames Smart lport = (struct lpfc_nvme_lport *)vport->localport->private; 52566a210ffSJames Smart if (lport) { 52666a210ffSJames Smart atomic_inc(&lport->fc4NvmeLsCmpls); 5274b056682SJames Smart if (status) { 5284b056682SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe)) 5294b056682SJames Smart atomic_inc(&lport->cmpl_ls_xb); 5304b056682SJames Smart atomic_inc(&lport->cmpl_ls_err); 5314b056682SJames Smart } 53266a210ffSJames Smart } 53366a85155SJames Smart } 5344b056682SJames Smart 5356514b25dSJames Smart __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); 53601649561SJames Smart } 53701649561SJames Smart 53801649561SJames Smart static int 53901649561SJames Smart lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 54001649561SJames Smart struct lpfc_dmabuf *inp, 54101649561SJames Smart struct nvmefc_ls_req *pnvme_lsreq, 54201649561SJames Smart void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 54301649561SJames Smart struct lpfc_wcqe_complete *), 54401649561SJames Smart struct lpfc_nodelist *ndlp, uint32_t num_entry, 54501649561SJames Smart uint32_t tmo, uint8_t retry) 54601649561SJames Smart { 54701649561SJames Smart struct lpfc_hba *phba = vport->phba; 548205e8240SJames Smart union lpfc_wqe128 *wqe; 54901649561SJames Smart struct lpfc_iocbq *genwqe; 55001649561SJames Smart struct ulp_bde64 *bpl; 55101649561SJames Smart struct ulp_bde64 bde; 55201649561SJames Smart int i, rc, xmit_len, first_len; 55301649561SJames Smart 55401649561SJames Smart /* Allocate buffer for command WQE */ 55501649561SJames Smart genwqe = lpfc_sli_get_iocbq(phba); 55601649561SJames Smart if (genwqe == NULL) 55701649561SJames Smart return 1; 55801649561SJames Smart 55901649561SJames Smart wqe = &genwqe->wqe; 560d9f492a1SJames Smart /* Initialize only 64 bytes */ 56101649561SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe)); 56201649561SJames Smart 56301649561SJames Smart genwqe->context3 = (uint8_t *)bmp; 56401649561SJames Smart genwqe->iocb_flag |= LPFC_IO_NVME_LS; 56501649561SJames Smart 56601649561SJames Smart /* Save for completion so we can release these resources */ 56701649561SJames Smart genwqe->context1 = lpfc_nlp_get(ndlp); 56801649561SJames Smart genwqe->context2 = (uint8_t *)pnvme_lsreq; 56901649561SJames Smart /* Fill in payload, bp points to frame payload */ 57001649561SJames Smart 57101649561SJames Smart if (!tmo) 57201649561SJames Smart /* FC spec states we need 3 * ratov for CT requests */ 57301649561SJames Smart tmo = (3 * phba->fc_ratov); 57401649561SJames Smart 57501649561SJames Smart /* For this command calculate the xmit length of the request bde. */ 57601649561SJames Smart xmit_len = 0; 57701649561SJames Smart first_len = 0; 57801649561SJames Smart bpl = (struct ulp_bde64 *)bmp->virt; 57901649561SJames Smart for (i = 0; i < num_entry; i++) { 58001649561SJames Smart bde.tus.w = bpl[i].tus.w; 58101649561SJames Smart if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 58201649561SJames Smart break; 58301649561SJames Smart xmit_len += bde.tus.f.bdeSize; 58401649561SJames Smart if (i == 0) 58501649561SJames Smart first_len = xmit_len; 58601649561SJames Smart } 58701649561SJames Smart 58801649561SJames Smart genwqe->rsvd2 = num_entry; 58901649561SJames Smart genwqe->hba_wqidx = 0; 59001649561SJames Smart 59101649561SJames Smart /* Words 0 - 2 */ 59201649561SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 59301649561SJames Smart wqe->generic.bde.tus.f.bdeSize = first_len; 59401649561SJames Smart wqe->generic.bde.addrLow = bpl[0].addrLow; 59501649561SJames Smart wqe->generic.bde.addrHigh = bpl[0].addrHigh; 59601649561SJames Smart 59701649561SJames Smart /* Word 3 */ 59801649561SJames Smart wqe->gen_req.request_payload_len = first_len; 59901649561SJames Smart 60001649561SJames Smart /* Word 4 */ 60101649561SJames Smart 60201649561SJames Smart /* Word 5 */ 60301649561SJames Smart bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 60401649561SJames Smart bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 60501649561SJames Smart bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 6068b361639SJames Smart bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 60701649561SJames Smart bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 60801649561SJames Smart 60901649561SJames Smart /* Word 6 */ 61001649561SJames Smart bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 61101649561SJames Smart phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 61201649561SJames Smart bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 61301649561SJames Smart 61401649561SJames Smart /* Word 7 */ 61501649561SJames Smart bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 61601649561SJames Smart bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 61701649561SJames Smart bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 61801649561SJames Smart bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 61901649561SJames Smart 62001649561SJames Smart /* Word 8 */ 62101649561SJames Smart wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 62201649561SJames Smart 62301649561SJames Smart /* Word 9 */ 62401649561SJames Smart bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 62501649561SJames Smart 62601649561SJames Smart /* Word 10 */ 62701649561SJames Smart bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 62801649561SJames Smart bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 62901649561SJames Smart bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 63001649561SJames Smart bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 63101649561SJames Smart bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 63201649561SJames Smart 63301649561SJames Smart /* Word 11 */ 63401649561SJames Smart bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 63501649561SJames Smart bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 63601649561SJames Smart 63701649561SJames Smart 63801649561SJames Smart /* Issue GEN REQ WQE for NPORT <did> */ 63901649561SJames Smart genwqe->wqe_cmpl = cmpl; 64001649561SJames Smart genwqe->iocb_cmpl = NULL; 64101649561SJames Smart genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 64201649561SJames Smart genwqe->vport = vport; 64301649561SJames Smart genwqe->retry = retry; 64401649561SJames Smart 645bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 646bd2cdd5eSJames Smart genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 647bd2cdd5eSJames Smart 6481fbf9742SJames Smart rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 649cd22d605SDick Kennedy if (rc) { 6506514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC | LOG_ELS, 65101649561SJames Smart "6045 Issue GEN REQ WQE to NPORT x%x " 6526514b25dSJames Smart "Data: x%x x%x rc x%x\n", 65301649561SJames Smart ndlp->nlp_DID, genwqe->iotag, 6546514b25dSJames Smart vport->port_state, rc); 65501649561SJames Smart lpfc_sli_release_iocbq(phba, genwqe); 65601649561SJames Smart return 1; 65701649561SJames Smart } 6586514b25dSJames Smart 6596514b25dSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, 6606514b25dSJames Smart "6050 Issue GEN REQ WQE to NPORT x%x " 6616514b25dSJames Smart "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " 6626514b25dSJames Smart "bmp:x%px xmit:%d 1st:%d\n", 6636514b25dSJames Smart ndlp->nlp_DID, genwqe->sli4_xritag, 6646514b25dSJames Smart vport->port_state, 6656514b25dSJames Smart genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 66601649561SJames Smart return 0; 66701649561SJames Smart } 66801649561SJames Smart 6696514b25dSJames Smart 67001649561SJames Smart /** 6716514b25dSJames Smart * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request 6726514b25dSJames Smart * @vport: The local port issuing the LS 6736514b25dSJames Smart * @ndlp: The remote port to send the LS to 6746514b25dSJames Smart * @pnvme_lsreq: Pointer to LS request structure from the transport 67501649561SJames Smart * 6766514b25dSJames Smart * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST 6776514b25dSJames Smart * WQE to perform the LS operation. 67801649561SJames Smart * 67901649561SJames Smart * Return value : 68001649561SJames Smart * 0 - Success 6816514b25dSJames Smart * non-zero: various error codes, in form of -Exxx 68201649561SJames Smart **/ 6836514b25dSJames Smart int 6846514b25dSJames Smart __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 6856514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq, 6866514b25dSJames Smart void (*gen_req_cmp)(struct lpfc_hba *phba, 6876514b25dSJames Smart struct lpfc_iocbq *cmdwqe, 6886514b25dSJames Smart struct lpfc_wcqe_complete *wcqe)) 68901649561SJames Smart { 69001649561SJames Smart struct lpfc_dmabuf *bmp; 6916514b25dSJames Smart struct ulp_bde64 *bpl; 6926514b25dSJames Smart int ret; 693ba43c4d0SJames Smart uint16_t ntype, nstate; 69401649561SJames Smart 695ba43c4d0SJames Smart if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 6966514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, 6976514b25dSJames Smart LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, 6986514b25dSJames Smart "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " 6996514b25dSJames Smart "LS Req\n", 7006514b25dSJames Smart ndlp); 701ba43c4d0SJames Smart return -ENODEV; 702ba43c4d0SJames Smart } 703ba43c4d0SJames Smart 704ba43c4d0SJames Smart ntype = ndlp->nlp_type; 705ba43c4d0SJames Smart nstate = ndlp->nlp_state; 706ba43c4d0SJames Smart if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 707ba43c4d0SJames Smart (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 7086514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, 7096514b25dSJames Smart LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, 7106514b25dSJames Smart "6088 NVMEx LS REQ: Fail DID x%06x not " 7116514b25dSJames Smart "ready for IO. Type x%x, State x%x\n", 7126514b25dSJames Smart ndlp->nlp_DID, ntype, nstate); 713ba43c4d0SJames Smart return -ENODEV; 71401649561SJames Smart } 7156514b25dSJames Smart 7166514b25dSJames Smart /* 7176514b25dSJames Smart * there are two dma buf in the request, actually there is one and 7186514b25dSJames Smart * the second one is just the start address + cmd size. 7196514b25dSJames Smart * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 7206514b25dSJames Smart * in a lpfc_dmabuf struct. When freeing we just free the wrapper 7216514b25dSJames Smart * because the nvem layer owns the data bufs. 7226514b25dSJames Smart * We do not have to break these packets open, we don't care what is 7236514b25dSJames Smart * in them. And we do not have to look at the resonse data, we only 7246514b25dSJames Smart * care that we got a response. All of the caring is going to happen 7256514b25dSJames Smart * in the nvme-fc layer. 7266514b25dSJames Smart */ 7276514b25dSJames Smart 7286514b25dSJames Smart bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); 72901649561SJames Smart if (!bmp) { 73001649561SJames Smart 7316514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, 7326514b25dSJames Smart LOG_NVME_DISC | LOG_NVME_IOERR, 7336514b25dSJames Smart "6044 NVMEx LS REQ: Could not alloc LS buf " 7346514b25dSJames Smart "for DID %x\n", 7356514b25dSJames Smart ndlp->nlp_DID); 7366514b25dSJames Smart return -ENOMEM; 73701649561SJames Smart } 7386514b25dSJames Smart 73901649561SJames Smart bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 74001649561SJames Smart if (!bmp->virt) { 7416514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, 7426514b25dSJames Smart LOG_NVME_DISC | LOG_NVME_IOERR, 7436514b25dSJames Smart "6042 NVMEx LS REQ: Could not alloc mbuf " 7446514b25dSJames Smart "for DID %x\n", 7456514b25dSJames Smart ndlp->nlp_DID); 74601649561SJames Smart kfree(bmp); 7476514b25dSJames Smart return -ENOMEM; 74801649561SJames Smart } 7496514b25dSJames Smart 7506514b25dSJames Smart INIT_LIST_HEAD(&bmp->list); 7516514b25dSJames Smart 75201649561SJames Smart bpl = (struct ulp_bde64 *)bmp->virt; 75301649561SJames Smart bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 75401649561SJames Smart bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 75501649561SJames Smart bpl->tus.f.bdeFlags = 0; 75601649561SJames Smart bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 75701649561SJames Smart bpl->tus.w = le32_to_cpu(bpl->tus.w); 75801649561SJames Smart bpl++; 75901649561SJames Smart 76001649561SJames Smart bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 76101649561SJames Smart bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 76201649561SJames Smart bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 76301649561SJames Smart bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 76401649561SJames Smart bpl->tus.w = le32_to_cpu(bpl->tus.w); 76501649561SJames Smart 76601649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 7676514b25dSJames Smart "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " 7686514b25dSJames Smart "rqstlen:%d rsplen:%d %pad %pad\n", 7696514b25dSJames Smart ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, 770825c6abbSArnd Bergmann pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 771825c6abbSArnd Bergmann &pnvme_lsreq->rspdma); 77201649561SJames Smart 77301649561SJames Smart ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 7746514b25dSJames Smart pnvme_lsreq, gen_req_cmp, ndlp, 2, 7756514b25dSJames Smart LPFC_NVME_LS_TIMEOUT, 0); 77601649561SJames Smart if (ret != WQE_SUCCESS) { 7776514b25dSJames Smart lpfc_printf_vlog(vport, KERN_ERR, 7786514b25dSJames Smart LOG_NVME_DISC | LOG_NVME_IOERR, 7796514b25dSJames Smart "6052 NVMEx REQ: EXIT. issue ls wqe failed " 7806514b25dSJames Smart "lsreq x%px Status %x DID %x\n", 7816514b25dSJames Smart pnvme_lsreq, ret, ndlp->nlp_DID); 78201649561SJames Smart lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 78301649561SJames Smart kfree(bmp); 7846514b25dSJames Smart return -EIO; 78501649561SJames Smart } 78601649561SJames Smart 7876514b25dSJames Smart return 0; 7886514b25dSJames Smart } 7896514b25dSJames Smart 7906514b25dSJames Smart /** 7916514b25dSJames Smart * lpfc_nvme_ls_req - Issue an NVME Link Service request 7926514b25dSJames Smart * @lpfc_nvme_lport: Transport localport that LS is to be issued from. 7936514b25dSJames Smart * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. 7946514b25dSJames Smart * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 7956514b25dSJames Smart * 7966514b25dSJames Smart * Driver registers this routine to handle any link service request 7976514b25dSJames Smart * from the nvme_fc transport to a remote nvme-aware port. 7986514b25dSJames Smart * 7996514b25dSJames Smart * Return value : 8006514b25dSJames Smart * 0 - Success 8016514b25dSJames Smart * non-zero: various error codes, in form of -Exxx 8026514b25dSJames Smart **/ 8036514b25dSJames Smart static int 8046514b25dSJames Smart lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 8056514b25dSJames Smart struct nvme_fc_remote_port *pnvme_rport, 8066514b25dSJames Smart struct nvmefc_ls_req *pnvme_lsreq) 8076514b25dSJames Smart { 8086514b25dSJames Smart struct lpfc_nvme_lport *lport; 8096514b25dSJames Smart struct lpfc_nvme_rport *rport; 8106514b25dSJames Smart struct lpfc_vport *vport; 8116514b25dSJames Smart int ret; 8126514b25dSJames Smart 8136514b25dSJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 8146514b25dSJames Smart rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 8156514b25dSJames Smart if (unlikely(!lport) || unlikely(!rport)) 8166514b25dSJames Smart return -EINVAL; 8176514b25dSJames Smart 8186514b25dSJames Smart vport = lport->vport; 8196514b25dSJames Smart if (vport->load_flag & FC_UNLOADING) 8206514b25dSJames Smart return -ENODEV; 8216514b25dSJames Smart 8226514b25dSJames Smart atomic_inc(&lport->fc4NvmeLsRequests); 8236514b25dSJames Smart 8246514b25dSJames Smart ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, 8256514b25dSJames Smart lpfc_nvme_ls_req_cmp); 8266514b25dSJames Smart if (ret) 8276514b25dSJames Smart atomic_inc(&lport->xmt_ls_err); 8286514b25dSJames Smart 82901649561SJames Smart return ret; 83001649561SJames Smart } 83101649561SJames Smart 83201649561SJames Smart /** 833e96a22b0SJames Smart * __lpfc_nvme_ls_abort - Generic service routine to abort a prior 834e96a22b0SJames Smart * NVME LS request 835e96a22b0SJames Smart * @vport: The local port that issued the LS 836e96a22b0SJames Smart * @ndlp: The remote port the LS was sent to 837e96a22b0SJames Smart * @pnvme_lsreq: Pointer to LS request structure from the transport 83801649561SJames Smart * 839e96a22b0SJames Smart * The driver validates the ndlp, looks for the LS, and aborts the 840e96a22b0SJames Smart * LS if found. 84101649561SJames Smart * 842e96a22b0SJames Smart * Returns: 843e96a22b0SJames Smart * 0 : if LS found and aborted 844e96a22b0SJames Smart * non-zero: various error conditions in form -Exxx 845e96a22b0SJames Smart **/ 846e96a22b0SJames Smart int 847e96a22b0SJames Smart __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 848e96a22b0SJames Smart struct nvmefc_ls_req *pnvme_lsreq) 849e96a22b0SJames Smart { 850e96a22b0SJames Smart struct lpfc_hba *phba = vport->phba; 851e96a22b0SJames Smart struct lpfc_sli_ring *pring; 852e96a22b0SJames Smart struct lpfc_iocbq *wqe, *next_wqe; 853e96a22b0SJames Smart bool foundit = false; 854e96a22b0SJames Smart 855e96a22b0SJames Smart if (!ndlp) { 856e96a22b0SJames Smart lpfc_printf_log(phba, KERN_ERR, 857e96a22b0SJames Smart LOG_NVME_DISC | LOG_NODE | 858e96a22b0SJames Smart LOG_NVME_IOERR | LOG_NVME_ABTS, 859e96a22b0SJames Smart "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " 860e96a22b0SJames Smart "x%06x, Failing LS Req\n", 861e96a22b0SJames Smart ndlp, ndlp ? ndlp->nlp_DID : 0); 862e96a22b0SJames Smart return -EINVAL; 863e96a22b0SJames Smart } 864e96a22b0SJames Smart 865e96a22b0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 866e96a22b0SJames Smart "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " 867e96a22b0SJames Smart "x%p rqstlen:%d rsplen:%d %pad %pad\n", 868e96a22b0SJames Smart pnvme_lsreq, pnvme_lsreq->rqstlen, 869e96a22b0SJames Smart pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 870e96a22b0SJames Smart &pnvme_lsreq->rspdma); 871e96a22b0SJames Smart 872e96a22b0SJames Smart /* 873e96a22b0SJames Smart * Lock the ELS ring txcmplq and look for the wqe that matches 874e96a22b0SJames Smart * this ELS. If found, issue an abort on the wqe. 875e96a22b0SJames Smart */ 876e96a22b0SJames Smart pring = phba->sli4_hba.nvmels_wq->pring; 877e96a22b0SJames Smart spin_lock_irq(&phba->hbalock); 878e96a22b0SJames Smart spin_lock(&pring->ring_lock); 879e96a22b0SJames Smart list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 880e96a22b0SJames Smart if (wqe->context2 == pnvme_lsreq) { 881e96a22b0SJames Smart wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 882e96a22b0SJames Smart foundit = true; 883e96a22b0SJames Smart break; 884e96a22b0SJames Smart } 885e96a22b0SJames Smart } 886e96a22b0SJames Smart spin_unlock(&pring->ring_lock); 887e96a22b0SJames Smart 888e96a22b0SJames Smart if (foundit) 889e96a22b0SJames Smart lpfc_sli_issue_abort_iotag(phba, pring, wqe); 890e96a22b0SJames Smart spin_unlock_irq(&phba->hbalock); 891e96a22b0SJames Smart 892e96a22b0SJames Smart if (foundit) 893e96a22b0SJames Smart return 0; 894e96a22b0SJames Smart 895e96a22b0SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 896e96a22b0SJames Smart "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n", 897e96a22b0SJames Smart pnvme_lsreq); 898*6b6e8963SJames Smart return -EINVAL; 899e96a22b0SJames Smart } 900e96a22b0SJames Smart 9019aa09e98SJames Smart static int 9029aa09e98SJames Smart lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, 9039aa09e98SJames Smart struct nvme_fc_remote_port *remoteport, 9049aa09e98SJames Smart struct nvmefc_ls_rsp *ls_rsp) 9059aa09e98SJames Smart { 9069aa09e98SJames Smart struct lpfc_async_xchg_ctx *axchg = 9079aa09e98SJames Smart container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 9089aa09e98SJames Smart struct lpfc_nvme_lport *lport; 9099aa09e98SJames Smart int rc; 9109aa09e98SJames Smart 9119aa09e98SJames Smart if (axchg->phba->pport->load_flag & FC_UNLOADING) 9129aa09e98SJames Smart return -ENODEV; 9139aa09e98SJames Smart 9149aa09e98SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 9159aa09e98SJames Smart 9169aa09e98SJames Smart rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); 9179aa09e98SJames Smart 9189aa09e98SJames Smart if (rc) { 9199aa09e98SJames Smart /* 9209aa09e98SJames Smart * unless the failure is due to having already sent 9219aa09e98SJames Smart * the response, an abort will be generated for the 9229aa09e98SJames Smart * exchange if the rsp can't be sent. 9239aa09e98SJames Smart */ 9249aa09e98SJames Smart if (rc != -EALREADY) 9259aa09e98SJames Smart atomic_inc(&lport->xmt_ls_abort); 9269aa09e98SJames Smart return rc; 9279aa09e98SJames Smart } 9289aa09e98SJames Smart 9299aa09e98SJames Smart return 0; 9309aa09e98SJames Smart } 9319aa09e98SJames Smart 932e96a22b0SJames Smart /** 933e96a22b0SJames Smart * lpfc_nvme_ls_abort - Abort a prior NVME LS request 934e96a22b0SJames Smart * @lpfc_nvme_lport: Transport localport that LS is to be issued from. 935e96a22b0SJames Smart * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. 936e96a22b0SJames Smart * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 937e96a22b0SJames Smart * 938e96a22b0SJames Smart * Driver registers this routine to abort a NVME LS request that is 939e96a22b0SJames Smart * in progress (from the transports perspective). 94001649561SJames Smart **/ 94101649561SJames Smart static void 94201649561SJames Smart lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 94301649561SJames Smart struct nvme_fc_remote_port *pnvme_rport, 94401649561SJames Smart struct nvmefc_ls_req *pnvme_lsreq) 94501649561SJames Smart { 94601649561SJames Smart struct lpfc_nvme_lport *lport; 94701649561SJames Smart struct lpfc_vport *vport; 94801649561SJames Smart struct lpfc_hba *phba; 94901649561SJames Smart struct lpfc_nodelist *ndlp; 950e96a22b0SJames Smart int ret; 95101649561SJames Smart 95201649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 95366a210ffSJames Smart if (unlikely(!lport)) 95466a210ffSJames Smart return; 95501649561SJames Smart vport = lport->vport; 95601649561SJames Smart phba = vport->phba; 95701649561SJames Smart 9583386f4bdSJames Smart if (vport->load_flag & FC_UNLOADING) 9593386f4bdSJames Smart return; 9603386f4bdSJames Smart 96101649561SJames Smart ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 96201649561SJames Smart 963e96a22b0SJames Smart ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); 964e96a22b0SJames Smart if (!ret) 9654b056682SJames Smart atomic_inc(&lport->xmt_ls_abort); 96601649561SJames Smart } 96701649561SJames Smart 96801649561SJames Smart /* Fix up the existing sgls for NVME IO. */ 9695fd11085SJames Smart static inline void 97001649561SJames Smart lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 971c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd, 97201649561SJames Smart struct nvmefc_fcp_req *nCmd) 97301649561SJames Smart { 9744e565cf0SJames Smart struct lpfc_hba *phba = vport->phba; 97501649561SJames Smart struct sli4_sge *sgl; 97601649561SJames Smart union lpfc_wqe128 *wqe; 97701649561SJames Smart uint32_t *wptr, *dptr; 97801649561SJames Smart 97901649561SJames Smart /* 9804e565cf0SJames Smart * Get a local pointer to the built-in wqe and correct 9814e565cf0SJames Smart * the cmd size to match NVME's 96 bytes and fix 9824e565cf0SJames Smart * the dma address. 9834e565cf0SJames Smart */ 9844e565cf0SJames Smart 985205e8240SJames Smart wqe = &lpfc_ncmd->cur_iocbq.wqe; 9864e565cf0SJames Smart 9874e565cf0SJames Smart /* 98801649561SJames Smart * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 98901649561SJames Smart * match NVME. NVME sends 96 bytes. Also, use the 99001649561SJames Smart * nvme commands command and response dma addresses 99101649561SJames Smart * rather than the virtual memory to ease the restore 99201649561SJames Smart * operation. 99301649561SJames Smart */ 9940794d601SJames Smart sgl = lpfc_ncmd->dma_sgl; 99501649561SJames Smart sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 9964e565cf0SJames Smart if (phba->cfg_nvme_embed_cmd) { 9974e565cf0SJames Smart sgl->addr_hi = 0; 9984e565cf0SJames Smart sgl->addr_lo = 0; 9994e565cf0SJames Smart 10004e565cf0SJames Smart /* Word 0-2 - NVME CMND IU (embedded payload) */ 10014e565cf0SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 10024e565cf0SJames Smart wqe->generic.bde.tus.f.bdeSize = 56; 10034e565cf0SJames Smart wqe->generic.bde.addrHigh = 0; 10044e565cf0SJames Smart wqe->generic.bde.addrLow = 64; /* Word 16 */ 10054e565cf0SJames Smart 10065fd11085SJames Smart /* Word 10 - dbde is 0, wqes is 1 in template */ 100701649561SJames Smart 100801649561SJames Smart /* 100901649561SJames Smart * Embed the payload in the last half of the WQE 101001649561SJames Smart * WQE words 16-30 get the NVME CMD IU payload 101101649561SJames Smart * 1012b06a622fSJames Smart * WQE words 16-19 get payload Words 1-4 101301649561SJames Smart * WQE words 20-21 get payload Words 6-7 101401649561SJames Smart * WQE words 22-29 get payload Words 16-23 101501649561SJames Smart */ 1016b06a622fSJames Smart wptr = &wqe->words[16]; /* WQE ptr */ 101701649561SJames Smart dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 1018b06a622fSJames Smart dptr++; /* Skip Word 0 in payload */ 101901649561SJames Smart 1020b06a622fSJames Smart *wptr++ = *dptr++; /* Word 1 */ 102101649561SJames Smart *wptr++ = *dptr++; /* Word 2 */ 102201649561SJames Smart *wptr++ = *dptr++; /* Word 3 */ 102301649561SJames Smart *wptr++ = *dptr++; /* Word 4 */ 102401649561SJames Smart dptr++; /* Skip Word 5 in payload */ 102501649561SJames Smart *wptr++ = *dptr++; /* Word 6 */ 102601649561SJames Smart *wptr++ = *dptr++; /* Word 7 */ 102701649561SJames Smart dptr += 8; /* Skip Words 8-15 in payload */ 102801649561SJames Smart *wptr++ = *dptr++; /* Word 16 */ 102901649561SJames Smart *wptr++ = *dptr++; /* Word 17 */ 103001649561SJames Smart *wptr++ = *dptr++; /* Word 18 */ 103101649561SJames Smart *wptr++ = *dptr++; /* Word 19 */ 103201649561SJames Smart *wptr++ = *dptr++; /* Word 20 */ 103301649561SJames Smart *wptr++ = *dptr++; /* Word 21 */ 103401649561SJames Smart *wptr++ = *dptr++; /* Word 22 */ 103501649561SJames Smart *wptr = *dptr; /* Word 23 */ 10365fd11085SJames Smart } else { 10375fd11085SJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 10385fd11085SJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 10395fd11085SJames Smart 10405fd11085SJames Smart /* Word 0-2 - NVME CMND IU Inline BDE */ 10415fd11085SJames Smart wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 10425fd11085SJames Smart wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 10435fd11085SJames Smart wqe->generic.bde.addrHigh = sgl->addr_hi; 10445fd11085SJames Smart wqe->generic.bde.addrLow = sgl->addr_lo; 10455fd11085SJames Smart 10465fd11085SJames Smart /* Word 10 */ 10475fd11085SJames Smart bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 10485fd11085SJames Smart bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 10495fd11085SJames Smart } 10505fd11085SJames Smart 10515fd11085SJames Smart sgl++; 10525fd11085SJames Smart 10535fd11085SJames Smart /* Setup the physical region for the FCP RSP */ 10545fd11085SJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 10555fd11085SJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 10565fd11085SJames Smart sgl->word2 = le32_to_cpu(sgl->word2); 10575fd11085SJames Smart if (nCmd->sg_cnt) 10585fd11085SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0); 10595fd11085SJames Smart else 10605fd11085SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1); 10615fd11085SJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 10625fd11085SJames Smart sgl->sge_len = cpu_to_le32(nCmd->rsplen); 106301649561SJames Smart } 106401649561SJames Smart 1065bd2cdd5eSJames Smart 106601649561SJames Smart /** 106701649561SJames Smart * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 106801649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 106901649561SJames Smart * @lpfc_nvme_lport: Pointer to the driver's local port data 107001649561SJames Smart * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 107101649561SJames Smart * 107201649561SJames Smart * Driver registers this routine as it io request handler. This 107301649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 107401649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. 107501649561SJames Smart * 107601649561SJames Smart * Return value : 107701649561SJames Smart * 0 - Success 107801649561SJames Smart * TODO: What are the failure codes. 107901649561SJames Smart **/ 108001649561SJames Smart static void 108101649561SJames Smart lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 108201649561SJames Smart struct lpfc_wcqe_complete *wcqe) 108301649561SJames Smart { 1084c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd = 1085c490850aSJames Smart (struct lpfc_io_buf *)pwqeIn->context1; 108601649561SJames Smart struct lpfc_vport *vport = pwqeIn->vport; 108701649561SJames Smart struct nvmefc_fcp_req *nCmd; 108801649561SJames Smart struct nvme_fc_ersp_iu *ep; 108901649561SJames Smart struct nvme_fc_cmd_iu *cp; 109001649561SJames Smart struct lpfc_nodelist *ndlp; 1091bbe3012bSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 10924b056682SJames Smart struct lpfc_nvme_lport *lport; 1093352b205aSArnd Bergmann uint32_t code, status, idx; 109401649561SJames Smart uint16_t cid, sqhd, data; 109501649561SJames Smart uint32_t *ptr; 1096840eda96SJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1097840eda96SJames Smart int cpu; 1098840eda96SJames Smart #endif 109901649561SJames Smart 110001649561SJames Smart /* Sanity check on return of outstanding command */ 11014d5e789aSJames Smart if (!lpfc_ncmd) { 11024d5e789aSJames Smart lpfc_printf_vlog(vport, KERN_ERR, 11034d5e789aSJames Smart LOG_NODE | LOG_NVME_IOERR, 11044d5e789aSJames Smart "6071 Null lpfc_ncmd pointer. No " 11054d5e789aSJames Smart "release, skip completion\n"); 11064d5e789aSJames Smart return; 11074d5e789aSJames Smart } 11084d5e789aSJames Smart 1109c2017260SJames Smart /* Guard against abort handler being called at same time */ 1110c2017260SJames Smart spin_lock(&lpfc_ncmd->buf_lock); 1111c2017260SJames Smart 1112c2017260SJames Smart if (!lpfc_ncmd->nvmeCmd) { 1113c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 111401649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 111532350664SJames Smart "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 111632350664SJames Smart "nvmeCmd x%px\n", 11170b05e9feSJames Smart lpfc_ncmd, lpfc_ncmd->nvmeCmd); 11184d5e789aSJames Smart 11194d5e789aSJames Smart /* Release the lpfc_ncmd regardless of the missing elements. */ 11204d5e789aSJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 112101649561SJames Smart return; 112201649561SJames Smart } 112301649561SJames Smart nCmd = lpfc_ncmd->nvmeCmd; 11244b056682SJames Smart status = bf_get(lpfc_wcqe_c_status, wcqe); 112566a210ffSJames Smart 11264c47efc1SJames Smart idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 11274c47efc1SJames Smart phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 11284c47efc1SJames Smart 11291df09449SJames Smart if (unlikely(status && vport->localport)) { 11304b056682SJames Smart lport = (struct lpfc_nvme_lport *)vport->localport->private; 11311df09449SJames Smart if (lport) { 11324b056682SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe)) 11334b056682SJames Smart atomic_inc(&lport->cmpl_fcp_xb); 11344b056682SJames Smart atomic_inc(&lport->cmpl_fcp_err); 11354b056682SJames Smart } 113666a210ffSJames Smart } 113701649561SJames Smart 1138bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 1139bd2cdd5eSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 11404b056682SJames Smart status, wcqe->parameter); 114101649561SJames Smart /* 114201649561SJames Smart * Catch race where our node has transitioned, but the 114301649561SJames Smart * transport is still transitioning. 114401649561SJames Smart */ 11450b05e9feSJames Smart ndlp = lpfc_ncmd->ndlp; 114601649561SJames Smart if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 114701649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 114801649561SJames Smart "6062 Ignoring NVME cmpl. No ndlp\n"); 114901649561SJames Smart goto out_err; 115001649561SJames Smart } 115101649561SJames Smart 115201649561SJames Smart code = bf_get(lpfc_wcqe_c_code, wcqe); 115301649561SJames Smart if (code == CQE_CODE_NVME_ERSP) { 115401649561SJames Smart /* For this type of CQE, we need to rebuild the rsp */ 115501649561SJames Smart ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 115601649561SJames Smart 115701649561SJames Smart /* 115801649561SJames Smart * Get Command Id from cmd to plug into response. This 115901649561SJames Smart * code is not needed in the next NVME Transport drop. 116001649561SJames Smart */ 116101649561SJames Smart cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 116201649561SJames Smart cid = cp->sqe.common.command_id; 116301649561SJames Smart 116401649561SJames Smart /* 116501649561SJames Smart * RSN is in CQE word 2 116601649561SJames Smart * SQHD is in CQE Word 3 bits 15:0 116701649561SJames Smart * Cmd Specific info is in CQE Word 1 116801649561SJames Smart * and in CQE Word 0 bits 15:0 116901649561SJames Smart */ 117001649561SJames Smart sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 117101649561SJames Smart 117201649561SJames Smart /* Now lets build the NVME ERSP IU */ 117301649561SJames Smart ep->iu_len = cpu_to_be16(8); 117401649561SJames Smart ep->rsn = wcqe->parameter; 117501649561SJames Smart ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 117601649561SJames Smart ep->rsvd12 = 0; 117701649561SJames Smart ptr = (uint32_t *)&ep->cqe.result.u64; 117801649561SJames Smart *ptr++ = wcqe->total_data_placed; 117901649561SJames Smart data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 118001649561SJames Smart *ptr = (uint32_t)data; 118101649561SJames Smart ep->cqe.sq_head = sqhd; 118201649561SJames Smart ep->cqe.sq_id = nCmd->sqid; 118301649561SJames Smart ep->cqe.command_id = cid; 118401649561SJames Smart ep->cqe.status = 0; 118501649561SJames Smart 118601649561SJames Smart lpfc_ncmd->status = IOSTAT_SUCCESS; 118701649561SJames Smart lpfc_ncmd->result = 0; 118801649561SJames Smart nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 118901649561SJames Smart nCmd->transferred_length = nCmd->payload_length; 119001649561SJames Smart } else { 11914b056682SJames Smart lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1192952c303bSDick Kennedy lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 119301649561SJames Smart 119401649561SJames Smart /* For NVME, the only failure path that results in an 119501649561SJames Smart * IO error is when the adapter rejects it. All other 119601649561SJames Smart * conditions are a success case and resolved by the 119701649561SJames Smart * transport. 119801649561SJames Smart * IOSTAT_FCP_RSP_ERROR means: 119901649561SJames Smart * 1. Length of data received doesn't match total 120001649561SJames Smart * transfer length in WQE 120101649561SJames Smart * 2. If the RSP payload does NOT match these cases: 120201649561SJames Smart * a. RSP length 12/24 bytes and all zeros 120301649561SJames Smart * b. NVME ERSP 120401649561SJames Smart */ 120501649561SJames Smart switch (lpfc_ncmd->status) { 120601649561SJames Smart case IOSTAT_SUCCESS: 120701649561SJames Smart nCmd->transferred_length = wcqe->total_data_placed; 120801649561SJames Smart nCmd->rcv_rsplen = 0; 120901649561SJames Smart nCmd->status = 0; 121001649561SJames Smart break; 121101649561SJames Smart case IOSTAT_FCP_RSP_ERROR: 121201649561SJames Smart nCmd->transferred_length = wcqe->total_data_placed; 121301649561SJames Smart nCmd->rcv_rsplen = wcqe->parameter; 121401649561SJames Smart nCmd->status = 0; 121501649561SJames Smart /* Sanity check */ 121601649561SJames Smart if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 121701649561SJames Smart break; 121801649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 121901649561SJames Smart "6081 NVME Completion Protocol Error: " 122086c67379SJames Smart "xri %x status x%x result x%x " 122186c67379SJames Smart "placed x%x\n", 122286c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 122301649561SJames Smart lpfc_ncmd->status, lpfc_ncmd->result, 122401649561SJames Smart wcqe->total_data_placed); 122501649561SJames Smart break; 1226952c303bSDick Kennedy case IOSTAT_LOCAL_REJECT: 1227952c303bSDick Kennedy /* Let fall through to set command final state. */ 1228952c303bSDick Kennedy if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1229952c303bSDick Kennedy lpfc_printf_vlog(vport, KERN_INFO, 1230952c303bSDick Kennedy LOG_NVME_IOERR, 123132350664SJames Smart "6032 Delay Aborted cmd x%px " 123232350664SJames Smart "nvme cmd x%px, xri x%x, " 1233952c303bSDick Kennedy "xb %d\n", 1234952c303bSDick Kennedy lpfc_ncmd, nCmd, 1235952c303bSDick Kennedy lpfc_ncmd->cur_iocbq.sli4_xritag, 1236952c303bSDick Kennedy bf_get(lpfc_wcqe_c_xb, wcqe)); 1237cd05c155SBart Van Assche /* fall through */ 123801649561SJames Smart default: 123901649561SJames Smart out_err: 1240e3246a12SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 124186c67379SJames Smart "6072 NVME Completion Error: xri %x " 12422c013a3aSJames Smart "status x%x result x%x [x%x] " 12432c013a3aSJames Smart "placed x%x\n", 124486c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 124501649561SJames Smart lpfc_ncmd->status, lpfc_ncmd->result, 12462c013a3aSJames Smart wcqe->parameter, 124701649561SJames Smart wcqe->total_data_placed); 124801649561SJames Smart nCmd->transferred_length = 0; 124901649561SJames Smart nCmd->rcv_rsplen = 0; 12508e009ce8SJames Smart nCmd->status = NVME_SC_INTERNAL; 125101649561SJames Smart } 125201649561SJames Smart } 125301649561SJames Smart 125401649561SJames Smart /* pick up SLI4 exhange busy condition */ 125501649561SJames Smart if (bf_get(lpfc_wcqe_c_xb, wcqe)) 125601649561SJames Smart lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 125701649561SJames Smart else 125801649561SJames Smart lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 125901649561SJames Smart 126001649561SJames Smart /* Update stats and complete the IO. There is 126101649561SJames Smart * no need for dma unprep because the nvme_transport 126201649561SJames Smart * owns the dma address. 126301649561SJames Smart */ 1264bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1265c8a4ce0bSDick Kennedy if (lpfc_ncmd->ts_cmd_start) { 1266bd2cdd5eSJames Smart lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 12672fcbc569SJames Smart lpfc_ncmd->ts_data_io = ktime_get_ns(); 12682fcbc569SJames Smart phba->ktime_last_cmd = lpfc_ncmd->ts_data_io; 12692fcbc569SJames Smart lpfc_io_ktime(phba, lpfc_ncmd); 1270bd2cdd5eSJames Smart } 1271840eda96SJames Smart if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) { 1272d6d189ceSBart Van Assche cpu = raw_smp_processor_id(); 1273840eda96SJames Smart this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 127463df6d63SJames Smart if (lpfc_ncmd->cpu != cpu) 127563df6d63SJames Smart lpfc_printf_vlog(vport, 127663df6d63SJames Smart KERN_INFO, LOG_NVME_IOERR, 1277bd2cdd5eSJames Smart "6701 CPU Check cmpl: " 1278bd2cdd5eSJames Smart "cpu %d expect %d\n", 127963df6d63SJames Smart cpu, lpfc_ncmd->cpu); 1280bd2cdd5eSJames Smart } 1281bd2cdd5eSJames Smart #endif 1282952c303bSDick Kennedy 1283952c303bSDick Kennedy /* NVME targets need completion held off until the abort exchange 1284add9d6beSJames Smart * completes unless the NVME Rport is getting unregistered. 1285952c303bSDick Kennedy */ 1286add9d6beSJames Smart 12873fd78355SJames Smart if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 128891455b85SJames Smart freqpriv = nCmd->private; 128991455b85SJames Smart freqpriv->nvme_buf = NULL; 12903fd78355SJames Smart lpfc_ncmd->nvmeCmd = NULL; 1291c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 1292c2017260SJames Smart nCmd->done(nCmd); 1293c2017260SJames Smart } else 1294c2017260SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 129501649561SJames Smart 1296952c303bSDick Kennedy /* Call release with XB=1 to queue the IO into the abort list. */ 129701649561SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 129801649561SJames Smart } 129901649561SJames Smart 130001649561SJames Smart 130101649561SJames Smart /** 130201649561SJames Smart * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 130301649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 130401649561SJames Smart * @lpfc_nvme_lport: Pointer to the driver's local port data 130501649561SJames Smart * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 130601649561SJames Smart * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 130701649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 130801649561SJames Smart * 130901649561SJames Smart * Driver registers this routine as it io request handler. This 131001649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 131101649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. 131201649561SJames Smart * 131301649561SJames Smart * Return value : 131401649561SJames Smart * 0 - Success 131501649561SJames Smart * TODO: What are the failure codes. 131601649561SJames Smart **/ 131701649561SJames Smart static int 131801649561SJames Smart lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1319c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd, 132066a210ffSJames Smart struct lpfc_nodelist *pnode, 13214c47efc1SJames Smart struct lpfc_fc4_ctrl_stat *cstat) 132201649561SJames Smart { 132301649561SJames Smart struct lpfc_hba *phba = vport->phba; 132401649561SJames Smart struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 132501649561SJames Smart struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 1326205e8240SJames Smart union lpfc_wqe128 *wqe = &pwqeq->wqe; 132701649561SJames Smart uint32_t req_len; 132801649561SJames Smart 1329e960f5abSJames Smart if (!NLP_CHK_NODE_ACT(pnode)) 133001649561SJames Smart return -EINVAL; 133101649561SJames Smart 133201649561SJames Smart /* 133301649561SJames Smart * There are three possibilities here - use scatter-gather segment, use 133401649561SJames Smart * the single mapping, or neither. 133501649561SJames Smart */ 133601649561SJames Smart if (nCmd->sg_cnt) { 133701649561SJames Smart if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 13385fd11085SJames Smart /* From the iwrite template, initialize words 7 - 11 */ 13395fd11085SJames Smart memcpy(&wqe->words[7], 13405fd11085SJames Smart &lpfc_iwrite_cmd_template.words[7], 13415fd11085SJames Smart sizeof(uint32_t) * 5); 13425fd11085SJames Smart 13435fd11085SJames Smart /* Word 4 */ 13445fd11085SJames Smart wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 13455fd11085SJames Smart 134601649561SJames Smart /* Word 5 */ 134701649561SJames Smart if ((phba->cfg_nvme_enable_fb) && 134801649561SJames Smart (pnode->nlp_flag & NLP_FIRSTBURST)) { 134901649561SJames Smart req_len = lpfc_ncmd->nvmeCmd->payload_length; 135001649561SJames Smart if (req_len < pnode->nvme_fb_size) 135101649561SJames Smart wqe->fcp_iwrite.initial_xfer_len = 135201649561SJames Smart req_len; 135301649561SJames Smart else 135401649561SJames Smart wqe->fcp_iwrite.initial_xfer_len = 135501649561SJames Smart pnode->nvme_fb_size; 13565fd11085SJames Smart } else { 13575fd11085SJames Smart wqe->fcp_iwrite.initial_xfer_len = 0; 135801649561SJames Smart } 13594c47efc1SJames Smart cstat->output_requests++; 136001649561SJames Smart } else { 13615fd11085SJames Smart /* From the iread template, initialize words 7 - 11 */ 13625fd11085SJames Smart memcpy(&wqe->words[7], 13635fd11085SJames Smart &lpfc_iread_cmd_template.words[7], 13645fd11085SJames Smart sizeof(uint32_t) * 5); 136501649561SJames Smart 13665fd11085SJames Smart /* Word 4 */ 13675fd11085SJames Smart wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 136801649561SJames Smart 13695fd11085SJames Smart /* Word 5 */ 13705fd11085SJames Smart wqe->fcp_iread.rsrvd5 = 0; 137101649561SJames Smart 13724c47efc1SJames Smart cstat->input_requests++; 137301649561SJames Smart } 137401649561SJames Smart } else { 13755fd11085SJames Smart /* From the icmnd template, initialize words 4 - 11 */ 13765fd11085SJames Smart memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 13775fd11085SJames Smart sizeof(uint32_t) * 8); 13784c47efc1SJames Smart cstat->control_requests++; 137901649561SJames Smart } 13800d8af096SJames Smart 13810d8af096SJames Smart if (pnode->nlp_nvme_info & NLP_NVME_NSLER) 13820d8af096SJames Smart bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 138301649561SJames Smart /* 138401649561SJames Smart * Finish initializing those WQE fields that are independent 138501649561SJames Smart * of the nvme_cmnd request_buffer 138601649561SJames Smart */ 138701649561SJames Smart 13885fd11085SJames Smart /* Word 3 */ 13895fd11085SJames Smart bf_set(payload_offset_len, &wqe->fcp_icmd, 13905fd11085SJames Smart (nCmd->rsplen + nCmd->cmdlen)); 13915fd11085SJames Smart 139201649561SJames Smart /* Word 6 */ 139301649561SJames Smart bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 139401649561SJames Smart phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 139501649561SJames Smart bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 139601649561SJames Smart 139701649561SJames Smart /* Word 8 */ 139801649561SJames Smart wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 139901649561SJames Smart 140001649561SJames Smart /* Word 9 */ 140101649561SJames Smart bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 140201649561SJames Smart 1403414abe0aSJames Smart /* Words 13 14 15 are for PBDE support */ 1404414abe0aSJames Smart 140501649561SJames Smart pwqeq->vport = vport; 140601649561SJames Smart return 0; 140701649561SJames Smart } 140801649561SJames Smart 140901649561SJames Smart 141001649561SJames Smart /** 141101649561SJames Smart * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 141201649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 141301649561SJames Smart * @lpfc_nvme_lport: Pointer to the driver's local port data 141401649561SJames Smart * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 141501649561SJames Smart * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 141601649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 141701649561SJames Smart * 141801649561SJames Smart * Driver registers this routine as it io request handler. This 141901649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 142001649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. 142101649561SJames Smart * 142201649561SJames Smart * Return value : 142301649561SJames Smart * 0 - Success 142401649561SJames Smart * TODO: What are the failure codes. 142501649561SJames Smart **/ 142601649561SJames Smart static int 142701649561SJames Smart lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1428c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd) 142901649561SJames Smart { 143001649561SJames Smart struct lpfc_hba *phba = vport->phba; 143101649561SJames Smart struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1432205e8240SJames Smart union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 14330794d601SJames Smart struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1434d79c9e9dSJames Smart struct sli4_hybrid_sgl *sgl_xtra = NULL; 143501649561SJames Smart struct scatterlist *data_sg; 143601649561SJames Smart struct sli4_sge *first_data_sgl; 14370bc2b7c5SJames Smart struct ulp_bde64 *bde; 1438d79c9e9dSJames Smart dma_addr_t physaddr = 0; 143901649561SJames Smart uint32_t num_bde = 0; 1440d79c9e9dSJames Smart uint32_t dma_len = 0; 144101649561SJames Smart uint32_t dma_offset = 0; 1442d79c9e9dSJames Smart int nseg, i, j; 1443d79c9e9dSJames Smart bool lsp_just_set = false; 144401649561SJames Smart 144501649561SJames Smart /* Fix up the command and response DMA stuff. */ 144601649561SJames Smart lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 144701649561SJames Smart 144801649561SJames Smart /* 144901649561SJames Smart * There are three possibilities here - use scatter-gather segment, use 145001649561SJames Smart * the single mapping, or neither. 145101649561SJames Smart */ 145201649561SJames Smart if (nCmd->sg_cnt) { 145301649561SJames Smart /* 145401649561SJames Smart * Jump over the cmd and rsp SGEs. The fix routine 145501649561SJames Smart * has already adjusted for this. 145601649561SJames Smart */ 145701649561SJames Smart sgl += 2; 145801649561SJames Smart 145901649561SJames Smart first_data_sgl = sgl; 146001649561SJames Smart lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 146181e6a637SJames Smart if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 146201649561SJames Smart lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 146301649561SJames Smart "6058 Too many sg segments from " 146401649561SJames Smart "NVME Transport. Max %d, " 146501649561SJames Smart "nvmeIO sg_cnt %d\n", 146629bfd55aSJames Smart phba->cfg_nvme_seg_cnt + 1, 146701649561SJames Smart lpfc_ncmd->seg_cnt); 146801649561SJames Smart lpfc_ncmd->seg_cnt = 0; 146901649561SJames Smart return 1; 147001649561SJames Smart } 147101649561SJames Smart 147201649561SJames Smart /* 147301649561SJames Smart * The driver established a maximum scatter-gather segment count 147401649561SJames Smart * during probe that limits the number of sg elements in any 147501649561SJames Smart * single nvme command. Just run through the seg_cnt and format 147601649561SJames Smart * the sge's. 147701649561SJames Smart */ 147801649561SJames Smart nseg = nCmd->sg_cnt; 147901649561SJames Smart data_sg = nCmd->first_sgl; 1480d79c9e9dSJames Smart 1481d79c9e9dSJames Smart /* for tracking the segment boundaries */ 1482d79c9e9dSJames Smart j = 2; 148301649561SJames Smart for (i = 0; i < nseg; i++) { 148401649561SJames Smart if (data_sg == NULL) { 148501649561SJames Smart lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 148601649561SJames Smart "6059 dptr err %d, nseg %d\n", 148701649561SJames Smart i, nseg); 148801649561SJames Smart lpfc_ncmd->seg_cnt = 0; 148901649561SJames Smart return 1; 149001649561SJames Smart } 1491d79c9e9dSJames Smart 1492d79c9e9dSJames Smart sgl->word2 = 0; 1493d79c9e9dSJames Smart if ((num_bde + 1) == nseg) { 1494d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1); 1495d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl, 1496d79c9e9dSJames Smart LPFC_SGE_TYPE_DATA); 1497d79c9e9dSJames Smart } else { 1498d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0); 1499d79c9e9dSJames Smart 1500d79c9e9dSJames Smart /* expand the segment */ 1501d79c9e9dSJames Smart if (!lsp_just_set && 1502d79c9e9dSJames Smart !((j + 1) % phba->border_sge_num) && 1503d79c9e9dSJames Smart ((nseg - 1) != i)) { 1504d79c9e9dSJames Smart /* set LSP type */ 1505d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl, 1506d79c9e9dSJames Smart LPFC_SGE_TYPE_LSP); 1507d79c9e9dSJames Smart 1508d79c9e9dSJames Smart sgl_xtra = lpfc_get_sgl_per_hdwq( 1509d79c9e9dSJames Smart phba, lpfc_ncmd); 1510d79c9e9dSJames Smart 1511d79c9e9dSJames Smart if (unlikely(!sgl_xtra)) { 1512d79c9e9dSJames Smart lpfc_ncmd->seg_cnt = 0; 1513d79c9e9dSJames Smart return 1; 1514d79c9e9dSJames Smart } 1515d79c9e9dSJames Smart sgl->addr_lo = cpu_to_le32(putPaddrLow( 1516d79c9e9dSJames Smart sgl_xtra->dma_phys_sgl)); 1517d79c9e9dSJames Smart sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1518d79c9e9dSJames Smart sgl_xtra->dma_phys_sgl)); 1519d79c9e9dSJames Smart 1520d79c9e9dSJames Smart } else { 1521d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_type, sgl, 1522d79c9e9dSJames Smart LPFC_SGE_TYPE_DATA); 1523d79c9e9dSJames Smart } 1524d79c9e9dSJames Smart } 1525d79c9e9dSJames Smart 1526d79c9e9dSJames Smart if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1527d79c9e9dSJames Smart LPFC_SGE_TYPE_LSP)) { 1528d79c9e9dSJames Smart if ((nseg - 1) == i) 1529d79c9e9dSJames Smart bf_set(lpfc_sli4_sge_last, sgl, 1); 1530d79c9e9dSJames Smart 153101649561SJames Smart physaddr = data_sg->dma_address; 153201649561SJames Smart dma_len = data_sg->length; 1533d79c9e9dSJames Smart sgl->addr_lo = cpu_to_le32( 1534d79c9e9dSJames Smart putPaddrLow(physaddr)); 1535d79c9e9dSJames Smart sgl->addr_hi = cpu_to_le32( 1536d79c9e9dSJames Smart putPaddrHigh(physaddr)); 1537d79c9e9dSJames Smart 153801649561SJames Smart bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 153901649561SJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 154001649561SJames Smart sgl->sge_len = cpu_to_le32(dma_len); 154101649561SJames Smart 154201649561SJames Smart dma_offset += dma_len; 154301649561SJames Smart data_sg = sg_next(data_sg); 1544d79c9e9dSJames Smart 154501649561SJames Smart sgl++; 1546d79c9e9dSJames Smart 1547d79c9e9dSJames Smart lsp_just_set = false; 1548d79c9e9dSJames Smart } else { 1549d79c9e9dSJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 1550d79c9e9dSJames Smart 1551d79c9e9dSJames Smart sgl->sge_len = cpu_to_le32( 1552d79c9e9dSJames Smart phba->cfg_sg_dma_buf_size); 1553d79c9e9dSJames Smart 1554d79c9e9dSJames Smart sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1555d79c9e9dSJames Smart i = i - 1; 1556d79c9e9dSJames Smart 1557d79c9e9dSJames Smart lsp_just_set = true; 1558d79c9e9dSJames Smart } 1559d79c9e9dSJames Smart 1560d79c9e9dSJames Smart j++; 156101649561SJames Smart } 1562414abe0aSJames Smart if (phba->cfg_enable_pbde) { 15630bc2b7c5SJames Smart /* Use PBDE support for first SGL only, offset == 0 */ 15640bc2b7c5SJames Smart /* Words 13-15 */ 15650bc2b7c5SJames Smart bde = (struct ulp_bde64 *) 15660bc2b7c5SJames Smart &wqe->words[13]; 15670bc2b7c5SJames Smart bde->addrLow = first_data_sgl->addr_lo; 15680bc2b7c5SJames Smart bde->addrHigh = first_data_sgl->addr_hi; 15690bc2b7c5SJames Smart bde->tus.f.bdeSize = 15700bc2b7c5SJames Smart le32_to_cpu(first_data_sgl->sge_len); 15710bc2b7c5SJames Smart bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 15720bc2b7c5SJames Smart bde->tus.w = cpu_to_le32(bde->tus.w); 15735fd11085SJames Smart /* wqe_pbde is 1 in template */ 15745fd11085SJames Smart } else { 15755fd11085SJames Smart memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 15760bc2b7c5SJames Smart bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 15775fd11085SJames Smart } 15780bc2b7c5SJames Smart 1579414abe0aSJames Smart } else { 15800794d601SJames Smart lpfc_ncmd->seg_cnt = 0; 15810794d601SJames Smart 158201649561SJames Smart /* For this clause to be valid, the payload_length 158301649561SJames Smart * and sg_cnt must zero. 158401649561SJames Smart */ 158501649561SJames Smart if (nCmd->payload_length != 0) { 158601649561SJames Smart lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 158701649561SJames Smart "6063 NVME DMA Prep Err: sg_cnt %d " 158801649561SJames Smart "payload_length x%x\n", 158901649561SJames Smart nCmd->sg_cnt, nCmd->payload_length); 159001649561SJames Smart return 1; 159101649561SJames Smart } 159201649561SJames Smart } 159301649561SJames Smart return 0; 159401649561SJames Smart } 159501649561SJames Smart 159601649561SJames Smart /** 159701649561SJames Smart * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 159801649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 159901649561SJames Smart * @lpfc_nvme_lport: Pointer to the driver's local port data 160001649561SJames Smart * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 160101649561SJames Smart * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 160201649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 160301649561SJames Smart * 160401649561SJames Smart * Driver registers this routine as it io request handler. This 160501649561SJames Smart * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 160601649561SJames Smart * data structure to the rport 160701649561SJames Smart indicated in @lpfc_nvme_rport. 160801649561SJames Smart * 160901649561SJames Smart * Return value : 161001649561SJames Smart * 0 - Success 161101649561SJames Smart * TODO: What are the failure codes. 161201649561SJames Smart **/ 161301649561SJames Smart static int 161401649561SJames Smart lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 161501649561SJames Smart struct nvme_fc_remote_port *pnvme_rport, 161601649561SJames Smart void *hw_queue_handle, 161701649561SJames Smart struct nvmefc_fcp_req *pnvme_fcreq) 161801649561SJames Smart { 161901649561SJames Smart int ret = 0; 1620cf1a1d3eSJames Smart int expedite = 0; 162163df6d63SJames Smart int idx, cpu; 162201649561SJames Smart struct lpfc_nvme_lport *lport; 16234c47efc1SJames Smart struct lpfc_fc4_ctrl_stat *cstat; 162401649561SJames Smart struct lpfc_vport *vport; 162501649561SJames Smart struct lpfc_hba *phba; 162601649561SJames Smart struct lpfc_nodelist *ndlp; 1627c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd; 162801649561SJames Smart struct lpfc_nvme_rport *rport; 162901649561SJames Smart struct lpfc_nvme_qhandle *lpfc_queue_info; 1630c3725bdcSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 1631cf1a1d3eSJames Smart struct nvme_common_command *sqe; 1632bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1633bd2cdd5eSJames Smart uint64_t start = 0; 1634bd2cdd5eSJames Smart #endif 163501649561SJames Smart 1636c3725bdcSJames Smart /* Validate pointers. LLDD fault handling with transport does 1637c3725bdcSJames Smart * have timing races. 1638c3725bdcSJames Smart */ 163901649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1640c3725bdcSJames Smart if (unlikely(!lport)) { 1641c3725bdcSJames Smart ret = -EINVAL; 1642c3725bdcSJames Smart goto out_fail; 1643c3725bdcSJames Smart } 1644c3725bdcSJames Smart 164501649561SJames Smart vport = lport->vport; 1646c3725bdcSJames Smart 1647c3725bdcSJames Smart if (unlikely(!hw_queue_handle)) { 164844c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 164944c2757bSJames Smart "6117 Fail IO, NULL hw_queue_handle\n"); 165044c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 1651cd240071SJames Smart ret = -EBUSY; 1652c3725bdcSJames Smart goto out_fail; 1653c3725bdcSJames Smart } 1654c3725bdcSJames Smart 165501649561SJames Smart phba = vport->phba; 165601649561SJames Smart 16571abcb371SDick Kennedy if (vport->load_flag & FC_UNLOADING) { 16581abcb371SDick Kennedy ret = -ENODEV; 16591abcb371SDick Kennedy goto out_fail; 16601abcb371SDick Kennedy } 16611abcb371SDick Kennedy 16621df09449SJames Smart if (unlikely(vport->load_flag & FC_UNLOADING)) { 166344c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 166444c2757bSJames Smart "6124 Fail IO, Driver unload\n"); 166544c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 16663386f4bdSJames Smart ret = -ENODEV; 16673386f4bdSJames Smart goto out_fail; 16683386f4bdSJames Smart } 16693386f4bdSJames Smart 1670c3725bdcSJames Smart freqpriv = pnvme_fcreq->private; 1671c3725bdcSJames Smart if (unlikely(!freqpriv)) { 167244c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 167344c2757bSJames Smart "6158 Fail IO, NULL request data\n"); 167444c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 1675c3725bdcSJames Smart ret = -EINVAL; 1676b7672ae6SDick Kennedy goto out_fail; 1677b7672ae6SDick Kennedy } 1678b7672ae6SDick Kennedy 1679bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1680bd2cdd5eSJames Smart if (phba->ktime_on) 1681bd2cdd5eSJames Smart start = ktime_get_ns(); 1682bd2cdd5eSJames Smart #endif 168301649561SJames Smart rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 168401649561SJames Smart lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 168501649561SJames Smart 168601649561SJames Smart /* 168701649561SJames Smart * Catch race where our node has transitioned, but the 168801649561SJames Smart * transport is still transitioning. 168901649561SJames Smart */ 169001649561SJames Smart ndlp = rport->ndlp; 169101649561SJames Smart if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 169244c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 169332350664SJames Smart "6053 Busy IO, ndlp not ready: rport x%px " 169432350664SJames Smart "ndlp x%px, DID x%06x\n", 169501649561SJames Smart rport, ndlp, pnvme_rport->port_id); 169644c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 1697cd240071SJames Smart ret = -EBUSY; 169801649561SJames Smart goto out_fail; 169901649561SJames Smart } 170001649561SJames Smart 170101649561SJames Smart /* The remote node has to be a mapped target or it's an error. */ 170201649561SJames Smart if ((ndlp->nlp_type & NLP_NVME_TARGET) && 170301649561SJames Smart (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 170444c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 170544c2757bSJames Smart "6036 Fail IO, DID x%06x not ready for " 1706cd240071SJames Smart "IO. State x%x, Type x%x Flg x%x\n", 170744c2757bSJames Smart pnvme_rport->port_id, 1708cd240071SJames Smart ndlp->nlp_state, ndlp->nlp_type, 1709cd240071SJames Smart ndlp->upcall_flags); 17104b056682SJames Smart atomic_inc(&lport->xmt_fcp_bad_ndlp); 1711cd240071SJames Smart ret = -EBUSY; 171201649561SJames Smart goto out_fail; 171301649561SJames Smart 171401649561SJames Smart } 171501649561SJames Smart 1716cf1a1d3eSJames Smart /* Currently only NVME Keep alive commands should be expedited 1717cf1a1d3eSJames Smart * if the driver runs out of a resource. These should only be 1718cf1a1d3eSJames Smart * issued on the admin queue, qidx 0 1719cf1a1d3eSJames Smart */ 1720cf1a1d3eSJames Smart if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1721cf1a1d3eSJames Smart sqe = &((struct nvme_fc_cmd_iu *) 1722cf1a1d3eSJames Smart pnvme_fcreq->cmdaddr)->sqe.common; 1723cf1a1d3eSJames Smart if (sqe->opcode == nvme_admin_keep_alive) 1724cf1a1d3eSJames Smart expedite = 1; 1725cf1a1d3eSJames Smart } 1726cf1a1d3eSJames Smart 172701649561SJames Smart /* The node is shared with FCP IO, make sure the IO pending count does 172801649561SJames Smart * not exceed the programmed depth. 172901649561SJames Smart */ 17302a5b7d62SJames Smart if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1731cf1a1d3eSJames Smart if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1732cf1a1d3eSJames Smart !expedite) { 173344c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 173444c2757bSJames Smart "6174 Fail IO, ndlp qdepth exceeded: " 17354d5e789aSJames Smart "idx %d DID %x pend %d qdepth %d\n", 17364d5e789aSJames Smart lpfc_queue_info->index, ndlp->nlp_DID, 17374d5e789aSJames Smart atomic_read(&ndlp->cmd_pending), 17384d5e789aSJames Smart ndlp->cmd_qdepth); 17394b056682SJames Smart atomic_inc(&lport->xmt_fcp_qdepth); 1740cd22d605SDick Kennedy ret = -EBUSY; 174101649561SJames Smart goto out_fail; 174201649561SJames Smart } 17432a5b7d62SJames Smart } 174401649561SJames Smart 17456a828b0fSJames Smart /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 174645aa312eSJames Smart if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 174745aa312eSJames Smart idx = lpfc_queue_info->index; 174845aa312eSJames Smart } else { 1749d6d189ceSBart Van Assche cpu = raw_smp_processor_id(); 17506a828b0fSJames Smart idx = phba->sli4_hba.cpu_map[cpu].hdwq; 175145aa312eSJames Smart } 175245aa312eSJames Smart 175345aa312eSJames Smart lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 175401649561SJames Smart if (lpfc_ncmd == NULL) { 17554b056682SJames Smart atomic_inc(&lport->xmt_fcp_noxri); 175601649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 175744c2757bSJames Smart "6065 Fail IO, driver buffer pool is empty: " 175844c2757bSJames Smart "idx %d DID %x\n", 175944c2757bSJames Smart lpfc_queue_info->index, ndlp->nlp_DID); 1760cd22d605SDick Kennedy ret = -EBUSY; 176101649561SJames Smart goto out_fail; 176201649561SJames Smart } 1763bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1764c8a4ce0bSDick Kennedy if (start) { 1765bd2cdd5eSJames Smart lpfc_ncmd->ts_cmd_start = start; 1766bd2cdd5eSJames Smart lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1767c8a4ce0bSDick Kennedy } else { 1768c8a4ce0bSDick Kennedy lpfc_ncmd->ts_cmd_start = 0; 1769bd2cdd5eSJames Smart } 1770bd2cdd5eSJames Smart #endif 177101649561SJames Smart 177201649561SJames Smart /* 177301649561SJames Smart * Store the data needed by the driver to issue, abort, and complete 177401649561SJames Smart * an IO. 177501649561SJames Smart * Do not let the IO hang out forever. There is no midlayer issuing 177601649561SJames Smart * an abort so inform the FW of the maximum IO pending time. 177701649561SJames Smart */ 1778bbe3012bSJames Smart freqpriv->nvme_buf = lpfc_ncmd; 177901649561SJames Smart lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1780318083adSJames Smart lpfc_ncmd->ndlp = ndlp; 17810794d601SJames Smart lpfc_ncmd->qidx = lpfc_queue_info->qidx; 178201649561SJames Smart 178301649561SJames Smart /* 178401649561SJames Smart * Issue the IO on the WQ indicated by index in the hw_queue_handle. 178501649561SJames Smart * This identfier was create in our hardware queue create callback 178601649561SJames Smart * routine. The driver now is dependent on the IO queue steering from 178701649561SJames Smart * the transport. We are trusting the upper NVME layers know which 178801649561SJames Smart * index to use and that they have affinitized a CPU to this hardware 178901649561SJames Smart * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 179001649561SJames Smart */ 179166a210ffSJames Smart lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 17924c47efc1SJames Smart cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 179366a210ffSJames Smart 179466a210ffSJames Smart lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 179566a210ffSJames Smart ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 179666a210ffSJames Smart if (ret) { 179744c2757bSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 179844c2757bSJames Smart "6175 Fail IO, Prep DMA: " 179944c2757bSJames Smart "idx %d DID %x\n", 180044c2757bSJames Smart lpfc_queue_info->index, ndlp->nlp_DID); 180144c2757bSJames Smart atomic_inc(&lport->xmt_fcp_err); 180266a210ffSJames Smart ret = -ENOMEM; 180366a210ffSJames Smart goto out_free_nvme_buf; 180466a210ffSJames Smart } 180566a210ffSJames Smart 1806bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1807bd2cdd5eSJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 1808bd2cdd5eSJames Smart lpfc_queue_info->index, ndlp->nlp_DID); 1809bd2cdd5eSJames Smart 18101fbf9742SJames Smart ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 181101649561SJames Smart if (ret) { 18124b056682SJames Smart atomic_inc(&lport->xmt_fcp_wqerr); 1813e3246a12SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 181444c2757bSJames Smart "6113 Fail IO, Could not issue WQE err %x " 181501649561SJames Smart "sid: x%x did: x%x oxid: x%x\n", 181601649561SJames Smart ret, vport->fc_myDID, ndlp->nlp_DID, 181701649561SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag); 181801649561SJames Smart goto out_free_nvme_buf; 181901649561SJames Smart } 182001649561SJames Smart 1821c490850aSJames Smart if (phba->cfg_xri_rebalancing) 1822c490850aSJames Smart lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1823c490850aSJames Smart 1824bd2cdd5eSJames Smart #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1825c8a4ce0bSDick Kennedy if (lpfc_ncmd->ts_cmd_start) 1826bd2cdd5eSJames Smart lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1827bd2cdd5eSJames Smart 1828840eda96SJames Smart if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) { 1829d6d189ceSBart Van Assche cpu = raw_smp_processor_id(); 1830840eda96SJames Smart this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 183163df6d63SJames Smart lpfc_ncmd->cpu = cpu; 183263df6d63SJames Smart if (idx != cpu) 1833bd2cdd5eSJames Smart lpfc_printf_vlog(vport, 183463df6d63SJames Smart KERN_INFO, LOG_NVME_IOERR, 1835bd2cdd5eSJames Smart "6702 CPU Check cmd: " 1836bd2cdd5eSJames Smart "cpu %d wq %d\n", 1837bd2cdd5eSJames Smart lpfc_ncmd->cpu, 1838bd2cdd5eSJames Smart lpfc_queue_info->index); 1839bd2cdd5eSJames Smart } 1840bd2cdd5eSJames Smart #endif 184101649561SJames Smart return 0; 184201649561SJames Smart 184301649561SJames Smart out_free_nvme_buf: 18442cee7808SJames Smart if (lpfc_ncmd->nvmeCmd->sg_cnt) { 18452cee7808SJames Smart if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 18464c47efc1SJames Smart cstat->output_requests--; 18472cee7808SJames Smart else 18484c47efc1SJames Smart cstat->input_requests--; 18492cee7808SJames Smart } else 18504c47efc1SJames Smart cstat->control_requests--; 185101649561SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 185201649561SJames Smart out_fail: 185301649561SJames Smart return ret; 185401649561SJames Smart } 185501649561SJames Smart 185601649561SJames Smart /** 185701649561SJames Smart * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 185801649561SJames Smart * @phba: Pointer to HBA context object 185901649561SJames Smart * @cmdiocb: Pointer to command iocb object. 186001649561SJames Smart * @rspiocb: Pointer to response iocb object. 186101649561SJames Smart * 186201649561SJames Smart * This is the callback function for any NVME FCP IO that was aborted. 186301649561SJames Smart * 186401649561SJames Smart * Return value: 186501649561SJames Smart * None 186601649561SJames Smart **/ 186701649561SJames Smart void 186801649561SJames Smart lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 186901649561SJames Smart struct lpfc_wcqe_complete *abts_cmpl) 187001649561SJames Smart { 1871e3246a12SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 187201649561SJames Smart "6145 ABORT_XRI_CN completing on rpi x%x " 187301649561SJames Smart "original iotag x%x, abort cmd iotag x%x " 187401649561SJames Smart "req_tag x%x, status x%x, hwstatus x%x\n", 187501649561SJames Smart cmdiocb->iocb.un.acxri.abortContextTag, 187601649561SJames Smart cmdiocb->iocb.un.acxri.abortIoTag, 187701649561SJames Smart cmdiocb->iotag, 187801649561SJames Smart bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 187901649561SJames Smart bf_get(lpfc_wcqe_c_status, abts_cmpl), 188001649561SJames Smart bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 188101649561SJames Smart lpfc_sli_release_iocbq(phba, cmdiocb); 188201649561SJames Smart } 188301649561SJames Smart 188401649561SJames Smart /** 188501649561SJames Smart * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 188601649561SJames Smart * @lpfc_pnvme: Pointer to the driver's nvme instance data 188701649561SJames Smart * @lpfc_nvme_lport: Pointer to the driver's local port data 188801649561SJames Smart * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 188901649561SJames Smart * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 189001649561SJames Smart * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 189101649561SJames Smart * 189201649561SJames Smart * Driver registers this routine as its nvme request io abort handler. This 189301649561SJames Smart * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 189401649561SJames Smart * data structure to the rport indicated in @lpfc_nvme_rport. This routine 189501649561SJames Smart * is executed asynchronously - one the target is validated as "MAPPED" and 189601649561SJames Smart * ready for IO, the driver issues the abort request and returns. 189701649561SJames Smart * 189801649561SJames Smart * Return value: 189901649561SJames Smart * None 190001649561SJames Smart **/ 190101649561SJames Smart static void 190201649561SJames Smart lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 190301649561SJames Smart struct nvme_fc_remote_port *pnvme_rport, 190401649561SJames Smart void *hw_queue_handle, 190501649561SJames Smart struct nvmefc_fcp_req *pnvme_fcreq) 190601649561SJames Smart { 190701649561SJames Smart struct lpfc_nvme_lport *lport; 190801649561SJames Smart struct lpfc_vport *vport; 190901649561SJames Smart struct lpfc_hba *phba; 1910c490850aSJames Smart struct lpfc_io_buf *lpfc_nbuf; 191101649561SJames Smart struct lpfc_iocbq *abts_buf; 191201649561SJames Smart struct lpfc_iocbq *nvmereq_wqe; 1913c3725bdcSJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 191401649561SJames Smart unsigned long flags; 191501649561SJames Smart int ret_val; 191601649561SJames Smart 1917c3725bdcSJames Smart /* Validate pointers. LLDD fault handling with transport does 1918c3725bdcSJames Smart * have timing races. 1919c3725bdcSJames Smart */ 192001649561SJames Smart lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1921c3725bdcSJames Smart if (unlikely(!lport)) 1922c3725bdcSJames Smart return; 192301649561SJames Smart 1924c3725bdcSJames Smart vport = lport->vport; 1925c3725bdcSJames Smart 1926c3725bdcSJames Smart if (unlikely(!hw_queue_handle)) { 1927c3725bdcSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1928c3725bdcSJames Smart "6129 Fail Abort, HW Queue Handle NULL.\n"); 1929c3725bdcSJames Smart return; 1930c3725bdcSJames Smart } 1931c3725bdcSJames Smart 1932c3725bdcSJames Smart phba = vport->phba; 1933c3725bdcSJames Smart freqpriv = pnvme_fcreq->private; 1934c3725bdcSJames Smart 1935c3725bdcSJames Smart if (unlikely(!freqpriv)) 1936c3725bdcSJames Smart return; 19373386f4bdSJames Smart if (vport->load_flag & FC_UNLOADING) 19383386f4bdSJames Smart return; 19393386f4bdSJames Smart 194001649561SJames Smart /* Announce entry to new IO submit field. */ 194186c67379SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 194201649561SJames Smart "6002 Abort Request to rport DID x%06x " 194332350664SJames Smart "for nvme_fc_req x%px\n", 194401649561SJames Smart pnvme_rport->port_id, 194501649561SJames Smart pnvme_fcreq); 194601649561SJames Smart 194701649561SJames Smart /* If the hba is getting reset, this flag is set. It is 194801649561SJames Smart * cleared when the reset is complete and rings reestablished. 194901649561SJames Smart */ 195001649561SJames Smart spin_lock_irqsave(&phba->hbalock, flags); 195101649561SJames Smart /* driver queued commands are in process of being flushed */ 1952c00f62e6SJames Smart if (phba->hba_flag & HBA_IOQ_FLUSH) { 195301649561SJames Smart spin_unlock_irqrestore(&phba->hbalock, flags); 195486c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 195501649561SJames Smart "6139 Driver in reset cleanup - flushing " 195601649561SJames Smart "NVME Req now. hba_flag x%x\n", 195701649561SJames Smart phba->hba_flag); 195801649561SJames Smart return; 195901649561SJames Smart } 196001649561SJames Smart 1961bbe3012bSJames Smart lpfc_nbuf = freqpriv->nvme_buf; 196201649561SJames Smart if (!lpfc_nbuf) { 196301649561SJames Smart spin_unlock_irqrestore(&phba->hbalock, flags); 196486c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 196501649561SJames Smart "6140 NVME IO req has no matching lpfc nvme " 196601649561SJames Smart "io buffer. Skipping abort req.\n"); 196701649561SJames Smart return; 196801649561SJames Smart } else if (!lpfc_nbuf->nvmeCmd) { 196901649561SJames Smart spin_unlock_irqrestore(&phba->hbalock, flags); 197086c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 197101649561SJames Smart "6141 lpfc NVME IO req has no nvme_fcreq " 197201649561SJames Smart "io buffer. Skipping abort req.\n"); 197301649561SJames Smart return; 197401649561SJames Smart } 19752b7824d0SJames Smart nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 197601649561SJames Smart 1977c2017260SJames Smart /* Guard against IO completion being called at same time */ 1978c2017260SJames Smart spin_lock(&lpfc_nbuf->buf_lock); 1979c2017260SJames Smart 198001649561SJames Smart /* 198101649561SJames Smart * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 198201649561SJames Smart * state must match the nvme_fcreq passed by the nvme 198301649561SJames Smart * transport. If they don't match, it is likely the driver 198401649561SJames Smart * has already completed the NVME IO and the nvme transport 198501649561SJames Smart * has not seen it yet. 198601649561SJames Smart */ 198701649561SJames Smart if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 198886c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 198901649561SJames Smart "6143 NVME req mismatch: " 199032350664SJames Smart "lpfc_nbuf x%px nvmeCmd x%px, " 199132350664SJames Smart "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 199201649561SJames Smart lpfc_nbuf, lpfc_nbuf->nvmeCmd, 19932b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1994c2017260SJames Smart goto out_unlock; 199501649561SJames Smart } 199601649561SJames Smart 199701649561SJames Smart /* Don't abort IOs no longer on the pending queue. */ 199801649561SJames Smart if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 199986c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 200032350664SJames Smart "6142 NVME IO req x%px not queued - skipping " 20012b7824d0SJames Smart "abort req xri x%x\n", 20022b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag); 2003c2017260SJames Smart goto out_unlock; 200401649561SJames Smart } 200501649561SJames Smart 20064b056682SJames Smart atomic_inc(&lport->xmt_fcp_abort); 2007bd2cdd5eSJames Smart lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 2008bd2cdd5eSJames Smart nvmereq_wqe->sli4_xritag, 200900cefeb9SJames Smart nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 2010bd2cdd5eSJames Smart 201101649561SJames Smart /* Outstanding abort is in progress */ 201201649561SJames Smart if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 201386c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 201401649561SJames Smart "6144 Outstanding NVME I/O Abort Request " 201532350664SJames Smart "still pending on nvme_fcreq x%px, " 201632350664SJames Smart "lpfc_ncmd %px xri x%x\n", 20172b7824d0SJames Smart pnvme_fcreq, lpfc_nbuf, 20182b7824d0SJames Smart nvmereq_wqe->sli4_xritag); 2019c2017260SJames Smart goto out_unlock; 202001649561SJames Smart } 202101649561SJames Smart 202201649561SJames Smart abts_buf = __lpfc_sli_get_iocbq(phba); 202301649561SJames Smart if (!abts_buf) { 202486c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 202501649561SJames Smart "6136 No available abort wqes. Skipping " 202632350664SJames Smart "Abts req for nvme_fcreq x%px xri x%x\n", 20272b7824d0SJames Smart pnvme_fcreq, nvmereq_wqe->sli4_xritag); 2028c2017260SJames Smart goto out_unlock; 202901649561SJames Smart } 203001649561SJames Smart 203101649561SJames Smart /* Ready - mark outstanding as aborted by driver. */ 203201649561SJames Smart nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 203301649561SJames Smart 203451f8e43eSJames Smart lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0); 203501649561SJames Smart 203601649561SJames Smart /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 203701649561SJames Smart abts_buf->iocb_flag |= LPFC_IO_NVME; 203801649561SJames Smart abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 203901649561SJames Smart abts_buf->vport = vport; 204001649561SJames Smart abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 20411fbf9742SJames Smart ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf); 2042c2017260SJames Smart spin_unlock(&lpfc_nbuf->buf_lock); 204301649561SJames Smart spin_unlock_irqrestore(&phba->hbalock, flags); 2044cd22d605SDick Kennedy if (ret_val) { 204586c67379SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 204601649561SJames Smart "6137 Failed abts issue_wqe with status x%x " 204732350664SJames Smart "for nvme_fcreq x%px.\n", 204801649561SJames Smart ret_val, pnvme_fcreq); 204901649561SJames Smart lpfc_sli_release_iocbq(phba, abts_buf); 205001649561SJames Smart return; 205101649561SJames Smart } 205201649561SJames Smart 205386c67379SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 20542b7824d0SJames Smart "6138 Transport Abort NVME Request Issued for " 205501649561SJames Smart "ox_id x%x on reqtag x%x\n", 205601649561SJames Smart nvmereq_wqe->sli4_xritag, 205701649561SJames Smart abts_buf->iotag); 2058c2017260SJames Smart return; 2059c2017260SJames Smart 2060c2017260SJames Smart out_unlock: 2061c2017260SJames Smart spin_unlock(&lpfc_nbuf->buf_lock); 2062c2017260SJames Smart spin_unlock_irqrestore(&phba->hbalock, flags); 2063c2017260SJames Smart return; 206401649561SJames Smart } 206501649561SJames Smart 206601649561SJames Smart /* Declare and initialization an instance of the FC NVME template. */ 206701649561SJames Smart static struct nvme_fc_port_template lpfc_nvme_template = { 206801649561SJames Smart /* initiator-based functions */ 206901649561SJames Smart .localport_delete = lpfc_nvme_localport_delete, 207001649561SJames Smart .remoteport_delete = lpfc_nvme_remoteport_delete, 207101649561SJames Smart .create_queue = lpfc_nvme_create_queue, 207201649561SJames Smart .delete_queue = lpfc_nvme_delete_queue, 207301649561SJames Smart .ls_req = lpfc_nvme_ls_req, 207401649561SJames Smart .fcp_io = lpfc_nvme_fcp_io_submit, 207501649561SJames Smart .ls_abort = lpfc_nvme_ls_abort, 207601649561SJames Smart .fcp_abort = lpfc_nvme_fcp_abort, 20779aa09e98SJames Smart .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, 207801649561SJames Smart 207901649561SJames Smart .max_hw_queues = 1, 208001649561SJames Smart .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 208101649561SJames Smart .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 208201649561SJames Smart .dma_boundary = 0xFFFFFFFF, 208301649561SJames Smart 208401649561SJames Smart /* Sizes of additional private data for data structures. 208501649561SJames Smart * No use for the last two sizes at this time. 208601649561SJames Smart */ 208701649561SJames Smart .local_priv_sz = sizeof(struct lpfc_nvme_lport), 208801649561SJames Smart .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 208901649561SJames Smart .lsrqst_priv_sz = 0, 2090bbe3012bSJames Smart .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 209101649561SJames Smart }; 209201649561SJames Smart 209301649561SJames Smart /** 20945e5b511dSJames Smart * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 209501649561SJames Smart * @phba: The HBA for which this call is being executed. 209601649561SJames Smart * 20975e5b511dSJames Smart * This routine removes a nvme buffer from head of @hdwq io_buf_list 209801649561SJames Smart * and returns to caller. 209901649561SJames Smart * 210001649561SJames Smart * Return codes: 210101649561SJames Smart * NULL - Error 210201649561SJames Smart * Pointer to lpfc_nvme_buf - Success 210301649561SJames Smart **/ 2104c490850aSJames Smart static struct lpfc_io_buf * 2105cf1a1d3eSJames Smart lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 21065e5b511dSJames Smart int idx, int expedite) 210701649561SJames Smart { 2108c490850aSJames Smart struct lpfc_io_buf *lpfc_ncmd; 21095e5b511dSJames Smart struct lpfc_sli4_hdw_queue *qp; 21100794d601SJames Smart struct sli4_sge *sgl; 21110794d601SJames Smart struct lpfc_iocbq *pwqeq; 21120794d601SJames Smart union lpfc_wqe128 *wqe; 211301649561SJames Smart 2114c490850aSJames Smart lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 21152a5b7d62SJames Smart 21160794d601SJames Smart if (lpfc_ncmd) { 21170794d601SJames Smart pwqeq = &(lpfc_ncmd->cur_iocbq); 21180794d601SJames Smart wqe = &pwqeq->wqe; 21190794d601SJames Smart 21200794d601SJames Smart /* Setup key fields in buffer that may have been changed 21210794d601SJames Smart * if other protocols used this buffer. 21220794d601SJames Smart */ 21230794d601SJames Smart pwqeq->iocb_flag = LPFC_IO_NVME; 21240794d601SJames Smart pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 21250794d601SJames Smart lpfc_ncmd->start_time = jiffies; 21260794d601SJames Smart lpfc_ncmd->flags = 0; 21270794d601SJames Smart 21280794d601SJames Smart /* Rsp SGE will be filled in when we rcv an IO 21290794d601SJames Smart * from the NVME Layer to be sent. 21300794d601SJames Smart * The cmd is going to be embedded so we need a SKIP SGE. 21310794d601SJames Smart */ 21320794d601SJames Smart sgl = lpfc_ncmd->dma_sgl; 21330794d601SJames Smart bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 21340794d601SJames Smart bf_set(lpfc_sli4_sge_last, sgl, 0); 21350794d601SJames Smart sgl->word2 = cpu_to_le32(sgl->word2); 21360794d601SJames Smart /* Fill in word 3 / sgl_len during cmd submission */ 21370794d601SJames Smart 2138d9f492a1SJames Smart /* Initialize 64 bytes only */ 21390794d601SJames Smart memset(wqe, 0, sizeof(union lpfc_wqe)); 21400794d601SJames Smart 21410794d601SJames Smart if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 21422a5b7d62SJames Smart atomic_inc(&ndlp->cmd_pending); 2143c490850aSJames Smart lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 21442a5b7d62SJames Smart } 21455e5b511dSJames Smart 2146c490850aSJames Smart } else { 2147c490850aSJames Smart qp = &phba->sli4_hba.hdwq[idx]; 21485e5b511dSJames Smart qp->empty_io_bufs++; 2149c490850aSJames Smart } 21505e5b511dSJames Smart 215101649561SJames Smart return lpfc_ncmd; 215201649561SJames Smart } 215301649561SJames Smart 215401649561SJames Smart /** 215501649561SJames Smart * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 215601649561SJames Smart * @phba: The Hba for which this call is being executed. 215701649561SJames Smart * @lpfc_ncmd: The nvme buffer which is being released. 215801649561SJames Smart * 215901649561SJames Smart * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 21605e5b511dSJames Smart * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 216101649561SJames Smart * and cannot be reused for at least RA_TOV amount of time if it was 216201649561SJames Smart * aborted. 216301649561SJames Smart **/ 216401649561SJames Smart static void 2165c490850aSJames Smart lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 216601649561SJames Smart { 21675e5b511dSJames Smart struct lpfc_sli4_hdw_queue *qp; 216801649561SJames Smart unsigned long iflag = 0; 216901649561SJames Smart 2170c490850aSJames Smart if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 21712a5b7d62SJames Smart atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 21722a5b7d62SJames Smart 21732a5b7d62SJames Smart lpfc_ncmd->ndlp = NULL; 2174c490850aSJames Smart lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 21752a5b7d62SJames Smart 21761fbf9742SJames Smart qp = lpfc_ncmd->hdwq; 217722770cbaSJames Smart if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 217886c67379SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 217986c67379SJames Smart "6310 XB release deferred for " 218086c67379SJames Smart "ox_id x%x on reqtag x%x\n", 218186c67379SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag, 218286c67379SJames Smart lpfc_ncmd->cur_iocbq.iotag); 218386c67379SJames Smart 2184c00f62e6SJames Smart spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 218501649561SJames Smart list_add_tail(&lpfc_ncmd->list, 2186c00f62e6SJames Smart &qp->lpfc_abts_io_buf_list); 21875e5b511dSJames Smart qp->abts_nvme_io_bufs++; 2188c00f62e6SJames Smart spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2189c490850aSJames Smart } else 2190c490850aSJames Smart lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 219101649561SJames Smart } 219201649561SJames Smart 219301649561SJames Smart /** 219401649561SJames Smart * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 219501649561SJames Smart * @pvport - the lpfc_vport instance requesting a localport. 219601649561SJames Smart * 219701649561SJames Smart * This routine is invoked to create an nvme localport instance to bind 219801649561SJames Smart * to the nvme_fc_transport. It is called once during driver load 219901649561SJames Smart * like lpfc_create_shost after all other services are initialized. 220001649561SJames Smart * It requires a vport, vpi, and wwns at call time. Other localport 220101649561SJames Smart * parameters are modified as the driver's FCID and the Fabric WWN 220201649561SJames Smart * are established. 220301649561SJames Smart * 220401649561SJames Smart * Return codes 220501649561SJames Smart * 0 - successful 220601649561SJames Smart * -ENOMEM - no heap memory available 220701649561SJames Smart * other values - from nvme registration upcall 220801649561SJames Smart **/ 220901649561SJames Smart int 221001649561SJames Smart lpfc_nvme_create_localport(struct lpfc_vport *vport) 221101649561SJames Smart { 2212166d7211SJames Smart int ret = 0; 221301649561SJames Smart struct lpfc_hba *phba = vport->phba; 221401649561SJames Smart struct nvme_fc_port_info nfcp_info; 221501649561SJames Smart struct nvme_fc_local_port *localport; 221601649561SJames Smart struct lpfc_nvme_lport *lport; 221701649561SJames Smart 221801649561SJames Smart /* Initialize this localport instance. The vport wwn usage ensures 221901649561SJames Smart * that NPIV is accounted for. 222001649561SJames Smart */ 222101649561SJames Smart memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 222201649561SJames Smart nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 222301649561SJames Smart nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 222401649561SJames Smart nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 222501649561SJames Smart 22265b9e70b2SJames Smart /* We need to tell the transport layer + 1 because it takes page 22275b9e70b2SJames Smart * alignment into account. When space for the SGL is allocated we 22285b9e70b2SJames Smart * allocate + 3, one for cmd, one for rsp and one for this alignment 22294d4c4a4aSJames Smart */ 22304d4c4a4aSJames Smart lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 223101649561SJames Smart 2232542ddc9bSJames Smart /* Advertise how many hw queues we support based on cfg_hdw_queue, 2233542ddc9bSJames Smart * which will not exceed cpu count. 2234542ddc9bSJames Smart */ 2235cdb42becSJames Smart lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 223666a210ffSJames Smart 2237faf5a744SArnd Bergmann if (!IS_ENABLED(CONFIG_NVME_FC)) 2238faf5a744SArnd Bergmann return ret; 2239faf5a744SArnd Bergmann 224001649561SJames Smart /* localport is allocated from the stack, but the registration 224101649561SJames Smart * call allocates heap memory as well as the private area. 224201649561SJames Smart */ 2243faf5a744SArnd Bergmann 224401649561SJames Smart ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 224501649561SJames Smart &vport->phba->pcidev->dev, &localport); 224601649561SJames Smart if (!ret) { 224701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 224801649561SJames Smart "6005 Successfully registered local " 224932350664SJames Smart "NVME port num %d, localP x%px, private " 225032350664SJames Smart "x%px, sg_seg %d\n", 225101649561SJames Smart localport->port_num, localport, 225201649561SJames Smart localport->private, 225301649561SJames Smart lpfc_nvme_template.max_sgl_segments); 225401649561SJames Smart 225501649561SJames Smart /* Private is our lport size declared in the template. */ 225601649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 225701649561SJames Smart vport->localport = localport; 225801649561SJames Smart lport->vport = vport; 225901649561SJames Smart vport->nvmei_support = 1; 22606b486ce9SDick Kennedy 22614b056682SJames Smart atomic_set(&lport->xmt_fcp_noxri, 0); 22624b056682SJames Smart atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 22634b056682SJames Smart atomic_set(&lport->xmt_fcp_qdepth, 0); 226444c2757bSJames Smart atomic_set(&lport->xmt_fcp_err, 0); 22654b056682SJames Smart atomic_set(&lport->xmt_fcp_wqerr, 0); 22664b056682SJames Smart atomic_set(&lport->xmt_fcp_abort, 0); 22674b056682SJames Smart atomic_set(&lport->xmt_ls_abort, 0); 22684b056682SJames Smart atomic_set(&lport->xmt_ls_err, 0); 22694b056682SJames Smart atomic_set(&lport->cmpl_fcp_xb, 0); 22704b056682SJames Smart atomic_set(&lport->cmpl_fcp_err, 0); 22714b056682SJames Smart atomic_set(&lport->cmpl_ls_xb, 0); 22724b056682SJames Smart atomic_set(&lport->cmpl_ls_err, 0); 22739aa09e98SJames Smart 227466a210ffSJames Smart atomic_set(&lport->fc4NvmeLsRequests, 0); 227566a210ffSJames Smart atomic_set(&lport->fc4NvmeLsCmpls, 0); 22766b486ce9SDick Kennedy } 22773ebd9b47SJames Smart 227801649561SJames Smart return ret; 227901649561SJames Smart } 228001649561SJames Smart 2281d964b3e5SBart Van Assche #if (IS_ENABLED(CONFIG_NVME_FC)) 2282add9d6beSJames Smart /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2283add9d6beSJames Smart * 2284add9d6beSJames Smart * The driver has to wait for the host nvme transport to callback 2285add9d6beSJames Smart * indicating the localport has successfully unregistered all 2286add9d6beSJames Smart * resources. Since this is an uninterruptible wait, loop every ten 2287add9d6beSJames Smart * seconds and print a message indicating no progress. 2288add9d6beSJames Smart * 2289add9d6beSJames Smart * An uninterruptible wait is used because of the risk of transport-to- 2290add9d6beSJames Smart * driver state mismatch. 2291add9d6beSJames Smart */ 22923999df75SBart Van Assche static void 2293add9d6beSJames Smart lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 22947961cba6SEwan D. Milne struct lpfc_nvme_lport *lport, 22957961cba6SEwan D. Milne struct completion *lport_unreg_cmp) 2296add9d6beSJames Smart { 2297add9d6beSJames Smart u32 wait_tmo; 22982ab70c21SJames Smart int ret, i, pending = 0; 22992ab70c21SJames Smart struct lpfc_sli_ring *pring; 23002ab70c21SJames Smart struct lpfc_hba *phba = vport->phba; 2301add9d6beSJames Smart 2302add9d6beSJames Smart /* Host transport has to clean up and confirm requiring an indefinite 2303add9d6beSJames Smart * wait. Print a message if a 10 second wait expires and renew the 2304add9d6beSJames Smart * wait. This is unexpected. 2305add9d6beSJames Smart */ 2306add9d6beSJames Smart wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2307add9d6beSJames Smart while (true) { 23087961cba6SEwan D. Milne ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2309add9d6beSJames Smart if (unlikely(!ret)) { 23102ab70c21SJames Smart pending = 0; 23112ab70c21SJames Smart for (i = 0; i < phba->cfg_hdw_queue; i++) { 2312c00f62e6SJames Smart pring = phba->sli4_hba.hdwq[i].io_wq->pring; 23132ab70c21SJames Smart if (!pring) 23142ab70c21SJames Smart continue; 23152ab70c21SJames Smart if (pring->txcmplq_cnt) 23162ab70c21SJames Smart pending += pring->txcmplq_cnt; 23172ab70c21SJames Smart } 2318add9d6beSJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 231932350664SJames Smart "6176 Lport x%px Localport x%px wait " 23202ab70c21SJames Smart "timed out. Pending %d. Renewing.\n", 23212ab70c21SJames Smart lport, vport->localport, pending); 2322add9d6beSJames Smart continue; 2323add9d6beSJames Smart } 2324add9d6beSJames Smart break; 2325add9d6beSJames Smart } 2326add9d6beSJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 232732350664SJames Smart "6177 Lport x%px Localport x%px Complete Success\n", 2328add9d6beSJames Smart lport, vport->localport); 2329add9d6beSJames Smart } 2330d964b3e5SBart Van Assche #endif 2331add9d6beSJames Smart 233201649561SJames Smart /** 233301649561SJames Smart * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 233401649561SJames Smart * @pnvme: pointer to lpfc nvme data structure. 233501649561SJames Smart * 233601649561SJames Smart * This routine is invoked to destroy all lports bound to the phba. 233701649561SJames Smart * The lport memory was allocated by the nvme fc transport and is 233801649561SJames Smart * released there. This routine ensures all rports bound to the 233901649561SJames Smart * lport have been disconnected. 234001649561SJames Smart * 234101649561SJames Smart **/ 234201649561SJames Smart void 234301649561SJames Smart lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 234401649561SJames Smart { 23457d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 234601649561SJames Smart struct nvme_fc_local_port *localport; 234701649561SJames Smart struct lpfc_nvme_lport *lport; 234801649561SJames Smart int ret; 23497961cba6SEwan D. Milne DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 235001649561SJames Smart 235101649561SJames Smart if (vport->nvmei_support == 0) 235201649561SJames Smart return; 235301649561SJames Smart 235401649561SJames Smart localport = vport->localport; 235501649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 235601649561SJames Smart 235701649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 235832350664SJames Smart "6011 Destroying NVME localport x%px\n", 235901649561SJames Smart localport); 2360166d7211SJames Smart 236101649561SJames Smart /* lport's rport list is clear. Unregister 236201649561SJames Smart * lport and release resources. 236301649561SJames Smart */ 23647961cba6SEwan D. Milne lport->lport_unreg_cmp = &lport_unreg_cmp; 236501649561SJames Smart ret = nvme_fc_unregister_localport(localport); 2366add9d6beSJames Smart 2367add9d6beSJames Smart /* Wait for completion. This either blocks 2368add9d6beSJames Smart * indefinitely or succeeds 2369add9d6beSJames Smart */ 23707961cba6SEwan D. Milne lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 23717961cba6SEwan D. Milne vport->localport = NULL; 237201649561SJames Smart 237301649561SJames Smart /* Regardless of the unregister upcall response, clear 237401649561SJames Smart * nvmei_support. All rports are unregistered and the 237501649561SJames Smart * driver will clean up. 237601649561SJames Smart */ 237701649561SJames Smart vport->nvmei_support = 0; 237801649561SJames Smart if (ret == 0) { 237901649561SJames Smart lpfc_printf_vlog(vport, 238001649561SJames Smart KERN_INFO, LOG_NVME_DISC, 238101649561SJames Smart "6009 Unregistered lport Success\n"); 238201649561SJames Smart } else { 238301649561SJames Smart lpfc_printf_vlog(vport, 238401649561SJames Smart KERN_INFO, LOG_NVME_DISC, 238501649561SJames Smart "6010 Unregistered lport " 238601649561SJames Smart "Failed, status x%x\n", 238701649561SJames Smart ret); 238801649561SJames Smart } 2389166d7211SJames Smart #endif 239001649561SJames Smart } 239101649561SJames Smart 239201649561SJames Smart void 239301649561SJames Smart lpfc_nvme_update_localport(struct lpfc_vport *vport) 239401649561SJames Smart { 23954410a67aSJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 239601649561SJames Smart struct nvme_fc_local_port *localport; 239701649561SJames Smart struct lpfc_nvme_lport *lport; 239801649561SJames Smart 239901649561SJames Smart localport = vport->localport; 24004410a67aSJames Smart if (!localport) { 24014410a67aSJames Smart lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 24024410a67aSJames Smart "6710 Update NVME fail. No localport\n"); 24034410a67aSJames Smart return; 24044410a67aSJames Smart } 240501649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 24064410a67aSJames Smart if (!lport) { 24074410a67aSJames Smart lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 240832350664SJames Smart "6171 Update NVME fail. localP x%px, No lport\n", 24094410a67aSJames Smart localport); 24104410a67aSJames Smart return; 24114410a67aSJames Smart } 241201649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 241332350664SJames Smart "6012 Update NVME lport x%px did x%x\n", 241401649561SJames Smart localport, vport->fc_myDID); 241501649561SJames Smart 241601649561SJames Smart localport->port_id = vport->fc_myDID; 241701649561SJames Smart if (localport->port_id == 0) 241801649561SJames Smart localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 241901649561SJames Smart else 242001649561SJames Smart localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 242101649561SJames Smart 242201649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 242332350664SJames Smart "6030 bound lport x%px to DID x%06x\n", 242401649561SJames Smart lport, localport->port_id); 24254410a67aSJames Smart #endif 242601649561SJames Smart } 242701649561SJames Smart 242801649561SJames Smart int 242901649561SJames Smart lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 243001649561SJames Smart { 24317d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 243201649561SJames Smart int ret = 0; 243301649561SJames Smart struct nvme_fc_local_port *localport; 243401649561SJames Smart struct lpfc_nvme_lport *lport; 243501649561SJames Smart struct lpfc_nvme_rport *rport; 243601466024SJames Smart struct lpfc_nvme_rport *oldrport; 243701649561SJames Smart struct nvme_fc_remote_port *remote_port; 243801649561SJames Smart struct nvme_fc_port_info rpinfo; 243993a3922dSJames Smart struct lpfc_nodelist *prev_ndlp = NULL; 244001649561SJames Smart 244101649561SJames Smart lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 244201649561SJames Smart "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 244301649561SJames Smart ndlp->nlp_DID, ndlp->nlp_type); 244401649561SJames Smart 244501649561SJames Smart localport = vport->localport; 2446bb6a8a2cSDick Kennedy if (!localport) 2447bb6a8a2cSDick Kennedy return 0; 2448bb6a8a2cSDick Kennedy 244901649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 245001649561SJames Smart 24517a06dcd3SJames Smart /* NVME rports are not preserved across devloss. 24527a06dcd3SJames Smart * Just register this instance. Note, rpinfo->dev_loss_tmo 24537a06dcd3SJames Smart * is left 0 to indicate accept transport defaults. The 24547a06dcd3SJames Smart * driver communicates port role capabilities consistent 24557a06dcd3SJames Smart * with the PRLI response data. 245601649561SJames Smart */ 24577a06dcd3SJames Smart memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 24587a06dcd3SJames Smart rpinfo.port_id = ndlp->nlp_DID; 245901649561SJames Smart if (ndlp->nlp_type & NLP_NVME_TARGET) 24607a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 246101649561SJames Smart if (ndlp->nlp_type & NLP_NVME_INITIATOR) 24627a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 246301649561SJames Smart 24647a06dcd3SJames Smart if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 24657a06dcd3SJames Smart rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 24667a06dcd3SJames Smart 24677a06dcd3SJames Smart rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 24687a06dcd3SJames Smart rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 246901466024SJames Smart 24709e210178SJames Smart spin_lock_irq(&vport->phba->hbalock); 247101466024SJames Smart oldrport = lpfc_ndlp_get_nrport(ndlp); 247261184f17SJames Smart if (oldrport) { 247361184f17SJames Smart prev_ndlp = oldrport->ndlp; 24749e210178SJames Smart spin_unlock_irq(&vport->phba->hbalock); 247561184f17SJames Smart } else { 247661184f17SJames Smart spin_unlock_irq(&vport->phba->hbalock); 24773fd78355SJames Smart lpfc_nlp_get(ndlp); 247861184f17SJames Smart } 24793fd78355SJames Smart 24807a06dcd3SJames Smart ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 24817a06dcd3SJames Smart if (!ret) { 24827a06dcd3SJames Smart /* If the ndlp already has an nrport, this is just 24837a06dcd3SJames Smart * a resume of the existing rport. Else this is a 24847a06dcd3SJames Smart * new rport. 24857a06dcd3SJames Smart */ 2486b15bd3e6SJames Smart /* Guard against an unregister/reregister 2487b15bd3e6SJames Smart * race that leaves the WAIT flag set. 2488b15bd3e6SJames Smart */ 2489b15bd3e6SJames Smart spin_lock_irq(&vport->phba->hbalock); 2490b15bd3e6SJames Smart ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2491b15bd3e6SJames Smart spin_unlock_irq(&vport->phba->hbalock); 24927a06dcd3SJames Smart rport = remote_port->private; 249301466024SJames Smart if (oldrport) { 249493a3922dSJames Smart /* New remoteport record does not guarantee valid 249593a3922dSJames Smart * host private memory area. 249693a3922dSJames Smart */ 249701466024SJames Smart if (oldrport == remote_port->private) { 249893a3922dSJames Smart /* Same remoteport - ndlp should match. 249993a3922dSJames Smart * Just reuse. 250093a3922dSJames Smart */ 250101649561SJames Smart lpfc_printf_vlog(ndlp->vport, KERN_INFO, 250201649561SJames Smart LOG_NVME_DISC, 25036825b7bdSJames Smart "6014 Rebind lport to current " 250432350664SJames Smart "remoteport x%px wwpn 0x%llx, " 250532350664SJames Smart "Data: x%x x%x x%px x%px x%x " 250632350664SJames Smart " x%06x\n", 25073fd78355SJames Smart remote_port, 250801649561SJames Smart remote_port->port_name, 250901649561SJames Smart remote_port->port_id, 251001649561SJames Smart remote_port->port_role, 251161184f17SJames Smart oldrport->ndlp, 25123fd78355SJames Smart ndlp, 251301649561SJames Smart ndlp->nlp_type, 251401649561SJames Smart ndlp->nlp_DID); 25156825b7bdSJames Smart 25166825b7bdSJames Smart /* It's a complete rebind only if the driver 25176825b7bdSJames Smart * is registering with the same ndlp. Otherwise 25186825b7bdSJames Smart * the driver likely executed a node swap 25196825b7bdSJames Smart * prior to this registration and the ndlp to 25206825b7bdSJames Smart * remoteport binding needs to be redone. 25216825b7bdSJames Smart */ 25226825b7bdSJames Smart if (prev_ndlp == ndlp) 25233fd78355SJames Smart return 0; 25246825b7bdSJames Smart 25253fd78355SJames Smart } 25262b75d0f9SDick Kennedy 25273fd78355SJames Smart /* Sever the ndlp<->rport association 25283fd78355SJames Smart * before dropping the ndlp ref from 25293fd78355SJames Smart * register. 25302b75d0f9SDick Kennedy */ 25313fd78355SJames Smart spin_lock_irq(&vport->phba->hbalock); 25322b75d0f9SDick Kennedy ndlp->nrport = NULL; 253301466024SJames Smart ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 25343fd78355SJames Smart spin_unlock_irq(&vport->phba->hbalock); 25352b75d0f9SDick Kennedy rport->ndlp = NULL; 25363fd78355SJames Smart rport->remoteport = NULL; 2537b04744ceSJames Smart 2538b04744ceSJames Smart /* Reference only removed if previous NDLP is no longer 2539b04744ceSJames Smart * active. It might be just a swap and removing the 2540b04744ceSJames Smart * reference would cause a premature cleanup. 2541b04744ceSJames Smart */ 2542b04744ceSJames Smart if (prev_ndlp && prev_ndlp != ndlp) { 2543b04744ceSJames Smart if ((!NLP_CHK_NODE_ACT(prev_ndlp)) || 2544b04744ceSJames Smart (!prev_ndlp->nrport)) 2545b04744ceSJames Smart lpfc_nlp_put(prev_ndlp); 2546b04744ceSJames Smart } 25472b75d0f9SDick Kennedy } 25482b75d0f9SDick Kennedy 25492b75d0f9SDick Kennedy /* Clean bind the rport to the ndlp. */ 255001649561SJames Smart rport->remoteport = remote_port; 255101649561SJames Smart rport->lport = lport; 25523fd78355SJames Smart rport->ndlp = ndlp; 25533fd78355SJames Smart spin_lock_irq(&vport->phba->hbalock); 255401649561SJames Smart ndlp->nrport = rport; 25553fd78355SJames Smart spin_unlock_irq(&vport->phba->hbalock); 255601649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, 255701649561SJames Smart LOG_NVME_DISC | LOG_NODE, 25586825b7bdSJames Smart "6022 Bind lport x%px to remoteport x%px " 25596825b7bdSJames Smart "rport x%px WWNN 0x%llx, " 25607a06dcd3SJames Smart "Rport WWPN 0x%llx DID " 256132350664SJames Smart "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 256293a3922dSJames Smart lport, remote_port, rport, 256301649561SJames Smart rpinfo.node_name, rpinfo.port_name, 25643fd78355SJames Smart rpinfo.port_id, rpinfo.port_role, 256593a3922dSJames Smart ndlp, prev_ndlp); 256601649561SJames Smart } else { 256701649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, 256801649561SJames Smart LOG_NVME_DISC | LOG_NODE, 256901649561SJames Smart "6031 RemotePort Registration failed " 257001649561SJames Smart "err: %d, DID x%06x\n", 257101649561SJames Smart ret, ndlp->nlp_DID); 257201649561SJames Smart } 25737a06dcd3SJames Smart 257401649561SJames Smart return ret; 2575166d7211SJames Smart #else 2576166d7211SJames Smart return 0; 2577166d7211SJames Smart #endif 257801649561SJames Smart } 257901649561SJames Smart 25806f2589f4SJames Smart /** 25816f2589f4SJames Smart * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 25826f2589f4SJames Smart * 25836f2589f4SJames Smart * If the ndlp represents an NVME Target, that we are logged into, 25846f2589f4SJames Smart * ping the NVME FC Transport layer to initiate a device rescan 25856f2589f4SJames Smart * on this remote NPort. 25866f2589f4SJames Smart */ 25876f2589f4SJames Smart void 25886f2589f4SJames Smart lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 25896f2589f4SJames Smart { 25906f2589f4SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 2591a6d10f24SJames Smart struct lpfc_nvme_rport *nrport; 2592a6d10f24SJames Smart struct nvme_fc_remote_port *remoteport = NULL; 25936f2589f4SJames Smart 2594a6d10f24SJames Smart spin_lock_irq(&vport->phba->hbalock); 2595a6d10f24SJames Smart nrport = lpfc_ndlp_get_nrport(ndlp); 2596a6d10f24SJames Smart if (nrport) 2597a6d10f24SJames Smart remoteport = nrport->remoteport; 2598a6d10f24SJames Smart spin_unlock_irq(&vport->phba->hbalock); 25996f2589f4SJames Smart 26006f2589f4SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 26016f2589f4SJames Smart "6170 Rescan NPort DID x%06x type x%x " 2602a6d10f24SJames Smart "state x%x nrport x%px remoteport x%px\n", 2603a6d10f24SJames Smart ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2604a6d10f24SJames Smart nrport, remoteport); 2605a6d10f24SJames Smart 2606a6d10f24SJames Smart if (!nrport || !remoteport) 2607a6d10f24SJames Smart goto rescan_exit; 26086f2589f4SJames Smart 26096f2589f4SJames Smart /* Only rescan if we are an NVME target in the MAPPED state */ 26106f2589f4SJames Smart if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 26116f2589f4SJames Smart ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 26126f2589f4SJames Smart nvme_fc_rescan_remoteport(remoteport); 26136f2589f4SJames Smart 26146f2589f4SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 26156f2589f4SJames Smart "6172 NVME rescanned DID x%06x " 26166f2589f4SJames Smart "port_state x%x\n", 26176f2589f4SJames Smart ndlp->nlp_DID, remoteport->port_state); 26186f2589f4SJames Smart } 26196f2589f4SJames Smart return; 2620a6d10f24SJames Smart rescan_exit: 2621a6d10f24SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2622a6d10f24SJames Smart "6169 Skip NVME Rport Rescan, NVME remoteport " 2623a6d10f24SJames Smart "unregistered\n"); 26246f2589f4SJames Smart #endif 26256f2589f4SJames Smart } 26266f2589f4SJames Smart 262701649561SJames Smart /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 262801649561SJames Smart * 262901649561SJames Smart * There is no notion of Devloss or rport recovery from the current 263001649561SJames Smart * nvme_transport perspective. Loss of an rport just means IO cannot 263101649561SJames Smart * be sent and recovery is completely up to the initator. 263201649561SJames Smart * For now, the driver just unbinds the DID and port_role so that 263301649561SJames Smart * no further IO can be issued. Changes are planned for later. 263401649561SJames Smart * 263501649561SJames Smart * Notes - the ndlp reference count is not decremented here since 263601649561SJames Smart * since there is no nvme_transport api for devloss. Node ref count 263701649561SJames Smart * is only adjusted in driver unload. 263801649561SJames Smart */ 263901649561SJames Smart void 264001649561SJames Smart lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 264101649561SJames Smart { 26427d708033SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 264301649561SJames Smart int ret; 264401649561SJames Smart struct nvme_fc_local_port *localport; 264501649561SJames Smart struct lpfc_nvme_lport *lport; 264601649561SJames Smart struct lpfc_nvme_rport *rport; 26479e210178SJames Smart struct nvme_fc_remote_port *remoteport = NULL; 264801649561SJames Smart 264901649561SJames Smart localport = vport->localport; 265001649561SJames Smart 265101649561SJames Smart /* This is fundamental error. The localport is always 265201649561SJames Smart * available until driver unload. Just exit. 265301649561SJames Smart */ 265401649561SJames Smart if (!localport) 265501649561SJames Smart return; 265601649561SJames Smart 265701649561SJames Smart lport = (struct lpfc_nvme_lport *)localport->private; 265801649561SJames Smart if (!lport) 265901649561SJames Smart goto input_err; 266001649561SJames Smart 26619e210178SJames Smart spin_lock_irq(&vport->phba->hbalock); 266201466024SJames Smart rport = lpfc_ndlp_get_nrport(ndlp); 26639e210178SJames Smart if (rport) 26649e210178SJames Smart remoteport = rport->remoteport; 26659e210178SJames Smart spin_unlock_irq(&vport->phba->hbalock); 26669e210178SJames Smart if (!remoteport) 266701649561SJames Smart goto input_err; 266801649561SJames Smart 266901649561SJames Smart lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 267032350664SJames Smart "6033 Unreg nvme remoteport x%px, portname x%llx, " 267101649561SJames Smart "port_id x%06x, portstate x%x port type x%x\n", 267201649561SJames Smart remoteport, remoteport->port_name, 267301649561SJames Smart remoteport->port_id, remoteport->port_state, 267401649561SJames Smart ndlp->nlp_type); 267501649561SJames Smart 267601649561SJames Smart /* Sanity check ndlp type. Only call for NVME ports. Don't 267701649561SJames Smart * clear any rport state until the transport calls back. 267801649561SJames Smart */ 26793b5bde69SJames Smart 26803b5bde69SJames Smart if (ndlp->nlp_type & NLP_NVME_TARGET) { 26817a06dcd3SJames Smart /* No concern about the role change on the nvme remoteport. 26827a06dcd3SJames Smart * The transport will update it. 26837a06dcd3SJames Smart */ 2684add9d6beSJames Smart ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG; 26857438273fSJames Smart 26867438273fSJames Smart /* Don't let the host nvme transport keep sending keep-alives 26877438273fSJames Smart * on this remoteport. Vport is unloading, no recovery. The 26887438273fSJames Smart * return values is ignored. The upcall is a courtesy to the 26897438273fSJames Smart * transport. 26907438273fSJames Smart */ 26917438273fSJames Smart if (vport->load_flag & FC_UNLOADING) 26927438273fSJames Smart (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 26937438273fSJames Smart 269401649561SJames Smart ret = nvme_fc_unregister_remoteport(remoteport); 26953fd78355SJames Smart if (ret != 0) { 26963fd78355SJames Smart lpfc_nlp_put(ndlp); 269701649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 269801649561SJames Smart "6167 NVME unregister failed %d " 269901649561SJames Smart "port_state x%x\n", 270001649561SJames Smart ret, remoteport->port_state); 27013fd78355SJames Smart } 270201649561SJames Smart } 270301649561SJames Smart return; 270401649561SJames Smart 270501649561SJames Smart input_err: 2706166d7211SJames Smart #endif 270701649561SJames Smart lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 270832350664SJames Smart "6168 State error: lport x%px, rport x%px FCID x%06x\n", 270901649561SJames Smart vport->localport, ndlp->rport, ndlp->nlp_DID); 271001649561SJames Smart } 2711318083adSJames Smart 2712318083adSJames Smart /** 2713318083adSJames Smart * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2714318083adSJames Smart * @phba: pointer to lpfc hba data structure. 2715318083adSJames Smart * @axri: pointer to the fcp xri abort wcqe structure. 2716c00f62e6SJames Smart * @lpfc_ncmd: The nvme job structure for the request being aborted. 2717318083adSJames Smart * 2718318083adSJames Smart * This routine is invoked by the worker thread to process a SLI4 fast-path 2719952c303bSDick Kennedy * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2720952c303bSDick Kennedy * here. 2721318083adSJames Smart **/ 2722318083adSJames Smart void 2723318083adSJames Smart lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2724c00f62e6SJames Smart struct sli4_wcqe_xri_aborted *axri, 2725c00f62e6SJames Smart struct lpfc_io_buf *lpfc_ncmd) 2726318083adSJames Smart { 2727318083adSJames Smart uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2728952c303bSDick Kennedy struct nvmefc_fcp_req *nvme_cmd = NULL; 2729c00f62e6SJames Smart struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2730318083adSJames Smart 2731318083adSJames Smart 2732952c303bSDick Kennedy if (ndlp) 2733318083adSJames Smart lpfc_sli4_abts_err_handler(phba, ndlp, axri); 273486c67379SJames Smart 273586c67379SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2736c00f62e6SJames Smart "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2737c00f62e6SJames Smart "xri released\n", 2738952c303bSDick Kennedy lpfc_ncmd->nvmeCmd, xri, 2739952c303bSDick Kennedy lpfc_ncmd->cur_iocbq.iotag); 274086c67379SJames Smart 2741952c303bSDick Kennedy /* Aborted NVME commands are required to not complete 2742952c303bSDick Kennedy * before the abort exchange command fully completes. 2743952c303bSDick Kennedy * Once completed, it is available via the put list. 2744952c303bSDick Kennedy */ 27453fd78355SJames Smart if (lpfc_ncmd->nvmeCmd) { 2746952c303bSDick Kennedy nvme_cmd = lpfc_ncmd->nvmeCmd; 2747952c303bSDick Kennedy nvme_cmd->done(nvme_cmd); 27483fd78355SJames Smart lpfc_ncmd->nvmeCmd = NULL; 27493fd78355SJames Smart } 2750318083adSJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 2751318083adSJames Smart } 2752c3725bdcSJames Smart 2753c3725bdcSJames Smart /** 2754c3725bdcSJames Smart * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2755c3725bdcSJames Smart * @phba: Pointer to HBA context object. 2756c3725bdcSJames Smart * 2757c3725bdcSJames Smart * This function flushes all wqes in the nvme rings and frees all resources 2758c3725bdcSJames Smart * in the txcmplq. This function does not issue abort wqes for the IO 2759c3725bdcSJames Smart * commands in txcmplq, they will just be returned with 2760c3725bdcSJames Smart * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2761c3725bdcSJames Smart * slot has been permanently disabled. 2762c3725bdcSJames Smart **/ 2763c3725bdcSJames Smart void 2764c3725bdcSJames Smart lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2765c3725bdcSJames Smart { 2766c3725bdcSJames Smart struct lpfc_sli_ring *pring; 2767c3725bdcSJames Smart u32 i, wait_cnt = 0; 2768c3725bdcSJames Smart 2769cdb42becSJames Smart if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2770c3725bdcSJames Smart return; 2771c3725bdcSJames Smart 2772c00f62e6SJames Smart /* Cycle through all IO rings and make sure all outstanding 2773c3725bdcSJames Smart * WQEs have been removed from the txcmplqs. 2774c3725bdcSJames Smart */ 2775cdb42becSJames Smart for (i = 0; i < phba->cfg_hdw_queue; i++) { 2776c00f62e6SJames Smart if (!phba->sli4_hba.hdwq[i].io_wq) 27776a828b0fSJames Smart continue; 2778c00f62e6SJames Smart pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2779c3725bdcSJames Smart 2780d580c613SJames Smart if (!pring) 2781d580c613SJames Smart continue; 2782d580c613SJames Smart 2783c3725bdcSJames Smart /* Retrieve everything on the txcmplq */ 2784c3725bdcSJames Smart while (!list_empty(&pring->txcmplq)) { 2785c3725bdcSJames Smart msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2786c3725bdcSJames Smart wait_cnt++; 2787c3725bdcSJames Smart 2788c3725bdcSJames Smart /* The sleep is 10mS. Every ten seconds, 2789c3725bdcSJames Smart * dump a message. Something is wrong. 2790c3725bdcSJames Smart */ 2791c3725bdcSJames Smart if ((wait_cnt % 1000) == 0) { 2792c3725bdcSJames Smart lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2793c3725bdcSJames Smart "6178 NVME IO not empty, " 2794c3725bdcSJames Smart "cnt %d\n", wait_cnt); 2795c3725bdcSJames Smart } 2796c3725bdcSJames Smart } 2797c3725bdcSJames Smart } 2798c3725bdcSJames Smart } 279984f2ddf8SJames Smart 280084f2ddf8SJames Smart void 280184f2ddf8SJames Smart lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn) 280284f2ddf8SJames Smart { 280384f2ddf8SJames Smart #if (IS_ENABLED(CONFIG_NVME_FC)) 280484f2ddf8SJames Smart struct lpfc_io_buf *lpfc_ncmd; 280584f2ddf8SJames Smart struct nvmefc_fcp_req *nCmd; 280684f2ddf8SJames Smart struct lpfc_nvme_fcpreq_priv *freqpriv; 280784f2ddf8SJames Smart 280884f2ddf8SJames Smart if (!pwqeIn->context1) { 280984f2ddf8SJames Smart lpfc_sli_release_iocbq(phba, pwqeIn); 281084f2ddf8SJames Smart return; 281184f2ddf8SJames Smart } 281284f2ddf8SJames Smart /* For abort iocb just return, IO iocb will do a done call */ 281384f2ddf8SJames Smart if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 281484f2ddf8SJames Smart CMD_ABORT_XRI_CX) { 281584f2ddf8SJames Smart lpfc_sli_release_iocbq(phba, pwqeIn); 281684f2ddf8SJames Smart return; 281784f2ddf8SJames Smart } 281884f2ddf8SJames Smart lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1; 281984f2ddf8SJames Smart 282084f2ddf8SJames Smart spin_lock(&lpfc_ncmd->buf_lock); 282184f2ddf8SJames Smart if (!lpfc_ncmd->nvmeCmd) { 282284f2ddf8SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 282384f2ddf8SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 282484f2ddf8SJames Smart return; 282584f2ddf8SJames Smart } 282684f2ddf8SJames Smart 282784f2ddf8SJames Smart nCmd = lpfc_ncmd->nvmeCmd; 282884f2ddf8SJames Smart lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 282984f2ddf8SJames Smart "6194 NVME Cancel xri %x\n", 283084f2ddf8SJames Smart lpfc_ncmd->cur_iocbq.sli4_xritag); 283184f2ddf8SJames Smart 283284f2ddf8SJames Smart nCmd->transferred_length = 0; 283384f2ddf8SJames Smart nCmd->rcv_rsplen = 0; 283484f2ddf8SJames Smart nCmd->status = NVME_SC_INTERNAL; 283584f2ddf8SJames Smart freqpriv = nCmd->private; 283684f2ddf8SJames Smart freqpriv->nvme_buf = NULL; 283784f2ddf8SJames Smart lpfc_ncmd->nvmeCmd = NULL; 283884f2ddf8SJames Smart 283984f2ddf8SJames Smart spin_unlock(&lpfc_ncmd->buf_lock); 284084f2ddf8SJames Smart nCmd->done(nCmd); 284184f2ddf8SJames Smart 284284f2ddf8SJames Smart /* Call release with XB=1 to queue the IO into the abort list. */ 284384f2ddf8SJames Smart lpfc_release_nvme_buf(phba, lpfc_ncmd); 284484f2ddf8SJames Smart #endif 284584f2ddf8SJames Smart } 2846