1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 44c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 2595b60f07SNicholas Bellinger #include <linux/llist.h> 26b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 27b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 29b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 30b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 31531b7bf4SSagi Grimberg #include <linux/semaphore.h> 32b8d26b3bSNicholas Bellinger 33b8d26b3bSNicholas Bellinger #include "isert_proto.h" 34b8d26b3bSNicholas Bellinger #include "ib_isert.h" 35b8d26b3bSNicholas Bellinger 36b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 37b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 38b8d26b3bSNicholas Bellinger #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 39b8d26b3bSNicholas Bellinger 40b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 41b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 42b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_rx_wq; 43b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 44*b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 45b8d26b3bSNicholas Bellinger 46b8d26b3bSNicholas Bellinger static void 47d40945d8SVu Pham isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 48d40945d8SVu Pham static int 49d40945d8SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 50d40945d8SVu Pham struct isert_rdma_wr *wr); 5159464ef4SVu Pham static void 52a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 5359464ef4SVu Pham static int 54a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5559464ef4SVu Pham struct isert_rdma_wr *wr); 56f93f3a70SSagi Grimberg static int 57f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 582371e5daSSagi Grimberg static int 592371e5daSSagi Grimberg isert_rdma_post_recvl(struct isert_conn *isert_conn); 602371e5daSSagi Grimberg static int 612371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 62ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 63d40945d8SVu Pham 64d40945d8SVu Pham static void 65b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 66b8d26b3bSNicholas Bellinger { 67b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)context; 68b8d26b3bSNicholas Bellinger 69b8d26b3bSNicholas Bellinger pr_err("isert_qp_event_callback event: %d\n", e->event); 70b8d26b3bSNicholas Bellinger switch (e->event) { 71b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 72b8d26b3bSNicholas Bellinger rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 73b8d26b3bSNicholas Bellinger break; 74b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 75b8d26b3bSNicholas Bellinger pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 76b8d26b3bSNicholas Bellinger break; 77b8d26b3bSNicholas Bellinger default: 78b8d26b3bSNicholas Bellinger break; 79b8d26b3bSNicholas Bellinger } 80b8d26b3bSNicholas Bellinger } 81b8d26b3bSNicholas Bellinger 82b8d26b3bSNicholas Bellinger static int 83b8d26b3bSNicholas Bellinger isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 84b8d26b3bSNicholas Bellinger { 85b8d26b3bSNicholas Bellinger int ret; 86b8d26b3bSNicholas Bellinger 87b8d26b3bSNicholas Bellinger ret = ib_query_device(ib_dev, devattr); 88b8d26b3bSNicholas Bellinger if (ret) { 89b8d26b3bSNicholas Bellinger pr_err("ib_query_device() failed: %d\n", ret); 90b8d26b3bSNicholas Bellinger return ret; 91b8d26b3bSNicholas Bellinger } 92b8d26b3bSNicholas Bellinger pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 93b8d26b3bSNicholas Bellinger pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 94b8d26b3bSNicholas Bellinger 95b8d26b3bSNicholas Bellinger return 0; 96b8d26b3bSNicholas Bellinger } 97b8d26b3bSNicholas Bellinger 98b8d26b3bSNicholas Bellinger static int 99d3e125daSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id, 100d3e125daSSagi Grimberg u8 protection) 101b8d26b3bSNicholas Bellinger { 102b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 103b8d26b3bSNicholas Bellinger struct ib_qp_init_attr attr; 104b8d26b3bSNicholas Bellinger int ret, index, min_index = 0; 105b8d26b3bSNicholas Bellinger 106b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 107b8d26b3bSNicholas Bellinger for (index = 0; index < device->cqs_used; index++) 108b8d26b3bSNicholas Bellinger if (device->cq_active_qps[index] < 109b8d26b3bSNicholas Bellinger device->cq_active_qps[min_index]) 110b8d26b3bSNicholas Bellinger min_index = index; 111b8d26b3bSNicholas Bellinger device->cq_active_qps[min_index]++; 112b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 113b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 114b8d26b3bSNicholas Bellinger 115b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 116b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 117b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 118b8d26b3bSNicholas Bellinger attr.send_cq = device->dev_tx_cq[min_index]; 119b8d26b3bSNicholas Bellinger attr.recv_cq = device->dev_rx_cq[min_index]; 120b8d26b3bSNicholas Bellinger attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 121b8d26b3bSNicholas Bellinger attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 122b8d26b3bSNicholas Bellinger /* 123b8d26b3bSNicholas Bellinger * FIXME: Use devattr.max_sge - 2 for max_send_sge as 124f57915cfSOr Gerlitz * work-around for RDMA_READs with ConnectX-2. 125f57915cfSOr Gerlitz * 126f57915cfSOr Gerlitz * Also, still make sure to have at least two SGEs for 127f57915cfSOr Gerlitz * outgoing control PDU responses. 128b8d26b3bSNicholas Bellinger */ 129f57915cfSOr Gerlitz attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 130b8d26b3bSNicholas Bellinger isert_conn->max_sge = attr.cap.max_send_sge; 131b8d26b3bSNicholas Bellinger 132b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 133b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 134b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 135d3e125daSSagi Grimberg if (protection) 136d3e125daSSagi Grimberg attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 137b8d26b3bSNicholas Bellinger 138b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 139b8d26b3bSNicholas Bellinger cma_id->device); 140b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 141b8d26b3bSNicholas Bellinger isert_conn->conn_pd->device); 142b8d26b3bSNicholas Bellinger 143b8d26b3bSNicholas Bellinger ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 144b8d26b3bSNicholas Bellinger if (ret) { 145b8d26b3bSNicholas Bellinger pr_err("rdma_create_qp failed for cma_id %d\n", ret); 14619e2090fSSagi Grimberg goto err; 147b8d26b3bSNicholas Bellinger } 148b8d26b3bSNicholas Bellinger isert_conn->conn_qp = cma_id->qp; 149b8d26b3bSNicholas Bellinger pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 150b8d26b3bSNicholas Bellinger 151b8d26b3bSNicholas Bellinger return 0; 15219e2090fSSagi Grimberg err: 15319e2090fSSagi Grimberg mutex_lock(&device_list_mutex); 15419e2090fSSagi Grimberg device->cq_active_qps[min_index]--; 15519e2090fSSagi Grimberg mutex_unlock(&device_list_mutex); 15619e2090fSSagi Grimberg 15719e2090fSSagi Grimberg return ret; 158b8d26b3bSNicholas Bellinger } 159b8d26b3bSNicholas Bellinger 160b8d26b3bSNicholas Bellinger static void 161b8d26b3bSNicholas Bellinger isert_cq_event_callback(struct ib_event *e, void *context) 162b8d26b3bSNicholas Bellinger { 163b8d26b3bSNicholas Bellinger pr_debug("isert_cq_event_callback event: %d\n", e->event); 164b8d26b3bSNicholas Bellinger } 165b8d26b3bSNicholas Bellinger 166b8d26b3bSNicholas Bellinger static int 167b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 168b8d26b3bSNicholas Bellinger { 169b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 170b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 171b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 172b8d26b3bSNicholas Bellinger u64 dma_addr; 173b8d26b3bSNicholas Bellinger int i, j; 174b8d26b3bSNicholas Bellinger 175b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 176b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 177b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 178b8d26b3bSNicholas Bellinger goto fail; 179b8d26b3bSNicholas Bellinger 180b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 181b8d26b3bSNicholas Bellinger 182b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 183b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 184b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 185b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 186b8d26b3bSNicholas Bellinger goto dma_map_fail; 187b8d26b3bSNicholas Bellinger 188b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 189b8d26b3bSNicholas Bellinger 190b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 191b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 192b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 193b8d26b3bSNicholas Bellinger rx_sg->lkey = isert_conn->conn_mr->lkey; 194b8d26b3bSNicholas Bellinger } 195b8d26b3bSNicholas Bellinger 196b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = 0; 197b8d26b3bSNicholas Bellinger return 0; 198b8d26b3bSNicholas Bellinger 199b8d26b3bSNicholas Bellinger dma_map_fail: 200b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 201b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 202b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 203b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 204b8d26b3bSNicholas Bellinger } 205b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 206b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 207b8d26b3bSNicholas Bellinger fail: 208b8d26b3bSNicholas Bellinger return -ENOMEM; 209b8d26b3bSNicholas Bellinger } 210b8d26b3bSNicholas Bellinger 211b8d26b3bSNicholas Bellinger static void 212b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 213b8d26b3bSNicholas Bellinger { 214b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 215b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 216b8d26b3bSNicholas Bellinger int i; 217b8d26b3bSNicholas Bellinger 218b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 219b8d26b3bSNicholas Bellinger return; 220b8d26b3bSNicholas Bellinger 221b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 222b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 223b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 224b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 225b8d26b3bSNicholas Bellinger } 226b8d26b3bSNicholas Bellinger 227b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 228b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 229b8d26b3bSNicholas Bellinger } 230b8d26b3bSNicholas Bellinger 2312853c2b6SNicholas Bellinger static void isert_cq_tx_work(struct work_struct *); 232b8d26b3bSNicholas Bellinger static void isert_cq_tx_callback(struct ib_cq *, void *); 2332853c2b6SNicholas Bellinger static void isert_cq_rx_work(struct work_struct *); 234b8d26b3bSNicholas Bellinger static void isert_cq_rx_callback(struct ib_cq *, void *); 235b8d26b3bSNicholas Bellinger 236b8d26b3bSNicholas Bellinger static int 237b8d26b3bSNicholas Bellinger isert_create_device_ib_res(struct isert_device *device) 238b8d26b3bSNicholas Bellinger { 239b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = device->ib_device; 240b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc; 24159464ef4SVu Pham struct ib_device_attr *dev_attr; 242b8d26b3bSNicholas Bellinger int ret = 0, i, j; 243b1a5ad00SChris Moore int max_rx_cqe, max_tx_cqe; 244b8d26b3bSNicholas Bellinger 24559464ef4SVu Pham dev_attr = &device->dev_attr; 24659464ef4SVu Pham ret = isert_query_device(ib_dev, dev_attr); 24759464ef4SVu Pham if (ret) 24859464ef4SVu Pham return ret; 24959464ef4SVu Pham 250b1a5ad00SChris Moore max_rx_cqe = min(ISER_MAX_RX_CQ_LEN, dev_attr->max_cqe); 251b1a5ad00SChris Moore max_tx_cqe = min(ISER_MAX_TX_CQ_LEN, dev_attr->max_cqe); 252b1a5ad00SChris Moore 253d40945d8SVu Pham /* asign function handlers */ 254f2252258SSagi Grimberg if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 255f2252258SSagi Grimberg dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 256a3a5a826SSagi Grimberg device->use_fastreg = 1; 257a3a5a826SSagi Grimberg device->reg_rdma_mem = isert_reg_rdma; 258a3a5a826SSagi Grimberg device->unreg_rdma_mem = isert_unreg_rdma; 25959464ef4SVu Pham } else { 260a3a5a826SSagi Grimberg device->use_fastreg = 0; 261d40945d8SVu Pham device->reg_rdma_mem = isert_map_rdma; 262d40945d8SVu Pham device->unreg_rdma_mem = isert_unmap_cmd; 26359464ef4SVu Pham } 264d40945d8SVu Pham 265d3e125daSSagi Grimberg /* Check signature cap */ 266d3e125daSSagi Grimberg device->pi_capable = dev_attr->device_cap_flags & 267d3e125daSSagi Grimberg IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 268d3e125daSSagi Grimberg 269b8d26b3bSNicholas Bellinger device->cqs_used = min_t(int, num_online_cpus(), 270b8d26b3bSNicholas Bellinger device->ib_device->num_comp_vectors); 271b8d26b3bSNicholas Bellinger device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 272a3a5a826SSagi Grimberg pr_debug("Using %d CQs, device %s supports %d vectors support " 273d3e125daSSagi Grimberg "Fast registration %d pi_capable %d\n", 274b8d26b3bSNicholas Bellinger device->cqs_used, device->ib_device->name, 275d3e125daSSagi Grimberg device->ib_device->num_comp_vectors, device->use_fastreg, 276d3e125daSSagi Grimberg device->pi_capable); 277b8d26b3bSNicholas Bellinger device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 278b8d26b3bSNicholas Bellinger device->cqs_used, GFP_KERNEL); 279b8d26b3bSNicholas Bellinger if (!device->cq_desc) { 280b8d26b3bSNicholas Bellinger pr_err("Unable to allocate device->cq_desc\n"); 281b8d26b3bSNicholas Bellinger return -ENOMEM; 282b8d26b3bSNicholas Bellinger } 283b8d26b3bSNicholas Bellinger cq_desc = device->cq_desc; 284b8d26b3bSNicholas Bellinger 285b8d26b3bSNicholas Bellinger for (i = 0; i < device->cqs_used; i++) { 286b8d26b3bSNicholas Bellinger cq_desc[i].device = device; 287b8d26b3bSNicholas Bellinger cq_desc[i].cq_index = i; 288b8d26b3bSNicholas Bellinger 2892853c2b6SNicholas Bellinger INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 290b8d26b3bSNicholas Bellinger device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 291b8d26b3bSNicholas Bellinger isert_cq_rx_callback, 292b8d26b3bSNicholas Bellinger isert_cq_event_callback, 293b8d26b3bSNicholas Bellinger (void *)&cq_desc[i], 294b1a5ad00SChris Moore max_rx_cqe, i); 29594a71110SWei Yongjun if (IS_ERR(device->dev_rx_cq[i])) { 29694a71110SWei Yongjun ret = PTR_ERR(device->dev_rx_cq[i]); 29794a71110SWei Yongjun device->dev_rx_cq[i] = NULL; 298b8d26b3bSNicholas Bellinger goto out_cq; 29994a71110SWei Yongjun } 300b8d26b3bSNicholas Bellinger 3012853c2b6SNicholas Bellinger INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 302b8d26b3bSNicholas Bellinger device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 303b8d26b3bSNicholas Bellinger isert_cq_tx_callback, 304b8d26b3bSNicholas Bellinger isert_cq_event_callback, 305b8d26b3bSNicholas Bellinger (void *)&cq_desc[i], 306b1a5ad00SChris Moore max_tx_cqe, i); 30794a71110SWei Yongjun if (IS_ERR(device->dev_tx_cq[i])) { 30894a71110SWei Yongjun ret = PTR_ERR(device->dev_tx_cq[i]); 30994a71110SWei Yongjun device->dev_tx_cq[i] = NULL; 31094a71110SWei Yongjun goto out_cq; 31194a71110SWei Yongjun } 31294a71110SWei Yongjun 31394a71110SWei Yongjun ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 31494a71110SWei Yongjun if (ret) 315b8d26b3bSNicholas Bellinger goto out_cq; 316b8d26b3bSNicholas Bellinger 31794a71110SWei Yongjun ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 31894a71110SWei Yongjun if (ret) 319b8d26b3bSNicholas Bellinger goto out_cq; 320b8d26b3bSNicholas Bellinger } 321b8d26b3bSNicholas Bellinger 322b8d26b3bSNicholas Bellinger return 0; 323b8d26b3bSNicholas Bellinger 324b8d26b3bSNicholas Bellinger out_cq: 325b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++) { 326b8d26b3bSNicholas Bellinger cq_desc = &device->cq_desc[j]; 327b8d26b3bSNicholas Bellinger 328b8d26b3bSNicholas Bellinger if (device->dev_rx_cq[j]) { 329b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_rx_work); 330b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_rx_cq[j]); 331b8d26b3bSNicholas Bellinger } 332b8d26b3bSNicholas Bellinger if (device->dev_tx_cq[j]) { 333b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_tx_work); 334b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_tx_cq[j]); 335b8d26b3bSNicholas Bellinger } 336b8d26b3bSNicholas Bellinger } 337b8d26b3bSNicholas Bellinger kfree(device->cq_desc); 338b8d26b3bSNicholas Bellinger 339b8d26b3bSNicholas Bellinger return ret; 340b8d26b3bSNicholas Bellinger } 341b8d26b3bSNicholas Bellinger 342b8d26b3bSNicholas Bellinger static void 343b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 344b8d26b3bSNicholas Bellinger { 345b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc; 346b8d26b3bSNicholas Bellinger int i; 347b8d26b3bSNicholas Bellinger 348b8d26b3bSNicholas Bellinger for (i = 0; i < device->cqs_used; i++) { 349b8d26b3bSNicholas Bellinger cq_desc = &device->cq_desc[i]; 350b8d26b3bSNicholas Bellinger 351b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_rx_work); 352b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_tx_work); 353b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_rx_cq[i]); 354b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_tx_cq[i]); 355b8d26b3bSNicholas Bellinger device->dev_rx_cq[i] = NULL; 356b8d26b3bSNicholas Bellinger device->dev_tx_cq[i] = NULL; 357b8d26b3bSNicholas Bellinger } 358b8d26b3bSNicholas Bellinger 359b8d26b3bSNicholas Bellinger kfree(device->cq_desc); 360b8d26b3bSNicholas Bellinger } 361b8d26b3bSNicholas Bellinger 362b8d26b3bSNicholas Bellinger static void 363b8d26b3bSNicholas Bellinger isert_device_try_release(struct isert_device *device) 364b8d26b3bSNicholas Bellinger { 365b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 366b8d26b3bSNicholas Bellinger device->refcount--; 367b8d26b3bSNicholas Bellinger if (!device->refcount) { 368b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 369b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 370b8d26b3bSNicholas Bellinger kfree(device); 371b8d26b3bSNicholas Bellinger } 372b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 373b8d26b3bSNicholas Bellinger } 374b8d26b3bSNicholas Bellinger 375b8d26b3bSNicholas Bellinger static struct isert_device * 376b8d26b3bSNicholas Bellinger isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 377b8d26b3bSNicholas Bellinger { 378b8d26b3bSNicholas Bellinger struct isert_device *device; 379b8d26b3bSNicholas Bellinger int ret; 380b8d26b3bSNicholas Bellinger 381b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 382b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 383b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 384b8d26b3bSNicholas Bellinger device->refcount++; 385b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 386b8d26b3bSNicholas Bellinger return device; 387b8d26b3bSNicholas Bellinger } 388b8d26b3bSNicholas Bellinger } 389b8d26b3bSNicholas Bellinger 390b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 391b8d26b3bSNicholas Bellinger if (!device) { 392b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 393b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 394b8d26b3bSNicholas Bellinger } 395b8d26b3bSNicholas Bellinger 396b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 397b8d26b3bSNicholas Bellinger 398b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 399b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 400b8d26b3bSNicholas Bellinger if (ret) { 401b8d26b3bSNicholas Bellinger kfree(device); 402b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 403b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 404b8d26b3bSNicholas Bellinger } 405b8d26b3bSNicholas Bellinger 406b8d26b3bSNicholas Bellinger device->refcount++; 407b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 408b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 409b8d26b3bSNicholas Bellinger 410b8d26b3bSNicholas Bellinger return device; 411b8d26b3bSNicholas Bellinger } 412b8d26b3bSNicholas Bellinger 41359464ef4SVu Pham static void 414a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 41559464ef4SVu Pham { 41659464ef4SVu Pham struct fast_reg_descriptor *fr_desc, *tmp; 41759464ef4SVu Pham int i = 0; 41859464ef4SVu Pham 419a3a5a826SSagi Grimberg if (list_empty(&isert_conn->conn_fr_pool)) 42059464ef4SVu Pham return; 42159464ef4SVu Pham 422a3a5a826SSagi Grimberg pr_debug("Freeing conn %p fastreg pool", isert_conn); 42359464ef4SVu Pham 42459464ef4SVu Pham list_for_each_entry_safe(fr_desc, tmp, 425a3a5a826SSagi Grimberg &isert_conn->conn_fr_pool, list) { 42659464ef4SVu Pham list_del(&fr_desc->list); 42759464ef4SVu Pham ib_free_fast_reg_page_list(fr_desc->data_frpl); 42859464ef4SVu Pham ib_dereg_mr(fr_desc->data_mr); 429d3e125daSSagi Grimberg if (fr_desc->pi_ctx) { 430d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 431d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 432d3e125daSSagi Grimberg ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 433d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 434d3e125daSSagi Grimberg } 43559464ef4SVu Pham kfree(fr_desc); 43659464ef4SVu Pham ++i; 43759464ef4SVu Pham } 43859464ef4SVu Pham 439a3a5a826SSagi Grimberg if (i < isert_conn->conn_fr_pool_size) 44059464ef4SVu Pham pr_warn("Pool still has %d regions registered\n", 441a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size - i); 44259464ef4SVu Pham } 44359464ef4SVu Pham 44459464ef4SVu Pham static int 445dc87a90fSSagi Grimberg isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 446d3e125daSSagi Grimberg struct fast_reg_descriptor *fr_desc, u8 protection) 447dc87a90fSSagi Grimberg { 448d3e125daSSagi Grimberg int ret; 449d3e125daSSagi Grimberg 450dc87a90fSSagi Grimberg fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 451dc87a90fSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 452dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_frpl)) { 453dc87a90fSSagi Grimberg pr_err("Failed to allocate data frpl err=%ld\n", 454dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_frpl)); 455dc87a90fSSagi Grimberg return PTR_ERR(fr_desc->data_frpl); 456dc87a90fSSagi Grimberg } 457dc87a90fSSagi Grimberg 458dc87a90fSSagi Grimberg fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 459dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_mr)) { 460dc87a90fSSagi Grimberg pr_err("Failed to allocate data frmr err=%ld\n", 461dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_mr)); 462d3e125daSSagi Grimberg ret = PTR_ERR(fr_desc->data_mr); 463d3e125daSSagi Grimberg goto err_data_frpl; 464dc87a90fSSagi Grimberg } 465dc87a90fSSagi Grimberg pr_debug("Create fr_desc %p page_list %p\n", 466dc87a90fSSagi Grimberg fr_desc, fr_desc->data_frpl->page_list); 467d3e125daSSagi Grimberg fr_desc->ind |= ISERT_DATA_KEY_VALID; 468dc87a90fSSagi Grimberg 469d3e125daSSagi Grimberg if (protection) { 470d3e125daSSagi Grimberg struct ib_mr_init_attr mr_init_attr = {0}; 471d3e125daSSagi Grimberg struct pi_context *pi_ctx; 472d3e125daSSagi Grimberg 473d3e125daSSagi Grimberg fr_desc->pi_ctx = kzalloc(sizeof(*fr_desc->pi_ctx), GFP_KERNEL); 474d3e125daSSagi Grimberg if (!fr_desc->pi_ctx) { 475d3e125daSSagi Grimberg pr_err("Failed to allocate pi context\n"); 476d3e125daSSagi Grimberg ret = -ENOMEM; 477d3e125daSSagi Grimberg goto err_data_mr; 478d3e125daSSagi Grimberg } 479d3e125daSSagi Grimberg pi_ctx = fr_desc->pi_ctx; 480d3e125daSSagi Grimberg 481d3e125daSSagi Grimberg pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(ib_device, 482d3e125daSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 483d3e125daSSagi Grimberg if (IS_ERR(pi_ctx->prot_frpl)) { 484d3e125daSSagi Grimberg pr_err("Failed to allocate prot frpl err=%ld\n", 485d3e125daSSagi Grimberg PTR_ERR(pi_ctx->prot_frpl)); 486d3e125daSSagi Grimberg ret = PTR_ERR(pi_ctx->prot_frpl); 487d3e125daSSagi Grimberg goto err_pi_ctx; 488d3e125daSSagi Grimberg } 489d3e125daSSagi Grimberg 490d3e125daSSagi Grimberg pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 491d3e125daSSagi Grimberg if (IS_ERR(pi_ctx->prot_mr)) { 492d3e125daSSagi Grimberg pr_err("Failed to allocate prot frmr err=%ld\n", 493d3e125daSSagi Grimberg PTR_ERR(pi_ctx->prot_mr)); 494d3e125daSSagi Grimberg ret = PTR_ERR(pi_ctx->prot_mr); 495d3e125daSSagi Grimberg goto err_prot_frpl; 496d3e125daSSagi Grimberg } 497d3e125daSSagi Grimberg fr_desc->ind |= ISERT_PROT_KEY_VALID; 498d3e125daSSagi Grimberg 499d3e125daSSagi Grimberg mr_init_attr.max_reg_descriptors = 2; 500d3e125daSSagi Grimberg mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 501d3e125daSSagi Grimberg pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 502d3e125daSSagi Grimberg if (IS_ERR(pi_ctx->sig_mr)) { 503d3e125daSSagi Grimberg pr_err("Failed to allocate signature enabled mr err=%ld\n", 504d3e125daSSagi Grimberg PTR_ERR(pi_ctx->sig_mr)); 505d3e125daSSagi Grimberg ret = PTR_ERR(pi_ctx->sig_mr); 506d3e125daSSagi Grimberg goto err_prot_mr; 507d3e125daSSagi Grimberg } 508d3e125daSSagi Grimberg fr_desc->ind |= ISERT_SIG_KEY_VALID; 509d3e125daSSagi Grimberg } 510d3e125daSSagi Grimberg fr_desc->ind &= ~ISERT_PROTECTED; 511dc87a90fSSagi Grimberg 512dc87a90fSSagi Grimberg return 0; 513d3e125daSSagi Grimberg err_prot_mr: 514d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 515d3e125daSSagi Grimberg err_prot_frpl: 516d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 517d3e125daSSagi Grimberg err_pi_ctx: 518d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 519d3e125daSSagi Grimberg err_data_mr: 520d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->data_mr); 521d3e125daSSagi Grimberg err_data_frpl: 522d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->data_frpl); 523d3e125daSSagi Grimberg 524d3e125daSSagi Grimberg return ret; 525dc87a90fSSagi Grimberg } 526dc87a90fSSagi Grimberg 527dc87a90fSSagi Grimberg static int 528d3e125daSSagi Grimberg isert_conn_create_fastreg_pool(struct isert_conn *isert_conn, u8 pi_support) 52959464ef4SVu Pham { 53059464ef4SVu Pham struct fast_reg_descriptor *fr_desc; 53159464ef4SVu Pham struct isert_device *device = isert_conn->conn_device; 532f46d6a8aSNicholas Bellinger struct se_session *se_sess = isert_conn->conn->sess->se_sess; 533f46d6a8aSNicholas Bellinger struct se_node_acl *se_nacl = se_sess->se_node_acl; 534f46d6a8aSNicholas Bellinger int i, ret, tag_num; 535f46d6a8aSNicholas Bellinger /* 536f46d6a8aSNicholas Bellinger * Setup the number of FRMRs based upon the number of tags 537f46d6a8aSNicholas Bellinger * available to session in iscsi_target_locate_portal(). 538f46d6a8aSNicholas Bellinger */ 539f46d6a8aSNicholas Bellinger tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 540f46d6a8aSNicholas Bellinger tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 54159464ef4SVu Pham 542a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size = 0; 543f46d6a8aSNicholas Bellinger for (i = 0; i < tag_num; i++) { 54459464ef4SVu Pham fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 54559464ef4SVu Pham if (!fr_desc) { 54659464ef4SVu Pham pr_err("Failed to allocate fast_reg descriptor\n"); 54759464ef4SVu Pham ret = -ENOMEM; 54859464ef4SVu Pham goto err; 54959464ef4SVu Pham } 55059464ef4SVu Pham 551dc87a90fSSagi Grimberg ret = isert_create_fr_desc(device->ib_device, 552d3e125daSSagi Grimberg isert_conn->conn_pd, fr_desc, 553d3e125daSSagi Grimberg pi_support); 554dc87a90fSSagi Grimberg if (ret) { 555dc87a90fSSagi Grimberg pr_err("Failed to create fastreg descriptor err=%d\n", 556dc87a90fSSagi Grimberg ret); 557a80e21b3SNicholas Bellinger kfree(fr_desc); 55859464ef4SVu Pham goto err; 55959464ef4SVu Pham } 56059464ef4SVu Pham 561a3a5a826SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 562a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size++; 56359464ef4SVu Pham } 56459464ef4SVu Pham 565a3a5a826SSagi Grimberg pr_debug("Creating conn %p fastreg pool size=%d", 566a3a5a826SSagi Grimberg isert_conn, isert_conn->conn_fr_pool_size); 56759464ef4SVu Pham 56859464ef4SVu Pham return 0; 56959464ef4SVu Pham 57059464ef4SVu Pham err: 571a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 57259464ef4SVu Pham return ret; 57359464ef4SVu Pham } 57459464ef4SVu Pham 575b8d26b3bSNicholas Bellinger static int 576b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 577b8d26b3bSNicholas Bellinger { 578ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 579ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 580b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 581b8d26b3bSNicholas Bellinger struct isert_device *device; 582b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = cma_id->device; 583b8d26b3bSNicholas Bellinger int ret = 0; 58414f4b54fSSagi Grimberg u8 pi_support; 58514f4b54fSSagi Grimberg 58614f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 58714f4b54fSSagi Grimberg if (!np->enabled) { 58814f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 58914f4b54fSSagi Grimberg pr_debug("iscsi_np is not enabled, reject connect request\n"); 59014f4b54fSSagi Grimberg return rdma_reject(cma_id, NULL, 0); 59114f4b54fSSagi Grimberg } 59214f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 593b8d26b3bSNicholas Bellinger 594b8d26b3bSNicholas Bellinger pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 595b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 596b8d26b3bSNicholas Bellinger 597b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 598b8d26b3bSNicholas Bellinger if (!isert_conn) { 599b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_conn\n"); 600b8d26b3bSNicholas Bellinger return -ENOMEM; 601b8d26b3bSNicholas Bellinger } 602b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_INIT; 603b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_accept_node); 604b8d26b3bSNicholas Bellinger init_completion(&isert_conn->conn_login_comp); 6052371e5daSSagi Grimberg init_completion(&isert_conn->login_req_comp); 606defd8848SNicholas Bellinger init_completion(&isert_conn->conn_wait); 607defd8848SNicholas Bellinger init_completion(&isert_conn->conn_wait_comp_err); 608b8d26b3bSNicholas Bellinger kref_init(&isert_conn->conn_kref); 609b2cb9649SNicholas Bellinger mutex_init(&isert_conn->conn_mutex); 61059464ef4SVu Pham spin_lock_init(&isert_conn->conn_lock); 611f46d6a8aSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 612b8d26b3bSNicholas Bellinger 613b8d26b3bSNicholas Bellinger isert_conn->conn_cm_id = cma_id; 614b8d26b3bSNicholas Bellinger 615b8d26b3bSNicholas Bellinger isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 616b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, GFP_KERNEL); 617b8d26b3bSNicholas Bellinger if (!isert_conn->login_buf) { 618b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_conn->login_buf\n"); 619b8d26b3bSNicholas Bellinger ret = -ENOMEM; 620b8d26b3bSNicholas Bellinger goto out; 621b8d26b3bSNicholas Bellinger } 622b8d26b3bSNicholas Bellinger 623b8d26b3bSNicholas Bellinger isert_conn->login_req_buf = isert_conn->login_buf; 624b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf = isert_conn->login_buf + 625b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN; 626b8d26b3bSNicholas Bellinger pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 627b8d26b3bSNicholas Bellinger isert_conn->login_buf, isert_conn->login_req_buf, 628b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf); 629b8d26b3bSNicholas Bellinger 630b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 631b8d26b3bSNicholas Bellinger (void *)isert_conn->login_req_buf, 632b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 633b8d26b3bSNicholas Bellinger 634b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 635b8d26b3bSNicholas Bellinger if (ret) { 636b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 637b8d26b3bSNicholas Bellinger ret); 638b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = 0; 639b8d26b3bSNicholas Bellinger goto out_login_buf; 640b8d26b3bSNicholas Bellinger } 641b8d26b3bSNicholas Bellinger 642b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 643b8d26b3bSNicholas Bellinger (void *)isert_conn->login_rsp_buf, 644b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 645b8d26b3bSNicholas Bellinger 646b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 647b8d26b3bSNicholas Bellinger if (ret) { 648b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 649b8d26b3bSNicholas Bellinger ret); 650b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = 0; 651b8d26b3bSNicholas Bellinger goto out_req_dma_map; 652b8d26b3bSNicholas Bellinger } 653b8d26b3bSNicholas Bellinger 654b8d26b3bSNicholas Bellinger device = isert_device_find_by_ib_dev(cma_id); 655b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 656b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 657b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 658b8d26b3bSNicholas Bellinger } 659b8d26b3bSNicholas Bellinger 6601a92e17eSSagi Grimberg /* Set max inflight RDMA READ requests */ 6611a92e17eSSagi Grimberg isert_conn->initiator_depth = min_t(u8, 6621a92e17eSSagi Grimberg event->param.conn.initiator_depth, 6631a92e17eSSagi Grimberg device->dev_attr.max_qp_init_rd_atom); 6641a92e17eSSagi Grimberg pr_debug("Using initiator_depth: %u\n", isert_conn->initiator_depth); 6651a92e17eSSagi Grimberg 666b8d26b3bSNicholas Bellinger isert_conn->conn_device = device; 667eb6ab132SSagi Grimberg isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 668eb6ab132SSagi Grimberg if (IS_ERR(isert_conn->conn_pd)) { 669eb6ab132SSagi Grimberg ret = PTR_ERR(isert_conn->conn_pd); 670eb6ab132SSagi Grimberg pr_err("ib_alloc_pd failed for conn %p: ret=%d\n", 671eb6ab132SSagi Grimberg isert_conn, ret); 672eb6ab132SSagi Grimberg goto out_pd; 673eb6ab132SSagi Grimberg } 674b8d26b3bSNicholas Bellinger 675eb6ab132SSagi Grimberg isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 676eb6ab132SSagi Grimberg IB_ACCESS_LOCAL_WRITE); 677eb6ab132SSagi Grimberg if (IS_ERR(isert_conn->conn_mr)) { 678eb6ab132SSagi Grimberg ret = PTR_ERR(isert_conn->conn_mr); 679eb6ab132SSagi Grimberg pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 680eb6ab132SSagi Grimberg isert_conn, ret); 681eb6ab132SSagi Grimberg goto out_mr; 682eb6ab132SSagi Grimberg } 683b8d26b3bSNicholas Bellinger 68414f4b54fSSagi Grimberg pi_support = np->tpg_np->tpg->tpg_attrib.t10_pi; 685d3e125daSSagi Grimberg if (pi_support && !device->pi_capable) { 6865f80ff8eSSagi Grimberg pr_err("Protection information requested but not supported, " 6875f80ff8eSSagi Grimberg "rejecting connect request\n"); 6885f80ff8eSSagi Grimberg ret = rdma_reject(cma_id, NULL, 0); 689d3e125daSSagi Grimberg goto out_mr; 690d3e125daSSagi Grimberg } 691d3e125daSSagi Grimberg 692d3e125daSSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id, pi_support); 693b8d26b3bSNicholas Bellinger if (ret) 694b8d26b3bSNicholas Bellinger goto out_conn_dev; 695b8d26b3bSNicholas Bellinger 6962371e5daSSagi Grimberg ret = isert_rdma_post_recvl(isert_conn); 6972371e5daSSagi Grimberg if (ret) 6982371e5daSSagi Grimberg goto out_conn_dev; 6992371e5daSSagi Grimberg 7002371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 7012371e5daSSagi Grimberg if (ret) 7022371e5daSSagi Grimberg goto out_conn_dev; 7032371e5daSSagi Grimberg 704b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 7059fe63c88SSagi Grimberg list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list); 706b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 707b8d26b3bSNicholas Bellinger 708531b7bf4SSagi Grimberg pr_debug("isert_connect_request() up np_sem np: %p\n", np); 709531b7bf4SSagi Grimberg up(&isert_np->np_sem); 710b8d26b3bSNicholas Bellinger return 0; 711b8d26b3bSNicholas Bellinger 712b8d26b3bSNicholas Bellinger out_conn_dev: 713eb6ab132SSagi Grimberg ib_dereg_mr(isert_conn->conn_mr); 714eb6ab132SSagi Grimberg out_mr: 715eb6ab132SSagi Grimberg ib_dealloc_pd(isert_conn->conn_pd); 716eb6ab132SSagi Grimberg out_pd: 717b8d26b3bSNicholas Bellinger isert_device_try_release(device); 718b8d26b3bSNicholas Bellinger out_rsp_dma_map: 719b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 720b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 721b8d26b3bSNicholas Bellinger out_req_dma_map: 722b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 723b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 724b8d26b3bSNicholas Bellinger out_login_buf: 725b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 726b8d26b3bSNicholas Bellinger out: 727b8d26b3bSNicholas Bellinger kfree(isert_conn); 7282371e5daSSagi Grimberg rdma_reject(cma_id, NULL, 0); 729b8d26b3bSNicholas Bellinger return ret; 730b8d26b3bSNicholas Bellinger } 731b8d26b3bSNicholas Bellinger 732b8d26b3bSNicholas Bellinger static void 733b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 734b8d26b3bSNicholas Bellinger { 735b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 736b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 737b8d26b3bSNicholas Bellinger int cq_index; 738b8d26b3bSNicholas Bellinger 739b8d26b3bSNicholas Bellinger pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 740b8d26b3bSNicholas Bellinger 741a3a5a826SSagi Grimberg if (device && device->use_fastreg) 742a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 74359464ef4SVu Pham 74419e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 74519e2090fSSagi Grimberg rdma_destroy_id(isert_conn->conn_cm_id); 74619e2090fSSagi Grimberg 747b8d26b3bSNicholas Bellinger if (isert_conn->conn_qp) { 748b8d26b3bSNicholas Bellinger cq_index = ((struct isert_cq_desc *) 749b8d26b3bSNicholas Bellinger isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 750b8d26b3bSNicholas Bellinger pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 75119e2090fSSagi Grimberg mutex_lock(&device_list_mutex); 752b8d26b3bSNicholas Bellinger isert_conn->conn_device->cq_active_qps[cq_index]--; 75319e2090fSSagi Grimberg mutex_unlock(&device_list_mutex); 754b8d26b3bSNicholas Bellinger 75519e2090fSSagi Grimberg ib_destroy_qp(isert_conn->conn_qp); 756b8d26b3bSNicholas Bellinger } 757b8d26b3bSNicholas Bellinger 758eb6ab132SSagi Grimberg ib_dereg_mr(isert_conn->conn_mr); 759eb6ab132SSagi Grimberg ib_dealloc_pd(isert_conn->conn_pd); 760eb6ab132SSagi Grimberg 761b8d26b3bSNicholas Bellinger if (isert_conn->login_buf) { 762b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 763b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 764b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 765b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, 766b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 767b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 768b8d26b3bSNicholas Bellinger } 769b8d26b3bSNicholas Bellinger kfree(isert_conn); 770b8d26b3bSNicholas Bellinger 771b8d26b3bSNicholas Bellinger if (device) 772b8d26b3bSNicholas Bellinger isert_device_try_release(device); 773b8d26b3bSNicholas Bellinger 774b8d26b3bSNicholas Bellinger pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 775b8d26b3bSNicholas Bellinger } 776b8d26b3bSNicholas Bellinger 777b8d26b3bSNicholas Bellinger static void 778b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 779b8d26b3bSNicholas Bellinger { 78019e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 781c2f88b17SSagi Grimberg 782128e9cc8SSagi Grimberg pr_info("conn %p\n", isert_conn); 783128e9cc8SSagi Grimberg 7842371e5daSSagi Grimberg if (!kref_get_unless_zero(&isert_conn->conn_kref)) { 7852371e5daSSagi Grimberg pr_warn("conn %p connect_release is running\n", isert_conn); 7862371e5daSSagi Grimberg return; 7872371e5daSSagi Grimberg } 7882371e5daSSagi Grimberg 7892371e5daSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 7902371e5daSSagi Grimberg if (isert_conn->state != ISER_CONN_FULL_FEATURE) 791128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 7922371e5daSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 793b8d26b3bSNicholas Bellinger } 794b8d26b3bSNicholas Bellinger 795b8d26b3bSNicholas Bellinger static void 796b8d26b3bSNicholas Bellinger isert_release_conn_kref(struct kref *kref) 797b8d26b3bSNicholas Bellinger { 798b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 799b8d26b3bSNicholas Bellinger struct isert_conn, conn_kref); 800b8d26b3bSNicholas Bellinger 801b8d26b3bSNicholas Bellinger pr_debug("Calling isert_connect_release for final kref %s/%d\n", 802b8d26b3bSNicholas Bellinger current->comm, current->pid); 803b8d26b3bSNicholas Bellinger 804b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 805b8d26b3bSNicholas Bellinger } 806b8d26b3bSNicholas Bellinger 807b8d26b3bSNicholas Bellinger static void 808b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 809b8d26b3bSNicholas Bellinger { 810b8d26b3bSNicholas Bellinger kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 811b8d26b3bSNicholas Bellinger } 812b8d26b3bSNicholas Bellinger 813954f2372SSagi Grimberg /** 814954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 815954f2372SSagi Grimberg * @isert_conn: isert connection struct 816954f2372SSagi Grimberg * 817954f2372SSagi Grimberg * Notes: 818128e9cc8SSagi Grimberg * In case the connection state is FULL_FEATURE, move state 819954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 820128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 821954f2372SSagi Grimberg * 822954f2372SSagi Grimberg * This routine must be called with conn_mutex held. Thus it is 823954f2372SSagi Grimberg * safe to call multiple times. 824954f2372SSagi Grimberg */ 825954f2372SSagi Grimberg static void 826954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 827954f2372SSagi Grimberg { 828954f2372SSagi Grimberg int err; 829954f2372SSagi Grimberg 830128e9cc8SSagi Grimberg switch (isert_conn->state) { 831128e9cc8SSagi Grimberg case ISER_CONN_TERMINATING: 832128e9cc8SSagi Grimberg break; 833128e9cc8SSagi Grimberg case ISER_CONN_UP: 834128e9cc8SSagi Grimberg /* 835128e9cc8SSagi Grimberg * No flush completions will occur as we didn't 836128e9cc8SSagi Grimberg * get to ISER_CONN_FULL_FEATURE yet, complete 837128e9cc8SSagi Grimberg * to allow teardown progress. 838128e9cc8SSagi Grimberg */ 839128e9cc8SSagi Grimberg complete(&isert_conn->conn_wait_comp_err); 840128e9cc8SSagi Grimberg case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 841954f2372SSagi Grimberg pr_info("Terminating conn %p state %d\n", 842954f2372SSagi Grimberg isert_conn, isert_conn->state); 843128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 844954f2372SSagi Grimberg err = rdma_disconnect(isert_conn->conn_cm_id); 845954f2372SSagi Grimberg if (err) 846954f2372SSagi Grimberg pr_warn("Failed rdma_disconnect isert_conn %p\n", 847954f2372SSagi Grimberg isert_conn); 848128e9cc8SSagi Grimberg break; 849128e9cc8SSagi Grimberg default: 850128e9cc8SSagi Grimberg pr_warn("conn %p teminating in state %d\n", 851128e9cc8SSagi Grimberg isert_conn, isert_conn->state); 852954f2372SSagi Grimberg } 853954f2372SSagi Grimberg } 854954f2372SSagi Grimberg 8553b726ae2SSagi Grimberg static int 856ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 857ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 858b8d26b3bSNicholas Bellinger { 859ca6c1d82SSagi Grimberg pr_debug("isert np %p, handling event %d\n", isert_np, event); 8603b726ae2SSagi Grimberg 861ca6c1d82SSagi Grimberg switch (event) { 862ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 8633b726ae2SSagi Grimberg isert_np->np_cm_id = NULL; 864ca6c1d82SSagi Grimberg break; 865ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 866ca6c1d82SSagi Grimberg isert_np->np_cm_id = isert_setup_id(isert_np); 867ca6c1d82SSagi Grimberg if (IS_ERR(isert_np->np_cm_id)) { 868ca6c1d82SSagi Grimberg pr_err("isert np %p setup id failed: %ld\n", 869ca6c1d82SSagi Grimberg isert_np, PTR_ERR(isert_np->np_cm_id)); 870ca6c1d82SSagi Grimberg isert_np->np_cm_id = NULL; 871ca6c1d82SSagi Grimberg } 872ca6c1d82SSagi Grimberg break; 873ca6c1d82SSagi Grimberg default: 874ca6c1d82SSagi Grimberg pr_err("isert np %p Unexpected event %d\n", 875ca6c1d82SSagi Grimberg isert_np, event); 876ca6c1d82SSagi Grimberg } 877ca6c1d82SSagi Grimberg 8783b726ae2SSagi Grimberg return -1; 8793b726ae2SSagi Grimberg } 8803b726ae2SSagi Grimberg 881ca6c1d82SSagi Grimberg static int 882ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 883ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 884ca6c1d82SSagi Grimberg { 885ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 886ca6c1d82SSagi Grimberg struct isert_conn *isert_conn; 887ca6c1d82SSagi Grimberg 888ca6c1d82SSagi Grimberg if (isert_np->np_cm_id == cma_id) 889ca6c1d82SSagi Grimberg return isert_np_cma_handler(cma_id->context, event); 890ca6c1d82SSagi Grimberg 89119e2090fSSagi Grimberg isert_conn = cma_id->qp->qp_context; 892b8d26b3bSNicholas Bellinger 893128e9cc8SSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 894128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 895128e9cc8SSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 896128e9cc8SSagi Grimberg 897128e9cc8SSagi Grimberg pr_info("conn %p completing conn_wait\n", isert_conn); 898128e9cc8SSagi Grimberg complete(&isert_conn->conn_wait); 8993b726ae2SSagi Grimberg 9003b726ae2SSagi Grimberg return 0; 901b8d26b3bSNicholas Bellinger } 902b8d26b3bSNicholas Bellinger 903954f2372SSagi Grimberg static void 904954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 905954f2372SSagi Grimberg { 90619e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 907954f2372SSagi Grimberg 908954f2372SSagi Grimberg isert_put_conn(isert_conn); 909954f2372SSagi Grimberg } 910954f2372SSagi Grimberg 911b8d26b3bSNicholas Bellinger static int 912b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 913b8d26b3bSNicholas Bellinger { 914b8d26b3bSNicholas Bellinger int ret = 0; 915b8d26b3bSNicholas Bellinger 916b8d26b3bSNicholas Bellinger pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 917b8d26b3bSNicholas Bellinger event->event, event->status, cma_id->context, cma_id); 918b8d26b3bSNicholas Bellinger 919b8d26b3bSNicholas Bellinger switch (event->event) { 920b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 921b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 9223b726ae2SSagi Grimberg if (ret) 9233b726ae2SSagi Grimberg pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 9243b726ae2SSagi Grimberg event->event, ret); 925b8d26b3bSNicholas Bellinger break; 926b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 927b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 928b8d26b3bSNicholas Bellinger break; 92988c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 93088c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 93188c4015fSSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 93288c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 933ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 934b8d26b3bSNicholas Bellinger break; 935954f2372SSagi Grimberg case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 936954f2372SSagi Grimberg case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 937b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 938954f2372SSagi Grimberg isert_connect_error(cma_id); 939954f2372SSagi Grimberg break; 940b8d26b3bSNicholas Bellinger default: 94188c4015fSSagi Grimberg pr_err("Unhandled RDMA CMA event: %d\n", event->event); 942b8d26b3bSNicholas Bellinger break; 943b8d26b3bSNicholas Bellinger } 944b8d26b3bSNicholas Bellinger 945b8d26b3bSNicholas Bellinger return ret; 946b8d26b3bSNicholas Bellinger } 947b8d26b3bSNicholas Bellinger 948b8d26b3bSNicholas Bellinger static int 949b8d26b3bSNicholas Bellinger isert_post_recv(struct isert_conn *isert_conn, u32 count) 950b8d26b3bSNicholas Bellinger { 951b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 952b8d26b3bSNicholas Bellinger int i, ret; 953b8d26b3bSNicholas Bellinger unsigned int rx_head = isert_conn->conn_rx_desc_head; 954b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 955b8d26b3bSNicholas Bellinger 956b8d26b3bSNicholas Bellinger for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 957b8d26b3bSNicholas Bellinger rx_desc = &isert_conn->conn_rx_descs[rx_head]; 958b8d26b3bSNicholas Bellinger rx_wr->wr_id = (unsigned long)rx_desc; 959b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 960b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 961b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 962b8d26b3bSNicholas Bellinger rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 963b8d26b3bSNicholas Bellinger } 964b8d26b3bSNicholas Bellinger 965b8d26b3bSNicholas Bellinger rx_wr--; 966b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 967b8d26b3bSNicholas Bellinger 968b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count += count; 969b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 970b8d26b3bSNicholas Bellinger &rx_wr_failed); 971b8d26b3bSNicholas Bellinger if (ret) { 972b8d26b3bSNicholas Bellinger pr_err("ib_post_recv() failed with ret: %d\n", ret); 973b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count -= count; 974b8d26b3bSNicholas Bellinger } else { 975b8d26b3bSNicholas Bellinger pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 976b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = rx_head; 977b8d26b3bSNicholas Bellinger } 978b8d26b3bSNicholas Bellinger return ret; 979b8d26b3bSNicholas Bellinger } 980b8d26b3bSNicholas Bellinger 981b8d26b3bSNicholas Bellinger static int 982b8d26b3bSNicholas Bellinger isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 983b8d26b3bSNicholas Bellinger { 984b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 985b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 986b8d26b3bSNicholas Bellinger int ret; 987b8d26b3bSNicholas Bellinger 988b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 989b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 990b8d26b3bSNicholas Bellinger 991b8d26b3bSNicholas Bellinger send_wr.next = NULL; 992b8d26b3bSNicholas Bellinger send_wr.wr_id = (unsigned long)tx_desc; 993b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 994b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 995b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 996b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 997b8d26b3bSNicholas Bellinger 998b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 999b8d26b3bSNicholas Bellinger 1000b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 1001b8d26b3bSNicholas Bellinger if (ret) { 1002b8d26b3bSNicholas Bellinger pr_err("ib_post_send() failed, ret: %d\n", ret); 1003b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1004b8d26b3bSNicholas Bellinger } 1005b8d26b3bSNicholas Bellinger 1006b8d26b3bSNicholas Bellinger return ret; 1007b8d26b3bSNicholas Bellinger } 1008b8d26b3bSNicholas Bellinger 1009b8d26b3bSNicholas Bellinger static void 1010b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 1011b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1012b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1013b8d26b3bSNicholas Bellinger { 1014b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1015b8d26b3bSNicholas Bellinger 1016b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1017b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1018b8d26b3bSNicholas Bellinger 1019b8d26b3bSNicholas Bellinger memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 1020b8d26b3bSNicholas Bellinger tx_desc->iser_header.flags = ISER_VER; 1021b8d26b3bSNicholas Bellinger 1022b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 1023b8d26b3bSNicholas Bellinger tx_desc->isert_cmd = isert_cmd; 1024b8d26b3bSNicholas Bellinger 1025b8d26b3bSNicholas Bellinger if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 1026b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 1027b8d26b3bSNicholas Bellinger pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1028b8d26b3bSNicholas Bellinger } 1029b8d26b3bSNicholas Bellinger } 1030b8d26b3bSNicholas Bellinger 1031b8d26b3bSNicholas Bellinger static int 1032b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 1033b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1034b8d26b3bSNicholas Bellinger { 1035b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1036b8d26b3bSNicholas Bellinger u64 dma_addr; 1037b8d26b3bSNicholas Bellinger 1038b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1039b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1040b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1041b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error() failed\n"); 1042b8d26b3bSNicholas Bellinger return -ENOMEM; 1043b8d26b3bSNicholas Bellinger } 1044b8d26b3bSNicholas Bellinger 1045b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 1046b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1047b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1048b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 1049b8d26b3bSNicholas Bellinger 1050b8d26b3bSNicholas Bellinger pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 1051b8d26b3bSNicholas Bellinger " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 1052b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 1053b8d26b3bSNicholas Bellinger 1054b8d26b3bSNicholas Bellinger return 0; 1055b8d26b3bSNicholas Bellinger } 1056b8d26b3bSNicholas Bellinger 1057b8d26b3bSNicholas Bellinger static void 105895b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 105995b60f07SNicholas Bellinger struct ib_send_wr *send_wr, bool coalesce) 1060b8d26b3bSNicholas Bellinger { 106195b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 106295b60f07SNicholas Bellinger 1063b8d26b3bSNicholas Bellinger isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1064b8d26b3bSNicholas Bellinger send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 1065b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 106695b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 1067b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 106895b60f07SNicholas Bellinger /* 106995b60f07SNicholas Bellinger * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 107095b60f07SNicholas Bellinger * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 107195b60f07SNicholas Bellinger */ 1072ebbe4421SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 1073128e9cc8SSagi Grimberg if (coalesce && isert_conn->state == ISER_CONN_FULL_FEATURE && 107495b60f07SNicholas Bellinger ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 1075ebbe4421SNicholas Bellinger tx_desc->llnode_active = true; 107695b60f07SNicholas Bellinger llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 1077ebbe4421SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 107895b60f07SNicholas Bellinger return; 107995b60f07SNicholas Bellinger } 108095b60f07SNicholas Bellinger isert_conn->conn_comp_batch = 0; 108195b60f07SNicholas Bellinger tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 1082ebbe4421SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 108395b60f07SNicholas Bellinger 108495b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 1085b8d26b3bSNicholas Bellinger } 1086b8d26b3bSNicholas Bellinger 1087b8d26b3bSNicholas Bellinger static int 1088b8d26b3bSNicholas Bellinger isert_rdma_post_recvl(struct isert_conn *isert_conn) 1089b8d26b3bSNicholas Bellinger { 1090b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 1091b8d26b3bSNicholas Bellinger struct ib_sge sge; 1092b8d26b3bSNicholas Bellinger int ret; 1093b8d26b3bSNicholas Bellinger 1094b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 1095b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 1096b8d26b3bSNicholas Bellinger sge.length = ISER_RX_LOGIN_SIZE; 1097b8d26b3bSNicholas Bellinger sge.lkey = isert_conn->conn_mr->lkey; 1098b8d26b3bSNicholas Bellinger 1099b8d26b3bSNicholas Bellinger pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 1100b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 1101b8d26b3bSNicholas Bellinger 1102b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1103b8d26b3bSNicholas Bellinger rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 1104b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 1105b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 1106b8d26b3bSNicholas Bellinger 1107b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count++; 1108b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 1109b8d26b3bSNicholas Bellinger if (ret) { 1110b8d26b3bSNicholas Bellinger pr_err("ib_post_recv() failed: %d\n", ret); 1111b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1112b8d26b3bSNicholas Bellinger } 1113b8d26b3bSNicholas Bellinger 1114b8d26b3bSNicholas Bellinger pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 1115b8d26b3bSNicholas Bellinger return ret; 1116b8d26b3bSNicholas Bellinger } 1117b8d26b3bSNicholas Bellinger 1118b8d26b3bSNicholas Bellinger static int 1119b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1120b8d26b3bSNicholas Bellinger u32 length) 1121b8d26b3bSNicholas Bellinger { 1122b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 1123b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1124b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 1125b8d26b3bSNicholas Bellinger int ret; 1126b8d26b3bSNicholas Bellinger 1127b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 1128b8d26b3bSNicholas Bellinger 1129b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1130b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1131b8d26b3bSNicholas Bellinger 1132b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1133b8d26b3bSNicholas Bellinger 1134b8d26b3bSNicholas Bellinger if (length > 0) { 1135b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1136b8d26b3bSNicholas Bellinger 1137b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1138b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1139b8d26b3bSNicholas Bellinger 1140b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1141b8d26b3bSNicholas Bellinger 1142b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1143b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1144b8d26b3bSNicholas Bellinger 1145b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1146b8d26b3bSNicholas Bellinger tx_dsg->length = length; 1147b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 1148b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1149b8d26b3bSNicholas Bellinger } 1150b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1151b8d26b3bSNicholas Bellinger if (login->login_complete) { 1152e0546fc1SSagi Grimberg if (!conn->sess->sess_ops->SessionType && 1153e0546fc1SSagi Grimberg isert_conn->conn_device->use_fastreg) { 1154e0546fc1SSagi Grimberg /* Normal Session and fastreg is used */ 1155f46d6a8aSNicholas Bellinger u8 pi_support = login->np->tpg_np->tpg->tpg_attrib.t10_pi; 1156f46d6a8aSNicholas Bellinger 1157f46d6a8aSNicholas Bellinger ret = isert_conn_create_fastreg_pool(isert_conn, 1158f46d6a8aSNicholas Bellinger pi_support); 1159f46d6a8aSNicholas Bellinger if (ret) { 1160f46d6a8aSNicholas Bellinger pr_err("Conn: %p failed to create" 1161f46d6a8aSNicholas Bellinger " fastreg pool\n", isert_conn); 1162f46d6a8aSNicholas Bellinger return ret; 1163f46d6a8aSNicholas Bellinger } 1164f46d6a8aSNicholas Bellinger } 1165f46d6a8aSNicholas Bellinger 1166b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1167b8d26b3bSNicholas Bellinger if (ret) 1168b8d26b3bSNicholas Bellinger return ret; 1169b8d26b3bSNicholas Bellinger 1170b8d26b3bSNicholas Bellinger ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1171b8d26b3bSNicholas Bellinger if (ret) 1172b8d26b3bSNicholas Bellinger return ret; 1173b8d26b3bSNicholas Bellinger 1174128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 11752371e5daSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 1176128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 11772371e5daSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 1178b8d26b3bSNicholas Bellinger goto post_send; 1179b8d26b3bSNicholas Bellinger } 1180b8d26b3bSNicholas Bellinger 1181b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 1182b8d26b3bSNicholas Bellinger if (ret) 1183b8d26b3bSNicholas Bellinger return ret; 1184b8d26b3bSNicholas Bellinger } 1185b8d26b3bSNicholas Bellinger post_send: 1186b8d26b3bSNicholas Bellinger ret = isert_post_send(isert_conn, tx_desc); 1187b8d26b3bSNicholas Bellinger if (ret) 1188b8d26b3bSNicholas Bellinger return ret; 1189b8d26b3bSNicholas Bellinger 1190b8d26b3bSNicholas Bellinger return 0; 1191b8d26b3bSNicholas Bellinger } 1192b8d26b3bSNicholas Bellinger 1193b8d26b3bSNicholas Bellinger static void 11942371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1195b8d26b3bSNicholas Bellinger { 11962371e5daSSagi Grimberg struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 11972371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1198b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1199b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1200b8d26b3bSNicholas Bellinger int size; 1201b8d26b3bSNicholas Bellinger 12022371e5daSSagi Grimberg pr_info("conn %p\n", isert_conn); 12032371e5daSSagi Grimberg 12042371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1205b8d26b3bSNicholas Bellinger 1206b8d26b3bSNicholas Bellinger if (login->first_request) { 1207b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1208b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1209b8d26b3bSNicholas Bellinger /* 1210b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1211b8d26b3bSNicholas Bellinger * login request PDU. 1212b8d26b3bSNicholas Bellinger */ 1213b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1214b8d26b3bSNicholas Bellinger login->current_stage = 1215b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1216b8d26b3bSNicholas Bellinger >> 2; 1217b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1218b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1219b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1220b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1221b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1222b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1223b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1224b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1225b8d26b3bSNicholas Bellinger } 1226b8d26b3bSNicholas Bellinger 1227b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1228b8d26b3bSNicholas Bellinger 1229b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1230b8d26b3bSNicholas Bellinger pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1231b8d26b3bSNicholas Bellinger size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1232b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1233b8d26b3bSNicholas Bellinger 12346faaa85fSNicholas Bellinger if (login->first_request) { 1235b8d26b3bSNicholas Bellinger complete(&isert_conn->conn_login_comp); 12366faaa85fSNicholas Bellinger return; 12376faaa85fSNicholas Bellinger } 12386faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1239b8d26b3bSNicholas Bellinger } 1240b8d26b3bSNicholas Bellinger 1241b8d26b3bSNicholas Bellinger static struct iscsi_cmd 1242676687c6SNicholas Bellinger *isert_allocate_cmd(struct iscsi_conn *conn) 1243b8d26b3bSNicholas Bellinger { 1244b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1245b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1246d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1247b8d26b3bSNicholas Bellinger 1248676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1249d703ce2fSNicholas Bellinger if (!cmd) { 1250d703ce2fSNicholas Bellinger pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1251b8d26b3bSNicholas Bellinger return NULL; 1252b8d26b3bSNicholas Bellinger } 1253d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1254b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1255d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 1256b8d26b3bSNicholas Bellinger 1257d703ce2fSNicholas Bellinger return cmd; 1258b8d26b3bSNicholas Bellinger } 1259b8d26b3bSNicholas Bellinger 1260b8d26b3bSNicholas Bellinger static int 1261b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1262d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1263d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1264b8d26b3bSNicholas Bellinger { 1265b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1266b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1267b8d26b3bSNicholas Bellinger struct scatterlist *sg; 1268b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1269b8d26b3bSNicholas Bellinger bool dump_payload = false; 1270b8d26b3bSNicholas Bellinger 1271b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1272b8d26b3bSNicholas Bellinger if (rc < 0) 1273b8d26b3bSNicholas Bellinger return rc; 1274b8d26b3bSNicholas Bellinger 1275b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1276b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1277b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 1278b8d26b3bSNicholas Bellinger 1279b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1280b8d26b3bSNicholas Bellinger if (rc < 0) { 1281b8d26b3bSNicholas Bellinger return 0; 1282b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1283b8d26b3bSNicholas Bellinger dump_payload = true; 1284b8d26b3bSNicholas Bellinger goto sequence_cmd; 1285b8d26b3bSNicholas Bellinger } 1286b8d26b3bSNicholas Bellinger 1287b8d26b3bSNicholas Bellinger if (!imm_data) 1288b8d26b3bSNicholas Bellinger return 0; 1289b8d26b3bSNicholas Bellinger 1290b8d26b3bSNicholas Bellinger sg = &cmd->se_cmd.t_data_sg[0]; 1291b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1292b8d26b3bSNicholas Bellinger 1293b8d26b3bSNicholas Bellinger pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1294b8d26b3bSNicholas Bellinger sg, sg_nents, &rx_desc->data[0], imm_data_len); 1295b8d26b3bSNicholas Bellinger 1296b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1297b8d26b3bSNicholas Bellinger 1298b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1299b8d26b3bSNicholas Bellinger 1300b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1301b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1302b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1303b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1304b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1305b8d26b3bSNicholas Bellinger } 1306b8d26b3bSNicholas Bellinger 1307b8d26b3bSNicholas Bellinger sequence_cmd: 1308561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1309b8d26b3bSNicholas Bellinger 1310b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 1311b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 13126cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 13136cc44a6fSNicholas Bellinger target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); 1314b8d26b3bSNicholas Bellinger 1315b8d26b3bSNicholas Bellinger return 0; 1316b8d26b3bSNicholas Bellinger } 1317b8d26b3bSNicholas Bellinger 1318b8d26b3bSNicholas Bellinger static int 1319b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1320b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1321b8d26b3bSNicholas Bellinger { 1322b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1323b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1324b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1325b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1326b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1327b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1328b8d26b3bSNicholas Bellinger 1329b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1330b8d26b3bSNicholas Bellinger if (rc < 0) 1331b8d26b3bSNicholas Bellinger return rc; 1332b8d26b3bSNicholas Bellinger else if (!cmd) 1333b8d26b3bSNicholas Bellinger return 0; 1334b8d26b3bSNicholas Bellinger /* 1335b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1336b8d26b3bSNicholas Bellinger */ 1337b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 1338b8d26b3bSNicholas Bellinger pr_err("Received unexpected solicited data payload\n"); 1339b8d26b3bSNicholas Bellinger dump_stack(); 1340b8d26b3bSNicholas Bellinger return -1; 1341b8d26b3bSNicholas Bellinger } 1342b8d26b3bSNicholas Bellinger 1343b8d26b3bSNicholas Bellinger pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1344b8d26b3bSNicholas Bellinger unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1345b8d26b3bSNicholas Bellinger 1346b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1347b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1348b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1349b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1350b8d26b3bSNicholas Bellinger /* 1351b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1352b8d26b3bSNicholas Bellinger */ 1353b8d26b3bSNicholas Bellinger if (page_off) { 1354b8d26b3bSNicholas Bellinger pr_err("Received unexpected non-page aligned data payload\n"); 1355b8d26b3bSNicholas Bellinger dump_stack(); 1356b8d26b3bSNicholas Bellinger return -1; 1357b8d26b3bSNicholas Bellinger } 1358b8d26b3bSNicholas Bellinger pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1359b8d26b3bSNicholas Bellinger sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1360b8d26b3bSNicholas Bellinger 1361b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1362b8d26b3bSNicholas Bellinger unsol_data_len); 1363b8d26b3bSNicholas Bellinger 1364b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1365b8d26b3bSNicholas Bellinger if (rc < 0) 1366b8d26b3bSNicholas Bellinger return rc; 1367b8d26b3bSNicholas Bellinger 1368b8d26b3bSNicholas Bellinger return 0; 1369b8d26b3bSNicholas Bellinger } 1370b8d26b3bSNicholas Bellinger 1371b8d26b3bSNicholas Bellinger static int 1372778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1373d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1374d703ce2fSNicholas Bellinger unsigned char *buf) 1375778de368SNicholas Bellinger { 1376778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1377778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1378778de368SNicholas Bellinger int rc; 1379778de368SNicholas Bellinger 1380778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1381778de368SNicholas Bellinger if (rc < 0) 1382778de368SNicholas Bellinger return rc; 1383778de368SNicholas Bellinger /* 1384778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1385778de368SNicholas Bellinger */ 1386778de368SNicholas Bellinger 1387778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1388778de368SNicholas Bellinger } 1389778de368SNicholas Bellinger 1390778de368SNicholas Bellinger static int 1391adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1392d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1393d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1394adb54c29SNicholas Bellinger { 1395adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1396adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1397adb54c29SNicholas Bellinger int rc; 1398adb54c29SNicholas Bellinger unsigned char *text_in; 1399adb54c29SNicholas Bellinger 1400adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1401adb54c29SNicholas Bellinger if (rc < 0) 1402adb54c29SNicholas Bellinger return rc; 1403adb54c29SNicholas Bellinger 1404adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 1405adb54c29SNicholas Bellinger if (!text_in) { 1406adb54c29SNicholas Bellinger pr_err("Unable to allocate text_in of payload_length: %u\n", 1407adb54c29SNicholas Bellinger payload_length); 1408adb54c29SNicholas Bellinger return -ENOMEM; 1409adb54c29SNicholas Bellinger } 1410adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1411adb54c29SNicholas Bellinger 1412adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1413adb54c29SNicholas Bellinger 1414adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1415adb54c29SNicholas Bellinger } 1416adb54c29SNicholas Bellinger 1417adb54c29SNicholas Bellinger static int 1418b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1419b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1420b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1421b8d26b3bSNicholas Bellinger { 1422b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1423b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1424ca40d24eSNicholas Bellinger struct iscsi_session *sess = conn->sess; 1425b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1426b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1427b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1428b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1429b8d26b3bSNicholas Bellinger 1430ca40d24eSNicholas Bellinger if (sess->sess_ops->SessionType && 1431ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1432ca40d24eSNicholas Bellinger pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1433ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1434ca40d24eSNicholas Bellinger return 0; 1435ca40d24eSNicholas Bellinger } 1436ca40d24eSNicholas Bellinger 1437b8d26b3bSNicholas Bellinger switch (opcode) { 1438b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1439676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1440b8d26b3bSNicholas Bellinger if (!cmd) 1441b8d26b3bSNicholas Bellinger break; 1442b8d26b3bSNicholas Bellinger 1443d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1444b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1445b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1446b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1447b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1448b8d26b3bSNicholas Bellinger 1449d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1450b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1451b8d26b3bSNicholas Bellinger break; 1452b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1453676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1454b8d26b3bSNicholas Bellinger if (!cmd) 1455b8d26b3bSNicholas Bellinger break; 1456b8d26b3bSNicholas Bellinger 1457d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1458d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1459778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1460b8d26b3bSNicholas Bellinger break; 1461b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1462b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1463b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1464b8d26b3bSNicholas Bellinger break; 1465b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1466676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1467b8d26b3bSNicholas Bellinger if (!cmd) 1468b8d26b3bSNicholas Bellinger break; 1469b8d26b3bSNicholas Bellinger 1470b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1471b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1472b8d26b3bSNicholas Bellinger break; 1473b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 1474676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1475b8d26b3bSNicholas Bellinger if (!cmd) 1476b8d26b3bSNicholas Bellinger break; 1477b8d26b3bSNicholas Bellinger 1478b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1479b8d26b3bSNicholas Bellinger if (ret > 0) 1480b8d26b3bSNicholas Bellinger wait_for_completion_timeout(&conn->conn_logout_comp, 1481b8d26b3bSNicholas Bellinger SECONDS_FOR_LOGOUT_COMP * 1482b8d26b3bSNicholas Bellinger HZ); 1483b8d26b3bSNicholas Bellinger break; 1484adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1485676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1486adb54c29SNicholas Bellinger if (!cmd) 1487adb54c29SNicholas Bellinger break; 1488adb54c29SNicholas Bellinger 1489d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1490d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1491adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1492adb54c29SNicholas Bellinger break; 1493b8d26b3bSNicholas Bellinger default: 1494b8d26b3bSNicholas Bellinger pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1495b8d26b3bSNicholas Bellinger dump_stack(); 1496b8d26b3bSNicholas Bellinger break; 1497b8d26b3bSNicholas Bellinger } 1498b8d26b3bSNicholas Bellinger 1499b8d26b3bSNicholas Bellinger return ret; 1500b8d26b3bSNicholas Bellinger } 1501b8d26b3bSNicholas Bellinger 1502b8d26b3bSNicholas Bellinger static void 1503b8d26b3bSNicholas Bellinger isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1504b8d26b3bSNicholas Bellinger { 1505b8d26b3bSNicholas Bellinger struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1506b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1507b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1508b8d26b3bSNicholas Bellinger int rc; 1509b8d26b3bSNicholas Bellinger 1510b8d26b3bSNicholas Bellinger switch (iser_hdr->flags & 0xF0) { 1511b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1512b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_RSV) { 1513b8d26b3bSNicholas Bellinger read_stag = be32_to_cpu(iser_hdr->read_stag); 1514b8d26b3bSNicholas Bellinger read_va = be64_to_cpu(iser_hdr->read_va); 1515b8d26b3bSNicholas Bellinger pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1516b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1517b8d26b3bSNicholas Bellinger } 1518b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_WSV) { 1519b8d26b3bSNicholas Bellinger write_stag = be32_to_cpu(iser_hdr->write_stag); 1520b8d26b3bSNicholas Bellinger write_va = be64_to_cpu(iser_hdr->write_va); 1521b8d26b3bSNicholas Bellinger pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1522b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1523b8d26b3bSNicholas Bellinger } 1524b8d26b3bSNicholas Bellinger 1525b8d26b3bSNicholas Bellinger pr_debug("ISER ISCSI_CTRL PDU\n"); 1526b8d26b3bSNicholas Bellinger break; 1527b8d26b3bSNicholas Bellinger case ISER_HELLO: 1528b8d26b3bSNicholas Bellinger pr_err("iSER Hello message\n"); 1529b8d26b3bSNicholas Bellinger break; 1530b8d26b3bSNicholas Bellinger default: 1531b8d26b3bSNicholas Bellinger pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1532b8d26b3bSNicholas Bellinger break; 1533b8d26b3bSNicholas Bellinger } 1534b8d26b3bSNicholas Bellinger 1535b8d26b3bSNicholas Bellinger rc = isert_rx_opcode(isert_conn, rx_desc, 1536b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 1537b8d26b3bSNicholas Bellinger } 1538b8d26b3bSNicholas Bellinger 1539b8d26b3bSNicholas Bellinger static void 1540b8d26b3bSNicholas Bellinger isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1541b8d26b3bSNicholas Bellinger unsigned long xfer_len) 1542b8d26b3bSNicholas Bellinger { 1543b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1544b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr; 1545b8d26b3bSNicholas Bellinger u64 rx_dma; 1546b8d26b3bSNicholas Bellinger int rx_buflen, outstanding; 1547b8d26b3bSNicholas Bellinger 1548b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) { 1549b8d26b3bSNicholas Bellinger rx_dma = isert_conn->login_req_dma; 1550b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_LOGIN_SIZE; 1551b8d26b3bSNicholas Bellinger pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1552b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1553b8d26b3bSNicholas Bellinger } else { 1554b8d26b3bSNicholas Bellinger rx_dma = desc->dma_addr; 1555b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_PAYLOAD_SIZE; 1556b8d26b3bSNicholas Bellinger pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1557b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1558b8d26b3bSNicholas Bellinger } 1559b8d26b3bSNicholas Bellinger 1560b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1561b8d26b3bSNicholas Bellinger 1562b8d26b3bSNicholas Bellinger hdr = &desc->iscsi_header; 1563b8d26b3bSNicholas Bellinger pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1564b8d26b3bSNicholas Bellinger hdr->opcode, hdr->itt, hdr->flags, 1565b8d26b3bSNicholas Bellinger (int)(xfer_len - ISER_HEADERS_LEN)); 1566b8d26b3bSNicholas Bellinger 15672371e5daSSagi Grimberg if ((char *)desc == isert_conn->login_req_buf) { 15682371e5daSSagi Grimberg isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 15692371e5daSSagi Grimberg if (isert_conn->conn) { 15702371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 15712371e5daSSagi Grimberg 15722371e5daSSagi Grimberg if (login && !login->first_request) 15732371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 15742371e5daSSagi Grimberg } 15752371e5daSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 15762371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 15772371e5daSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 15782371e5daSSagi Grimberg } else { 1579b8d26b3bSNicholas Bellinger isert_rx_do_work(desc, isert_conn); 15802371e5daSSagi Grimberg } 1581b8d26b3bSNicholas Bellinger 1582b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1583b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 1584b8d26b3bSNicholas Bellinger 1585b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1586b8d26b3bSNicholas Bellinger pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1587b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count); 1588b8d26b3bSNicholas Bellinger 1589b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) 1590b8d26b3bSNicholas Bellinger return; 1591b8d26b3bSNicholas Bellinger 1592b8d26b3bSNicholas Bellinger outstanding = isert_conn->post_recv_buf_count; 1593b8d26b3bSNicholas Bellinger if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1594b8d26b3bSNicholas Bellinger int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1595b8d26b3bSNicholas Bellinger ISERT_MIN_POSTED_RX); 1596b8d26b3bSNicholas Bellinger err = isert_post_recv(isert_conn, count); 1597b8d26b3bSNicholas Bellinger if (err) { 1598b8d26b3bSNicholas Bellinger pr_err("isert_post_recv() count: %d failed, %d\n", 1599b8d26b3bSNicholas Bellinger count, err); 1600b8d26b3bSNicholas Bellinger } 1601b8d26b3bSNicholas Bellinger } 1602b8d26b3bSNicholas Bellinger } 1603b8d26b3bSNicholas Bellinger 1604e3d7e4c3SSagi Grimberg static int 1605e3d7e4c3SSagi Grimberg isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1606e3d7e4c3SSagi Grimberg struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1607e3d7e4c3SSagi Grimberg enum iser_ib_op_code op, struct isert_data_buf *data) 1608e3d7e4c3SSagi Grimberg { 1609e3d7e4c3SSagi Grimberg struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1610e3d7e4c3SSagi Grimberg 1611e3d7e4c3SSagi Grimberg data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1612e3d7e4c3SSagi Grimberg DMA_TO_DEVICE : DMA_FROM_DEVICE; 1613e3d7e4c3SSagi Grimberg 1614e3d7e4c3SSagi Grimberg data->len = length - offset; 1615e3d7e4c3SSagi Grimberg data->offset = offset; 1616e3d7e4c3SSagi Grimberg data->sg_off = data->offset / PAGE_SIZE; 1617e3d7e4c3SSagi Grimberg 1618e3d7e4c3SSagi Grimberg data->sg = &sg[data->sg_off]; 1619e3d7e4c3SSagi Grimberg data->nents = min_t(unsigned int, nents - data->sg_off, 1620e3d7e4c3SSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 1621e3d7e4c3SSagi Grimberg data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1622e3d7e4c3SSagi Grimberg PAGE_SIZE); 1623e3d7e4c3SSagi Grimberg 1624e3d7e4c3SSagi Grimberg data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1625e3d7e4c3SSagi Grimberg data->dma_dir); 1626e3d7e4c3SSagi Grimberg if (unlikely(!data->dma_nents)) { 1627e3d7e4c3SSagi Grimberg pr_err("Cmd: unable to dma map SGs %p\n", sg); 1628e3d7e4c3SSagi Grimberg return -EINVAL; 1629e3d7e4c3SSagi Grimberg } 1630e3d7e4c3SSagi Grimberg 1631e3d7e4c3SSagi Grimberg pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1632e3d7e4c3SSagi Grimberg isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1633e3d7e4c3SSagi Grimberg 1634e3d7e4c3SSagi Grimberg return 0; 1635e3d7e4c3SSagi Grimberg } 1636e3d7e4c3SSagi Grimberg 1637e3d7e4c3SSagi Grimberg static void 1638e3d7e4c3SSagi Grimberg isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1639e3d7e4c3SSagi Grimberg { 1640e3d7e4c3SSagi Grimberg struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1641e3d7e4c3SSagi Grimberg 1642e3d7e4c3SSagi Grimberg ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1643e3d7e4c3SSagi Grimberg memset(data, 0, sizeof(*data)); 1644e3d7e4c3SSagi Grimberg } 1645e3d7e4c3SSagi Grimberg 1646e3d7e4c3SSagi Grimberg 1647e3d7e4c3SSagi Grimberg 1648b8d26b3bSNicholas Bellinger static void 1649b8d26b3bSNicholas Bellinger isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1650b8d26b3bSNicholas Bellinger { 1651b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1652b8d26b3bSNicholas Bellinger 165390ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1654e3d7e4c3SSagi Grimberg 1655e3d7e4c3SSagi Grimberg if (wr->data.sg) { 165690ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1657e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 1658b8d26b3bSNicholas Bellinger } 1659b8d26b3bSNicholas Bellinger 166090ecc6e2SVu Pham if (wr->send_wr) { 166190ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1662b8d26b3bSNicholas Bellinger kfree(wr->send_wr); 1663b8d26b3bSNicholas Bellinger wr->send_wr = NULL; 166490ecc6e2SVu Pham } 1665b8d26b3bSNicholas Bellinger 166690ecc6e2SVu Pham if (wr->ib_sge) { 166790ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 166890ecc6e2SVu Pham kfree(wr->ib_sge); 166990ecc6e2SVu Pham wr->ib_sge = NULL; 167090ecc6e2SVu Pham } 1671b8d26b3bSNicholas Bellinger } 1672b8d26b3bSNicholas Bellinger 1673b8d26b3bSNicholas Bellinger static void 1674a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 167559464ef4SVu Pham { 167659464ef4SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 167759464ef4SVu Pham LIST_HEAD(unmap_list); 167859464ef4SVu Pham 1679a3a5a826SSagi Grimberg pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd); 168059464ef4SVu Pham 168159464ef4SVu Pham if (wr->fr_desc) { 1682a3a5a826SSagi Grimberg pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n", 168359464ef4SVu Pham isert_cmd, wr->fr_desc); 16849e961ae7SSagi Grimberg if (wr->fr_desc->ind & ISERT_PROTECTED) { 16859e961ae7SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 16869e961ae7SSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 16879e961ae7SSagi Grimberg } 168859464ef4SVu Pham spin_lock_bh(&isert_conn->conn_lock); 1689a3a5a826SSagi Grimberg list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 169059464ef4SVu Pham spin_unlock_bh(&isert_conn->conn_lock); 169159464ef4SVu Pham wr->fr_desc = NULL; 169259464ef4SVu Pham } 169359464ef4SVu Pham 1694e3d7e4c3SSagi Grimberg if (wr->data.sg) { 1695a3a5a826SSagi Grimberg pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1696e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 169759464ef4SVu Pham } 169859464ef4SVu Pham 169959464ef4SVu Pham wr->ib_sge = NULL; 170059464ef4SVu Pham wr->send_wr = NULL; 170159464ef4SVu Pham } 170259464ef4SVu Pham 170359464ef4SVu Pham static void 170403e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1705b8d26b3bSNicholas Bellinger { 1706d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1707b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1708186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1709d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 1710b8d26b3bSNicholas Bellinger 1711b8d26b3bSNicholas Bellinger pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1712b8d26b3bSNicholas Bellinger 1713b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1714b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1715b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1716b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17175159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1718b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1719b8d26b3bSNicholas Bellinger 172003e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1721b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 172203e7848aSNicholas Bellinger /* 172303e7848aSNicholas Bellinger * Check for special case during comp_err where 172403e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 172503e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 172603e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 172703e7848aSNicholas Bellinger */ 172803e7848aSNicholas Bellinger if (comp_err && 172903e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 173003e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 173103e7848aSNicholas Bellinger 173203e7848aSNicholas Bellinger target_put_sess_cmd(se_cmd->se_sess, se_cmd); 173303e7848aSNicholas Bellinger } 173403e7848aSNicholas Bellinger } 1735b8d26b3bSNicholas Bellinger 1736d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1737186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1738186a9647SNicholas Bellinger break; 1739b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1740186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1741186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17425159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1743186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1744186a9647SNicholas Bellinger 1745b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1746b8d26b3bSNicholas Bellinger break; 1747b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1748b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1749adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1750b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1751b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17525159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1753b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1754b8d26b3bSNicholas Bellinger 1755b8d26b3bSNicholas Bellinger /* 1756b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1757b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1758b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1759b8d26b3bSNicholas Bellinger */ 1760b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 17613df8f68aSNicholas Bellinger pr_debug("Calling transport_generic_free_cmd from" 17623df8f68aSNicholas Bellinger " isert_put_cmd for 0x%02x\n", 17633df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1764b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1765b8d26b3bSNicholas Bellinger break; 1766b8d26b3bSNicholas Bellinger } 1767b8d26b3bSNicholas Bellinger /* 1768b8d26b3bSNicholas Bellinger * Fall-through 1769b8d26b3bSNicholas Bellinger */ 1770b8d26b3bSNicholas Bellinger default: 1771d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1772b8d26b3bSNicholas Bellinger break; 1773b8d26b3bSNicholas Bellinger } 1774b8d26b3bSNicholas Bellinger } 1775b8d26b3bSNicholas Bellinger 1776b8d26b3bSNicholas Bellinger static void 1777b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1778b8d26b3bSNicholas Bellinger { 1779b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 1780b8d26b3bSNicholas Bellinger pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1781b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1782b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1783b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1784b8d26b3bSNicholas Bellinger } 1785b8d26b3bSNicholas Bellinger } 1786b8d26b3bSNicholas Bellinger 1787b8d26b3bSNicholas Bellinger static void 1788b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 178903e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1790b8d26b3bSNicholas Bellinger { 1791dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 1792dbbc5d11SNicholas Bellinger pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1793dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1794dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1795dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1796b8d26b3bSNicholas Bellinger } 1797b8d26b3bSNicholas Bellinger 1798b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 179903e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1800b8d26b3bSNicholas Bellinger } 1801b8d26b3bSNicholas Bellinger 180296b7973eSSagi Grimberg static int 180396b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1804f93f3a70SSagi Grimberg { 18059e961ae7SSagi Grimberg struct ib_mr_status mr_status; 180696b7973eSSagi Grimberg int ret; 1807f93f3a70SSagi Grimberg 180896b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 18099e961ae7SSagi Grimberg if (ret) { 18109e961ae7SSagi Grimberg pr_err("ib_check_mr_status failed, ret %d\n", ret); 18119e961ae7SSagi Grimberg goto fail_mr_status; 18129e961ae7SSagi Grimberg } 181396b7973eSSagi Grimberg 18149e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 181596b7973eSSagi Grimberg u64 sec_offset_err; 18169e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 18179e961ae7SSagi Grimberg 18189e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 18199e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 18209e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 18219e961ae7SSagi Grimberg break; 18229e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 18239e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 18249e961ae7SSagi Grimberg break; 18259e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 18269e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 18279e961ae7SSagi Grimberg break; 18289e961ae7SSagi Grimberg } 182996b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 183096b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 183196b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 18329e961ae7SSagi Grimberg 18339e961ae7SSagi Grimberg pr_err("isert: PI error found type %d at sector 0x%llx " 18349e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 18359e961ae7SSagi Grimberg mr_status.sig_err.err_type, 18369e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 18379e961ae7SSagi Grimberg mr_status.sig_err.expected, 18389e961ae7SSagi Grimberg mr_status.sig_err.actual); 18399e961ae7SSagi Grimberg ret = 1; 18409e961ae7SSagi Grimberg } 18419e961ae7SSagi Grimberg 18429e961ae7SSagi Grimberg fail_mr_status: 184396b7973eSSagi Grimberg return ret; 184496b7973eSSagi Grimberg } 184596b7973eSSagi Grimberg 184696b7973eSSagi Grimberg static void 184796b7973eSSagi Grimberg isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 184896b7973eSSagi Grimberg struct isert_cmd *isert_cmd) 184996b7973eSSagi Grimberg { 185096b7973eSSagi Grimberg struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 185196b7973eSSagi Grimberg struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 185296b7973eSSagi Grimberg struct se_cmd *se_cmd = &cmd->se_cmd; 185396b7973eSSagi Grimberg struct isert_conn *isert_conn = isert_cmd->conn; 185496b7973eSSagi Grimberg struct isert_device *device = isert_conn->conn_device; 185596b7973eSSagi Grimberg int ret = 0; 185696b7973eSSagi Grimberg 185796b7973eSSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 185896b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 185996b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 186096b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 186196b7973eSSagi Grimberg } 186296b7973eSSagi Grimberg 1863f93f3a70SSagi Grimberg device->unreg_rdma_mem(isert_cmd, isert_conn); 1864897bb2c9SSagi Grimberg wr->send_wr_num = 0; 18659e961ae7SSagi Grimberg if (ret) 18669e961ae7SSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 18679e961ae7SSagi Grimberg se_cmd->pi_err, 0); 18689e961ae7SSagi Grimberg else 1869f93f3a70SSagi Grimberg isert_put_response(isert_conn->conn, cmd); 1870f93f3a70SSagi Grimberg } 1871f93f3a70SSagi Grimberg 1872f93f3a70SSagi Grimberg static void 1873b8d26b3bSNicholas Bellinger isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1874b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd) 1875b8d26b3bSNicholas Bellinger { 1876b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1877d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1878b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 187990ecc6e2SVu Pham struct isert_conn *isert_conn = isert_cmd->conn; 1880d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 18815bac4b1aSSagi Grimberg int ret = 0; 1882b8d26b3bSNicholas Bellinger 18839e961ae7SSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 188496b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 188596b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 188696b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 18879e961ae7SSagi Grimberg } 18889e961ae7SSagi Grimberg 1889b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1890d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1891e3d7e4c3SSagi Grimberg cmd->write_data_done = wr->data.len; 1892b6b87a1dSNicholas Bellinger wr->send_wr_num = 0; 1893b8d26b3bSNicholas Bellinger 189490ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1895b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1896b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1897b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1898b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1899b8d26b3bSNicholas Bellinger 19005bac4b1aSSagi Grimberg if (ret) 19015bac4b1aSSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 19025bac4b1aSSagi Grimberg se_cmd->pi_err, 0); 19035bac4b1aSSagi Grimberg else 1904b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1905b8d26b3bSNicholas Bellinger } 1906b8d26b3bSNicholas Bellinger 1907b8d26b3bSNicholas Bellinger static void 1908b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1909b8d26b3bSNicholas Bellinger { 1910b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1911b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1912b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1913b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1914d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1915b8d26b3bSNicholas Bellinger 1916b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1917b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1918b8d26b3bSNicholas Bellinger pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1919b8d26b3bSNicholas Bellinger 1920b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1921b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1922b8d26b3bSNicholas Bellinger 1923b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 192403e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1925b8d26b3bSNicholas Bellinger break; 1926b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 1927b8d26b3bSNicholas Bellinger pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1928b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1929b8d26b3bSNicholas Bellinger 1930b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 193103e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 19323df8f68aSNicholas Bellinger break; 1933b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1934b8d26b3bSNicholas Bellinger pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 19359d49f5e2SSagi Grimberg 19369d49f5e2SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 1937b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1938b8d26b3bSNicholas Bellinger break; 1939adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 1940adb54c29SNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1941adb54c29SNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 194203e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1943adb54c29SNicholas Bellinger break; 1944b8d26b3bSNicholas Bellinger default: 1945b8d26b3bSNicholas Bellinger pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1946b8d26b3bSNicholas Bellinger dump_stack(); 1947b8d26b3bSNicholas Bellinger break; 1948b8d26b3bSNicholas Bellinger } 1949b8d26b3bSNicholas Bellinger } 1950b8d26b3bSNicholas Bellinger 1951b8d26b3bSNicholas Bellinger static void 1952b8d26b3bSNicholas Bellinger isert_response_completion(struct iser_tx_desc *tx_desc, 1953b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1954b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn, 1955b8d26b3bSNicholas Bellinger struct ib_device *ib_dev) 1956b8d26b3bSNicholas Bellinger { 1957d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1958b6b87a1dSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1959b8d26b3bSNicholas Bellinger 1960b8d26b3bSNicholas Bellinger if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 19613df8f68aSNicholas Bellinger cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1962adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_REJECT || 1963adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_TEXTRSP) { 1964b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1965b8d26b3bSNicholas Bellinger 1966b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1967b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1968b8d26b3bSNicholas Bellinger return; 1969b8d26b3bSNicholas Bellinger } 1970897bb2c9SSagi Grimberg 1971897bb2c9SSagi Grimberg /** 1972897bb2c9SSagi Grimberg * If send_wr_num is 0 this means that we got 1973897bb2c9SSagi Grimberg * RDMA completion and we cleared it and we should 1974897bb2c9SSagi Grimberg * simply decrement the response post. else the 1975897bb2c9SSagi Grimberg * response is incorporated in send_wr_num, just 1976897bb2c9SSagi Grimberg * sub it. 1977897bb2c9SSagi Grimberg **/ 1978897bb2c9SSagi Grimberg if (wr->send_wr_num) 1979897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1980897bb2c9SSagi Grimberg else 1981897bb2c9SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 1982b8d26b3bSNicholas Bellinger 1983b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 198403e7848aSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1985b8d26b3bSNicholas Bellinger } 1986b8d26b3bSNicholas Bellinger 1987b8d26b3bSNicholas Bellinger static void 198895b60f07SNicholas Bellinger __isert_send_completion(struct iser_tx_desc *tx_desc, 1989b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 1990b8d26b3bSNicholas Bellinger { 1991b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1992b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1993b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr; 1994b8d26b3bSNicholas Bellinger 1995b8d26b3bSNicholas Bellinger if (!isert_cmd) { 1996b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1997b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1998b8d26b3bSNicholas Bellinger return; 1999b8d26b3bSNicholas Bellinger } 2000b8d26b3bSNicholas Bellinger wr = &isert_cmd->rdma_wr; 2001b8d26b3bSNicholas Bellinger 2002b8d26b3bSNicholas Bellinger switch (wr->iser_ib_op) { 2003b8d26b3bSNicholas Bellinger case ISER_IB_RECV: 2004b8d26b3bSNicholas Bellinger pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 2005b8d26b3bSNicholas Bellinger dump_stack(); 2006b8d26b3bSNicholas Bellinger break; 2007b8d26b3bSNicholas Bellinger case ISER_IB_SEND: 2008b8d26b3bSNicholas Bellinger pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 2009b8d26b3bSNicholas Bellinger isert_response_completion(tx_desc, isert_cmd, 2010b8d26b3bSNicholas Bellinger isert_conn, ib_dev); 2011b8d26b3bSNicholas Bellinger break; 2012b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_WRITE: 2013f93f3a70SSagi Grimberg pr_debug("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 2014897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2015f93f3a70SSagi Grimberg isert_completion_rdma_write(tx_desc, isert_cmd); 2016b8d26b3bSNicholas Bellinger break; 2017b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_READ: 2018b8d26b3bSNicholas Bellinger pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 2019b8d26b3bSNicholas Bellinger 2020b6b87a1dSNicholas Bellinger atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2021b8d26b3bSNicholas Bellinger isert_completion_rdma_read(tx_desc, isert_cmd); 2022b8d26b3bSNicholas Bellinger break; 2023b8d26b3bSNicholas Bellinger default: 2024b8d26b3bSNicholas Bellinger pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 2025b8d26b3bSNicholas Bellinger dump_stack(); 2026b8d26b3bSNicholas Bellinger break; 2027b8d26b3bSNicholas Bellinger } 2028b8d26b3bSNicholas Bellinger } 2029b8d26b3bSNicholas Bellinger 2030b8d26b3bSNicholas Bellinger static void 203195b60f07SNicholas Bellinger isert_send_completion(struct iser_tx_desc *tx_desc, 203295b60f07SNicholas Bellinger struct isert_conn *isert_conn) 203395b60f07SNicholas Bellinger { 203495b60f07SNicholas Bellinger struct llist_node *llnode = tx_desc->comp_llnode_batch; 203595b60f07SNicholas Bellinger struct iser_tx_desc *t; 203695b60f07SNicholas Bellinger /* 203795b60f07SNicholas Bellinger * Drain coalesced completion llist starting from comp_llnode_batch 203895b60f07SNicholas Bellinger * setup in isert_init_send_wr(), and then complete trailing tx_desc. 203995b60f07SNicholas Bellinger */ 204095b60f07SNicholas Bellinger while (llnode) { 204195b60f07SNicholas Bellinger t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 204295b60f07SNicholas Bellinger llnode = llist_next(llnode); 204395b60f07SNicholas Bellinger __isert_send_completion(t, isert_conn); 204495b60f07SNicholas Bellinger } 204595b60f07SNicholas Bellinger __isert_send_completion(tx_desc, isert_conn); 204695b60f07SNicholas Bellinger } 204795b60f07SNicholas Bellinger 204895b60f07SNicholas Bellinger static void 2049ebbe4421SNicholas Bellinger isert_cq_drain_comp_llist(struct isert_conn *isert_conn, struct ib_device *ib_dev) 2050ebbe4421SNicholas Bellinger { 2051ebbe4421SNicholas Bellinger struct llist_node *llnode; 2052ebbe4421SNicholas Bellinger struct isert_rdma_wr *wr; 2053ebbe4421SNicholas Bellinger struct iser_tx_desc *t; 2054ebbe4421SNicholas Bellinger 2055ebbe4421SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 2056ebbe4421SNicholas Bellinger llnode = llist_del_all(&isert_conn->conn_comp_llist); 2057ebbe4421SNicholas Bellinger isert_conn->conn_comp_batch = 0; 2058ebbe4421SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2059ebbe4421SNicholas Bellinger 2060ebbe4421SNicholas Bellinger while (llnode) { 2061ebbe4421SNicholas Bellinger t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 2062ebbe4421SNicholas Bellinger llnode = llist_next(llnode); 2063ebbe4421SNicholas Bellinger wr = &t->isert_cmd->rdma_wr; 2064ebbe4421SNicholas Bellinger 2065897bb2c9SSagi Grimberg /** 2066897bb2c9SSagi Grimberg * If send_wr_num is 0 this means that we got 2067897bb2c9SSagi Grimberg * RDMA completion and we cleared it and we should 2068897bb2c9SSagi Grimberg * simply decrement the response post. else the 2069897bb2c9SSagi Grimberg * response is incorporated in send_wr_num, just 2070897bb2c9SSagi Grimberg * sub it. 2071897bb2c9SSagi Grimberg **/ 2072897bb2c9SSagi Grimberg if (wr->send_wr_num) 2073897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, 2074897bb2c9SSagi Grimberg &isert_conn->post_send_buf_count); 2075897bb2c9SSagi Grimberg else 2076897bb2c9SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 207703e7848aSNicholas Bellinger 207803e7848aSNicholas Bellinger isert_completion_put(t, t->isert_cmd, ib_dev, true); 2079ebbe4421SNicholas Bellinger } 2080ebbe4421SNicholas Bellinger } 2081ebbe4421SNicholas Bellinger 2082ebbe4421SNicholas Bellinger static void 2083defd8848SNicholas Bellinger isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 2084b8d26b3bSNicholas Bellinger { 2085b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2086b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2087ebbe4421SNicholas Bellinger struct llist_node *llnode = tx_desc->comp_llnode_batch; 2088ebbe4421SNicholas Bellinger struct isert_rdma_wr *wr; 2089ebbe4421SNicholas Bellinger struct iser_tx_desc *t; 2090ebbe4421SNicholas Bellinger 2091ebbe4421SNicholas Bellinger while (llnode) { 2092ebbe4421SNicholas Bellinger t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 2093ebbe4421SNicholas Bellinger llnode = llist_next(llnode); 2094ebbe4421SNicholas Bellinger wr = &t->isert_cmd->rdma_wr; 2095ebbe4421SNicholas Bellinger 2096897bb2c9SSagi Grimberg /** 2097897bb2c9SSagi Grimberg * If send_wr_num is 0 this means that we got 2098897bb2c9SSagi Grimberg * RDMA completion and we cleared it and we should 2099897bb2c9SSagi Grimberg * simply decrement the response post. else the 2100897bb2c9SSagi Grimberg * response is incorporated in send_wr_num, just 2101897bb2c9SSagi Grimberg * sub it. 2102897bb2c9SSagi Grimberg **/ 2103897bb2c9SSagi Grimberg if (wr->send_wr_num) 2104897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, 2105897bb2c9SSagi Grimberg &isert_conn->post_send_buf_count); 2106897bb2c9SSagi Grimberg else 2107897bb2c9SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 210803e7848aSNicholas Bellinger 210903e7848aSNicholas Bellinger isert_completion_put(t, t->isert_cmd, ib_dev, true); 2110ebbe4421SNicholas Bellinger } 2111ebbe4421SNicholas Bellinger tx_desc->comp_llnode_batch = NULL; 2112b8d26b3bSNicholas Bellinger 2113b8d26b3bSNicholas Bellinger if (!isert_cmd) 2114b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 2115b8d26b3bSNicholas Bellinger else 211603e7848aSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 2117b8d26b3bSNicholas Bellinger } 2118b8d26b3bSNicholas Bellinger 2119defd8848SNicholas Bellinger static void 2120defd8848SNicholas Bellinger isert_cq_rx_comp_err(struct isert_conn *isert_conn) 2121defd8848SNicholas Bellinger { 2122defd8848SNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2123defd8848SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 2124defd8848SNicholas Bellinger 2125defd8848SNicholas Bellinger if (isert_conn->post_recv_buf_count) 2126defd8848SNicholas Bellinger return; 2127defd8848SNicholas Bellinger 2128ebbe4421SNicholas Bellinger isert_cq_drain_comp_llist(isert_conn, ib_dev); 2129ebbe4421SNicholas Bellinger 2130defd8848SNicholas Bellinger if (conn->sess) { 2131defd8848SNicholas Bellinger target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2132defd8848SNicholas Bellinger target_wait_for_sess_cmds(conn->sess->se_sess); 2133defd8848SNicholas Bellinger } 2134defd8848SNicholas Bellinger 2135defd8848SNicholas Bellinger while (atomic_read(&isert_conn->post_send_buf_count)) 2136defd8848SNicholas Bellinger msleep(3000); 2137b8d26b3bSNicholas Bellinger 2138b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 2139954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 2140b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2141b2cb9649SNicholas Bellinger 21429d49f5e2SSagi Grimberg iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 21439d49f5e2SSagi Grimberg 2144defd8848SNicholas Bellinger complete(&isert_conn->conn_wait_comp_err); 2145b8d26b3bSNicholas Bellinger } 2146b8d26b3bSNicholas Bellinger 2147b8d26b3bSNicholas Bellinger static void 2148b8d26b3bSNicholas Bellinger isert_cq_tx_work(struct work_struct *work) 2149b8d26b3bSNicholas Bellinger { 2150b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = container_of(work, 2151b8d26b3bSNicholas Bellinger struct isert_cq_desc, cq_tx_work); 2152b8d26b3bSNicholas Bellinger struct isert_device *device = cq_desc->device; 2153b8d26b3bSNicholas Bellinger int cq_index = cq_desc->cq_index; 2154b8d26b3bSNicholas Bellinger struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 2155b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2156b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc; 2157b8d26b3bSNicholas Bellinger struct ib_wc wc; 2158b8d26b3bSNicholas Bellinger 2159b8d26b3bSNicholas Bellinger while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 2160b8d26b3bSNicholas Bellinger tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 2161b8d26b3bSNicholas Bellinger isert_conn = wc.qp->qp_context; 2162b8d26b3bSNicholas Bellinger 2163b8d26b3bSNicholas Bellinger if (wc.status == IB_WC_SUCCESS) { 2164b8d26b3bSNicholas Bellinger isert_send_completion(tx_desc, isert_conn); 2165b8d26b3bSNicholas Bellinger } else { 2166b8d26b3bSNicholas Bellinger pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2167b8d26b3bSNicholas Bellinger pr_debug("TX wc.status: 0x%08x\n", wc.status); 2168c5a2adbfSNicholas Bellinger pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 2169defd8848SNicholas Bellinger 21709bb4ca68SNicholas Bellinger if (wc.wr_id != ISER_FASTREG_LI_WRID) { 2171ebbe4421SNicholas Bellinger if (tx_desc->llnode_active) 2172ebbe4421SNicholas Bellinger continue; 2173ebbe4421SNicholas Bellinger 2174b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 2175defd8848SNicholas Bellinger isert_cq_tx_comp_err(tx_desc, isert_conn); 2176b8d26b3bSNicholas Bellinger } 2177b8d26b3bSNicholas Bellinger } 21789bb4ca68SNicholas Bellinger } 2179b8d26b3bSNicholas Bellinger 2180b8d26b3bSNicholas Bellinger ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 2181b8d26b3bSNicholas Bellinger } 2182b8d26b3bSNicholas Bellinger 2183b8d26b3bSNicholas Bellinger static void 2184b8d26b3bSNicholas Bellinger isert_cq_tx_callback(struct ib_cq *cq, void *context) 2185b8d26b3bSNicholas Bellinger { 2186b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2187b8d26b3bSNicholas Bellinger 2188b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 2189b8d26b3bSNicholas Bellinger } 2190b8d26b3bSNicholas Bellinger 2191b8d26b3bSNicholas Bellinger static void 2192b8d26b3bSNicholas Bellinger isert_cq_rx_work(struct work_struct *work) 2193b8d26b3bSNicholas Bellinger { 2194b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = container_of(work, 2195b8d26b3bSNicholas Bellinger struct isert_cq_desc, cq_rx_work); 2196b8d26b3bSNicholas Bellinger struct isert_device *device = cq_desc->device; 2197b8d26b3bSNicholas Bellinger int cq_index = cq_desc->cq_index; 2198b8d26b3bSNicholas Bellinger struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 2199b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2200b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 2201b8d26b3bSNicholas Bellinger struct ib_wc wc; 2202b8d26b3bSNicholas Bellinger unsigned long xfer_len; 2203b8d26b3bSNicholas Bellinger 2204b8d26b3bSNicholas Bellinger while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 2205b8d26b3bSNicholas Bellinger rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 2206b8d26b3bSNicholas Bellinger isert_conn = wc.qp->qp_context; 2207b8d26b3bSNicholas Bellinger 2208b8d26b3bSNicholas Bellinger if (wc.status == IB_WC_SUCCESS) { 2209b8d26b3bSNicholas Bellinger xfer_len = (unsigned long)wc.byte_len; 2210b8d26b3bSNicholas Bellinger isert_rx_completion(rx_desc, isert_conn, xfer_len); 2211b8d26b3bSNicholas Bellinger } else { 2212b8d26b3bSNicholas Bellinger pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2213c5a2adbfSNicholas Bellinger if (wc.status != IB_WC_WR_FLUSH_ERR) { 2214b8d26b3bSNicholas Bellinger pr_debug("RX wc.status: 0x%08x\n", wc.status); 2215c5a2adbfSNicholas Bellinger pr_debug("RX wc.vendor_err: 0x%08x\n", 2216c5a2adbfSNicholas Bellinger wc.vendor_err); 2217c5a2adbfSNicholas Bellinger } 2218b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 2219defd8848SNicholas Bellinger isert_cq_rx_comp_err(isert_conn); 2220b8d26b3bSNicholas Bellinger } 2221b8d26b3bSNicholas Bellinger } 2222b8d26b3bSNicholas Bellinger 2223b8d26b3bSNicholas Bellinger ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 2224b8d26b3bSNicholas Bellinger } 2225b8d26b3bSNicholas Bellinger 2226b8d26b3bSNicholas Bellinger static void 2227b8d26b3bSNicholas Bellinger isert_cq_rx_callback(struct ib_cq *cq, void *context) 2228b8d26b3bSNicholas Bellinger { 2229b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2230b8d26b3bSNicholas Bellinger 2231b8d26b3bSNicholas Bellinger queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 2232b8d26b3bSNicholas Bellinger } 2233b8d26b3bSNicholas Bellinger 2234b8d26b3bSNicholas Bellinger static int 2235b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2236b8d26b3bSNicholas Bellinger { 2237b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 2238b8d26b3bSNicholas Bellinger int ret; 2239b8d26b3bSNicholas Bellinger 2240b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 2241b8d26b3bSNicholas Bellinger 2242b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 2243b8d26b3bSNicholas Bellinger &wr_failed); 2244b8d26b3bSNicholas Bellinger if (ret) { 2245b8d26b3bSNicholas Bellinger pr_err("ib_post_send failed with %d\n", ret); 2246b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 2247b8d26b3bSNicholas Bellinger return ret; 2248b8d26b3bSNicholas Bellinger } 2249b8d26b3bSNicholas Bellinger return ret; 2250b8d26b3bSNicholas Bellinger } 2251b8d26b3bSNicholas Bellinger 2252b8d26b3bSNicholas Bellinger static int 2253b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2254b8d26b3bSNicholas Bellinger { 2255d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2256b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2257b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2258b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2259b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 2260b8d26b3bSNicholas Bellinger 2261b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2262b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2263b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2264b8d26b3bSNicholas Bellinger /* 2265b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 2266b8d26b3bSNicholas Bellinger */ 2267b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 2268b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2269b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2270b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2271b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2272dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 2273b8d26b3bSNicholas Bellinger 2274b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2275b8d26b3bSNicholas Bellinger cmd->sense_buffer); 2276b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2277b8d26b3bSNicholas Bellinger 2278b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2279b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2280dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2281b8d26b3bSNicholas Bellinger 2282dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2283dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 2284b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 2285b8d26b3bSNicholas Bellinger 2286dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 2287dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2288dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 2289b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 2290b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2291b8d26b3bSNicholas Bellinger } 2292b8d26b3bSNicholas Bellinger 22930d0f660dSNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2294b8d26b3bSNicholas Bellinger 2295b8d26b3bSNicholas Bellinger pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2296b8d26b3bSNicholas Bellinger 2297b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2298b8d26b3bSNicholas Bellinger } 2299b8d26b3bSNicholas Bellinger 2300131e6abcSNicholas Bellinger static void 2301131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2302131e6abcSNicholas Bellinger { 2303131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2304131e6abcSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2305131e6abcSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 2306131e6abcSNicholas Bellinger 2307131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 2308131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 2309131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 2310131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 2311131e6abcSNicholas Bellinger 2312131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 2313131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 2314131e6abcSNicholas Bellinger 2315131e6abcSNicholas Bellinger device->unreg_rdma_mem(isert_cmd, isert_conn); 2316131e6abcSNicholas Bellinger } 2317131e6abcSNicholas Bellinger 2318e70beee7SNicholas Bellinger static enum target_prot_op 2319e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 2320e70beee7SNicholas Bellinger { 2321e70beee7SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2322e70beee7SNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 2323e70beee7SNicholas Bellinger 2324e70beee7SNicholas Bellinger if (device->pi_capable) 2325e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 2326e70beee7SNicholas Bellinger 2327e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 2328e70beee7SNicholas Bellinger } 2329e70beee7SNicholas Bellinger 2330b8d26b3bSNicholas Bellinger static int 2331b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2332b8d26b3bSNicholas Bellinger bool nopout_response) 2333b8d26b3bSNicholas Bellinger { 2334d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2335b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2336b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2337b8d26b3bSNicholas Bellinger 2338b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2339b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2340b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 2341b8d26b3bSNicholas Bellinger nopout_response); 2342b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 234395b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2344b8d26b3bSNicholas Bellinger 23458b513d0cSMasanari Iida pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2346b8d26b3bSNicholas Bellinger 2347b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2348b8d26b3bSNicholas Bellinger } 2349b8d26b3bSNicholas Bellinger 2350b8d26b3bSNicholas Bellinger static int 2351b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2352b8d26b3bSNicholas Bellinger { 2353d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2354b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2355b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2356b8d26b3bSNicholas Bellinger 2357b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2358b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2359b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2360b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 236195b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2362b8d26b3bSNicholas Bellinger 2363b8d26b3bSNicholas Bellinger pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2364b8d26b3bSNicholas Bellinger 2365b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2366b8d26b3bSNicholas Bellinger } 2367b8d26b3bSNicholas Bellinger 2368b8d26b3bSNicholas Bellinger static int 2369b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2370b8d26b3bSNicholas Bellinger { 2371d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2372b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2373b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2374b8d26b3bSNicholas Bellinger 2375b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2376b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2377b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2378b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 237995b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2380b8d26b3bSNicholas Bellinger 2381b8d26b3bSNicholas Bellinger pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2382b8d26b3bSNicholas Bellinger 2383b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2384b8d26b3bSNicholas Bellinger } 2385b8d26b3bSNicholas Bellinger 2386b8d26b3bSNicholas Bellinger static int 2387b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2388b8d26b3bSNicholas Bellinger { 2389d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2390b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2391b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 23923df8f68aSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 23933df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 23943df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 23953df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2396b8d26b3bSNicholas Bellinger 2397b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 23983df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 2399b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 24003df8f68aSNicholas Bellinger 24013df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2402dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 24033df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 24043df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2405dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2406dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 24073df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 24083df8f68aSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 24093df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 24103df8f68aSNicholas Bellinger 241195b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2412b8d26b3bSNicholas Bellinger 2413b8d26b3bSNicholas Bellinger pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2414b8d26b3bSNicholas Bellinger 2415b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2416b8d26b3bSNicholas Bellinger } 2417b8d26b3bSNicholas Bellinger 2418b8d26b3bSNicholas Bellinger static int 2419adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2420adb54c29SNicholas Bellinger { 2421d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2422adb54c29SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2423adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2424adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2425adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2426adb54c29SNicholas Bellinger u32 txt_rsp_len; 2427adb54c29SNicholas Bellinger int rc; 2428adb54c29SNicholas Bellinger 2429adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 243022c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2431adb54c29SNicholas Bellinger if (rc < 0) 2432adb54c29SNicholas Bellinger return rc; 2433adb54c29SNicholas Bellinger 2434adb54c29SNicholas Bellinger txt_rsp_len = rc; 2435adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2436adb54c29SNicholas Bellinger 2437adb54c29SNicholas Bellinger if (txt_rsp_len) { 2438adb54c29SNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2439adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2440adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2441adb54c29SNicholas Bellinger 2442adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2443adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2444adb54c29SNicholas Bellinger 2445adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2446adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2447adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 2448adb54c29SNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 2449adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2450adb54c29SNicholas Bellinger } 245195b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2452adb54c29SNicholas Bellinger 2453adb54c29SNicholas Bellinger pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2454adb54c29SNicholas Bellinger 2455adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2456adb54c29SNicholas Bellinger } 2457adb54c29SNicholas Bellinger 2458adb54c29SNicholas Bellinger static int 2459b8d26b3bSNicholas Bellinger isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2460b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2461b8d26b3bSNicholas Bellinger u32 data_left, u32 offset) 2462b8d26b3bSNicholas Bellinger { 2463d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2464b8d26b3bSNicholas Bellinger struct scatterlist *sg_start, *tmp_sg; 2465b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2466b8d26b3bSNicholas Bellinger u32 sg_off, page_off; 2467b8d26b3bSNicholas Bellinger int i = 0, sg_nents; 2468b8d26b3bSNicholas Bellinger 2469b8d26b3bSNicholas Bellinger sg_off = offset / PAGE_SIZE; 2470b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2471b8d26b3bSNicholas Bellinger sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2472b8d26b3bSNicholas Bellinger page_off = offset % PAGE_SIZE; 2473b8d26b3bSNicholas Bellinger 2474b8d26b3bSNicholas Bellinger send_wr->sg_list = ib_sge; 2475b8d26b3bSNicholas Bellinger send_wr->num_sge = sg_nents; 2476b8d26b3bSNicholas Bellinger send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2477b8d26b3bSNicholas Bellinger /* 2478b8d26b3bSNicholas Bellinger * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2479b8d26b3bSNicholas Bellinger */ 2480b8d26b3bSNicholas Bellinger for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2481b8d26b3bSNicholas Bellinger pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2482b8d26b3bSNicholas Bellinger (unsigned long long)tmp_sg->dma_address, 2483b8d26b3bSNicholas Bellinger tmp_sg->length, page_off); 2484b8d26b3bSNicholas Bellinger 2485b8d26b3bSNicholas Bellinger ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2486b8d26b3bSNicholas Bellinger ib_sge->length = min_t(u32, data_left, 2487b8d26b3bSNicholas Bellinger ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2488b8d26b3bSNicholas Bellinger ib_sge->lkey = isert_conn->conn_mr->lkey; 2489b8d26b3bSNicholas Bellinger 249090ecc6e2SVu Pham pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 249190ecc6e2SVu Pham ib_sge->addr, ib_sge->length, ib_sge->lkey); 2492b8d26b3bSNicholas Bellinger page_off = 0; 2493b8d26b3bSNicholas Bellinger data_left -= ib_sge->length; 2494b8d26b3bSNicholas Bellinger ib_sge++; 2495b8d26b3bSNicholas Bellinger pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2496b8d26b3bSNicholas Bellinger } 2497b8d26b3bSNicholas Bellinger 2498b8d26b3bSNicholas Bellinger pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2499b8d26b3bSNicholas Bellinger send_wr->sg_list, send_wr->num_sge); 2500b8d26b3bSNicholas Bellinger 2501b8d26b3bSNicholas Bellinger return sg_nents; 2502b8d26b3bSNicholas Bellinger } 2503b8d26b3bSNicholas Bellinger 2504b8d26b3bSNicholas Bellinger static int 250590ecc6e2SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 250690ecc6e2SVu Pham struct isert_rdma_wr *wr) 2507b8d26b3bSNicholas Bellinger { 2508b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2509d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2510b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2511e3d7e4c3SSagi Grimberg struct isert_data_buf *data = &wr->data; 251290ecc6e2SVu Pham struct ib_send_wr *send_wr; 2513b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge; 2514e3d7e4c3SSagi Grimberg u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2515e3d7e4c3SSagi Grimberg int ret = 0, i, ib_sge_cnt; 2516b8d26b3bSNicholas Bellinger 251790ecc6e2SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2518b8d26b3bSNicholas Bellinger 2519e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2520e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2521e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2522e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2523e3d7e4c3SSagi Grimberg if (ret) 2524e3d7e4c3SSagi Grimberg return ret; 2525b8d26b3bSNicholas Bellinger 2526e3d7e4c3SSagi Grimberg data_left = data->len; 2527e3d7e4c3SSagi Grimberg offset = data->offset; 2528b8d26b3bSNicholas Bellinger 2529e3d7e4c3SSagi Grimberg ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2530b8d26b3bSNicholas Bellinger if (!ib_sge) { 253190ecc6e2SVu Pham pr_warn("Unable to allocate ib_sge\n"); 2532b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2533e3d7e4c3SSagi Grimberg goto unmap_cmd; 2534b8d26b3bSNicholas Bellinger } 253590ecc6e2SVu Pham wr->ib_sge = ib_sge; 2536b8d26b3bSNicholas Bellinger 2537e3d7e4c3SSagi Grimberg wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2538b8d26b3bSNicholas Bellinger wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2539b8d26b3bSNicholas Bellinger GFP_KERNEL); 2540b8d26b3bSNicholas Bellinger if (!wr->send_wr) { 254190ecc6e2SVu Pham pr_debug("Unable to allocate wr->send_wr\n"); 2542b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2543e3d7e4c3SSagi Grimberg goto unmap_cmd; 2544b8d26b3bSNicholas Bellinger } 2545b8d26b3bSNicholas Bellinger 2546b8d26b3bSNicholas Bellinger wr->isert_cmd = isert_cmd; 2547b8d26b3bSNicholas Bellinger rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2548b8d26b3bSNicholas Bellinger 2549b8d26b3bSNicholas Bellinger for (i = 0; i < wr->send_wr_num; i++) { 2550b8d26b3bSNicholas Bellinger send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2551b8d26b3bSNicholas Bellinger data_len = min(data_left, rdma_write_max); 2552b8d26b3bSNicholas Bellinger 2553b8d26b3bSNicholas Bellinger send_wr->send_flags = 0; 255490ecc6e2SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 255590ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 2556b8d26b3bSNicholas Bellinger send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2557b8d26b3bSNicholas Bellinger send_wr->wr.rdma.rkey = isert_cmd->read_stag; 255890ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 255990ecc6e2SVu Pham send_wr->next = &isert_cmd->tx_desc.send_wr; 256090ecc6e2SVu Pham else 256190ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 256290ecc6e2SVu Pham } else { 256390ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 256490ecc6e2SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 256590ecc6e2SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 256690ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 256790ecc6e2SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 256890ecc6e2SVu Pham else 256990ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 257090ecc6e2SVu Pham } 2571b8d26b3bSNicholas Bellinger 2572b8d26b3bSNicholas Bellinger ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2573b8d26b3bSNicholas Bellinger send_wr, data_len, offset); 2574b8d26b3bSNicholas Bellinger ib_sge += ib_sge_cnt; 2575b8d26b3bSNicholas Bellinger 2576b8d26b3bSNicholas Bellinger offset += data_len; 257790ecc6e2SVu Pham va_offset += data_len; 2578b8d26b3bSNicholas Bellinger data_left -= data_len; 2579b8d26b3bSNicholas Bellinger } 258090ecc6e2SVu Pham 258190ecc6e2SVu Pham return 0; 2582e3d7e4c3SSagi Grimberg unmap_cmd: 2583e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, data); 2584e3d7e4c3SSagi Grimberg 258590ecc6e2SVu Pham return ret; 258690ecc6e2SVu Pham } 258790ecc6e2SVu Pham 258890ecc6e2SVu Pham static int 258959464ef4SVu Pham isert_map_fr_pagelist(struct ib_device *ib_dev, 259059464ef4SVu Pham struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 259159464ef4SVu Pham { 259259464ef4SVu Pham u64 start_addr, end_addr, page, chunk_start = 0; 259359464ef4SVu Pham struct scatterlist *tmp_sg; 259459464ef4SVu Pham int i = 0, new_chunk, last_ent, n_pages; 259559464ef4SVu Pham 259659464ef4SVu Pham n_pages = 0; 259759464ef4SVu Pham new_chunk = 1; 259859464ef4SVu Pham last_ent = sg_nents - 1; 259959464ef4SVu Pham for_each_sg(sg_start, tmp_sg, sg_nents, i) { 260059464ef4SVu Pham start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 260159464ef4SVu Pham if (new_chunk) 260259464ef4SVu Pham chunk_start = start_addr; 260359464ef4SVu Pham end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 260459464ef4SVu Pham 260559464ef4SVu Pham pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 260659464ef4SVu Pham i, (unsigned long long)tmp_sg->dma_address, 260759464ef4SVu Pham tmp_sg->length); 260859464ef4SVu Pham 260959464ef4SVu Pham if ((end_addr & ~PAGE_MASK) && i < last_ent) { 261059464ef4SVu Pham new_chunk = 0; 261159464ef4SVu Pham continue; 261259464ef4SVu Pham } 261359464ef4SVu Pham new_chunk = 1; 261459464ef4SVu Pham 261559464ef4SVu Pham page = chunk_start & PAGE_MASK; 261659464ef4SVu Pham do { 261759464ef4SVu Pham fr_pl[n_pages++] = page; 261859464ef4SVu Pham pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 261959464ef4SVu Pham n_pages - 1, page); 262059464ef4SVu Pham page += PAGE_SIZE; 262159464ef4SVu Pham } while (page < end_addr); 262259464ef4SVu Pham } 262359464ef4SVu Pham 262459464ef4SVu Pham return n_pages; 262559464ef4SVu Pham } 262659464ef4SVu Pham 262759464ef4SVu Pham static int 2628e3d7e4c3SSagi Grimberg isert_fast_reg_mr(struct isert_conn *isert_conn, 2629e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc, 2630e3d7e4c3SSagi Grimberg struct isert_data_buf *mem, 26319e961ae7SSagi Grimberg enum isert_indicator ind, 2632e3d7e4c3SSagi Grimberg struct ib_sge *sge) 263359464ef4SVu Pham { 263459464ef4SVu Pham struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 26359e961ae7SSagi Grimberg struct ib_mr *mr; 26369e961ae7SSagi Grimberg struct ib_fast_reg_page_list *frpl; 263759464ef4SVu Pham struct ib_send_wr fr_wr, inv_wr; 263859464ef4SVu Pham struct ib_send_wr *bad_wr, *wr = NULL; 26399bd626e7SSagi Grimberg int ret, pagelist_len; 26409bd626e7SSagi Grimberg u32 page_off; 264159464ef4SVu Pham u8 key; 264259464ef4SVu Pham 2643e3d7e4c3SSagi Grimberg if (mem->dma_nents == 1) { 2644e3d7e4c3SSagi Grimberg sge->lkey = isert_conn->conn_mr->lkey; 2645e3d7e4c3SSagi Grimberg sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2646e3d7e4c3SSagi Grimberg sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 26479e961ae7SSagi Grimberg pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 26489e961ae7SSagi Grimberg __func__, __LINE__, sge->addr, sge->length, 26499e961ae7SSagi Grimberg sge->lkey); 2650e3d7e4c3SSagi Grimberg return 0; 2651e3d7e4c3SSagi Grimberg } 2652e3d7e4c3SSagi Grimberg 26539e961ae7SSagi Grimberg if (ind == ISERT_DATA_KEY_VALID) { 26549e961ae7SSagi Grimberg /* Registering data buffer */ 26559e961ae7SSagi Grimberg mr = fr_desc->data_mr; 26569e961ae7SSagi Grimberg frpl = fr_desc->data_frpl; 26579e961ae7SSagi Grimberg } else { 26589e961ae7SSagi Grimberg /* Registering protection buffer */ 26599e961ae7SSagi Grimberg mr = fr_desc->pi_ctx->prot_mr; 26609e961ae7SSagi Grimberg frpl = fr_desc->pi_ctx->prot_frpl; 26619e961ae7SSagi Grimberg } 26629e961ae7SSagi Grimberg 2663e3d7e4c3SSagi Grimberg page_off = mem->offset % PAGE_SIZE; 266459464ef4SVu Pham 26659bd626e7SSagi Grimberg pr_debug("Use fr_desc %p sg_nents %d offset %u\n", 2666e3d7e4c3SSagi Grimberg fr_desc, mem->nents, mem->offset); 266759464ef4SVu Pham 2668e3d7e4c3SSagi Grimberg pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 26699e961ae7SSagi Grimberg &frpl->page_list[0]); 267059464ef4SVu Pham 2671d3e125daSSagi Grimberg if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) { 267259464ef4SVu Pham memset(&inv_wr, 0, sizeof(inv_wr)); 26739bb4ca68SNicholas Bellinger inv_wr.wr_id = ISER_FASTREG_LI_WRID; 267459464ef4SVu Pham inv_wr.opcode = IB_WR_LOCAL_INV; 26759e961ae7SSagi Grimberg inv_wr.ex.invalidate_rkey = mr->rkey; 267659464ef4SVu Pham wr = &inv_wr; 267759464ef4SVu Pham /* Bump the key */ 26789e961ae7SSagi Grimberg key = (u8)(mr->rkey & 0x000000FF); 26799e961ae7SSagi Grimberg ib_update_fast_reg_key(mr, ++key); 268059464ef4SVu Pham } 268159464ef4SVu Pham 268259464ef4SVu Pham /* Prepare FASTREG WR */ 268359464ef4SVu Pham memset(&fr_wr, 0, sizeof(fr_wr)); 26849bb4ca68SNicholas Bellinger fr_wr.wr_id = ISER_FASTREG_LI_WRID; 268559464ef4SVu Pham fr_wr.opcode = IB_WR_FAST_REG_MR; 26869e961ae7SSagi Grimberg fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 26879e961ae7SSagi Grimberg fr_wr.wr.fast_reg.page_list = frpl; 268859464ef4SVu Pham fr_wr.wr.fast_reg.page_list_len = pagelist_len; 268959464ef4SVu Pham fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2690e3d7e4c3SSagi Grimberg fr_wr.wr.fast_reg.length = mem->len; 26919e961ae7SSagi Grimberg fr_wr.wr.fast_reg.rkey = mr->rkey; 269259464ef4SVu Pham fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 269359464ef4SVu Pham 269459464ef4SVu Pham if (!wr) 269559464ef4SVu Pham wr = &fr_wr; 269659464ef4SVu Pham else 269759464ef4SVu Pham wr->next = &fr_wr; 269859464ef4SVu Pham 269959464ef4SVu Pham ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 270059464ef4SVu Pham if (ret) { 270159464ef4SVu Pham pr_err("fast registration failed, ret:%d\n", ret); 270259464ef4SVu Pham return ret; 270359464ef4SVu Pham } 27049e961ae7SSagi Grimberg fr_desc->ind &= ~ind; 270559464ef4SVu Pham 27069e961ae7SSagi Grimberg sge->lkey = mr->lkey; 27079e961ae7SSagi Grimberg sge->addr = frpl->page_list[0] + page_off; 2708e3d7e4c3SSagi Grimberg sge->length = mem->len; 270959464ef4SVu Pham 27109e961ae7SSagi Grimberg pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 27119e961ae7SSagi Grimberg __func__, __LINE__, sge->addr, sge->length, 27129e961ae7SSagi Grimberg sge->lkey); 271359464ef4SVu Pham 271459464ef4SVu Pham return ret; 271559464ef4SVu Pham } 271659464ef4SVu Pham 27173d73cf1aSSagi Grimberg static inline void 27183d73cf1aSSagi Grimberg isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 27193d73cf1aSSagi Grimberg struct ib_sig_domain *domain) 27209e961ae7SSagi Grimberg { 272178eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 27223d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 27233d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 27243d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 272578eda2bbSSagi Grimberg /* 272678eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 272778eda2bbSSagi Grimberg * the target core would like to use it, we will take it 272878eda2bbSSagi Grimberg * from se_cmd. 272978eda2bbSSagi Grimberg */ 273078eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 273178eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 273278eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 273378eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 273478eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 273578eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 27363d73cf1aSSagi Grimberg }; 27373d73cf1aSSagi Grimberg 27383d73cf1aSSagi Grimberg static int 27393d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 27403d73cf1aSSagi Grimberg { 27419e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 27429e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 27439e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 274478eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 27453d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 27469e961ae7SSagi Grimberg break; 27479e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 27489e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 274978eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 27503d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 27519e961ae7SSagi Grimberg break; 27529e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 27539e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 27543d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 27553d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 27569e961ae7SSagi Grimberg break; 27579e961ae7SSagi Grimberg default: 27589e961ae7SSagi Grimberg pr_err("Unsupported PI operation %d\n", se_cmd->prot_op); 27599e961ae7SSagi Grimberg return -EINVAL; 27609e961ae7SSagi Grimberg } 27619e961ae7SSagi Grimberg 27629e961ae7SSagi Grimberg return 0; 27639e961ae7SSagi Grimberg } 27649e961ae7SSagi Grimberg 27659e961ae7SSagi Grimberg static inline u8 27669e961ae7SSagi Grimberg isert_set_prot_checks(u8 prot_checks) 27679e961ae7SSagi Grimberg { 27689e961ae7SSagi Grimberg return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 27699e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 27709e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 27719e961ae7SSagi Grimberg } 27729e961ae7SSagi Grimberg 27739e961ae7SSagi Grimberg static int 27749e961ae7SSagi Grimberg isert_reg_sig_mr(struct isert_conn *isert_conn, struct se_cmd *se_cmd, 27759e961ae7SSagi Grimberg struct fast_reg_descriptor *fr_desc, 27769e961ae7SSagi Grimberg struct ib_sge *data_sge, struct ib_sge *prot_sge, 27779e961ae7SSagi Grimberg struct ib_sge *sig_sge) 27789e961ae7SSagi Grimberg { 27799e961ae7SSagi Grimberg struct ib_send_wr sig_wr, inv_wr; 27809e961ae7SSagi Grimberg struct ib_send_wr *bad_wr, *wr = NULL; 27819e961ae7SSagi Grimberg struct pi_context *pi_ctx = fr_desc->pi_ctx; 27829e961ae7SSagi Grimberg struct ib_sig_attrs sig_attrs; 27839e961ae7SSagi Grimberg int ret; 27849e961ae7SSagi Grimberg u32 key; 27859e961ae7SSagi Grimberg 27869e961ae7SSagi Grimberg memset(&sig_attrs, 0, sizeof(sig_attrs)); 27879e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 27889e961ae7SSagi Grimberg if (ret) 27899e961ae7SSagi Grimberg goto err; 27909e961ae7SSagi Grimberg 27919e961ae7SSagi Grimberg sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 27929e961ae7SSagi Grimberg 27939e961ae7SSagi Grimberg if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 27949e961ae7SSagi Grimberg memset(&inv_wr, 0, sizeof(inv_wr)); 27959e961ae7SSagi Grimberg inv_wr.opcode = IB_WR_LOCAL_INV; 2796c2caa207SSagi Grimberg inv_wr.wr_id = ISER_FASTREG_LI_WRID; 27979e961ae7SSagi Grimberg inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; 27989e961ae7SSagi Grimberg wr = &inv_wr; 27999e961ae7SSagi Grimberg /* Bump the key */ 28009e961ae7SSagi Grimberg key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); 28019e961ae7SSagi Grimberg ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); 28029e961ae7SSagi Grimberg } 28039e961ae7SSagi Grimberg 28049e961ae7SSagi Grimberg memset(&sig_wr, 0, sizeof(sig_wr)); 28059e961ae7SSagi Grimberg sig_wr.opcode = IB_WR_REG_SIG_MR; 2806c2caa207SSagi Grimberg sig_wr.wr_id = ISER_FASTREG_LI_WRID; 28079e961ae7SSagi Grimberg sig_wr.sg_list = data_sge; 28089e961ae7SSagi Grimberg sig_wr.num_sge = 1; 28099e961ae7SSagi Grimberg sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 28109e961ae7SSagi Grimberg sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 28119e961ae7SSagi Grimberg sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 28129e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 28139e961ae7SSagi Grimberg sig_wr.wr.sig_handover.prot = prot_sge; 28149e961ae7SSagi Grimberg 28159e961ae7SSagi Grimberg if (!wr) 28169e961ae7SSagi Grimberg wr = &sig_wr; 28179e961ae7SSagi Grimberg else 28189e961ae7SSagi Grimberg wr->next = &sig_wr; 28199e961ae7SSagi Grimberg 28209e961ae7SSagi Grimberg ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 28219e961ae7SSagi Grimberg if (ret) { 28229e961ae7SSagi Grimberg pr_err("fast registration failed, ret:%d\n", ret); 28239e961ae7SSagi Grimberg goto err; 28249e961ae7SSagi Grimberg } 28259e961ae7SSagi Grimberg fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 28269e961ae7SSagi Grimberg 28279e961ae7SSagi Grimberg sig_sge->lkey = pi_ctx->sig_mr->lkey; 28289e961ae7SSagi Grimberg sig_sge->addr = 0; 28299e961ae7SSagi Grimberg sig_sge->length = se_cmd->data_length; 28309e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 28319e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 28329e961ae7SSagi Grimberg /* 28339e961ae7SSagi Grimberg * We have protection guards on the wire 28349e961ae7SSagi Grimberg * so we need to set a larget transfer 28359e961ae7SSagi Grimberg */ 28369e961ae7SSagi Grimberg sig_sge->length += se_cmd->prot_length; 28379e961ae7SSagi Grimberg 28389e961ae7SSagi Grimberg pr_debug("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 28399e961ae7SSagi Grimberg sig_sge->addr, sig_sge->length, 28409e961ae7SSagi Grimberg sig_sge->lkey); 28419e961ae7SSagi Grimberg err: 28429e961ae7SSagi Grimberg return ret; 28439e961ae7SSagi Grimberg } 28449e961ae7SSagi Grimberg 284559464ef4SVu Pham static int 2846a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 284759464ef4SVu Pham struct isert_rdma_wr *wr) 284859464ef4SVu Pham { 284959464ef4SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 285059464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2851e3d7e4c3SSagi Grimberg struct isert_conn *isert_conn = conn->context; 28529e961ae7SSagi Grimberg struct ib_sge data_sge; 285359464ef4SVu Pham struct ib_send_wr *send_wr; 2854e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc = NULL; 2855e3d7e4c3SSagi Grimberg u32 offset; 2856e3d7e4c3SSagi Grimberg int ret = 0; 285759464ef4SVu Pham unsigned long flags; 285859464ef4SVu Pham 285959464ef4SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2860e3d7e4c3SSagi Grimberg 2861e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2862e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2863e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2864e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2865e3d7e4c3SSagi Grimberg if (ret) 2866e3d7e4c3SSagi Grimberg return ret; 2867e3d7e4c3SSagi Grimberg 28689e961ae7SSagi Grimberg if (wr->data.dma_nents != 1 || 28699e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_NORMAL) { 2870e3d7e4c3SSagi Grimberg spin_lock_irqsave(&isert_conn->conn_lock, flags); 2871e3d7e4c3SSagi Grimberg fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2872e3d7e4c3SSagi Grimberg struct fast_reg_descriptor, list); 2873e3d7e4c3SSagi Grimberg list_del(&fr_desc->list); 2874e3d7e4c3SSagi Grimberg spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2875e3d7e4c3SSagi Grimberg wr->fr_desc = fr_desc; 287659464ef4SVu Pham } 287759464ef4SVu Pham 28789e961ae7SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 28799e961ae7SSagi Grimberg ISERT_DATA_KEY_VALID, &data_sge); 2880e3d7e4c3SSagi Grimberg if (ret) 2881e3d7e4c3SSagi Grimberg goto unmap_cmd; 288259464ef4SVu Pham 28839e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_NORMAL) { 28849e961ae7SSagi Grimberg struct ib_sge prot_sge, sig_sge; 28859e961ae7SSagi Grimberg 28869e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) { 28879e961ae7SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, 28889e961ae7SSagi Grimberg se_cmd->t_prot_sg, 28899e961ae7SSagi Grimberg se_cmd->t_prot_nents, 28909e961ae7SSagi Grimberg se_cmd->prot_length, 28919e961ae7SSagi Grimberg 0, wr->iser_ib_op, &wr->prot); 28929e961ae7SSagi Grimberg if (ret) 28939e961ae7SSagi Grimberg goto unmap_cmd; 28949e961ae7SSagi Grimberg 28959e961ae7SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->prot, 28969e961ae7SSagi Grimberg ISERT_PROT_KEY_VALID, &prot_sge); 28979e961ae7SSagi Grimberg if (ret) 28989e961ae7SSagi Grimberg goto unmap_prot_cmd; 28999e961ae7SSagi Grimberg } 29009e961ae7SSagi Grimberg 29019e961ae7SSagi Grimberg ret = isert_reg_sig_mr(isert_conn, se_cmd, fr_desc, 29029e961ae7SSagi Grimberg &data_sge, &prot_sge, &sig_sge); 29039e961ae7SSagi Grimberg if (ret) 29049e961ae7SSagi Grimberg goto unmap_prot_cmd; 29059e961ae7SSagi Grimberg 29069e961ae7SSagi Grimberg fr_desc->ind |= ISERT_PROTECTED; 29079e961ae7SSagi Grimberg memcpy(&wr->s_ib_sge, &sig_sge, sizeof(sig_sge)); 29089e961ae7SSagi Grimberg } else 29099e961ae7SSagi Grimberg memcpy(&wr->s_ib_sge, &data_sge, sizeof(data_sge)); 29109e961ae7SSagi Grimberg 2911e3d7e4c3SSagi Grimberg wr->ib_sge = &wr->s_ib_sge; 291259464ef4SVu Pham wr->send_wr_num = 1; 291359464ef4SVu Pham memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 291459464ef4SVu Pham wr->send_wr = &wr->s_send_wr; 291559464ef4SVu Pham wr->isert_cmd = isert_cmd; 291659464ef4SVu Pham 291759464ef4SVu Pham send_wr = &isert_cmd->rdma_wr.s_send_wr; 2918e3d7e4c3SSagi Grimberg send_wr->sg_list = &wr->s_ib_sge; 291959464ef4SVu Pham send_wr->num_sge = 1; 292059464ef4SVu Pham send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 292159464ef4SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 292259464ef4SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 292359464ef4SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 292459464ef4SVu Pham send_wr->wr.rdma.rkey = isert_cmd->read_stag; 29259e961ae7SSagi Grimberg send_wr->send_flags = se_cmd->prot_op == TARGET_PROT_NORMAL ? 29269e961ae7SSagi Grimberg 0 : IB_SEND_SIGNALED; 292759464ef4SVu Pham } else { 292859464ef4SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 292959464ef4SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 293059464ef4SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 293159464ef4SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 293259464ef4SVu Pham } 293359464ef4SVu Pham 293459464ef4SVu Pham return 0; 29359e961ae7SSagi Grimberg unmap_prot_cmd: 29369e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 29379e961ae7SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 2938e3d7e4c3SSagi Grimberg unmap_cmd: 2939e3d7e4c3SSagi Grimberg if (fr_desc) { 2940e3d7e4c3SSagi Grimberg spin_lock_irqsave(&isert_conn->conn_lock, flags); 2941e3d7e4c3SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2942e3d7e4c3SSagi Grimberg spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2943e3d7e4c3SSagi Grimberg } 2944e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 294559464ef4SVu Pham 294659464ef4SVu Pham return ret; 294759464ef4SVu Pham } 294859464ef4SVu Pham 294959464ef4SVu Pham static int 295090ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 295190ecc6e2SVu Pham { 295290ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 295359464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 295490ecc6e2SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 295590ecc6e2SVu Pham struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2956d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 295790ecc6e2SVu Pham struct ib_send_wr *wr_failed; 295890ecc6e2SVu Pham int rc; 295990ecc6e2SVu Pham 296090ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 296190ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 296290ecc6e2SVu Pham wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2963d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 296490ecc6e2SVu Pham if (rc) { 296590ecc6e2SVu Pham pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 296690ecc6e2SVu Pham return rc; 296790ecc6e2SVu Pham } 296890ecc6e2SVu Pham 29699e961ae7SSagi Grimberg if (se_cmd->prot_op == TARGET_PROT_NORMAL) { 2970b8d26b3bSNicholas Bellinger /* 2971b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2972b8d26b3bSNicholas Bellinger */ 29739e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 29749e961ae7SSagi Grimberg &isert_cmd->tx_desc); 297504d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2976b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2977b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 297895b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 29790d0f660dSNicholas Bellinger &isert_cmd->tx_desc.send_wr, false); 29809e961ae7SSagi Grimberg isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2981897bb2c9SSagi Grimberg wr->send_wr_num += 1; 29829e961ae7SSagi Grimberg } 2983b8d26b3bSNicholas Bellinger 2984897bb2c9SSagi Grimberg atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2985b8d26b3bSNicholas Bellinger 2986b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2987b8d26b3bSNicholas Bellinger if (rc) { 2988b8d26b3bSNicholas Bellinger pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2989897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2990b8d26b3bSNicholas Bellinger } 29919e961ae7SSagi Grimberg 29929e961ae7SSagi Grimberg if (se_cmd->prot_op == TARGET_PROT_NORMAL) 29939e961ae7SSagi Grimberg pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 29949e961ae7SSagi Grimberg "READ\n", isert_cmd); 29959e961ae7SSagi Grimberg else 29969e961ae7SSagi Grimberg pr_debug("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 299790ecc6e2SVu Pham isert_cmd); 2998b8d26b3bSNicholas Bellinger 299990ecc6e2SVu Pham return 1; 3000b8d26b3bSNicholas Bellinger } 3001b8d26b3bSNicholas Bellinger 3002b8d26b3bSNicholas Bellinger static int 3003b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 3004b8d26b3bSNicholas Bellinger { 3005b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 3006d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 3007b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 3008b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3009d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 301090ecc6e2SVu Pham struct ib_send_wr *wr_failed; 301190ecc6e2SVu Pham int rc; 3012b8d26b3bSNicholas Bellinger 301390ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 301490ecc6e2SVu Pham isert_cmd, se_cmd->data_length, cmd->write_data_done); 3015b8d26b3bSNicholas Bellinger wr->iser_ib_op = ISER_IB_RDMA_READ; 3016d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 301790ecc6e2SVu Pham if (rc) { 301890ecc6e2SVu Pham pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 301990ecc6e2SVu Pham return rc; 3020b8d26b3bSNicholas Bellinger } 3021b8d26b3bSNicholas Bellinger 3022b6b87a1dSNicholas Bellinger atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 3023b8d26b3bSNicholas Bellinger 3024b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 3025b8d26b3bSNicholas Bellinger if (rc) { 3026b8d26b3bSNicholas Bellinger pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 3027b6b87a1dSNicholas Bellinger atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 3028b8d26b3bSNicholas Bellinger } 302990ecc6e2SVu Pham pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 303090ecc6e2SVu Pham isert_cmd); 3031b8d26b3bSNicholas Bellinger 303290ecc6e2SVu Pham return 0; 3033b8d26b3bSNicholas Bellinger } 3034b8d26b3bSNicholas Bellinger 3035b8d26b3bSNicholas Bellinger static int 3036b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3037b8d26b3bSNicholas Bellinger { 3038b8d26b3bSNicholas Bellinger int ret; 3039b8d26b3bSNicholas Bellinger 3040b8d26b3bSNicholas Bellinger switch (state) { 3041b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3042b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 3043b8d26b3bSNicholas Bellinger break; 3044b8d26b3bSNicholas Bellinger default: 3045b8d26b3bSNicholas Bellinger pr_err("Unknown immediate state: 0x%02x\n", state); 3046b8d26b3bSNicholas Bellinger ret = -EINVAL; 3047b8d26b3bSNicholas Bellinger break; 3048b8d26b3bSNicholas Bellinger } 3049b8d26b3bSNicholas Bellinger 3050b8d26b3bSNicholas Bellinger return ret; 3051b8d26b3bSNicholas Bellinger } 3052b8d26b3bSNicholas Bellinger 3053b8d26b3bSNicholas Bellinger static int 3054b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3055b8d26b3bSNicholas Bellinger { 3056b8d26b3bSNicholas Bellinger int ret; 3057b8d26b3bSNicholas Bellinger 3058b8d26b3bSNicholas Bellinger switch (state) { 3059b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 3060b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 3061b8d26b3bSNicholas Bellinger if (!ret) { 3062b8d26b3bSNicholas Bellinger pr_debug("Returning iSER Logout -EAGAIN\n"); 3063b8d26b3bSNicholas Bellinger ret = -EAGAIN; 3064b8d26b3bSNicholas Bellinger } 3065b8d26b3bSNicholas Bellinger break; 3066b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 3067b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 3068b8d26b3bSNicholas Bellinger break; 3069b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 3070b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 3071b8d26b3bSNicholas Bellinger break; 3072b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 3073b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 3074b8d26b3bSNicholas Bellinger break; 3075adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 3076adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 3077adb54c29SNicholas Bellinger break; 3078b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 3079b8d26b3bSNicholas Bellinger /* 3080b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 3081b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 3082b8d26b3bSNicholas Bellinger */ 3083b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 3084b8d26b3bSNicholas Bellinger break; 3085b8d26b3bSNicholas Bellinger default: 3086b8d26b3bSNicholas Bellinger pr_err("Unknown response state: 0x%02x\n", state); 3087b8d26b3bSNicholas Bellinger ret = -EINVAL; 3088b8d26b3bSNicholas Bellinger break; 3089b8d26b3bSNicholas Bellinger } 3090b8d26b3bSNicholas Bellinger 3091b8d26b3bSNicholas Bellinger return ret; 3092b8d26b3bSNicholas Bellinger } 3093b8d26b3bSNicholas Bellinger 3094ca6c1d82SSagi Grimberg struct rdma_cm_id * 3095ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 3096ca6c1d82SSagi Grimberg { 3097ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 3098ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 3099ca6c1d82SSagi Grimberg struct sockaddr *sa; 3100ca6c1d82SSagi Grimberg int ret; 3101ca6c1d82SSagi Grimberg 3102ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 3103ca6c1d82SSagi Grimberg pr_debug("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3104ca6c1d82SSagi Grimberg 3105ca6c1d82SSagi Grimberg id = rdma_create_id(isert_cma_handler, isert_np, 3106ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 3107ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 3108ca6c1d82SSagi Grimberg pr_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3109ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 3110ca6c1d82SSagi Grimberg goto out; 3111ca6c1d82SSagi Grimberg } 3112ca6c1d82SSagi Grimberg pr_debug("id %p context %p\n", id, id->context); 3113ca6c1d82SSagi Grimberg 3114ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 3115ca6c1d82SSagi Grimberg if (ret) { 3116ca6c1d82SSagi Grimberg pr_err("rdma_bind_addr() failed: %d\n", ret); 3117ca6c1d82SSagi Grimberg goto out_id; 3118ca6c1d82SSagi Grimberg } 3119ca6c1d82SSagi Grimberg 3120ca6c1d82SSagi Grimberg ret = rdma_listen(id, ISERT_RDMA_LISTEN_BACKLOG); 3121ca6c1d82SSagi Grimberg if (ret) { 3122ca6c1d82SSagi Grimberg pr_err("rdma_listen() failed: %d\n", ret); 3123ca6c1d82SSagi Grimberg goto out_id; 3124ca6c1d82SSagi Grimberg } 3125ca6c1d82SSagi Grimberg 3126ca6c1d82SSagi Grimberg return id; 3127ca6c1d82SSagi Grimberg out_id: 3128ca6c1d82SSagi Grimberg rdma_destroy_id(id); 3129ca6c1d82SSagi Grimberg out: 3130ca6c1d82SSagi Grimberg return ERR_PTR(ret); 3131ca6c1d82SSagi Grimberg } 3132ca6c1d82SSagi Grimberg 3133b8d26b3bSNicholas Bellinger static int 3134b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 3135b8d26b3bSNicholas Bellinger struct __kernel_sockaddr_storage *ksockaddr) 3136b8d26b3bSNicholas Bellinger { 3137b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 3138b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 3139b8d26b3bSNicholas Bellinger int ret; 3140b8d26b3bSNicholas Bellinger 3141b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3142b8d26b3bSNicholas Bellinger if (!isert_np) { 3143b8d26b3bSNicholas Bellinger pr_err("Unable to allocate struct isert_np\n"); 3144b8d26b3bSNicholas Bellinger return -ENOMEM; 3145b8d26b3bSNicholas Bellinger } 3146531b7bf4SSagi Grimberg sema_init(&isert_np->np_sem, 0); 3147b8d26b3bSNicholas Bellinger mutex_init(&isert_np->np_accept_mutex); 3148b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_np->np_accept_list); 3149b8d26b3bSNicholas Bellinger init_completion(&isert_np->np_login_comp); 3150ca6c1d82SSagi Grimberg isert_np->np = np; 3151b8d26b3bSNicholas Bellinger 3152b8d26b3bSNicholas Bellinger /* 3153b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 3154b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 3155b8d26b3bSNicholas Bellinger */ 3156b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 3157b8d26b3bSNicholas Bellinger sizeof(struct __kernel_sockaddr_storage)); 3158b8d26b3bSNicholas Bellinger 3159ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 3160b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 3161b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 3162b8d26b3bSNicholas Bellinger goto out; 3163b8d26b3bSNicholas Bellinger } 3164b8d26b3bSNicholas Bellinger 3165b8d26b3bSNicholas Bellinger isert_np->np_cm_id = isert_lid; 3166b8d26b3bSNicholas Bellinger np->np_context = isert_np; 3167b8d26b3bSNicholas Bellinger 3168b8d26b3bSNicholas Bellinger return 0; 3169b8d26b3bSNicholas Bellinger 3170b8d26b3bSNicholas Bellinger out: 3171b8d26b3bSNicholas Bellinger kfree(isert_np); 3172ca6c1d82SSagi Grimberg 3173b8d26b3bSNicholas Bellinger return ret; 3174b8d26b3bSNicholas Bellinger } 3175b8d26b3bSNicholas Bellinger 3176b8d26b3bSNicholas Bellinger static int 3177b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 3178b8d26b3bSNicholas Bellinger { 3179b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3180b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 3181b8d26b3bSNicholas Bellinger int ret; 3182b8d26b3bSNicholas Bellinger 3183b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 3184b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 3185b8d26b3bSNicholas Bellinger cp.retry_count = 7; 3186b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 3187b8d26b3bSNicholas Bellinger 3188b8d26b3bSNicholas Bellinger pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 3189b8d26b3bSNicholas Bellinger 3190b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 3191b8d26b3bSNicholas Bellinger if (ret) { 3192b8d26b3bSNicholas Bellinger pr_err("rdma_accept() failed with: %d\n", ret); 3193b8d26b3bSNicholas Bellinger return ret; 3194b8d26b3bSNicholas Bellinger } 3195b8d26b3bSNicholas Bellinger 3196b8d26b3bSNicholas Bellinger pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 3197b8d26b3bSNicholas Bellinger 3198b8d26b3bSNicholas Bellinger return 0; 3199b8d26b3bSNicholas Bellinger } 3200b8d26b3bSNicholas Bellinger 3201b8d26b3bSNicholas Bellinger static int 3202b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3203b8d26b3bSNicholas Bellinger { 3204b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3205b8d26b3bSNicholas Bellinger int ret; 3206b8d26b3bSNicholas Bellinger 32072371e5daSSagi Grimberg pr_info("before login_req comp conn: %p\n", isert_conn); 32082371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 32092371e5daSSagi Grimberg if (ret) { 32102371e5daSSagi Grimberg pr_err("isert_conn %p interrupted before got login req\n", 32112371e5daSSagi Grimberg isert_conn); 32122371e5daSSagi Grimberg return ret; 32132371e5daSSagi Grimberg } 32142371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 32152371e5daSSagi Grimberg 32166faaa85fSNicholas Bellinger /* 32176faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 32186faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 32196faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 32206faaa85fSNicholas Bellinger * into a NOP. 32216faaa85fSNicholas Bellinger */ 32226faaa85fSNicholas Bellinger if (!login->first_request) 32236faaa85fSNicholas Bellinger return 0; 3224b8d26b3bSNicholas Bellinger 32252371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 32262371e5daSSagi Grimberg 32272371e5daSSagi Grimberg pr_info("before conn_login_comp conn: %p\n", conn); 3228b8d26b3bSNicholas Bellinger ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 3229b8d26b3bSNicholas Bellinger if (ret) 3230b8d26b3bSNicholas Bellinger return ret; 3231b8d26b3bSNicholas Bellinger 32322371e5daSSagi Grimberg pr_info("processing login->req: %p\n", login->req); 32332371e5daSSagi Grimberg 3234b8d26b3bSNicholas Bellinger return 0; 3235b8d26b3bSNicholas Bellinger } 3236b8d26b3bSNicholas Bellinger 3237b8d26b3bSNicholas Bellinger static void 3238b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3239b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 3240b8d26b3bSNicholas Bellinger { 3241b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3242b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 3243b8d26b3bSNicholas Bellinger struct sockaddr_in *sock_in; 3244b8d26b3bSNicholas Bellinger struct sockaddr_in6 *sock_in6; 3245b8d26b3bSNicholas Bellinger 3246b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 3247b8d26b3bSNicholas Bellinger 3248b8d26b3bSNicholas Bellinger if (np->np_sockaddr.ss_family == AF_INET6) { 3249b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3250b8d26b3bSNicholas Bellinger snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3251b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 3252b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in6->sin6_port); 3253b8d26b3bSNicholas Bellinger 3254b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3255b8d26b3bSNicholas Bellinger snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3256b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 3257b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in6->sin6_port); 3258b8d26b3bSNicholas Bellinger } else { 3259b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3260b8d26b3bSNicholas Bellinger sprintf(conn->login_ip, "%pI4", 3261b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 3262b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in->sin_port); 3263b8d26b3bSNicholas Bellinger 3264b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3265b8d26b3bSNicholas Bellinger sprintf(conn->local_ip, "%pI4", 3266b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 3267b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in->sin_port); 3268b8d26b3bSNicholas Bellinger } 3269b8d26b3bSNicholas Bellinger } 3270b8d26b3bSNicholas Bellinger 3271b8d26b3bSNicholas Bellinger static int 3272b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3273b8d26b3bSNicholas Bellinger { 3274b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 3275b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 3276b8d26b3bSNicholas Bellinger int max_accept = 0, ret; 3277b8d26b3bSNicholas Bellinger 3278b8d26b3bSNicholas Bellinger accept_wait: 3279531b7bf4SSagi Grimberg ret = down_interruptible(&isert_np->np_sem); 32801acff63fSSagi Grimberg if (ret || max_accept > 5) 3281b8d26b3bSNicholas Bellinger return -ENODEV; 3282b8d26b3bSNicholas Bellinger 3283b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 3284e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3285b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3286e346ab34SSagi Grimberg pr_debug("np_thread_state %d for isert_accept_np\n", 3287e346ab34SSagi Grimberg np->np_thread_state); 3288e346ab34SSagi Grimberg /** 3289e346ab34SSagi Grimberg * No point in stalling here when np_thread 3290e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 3291e346ab34SSagi Grimberg **/ 3292b8d26b3bSNicholas Bellinger return -ENODEV; 3293b8d26b3bSNicholas Bellinger } 3294b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3295b8d26b3bSNicholas Bellinger 3296b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 3297b8d26b3bSNicholas Bellinger if (list_empty(&isert_np->np_accept_list)) { 3298b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3299b8d26b3bSNicholas Bellinger max_accept++; 3300b8d26b3bSNicholas Bellinger goto accept_wait; 3301b8d26b3bSNicholas Bellinger } 3302b8d26b3bSNicholas Bellinger isert_conn = list_first_entry(&isert_np->np_accept_list, 3303b8d26b3bSNicholas Bellinger struct isert_conn, conn_accept_node); 3304b8d26b3bSNicholas Bellinger list_del_init(&isert_conn->conn_accept_node); 3305b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3306b8d26b3bSNicholas Bellinger 3307b8d26b3bSNicholas Bellinger conn->context = isert_conn; 3308b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 3309b8d26b3bSNicholas Bellinger max_accept = 0; 3310b8d26b3bSNicholas Bellinger 3311b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 3312b8d26b3bSNicholas Bellinger 33132371e5daSSagi Grimberg pr_debug("Processing isert_conn: %p\n", isert_conn); 33142371e5daSSagi Grimberg 3315b8d26b3bSNicholas Bellinger return 0; 3316b8d26b3bSNicholas Bellinger } 3317b8d26b3bSNicholas Bellinger 3318b8d26b3bSNicholas Bellinger static void 3319b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 3320b8d26b3bSNicholas Bellinger { 3321b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 3322b8d26b3bSNicholas Bellinger 33233b726ae2SSagi Grimberg if (isert_np->np_cm_id) 3324b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_np->np_cm_id); 3325b8d26b3bSNicholas Bellinger 3326b8d26b3bSNicholas Bellinger np->np_context = NULL; 3327b8d26b3bSNicholas Bellinger kfree(isert_np); 3328b8d26b3bSNicholas Bellinger } 3329b8d26b3bSNicholas Bellinger 3330*b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 3331*b02efbfcSSagi Grimberg { 3332*b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 3333*b02efbfcSSagi Grimberg struct isert_conn, 3334*b02efbfcSSagi Grimberg release_work); 3335*b02efbfcSSagi Grimberg 3336*b02efbfcSSagi Grimberg pr_info("Starting release conn %p\n", isert_conn); 3337*b02efbfcSSagi Grimberg 3338*b02efbfcSSagi Grimberg wait_for_completion(&isert_conn->conn_wait); 3339*b02efbfcSSagi Grimberg 3340*b02efbfcSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 3341*b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 3342*b02efbfcSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 3343*b02efbfcSSagi Grimberg 3344*b02efbfcSSagi Grimberg pr_info("Destroying conn %p\n", isert_conn); 3345*b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 3346*b02efbfcSSagi Grimberg } 3347*b02efbfcSSagi Grimberg 3348defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 3349b8d26b3bSNicholas Bellinger { 3350b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3351b8d26b3bSNicholas Bellinger 3352defd8848SNicholas Bellinger pr_debug("isert_wait_conn: Starting \n"); 3353b8d26b3bSNicholas Bellinger 33549d49f5e2SSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 3355b8d26b3bSNicholas Bellinger /* 3356b8d26b3bSNicholas Bellinger * Only wait for conn_wait_comp_err if the isert_conn made it 3357b8d26b3bSNicholas Bellinger * into full feature phase.. 3358b8d26b3bSNicholas Bellinger */ 3359b2cb9649SNicholas Bellinger if (isert_conn->state == ISER_CONN_INIT) { 3360b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 3361b2cb9649SNicholas Bellinger return; 3362b2cb9649SNicholas Bellinger } 3363954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 3364b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 3365b2cb9649SNicholas Bellinger 3366defd8848SNicholas Bellinger wait_for_completion(&isert_conn->conn_wait_comp_err); 3367954f2372SSagi Grimberg 3368*b02efbfcSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 3369*b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 3370defd8848SNicholas Bellinger } 3371defd8848SNicholas Bellinger 3372defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 3373defd8848SNicholas Bellinger { 3374defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3375b8d26b3bSNicholas Bellinger 3376b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 3377b8d26b3bSNicholas Bellinger } 3378b8d26b3bSNicholas Bellinger 3379b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 3380b8d26b3bSNicholas Bellinger .name = "IB/iSER", 3381b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 3382d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 3383b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 3384b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 3385b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 3386b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 3387defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 3388b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 3389b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 3390b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 3391b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 3392b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 3393b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 3394b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 3395b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 3396131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 3397e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3398b8d26b3bSNicholas Bellinger }; 3399b8d26b3bSNicholas Bellinger 3400b8d26b3bSNicholas Bellinger static int __init isert_init(void) 3401b8d26b3bSNicholas Bellinger { 3402b8d26b3bSNicholas Bellinger int ret; 3403b8d26b3bSNicholas Bellinger 3404b8d26b3bSNicholas Bellinger isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 3405b8d26b3bSNicholas Bellinger if (!isert_rx_wq) { 3406b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_rx_wq\n"); 3407b8d26b3bSNicholas Bellinger return -ENOMEM; 3408b8d26b3bSNicholas Bellinger } 3409b8d26b3bSNicholas Bellinger 3410b8d26b3bSNicholas Bellinger isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 3411b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 3412b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_comp_wq\n"); 3413b8d26b3bSNicholas Bellinger ret = -ENOMEM; 3414b8d26b3bSNicholas Bellinger goto destroy_rx_wq; 3415b8d26b3bSNicholas Bellinger } 3416b8d26b3bSNicholas Bellinger 3417*b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3418*b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 3419*b02efbfcSSagi Grimberg if (!isert_release_wq) { 3420*b02efbfcSSagi Grimberg pr_err("Unable to allocate isert_release_wq\n"); 3421*b02efbfcSSagi Grimberg ret = -ENOMEM; 3422*b02efbfcSSagi Grimberg goto destroy_comp_wq; 3423*b02efbfcSSagi Grimberg } 3424*b02efbfcSSagi Grimberg 3425b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 3426*b02efbfcSSagi Grimberg pr_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3427*b02efbfcSSagi Grimberg 3428b8d26b3bSNicholas Bellinger return 0; 3429b8d26b3bSNicholas Bellinger 3430*b02efbfcSSagi Grimberg destroy_comp_wq: 3431*b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 3432b8d26b3bSNicholas Bellinger destroy_rx_wq: 3433b8d26b3bSNicholas Bellinger destroy_workqueue(isert_rx_wq); 3434b8d26b3bSNicholas Bellinger return ret; 3435b8d26b3bSNicholas Bellinger } 3436b8d26b3bSNicholas Bellinger 3437b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 3438b8d26b3bSNicholas Bellinger { 3439f5ebec96SSagi Grimberg flush_scheduled_work(); 3440*b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 3441b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 3442b8d26b3bSNicholas Bellinger destroy_workqueue(isert_rx_wq); 3443b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 3444b8d26b3bSNicholas Bellinger pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 3445b8d26b3bSNicholas Bellinger } 3446b8d26b3bSNicholas Bellinger 3447b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3448b8d26b3bSNicholas Bellinger MODULE_VERSION("0.1"); 3449b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3450b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 3451b8d26b3bSNicholas Bellinger 3452b8d26b3bSNicholas Bellinger module_init(isert_init); 3453b8d26b3bSNicholas Bellinger module_exit(isert_exit); 3454