1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 44c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 2595b60f07SNicholas Bellinger #include <linux/llist.h> 26b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 27b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 29b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 30b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 31b8d26b3bSNicholas Bellinger 32b8d26b3bSNicholas Bellinger #include "isert_proto.h" 33b8d26b3bSNicholas Bellinger #include "ib_isert.h" 34b8d26b3bSNicholas Bellinger 35b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 36b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37b8d26b3bSNicholas Bellinger #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38b8d26b3bSNicholas Bellinger 39b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 40b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 41b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_rx_wq; 42b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 43b8d26b3bSNicholas Bellinger 44b8d26b3bSNicholas Bellinger static void 45d40945d8SVu Pham isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 46d40945d8SVu Pham static int 47d40945d8SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 48d40945d8SVu Pham struct isert_rdma_wr *wr); 4959464ef4SVu Pham static void 50a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 5159464ef4SVu Pham static int 52a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5359464ef4SVu Pham struct isert_rdma_wr *wr); 54f93f3a70SSagi Grimberg static int 55f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 56d40945d8SVu Pham 57d40945d8SVu Pham static void 58b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 59b8d26b3bSNicholas Bellinger { 60b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)context; 61b8d26b3bSNicholas Bellinger 62b8d26b3bSNicholas Bellinger pr_err("isert_qp_event_callback event: %d\n", e->event); 63b8d26b3bSNicholas Bellinger switch (e->event) { 64b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 65b8d26b3bSNicholas Bellinger rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 66b8d26b3bSNicholas Bellinger break; 67b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 68b8d26b3bSNicholas Bellinger pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 69b8d26b3bSNicholas Bellinger break; 70b8d26b3bSNicholas Bellinger default: 71b8d26b3bSNicholas Bellinger break; 72b8d26b3bSNicholas Bellinger } 73b8d26b3bSNicholas Bellinger } 74b8d26b3bSNicholas Bellinger 75b8d26b3bSNicholas Bellinger static int 76b8d26b3bSNicholas Bellinger isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 77b8d26b3bSNicholas Bellinger { 78b8d26b3bSNicholas Bellinger int ret; 79b8d26b3bSNicholas Bellinger 80b8d26b3bSNicholas Bellinger ret = ib_query_device(ib_dev, devattr); 81b8d26b3bSNicholas Bellinger if (ret) { 82b8d26b3bSNicholas Bellinger pr_err("ib_query_device() failed: %d\n", ret); 83b8d26b3bSNicholas Bellinger return ret; 84b8d26b3bSNicholas Bellinger } 85b8d26b3bSNicholas Bellinger pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 86b8d26b3bSNicholas Bellinger pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 87b8d26b3bSNicholas Bellinger 88b8d26b3bSNicholas Bellinger return 0; 89b8d26b3bSNicholas Bellinger } 90b8d26b3bSNicholas Bellinger 91b8d26b3bSNicholas Bellinger static int 92d3e125daSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id, 93d3e125daSSagi Grimberg u8 protection) 94b8d26b3bSNicholas Bellinger { 95b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 96b8d26b3bSNicholas Bellinger struct ib_qp_init_attr attr; 97b8d26b3bSNicholas Bellinger int ret, index, min_index = 0; 98b8d26b3bSNicholas Bellinger 99b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 100b8d26b3bSNicholas Bellinger for (index = 0; index < device->cqs_used; index++) 101b8d26b3bSNicholas Bellinger if (device->cq_active_qps[index] < 102b8d26b3bSNicholas Bellinger device->cq_active_qps[min_index]) 103b8d26b3bSNicholas Bellinger min_index = index; 104b8d26b3bSNicholas Bellinger device->cq_active_qps[min_index]++; 105b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 106b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 107b8d26b3bSNicholas Bellinger 108b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 109b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 110b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 111b8d26b3bSNicholas Bellinger attr.send_cq = device->dev_tx_cq[min_index]; 112b8d26b3bSNicholas Bellinger attr.recv_cq = device->dev_rx_cq[min_index]; 113b8d26b3bSNicholas Bellinger attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 114b8d26b3bSNicholas Bellinger attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 115b8d26b3bSNicholas Bellinger /* 116b8d26b3bSNicholas Bellinger * FIXME: Use devattr.max_sge - 2 for max_send_sge as 117b8d26b3bSNicholas Bellinger * work-around for RDMA_READ.. 118b8d26b3bSNicholas Bellinger */ 11959464ef4SVu Pham attr.cap.max_send_sge = device->dev_attr.max_sge - 2; 120b8d26b3bSNicholas Bellinger isert_conn->max_sge = attr.cap.max_send_sge; 121b8d26b3bSNicholas Bellinger 122b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 123b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 124b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 125d3e125daSSagi Grimberg if (protection) 126d3e125daSSagi Grimberg attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 127b8d26b3bSNicholas Bellinger 128b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 129b8d26b3bSNicholas Bellinger cma_id->device); 130b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 131b8d26b3bSNicholas Bellinger isert_conn->conn_pd->device); 132b8d26b3bSNicholas Bellinger 133b8d26b3bSNicholas Bellinger ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 134b8d26b3bSNicholas Bellinger if (ret) { 135b8d26b3bSNicholas Bellinger pr_err("rdma_create_qp failed for cma_id %d\n", ret); 136b8d26b3bSNicholas Bellinger return ret; 137b8d26b3bSNicholas Bellinger } 138b8d26b3bSNicholas Bellinger isert_conn->conn_qp = cma_id->qp; 139b8d26b3bSNicholas Bellinger pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 140b8d26b3bSNicholas Bellinger 141b8d26b3bSNicholas Bellinger return 0; 142b8d26b3bSNicholas Bellinger } 143b8d26b3bSNicholas Bellinger 144b8d26b3bSNicholas Bellinger static void 145b8d26b3bSNicholas Bellinger isert_cq_event_callback(struct ib_event *e, void *context) 146b8d26b3bSNicholas Bellinger { 147b8d26b3bSNicholas Bellinger pr_debug("isert_cq_event_callback event: %d\n", e->event); 148b8d26b3bSNicholas Bellinger } 149b8d26b3bSNicholas Bellinger 150b8d26b3bSNicholas Bellinger static int 151b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 152b8d26b3bSNicholas Bellinger { 153b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 154b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 155b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 156b8d26b3bSNicholas Bellinger u64 dma_addr; 157b8d26b3bSNicholas Bellinger int i, j; 158b8d26b3bSNicholas Bellinger 159b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 160b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 161b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 162b8d26b3bSNicholas Bellinger goto fail; 163b8d26b3bSNicholas Bellinger 164b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 165b8d26b3bSNicholas Bellinger 166b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 167b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 168b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 169b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 170b8d26b3bSNicholas Bellinger goto dma_map_fail; 171b8d26b3bSNicholas Bellinger 172b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 173b8d26b3bSNicholas Bellinger 174b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 175b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 176b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 177b8d26b3bSNicholas Bellinger rx_sg->lkey = isert_conn->conn_mr->lkey; 178b8d26b3bSNicholas Bellinger } 179b8d26b3bSNicholas Bellinger 180b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = 0; 181b8d26b3bSNicholas Bellinger return 0; 182b8d26b3bSNicholas Bellinger 183b8d26b3bSNicholas Bellinger dma_map_fail: 184b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 185b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 186b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 187b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 188b8d26b3bSNicholas Bellinger } 189b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 190b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 191b8d26b3bSNicholas Bellinger fail: 192b8d26b3bSNicholas Bellinger return -ENOMEM; 193b8d26b3bSNicholas Bellinger } 194b8d26b3bSNicholas Bellinger 195b8d26b3bSNicholas Bellinger static void 196b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 197b8d26b3bSNicholas Bellinger { 198b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 199b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 200b8d26b3bSNicholas Bellinger int i; 201b8d26b3bSNicholas Bellinger 202b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 203b8d26b3bSNicholas Bellinger return; 204b8d26b3bSNicholas Bellinger 205b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 206b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 207b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 208b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 209b8d26b3bSNicholas Bellinger } 210b8d26b3bSNicholas Bellinger 211b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 212b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 213b8d26b3bSNicholas Bellinger } 214b8d26b3bSNicholas Bellinger 2152853c2b6SNicholas Bellinger static void isert_cq_tx_work(struct work_struct *); 216b8d26b3bSNicholas Bellinger static void isert_cq_tx_callback(struct ib_cq *, void *); 2172853c2b6SNicholas Bellinger static void isert_cq_rx_work(struct work_struct *); 218b8d26b3bSNicholas Bellinger static void isert_cq_rx_callback(struct ib_cq *, void *); 219b8d26b3bSNicholas Bellinger 220b8d26b3bSNicholas Bellinger static int 221b8d26b3bSNicholas Bellinger isert_create_device_ib_res(struct isert_device *device) 222b8d26b3bSNicholas Bellinger { 223b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = device->ib_device; 224b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc; 22559464ef4SVu Pham struct ib_device_attr *dev_attr; 226b8d26b3bSNicholas Bellinger int ret = 0, i, j; 227b8d26b3bSNicholas Bellinger 22859464ef4SVu Pham dev_attr = &device->dev_attr; 22959464ef4SVu Pham ret = isert_query_device(ib_dev, dev_attr); 23059464ef4SVu Pham if (ret) 23159464ef4SVu Pham return ret; 23259464ef4SVu Pham 233d40945d8SVu Pham /* asign function handlers */ 234f2252258SSagi Grimberg if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 235f2252258SSagi Grimberg dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 236a3a5a826SSagi Grimberg device->use_fastreg = 1; 237a3a5a826SSagi Grimberg device->reg_rdma_mem = isert_reg_rdma; 238a3a5a826SSagi Grimberg device->unreg_rdma_mem = isert_unreg_rdma; 23959464ef4SVu Pham } else { 240a3a5a826SSagi Grimberg device->use_fastreg = 0; 241d40945d8SVu Pham device->reg_rdma_mem = isert_map_rdma; 242d40945d8SVu Pham device->unreg_rdma_mem = isert_unmap_cmd; 24359464ef4SVu Pham } 244d40945d8SVu Pham 245d3e125daSSagi Grimberg /* Check signature cap */ 246d3e125daSSagi Grimberg device->pi_capable = dev_attr->device_cap_flags & 247d3e125daSSagi Grimberg IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 248d3e125daSSagi Grimberg 249b8d26b3bSNicholas Bellinger device->cqs_used = min_t(int, num_online_cpus(), 250b8d26b3bSNicholas Bellinger device->ib_device->num_comp_vectors); 251b8d26b3bSNicholas Bellinger device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 252a3a5a826SSagi Grimberg pr_debug("Using %d CQs, device %s supports %d vectors support " 253d3e125daSSagi Grimberg "Fast registration %d pi_capable %d\n", 254b8d26b3bSNicholas Bellinger device->cqs_used, device->ib_device->name, 255d3e125daSSagi Grimberg device->ib_device->num_comp_vectors, device->use_fastreg, 256d3e125daSSagi Grimberg device->pi_capable); 257b8d26b3bSNicholas Bellinger device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 258b8d26b3bSNicholas Bellinger device->cqs_used, GFP_KERNEL); 259b8d26b3bSNicholas Bellinger if (!device->cq_desc) { 260b8d26b3bSNicholas Bellinger pr_err("Unable to allocate device->cq_desc\n"); 261b8d26b3bSNicholas Bellinger return -ENOMEM; 262b8d26b3bSNicholas Bellinger } 263b8d26b3bSNicholas Bellinger cq_desc = device->cq_desc; 264b8d26b3bSNicholas Bellinger 265b8d26b3bSNicholas Bellinger for (i = 0; i < device->cqs_used; i++) { 266b8d26b3bSNicholas Bellinger cq_desc[i].device = device; 267b8d26b3bSNicholas Bellinger cq_desc[i].cq_index = i; 268b8d26b3bSNicholas Bellinger 2692853c2b6SNicholas Bellinger INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 270b8d26b3bSNicholas Bellinger device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 271b8d26b3bSNicholas Bellinger isert_cq_rx_callback, 272b8d26b3bSNicholas Bellinger isert_cq_event_callback, 273b8d26b3bSNicholas Bellinger (void *)&cq_desc[i], 274b8d26b3bSNicholas Bellinger ISER_MAX_RX_CQ_LEN, i); 27594a71110SWei Yongjun if (IS_ERR(device->dev_rx_cq[i])) { 27694a71110SWei Yongjun ret = PTR_ERR(device->dev_rx_cq[i]); 27794a71110SWei Yongjun device->dev_rx_cq[i] = NULL; 278b8d26b3bSNicholas Bellinger goto out_cq; 27994a71110SWei Yongjun } 280b8d26b3bSNicholas Bellinger 2812853c2b6SNicholas Bellinger INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 282b8d26b3bSNicholas Bellinger device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 283b8d26b3bSNicholas Bellinger isert_cq_tx_callback, 284b8d26b3bSNicholas Bellinger isert_cq_event_callback, 285b8d26b3bSNicholas Bellinger (void *)&cq_desc[i], 286b8d26b3bSNicholas Bellinger ISER_MAX_TX_CQ_LEN, i); 28794a71110SWei Yongjun if (IS_ERR(device->dev_tx_cq[i])) { 28894a71110SWei Yongjun ret = PTR_ERR(device->dev_tx_cq[i]); 28994a71110SWei Yongjun device->dev_tx_cq[i] = NULL; 29094a71110SWei Yongjun goto out_cq; 29194a71110SWei Yongjun } 29294a71110SWei Yongjun 29394a71110SWei Yongjun ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 29494a71110SWei Yongjun if (ret) 295b8d26b3bSNicholas Bellinger goto out_cq; 296b8d26b3bSNicholas Bellinger 29794a71110SWei Yongjun ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 29894a71110SWei Yongjun if (ret) 299b8d26b3bSNicholas Bellinger goto out_cq; 300b8d26b3bSNicholas Bellinger } 301b8d26b3bSNicholas Bellinger 302b8d26b3bSNicholas Bellinger return 0; 303b8d26b3bSNicholas Bellinger 304b8d26b3bSNicholas Bellinger out_cq: 305b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++) { 306b8d26b3bSNicholas Bellinger cq_desc = &device->cq_desc[j]; 307b8d26b3bSNicholas Bellinger 308b8d26b3bSNicholas Bellinger if (device->dev_rx_cq[j]) { 309b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_rx_work); 310b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_rx_cq[j]); 311b8d26b3bSNicholas Bellinger } 312b8d26b3bSNicholas Bellinger if (device->dev_tx_cq[j]) { 313b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_tx_work); 314b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_tx_cq[j]); 315b8d26b3bSNicholas Bellinger } 316b8d26b3bSNicholas Bellinger } 317b8d26b3bSNicholas Bellinger kfree(device->cq_desc); 318b8d26b3bSNicholas Bellinger 319b8d26b3bSNicholas Bellinger return ret; 320b8d26b3bSNicholas Bellinger } 321b8d26b3bSNicholas Bellinger 322b8d26b3bSNicholas Bellinger static void 323b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 324b8d26b3bSNicholas Bellinger { 325b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc; 326b8d26b3bSNicholas Bellinger int i; 327b8d26b3bSNicholas Bellinger 328b8d26b3bSNicholas Bellinger for (i = 0; i < device->cqs_used; i++) { 329b8d26b3bSNicholas Bellinger cq_desc = &device->cq_desc[i]; 330b8d26b3bSNicholas Bellinger 331b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_rx_work); 332b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_tx_work); 333b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_rx_cq[i]); 334b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_tx_cq[i]); 335b8d26b3bSNicholas Bellinger device->dev_rx_cq[i] = NULL; 336b8d26b3bSNicholas Bellinger device->dev_tx_cq[i] = NULL; 337b8d26b3bSNicholas Bellinger } 338b8d26b3bSNicholas Bellinger 339b8d26b3bSNicholas Bellinger kfree(device->cq_desc); 340b8d26b3bSNicholas Bellinger } 341b8d26b3bSNicholas Bellinger 342b8d26b3bSNicholas Bellinger static void 343b8d26b3bSNicholas Bellinger isert_device_try_release(struct isert_device *device) 344b8d26b3bSNicholas Bellinger { 345b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 346b8d26b3bSNicholas Bellinger device->refcount--; 347b8d26b3bSNicholas Bellinger if (!device->refcount) { 348b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 349b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 350b8d26b3bSNicholas Bellinger kfree(device); 351b8d26b3bSNicholas Bellinger } 352b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 353b8d26b3bSNicholas Bellinger } 354b8d26b3bSNicholas Bellinger 355b8d26b3bSNicholas Bellinger static struct isert_device * 356b8d26b3bSNicholas Bellinger isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 357b8d26b3bSNicholas Bellinger { 358b8d26b3bSNicholas Bellinger struct isert_device *device; 359b8d26b3bSNicholas Bellinger int ret; 360b8d26b3bSNicholas Bellinger 361b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 362b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 363b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 364b8d26b3bSNicholas Bellinger device->refcount++; 365b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 366b8d26b3bSNicholas Bellinger return device; 367b8d26b3bSNicholas Bellinger } 368b8d26b3bSNicholas Bellinger } 369b8d26b3bSNicholas Bellinger 370b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 371b8d26b3bSNicholas Bellinger if (!device) { 372b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 373b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 374b8d26b3bSNicholas Bellinger } 375b8d26b3bSNicholas Bellinger 376b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 377b8d26b3bSNicholas Bellinger 378b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 379b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 380b8d26b3bSNicholas Bellinger if (ret) { 381b8d26b3bSNicholas Bellinger kfree(device); 382b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 383b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 384b8d26b3bSNicholas Bellinger } 385b8d26b3bSNicholas Bellinger 386b8d26b3bSNicholas Bellinger device->refcount++; 387b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 388b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 389b8d26b3bSNicholas Bellinger 390b8d26b3bSNicholas Bellinger return device; 391b8d26b3bSNicholas Bellinger } 392b8d26b3bSNicholas Bellinger 39359464ef4SVu Pham static void 394a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 39559464ef4SVu Pham { 39659464ef4SVu Pham struct fast_reg_descriptor *fr_desc, *tmp; 39759464ef4SVu Pham int i = 0; 39859464ef4SVu Pham 399a3a5a826SSagi Grimberg if (list_empty(&isert_conn->conn_fr_pool)) 40059464ef4SVu Pham return; 40159464ef4SVu Pham 402a3a5a826SSagi Grimberg pr_debug("Freeing conn %p fastreg pool", isert_conn); 40359464ef4SVu Pham 40459464ef4SVu Pham list_for_each_entry_safe(fr_desc, tmp, 405a3a5a826SSagi Grimberg &isert_conn->conn_fr_pool, list) { 40659464ef4SVu Pham list_del(&fr_desc->list); 40759464ef4SVu Pham ib_free_fast_reg_page_list(fr_desc->data_frpl); 40859464ef4SVu Pham ib_dereg_mr(fr_desc->data_mr); 409d3e125daSSagi Grimberg if (fr_desc->pi_ctx) { 410d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 411d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 412d3e125daSSagi Grimberg ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 413d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 414d3e125daSSagi Grimberg } 41559464ef4SVu Pham kfree(fr_desc); 41659464ef4SVu Pham ++i; 41759464ef4SVu Pham } 41859464ef4SVu Pham 419a3a5a826SSagi Grimberg if (i < isert_conn->conn_fr_pool_size) 42059464ef4SVu Pham pr_warn("Pool still has %d regions registered\n", 421a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size - i); 42259464ef4SVu Pham } 42359464ef4SVu Pham 42459464ef4SVu Pham static int 425dc87a90fSSagi Grimberg isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 426d3e125daSSagi Grimberg struct fast_reg_descriptor *fr_desc, u8 protection) 427dc87a90fSSagi Grimberg { 428d3e125daSSagi Grimberg int ret; 429d3e125daSSagi Grimberg 430dc87a90fSSagi Grimberg fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 431dc87a90fSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 432dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_frpl)) { 433dc87a90fSSagi Grimberg pr_err("Failed to allocate data frpl err=%ld\n", 434dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_frpl)); 435dc87a90fSSagi Grimberg return PTR_ERR(fr_desc->data_frpl); 436dc87a90fSSagi Grimberg } 437dc87a90fSSagi Grimberg 438dc87a90fSSagi Grimberg fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 439dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_mr)) { 440dc87a90fSSagi Grimberg pr_err("Failed to allocate data frmr err=%ld\n", 441dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_mr)); 442d3e125daSSagi Grimberg ret = PTR_ERR(fr_desc->data_mr); 443d3e125daSSagi Grimberg goto err_data_frpl; 444dc87a90fSSagi Grimberg } 445dc87a90fSSagi Grimberg pr_debug("Create fr_desc %p page_list %p\n", 446dc87a90fSSagi Grimberg fr_desc, fr_desc->data_frpl->page_list); 447d3e125daSSagi Grimberg fr_desc->ind |= ISERT_DATA_KEY_VALID; 448dc87a90fSSagi Grimberg 449d3e125daSSagi Grimberg if (protection) { 450d3e125daSSagi Grimberg struct ib_mr_init_attr mr_init_attr = {0}; 451d3e125daSSagi Grimberg struct pi_context *pi_ctx; 452d3e125daSSagi Grimberg 453d3e125daSSagi Grimberg fr_desc->pi_ctx = kzalloc(sizeof(*fr_desc->pi_ctx), GFP_KERNEL); 454d3e125daSSagi Grimberg if (!fr_desc->pi_ctx) { 455d3e125daSSagi Grimberg pr_err("Failed to allocate pi context\n"); 456d3e125daSSagi Grimberg ret = -ENOMEM; 457d3e125daSSagi Grimberg goto err_data_mr; 458d3e125daSSagi Grimberg } 459d3e125daSSagi Grimberg pi_ctx = fr_desc->pi_ctx; 460d3e125daSSagi Grimberg 461d3e125daSSagi Grimberg pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(ib_device, 462d3e125daSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 463d3e125daSSagi Grimberg if (IS_ERR(pi_ctx->prot_frpl)) { 464d3e125daSSagi Grimberg pr_err("Failed to allocate prot frpl err=%ld\n", 465d3e125daSSagi Grimberg PTR_ERR(pi_ctx->prot_frpl)); 466d3e125daSSagi Grimberg ret = PTR_ERR(pi_ctx->prot_frpl); 467d3e125daSSagi Grimberg goto err_pi_ctx; 468d3e125daSSagi Grimberg } 469d3e125daSSagi Grimberg 470d3e125daSSagi Grimberg pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 471d3e125daSSagi Grimberg if (IS_ERR(pi_ctx->prot_mr)) { 472d3e125daSSagi Grimberg pr_err("Failed to allocate prot frmr err=%ld\n", 473d3e125daSSagi Grimberg PTR_ERR(pi_ctx->prot_mr)); 474d3e125daSSagi Grimberg ret = PTR_ERR(pi_ctx->prot_mr); 475d3e125daSSagi Grimberg goto err_prot_frpl; 476d3e125daSSagi Grimberg } 477d3e125daSSagi Grimberg fr_desc->ind |= ISERT_PROT_KEY_VALID; 478d3e125daSSagi Grimberg 479d3e125daSSagi Grimberg mr_init_attr.max_reg_descriptors = 2; 480d3e125daSSagi Grimberg mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 481d3e125daSSagi Grimberg pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 482d3e125daSSagi Grimberg if (IS_ERR(pi_ctx->sig_mr)) { 483d3e125daSSagi Grimberg pr_err("Failed to allocate signature enabled mr err=%ld\n", 484d3e125daSSagi Grimberg PTR_ERR(pi_ctx->sig_mr)); 485d3e125daSSagi Grimberg ret = PTR_ERR(pi_ctx->sig_mr); 486d3e125daSSagi Grimberg goto err_prot_mr; 487d3e125daSSagi Grimberg } 488d3e125daSSagi Grimberg fr_desc->ind |= ISERT_SIG_KEY_VALID; 489d3e125daSSagi Grimberg } 490d3e125daSSagi Grimberg fr_desc->ind &= ~ISERT_PROTECTED; 491dc87a90fSSagi Grimberg 492dc87a90fSSagi Grimberg return 0; 493d3e125daSSagi Grimberg err_prot_mr: 494d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 495d3e125daSSagi Grimberg err_prot_frpl: 496d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 497d3e125daSSagi Grimberg err_pi_ctx: 498d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 499d3e125daSSagi Grimberg err_data_mr: 500d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->data_mr); 501d3e125daSSagi Grimberg err_data_frpl: 502d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->data_frpl); 503d3e125daSSagi Grimberg 504d3e125daSSagi Grimberg return ret; 505dc87a90fSSagi Grimberg } 506dc87a90fSSagi Grimberg 507dc87a90fSSagi Grimberg static int 508d3e125daSSagi Grimberg isert_conn_create_fastreg_pool(struct isert_conn *isert_conn, u8 pi_support) 50959464ef4SVu Pham { 51059464ef4SVu Pham struct fast_reg_descriptor *fr_desc; 51159464ef4SVu Pham struct isert_device *device = isert_conn->conn_device; 512f46d6a8aSNicholas Bellinger struct se_session *se_sess = isert_conn->conn->sess->se_sess; 513f46d6a8aSNicholas Bellinger struct se_node_acl *se_nacl = se_sess->se_node_acl; 514f46d6a8aSNicholas Bellinger int i, ret, tag_num; 515f46d6a8aSNicholas Bellinger /* 516f46d6a8aSNicholas Bellinger * Setup the number of FRMRs based upon the number of tags 517f46d6a8aSNicholas Bellinger * available to session in iscsi_target_locate_portal(). 518f46d6a8aSNicholas Bellinger */ 519f46d6a8aSNicholas Bellinger tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 520f46d6a8aSNicholas Bellinger tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 52159464ef4SVu Pham 522a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size = 0; 523f46d6a8aSNicholas Bellinger for (i = 0; i < tag_num; i++) { 52459464ef4SVu Pham fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 52559464ef4SVu Pham if (!fr_desc) { 52659464ef4SVu Pham pr_err("Failed to allocate fast_reg descriptor\n"); 52759464ef4SVu Pham ret = -ENOMEM; 52859464ef4SVu Pham goto err; 52959464ef4SVu Pham } 53059464ef4SVu Pham 531dc87a90fSSagi Grimberg ret = isert_create_fr_desc(device->ib_device, 532d3e125daSSagi Grimberg isert_conn->conn_pd, fr_desc, 533d3e125daSSagi Grimberg pi_support); 534dc87a90fSSagi Grimberg if (ret) { 535dc87a90fSSagi Grimberg pr_err("Failed to create fastreg descriptor err=%d\n", 536dc87a90fSSagi Grimberg ret); 537a80e21b3SNicholas Bellinger kfree(fr_desc); 53859464ef4SVu Pham goto err; 53959464ef4SVu Pham } 54059464ef4SVu Pham 541a3a5a826SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 542a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size++; 54359464ef4SVu Pham } 54459464ef4SVu Pham 545a3a5a826SSagi Grimberg pr_debug("Creating conn %p fastreg pool size=%d", 546a3a5a826SSagi Grimberg isert_conn, isert_conn->conn_fr_pool_size); 54759464ef4SVu Pham 54859464ef4SVu Pham return 0; 54959464ef4SVu Pham 55059464ef4SVu Pham err: 551a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 55259464ef4SVu Pham return ret; 55359464ef4SVu Pham } 55459464ef4SVu Pham 555b8d26b3bSNicholas Bellinger static int 556b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 557b8d26b3bSNicholas Bellinger { 558b8d26b3bSNicholas Bellinger struct iscsi_np *np = cma_id->context; 559b8d26b3bSNicholas Bellinger struct isert_np *isert_np = np->np_context; 560b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 561b8d26b3bSNicholas Bellinger struct isert_device *device; 562b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = cma_id->device; 563b8d26b3bSNicholas Bellinger int ret = 0; 564d3e125daSSagi Grimberg u8 pi_support = np->tpg_np->tpg->tpg_attrib.t10_pi; 565b8d26b3bSNicholas Bellinger 566b8d26b3bSNicholas Bellinger pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 567b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 568b8d26b3bSNicholas Bellinger 569b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 570b8d26b3bSNicholas Bellinger if (!isert_conn) { 571b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_conn\n"); 572b8d26b3bSNicholas Bellinger return -ENOMEM; 573b8d26b3bSNicholas Bellinger } 574b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_INIT; 575b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_accept_node); 576b8d26b3bSNicholas Bellinger init_completion(&isert_conn->conn_login_comp); 577defd8848SNicholas Bellinger init_completion(&isert_conn->conn_wait); 578defd8848SNicholas Bellinger init_completion(&isert_conn->conn_wait_comp_err); 579b8d26b3bSNicholas Bellinger kref_init(&isert_conn->conn_kref); 580b8d26b3bSNicholas Bellinger kref_get(&isert_conn->conn_kref); 581b2cb9649SNicholas Bellinger mutex_init(&isert_conn->conn_mutex); 58259464ef4SVu Pham spin_lock_init(&isert_conn->conn_lock); 583f46d6a8aSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 584b8d26b3bSNicholas Bellinger 585b8d26b3bSNicholas Bellinger cma_id->context = isert_conn; 586b8d26b3bSNicholas Bellinger isert_conn->conn_cm_id = cma_id; 587b8d26b3bSNicholas Bellinger isert_conn->responder_resources = event->param.conn.responder_resources; 588b8d26b3bSNicholas Bellinger isert_conn->initiator_depth = event->param.conn.initiator_depth; 589b8d26b3bSNicholas Bellinger pr_debug("Using responder_resources: %u initiator_depth: %u\n", 590b8d26b3bSNicholas Bellinger isert_conn->responder_resources, isert_conn->initiator_depth); 591b8d26b3bSNicholas Bellinger 592b8d26b3bSNicholas Bellinger isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 593b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, GFP_KERNEL); 594b8d26b3bSNicholas Bellinger if (!isert_conn->login_buf) { 595b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_conn->login_buf\n"); 596b8d26b3bSNicholas Bellinger ret = -ENOMEM; 597b8d26b3bSNicholas Bellinger goto out; 598b8d26b3bSNicholas Bellinger } 599b8d26b3bSNicholas Bellinger 600b8d26b3bSNicholas Bellinger isert_conn->login_req_buf = isert_conn->login_buf; 601b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf = isert_conn->login_buf + 602b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN; 603b8d26b3bSNicholas Bellinger pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 604b8d26b3bSNicholas Bellinger isert_conn->login_buf, isert_conn->login_req_buf, 605b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf); 606b8d26b3bSNicholas Bellinger 607b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 608b8d26b3bSNicholas Bellinger (void *)isert_conn->login_req_buf, 609b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 610b8d26b3bSNicholas Bellinger 611b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 612b8d26b3bSNicholas Bellinger if (ret) { 613b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 614b8d26b3bSNicholas Bellinger ret); 615b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = 0; 616b8d26b3bSNicholas Bellinger goto out_login_buf; 617b8d26b3bSNicholas Bellinger } 618b8d26b3bSNicholas Bellinger 619b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 620b8d26b3bSNicholas Bellinger (void *)isert_conn->login_rsp_buf, 621b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 622b8d26b3bSNicholas Bellinger 623b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 624b8d26b3bSNicholas Bellinger if (ret) { 625b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 626b8d26b3bSNicholas Bellinger ret); 627b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = 0; 628b8d26b3bSNicholas Bellinger goto out_req_dma_map; 629b8d26b3bSNicholas Bellinger } 630b8d26b3bSNicholas Bellinger 631b8d26b3bSNicholas Bellinger device = isert_device_find_by_ib_dev(cma_id); 632b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 633b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 634b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 635b8d26b3bSNicholas Bellinger } 636b8d26b3bSNicholas Bellinger 637b8d26b3bSNicholas Bellinger isert_conn->conn_device = device; 638eb6ab132SSagi Grimberg isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 639eb6ab132SSagi Grimberg if (IS_ERR(isert_conn->conn_pd)) { 640eb6ab132SSagi Grimberg ret = PTR_ERR(isert_conn->conn_pd); 641eb6ab132SSagi Grimberg pr_err("ib_alloc_pd failed for conn %p: ret=%d\n", 642eb6ab132SSagi Grimberg isert_conn, ret); 643eb6ab132SSagi Grimberg goto out_pd; 644eb6ab132SSagi Grimberg } 645b8d26b3bSNicholas Bellinger 646eb6ab132SSagi Grimberg isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 647eb6ab132SSagi Grimberg IB_ACCESS_LOCAL_WRITE); 648eb6ab132SSagi Grimberg if (IS_ERR(isert_conn->conn_mr)) { 649eb6ab132SSagi Grimberg ret = PTR_ERR(isert_conn->conn_mr); 650eb6ab132SSagi Grimberg pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 651eb6ab132SSagi Grimberg isert_conn, ret); 652eb6ab132SSagi Grimberg goto out_mr; 653eb6ab132SSagi Grimberg } 654b8d26b3bSNicholas Bellinger 655d3e125daSSagi Grimberg if (pi_support && !device->pi_capable) { 656d3e125daSSagi Grimberg pr_err("Protection information requested but not supported\n"); 657d3e125daSSagi Grimberg ret = -EINVAL; 658d3e125daSSagi Grimberg goto out_mr; 659d3e125daSSagi Grimberg } 660d3e125daSSagi Grimberg 661d3e125daSSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id, pi_support); 662b8d26b3bSNicholas Bellinger if (ret) 663b8d26b3bSNicholas Bellinger goto out_conn_dev; 664b8d26b3bSNicholas Bellinger 665b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 666b8d26b3bSNicholas Bellinger list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 667b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 668b8d26b3bSNicholas Bellinger 669b8d26b3bSNicholas Bellinger pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 670b8d26b3bSNicholas Bellinger wake_up(&isert_np->np_accept_wq); 671b8d26b3bSNicholas Bellinger return 0; 672b8d26b3bSNicholas Bellinger 673b8d26b3bSNicholas Bellinger out_conn_dev: 674eb6ab132SSagi Grimberg ib_dereg_mr(isert_conn->conn_mr); 675eb6ab132SSagi Grimberg out_mr: 676eb6ab132SSagi Grimberg ib_dealloc_pd(isert_conn->conn_pd); 677eb6ab132SSagi Grimberg out_pd: 678b8d26b3bSNicholas Bellinger isert_device_try_release(device); 679b8d26b3bSNicholas Bellinger out_rsp_dma_map: 680b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 681b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 682b8d26b3bSNicholas Bellinger out_req_dma_map: 683b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 684b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 685b8d26b3bSNicholas Bellinger out_login_buf: 686b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 687b8d26b3bSNicholas Bellinger out: 688b8d26b3bSNicholas Bellinger kfree(isert_conn); 689b8d26b3bSNicholas Bellinger return ret; 690b8d26b3bSNicholas Bellinger } 691b8d26b3bSNicholas Bellinger 692b8d26b3bSNicholas Bellinger static void 693b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 694b8d26b3bSNicholas Bellinger { 695b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 696b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 697b8d26b3bSNicholas Bellinger int cq_index; 698b8d26b3bSNicholas Bellinger 699b8d26b3bSNicholas Bellinger pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 700b8d26b3bSNicholas Bellinger 701a3a5a826SSagi Grimberg if (device && device->use_fastreg) 702a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 70359464ef4SVu Pham 704b8d26b3bSNicholas Bellinger if (isert_conn->conn_qp) { 705b8d26b3bSNicholas Bellinger cq_index = ((struct isert_cq_desc *) 706b8d26b3bSNicholas Bellinger isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 707b8d26b3bSNicholas Bellinger pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 708b8d26b3bSNicholas Bellinger isert_conn->conn_device->cq_active_qps[cq_index]--; 709b8d26b3bSNicholas Bellinger 710b8d26b3bSNicholas Bellinger rdma_destroy_qp(isert_conn->conn_cm_id); 711b8d26b3bSNicholas Bellinger } 712b8d26b3bSNicholas Bellinger 713b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(isert_conn); 714b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_conn->conn_cm_id); 715b8d26b3bSNicholas Bellinger 716eb6ab132SSagi Grimberg ib_dereg_mr(isert_conn->conn_mr); 717eb6ab132SSagi Grimberg ib_dealloc_pd(isert_conn->conn_pd); 718eb6ab132SSagi Grimberg 719b8d26b3bSNicholas Bellinger if (isert_conn->login_buf) { 720b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 721b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 722b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 723b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, 724b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 725b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 726b8d26b3bSNicholas Bellinger } 727b8d26b3bSNicholas Bellinger kfree(isert_conn); 728b8d26b3bSNicholas Bellinger 729b8d26b3bSNicholas Bellinger if (device) 730b8d26b3bSNicholas Bellinger isert_device_try_release(device); 731b8d26b3bSNicholas Bellinger 732b8d26b3bSNicholas Bellinger pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 733b8d26b3bSNicholas Bellinger } 734b8d26b3bSNicholas Bellinger 735b8d26b3bSNicholas Bellinger static void 736b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 737b8d26b3bSNicholas Bellinger { 738b8d26b3bSNicholas Bellinger return; 739b8d26b3bSNicholas Bellinger } 740b8d26b3bSNicholas Bellinger 741b8d26b3bSNicholas Bellinger static void 742b8d26b3bSNicholas Bellinger isert_release_conn_kref(struct kref *kref) 743b8d26b3bSNicholas Bellinger { 744b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 745b8d26b3bSNicholas Bellinger struct isert_conn, conn_kref); 746b8d26b3bSNicholas Bellinger 747b8d26b3bSNicholas Bellinger pr_debug("Calling isert_connect_release for final kref %s/%d\n", 748b8d26b3bSNicholas Bellinger current->comm, current->pid); 749b8d26b3bSNicholas Bellinger 750b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 751b8d26b3bSNicholas Bellinger } 752b8d26b3bSNicholas Bellinger 753b8d26b3bSNicholas Bellinger static void 754b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 755b8d26b3bSNicholas Bellinger { 756b8d26b3bSNicholas Bellinger kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 757b8d26b3bSNicholas Bellinger } 758b8d26b3bSNicholas Bellinger 759b8d26b3bSNicholas Bellinger static void 760b8d26b3bSNicholas Bellinger isert_disconnect_work(struct work_struct *work) 761b8d26b3bSNicholas Bellinger { 762b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(work, 763b8d26b3bSNicholas Bellinger struct isert_conn, conn_logout_work); 764b8d26b3bSNicholas Bellinger 765b8d26b3bSNicholas Bellinger pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 766b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 767defd8848SNicholas Bellinger if (isert_conn->state == ISER_CONN_UP) 768defd8848SNicholas Bellinger isert_conn->state = ISER_CONN_TERMINATING; 769b8d26b3bSNicholas Bellinger 770b8d26b3bSNicholas Bellinger if (isert_conn->post_recv_buf_count == 0 && 771b8d26b3bSNicholas Bellinger atomic_read(&isert_conn->post_send_buf_count) == 0) { 772b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 773b2cb9649SNicholas Bellinger goto wake_up; 774b8d26b3bSNicholas Bellinger } 775b2cb9649SNicholas Bellinger if (!isert_conn->conn_cm_id) { 776b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 777b2cb9649SNicholas Bellinger isert_put_conn(isert_conn); 778b2cb9649SNicholas Bellinger return; 779b2cb9649SNicholas Bellinger } 780b2cb9649SNicholas Bellinger if (!isert_conn->logout_posted) { 781b2cb9649SNicholas Bellinger pr_debug("Calling rdma_disconnect for !logout_posted from" 782b2cb9649SNicholas Bellinger " isert_disconnect_work\n"); 783b2cb9649SNicholas Bellinger rdma_disconnect(isert_conn->conn_cm_id); 784b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 785b2cb9649SNicholas Bellinger iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 786b2cb9649SNicholas Bellinger goto wake_up; 787b2cb9649SNicholas Bellinger } 788b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 789b8d26b3bSNicholas Bellinger 790b2cb9649SNicholas Bellinger wake_up: 791defd8848SNicholas Bellinger complete(&isert_conn->conn_wait); 792b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 793b8d26b3bSNicholas Bellinger } 794b8d26b3bSNicholas Bellinger 795b8d26b3bSNicholas Bellinger static void 796b8d26b3bSNicholas Bellinger isert_disconnected_handler(struct rdma_cm_id *cma_id) 797b8d26b3bSNicholas Bellinger { 798b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 799b8d26b3bSNicholas Bellinger 800b8d26b3bSNicholas Bellinger INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 801b8d26b3bSNicholas Bellinger schedule_work(&isert_conn->conn_logout_work); 802b8d26b3bSNicholas Bellinger } 803b8d26b3bSNicholas Bellinger 804b8d26b3bSNicholas Bellinger static int 805b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 806b8d26b3bSNicholas Bellinger { 807b8d26b3bSNicholas Bellinger int ret = 0; 808b8d26b3bSNicholas Bellinger 809b8d26b3bSNicholas Bellinger pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 810b8d26b3bSNicholas Bellinger event->event, event->status, cma_id->context, cma_id); 811b8d26b3bSNicholas Bellinger 812b8d26b3bSNicholas Bellinger switch (event->event) { 813b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 814b8d26b3bSNicholas Bellinger pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 815b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 816b8d26b3bSNicholas Bellinger break; 817b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 818b8d26b3bSNicholas Bellinger pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 819b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 820b8d26b3bSNicholas Bellinger break; 821b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_DISCONNECTED: 822b8d26b3bSNicholas Bellinger pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 823b8d26b3bSNicholas Bellinger isert_disconnected_handler(cma_id); 824b8d26b3bSNicholas Bellinger break; 825b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_DEVICE_REMOVAL: 826b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ADDR_CHANGE: 827b8d26b3bSNicholas Bellinger break; 828b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 829b8d26b3bSNicholas Bellinger default: 830b8d26b3bSNicholas Bellinger pr_err("Unknown RDMA CMA event: %d\n", event->event); 831b8d26b3bSNicholas Bellinger break; 832b8d26b3bSNicholas Bellinger } 833b8d26b3bSNicholas Bellinger 834b8d26b3bSNicholas Bellinger if (ret != 0) { 835b8d26b3bSNicholas Bellinger pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 836b8d26b3bSNicholas Bellinger event->event, ret); 837b8d26b3bSNicholas Bellinger dump_stack(); 838b8d26b3bSNicholas Bellinger } 839b8d26b3bSNicholas Bellinger 840b8d26b3bSNicholas Bellinger return ret; 841b8d26b3bSNicholas Bellinger } 842b8d26b3bSNicholas Bellinger 843b8d26b3bSNicholas Bellinger static int 844b8d26b3bSNicholas Bellinger isert_post_recv(struct isert_conn *isert_conn, u32 count) 845b8d26b3bSNicholas Bellinger { 846b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 847b8d26b3bSNicholas Bellinger int i, ret; 848b8d26b3bSNicholas Bellinger unsigned int rx_head = isert_conn->conn_rx_desc_head; 849b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 850b8d26b3bSNicholas Bellinger 851b8d26b3bSNicholas Bellinger for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 852b8d26b3bSNicholas Bellinger rx_desc = &isert_conn->conn_rx_descs[rx_head]; 853b8d26b3bSNicholas Bellinger rx_wr->wr_id = (unsigned long)rx_desc; 854b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 855b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 856b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 857b8d26b3bSNicholas Bellinger rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 858b8d26b3bSNicholas Bellinger } 859b8d26b3bSNicholas Bellinger 860b8d26b3bSNicholas Bellinger rx_wr--; 861b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 862b8d26b3bSNicholas Bellinger 863b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count += count; 864b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 865b8d26b3bSNicholas Bellinger &rx_wr_failed); 866b8d26b3bSNicholas Bellinger if (ret) { 867b8d26b3bSNicholas Bellinger pr_err("ib_post_recv() failed with ret: %d\n", ret); 868b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count -= count; 869b8d26b3bSNicholas Bellinger } else { 870b8d26b3bSNicholas Bellinger pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 871b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = rx_head; 872b8d26b3bSNicholas Bellinger } 873b8d26b3bSNicholas Bellinger return ret; 874b8d26b3bSNicholas Bellinger } 875b8d26b3bSNicholas Bellinger 876b8d26b3bSNicholas Bellinger static int 877b8d26b3bSNicholas Bellinger isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 878b8d26b3bSNicholas Bellinger { 879b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 880b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 881b8d26b3bSNicholas Bellinger int ret; 882b8d26b3bSNicholas Bellinger 883b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 884b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 885b8d26b3bSNicholas Bellinger 886b8d26b3bSNicholas Bellinger send_wr.next = NULL; 887b8d26b3bSNicholas Bellinger send_wr.wr_id = (unsigned long)tx_desc; 888b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 889b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 890b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 891b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 892b8d26b3bSNicholas Bellinger 893b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 894b8d26b3bSNicholas Bellinger 895b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 896b8d26b3bSNicholas Bellinger if (ret) { 897b8d26b3bSNicholas Bellinger pr_err("ib_post_send() failed, ret: %d\n", ret); 898b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 899b8d26b3bSNicholas Bellinger } 900b8d26b3bSNicholas Bellinger 901b8d26b3bSNicholas Bellinger return ret; 902b8d26b3bSNicholas Bellinger } 903b8d26b3bSNicholas Bellinger 904b8d26b3bSNicholas Bellinger static void 905b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 906b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 907b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 908b8d26b3bSNicholas Bellinger { 909b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 910b8d26b3bSNicholas Bellinger 911b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 912b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 913b8d26b3bSNicholas Bellinger 914b8d26b3bSNicholas Bellinger memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 915b8d26b3bSNicholas Bellinger tx_desc->iser_header.flags = ISER_VER; 916b8d26b3bSNicholas Bellinger 917b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 918b8d26b3bSNicholas Bellinger tx_desc->isert_cmd = isert_cmd; 919b8d26b3bSNicholas Bellinger 920b8d26b3bSNicholas Bellinger if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 921b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 922b8d26b3bSNicholas Bellinger pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 923b8d26b3bSNicholas Bellinger } 924b8d26b3bSNicholas Bellinger } 925b8d26b3bSNicholas Bellinger 926b8d26b3bSNicholas Bellinger static int 927b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 928b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 929b8d26b3bSNicholas Bellinger { 930b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 931b8d26b3bSNicholas Bellinger u64 dma_addr; 932b8d26b3bSNicholas Bellinger 933b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 934b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 935b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 936b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error() failed\n"); 937b8d26b3bSNicholas Bellinger return -ENOMEM; 938b8d26b3bSNicholas Bellinger } 939b8d26b3bSNicholas Bellinger 940b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 941b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 942b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 943b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 944b8d26b3bSNicholas Bellinger 945b8d26b3bSNicholas Bellinger pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 946b8d26b3bSNicholas Bellinger " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 947b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 948b8d26b3bSNicholas Bellinger 949b8d26b3bSNicholas Bellinger return 0; 950b8d26b3bSNicholas Bellinger } 951b8d26b3bSNicholas Bellinger 952b8d26b3bSNicholas Bellinger static void 95395b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 95495b60f07SNicholas Bellinger struct ib_send_wr *send_wr, bool coalesce) 955b8d26b3bSNicholas Bellinger { 95695b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 95795b60f07SNicholas Bellinger 958b8d26b3bSNicholas Bellinger isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 959b8d26b3bSNicholas Bellinger send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 960b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 96195b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 962b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 96395b60f07SNicholas Bellinger /* 96495b60f07SNicholas Bellinger * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 96595b60f07SNicholas Bellinger * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 96695b60f07SNicholas Bellinger */ 967ebbe4421SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 968ebbe4421SNicholas Bellinger if (coalesce && isert_conn->state == ISER_CONN_UP && 96995b60f07SNicholas Bellinger ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 970ebbe4421SNicholas Bellinger tx_desc->llnode_active = true; 97195b60f07SNicholas Bellinger llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 972ebbe4421SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 97395b60f07SNicholas Bellinger return; 97495b60f07SNicholas Bellinger } 97595b60f07SNicholas Bellinger isert_conn->conn_comp_batch = 0; 97695b60f07SNicholas Bellinger tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 977ebbe4421SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 97895b60f07SNicholas Bellinger 97995b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 980b8d26b3bSNicholas Bellinger } 981b8d26b3bSNicholas Bellinger 982b8d26b3bSNicholas Bellinger static int 983b8d26b3bSNicholas Bellinger isert_rdma_post_recvl(struct isert_conn *isert_conn) 984b8d26b3bSNicholas Bellinger { 985b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 986b8d26b3bSNicholas Bellinger struct ib_sge sge; 987b8d26b3bSNicholas Bellinger int ret; 988b8d26b3bSNicholas Bellinger 989b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 990b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 991b8d26b3bSNicholas Bellinger sge.length = ISER_RX_LOGIN_SIZE; 992b8d26b3bSNicholas Bellinger sge.lkey = isert_conn->conn_mr->lkey; 993b8d26b3bSNicholas Bellinger 994b8d26b3bSNicholas Bellinger pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 995b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 996b8d26b3bSNicholas Bellinger 997b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 998b8d26b3bSNicholas Bellinger rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 999b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 1000b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 1001b8d26b3bSNicholas Bellinger 1002b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count++; 1003b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 1004b8d26b3bSNicholas Bellinger if (ret) { 1005b8d26b3bSNicholas Bellinger pr_err("ib_post_recv() failed: %d\n", ret); 1006b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1007b8d26b3bSNicholas Bellinger } 1008b8d26b3bSNicholas Bellinger 1009b8d26b3bSNicholas Bellinger pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 1010b8d26b3bSNicholas Bellinger return ret; 1011b8d26b3bSNicholas Bellinger } 1012b8d26b3bSNicholas Bellinger 1013b8d26b3bSNicholas Bellinger static int 1014b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1015b8d26b3bSNicholas Bellinger u32 length) 1016b8d26b3bSNicholas Bellinger { 1017b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 1018b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1019b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 1020b8d26b3bSNicholas Bellinger int ret; 1021b8d26b3bSNicholas Bellinger 1022b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 1023b8d26b3bSNicholas Bellinger 1024b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1025b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1026b8d26b3bSNicholas Bellinger 1027b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1028b8d26b3bSNicholas Bellinger 1029b8d26b3bSNicholas Bellinger if (length > 0) { 1030b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1031b8d26b3bSNicholas Bellinger 1032b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1033b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1034b8d26b3bSNicholas Bellinger 1035b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1036b8d26b3bSNicholas Bellinger 1037b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1038b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1039b8d26b3bSNicholas Bellinger 1040b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1041b8d26b3bSNicholas Bellinger tx_dsg->length = length; 1042b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 1043b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1044b8d26b3bSNicholas Bellinger } 1045b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1046b8d26b3bSNicholas Bellinger if (login->login_complete) { 1047f46d6a8aSNicholas Bellinger if (isert_conn->conn_device->use_fastreg) { 1048f46d6a8aSNicholas Bellinger u8 pi_support = login->np->tpg_np->tpg->tpg_attrib.t10_pi; 1049f46d6a8aSNicholas Bellinger 1050f46d6a8aSNicholas Bellinger ret = isert_conn_create_fastreg_pool(isert_conn, 1051f46d6a8aSNicholas Bellinger pi_support); 1052f46d6a8aSNicholas Bellinger if (ret) { 1053f46d6a8aSNicholas Bellinger pr_err("Conn: %p failed to create" 1054f46d6a8aSNicholas Bellinger " fastreg pool\n", isert_conn); 1055f46d6a8aSNicholas Bellinger return ret; 1056f46d6a8aSNicholas Bellinger } 1057f46d6a8aSNicholas Bellinger } 1058f46d6a8aSNicholas Bellinger 1059b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1060b8d26b3bSNicholas Bellinger if (ret) 1061b8d26b3bSNicholas Bellinger return ret; 1062b8d26b3bSNicholas Bellinger 1063b8d26b3bSNicholas Bellinger ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1064b8d26b3bSNicholas Bellinger if (ret) 1065b8d26b3bSNicholas Bellinger return ret; 1066b8d26b3bSNicholas Bellinger 1067b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_UP; 1068b8d26b3bSNicholas Bellinger goto post_send; 1069b8d26b3bSNicholas Bellinger } 1070b8d26b3bSNicholas Bellinger 1071b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 1072b8d26b3bSNicholas Bellinger if (ret) 1073b8d26b3bSNicholas Bellinger return ret; 1074b8d26b3bSNicholas Bellinger } 1075b8d26b3bSNicholas Bellinger post_send: 1076b8d26b3bSNicholas Bellinger ret = isert_post_send(isert_conn, tx_desc); 1077b8d26b3bSNicholas Bellinger if (ret) 1078b8d26b3bSNicholas Bellinger return ret; 1079b8d26b3bSNicholas Bellinger 1080b8d26b3bSNicholas Bellinger return 0; 1081b8d26b3bSNicholas Bellinger } 1082b8d26b3bSNicholas Bellinger 1083b8d26b3bSNicholas Bellinger static void 1084b8d26b3bSNicholas Bellinger isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 1085b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 1086b8d26b3bSNicholas Bellinger { 1087b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1088b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1089b8d26b3bSNicholas Bellinger int size; 1090b8d26b3bSNicholas Bellinger 1091b8d26b3bSNicholas Bellinger if (!login) { 1092b8d26b3bSNicholas Bellinger pr_err("conn->conn_login is NULL\n"); 1093b8d26b3bSNicholas Bellinger dump_stack(); 1094b8d26b3bSNicholas Bellinger return; 1095b8d26b3bSNicholas Bellinger } 1096b8d26b3bSNicholas Bellinger 1097b8d26b3bSNicholas Bellinger if (login->first_request) { 1098b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1099b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1100b8d26b3bSNicholas Bellinger /* 1101b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1102b8d26b3bSNicholas Bellinger * login request PDU. 1103b8d26b3bSNicholas Bellinger */ 1104b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1105b8d26b3bSNicholas Bellinger login->current_stage = 1106b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1107b8d26b3bSNicholas Bellinger >> 2; 1108b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1109b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1110b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1111b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1112b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1113b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1114b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1115b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1116b8d26b3bSNicholas Bellinger } 1117b8d26b3bSNicholas Bellinger 1118b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1119b8d26b3bSNicholas Bellinger 1120b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1121b8d26b3bSNicholas Bellinger pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1122b8d26b3bSNicholas Bellinger size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1123b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1124b8d26b3bSNicholas Bellinger 11256faaa85fSNicholas Bellinger if (login->first_request) { 1126b8d26b3bSNicholas Bellinger complete(&isert_conn->conn_login_comp); 11276faaa85fSNicholas Bellinger return; 11286faaa85fSNicholas Bellinger } 11296faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1130b8d26b3bSNicholas Bellinger } 1131b8d26b3bSNicholas Bellinger 1132b8d26b3bSNicholas Bellinger static struct iscsi_cmd 1133676687c6SNicholas Bellinger *isert_allocate_cmd(struct iscsi_conn *conn) 1134b8d26b3bSNicholas Bellinger { 1135b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1136b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1137d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1138b8d26b3bSNicholas Bellinger 1139676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1140d703ce2fSNicholas Bellinger if (!cmd) { 1141d703ce2fSNicholas Bellinger pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1142b8d26b3bSNicholas Bellinger return NULL; 1143b8d26b3bSNicholas Bellinger } 1144d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1145b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1146d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 1147b8d26b3bSNicholas Bellinger 1148d703ce2fSNicholas Bellinger return cmd; 1149b8d26b3bSNicholas Bellinger } 1150b8d26b3bSNicholas Bellinger 1151b8d26b3bSNicholas Bellinger static int 1152b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1153d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1154d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1155b8d26b3bSNicholas Bellinger { 1156b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1157b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1158b8d26b3bSNicholas Bellinger struct scatterlist *sg; 1159b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1160b8d26b3bSNicholas Bellinger bool dump_payload = false; 1161b8d26b3bSNicholas Bellinger 1162b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1163b8d26b3bSNicholas Bellinger if (rc < 0) 1164b8d26b3bSNicholas Bellinger return rc; 1165b8d26b3bSNicholas Bellinger 1166b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1167b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1168b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 1169b8d26b3bSNicholas Bellinger 1170b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1171b8d26b3bSNicholas Bellinger if (rc < 0) { 1172b8d26b3bSNicholas Bellinger return 0; 1173b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1174b8d26b3bSNicholas Bellinger dump_payload = true; 1175b8d26b3bSNicholas Bellinger goto sequence_cmd; 1176b8d26b3bSNicholas Bellinger } 1177b8d26b3bSNicholas Bellinger 1178b8d26b3bSNicholas Bellinger if (!imm_data) 1179b8d26b3bSNicholas Bellinger return 0; 1180b8d26b3bSNicholas Bellinger 1181b8d26b3bSNicholas Bellinger sg = &cmd->se_cmd.t_data_sg[0]; 1182b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1183b8d26b3bSNicholas Bellinger 1184b8d26b3bSNicholas Bellinger pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1185b8d26b3bSNicholas Bellinger sg, sg_nents, &rx_desc->data[0], imm_data_len); 1186b8d26b3bSNicholas Bellinger 1187b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1188b8d26b3bSNicholas Bellinger 1189b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1190b8d26b3bSNicholas Bellinger 1191b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1192b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1193b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1194b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1195b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1196b8d26b3bSNicholas Bellinger } 1197b8d26b3bSNicholas Bellinger 1198b8d26b3bSNicholas Bellinger sequence_cmd: 1199561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1200b8d26b3bSNicholas Bellinger 1201b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 1202b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 1203b8d26b3bSNicholas Bellinger 1204b8d26b3bSNicholas Bellinger return 0; 1205b8d26b3bSNicholas Bellinger } 1206b8d26b3bSNicholas Bellinger 1207b8d26b3bSNicholas Bellinger static int 1208b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1209b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1210b8d26b3bSNicholas Bellinger { 1211b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1212b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1213b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1214b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1215b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1216b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1217b8d26b3bSNicholas Bellinger 1218b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1219b8d26b3bSNicholas Bellinger if (rc < 0) 1220b8d26b3bSNicholas Bellinger return rc; 1221b8d26b3bSNicholas Bellinger else if (!cmd) 1222b8d26b3bSNicholas Bellinger return 0; 1223b8d26b3bSNicholas Bellinger /* 1224b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1225b8d26b3bSNicholas Bellinger */ 1226b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 1227b8d26b3bSNicholas Bellinger pr_err("Received unexpected solicited data payload\n"); 1228b8d26b3bSNicholas Bellinger dump_stack(); 1229b8d26b3bSNicholas Bellinger return -1; 1230b8d26b3bSNicholas Bellinger } 1231b8d26b3bSNicholas Bellinger 1232b8d26b3bSNicholas Bellinger pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1233b8d26b3bSNicholas Bellinger unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1234b8d26b3bSNicholas Bellinger 1235b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1236b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1237b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1238b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1239b8d26b3bSNicholas Bellinger /* 1240b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1241b8d26b3bSNicholas Bellinger */ 1242b8d26b3bSNicholas Bellinger if (page_off) { 1243b8d26b3bSNicholas Bellinger pr_err("Received unexpected non-page aligned data payload\n"); 1244b8d26b3bSNicholas Bellinger dump_stack(); 1245b8d26b3bSNicholas Bellinger return -1; 1246b8d26b3bSNicholas Bellinger } 1247b8d26b3bSNicholas Bellinger pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1248b8d26b3bSNicholas Bellinger sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1249b8d26b3bSNicholas Bellinger 1250b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1251b8d26b3bSNicholas Bellinger unsol_data_len); 1252b8d26b3bSNicholas Bellinger 1253b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1254b8d26b3bSNicholas Bellinger if (rc < 0) 1255b8d26b3bSNicholas Bellinger return rc; 1256b8d26b3bSNicholas Bellinger 1257b8d26b3bSNicholas Bellinger return 0; 1258b8d26b3bSNicholas Bellinger } 1259b8d26b3bSNicholas Bellinger 1260b8d26b3bSNicholas Bellinger static int 1261778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1262d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1263d703ce2fSNicholas Bellinger unsigned char *buf) 1264778de368SNicholas Bellinger { 1265778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1266778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1267778de368SNicholas Bellinger int rc; 1268778de368SNicholas Bellinger 1269778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1270778de368SNicholas Bellinger if (rc < 0) 1271778de368SNicholas Bellinger return rc; 1272778de368SNicholas Bellinger /* 1273778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1274778de368SNicholas Bellinger */ 1275778de368SNicholas Bellinger 1276778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1277778de368SNicholas Bellinger } 1278778de368SNicholas Bellinger 1279778de368SNicholas Bellinger static int 1280adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1281d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1282d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1283adb54c29SNicholas Bellinger { 1284adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1285adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1286adb54c29SNicholas Bellinger int rc; 1287adb54c29SNicholas Bellinger unsigned char *text_in; 1288adb54c29SNicholas Bellinger 1289adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1290adb54c29SNicholas Bellinger if (rc < 0) 1291adb54c29SNicholas Bellinger return rc; 1292adb54c29SNicholas Bellinger 1293adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 1294adb54c29SNicholas Bellinger if (!text_in) { 1295adb54c29SNicholas Bellinger pr_err("Unable to allocate text_in of payload_length: %u\n", 1296adb54c29SNicholas Bellinger payload_length); 1297adb54c29SNicholas Bellinger return -ENOMEM; 1298adb54c29SNicholas Bellinger } 1299adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1300adb54c29SNicholas Bellinger 1301adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1302adb54c29SNicholas Bellinger 1303adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1304adb54c29SNicholas Bellinger } 1305adb54c29SNicholas Bellinger 1306adb54c29SNicholas Bellinger static int 1307b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1308b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1309b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1310b8d26b3bSNicholas Bellinger { 1311b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1312b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1313ca40d24eSNicholas Bellinger struct iscsi_session *sess = conn->sess; 1314b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1315b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1316b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1317b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1318b8d26b3bSNicholas Bellinger 1319ca40d24eSNicholas Bellinger if (sess->sess_ops->SessionType && 1320ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1321ca40d24eSNicholas Bellinger pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1322ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1323ca40d24eSNicholas Bellinger return 0; 1324ca40d24eSNicholas Bellinger } 1325ca40d24eSNicholas Bellinger 1326b8d26b3bSNicholas Bellinger switch (opcode) { 1327b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1328676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1329b8d26b3bSNicholas Bellinger if (!cmd) 1330b8d26b3bSNicholas Bellinger break; 1331b8d26b3bSNicholas Bellinger 1332d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1333b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1334b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1335b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1336b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1337b8d26b3bSNicholas Bellinger 1338d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1339b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1340b8d26b3bSNicholas Bellinger break; 1341b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1342676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1343b8d26b3bSNicholas Bellinger if (!cmd) 1344b8d26b3bSNicholas Bellinger break; 1345b8d26b3bSNicholas Bellinger 1346d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1347d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1348778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1349b8d26b3bSNicholas Bellinger break; 1350b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1351b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1352b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1353b8d26b3bSNicholas Bellinger break; 1354b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1355676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1356b8d26b3bSNicholas Bellinger if (!cmd) 1357b8d26b3bSNicholas Bellinger break; 1358b8d26b3bSNicholas Bellinger 1359b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1360b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1361b8d26b3bSNicholas Bellinger break; 1362b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 1363676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1364b8d26b3bSNicholas Bellinger if (!cmd) 1365b8d26b3bSNicholas Bellinger break; 1366b8d26b3bSNicholas Bellinger 1367b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1368b8d26b3bSNicholas Bellinger if (ret > 0) 1369b8d26b3bSNicholas Bellinger wait_for_completion_timeout(&conn->conn_logout_comp, 1370b8d26b3bSNicholas Bellinger SECONDS_FOR_LOGOUT_COMP * 1371b8d26b3bSNicholas Bellinger HZ); 1372b8d26b3bSNicholas Bellinger break; 1373adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1374676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1375adb54c29SNicholas Bellinger if (!cmd) 1376adb54c29SNicholas Bellinger break; 1377adb54c29SNicholas Bellinger 1378d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1379d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1380adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1381adb54c29SNicholas Bellinger break; 1382b8d26b3bSNicholas Bellinger default: 1383b8d26b3bSNicholas Bellinger pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1384b8d26b3bSNicholas Bellinger dump_stack(); 1385b8d26b3bSNicholas Bellinger break; 1386b8d26b3bSNicholas Bellinger } 1387b8d26b3bSNicholas Bellinger 1388b8d26b3bSNicholas Bellinger return ret; 1389b8d26b3bSNicholas Bellinger } 1390b8d26b3bSNicholas Bellinger 1391b8d26b3bSNicholas Bellinger static void 1392b8d26b3bSNicholas Bellinger isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1393b8d26b3bSNicholas Bellinger { 1394b8d26b3bSNicholas Bellinger struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1395b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1396b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1397b8d26b3bSNicholas Bellinger int rc; 1398b8d26b3bSNicholas Bellinger 1399b8d26b3bSNicholas Bellinger switch (iser_hdr->flags & 0xF0) { 1400b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1401b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_RSV) { 1402b8d26b3bSNicholas Bellinger read_stag = be32_to_cpu(iser_hdr->read_stag); 1403b8d26b3bSNicholas Bellinger read_va = be64_to_cpu(iser_hdr->read_va); 1404b8d26b3bSNicholas Bellinger pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1405b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1406b8d26b3bSNicholas Bellinger } 1407b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_WSV) { 1408b8d26b3bSNicholas Bellinger write_stag = be32_to_cpu(iser_hdr->write_stag); 1409b8d26b3bSNicholas Bellinger write_va = be64_to_cpu(iser_hdr->write_va); 1410b8d26b3bSNicholas Bellinger pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1411b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1412b8d26b3bSNicholas Bellinger } 1413b8d26b3bSNicholas Bellinger 1414b8d26b3bSNicholas Bellinger pr_debug("ISER ISCSI_CTRL PDU\n"); 1415b8d26b3bSNicholas Bellinger break; 1416b8d26b3bSNicholas Bellinger case ISER_HELLO: 1417b8d26b3bSNicholas Bellinger pr_err("iSER Hello message\n"); 1418b8d26b3bSNicholas Bellinger break; 1419b8d26b3bSNicholas Bellinger default: 1420b8d26b3bSNicholas Bellinger pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1421b8d26b3bSNicholas Bellinger break; 1422b8d26b3bSNicholas Bellinger } 1423b8d26b3bSNicholas Bellinger 1424b8d26b3bSNicholas Bellinger rc = isert_rx_opcode(isert_conn, rx_desc, 1425b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 1426b8d26b3bSNicholas Bellinger } 1427b8d26b3bSNicholas Bellinger 1428b8d26b3bSNicholas Bellinger static void 1429b8d26b3bSNicholas Bellinger isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1430b8d26b3bSNicholas Bellinger unsigned long xfer_len) 1431b8d26b3bSNicholas Bellinger { 1432b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1433b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr; 1434b8d26b3bSNicholas Bellinger u64 rx_dma; 1435b8d26b3bSNicholas Bellinger int rx_buflen, outstanding; 1436b8d26b3bSNicholas Bellinger 1437b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) { 1438b8d26b3bSNicholas Bellinger rx_dma = isert_conn->login_req_dma; 1439b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_LOGIN_SIZE; 1440b8d26b3bSNicholas Bellinger pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1441b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1442b8d26b3bSNicholas Bellinger } else { 1443b8d26b3bSNicholas Bellinger rx_dma = desc->dma_addr; 1444b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_PAYLOAD_SIZE; 1445b8d26b3bSNicholas Bellinger pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1446b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1447b8d26b3bSNicholas Bellinger } 1448b8d26b3bSNicholas Bellinger 1449b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1450b8d26b3bSNicholas Bellinger 1451b8d26b3bSNicholas Bellinger hdr = &desc->iscsi_header; 1452b8d26b3bSNicholas Bellinger pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1453b8d26b3bSNicholas Bellinger hdr->opcode, hdr->itt, hdr->flags, 1454b8d26b3bSNicholas Bellinger (int)(xfer_len - ISER_HEADERS_LEN)); 1455b8d26b3bSNicholas Bellinger 1456b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) 1457b8d26b3bSNicholas Bellinger isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1458b8d26b3bSNicholas Bellinger isert_conn); 1459b8d26b3bSNicholas Bellinger else 1460b8d26b3bSNicholas Bellinger isert_rx_do_work(desc, isert_conn); 1461b8d26b3bSNicholas Bellinger 1462b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1463b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 1464b8d26b3bSNicholas Bellinger 1465b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1466b8d26b3bSNicholas Bellinger pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1467b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count); 1468b8d26b3bSNicholas Bellinger 1469b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) 1470b8d26b3bSNicholas Bellinger return; 1471b8d26b3bSNicholas Bellinger 1472b8d26b3bSNicholas Bellinger outstanding = isert_conn->post_recv_buf_count; 1473b8d26b3bSNicholas Bellinger if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1474b8d26b3bSNicholas Bellinger int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1475b8d26b3bSNicholas Bellinger ISERT_MIN_POSTED_RX); 1476b8d26b3bSNicholas Bellinger err = isert_post_recv(isert_conn, count); 1477b8d26b3bSNicholas Bellinger if (err) { 1478b8d26b3bSNicholas Bellinger pr_err("isert_post_recv() count: %d failed, %d\n", 1479b8d26b3bSNicholas Bellinger count, err); 1480b8d26b3bSNicholas Bellinger } 1481b8d26b3bSNicholas Bellinger } 1482b8d26b3bSNicholas Bellinger } 1483b8d26b3bSNicholas Bellinger 1484e3d7e4c3SSagi Grimberg static int 1485e3d7e4c3SSagi Grimberg isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1486e3d7e4c3SSagi Grimberg struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1487e3d7e4c3SSagi Grimberg enum iser_ib_op_code op, struct isert_data_buf *data) 1488e3d7e4c3SSagi Grimberg { 1489e3d7e4c3SSagi Grimberg struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1490e3d7e4c3SSagi Grimberg 1491e3d7e4c3SSagi Grimberg data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1492e3d7e4c3SSagi Grimberg DMA_TO_DEVICE : DMA_FROM_DEVICE; 1493e3d7e4c3SSagi Grimberg 1494e3d7e4c3SSagi Grimberg data->len = length - offset; 1495e3d7e4c3SSagi Grimberg data->offset = offset; 1496e3d7e4c3SSagi Grimberg data->sg_off = data->offset / PAGE_SIZE; 1497e3d7e4c3SSagi Grimberg 1498e3d7e4c3SSagi Grimberg data->sg = &sg[data->sg_off]; 1499e3d7e4c3SSagi Grimberg data->nents = min_t(unsigned int, nents - data->sg_off, 1500e3d7e4c3SSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 1501e3d7e4c3SSagi Grimberg data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1502e3d7e4c3SSagi Grimberg PAGE_SIZE); 1503e3d7e4c3SSagi Grimberg 1504e3d7e4c3SSagi Grimberg data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1505e3d7e4c3SSagi Grimberg data->dma_dir); 1506e3d7e4c3SSagi Grimberg if (unlikely(!data->dma_nents)) { 1507e3d7e4c3SSagi Grimberg pr_err("Cmd: unable to dma map SGs %p\n", sg); 1508e3d7e4c3SSagi Grimberg return -EINVAL; 1509e3d7e4c3SSagi Grimberg } 1510e3d7e4c3SSagi Grimberg 1511e3d7e4c3SSagi Grimberg pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1512e3d7e4c3SSagi Grimberg isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1513e3d7e4c3SSagi Grimberg 1514e3d7e4c3SSagi Grimberg return 0; 1515e3d7e4c3SSagi Grimberg } 1516e3d7e4c3SSagi Grimberg 1517e3d7e4c3SSagi Grimberg static void 1518e3d7e4c3SSagi Grimberg isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1519e3d7e4c3SSagi Grimberg { 1520e3d7e4c3SSagi Grimberg struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1521e3d7e4c3SSagi Grimberg 1522e3d7e4c3SSagi Grimberg ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1523e3d7e4c3SSagi Grimberg memset(data, 0, sizeof(*data)); 1524e3d7e4c3SSagi Grimberg } 1525e3d7e4c3SSagi Grimberg 1526e3d7e4c3SSagi Grimberg 1527e3d7e4c3SSagi Grimberg 1528b8d26b3bSNicholas Bellinger static void 1529b8d26b3bSNicholas Bellinger isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1530b8d26b3bSNicholas Bellinger { 1531b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1532b8d26b3bSNicholas Bellinger 153390ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1534e3d7e4c3SSagi Grimberg 1535e3d7e4c3SSagi Grimberg if (wr->data.sg) { 153690ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1537e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 1538b8d26b3bSNicholas Bellinger } 1539b8d26b3bSNicholas Bellinger 154090ecc6e2SVu Pham if (wr->send_wr) { 154190ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1542b8d26b3bSNicholas Bellinger kfree(wr->send_wr); 1543b8d26b3bSNicholas Bellinger wr->send_wr = NULL; 154490ecc6e2SVu Pham } 1545b8d26b3bSNicholas Bellinger 154690ecc6e2SVu Pham if (wr->ib_sge) { 154790ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 154890ecc6e2SVu Pham kfree(wr->ib_sge); 154990ecc6e2SVu Pham wr->ib_sge = NULL; 155090ecc6e2SVu Pham } 1551b8d26b3bSNicholas Bellinger } 1552b8d26b3bSNicholas Bellinger 1553b8d26b3bSNicholas Bellinger static void 1554a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 155559464ef4SVu Pham { 155659464ef4SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 155759464ef4SVu Pham LIST_HEAD(unmap_list); 155859464ef4SVu Pham 1559a3a5a826SSagi Grimberg pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd); 156059464ef4SVu Pham 156159464ef4SVu Pham if (wr->fr_desc) { 1562a3a5a826SSagi Grimberg pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n", 156359464ef4SVu Pham isert_cmd, wr->fr_desc); 15649e961ae7SSagi Grimberg if (wr->fr_desc->ind & ISERT_PROTECTED) { 15659e961ae7SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 15669e961ae7SSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 15679e961ae7SSagi Grimberg } 156859464ef4SVu Pham spin_lock_bh(&isert_conn->conn_lock); 1569a3a5a826SSagi Grimberg list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 157059464ef4SVu Pham spin_unlock_bh(&isert_conn->conn_lock); 157159464ef4SVu Pham wr->fr_desc = NULL; 157259464ef4SVu Pham } 157359464ef4SVu Pham 1574e3d7e4c3SSagi Grimberg if (wr->data.sg) { 1575a3a5a826SSagi Grimberg pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1576e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 157759464ef4SVu Pham } 157859464ef4SVu Pham 157959464ef4SVu Pham wr->ib_sge = NULL; 158059464ef4SVu Pham wr->send_wr = NULL; 158159464ef4SVu Pham } 158259464ef4SVu Pham 158359464ef4SVu Pham static void 158403e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1585b8d26b3bSNicholas Bellinger { 1586d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1587b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1588186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1589d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 1590b8d26b3bSNicholas Bellinger 1591b8d26b3bSNicholas Bellinger pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1592b8d26b3bSNicholas Bellinger 1593b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1594b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1595b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1596b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15975159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1598b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1599b8d26b3bSNicholas Bellinger 160003e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1601b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 160203e7848aSNicholas Bellinger /* 160303e7848aSNicholas Bellinger * Check for special case during comp_err where 160403e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 160503e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 160603e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 160703e7848aSNicholas Bellinger */ 160803e7848aSNicholas Bellinger if (comp_err && 160903e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 161003e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 161103e7848aSNicholas Bellinger 161203e7848aSNicholas Bellinger target_put_sess_cmd(se_cmd->se_sess, se_cmd); 161303e7848aSNicholas Bellinger } 161403e7848aSNicholas Bellinger } 1615b8d26b3bSNicholas Bellinger 1616d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1617186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1618186a9647SNicholas Bellinger break; 1619b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1620186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1621186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 16225159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1623186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1624186a9647SNicholas Bellinger 1625b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1626b8d26b3bSNicholas Bellinger break; 1627b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1628b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1629adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1630b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1631b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 16325159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1633b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1634b8d26b3bSNicholas Bellinger 1635b8d26b3bSNicholas Bellinger /* 1636b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1637b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1638b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1639b8d26b3bSNicholas Bellinger */ 1640b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 16413df8f68aSNicholas Bellinger pr_debug("Calling transport_generic_free_cmd from" 16423df8f68aSNicholas Bellinger " isert_put_cmd for 0x%02x\n", 16433df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1644b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1645b8d26b3bSNicholas Bellinger break; 1646b8d26b3bSNicholas Bellinger } 1647b8d26b3bSNicholas Bellinger /* 1648b8d26b3bSNicholas Bellinger * Fall-through 1649b8d26b3bSNicholas Bellinger */ 1650b8d26b3bSNicholas Bellinger default: 1651d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1652b8d26b3bSNicholas Bellinger break; 1653b8d26b3bSNicholas Bellinger } 1654b8d26b3bSNicholas Bellinger } 1655b8d26b3bSNicholas Bellinger 1656b8d26b3bSNicholas Bellinger static void 1657b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1658b8d26b3bSNicholas Bellinger { 1659b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 1660b8d26b3bSNicholas Bellinger pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1661b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1662b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1663b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1664b8d26b3bSNicholas Bellinger } 1665b8d26b3bSNicholas Bellinger } 1666b8d26b3bSNicholas Bellinger 1667b8d26b3bSNicholas Bellinger static void 1668b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 166903e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1670b8d26b3bSNicholas Bellinger { 1671dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 1672dbbc5d11SNicholas Bellinger pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1673dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1674dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1675dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1676b8d26b3bSNicholas Bellinger } 1677b8d26b3bSNicholas Bellinger 1678b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 167903e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1680b8d26b3bSNicholas Bellinger } 1681b8d26b3bSNicholas Bellinger 168296b7973eSSagi Grimberg static int 168396b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1684f93f3a70SSagi Grimberg { 16859e961ae7SSagi Grimberg struct ib_mr_status mr_status; 168696b7973eSSagi Grimberg int ret; 1687f93f3a70SSagi Grimberg 168896b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 16899e961ae7SSagi Grimberg if (ret) { 16909e961ae7SSagi Grimberg pr_err("ib_check_mr_status failed, ret %d\n", ret); 16919e961ae7SSagi Grimberg goto fail_mr_status; 16929e961ae7SSagi Grimberg } 169396b7973eSSagi Grimberg 16949e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 169596b7973eSSagi Grimberg u64 sec_offset_err; 16969e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 16979e961ae7SSagi Grimberg 16989e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 16999e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 17009e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 17019e961ae7SSagi Grimberg break; 17029e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 17039e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 17049e961ae7SSagi Grimberg break; 17059e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 17069e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 17079e961ae7SSagi Grimberg break; 17089e961ae7SSagi Grimberg } 170996b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 171096b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 171196b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 17129e961ae7SSagi Grimberg 17139e961ae7SSagi Grimberg pr_err("isert: PI error found type %d at sector 0x%llx " 17149e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 17159e961ae7SSagi Grimberg mr_status.sig_err.err_type, 17169e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 17179e961ae7SSagi Grimberg mr_status.sig_err.expected, 17189e961ae7SSagi Grimberg mr_status.sig_err.actual); 17199e961ae7SSagi Grimberg ret = 1; 17209e961ae7SSagi Grimberg } 17219e961ae7SSagi Grimberg 17229e961ae7SSagi Grimberg fail_mr_status: 172396b7973eSSagi Grimberg return ret; 172496b7973eSSagi Grimberg } 172596b7973eSSagi Grimberg 172696b7973eSSagi Grimberg static void 172796b7973eSSagi Grimberg isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 172896b7973eSSagi Grimberg struct isert_cmd *isert_cmd) 172996b7973eSSagi Grimberg { 173096b7973eSSagi Grimberg struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 173196b7973eSSagi Grimberg struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 173296b7973eSSagi Grimberg struct se_cmd *se_cmd = &cmd->se_cmd; 173396b7973eSSagi Grimberg struct isert_conn *isert_conn = isert_cmd->conn; 173496b7973eSSagi Grimberg struct isert_device *device = isert_conn->conn_device; 173596b7973eSSagi Grimberg int ret = 0; 173696b7973eSSagi Grimberg 173796b7973eSSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 173896b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 173996b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 174096b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 174196b7973eSSagi Grimberg } 174296b7973eSSagi Grimberg 1743f93f3a70SSagi Grimberg device->unreg_rdma_mem(isert_cmd, isert_conn); 1744897bb2c9SSagi Grimberg wr->send_wr_num = 0; 17459e961ae7SSagi Grimberg if (ret) 17469e961ae7SSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 17479e961ae7SSagi Grimberg se_cmd->pi_err, 0); 17489e961ae7SSagi Grimberg else 1749f93f3a70SSagi Grimberg isert_put_response(isert_conn->conn, cmd); 1750f93f3a70SSagi Grimberg } 1751f93f3a70SSagi Grimberg 1752f93f3a70SSagi Grimberg static void 1753b8d26b3bSNicholas Bellinger isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1754b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd) 1755b8d26b3bSNicholas Bellinger { 1756b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1757d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1758b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 175990ecc6e2SVu Pham struct isert_conn *isert_conn = isert_cmd->conn; 1760d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 17615bac4b1aSSagi Grimberg int ret = 0; 1762b8d26b3bSNicholas Bellinger 17639e961ae7SSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 176496b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 176596b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 176696b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 17679e961ae7SSagi Grimberg } 17689e961ae7SSagi Grimberg 1769b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1770d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1771e3d7e4c3SSagi Grimberg cmd->write_data_done = wr->data.len; 1772b6b87a1dSNicholas Bellinger wr->send_wr_num = 0; 1773b8d26b3bSNicholas Bellinger 177490ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1775b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1776b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1777b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1778b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1779b8d26b3bSNicholas Bellinger 17805bac4b1aSSagi Grimberg if (ret) 17815bac4b1aSSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 17825bac4b1aSSagi Grimberg se_cmd->pi_err, 0); 17835bac4b1aSSagi Grimberg else 1784b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1785b8d26b3bSNicholas Bellinger } 1786b8d26b3bSNicholas Bellinger 1787b8d26b3bSNicholas Bellinger static void 1788b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1789b8d26b3bSNicholas Bellinger { 1790b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1791b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1792b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1793b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1794d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1795b8d26b3bSNicholas Bellinger 1796b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1797b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1798b8d26b3bSNicholas Bellinger pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1799b8d26b3bSNicholas Bellinger 1800b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1801b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1802b8d26b3bSNicholas Bellinger 1803b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 180403e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1805b8d26b3bSNicholas Bellinger break; 1806b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 1807b8d26b3bSNicholas Bellinger pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1808b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1809b8d26b3bSNicholas Bellinger 1810b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 181103e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 18123df8f68aSNicholas Bellinger break; 1813b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1814b8d26b3bSNicholas Bellinger pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1815b8d26b3bSNicholas Bellinger /* 1816b8d26b3bSNicholas Bellinger * Call atomic_dec(&isert_conn->post_send_buf_count) 1817defd8848SNicholas Bellinger * from isert_wait_conn() 1818b8d26b3bSNicholas Bellinger */ 1819b8d26b3bSNicholas Bellinger isert_conn->logout_posted = true; 1820b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1821b8d26b3bSNicholas Bellinger break; 1822adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 1823adb54c29SNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1824adb54c29SNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 182503e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1826adb54c29SNicholas Bellinger break; 1827b8d26b3bSNicholas Bellinger default: 1828b8d26b3bSNicholas Bellinger pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1829b8d26b3bSNicholas Bellinger dump_stack(); 1830b8d26b3bSNicholas Bellinger break; 1831b8d26b3bSNicholas Bellinger } 1832b8d26b3bSNicholas Bellinger } 1833b8d26b3bSNicholas Bellinger 1834b8d26b3bSNicholas Bellinger static void 1835b8d26b3bSNicholas Bellinger isert_response_completion(struct iser_tx_desc *tx_desc, 1836b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1837b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn, 1838b8d26b3bSNicholas Bellinger struct ib_device *ib_dev) 1839b8d26b3bSNicholas Bellinger { 1840d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1841b6b87a1dSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1842b8d26b3bSNicholas Bellinger 1843b8d26b3bSNicholas Bellinger if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 18443df8f68aSNicholas Bellinger cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1845adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_REJECT || 1846adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_TEXTRSP) { 1847b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1848b8d26b3bSNicholas Bellinger 1849b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1850b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1851b8d26b3bSNicholas Bellinger return; 1852b8d26b3bSNicholas Bellinger } 1853897bb2c9SSagi Grimberg 1854897bb2c9SSagi Grimberg /** 1855897bb2c9SSagi Grimberg * If send_wr_num is 0 this means that we got 1856897bb2c9SSagi Grimberg * RDMA completion and we cleared it and we should 1857897bb2c9SSagi Grimberg * simply decrement the response post. else the 1858897bb2c9SSagi Grimberg * response is incorporated in send_wr_num, just 1859897bb2c9SSagi Grimberg * sub it. 1860897bb2c9SSagi Grimberg **/ 1861897bb2c9SSagi Grimberg if (wr->send_wr_num) 1862897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1863897bb2c9SSagi Grimberg else 1864897bb2c9SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 1865b8d26b3bSNicholas Bellinger 1866b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 186703e7848aSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1868b8d26b3bSNicholas Bellinger } 1869b8d26b3bSNicholas Bellinger 1870b8d26b3bSNicholas Bellinger static void 187195b60f07SNicholas Bellinger __isert_send_completion(struct iser_tx_desc *tx_desc, 1872b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 1873b8d26b3bSNicholas Bellinger { 1874b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1875b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1876b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr; 1877b8d26b3bSNicholas Bellinger 1878b8d26b3bSNicholas Bellinger if (!isert_cmd) { 1879b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1880b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1881b8d26b3bSNicholas Bellinger return; 1882b8d26b3bSNicholas Bellinger } 1883b8d26b3bSNicholas Bellinger wr = &isert_cmd->rdma_wr; 1884b8d26b3bSNicholas Bellinger 1885b8d26b3bSNicholas Bellinger switch (wr->iser_ib_op) { 1886b8d26b3bSNicholas Bellinger case ISER_IB_RECV: 1887b8d26b3bSNicholas Bellinger pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1888b8d26b3bSNicholas Bellinger dump_stack(); 1889b8d26b3bSNicholas Bellinger break; 1890b8d26b3bSNicholas Bellinger case ISER_IB_SEND: 1891b8d26b3bSNicholas Bellinger pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1892b8d26b3bSNicholas Bellinger isert_response_completion(tx_desc, isert_cmd, 1893b8d26b3bSNicholas Bellinger isert_conn, ib_dev); 1894b8d26b3bSNicholas Bellinger break; 1895b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_WRITE: 1896f93f3a70SSagi Grimberg pr_debug("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1897897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1898f93f3a70SSagi Grimberg isert_completion_rdma_write(tx_desc, isert_cmd); 1899b8d26b3bSNicholas Bellinger break; 1900b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_READ: 1901b8d26b3bSNicholas Bellinger pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1902b8d26b3bSNicholas Bellinger 1903b6b87a1dSNicholas Bellinger atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1904b8d26b3bSNicholas Bellinger isert_completion_rdma_read(tx_desc, isert_cmd); 1905b8d26b3bSNicholas Bellinger break; 1906b8d26b3bSNicholas Bellinger default: 1907b8d26b3bSNicholas Bellinger pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1908b8d26b3bSNicholas Bellinger dump_stack(); 1909b8d26b3bSNicholas Bellinger break; 1910b8d26b3bSNicholas Bellinger } 1911b8d26b3bSNicholas Bellinger } 1912b8d26b3bSNicholas Bellinger 1913b8d26b3bSNicholas Bellinger static void 191495b60f07SNicholas Bellinger isert_send_completion(struct iser_tx_desc *tx_desc, 191595b60f07SNicholas Bellinger struct isert_conn *isert_conn) 191695b60f07SNicholas Bellinger { 191795b60f07SNicholas Bellinger struct llist_node *llnode = tx_desc->comp_llnode_batch; 191895b60f07SNicholas Bellinger struct iser_tx_desc *t; 191995b60f07SNicholas Bellinger /* 192095b60f07SNicholas Bellinger * Drain coalesced completion llist starting from comp_llnode_batch 192195b60f07SNicholas Bellinger * setup in isert_init_send_wr(), and then complete trailing tx_desc. 192295b60f07SNicholas Bellinger */ 192395b60f07SNicholas Bellinger while (llnode) { 192495b60f07SNicholas Bellinger t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 192595b60f07SNicholas Bellinger llnode = llist_next(llnode); 192695b60f07SNicholas Bellinger __isert_send_completion(t, isert_conn); 192795b60f07SNicholas Bellinger } 192895b60f07SNicholas Bellinger __isert_send_completion(tx_desc, isert_conn); 192995b60f07SNicholas Bellinger } 193095b60f07SNicholas Bellinger 193195b60f07SNicholas Bellinger static void 1932ebbe4421SNicholas Bellinger isert_cq_drain_comp_llist(struct isert_conn *isert_conn, struct ib_device *ib_dev) 1933ebbe4421SNicholas Bellinger { 1934ebbe4421SNicholas Bellinger struct llist_node *llnode; 1935ebbe4421SNicholas Bellinger struct isert_rdma_wr *wr; 1936ebbe4421SNicholas Bellinger struct iser_tx_desc *t; 1937ebbe4421SNicholas Bellinger 1938ebbe4421SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 1939ebbe4421SNicholas Bellinger llnode = llist_del_all(&isert_conn->conn_comp_llist); 1940ebbe4421SNicholas Bellinger isert_conn->conn_comp_batch = 0; 1941ebbe4421SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 1942ebbe4421SNicholas Bellinger 1943ebbe4421SNicholas Bellinger while (llnode) { 1944ebbe4421SNicholas Bellinger t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1945ebbe4421SNicholas Bellinger llnode = llist_next(llnode); 1946ebbe4421SNicholas Bellinger wr = &t->isert_cmd->rdma_wr; 1947ebbe4421SNicholas Bellinger 1948897bb2c9SSagi Grimberg /** 1949897bb2c9SSagi Grimberg * If send_wr_num is 0 this means that we got 1950897bb2c9SSagi Grimberg * RDMA completion and we cleared it and we should 1951897bb2c9SSagi Grimberg * simply decrement the response post. else the 1952897bb2c9SSagi Grimberg * response is incorporated in send_wr_num, just 1953897bb2c9SSagi Grimberg * sub it. 1954897bb2c9SSagi Grimberg **/ 1955897bb2c9SSagi Grimberg if (wr->send_wr_num) 1956897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, 1957897bb2c9SSagi Grimberg &isert_conn->post_send_buf_count); 1958897bb2c9SSagi Grimberg else 1959897bb2c9SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 196003e7848aSNicholas Bellinger 196103e7848aSNicholas Bellinger isert_completion_put(t, t->isert_cmd, ib_dev, true); 1962ebbe4421SNicholas Bellinger } 1963ebbe4421SNicholas Bellinger } 1964ebbe4421SNicholas Bellinger 1965ebbe4421SNicholas Bellinger static void 1966defd8848SNicholas Bellinger isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1967b8d26b3bSNicholas Bellinger { 1968b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1969b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1970ebbe4421SNicholas Bellinger struct llist_node *llnode = tx_desc->comp_llnode_batch; 1971ebbe4421SNicholas Bellinger struct isert_rdma_wr *wr; 1972ebbe4421SNicholas Bellinger struct iser_tx_desc *t; 1973ebbe4421SNicholas Bellinger 1974ebbe4421SNicholas Bellinger while (llnode) { 1975ebbe4421SNicholas Bellinger t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1976ebbe4421SNicholas Bellinger llnode = llist_next(llnode); 1977ebbe4421SNicholas Bellinger wr = &t->isert_cmd->rdma_wr; 1978ebbe4421SNicholas Bellinger 1979897bb2c9SSagi Grimberg /** 1980897bb2c9SSagi Grimberg * If send_wr_num is 0 this means that we got 1981897bb2c9SSagi Grimberg * RDMA completion and we cleared it and we should 1982897bb2c9SSagi Grimberg * simply decrement the response post. else the 1983897bb2c9SSagi Grimberg * response is incorporated in send_wr_num, just 1984897bb2c9SSagi Grimberg * sub it. 1985897bb2c9SSagi Grimberg **/ 1986897bb2c9SSagi Grimberg if (wr->send_wr_num) 1987897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, 1988897bb2c9SSagi Grimberg &isert_conn->post_send_buf_count); 1989897bb2c9SSagi Grimberg else 1990897bb2c9SSagi Grimberg atomic_dec(&isert_conn->post_send_buf_count); 199103e7848aSNicholas Bellinger 199203e7848aSNicholas Bellinger isert_completion_put(t, t->isert_cmd, ib_dev, true); 1993ebbe4421SNicholas Bellinger } 1994ebbe4421SNicholas Bellinger tx_desc->comp_llnode_batch = NULL; 1995b8d26b3bSNicholas Bellinger 1996b8d26b3bSNicholas Bellinger if (!isert_cmd) 1997b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1998b8d26b3bSNicholas Bellinger else 199903e7848aSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 2000b8d26b3bSNicholas Bellinger } 2001b8d26b3bSNicholas Bellinger 2002defd8848SNicholas Bellinger static void 2003defd8848SNicholas Bellinger isert_cq_rx_comp_err(struct isert_conn *isert_conn) 2004defd8848SNicholas Bellinger { 2005defd8848SNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2006defd8848SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 2007defd8848SNicholas Bellinger 2008defd8848SNicholas Bellinger if (isert_conn->post_recv_buf_count) 2009defd8848SNicholas Bellinger return; 2010defd8848SNicholas Bellinger 2011ebbe4421SNicholas Bellinger isert_cq_drain_comp_llist(isert_conn, ib_dev); 2012ebbe4421SNicholas Bellinger 2013defd8848SNicholas Bellinger if (conn->sess) { 2014defd8848SNicholas Bellinger target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2015defd8848SNicholas Bellinger target_wait_for_sess_cmds(conn->sess->se_sess); 2016defd8848SNicholas Bellinger } 2017defd8848SNicholas Bellinger 2018defd8848SNicholas Bellinger while (atomic_read(&isert_conn->post_send_buf_count)) 2019defd8848SNicholas Bellinger msleep(3000); 2020b8d26b3bSNicholas Bellinger 2021b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 2022defd8848SNicholas Bellinger isert_conn->state = ISER_CONN_DOWN; 2023b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2024b2cb9649SNicholas Bellinger 2025defd8848SNicholas Bellinger complete(&isert_conn->conn_wait_comp_err); 2026b8d26b3bSNicholas Bellinger } 2027b8d26b3bSNicholas Bellinger 2028b8d26b3bSNicholas Bellinger static void 2029b8d26b3bSNicholas Bellinger isert_cq_tx_work(struct work_struct *work) 2030b8d26b3bSNicholas Bellinger { 2031b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = container_of(work, 2032b8d26b3bSNicholas Bellinger struct isert_cq_desc, cq_tx_work); 2033b8d26b3bSNicholas Bellinger struct isert_device *device = cq_desc->device; 2034b8d26b3bSNicholas Bellinger int cq_index = cq_desc->cq_index; 2035b8d26b3bSNicholas Bellinger struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 2036b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2037b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc; 2038b8d26b3bSNicholas Bellinger struct ib_wc wc; 2039b8d26b3bSNicholas Bellinger 2040b8d26b3bSNicholas Bellinger while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 2041b8d26b3bSNicholas Bellinger tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 2042b8d26b3bSNicholas Bellinger isert_conn = wc.qp->qp_context; 2043b8d26b3bSNicholas Bellinger 2044b8d26b3bSNicholas Bellinger if (wc.status == IB_WC_SUCCESS) { 2045b8d26b3bSNicholas Bellinger isert_send_completion(tx_desc, isert_conn); 2046b8d26b3bSNicholas Bellinger } else { 2047b8d26b3bSNicholas Bellinger pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2048b8d26b3bSNicholas Bellinger pr_debug("TX wc.status: 0x%08x\n", wc.status); 2049c5a2adbfSNicholas Bellinger pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 2050defd8848SNicholas Bellinger 20519bb4ca68SNicholas Bellinger if (wc.wr_id != ISER_FASTREG_LI_WRID) { 2052ebbe4421SNicholas Bellinger if (tx_desc->llnode_active) 2053ebbe4421SNicholas Bellinger continue; 2054ebbe4421SNicholas Bellinger 2055b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 2056defd8848SNicholas Bellinger isert_cq_tx_comp_err(tx_desc, isert_conn); 2057b8d26b3bSNicholas Bellinger } 2058b8d26b3bSNicholas Bellinger } 20599bb4ca68SNicholas Bellinger } 2060b8d26b3bSNicholas Bellinger 2061b8d26b3bSNicholas Bellinger ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 2062b8d26b3bSNicholas Bellinger } 2063b8d26b3bSNicholas Bellinger 2064b8d26b3bSNicholas Bellinger static void 2065b8d26b3bSNicholas Bellinger isert_cq_tx_callback(struct ib_cq *cq, void *context) 2066b8d26b3bSNicholas Bellinger { 2067b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2068b8d26b3bSNicholas Bellinger 2069b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 2070b8d26b3bSNicholas Bellinger } 2071b8d26b3bSNicholas Bellinger 2072b8d26b3bSNicholas Bellinger static void 2073b8d26b3bSNicholas Bellinger isert_cq_rx_work(struct work_struct *work) 2074b8d26b3bSNicholas Bellinger { 2075b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = container_of(work, 2076b8d26b3bSNicholas Bellinger struct isert_cq_desc, cq_rx_work); 2077b8d26b3bSNicholas Bellinger struct isert_device *device = cq_desc->device; 2078b8d26b3bSNicholas Bellinger int cq_index = cq_desc->cq_index; 2079b8d26b3bSNicholas Bellinger struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 2080b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2081b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 2082b8d26b3bSNicholas Bellinger struct ib_wc wc; 2083b8d26b3bSNicholas Bellinger unsigned long xfer_len; 2084b8d26b3bSNicholas Bellinger 2085b8d26b3bSNicholas Bellinger while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 2086b8d26b3bSNicholas Bellinger rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 2087b8d26b3bSNicholas Bellinger isert_conn = wc.qp->qp_context; 2088b8d26b3bSNicholas Bellinger 2089b8d26b3bSNicholas Bellinger if (wc.status == IB_WC_SUCCESS) { 2090b8d26b3bSNicholas Bellinger xfer_len = (unsigned long)wc.byte_len; 2091b8d26b3bSNicholas Bellinger isert_rx_completion(rx_desc, isert_conn, xfer_len); 2092b8d26b3bSNicholas Bellinger } else { 2093b8d26b3bSNicholas Bellinger pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2094c5a2adbfSNicholas Bellinger if (wc.status != IB_WC_WR_FLUSH_ERR) { 2095b8d26b3bSNicholas Bellinger pr_debug("RX wc.status: 0x%08x\n", wc.status); 2096c5a2adbfSNicholas Bellinger pr_debug("RX wc.vendor_err: 0x%08x\n", 2097c5a2adbfSNicholas Bellinger wc.vendor_err); 2098c5a2adbfSNicholas Bellinger } 2099b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 2100defd8848SNicholas Bellinger isert_cq_rx_comp_err(isert_conn); 2101b8d26b3bSNicholas Bellinger } 2102b8d26b3bSNicholas Bellinger } 2103b8d26b3bSNicholas Bellinger 2104b8d26b3bSNicholas Bellinger ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 2105b8d26b3bSNicholas Bellinger } 2106b8d26b3bSNicholas Bellinger 2107b8d26b3bSNicholas Bellinger static void 2108b8d26b3bSNicholas Bellinger isert_cq_rx_callback(struct ib_cq *cq, void *context) 2109b8d26b3bSNicholas Bellinger { 2110b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2111b8d26b3bSNicholas Bellinger 2112b8d26b3bSNicholas Bellinger queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 2113b8d26b3bSNicholas Bellinger } 2114b8d26b3bSNicholas Bellinger 2115b8d26b3bSNicholas Bellinger static int 2116b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2117b8d26b3bSNicholas Bellinger { 2118b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 2119b8d26b3bSNicholas Bellinger int ret; 2120b8d26b3bSNicholas Bellinger 2121b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 2122b8d26b3bSNicholas Bellinger 2123b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 2124b8d26b3bSNicholas Bellinger &wr_failed); 2125b8d26b3bSNicholas Bellinger if (ret) { 2126b8d26b3bSNicholas Bellinger pr_err("ib_post_send failed with %d\n", ret); 2127b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 2128b8d26b3bSNicholas Bellinger return ret; 2129b8d26b3bSNicholas Bellinger } 2130b8d26b3bSNicholas Bellinger return ret; 2131b8d26b3bSNicholas Bellinger } 2132b8d26b3bSNicholas Bellinger 2133b8d26b3bSNicholas Bellinger static int 2134b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2135b8d26b3bSNicholas Bellinger { 2136d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2137b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2138b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2139b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2140b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 2141b8d26b3bSNicholas Bellinger 2142b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2143b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2144b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2145b8d26b3bSNicholas Bellinger /* 2146b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 2147b8d26b3bSNicholas Bellinger */ 2148b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 2149b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2150b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2151b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2152b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2153dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 2154b8d26b3bSNicholas Bellinger 2155b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2156b8d26b3bSNicholas Bellinger cmd->sense_buffer); 2157b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2158b8d26b3bSNicholas Bellinger 2159b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2160b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2161dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2162b8d26b3bSNicholas Bellinger 2163dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2164dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 2165b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 2166b8d26b3bSNicholas Bellinger 2167dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 2168dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2169dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 2170b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 2171b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2172b8d26b3bSNicholas Bellinger } 2173b8d26b3bSNicholas Bellinger 217495b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, true); 2175b8d26b3bSNicholas Bellinger 2176b8d26b3bSNicholas Bellinger pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2177b8d26b3bSNicholas Bellinger 2178b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2179b8d26b3bSNicholas Bellinger } 2180b8d26b3bSNicholas Bellinger 2181131e6abcSNicholas Bellinger static void 2182131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2183131e6abcSNicholas Bellinger { 2184131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2185131e6abcSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2186131e6abcSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 2187131e6abcSNicholas Bellinger 2188131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 2189131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 2190131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 2191131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 2192131e6abcSNicholas Bellinger 2193131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 2194131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 2195131e6abcSNicholas Bellinger 2196131e6abcSNicholas Bellinger device->unreg_rdma_mem(isert_cmd, isert_conn); 2197131e6abcSNicholas Bellinger } 2198131e6abcSNicholas Bellinger 2199*e70beee7SNicholas Bellinger static enum target_prot_op 2200*e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 2201*e70beee7SNicholas Bellinger { 2202*e70beee7SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2203*e70beee7SNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 2204*e70beee7SNicholas Bellinger 2205*e70beee7SNicholas Bellinger if (device->pi_capable) 2206*e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 2207*e70beee7SNicholas Bellinger 2208*e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 2209*e70beee7SNicholas Bellinger } 2210*e70beee7SNicholas Bellinger 2211b8d26b3bSNicholas Bellinger static int 2212b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2213b8d26b3bSNicholas Bellinger bool nopout_response) 2214b8d26b3bSNicholas Bellinger { 2215d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2216b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2217b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2218b8d26b3bSNicholas Bellinger 2219b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2220b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2221b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 2222b8d26b3bSNicholas Bellinger nopout_response); 2223b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 222495b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2225b8d26b3bSNicholas Bellinger 22268b513d0cSMasanari Iida pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2227b8d26b3bSNicholas Bellinger 2228b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2229b8d26b3bSNicholas Bellinger } 2230b8d26b3bSNicholas Bellinger 2231b8d26b3bSNicholas Bellinger static int 2232b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2233b8d26b3bSNicholas Bellinger { 2234d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2235b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2236b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2237b8d26b3bSNicholas Bellinger 2238b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2239b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2240b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2241b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 224295b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2243b8d26b3bSNicholas Bellinger 2244b8d26b3bSNicholas Bellinger pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2245b8d26b3bSNicholas Bellinger 2246b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2247b8d26b3bSNicholas Bellinger } 2248b8d26b3bSNicholas Bellinger 2249b8d26b3bSNicholas Bellinger static int 2250b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2251b8d26b3bSNicholas Bellinger { 2252d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2253b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2254b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2255b8d26b3bSNicholas Bellinger 2256b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2257b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2258b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2259b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 226095b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2261b8d26b3bSNicholas Bellinger 2262b8d26b3bSNicholas Bellinger pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2263b8d26b3bSNicholas Bellinger 2264b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2265b8d26b3bSNicholas Bellinger } 2266b8d26b3bSNicholas Bellinger 2267b8d26b3bSNicholas Bellinger static int 2268b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2269b8d26b3bSNicholas Bellinger { 2270d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2271b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2272b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 22733df8f68aSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 22743df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 22753df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 22763df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2277b8d26b3bSNicholas Bellinger 2278b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 22793df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 2280b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 22813df8f68aSNicholas Bellinger 22823df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2283dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 22843df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 22853df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2286dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2287dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 22883df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 22893df8f68aSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 22903df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 22913df8f68aSNicholas Bellinger 229295b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2293b8d26b3bSNicholas Bellinger 2294b8d26b3bSNicholas Bellinger pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2295b8d26b3bSNicholas Bellinger 2296b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2297b8d26b3bSNicholas Bellinger } 2298b8d26b3bSNicholas Bellinger 2299b8d26b3bSNicholas Bellinger static int 2300adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2301adb54c29SNicholas Bellinger { 2302d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2303adb54c29SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2304adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2305adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2306adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2307adb54c29SNicholas Bellinger u32 txt_rsp_len; 2308adb54c29SNicholas Bellinger int rc; 2309adb54c29SNicholas Bellinger 2310adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2311adb54c29SNicholas Bellinger rc = iscsit_build_text_rsp(cmd, conn, hdr); 2312adb54c29SNicholas Bellinger if (rc < 0) 2313adb54c29SNicholas Bellinger return rc; 2314adb54c29SNicholas Bellinger 2315adb54c29SNicholas Bellinger txt_rsp_len = rc; 2316adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2317adb54c29SNicholas Bellinger 2318adb54c29SNicholas Bellinger if (txt_rsp_len) { 2319adb54c29SNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2320adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2321adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2322adb54c29SNicholas Bellinger 2323adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2324adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2325adb54c29SNicholas Bellinger 2326adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2327adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2328adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 2329adb54c29SNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 2330adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2331adb54c29SNicholas Bellinger } 233295b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2333adb54c29SNicholas Bellinger 2334adb54c29SNicholas Bellinger pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2335adb54c29SNicholas Bellinger 2336adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2337adb54c29SNicholas Bellinger } 2338adb54c29SNicholas Bellinger 2339adb54c29SNicholas Bellinger static int 2340b8d26b3bSNicholas Bellinger isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2341b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2342b8d26b3bSNicholas Bellinger u32 data_left, u32 offset) 2343b8d26b3bSNicholas Bellinger { 2344d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2345b8d26b3bSNicholas Bellinger struct scatterlist *sg_start, *tmp_sg; 2346b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2347b8d26b3bSNicholas Bellinger u32 sg_off, page_off; 2348b8d26b3bSNicholas Bellinger int i = 0, sg_nents; 2349b8d26b3bSNicholas Bellinger 2350b8d26b3bSNicholas Bellinger sg_off = offset / PAGE_SIZE; 2351b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2352b8d26b3bSNicholas Bellinger sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2353b8d26b3bSNicholas Bellinger page_off = offset % PAGE_SIZE; 2354b8d26b3bSNicholas Bellinger 2355b8d26b3bSNicholas Bellinger send_wr->sg_list = ib_sge; 2356b8d26b3bSNicholas Bellinger send_wr->num_sge = sg_nents; 2357b8d26b3bSNicholas Bellinger send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2358b8d26b3bSNicholas Bellinger /* 2359b8d26b3bSNicholas Bellinger * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2360b8d26b3bSNicholas Bellinger */ 2361b8d26b3bSNicholas Bellinger for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2362b8d26b3bSNicholas Bellinger pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2363b8d26b3bSNicholas Bellinger (unsigned long long)tmp_sg->dma_address, 2364b8d26b3bSNicholas Bellinger tmp_sg->length, page_off); 2365b8d26b3bSNicholas Bellinger 2366b8d26b3bSNicholas Bellinger ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2367b8d26b3bSNicholas Bellinger ib_sge->length = min_t(u32, data_left, 2368b8d26b3bSNicholas Bellinger ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2369b8d26b3bSNicholas Bellinger ib_sge->lkey = isert_conn->conn_mr->lkey; 2370b8d26b3bSNicholas Bellinger 237190ecc6e2SVu Pham pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 237290ecc6e2SVu Pham ib_sge->addr, ib_sge->length, ib_sge->lkey); 2373b8d26b3bSNicholas Bellinger page_off = 0; 2374b8d26b3bSNicholas Bellinger data_left -= ib_sge->length; 2375b8d26b3bSNicholas Bellinger ib_sge++; 2376b8d26b3bSNicholas Bellinger pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2377b8d26b3bSNicholas Bellinger } 2378b8d26b3bSNicholas Bellinger 2379b8d26b3bSNicholas Bellinger pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2380b8d26b3bSNicholas Bellinger send_wr->sg_list, send_wr->num_sge); 2381b8d26b3bSNicholas Bellinger 2382b8d26b3bSNicholas Bellinger return sg_nents; 2383b8d26b3bSNicholas Bellinger } 2384b8d26b3bSNicholas Bellinger 2385b8d26b3bSNicholas Bellinger static int 238690ecc6e2SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 238790ecc6e2SVu Pham struct isert_rdma_wr *wr) 2388b8d26b3bSNicholas Bellinger { 2389b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2390d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2391b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2392e3d7e4c3SSagi Grimberg struct isert_data_buf *data = &wr->data; 239390ecc6e2SVu Pham struct ib_send_wr *send_wr; 2394b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge; 2395e3d7e4c3SSagi Grimberg u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2396e3d7e4c3SSagi Grimberg int ret = 0, i, ib_sge_cnt; 2397b8d26b3bSNicholas Bellinger 239890ecc6e2SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2399b8d26b3bSNicholas Bellinger 2400e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2401e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2402e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2403e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2404e3d7e4c3SSagi Grimberg if (ret) 2405e3d7e4c3SSagi Grimberg return ret; 2406b8d26b3bSNicholas Bellinger 2407e3d7e4c3SSagi Grimberg data_left = data->len; 2408e3d7e4c3SSagi Grimberg offset = data->offset; 2409b8d26b3bSNicholas Bellinger 2410e3d7e4c3SSagi Grimberg ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2411b8d26b3bSNicholas Bellinger if (!ib_sge) { 241290ecc6e2SVu Pham pr_warn("Unable to allocate ib_sge\n"); 2413b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2414e3d7e4c3SSagi Grimberg goto unmap_cmd; 2415b8d26b3bSNicholas Bellinger } 241690ecc6e2SVu Pham wr->ib_sge = ib_sge; 2417b8d26b3bSNicholas Bellinger 2418e3d7e4c3SSagi Grimberg wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2419b8d26b3bSNicholas Bellinger wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2420b8d26b3bSNicholas Bellinger GFP_KERNEL); 2421b8d26b3bSNicholas Bellinger if (!wr->send_wr) { 242290ecc6e2SVu Pham pr_debug("Unable to allocate wr->send_wr\n"); 2423b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2424e3d7e4c3SSagi Grimberg goto unmap_cmd; 2425b8d26b3bSNicholas Bellinger } 2426b8d26b3bSNicholas Bellinger 2427b8d26b3bSNicholas Bellinger wr->isert_cmd = isert_cmd; 2428b8d26b3bSNicholas Bellinger rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2429b8d26b3bSNicholas Bellinger 2430b8d26b3bSNicholas Bellinger for (i = 0; i < wr->send_wr_num; i++) { 2431b8d26b3bSNicholas Bellinger send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2432b8d26b3bSNicholas Bellinger data_len = min(data_left, rdma_write_max); 2433b8d26b3bSNicholas Bellinger 2434b8d26b3bSNicholas Bellinger send_wr->send_flags = 0; 243590ecc6e2SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 243690ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 2437b8d26b3bSNicholas Bellinger send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2438b8d26b3bSNicholas Bellinger send_wr->wr.rdma.rkey = isert_cmd->read_stag; 243990ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 244090ecc6e2SVu Pham send_wr->next = &isert_cmd->tx_desc.send_wr; 244190ecc6e2SVu Pham else 244290ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 244390ecc6e2SVu Pham } else { 244490ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 244590ecc6e2SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 244690ecc6e2SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 244790ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 244890ecc6e2SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 244990ecc6e2SVu Pham else 245090ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 245190ecc6e2SVu Pham } 2452b8d26b3bSNicholas Bellinger 2453b8d26b3bSNicholas Bellinger ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2454b8d26b3bSNicholas Bellinger send_wr, data_len, offset); 2455b8d26b3bSNicholas Bellinger ib_sge += ib_sge_cnt; 2456b8d26b3bSNicholas Bellinger 2457b8d26b3bSNicholas Bellinger offset += data_len; 245890ecc6e2SVu Pham va_offset += data_len; 2459b8d26b3bSNicholas Bellinger data_left -= data_len; 2460b8d26b3bSNicholas Bellinger } 246190ecc6e2SVu Pham 246290ecc6e2SVu Pham return 0; 2463e3d7e4c3SSagi Grimberg unmap_cmd: 2464e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, data); 2465e3d7e4c3SSagi Grimberg 246690ecc6e2SVu Pham return ret; 246790ecc6e2SVu Pham } 246890ecc6e2SVu Pham 246990ecc6e2SVu Pham static int 247059464ef4SVu Pham isert_map_fr_pagelist(struct ib_device *ib_dev, 247159464ef4SVu Pham struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 247259464ef4SVu Pham { 247359464ef4SVu Pham u64 start_addr, end_addr, page, chunk_start = 0; 247459464ef4SVu Pham struct scatterlist *tmp_sg; 247559464ef4SVu Pham int i = 0, new_chunk, last_ent, n_pages; 247659464ef4SVu Pham 247759464ef4SVu Pham n_pages = 0; 247859464ef4SVu Pham new_chunk = 1; 247959464ef4SVu Pham last_ent = sg_nents - 1; 248059464ef4SVu Pham for_each_sg(sg_start, tmp_sg, sg_nents, i) { 248159464ef4SVu Pham start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 248259464ef4SVu Pham if (new_chunk) 248359464ef4SVu Pham chunk_start = start_addr; 248459464ef4SVu Pham end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 248559464ef4SVu Pham 248659464ef4SVu Pham pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 248759464ef4SVu Pham i, (unsigned long long)tmp_sg->dma_address, 248859464ef4SVu Pham tmp_sg->length); 248959464ef4SVu Pham 249059464ef4SVu Pham if ((end_addr & ~PAGE_MASK) && i < last_ent) { 249159464ef4SVu Pham new_chunk = 0; 249259464ef4SVu Pham continue; 249359464ef4SVu Pham } 249459464ef4SVu Pham new_chunk = 1; 249559464ef4SVu Pham 249659464ef4SVu Pham page = chunk_start & PAGE_MASK; 249759464ef4SVu Pham do { 249859464ef4SVu Pham fr_pl[n_pages++] = page; 249959464ef4SVu Pham pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 250059464ef4SVu Pham n_pages - 1, page); 250159464ef4SVu Pham page += PAGE_SIZE; 250259464ef4SVu Pham } while (page < end_addr); 250359464ef4SVu Pham } 250459464ef4SVu Pham 250559464ef4SVu Pham return n_pages; 250659464ef4SVu Pham } 250759464ef4SVu Pham 250859464ef4SVu Pham static int 2509e3d7e4c3SSagi Grimberg isert_fast_reg_mr(struct isert_conn *isert_conn, 2510e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc, 2511e3d7e4c3SSagi Grimberg struct isert_data_buf *mem, 25129e961ae7SSagi Grimberg enum isert_indicator ind, 2513e3d7e4c3SSagi Grimberg struct ib_sge *sge) 251459464ef4SVu Pham { 251559464ef4SVu Pham struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 25169e961ae7SSagi Grimberg struct ib_mr *mr; 25179e961ae7SSagi Grimberg struct ib_fast_reg_page_list *frpl; 251859464ef4SVu Pham struct ib_send_wr fr_wr, inv_wr; 251959464ef4SVu Pham struct ib_send_wr *bad_wr, *wr = NULL; 25209bd626e7SSagi Grimberg int ret, pagelist_len; 25219bd626e7SSagi Grimberg u32 page_off; 252259464ef4SVu Pham u8 key; 252359464ef4SVu Pham 2524e3d7e4c3SSagi Grimberg if (mem->dma_nents == 1) { 2525e3d7e4c3SSagi Grimberg sge->lkey = isert_conn->conn_mr->lkey; 2526e3d7e4c3SSagi Grimberg sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2527e3d7e4c3SSagi Grimberg sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 25289e961ae7SSagi Grimberg pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 25299e961ae7SSagi Grimberg __func__, __LINE__, sge->addr, sge->length, 25309e961ae7SSagi Grimberg sge->lkey); 2531e3d7e4c3SSagi Grimberg return 0; 2532e3d7e4c3SSagi Grimberg } 2533e3d7e4c3SSagi Grimberg 25349e961ae7SSagi Grimberg if (ind == ISERT_DATA_KEY_VALID) { 25359e961ae7SSagi Grimberg /* Registering data buffer */ 25369e961ae7SSagi Grimberg mr = fr_desc->data_mr; 25379e961ae7SSagi Grimberg frpl = fr_desc->data_frpl; 25389e961ae7SSagi Grimberg } else { 25399e961ae7SSagi Grimberg /* Registering protection buffer */ 25409e961ae7SSagi Grimberg mr = fr_desc->pi_ctx->prot_mr; 25419e961ae7SSagi Grimberg frpl = fr_desc->pi_ctx->prot_frpl; 25429e961ae7SSagi Grimberg } 25439e961ae7SSagi Grimberg 2544e3d7e4c3SSagi Grimberg page_off = mem->offset % PAGE_SIZE; 254559464ef4SVu Pham 25469bd626e7SSagi Grimberg pr_debug("Use fr_desc %p sg_nents %d offset %u\n", 2547e3d7e4c3SSagi Grimberg fr_desc, mem->nents, mem->offset); 254859464ef4SVu Pham 2549e3d7e4c3SSagi Grimberg pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 25509e961ae7SSagi Grimberg &frpl->page_list[0]); 255159464ef4SVu Pham 2552d3e125daSSagi Grimberg if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) { 255359464ef4SVu Pham memset(&inv_wr, 0, sizeof(inv_wr)); 25549bb4ca68SNicholas Bellinger inv_wr.wr_id = ISER_FASTREG_LI_WRID; 255559464ef4SVu Pham inv_wr.opcode = IB_WR_LOCAL_INV; 25569e961ae7SSagi Grimberg inv_wr.ex.invalidate_rkey = mr->rkey; 255759464ef4SVu Pham wr = &inv_wr; 255859464ef4SVu Pham /* Bump the key */ 25599e961ae7SSagi Grimberg key = (u8)(mr->rkey & 0x000000FF); 25609e961ae7SSagi Grimberg ib_update_fast_reg_key(mr, ++key); 256159464ef4SVu Pham } 256259464ef4SVu Pham 256359464ef4SVu Pham /* Prepare FASTREG WR */ 256459464ef4SVu Pham memset(&fr_wr, 0, sizeof(fr_wr)); 25659bb4ca68SNicholas Bellinger fr_wr.wr_id = ISER_FASTREG_LI_WRID; 256659464ef4SVu Pham fr_wr.opcode = IB_WR_FAST_REG_MR; 25679e961ae7SSagi Grimberg fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 25689e961ae7SSagi Grimberg fr_wr.wr.fast_reg.page_list = frpl; 256959464ef4SVu Pham fr_wr.wr.fast_reg.page_list_len = pagelist_len; 257059464ef4SVu Pham fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2571e3d7e4c3SSagi Grimberg fr_wr.wr.fast_reg.length = mem->len; 25729e961ae7SSagi Grimberg fr_wr.wr.fast_reg.rkey = mr->rkey; 257359464ef4SVu Pham fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 257459464ef4SVu Pham 257559464ef4SVu Pham if (!wr) 257659464ef4SVu Pham wr = &fr_wr; 257759464ef4SVu Pham else 257859464ef4SVu Pham wr->next = &fr_wr; 257959464ef4SVu Pham 258059464ef4SVu Pham ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 258159464ef4SVu Pham if (ret) { 258259464ef4SVu Pham pr_err("fast registration failed, ret:%d\n", ret); 258359464ef4SVu Pham return ret; 258459464ef4SVu Pham } 25859e961ae7SSagi Grimberg fr_desc->ind &= ~ind; 258659464ef4SVu Pham 25879e961ae7SSagi Grimberg sge->lkey = mr->lkey; 25889e961ae7SSagi Grimberg sge->addr = frpl->page_list[0] + page_off; 2589e3d7e4c3SSagi Grimberg sge->length = mem->len; 259059464ef4SVu Pham 25919e961ae7SSagi Grimberg pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 25929e961ae7SSagi Grimberg __func__, __LINE__, sge->addr, sge->length, 25939e961ae7SSagi Grimberg sge->lkey); 259459464ef4SVu Pham 259559464ef4SVu Pham return ret; 259659464ef4SVu Pham } 259759464ef4SVu Pham 25989e961ae7SSagi Grimberg static inline enum ib_t10_dif_type 25999e961ae7SSagi Grimberg se2ib_prot_type(enum target_prot_type prot_type) 26009e961ae7SSagi Grimberg { 26019e961ae7SSagi Grimberg switch (prot_type) { 26029e961ae7SSagi Grimberg case TARGET_DIF_TYPE0_PROT: 26039e961ae7SSagi Grimberg return IB_T10DIF_NONE; 26049e961ae7SSagi Grimberg case TARGET_DIF_TYPE1_PROT: 26059e961ae7SSagi Grimberg return IB_T10DIF_TYPE1; 26069e961ae7SSagi Grimberg case TARGET_DIF_TYPE2_PROT: 26079e961ae7SSagi Grimberg return IB_T10DIF_TYPE2; 26089e961ae7SSagi Grimberg case TARGET_DIF_TYPE3_PROT: 26099e961ae7SSagi Grimberg return IB_T10DIF_TYPE3; 26109e961ae7SSagi Grimberg default: 26119e961ae7SSagi Grimberg return IB_T10DIF_NONE; 26129e961ae7SSagi Grimberg } 26139e961ae7SSagi Grimberg } 26149e961ae7SSagi Grimberg 26159e961ae7SSagi Grimberg static int 26169e961ae7SSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 26179e961ae7SSagi Grimberg { 26189e961ae7SSagi Grimberg enum ib_t10_dif_type ib_prot_type = se2ib_prot_type(se_cmd->prot_type); 26199e961ae7SSagi Grimberg 26209e961ae7SSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF; 26219e961ae7SSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF; 26229e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.pi_interval = 26239e961ae7SSagi Grimberg se_cmd->se_dev->dev_attrib.block_size; 26249e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.pi_interval = 26259e961ae7SSagi Grimberg se_cmd->se_dev->dev_attrib.block_size; 26269e961ae7SSagi Grimberg 26279e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 26289e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 26299e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 26309e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE; 26319e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.type = ib_prot_type; 26329e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 26339e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed; 26349e961ae7SSagi Grimberg break; 26359e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 26369e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 26379e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.type = ib_prot_type; 26389e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 26399e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed; 26409e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE; 26419e961ae7SSagi Grimberg break; 26429e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 26439e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 26449e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.type = ib_prot_type; 26459e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 26469e961ae7SSagi Grimberg sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed; 26479e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.type = ib_prot_type; 26489e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 26499e961ae7SSagi Grimberg sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed; 26509e961ae7SSagi Grimberg break; 26519e961ae7SSagi Grimberg default: 26529e961ae7SSagi Grimberg pr_err("Unsupported PI operation %d\n", se_cmd->prot_op); 26539e961ae7SSagi Grimberg return -EINVAL; 26549e961ae7SSagi Grimberg } 26559e961ae7SSagi Grimberg 26569e961ae7SSagi Grimberg return 0; 26579e961ae7SSagi Grimberg } 26589e961ae7SSagi Grimberg 26599e961ae7SSagi Grimberg static inline u8 26609e961ae7SSagi Grimberg isert_set_prot_checks(u8 prot_checks) 26619e961ae7SSagi Grimberg { 26629e961ae7SSagi Grimberg return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 26639e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 26649e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 26659e961ae7SSagi Grimberg } 26669e961ae7SSagi Grimberg 26679e961ae7SSagi Grimberg static int 26689e961ae7SSagi Grimberg isert_reg_sig_mr(struct isert_conn *isert_conn, struct se_cmd *se_cmd, 26699e961ae7SSagi Grimberg struct fast_reg_descriptor *fr_desc, 26709e961ae7SSagi Grimberg struct ib_sge *data_sge, struct ib_sge *prot_sge, 26719e961ae7SSagi Grimberg struct ib_sge *sig_sge) 26729e961ae7SSagi Grimberg { 26739e961ae7SSagi Grimberg struct ib_send_wr sig_wr, inv_wr; 26749e961ae7SSagi Grimberg struct ib_send_wr *bad_wr, *wr = NULL; 26759e961ae7SSagi Grimberg struct pi_context *pi_ctx = fr_desc->pi_ctx; 26769e961ae7SSagi Grimberg struct ib_sig_attrs sig_attrs; 26779e961ae7SSagi Grimberg int ret; 26789e961ae7SSagi Grimberg u32 key; 26799e961ae7SSagi Grimberg 26809e961ae7SSagi Grimberg memset(&sig_attrs, 0, sizeof(sig_attrs)); 26819e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 26829e961ae7SSagi Grimberg if (ret) 26839e961ae7SSagi Grimberg goto err; 26849e961ae7SSagi Grimberg 26859e961ae7SSagi Grimberg sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 26869e961ae7SSagi Grimberg 26879e961ae7SSagi Grimberg if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 26889e961ae7SSagi Grimberg memset(&inv_wr, 0, sizeof(inv_wr)); 26899e961ae7SSagi Grimberg inv_wr.opcode = IB_WR_LOCAL_INV; 2690c2caa207SSagi Grimberg inv_wr.wr_id = ISER_FASTREG_LI_WRID; 26919e961ae7SSagi Grimberg inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; 26929e961ae7SSagi Grimberg wr = &inv_wr; 26939e961ae7SSagi Grimberg /* Bump the key */ 26949e961ae7SSagi Grimberg key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); 26959e961ae7SSagi Grimberg ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); 26969e961ae7SSagi Grimberg } 26979e961ae7SSagi Grimberg 26989e961ae7SSagi Grimberg memset(&sig_wr, 0, sizeof(sig_wr)); 26999e961ae7SSagi Grimberg sig_wr.opcode = IB_WR_REG_SIG_MR; 2700c2caa207SSagi Grimberg sig_wr.wr_id = ISER_FASTREG_LI_WRID; 27019e961ae7SSagi Grimberg sig_wr.sg_list = data_sge; 27029e961ae7SSagi Grimberg sig_wr.num_sge = 1; 27039e961ae7SSagi Grimberg sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 27049e961ae7SSagi Grimberg sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 27059e961ae7SSagi Grimberg sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 27069e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 27079e961ae7SSagi Grimberg sig_wr.wr.sig_handover.prot = prot_sge; 27089e961ae7SSagi Grimberg 27099e961ae7SSagi Grimberg if (!wr) 27109e961ae7SSagi Grimberg wr = &sig_wr; 27119e961ae7SSagi Grimberg else 27129e961ae7SSagi Grimberg wr->next = &sig_wr; 27139e961ae7SSagi Grimberg 27149e961ae7SSagi Grimberg ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 27159e961ae7SSagi Grimberg if (ret) { 27169e961ae7SSagi Grimberg pr_err("fast registration failed, ret:%d\n", ret); 27179e961ae7SSagi Grimberg goto err; 27189e961ae7SSagi Grimberg } 27199e961ae7SSagi Grimberg fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 27209e961ae7SSagi Grimberg 27219e961ae7SSagi Grimberg sig_sge->lkey = pi_ctx->sig_mr->lkey; 27229e961ae7SSagi Grimberg sig_sge->addr = 0; 27239e961ae7SSagi Grimberg sig_sge->length = se_cmd->data_length; 27249e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 27259e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 27269e961ae7SSagi Grimberg /* 27279e961ae7SSagi Grimberg * We have protection guards on the wire 27289e961ae7SSagi Grimberg * so we need to set a larget transfer 27299e961ae7SSagi Grimberg */ 27309e961ae7SSagi Grimberg sig_sge->length += se_cmd->prot_length; 27319e961ae7SSagi Grimberg 27329e961ae7SSagi Grimberg pr_debug("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 27339e961ae7SSagi Grimberg sig_sge->addr, sig_sge->length, 27349e961ae7SSagi Grimberg sig_sge->lkey); 27359e961ae7SSagi Grimberg err: 27369e961ae7SSagi Grimberg return ret; 27379e961ae7SSagi Grimberg } 27389e961ae7SSagi Grimberg 273959464ef4SVu Pham static int 2740a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 274159464ef4SVu Pham struct isert_rdma_wr *wr) 274259464ef4SVu Pham { 274359464ef4SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 274459464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2745e3d7e4c3SSagi Grimberg struct isert_conn *isert_conn = conn->context; 27469e961ae7SSagi Grimberg struct ib_sge data_sge; 274759464ef4SVu Pham struct ib_send_wr *send_wr; 2748e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc = NULL; 2749e3d7e4c3SSagi Grimberg u32 offset; 2750e3d7e4c3SSagi Grimberg int ret = 0; 275159464ef4SVu Pham unsigned long flags; 275259464ef4SVu Pham 275359464ef4SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2754e3d7e4c3SSagi Grimberg 2755e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2756e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2757e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2758e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2759e3d7e4c3SSagi Grimberg if (ret) 2760e3d7e4c3SSagi Grimberg return ret; 2761e3d7e4c3SSagi Grimberg 27629e961ae7SSagi Grimberg if (wr->data.dma_nents != 1 || 27639e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_NORMAL) { 2764e3d7e4c3SSagi Grimberg spin_lock_irqsave(&isert_conn->conn_lock, flags); 2765e3d7e4c3SSagi Grimberg fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2766e3d7e4c3SSagi Grimberg struct fast_reg_descriptor, list); 2767e3d7e4c3SSagi Grimberg list_del(&fr_desc->list); 2768e3d7e4c3SSagi Grimberg spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2769e3d7e4c3SSagi Grimberg wr->fr_desc = fr_desc; 277059464ef4SVu Pham } 277159464ef4SVu Pham 27729e961ae7SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 27739e961ae7SSagi Grimberg ISERT_DATA_KEY_VALID, &data_sge); 2774e3d7e4c3SSagi Grimberg if (ret) 2775e3d7e4c3SSagi Grimberg goto unmap_cmd; 277659464ef4SVu Pham 27779e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_NORMAL) { 27789e961ae7SSagi Grimberg struct ib_sge prot_sge, sig_sge; 27799e961ae7SSagi Grimberg 27809e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) { 27819e961ae7SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, 27829e961ae7SSagi Grimberg se_cmd->t_prot_sg, 27839e961ae7SSagi Grimberg se_cmd->t_prot_nents, 27849e961ae7SSagi Grimberg se_cmd->prot_length, 27859e961ae7SSagi Grimberg 0, wr->iser_ib_op, &wr->prot); 27869e961ae7SSagi Grimberg if (ret) 27879e961ae7SSagi Grimberg goto unmap_cmd; 27889e961ae7SSagi Grimberg 27899e961ae7SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->prot, 27909e961ae7SSagi Grimberg ISERT_PROT_KEY_VALID, &prot_sge); 27919e961ae7SSagi Grimberg if (ret) 27929e961ae7SSagi Grimberg goto unmap_prot_cmd; 27939e961ae7SSagi Grimberg } 27949e961ae7SSagi Grimberg 27959e961ae7SSagi Grimberg ret = isert_reg_sig_mr(isert_conn, se_cmd, fr_desc, 27969e961ae7SSagi Grimberg &data_sge, &prot_sge, &sig_sge); 27979e961ae7SSagi Grimberg if (ret) 27989e961ae7SSagi Grimberg goto unmap_prot_cmd; 27999e961ae7SSagi Grimberg 28009e961ae7SSagi Grimberg fr_desc->ind |= ISERT_PROTECTED; 28019e961ae7SSagi Grimberg memcpy(&wr->s_ib_sge, &sig_sge, sizeof(sig_sge)); 28029e961ae7SSagi Grimberg } else 28039e961ae7SSagi Grimberg memcpy(&wr->s_ib_sge, &data_sge, sizeof(data_sge)); 28049e961ae7SSagi Grimberg 2805e3d7e4c3SSagi Grimberg wr->ib_sge = &wr->s_ib_sge; 280659464ef4SVu Pham wr->send_wr_num = 1; 280759464ef4SVu Pham memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 280859464ef4SVu Pham wr->send_wr = &wr->s_send_wr; 280959464ef4SVu Pham wr->isert_cmd = isert_cmd; 281059464ef4SVu Pham 281159464ef4SVu Pham send_wr = &isert_cmd->rdma_wr.s_send_wr; 2812e3d7e4c3SSagi Grimberg send_wr->sg_list = &wr->s_ib_sge; 281359464ef4SVu Pham send_wr->num_sge = 1; 281459464ef4SVu Pham send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 281559464ef4SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 281659464ef4SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 281759464ef4SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 281859464ef4SVu Pham send_wr->wr.rdma.rkey = isert_cmd->read_stag; 28199e961ae7SSagi Grimberg send_wr->send_flags = se_cmd->prot_op == TARGET_PROT_NORMAL ? 28209e961ae7SSagi Grimberg 0 : IB_SEND_SIGNALED; 282159464ef4SVu Pham } else { 282259464ef4SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 282359464ef4SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 282459464ef4SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 282559464ef4SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 282659464ef4SVu Pham } 282759464ef4SVu Pham 282859464ef4SVu Pham return 0; 28299e961ae7SSagi Grimberg unmap_prot_cmd: 28309e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 28319e961ae7SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 2832e3d7e4c3SSagi Grimberg unmap_cmd: 2833e3d7e4c3SSagi Grimberg if (fr_desc) { 2834e3d7e4c3SSagi Grimberg spin_lock_irqsave(&isert_conn->conn_lock, flags); 2835e3d7e4c3SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2836e3d7e4c3SSagi Grimberg spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2837e3d7e4c3SSagi Grimberg } 2838e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 283959464ef4SVu Pham 284059464ef4SVu Pham return ret; 284159464ef4SVu Pham } 284259464ef4SVu Pham 284359464ef4SVu Pham static int 284490ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 284590ecc6e2SVu Pham { 284690ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 284759464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 284890ecc6e2SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 284990ecc6e2SVu Pham struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2850d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 285190ecc6e2SVu Pham struct ib_send_wr *wr_failed; 285290ecc6e2SVu Pham int rc; 285390ecc6e2SVu Pham 285490ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 285590ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 285690ecc6e2SVu Pham wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2857d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 285890ecc6e2SVu Pham if (rc) { 285990ecc6e2SVu Pham pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 286090ecc6e2SVu Pham return rc; 286190ecc6e2SVu Pham } 286290ecc6e2SVu Pham 28639e961ae7SSagi Grimberg if (se_cmd->prot_op == TARGET_PROT_NORMAL) { 2864b8d26b3bSNicholas Bellinger /* 2865b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2866b8d26b3bSNicholas Bellinger */ 28679e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 28689e961ae7SSagi Grimberg &isert_cmd->tx_desc); 286904d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2870b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2871b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 287295b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 287395b60f07SNicholas Bellinger &isert_cmd->tx_desc.send_wr, true); 28749e961ae7SSagi Grimberg isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2875897bb2c9SSagi Grimberg wr->send_wr_num += 1; 28769e961ae7SSagi Grimberg } 2877b8d26b3bSNicholas Bellinger 2878897bb2c9SSagi Grimberg atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2879b8d26b3bSNicholas Bellinger 2880b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2881b8d26b3bSNicholas Bellinger if (rc) { 2882b8d26b3bSNicholas Bellinger pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2883897bb2c9SSagi Grimberg atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2884b8d26b3bSNicholas Bellinger } 28859e961ae7SSagi Grimberg 28869e961ae7SSagi Grimberg if (se_cmd->prot_op == TARGET_PROT_NORMAL) 28879e961ae7SSagi Grimberg pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 28889e961ae7SSagi Grimberg "READ\n", isert_cmd); 28899e961ae7SSagi Grimberg else 28909e961ae7SSagi Grimberg pr_debug("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 289190ecc6e2SVu Pham isert_cmd); 2892b8d26b3bSNicholas Bellinger 289390ecc6e2SVu Pham return 1; 2894b8d26b3bSNicholas Bellinger } 2895b8d26b3bSNicholas Bellinger 2896b8d26b3bSNicholas Bellinger static int 2897b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2898b8d26b3bSNicholas Bellinger { 2899b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2900d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2901b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2902b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2903d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 290490ecc6e2SVu Pham struct ib_send_wr *wr_failed; 290590ecc6e2SVu Pham int rc; 2906b8d26b3bSNicholas Bellinger 290790ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 290890ecc6e2SVu Pham isert_cmd, se_cmd->data_length, cmd->write_data_done); 2909b8d26b3bSNicholas Bellinger wr->iser_ib_op = ISER_IB_RDMA_READ; 2910d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 291190ecc6e2SVu Pham if (rc) { 291290ecc6e2SVu Pham pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 291390ecc6e2SVu Pham return rc; 2914b8d26b3bSNicholas Bellinger } 2915b8d26b3bSNicholas Bellinger 2916b6b87a1dSNicholas Bellinger atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2917b8d26b3bSNicholas Bellinger 2918b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2919b8d26b3bSNicholas Bellinger if (rc) { 2920b8d26b3bSNicholas Bellinger pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2921b6b87a1dSNicholas Bellinger atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2922b8d26b3bSNicholas Bellinger } 292390ecc6e2SVu Pham pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 292490ecc6e2SVu Pham isert_cmd); 2925b8d26b3bSNicholas Bellinger 292690ecc6e2SVu Pham return 0; 2927b8d26b3bSNicholas Bellinger } 2928b8d26b3bSNicholas Bellinger 2929b8d26b3bSNicholas Bellinger static int 2930b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2931b8d26b3bSNicholas Bellinger { 2932b8d26b3bSNicholas Bellinger int ret; 2933b8d26b3bSNicholas Bellinger 2934b8d26b3bSNicholas Bellinger switch (state) { 2935b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2936b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2937b8d26b3bSNicholas Bellinger break; 2938b8d26b3bSNicholas Bellinger default: 2939b8d26b3bSNicholas Bellinger pr_err("Unknown immediate state: 0x%02x\n", state); 2940b8d26b3bSNicholas Bellinger ret = -EINVAL; 2941b8d26b3bSNicholas Bellinger break; 2942b8d26b3bSNicholas Bellinger } 2943b8d26b3bSNicholas Bellinger 2944b8d26b3bSNicholas Bellinger return ret; 2945b8d26b3bSNicholas Bellinger } 2946b8d26b3bSNicholas Bellinger 2947b8d26b3bSNicholas Bellinger static int 2948b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2949b8d26b3bSNicholas Bellinger { 2950b8d26b3bSNicholas Bellinger int ret; 2951b8d26b3bSNicholas Bellinger 2952b8d26b3bSNicholas Bellinger switch (state) { 2953b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2954b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2955b8d26b3bSNicholas Bellinger if (!ret) { 2956b8d26b3bSNicholas Bellinger pr_debug("Returning iSER Logout -EAGAIN\n"); 2957b8d26b3bSNicholas Bellinger ret = -EAGAIN; 2958b8d26b3bSNicholas Bellinger } 2959b8d26b3bSNicholas Bellinger break; 2960b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2961b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2962b8d26b3bSNicholas Bellinger break; 2963b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2964b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2965b8d26b3bSNicholas Bellinger break; 2966b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2967b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2968b8d26b3bSNicholas Bellinger break; 2969adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2970adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2971adb54c29SNicholas Bellinger break; 2972b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2973b8d26b3bSNicholas Bellinger /* 2974b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2975b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2976b8d26b3bSNicholas Bellinger */ 2977b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2978b8d26b3bSNicholas Bellinger break; 2979b8d26b3bSNicholas Bellinger default: 2980b8d26b3bSNicholas Bellinger pr_err("Unknown response state: 0x%02x\n", state); 2981b8d26b3bSNicholas Bellinger ret = -EINVAL; 2982b8d26b3bSNicholas Bellinger break; 2983b8d26b3bSNicholas Bellinger } 2984b8d26b3bSNicholas Bellinger 2985b8d26b3bSNicholas Bellinger return ret; 2986b8d26b3bSNicholas Bellinger } 2987b8d26b3bSNicholas Bellinger 2988b8d26b3bSNicholas Bellinger static int 2989b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 2990b8d26b3bSNicholas Bellinger struct __kernel_sockaddr_storage *ksockaddr) 2991b8d26b3bSNicholas Bellinger { 2992b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2993b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2994b8d26b3bSNicholas Bellinger struct sockaddr *sa; 2995b8d26b3bSNicholas Bellinger int ret; 2996b8d26b3bSNicholas Bellinger 2997b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2998b8d26b3bSNicholas Bellinger if (!isert_np) { 2999b8d26b3bSNicholas Bellinger pr_err("Unable to allocate struct isert_np\n"); 3000b8d26b3bSNicholas Bellinger return -ENOMEM; 3001b8d26b3bSNicholas Bellinger } 3002b8d26b3bSNicholas Bellinger init_waitqueue_head(&isert_np->np_accept_wq); 3003b8d26b3bSNicholas Bellinger mutex_init(&isert_np->np_accept_mutex); 3004b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_np->np_accept_list); 3005b8d26b3bSNicholas Bellinger init_completion(&isert_np->np_login_comp); 3006b8d26b3bSNicholas Bellinger 3007b8d26b3bSNicholas Bellinger sa = (struct sockaddr *)ksockaddr; 3008b8d26b3bSNicholas Bellinger pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 3009b8d26b3bSNicholas Bellinger /* 3010b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 3011b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 3012b8d26b3bSNicholas Bellinger */ 3013b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 3014b8d26b3bSNicholas Bellinger sizeof(struct __kernel_sockaddr_storage)); 3015b8d26b3bSNicholas Bellinger 3016b8d26b3bSNicholas Bellinger isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 3017b8d26b3bSNicholas Bellinger IB_QPT_RC); 3018b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 3019b8d26b3bSNicholas Bellinger pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 3020b8d26b3bSNicholas Bellinger PTR_ERR(isert_lid)); 3021b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 3022b8d26b3bSNicholas Bellinger goto out; 3023b8d26b3bSNicholas Bellinger } 3024b8d26b3bSNicholas Bellinger 3025b8d26b3bSNicholas Bellinger ret = rdma_bind_addr(isert_lid, sa); 3026b8d26b3bSNicholas Bellinger if (ret) { 3027b8d26b3bSNicholas Bellinger pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 3028b8d26b3bSNicholas Bellinger goto out_lid; 3029b8d26b3bSNicholas Bellinger } 3030b8d26b3bSNicholas Bellinger 3031b8d26b3bSNicholas Bellinger ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 3032b8d26b3bSNicholas Bellinger if (ret) { 3033b8d26b3bSNicholas Bellinger pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 3034b8d26b3bSNicholas Bellinger goto out_lid; 3035b8d26b3bSNicholas Bellinger } 3036b8d26b3bSNicholas Bellinger 3037b8d26b3bSNicholas Bellinger isert_np->np_cm_id = isert_lid; 3038b8d26b3bSNicholas Bellinger np->np_context = isert_np; 3039b8d26b3bSNicholas Bellinger pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 3040b8d26b3bSNicholas Bellinger 3041b8d26b3bSNicholas Bellinger return 0; 3042b8d26b3bSNicholas Bellinger 3043b8d26b3bSNicholas Bellinger out_lid: 3044b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_lid); 3045b8d26b3bSNicholas Bellinger out: 3046b8d26b3bSNicholas Bellinger kfree(isert_np); 3047b8d26b3bSNicholas Bellinger return ret; 3048b8d26b3bSNicholas Bellinger } 3049b8d26b3bSNicholas Bellinger 3050b8d26b3bSNicholas Bellinger static int 3051b8d26b3bSNicholas Bellinger isert_check_accept_queue(struct isert_np *isert_np) 3052b8d26b3bSNicholas Bellinger { 3053b8d26b3bSNicholas Bellinger int empty; 3054b8d26b3bSNicholas Bellinger 3055b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 3056b8d26b3bSNicholas Bellinger empty = list_empty(&isert_np->np_accept_list); 3057b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3058b8d26b3bSNicholas Bellinger 3059b8d26b3bSNicholas Bellinger return empty; 3060b8d26b3bSNicholas Bellinger } 3061b8d26b3bSNicholas Bellinger 3062b8d26b3bSNicholas Bellinger static int 3063b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 3064b8d26b3bSNicholas Bellinger { 3065b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3066b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 3067b8d26b3bSNicholas Bellinger int ret; 3068b8d26b3bSNicholas Bellinger 3069b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 3070b8d26b3bSNicholas Bellinger cp.responder_resources = isert_conn->responder_resources; 3071b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 3072b8d26b3bSNicholas Bellinger cp.retry_count = 7; 3073b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 3074b8d26b3bSNicholas Bellinger 3075b8d26b3bSNicholas Bellinger pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 3076b8d26b3bSNicholas Bellinger 3077b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 3078b8d26b3bSNicholas Bellinger if (ret) { 3079b8d26b3bSNicholas Bellinger pr_err("rdma_accept() failed with: %d\n", ret); 3080b8d26b3bSNicholas Bellinger return ret; 3081b8d26b3bSNicholas Bellinger } 3082b8d26b3bSNicholas Bellinger 3083b8d26b3bSNicholas Bellinger pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 3084b8d26b3bSNicholas Bellinger 3085b8d26b3bSNicholas Bellinger return 0; 3086b8d26b3bSNicholas Bellinger } 3087b8d26b3bSNicholas Bellinger 3088b8d26b3bSNicholas Bellinger static int 3089b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3090b8d26b3bSNicholas Bellinger { 3091b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3092b8d26b3bSNicholas Bellinger int ret; 3093b8d26b3bSNicholas Bellinger 3094b8d26b3bSNicholas Bellinger pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 30956faaa85fSNicholas Bellinger /* 30966faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 30976faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 30986faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 30996faaa85fSNicholas Bellinger * into a NOP. 31006faaa85fSNicholas Bellinger */ 31016faaa85fSNicholas Bellinger if (!login->first_request) 31026faaa85fSNicholas Bellinger return 0; 3103b8d26b3bSNicholas Bellinger 3104b8d26b3bSNicholas Bellinger ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 3105b8d26b3bSNicholas Bellinger if (ret) 3106b8d26b3bSNicholas Bellinger return ret; 3107b8d26b3bSNicholas Bellinger 3108b8d26b3bSNicholas Bellinger pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 3109b8d26b3bSNicholas Bellinger return 0; 3110b8d26b3bSNicholas Bellinger } 3111b8d26b3bSNicholas Bellinger 3112b8d26b3bSNicholas Bellinger static void 3113b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3114b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 3115b8d26b3bSNicholas Bellinger { 3116b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3117b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 3118b8d26b3bSNicholas Bellinger struct sockaddr_in *sock_in; 3119b8d26b3bSNicholas Bellinger struct sockaddr_in6 *sock_in6; 3120b8d26b3bSNicholas Bellinger 3121b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 3122b8d26b3bSNicholas Bellinger 3123b8d26b3bSNicholas Bellinger if (np->np_sockaddr.ss_family == AF_INET6) { 3124b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3125b8d26b3bSNicholas Bellinger snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3126b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 3127b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in6->sin6_port); 3128b8d26b3bSNicholas Bellinger 3129b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3130b8d26b3bSNicholas Bellinger snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3131b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 3132b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in6->sin6_port); 3133b8d26b3bSNicholas Bellinger } else { 3134b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3135b8d26b3bSNicholas Bellinger sprintf(conn->login_ip, "%pI4", 3136b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 3137b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in->sin_port); 3138b8d26b3bSNicholas Bellinger 3139b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3140b8d26b3bSNicholas Bellinger sprintf(conn->local_ip, "%pI4", 3141b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 3142b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in->sin_port); 3143b8d26b3bSNicholas Bellinger } 3144b8d26b3bSNicholas Bellinger } 3145b8d26b3bSNicholas Bellinger 3146b8d26b3bSNicholas Bellinger static int 3147b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3148b8d26b3bSNicholas Bellinger { 3149b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 3150b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 3151b8d26b3bSNicholas Bellinger int max_accept = 0, ret; 3152b8d26b3bSNicholas Bellinger 3153b8d26b3bSNicholas Bellinger accept_wait: 3154b8d26b3bSNicholas Bellinger ret = wait_event_interruptible(isert_np->np_accept_wq, 3155b8d26b3bSNicholas Bellinger !isert_check_accept_queue(isert_np) || 3156b8d26b3bSNicholas Bellinger np->np_thread_state == ISCSI_NP_THREAD_RESET); 3157b8d26b3bSNicholas Bellinger if (max_accept > 5) 3158b8d26b3bSNicholas Bellinger return -ENODEV; 3159b8d26b3bSNicholas Bellinger 3160b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 3161b8d26b3bSNicholas Bellinger if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 3162b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3163b8d26b3bSNicholas Bellinger pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 3164b8d26b3bSNicholas Bellinger return -ENODEV; 3165b8d26b3bSNicholas Bellinger } 3166b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3167b8d26b3bSNicholas Bellinger 3168b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 3169b8d26b3bSNicholas Bellinger if (list_empty(&isert_np->np_accept_list)) { 3170b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3171b8d26b3bSNicholas Bellinger max_accept++; 3172b8d26b3bSNicholas Bellinger goto accept_wait; 3173b8d26b3bSNicholas Bellinger } 3174b8d26b3bSNicholas Bellinger isert_conn = list_first_entry(&isert_np->np_accept_list, 3175b8d26b3bSNicholas Bellinger struct isert_conn, conn_accept_node); 3176b8d26b3bSNicholas Bellinger list_del_init(&isert_conn->conn_accept_node); 3177b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3178b8d26b3bSNicholas Bellinger 3179b8d26b3bSNicholas Bellinger conn->context = isert_conn; 3180b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 3181b8d26b3bSNicholas Bellinger max_accept = 0; 3182b8d26b3bSNicholas Bellinger 3183b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 3184b8d26b3bSNicholas Bellinger if (ret) 3185b8d26b3bSNicholas Bellinger return ret; 3186b8d26b3bSNicholas Bellinger 3187b8d26b3bSNicholas Bellinger ret = isert_rdma_accept(isert_conn); 3188b8d26b3bSNicholas Bellinger if (ret) 3189b8d26b3bSNicholas Bellinger return ret; 3190b8d26b3bSNicholas Bellinger 3191b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 3192b8d26b3bSNicholas Bellinger 3193b8d26b3bSNicholas Bellinger pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 3194b8d26b3bSNicholas Bellinger return 0; 3195b8d26b3bSNicholas Bellinger } 3196b8d26b3bSNicholas Bellinger 3197b8d26b3bSNicholas Bellinger static void 3198b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 3199b8d26b3bSNicholas Bellinger { 3200b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 3201b8d26b3bSNicholas Bellinger 3202b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_np->np_cm_id); 3203b8d26b3bSNicholas Bellinger 3204b8d26b3bSNicholas Bellinger np->np_context = NULL; 3205b8d26b3bSNicholas Bellinger kfree(isert_np); 3206b8d26b3bSNicholas Bellinger } 3207b8d26b3bSNicholas Bellinger 3208defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 3209b8d26b3bSNicholas Bellinger { 3210b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3211b8d26b3bSNicholas Bellinger 3212defd8848SNicholas Bellinger pr_debug("isert_wait_conn: Starting \n"); 3213b8d26b3bSNicholas Bellinger /* 3214b8d26b3bSNicholas Bellinger * Decrement post_send_buf_count for special case when called 3215b8d26b3bSNicholas Bellinger * from isert_do_control_comp() -> iscsit_logout_post_handler() 3216b8d26b3bSNicholas Bellinger */ 3217b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 3218b8d26b3bSNicholas Bellinger if (isert_conn->logout_posted) 3219b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 3220b8d26b3bSNicholas Bellinger 3221b2cb9649SNicholas Bellinger if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 3222defd8848SNicholas Bellinger pr_debug("Calling rdma_disconnect from isert_wait_conn\n"); 3223b8d26b3bSNicholas Bellinger rdma_disconnect(isert_conn->conn_cm_id); 3224b2cb9649SNicholas Bellinger } 3225b8d26b3bSNicholas Bellinger /* 3226b8d26b3bSNicholas Bellinger * Only wait for conn_wait_comp_err if the isert_conn made it 3227b8d26b3bSNicholas Bellinger * into full feature phase.. 3228b8d26b3bSNicholas Bellinger */ 3229b2cb9649SNicholas Bellinger if (isert_conn->state == ISER_CONN_INIT) { 3230b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 3231b2cb9649SNicholas Bellinger return; 3232b2cb9649SNicholas Bellinger } 3233defd8848SNicholas Bellinger if (isert_conn->state == ISER_CONN_UP) 3234defd8848SNicholas Bellinger isert_conn->state = ISER_CONN_TERMINATING; 3235b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 3236b2cb9649SNicholas Bellinger 3237defd8848SNicholas Bellinger wait_for_completion(&isert_conn->conn_wait_comp_err); 3238defd8848SNicholas Bellinger 3239defd8848SNicholas Bellinger wait_for_completion(&isert_conn->conn_wait); 3240defd8848SNicholas Bellinger } 3241defd8848SNicholas Bellinger 3242defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 3243defd8848SNicholas Bellinger { 3244defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3245b8d26b3bSNicholas Bellinger 3246b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 3247b8d26b3bSNicholas Bellinger } 3248b8d26b3bSNicholas Bellinger 3249b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 3250b8d26b3bSNicholas Bellinger .name = "IB/iSER", 3251b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 3252d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 3253b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 3254b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 3255b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 3256b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 3257defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 3258b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 3259b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 3260b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 3261b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 3262b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 3263b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 3264b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 3265b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 3266131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 3267*e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3268b8d26b3bSNicholas Bellinger }; 3269b8d26b3bSNicholas Bellinger 3270b8d26b3bSNicholas Bellinger static int __init isert_init(void) 3271b8d26b3bSNicholas Bellinger { 3272b8d26b3bSNicholas Bellinger int ret; 3273b8d26b3bSNicholas Bellinger 3274b8d26b3bSNicholas Bellinger isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 3275b8d26b3bSNicholas Bellinger if (!isert_rx_wq) { 3276b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_rx_wq\n"); 3277b8d26b3bSNicholas Bellinger return -ENOMEM; 3278b8d26b3bSNicholas Bellinger } 3279b8d26b3bSNicholas Bellinger 3280b8d26b3bSNicholas Bellinger isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 3281b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 3282b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_comp_wq\n"); 3283b8d26b3bSNicholas Bellinger ret = -ENOMEM; 3284b8d26b3bSNicholas Bellinger goto destroy_rx_wq; 3285b8d26b3bSNicholas Bellinger } 3286b8d26b3bSNicholas Bellinger 3287b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 3288b8d26b3bSNicholas Bellinger pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3289b8d26b3bSNicholas Bellinger return 0; 3290b8d26b3bSNicholas Bellinger 3291b8d26b3bSNicholas Bellinger destroy_rx_wq: 3292b8d26b3bSNicholas Bellinger destroy_workqueue(isert_rx_wq); 3293b8d26b3bSNicholas Bellinger return ret; 3294b8d26b3bSNicholas Bellinger } 3295b8d26b3bSNicholas Bellinger 3296b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 3297b8d26b3bSNicholas Bellinger { 3298b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 3299b8d26b3bSNicholas Bellinger destroy_workqueue(isert_rx_wq); 3300b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 3301b8d26b3bSNicholas Bellinger pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 3302b8d26b3bSNicholas Bellinger } 3303b8d26b3bSNicholas Bellinger 3304b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3305b8d26b3bSNicholas Bellinger MODULE_VERSION("0.1"); 3306b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3307b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 3308b8d26b3bSNicholas Bellinger 3309b8d26b3bSNicholas Bellinger module_init(isert_init); 3310b8d26b3bSNicholas Bellinger module_exit(isert_exit); 3311