1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 44c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 25b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 26b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 27b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 29b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 30531b7bf4SSagi Grimberg #include <linux/semaphore.h> 31b8d26b3bSNicholas Bellinger 32b8d26b3bSNicholas Bellinger #include "isert_proto.h" 33b8d26b3bSNicholas Bellinger #include "ib_isert.h" 34b8d26b3bSNicholas Bellinger 35b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 36b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37b8d26b3bSNicholas Bellinger #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39bdf20e72SSagi Grimberg ISERT_MAX_CONN) 40b8d26b3bSNicholas Bellinger 41*24f412ddSSagi Grimberg int isert_debug_level = 0; 42*24f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 43*24f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44*24f412ddSSagi Grimberg 45b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 46b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 47b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 48b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 49b8d26b3bSNicholas Bellinger 50b8d26b3bSNicholas Bellinger static void 51d40945d8SVu Pham isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52d40945d8SVu Pham static int 53d40945d8SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54d40945d8SVu Pham struct isert_rdma_wr *wr); 5559464ef4SVu Pham static void 56a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 5759464ef4SVu Pham static int 58a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5959464ef4SVu Pham struct isert_rdma_wr *wr); 60f93f3a70SSagi Grimberg static int 61f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 622371e5daSSagi Grimberg static int 632371e5daSSagi Grimberg isert_rdma_post_recvl(struct isert_conn *isert_conn); 642371e5daSSagi Grimberg static int 652371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 66ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 67d40945d8SVu Pham 68302cc7c3SSagi Grimberg static inline bool 69302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 70302cc7c3SSagi Grimberg { 7123a548eeSSagi Grimberg return (conn->pi_support && 72302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 73302cc7c3SSagi Grimberg } 74302cc7c3SSagi Grimberg 75302cc7c3SSagi Grimberg 76d40945d8SVu Pham static void 77b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 78b8d26b3bSNicholas Bellinger { 79b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)context; 80b8d26b3bSNicholas Bellinger 81*24f412ddSSagi Grimberg isert_err("isert_qp_event_callback event: %d\n", e->event); 82b8d26b3bSNicholas Bellinger switch (e->event) { 83b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 84b8d26b3bSNicholas Bellinger rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 85b8d26b3bSNicholas Bellinger break; 86b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 87*24f412ddSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 88b8d26b3bSNicholas Bellinger break; 89b8d26b3bSNicholas Bellinger default: 90b8d26b3bSNicholas Bellinger break; 91b8d26b3bSNicholas Bellinger } 92b8d26b3bSNicholas Bellinger } 93b8d26b3bSNicholas Bellinger 94b8d26b3bSNicholas Bellinger static int 95b8d26b3bSNicholas Bellinger isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 96b8d26b3bSNicholas Bellinger { 97b8d26b3bSNicholas Bellinger int ret; 98b8d26b3bSNicholas Bellinger 99b8d26b3bSNicholas Bellinger ret = ib_query_device(ib_dev, devattr); 100b8d26b3bSNicholas Bellinger if (ret) { 101*24f412ddSSagi Grimberg isert_err("ib_query_device() failed: %d\n", ret); 102b8d26b3bSNicholas Bellinger return ret; 103b8d26b3bSNicholas Bellinger } 104*24f412ddSSagi Grimberg isert_dbg("devattr->max_sge: %d\n", devattr->max_sge); 105*24f412ddSSagi Grimberg isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 106b8d26b3bSNicholas Bellinger 107b8d26b3bSNicholas Bellinger return 0; 108b8d26b3bSNicholas Bellinger } 109b8d26b3bSNicholas Bellinger 110b8d26b3bSNicholas Bellinger static int 111570db170SSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 112b8d26b3bSNicholas Bellinger { 113b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 114b8d26b3bSNicholas Bellinger struct ib_qp_init_attr attr; 1154a295baeSSagi Grimberg struct isert_comp *comp; 1164a295baeSSagi Grimberg int ret, i, min = 0; 117b8d26b3bSNicholas Bellinger 118b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 1194a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 1204a295baeSSagi Grimberg if (device->comps[i].active_qps < 1214a295baeSSagi Grimberg device->comps[min].active_qps) 1224a295baeSSagi Grimberg min = i; 1234a295baeSSagi Grimberg comp = &device->comps[min]; 1244a295baeSSagi Grimberg comp->active_qps++; 125*24f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1264a295baeSSagi Grimberg isert_conn, comp, min); 127b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 128b8d26b3bSNicholas Bellinger 129b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 130b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 131b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1326f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1336f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 134b8d26b3bSNicholas Bellinger attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 135bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 136b8d26b3bSNicholas Bellinger /* 137b8d26b3bSNicholas Bellinger * FIXME: Use devattr.max_sge - 2 for max_send_sge as 138f57915cfSOr Gerlitz * work-around for RDMA_READs with ConnectX-2. 139f57915cfSOr Gerlitz * 140f57915cfSOr Gerlitz * Also, still make sure to have at least two SGEs for 141f57915cfSOr Gerlitz * outgoing control PDU responses. 142b8d26b3bSNicholas Bellinger */ 143f57915cfSOr Gerlitz attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 144b8d26b3bSNicholas Bellinger isert_conn->max_sge = attr.cap.max_send_sge; 145b8d26b3bSNicholas Bellinger 146b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 147b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 148b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 149570db170SSagi Grimberg if (device->pi_capable) 150d3e125daSSagi Grimberg attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 151b8d26b3bSNicholas Bellinger 152*24f412ddSSagi Grimberg isert_dbg("isert_conn_setup_qp cma_id->device: %p\n", 153b8d26b3bSNicholas Bellinger cma_id->device); 154*24f412ddSSagi Grimberg isert_dbg("isert_conn_setup_qp conn_pd->device: %p\n", 155b8d26b3bSNicholas Bellinger isert_conn->conn_pd->device); 156b8d26b3bSNicholas Bellinger 157b8d26b3bSNicholas Bellinger ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 158b8d26b3bSNicholas Bellinger if (ret) { 159*24f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 16019e2090fSSagi Grimberg goto err; 161b8d26b3bSNicholas Bellinger } 162b8d26b3bSNicholas Bellinger isert_conn->conn_qp = cma_id->qp; 163*24f412ddSSagi Grimberg isert_dbg("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 164b8d26b3bSNicholas Bellinger 165b8d26b3bSNicholas Bellinger return 0; 16619e2090fSSagi Grimberg err: 16719e2090fSSagi Grimberg mutex_lock(&device_list_mutex); 1684a295baeSSagi Grimberg comp->active_qps--; 16919e2090fSSagi Grimberg mutex_unlock(&device_list_mutex); 17019e2090fSSagi Grimberg 17119e2090fSSagi Grimberg return ret; 172b8d26b3bSNicholas Bellinger } 173b8d26b3bSNicholas Bellinger 174b8d26b3bSNicholas Bellinger static void 175b8d26b3bSNicholas Bellinger isert_cq_event_callback(struct ib_event *e, void *context) 176b8d26b3bSNicholas Bellinger { 177*24f412ddSSagi Grimberg isert_dbg("isert_cq_event_callback event: %d\n", e->event); 178b8d26b3bSNicholas Bellinger } 179b8d26b3bSNicholas Bellinger 180b8d26b3bSNicholas Bellinger static int 181b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 182b8d26b3bSNicholas Bellinger { 183b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 184b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 185b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 186b8d26b3bSNicholas Bellinger u64 dma_addr; 187b8d26b3bSNicholas Bellinger int i, j; 188b8d26b3bSNicholas Bellinger 189b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 190b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 191b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 192b8d26b3bSNicholas Bellinger goto fail; 193b8d26b3bSNicholas Bellinger 194b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 195b8d26b3bSNicholas Bellinger 196b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 197b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 198b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 199b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 200b8d26b3bSNicholas Bellinger goto dma_map_fail; 201b8d26b3bSNicholas Bellinger 202b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 203b8d26b3bSNicholas Bellinger 204b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 205b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 206b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 207b8d26b3bSNicholas Bellinger rx_sg->lkey = isert_conn->conn_mr->lkey; 208b8d26b3bSNicholas Bellinger } 209b8d26b3bSNicholas Bellinger 210b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = 0; 211b8d26b3bSNicholas Bellinger return 0; 212b8d26b3bSNicholas Bellinger 213b8d26b3bSNicholas Bellinger dma_map_fail: 214b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 215b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 216b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 217b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 218b8d26b3bSNicholas Bellinger } 219b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 220b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 221b8d26b3bSNicholas Bellinger fail: 222b8d26b3bSNicholas Bellinger return -ENOMEM; 223b8d26b3bSNicholas Bellinger } 224b8d26b3bSNicholas Bellinger 225b8d26b3bSNicholas Bellinger static void 226b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 227b8d26b3bSNicholas Bellinger { 228b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 229b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 230b8d26b3bSNicholas Bellinger int i; 231b8d26b3bSNicholas Bellinger 232b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 233b8d26b3bSNicholas Bellinger return; 234b8d26b3bSNicholas Bellinger 235b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 236b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 237b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 238b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 239b8d26b3bSNicholas Bellinger } 240b8d26b3bSNicholas Bellinger 241b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 242b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 243b8d26b3bSNicholas Bellinger } 244b8d26b3bSNicholas Bellinger 2456f0fae3dSSagi Grimberg static void isert_cq_work(struct work_struct *); 2466f0fae3dSSagi Grimberg static void isert_cq_callback(struct ib_cq *, void *); 247b8d26b3bSNicholas Bellinger 248b8d26b3bSNicholas Bellinger static int 249b8d26b3bSNicholas Bellinger isert_create_device_ib_res(struct isert_device *device) 250b8d26b3bSNicholas Bellinger { 251b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = device->ib_device; 25259464ef4SVu Pham struct ib_device_attr *dev_attr; 2534a295baeSSagi Grimberg int ret = 0, i; 2546f0fae3dSSagi Grimberg int max_cqe; 255b8d26b3bSNicholas Bellinger 25659464ef4SVu Pham dev_attr = &device->dev_attr; 25759464ef4SVu Pham ret = isert_query_device(ib_dev, dev_attr); 25859464ef4SVu Pham if (ret) 25959464ef4SVu Pham return ret; 26059464ef4SVu Pham 2616f0fae3dSSagi Grimberg max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe); 262b1a5ad00SChris Moore 263d40945d8SVu Pham /* asign function handlers */ 264f2252258SSagi Grimberg if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 265f2252258SSagi Grimberg dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 266a3a5a826SSagi Grimberg device->use_fastreg = 1; 267a3a5a826SSagi Grimberg device->reg_rdma_mem = isert_reg_rdma; 268a3a5a826SSagi Grimberg device->unreg_rdma_mem = isert_unreg_rdma; 26959464ef4SVu Pham } else { 270a3a5a826SSagi Grimberg device->use_fastreg = 0; 271d40945d8SVu Pham device->reg_rdma_mem = isert_map_rdma; 272d40945d8SVu Pham device->unreg_rdma_mem = isert_unmap_cmd; 27359464ef4SVu Pham } 274d40945d8SVu Pham 275d3e125daSSagi Grimberg /* Check signature cap */ 276d3e125daSSagi Grimberg device->pi_capable = dev_attr->device_cap_flags & 277d3e125daSSagi Grimberg IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 278d3e125daSSagi Grimberg 2794a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2804a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 281*24f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 282d3e125daSSagi Grimberg "Fast registration %d pi_capable %d\n", 2834a295baeSSagi Grimberg device->comps_used, device->ib_device->name, 284d3e125daSSagi Grimberg device->ib_device->num_comp_vectors, device->use_fastreg, 285d3e125daSSagi Grimberg device->pi_capable); 2864a295baeSSagi Grimberg 2874a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2884a295baeSSagi Grimberg GFP_KERNEL); 2894a295baeSSagi Grimberg if (!device->comps) { 290*24f412ddSSagi Grimberg isert_err("Unable to allocate completion contexts\n"); 291b8d26b3bSNicholas Bellinger return -ENOMEM; 292b8d26b3bSNicholas Bellinger } 293b8d26b3bSNicholas Bellinger 2944a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2954a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 296b8d26b3bSNicholas Bellinger 2974a295baeSSagi Grimberg comp->device = device; 2986f0fae3dSSagi Grimberg INIT_WORK(&comp->work, isert_cq_work); 2996f0fae3dSSagi Grimberg comp->cq = ib_create_cq(device->ib_device, 3006f0fae3dSSagi Grimberg isert_cq_callback, 301b8d26b3bSNicholas Bellinger isert_cq_event_callback, 3024a295baeSSagi Grimberg (void *)comp, 3036f0fae3dSSagi Grimberg max_cqe, i); 3046f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 3056f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 3066f0fae3dSSagi Grimberg comp->cq = NULL; 307b8d26b3bSNicholas Bellinger goto out_cq; 30894a71110SWei Yongjun } 309b8d26b3bSNicholas Bellinger 3106f0fae3dSSagi Grimberg ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 31194a71110SWei Yongjun if (ret) 312b8d26b3bSNicholas Bellinger goto out_cq; 313b8d26b3bSNicholas Bellinger } 314b8d26b3bSNicholas Bellinger 315b8d26b3bSNicholas Bellinger return 0; 316b8d26b3bSNicholas Bellinger 317b8d26b3bSNicholas Bellinger out_cq: 3184a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 3194a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 320b8d26b3bSNicholas Bellinger 3216f0fae3dSSagi Grimberg if (comp->cq) { 3226f0fae3dSSagi Grimberg cancel_work_sync(&comp->work); 3236f0fae3dSSagi Grimberg ib_destroy_cq(comp->cq); 324b8d26b3bSNicholas Bellinger } 325b8d26b3bSNicholas Bellinger } 3264a295baeSSagi Grimberg kfree(device->comps); 327b8d26b3bSNicholas Bellinger 328b8d26b3bSNicholas Bellinger return ret; 329b8d26b3bSNicholas Bellinger } 330b8d26b3bSNicholas Bellinger 331b8d26b3bSNicholas Bellinger static void 332b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 333b8d26b3bSNicholas Bellinger { 334b8d26b3bSNicholas Bellinger int i; 335b8d26b3bSNicholas Bellinger 336*24f412ddSSagi Grimberg isert_info("device %p\n", device); 337b8d26b3bSNicholas Bellinger 3384a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 3394a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 3404a295baeSSagi Grimberg 3416f0fae3dSSagi Grimberg cancel_work_sync(&comp->work); 3426f0fae3dSSagi Grimberg ib_destroy_cq(comp->cq); 3436f0fae3dSSagi Grimberg comp->cq = NULL; 344b8d26b3bSNicholas Bellinger } 3454a295baeSSagi Grimberg kfree(device->comps); 346b8d26b3bSNicholas Bellinger } 347b8d26b3bSNicholas Bellinger 348b8d26b3bSNicholas Bellinger static void 349b8d26b3bSNicholas Bellinger isert_device_try_release(struct isert_device *device) 350b8d26b3bSNicholas Bellinger { 351b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 352b8d26b3bSNicholas Bellinger device->refcount--; 353b8d26b3bSNicholas Bellinger if (!device->refcount) { 354b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 355b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 356b8d26b3bSNicholas Bellinger kfree(device); 357b8d26b3bSNicholas Bellinger } 358b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 359b8d26b3bSNicholas Bellinger } 360b8d26b3bSNicholas Bellinger 361b8d26b3bSNicholas Bellinger static struct isert_device * 362b8d26b3bSNicholas Bellinger isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 363b8d26b3bSNicholas Bellinger { 364b8d26b3bSNicholas Bellinger struct isert_device *device; 365b8d26b3bSNicholas Bellinger int ret; 366b8d26b3bSNicholas Bellinger 367b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 368b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 369b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 370b8d26b3bSNicholas Bellinger device->refcount++; 371b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 372b8d26b3bSNicholas Bellinger return device; 373b8d26b3bSNicholas Bellinger } 374b8d26b3bSNicholas Bellinger } 375b8d26b3bSNicholas Bellinger 376b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 377b8d26b3bSNicholas Bellinger if (!device) { 378b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 379b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 380b8d26b3bSNicholas Bellinger } 381b8d26b3bSNicholas Bellinger 382b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 383b8d26b3bSNicholas Bellinger 384b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 385b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 386b8d26b3bSNicholas Bellinger if (ret) { 387b8d26b3bSNicholas Bellinger kfree(device); 388b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 389b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 390b8d26b3bSNicholas Bellinger } 391b8d26b3bSNicholas Bellinger 392b8d26b3bSNicholas Bellinger device->refcount++; 393b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 394b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 395b8d26b3bSNicholas Bellinger 396b8d26b3bSNicholas Bellinger return device; 397b8d26b3bSNicholas Bellinger } 398b8d26b3bSNicholas Bellinger 39959464ef4SVu Pham static void 400a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 40159464ef4SVu Pham { 40259464ef4SVu Pham struct fast_reg_descriptor *fr_desc, *tmp; 40359464ef4SVu Pham int i = 0; 40459464ef4SVu Pham 405a3a5a826SSagi Grimberg if (list_empty(&isert_conn->conn_fr_pool)) 40659464ef4SVu Pham return; 40759464ef4SVu Pham 408*24f412ddSSagi Grimberg isert_dbg("Freeing conn %p fastreg pool", isert_conn); 40959464ef4SVu Pham 41059464ef4SVu Pham list_for_each_entry_safe(fr_desc, tmp, 411a3a5a826SSagi Grimberg &isert_conn->conn_fr_pool, list) { 41259464ef4SVu Pham list_del(&fr_desc->list); 41359464ef4SVu Pham ib_free_fast_reg_page_list(fr_desc->data_frpl); 41459464ef4SVu Pham ib_dereg_mr(fr_desc->data_mr); 415d3e125daSSagi Grimberg if (fr_desc->pi_ctx) { 416d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 417d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 418d3e125daSSagi Grimberg ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 419d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 420d3e125daSSagi Grimberg } 42159464ef4SVu Pham kfree(fr_desc); 42259464ef4SVu Pham ++i; 42359464ef4SVu Pham } 42459464ef4SVu Pham 425a3a5a826SSagi Grimberg if (i < isert_conn->conn_fr_pool_size) 426*24f412ddSSagi Grimberg isert_warn("Pool still has %d regions registered\n", 427a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size - i); 42859464ef4SVu Pham } 42959464ef4SVu Pham 43059464ef4SVu Pham static int 431570db170SSagi Grimberg isert_create_pi_ctx(struct fast_reg_descriptor *desc, 432570db170SSagi Grimberg struct ib_device *device, 433570db170SSagi Grimberg struct ib_pd *pd) 434570db170SSagi Grimberg { 435570db170SSagi Grimberg struct ib_mr_init_attr mr_init_attr; 436570db170SSagi Grimberg struct pi_context *pi_ctx; 437570db170SSagi Grimberg int ret; 438570db170SSagi Grimberg 439570db170SSagi Grimberg pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 440570db170SSagi Grimberg if (!pi_ctx) { 441*24f412ddSSagi Grimberg isert_err("Failed to allocate pi context\n"); 442570db170SSagi Grimberg return -ENOMEM; 443570db170SSagi Grimberg } 444570db170SSagi Grimberg 445570db170SSagi Grimberg pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(device, 446570db170SSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 447570db170SSagi Grimberg if (IS_ERR(pi_ctx->prot_frpl)) { 448*24f412ddSSagi Grimberg isert_err("Failed to allocate prot frpl err=%ld\n", 449570db170SSagi Grimberg PTR_ERR(pi_ctx->prot_frpl)); 450570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->prot_frpl); 451570db170SSagi Grimberg goto err_pi_ctx; 452570db170SSagi Grimberg } 453570db170SSagi Grimberg 454570db170SSagi Grimberg pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 455570db170SSagi Grimberg if (IS_ERR(pi_ctx->prot_mr)) { 456*24f412ddSSagi Grimberg isert_err("Failed to allocate prot frmr err=%ld\n", 457570db170SSagi Grimberg PTR_ERR(pi_ctx->prot_mr)); 458570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->prot_mr); 459570db170SSagi Grimberg goto err_prot_frpl; 460570db170SSagi Grimberg } 461570db170SSagi Grimberg desc->ind |= ISERT_PROT_KEY_VALID; 462570db170SSagi Grimberg 463570db170SSagi Grimberg memset(&mr_init_attr, 0, sizeof(mr_init_attr)); 464570db170SSagi Grimberg mr_init_attr.max_reg_descriptors = 2; 465570db170SSagi Grimberg mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 466570db170SSagi Grimberg pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 467570db170SSagi Grimberg if (IS_ERR(pi_ctx->sig_mr)) { 468*24f412ddSSagi Grimberg isert_err("Failed to allocate signature enabled mr err=%ld\n", 469570db170SSagi Grimberg PTR_ERR(pi_ctx->sig_mr)); 470570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->sig_mr); 471570db170SSagi Grimberg goto err_prot_mr; 472570db170SSagi Grimberg } 473570db170SSagi Grimberg 474570db170SSagi Grimberg desc->pi_ctx = pi_ctx; 475570db170SSagi Grimberg desc->ind |= ISERT_SIG_KEY_VALID; 476570db170SSagi Grimberg desc->ind &= ~ISERT_PROTECTED; 477570db170SSagi Grimberg 478570db170SSagi Grimberg return 0; 479570db170SSagi Grimberg 480570db170SSagi Grimberg err_prot_mr: 481570db170SSagi Grimberg ib_dereg_mr(desc->pi_ctx->prot_mr); 482570db170SSagi Grimberg err_prot_frpl: 483570db170SSagi Grimberg ib_free_fast_reg_page_list(desc->pi_ctx->prot_frpl); 484570db170SSagi Grimberg err_pi_ctx: 485570db170SSagi Grimberg kfree(desc->pi_ctx); 486570db170SSagi Grimberg 487570db170SSagi Grimberg return ret; 488570db170SSagi Grimberg } 489570db170SSagi Grimberg 490570db170SSagi Grimberg static int 491dc87a90fSSagi Grimberg isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 492570db170SSagi Grimberg struct fast_reg_descriptor *fr_desc) 493dc87a90fSSagi Grimberg { 494d3e125daSSagi Grimberg int ret; 495d3e125daSSagi Grimberg 496dc87a90fSSagi Grimberg fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 497dc87a90fSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 498dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_frpl)) { 499*24f412ddSSagi Grimberg isert_err("Failed to allocate data frpl err=%ld\n", 500dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_frpl)); 501dc87a90fSSagi Grimberg return PTR_ERR(fr_desc->data_frpl); 502dc87a90fSSagi Grimberg } 503dc87a90fSSagi Grimberg 504dc87a90fSSagi Grimberg fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 505dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_mr)) { 506*24f412ddSSagi Grimberg isert_err("Failed to allocate data frmr err=%ld\n", 507dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_mr)); 508d3e125daSSagi Grimberg ret = PTR_ERR(fr_desc->data_mr); 509d3e125daSSagi Grimberg goto err_data_frpl; 510dc87a90fSSagi Grimberg } 511d3e125daSSagi Grimberg fr_desc->ind |= ISERT_DATA_KEY_VALID; 512dc87a90fSSagi Grimberg 513*24f412ddSSagi Grimberg isert_dbg("Created fr_desc %p\n", fr_desc); 514dc87a90fSSagi Grimberg 515dc87a90fSSagi Grimberg return 0; 516570db170SSagi Grimberg 517d3e125daSSagi Grimberg err_data_frpl: 518d3e125daSSagi Grimberg ib_free_fast_reg_page_list(fr_desc->data_frpl); 519d3e125daSSagi Grimberg 520d3e125daSSagi Grimberg return ret; 521dc87a90fSSagi Grimberg } 522dc87a90fSSagi Grimberg 523dc87a90fSSagi Grimberg static int 524570db170SSagi Grimberg isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 52559464ef4SVu Pham { 52659464ef4SVu Pham struct fast_reg_descriptor *fr_desc; 52759464ef4SVu Pham struct isert_device *device = isert_conn->conn_device; 528f46d6a8aSNicholas Bellinger struct se_session *se_sess = isert_conn->conn->sess->se_sess; 529f46d6a8aSNicholas Bellinger struct se_node_acl *se_nacl = se_sess->se_node_acl; 530f46d6a8aSNicholas Bellinger int i, ret, tag_num; 531f46d6a8aSNicholas Bellinger /* 532f46d6a8aSNicholas Bellinger * Setup the number of FRMRs based upon the number of tags 533f46d6a8aSNicholas Bellinger * available to session in iscsi_target_locate_portal(). 534f46d6a8aSNicholas Bellinger */ 535f46d6a8aSNicholas Bellinger tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 536f46d6a8aSNicholas Bellinger tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 53759464ef4SVu Pham 538a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size = 0; 539f46d6a8aSNicholas Bellinger for (i = 0; i < tag_num; i++) { 54059464ef4SVu Pham fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 54159464ef4SVu Pham if (!fr_desc) { 542*24f412ddSSagi Grimberg isert_err("Failed to allocate fast_reg descriptor\n"); 54359464ef4SVu Pham ret = -ENOMEM; 54459464ef4SVu Pham goto err; 54559464ef4SVu Pham } 54659464ef4SVu Pham 547dc87a90fSSagi Grimberg ret = isert_create_fr_desc(device->ib_device, 548570db170SSagi Grimberg isert_conn->conn_pd, fr_desc); 549dc87a90fSSagi Grimberg if (ret) { 550*24f412ddSSagi Grimberg isert_err("Failed to create fastreg descriptor err=%d\n", 551dc87a90fSSagi Grimberg ret); 552a80e21b3SNicholas Bellinger kfree(fr_desc); 55359464ef4SVu Pham goto err; 55459464ef4SVu Pham } 55559464ef4SVu Pham 556a3a5a826SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 557a3a5a826SSagi Grimberg isert_conn->conn_fr_pool_size++; 55859464ef4SVu Pham } 55959464ef4SVu Pham 560*24f412ddSSagi Grimberg isert_dbg("Creating conn %p fastreg pool size=%d", 561a3a5a826SSagi Grimberg isert_conn, isert_conn->conn_fr_pool_size); 56259464ef4SVu Pham 56359464ef4SVu Pham return 0; 56459464ef4SVu Pham 56559464ef4SVu Pham err: 566a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 56759464ef4SVu Pham return ret; 56859464ef4SVu Pham } 56959464ef4SVu Pham 570b8d26b3bSNicholas Bellinger static int 571b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 572b8d26b3bSNicholas Bellinger { 573ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 574ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 575b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 576b8d26b3bSNicholas Bellinger struct isert_device *device; 577b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = cma_id->device; 578b8d26b3bSNicholas Bellinger int ret = 0; 57914f4b54fSSagi Grimberg 58014f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 58114f4b54fSSagi Grimberg if (!np->enabled) { 58214f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 583*24f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 58414f4b54fSSagi Grimberg return rdma_reject(cma_id, NULL, 0); 58514f4b54fSSagi Grimberg } 58614f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 587b8d26b3bSNicholas Bellinger 588*24f412ddSSagi Grimberg isert_dbg("Entering isert_connect_request cma_id: %p, context: %p\n", 589b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 590b8d26b3bSNicholas Bellinger 591b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 592b8d26b3bSNicholas Bellinger if (!isert_conn) { 593*24f412ddSSagi Grimberg isert_err("Unable to allocate isert_conn\n"); 594b8d26b3bSNicholas Bellinger return -ENOMEM; 595b8d26b3bSNicholas Bellinger } 596b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_INIT; 597b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_accept_node); 598b8d26b3bSNicholas Bellinger init_completion(&isert_conn->conn_login_comp); 5992371e5daSSagi Grimberg init_completion(&isert_conn->login_req_comp); 600defd8848SNicholas Bellinger init_completion(&isert_conn->conn_wait); 601b8d26b3bSNicholas Bellinger kref_init(&isert_conn->conn_kref); 602b2cb9649SNicholas Bellinger mutex_init(&isert_conn->conn_mutex); 60359464ef4SVu Pham spin_lock_init(&isert_conn->conn_lock); 604f46d6a8aSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 605b8d26b3bSNicholas Bellinger 606b8d26b3bSNicholas Bellinger isert_conn->conn_cm_id = cma_id; 607b8d26b3bSNicholas Bellinger 608b8d26b3bSNicholas Bellinger isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 609b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, GFP_KERNEL); 610b8d26b3bSNicholas Bellinger if (!isert_conn->login_buf) { 611*24f412ddSSagi Grimberg isert_err("Unable to allocate isert_conn->login_buf\n"); 612b8d26b3bSNicholas Bellinger ret = -ENOMEM; 613b8d26b3bSNicholas Bellinger goto out; 614b8d26b3bSNicholas Bellinger } 615b8d26b3bSNicholas Bellinger 616b8d26b3bSNicholas Bellinger isert_conn->login_req_buf = isert_conn->login_buf; 617b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf = isert_conn->login_buf + 618b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN; 619*24f412ddSSagi Grimberg isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 620b8d26b3bSNicholas Bellinger isert_conn->login_buf, isert_conn->login_req_buf, 621b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf); 622b8d26b3bSNicholas Bellinger 623b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 624b8d26b3bSNicholas Bellinger (void *)isert_conn->login_req_buf, 625b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 626b8d26b3bSNicholas Bellinger 627b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 628b8d26b3bSNicholas Bellinger if (ret) { 629*24f412ddSSagi Grimberg isert_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 630b8d26b3bSNicholas Bellinger ret); 631b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = 0; 632b8d26b3bSNicholas Bellinger goto out_login_buf; 633b8d26b3bSNicholas Bellinger } 634b8d26b3bSNicholas Bellinger 635b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 636b8d26b3bSNicholas Bellinger (void *)isert_conn->login_rsp_buf, 637b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 638b8d26b3bSNicholas Bellinger 639b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 640b8d26b3bSNicholas Bellinger if (ret) { 641*24f412ddSSagi Grimberg isert_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 642b8d26b3bSNicholas Bellinger ret); 643b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = 0; 644b8d26b3bSNicholas Bellinger goto out_req_dma_map; 645b8d26b3bSNicholas Bellinger } 646b8d26b3bSNicholas Bellinger 647b8d26b3bSNicholas Bellinger device = isert_device_find_by_ib_dev(cma_id); 648b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 649b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 650b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 651b8d26b3bSNicholas Bellinger } 652b8d26b3bSNicholas Bellinger 6531a92e17eSSagi Grimberg /* Set max inflight RDMA READ requests */ 6541a92e17eSSagi Grimberg isert_conn->initiator_depth = min_t(u8, 6551a92e17eSSagi Grimberg event->param.conn.initiator_depth, 6561a92e17eSSagi Grimberg device->dev_attr.max_qp_init_rd_atom); 657*24f412ddSSagi Grimberg isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 6581a92e17eSSagi Grimberg 659b8d26b3bSNicholas Bellinger isert_conn->conn_device = device; 660eb6ab132SSagi Grimberg isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 661eb6ab132SSagi Grimberg if (IS_ERR(isert_conn->conn_pd)) { 662eb6ab132SSagi Grimberg ret = PTR_ERR(isert_conn->conn_pd); 663*24f412ddSSagi Grimberg isert_err("ib_alloc_pd failed for conn %p: ret=%d\n", 664eb6ab132SSagi Grimberg isert_conn, ret); 665eb6ab132SSagi Grimberg goto out_pd; 666eb6ab132SSagi Grimberg } 667b8d26b3bSNicholas Bellinger 668eb6ab132SSagi Grimberg isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 669eb6ab132SSagi Grimberg IB_ACCESS_LOCAL_WRITE); 670eb6ab132SSagi Grimberg if (IS_ERR(isert_conn->conn_mr)) { 671eb6ab132SSagi Grimberg ret = PTR_ERR(isert_conn->conn_mr); 672*24f412ddSSagi Grimberg isert_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 673eb6ab132SSagi Grimberg isert_conn, ret); 674eb6ab132SSagi Grimberg goto out_mr; 675eb6ab132SSagi Grimberg } 676b8d26b3bSNicholas Bellinger 677570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 678b8d26b3bSNicholas Bellinger if (ret) 679b8d26b3bSNicholas Bellinger goto out_conn_dev; 680b8d26b3bSNicholas Bellinger 6812371e5daSSagi Grimberg ret = isert_rdma_post_recvl(isert_conn); 6822371e5daSSagi Grimberg if (ret) 6832371e5daSSagi Grimberg goto out_conn_dev; 6842371e5daSSagi Grimberg 6852371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 6862371e5daSSagi Grimberg if (ret) 6872371e5daSSagi Grimberg goto out_conn_dev; 6882371e5daSSagi Grimberg 689b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 6909fe63c88SSagi Grimberg list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list); 691b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 692b8d26b3bSNicholas Bellinger 693*24f412ddSSagi Grimberg isert_info("np %p: Allow accept_np to continue\n", np); 694531b7bf4SSagi Grimberg up(&isert_np->np_sem); 695b8d26b3bSNicholas Bellinger return 0; 696b8d26b3bSNicholas Bellinger 697b8d26b3bSNicholas Bellinger out_conn_dev: 698eb6ab132SSagi Grimberg ib_dereg_mr(isert_conn->conn_mr); 699eb6ab132SSagi Grimberg out_mr: 700eb6ab132SSagi Grimberg ib_dealloc_pd(isert_conn->conn_pd); 701eb6ab132SSagi Grimberg out_pd: 702b8d26b3bSNicholas Bellinger isert_device_try_release(device); 703b8d26b3bSNicholas Bellinger out_rsp_dma_map: 704b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 705b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 706b8d26b3bSNicholas Bellinger out_req_dma_map: 707b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 708b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 709b8d26b3bSNicholas Bellinger out_login_buf: 710b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 711b8d26b3bSNicholas Bellinger out: 712b8d26b3bSNicholas Bellinger kfree(isert_conn); 7132371e5daSSagi Grimberg rdma_reject(cma_id, NULL, 0); 714b8d26b3bSNicholas Bellinger return ret; 715b8d26b3bSNicholas Bellinger } 716b8d26b3bSNicholas Bellinger 717b8d26b3bSNicholas Bellinger static void 718b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 719b8d26b3bSNicholas Bellinger { 720b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 721b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 722b8d26b3bSNicholas Bellinger 723*24f412ddSSagi Grimberg isert_dbg("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 724b8d26b3bSNicholas Bellinger 725a3a5a826SSagi Grimberg if (device && device->use_fastreg) 726a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 72759464ef4SVu Pham 72819e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 72919e2090fSSagi Grimberg rdma_destroy_id(isert_conn->conn_cm_id); 73019e2090fSSagi Grimberg 731b8d26b3bSNicholas Bellinger if (isert_conn->conn_qp) { 7324a295baeSSagi Grimberg struct isert_comp *comp = isert_conn->conn_qp->recv_cq->cq_context; 7334a295baeSSagi Grimberg 734*24f412ddSSagi Grimberg isert_dbg("dec completion context %p active_qps\n", comp); 73519e2090fSSagi Grimberg mutex_lock(&device_list_mutex); 7364a295baeSSagi Grimberg comp->active_qps--; 73719e2090fSSagi Grimberg mutex_unlock(&device_list_mutex); 738b8d26b3bSNicholas Bellinger 73919e2090fSSagi Grimberg ib_destroy_qp(isert_conn->conn_qp); 740b8d26b3bSNicholas Bellinger } 741b8d26b3bSNicholas Bellinger 742eb6ab132SSagi Grimberg ib_dereg_mr(isert_conn->conn_mr); 743eb6ab132SSagi Grimberg ib_dealloc_pd(isert_conn->conn_pd); 744eb6ab132SSagi Grimberg 745b8d26b3bSNicholas Bellinger if (isert_conn->login_buf) { 746b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 747b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 748b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 749b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, 750b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 751b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 752b8d26b3bSNicholas Bellinger } 753b8d26b3bSNicholas Bellinger kfree(isert_conn); 754b8d26b3bSNicholas Bellinger 755b8d26b3bSNicholas Bellinger if (device) 756b8d26b3bSNicholas Bellinger isert_device_try_release(device); 757b8d26b3bSNicholas Bellinger 758*24f412ddSSagi Grimberg isert_dbg("Leaving isert_connect_release >>>>>>>>>>>>\n"); 759b8d26b3bSNicholas Bellinger } 760b8d26b3bSNicholas Bellinger 761b8d26b3bSNicholas Bellinger static void 762b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 763b8d26b3bSNicholas Bellinger { 76419e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 765c2f88b17SSagi Grimberg 766*24f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 767128e9cc8SSagi Grimberg 7682371e5daSSagi Grimberg if (!kref_get_unless_zero(&isert_conn->conn_kref)) { 769*24f412ddSSagi Grimberg isert_warn("conn %p connect_release is running\n", isert_conn); 7702371e5daSSagi Grimberg return; 7712371e5daSSagi Grimberg } 7722371e5daSSagi Grimberg 7732371e5daSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 7742371e5daSSagi Grimberg if (isert_conn->state != ISER_CONN_FULL_FEATURE) 775128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 7762371e5daSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 777b8d26b3bSNicholas Bellinger } 778b8d26b3bSNicholas Bellinger 779b8d26b3bSNicholas Bellinger static void 780b8d26b3bSNicholas Bellinger isert_release_conn_kref(struct kref *kref) 781b8d26b3bSNicholas Bellinger { 782b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 783b8d26b3bSNicholas Bellinger struct isert_conn, conn_kref); 784b8d26b3bSNicholas Bellinger 785*24f412ddSSagi Grimberg isert_dbg("Calling isert_connect_release for final kref %s/%d\n", 786b8d26b3bSNicholas Bellinger current->comm, current->pid); 787b8d26b3bSNicholas Bellinger 788b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 789b8d26b3bSNicholas Bellinger } 790b8d26b3bSNicholas Bellinger 791b8d26b3bSNicholas Bellinger static void 792b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 793b8d26b3bSNicholas Bellinger { 794b8d26b3bSNicholas Bellinger kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 795b8d26b3bSNicholas Bellinger } 796b8d26b3bSNicholas Bellinger 797954f2372SSagi Grimberg /** 798954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 799954f2372SSagi Grimberg * @isert_conn: isert connection struct 800954f2372SSagi Grimberg * 801954f2372SSagi Grimberg * Notes: 802128e9cc8SSagi Grimberg * In case the connection state is FULL_FEATURE, move state 803954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 804128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 805954f2372SSagi Grimberg * 806954f2372SSagi Grimberg * This routine must be called with conn_mutex held. Thus it is 807954f2372SSagi Grimberg * safe to call multiple times. 808954f2372SSagi Grimberg */ 809954f2372SSagi Grimberg static void 810954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 811954f2372SSagi Grimberg { 812954f2372SSagi Grimberg int err; 813954f2372SSagi Grimberg 814128e9cc8SSagi Grimberg switch (isert_conn->state) { 815128e9cc8SSagi Grimberg case ISER_CONN_TERMINATING: 816128e9cc8SSagi Grimberg break; 817128e9cc8SSagi Grimberg case ISER_CONN_UP: 818128e9cc8SSagi Grimberg case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 819*24f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 820954f2372SSagi Grimberg isert_conn, isert_conn->state); 821128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 822954f2372SSagi Grimberg err = rdma_disconnect(isert_conn->conn_cm_id); 823954f2372SSagi Grimberg if (err) 824*24f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 825954f2372SSagi Grimberg isert_conn); 826128e9cc8SSagi Grimberg break; 827128e9cc8SSagi Grimberg default: 828*24f412ddSSagi Grimberg isert_warn("conn %p teminating in state %d\n", 829128e9cc8SSagi Grimberg isert_conn, isert_conn->state); 830954f2372SSagi Grimberg } 831954f2372SSagi Grimberg } 832954f2372SSagi Grimberg 8333b726ae2SSagi Grimberg static int 834ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 835ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 836b8d26b3bSNicholas Bellinger { 837*24f412ddSSagi Grimberg isert_dbg("isert np %p, handling event %d\n", isert_np, event); 8383b726ae2SSagi Grimberg 839ca6c1d82SSagi Grimberg switch (event) { 840ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 8413b726ae2SSagi Grimberg isert_np->np_cm_id = NULL; 842ca6c1d82SSagi Grimberg break; 843ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 844ca6c1d82SSagi Grimberg isert_np->np_cm_id = isert_setup_id(isert_np); 845ca6c1d82SSagi Grimberg if (IS_ERR(isert_np->np_cm_id)) { 846*24f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 847ca6c1d82SSagi Grimberg isert_np, PTR_ERR(isert_np->np_cm_id)); 848ca6c1d82SSagi Grimberg isert_np->np_cm_id = NULL; 849ca6c1d82SSagi Grimberg } 850ca6c1d82SSagi Grimberg break; 851ca6c1d82SSagi Grimberg default: 852*24f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 853ca6c1d82SSagi Grimberg isert_np, event); 854ca6c1d82SSagi Grimberg } 855ca6c1d82SSagi Grimberg 8563b726ae2SSagi Grimberg return -1; 8573b726ae2SSagi Grimberg } 8583b726ae2SSagi Grimberg 859ca6c1d82SSagi Grimberg static int 860ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 861ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 862ca6c1d82SSagi Grimberg { 863ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 864ca6c1d82SSagi Grimberg struct isert_conn *isert_conn; 865ca6c1d82SSagi Grimberg 866ca6c1d82SSagi Grimberg if (isert_np->np_cm_id == cma_id) 867ca6c1d82SSagi Grimberg return isert_np_cma_handler(cma_id->context, event); 868ca6c1d82SSagi Grimberg 86919e2090fSSagi Grimberg isert_conn = cma_id->qp->qp_context; 870b8d26b3bSNicholas Bellinger 871128e9cc8SSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 872128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 873128e9cc8SSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 874128e9cc8SSagi Grimberg 875*24f412ddSSagi Grimberg isert_info("conn %p completing conn_wait\n", isert_conn); 876128e9cc8SSagi Grimberg complete(&isert_conn->conn_wait); 8773b726ae2SSagi Grimberg 8783b726ae2SSagi Grimberg return 0; 879b8d26b3bSNicholas Bellinger } 880b8d26b3bSNicholas Bellinger 881954f2372SSagi Grimberg static void 882954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 883954f2372SSagi Grimberg { 88419e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 885954f2372SSagi Grimberg 886954f2372SSagi Grimberg isert_put_conn(isert_conn); 887954f2372SSagi Grimberg } 888954f2372SSagi Grimberg 889b8d26b3bSNicholas Bellinger static int 890b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 891b8d26b3bSNicholas Bellinger { 892b8d26b3bSNicholas Bellinger int ret = 0; 893b8d26b3bSNicholas Bellinger 894*24f412ddSSagi Grimberg isert_dbg("isert_cma_handler: event %d status %d conn %p id %p\n", 895b8d26b3bSNicholas Bellinger event->event, event->status, cma_id->context, cma_id); 896b8d26b3bSNicholas Bellinger 897b8d26b3bSNicholas Bellinger switch (event->event) { 898b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 899b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 9003b726ae2SSagi Grimberg if (ret) 901*24f412ddSSagi Grimberg isert_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 9023b726ae2SSagi Grimberg event->event, ret); 903b8d26b3bSNicholas Bellinger break; 904b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 905b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 906b8d26b3bSNicholas Bellinger break; 90788c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 90888c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 90988c4015fSSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 91088c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 911ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 912b8d26b3bSNicholas Bellinger break; 913954f2372SSagi Grimberg case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 914954f2372SSagi Grimberg case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 915b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 916954f2372SSagi Grimberg isert_connect_error(cma_id); 917954f2372SSagi Grimberg break; 918b8d26b3bSNicholas Bellinger default: 919*24f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 920b8d26b3bSNicholas Bellinger break; 921b8d26b3bSNicholas Bellinger } 922b8d26b3bSNicholas Bellinger 923b8d26b3bSNicholas Bellinger return ret; 924b8d26b3bSNicholas Bellinger } 925b8d26b3bSNicholas Bellinger 926b8d26b3bSNicholas Bellinger static int 927b8d26b3bSNicholas Bellinger isert_post_recv(struct isert_conn *isert_conn, u32 count) 928b8d26b3bSNicholas Bellinger { 929b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 930b8d26b3bSNicholas Bellinger int i, ret; 931b8d26b3bSNicholas Bellinger unsigned int rx_head = isert_conn->conn_rx_desc_head; 932b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 933b8d26b3bSNicholas Bellinger 934b8d26b3bSNicholas Bellinger for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 935b8d26b3bSNicholas Bellinger rx_desc = &isert_conn->conn_rx_descs[rx_head]; 936b0a191e7SSagi Grimberg rx_wr->wr_id = (uintptr_t)rx_desc; 937b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 938b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 939b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 940b8d26b3bSNicholas Bellinger rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 941b8d26b3bSNicholas Bellinger } 942b8d26b3bSNicholas Bellinger 943b8d26b3bSNicholas Bellinger rx_wr--; 944b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 945b8d26b3bSNicholas Bellinger 946b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count += count; 947b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 948b8d26b3bSNicholas Bellinger &rx_wr_failed); 949b8d26b3bSNicholas Bellinger if (ret) { 950*24f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 951b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count -= count; 952b8d26b3bSNicholas Bellinger } else { 953*24f412ddSSagi Grimberg isert_dbg("isert_post_recv(): Posted %d RX buffers\n", count); 954b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = rx_head; 955b8d26b3bSNicholas Bellinger } 956b8d26b3bSNicholas Bellinger return ret; 957b8d26b3bSNicholas Bellinger } 958b8d26b3bSNicholas Bellinger 959b8d26b3bSNicholas Bellinger static int 960b8d26b3bSNicholas Bellinger isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 961b8d26b3bSNicholas Bellinger { 962b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 963b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 964b8d26b3bSNicholas Bellinger int ret; 965b8d26b3bSNicholas Bellinger 966b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 967b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 968b8d26b3bSNicholas Bellinger 969b8d26b3bSNicholas Bellinger send_wr.next = NULL; 970b0a191e7SSagi Grimberg send_wr.wr_id = (uintptr_t)tx_desc; 971b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 972b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 973b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 974b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 975b8d26b3bSNicholas Bellinger 976b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 977bdf20e72SSagi Grimberg if (ret) 978*24f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 979b8d26b3bSNicholas Bellinger 980b8d26b3bSNicholas Bellinger return ret; 981b8d26b3bSNicholas Bellinger } 982b8d26b3bSNicholas Bellinger 983b8d26b3bSNicholas Bellinger static void 984b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 985b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 986b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 987b8d26b3bSNicholas Bellinger { 988b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 989b8d26b3bSNicholas Bellinger 990b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 991b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 992b8d26b3bSNicholas Bellinger 993b8d26b3bSNicholas Bellinger memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 994b8d26b3bSNicholas Bellinger tx_desc->iser_header.flags = ISER_VER; 995b8d26b3bSNicholas Bellinger 996b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 997b8d26b3bSNicholas Bellinger tx_desc->isert_cmd = isert_cmd; 998b8d26b3bSNicholas Bellinger 999b8d26b3bSNicholas Bellinger if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 1000b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 1001*24f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1002b8d26b3bSNicholas Bellinger } 1003b8d26b3bSNicholas Bellinger } 1004b8d26b3bSNicholas Bellinger 1005b8d26b3bSNicholas Bellinger static int 1006b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 1007b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1008b8d26b3bSNicholas Bellinger { 1009b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1010b8d26b3bSNicholas Bellinger u64 dma_addr; 1011b8d26b3bSNicholas Bellinger 1012b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1013b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1014b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1015*24f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 1016b8d26b3bSNicholas Bellinger return -ENOMEM; 1017b8d26b3bSNicholas Bellinger } 1018b8d26b3bSNicholas Bellinger 1019b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 1020b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1021b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1022b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 1023b8d26b3bSNicholas Bellinger 1024*24f412ddSSagi Grimberg isert_dbg("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 1025b8d26b3bSNicholas Bellinger " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 1026b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 1027b8d26b3bSNicholas Bellinger 1028b8d26b3bSNicholas Bellinger return 0; 1029b8d26b3bSNicholas Bellinger } 1030b8d26b3bSNicholas Bellinger 1031b8d26b3bSNicholas Bellinger static void 103295b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 103368a86deeSSagi Grimberg struct ib_send_wr *send_wr) 1034b8d26b3bSNicholas Bellinger { 103595b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 103695b60f07SNicholas Bellinger 1037b8d26b3bSNicholas Bellinger isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1038b0a191e7SSagi Grimberg send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1039b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 104095b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 1041b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 104295b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 1043b8d26b3bSNicholas Bellinger } 1044b8d26b3bSNicholas Bellinger 1045b8d26b3bSNicholas Bellinger static int 1046b8d26b3bSNicholas Bellinger isert_rdma_post_recvl(struct isert_conn *isert_conn) 1047b8d26b3bSNicholas Bellinger { 1048b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 1049b8d26b3bSNicholas Bellinger struct ib_sge sge; 1050b8d26b3bSNicholas Bellinger int ret; 1051b8d26b3bSNicholas Bellinger 1052b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 1053b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 1054b8d26b3bSNicholas Bellinger sge.length = ISER_RX_LOGIN_SIZE; 1055b8d26b3bSNicholas Bellinger sge.lkey = isert_conn->conn_mr->lkey; 1056b8d26b3bSNicholas Bellinger 1057*24f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1058b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 1059b8d26b3bSNicholas Bellinger 1060b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1061b0a191e7SSagi Grimberg rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1062b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 1063b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 1064b8d26b3bSNicholas Bellinger 1065b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count++; 1066b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 1067b8d26b3bSNicholas Bellinger if (ret) { 1068*24f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 1069b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1070b8d26b3bSNicholas Bellinger } 1071b8d26b3bSNicholas Bellinger 1072*24f412ddSSagi Grimberg isert_dbg("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 1073b8d26b3bSNicholas Bellinger return ret; 1074b8d26b3bSNicholas Bellinger } 1075b8d26b3bSNicholas Bellinger 1076b8d26b3bSNicholas Bellinger static int 1077b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1078b8d26b3bSNicholas Bellinger u32 length) 1079b8d26b3bSNicholas Bellinger { 1080b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 1081b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1082b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 1083b8d26b3bSNicholas Bellinger int ret; 1084b8d26b3bSNicholas Bellinger 1085b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 1086b8d26b3bSNicholas Bellinger 1087b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1088b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1089b8d26b3bSNicholas Bellinger 1090b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1091b8d26b3bSNicholas Bellinger 1092b8d26b3bSNicholas Bellinger if (length > 0) { 1093b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1094b8d26b3bSNicholas Bellinger 1095b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1096b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1097b8d26b3bSNicholas Bellinger 1098b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1099b8d26b3bSNicholas Bellinger 1100b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1101b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1102b8d26b3bSNicholas Bellinger 1103b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1104b8d26b3bSNicholas Bellinger tx_dsg->length = length; 1105b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 1106b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1107b8d26b3bSNicholas Bellinger } 1108b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1109b8d26b3bSNicholas Bellinger if (login->login_complete) { 1110e0546fc1SSagi Grimberg if (!conn->sess->sess_ops->SessionType && 1111e0546fc1SSagi Grimberg isert_conn->conn_device->use_fastreg) { 1112570db170SSagi Grimberg ret = isert_conn_create_fastreg_pool(isert_conn); 1113f46d6a8aSNicholas Bellinger if (ret) { 1114*24f412ddSSagi Grimberg isert_err("Conn: %p failed to create" 1115f46d6a8aSNicholas Bellinger " fastreg pool\n", isert_conn); 1116f46d6a8aSNicholas Bellinger return ret; 1117f46d6a8aSNicholas Bellinger } 1118f46d6a8aSNicholas Bellinger } 1119f46d6a8aSNicholas Bellinger 1120b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1121b8d26b3bSNicholas Bellinger if (ret) 1122b8d26b3bSNicholas Bellinger return ret; 1123b8d26b3bSNicholas Bellinger 1124b8d26b3bSNicholas Bellinger ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1125b8d26b3bSNicholas Bellinger if (ret) 1126b8d26b3bSNicholas Bellinger return ret; 1127b8d26b3bSNicholas Bellinger 1128128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 11292371e5daSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 1130128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 11312371e5daSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 1132b8d26b3bSNicholas Bellinger goto post_send; 1133b8d26b3bSNicholas Bellinger } 1134b8d26b3bSNicholas Bellinger 1135b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 1136b8d26b3bSNicholas Bellinger if (ret) 1137b8d26b3bSNicholas Bellinger return ret; 1138b8d26b3bSNicholas Bellinger } 1139b8d26b3bSNicholas Bellinger post_send: 1140b8d26b3bSNicholas Bellinger ret = isert_post_send(isert_conn, tx_desc); 1141b8d26b3bSNicholas Bellinger if (ret) 1142b8d26b3bSNicholas Bellinger return ret; 1143b8d26b3bSNicholas Bellinger 1144b8d26b3bSNicholas Bellinger return 0; 1145b8d26b3bSNicholas Bellinger } 1146b8d26b3bSNicholas Bellinger 1147b8d26b3bSNicholas Bellinger static void 11482371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1149b8d26b3bSNicholas Bellinger { 11502371e5daSSagi Grimberg struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 11512371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1152b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1153b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1154b8d26b3bSNicholas Bellinger int size; 1155b8d26b3bSNicholas Bellinger 1156*24f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 11572371e5daSSagi Grimberg 11582371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1159b8d26b3bSNicholas Bellinger 1160b8d26b3bSNicholas Bellinger if (login->first_request) { 1161b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1162b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1163b8d26b3bSNicholas Bellinger /* 1164b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1165b8d26b3bSNicholas Bellinger * login request PDU. 1166b8d26b3bSNicholas Bellinger */ 1167b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1168b8d26b3bSNicholas Bellinger login->current_stage = 1169b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1170b8d26b3bSNicholas Bellinger >> 2; 1171b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1172b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1173b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1174b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1175b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1176b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1177b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1178b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1179b8d26b3bSNicholas Bellinger } 1180b8d26b3bSNicholas Bellinger 1181b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1182b8d26b3bSNicholas Bellinger 1183b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1184*24f412ddSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1185b8d26b3bSNicholas Bellinger size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1186b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1187b8d26b3bSNicholas Bellinger 11886faaa85fSNicholas Bellinger if (login->first_request) { 1189b8d26b3bSNicholas Bellinger complete(&isert_conn->conn_login_comp); 11906faaa85fSNicholas Bellinger return; 11916faaa85fSNicholas Bellinger } 11926faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1193b8d26b3bSNicholas Bellinger } 1194b8d26b3bSNicholas Bellinger 1195b8d26b3bSNicholas Bellinger static struct iscsi_cmd 1196676687c6SNicholas Bellinger *isert_allocate_cmd(struct iscsi_conn *conn) 1197b8d26b3bSNicholas Bellinger { 1198b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1199b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1200d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1201b8d26b3bSNicholas Bellinger 1202676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1203d703ce2fSNicholas Bellinger if (!cmd) { 1204*24f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1205b8d26b3bSNicholas Bellinger return NULL; 1206b8d26b3bSNicholas Bellinger } 1207d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1208b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1209d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 1210b8d26b3bSNicholas Bellinger 1211d703ce2fSNicholas Bellinger return cmd; 1212b8d26b3bSNicholas Bellinger } 1213b8d26b3bSNicholas Bellinger 1214b8d26b3bSNicholas Bellinger static int 1215b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1216d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1217d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1218b8d26b3bSNicholas Bellinger { 1219b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1220b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1221b8d26b3bSNicholas Bellinger struct scatterlist *sg; 1222b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1223b8d26b3bSNicholas Bellinger bool dump_payload = false; 1224b8d26b3bSNicholas Bellinger 1225b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1226b8d26b3bSNicholas Bellinger if (rc < 0) 1227b8d26b3bSNicholas Bellinger return rc; 1228b8d26b3bSNicholas Bellinger 1229b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1230b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1231b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 1232b8d26b3bSNicholas Bellinger 1233b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1234b8d26b3bSNicholas Bellinger if (rc < 0) { 1235b8d26b3bSNicholas Bellinger return 0; 1236b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1237b8d26b3bSNicholas Bellinger dump_payload = true; 1238b8d26b3bSNicholas Bellinger goto sequence_cmd; 1239b8d26b3bSNicholas Bellinger } 1240b8d26b3bSNicholas Bellinger 1241b8d26b3bSNicholas Bellinger if (!imm_data) 1242b8d26b3bSNicholas Bellinger return 0; 1243b8d26b3bSNicholas Bellinger 1244b8d26b3bSNicholas Bellinger sg = &cmd->se_cmd.t_data_sg[0]; 1245b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1246b8d26b3bSNicholas Bellinger 1247*24f412ddSSagi Grimberg isert_dbg("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1248b8d26b3bSNicholas Bellinger sg, sg_nents, &rx_desc->data[0], imm_data_len); 1249b8d26b3bSNicholas Bellinger 1250b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1251b8d26b3bSNicholas Bellinger 1252b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1253b8d26b3bSNicholas Bellinger 1254b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1255b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1256b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1257b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1258b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1259b8d26b3bSNicholas Bellinger } 1260b8d26b3bSNicholas Bellinger 1261b8d26b3bSNicholas Bellinger sequence_cmd: 1262561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1263b8d26b3bSNicholas Bellinger 1264b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 1265b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 12666cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 12676cc44a6fSNicholas Bellinger target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); 1268b8d26b3bSNicholas Bellinger 1269b8d26b3bSNicholas Bellinger return 0; 1270b8d26b3bSNicholas Bellinger } 1271b8d26b3bSNicholas Bellinger 1272b8d26b3bSNicholas Bellinger static int 1273b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1274b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1275b8d26b3bSNicholas Bellinger { 1276b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1277b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1278b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1279b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1280b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1281b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1282b8d26b3bSNicholas Bellinger 1283b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1284b8d26b3bSNicholas Bellinger if (rc < 0) 1285b8d26b3bSNicholas Bellinger return rc; 1286b8d26b3bSNicholas Bellinger else if (!cmd) 1287b8d26b3bSNicholas Bellinger return 0; 1288b8d26b3bSNicholas Bellinger /* 1289b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1290b8d26b3bSNicholas Bellinger */ 1291b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 1292*24f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1293b8d26b3bSNicholas Bellinger dump_stack(); 1294b8d26b3bSNicholas Bellinger return -1; 1295b8d26b3bSNicholas Bellinger } 1296b8d26b3bSNicholas Bellinger 1297*24f412ddSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1298b8d26b3bSNicholas Bellinger unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1299b8d26b3bSNicholas Bellinger 1300b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1301b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1302b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1303b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1304b8d26b3bSNicholas Bellinger /* 1305b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1306b8d26b3bSNicholas Bellinger */ 1307b8d26b3bSNicholas Bellinger if (page_off) { 1308*24f412ddSSagi Grimberg isert_err("Received unexpected non-page aligned data payload\n"); 1309b8d26b3bSNicholas Bellinger dump_stack(); 1310b8d26b3bSNicholas Bellinger return -1; 1311b8d26b3bSNicholas Bellinger } 1312*24f412ddSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1313b8d26b3bSNicholas Bellinger sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1314b8d26b3bSNicholas Bellinger 1315b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1316b8d26b3bSNicholas Bellinger unsol_data_len); 1317b8d26b3bSNicholas Bellinger 1318b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1319b8d26b3bSNicholas Bellinger if (rc < 0) 1320b8d26b3bSNicholas Bellinger return rc; 1321b8d26b3bSNicholas Bellinger 1322b8d26b3bSNicholas Bellinger return 0; 1323b8d26b3bSNicholas Bellinger } 1324b8d26b3bSNicholas Bellinger 1325b8d26b3bSNicholas Bellinger static int 1326778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1327d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1328d703ce2fSNicholas Bellinger unsigned char *buf) 1329778de368SNicholas Bellinger { 1330778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1331778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1332778de368SNicholas Bellinger int rc; 1333778de368SNicholas Bellinger 1334778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1335778de368SNicholas Bellinger if (rc < 0) 1336778de368SNicholas Bellinger return rc; 1337778de368SNicholas Bellinger /* 1338778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1339778de368SNicholas Bellinger */ 1340778de368SNicholas Bellinger 1341778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1342778de368SNicholas Bellinger } 1343778de368SNicholas Bellinger 1344778de368SNicholas Bellinger static int 1345adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1346d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1347d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1348adb54c29SNicholas Bellinger { 1349adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1350adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1351adb54c29SNicholas Bellinger int rc; 1352adb54c29SNicholas Bellinger unsigned char *text_in; 1353adb54c29SNicholas Bellinger 1354adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1355adb54c29SNicholas Bellinger if (rc < 0) 1356adb54c29SNicholas Bellinger return rc; 1357adb54c29SNicholas Bellinger 1358adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 1359adb54c29SNicholas Bellinger if (!text_in) { 1360*24f412ddSSagi Grimberg isert_err("Unable to allocate text_in of payload_length: %u\n", 1361adb54c29SNicholas Bellinger payload_length); 1362adb54c29SNicholas Bellinger return -ENOMEM; 1363adb54c29SNicholas Bellinger } 1364adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1365adb54c29SNicholas Bellinger 1366adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1367adb54c29SNicholas Bellinger 1368adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1369adb54c29SNicholas Bellinger } 1370adb54c29SNicholas Bellinger 1371adb54c29SNicholas Bellinger static int 1372b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1373b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1374b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1375b8d26b3bSNicholas Bellinger { 1376b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1377b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1378ca40d24eSNicholas Bellinger struct iscsi_session *sess = conn->sess; 1379b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1380b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1381b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1382b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1383b8d26b3bSNicholas Bellinger 1384ca40d24eSNicholas Bellinger if (sess->sess_ops->SessionType && 1385ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1386*24f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1387ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1388ca40d24eSNicholas Bellinger return 0; 1389ca40d24eSNicholas Bellinger } 1390ca40d24eSNicholas Bellinger 1391b8d26b3bSNicholas Bellinger switch (opcode) { 1392b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1393676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1394b8d26b3bSNicholas Bellinger if (!cmd) 1395b8d26b3bSNicholas Bellinger break; 1396b8d26b3bSNicholas Bellinger 1397d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1398b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1399b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1400b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1401b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1402b8d26b3bSNicholas Bellinger 1403d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1404b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1405b8d26b3bSNicholas Bellinger break; 1406b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1407676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1408b8d26b3bSNicholas Bellinger if (!cmd) 1409b8d26b3bSNicholas Bellinger break; 1410b8d26b3bSNicholas Bellinger 1411d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1412d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1413778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1414b8d26b3bSNicholas Bellinger break; 1415b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1416b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1417b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1418b8d26b3bSNicholas Bellinger break; 1419b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1420676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1421b8d26b3bSNicholas Bellinger if (!cmd) 1422b8d26b3bSNicholas Bellinger break; 1423b8d26b3bSNicholas Bellinger 1424b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1425b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1426b8d26b3bSNicholas Bellinger break; 1427b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 1428676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1429b8d26b3bSNicholas Bellinger if (!cmd) 1430b8d26b3bSNicholas Bellinger break; 1431b8d26b3bSNicholas Bellinger 1432b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1433b8d26b3bSNicholas Bellinger break; 1434adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1435676687c6SNicholas Bellinger cmd = isert_allocate_cmd(conn); 1436adb54c29SNicholas Bellinger if (!cmd) 1437adb54c29SNicholas Bellinger break; 1438adb54c29SNicholas Bellinger 1439d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1440d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1441adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1442adb54c29SNicholas Bellinger break; 1443b8d26b3bSNicholas Bellinger default: 1444*24f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1445b8d26b3bSNicholas Bellinger dump_stack(); 1446b8d26b3bSNicholas Bellinger break; 1447b8d26b3bSNicholas Bellinger } 1448b8d26b3bSNicholas Bellinger 1449b8d26b3bSNicholas Bellinger return ret; 1450b8d26b3bSNicholas Bellinger } 1451b8d26b3bSNicholas Bellinger 1452b8d26b3bSNicholas Bellinger static void 1453b8d26b3bSNicholas Bellinger isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1454b8d26b3bSNicholas Bellinger { 1455b8d26b3bSNicholas Bellinger struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1456b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1457b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1458b8d26b3bSNicholas Bellinger int rc; 1459b8d26b3bSNicholas Bellinger 1460b8d26b3bSNicholas Bellinger switch (iser_hdr->flags & 0xF0) { 1461b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1462b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_RSV) { 1463b8d26b3bSNicholas Bellinger read_stag = be32_to_cpu(iser_hdr->read_stag); 1464b8d26b3bSNicholas Bellinger read_va = be64_to_cpu(iser_hdr->read_va); 1465*24f412ddSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1466b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1467b8d26b3bSNicholas Bellinger } 1468b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_WSV) { 1469b8d26b3bSNicholas Bellinger write_stag = be32_to_cpu(iser_hdr->write_stag); 1470b8d26b3bSNicholas Bellinger write_va = be64_to_cpu(iser_hdr->write_va); 1471*24f412ddSSagi Grimberg isert_dbg("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1472b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1473b8d26b3bSNicholas Bellinger } 1474b8d26b3bSNicholas Bellinger 1475*24f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1476b8d26b3bSNicholas Bellinger break; 1477b8d26b3bSNicholas Bellinger case ISER_HELLO: 1478*24f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1479b8d26b3bSNicholas Bellinger break; 1480b8d26b3bSNicholas Bellinger default: 1481*24f412ddSSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1482b8d26b3bSNicholas Bellinger break; 1483b8d26b3bSNicholas Bellinger } 1484b8d26b3bSNicholas Bellinger 1485b8d26b3bSNicholas Bellinger rc = isert_rx_opcode(isert_conn, rx_desc, 1486b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 1487b8d26b3bSNicholas Bellinger } 1488b8d26b3bSNicholas Bellinger 1489b8d26b3bSNicholas Bellinger static void 1490b8d26b3bSNicholas Bellinger isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 14914a295baeSSagi Grimberg u32 xfer_len) 1492b8d26b3bSNicholas Bellinger { 1493b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1494b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr; 1495b8d26b3bSNicholas Bellinger u64 rx_dma; 1496b8d26b3bSNicholas Bellinger int rx_buflen, outstanding; 1497b8d26b3bSNicholas Bellinger 1498b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) { 1499b8d26b3bSNicholas Bellinger rx_dma = isert_conn->login_req_dma; 1500b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_LOGIN_SIZE; 1501*24f412ddSSagi Grimberg isert_dbg("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1502b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1503b8d26b3bSNicholas Bellinger } else { 1504b8d26b3bSNicholas Bellinger rx_dma = desc->dma_addr; 1505b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_PAYLOAD_SIZE; 1506*24f412ddSSagi Grimberg isert_dbg("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1507b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1508b8d26b3bSNicholas Bellinger } 1509b8d26b3bSNicholas Bellinger 1510b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1511b8d26b3bSNicholas Bellinger 1512b8d26b3bSNicholas Bellinger hdr = &desc->iscsi_header; 1513*24f412ddSSagi Grimberg isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1514b8d26b3bSNicholas Bellinger hdr->opcode, hdr->itt, hdr->flags, 1515b8d26b3bSNicholas Bellinger (int)(xfer_len - ISER_HEADERS_LEN)); 1516b8d26b3bSNicholas Bellinger 15172371e5daSSagi Grimberg if ((char *)desc == isert_conn->login_req_buf) { 15182371e5daSSagi Grimberg isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 15192371e5daSSagi Grimberg if (isert_conn->conn) { 15202371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 15212371e5daSSagi Grimberg 15222371e5daSSagi Grimberg if (login && !login->first_request) 15232371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 15242371e5daSSagi Grimberg } 15252371e5daSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 15262371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 15272371e5daSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 15282371e5daSSagi Grimberg } else { 1529b8d26b3bSNicholas Bellinger isert_rx_do_work(desc, isert_conn); 15302371e5daSSagi Grimberg } 1531b8d26b3bSNicholas Bellinger 1532b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1533b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 1534b8d26b3bSNicholas Bellinger 1535b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1536*24f412ddSSagi Grimberg isert_dbg("iSERT: Decremented post_recv_buf_count: %d\n", 1537b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count); 1538b8d26b3bSNicholas Bellinger 1539b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) 1540b8d26b3bSNicholas Bellinger return; 1541b8d26b3bSNicholas Bellinger 1542b8d26b3bSNicholas Bellinger outstanding = isert_conn->post_recv_buf_count; 1543b8d26b3bSNicholas Bellinger if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1544b8d26b3bSNicholas Bellinger int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1545b8d26b3bSNicholas Bellinger ISERT_MIN_POSTED_RX); 1546b8d26b3bSNicholas Bellinger err = isert_post_recv(isert_conn, count); 1547b8d26b3bSNicholas Bellinger if (err) { 1548*24f412ddSSagi Grimberg isert_err("isert_post_recv() count: %d failed, %d\n", 1549b8d26b3bSNicholas Bellinger count, err); 1550b8d26b3bSNicholas Bellinger } 1551b8d26b3bSNicholas Bellinger } 1552b8d26b3bSNicholas Bellinger } 1553b8d26b3bSNicholas Bellinger 1554e3d7e4c3SSagi Grimberg static int 1555e3d7e4c3SSagi Grimberg isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1556e3d7e4c3SSagi Grimberg struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1557e3d7e4c3SSagi Grimberg enum iser_ib_op_code op, struct isert_data_buf *data) 1558e3d7e4c3SSagi Grimberg { 1559e3d7e4c3SSagi Grimberg struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1560e3d7e4c3SSagi Grimberg 1561e3d7e4c3SSagi Grimberg data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1562e3d7e4c3SSagi Grimberg DMA_TO_DEVICE : DMA_FROM_DEVICE; 1563e3d7e4c3SSagi Grimberg 1564e3d7e4c3SSagi Grimberg data->len = length - offset; 1565e3d7e4c3SSagi Grimberg data->offset = offset; 1566e3d7e4c3SSagi Grimberg data->sg_off = data->offset / PAGE_SIZE; 1567e3d7e4c3SSagi Grimberg 1568e3d7e4c3SSagi Grimberg data->sg = &sg[data->sg_off]; 1569e3d7e4c3SSagi Grimberg data->nents = min_t(unsigned int, nents - data->sg_off, 1570e3d7e4c3SSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 1571e3d7e4c3SSagi Grimberg data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1572e3d7e4c3SSagi Grimberg PAGE_SIZE); 1573e3d7e4c3SSagi Grimberg 1574e3d7e4c3SSagi Grimberg data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1575e3d7e4c3SSagi Grimberg data->dma_dir); 1576e3d7e4c3SSagi Grimberg if (unlikely(!data->dma_nents)) { 1577*24f412ddSSagi Grimberg isert_err("Cmd: unable to dma map SGs %p\n", sg); 1578e3d7e4c3SSagi Grimberg return -EINVAL; 1579e3d7e4c3SSagi Grimberg } 1580e3d7e4c3SSagi Grimberg 1581*24f412ddSSagi Grimberg isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1582e3d7e4c3SSagi Grimberg isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1583e3d7e4c3SSagi Grimberg 1584e3d7e4c3SSagi Grimberg return 0; 1585e3d7e4c3SSagi Grimberg } 1586e3d7e4c3SSagi Grimberg 1587e3d7e4c3SSagi Grimberg static void 1588e3d7e4c3SSagi Grimberg isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1589e3d7e4c3SSagi Grimberg { 1590e3d7e4c3SSagi Grimberg struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1591e3d7e4c3SSagi Grimberg 1592e3d7e4c3SSagi Grimberg ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1593e3d7e4c3SSagi Grimberg memset(data, 0, sizeof(*data)); 1594e3d7e4c3SSagi Grimberg } 1595e3d7e4c3SSagi Grimberg 1596e3d7e4c3SSagi Grimberg 1597e3d7e4c3SSagi Grimberg 1598b8d26b3bSNicholas Bellinger static void 1599b8d26b3bSNicholas Bellinger isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1600b8d26b3bSNicholas Bellinger { 1601b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1602b8d26b3bSNicholas Bellinger 1603*24f412ddSSagi Grimberg isert_dbg("isert_unmap_cmd: %p\n", isert_cmd); 1604e3d7e4c3SSagi Grimberg 1605e3d7e4c3SSagi Grimberg if (wr->data.sg) { 1606*24f412ddSSagi Grimberg isert_dbg("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1607e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 1608b8d26b3bSNicholas Bellinger } 1609b8d26b3bSNicholas Bellinger 161090ecc6e2SVu Pham if (wr->send_wr) { 1611*24f412ddSSagi Grimberg isert_dbg("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1612b8d26b3bSNicholas Bellinger kfree(wr->send_wr); 1613b8d26b3bSNicholas Bellinger wr->send_wr = NULL; 161490ecc6e2SVu Pham } 1615b8d26b3bSNicholas Bellinger 161690ecc6e2SVu Pham if (wr->ib_sge) { 1617*24f412ddSSagi Grimberg isert_dbg("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 161890ecc6e2SVu Pham kfree(wr->ib_sge); 161990ecc6e2SVu Pham wr->ib_sge = NULL; 162090ecc6e2SVu Pham } 1621b8d26b3bSNicholas Bellinger } 1622b8d26b3bSNicholas Bellinger 1623b8d26b3bSNicholas Bellinger static void 1624a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 162559464ef4SVu Pham { 162659464ef4SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 162759464ef4SVu Pham LIST_HEAD(unmap_list); 162859464ef4SVu Pham 1629*24f412ddSSagi Grimberg isert_dbg("unreg_fastreg_cmd: %p\n", isert_cmd); 163059464ef4SVu Pham 163159464ef4SVu Pham if (wr->fr_desc) { 1632*24f412ddSSagi Grimberg isert_dbg("unreg_fastreg_cmd: %p free fr_desc %p\n", 163359464ef4SVu Pham isert_cmd, wr->fr_desc); 16349e961ae7SSagi Grimberg if (wr->fr_desc->ind & ISERT_PROTECTED) { 16359e961ae7SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 16369e961ae7SSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 16379e961ae7SSagi Grimberg } 163859464ef4SVu Pham spin_lock_bh(&isert_conn->conn_lock); 1639a3a5a826SSagi Grimberg list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 164059464ef4SVu Pham spin_unlock_bh(&isert_conn->conn_lock); 164159464ef4SVu Pham wr->fr_desc = NULL; 164259464ef4SVu Pham } 164359464ef4SVu Pham 1644e3d7e4c3SSagi Grimberg if (wr->data.sg) { 1645*24f412ddSSagi Grimberg isert_dbg("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1646e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 164759464ef4SVu Pham } 164859464ef4SVu Pham 164959464ef4SVu Pham wr->ib_sge = NULL; 165059464ef4SVu Pham wr->send_wr = NULL; 165159464ef4SVu Pham } 165259464ef4SVu Pham 165359464ef4SVu Pham static void 165403e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1655b8d26b3bSNicholas Bellinger { 1656d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1657b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1658186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1659d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 1660b8d26b3bSNicholas Bellinger 1661*24f412ddSSagi Grimberg isert_dbg("Entering isert_put_cmd: %p\n", isert_cmd); 1662b8d26b3bSNicholas Bellinger 1663b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1664b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1665b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1666b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 16675159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1668b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1669b8d26b3bSNicholas Bellinger 167003e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1671b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 167203e7848aSNicholas Bellinger /* 167303e7848aSNicholas Bellinger * Check for special case during comp_err where 167403e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 167503e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 167603e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 167703e7848aSNicholas Bellinger */ 167803e7848aSNicholas Bellinger if (comp_err && 167903e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 168003e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 168103e7848aSNicholas Bellinger 168203e7848aSNicholas Bellinger target_put_sess_cmd(se_cmd->se_sess, se_cmd); 168303e7848aSNicholas Bellinger } 168403e7848aSNicholas Bellinger } 1685b8d26b3bSNicholas Bellinger 1686d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1687186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1688186a9647SNicholas Bellinger break; 1689b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1690186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1691186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 16925159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1693186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1694186a9647SNicholas Bellinger 1695b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1696b8d26b3bSNicholas Bellinger break; 1697b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1698b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1699adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1700b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1701b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17025159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1703b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1704b8d26b3bSNicholas Bellinger 1705b8d26b3bSNicholas Bellinger /* 1706b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1707b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1708b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1709b8d26b3bSNicholas Bellinger */ 1710b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 1711*24f412ddSSagi Grimberg isert_dbg("Calling transport_generic_free_cmd from" 17123df8f68aSNicholas Bellinger " isert_put_cmd for 0x%02x\n", 17133df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1714b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1715b8d26b3bSNicholas Bellinger break; 1716b8d26b3bSNicholas Bellinger } 1717b8d26b3bSNicholas Bellinger /* 1718b8d26b3bSNicholas Bellinger * Fall-through 1719b8d26b3bSNicholas Bellinger */ 1720b8d26b3bSNicholas Bellinger default: 1721d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1722b8d26b3bSNicholas Bellinger break; 1723b8d26b3bSNicholas Bellinger } 1724b8d26b3bSNicholas Bellinger } 1725b8d26b3bSNicholas Bellinger 1726b8d26b3bSNicholas Bellinger static void 1727b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1728b8d26b3bSNicholas Bellinger { 1729b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 1730*24f412ddSSagi Grimberg isert_dbg("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1731b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1732b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1733b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1734b8d26b3bSNicholas Bellinger } 1735b8d26b3bSNicholas Bellinger } 1736b8d26b3bSNicholas Bellinger 1737b8d26b3bSNicholas Bellinger static void 1738b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 173903e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1740b8d26b3bSNicholas Bellinger { 1741dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 1742*24f412ddSSagi Grimberg isert_dbg("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1743dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1744dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1745dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1746b8d26b3bSNicholas Bellinger } 1747b8d26b3bSNicholas Bellinger 1748b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 174903e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1750b8d26b3bSNicholas Bellinger } 1751b8d26b3bSNicholas Bellinger 175296b7973eSSagi Grimberg static int 175396b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1754f93f3a70SSagi Grimberg { 17559e961ae7SSagi Grimberg struct ib_mr_status mr_status; 175696b7973eSSagi Grimberg int ret; 1757f93f3a70SSagi Grimberg 175896b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 17599e961ae7SSagi Grimberg if (ret) { 1760*24f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 17619e961ae7SSagi Grimberg goto fail_mr_status; 17629e961ae7SSagi Grimberg } 176396b7973eSSagi Grimberg 17649e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 176596b7973eSSagi Grimberg u64 sec_offset_err; 17669e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 17679e961ae7SSagi Grimberg 17689e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 17699e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 17709e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 17719e961ae7SSagi Grimberg break; 17729e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 17739e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 17749e961ae7SSagi Grimberg break; 17759e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 17769e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 17779e961ae7SSagi Grimberg break; 17789e961ae7SSagi Grimberg } 177996b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 178096b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 178196b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 17829e961ae7SSagi Grimberg 1783*24f412ddSSagi Grimberg isert_err("isert: PI error found type %d at sector 0x%llx " 17849e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 17859e961ae7SSagi Grimberg mr_status.sig_err.err_type, 17869e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 17879e961ae7SSagi Grimberg mr_status.sig_err.expected, 17889e961ae7SSagi Grimberg mr_status.sig_err.actual); 17899e961ae7SSagi Grimberg ret = 1; 17909e961ae7SSagi Grimberg } 17919e961ae7SSagi Grimberg 17929e961ae7SSagi Grimberg fail_mr_status: 179396b7973eSSagi Grimberg return ret; 179496b7973eSSagi Grimberg } 179596b7973eSSagi Grimberg 179696b7973eSSagi Grimberg static void 179796b7973eSSagi Grimberg isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 179896b7973eSSagi Grimberg struct isert_cmd *isert_cmd) 179996b7973eSSagi Grimberg { 180096b7973eSSagi Grimberg struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 180196b7973eSSagi Grimberg struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 180296b7973eSSagi Grimberg struct se_cmd *se_cmd = &cmd->se_cmd; 180396b7973eSSagi Grimberg struct isert_conn *isert_conn = isert_cmd->conn; 180496b7973eSSagi Grimberg struct isert_device *device = isert_conn->conn_device; 180596b7973eSSagi Grimberg int ret = 0; 180696b7973eSSagi Grimberg 180796b7973eSSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 180896b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 180996b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 181096b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 181196b7973eSSagi Grimberg } 181296b7973eSSagi Grimberg 1813f93f3a70SSagi Grimberg device->unreg_rdma_mem(isert_cmd, isert_conn); 1814897bb2c9SSagi Grimberg wr->send_wr_num = 0; 18159e961ae7SSagi Grimberg if (ret) 18169e961ae7SSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 18179e961ae7SSagi Grimberg se_cmd->pi_err, 0); 18189e961ae7SSagi Grimberg else 1819f93f3a70SSagi Grimberg isert_put_response(isert_conn->conn, cmd); 1820f93f3a70SSagi Grimberg } 1821f93f3a70SSagi Grimberg 1822f93f3a70SSagi Grimberg static void 1823b8d26b3bSNicholas Bellinger isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1824b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd) 1825b8d26b3bSNicholas Bellinger { 1826b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1827d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1828b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 182990ecc6e2SVu Pham struct isert_conn *isert_conn = isert_cmd->conn; 1830d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 18315bac4b1aSSagi Grimberg int ret = 0; 1832b8d26b3bSNicholas Bellinger 18339e961ae7SSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 183496b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 183596b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 183696b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 18379e961ae7SSagi Grimberg } 18389e961ae7SSagi Grimberg 1839b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1840d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1841e3d7e4c3SSagi Grimberg cmd->write_data_done = wr->data.len; 1842b6b87a1dSNicholas Bellinger wr->send_wr_num = 0; 1843b8d26b3bSNicholas Bellinger 1844*24f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1845b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1846b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1847b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1848b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1849b8d26b3bSNicholas Bellinger 18505bac4b1aSSagi Grimberg if (ret) 18515bac4b1aSSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 18525bac4b1aSSagi Grimberg se_cmd->pi_err, 0); 18535bac4b1aSSagi Grimberg else 1854b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1855b8d26b3bSNicholas Bellinger } 1856b8d26b3bSNicholas Bellinger 1857b8d26b3bSNicholas Bellinger static void 1858b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1859b8d26b3bSNicholas Bellinger { 1860b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1861b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1862b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1863b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1864d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1865b8d26b3bSNicholas Bellinger 1866b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1867b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1868*24f412ddSSagi Grimberg isert_dbg("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1869b8d26b3bSNicholas Bellinger 1870b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1871b8d26b3bSNicholas Bellinger 1872b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 187303e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1874b8d26b3bSNicholas Bellinger break; 1875b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 1876*24f412ddSSagi Grimberg isert_dbg("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1877b8d26b3bSNicholas Bellinger 1878b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 187903e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 18803df8f68aSNicholas Bellinger break; 1881b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1882*24f412ddSSagi Grimberg isert_dbg("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 18839d49f5e2SSagi Grimberg 1884b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1885b8d26b3bSNicholas Bellinger break; 1886adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 1887adb54c29SNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 188803e7848aSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1889adb54c29SNicholas Bellinger break; 1890b8d26b3bSNicholas Bellinger default: 1891*24f412ddSSagi Grimberg isert_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1892b8d26b3bSNicholas Bellinger dump_stack(); 1893b8d26b3bSNicholas Bellinger break; 1894b8d26b3bSNicholas Bellinger } 1895b8d26b3bSNicholas Bellinger } 1896b8d26b3bSNicholas Bellinger 1897b8d26b3bSNicholas Bellinger static void 1898b8d26b3bSNicholas Bellinger isert_response_completion(struct iser_tx_desc *tx_desc, 1899b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1900b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn, 1901b8d26b3bSNicholas Bellinger struct ib_device *ib_dev) 1902b8d26b3bSNicholas Bellinger { 1903d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1904b8d26b3bSNicholas Bellinger 1905b8d26b3bSNicholas Bellinger if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 19063df8f68aSNicholas Bellinger cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1907adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_REJECT || 1908adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_TEXTRSP) { 1909b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1910b8d26b3bSNicholas Bellinger 1911b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1912b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1913b8d26b3bSNicholas Bellinger return; 1914b8d26b3bSNicholas Bellinger } 1915897bb2c9SSagi Grimberg 1916b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 191703e7848aSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1918b8d26b3bSNicholas Bellinger } 1919b8d26b3bSNicholas Bellinger 1920b8d26b3bSNicholas Bellinger static void 192168a86deeSSagi Grimberg isert_send_completion(struct iser_tx_desc *tx_desc, 1922b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 1923b8d26b3bSNicholas Bellinger { 1924b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1925b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1926b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr; 1927b8d26b3bSNicholas Bellinger 1928b8d26b3bSNicholas Bellinger if (!isert_cmd) { 1929b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1930b8d26b3bSNicholas Bellinger return; 1931b8d26b3bSNicholas Bellinger } 1932b8d26b3bSNicholas Bellinger wr = &isert_cmd->rdma_wr; 1933b8d26b3bSNicholas Bellinger 1934b8d26b3bSNicholas Bellinger switch (wr->iser_ib_op) { 1935b8d26b3bSNicholas Bellinger case ISER_IB_RECV: 1936*24f412ddSSagi Grimberg isert_err("isert_send_completion: Got ISER_IB_RECV\n"); 1937b8d26b3bSNicholas Bellinger dump_stack(); 1938b8d26b3bSNicholas Bellinger break; 1939b8d26b3bSNicholas Bellinger case ISER_IB_SEND: 1940*24f412ddSSagi Grimberg isert_dbg("isert_send_completion: Got ISER_IB_SEND\n"); 1941b8d26b3bSNicholas Bellinger isert_response_completion(tx_desc, isert_cmd, 1942b8d26b3bSNicholas Bellinger isert_conn, ib_dev); 1943b8d26b3bSNicholas Bellinger break; 1944b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_WRITE: 1945*24f412ddSSagi Grimberg isert_dbg("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1946f93f3a70SSagi Grimberg isert_completion_rdma_write(tx_desc, isert_cmd); 1947b8d26b3bSNicholas Bellinger break; 1948b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_READ: 1949*24f412ddSSagi Grimberg isert_dbg("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1950b8d26b3bSNicholas Bellinger 1951b8d26b3bSNicholas Bellinger isert_completion_rdma_read(tx_desc, isert_cmd); 1952b8d26b3bSNicholas Bellinger break; 1953b8d26b3bSNicholas Bellinger default: 1954*24f412ddSSagi Grimberg isert_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1955b8d26b3bSNicholas Bellinger dump_stack(); 1956b8d26b3bSNicholas Bellinger break; 1957b8d26b3bSNicholas Bellinger } 1958b8d26b3bSNicholas Bellinger } 1959b8d26b3bSNicholas Bellinger 19606f0fae3dSSagi Grimberg /** 19616f0fae3dSSagi Grimberg * is_isert_tx_desc() - Indicate if the completion wr_id 19626f0fae3dSSagi Grimberg * is a TX descriptor or not. 19636f0fae3dSSagi Grimberg * @isert_conn: iser connection 19646f0fae3dSSagi Grimberg * @wr_id: completion WR identifier 19656f0fae3dSSagi Grimberg * 19666f0fae3dSSagi Grimberg * Since we cannot rely on wc opcode in FLUSH errors 19676f0fae3dSSagi Grimberg * we must work around it by checking if the wr_id address 19686f0fae3dSSagi Grimberg * falls in the iser connection rx_descs buffer. If so 19696f0fae3dSSagi Grimberg * it is an RX descriptor, otherwize it is a TX. 19706f0fae3dSSagi Grimberg */ 19716f0fae3dSSagi Grimberg static inline bool 19726f0fae3dSSagi Grimberg is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 1973b8d26b3bSNicholas Bellinger { 19746f0fae3dSSagi Grimberg void *start = isert_conn->conn_rx_descs; 19756f0fae3dSSagi Grimberg int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->conn_rx_descs); 19766f0fae3dSSagi Grimberg 19776f0fae3dSSagi Grimberg if (wr_id >= start && wr_id < start + len) 19786f0fae3dSSagi Grimberg return false; 19796f0fae3dSSagi Grimberg 19806f0fae3dSSagi Grimberg return true; 19816f0fae3dSSagi Grimberg } 19826f0fae3dSSagi Grimberg 19836f0fae3dSSagi Grimberg static void 19846f0fae3dSSagi Grimberg isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 19856f0fae3dSSagi Grimberg { 1986bdf20e72SSagi Grimberg if (wc->wr_id == ISER_BEACON_WRID) { 1987*24f412ddSSagi Grimberg isert_info("conn %p completing conn_wait_comp_err\n", 1988bdf20e72SSagi Grimberg isert_conn); 1989bdf20e72SSagi Grimberg complete(&isert_conn->conn_wait_comp_err); 1990bdf20e72SSagi Grimberg } else 19916f0fae3dSSagi Grimberg if (is_isert_tx_desc(isert_conn, (void *)wc->wr_id)) { 1992b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1993df43debdSSagi Grimberg struct isert_cmd *isert_cmd; 19946f0fae3dSSagi Grimberg struct iser_tx_desc *desc; 1995b8d26b3bSNicholas Bellinger 19966f0fae3dSSagi Grimberg desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 19976f0fae3dSSagi Grimberg isert_cmd = desc->isert_cmd; 1998b8d26b3bSNicholas Bellinger if (!isert_cmd) 1999df43debdSSagi Grimberg isert_unmap_tx_desc(desc, ib_dev); 2000b8d26b3bSNicholas Bellinger else 2001df43debdSSagi Grimberg isert_completion_put(desc, isert_cmd, ib_dev, true); 2002df43debdSSagi Grimberg } else { 2003df43debdSSagi Grimberg isert_conn->post_recv_buf_count--; 2004bdf20e72SSagi Grimberg if (!isert_conn->post_recv_buf_count) 20059d49f5e2SSagi Grimberg iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2006b8d26b3bSNicholas Bellinger } 2007df43debdSSagi Grimberg } 2008b8d26b3bSNicholas Bellinger 2009b8d26b3bSNicholas Bellinger static void 20106f0fae3dSSagi Grimberg isert_handle_wc(struct ib_wc *wc) 2011b8d26b3bSNicholas Bellinger { 2012b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2013b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc; 20146f0fae3dSSagi Grimberg struct iser_rx_desc *rx_desc; 2015b8d26b3bSNicholas Bellinger 20166f0fae3dSSagi Grimberg isert_conn = wc->qp->qp_context; 20176f0fae3dSSagi Grimberg if (likely(wc->status == IB_WC_SUCCESS)) { 20186f0fae3dSSagi Grimberg if (wc->opcode == IB_WC_RECV) { 20196f0fae3dSSagi Grimberg rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 20206f0fae3dSSagi Grimberg isert_rx_completion(rx_desc, isert_conn, wc->byte_len); 2021b8d26b3bSNicholas Bellinger } else { 20226f0fae3dSSagi Grimberg tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 20236f0fae3dSSagi Grimberg isert_send_completion(tx_desc, isert_conn); 2024b8d26b3bSNicholas Bellinger } 20256f0fae3dSSagi Grimberg } else { 20266f0fae3dSSagi Grimberg if (wc->status != IB_WC_WR_FLUSH_ERR) 2027*24f412ddSSagi Grimberg isert_err("wr id %llx status %d vend_err %x\n", 20286f0fae3dSSagi Grimberg wc->wr_id, wc->status, wc->vendor_err); 20296f0fae3dSSagi Grimberg else 2030*24f412ddSSagi Grimberg isert_dbg("flush error: wr id %llx\n", wc->wr_id); 2031b8d26b3bSNicholas Bellinger 20326f0fae3dSSagi Grimberg if (wc->wr_id != ISER_FASTREG_LI_WRID) 20336f0fae3dSSagi Grimberg isert_cq_comp_err(isert_conn, wc); 20346f0fae3dSSagi Grimberg } 2035b8d26b3bSNicholas Bellinger } 2036b8d26b3bSNicholas Bellinger 2037b8d26b3bSNicholas Bellinger static void 20386f0fae3dSSagi Grimberg isert_cq_work(struct work_struct *work) 2039b8d26b3bSNicholas Bellinger { 204037d9fe80SSagi Grimberg enum { isert_poll_budget = 65536 }; 20414a295baeSSagi Grimberg struct isert_comp *comp = container_of(work, struct isert_comp, 20426f0fae3dSSagi Grimberg work); 204336ea63b5SSagi Grimberg struct ib_wc *const wcs = comp->wcs; 204436ea63b5SSagi Grimberg int i, n, completed = 0; 2045b8d26b3bSNicholas Bellinger 204636ea63b5SSagi Grimberg while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 204736ea63b5SSagi Grimberg for (i = 0; i < n; i++) 204836ea63b5SSagi Grimberg isert_handle_wc(&wcs[i]); 2049b8d26b3bSNicholas Bellinger 205036ea63b5SSagi Grimberg completed += n; 205136ea63b5SSagi Grimberg if (completed >= isert_poll_budget) 205237d9fe80SSagi Grimberg break; 205337d9fe80SSagi Grimberg } 205437d9fe80SSagi Grimberg 20556f0fae3dSSagi Grimberg ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2056b8d26b3bSNicholas Bellinger } 2057b8d26b3bSNicholas Bellinger 2058b8d26b3bSNicholas Bellinger static void 20596f0fae3dSSagi Grimberg isert_cq_callback(struct ib_cq *cq, void *context) 2060b8d26b3bSNicholas Bellinger { 20614a295baeSSagi Grimberg struct isert_comp *comp = context; 2062b8d26b3bSNicholas Bellinger 20636f0fae3dSSagi Grimberg queue_work(isert_comp_wq, &comp->work); 2064b8d26b3bSNicholas Bellinger } 2065b8d26b3bSNicholas Bellinger 2066b8d26b3bSNicholas Bellinger static int 2067b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2068b8d26b3bSNicholas Bellinger { 2069b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 2070b8d26b3bSNicholas Bellinger int ret; 2071b8d26b3bSNicholas Bellinger 2072b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 2073b8d26b3bSNicholas Bellinger &wr_failed); 2074b8d26b3bSNicholas Bellinger if (ret) { 2075*24f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 2076b8d26b3bSNicholas Bellinger return ret; 2077b8d26b3bSNicholas Bellinger } 2078b8d26b3bSNicholas Bellinger return ret; 2079b8d26b3bSNicholas Bellinger } 2080b8d26b3bSNicholas Bellinger 2081b8d26b3bSNicholas Bellinger static int 2082b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2083b8d26b3bSNicholas Bellinger { 2084d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2085b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2086b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2087b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2088b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 2089b8d26b3bSNicholas Bellinger 2090b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2091b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2092b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2093b8d26b3bSNicholas Bellinger /* 2094b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 2095b8d26b3bSNicholas Bellinger */ 2096b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 2097b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2098b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2099b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2100b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2101dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 2102b8d26b3bSNicholas Bellinger 2103b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2104b8d26b3bSNicholas Bellinger cmd->sense_buffer); 2105b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2106b8d26b3bSNicholas Bellinger 2107b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2108b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2109dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2110b8d26b3bSNicholas Bellinger 2111dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2112dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 2113b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 2114b8d26b3bSNicholas Bellinger 2115dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 2116dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2117dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 2118b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 2119b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2120b8d26b3bSNicholas Bellinger } 2121b8d26b3bSNicholas Bellinger 212268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2123b8d26b3bSNicholas Bellinger 2124*24f412ddSSagi Grimberg isert_dbg("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2125b8d26b3bSNicholas Bellinger 2126b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2127b8d26b3bSNicholas Bellinger } 2128b8d26b3bSNicholas Bellinger 2129131e6abcSNicholas Bellinger static void 2130131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2131131e6abcSNicholas Bellinger { 2132131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2133131e6abcSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2134131e6abcSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 2135131e6abcSNicholas Bellinger 2136131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 2137131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 2138131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 2139131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 2140131e6abcSNicholas Bellinger 2141131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 2142131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 2143131e6abcSNicholas Bellinger 2144131e6abcSNicholas Bellinger device->unreg_rdma_mem(isert_cmd, isert_conn); 2145131e6abcSNicholas Bellinger } 2146131e6abcSNicholas Bellinger 2147e70beee7SNicholas Bellinger static enum target_prot_op 2148e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 2149e70beee7SNicholas Bellinger { 2150e70beee7SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2151e70beee7SNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 2152e70beee7SNicholas Bellinger 215323a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 215423a548eeSSagi Grimberg if (device->pi_capable) { 2155*24f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 215623a548eeSSagi Grimberg isert_conn->pi_support = true; 2157e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 215823a548eeSSagi Grimberg } 215923a548eeSSagi Grimberg } 216023a548eeSSagi Grimberg 2161*24f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 216223a548eeSSagi Grimberg isert_conn->pi_support = false; 2163e70beee7SNicholas Bellinger 2164e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 2165e70beee7SNicholas Bellinger } 2166e70beee7SNicholas Bellinger 2167b8d26b3bSNicholas Bellinger static int 2168b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2169b8d26b3bSNicholas Bellinger bool nopout_response) 2170b8d26b3bSNicholas Bellinger { 2171d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2172b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2173b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2174b8d26b3bSNicholas Bellinger 2175b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2176b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2177b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 2178b8d26b3bSNicholas Bellinger nopout_response); 2179b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 218068a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2181b8d26b3bSNicholas Bellinger 2182*24f412ddSSagi Grimberg isert_dbg("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2183b8d26b3bSNicholas Bellinger 2184b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2185b8d26b3bSNicholas Bellinger } 2186b8d26b3bSNicholas Bellinger 2187b8d26b3bSNicholas Bellinger static int 2188b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2189b8d26b3bSNicholas Bellinger { 2190d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2191b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2192b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2193b8d26b3bSNicholas Bellinger 2194b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2195b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2196b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2197b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 219868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2199b8d26b3bSNicholas Bellinger 2200*24f412ddSSagi Grimberg isert_dbg("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2201b8d26b3bSNicholas Bellinger 2202b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2203b8d26b3bSNicholas Bellinger } 2204b8d26b3bSNicholas Bellinger 2205b8d26b3bSNicholas Bellinger static int 2206b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2207b8d26b3bSNicholas Bellinger { 2208d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2209b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2210b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2211b8d26b3bSNicholas Bellinger 2212b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2213b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2214b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2215b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 221668a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2217b8d26b3bSNicholas Bellinger 2218*24f412ddSSagi Grimberg isert_dbg("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2219b8d26b3bSNicholas Bellinger 2220b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2221b8d26b3bSNicholas Bellinger } 2222b8d26b3bSNicholas Bellinger 2223b8d26b3bSNicholas Bellinger static int 2224b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2225b8d26b3bSNicholas Bellinger { 2226d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2227b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2228b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 22293df8f68aSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 22303df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 22313df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 22323df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2233b8d26b3bSNicholas Bellinger 2234b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 22353df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 2236b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 22373df8f68aSNicholas Bellinger 22383df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2239dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 22403df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 22413df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2242dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2243dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 22443df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 22453df8f68aSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 22463df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 22473df8f68aSNicholas Bellinger 224868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2249b8d26b3bSNicholas Bellinger 2250*24f412ddSSagi Grimberg isert_dbg("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2251b8d26b3bSNicholas Bellinger 2252b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2253b8d26b3bSNicholas Bellinger } 2254b8d26b3bSNicholas Bellinger 2255b8d26b3bSNicholas Bellinger static int 2256adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2257adb54c29SNicholas Bellinger { 2258d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2259adb54c29SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2260adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2261adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2262adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2263adb54c29SNicholas Bellinger u32 txt_rsp_len; 2264adb54c29SNicholas Bellinger int rc; 2265adb54c29SNicholas Bellinger 2266adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 226722c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2268adb54c29SNicholas Bellinger if (rc < 0) 2269adb54c29SNicholas Bellinger return rc; 2270adb54c29SNicholas Bellinger 2271adb54c29SNicholas Bellinger txt_rsp_len = rc; 2272adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2273adb54c29SNicholas Bellinger 2274adb54c29SNicholas Bellinger if (txt_rsp_len) { 2275adb54c29SNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2276adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2277adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2278adb54c29SNicholas Bellinger 2279adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2280adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2281adb54c29SNicholas Bellinger 2282adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2283adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2284adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 2285adb54c29SNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 2286adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2287adb54c29SNicholas Bellinger } 228868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2289adb54c29SNicholas Bellinger 2290*24f412ddSSagi Grimberg isert_dbg("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2291adb54c29SNicholas Bellinger 2292adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2293adb54c29SNicholas Bellinger } 2294adb54c29SNicholas Bellinger 2295adb54c29SNicholas Bellinger static int 2296b8d26b3bSNicholas Bellinger isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2297b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2298b8d26b3bSNicholas Bellinger u32 data_left, u32 offset) 2299b8d26b3bSNicholas Bellinger { 2300d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2301b8d26b3bSNicholas Bellinger struct scatterlist *sg_start, *tmp_sg; 2302b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2303b8d26b3bSNicholas Bellinger u32 sg_off, page_off; 2304b8d26b3bSNicholas Bellinger int i = 0, sg_nents; 2305b8d26b3bSNicholas Bellinger 2306b8d26b3bSNicholas Bellinger sg_off = offset / PAGE_SIZE; 2307b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2308b8d26b3bSNicholas Bellinger sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2309b8d26b3bSNicholas Bellinger page_off = offset % PAGE_SIZE; 2310b8d26b3bSNicholas Bellinger 2311b8d26b3bSNicholas Bellinger send_wr->sg_list = ib_sge; 2312b8d26b3bSNicholas Bellinger send_wr->num_sge = sg_nents; 2313b0a191e7SSagi Grimberg send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2314b8d26b3bSNicholas Bellinger /* 2315b8d26b3bSNicholas Bellinger * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2316b8d26b3bSNicholas Bellinger */ 2317b8d26b3bSNicholas Bellinger for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2318*24f412ddSSagi Grimberg isert_dbg("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2319b8d26b3bSNicholas Bellinger (unsigned long long)tmp_sg->dma_address, 2320b8d26b3bSNicholas Bellinger tmp_sg->length, page_off); 2321b8d26b3bSNicholas Bellinger 2322b8d26b3bSNicholas Bellinger ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2323b8d26b3bSNicholas Bellinger ib_sge->length = min_t(u32, data_left, 2324b8d26b3bSNicholas Bellinger ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2325b8d26b3bSNicholas Bellinger ib_sge->lkey = isert_conn->conn_mr->lkey; 2326b8d26b3bSNicholas Bellinger 2327*24f412ddSSagi Grimberg isert_dbg("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 232890ecc6e2SVu Pham ib_sge->addr, ib_sge->length, ib_sge->lkey); 2329b8d26b3bSNicholas Bellinger page_off = 0; 2330b8d26b3bSNicholas Bellinger data_left -= ib_sge->length; 2331b8d26b3bSNicholas Bellinger ib_sge++; 2332*24f412ddSSagi Grimberg isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2333b8d26b3bSNicholas Bellinger } 2334b8d26b3bSNicholas Bellinger 2335*24f412ddSSagi Grimberg isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2336b8d26b3bSNicholas Bellinger send_wr->sg_list, send_wr->num_sge); 2337b8d26b3bSNicholas Bellinger 2338b8d26b3bSNicholas Bellinger return sg_nents; 2339b8d26b3bSNicholas Bellinger } 2340b8d26b3bSNicholas Bellinger 2341b8d26b3bSNicholas Bellinger static int 234290ecc6e2SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 234390ecc6e2SVu Pham struct isert_rdma_wr *wr) 2344b8d26b3bSNicholas Bellinger { 2345b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2346d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2347b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2348e3d7e4c3SSagi Grimberg struct isert_data_buf *data = &wr->data; 234990ecc6e2SVu Pham struct ib_send_wr *send_wr; 2350b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge; 2351e3d7e4c3SSagi Grimberg u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2352e3d7e4c3SSagi Grimberg int ret = 0, i, ib_sge_cnt; 2353b8d26b3bSNicholas Bellinger 235490ecc6e2SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2355b8d26b3bSNicholas Bellinger 2356e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2357e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2358e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2359e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2360e3d7e4c3SSagi Grimberg if (ret) 2361e3d7e4c3SSagi Grimberg return ret; 2362b8d26b3bSNicholas Bellinger 2363e3d7e4c3SSagi Grimberg data_left = data->len; 2364e3d7e4c3SSagi Grimberg offset = data->offset; 2365b8d26b3bSNicholas Bellinger 2366e3d7e4c3SSagi Grimberg ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2367b8d26b3bSNicholas Bellinger if (!ib_sge) { 2368*24f412ddSSagi Grimberg isert_warn("Unable to allocate ib_sge\n"); 2369b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2370e3d7e4c3SSagi Grimberg goto unmap_cmd; 2371b8d26b3bSNicholas Bellinger } 237290ecc6e2SVu Pham wr->ib_sge = ib_sge; 2373b8d26b3bSNicholas Bellinger 2374e3d7e4c3SSagi Grimberg wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2375b8d26b3bSNicholas Bellinger wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2376b8d26b3bSNicholas Bellinger GFP_KERNEL); 2377b8d26b3bSNicholas Bellinger if (!wr->send_wr) { 2378*24f412ddSSagi Grimberg isert_dbg("Unable to allocate wr->send_wr\n"); 2379b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2380e3d7e4c3SSagi Grimberg goto unmap_cmd; 2381b8d26b3bSNicholas Bellinger } 2382b8d26b3bSNicholas Bellinger 2383b8d26b3bSNicholas Bellinger wr->isert_cmd = isert_cmd; 2384b8d26b3bSNicholas Bellinger rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2385b8d26b3bSNicholas Bellinger 2386b8d26b3bSNicholas Bellinger for (i = 0; i < wr->send_wr_num; i++) { 2387b8d26b3bSNicholas Bellinger send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2388b8d26b3bSNicholas Bellinger data_len = min(data_left, rdma_write_max); 2389b8d26b3bSNicholas Bellinger 2390b8d26b3bSNicholas Bellinger send_wr->send_flags = 0; 239190ecc6e2SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 239290ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 2393b8d26b3bSNicholas Bellinger send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2394b8d26b3bSNicholas Bellinger send_wr->wr.rdma.rkey = isert_cmd->read_stag; 239590ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 239690ecc6e2SVu Pham send_wr->next = &isert_cmd->tx_desc.send_wr; 239790ecc6e2SVu Pham else 239890ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 239990ecc6e2SVu Pham } else { 240090ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 240190ecc6e2SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 240290ecc6e2SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 240390ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 240490ecc6e2SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 240590ecc6e2SVu Pham else 240690ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 240790ecc6e2SVu Pham } 2408b8d26b3bSNicholas Bellinger 2409b8d26b3bSNicholas Bellinger ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2410b8d26b3bSNicholas Bellinger send_wr, data_len, offset); 2411b8d26b3bSNicholas Bellinger ib_sge += ib_sge_cnt; 2412b8d26b3bSNicholas Bellinger 2413b8d26b3bSNicholas Bellinger offset += data_len; 241490ecc6e2SVu Pham va_offset += data_len; 2415b8d26b3bSNicholas Bellinger data_left -= data_len; 2416b8d26b3bSNicholas Bellinger } 241790ecc6e2SVu Pham 241890ecc6e2SVu Pham return 0; 2419e3d7e4c3SSagi Grimberg unmap_cmd: 2420e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, data); 2421e3d7e4c3SSagi Grimberg 242290ecc6e2SVu Pham return ret; 242390ecc6e2SVu Pham } 242490ecc6e2SVu Pham 242590ecc6e2SVu Pham static int 242659464ef4SVu Pham isert_map_fr_pagelist(struct ib_device *ib_dev, 242759464ef4SVu Pham struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 242859464ef4SVu Pham { 242959464ef4SVu Pham u64 start_addr, end_addr, page, chunk_start = 0; 243059464ef4SVu Pham struct scatterlist *tmp_sg; 243159464ef4SVu Pham int i = 0, new_chunk, last_ent, n_pages; 243259464ef4SVu Pham 243359464ef4SVu Pham n_pages = 0; 243459464ef4SVu Pham new_chunk = 1; 243559464ef4SVu Pham last_ent = sg_nents - 1; 243659464ef4SVu Pham for_each_sg(sg_start, tmp_sg, sg_nents, i) { 243759464ef4SVu Pham start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 243859464ef4SVu Pham if (new_chunk) 243959464ef4SVu Pham chunk_start = start_addr; 244059464ef4SVu Pham end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 244159464ef4SVu Pham 2442*24f412ddSSagi Grimberg isert_dbg("SGL[%d] dma_addr: 0x%16llx len: %u\n", 244359464ef4SVu Pham i, (unsigned long long)tmp_sg->dma_address, 244459464ef4SVu Pham tmp_sg->length); 244559464ef4SVu Pham 244659464ef4SVu Pham if ((end_addr & ~PAGE_MASK) && i < last_ent) { 244759464ef4SVu Pham new_chunk = 0; 244859464ef4SVu Pham continue; 244959464ef4SVu Pham } 245059464ef4SVu Pham new_chunk = 1; 245159464ef4SVu Pham 245259464ef4SVu Pham page = chunk_start & PAGE_MASK; 245359464ef4SVu Pham do { 245459464ef4SVu Pham fr_pl[n_pages++] = page; 2455*24f412ddSSagi Grimberg isert_dbg("Mapped page_list[%d] page_addr: 0x%16llx\n", 245659464ef4SVu Pham n_pages - 1, page); 245759464ef4SVu Pham page += PAGE_SIZE; 245859464ef4SVu Pham } while (page < end_addr); 245959464ef4SVu Pham } 246059464ef4SVu Pham 246159464ef4SVu Pham return n_pages; 246259464ef4SVu Pham } 246359464ef4SVu Pham 246459464ef4SVu Pham static int 2465e3d7e4c3SSagi Grimberg isert_fast_reg_mr(struct isert_conn *isert_conn, 2466e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc, 2467e3d7e4c3SSagi Grimberg struct isert_data_buf *mem, 24689e961ae7SSagi Grimberg enum isert_indicator ind, 2469e3d7e4c3SSagi Grimberg struct ib_sge *sge) 247059464ef4SVu Pham { 247159464ef4SVu Pham struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 24729e961ae7SSagi Grimberg struct ib_mr *mr; 24739e961ae7SSagi Grimberg struct ib_fast_reg_page_list *frpl; 247459464ef4SVu Pham struct ib_send_wr fr_wr, inv_wr; 247559464ef4SVu Pham struct ib_send_wr *bad_wr, *wr = NULL; 24769bd626e7SSagi Grimberg int ret, pagelist_len; 24779bd626e7SSagi Grimberg u32 page_off; 247859464ef4SVu Pham u8 key; 247959464ef4SVu Pham 2480e3d7e4c3SSagi Grimberg if (mem->dma_nents == 1) { 2481e3d7e4c3SSagi Grimberg sge->lkey = isert_conn->conn_mr->lkey; 2482e3d7e4c3SSagi Grimberg sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2483e3d7e4c3SSagi Grimberg sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2484*24f412ddSSagi Grimberg isert_dbg("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 24859e961ae7SSagi Grimberg __func__, __LINE__, sge->addr, sge->length, 24869e961ae7SSagi Grimberg sge->lkey); 2487e3d7e4c3SSagi Grimberg return 0; 2488e3d7e4c3SSagi Grimberg } 2489e3d7e4c3SSagi Grimberg 24909e961ae7SSagi Grimberg if (ind == ISERT_DATA_KEY_VALID) { 24919e961ae7SSagi Grimberg /* Registering data buffer */ 24929e961ae7SSagi Grimberg mr = fr_desc->data_mr; 24939e961ae7SSagi Grimberg frpl = fr_desc->data_frpl; 24949e961ae7SSagi Grimberg } else { 24959e961ae7SSagi Grimberg /* Registering protection buffer */ 24969e961ae7SSagi Grimberg mr = fr_desc->pi_ctx->prot_mr; 24979e961ae7SSagi Grimberg frpl = fr_desc->pi_ctx->prot_frpl; 24989e961ae7SSagi Grimberg } 24999e961ae7SSagi Grimberg 2500e3d7e4c3SSagi Grimberg page_off = mem->offset % PAGE_SIZE; 250159464ef4SVu Pham 2502*24f412ddSSagi Grimberg isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2503e3d7e4c3SSagi Grimberg fr_desc, mem->nents, mem->offset); 250459464ef4SVu Pham 2505e3d7e4c3SSagi Grimberg pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 25069e961ae7SSagi Grimberg &frpl->page_list[0]); 250759464ef4SVu Pham 2508d3e125daSSagi Grimberg if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) { 250959464ef4SVu Pham memset(&inv_wr, 0, sizeof(inv_wr)); 25109bb4ca68SNicholas Bellinger inv_wr.wr_id = ISER_FASTREG_LI_WRID; 251159464ef4SVu Pham inv_wr.opcode = IB_WR_LOCAL_INV; 25129e961ae7SSagi Grimberg inv_wr.ex.invalidate_rkey = mr->rkey; 251359464ef4SVu Pham wr = &inv_wr; 251459464ef4SVu Pham /* Bump the key */ 25159e961ae7SSagi Grimberg key = (u8)(mr->rkey & 0x000000FF); 25169e961ae7SSagi Grimberg ib_update_fast_reg_key(mr, ++key); 251759464ef4SVu Pham } 251859464ef4SVu Pham 251959464ef4SVu Pham /* Prepare FASTREG WR */ 252059464ef4SVu Pham memset(&fr_wr, 0, sizeof(fr_wr)); 25219bb4ca68SNicholas Bellinger fr_wr.wr_id = ISER_FASTREG_LI_WRID; 252259464ef4SVu Pham fr_wr.opcode = IB_WR_FAST_REG_MR; 25239e961ae7SSagi Grimberg fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 25249e961ae7SSagi Grimberg fr_wr.wr.fast_reg.page_list = frpl; 252559464ef4SVu Pham fr_wr.wr.fast_reg.page_list_len = pagelist_len; 252659464ef4SVu Pham fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2527e3d7e4c3SSagi Grimberg fr_wr.wr.fast_reg.length = mem->len; 25289e961ae7SSagi Grimberg fr_wr.wr.fast_reg.rkey = mr->rkey; 252959464ef4SVu Pham fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 253059464ef4SVu Pham 253159464ef4SVu Pham if (!wr) 253259464ef4SVu Pham wr = &fr_wr; 253359464ef4SVu Pham else 253459464ef4SVu Pham wr->next = &fr_wr; 253559464ef4SVu Pham 253659464ef4SVu Pham ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 253759464ef4SVu Pham if (ret) { 2538*24f412ddSSagi Grimberg isert_err("fast registration failed, ret:%d\n", ret); 253959464ef4SVu Pham return ret; 254059464ef4SVu Pham } 25419e961ae7SSagi Grimberg fr_desc->ind &= ~ind; 254259464ef4SVu Pham 25439e961ae7SSagi Grimberg sge->lkey = mr->lkey; 25449e961ae7SSagi Grimberg sge->addr = frpl->page_list[0] + page_off; 2545e3d7e4c3SSagi Grimberg sge->length = mem->len; 254659464ef4SVu Pham 2547*24f412ddSSagi Grimberg isert_dbg("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 25489e961ae7SSagi Grimberg __func__, __LINE__, sge->addr, sge->length, 25499e961ae7SSagi Grimberg sge->lkey); 255059464ef4SVu Pham 255159464ef4SVu Pham return ret; 255259464ef4SVu Pham } 255359464ef4SVu Pham 25543d73cf1aSSagi Grimberg static inline void 25553d73cf1aSSagi Grimberg isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 25563d73cf1aSSagi Grimberg struct ib_sig_domain *domain) 25579e961ae7SSagi Grimberg { 255878eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 25593d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 25603d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 25613d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 256278eda2bbSSagi Grimberg /* 256378eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 256478eda2bbSSagi Grimberg * the target core would like to use it, we will take it 256578eda2bbSSagi Grimberg * from se_cmd. 256678eda2bbSSagi Grimberg */ 256778eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 256878eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 256978eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 257078eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 257178eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 257278eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 25733d73cf1aSSagi Grimberg }; 25743d73cf1aSSagi Grimberg 25753d73cf1aSSagi Grimberg static int 25763d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 25773d73cf1aSSagi Grimberg { 25789e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 25799e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 25809e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 258178eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 25823d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 25839e961ae7SSagi Grimberg break; 25849e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 25859e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 258678eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 25873d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 25889e961ae7SSagi Grimberg break; 25899e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 25909e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 25913d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 25923d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 25939e961ae7SSagi Grimberg break; 25949e961ae7SSagi Grimberg default: 2595*24f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 25969e961ae7SSagi Grimberg return -EINVAL; 25979e961ae7SSagi Grimberg } 25989e961ae7SSagi Grimberg 25999e961ae7SSagi Grimberg return 0; 26009e961ae7SSagi Grimberg } 26019e961ae7SSagi Grimberg 26029e961ae7SSagi Grimberg static inline u8 26039e961ae7SSagi Grimberg isert_set_prot_checks(u8 prot_checks) 26049e961ae7SSagi Grimberg { 26059e961ae7SSagi Grimberg return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 26069e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 26079e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 26089e961ae7SSagi Grimberg } 26099e961ae7SSagi Grimberg 26109e961ae7SSagi Grimberg static int 2611570db170SSagi Grimberg isert_reg_sig_mr(struct isert_conn *isert_conn, 2612570db170SSagi Grimberg struct se_cmd *se_cmd, 2613570db170SSagi Grimberg struct isert_rdma_wr *rdma_wr, 2614570db170SSagi Grimberg struct fast_reg_descriptor *fr_desc) 26159e961ae7SSagi Grimberg { 26169e961ae7SSagi Grimberg struct ib_send_wr sig_wr, inv_wr; 26179e961ae7SSagi Grimberg struct ib_send_wr *bad_wr, *wr = NULL; 26189e961ae7SSagi Grimberg struct pi_context *pi_ctx = fr_desc->pi_ctx; 26199e961ae7SSagi Grimberg struct ib_sig_attrs sig_attrs; 26209e961ae7SSagi Grimberg int ret; 26219e961ae7SSagi Grimberg u32 key; 26229e961ae7SSagi Grimberg 26239e961ae7SSagi Grimberg memset(&sig_attrs, 0, sizeof(sig_attrs)); 26249e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 26259e961ae7SSagi Grimberg if (ret) 26269e961ae7SSagi Grimberg goto err; 26279e961ae7SSagi Grimberg 26289e961ae7SSagi Grimberg sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 26299e961ae7SSagi Grimberg 26309e961ae7SSagi Grimberg if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 26319e961ae7SSagi Grimberg memset(&inv_wr, 0, sizeof(inv_wr)); 26329e961ae7SSagi Grimberg inv_wr.opcode = IB_WR_LOCAL_INV; 2633c2caa207SSagi Grimberg inv_wr.wr_id = ISER_FASTREG_LI_WRID; 26349e961ae7SSagi Grimberg inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; 26359e961ae7SSagi Grimberg wr = &inv_wr; 26369e961ae7SSagi Grimberg /* Bump the key */ 26379e961ae7SSagi Grimberg key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); 26389e961ae7SSagi Grimberg ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); 26399e961ae7SSagi Grimberg } 26409e961ae7SSagi Grimberg 26419e961ae7SSagi Grimberg memset(&sig_wr, 0, sizeof(sig_wr)); 26429e961ae7SSagi Grimberg sig_wr.opcode = IB_WR_REG_SIG_MR; 2643c2caa207SSagi Grimberg sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2644570db170SSagi Grimberg sig_wr.sg_list = &rdma_wr->ib_sg[DATA]; 26459e961ae7SSagi Grimberg sig_wr.num_sge = 1; 26469e961ae7SSagi Grimberg sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 26479e961ae7SSagi Grimberg sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 26489e961ae7SSagi Grimberg sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 26499e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 2650570db170SSagi Grimberg sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT]; 26519e961ae7SSagi Grimberg 26529e961ae7SSagi Grimberg if (!wr) 26539e961ae7SSagi Grimberg wr = &sig_wr; 26549e961ae7SSagi Grimberg else 26559e961ae7SSagi Grimberg wr->next = &sig_wr; 26569e961ae7SSagi Grimberg 26579e961ae7SSagi Grimberg ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 26589e961ae7SSagi Grimberg if (ret) { 2659*24f412ddSSagi Grimberg isert_err("fast registration failed, ret:%d\n", ret); 26609e961ae7SSagi Grimberg goto err; 26619e961ae7SSagi Grimberg } 26629e961ae7SSagi Grimberg fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 26639e961ae7SSagi Grimberg 2664570db170SSagi Grimberg rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2665570db170SSagi Grimberg rdma_wr->ib_sg[SIG].addr = 0; 2666570db170SSagi Grimberg rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 26679e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 26689e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 26699e961ae7SSagi Grimberg /* 26709e961ae7SSagi Grimberg * We have protection guards on the wire 26719e961ae7SSagi Grimberg * so we need to set a larget transfer 26729e961ae7SSagi Grimberg */ 2673570db170SSagi Grimberg rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 26749e961ae7SSagi Grimberg 2675*24f412ddSSagi Grimberg isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2676570db170SSagi Grimberg rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2677570db170SSagi Grimberg rdma_wr->ib_sg[SIG].lkey); 26789e961ae7SSagi Grimberg err: 26799e961ae7SSagi Grimberg return ret; 26809e961ae7SSagi Grimberg } 26819e961ae7SSagi Grimberg 268259464ef4SVu Pham static int 2683570db170SSagi Grimberg isert_handle_prot_cmd(struct isert_conn *isert_conn, 2684570db170SSagi Grimberg struct isert_cmd *isert_cmd, 2685570db170SSagi Grimberg struct isert_rdma_wr *wr) 2686570db170SSagi Grimberg { 2687570db170SSagi Grimberg struct isert_device *device = isert_conn->conn_device; 2688570db170SSagi Grimberg struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2689570db170SSagi Grimberg int ret; 2690570db170SSagi Grimberg 2691570db170SSagi Grimberg if (!wr->fr_desc->pi_ctx) { 2692570db170SSagi Grimberg ret = isert_create_pi_ctx(wr->fr_desc, 2693570db170SSagi Grimberg device->ib_device, 2694570db170SSagi Grimberg isert_conn->conn_pd); 2695570db170SSagi Grimberg if (ret) { 2696*24f412ddSSagi Grimberg isert_err("conn %p failed to allocate pi_ctx\n", 2697570db170SSagi Grimberg isert_conn); 2698570db170SSagi Grimberg return ret; 2699570db170SSagi Grimberg } 2700570db170SSagi Grimberg } 2701570db170SSagi Grimberg 2702570db170SSagi Grimberg if (se_cmd->t_prot_sg) { 2703570db170SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, 2704570db170SSagi Grimberg se_cmd->t_prot_sg, 2705570db170SSagi Grimberg se_cmd->t_prot_nents, 2706570db170SSagi Grimberg se_cmd->prot_length, 2707570db170SSagi Grimberg 0, wr->iser_ib_op, &wr->prot); 2708570db170SSagi Grimberg if (ret) { 2709*24f412ddSSagi Grimberg isert_err("conn %p failed to map protection buffer\n", 2710570db170SSagi Grimberg isert_conn); 2711570db170SSagi Grimberg return ret; 2712570db170SSagi Grimberg } 2713570db170SSagi Grimberg 2714570db170SSagi Grimberg memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2715570db170SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2716570db170SSagi Grimberg ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2717570db170SSagi Grimberg if (ret) { 2718*24f412ddSSagi Grimberg isert_err("conn %p failed to fast reg mr\n", 2719570db170SSagi Grimberg isert_conn); 2720570db170SSagi Grimberg goto unmap_prot_cmd; 2721570db170SSagi Grimberg } 2722570db170SSagi Grimberg } 2723570db170SSagi Grimberg 2724570db170SSagi Grimberg ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2725570db170SSagi Grimberg if (ret) { 2726*24f412ddSSagi Grimberg isert_err("conn %p failed to fast reg mr\n", 2727570db170SSagi Grimberg isert_conn); 2728570db170SSagi Grimberg goto unmap_prot_cmd; 2729570db170SSagi Grimberg } 2730570db170SSagi Grimberg wr->fr_desc->ind |= ISERT_PROTECTED; 2731570db170SSagi Grimberg 2732570db170SSagi Grimberg return 0; 2733570db170SSagi Grimberg 2734570db170SSagi Grimberg unmap_prot_cmd: 2735570db170SSagi Grimberg if (se_cmd->t_prot_sg) 2736570db170SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 2737570db170SSagi Grimberg 2738570db170SSagi Grimberg return ret; 2739570db170SSagi Grimberg } 2740570db170SSagi Grimberg 2741570db170SSagi Grimberg static int 2742a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 274359464ef4SVu Pham struct isert_rdma_wr *wr) 274459464ef4SVu Pham { 274559464ef4SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 274659464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2747e3d7e4c3SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2748e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc = NULL; 2749570db170SSagi Grimberg struct ib_send_wr *send_wr; 2750570db170SSagi Grimberg struct ib_sge *ib_sg; 2751e3d7e4c3SSagi Grimberg u32 offset; 2752e3d7e4c3SSagi Grimberg int ret = 0; 275359464ef4SVu Pham unsigned long flags; 275459464ef4SVu Pham 275559464ef4SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2756e3d7e4c3SSagi Grimberg 2757e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2758e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2759e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2760e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2761e3d7e4c3SSagi Grimberg if (ret) 2762e3d7e4c3SSagi Grimberg return ret; 2763e3d7e4c3SSagi Grimberg 2764302cc7c3SSagi Grimberg if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2765e3d7e4c3SSagi Grimberg spin_lock_irqsave(&isert_conn->conn_lock, flags); 2766e3d7e4c3SSagi Grimberg fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2767e3d7e4c3SSagi Grimberg struct fast_reg_descriptor, list); 2768e3d7e4c3SSagi Grimberg list_del(&fr_desc->list); 2769e3d7e4c3SSagi Grimberg spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2770e3d7e4c3SSagi Grimberg wr->fr_desc = fr_desc; 277159464ef4SVu Pham } 277259464ef4SVu Pham 27739e961ae7SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2774570db170SSagi Grimberg ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2775e3d7e4c3SSagi Grimberg if (ret) 2776e3d7e4c3SSagi Grimberg goto unmap_cmd; 277759464ef4SVu Pham 2778302cc7c3SSagi Grimberg if (isert_prot_cmd(isert_conn, se_cmd)) { 2779570db170SSagi Grimberg ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 27809e961ae7SSagi Grimberg if (ret) 27819e961ae7SSagi Grimberg goto unmap_cmd; 27829e961ae7SSagi Grimberg 2783570db170SSagi Grimberg ib_sg = &wr->ib_sg[SIG]; 2784570db170SSagi Grimberg } else { 2785570db170SSagi Grimberg ib_sg = &wr->ib_sg[DATA]; 27869e961ae7SSagi Grimberg } 27879e961ae7SSagi Grimberg 2788570db170SSagi Grimberg memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2789e3d7e4c3SSagi Grimberg wr->ib_sge = &wr->s_ib_sge; 279059464ef4SVu Pham wr->send_wr_num = 1; 279159464ef4SVu Pham memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 279259464ef4SVu Pham wr->send_wr = &wr->s_send_wr; 279359464ef4SVu Pham wr->isert_cmd = isert_cmd; 279459464ef4SVu Pham 279559464ef4SVu Pham send_wr = &isert_cmd->rdma_wr.s_send_wr; 2796e3d7e4c3SSagi Grimberg send_wr->sg_list = &wr->s_ib_sge; 279759464ef4SVu Pham send_wr->num_sge = 1; 2798b0a191e7SSagi Grimberg send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 279959464ef4SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 280059464ef4SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 280159464ef4SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 280259464ef4SVu Pham send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2803302cc7c3SSagi Grimberg send_wr->send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 28049e961ae7SSagi Grimberg 0 : IB_SEND_SIGNALED; 280559464ef4SVu Pham } else { 280659464ef4SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 280759464ef4SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 280859464ef4SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 280959464ef4SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 281059464ef4SVu Pham } 281159464ef4SVu Pham 281259464ef4SVu Pham return 0; 2813570db170SSagi Grimberg 2814e3d7e4c3SSagi Grimberg unmap_cmd: 2815e3d7e4c3SSagi Grimberg if (fr_desc) { 2816e3d7e4c3SSagi Grimberg spin_lock_irqsave(&isert_conn->conn_lock, flags); 2817e3d7e4c3SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2818e3d7e4c3SSagi Grimberg spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2819e3d7e4c3SSagi Grimberg } 2820e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 282159464ef4SVu Pham 282259464ef4SVu Pham return ret; 282359464ef4SVu Pham } 282459464ef4SVu Pham 282559464ef4SVu Pham static int 282690ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 282790ecc6e2SVu Pham { 282890ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 282959464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 283090ecc6e2SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 283190ecc6e2SVu Pham struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2832d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 283390ecc6e2SVu Pham struct ib_send_wr *wr_failed; 283490ecc6e2SVu Pham int rc; 283590ecc6e2SVu Pham 2836*24f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 283790ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 283890ecc6e2SVu Pham wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2839d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 284090ecc6e2SVu Pham if (rc) { 2841*24f412ddSSagi Grimberg isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 284290ecc6e2SVu Pham return rc; 284390ecc6e2SVu Pham } 284490ecc6e2SVu Pham 2845302cc7c3SSagi Grimberg if (!isert_prot_cmd(isert_conn, se_cmd)) { 2846b8d26b3bSNicholas Bellinger /* 2847b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2848b8d26b3bSNicholas Bellinger */ 28499e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 28509e961ae7SSagi Grimberg &isert_cmd->tx_desc); 285104d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2852b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2853b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 285495b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 285568a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 28569e961ae7SSagi Grimberg isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2857897bb2c9SSagi Grimberg wr->send_wr_num += 1; 28589e961ae7SSagi Grimberg } 2859b8d26b3bSNicholas Bellinger 2860b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2861bdf20e72SSagi Grimberg if (rc) 2862*24f412ddSSagi Grimberg isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 28639e961ae7SSagi Grimberg 2864302cc7c3SSagi Grimberg if (!isert_prot_cmd(isert_conn, se_cmd)) 2865*24f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 28669e961ae7SSagi Grimberg "READ\n", isert_cmd); 28679e961ae7SSagi Grimberg else 2868*24f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 286990ecc6e2SVu Pham isert_cmd); 2870b8d26b3bSNicholas Bellinger 287190ecc6e2SVu Pham return 1; 2872b8d26b3bSNicholas Bellinger } 2873b8d26b3bSNicholas Bellinger 2874b8d26b3bSNicholas Bellinger static int 2875b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2876b8d26b3bSNicholas Bellinger { 2877b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2878d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2879b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2880b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2881d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 288290ecc6e2SVu Pham struct ib_send_wr *wr_failed; 288390ecc6e2SVu Pham int rc; 2884b8d26b3bSNicholas Bellinger 2885*24f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 288690ecc6e2SVu Pham isert_cmd, se_cmd->data_length, cmd->write_data_done); 2887b8d26b3bSNicholas Bellinger wr->iser_ib_op = ISER_IB_RDMA_READ; 2888d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 288990ecc6e2SVu Pham if (rc) { 2890*24f412ddSSagi Grimberg isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 289190ecc6e2SVu Pham return rc; 2892b8d26b3bSNicholas Bellinger } 2893b8d26b3bSNicholas Bellinger 2894b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2895bdf20e72SSagi Grimberg if (rc) 2896*24f412ddSSagi Grimberg isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2897bdf20e72SSagi Grimberg 2898*24f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 289990ecc6e2SVu Pham isert_cmd); 2900b8d26b3bSNicholas Bellinger 290190ecc6e2SVu Pham return 0; 2902b8d26b3bSNicholas Bellinger } 2903b8d26b3bSNicholas Bellinger 2904b8d26b3bSNicholas Bellinger static int 2905b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2906b8d26b3bSNicholas Bellinger { 2907b8d26b3bSNicholas Bellinger int ret; 2908b8d26b3bSNicholas Bellinger 2909b8d26b3bSNicholas Bellinger switch (state) { 2910b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2911b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2912b8d26b3bSNicholas Bellinger break; 2913b8d26b3bSNicholas Bellinger default: 2914*24f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2915b8d26b3bSNicholas Bellinger ret = -EINVAL; 2916b8d26b3bSNicholas Bellinger break; 2917b8d26b3bSNicholas Bellinger } 2918b8d26b3bSNicholas Bellinger 2919b8d26b3bSNicholas Bellinger return ret; 2920b8d26b3bSNicholas Bellinger } 2921b8d26b3bSNicholas Bellinger 2922b8d26b3bSNicholas Bellinger static int 2923b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2924b8d26b3bSNicholas Bellinger { 2925991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2926b8d26b3bSNicholas Bellinger int ret; 2927b8d26b3bSNicholas Bellinger 2928b8d26b3bSNicholas Bellinger switch (state) { 2929b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2930b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2931991bb764SSagi Grimberg if (!ret) 2932991bb764SSagi Grimberg isert_conn->logout_posted = true; 2933b8d26b3bSNicholas Bellinger break; 2934b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2935b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2936b8d26b3bSNicholas Bellinger break; 2937b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2938b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2939b8d26b3bSNicholas Bellinger break; 2940b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2941b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2942b8d26b3bSNicholas Bellinger break; 2943adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2944adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2945adb54c29SNicholas Bellinger break; 2946b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2947b8d26b3bSNicholas Bellinger /* 2948b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2949b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2950b8d26b3bSNicholas Bellinger */ 2951b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2952b8d26b3bSNicholas Bellinger break; 2953b8d26b3bSNicholas Bellinger default: 2954*24f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 2955b8d26b3bSNicholas Bellinger ret = -EINVAL; 2956b8d26b3bSNicholas Bellinger break; 2957b8d26b3bSNicholas Bellinger } 2958b8d26b3bSNicholas Bellinger 2959b8d26b3bSNicholas Bellinger return ret; 2960b8d26b3bSNicholas Bellinger } 2961b8d26b3bSNicholas Bellinger 2962ca6c1d82SSagi Grimberg struct rdma_cm_id * 2963ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 2964ca6c1d82SSagi Grimberg { 2965ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 2966ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 2967ca6c1d82SSagi Grimberg struct sockaddr *sa; 2968ca6c1d82SSagi Grimberg int ret; 2969ca6c1d82SSagi Grimberg 2970ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 2971*24f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2972ca6c1d82SSagi Grimberg 2973ca6c1d82SSagi Grimberg id = rdma_create_id(isert_cma_handler, isert_np, 2974ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 2975ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 2976*24f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2977ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 2978ca6c1d82SSagi Grimberg goto out; 2979ca6c1d82SSagi Grimberg } 2980*24f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 2981ca6c1d82SSagi Grimberg 2982ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 2983ca6c1d82SSagi Grimberg if (ret) { 2984*24f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 2985ca6c1d82SSagi Grimberg goto out_id; 2986ca6c1d82SSagi Grimberg } 2987ca6c1d82SSagi Grimberg 2988ca6c1d82SSagi Grimberg ret = rdma_listen(id, ISERT_RDMA_LISTEN_BACKLOG); 2989ca6c1d82SSagi Grimberg if (ret) { 2990*24f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 2991ca6c1d82SSagi Grimberg goto out_id; 2992ca6c1d82SSagi Grimberg } 2993ca6c1d82SSagi Grimberg 2994ca6c1d82SSagi Grimberg return id; 2995ca6c1d82SSagi Grimberg out_id: 2996ca6c1d82SSagi Grimberg rdma_destroy_id(id); 2997ca6c1d82SSagi Grimberg out: 2998ca6c1d82SSagi Grimberg return ERR_PTR(ret); 2999ca6c1d82SSagi Grimberg } 3000ca6c1d82SSagi Grimberg 3001b8d26b3bSNicholas Bellinger static int 3002b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 3003b8d26b3bSNicholas Bellinger struct __kernel_sockaddr_storage *ksockaddr) 3004b8d26b3bSNicholas Bellinger { 3005b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 3006b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 3007b8d26b3bSNicholas Bellinger int ret; 3008b8d26b3bSNicholas Bellinger 3009b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3010b8d26b3bSNicholas Bellinger if (!isert_np) { 3011*24f412ddSSagi Grimberg isert_err("Unable to allocate struct isert_np\n"); 3012b8d26b3bSNicholas Bellinger return -ENOMEM; 3013b8d26b3bSNicholas Bellinger } 3014531b7bf4SSagi Grimberg sema_init(&isert_np->np_sem, 0); 3015b8d26b3bSNicholas Bellinger mutex_init(&isert_np->np_accept_mutex); 3016b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_np->np_accept_list); 3017b8d26b3bSNicholas Bellinger init_completion(&isert_np->np_login_comp); 3018ca6c1d82SSagi Grimberg isert_np->np = np; 3019b8d26b3bSNicholas Bellinger 3020b8d26b3bSNicholas Bellinger /* 3021b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 3022b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 3023b8d26b3bSNicholas Bellinger */ 3024b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 3025b8d26b3bSNicholas Bellinger sizeof(struct __kernel_sockaddr_storage)); 3026b8d26b3bSNicholas Bellinger 3027ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 3028b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 3029b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 3030b8d26b3bSNicholas Bellinger goto out; 3031b8d26b3bSNicholas Bellinger } 3032b8d26b3bSNicholas Bellinger 3033b8d26b3bSNicholas Bellinger isert_np->np_cm_id = isert_lid; 3034b8d26b3bSNicholas Bellinger np->np_context = isert_np; 3035b8d26b3bSNicholas Bellinger 3036b8d26b3bSNicholas Bellinger return 0; 3037b8d26b3bSNicholas Bellinger 3038b8d26b3bSNicholas Bellinger out: 3039b8d26b3bSNicholas Bellinger kfree(isert_np); 3040ca6c1d82SSagi Grimberg 3041b8d26b3bSNicholas Bellinger return ret; 3042b8d26b3bSNicholas Bellinger } 3043b8d26b3bSNicholas Bellinger 3044b8d26b3bSNicholas Bellinger static int 3045b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 3046b8d26b3bSNicholas Bellinger { 3047b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3048b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 3049b8d26b3bSNicholas Bellinger int ret; 3050b8d26b3bSNicholas Bellinger 3051b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 3052b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 3053b8d26b3bSNicholas Bellinger cp.retry_count = 7; 3054b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 3055b8d26b3bSNicholas Bellinger 3056*24f412ddSSagi Grimberg isert_dbg("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 3057b8d26b3bSNicholas Bellinger 3058b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 3059b8d26b3bSNicholas Bellinger if (ret) { 3060*24f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 3061b8d26b3bSNicholas Bellinger return ret; 3062b8d26b3bSNicholas Bellinger } 3063b8d26b3bSNicholas Bellinger 3064*24f412ddSSagi Grimberg isert_dbg("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 3065b8d26b3bSNicholas Bellinger 3066b8d26b3bSNicholas Bellinger return 0; 3067b8d26b3bSNicholas Bellinger } 3068b8d26b3bSNicholas Bellinger 3069b8d26b3bSNicholas Bellinger static int 3070b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3071b8d26b3bSNicholas Bellinger { 3072b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3073b8d26b3bSNicholas Bellinger int ret; 3074b8d26b3bSNicholas Bellinger 3075*24f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 30762371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 30772371e5daSSagi Grimberg if (ret) { 3078*24f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 30792371e5daSSagi Grimberg isert_conn); 30802371e5daSSagi Grimberg return ret; 30812371e5daSSagi Grimberg } 30822371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 30832371e5daSSagi Grimberg 30846faaa85fSNicholas Bellinger /* 30856faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 30866faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 30876faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 30886faaa85fSNicholas Bellinger * into a NOP. 30896faaa85fSNicholas Bellinger */ 30906faaa85fSNicholas Bellinger if (!login->first_request) 30916faaa85fSNicholas Bellinger return 0; 3092b8d26b3bSNicholas Bellinger 30932371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 30942371e5daSSagi Grimberg 3095*24f412ddSSagi Grimberg isert_info("before conn_login_comp conn: %p\n", conn); 3096b8d26b3bSNicholas Bellinger ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 3097b8d26b3bSNicholas Bellinger if (ret) 3098b8d26b3bSNicholas Bellinger return ret; 3099b8d26b3bSNicholas Bellinger 3100*24f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 31012371e5daSSagi Grimberg 3102b8d26b3bSNicholas Bellinger return 0; 3103b8d26b3bSNicholas Bellinger } 3104b8d26b3bSNicholas Bellinger 3105b8d26b3bSNicholas Bellinger static void 3106b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3107b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 3108b8d26b3bSNicholas Bellinger { 3109b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3110b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 3111b8d26b3bSNicholas Bellinger struct sockaddr_in *sock_in; 3112b8d26b3bSNicholas Bellinger struct sockaddr_in6 *sock_in6; 3113b8d26b3bSNicholas Bellinger 3114b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 3115b8d26b3bSNicholas Bellinger 3116b8d26b3bSNicholas Bellinger if (np->np_sockaddr.ss_family == AF_INET6) { 3117b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3118b8d26b3bSNicholas Bellinger snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3119b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 3120b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in6->sin6_port); 3121b8d26b3bSNicholas Bellinger 3122b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3123b8d26b3bSNicholas Bellinger snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3124b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 3125b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in6->sin6_port); 3126b8d26b3bSNicholas Bellinger } else { 3127b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3128b8d26b3bSNicholas Bellinger sprintf(conn->login_ip, "%pI4", 3129b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 3130b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in->sin_port); 3131b8d26b3bSNicholas Bellinger 3132b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3133b8d26b3bSNicholas Bellinger sprintf(conn->local_ip, "%pI4", 3134b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 3135b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in->sin_port); 3136b8d26b3bSNicholas Bellinger } 3137b8d26b3bSNicholas Bellinger } 3138b8d26b3bSNicholas Bellinger 3139b8d26b3bSNicholas Bellinger static int 3140b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3141b8d26b3bSNicholas Bellinger { 3142b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 3143b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 3144b8d26b3bSNicholas Bellinger int max_accept = 0, ret; 3145b8d26b3bSNicholas Bellinger 3146b8d26b3bSNicholas Bellinger accept_wait: 3147531b7bf4SSagi Grimberg ret = down_interruptible(&isert_np->np_sem); 31481acff63fSSagi Grimberg if (ret || max_accept > 5) 3149b8d26b3bSNicholas Bellinger return -ENODEV; 3150b8d26b3bSNicholas Bellinger 3151b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 3152e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3153b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3154*24f412ddSSagi Grimberg isert_dbg("np_thread_state %d for isert_accept_np\n", 3155e346ab34SSagi Grimberg np->np_thread_state); 3156e346ab34SSagi Grimberg /** 3157e346ab34SSagi Grimberg * No point in stalling here when np_thread 3158e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 3159e346ab34SSagi Grimberg **/ 3160b8d26b3bSNicholas Bellinger return -ENODEV; 3161b8d26b3bSNicholas Bellinger } 3162b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3163b8d26b3bSNicholas Bellinger 3164b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 3165b8d26b3bSNicholas Bellinger if (list_empty(&isert_np->np_accept_list)) { 3166b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3167b8d26b3bSNicholas Bellinger max_accept++; 3168b8d26b3bSNicholas Bellinger goto accept_wait; 3169b8d26b3bSNicholas Bellinger } 3170b8d26b3bSNicholas Bellinger isert_conn = list_first_entry(&isert_np->np_accept_list, 3171b8d26b3bSNicholas Bellinger struct isert_conn, conn_accept_node); 3172b8d26b3bSNicholas Bellinger list_del_init(&isert_conn->conn_accept_node); 3173b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 3174b8d26b3bSNicholas Bellinger 3175b8d26b3bSNicholas Bellinger conn->context = isert_conn; 3176b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 3177b8d26b3bSNicholas Bellinger max_accept = 0; 3178b8d26b3bSNicholas Bellinger 3179b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 3180b8d26b3bSNicholas Bellinger 3181*24f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 31822371e5daSSagi Grimberg 3183b8d26b3bSNicholas Bellinger return 0; 3184b8d26b3bSNicholas Bellinger } 3185b8d26b3bSNicholas Bellinger 3186b8d26b3bSNicholas Bellinger static void 3187b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 3188b8d26b3bSNicholas Bellinger { 3189b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 3190268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 3191b8d26b3bSNicholas Bellinger 31923b726ae2SSagi Grimberg if (isert_np->np_cm_id) 3193b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_np->np_cm_id); 3194b8d26b3bSNicholas Bellinger 3195268e6811SSagi Grimberg /* 3196268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 3197268e6811SSagi Grimberg * that at this point we don't have hanging connections that 3198268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 3199268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 3200268e6811SSagi Grimberg * up in np_accept_list. 3201268e6811SSagi Grimberg */ 3202268e6811SSagi Grimberg mutex_lock(&isert_np->np_accept_mutex); 3203268e6811SSagi Grimberg if (!list_empty(&isert_np->np_accept_list)) { 3204*24f412ddSSagi Grimberg isert_info("Still have isert connections, cleaning up...\n"); 3205268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 3206268e6811SSagi Grimberg &isert_np->np_accept_list, 3207268e6811SSagi Grimberg conn_accept_node) { 3208*24f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 3209268e6811SSagi Grimberg isert_conn, isert_conn->state); 3210268e6811SSagi Grimberg isert_connect_release(isert_conn); 3211268e6811SSagi Grimberg } 3212268e6811SSagi Grimberg } 3213268e6811SSagi Grimberg mutex_unlock(&isert_np->np_accept_mutex); 3214268e6811SSagi Grimberg 3215b8d26b3bSNicholas Bellinger np->np_context = NULL; 3216b8d26b3bSNicholas Bellinger kfree(isert_np); 3217b8d26b3bSNicholas Bellinger } 3218b8d26b3bSNicholas Bellinger 3219b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 3220b02efbfcSSagi Grimberg { 3221b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 3222b02efbfcSSagi Grimberg struct isert_conn, 3223b02efbfcSSagi Grimberg release_work); 3224b02efbfcSSagi Grimberg 3225*24f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 3226b02efbfcSSagi Grimberg 3227b02efbfcSSagi Grimberg wait_for_completion(&isert_conn->conn_wait); 3228b02efbfcSSagi Grimberg 3229b02efbfcSSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 3230b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 3231b02efbfcSSagi Grimberg mutex_unlock(&isert_conn->conn_mutex); 3232b02efbfcSSagi Grimberg 3233*24f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 3234b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 3235b02efbfcSSagi Grimberg } 3236b02efbfcSSagi Grimberg 3237bdf20e72SSagi Grimberg static void 3238991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 3239991bb764SSagi Grimberg { 3240991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 3241991bb764SSagi Grimberg 3242991bb764SSagi Grimberg if (isert_conn->logout_posted) { 3243*24f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3244991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 3245991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 3246991bb764SSagi Grimberg } 3247991bb764SSagi Grimberg } 3248991bb764SSagi Grimberg 3249991bb764SSagi Grimberg static void 3250c7e160eeSSagi Grimberg isert_wait4cmds(struct iscsi_conn *conn) 3251c7e160eeSSagi Grimberg { 3252c7e160eeSSagi Grimberg if (conn->sess) { 3253c7e160eeSSagi Grimberg target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3254c7e160eeSSagi Grimberg target_wait_for_sess_cmds(conn->sess->se_sess); 3255c7e160eeSSagi Grimberg } 3256c7e160eeSSagi Grimberg } 3257c7e160eeSSagi Grimberg 3258c7e160eeSSagi Grimberg static void 3259bdf20e72SSagi Grimberg isert_wait4flush(struct isert_conn *isert_conn) 3260bdf20e72SSagi Grimberg { 3261bdf20e72SSagi Grimberg struct ib_recv_wr *bad_wr; 3262bdf20e72SSagi Grimberg 3263bdf20e72SSagi Grimberg init_completion(&isert_conn->conn_wait_comp_err); 3264bdf20e72SSagi Grimberg isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3265bdf20e72SSagi Grimberg /* post an indication that all flush errors were consumed */ 3266bdf20e72SSagi Grimberg if (ib_post_recv(isert_conn->conn_qp, &isert_conn->beacon, &bad_wr)) { 3267*24f412ddSSagi Grimberg isert_err("conn %p failed to post beacon", isert_conn); 3268bdf20e72SSagi Grimberg return; 3269bdf20e72SSagi Grimberg } 3270bdf20e72SSagi Grimberg 3271bdf20e72SSagi Grimberg wait_for_completion(&isert_conn->conn_wait_comp_err); 3272bdf20e72SSagi Grimberg } 3273bdf20e72SSagi Grimberg 3274defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 3275b8d26b3bSNicholas Bellinger { 3276b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3277b8d26b3bSNicholas Bellinger 3278*24f412ddSSagi Grimberg isert_dbg("isert_wait_conn: Starting\n"); 3279b8d26b3bSNicholas Bellinger 32809d49f5e2SSagi Grimberg mutex_lock(&isert_conn->conn_mutex); 3281b8d26b3bSNicholas Bellinger /* 3282b8d26b3bSNicholas Bellinger * Only wait for conn_wait_comp_err if the isert_conn made it 3283b8d26b3bSNicholas Bellinger * into full feature phase.. 3284b8d26b3bSNicholas Bellinger */ 3285b2cb9649SNicholas Bellinger if (isert_conn->state == ISER_CONN_INIT) { 3286b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 3287b2cb9649SNicholas Bellinger return; 3288b2cb9649SNicholas Bellinger } 3289954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 3290b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 3291b2cb9649SNicholas Bellinger 3292c7e160eeSSagi Grimberg isert_wait4cmds(conn); 3293bdf20e72SSagi Grimberg isert_wait4flush(isert_conn); 3294991bb764SSagi Grimberg isert_wait4logout(isert_conn); 3295954f2372SSagi Grimberg 3296b02efbfcSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 3297b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 3298defd8848SNicholas Bellinger } 3299defd8848SNicholas Bellinger 3300defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 3301defd8848SNicholas Bellinger { 3302defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3303b8d26b3bSNicholas Bellinger 3304b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 3305b8d26b3bSNicholas Bellinger } 3306b8d26b3bSNicholas Bellinger 3307b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 3308b8d26b3bSNicholas Bellinger .name = "IB/iSER", 3309b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 3310d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 3311b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 3312b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 3313b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 3314b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 3315defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 3316b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 3317b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 3318b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 3319b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 3320b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 3321b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 3322b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 3323b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 3324131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 3325e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3326b8d26b3bSNicholas Bellinger }; 3327b8d26b3bSNicholas Bellinger 3328b8d26b3bSNicholas Bellinger static int __init isert_init(void) 3329b8d26b3bSNicholas Bellinger { 3330b8d26b3bSNicholas Bellinger int ret; 3331b8d26b3bSNicholas Bellinger 3332b8d26b3bSNicholas Bellinger isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 3333b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 3334*24f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 3335b8d26b3bSNicholas Bellinger ret = -ENOMEM; 33366f0fae3dSSagi Grimberg return -ENOMEM; 3337b8d26b3bSNicholas Bellinger } 3338b8d26b3bSNicholas Bellinger 3339b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3340b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 3341b02efbfcSSagi Grimberg if (!isert_release_wq) { 3342*24f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 3343b02efbfcSSagi Grimberg ret = -ENOMEM; 3344b02efbfcSSagi Grimberg goto destroy_comp_wq; 3345b02efbfcSSagi Grimberg } 3346b02efbfcSSagi Grimberg 3347b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 3348*24f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3349b02efbfcSSagi Grimberg 3350b8d26b3bSNicholas Bellinger return 0; 3351b8d26b3bSNicholas Bellinger 3352b02efbfcSSagi Grimberg destroy_comp_wq: 3353b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 33546f0fae3dSSagi Grimberg 3355b8d26b3bSNicholas Bellinger return ret; 3356b8d26b3bSNicholas Bellinger } 3357b8d26b3bSNicholas Bellinger 3358b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 3359b8d26b3bSNicholas Bellinger { 3360f5ebec96SSagi Grimberg flush_scheduled_work(); 3361b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 3362b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 3363b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 3364*24f412ddSSagi Grimberg isert_dbg("iSER_TARGET[0] - Released iser_target_transport\n"); 3365b8d26b3bSNicholas Bellinger } 3366b8d26b3bSNicholas Bellinger 3367b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3368b8d26b3bSNicholas Bellinger MODULE_VERSION("0.1"); 3369b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3370b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 3371b8d26b3bSNicholas Bellinger 3372b8d26b3bSNicholas Bellinger module_init(isert_init); 3373b8d26b3bSNicholas Bellinger module_exit(isert_exit); 3374