1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 4b8d26b3bSNicholas Bellinger * (c) Copyright 2013 RisingTide Systems LLC. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 25b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 26b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 27b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 29b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 30b8d26b3bSNicholas Bellinger 31b8d26b3bSNicholas Bellinger #include "isert_proto.h" 32b8d26b3bSNicholas Bellinger #include "ib_isert.h" 33b8d26b3bSNicholas Bellinger 34b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 35b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36b8d26b3bSNicholas Bellinger #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 37b8d26b3bSNicholas Bellinger 38b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 39b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 40b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_rx_wq; 41b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 42b8d26b3bSNicholas Bellinger 43b8d26b3bSNicholas Bellinger static void 44*d40945d8SVu Pham isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 45*d40945d8SVu Pham static int 46*d40945d8SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 47*d40945d8SVu Pham struct isert_rdma_wr *wr); 48*d40945d8SVu Pham 49*d40945d8SVu Pham static void 50b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 51b8d26b3bSNicholas Bellinger { 52b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)context; 53b8d26b3bSNicholas Bellinger 54b8d26b3bSNicholas Bellinger pr_err("isert_qp_event_callback event: %d\n", e->event); 55b8d26b3bSNicholas Bellinger switch (e->event) { 56b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 57b8d26b3bSNicholas Bellinger rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 58b8d26b3bSNicholas Bellinger break; 59b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 60b8d26b3bSNicholas Bellinger pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 61b8d26b3bSNicholas Bellinger break; 62b8d26b3bSNicholas Bellinger default: 63b8d26b3bSNicholas Bellinger break; 64b8d26b3bSNicholas Bellinger } 65b8d26b3bSNicholas Bellinger } 66b8d26b3bSNicholas Bellinger 67b8d26b3bSNicholas Bellinger static int 68b8d26b3bSNicholas Bellinger isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 69b8d26b3bSNicholas Bellinger { 70b8d26b3bSNicholas Bellinger int ret; 71b8d26b3bSNicholas Bellinger 72b8d26b3bSNicholas Bellinger ret = ib_query_device(ib_dev, devattr); 73b8d26b3bSNicholas Bellinger if (ret) { 74b8d26b3bSNicholas Bellinger pr_err("ib_query_device() failed: %d\n", ret); 75b8d26b3bSNicholas Bellinger return ret; 76b8d26b3bSNicholas Bellinger } 77b8d26b3bSNicholas Bellinger pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 78b8d26b3bSNicholas Bellinger pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 79b8d26b3bSNicholas Bellinger 80b8d26b3bSNicholas Bellinger return 0; 81b8d26b3bSNicholas Bellinger } 82b8d26b3bSNicholas Bellinger 83b8d26b3bSNicholas Bellinger static int 84b8d26b3bSNicholas Bellinger isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 85b8d26b3bSNicholas Bellinger { 86b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 87b8d26b3bSNicholas Bellinger struct ib_qp_init_attr attr; 88b8d26b3bSNicholas Bellinger struct ib_device_attr devattr; 89b8d26b3bSNicholas Bellinger int ret, index, min_index = 0; 90b8d26b3bSNicholas Bellinger 91b8d26b3bSNicholas Bellinger memset(&devattr, 0, sizeof(struct ib_device_attr)); 92b8d26b3bSNicholas Bellinger ret = isert_query_device(cma_id->device, &devattr); 93b8d26b3bSNicholas Bellinger if (ret) 94b8d26b3bSNicholas Bellinger return ret; 95b8d26b3bSNicholas Bellinger 96b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 97b8d26b3bSNicholas Bellinger for (index = 0; index < device->cqs_used; index++) 98b8d26b3bSNicholas Bellinger if (device->cq_active_qps[index] < 99b8d26b3bSNicholas Bellinger device->cq_active_qps[min_index]) 100b8d26b3bSNicholas Bellinger min_index = index; 101b8d26b3bSNicholas Bellinger device->cq_active_qps[min_index]++; 102b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 103b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 104b8d26b3bSNicholas Bellinger 105b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 106b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 107b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 108b8d26b3bSNicholas Bellinger attr.send_cq = device->dev_tx_cq[min_index]; 109b8d26b3bSNicholas Bellinger attr.recv_cq = device->dev_rx_cq[min_index]; 110b8d26b3bSNicholas Bellinger attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 111b8d26b3bSNicholas Bellinger attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 112b8d26b3bSNicholas Bellinger /* 113b8d26b3bSNicholas Bellinger * FIXME: Use devattr.max_sge - 2 for max_send_sge as 114b8d26b3bSNicholas Bellinger * work-around for RDMA_READ.. 115b8d26b3bSNicholas Bellinger */ 116b8d26b3bSNicholas Bellinger attr.cap.max_send_sge = devattr.max_sge - 2; 117b8d26b3bSNicholas Bellinger isert_conn->max_sge = attr.cap.max_send_sge; 118b8d26b3bSNicholas Bellinger 119b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 120b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 121b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 122b8d26b3bSNicholas Bellinger 123b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 124b8d26b3bSNicholas Bellinger cma_id->device); 125b8d26b3bSNicholas Bellinger pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 126b8d26b3bSNicholas Bellinger isert_conn->conn_pd->device); 127b8d26b3bSNicholas Bellinger 128b8d26b3bSNicholas Bellinger ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 129b8d26b3bSNicholas Bellinger if (ret) { 130b8d26b3bSNicholas Bellinger pr_err("rdma_create_qp failed for cma_id %d\n", ret); 131b8d26b3bSNicholas Bellinger return ret; 132b8d26b3bSNicholas Bellinger } 133b8d26b3bSNicholas Bellinger isert_conn->conn_qp = cma_id->qp; 134b8d26b3bSNicholas Bellinger pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 135b8d26b3bSNicholas Bellinger 136b8d26b3bSNicholas Bellinger return 0; 137b8d26b3bSNicholas Bellinger } 138b8d26b3bSNicholas Bellinger 139b8d26b3bSNicholas Bellinger static void 140b8d26b3bSNicholas Bellinger isert_cq_event_callback(struct ib_event *e, void *context) 141b8d26b3bSNicholas Bellinger { 142b8d26b3bSNicholas Bellinger pr_debug("isert_cq_event_callback event: %d\n", e->event); 143b8d26b3bSNicholas Bellinger } 144b8d26b3bSNicholas Bellinger 145b8d26b3bSNicholas Bellinger static int 146b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 147b8d26b3bSNicholas Bellinger { 148b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 149b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 150b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 151b8d26b3bSNicholas Bellinger u64 dma_addr; 152b8d26b3bSNicholas Bellinger int i, j; 153b8d26b3bSNicholas Bellinger 154b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 155b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 156b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 157b8d26b3bSNicholas Bellinger goto fail; 158b8d26b3bSNicholas Bellinger 159b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 160b8d26b3bSNicholas Bellinger 161b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 162b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 163b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 164b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 165b8d26b3bSNicholas Bellinger goto dma_map_fail; 166b8d26b3bSNicholas Bellinger 167b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 168b8d26b3bSNicholas Bellinger 169b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 170b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 171b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 172b8d26b3bSNicholas Bellinger rx_sg->lkey = isert_conn->conn_mr->lkey; 173b8d26b3bSNicholas Bellinger } 174b8d26b3bSNicholas Bellinger 175b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = 0; 176b8d26b3bSNicholas Bellinger return 0; 177b8d26b3bSNicholas Bellinger 178b8d26b3bSNicholas Bellinger dma_map_fail: 179b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 180b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 181b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 182b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 183b8d26b3bSNicholas Bellinger } 184b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 185b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 186b8d26b3bSNicholas Bellinger fail: 187b8d26b3bSNicholas Bellinger return -ENOMEM; 188b8d26b3bSNicholas Bellinger } 189b8d26b3bSNicholas Bellinger 190b8d26b3bSNicholas Bellinger static void 191b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 192b8d26b3bSNicholas Bellinger { 193b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 194b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 195b8d26b3bSNicholas Bellinger int i; 196b8d26b3bSNicholas Bellinger 197b8d26b3bSNicholas Bellinger if (!isert_conn->conn_rx_descs) 198b8d26b3bSNicholas Bellinger return; 199b8d26b3bSNicholas Bellinger 200b8d26b3bSNicholas Bellinger rx_desc = isert_conn->conn_rx_descs; 201b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 202b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 203b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 204b8d26b3bSNicholas Bellinger } 205b8d26b3bSNicholas Bellinger 206b8d26b3bSNicholas Bellinger kfree(isert_conn->conn_rx_descs); 207b8d26b3bSNicholas Bellinger isert_conn->conn_rx_descs = NULL; 208b8d26b3bSNicholas Bellinger } 209b8d26b3bSNicholas Bellinger 210b8d26b3bSNicholas Bellinger static void isert_cq_tx_callback(struct ib_cq *, void *); 211b8d26b3bSNicholas Bellinger static void isert_cq_rx_callback(struct ib_cq *, void *); 212b8d26b3bSNicholas Bellinger 213b8d26b3bSNicholas Bellinger static int 214b8d26b3bSNicholas Bellinger isert_create_device_ib_res(struct isert_device *device) 215b8d26b3bSNicholas Bellinger { 216b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = device->ib_device; 217b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc; 218b8d26b3bSNicholas Bellinger int ret = 0, i, j; 219b8d26b3bSNicholas Bellinger 220*d40945d8SVu Pham /* asign function handlers */ 221*d40945d8SVu Pham device->reg_rdma_mem = isert_map_rdma; 222*d40945d8SVu Pham device->unreg_rdma_mem = isert_unmap_cmd; 223*d40945d8SVu Pham 224b8d26b3bSNicholas Bellinger device->cqs_used = min_t(int, num_online_cpus(), 225b8d26b3bSNicholas Bellinger device->ib_device->num_comp_vectors); 226b8d26b3bSNicholas Bellinger device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 227b8d26b3bSNicholas Bellinger pr_debug("Using %d CQs, device %s supports %d vectors\n", 228b8d26b3bSNicholas Bellinger device->cqs_used, device->ib_device->name, 229b8d26b3bSNicholas Bellinger device->ib_device->num_comp_vectors); 230b8d26b3bSNicholas Bellinger device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 231b8d26b3bSNicholas Bellinger device->cqs_used, GFP_KERNEL); 232b8d26b3bSNicholas Bellinger if (!device->cq_desc) { 233b8d26b3bSNicholas Bellinger pr_err("Unable to allocate device->cq_desc\n"); 234b8d26b3bSNicholas Bellinger return -ENOMEM; 235b8d26b3bSNicholas Bellinger } 236b8d26b3bSNicholas Bellinger cq_desc = device->cq_desc; 237b8d26b3bSNicholas Bellinger 238b8d26b3bSNicholas Bellinger device->dev_pd = ib_alloc_pd(ib_dev); 239b8d26b3bSNicholas Bellinger if (IS_ERR(device->dev_pd)) { 240b8d26b3bSNicholas Bellinger ret = PTR_ERR(device->dev_pd); 241b8d26b3bSNicholas Bellinger pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret); 242b8d26b3bSNicholas Bellinger goto out_cq_desc; 243b8d26b3bSNicholas Bellinger } 244b8d26b3bSNicholas Bellinger 245b8d26b3bSNicholas Bellinger for (i = 0; i < device->cqs_used; i++) { 246b8d26b3bSNicholas Bellinger cq_desc[i].device = device; 247b8d26b3bSNicholas Bellinger cq_desc[i].cq_index = i; 248b8d26b3bSNicholas Bellinger 249b8d26b3bSNicholas Bellinger device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 250b8d26b3bSNicholas Bellinger isert_cq_rx_callback, 251b8d26b3bSNicholas Bellinger isert_cq_event_callback, 252b8d26b3bSNicholas Bellinger (void *)&cq_desc[i], 253b8d26b3bSNicholas Bellinger ISER_MAX_RX_CQ_LEN, i); 254b8d26b3bSNicholas Bellinger if (IS_ERR(device->dev_rx_cq[i])) 255b8d26b3bSNicholas Bellinger goto out_cq; 256b8d26b3bSNicholas Bellinger 257b8d26b3bSNicholas Bellinger device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 258b8d26b3bSNicholas Bellinger isert_cq_tx_callback, 259b8d26b3bSNicholas Bellinger isert_cq_event_callback, 260b8d26b3bSNicholas Bellinger (void *)&cq_desc[i], 261b8d26b3bSNicholas Bellinger ISER_MAX_TX_CQ_LEN, i); 262b8d26b3bSNicholas Bellinger if (IS_ERR(device->dev_tx_cq[i])) 263b8d26b3bSNicholas Bellinger goto out_cq; 264b8d26b3bSNicholas Bellinger 265b8d26b3bSNicholas Bellinger if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP)) 266b8d26b3bSNicholas Bellinger goto out_cq; 267b8d26b3bSNicholas Bellinger 268b8d26b3bSNicholas Bellinger if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP)) 269b8d26b3bSNicholas Bellinger goto out_cq; 270b8d26b3bSNicholas Bellinger } 271b8d26b3bSNicholas Bellinger 272b8d26b3bSNicholas Bellinger device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE); 273b8d26b3bSNicholas Bellinger if (IS_ERR(device->dev_mr)) { 274b8d26b3bSNicholas Bellinger ret = PTR_ERR(device->dev_mr); 275b8d26b3bSNicholas Bellinger pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret); 276b8d26b3bSNicholas Bellinger goto out_cq; 277b8d26b3bSNicholas Bellinger } 278b8d26b3bSNicholas Bellinger 279b8d26b3bSNicholas Bellinger return 0; 280b8d26b3bSNicholas Bellinger 281b8d26b3bSNicholas Bellinger out_cq: 282b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++) { 283b8d26b3bSNicholas Bellinger cq_desc = &device->cq_desc[j]; 284b8d26b3bSNicholas Bellinger 285b8d26b3bSNicholas Bellinger if (device->dev_rx_cq[j]) { 286b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_rx_work); 287b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_rx_cq[j]); 288b8d26b3bSNicholas Bellinger } 289b8d26b3bSNicholas Bellinger if (device->dev_tx_cq[j]) { 290b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_tx_work); 291b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_tx_cq[j]); 292b8d26b3bSNicholas Bellinger } 293b8d26b3bSNicholas Bellinger } 294b8d26b3bSNicholas Bellinger ib_dealloc_pd(device->dev_pd); 295b8d26b3bSNicholas Bellinger 296b8d26b3bSNicholas Bellinger out_cq_desc: 297b8d26b3bSNicholas Bellinger kfree(device->cq_desc); 298b8d26b3bSNicholas Bellinger 299b8d26b3bSNicholas Bellinger return ret; 300b8d26b3bSNicholas Bellinger } 301b8d26b3bSNicholas Bellinger 302b8d26b3bSNicholas Bellinger static void 303b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 304b8d26b3bSNicholas Bellinger { 305b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc; 306b8d26b3bSNicholas Bellinger int i; 307b8d26b3bSNicholas Bellinger 308b8d26b3bSNicholas Bellinger for (i = 0; i < device->cqs_used; i++) { 309b8d26b3bSNicholas Bellinger cq_desc = &device->cq_desc[i]; 310b8d26b3bSNicholas Bellinger 311b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_rx_work); 312b8d26b3bSNicholas Bellinger cancel_work_sync(&cq_desc->cq_tx_work); 313b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_rx_cq[i]); 314b8d26b3bSNicholas Bellinger ib_destroy_cq(device->dev_tx_cq[i]); 315b8d26b3bSNicholas Bellinger device->dev_rx_cq[i] = NULL; 316b8d26b3bSNicholas Bellinger device->dev_tx_cq[i] = NULL; 317b8d26b3bSNicholas Bellinger } 318b8d26b3bSNicholas Bellinger 319b8d26b3bSNicholas Bellinger ib_dereg_mr(device->dev_mr); 320b8d26b3bSNicholas Bellinger ib_dealloc_pd(device->dev_pd); 321b8d26b3bSNicholas Bellinger kfree(device->cq_desc); 322b8d26b3bSNicholas Bellinger } 323b8d26b3bSNicholas Bellinger 324b8d26b3bSNicholas Bellinger static void 325b8d26b3bSNicholas Bellinger isert_device_try_release(struct isert_device *device) 326b8d26b3bSNicholas Bellinger { 327b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 328b8d26b3bSNicholas Bellinger device->refcount--; 329b8d26b3bSNicholas Bellinger if (!device->refcount) { 330b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 331b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 332b8d26b3bSNicholas Bellinger kfree(device); 333b8d26b3bSNicholas Bellinger } 334b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 335b8d26b3bSNicholas Bellinger } 336b8d26b3bSNicholas Bellinger 337b8d26b3bSNicholas Bellinger static struct isert_device * 338b8d26b3bSNicholas Bellinger isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 339b8d26b3bSNicholas Bellinger { 340b8d26b3bSNicholas Bellinger struct isert_device *device; 341b8d26b3bSNicholas Bellinger int ret; 342b8d26b3bSNicholas Bellinger 343b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 344b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 345b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 346b8d26b3bSNicholas Bellinger device->refcount++; 347b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 348b8d26b3bSNicholas Bellinger return device; 349b8d26b3bSNicholas Bellinger } 350b8d26b3bSNicholas Bellinger } 351b8d26b3bSNicholas Bellinger 352b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 353b8d26b3bSNicholas Bellinger if (!device) { 354b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 355b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 356b8d26b3bSNicholas Bellinger } 357b8d26b3bSNicholas Bellinger 358b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 359b8d26b3bSNicholas Bellinger 360b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 361b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 362b8d26b3bSNicholas Bellinger if (ret) { 363b8d26b3bSNicholas Bellinger kfree(device); 364b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 365b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 366b8d26b3bSNicholas Bellinger } 367b8d26b3bSNicholas Bellinger 368b8d26b3bSNicholas Bellinger device->refcount++; 369b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 370b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 371b8d26b3bSNicholas Bellinger 372b8d26b3bSNicholas Bellinger return device; 373b8d26b3bSNicholas Bellinger } 374b8d26b3bSNicholas Bellinger 375b8d26b3bSNicholas Bellinger static int 376b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 377b8d26b3bSNicholas Bellinger { 378b8d26b3bSNicholas Bellinger struct iscsi_np *np = cma_id->context; 379b8d26b3bSNicholas Bellinger struct isert_np *isert_np = np->np_context; 380b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 381b8d26b3bSNicholas Bellinger struct isert_device *device; 382b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = cma_id->device; 383b8d26b3bSNicholas Bellinger int ret = 0; 384b8d26b3bSNicholas Bellinger 385b8d26b3bSNicholas Bellinger pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 386b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 387b8d26b3bSNicholas Bellinger 388b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 389b8d26b3bSNicholas Bellinger if (!isert_conn) { 390b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_conn\n"); 391b8d26b3bSNicholas Bellinger return -ENOMEM; 392b8d26b3bSNicholas Bellinger } 393b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_INIT; 394b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_conn->conn_accept_node); 395b8d26b3bSNicholas Bellinger init_completion(&isert_conn->conn_login_comp); 396b8d26b3bSNicholas Bellinger init_waitqueue_head(&isert_conn->conn_wait); 397b8d26b3bSNicholas Bellinger init_waitqueue_head(&isert_conn->conn_wait_comp_err); 398b8d26b3bSNicholas Bellinger kref_init(&isert_conn->conn_kref); 399b8d26b3bSNicholas Bellinger kref_get(&isert_conn->conn_kref); 400b2cb9649SNicholas Bellinger mutex_init(&isert_conn->conn_mutex); 401b8d26b3bSNicholas Bellinger 402b8d26b3bSNicholas Bellinger cma_id->context = isert_conn; 403b8d26b3bSNicholas Bellinger isert_conn->conn_cm_id = cma_id; 404b8d26b3bSNicholas Bellinger isert_conn->responder_resources = event->param.conn.responder_resources; 405b8d26b3bSNicholas Bellinger isert_conn->initiator_depth = event->param.conn.initiator_depth; 406b8d26b3bSNicholas Bellinger pr_debug("Using responder_resources: %u initiator_depth: %u\n", 407b8d26b3bSNicholas Bellinger isert_conn->responder_resources, isert_conn->initiator_depth); 408b8d26b3bSNicholas Bellinger 409b8d26b3bSNicholas Bellinger isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 410b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, GFP_KERNEL); 411b8d26b3bSNicholas Bellinger if (!isert_conn->login_buf) { 412b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_conn->login_buf\n"); 413b8d26b3bSNicholas Bellinger ret = -ENOMEM; 414b8d26b3bSNicholas Bellinger goto out; 415b8d26b3bSNicholas Bellinger } 416b8d26b3bSNicholas Bellinger 417b8d26b3bSNicholas Bellinger isert_conn->login_req_buf = isert_conn->login_buf; 418b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf = isert_conn->login_buf + 419b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN; 420b8d26b3bSNicholas Bellinger pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 421b8d26b3bSNicholas Bellinger isert_conn->login_buf, isert_conn->login_req_buf, 422b8d26b3bSNicholas Bellinger isert_conn->login_rsp_buf); 423b8d26b3bSNicholas Bellinger 424b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 425b8d26b3bSNicholas Bellinger (void *)isert_conn->login_req_buf, 426b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 427b8d26b3bSNicholas Bellinger 428b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 429b8d26b3bSNicholas Bellinger if (ret) { 430b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 431b8d26b3bSNicholas Bellinger ret); 432b8d26b3bSNicholas Bellinger isert_conn->login_req_dma = 0; 433b8d26b3bSNicholas Bellinger goto out_login_buf; 434b8d26b3bSNicholas Bellinger } 435b8d26b3bSNicholas Bellinger 436b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 437b8d26b3bSNicholas Bellinger (void *)isert_conn->login_rsp_buf, 438b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 439b8d26b3bSNicholas Bellinger 440b8d26b3bSNicholas Bellinger ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 441b8d26b3bSNicholas Bellinger if (ret) { 442b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 443b8d26b3bSNicholas Bellinger ret); 444b8d26b3bSNicholas Bellinger isert_conn->login_rsp_dma = 0; 445b8d26b3bSNicholas Bellinger goto out_req_dma_map; 446b8d26b3bSNicholas Bellinger } 447b8d26b3bSNicholas Bellinger 448b8d26b3bSNicholas Bellinger device = isert_device_find_by_ib_dev(cma_id); 449b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 450b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 451b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 452b8d26b3bSNicholas Bellinger } 453b8d26b3bSNicholas Bellinger 454b8d26b3bSNicholas Bellinger isert_conn->conn_device = device; 455b8d26b3bSNicholas Bellinger isert_conn->conn_pd = device->dev_pd; 456b8d26b3bSNicholas Bellinger isert_conn->conn_mr = device->dev_mr; 457b8d26b3bSNicholas Bellinger 458b8d26b3bSNicholas Bellinger ret = isert_conn_setup_qp(isert_conn, cma_id); 459b8d26b3bSNicholas Bellinger if (ret) 460b8d26b3bSNicholas Bellinger goto out_conn_dev; 461b8d26b3bSNicholas Bellinger 462b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 463b8d26b3bSNicholas Bellinger list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 464b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 465b8d26b3bSNicholas Bellinger 466b8d26b3bSNicholas Bellinger pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 467b8d26b3bSNicholas Bellinger wake_up(&isert_np->np_accept_wq); 468b8d26b3bSNicholas Bellinger return 0; 469b8d26b3bSNicholas Bellinger 470b8d26b3bSNicholas Bellinger out_conn_dev: 471b8d26b3bSNicholas Bellinger isert_device_try_release(device); 472b8d26b3bSNicholas Bellinger out_rsp_dma_map: 473b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 474b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 475b8d26b3bSNicholas Bellinger out_req_dma_map: 476b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 477b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 478b8d26b3bSNicholas Bellinger out_login_buf: 479b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 480b8d26b3bSNicholas Bellinger out: 481b8d26b3bSNicholas Bellinger kfree(isert_conn); 482b8d26b3bSNicholas Bellinger return ret; 483b8d26b3bSNicholas Bellinger } 484b8d26b3bSNicholas Bellinger 485b8d26b3bSNicholas Bellinger static void 486b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 487b8d26b3bSNicholas Bellinger { 488b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 489b8d26b3bSNicholas Bellinger struct isert_device *device = isert_conn->conn_device; 490b8d26b3bSNicholas Bellinger int cq_index; 491b8d26b3bSNicholas Bellinger 492b8d26b3bSNicholas Bellinger pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 493b8d26b3bSNicholas Bellinger 494b8d26b3bSNicholas Bellinger if (isert_conn->conn_qp) { 495b8d26b3bSNicholas Bellinger cq_index = ((struct isert_cq_desc *) 496b8d26b3bSNicholas Bellinger isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 497b8d26b3bSNicholas Bellinger pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 498b8d26b3bSNicholas Bellinger isert_conn->conn_device->cq_active_qps[cq_index]--; 499b8d26b3bSNicholas Bellinger 500b8d26b3bSNicholas Bellinger rdma_destroy_qp(isert_conn->conn_cm_id); 501b8d26b3bSNicholas Bellinger } 502b8d26b3bSNicholas Bellinger 503b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(isert_conn); 504b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_conn->conn_cm_id); 505b8d26b3bSNicholas Bellinger 506b8d26b3bSNicholas Bellinger if (isert_conn->login_buf) { 507b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 508b8d26b3bSNicholas Bellinger ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 509b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 510b8d26b3bSNicholas Bellinger ISCSI_DEF_MAX_RECV_SEG_LEN, 511b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 512b8d26b3bSNicholas Bellinger kfree(isert_conn->login_buf); 513b8d26b3bSNicholas Bellinger } 514b8d26b3bSNicholas Bellinger kfree(isert_conn); 515b8d26b3bSNicholas Bellinger 516b8d26b3bSNicholas Bellinger if (device) 517b8d26b3bSNicholas Bellinger isert_device_try_release(device); 518b8d26b3bSNicholas Bellinger 519b8d26b3bSNicholas Bellinger pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 520b8d26b3bSNicholas Bellinger } 521b8d26b3bSNicholas Bellinger 522b8d26b3bSNicholas Bellinger static void 523b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 524b8d26b3bSNicholas Bellinger { 525b8d26b3bSNicholas Bellinger return; 526b8d26b3bSNicholas Bellinger } 527b8d26b3bSNicholas Bellinger 528b8d26b3bSNicholas Bellinger static void 529b8d26b3bSNicholas Bellinger isert_release_conn_kref(struct kref *kref) 530b8d26b3bSNicholas Bellinger { 531b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 532b8d26b3bSNicholas Bellinger struct isert_conn, conn_kref); 533b8d26b3bSNicholas Bellinger 534b8d26b3bSNicholas Bellinger pr_debug("Calling isert_connect_release for final kref %s/%d\n", 535b8d26b3bSNicholas Bellinger current->comm, current->pid); 536b8d26b3bSNicholas Bellinger 537b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 538b8d26b3bSNicholas Bellinger } 539b8d26b3bSNicholas Bellinger 540b8d26b3bSNicholas Bellinger static void 541b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 542b8d26b3bSNicholas Bellinger { 543b8d26b3bSNicholas Bellinger kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 544b8d26b3bSNicholas Bellinger } 545b8d26b3bSNicholas Bellinger 546b8d26b3bSNicholas Bellinger static void 547b8d26b3bSNicholas Bellinger isert_disconnect_work(struct work_struct *work) 548b8d26b3bSNicholas Bellinger { 549b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(work, 550b8d26b3bSNicholas Bellinger struct isert_conn, conn_logout_work); 551b8d26b3bSNicholas Bellinger 552b8d26b3bSNicholas Bellinger pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 553b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 554b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_DOWN; 555b8d26b3bSNicholas Bellinger 556b8d26b3bSNicholas Bellinger if (isert_conn->post_recv_buf_count == 0 && 557b8d26b3bSNicholas Bellinger atomic_read(&isert_conn->post_send_buf_count) == 0) { 558b8d26b3bSNicholas Bellinger pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); 559b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 560b2cb9649SNicholas Bellinger goto wake_up; 561b8d26b3bSNicholas Bellinger } 562b2cb9649SNicholas Bellinger if (!isert_conn->conn_cm_id) { 563b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 564b2cb9649SNicholas Bellinger isert_put_conn(isert_conn); 565b2cb9649SNicholas Bellinger return; 566b2cb9649SNicholas Bellinger } 567b2cb9649SNicholas Bellinger if (!isert_conn->logout_posted) { 568b2cb9649SNicholas Bellinger pr_debug("Calling rdma_disconnect for !logout_posted from" 569b2cb9649SNicholas Bellinger " isert_disconnect_work\n"); 570b2cb9649SNicholas Bellinger rdma_disconnect(isert_conn->conn_cm_id); 571b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 572b2cb9649SNicholas Bellinger iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 573b2cb9649SNicholas Bellinger goto wake_up; 574b2cb9649SNicholas Bellinger } 575b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 576b8d26b3bSNicholas Bellinger 577b2cb9649SNicholas Bellinger wake_up: 578b2cb9649SNicholas Bellinger wake_up(&isert_conn->conn_wait); 579b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 580b8d26b3bSNicholas Bellinger } 581b8d26b3bSNicholas Bellinger 582b8d26b3bSNicholas Bellinger static void 583b8d26b3bSNicholas Bellinger isert_disconnected_handler(struct rdma_cm_id *cma_id) 584b8d26b3bSNicholas Bellinger { 585b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 586b8d26b3bSNicholas Bellinger 587b8d26b3bSNicholas Bellinger INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 588b8d26b3bSNicholas Bellinger schedule_work(&isert_conn->conn_logout_work); 589b8d26b3bSNicholas Bellinger } 590b8d26b3bSNicholas Bellinger 591b8d26b3bSNicholas Bellinger static int 592b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 593b8d26b3bSNicholas Bellinger { 594b8d26b3bSNicholas Bellinger int ret = 0; 595b8d26b3bSNicholas Bellinger 596b8d26b3bSNicholas Bellinger pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 597b8d26b3bSNicholas Bellinger event->event, event->status, cma_id->context, cma_id); 598b8d26b3bSNicholas Bellinger 599b8d26b3bSNicholas Bellinger switch (event->event) { 600b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 601b8d26b3bSNicholas Bellinger pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 602b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 603b8d26b3bSNicholas Bellinger break; 604b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 605b8d26b3bSNicholas Bellinger pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 606b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 607b8d26b3bSNicholas Bellinger break; 608b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_DISCONNECTED: 609b8d26b3bSNicholas Bellinger pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 610b8d26b3bSNicholas Bellinger isert_disconnected_handler(cma_id); 611b8d26b3bSNicholas Bellinger break; 612b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_DEVICE_REMOVAL: 613b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ADDR_CHANGE: 614b8d26b3bSNicholas Bellinger break; 615b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 616b8d26b3bSNicholas Bellinger default: 617b8d26b3bSNicholas Bellinger pr_err("Unknown RDMA CMA event: %d\n", event->event); 618b8d26b3bSNicholas Bellinger break; 619b8d26b3bSNicholas Bellinger } 620b8d26b3bSNicholas Bellinger 621b8d26b3bSNicholas Bellinger if (ret != 0) { 622b8d26b3bSNicholas Bellinger pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 623b8d26b3bSNicholas Bellinger event->event, ret); 624b8d26b3bSNicholas Bellinger dump_stack(); 625b8d26b3bSNicholas Bellinger } 626b8d26b3bSNicholas Bellinger 627b8d26b3bSNicholas Bellinger return ret; 628b8d26b3bSNicholas Bellinger } 629b8d26b3bSNicholas Bellinger 630b8d26b3bSNicholas Bellinger static int 631b8d26b3bSNicholas Bellinger isert_post_recv(struct isert_conn *isert_conn, u32 count) 632b8d26b3bSNicholas Bellinger { 633b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 634b8d26b3bSNicholas Bellinger int i, ret; 635b8d26b3bSNicholas Bellinger unsigned int rx_head = isert_conn->conn_rx_desc_head; 636b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 637b8d26b3bSNicholas Bellinger 638b8d26b3bSNicholas Bellinger for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 639b8d26b3bSNicholas Bellinger rx_desc = &isert_conn->conn_rx_descs[rx_head]; 640b8d26b3bSNicholas Bellinger rx_wr->wr_id = (unsigned long)rx_desc; 641b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 642b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 643b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 644b8d26b3bSNicholas Bellinger rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 645b8d26b3bSNicholas Bellinger } 646b8d26b3bSNicholas Bellinger 647b8d26b3bSNicholas Bellinger rx_wr--; 648b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 649b8d26b3bSNicholas Bellinger 650b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count += count; 651b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 652b8d26b3bSNicholas Bellinger &rx_wr_failed); 653b8d26b3bSNicholas Bellinger if (ret) { 654b8d26b3bSNicholas Bellinger pr_err("ib_post_recv() failed with ret: %d\n", ret); 655b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count -= count; 656b8d26b3bSNicholas Bellinger } else { 657b8d26b3bSNicholas Bellinger pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 658b8d26b3bSNicholas Bellinger isert_conn->conn_rx_desc_head = rx_head; 659b8d26b3bSNicholas Bellinger } 660b8d26b3bSNicholas Bellinger return ret; 661b8d26b3bSNicholas Bellinger } 662b8d26b3bSNicholas Bellinger 663b8d26b3bSNicholas Bellinger static int 664b8d26b3bSNicholas Bellinger isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 665b8d26b3bSNicholas Bellinger { 666b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 667b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 668b8d26b3bSNicholas Bellinger int ret; 669b8d26b3bSNicholas Bellinger 670b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 671b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 672b8d26b3bSNicholas Bellinger 673b8d26b3bSNicholas Bellinger send_wr.next = NULL; 674b8d26b3bSNicholas Bellinger send_wr.wr_id = (unsigned long)tx_desc; 675b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 676b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 677b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 678b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 679b8d26b3bSNicholas Bellinger 680b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 681b8d26b3bSNicholas Bellinger 682b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 683b8d26b3bSNicholas Bellinger if (ret) { 684b8d26b3bSNicholas Bellinger pr_err("ib_post_send() failed, ret: %d\n", ret); 685b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 686b8d26b3bSNicholas Bellinger } 687b8d26b3bSNicholas Bellinger 688b8d26b3bSNicholas Bellinger return ret; 689b8d26b3bSNicholas Bellinger } 690b8d26b3bSNicholas Bellinger 691b8d26b3bSNicholas Bellinger static void 692b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 693b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 694b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 695b8d26b3bSNicholas Bellinger { 696b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 697b8d26b3bSNicholas Bellinger 698b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 699b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 700b8d26b3bSNicholas Bellinger 701b8d26b3bSNicholas Bellinger memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 702b8d26b3bSNicholas Bellinger tx_desc->iser_header.flags = ISER_VER; 703b8d26b3bSNicholas Bellinger 704b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 705b8d26b3bSNicholas Bellinger tx_desc->isert_cmd = isert_cmd; 706b8d26b3bSNicholas Bellinger 707b8d26b3bSNicholas Bellinger if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 708b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 709b8d26b3bSNicholas Bellinger pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 710b8d26b3bSNicholas Bellinger } 711b8d26b3bSNicholas Bellinger } 712b8d26b3bSNicholas Bellinger 713b8d26b3bSNicholas Bellinger static int 714b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 715b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 716b8d26b3bSNicholas Bellinger { 717b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 718b8d26b3bSNicholas Bellinger u64 dma_addr; 719b8d26b3bSNicholas Bellinger 720b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 721b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 722b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 723b8d26b3bSNicholas Bellinger pr_err("ib_dma_mapping_error() failed\n"); 724b8d26b3bSNicholas Bellinger return -ENOMEM; 725b8d26b3bSNicholas Bellinger } 726b8d26b3bSNicholas Bellinger 727b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 728b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 729b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 730b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 731b8d26b3bSNicholas Bellinger 732b8d26b3bSNicholas Bellinger pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 733b8d26b3bSNicholas Bellinger " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 734b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 735b8d26b3bSNicholas Bellinger 736b8d26b3bSNicholas Bellinger return 0; 737b8d26b3bSNicholas Bellinger } 738b8d26b3bSNicholas Bellinger 739b8d26b3bSNicholas Bellinger static void 740b8d26b3bSNicholas Bellinger isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr) 741b8d26b3bSNicholas Bellinger { 742b8d26b3bSNicholas Bellinger isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 743b8d26b3bSNicholas Bellinger send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 744b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 745b8d26b3bSNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 746b8d26b3bSNicholas Bellinger send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0]; 747b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 748b8d26b3bSNicholas Bellinger } 749b8d26b3bSNicholas Bellinger 750b8d26b3bSNicholas Bellinger static int 751b8d26b3bSNicholas Bellinger isert_rdma_post_recvl(struct isert_conn *isert_conn) 752b8d26b3bSNicholas Bellinger { 753b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 754b8d26b3bSNicholas Bellinger struct ib_sge sge; 755b8d26b3bSNicholas Bellinger int ret; 756b8d26b3bSNicholas Bellinger 757b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 758b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 759b8d26b3bSNicholas Bellinger sge.length = ISER_RX_LOGIN_SIZE; 760b8d26b3bSNicholas Bellinger sge.lkey = isert_conn->conn_mr->lkey; 761b8d26b3bSNicholas Bellinger 762b8d26b3bSNicholas Bellinger pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 763b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 764b8d26b3bSNicholas Bellinger 765b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 766b8d26b3bSNicholas Bellinger rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 767b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 768b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 769b8d26b3bSNicholas Bellinger 770b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count++; 771b8d26b3bSNicholas Bellinger ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 772b8d26b3bSNicholas Bellinger if (ret) { 773b8d26b3bSNicholas Bellinger pr_err("ib_post_recv() failed: %d\n", ret); 774b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 775b8d26b3bSNicholas Bellinger } 776b8d26b3bSNicholas Bellinger 777b8d26b3bSNicholas Bellinger pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 778b8d26b3bSNicholas Bellinger return ret; 779b8d26b3bSNicholas Bellinger } 780b8d26b3bSNicholas Bellinger 781b8d26b3bSNicholas Bellinger static int 782b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 783b8d26b3bSNicholas Bellinger u32 length) 784b8d26b3bSNicholas Bellinger { 785b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 786b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 787b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 788b8d26b3bSNicholas Bellinger int ret; 789b8d26b3bSNicholas Bellinger 790b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 791b8d26b3bSNicholas Bellinger 792b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 793b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 794b8d26b3bSNicholas Bellinger 795b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 796b8d26b3bSNicholas Bellinger 797b8d26b3bSNicholas Bellinger if (length > 0) { 798b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 799b8d26b3bSNicholas Bellinger 800b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 801b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 802b8d26b3bSNicholas Bellinger 803b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 804b8d26b3bSNicholas Bellinger 805b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 806b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 807b8d26b3bSNicholas Bellinger 808b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 809b8d26b3bSNicholas Bellinger tx_dsg->length = length; 810b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 811b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 812b8d26b3bSNicholas Bellinger } 813b8d26b3bSNicholas Bellinger if (!login->login_failed) { 814b8d26b3bSNicholas Bellinger if (login->login_complete) { 815b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 816b8d26b3bSNicholas Bellinger if (ret) 817b8d26b3bSNicholas Bellinger return ret; 818b8d26b3bSNicholas Bellinger 819b8d26b3bSNicholas Bellinger ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 820b8d26b3bSNicholas Bellinger if (ret) 821b8d26b3bSNicholas Bellinger return ret; 822b8d26b3bSNicholas Bellinger 823b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_UP; 824b8d26b3bSNicholas Bellinger goto post_send; 825b8d26b3bSNicholas Bellinger } 826b8d26b3bSNicholas Bellinger 827b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 828b8d26b3bSNicholas Bellinger if (ret) 829b8d26b3bSNicholas Bellinger return ret; 830b8d26b3bSNicholas Bellinger } 831b8d26b3bSNicholas Bellinger post_send: 832b8d26b3bSNicholas Bellinger ret = isert_post_send(isert_conn, tx_desc); 833b8d26b3bSNicholas Bellinger if (ret) 834b8d26b3bSNicholas Bellinger return ret; 835b8d26b3bSNicholas Bellinger 836b8d26b3bSNicholas Bellinger return 0; 837b8d26b3bSNicholas Bellinger } 838b8d26b3bSNicholas Bellinger 839b8d26b3bSNicholas Bellinger static void 840b8d26b3bSNicholas Bellinger isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 841b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 842b8d26b3bSNicholas Bellinger { 843b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 844b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 845b8d26b3bSNicholas Bellinger int size; 846b8d26b3bSNicholas Bellinger 847b8d26b3bSNicholas Bellinger if (!login) { 848b8d26b3bSNicholas Bellinger pr_err("conn->conn_login is NULL\n"); 849b8d26b3bSNicholas Bellinger dump_stack(); 850b8d26b3bSNicholas Bellinger return; 851b8d26b3bSNicholas Bellinger } 852b8d26b3bSNicholas Bellinger 853b8d26b3bSNicholas Bellinger if (login->first_request) { 854b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 855b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 856b8d26b3bSNicholas Bellinger /* 857b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 858b8d26b3bSNicholas Bellinger * login request PDU. 859b8d26b3bSNicholas Bellinger */ 860b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 861b8d26b3bSNicholas Bellinger login->current_stage = 862b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 863b8d26b3bSNicholas Bellinger >> 2; 864b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 865b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 866b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 867b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 868b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 869b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 870b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 871b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 872b8d26b3bSNicholas Bellinger } 873b8d26b3bSNicholas Bellinger 874b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 875b8d26b3bSNicholas Bellinger 876b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 877b8d26b3bSNicholas Bellinger pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 878b8d26b3bSNicholas Bellinger size, rx_buflen, MAX_KEY_VALUE_PAIRS); 879b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 880b8d26b3bSNicholas Bellinger 8816faaa85fSNicholas Bellinger if (login->first_request) { 882b8d26b3bSNicholas Bellinger complete(&isert_conn->conn_login_comp); 8836faaa85fSNicholas Bellinger return; 8846faaa85fSNicholas Bellinger } 8856faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 886b8d26b3bSNicholas Bellinger } 887b8d26b3bSNicholas Bellinger 888b8d26b3bSNicholas Bellinger static struct iscsi_cmd 889d703ce2fSNicholas Bellinger *isert_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp) 890b8d26b3bSNicholas Bellinger { 891b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 892b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 893d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 894b8d26b3bSNicholas Bellinger 895d703ce2fSNicholas Bellinger cmd = iscsit_allocate_cmd(conn, gfp); 896d703ce2fSNicholas Bellinger if (!cmd) { 897d703ce2fSNicholas Bellinger pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 898b8d26b3bSNicholas Bellinger return NULL; 899b8d26b3bSNicholas Bellinger } 900d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 901b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 902d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 903b8d26b3bSNicholas Bellinger 904d703ce2fSNicholas Bellinger return cmd; 905b8d26b3bSNicholas Bellinger } 906b8d26b3bSNicholas Bellinger 907b8d26b3bSNicholas Bellinger static int 908b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 909d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 910d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 911b8d26b3bSNicholas Bellinger { 912b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 913b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 914b8d26b3bSNicholas Bellinger struct scatterlist *sg; 915b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 916b8d26b3bSNicholas Bellinger bool dump_payload = false; 917b8d26b3bSNicholas Bellinger 918b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 919b8d26b3bSNicholas Bellinger if (rc < 0) 920b8d26b3bSNicholas Bellinger return rc; 921b8d26b3bSNicholas Bellinger 922b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 923b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 924b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 925b8d26b3bSNicholas Bellinger 926b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 927b8d26b3bSNicholas Bellinger if (rc < 0) { 928b8d26b3bSNicholas Bellinger return 0; 929b8d26b3bSNicholas Bellinger } else if (rc > 0) { 930b8d26b3bSNicholas Bellinger dump_payload = true; 931b8d26b3bSNicholas Bellinger goto sequence_cmd; 932b8d26b3bSNicholas Bellinger } 933b8d26b3bSNicholas Bellinger 934b8d26b3bSNicholas Bellinger if (!imm_data) 935b8d26b3bSNicholas Bellinger return 0; 936b8d26b3bSNicholas Bellinger 937b8d26b3bSNicholas Bellinger sg = &cmd->se_cmd.t_data_sg[0]; 938b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 939b8d26b3bSNicholas Bellinger 940b8d26b3bSNicholas Bellinger pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 941b8d26b3bSNicholas Bellinger sg, sg_nents, &rx_desc->data[0], imm_data_len); 942b8d26b3bSNicholas Bellinger 943b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 944b8d26b3bSNicholas Bellinger 945b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 946b8d26b3bSNicholas Bellinger 947b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 948b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 949b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 950b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 951b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 952b8d26b3bSNicholas Bellinger } 953b8d26b3bSNicholas Bellinger 954b8d26b3bSNicholas Bellinger sequence_cmd: 955561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 956b8d26b3bSNicholas Bellinger 957b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 958b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 959b8d26b3bSNicholas Bellinger 960b8d26b3bSNicholas Bellinger return 0; 961b8d26b3bSNicholas Bellinger } 962b8d26b3bSNicholas Bellinger 963b8d26b3bSNicholas Bellinger static int 964b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 965b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 966b8d26b3bSNicholas Bellinger { 967b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 968b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 969b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 970b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 971b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 972b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 973b8d26b3bSNicholas Bellinger 974b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 975b8d26b3bSNicholas Bellinger if (rc < 0) 976b8d26b3bSNicholas Bellinger return rc; 977b8d26b3bSNicholas Bellinger else if (!cmd) 978b8d26b3bSNicholas Bellinger return 0; 979b8d26b3bSNicholas Bellinger /* 980b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 981b8d26b3bSNicholas Bellinger */ 982b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 983b8d26b3bSNicholas Bellinger pr_err("Received unexpected solicited data payload\n"); 984b8d26b3bSNicholas Bellinger dump_stack(); 985b8d26b3bSNicholas Bellinger return -1; 986b8d26b3bSNicholas Bellinger } 987b8d26b3bSNicholas Bellinger 988b8d26b3bSNicholas Bellinger pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 989b8d26b3bSNicholas Bellinger unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 990b8d26b3bSNicholas Bellinger 991b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 992b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 993b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 994b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 995b8d26b3bSNicholas Bellinger /* 996b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 997b8d26b3bSNicholas Bellinger */ 998b8d26b3bSNicholas Bellinger if (page_off) { 999b8d26b3bSNicholas Bellinger pr_err("Received unexpected non-page aligned data payload\n"); 1000b8d26b3bSNicholas Bellinger dump_stack(); 1001b8d26b3bSNicholas Bellinger return -1; 1002b8d26b3bSNicholas Bellinger } 1003b8d26b3bSNicholas Bellinger pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1004b8d26b3bSNicholas Bellinger sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1005b8d26b3bSNicholas Bellinger 1006b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1007b8d26b3bSNicholas Bellinger unsol_data_len); 1008b8d26b3bSNicholas Bellinger 1009b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1010b8d26b3bSNicholas Bellinger if (rc < 0) 1011b8d26b3bSNicholas Bellinger return rc; 1012b8d26b3bSNicholas Bellinger 1013b8d26b3bSNicholas Bellinger return 0; 1014b8d26b3bSNicholas Bellinger } 1015b8d26b3bSNicholas Bellinger 1016b8d26b3bSNicholas Bellinger static int 1017778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1018d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1019d703ce2fSNicholas Bellinger unsigned char *buf) 1020778de368SNicholas Bellinger { 1021778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1022778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1023778de368SNicholas Bellinger int rc; 1024778de368SNicholas Bellinger 1025778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1026778de368SNicholas Bellinger if (rc < 0) 1027778de368SNicholas Bellinger return rc; 1028778de368SNicholas Bellinger /* 1029778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1030778de368SNicholas Bellinger */ 1031778de368SNicholas Bellinger 1032778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1033778de368SNicholas Bellinger } 1034778de368SNicholas Bellinger 1035778de368SNicholas Bellinger static int 1036adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1037d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1038d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1039adb54c29SNicholas Bellinger { 1040adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1041adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1042adb54c29SNicholas Bellinger int rc; 1043adb54c29SNicholas Bellinger unsigned char *text_in; 1044adb54c29SNicholas Bellinger 1045adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1046adb54c29SNicholas Bellinger if (rc < 0) 1047adb54c29SNicholas Bellinger return rc; 1048adb54c29SNicholas Bellinger 1049adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 1050adb54c29SNicholas Bellinger if (!text_in) { 1051adb54c29SNicholas Bellinger pr_err("Unable to allocate text_in of payload_length: %u\n", 1052adb54c29SNicholas Bellinger payload_length); 1053adb54c29SNicholas Bellinger return -ENOMEM; 1054adb54c29SNicholas Bellinger } 1055adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1056adb54c29SNicholas Bellinger 1057adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1058adb54c29SNicholas Bellinger 1059adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1060adb54c29SNicholas Bellinger } 1061adb54c29SNicholas Bellinger 1062adb54c29SNicholas Bellinger static int 1063b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1064b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1065b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1066b8d26b3bSNicholas Bellinger { 1067b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1068b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1069ca40d24eSNicholas Bellinger struct iscsi_session *sess = conn->sess; 1070b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1071b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1072b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1073b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1074b8d26b3bSNicholas Bellinger 1075ca40d24eSNicholas Bellinger if (sess->sess_ops->SessionType && 1076ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1077ca40d24eSNicholas Bellinger pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1078ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1079ca40d24eSNicholas Bellinger return 0; 1080ca40d24eSNicholas Bellinger } 1081ca40d24eSNicholas Bellinger 1082b8d26b3bSNicholas Bellinger switch (opcode) { 1083b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1084d703ce2fSNicholas Bellinger cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1085b8d26b3bSNicholas Bellinger if (!cmd) 1086b8d26b3bSNicholas Bellinger break; 1087b8d26b3bSNicholas Bellinger 1088d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1089b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1090b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1091b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1092b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1093b8d26b3bSNicholas Bellinger 1094d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1095b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1096b8d26b3bSNicholas Bellinger break; 1097b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1098d703ce2fSNicholas Bellinger cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1099b8d26b3bSNicholas Bellinger if (!cmd) 1100b8d26b3bSNicholas Bellinger break; 1101b8d26b3bSNicholas Bellinger 1102d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1103d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1104778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1105b8d26b3bSNicholas Bellinger break; 1106b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1107b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1108b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1109b8d26b3bSNicholas Bellinger break; 1110b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1111d703ce2fSNicholas Bellinger cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1112b8d26b3bSNicholas Bellinger if (!cmd) 1113b8d26b3bSNicholas Bellinger break; 1114b8d26b3bSNicholas Bellinger 1115b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1116b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1117b8d26b3bSNicholas Bellinger break; 1118b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 1119d703ce2fSNicholas Bellinger cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1120b8d26b3bSNicholas Bellinger if (!cmd) 1121b8d26b3bSNicholas Bellinger break; 1122b8d26b3bSNicholas Bellinger 1123b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1124b8d26b3bSNicholas Bellinger if (ret > 0) 1125b8d26b3bSNicholas Bellinger wait_for_completion_timeout(&conn->conn_logout_comp, 1126b8d26b3bSNicholas Bellinger SECONDS_FOR_LOGOUT_COMP * 1127b8d26b3bSNicholas Bellinger HZ); 1128b8d26b3bSNicholas Bellinger break; 1129adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1130d703ce2fSNicholas Bellinger cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1131adb54c29SNicholas Bellinger if (!cmd) 1132adb54c29SNicholas Bellinger break; 1133adb54c29SNicholas Bellinger 1134d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1135d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1136adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1137adb54c29SNicholas Bellinger break; 1138b8d26b3bSNicholas Bellinger default: 1139b8d26b3bSNicholas Bellinger pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1140b8d26b3bSNicholas Bellinger dump_stack(); 1141b8d26b3bSNicholas Bellinger break; 1142b8d26b3bSNicholas Bellinger } 1143b8d26b3bSNicholas Bellinger 1144b8d26b3bSNicholas Bellinger return ret; 1145b8d26b3bSNicholas Bellinger } 1146b8d26b3bSNicholas Bellinger 1147b8d26b3bSNicholas Bellinger static void 1148b8d26b3bSNicholas Bellinger isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1149b8d26b3bSNicholas Bellinger { 1150b8d26b3bSNicholas Bellinger struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1151b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1152b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1153b8d26b3bSNicholas Bellinger int rc; 1154b8d26b3bSNicholas Bellinger 1155b8d26b3bSNicholas Bellinger switch (iser_hdr->flags & 0xF0) { 1156b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1157b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_RSV) { 1158b8d26b3bSNicholas Bellinger read_stag = be32_to_cpu(iser_hdr->read_stag); 1159b8d26b3bSNicholas Bellinger read_va = be64_to_cpu(iser_hdr->read_va); 1160b8d26b3bSNicholas Bellinger pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1161b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1162b8d26b3bSNicholas Bellinger } 1163b8d26b3bSNicholas Bellinger if (iser_hdr->flags & ISER_WSV) { 1164b8d26b3bSNicholas Bellinger write_stag = be32_to_cpu(iser_hdr->write_stag); 1165b8d26b3bSNicholas Bellinger write_va = be64_to_cpu(iser_hdr->write_va); 1166b8d26b3bSNicholas Bellinger pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1167b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1168b8d26b3bSNicholas Bellinger } 1169b8d26b3bSNicholas Bellinger 1170b8d26b3bSNicholas Bellinger pr_debug("ISER ISCSI_CTRL PDU\n"); 1171b8d26b3bSNicholas Bellinger break; 1172b8d26b3bSNicholas Bellinger case ISER_HELLO: 1173b8d26b3bSNicholas Bellinger pr_err("iSER Hello message\n"); 1174b8d26b3bSNicholas Bellinger break; 1175b8d26b3bSNicholas Bellinger default: 1176b8d26b3bSNicholas Bellinger pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1177b8d26b3bSNicholas Bellinger break; 1178b8d26b3bSNicholas Bellinger } 1179b8d26b3bSNicholas Bellinger 1180b8d26b3bSNicholas Bellinger rc = isert_rx_opcode(isert_conn, rx_desc, 1181b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 1182b8d26b3bSNicholas Bellinger } 1183b8d26b3bSNicholas Bellinger 1184b8d26b3bSNicholas Bellinger static void 1185b8d26b3bSNicholas Bellinger isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1186b8d26b3bSNicholas Bellinger unsigned long xfer_len) 1187b8d26b3bSNicholas Bellinger { 1188b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1189b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr; 1190b8d26b3bSNicholas Bellinger u64 rx_dma; 1191b8d26b3bSNicholas Bellinger int rx_buflen, outstanding; 1192b8d26b3bSNicholas Bellinger 1193b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) { 1194b8d26b3bSNicholas Bellinger rx_dma = isert_conn->login_req_dma; 1195b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_LOGIN_SIZE; 1196b8d26b3bSNicholas Bellinger pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1197b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1198b8d26b3bSNicholas Bellinger } else { 1199b8d26b3bSNicholas Bellinger rx_dma = desc->dma_addr; 1200b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_PAYLOAD_SIZE; 1201b8d26b3bSNicholas Bellinger pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1202b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1203b8d26b3bSNicholas Bellinger } 1204b8d26b3bSNicholas Bellinger 1205b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1206b8d26b3bSNicholas Bellinger 1207b8d26b3bSNicholas Bellinger hdr = &desc->iscsi_header; 1208b8d26b3bSNicholas Bellinger pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1209b8d26b3bSNicholas Bellinger hdr->opcode, hdr->itt, hdr->flags, 1210b8d26b3bSNicholas Bellinger (int)(xfer_len - ISER_HEADERS_LEN)); 1211b8d26b3bSNicholas Bellinger 1212b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) 1213b8d26b3bSNicholas Bellinger isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1214b8d26b3bSNicholas Bellinger isert_conn); 1215b8d26b3bSNicholas Bellinger else 1216b8d26b3bSNicholas Bellinger isert_rx_do_work(desc, isert_conn); 1217b8d26b3bSNicholas Bellinger 1218b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1219b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 1220b8d26b3bSNicholas Bellinger 1221b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1222b8d26b3bSNicholas Bellinger pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1223b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count); 1224b8d26b3bSNicholas Bellinger 1225b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) 1226b8d26b3bSNicholas Bellinger return; 1227b8d26b3bSNicholas Bellinger 1228b8d26b3bSNicholas Bellinger outstanding = isert_conn->post_recv_buf_count; 1229b8d26b3bSNicholas Bellinger if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1230b8d26b3bSNicholas Bellinger int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1231b8d26b3bSNicholas Bellinger ISERT_MIN_POSTED_RX); 1232b8d26b3bSNicholas Bellinger err = isert_post_recv(isert_conn, count); 1233b8d26b3bSNicholas Bellinger if (err) { 1234b8d26b3bSNicholas Bellinger pr_err("isert_post_recv() count: %d failed, %d\n", 1235b8d26b3bSNicholas Bellinger count, err); 1236b8d26b3bSNicholas Bellinger } 1237b8d26b3bSNicholas Bellinger } 1238b8d26b3bSNicholas Bellinger } 1239b8d26b3bSNicholas Bellinger 1240b8d26b3bSNicholas Bellinger static void 1241b8d26b3bSNicholas Bellinger isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1242b8d26b3bSNicholas Bellinger { 1243b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1244b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1245b8d26b3bSNicholas Bellinger 124690ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1247b8d26b3bSNicholas Bellinger if (wr->sge) { 124890ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 124990ecc6e2SVu Pham ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 125090ecc6e2SVu Pham (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 125190ecc6e2SVu Pham DMA_TO_DEVICE : DMA_FROM_DEVICE); 1252b8d26b3bSNicholas Bellinger wr->sge = NULL; 1253b8d26b3bSNicholas Bellinger } 1254b8d26b3bSNicholas Bellinger 125590ecc6e2SVu Pham if (wr->send_wr) { 125690ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1257b8d26b3bSNicholas Bellinger kfree(wr->send_wr); 1258b8d26b3bSNicholas Bellinger wr->send_wr = NULL; 125990ecc6e2SVu Pham } 1260b8d26b3bSNicholas Bellinger 126190ecc6e2SVu Pham if (wr->ib_sge) { 126290ecc6e2SVu Pham pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 126390ecc6e2SVu Pham kfree(wr->ib_sge); 126490ecc6e2SVu Pham wr->ib_sge = NULL; 126590ecc6e2SVu Pham } 1266b8d26b3bSNicholas Bellinger } 1267b8d26b3bSNicholas Bellinger 1268b8d26b3bSNicholas Bellinger static void 1269b8d26b3bSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd) 1270b8d26b3bSNicholas Bellinger { 1271d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1272b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1273186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1274*d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 1275b8d26b3bSNicholas Bellinger 1276b8d26b3bSNicholas Bellinger pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1277b8d26b3bSNicholas Bellinger 1278b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1279b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1280b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1281b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1282b8d26b3bSNicholas Bellinger list_del(&cmd->i_conn_node); 1283b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1284b8d26b3bSNicholas Bellinger 1285b8d26b3bSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 1286b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1287b8d26b3bSNicholas Bellinger 1288*d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1289186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1290186a9647SNicholas Bellinger break; 1291b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1292186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1293186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1294186a9647SNicholas Bellinger list_del(&cmd->i_conn_node); 1295186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1296186a9647SNicholas Bellinger 1297b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1298b8d26b3bSNicholas Bellinger break; 1299b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1300b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1301adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1302b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1303b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1304b8d26b3bSNicholas Bellinger list_del(&cmd->i_conn_node); 1305b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1306b8d26b3bSNicholas Bellinger 1307b8d26b3bSNicholas Bellinger /* 1308b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1309b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1310b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1311b8d26b3bSNicholas Bellinger */ 1312b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 13133df8f68aSNicholas Bellinger pr_debug("Calling transport_generic_free_cmd from" 13143df8f68aSNicholas Bellinger " isert_put_cmd for 0x%02x\n", 13153df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1316b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1317b8d26b3bSNicholas Bellinger break; 1318b8d26b3bSNicholas Bellinger } 1319b8d26b3bSNicholas Bellinger /* 1320b8d26b3bSNicholas Bellinger * Fall-through 1321b8d26b3bSNicholas Bellinger */ 1322b8d26b3bSNicholas Bellinger default: 1323d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1324b8d26b3bSNicholas Bellinger break; 1325b8d26b3bSNicholas Bellinger } 1326b8d26b3bSNicholas Bellinger } 1327b8d26b3bSNicholas Bellinger 1328b8d26b3bSNicholas Bellinger static void 1329b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1330b8d26b3bSNicholas Bellinger { 1331b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 1332b8d26b3bSNicholas Bellinger pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1333b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1334b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1335b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1336b8d26b3bSNicholas Bellinger } 1337b8d26b3bSNicholas Bellinger } 1338b8d26b3bSNicholas Bellinger 1339b8d26b3bSNicholas Bellinger static void 1340b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1341b8d26b3bSNicholas Bellinger struct ib_device *ib_dev) 1342b8d26b3bSNicholas Bellinger { 1343dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 1344dbbc5d11SNicholas Bellinger pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1345dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1346dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1347dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1348b8d26b3bSNicholas Bellinger } 1349b8d26b3bSNicholas Bellinger 1350b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1351b8d26b3bSNicholas Bellinger isert_put_cmd(isert_cmd); 1352b8d26b3bSNicholas Bellinger } 1353b8d26b3bSNicholas Bellinger 1354b8d26b3bSNicholas Bellinger static void 1355b8d26b3bSNicholas Bellinger isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1356b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd) 1357b8d26b3bSNicholas Bellinger { 1358b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1359d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1360b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 136190ecc6e2SVu Pham struct isert_conn *isert_conn = isert_cmd->conn; 1362*d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 1363b8d26b3bSNicholas Bellinger 1364b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1365*d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 136690ecc6e2SVu Pham cmd->write_data_done = wr->cur_rdma_length; 1367b8d26b3bSNicholas Bellinger 136890ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1369b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1370b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1371b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1372b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1373b8d26b3bSNicholas Bellinger 1374b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1375b8d26b3bSNicholas Bellinger } 1376b8d26b3bSNicholas Bellinger 1377b8d26b3bSNicholas Bellinger static void 1378b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1379b8d26b3bSNicholas Bellinger { 1380b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1381b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1382b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1383b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1384d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1385b8d26b3bSNicholas Bellinger 1386b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1387b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1388b8d26b3bSNicholas Bellinger pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1389b8d26b3bSNicholas Bellinger 1390b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1391b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1392b8d26b3bSNicholas Bellinger 1393b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 1394b8d26b3bSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1395b8d26b3bSNicholas Bellinger break; 1396b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 1397b8d26b3bSNicholas Bellinger pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1398b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1399b8d26b3bSNicholas Bellinger 1400b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 1401b8d26b3bSNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 14023df8f68aSNicholas Bellinger break; 1403b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1404b8d26b3bSNicholas Bellinger pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1405b8d26b3bSNicholas Bellinger /* 1406b8d26b3bSNicholas Bellinger * Call atomic_dec(&isert_conn->post_send_buf_count) 1407b8d26b3bSNicholas Bellinger * from isert_free_conn() 1408b8d26b3bSNicholas Bellinger */ 1409b8d26b3bSNicholas Bellinger isert_conn->logout_posted = true; 1410b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1411b8d26b3bSNicholas Bellinger break; 1412adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 1413adb54c29SNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1414adb54c29SNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 1415adb54c29SNicholas Bellinger isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1416adb54c29SNicholas Bellinger break; 1417b8d26b3bSNicholas Bellinger default: 1418b8d26b3bSNicholas Bellinger pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1419b8d26b3bSNicholas Bellinger dump_stack(); 1420b8d26b3bSNicholas Bellinger break; 1421b8d26b3bSNicholas Bellinger } 1422b8d26b3bSNicholas Bellinger } 1423b8d26b3bSNicholas Bellinger 1424b8d26b3bSNicholas Bellinger static void 1425b8d26b3bSNicholas Bellinger isert_response_completion(struct iser_tx_desc *tx_desc, 1426b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1427b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn, 1428b8d26b3bSNicholas Bellinger struct ib_device *ib_dev) 1429b8d26b3bSNicholas Bellinger { 1430d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1431b8d26b3bSNicholas Bellinger 1432b8d26b3bSNicholas Bellinger if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 14333df8f68aSNicholas Bellinger cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1434adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_REJECT || 1435adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_TEXTRSP) { 1436b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1437b8d26b3bSNicholas Bellinger 1438b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1439b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1440b8d26b3bSNicholas Bellinger return; 1441b8d26b3bSNicholas Bellinger } 1442b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1443b8d26b3bSNicholas Bellinger 1444b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 1445b8d26b3bSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev); 1446b8d26b3bSNicholas Bellinger } 1447b8d26b3bSNicholas Bellinger 1448b8d26b3bSNicholas Bellinger static void 1449b8d26b3bSNicholas Bellinger isert_send_completion(struct iser_tx_desc *tx_desc, 1450b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 1451b8d26b3bSNicholas Bellinger { 1452b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1453b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1454b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr; 1455b8d26b3bSNicholas Bellinger 1456b8d26b3bSNicholas Bellinger if (!isert_cmd) { 1457b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1458b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1459b8d26b3bSNicholas Bellinger return; 1460b8d26b3bSNicholas Bellinger } 1461b8d26b3bSNicholas Bellinger wr = &isert_cmd->rdma_wr; 1462b8d26b3bSNicholas Bellinger 1463b8d26b3bSNicholas Bellinger switch (wr->iser_ib_op) { 1464b8d26b3bSNicholas Bellinger case ISER_IB_RECV: 1465b8d26b3bSNicholas Bellinger pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1466b8d26b3bSNicholas Bellinger dump_stack(); 1467b8d26b3bSNicholas Bellinger break; 1468b8d26b3bSNicholas Bellinger case ISER_IB_SEND: 1469b8d26b3bSNicholas Bellinger pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1470b8d26b3bSNicholas Bellinger isert_response_completion(tx_desc, isert_cmd, 1471b8d26b3bSNicholas Bellinger isert_conn, ib_dev); 1472b8d26b3bSNicholas Bellinger break; 1473b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_WRITE: 1474b8d26b3bSNicholas Bellinger pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1475b8d26b3bSNicholas Bellinger dump_stack(); 1476b8d26b3bSNicholas Bellinger break; 1477b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_READ: 1478b8d26b3bSNicholas Bellinger pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1479b8d26b3bSNicholas Bellinger 1480b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1481b8d26b3bSNicholas Bellinger isert_completion_rdma_read(tx_desc, isert_cmd); 1482b8d26b3bSNicholas Bellinger break; 1483b8d26b3bSNicholas Bellinger default: 1484b8d26b3bSNicholas Bellinger pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1485b8d26b3bSNicholas Bellinger dump_stack(); 1486b8d26b3bSNicholas Bellinger break; 1487b8d26b3bSNicholas Bellinger } 1488b8d26b3bSNicholas Bellinger } 1489b8d26b3bSNicholas Bellinger 1490b8d26b3bSNicholas Bellinger static void 1491b8d26b3bSNicholas Bellinger isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1492b8d26b3bSNicholas Bellinger { 1493b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1494b8d26b3bSNicholas Bellinger 1495b8d26b3bSNicholas Bellinger if (tx_desc) { 1496b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1497b8d26b3bSNicholas Bellinger 1498b8d26b3bSNicholas Bellinger if (!isert_cmd) 1499b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1500b8d26b3bSNicholas Bellinger else 1501b8d26b3bSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev); 1502b8d26b3bSNicholas Bellinger } 1503b8d26b3bSNicholas Bellinger 1504b8d26b3bSNicholas Bellinger if (isert_conn->post_recv_buf_count == 0 && 1505b8d26b3bSNicholas Bellinger atomic_read(&isert_conn->post_send_buf_count) == 0) { 1506b8d26b3bSNicholas Bellinger pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 1507b8d26b3bSNicholas Bellinger pr_debug("Calling wake_up from isert_cq_comp_err\n"); 1508b8d26b3bSNicholas Bellinger 1509b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 1510b2cb9649SNicholas Bellinger if (isert_conn->state != ISER_CONN_DOWN) 1511b8d26b3bSNicholas Bellinger isert_conn->state = ISER_CONN_TERMINATING; 1512b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 1513b2cb9649SNicholas Bellinger 1514b8d26b3bSNicholas Bellinger wake_up(&isert_conn->conn_wait_comp_err); 1515b8d26b3bSNicholas Bellinger } 1516b8d26b3bSNicholas Bellinger } 1517b8d26b3bSNicholas Bellinger 1518b8d26b3bSNicholas Bellinger static void 1519b8d26b3bSNicholas Bellinger isert_cq_tx_work(struct work_struct *work) 1520b8d26b3bSNicholas Bellinger { 1521b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = container_of(work, 1522b8d26b3bSNicholas Bellinger struct isert_cq_desc, cq_tx_work); 1523b8d26b3bSNicholas Bellinger struct isert_device *device = cq_desc->device; 1524b8d26b3bSNicholas Bellinger int cq_index = cq_desc->cq_index; 1525b8d26b3bSNicholas Bellinger struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 1526b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 1527b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc; 1528b8d26b3bSNicholas Bellinger struct ib_wc wc; 1529b8d26b3bSNicholas Bellinger 1530b8d26b3bSNicholas Bellinger while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 1531b8d26b3bSNicholas Bellinger tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 1532b8d26b3bSNicholas Bellinger isert_conn = wc.qp->qp_context; 1533b8d26b3bSNicholas Bellinger 1534b8d26b3bSNicholas Bellinger if (wc.status == IB_WC_SUCCESS) { 1535b8d26b3bSNicholas Bellinger isert_send_completion(tx_desc, isert_conn); 1536b8d26b3bSNicholas Bellinger } else { 1537b8d26b3bSNicholas Bellinger pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1538b8d26b3bSNicholas Bellinger pr_debug("TX wc.status: 0x%08x\n", wc.status); 1539c5a2adbfSNicholas Bellinger pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 1540b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1541b8d26b3bSNicholas Bellinger isert_cq_comp_err(tx_desc, isert_conn); 1542b8d26b3bSNicholas Bellinger } 1543b8d26b3bSNicholas Bellinger } 1544b8d26b3bSNicholas Bellinger 1545b8d26b3bSNicholas Bellinger ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 1546b8d26b3bSNicholas Bellinger } 1547b8d26b3bSNicholas Bellinger 1548b8d26b3bSNicholas Bellinger static void 1549b8d26b3bSNicholas Bellinger isert_cq_tx_callback(struct ib_cq *cq, void *context) 1550b8d26b3bSNicholas Bellinger { 1551b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1552b8d26b3bSNicholas Bellinger 1553b8d26b3bSNicholas Bellinger INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work); 1554b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 1555b8d26b3bSNicholas Bellinger } 1556b8d26b3bSNicholas Bellinger 1557b8d26b3bSNicholas Bellinger static void 1558b8d26b3bSNicholas Bellinger isert_cq_rx_work(struct work_struct *work) 1559b8d26b3bSNicholas Bellinger { 1560b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = container_of(work, 1561b8d26b3bSNicholas Bellinger struct isert_cq_desc, cq_rx_work); 1562b8d26b3bSNicholas Bellinger struct isert_device *device = cq_desc->device; 1563b8d26b3bSNicholas Bellinger int cq_index = cq_desc->cq_index; 1564b8d26b3bSNicholas Bellinger struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 1565b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 1566b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 1567b8d26b3bSNicholas Bellinger struct ib_wc wc; 1568b8d26b3bSNicholas Bellinger unsigned long xfer_len; 1569b8d26b3bSNicholas Bellinger 1570b8d26b3bSNicholas Bellinger while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 1571b8d26b3bSNicholas Bellinger rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 1572b8d26b3bSNicholas Bellinger isert_conn = wc.qp->qp_context; 1573b8d26b3bSNicholas Bellinger 1574b8d26b3bSNicholas Bellinger if (wc.status == IB_WC_SUCCESS) { 1575b8d26b3bSNicholas Bellinger xfer_len = (unsigned long)wc.byte_len; 1576b8d26b3bSNicholas Bellinger isert_rx_completion(rx_desc, isert_conn, xfer_len); 1577b8d26b3bSNicholas Bellinger } else { 1578b8d26b3bSNicholas Bellinger pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1579c5a2adbfSNicholas Bellinger if (wc.status != IB_WC_WR_FLUSH_ERR) { 1580b8d26b3bSNicholas Bellinger pr_debug("RX wc.status: 0x%08x\n", wc.status); 1581c5a2adbfSNicholas Bellinger pr_debug("RX wc.vendor_err: 0x%08x\n", 1582c5a2adbfSNicholas Bellinger wc.vendor_err); 1583c5a2adbfSNicholas Bellinger } 1584b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1585b8d26b3bSNicholas Bellinger isert_cq_comp_err(NULL, isert_conn); 1586b8d26b3bSNicholas Bellinger } 1587b8d26b3bSNicholas Bellinger } 1588b8d26b3bSNicholas Bellinger 1589b8d26b3bSNicholas Bellinger ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 1590b8d26b3bSNicholas Bellinger } 1591b8d26b3bSNicholas Bellinger 1592b8d26b3bSNicholas Bellinger static void 1593b8d26b3bSNicholas Bellinger isert_cq_rx_callback(struct ib_cq *cq, void *context) 1594b8d26b3bSNicholas Bellinger { 1595b8d26b3bSNicholas Bellinger struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1596b8d26b3bSNicholas Bellinger 1597b8d26b3bSNicholas Bellinger INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work); 1598b8d26b3bSNicholas Bellinger queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 1599b8d26b3bSNicholas Bellinger } 1600b8d26b3bSNicholas Bellinger 1601b8d26b3bSNicholas Bellinger static int 1602b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1603b8d26b3bSNicholas Bellinger { 1604b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 1605b8d26b3bSNicholas Bellinger int ret; 1606b8d26b3bSNicholas Bellinger 1607b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 1608b8d26b3bSNicholas Bellinger 1609b8d26b3bSNicholas Bellinger ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 1610b8d26b3bSNicholas Bellinger &wr_failed); 1611b8d26b3bSNicholas Bellinger if (ret) { 1612b8d26b3bSNicholas Bellinger pr_err("ib_post_send failed with %d\n", ret); 1613b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1614b8d26b3bSNicholas Bellinger return ret; 1615b8d26b3bSNicholas Bellinger } 1616b8d26b3bSNicholas Bellinger return ret; 1617b8d26b3bSNicholas Bellinger } 1618b8d26b3bSNicholas Bellinger 1619b8d26b3bSNicholas Bellinger static int 1620b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1621b8d26b3bSNicholas Bellinger { 1622d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1623b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1624b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1625b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1626b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 1627b8d26b3bSNicholas Bellinger 1628b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1629b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1630b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1631b8d26b3bSNicholas Bellinger /* 1632b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 1633b8d26b3bSNicholas Bellinger */ 1634b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 1635b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1636b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1637b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1638b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1639dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 1640b8d26b3bSNicholas Bellinger 1641b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1642b8d26b3bSNicholas Bellinger cmd->sense_buffer); 1643b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1644b8d26b3bSNicholas Bellinger 1645b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1646b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1647dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1648b8d26b3bSNicholas Bellinger 1649dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1650dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 1651b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 1652b8d26b3bSNicholas Bellinger 1653dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 1654dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 1655dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 1656b8d26b3bSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 1657b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 1658b8d26b3bSNicholas Bellinger } 1659b8d26b3bSNicholas Bellinger 1660b8d26b3bSNicholas Bellinger isert_init_send_wr(isert_cmd, send_wr); 1661b8d26b3bSNicholas Bellinger 1662b8d26b3bSNicholas Bellinger pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1663b8d26b3bSNicholas Bellinger 1664b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1665b8d26b3bSNicholas Bellinger } 1666b8d26b3bSNicholas Bellinger 1667b8d26b3bSNicholas Bellinger static int 1668b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1669b8d26b3bSNicholas Bellinger bool nopout_response) 1670b8d26b3bSNicholas Bellinger { 1671d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1672b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1673b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1674b8d26b3bSNicholas Bellinger 1675b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1676b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1677b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 1678b8d26b3bSNicholas Bellinger nopout_response); 1679b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1680b8d26b3bSNicholas Bellinger isert_init_send_wr(isert_cmd, send_wr); 1681b8d26b3bSNicholas Bellinger 16828b513d0cSMasanari Iida pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1683b8d26b3bSNicholas Bellinger 1684b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1685b8d26b3bSNicholas Bellinger } 1686b8d26b3bSNicholas Bellinger 1687b8d26b3bSNicholas Bellinger static int 1688b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1689b8d26b3bSNicholas Bellinger { 1690d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1691b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1692b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1693b8d26b3bSNicholas Bellinger 1694b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1695b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1696b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1697b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1698b8d26b3bSNicholas Bellinger isert_init_send_wr(isert_cmd, send_wr); 1699b8d26b3bSNicholas Bellinger 1700b8d26b3bSNicholas Bellinger pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1701b8d26b3bSNicholas Bellinger 1702b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1703b8d26b3bSNicholas Bellinger } 1704b8d26b3bSNicholas Bellinger 1705b8d26b3bSNicholas Bellinger static int 1706b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1707b8d26b3bSNicholas Bellinger { 1708d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1709b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1710b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1711b8d26b3bSNicholas Bellinger 1712b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1713b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1714b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1715b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1716b8d26b3bSNicholas Bellinger isert_init_send_wr(isert_cmd, send_wr); 1717b8d26b3bSNicholas Bellinger 1718b8d26b3bSNicholas Bellinger pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1719b8d26b3bSNicholas Bellinger 1720b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1721b8d26b3bSNicholas Bellinger } 1722b8d26b3bSNicholas Bellinger 1723b8d26b3bSNicholas Bellinger static int 1724b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1725b8d26b3bSNicholas Bellinger { 1726d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1727b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1728b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 17293df8f68aSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 17303df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 17313df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 17323df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1733b8d26b3bSNicholas Bellinger 1734b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 17353df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 1736b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 17373df8f68aSNicholas Bellinger 17383df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 1739dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 17403df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 17413df8f68aSNicholas Bellinger DMA_TO_DEVICE); 1742dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1743dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 17443df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 17453df8f68aSNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 17463df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 17473df8f68aSNicholas Bellinger 1748b8d26b3bSNicholas Bellinger isert_init_send_wr(isert_cmd, send_wr); 1749b8d26b3bSNicholas Bellinger 1750b8d26b3bSNicholas Bellinger pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1751b8d26b3bSNicholas Bellinger 1752b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1753b8d26b3bSNicholas Bellinger } 1754b8d26b3bSNicholas Bellinger 1755b8d26b3bSNicholas Bellinger static int 1756adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1757adb54c29SNicholas Bellinger { 1758d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1759adb54c29SNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1760adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1761adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 1762adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1763adb54c29SNicholas Bellinger u32 txt_rsp_len; 1764adb54c29SNicholas Bellinger int rc; 1765adb54c29SNicholas Bellinger 1766adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1767adb54c29SNicholas Bellinger rc = iscsit_build_text_rsp(cmd, conn, hdr); 1768adb54c29SNicholas Bellinger if (rc < 0) 1769adb54c29SNicholas Bellinger return rc; 1770adb54c29SNicholas Bellinger 1771adb54c29SNicholas Bellinger txt_rsp_len = rc; 1772adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1773adb54c29SNicholas Bellinger 1774adb54c29SNicholas Bellinger if (txt_rsp_len) { 1775adb54c29SNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1776adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1777adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 1778adb54c29SNicholas Bellinger 1779adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1780adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1781adb54c29SNicholas Bellinger 1782adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 1783adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 1784adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 1785adb54c29SNicholas Bellinger tx_dsg->lkey = isert_conn->conn_mr->lkey; 1786adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 1787adb54c29SNicholas Bellinger } 1788adb54c29SNicholas Bellinger isert_init_send_wr(isert_cmd, send_wr); 1789adb54c29SNicholas Bellinger 1790adb54c29SNicholas Bellinger pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1791adb54c29SNicholas Bellinger 1792adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1793adb54c29SNicholas Bellinger } 1794adb54c29SNicholas Bellinger 1795adb54c29SNicholas Bellinger static int 1796b8d26b3bSNicholas Bellinger isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1797b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 1798b8d26b3bSNicholas Bellinger u32 data_left, u32 offset) 1799b8d26b3bSNicholas Bellinger { 1800d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1801b8d26b3bSNicholas Bellinger struct scatterlist *sg_start, *tmp_sg; 1802b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1803b8d26b3bSNicholas Bellinger u32 sg_off, page_off; 1804b8d26b3bSNicholas Bellinger int i = 0, sg_nents; 1805b8d26b3bSNicholas Bellinger 1806b8d26b3bSNicholas Bellinger sg_off = offset / PAGE_SIZE; 1807b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1808b8d26b3bSNicholas Bellinger sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 1809b8d26b3bSNicholas Bellinger page_off = offset % PAGE_SIZE; 1810b8d26b3bSNicholas Bellinger 1811b8d26b3bSNicholas Bellinger send_wr->sg_list = ib_sge; 1812b8d26b3bSNicholas Bellinger send_wr->num_sge = sg_nents; 1813b8d26b3bSNicholas Bellinger send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 1814b8d26b3bSNicholas Bellinger /* 1815b8d26b3bSNicholas Bellinger * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 1816b8d26b3bSNicholas Bellinger */ 1817b8d26b3bSNicholas Bellinger for_each_sg(sg_start, tmp_sg, sg_nents, i) { 1818b8d26b3bSNicholas Bellinger pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 1819b8d26b3bSNicholas Bellinger (unsigned long long)tmp_sg->dma_address, 1820b8d26b3bSNicholas Bellinger tmp_sg->length, page_off); 1821b8d26b3bSNicholas Bellinger 1822b8d26b3bSNicholas Bellinger ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 1823b8d26b3bSNicholas Bellinger ib_sge->length = min_t(u32, data_left, 1824b8d26b3bSNicholas Bellinger ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 1825b8d26b3bSNicholas Bellinger ib_sge->lkey = isert_conn->conn_mr->lkey; 1826b8d26b3bSNicholas Bellinger 182790ecc6e2SVu Pham pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 182890ecc6e2SVu Pham ib_sge->addr, ib_sge->length, ib_sge->lkey); 1829b8d26b3bSNicholas Bellinger page_off = 0; 1830b8d26b3bSNicholas Bellinger data_left -= ib_sge->length; 1831b8d26b3bSNicholas Bellinger ib_sge++; 1832b8d26b3bSNicholas Bellinger pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 1833b8d26b3bSNicholas Bellinger } 1834b8d26b3bSNicholas Bellinger 1835b8d26b3bSNicholas Bellinger pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 1836b8d26b3bSNicholas Bellinger send_wr->sg_list, send_wr->num_sge); 1837b8d26b3bSNicholas Bellinger 1838b8d26b3bSNicholas Bellinger return sg_nents; 1839b8d26b3bSNicholas Bellinger } 1840b8d26b3bSNicholas Bellinger 1841b8d26b3bSNicholas Bellinger static int 184290ecc6e2SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 184390ecc6e2SVu Pham struct isert_rdma_wr *wr) 1844b8d26b3bSNicholas Bellinger { 1845b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 1846d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1847b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1848b8d26b3bSNicholas Bellinger struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 184990ecc6e2SVu Pham struct ib_send_wr *send_wr; 1850b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge; 185190ecc6e2SVu Pham struct scatterlist *sg_start; 185290ecc6e2SVu Pham u32 sg_off = 0, sg_nents; 185390ecc6e2SVu Pham u32 offset = 0, data_len, data_left, rdma_write_max, va_offset = 0; 185490ecc6e2SVu Pham int ret = 0, count, i, ib_sge_cnt; 1855b8d26b3bSNicholas Bellinger 185690ecc6e2SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 185790ecc6e2SVu Pham data_left = se_cmd->data_length; 185890ecc6e2SVu Pham iscsit_increment_maxcmdsn(cmd, conn->sess); 185990ecc6e2SVu Pham cmd->stat_sn = conn->stat_sn++; 186090ecc6e2SVu Pham } else { 186190ecc6e2SVu Pham sg_off = cmd->write_data_done / PAGE_SIZE; 186290ecc6e2SVu Pham data_left = se_cmd->data_length - cmd->write_data_done; 186390ecc6e2SVu Pham offset = cmd->write_data_done; 186490ecc6e2SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 186590ecc6e2SVu Pham } 1866b8d26b3bSNicholas Bellinger 186790ecc6e2SVu Pham sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 186890ecc6e2SVu Pham sg_nents = se_cmd->t_data_nents - sg_off; 1869b8d26b3bSNicholas Bellinger 187090ecc6e2SVu Pham count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 187190ecc6e2SVu Pham (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 187290ecc6e2SVu Pham DMA_TO_DEVICE : DMA_FROM_DEVICE); 1873b8d26b3bSNicholas Bellinger if (unlikely(!count)) { 187490ecc6e2SVu Pham pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 1875b8d26b3bSNicholas Bellinger return -EINVAL; 1876b8d26b3bSNicholas Bellinger } 187790ecc6e2SVu Pham wr->sge = sg_start; 1878b8d26b3bSNicholas Bellinger wr->num_sge = sg_nents; 187990ecc6e2SVu Pham wr->cur_rdma_length = data_left; 188090ecc6e2SVu Pham pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 188190ecc6e2SVu Pham isert_cmd, count, sg_start, sg_nents, data_left); 1882b8d26b3bSNicholas Bellinger 1883b8d26b3bSNicholas Bellinger ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 1884b8d26b3bSNicholas Bellinger if (!ib_sge) { 188590ecc6e2SVu Pham pr_warn("Unable to allocate ib_sge\n"); 1886b8d26b3bSNicholas Bellinger ret = -ENOMEM; 1887b8d26b3bSNicholas Bellinger goto unmap_sg; 1888b8d26b3bSNicholas Bellinger } 188990ecc6e2SVu Pham wr->ib_sge = ib_sge; 1890b8d26b3bSNicholas Bellinger 1891b8d26b3bSNicholas Bellinger wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 1892b8d26b3bSNicholas Bellinger wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 1893b8d26b3bSNicholas Bellinger GFP_KERNEL); 1894b8d26b3bSNicholas Bellinger if (!wr->send_wr) { 189590ecc6e2SVu Pham pr_debug("Unable to allocate wr->send_wr\n"); 1896b8d26b3bSNicholas Bellinger ret = -ENOMEM; 1897b8d26b3bSNicholas Bellinger goto unmap_sg; 1898b8d26b3bSNicholas Bellinger } 1899b8d26b3bSNicholas Bellinger 1900b8d26b3bSNicholas Bellinger wr->isert_cmd = isert_cmd; 1901b8d26b3bSNicholas Bellinger rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 1902b8d26b3bSNicholas Bellinger 1903b8d26b3bSNicholas Bellinger for (i = 0; i < wr->send_wr_num; i++) { 1904b8d26b3bSNicholas Bellinger send_wr = &isert_cmd->rdma_wr.send_wr[i]; 1905b8d26b3bSNicholas Bellinger data_len = min(data_left, rdma_write_max); 1906b8d26b3bSNicholas Bellinger 1907b8d26b3bSNicholas Bellinger send_wr->send_flags = 0; 190890ecc6e2SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 190990ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_WRITE; 1910b8d26b3bSNicholas Bellinger send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 1911b8d26b3bSNicholas Bellinger send_wr->wr.rdma.rkey = isert_cmd->read_stag; 191290ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 191390ecc6e2SVu Pham send_wr->next = &isert_cmd->tx_desc.send_wr; 191490ecc6e2SVu Pham else 191590ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 191690ecc6e2SVu Pham } else { 191790ecc6e2SVu Pham send_wr->opcode = IB_WR_RDMA_READ; 191890ecc6e2SVu Pham send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 191990ecc6e2SVu Pham send_wr->wr.rdma.rkey = isert_cmd->write_stag; 192090ecc6e2SVu Pham if (i + 1 == wr->send_wr_num) 192190ecc6e2SVu Pham send_wr->send_flags = IB_SEND_SIGNALED; 192290ecc6e2SVu Pham else 192390ecc6e2SVu Pham send_wr->next = &wr->send_wr[i + 1]; 192490ecc6e2SVu Pham } 1925b8d26b3bSNicholas Bellinger 1926b8d26b3bSNicholas Bellinger ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 1927b8d26b3bSNicholas Bellinger send_wr, data_len, offset); 1928b8d26b3bSNicholas Bellinger ib_sge += ib_sge_cnt; 1929b8d26b3bSNicholas Bellinger 1930b8d26b3bSNicholas Bellinger offset += data_len; 193190ecc6e2SVu Pham va_offset += data_len; 1932b8d26b3bSNicholas Bellinger data_left -= data_len; 1933b8d26b3bSNicholas Bellinger } 193490ecc6e2SVu Pham 193590ecc6e2SVu Pham return 0; 193690ecc6e2SVu Pham unmap_sg: 193790ecc6e2SVu Pham ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 193890ecc6e2SVu Pham (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 193990ecc6e2SVu Pham DMA_TO_DEVICE : DMA_FROM_DEVICE); 194090ecc6e2SVu Pham return ret; 194190ecc6e2SVu Pham } 194290ecc6e2SVu Pham 194390ecc6e2SVu Pham static int 194490ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 194590ecc6e2SVu Pham { 194690ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 194790ecc6e2SVu Pham struct isert_cmd *isert_cmd = container_of(cmd, 194890ecc6e2SVu Pham struct isert_cmd, iscsi_cmd); 194990ecc6e2SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 195090ecc6e2SVu Pham struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1951*d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 195290ecc6e2SVu Pham struct ib_send_wr *wr_failed; 195390ecc6e2SVu Pham int rc; 195490ecc6e2SVu Pham 195590ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 195690ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 195790ecc6e2SVu Pham wr->iser_ib_op = ISER_IB_RDMA_WRITE; 1958*d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 195990ecc6e2SVu Pham if (rc) { 196090ecc6e2SVu Pham pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 196190ecc6e2SVu Pham return rc; 196290ecc6e2SVu Pham } 196390ecc6e2SVu Pham 1964b8d26b3bSNicholas Bellinger /* 1965b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 1966b8d26b3bSNicholas Bellinger */ 1967b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1968b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *) 1969b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1970b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1971b8d26b3bSNicholas Bellinger isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr); 1972b8d26b3bSNicholas Bellinger 1973b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 1974b8d26b3bSNicholas Bellinger 1975b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 1976b8d26b3bSNicholas Bellinger if (rc) { 1977b8d26b3bSNicholas Bellinger pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 1978b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 1979b8d26b3bSNicholas Bellinger } 198090ecc6e2SVu Pham pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data READ\n", 198190ecc6e2SVu Pham isert_cmd); 1982b8d26b3bSNicholas Bellinger 198390ecc6e2SVu Pham return 1; 1984b8d26b3bSNicholas Bellinger } 1985b8d26b3bSNicholas Bellinger 1986b8d26b3bSNicholas Bellinger static int 1987b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 1988b8d26b3bSNicholas Bellinger { 1989b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 1990d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1991b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1992b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1993*d40945d8SVu Pham struct isert_device *device = isert_conn->conn_device; 199490ecc6e2SVu Pham struct ib_send_wr *wr_failed; 199590ecc6e2SVu Pham int rc; 1996b8d26b3bSNicholas Bellinger 199790ecc6e2SVu Pham pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 199890ecc6e2SVu Pham isert_cmd, se_cmd->data_length, cmd->write_data_done); 1999b8d26b3bSNicholas Bellinger wr->iser_ib_op = ISER_IB_RDMA_READ; 2000*d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 200190ecc6e2SVu Pham if (rc) { 200290ecc6e2SVu Pham pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 200390ecc6e2SVu Pham return rc; 2004b8d26b3bSNicholas Bellinger } 2005b8d26b3bSNicholas Bellinger 2006b8d26b3bSNicholas Bellinger atomic_inc(&isert_conn->post_send_buf_count); 2007b8d26b3bSNicholas Bellinger 2008b8d26b3bSNicholas Bellinger rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2009b8d26b3bSNicholas Bellinger if (rc) { 2010b8d26b3bSNicholas Bellinger pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2011b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 2012b8d26b3bSNicholas Bellinger } 201390ecc6e2SVu Pham pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 201490ecc6e2SVu Pham isert_cmd); 2015b8d26b3bSNicholas Bellinger 201690ecc6e2SVu Pham return 0; 2017b8d26b3bSNicholas Bellinger } 2018b8d26b3bSNicholas Bellinger 2019b8d26b3bSNicholas Bellinger static int 2020b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2021b8d26b3bSNicholas Bellinger { 2022b8d26b3bSNicholas Bellinger int ret; 2023b8d26b3bSNicholas Bellinger 2024b8d26b3bSNicholas Bellinger switch (state) { 2025b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2026b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2027b8d26b3bSNicholas Bellinger break; 2028b8d26b3bSNicholas Bellinger default: 2029b8d26b3bSNicholas Bellinger pr_err("Unknown immediate state: 0x%02x\n", state); 2030b8d26b3bSNicholas Bellinger ret = -EINVAL; 2031b8d26b3bSNicholas Bellinger break; 2032b8d26b3bSNicholas Bellinger } 2033b8d26b3bSNicholas Bellinger 2034b8d26b3bSNicholas Bellinger return ret; 2035b8d26b3bSNicholas Bellinger } 2036b8d26b3bSNicholas Bellinger 2037b8d26b3bSNicholas Bellinger static int 2038b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2039b8d26b3bSNicholas Bellinger { 2040b8d26b3bSNicholas Bellinger int ret; 2041b8d26b3bSNicholas Bellinger 2042b8d26b3bSNicholas Bellinger switch (state) { 2043b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2044b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2045b8d26b3bSNicholas Bellinger if (!ret) { 2046b8d26b3bSNicholas Bellinger pr_debug("Returning iSER Logout -EAGAIN\n"); 2047b8d26b3bSNicholas Bellinger ret = -EAGAIN; 2048b8d26b3bSNicholas Bellinger } 2049b8d26b3bSNicholas Bellinger break; 2050b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2051b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2052b8d26b3bSNicholas Bellinger break; 2053b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2054b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2055b8d26b3bSNicholas Bellinger break; 2056b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2057b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2058b8d26b3bSNicholas Bellinger break; 2059adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2060adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2061adb54c29SNicholas Bellinger break; 2062b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2063b8d26b3bSNicholas Bellinger /* 2064b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2065b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2066b8d26b3bSNicholas Bellinger */ 2067b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2068b8d26b3bSNicholas Bellinger break; 2069b8d26b3bSNicholas Bellinger default: 2070b8d26b3bSNicholas Bellinger pr_err("Unknown response state: 0x%02x\n", state); 2071b8d26b3bSNicholas Bellinger ret = -EINVAL; 2072b8d26b3bSNicholas Bellinger break; 2073b8d26b3bSNicholas Bellinger } 2074b8d26b3bSNicholas Bellinger 2075b8d26b3bSNicholas Bellinger return ret; 2076b8d26b3bSNicholas Bellinger } 2077b8d26b3bSNicholas Bellinger 2078b8d26b3bSNicholas Bellinger static int 2079b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 2080b8d26b3bSNicholas Bellinger struct __kernel_sockaddr_storage *ksockaddr) 2081b8d26b3bSNicholas Bellinger { 2082b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2083b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2084b8d26b3bSNicholas Bellinger struct sockaddr *sa; 2085b8d26b3bSNicholas Bellinger int ret; 2086b8d26b3bSNicholas Bellinger 2087b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2088b8d26b3bSNicholas Bellinger if (!isert_np) { 2089b8d26b3bSNicholas Bellinger pr_err("Unable to allocate struct isert_np\n"); 2090b8d26b3bSNicholas Bellinger return -ENOMEM; 2091b8d26b3bSNicholas Bellinger } 2092b8d26b3bSNicholas Bellinger init_waitqueue_head(&isert_np->np_accept_wq); 2093b8d26b3bSNicholas Bellinger mutex_init(&isert_np->np_accept_mutex); 2094b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&isert_np->np_accept_list); 2095b8d26b3bSNicholas Bellinger init_completion(&isert_np->np_login_comp); 2096b8d26b3bSNicholas Bellinger 2097b8d26b3bSNicholas Bellinger sa = (struct sockaddr *)ksockaddr; 2098b8d26b3bSNicholas Bellinger pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 2099b8d26b3bSNicholas Bellinger /* 2100b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 2101b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 2102b8d26b3bSNicholas Bellinger */ 2103b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 2104b8d26b3bSNicholas Bellinger sizeof(struct __kernel_sockaddr_storage)); 2105b8d26b3bSNicholas Bellinger 2106b8d26b3bSNicholas Bellinger isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 2107b8d26b3bSNicholas Bellinger IB_QPT_RC); 2108b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 2109b8d26b3bSNicholas Bellinger pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 2110b8d26b3bSNicholas Bellinger PTR_ERR(isert_lid)); 2111b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 2112b8d26b3bSNicholas Bellinger goto out; 2113b8d26b3bSNicholas Bellinger } 2114b8d26b3bSNicholas Bellinger 2115b8d26b3bSNicholas Bellinger ret = rdma_bind_addr(isert_lid, sa); 2116b8d26b3bSNicholas Bellinger if (ret) { 2117b8d26b3bSNicholas Bellinger pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 2118b8d26b3bSNicholas Bellinger goto out_lid; 2119b8d26b3bSNicholas Bellinger } 2120b8d26b3bSNicholas Bellinger 2121b8d26b3bSNicholas Bellinger ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 2122b8d26b3bSNicholas Bellinger if (ret) { 2123b8d26b3bSNicholas Bellinger pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 2124b8d26b3bSNicholas Bellinger goto out_lid; 2125b8d26b3bSNicholas Bellinger } 2126b8d26b3bSNicholas Bellinger 2127b8d26b3bSNicholas Bellinger isert_np->np_cm_id = isert_lid; 2128b8d26b3bSNicholas Bellinger np->np_context = isert_np; 2129b8d26b3bSNicholas Bellinger pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 2130b8d26b3bSNicholas Bellinger 2131b8d26b3bSNicholas Bellinger return 0; 2132b8d26b3bSNicholas Bellinger 2133b8d26b3bSNicholas Bellinger out_lid: 2134b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_lid); 2135b8d26b3bSNicholas Bellinger out: 2136b8d26b3bSNicholas Bellinger kfree(isert_np); 2137b8d26b3bSNicholas Bellinger return ret; 2138b8d26b3bSNicholas Bellinger } 2139b8d26b3bSNicholas Bellinger 2140b8d26b3bSNicholas Bellinger static int 2141b8d26b3bSNicholas Bellinger isert_check_accept_queue(struct isert_np *isert_np) 2142b8d26b3bSNicholas Bellinger { 2143b8d26b3bSNicholas Bellinger int empty; 2144b8d26b3bSNicholas Bellinger 2145b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 2146b8d26b3bSNicholas Bellinger empty = list_empty(&isert_np->np_accept_list); 2147b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 2148b8d26b3bSNicholas Bellinger 2149b8d26b3bSNicholas Bellinger return empty; 2150b8d26b3bSNicholas Bellinger } 2151b8d26b3bSNicholas Bellinger 2152b8d26b3bSNicholas Bellinger static int 2153b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 2154b8d26b3bSNicholas Bellinger { 2155b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2156b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 2157b8d26b3bSNicholas Bellinger int ret; 2158b8d26b3bSNicholas Bellinger 2159b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 2160b8d26b3bSNicholas Bellinger cp.responder_resources = isert_conn->responder_resources; 2161b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 2162b8d26b3bSNicholas Bellinger cp.retry_count = 7; 2163b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 2164b8d26b3bSNicholas Bellinger 2165b8d26b3bSNicholas Bellinger pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 2166b8d26b3bSNicholas Bellinger 2167b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 2168b8d26b3bSNicholas Bellinger if (ret) { 2169b8d26b3bSNicholas Bellinger pr_err("rdma_accept() failed with: %d\n", ret); 2170b8d26b3bSNicholas Bellinger return ret; 2171b8d26b3bSNicholas Bellinger } 2172b8d26b3bSNicholas Bellinger 2173b8d26b3bSNicholas Bellinger pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 2174b8d26b3bSNicholas Bellinger 2175b8d26b3bSNicholas Bellinger return 0; 2176b8d26b3bSNicholas Bellinger } 2177b8d26b3bSNicholas Bellinger 2178b8d26b3bSNicholas Bellinger static int 2179b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2180b8d26b3bSNicholas Bellinger { 2181b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2182b8d26b3bSNicholas Bellinger int ret; 2183b8d26b3bSNicholas Bellinger 2184b8d26b3bSNicholas Bellinger pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 21856faaa85fSNicholas Bellinger /* 21866faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 21876faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 21886faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 21896faaa85fSNicholas Bellinger * into a NOP. 21906faaa85fSNicholas Bellinger */ 21916faaa85fSNicholas Bellinger if (!login->first_request) 21926faaa85fSNicholas Bellinger return 0; 2193b8d26b3bSNicholas Bellinger 2194b8d26b3bSNicholas Bellinger ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 2195b8d26b3bSNicholas Bellinger if (ret) 2196b8d26b3bSNicholas Bellinger return ret; 2197b8d26b3bSNicholas Bellinger 2198b8d26b3bSNicholas Bellinger pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 2199b8d26b3bSNicholas Bellinger return 0; 2200b8d26b3bSNicholas Bellinger } 2201b8d26b3bSNicholas Bellinger 2202b8d26b3bSNicholas Bellinger static void 2203b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2204b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 2205b8d26b3bSNicholas Bellinger { 2206b8d26b3bSNicholas Bellinger struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2207b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 2208b8d26b3bSNicholas Bellinger struct sockaddr_in *sock_in; 2209b8d26b3bSNicholas Bellinger struct sockaddr_in6 *sock_in6; 2210b8d26b3bSNicholas Bellinger 2211b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 2212b8d26b3bSNicholas Bellinger 2213b8d26b3bSNicholas Bellinger if (np->np_sockaddr.ss_family == AF_INET6) { 2214b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 2215b8d26b3bSNicholas Bellinger snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 2216b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 2217b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in6->sin6_port); 2218b8d26b3bSNicholas Bellinger 2219b8d26b3bSNicholas Bellinger sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 2220b8d26b3bSNicholas Bellinger snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 2221b8d26b3bSNicholas Bellinger &sock_in6->sin6_addr.in6_u); 2222b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in6->sin6_port); 2223b8d26b3bSNicholas Bellinger } else { 2224b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 2225b8d26b3bSNicholas Bellinger sprintf(conn->login_ip, "%pI4", 2226b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 2227b8d26b3bSNicholas Bellinger conn->login_port = ntohs(sock_in->sin_port); 2228b8d26b3bSNicholas Bellinger 2229b8d26b3bSNicholas Bellinger sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 2230b8d26b3bSNicholas Bellinger sprintf(conn->local_ip, "%pI4", 2231b8d26b3bSNicholas Bellinger &sock_in->sin_addr.s_addr); 2232b8d26b3bSNicholas Bellinger conn->local_port = ntohs(sock_in->sin_port); 2233b8d26b3bSNicholas Bellinger } 2234b8d26b3bSNicholas Bellinger } 2235b8d26b3bSNicholas Bellinger 2236b8d26b3bSNicholas Bellinger static int 2237b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2238b8d26b3bSNicholas Bellinger { 2239b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 2240b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2241b8d26b3bSNicholas Bellinger int max_accept = 0, ret; 2242b8d26b3bSNicholas Bellinger 2243b8d26b3bSNicholas Bellinger accept_wait: 2244b8d26b3bSNicholas Bellinger ret = wait_event_interruptible(isert_np->np_accept_wq, 2245b8d26b3bSNicholas Bellinger !isert_check_accept_queue(isert_np) || 2246b8d26b3bSNicholas Bellinger np->np_thread_state == ISCSI_NP_THREAD_RESET); 2247b8d26b3bSNicholas Bellinger if (max_accept > 5) 2248b8d26b3bSNicholas Bellinger return -ENODEV; 2249b8d26b3bSNicholas Bellinger 2250b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 2251b8d26b3bSNicholas Bellinger if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 2252b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 2253b8d26b3bSNicholas Bellinger pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 2254b8d26b3bSNicholas Bellinger return -ENODEV; 2255b8d26b3bSNicholas Bellinger } 2256b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 2257b8d26b3bSNicholas Bellinger 2258b8d26b3bSNicholas Bellinger mutex_lock(&isert_np->np_accept_mutex); 2259b8d26b3bSNicholas Bellinger if (list_empty(&isert_np->np_accept_list)) { 2260b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 2261b8d26b3bSNicholas Bellinger max_accept++; 2262b8d26b3bSNicholas Bellinger goto accept_wait; 2263b8d26b3bSNicholas Bellinger } 2264b8d26b3bSNicholas Bellinger isert_conn = list_first_entry(&isert_np->np_accept_list, 2265b8d26b3bSNicholas Bellinger struct isert_conn, conn_accept_node); 2266b8d26b3bSNicholas Bellinger list_del_init(&isert_conn->conn_accept_node); 2267b8d26b3bSNicholas Bellinger mutex_unlock(&isert_np->np_accept_mutex); 2268b8d26b3bSNicholas Bellinger 2269b8d26b3bSNicholas Bellinger conn->context = isert_conn; 2270b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 2271b8d26b3bSNicholas Bellinger max_accept = 0; 2272b8d26b3bSNicholas Bellinger 2273b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 2274b8d26b3bSNicholas Bellinger if (ret) 2275b8d26b3bSNicholas Bellinger return ret; 2276b8d26b3bSNicholas Bellinger 2277b8d26b3bSNicholas Bellinger ret = isert_rdma_accept(isert_conn); 2278b8d26b3bSNicholas Bellinger if (ret) 2279b8d26b3bSNicholas Bellinger return ret; 2280b8d26b3bSNicholas Bellinger 2281b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 2282b8d26b3bSNicholas Bellinger 2283b8d26b3bSNicholas Bellinger pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 2284b8d26b3bSNicholas Bellinger return 0; 2285b8d26b3bSNicholas Bellinger } 2286b8d26b3bSNicholas Bellinger 2287b8d26b3bSNicholas Bellinger static void 2288b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 2289b8d26b3bSNicholas Bellinger { 2290b8d26b3bSNicholas Bellinger struct isert_np *isert_np = (struct isert_np *)np->np_context; 2291b8d26b3bSNicholas Bellinger 2292b8d26b3bSNicholas Bellinger rdma_destroy_id(isert_np->np_cm_id); 2293b8d26b3bSNicholas Bellinger 2294b8d26b3bSNicholas Bellinger np->np_context = NULL; 2295b8d26b3bSNicholas Bellinger kfree(isert_np); 2296b8d26b3bSNicholas Bellinger } 2297b8d26b3bSNicholas Bellinger 2298b2cb9649SNicholas Bellinger static int isert_check_state(struct isert_conn *isert_conn, int state) 2299b2cb9649SNicholas Bellinger { 2300b2cb9649SNicholas Bellinger int ret; 2301b2cb9649SNicholas Bellinger 2302b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 2303b2cb9649SNicholas Bellinger ret = (isert_conn->state == state); 2304b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2305b2cb9649SNicholas Bellinger 2306b2cb9649SNicholas Bellinger return ret; 2307b2cb9649SNicholas Bellinger } 2308b2cb9649SNicholas Bellinger 2309b8d26b3bSNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 2310b8d26b3bSNicholas Bellinger { 2311b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2312b8d26b3bSNicholas Bellinger 2313b8d26b3bSNicholas Bellinger pr_debug("isert_free_conn: Starting \n"); 2314b8d26b3bSNicholas Bellinger /* 2315b8d26b3bSNicholas Bellinger * Decrement post_send_buf_count for special case when called 2316b8d26b3bSNicholas Bellinger * from isert_do_control_comp() -> iscsit_logout_post_handler() 2317b8d26b3bSNicholas Bellinger */ 2318b2cb9649SNicholas Bellinger mutex_lock(&isert_conn->conn_mutex); 2319b8d26b3bSNicholas Bellinger if (isert_conn->logout_posted) 2320b8d26b3bSNicholas Bellinger atomic_dec(&isert_conn->post_send_buf_count); 2321b8d26b3bSNicholas Bellinger 2322b2cb9649SNicholas Bellinger if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 2323b2cb9649SNicholas Bellinger pr_debug("Calling rdma_disconnect from isert_free_conn\n"); 2324b8d26b3bSNicholas Bellinger rdma_disconnect(isert_conn->conn_cm_id); 2325b2cb9649SNicholas Bellinger } 2326b8d26b3bSNicholas Bellinger /* 2327b8d26b3bSNicholas Bellinger * Only wait for conn_wait_comp_err if the isert_conn made it 2328b8d26b3bSNicholas Bellinger * into full feature phase.. 2329b8d26b3bSNicholas Bellinger */ 2330b2cb9649SNicholas Bellinger if (isert_conn->state == ISER_CONN_UP) { 2331b8d26b3bSNicholas Bellinger pr_debug("isert_free_conn: Before wait_event comp_err %d\n", 2332b8d26b3bSNicholas Bellinger isert_conn->state); 2333b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2334b8d26b3bSNicholas Bellinger 2335b2cb9649SNicholas Bellinger wait_event(isert_conn->conn_wait_comp_err, 2336b2cb9649SNicholas Bellinger (isert_check_state(isert_conn, ISER_CONN_TERMINATING))); 2337b2cb9649SNicholas Bellinger 2338b2cb9649SNicholas Bellinger wait_event(isert_conn->conn_wait, 2339b2cb9649SNicholas Bellinger (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2340b2cb9649SNicholas Bellinger 2341b2cb9649SNicholas Bellinger isert_put_conn(isert_conn); 2342b2cb9649SNicholas Bellinger return; 2343b2cb9649SNicholas Bellinger } 2344b2cb9649SNicholas Bellinger if (isert_conn->state == ISER_CONN_INIT) { 2345b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2346b2cb9649SNicholas Bellinger isert_put_conn(isert_conn); 2347b2cb9649SNicholas Bellinger return; 2348b2cb9649SNicholas Bellinger } 2349b2cb9649SNicholas Bellinger pr_debug("isert_free_conn: wait_event conn_wait %d\n", 2350b2cb9649SNicholas Bellinger isert_conn->state); 2351b2cb9649SNicholas Bellinger mutex_unlock(&isert_conn->conn_mutex); 2352b2cb9649SNicholas Bellinger 2353b2cb9649SNicholas Bellinger wait_event(isert_conn->conn_wait, 2354b2cb9649SNicholas Bellinger (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2355b8d26b3bSNicholas Bellinger 2356b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 2357b8d26b3bSNicholas Bellinger } 2358b8d26b3bSNicholas Bellinger 2359b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 2360b8d26b3bSNicholas Bellinger .name = "IB/iSER", 2361b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 2362d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 2363b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 2364b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 2365b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 2366b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 2367b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 2368b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 2369b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 2370b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 2371b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 2372b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 2373b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 2374b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 2375b8d26b3bSNicholas Bellinger }; 2376b8d26b3bSNicholas Bellinger 2377b8d26b3bSNicholas Bellinger static int __init isert_init(void) 2378b8d26b3bSNicholas Bellinger { 2379b8d26b3bSNicholas Bellinger int ret; 2380b8d26b3bSNicholas Bellinger 2381b8d26b3bSNicholas Bellinger isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 2382b8d26b3bSNicholas Bellinger if (!isert_rx_wq) { 2383b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_rx_wq\n"); 2384b8d26b3bSNicholas Bellinger return -ENOMEM; 2385b8d26b3bSNicholas Bellinger } 2386b8d26b3bSNicholas Bellinger 2387b8d26b3bSNicholas Bellinger isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 2388b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 2389b8d26b3bSNicholas Bellinger pr_err("Unable to allocate isert_comp_wq\n"); 2390b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2391b8d26b3bSNicholas Bellinger goto destroy_rx_wq; 2392b8d26b3bSNicholas Bellinger } 2393b8d26b3bSNicholas Bellinger 2394b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 2395b8d26b3bSNicholas Bellinger pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2396b8d26b3bSNicholas Bellinger return 0; 2397b8d26b3bSNicholas Bellinger 2398b8d26b3bSNicholas Bellinger destroy_rx_wq: 2399b8d26b3bSNicholas Bellinger destroy_workqueue(isert_rx_wq); 2400b8d26b3bSNicholas Bellinger return ret; 2401b8d26b3bSNicholas Bellinger } 2402b8d26b3bSNicholas Bellinger 2403b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 2404b8d26b3bSNicholas Bellinger { 2405b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 2406b8d26b3bSNicholas Bellinger destroy_workqueue(isert_rx_wq); 2407b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 2408b8d26b3bSNicholas Bellinger pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 2409b8d26b3bSNicholas Bellinger } 2410b8d26b3bSNicholas Bellinger 2411b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2412b8d26b3bSNicholas Bellinger MODULE_VERSION("0.1"); 2413b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2414b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 2415b8d26b3bSNicholas Bellinger 2416b8d26b3bSNicholas Bellinger module_init(isert_init); 2417b8d26b3bSNicholas Bellinger module_exit(isert_exit); 2418