1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 44c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 25b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 26b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 27b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 29b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 30531b7bf4SSagi Grimberg #include <linux/semaphore.h> 31b8d26b3bSNicholas Bellinger 32b8d26b3bSNicholas Bellinger #include "ib_isert.h" 33b8d26b3bSNicholas Bellinger 34b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 35b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36b8d26b3bSNicholas Bellinger #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 37bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 38bdf20e72SSagi Grimberg ISERT_MAX_CONN) 39b8d26b3bSNicholas Bellinger 4045678b6bSSagi Grimberg static int isert_debug_level; 4124f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 4224f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 4324f412ddSSagi Grimberg 44b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 45b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 46b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 47b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 48b8d26b3bSNicholas Bellinger 49b8d26b3bSNicholas Bellinger static void 50d40945d8SVu Pham isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51d40945d8SVu Pham static int 52d40945d8SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 53d40945d8SVu Pham struct isert_rdma_wr *wr); 5459464ef4SVu Pham static void 55a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 5659464ef4SVu Pham static int 57a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5859464ef4SVu Pham struct isert_rdma_wr *wr); 59f93f3a70SSagi Grimberg static int 60f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 612371e5daSSagi Grimberg static int 622371e5daSSagi Grimberg isert_rdma_post_recvl(struct isert_conn *isert_conn); 632371e5daSSagi Grimberg static int 642371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 65ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 66d40945d8SVu Pham 672f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work); 682f1b6b7dSSagi Grimberg 69302cc7c3SSagi Grimberg static inline bool 70302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 71302cc7c3SSagi Grimberg { 7223a548eeSSagi Grimberg return (conn->pi_support && 73302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 74302cc7c3SSagi Grimberg } 75302cc7c3SSagi Grimberg 76302cc7c3SSagi Grimberg 77d40945d8SVu Pham static void 78b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 79b8d26b3bSNicholas Bellinger { 806700425eSSagi Grimberg struct isert_conn *isert_conn = context; 81b8d26b3bSNicholas Bellinger 82ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n", 83ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn); 84ea8a1616SSagi Grimberg 85b8d26b3bSNicholas Bellinger switch (e->event) { 86b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 87dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 88b8d26b3bSNicholas Bellinger break; 89b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 904c22e07fSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 91b8d26b3bSNicholas Bellinger break; 92b8d26b3bSNicholas Bellinger default: 93b8d26b3bSNicholas Bellinger break; 94b8d26b3bSNicholas Bellinger } 95b8d26b3bSNicholas Bellinger } 96b8d26b3bSNicholas Bellinger 9740fc069aSSagi Grimberg static struct isert_comp * 9840fc069aSSagi Grimberg isert_comp_get(struct isert_conn *isert_conn) 99b8d26b3bSNicholas Bellinger { 100dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1014a295baeSSagi Grimberg struct isert_comp *comp; 10240fc069aSSagi Grimberg int i, min = 0; 103b8d26b3bSNicholas Bellinger 104b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 1054a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 1064a295baeSSagi Grimberg if (device->comps[i].active_qps < 1074a295baeSSagi Grimberg device->comps[min].active_qps) 1084a295baeSSagi Grimberg min = i; 1094a295baeSSagi Grimberg comp = &device->comps[min]; 1104a295baeSSagi Grimberg comp->active_qps++; 11140fc069aSSagi Grimberg mutex_unlock(&device_list_mutex); 11240fc069aSSagi Grimberg 11324f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1144a295baeSSagi Grimberg isert_conn, comp, min); 11540fc069aSSagi Grimberg 11640fc069aSSagi Grimberg return comp; 11740fc069aSSagi Grimberg } 11840fc069aSSagi Grimberg 11940fc069aSSagi Grimberg static void 12040fc069aSSagi Grimberg isert_comp_put(struct isert_comp *comp) 12140fc069aSSagi Grimberg { 12240fc069aSSagi Grimberg mutex_lock(&device_list_mutex); 12340fc069aSSagi Grimberg comp->active_qps--; 124b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 12540fc069aSSagi Grimberg } 12640fc069aSSagi Grimberg 12740fc069aSSagi Grimberg static struct ib_qp * 12840fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn, 12940fc069aSSagi Grimberg struct isert_comp *comp, 13040fc069aSSagi Grimberg struct rdma_cm_id *cma_id) 13140fc069aSSagi Grimberg { 132dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 13340fc069aSSagi Grimberg struct ib_qp_init_attr attr; 13440fc069aSSagi Grimberg int ret; 135b8d26b3bSNicholas Bellinger 136b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 137b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 138b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1396f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1406f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 141b8d26b3bSNicholas Bellinger attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 142bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 1434a061b28SOr Gerlitz attr.cap.max_send_sge = device->ib_device->attrs.max_sge; 1444a061b28SOr Gerlitz isert_conn->max_sge = min(device->ib_device->attrs.max_sge, 1454a061b28SOr Gerlitz device->ib_device->attrs.max_sge_rd); 146b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 147b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 148b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 149570db170SSagi Grimberg if (device->pi_capable) 150d3e125daSSagi Grimberg attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 151b8d26b3bSNicholas Bellinger 15267cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr); 153b8d26b3bSNicholas Bellinger if (ret) { 15424f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 15540fc069aSSagi Grimberg return ERR_PTR(ret); 15640fc069aSSagi Grimberg } 15740fc069aSSagi Grimberg 15840fc069aSSagi Grimberg return cma_id->qp; 15940fc069aSSagi Grimberg } 16040fc069aSSagi Grimberg 16140fc069aSSagi Grimberg static int 16240fc069aSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 16340fc069aSSagi Grimberg { 16440fc069aSSagi Grimberg struct isert_comp *comp; 16540fc069aSSagi Grimberg int ret; 16640fc069aSSagi Grimberg 16740fc069aSSagi Grimberg comp = isert_comp_get(isert_conn); 168dac6ab30SSagi Grimberg isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 169dac6ab30SSagi Grimberg if (IS_ERR(isert_conn->qp)) { 170dac6ab30SSagi Grimberg ret = PTR_ERR(isert_conn->qp); 17119e2090fSSagi Grimberg goto err; 172b8d26b3bSNicholas Bellinger } 173b8d26b3bSNicholas Bellinger 174b8d26b3bSNicholas Bellinger return 0; 17519e2090fSSagi Grimberg err: 17640fc069aSSagi Grimberg isert_comp_put(comp); 17719e2090fSSagi Grimberg return ret; 178b8d26b3bSNicholas Bellinger } 179b8d26b3bSNicholas Bellinger 180b8d26b3bSNicholas Bellinger static void 181b8d26b3bSNicholas Bellinger isert_cq_event_callback(struct ib_event *e, void *context) 182b8d26b3bSNicholas Bellinger { 1834c22e07fSSagi Grimberg isert_dbg("event: %d\n", e->event); 184b8d26b3bSNicholas Bellinger } 185b8d26b3bSNicholas Bellinger 186b8d26b3bSNicholas Bellinger static int 187b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 188b8d26b3bSNicholas Bellinger { 189dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 19067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 191b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 192b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 193b8d26b3bSNicholas Bellinger u64 dma_addr; 194b8d26b3bSNicholas Bellinger int i, j; 195b8d26b3bSNicholas Bellinger 196dac6ab30SSagi Grimberg isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 197b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 198dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 199b8d26b3bSNicholas Bellinger goto fail; 200b8d26b3bSNicholas Bellinger 201dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 202b8d26b3bSNicholas Bellinger 203b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 204b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 205b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 206b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 207b8d26b3bSNicholas Bellinger goto dma_map_fail; 208b8d26b3bSNicholas Bellinger 209b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 210b8d26b3bSNicholas Bellinger 211b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 212b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 213b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 21434efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey; 215b8d26b3bSNicholas Bellinger } 216b8d26b3bSNicholas Bellinger 217b8d26b3bSNicholas Bellinger return 0; 218b8d26b3bSNicholas Bellinger 219b8d26b3bSNicholas Bellinger dma_map_fail: 220dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 221b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 222b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 223b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 224b8d26b3bSNicholas Bellinger } 225dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 226dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 227b8d26b3bSNicholas Bellinger fail: 2284c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 2294c22e07fSSagi Grimberg 230b8d26b3bSNicholas Bellinger return -ENOMEM; 231b8d26b3bSNicholas Bellinger } 232b8d26b3bSNicholas Bellinger 233b8d26b3bSNicholas Bellinger static void 234b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 235b8d26b3bSNicholas Bellinger { 236dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 237b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 238b8d26b3bSNicholas Bellinger int i; 239b8d26b3bSNicholas Bellinger 240dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 241b8d26b3bSNicholas Bellinger return; 242b8d26b3bSNicholas Bellinger 243dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 244b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 245b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 246b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 247b8d26b3bSNicholas Bellinger } 248b8d26b3bSNicholas Bellinger 249dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 250dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 251b8d26b3bSNicholas Bellinger } 252b8d26b3bSNicholas Bellinger 2536f0fae3dSSagi Grimberg static void isert_cq_work(struct work_struct *); 2546f0fae3dSSagi Grimberg static void isert_cq_callback(struct ib_cq *, void *); 255b8d26b3bSNicholas Bellinger 256172369c5SSagi Grimberg static void 257172369c5SSagi Grimberg isert_free_comps(struct isert_device *device) 258b8d26b3bSNicholas Bellinger { 259172369c5SSagi Grimberg int i; 260b8d26b3bSNicholas Bellinger 261172369c5SSagi Grimberg for (i = 0; i < device->comps_used; i++) { 262172369c5SSagi Grimberg struct isert_comp *comp = &device->comps[i]; 26359464ef4SVu Pham 264172369c5SSagi Grimberg if (comp->cq) { 265172369c5SSagi Grimberg cancel_work_sync(&comp->work); 266172369c5SSagi Grimberg ib_destroy_cq(comp->cq); 267172369c5SSagi Grimberg } 268172369c5SSagi Grimberg } 269172369c5SSagi Grimberg kfree(device->comps); 27059464ef4SVu Pham } 271d40945d8SVu Pham 272172369c5SSagi Grimberg static int 2734a061b28SOr Gerlitz isert_alloc_comps(struct isert_device *device) 274172369c5SSagi Grimberg { 275172369c5SSagi Grimberg int i, max_cqe, ret = 0; 276d3e125daSSagi Grimberg 2774a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2784a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 279172369c5SSagi Grimberg 28024f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 281d3e125daSSagi Grimberg "Fast registration %d pi_capable %d\n", 2824a295baeSSagi Grimberg device->comps_used, device->ib_device->name, 283d3e125daSSagi Grimberg device->ib_device->num_comp_vectors, device->use_fastreg, 284d3e125daSSagi Grimberg device->pi_capable); 2854a295baeSSagi Grimberg 2864a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2874a295baeSSagi Grimberg GFP_KERNEL); 2884a295baeSSagi Grimberg if (!device->comps) { 28924f412ddSSagi Grimberg isert_err("Unable to allocate completion contexts\n"); 290b8d26b3bSNicholas Bellinger return -ENOMEM; 291b8d26b3bSNicholas Bellinger } 292b8d26b3bSNicholas Bellinger 2934a061b28SOr Gerlitz max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 294172369c5SSagi Grimberg 2954a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2968e37210bSMatan Barak struct ib_cq_init_attr cq_attr = {}; 2974a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 298b8d26b3bSNicholas Bellinger 2994a295baeSSagi Grimberg comp->device = device; 3006f0fae3dSSagi Grimberg INIT_WORK(&comp->work, isert_cq_work); 3018e37210bSMatan Barak cq_attr.cqe = max_cqe; 3028e37210bSMatan Barak cq_attr.comp_vector = i; 3036f0fae3dSSagi Grimberg comp->cq = ib_create_cq(device->ib_device, 3046f0fae3dSSagi Grimberg isert_cq_callback, 305b8d26b3bSNicholas Bellinger isert_cq_event_callback, 3064a295baeSSagi Grimberg (void *)comp, 3078e37210bSMatan Barak &cq_attr); 3086f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 309172369c5SSagi Grimberg isert_err("Unable to allocate cq\n"); 3106f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 3116f0fae3dSSagi Grimberg comp->cq = NULL; 312b8d26b3bSNicholas Bellinger goto out_cq; 31394a71110SWei Yongjun } 314b8d26b3bSNicholas Bellinger 3156f0fae3dSSagi Grimberg ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 31694a71110SWei Yongjun if (ret) 317b8d26b3bSNicholas Bellinger goto out_cq; 318b8d26b3bSNicholas Bellinger } 319b8d26b3bSNicholas Bellinger 320172369c5SSagi Grimberg return 0; 321172369c5SSagi Grimberg out_cq: 322172369c5SSagi Grimberg isert_free_comps(device); 323172369c5SSagi Grimberg return ret; 324172369c5SSagi Grimberg } 325172369c5SSagi Grimberg 326172369c5SSagi Grimberg static int 327172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device) 328172369c5SSagi Grimberg { 3294a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device; 330fd8205e8SSagi Grimberg int ret; 331172369c5SSagi Grimberg 3324a061b28SOr Gerlitz isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge); 3334a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 334172369c5SSagi Grimberg 335172369c5SSagi Grimberg /* asign function handlers */ 3364a061b28SOr Gerlitz if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 3374a061b28SOr Gerlitz ib_dev->attrs.device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 338172369c5SSagi Grimberg device->use_fastreg = 1; 339172369c5SSagi Grimberg device->reg_rdma_mem = isert_reg_rdma; 340172369c5SSagi Grimberg device->unreg_rdma_mem = isert_unreg_rdma; 341172369c5SSagi Grimberg } else { 342172369c5SSagi Grimberg device->use_fastreg = 0; 343172369c5SSagi Grimberg device->reg_rdma_mem = isert_map_rdma; 344172369c5SSagi Grimberg device->unreg_rdma_mem = isert_unmap_cmd; 345172369c5SSagi Grimberg } 346172369c5SSagi Grimberg 3474a061b28SOr Gerlitz ret = isert_alloc_comps(device); 348172369c5SSagi Grimberg if (ret) 349172369c5SSagi Grimberg return ret; 350172369c5SSagi Grimberg 3514a061b28SOr Gerlitz device->pd = ib_alloc_pd(ib_dev); 35267cb3949SSagi Grimberg if (IS_ERR(device->pd)) { 35367cb3949SSagi Grimberg ret = PTR_ERR(device->pd); 35467cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n", 35567cb3949SSagi Grimberg device, ret); 35667cb3949SSagi Grimberg goto out_cq; 35767cb3949SSagi Grimberg } 35867cb3949SSagi Grimberg 359172369c5SSagi Grimberg /* Check signature cap */ 3604a061b28SOr Gerlitz device->pi_capable = ib_dev->attrs.device_cap_flags & 361172369c5SSagi Grimberg IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 36267cb3949SSagi Grimberg 363b8d26b3bSNicholas Bellinger return 0; 364b8d26b3bSNicholas Bellinger 365b8d26b3bSNicholas Bellinger out_cq: 366172369c5SSagi Grimberg isert_free_comps(device); 367b8d26b3bSNicholas Bellinger return ret; 368b8d26b3bSNicholas Bellinger } 369b8d26b3bSNicholas Bellinger 370b8d26b3bSNicholas Bellinger static void 371b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 372b8d26b3bSNicholas Bellinger { 37324f412ddSSagi Grimberg isert_info("device %p\n", device); 374b8d26b3bSNicholas Bellinger 37567cb3949SSagi Grimberg ib_dealloc_pd(device->pd); 376172369c5SSagi Grimberg isert_free_comps(device); 377b8d26b3bSNicholas Bellinger } 378b8d26b3bSNicholas Bellinger 379b8d26b3bSNicholas Bellinger static void 380cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device) 381b8d26b3bSNicholas Bellinger { 382b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 383b8d26b3bSNicholas Bellinger device->refcount--; 3844c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount); 385b8d26b3bSNicholas Bellinger if (!device->refcount) { 386b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 387b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 388b8d26b3bSNicholas Bellinger kfree(device); 389b8d26b3bSNicholas Bellinger } 390b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 391b8d26b3bSNicholas Bellinger } 392b8d26b3bSNicholas Bellinger 393b8d26b3bSNicholas Bellinger static struct isert_device * 394cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id) 395b8d26b3bSNicholas Bellinger { 396b8d26b3bSNicholas Bellinger struct isert_device *device; 397b8d26b3bSNicholas Bellinger int ret; 398b8d26b3bSNicholas Bellinger 399b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 400b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 401b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 402b8d26b3bSNicholas Bellinger device->refcount++; 4034c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n", 4044c22e07fSSagi Grimberg device, device->refcount); 405b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 406b8d26b3bSNicholas Bellinger return device; 407b8d26b3bSNicholas Bellinger } 408b8d26b3bSNicholas Bellinger } 409b8d26b3bSNicholas Bellinger 410b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 411b8d26b3bSNicholas Bellinger if (!device) { 412b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 413b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 414b8d26b3bSNicholas Bellinger } 415b8d26b3bSNicholas Bellinger 416b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 417b8d26b3bSNicholas Bellinger 418b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 419b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 420b8d26b3bSNicholas Bellinger if (ret) { 421b8d26b3bSNicholas Bellinger kfree(device); 422b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 423b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 424b8d26b3bSNicholas Bellinger } 425b8d26b3bSNicholas Bellinger 426b8d26b3bSNicholas Bellinger device->refcount++; 427b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 4284c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n", 4294c22e07fSSagi Grimberg device, device->refcount); 430b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 431b8d26b3bSNicholas Bellinger 432b8d26b3bSNicholas Bellinger return device; 433b8d26b3bSNicholas Bellinger } 434b8d26b3bSNicholas Bellinger 43559464ef4SVu Pham static void 436a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 43759464ef4SVu Pham { 43859464ef4SVu Pham struct fast_reg_descriptor *fr_desc, *tmp; 43959464ef4SVu Pham int i = 0; 44059464ef4SVu Pham 441dac6ab30SSagi Grimberg if (list_empty(&isert_conn->fr_pool)) 44259464ef4SVu Pham return; 44359464ef4SVu Pham 4444c22e07fSSagi Grimberg isert_info("Freeing conn %p fastreg pool", isert_conn); 44559464ef4SVu Pham 44659464ef4SVu Pham list_for_each_entry_safe(fr_desc, tmp, 447dac6ab30SSagi Grimberg &isert_conn->fr_pool, list) { 44859464ef4SVu Pham list_del(&fr_desc->list); 44959464ef4SVu Pham ib_dereg_mr(fr_desc->data_mr); 450d3e125daSSagi Grimberg if (fr_desc->pi_ctx) { 451d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 4528b91ffc1SSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->sig_mr); 453d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 454d3e125daSSagi Grimberg } 45559464ef4SVu Pham kfree(fr_desc); 45659464ef4SVu Pham ++i; 45759464ef4SVu Pham } 45859464ef4SVu Pham 459dac6ab30SSagi Grimberg if (i < isert_conn->fr_pool_size) 46024f412ddSSagi Grimberg isert_warn("Pool still has %d regions registered\n", 461dac6ab30SSagi Grimberg isert_conn->fr_pool_size - i); 46259464ef4SVu Pham } 46359464ef4SVu Pham 46459464ef4SVu Pham static int 465570db170SSagi Grimberg isert_create_pi_ctx(struct fast_reg_descriptor *desc, 466570db170SSagi Grimberg struct ib_device *device, 467570db170SSagi Grimberg struct ib_pd *pd) 468570db170SSagi Grimberg { 469570db170SSagi Grimberg struct pi_context *pi_ctx; 470570db170SSagi Grimberg int ret; 471570db170SSagi Grimberg 472570db170SSagi Grimberg pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 473570db170SSagi Grimberg if (!pi_ctx) { 47424f412ddSSagi Grimberg isert_err("Failed to allocate pi context\n"); 475570db170SSagi Grimberg return -ENOMEM; 476570db170SSagi Grimberg } 477570db170SSagi Grimberg 478a89be2ccSSagi Grimberg pi_ctx->prot_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 479a89be2ccSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 480570db170SSagi Grimberg if (IS_ERR(pi_ctx->prot_mr)) { 48124f412ddSSagi Grimberg isert_err("Failed to allocate prot frmr err=%ld\n", 482570db170SSagi Grimberg PTR_ERR(pi_ctx->prot_mr)); 483570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->prot_mr); 48416c2d702SSagi Grimberg goto err_pi_ctx; 485570db170SSagi Grimberg } 486570db170SSagi Grimberg desc->ind |= ISERT_PROT_KEY_VALID; 487570db170SSagi Grimberg 4889bee178bSSagi Grimberg pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2); 489570db170SSagi Grimberg if (IS_ERR(pi_ctx->sig_mr)) { 49024f412ddSSagi Grimberg isert_err("Failed to allocate signature enabled mr err=%ld\n", 491570db170SSagi Grimberg PTR_ERR(pi_ctx->sig_mr)); 492570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->sig_mr); 493570db170SSagi Grimberg goto err_prot_mr; 494570db170SSagi Grimberg } 495570db170SSagi Grimberg 496570db170SSagi Grimberg desc->pi_ctx = pi_ctx; 497570db170SSagi Grimberg desc->ind |= ISERT_SIG_KEY_VALID; 498570db170SSagi Grimberg desc->ind &= ~ISERT_PROTECTED; 499570db170SSagi Grimberg 500570db170SSagi Grimberg return 0; 501570db170SSagi Grimberg 502570db170SSagi Grimberg err_prot_mr: 503b2feda4fSRoland Dreier ib_dereg_mr(pi_ctx->prot_mr); 504570db170SSagi Grimberg err_pi_ctx: 505b2feda4fSRoland Dreier kfree(pi_ctx); 506570db170SSagi Grimberg 507570db170SSagi Grimberg return ret; 508570db170SSagi Grimberg } 509570db170SSagi Grimberg 510570db170SSagi Grimberg static int 511dc87a90fSSagi Grimberg isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 512570db170SSagi Grimberg struct fast_reg_descriptor *fr_desc) 513dc87a90fSSagi Grimberg { 514a89be2ccSSagi Grimberg fr_desc->data_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 515a89be2ccSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 516dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_mr)) { 51724f412ddSSagi Grimberg isert_err("Failed to allocate data frmr err=%ld\n", 518dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_mr)); 51916c2d702SSagi Grimberg return PTR_ERR(fr_desc->data_mr); 520dc87a90fSSagi Grimberg } 521d3e125daSSagi Grimberg fr_desc->ind |= ISERT_DATA_KEY_VALID; 522dc87a90fSSagi Grimberg 52324f412ddSSagi Grimberg isert_dbg("Created fr_desc %p\n", fr_desc); 524dc87a90fSSagi Grimberg 525dc87a90fSSagi Grimberg return 0; 526dc87a90fSSagi Grimberg } 527dc87a90fSSagi Grimberg 528dc87a90fSSagi Grimberg static int 529570db170SSagi Grimberg isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 53059464ef4SVu Pham { 53159464ef4SVu Pham struct fast_reg_descriptor *fr_desc; 532dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 533f46d6a8aSNicholas Bellinger struct se_session *se_sess = isert_conn->conn->sess->se_sess; 534f46d6a8aSNicholas Bellinger struct se_node_acl *se_nacl = se_sess->se_node_acl; 535f46d6a8aSNicholas Bellinger int i, ret, tag_num; 536f46d6a8aSNicholas Bellinger /* 537f46d6a8aSNicholas Bellinger * Setup the number of FRMRs based upon the number of tags 538f46d6a8aSNicholas Bellinger * available to session in iscsi_target_locate_portal(). 539f46d6a8aSNicholas Bellinger */ 540f46d6a8aSNicholas Bellinger tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 541f46d6a8aSNicholas Bellinger tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 54259464ef4SVu Pham 543dac6ab30SSagi Grimberg isert_conn->fr_pool_size = 0; 544f46d6a8aSNicholas Bellinger for (i = 0; i < tag_num; i++) { 54559464ef4SVu Pham fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 54659464ef4SVu Pham if (!fr_desc) { 54724f412ddSSagi Grimberg isert_err("Failed to allocate fast_reg descriptor\n"); 54859464ef4SVu Pham ret = -ENOMEM; 54959464ef4SVu Pham goto err; 55059464ef4SVu Pham } 55159464ef4SVu Pham 552dc87a90fSSagi Grimberg ret = isert_create_fr_desc(device->ib_device, 55367cb3949SSagi Grimberg device->pd, fr_desc); 554dc87a90fSSagi Grimberg if (ret) { 55524f412ddSSagi Grimberg isert_err("Failed to create fastreg descriptor err=%d\n", 556dc87a90fSSagi Grimberg ret); 557a80e21b3SNicholas Bellinger kfree(fr_desc); 55859464ef4SVu Pham goto err; 55959464ef4SVu Pham } 56059464ef4SVu Pham 561dac6ab30SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 562dac6ab30SSagi Grimberg isert_conn->fr_pool_size++; 56359464ef4SVu Pham } 56459464ef4SVu Pham 56524f412ddSSagi Grimberg isert_dbg("Creating conn %p fastreg pool size=%d", 566dac6ab30SSagi Grimberg isert_conn, isert_conn->fr_pool_size); 56759464ef4SVu Pham 56859464ef4SVu Pham return 0; 56959464ef4SVu Pham 57059464ef4SVu Pham err: 571a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 57259464ef4SVu Pham return ret; 57359464ef4SVu Pham } 57459464ef4SVu Pham 575ae9ea9edSSagi Grimberg static void 576ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn) 577ae9ea9edSSagi Grimberg { 578ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT; 579bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node); 580dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp); 581ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp); 582dac6ab30SSagi Grimberg init_completion(&isert_conn->wait); 583dac6ab30SSagi Grimberg kref_init(&isert_conn->kref); 584dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex); 585dac6ab30SSagi Grimberg spin_lock_init(&isert_conn->pool_lock); 586dac6ab30SSagi Grimberg INIT_LIST_HEAD(&isert_conn->fr_pool); 5872f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 588ae9ea9edSSagi Grimberg } 589ae9ea9edSSagi Grimberg 590ae9ea9edSSagi Grimberg static void 591ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn) 592ae9ea9edSSagi Grimberg { 593dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 594ae9ea9edSSagi Grimberg 595ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 596ae9ea9edSSagi Grimberg ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 597ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 598ae9ea9edSSagi Grimberg ISCSI_DEF_MAX_RECV_SEG_LEN, 599ae9ea9edSSagi Grimberg DMA_FROM_DEVICE); 600ae9ea9edSSagi Grimberg kfree(isert_conn->login_buf); 601ae9ea9edSSagi Grimberg } 602ae9ea9edSSagi Grimberg 603ae9ea9edSSagi Grimberg static int 604ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn, 605ae9ea9edSSagi Grimberg struct ib_device *ib_dev) 606ae9ea9edSSagi Grimberg { 607ae9ea9edSSagi Grimberg int ret; 608ae9ea9edSSagi Grimberg 609ae9ea9edSSagi Grimberg isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 610ae9ea9edSSagi Grimberg ISER_RX_LOGIN_SIZE, GFP_KERNEL); 611ae9ea9edSSagi Grimberg if (!isert_conn->login_buf) { 612ae9ea9edSSagi Grimberg isert_err("Unable to allocate isert_conn->login_buf\n"); 613ae9ea9edSSagi Grimberg return -ENOMEM; 614ae9ea9edSSagi Grimberg } 615ae9ea9edSSagi Grimberg 616ae9ea9edSSagi Grimberg isert_conn->login_req_buf = isert_conn->login_buf; 617ae9ea9edSSagi Grimberg isert_conn->login_rsp_buf = isert_conn->login_buf + 618ae9ea9edSSagi Grimberg ISCSI_DEF_MAX_RECV_SEG_LEN; 619ae9ea9edSSagi Grimberg 620ae9ea9edSSagi Grimberg isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 621ae9ea9edSSagi Grimberg isert_conn->login_buf, isert_conn->login_req_buf, 622ae9ea9edSSagi Grimberg isert_conn->login_rsp_buf); 623ae9ea9edSSagi Grimberg 624ae9ea9edSSagi Grimberg isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 625ae9ea9edSSagi Grimberg (void *)isert_conn->login_req_buf, 626ae9ea9edSSagi Grimberg ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 627ae9ea9edSSagi Grimberg 628ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 629ae9ea9edSSagi Grimberg if (ret) { 630ae9ea9edSSagi Grimberg isert_err("login_req_dma mapping error: %d\n", ret); 631ae9ea9edSSagi Grimberg isert_conn->login_req_dma = 0; 632ae9ea9edSSagi Grimberg goto out_login_buf; 633ae9ea9edSSagi Grimberg } 634ae9ea9edSSagi Grimberg 635ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 636ae9ea9edSSagi Grimberg (void *)isert_conn->login_rsp_buf, 637ae9ea9edSSagi Grimberg ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 638ae9ea9edSSagi Grimberg 639ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 640ae9ea9edSSagi Grimberg if (ret) { 641ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret); 642ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0; 643ae9ea9edSSagi Grimberg goto out_req_dma_map; 644ae9ea9edSSagi Grimberg } 645ae9ea9edSSagi Grimberg 646ae9ea9edSSagi Grimberg return 0; 647ae9ea9edSSagi Grimberg 648ae9ea9edSSagi Grimberg out_req_dma_map: 649ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 650ae9ea9edSSagi Grimberg ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 651ae9ea9edSSagi Grimberg out_login_buf: 652ae9ea9edSSagi Grimberg kfree(isert_conn->login_buf); 653ae9ea9edSSagi Grimberg return ret; 654ae9ea9edSSagi Grimberg } 655ae9ea9edSSagi Grimberg 656*422bd0acSJenny Derzhavetz static void 657*422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn, 658*422bd0acSJenny Derzhavetz struct rdma_conn_param *param) 659*422bd0acSJenny Derzhavetz { 660*422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 661*422bd0acSJenny Derzhavetz 662*422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */ 663*422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 664*422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom); 665*422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 666*422bd0acSJenny Derzhavetz 667*422bd0acSJenny Derzhavetz if (param->private_data) { 668*422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data; 669*422bd0acSJenny Derzhavetz 670*422bd0acSJenny Derzhavetz /* 671*422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator 672*422bd0acSJenny Derzhavetz * and the HCA support it 673*422bd0acSJenny Derzhavetz */ 674*422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 675*422bd0acSJenny Derzhavetz (attr->device_cap_flags & 676*422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS); 677*422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv) 678*422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n"); 679*422bd0acSJenny Derzhavetz } 680*422bd0acSJenny Derzhavetz } 681*422bd0acSJenny Derzhavetz 682b8d26b3bSNicholas Bellinger static int 683b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 684b8d26b3bSNicholas Bellinger { 685ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 686ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 687b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 688b8d26b3bSNicholas Bellinger struct isert_device *device; 689b8d26b3bSNicholas Bellinger int ret = 0; 69014f4b54fSSagi Grimberg 69114f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 69214f4b54fSSagi Grimberg if (!np->enabled) { 69314f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 69424f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 69514f4b54fSSagi Grimberg return rdma_reject(cma_id, NULL, 0); 69614f4b54fSSagi Grimberg } 69714f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 698b8d26b3bSNicholas Bellinger 6994c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n", 700b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 701b8d26b3bSNicholas Bellinger 702b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 703ae9ea9edSSagi Grimberg if (!isert_conn) 704b8d26b3bSNicholas Bellinger return -ENOMEM; 705b8d26b3bSNicholas Bellinger 706ae9ea9edSSagi Grimberg isert_init_conn(isert_conn); 707dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id; 708b8d26b3bSNicholas Bellinger 709ae9ea9edSSagi Grimberg ret = isert_alloc_login_buf(isert_conn, cma_id->device); 710ae9ea9edSSagi Grimberg if (ret) 711b8d26b3bSNicholas Bellinger goto out; 712b8d26b3bSNicholas Bellinger 713cf8ae958SSagi Grimberg device = isert_device_get(cma_id); 714b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 715b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 716b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 717b8d26b3bSNicholas Bellinger } 718dac6ab30SSagi Grimberg isert_conn->device = device; 719b8d26b3bSNicholas Bellinger 720*422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn); 7211a92e17eSSagi Grimberg 722570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 723b8d26b3bSNicholas Bellinger if (ret) 724b8d26b3bSNicholas Bellinger goto out_conn_dev; 725b8d26b3bSNicholas Bellinger 7262371e5daSSagi Grimberg ret = isert_rdma_post_recvl(isert_conn); 7272371e5daSSagi Grimberg if (ret) 7282371e5daSSagi Grimberg goto out_conn_dev; 7292371e5daSSagi Grimberg 7302371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 7312371e5daSSagi Grimberg if (ret) 7322371e5daSSagi Grimberg goto out_conn_dev; 7332371e5daSSagi Grimberg 734ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 735bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted); 736ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 737b8d26b3bSNicholas Bellinger 738b8d26b3bSNicholas Bellinger return 0; 739b8d26b3bSNicholas Bellinger 740b8d26b3bSNicholas Bellinger out_conn_dev: 741cf8ae958SSagi Grimberg isert_device_put(device); 742b8d26b3bSNicholas Bellinger out_rsp_dma_map: 743ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 744b8d26b3bSNicholas Bellinger out: 745b8d26b3bSNicholas Bellinger kfree(isert_conn); 7462371e5daSSagi Grimberg rdma_reject(cma_id, NULL, 0); 747b8d26b3bSNicholas Bellinger return ret; 748b8d26b3bSNicholas Bellinger } 749b8d26b3bSNicholas Bellinger 750b8d26b3bSNicholas Bellinger static void 751b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 752b8d26b3bSNicholas Bellinger { 753dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 754b8d26b3bSNicholas Bellinger 7554c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn); 756b8d26b3bSNicholas Bellinger 75757df81e3SSagi Grimberg BUG_ON(!device); 75857df81e3SSagi Grimberg 75957df81e3SSagi Grimberg if (device->use_fastreg) 760a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 76159464ef4SVu Pham 76219e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 763dac6ab30SSagi Grimberg if (isert_conn->cm_id) 764dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id); 76519e2090fSSagi Grimberg 766dac6ab30SSagi Grimberg if (isert_conn->qp) { 767dac6ab30SSagi Grimberg struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 7684a295baeSSagi Grimberg 76940fc069aSSagi Grimberg isert_comp_put(comp); 770dac6ab30SSagi Grimberg ib_destroy_qp(isert_conn->qp); 771b8d26b3bSNicholas Bellinger } 772b8d26b3bSNicholas Bellinger 773ae9ea9edSSagi Grimberg if (isert_conn->login_buf) 774ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 775ae9ea9edSSagi Grimberg 776cf8ae958SSagi Grimberg isert_device_put(device); 77757df81e3SSagi Grimberg 77857df81e3SSagi Grimberg kfree(isert_conn); 779b8d26b3bSNicholas Bellinger } 780b8d26b3bSNicholas Bellinger 781b8d26b3bSNicholas Bellinger static void 782b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 783b8d26b3bSNicholas Bellinger { 78419e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 785bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 786c2f88b17SSagi Grimberg 78724f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 788128e9cc8SSagi Grimberg 789dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 790128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 791bd379220SJenny Derzhavetz kref_get(&isert_conn->kref); 792dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 793bd379220SJenny Derzhavetz 794bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex); 795bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending); 796bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 797bd379220SJenny Derzhavetz 798bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np); 799bd379220SJenny Derzhavetz up(&isert_np->sem); 800b8d26b3bSNicholas Bellinger } 801b8d26b3bSNicholas Bellinger 802b8d26b3bSNicholas Bellinger static void 803dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref) 804b8d26b3bSNicholas Bellinger { 805b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 806dac6ab30SSagi Grimberg struct isert_conn, kref); 807b8d26b3bSNicholas Bellinger 8084c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 8094c22e07fSSagi Grimberg current->pid); 810b8d26b3bSNicholas Bellinger 811b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 812b8d26b3bSNicholas Bellinger } 813b8d26b3bSNicholas Bellinger 814b8d26b3bSNicholas Bellinger static void 815b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 816b8d26b3bSNicholas Bellinger { 817dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref); 818b8d26b3bSNicholas Bellinger } 819b8d26b3bSNicholas Bellinger 820954f2372SSagi Grimberg /** 821954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 822954f2372SSagi Grimberg * @isert_conn: isert connection struct 823954f2372SSagi Grimberg * 824954f2372SSagi Grimberg * Notes: 825128e9cc8SSagi Grimberg * In case the connection state is FULL_FEATURE, move state 826954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 827128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 828954f2372SSagi Grimberg * 829dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is 830954f2372SSagi Grimberg * safe to call multiple times. 831954f2372SSagi Grimberg */ 832954f2372SSagi Grimberg static void 833954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 834954f2372SSagi Grimberg { 835954f2372SSagi Grimberg int err; 836954f2372SSagi Grimberg 837128e9cc8SSagi Grimberg switch (isert_conn->state) { 838128e9cc8SSagi Grimberg case ISER_CONN_TERMINATING: 839128e9cc8SSagi Grimberg break; 840128e9cc8SSagi Grimberg case ISER_CONN_UP: 841128e9cc8SSagi Grimberg case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 84224f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 843954f2372SSagi Grimberg isert_conn, isert_conn->state); 844128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 845dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id); 846954f2372SSagi Grimberg if (err) 84724f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 848954f2372SSagi Grimberg isert_conn); 849128e9cc8SSagi Grimberg break; 850128e9cc8SSagi Grimberg default: 85124f412ddSSagi Grimberg isert_warn("conn %p teminating in state %d\n", 852128e9cc8SSagi Grimberg isert_conn, isert_conn->state); 853954f2372SSagi Grimberg } 854954f2372SSagi Grimberg } 855954f2372SSagi Grimberg 8563b726ae2SSagi Grimberg static int 857ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 858ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 859b8d26b3bSNicholas Bellinger { 860ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n", 861ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np); 8623b726ae2SSagi Grimberg 863ca6c1d82SSagi Grimberg switch (event) { 864ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 865ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 866ca6c1d82SSagi Grimberg break; 867ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 868ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np); 869ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) { 87024f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 871ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id)); 872ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 873ca6c1d82SSagi Grimberg } 874ca6c1d82SSagi Grimberg break; 875ca6c1d82SSagi Grimberg default: 87624f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 877ca6c1d82SSagi Grimberg isert_np, event); 878ca6c1d82SSagi Grimberg } 879ca6c1d82SSagi Grimberg 8803b726ae2SSagi Grimberg return -1; 8813b726ae2SSagi Grimberg } 8823b726ae2SSagi Grimberg 883ca6c1d82SSagi Grimberg static int 884ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 885ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 886ca6c1d82SSagi Grimberg { 887ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 888ca6c1d82SSagi Grimberg struct isert_conn *isert_conn; 8892f1b6b7dSSagi Grimberg bool terminating = false; 890ca6c1d82SSagi Grimberg 891ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id == cma_id) 892ca6c1d82SSagi Grimberg return isert_np_cma_handler(cma_id->context, event); 893ca6c1d82SSagi Grimberg 89419e2090fSSagi Grimberg isert_conn = cma_id->qp->qp_context; 895b8d26b3bSNicholas Bellinger 896dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 8972f1b6b7dSSagi Grimberg terminating = (isert_conn->state == ISER_CONN_TERMINATING); 898128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 899dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 900128e9cc8SSagi Grimberg 901dac6ab30SSagi Grimberg isert_info("conn %p completing wait\n", isert_conn); 902dac6ab30SSagi Grimberg complete(&isert_conn->wait); 9033b726ae2SSagi Grimberg 9042f1b6b7dSSagi Grimberg if (terminating) 9052f1b6b7dSSagi Grimberg goto out; 9062f1b6b7dSSagi Grimberg 907ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 908bd379220SJenny Derzhavetz if (!list_empty(&isert_conn->node)) { 909bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 9102f1b6b7dSSagi Grimberg isert_put_conn(isert_conn); 9112f1b6b7dSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 9122f1b6b7dSSagi Grimberg } 913ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 9142f1b6b7dSSagi Grimberg 9152f1b6b7dSSagi Grimberg out: 9163b726ae2SSagi Grimberg return 0; 917b8d26b3bSNicholas Bellinger } 918b8d26b3bSNicholas Bellinger 9194a579da2SSagi Grimberg static int 920954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 921954f2372SSagi Grimberg { 92219e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 923954f2372SSagi Grimberg 924bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 925dac6ab30SSagi Grimberg isert_conn->cm_id = NULL; 926954f2372SSagi Grimberg isert_put_conn(isert_conn); 9274a579da2SSagi Grimberg 9284a579da2SSagi Grimberg return -1; 929954f2372SSagi Grimberg } 930954f2372SSagi Grimberg 931b8d26b3bSNicholas Bellinger static int 932b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 933b8d26b3bSNicholas Bellinger { 934b8d26b3bSNicholas Bellinger int ret = 0; 935b8d26b3bSNicholas Bellinger 936ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n", 937ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event, 9384c22e07fSSagi Grimberg event->status, cma_id, cma_id->context); 939b8d26b3bSNicholas Bellinger 940b8d26b3bSNicholas Bellinger switch (event->event) { 941b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 942b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 9433b726ae2SSagi Grimberg if (ret) 9444c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret); 945b8d26b3bSNicholas Bellinger break; 946b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 947b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 948b8d26b3bSNicholas Bellinger break; 94988c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 95088c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 95188c4015fSSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 95288c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 953ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 954b8d26b3bSNicholas Bellinger break; 955954f2372SSagi Grimberg case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 956954f2372SSagi Grimberg case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 957b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 9584a579da2SSagi Grimberg ret = isert_connect_error(cma_id); 959954f2372SSagi Grimberg break; 960b8d26b3bSNicholas Bellinger default: 96124f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 962b8d26b3bSNicholas Bellinger break; 963b8d26b3bSNicholas Bellinger } 964b8d26b3bSNicholas Bellinger 965b8d26b3bSNicholas Bellinger return ret; 966b8d26b3bSNicholas Bellinger } 967b8d26b3bSNicholas Bellinger 968b8d26b3bSNicholas Bellinger static int 9694366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count) 970b8d26b3bSNicholas Bellinger { 971b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 972b8d26b3bSNicholas Bellinger int i, ret; 973b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 974b8d26b3bSNicholas Bellinger 975dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 9764366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i]; 977b0a191e7SSagi Grimberg rx_wr->wr_id = (uintptr_t)rx_desc; 978b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 979b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 980b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 981b8d26b3bSNicholas Bellinger } 982b8d26b3bSNicholas Bellinger rx_wr--; 983b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 984b8d26b3bSNicholas Bellinger 985b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count += count; 986dac6ab30SSagi Grimberg ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 987b8d26b3bSNicholas Bellinger &rx_wr_failed); 988b8d26b3bSNicholas Bellinger if (ret) { 98924f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 990b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count -= count; 991b8d26b3bSNicholas Bellinger } 9924366b19cSJenny Derzhavetz 9934366b19cSJenny Derzhavetz return ret; 9944366b19cSJenny Derzhavetz } 9954366b19cSJenny Derzhavetz 9964366b19cSJenny Derzhavetz static int 9974366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 9984366b19cSJenny Derzhavetz { 9994366b19cSJenny Derzhavetz struct ib_recv_wr *rx_wr_failed, rx_wr; 10004366b19cSJenny Derzhavetz int ret; 10014366b19cSJenny Derzhavetz 10024366b19cSJenny Derzhavetz rx_wr.wr_id = (uintptr_t)rx_desc; 10034366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg; 10044366b19cSJenny Derzhavetz rx_wr.num_sge = 1; 10054366b19cSJenny Derzhavetz rx_wr.next = NULL; 10064366b19cSJenny Derzhavetz 10074366b19cSJenny Derzhavetz isert_conn->post_recv_buf_count++; 10084366b19cSJenny Derzhavetz ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 10094366b19cSJenny Derzhavetz if (ret) { 10104366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret); 10114366b19cSJenny Derzhavetz isert_conn->post_recv_buf_count--; 10124366b19cSJenny Derzhavetz } 10134366b19cSJenny Derzhavetz 1014b8d26b3bSNicholas Bellinger return ret; 1015b8d26b3bSNicholas Bellinger } 1016b8d26b3bSNicholas Bellinger 1017b8d26b3bSNicholas Bellinger static int 1018b8d26b3bSNicholas Bellinger isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1019b8d26b3bSNicholas Bellinger { 1020dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1021b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 1022b8d26b3bSNicholas Bellinger int ret; 1023b8d26b3bSNicholas Bellinger 1024b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1025b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1026b8d26b3bSNicholas Bellinger 1027b8d26b3bSNicholas Bellinger send_wr.next = NULL; 1028b0a191e7SSagi Grimberg send_wr.wr_id = (uintptr_t)tx_desc; 1029b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 1030b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 1031b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 1032b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 1033b8d26b3bSNicholas Bellinger 1034dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1035bdf20e72SSagi Grimberg if (ret) 103624f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 1037b8d26b3bSNicholas Bellinger 1038b8d26b3bSNicholas Bellinger return ret; 1039b8d26b3bSNicholas Bellinger } 1040b8d26b3bSNicholas Bellinger 1041b8d26b3bSNicholas Bellinger static void 1042b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 1043b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1044b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1045b8d26b3bSNicholas Bellinger { 1046dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 104767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1048b8d26b3bSNicholas Bellinger 1049b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1050b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1051b8d26b3bSNicholas Bellinger 1052d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 1053d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL; 1054b8d26b3bSNicholas Bellinger 1055b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 1056b8d26b3bSNicholas Bellinger tx_desc->isert_cmd = isert_cmd; 1057b8d26b3bSNicholas Bellinger 105834efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 105934efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 106024f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1061b8d26b3bSNicholas Bellinger } 1062b8d26b3bSNicholas Bellinger } 1063b8d26b3bSNicholas Bellinger 1064b8d26b3bSNicholas Bellinger static int 1065b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 1066b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1067b8d26b3bSNicholas Bellinger { 1068dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 106967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1070b8d26b3bSNicholas Bellinger u64 dma_addr; 1071b8d26b3bSNicholas Bellinger 1072b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1073b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1074b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 107524f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 1076b8d26b3bSNicholas Bellinger return -ENOMEM; 1077b8d26b3bSNicholas Bellinger } 1078b8d26b3bSNicholas Bellinger 1079b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 1080b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1081b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 108234efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1083b8d26b3bSNicholas Bellinger 10844c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 10854c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 10864c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey); 1087b8d26b3bSNicholas Bellinger 1088b8d26b3bSNicholas Bellinger return 0; 1089b8d26b3bSNicholas Bellinger } 1090b8d26b3bSNicholas Bellinger 1091b8d26b3bSNicholas Bellinger static void 109295b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 109368a86deeSSagi Grimberg struct ib_send_wr *send_wr) 1094b8d26b3bSNicholas Bellinger { 109595b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 109695b60f07SNicholas Bellinger 1097b8d26b3bSNicholas Bellinger isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1098b0a191e7SSagi Grimberg send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1099*422bd0acSJenny Derzhavetz 1100*422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 1101*422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV; 1102*422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 1103*422bd0acSJenny Derzhavetz } else { 1104b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 1105*422bd0acSJenny Derzhavetz } 1106*422bd0acSJenny Derzhavetz 110795b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 1108b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 110995b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 1110b8d26b3bSNicholas Bellinger } 1111b8d26b3bSNicholas Bellinger 1112b8d26b3bSNicholas Bellinger static int 1113b8d26b3bSNicholas Bellinger isert_rdma_post_recvl(struct isert_conn *isert_conn) 1114b8d26b3bSNicholas Bellinger { 1115b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 1116b8d26b3bSNicholas Bellinger struct ib_sge sge; 1117b8d26b3bSNicholas Bellinger int ret; 1118b8d26b3bSNicholas Bellinger 1119b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 1120b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 1121b8d26b3bSNicholas Bellinger sge.length = ISER_RX_LOGIN_SIZE; 112234efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey; 1123b8d26b3bSNicholas Bellinger 112424f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1125b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 1126b8d26b3bSNicholas Bellinger 1127b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1128b0a191e7SSagi Grimberg rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1129b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 1130b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 1131b8d26b3bSNicholas Bellinger 1132b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count++; 1133dac6ab30SSagi Grimberg ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 1134b8d26b3bSNicholas Bellinger if (ret) { 113524f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 1136b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1137b8d26b3bSNicholas Bellinger } 1138b8d26b3bSNicholas Bellinger 1139b8d26b3bSNicholas Bellinger return ret; 1140b8d26b3bSNicholas Bellinger } 1141b8d26b3bSNicholas Bellinger 1142b8d26b3bSNicholas Bellinger static int 1143b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1144b8d26b3bSNicholas Bellinger u32 length) 1145b8d26b3bSNicholas Bellinger { 1146b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 1147dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 114867cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1149dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1150b8d26b3bSNicholas Bellinger int ret; 1151b8d26b3bSNicholas Bellinger 1152b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 1153b8d26b3bSNicholas Bellinger 1154b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1155b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1156b8d26b3bSNicholas Bellinger 1157b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1158b8d26b3bSNicholas Bellinger 1159b8d26b3bSNicholas Bellinger if (length > 0) { 1160b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1161b8d26b3bSNicholas Bellinger 1162b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1163b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1164b8d26b3bSNicholas Bellinger 1165b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1166b8d26b3bSNicholas Bellinger 1167b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1168b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1169b8d26b3bSNicholas Bellinger 1170b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1171b8d26b3bSNicholas Bellinger tx_dsg->length = length; 117234efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1173b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1174b8d26b3bSNicholas Bellinger } 1175b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1176b8d26b3bSNicholas Bellinger if (login->login_complete) { 1177e0546fc1SSagi Grimberg if (!conn->sess->sess_ops->SessionType && 1178dac6ab30SSagi Grimberg isert_conn->device->use_fastreg) { 1179570db170SSagi Grimberg ret = isert_conn_create_fastreg_pool(isert_conn); 1180f46d6a8aSNicholas Bellinger if (ret) { 118124f412ddSSagi Grimberg isert_err("Conn: %p failed to create" 1182f46d6a8aSNicholas Bellinger " fastreg pool\n", isert_conn); 1183f46d6a8aSNicholas Bellinger return ret; 1184f46d6a8aSNicholas Bellinger } 1185f46d6a8aSNicholas Bellinger } 1186f46d6a8aSNicholas Bellinger 1187b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1188b8d26b3bSNicholas Bellinger if (ret) 1189b8d26b3bSNicholas Bellinger return ret; 1190b8d26b3bSNicholas Bellinger 11914366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn, 11924366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS); 1193b8d26b3bSNicholas Bellinger if (ret) 1194b8d26b3bSNicholas Bellinger return ret; 1195b8d26b3bSNicholas Bellinger 1196128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 1197dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 1198128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 1199dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1200b8d26b3bSNicholas Bellinger goto post_send; 1201b8d26b3bSNicholas Bellinger } 1202b8d26b3bSNicholas Bellinger 1203b8d26b3bSNicholas Bellinger ret = isert_rdma_post_recvl(isert_conn); 1204b8d26b3bSNicholas Bellinger if (ret) 1205b8d26b3bSNicholas Bellinger return ret; 1206b8d26b3bSNicholas Bellinger } 1207b8d26b3bSNicholas Bellinger post_send: 1208b8d26b3bSNicholas Bellinger ret = isert_post_send(isert_conn, tx_desc); 1209b8d26b3bSNicholas Bellinger if (ret) 1210b8d26b3bSNicholas Bellinger return ret; 1211b8d26b3bSNicholas Bellinger 1212b8d26b3bSNicholas Bellinger return 0; 1213b8d26b3bSNicholas Bellinger } 1214b8d26b3bSNicholas Bellinger 1215b8d26b3bSNicholas Bellinger static void 12162371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1217b8d26b3bSNicholas Bellinger { 12182371e5daSSagi Grimberg struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 12192371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1220b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1221b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1222b8d26b3bSNicholas Bellinger int size; 1223b8d26b3bSNicholas Bellinger 122424f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 12252371e5daSSagi Grimberg 12262371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1227b8d26b3bSNicholas Bellinger 1228b8d26b3bSNicholas Bellinger if (login->first_request) { 1229b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1230b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1231b8d26b3bSNicholas Bellinger /* 1232b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1233b8d26b3bSNicholas Bellinger * login request PDU. 1234b8d26b3bSNicholas Bellinger */ 1235b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1236b8d26b3bSNicholas Bellinger login->current_stage = 1237b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1238b8d26b3bSNicholas Bellinger >> 2; 1239b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1240b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1241b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1242b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1243b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1244b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1245b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1246b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1247b8d26b3bSNicholas Bellinger } 1248b8d26b3bSNicholas Bellinger 1249b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1250b8d26b3bSNicholas Bellinger 1251b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 12524c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d " 12534c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 12544c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS); 1255b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1256b8d26b3bSNicholas Bellinger 12576faaa85fSNicholas Bellinger if (login->first_request) { 1258dac6ab30SSagi Grimberg complete(&isert_conn->login_comp); 12596faaa85fSNicholas Bellinger return; 12606faaa85fSNicholas Bellinger } 12616faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1262b8d26b3bSNicholas Bellinger } 1263b8d26b3bSNicholas Bellinger 1264b8d26b3bSNicholas Bellinger static struct iscsi_cmd 12654366b19cSJenny Derzhavetz *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1266b8d26b3bSNicholas Bellinger { 12676700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1268b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1269d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1270b8d26b3bSNicholas Bellinger 1271676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1272d703ce2fSNicholas Bellinger if (!cmd) { 127324f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1274b8d26b3bSNicholas Bellinger return NULL; 1275b8d26b3bSNicholas Bellinger } 1276d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1277b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1278d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 12794366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc; 1280b8d26b3bSNicholas Bellinger 1281d703ce2fSNicholas Bellinger return cmd; 1282b8d26b3bSNicholas Bellinger } 1283b8d26b3bSNicholas Bellinger 1284b8d26b3bSNicholas Bellinger static int 1285b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1286d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1287d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1288b8d26b3bSNicholas Bellinger { 1289b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1290b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1291b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1292b8d26b3bSNicholas Bellinger bool dump_payload = false; 12939fd60088SJenny Derzhavetz unsigned int data_len; 1294b8d26b3bSNicholas Bellinger 1295b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1296b8d26b3bSNicholas Bellinger if (rc < 0) 1297b8d26b3bSNicholas Bellinger return rc; 1298b8d26b3bSNicholas Bellinger 1299b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1300b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1301b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 13029fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length; 1303b8d26b3bSNicholas Bellinger 13049fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len) 13059fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1306b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1307b8d26b3bSNicholas Bellinger if (rc < 0) { 1308b8d26b3bSNicholas Bellinger return 0; 1309b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1310b8d26b3bSNicholas Bellinger dump_payload = true; 1311b8d26b3bSNicholas Bellinger goto sequence_cmd; 1312b8d26b3bSNicholas Bellinger } 1313b8d26b3bSNicholas Bellinger 1314b8d26b3bSNicholas Bellinger if (!imm_data) 1315b8d26b3bSNicholas Bellinger return 0; 1316b8d26b3bSNicholas Bellinger 13179fd60088SJenny Derzhavetz if (imm_data_len != data_len) { 1318b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 13199fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 13209fd60088SJenny Derzhavetz &rx_desc->data[0], imm_data_len); 13219fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 13229fd60088SJenny Derzhavetz sg_nents, imm_data_len); 13239fd60088SJenny Derzhavetz } else { 13249fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1); 13259fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg; 13269fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1; 13279fd60088SJenny Derzhavetz sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 13289fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n", 13299fd60088SJenny Derzhavetz imm_data_len); 13309fd60088SJenny Derzhavetz } 1331b8d26b3bSNicholas Bellinger 1332b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1333b8d26b3bSNicholas Bellinger 1334b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1335b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1336b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1337b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1338b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1339b8d26b3bSNicholas Bellinger } 1340b8d26b3bSNicholas Bellinger 1341b8d26b3bSNicholas Bellinger sequence_cmd: 1342561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1343b8d26b3bSNicholas Bellinger 1344b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 1345b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 13466cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 1347afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd); 1348b8d26b3bSNicholas Bellinger 1349b8d26b3bSNicholas Bellinger return 0; 1350b8d26b3bSNicholas Bellinger } 1351b8d26b3bSNicholas Bellinger 1352b8d26b3bSNicholas Bellinger static int 1353b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1354b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1355b8d26b3bSNicholas Bellinger { 1356b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1357b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1358b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1359b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1360b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1361b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1362b8d26b3bSNicholas Bellinger 1363b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1364b8d26b3bSNicholas Bellinger if (rc < 0) 1365b8d26b3bSNicholas Bellinger return rc; 1366b8d26b3bSNicholas Bellinger else if (!cmd) 1367b8d26b3bSNicholas Bellinger return 0; 1368b8d26b3bSNicholas Bellinger /* 1369b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1370b8d26b3bSNicholas Bellinger */ 1371b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 137224f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1373b8d26b3bSNicholas Bellinger dump_stack(); 1374b8d26b3bSNicholas Bellinger return -1; 1375b8d26b3bSNicholas Bellinger } 1376b8d26b3bSNicholas Bellinger 13774c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 13784c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n", 13794c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done, 13804c22e07fSSagi Grimberg cmd->se_cmd.data_length); 1381b8d26b3bSNicholas Bellinger 1382b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1383b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1384b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1385b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1386b8d26b3bSNicholas Bellinger /* 1387b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1388b8d26b3bSNicholas Bellinger */ 1389b8d26b3bSNicholas Bellinger if (page_off) { 13904c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n"); 1391b8d26b3bSNicholas Bellinger dump_stack(); 1392b8d26b3bSNicholas Bellinger return -1; 1393b8d26b3bSNicholas Bellinger } 13944c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 13954c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off, 13964c22e07fSSagi Grimberg sg_nents, &rx_desc->data[0], unsol_data_len); 1397b8d26b3bSNicholas Bellinger 1398b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1399b8d26b3bSNicholas Bellinger unsol_data_len); 1400b8d26b3bSNicholas Bellinger 1401b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1402b8d26b3bSNicholas Bellinger if (rc < 0) 1403b8d26b3bSNicholas Bellinger return rc; 1404b8d26b3bSNicholas Bellinger 14054366b19cSJenny Derzhavetz /* 14064366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive - 14074366b19cSJenny Derzhavetz * so post the buffer before hand 14084366b19cSJenny Derzhavetz */ 14094366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, rx_desc); 14104366b19cSJenny Derzhavetz if (rc) { 14114366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 14124366b19cSJenny Derzhavetz return rc; 14134366b19cSJenny Derzhavetz } 1414b8d26b3bSNicholas Bellinger return 0; 1415b8d26b3bSNicholas Bellinger } 1416b8d26b3bSNicholas Bellinger 1417b8d26b3bSNicholas Bellinger static int 1418778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1419d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1420d703ce2fSNicholas Bellinger unsigned char *buf) 1421778de368SNicholas Bellinger { 1422778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1423778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1424778de368SNicholas Bellinger int rc; 1425778de368SNicholas Bellinger 1426778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1427778de368SNicholas Bellinger if (rc < 0) 1428778de368SNicholas Bellinger return rc; 1429778de368SNicholas Bellinger /* 1430778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1431778de368SNicholas Bellinger */ 1432778de368SNicholas Bellinger 1433778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1434778de368SNicholas Bellinger } 1435778de368SNicholas Bellinger 1436778de368SNicholas Bellinger static int 1437adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1438d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1439d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1440adb54c29SNicholas Bellinger { 1441adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1442adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1443adb54c29SNicholas Bellinger int rc; 1444b44a2b67SSagi Grimberg unsigned char *text_in = NULL; 1445adb54c29SNicholas Bellinger 1446adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1447adb54c29SNicholas Bellinger if (rc < 0) 1448adb54c29SNicholas Bellinger return rc; 1449adb54c29SNicholas Bellinger 1450b44a2b67SSagi Grimberg if (payload_length) { 1451adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 1452adb54c29SNicholas Bellinger if (!text_in) { 145324f412ddSSagi Grimberg isert_err("Unable to allocate text_in of payload_length: %u\n", 1454adb54c29SNicholas Bellinger payload_length); 1455adb54c29SNicholas Bellinger return -ENOMEM; 1456adb54c29SNicholas Bellinger } 1457b44a2b67SSagi Grimberg } 1458adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1459adb54c29SNicholas Bellinger 1460adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1461adb54c29SNicholas Bellinger 1462adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1463adb54c29SNicholas Bellinger } 1464adb54c29SNicholas Bellinger 1465adb54c29SNicholas Bellinger static int 1466b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1467b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1468b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1469b8d26b3bSNicholas Bellinger { 1470b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1471b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1472b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1473b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1474b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1475b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1476b8d26b3bSNicholas Bellinger 1477fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType && 1478ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 147924f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1480ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1481ca40d24eSNicholas Bellinger return 0; 1482ca40d24eSNicholas Bellinger } 1483ca40d24eSNicholas Bellinger 1484b8d26b3bSNicholas Bellinger switch (opcode) { 1485b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 14864366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1487b8d26b3bSNicholas Bellinger if (!cmd) 1488b8d26b3bSNicholas Bellinger break; 1489b8d26b3bSNicholas Bellinger 1490d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1491b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1492b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1493b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1494b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1495*422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1496b8d26b3bSNicholas Bellinger 1497d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1498b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1499b8d26b3bSNicholas Bellinger break; 1500b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 15014366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1502b8d26b3bSNicholas Bellinger if (!cmd) 1503b8d26b3bSNicholas Bellinger break; 1504b8d26b3bSNicholas Bellinger 1505d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1506d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1507778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1508b8d26b3bSNicholas Bellinger break; 1509b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1510b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1511b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1512b8d26b3bSNicholas Bellinger break; 1513b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 15144366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1515b8d26b3bSNicholas Bellinger if (!cmd) 1516b8d26b3bSNicholas Bellinger break; 1517b8d26b3bSNicholas Bellinger 1518b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1519b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1520b8d26b3bSNicholas Bellinger break; 1521b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 15224366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1523b8d26b3bSNicholas Bellinger if (!cmd) 1524b8d26b3bSNicholas Bellinger break; 1525b8d26b3bSNicholas Bellinger 1526b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1527b8d26b3bSNicholas Bellinger break; 1528adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 15294366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1530e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 15314366b19cSJenny Derzhavetz else 15324366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 15334366b19cSJenny Derzhavetz 1534e4f4e801SSagi Grimberg if (!cmd) 1535e4f4e801SSagi Grimberg break; 1536adb54c29SNicholas Bellinger 1537d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1538d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1539adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1540adb54c29SNicholas Bellinger break; 1541b8d26b3bSNicholas Bellinger default: 154224f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1543b8d26b3bSNicholas Bellinger dump_stack(); 1544b8d26b3bSNicholas Bellinger break; 1545b8d26b3bSNicholas Bellinger } 1546b8d26b3bSNicholas Bellinger 1547b8d26b3bSNicholas Bellinger return ret; 1548b8d26b3bSNicholas Bellinger } 1549b8d26b3bSNicholas Bellinger 1550b8d26b3bSNicholas Bellinger static void 1551b8d26b3bSNicholas Bellinger isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1552b8d26b3bSNicholas Bellinger { 1553d3cf81f9SSagi Grimberg struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1554b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1555b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1556b8d26b3bSNicholas Bellinger 1557d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) { 1558b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1559d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) { 1560d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag); 1561d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va); 15624c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1563b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1564b8d26b3bSNicholas Bellinger } 1565d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) { 1566d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag); 1567d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va); 15684c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1569b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1570b8d26b3bSNicholas Bellinger } 1571b8d26b3bSNicholas Bellinger 157224f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1573b8d26b3bSNicholas Bellinger break; 1574b8d26b3bSNicholas Bellinger case ISER_HELLO: 157524f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1576b8d26b3bSNicholas Bellinger break; 1577b8d26b3bSNicholas Bellinger default: 1578d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1579b8d26b3bSNicholas Bellinger break; 1580b8d26b3bSNicholas Bellinger } 1581b8d26b3bSNicholas Bellinger 15826c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc, 1583b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 1584b8d26b3bSNicholas Bellinger } 1585b8d26b3bSNicholas Bellinger 1586b8d26b3bSNicholas Bellinger static void 15877748681bSSagi Grimberg isert_rcv_completion(struct iser_rx_desc *desc, 15887748681bSSagi Grimberg struct isert_conn *isert_conn, 15894a295baeSSagi Grimberg u32 xfer_len) 1590b8d26b3bSNicholas Bellinger { 1591dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1592b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr; 1593b8d26b3bSNicholas Bellinger u64 rx_dma; 15944366b19cSJenny Derzhavetz int rx_buflen; 1595b8d26b3bSNicholas Bellinger 1596b8d26b3bSNicholas Bellinger if ((char *)desc == isert_conn->login_req_buf) { 1597b8d26b3bSNicholas Bellinger rx_dma = isert_conn->login_req_dma; 1598b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_LOGIN_SIZE; 15994c22e07fSSagi Grimberg isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1600b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1601b8d26b3bSNicholas Bellinger } else { 1602b8d26b3bSNicholas Bellinger rx_dma = desc->dma_addr; 1603b8d26b3bSNicholas Bellinger rx_buflen = ISER_RX_PAYLOAD_SIZE; 16044c22e07fSSagi Grimberg isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1605b8d26b3bSNicholas Bellinger rx_dma, rx_buflen); 1606b8d26b3bSNicholas Bellinger } 1607b8d26b3bSNicholas Bellinger 1608b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1609b8d26b3bSNicholas Bellinger 1610b8d26b3bSNicholas Bellinger hdr = &desc->iscsi_header; 161124f412ddSSagi Grimberg isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1612b8d26b3bSNicholas Bellinger hdr->opcode, hdr->itt, hdr->flags, 1613b8d26b3bSNicholas Bellinger (int)(xfer_len - ISER_HEADERS_LEN)); 1614b8d26b3bSNicholas Bellinger 16152371e5daSSagi Grimberg if ((char *)desc == isert_conn->login_req_buf) { 16162371e5daSSagi Grimberg isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 16172371e5daSSagi Grimberg if (isert_conn->conn) { 16182371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 16192371e5daSSagi Grimberg 16202371e5daSSagi Grimberg if (login && !login->first_request) 16212371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 16222371e5daSSagi Grimberg } 1623dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 16242371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 1625dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 16262371e5daSSagi Grimberg } else { 1627b8d26b3bSNicholas Bellinger isert_rx_do_work(desc, isert_conn); 16282371e5daSSagi Grimberg } 1629b8d26b3bSNicholas Bellinger 1630b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1631b8d26b3bSNicholas Bellinger DMA_FROM_DEVICE); 1632b8d26b3bSNicholas Bellinger 1633b8d26b3bSNicholas Bellinger isert_conn->post_recv_buf_count--; 1634b8d26b3bSNicholas Bellinger } 1635b8d26b3bSNicholas Bellinger 1636e3d7e4c3SSagi Grimberg static int 1637e3d7e4c3SSagi Grimberg isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1638e3d7e4c3SSagi Grimberg struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1639e3d7e4c3SSagi Grimberg enum iser_ib_op_code op, struct isert_data_buf *data) 1640e3d7e4c3SSagi Grimberg { 1641dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1642e3d7e4c3SSagi Grimberg 1643e3d7e4c3SSagi Grimberg data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1644e3d7e4c3SSagi Grimberg DMA_TO_DEVICE : DMA_FROM_DEVICE; 1645e3d7e4c3SSagi Grimberg 1646e3d7e4c3SSagi Grimberg data->len = length - offset; 1647e3d7e4c3SSagi Grimberg data->offset = offset; 1648e3d7e4c3SSagi Grimberg data->sg_off = data->offset / PAGE_SIZE; 1649e3d7e4c3SSagi Grimberg 1650e3d7e4c3SSagi Grimberg data->sg = &sg[data->sg_off]; 1651e3d7e4c3SSagi Grimberg data->nents = min_t(unsigned int, nents - data->sg_off, 1652e3d7e4c3SSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 1653e3d7e4c3SSagi Grimberg data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1654e3d7e4c3SSagi Grimberg PAGE_SIZE); 1655e3d7e4c3SSagi Grimberg 1656e3d7e4c3SSagi Grimberg data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1657e3d7e4c3SSagi Grimberg data->dma_dir); 1658e3d7e4c3SSagi Grimberg if (unlikely(!data->dma_nents)) { 165924f412ddSSagi Grimberg isert_err("Cmd: unable to dma map SGs %p\n", sg); 1660e3d7e4c3SSagi Grimberg return -EINVAL; 1661e3d7e4c3SSagi Grimberg } 1662e3d7e4c3SSagi Grimberg 166324f412ddSSagi Grimberg isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1664e3d7e4c3SSagi Grimberg isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1665e3d7e4c3SSagi Grimberg 1666e3d7e4c3SSagi Grimberg return 0; 1667e3d7e4c3SSagi Grimberg } 1668e3d7e4c3SSagi Grimberg 1669e3d7e4c3SSagi Grimberg static void 1670e3d7e4c3SSagi Grimberg isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1671e3d7e4c3SSagi Grimberg { 1672dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1673e3d7e4c3SSagi Grimberg 1674e3d7e4c3SSagi Grimberg ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1675e3d7e4c3SSagi Grimberg memset(data, 0, sizeof(*data)); 1676e3d7e4c3SSagi Grimberg } 1677e3d7e4c3SSagi Grimberg 1678e3d7e4c3SSagi Grimberg 1679e3d7e4c3SSagi Grimberg 1680b8d26b3bSNicholas Bellinger static void 1681b8d26b3bSNicholas Bellinger isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1682b8d26b3bSNicholas Bellinger { 1683b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1684b8d26b3bSNicholas Bellinger 16854c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1686e3d7e4c3SSagi Grimberg 1687e3d7e4c3SSagi Grimberg if (wr->data.sg) { 16884c22e07fSSagi Grimberg isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1689e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 1690b8d26b3bSNicholas Bellinger } 1691b8d26b3bSNicholas Bellinger 1692e622f2f4SChristoph Hellwig if (wr->rdma_wr) { 16934c22e07fSSagi Grimberg isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1694e622f2f4SChristoph Hellwig kfree(wr->rdma_wr); 1695e622f2f4SChristoph Hellwig wr->rdma_wr = NULL; 169690ecc6e2SVu Pham } 1697b8d26b3bSNicholas Bellinger 169890ecc6e2SVu Pham if (wr->ib_sge) { 16994c22e07fSSagi Grimberg isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 170090ecc6e2SVu Pham kfree(wr->ib_sge); 170190ecc6e2SVu Pham wr->ib_sge = NULL; 170290ecc6e2SVu Pham } 1703b8d26b3bSNicholas Bellinger } 1704b8d26b3bSNicholas Bellinger 1705b8d26b3bSNicholas Bellinger static void 1706a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 170759464ef4SVu Pham { 170859464ef4SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 170959464ef4SVu Pham 17104c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 171159464ef4SVu Pham 171259464ef4SVu Pham if (wr->fr_desc) { 17134c22e07fSSagi Grimberg isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 17149e961ae7SSagi Grimberg if (wr->fr_desc->ind & ISERT_PROTECTED) { 17159e961ae7SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 17169e961ae7SSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 17179e961ae7SSagi Grimberg } 1718dac6ab30SSagi Grimberg spin_lock_bh(&isert_conn->pool_lock); 1719dac6ab30SSagi Grimberg list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); 1720dac6ab30SSagi Grimberg spin_unlock_bh(&isert_conn->pool_lock); 172159464ef4SVu Pham wr->fr_desc = NULL; 172259464ef4SVu Pham } 172359464ef4SVu Pham 1724e3d7e4c3SSagi Grimberg if (wr->data.sg) { 17254c22e07fSSagi Grimberg isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1726e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 172759464ef4SVu Pham } 172859464ef4SVu Pham 172959464ef4SVu Pham wr->ib_sge = NULL; 1730e622f2f4SChristoph Hellwig wr->rdma_wr = NULL; 173159464ef4SVu Pham } 173259464ef4SVu Pham 173359464ef4SVu Pham static void 173403e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1735b8d26b3bSNicholas Bellinger { 1736d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1737b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1738186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1739dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1740e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr; 1741b8d26b3bSNicholas Bellinger 17424c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1743b8d26b3bSNicholas Bellinger 1744b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1745b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1746b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1747b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17485159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1749b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1750b8d26b3bSNicholas Bellinger 175103e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1752b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 175303e7848aSNicholas Bellinger /* 175403e7848aSNicholas Bellinger * Check for special case during comp_err where 175503e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 175603e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 175703e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 175803e7848aSNicholas Bellinger */ 175903e7848aSNicholas Bellinger if (comp_err && 176003e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 176103e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 176203e7848aSNicholas Bellinger 1763afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 176403e7848aSNicholas Bellinger } 176503e7848aSNicholas Bellinger } 1766b8d26b3bSNicholas Bellinger 1767d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1768186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1769186a9647SNicholas Bellinger break; 1770b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1771186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1772186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17735159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1774186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1775186a9647SNicholas Bellinger 1776b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1777b8d26b3bSNicholas Bellinger break; 1778b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1779b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1780adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1781e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1782e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */ 1783e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1784e4f4e801SSagi Grimberg break; 1785e4f4e801SSagi Grimberg 1786b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1787b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17885159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1789b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1790b8d26b3bSNicholas Bellinger 1791b8d26b3bSNicholas Bellinger /* 1792b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1793b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1794b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1795b8d26b3bSNicholas Bellinger */ 1796b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 179711378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 17983df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1799b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1800b8d26b3bSNicholas Bellinger break; 1801b8d26b3bSNicholas Bellinger } 1802b8d26b3bSNicholas Bellinger /* 1803b8d26b3bSNicholas Bellinger * Fall-through 1804b8d26b3bSNicholas Bellinger */ 1805b8d26b3bSNicholas Bellinger default: 1806d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1807b8d26b3bSNicholas Bellinger break; 1808b8d26b3bSNicholas Bellinger } 1809b8d26b3bSNicholas Bellinger } 1810b8d26b3bSNicholas Bellinger 1811b8d26b3bSNicholas Bellinger static void 1812b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1813b8d26b3bSNicholas Bellinger { 1814b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 18154c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n"); 1816b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1817b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1818b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1819b8d26b3bSNicholas Bellinger } 1820b8d26b3bSNicholas Bellinger } 1821b8d26b3bSNicholas Bellinger 1822b8d26b3bSNicholas Bellinger static void 1823b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 182403e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1825b8d26b3bSNicholas Bellinger { 1826dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 18274c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1828dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1829dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1830dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1831b8d26b3bSNicholas Bellinger } 1832b8d26b3bSNicholas Bellinger 1833b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 183403e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1835b8d26b3bSNicholas Bellinger } 1836b8d26b3bSNicholas Bellinger 183796b7973eSSagi Grimberg static int 183896b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1839f93f3a70SSagi Grimberg { 18409e961ae7SSagi Grimberg struct ib_mr_status mr_status; 184196b7973eSSagi Grimberg int ret; 1842f93f3a70SSagi Grimberg 184396b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 18449e961ae7SSagi Grimberg if (ret) { 184524f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 18469e961ae7SSagi Grimberg goto fail_mr_status; 18479e961ae7SSagi Grimberg } 184896b7973eSSagi Grimberg 18499e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 185096b7973eSSagi Grimberg u64 sec_offset_err; 18519e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 18529e961ae7SSagi Grimberg 18539e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 18549e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 18559e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 18569e961ae7SSagi Grimberg break; 18579e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 18589e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 18599e961ae7SSagi Grimberg break; 18609e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 18619e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 18629e961ae7SSagi Grimberg break; 18639e961ae7SSagi Grimberg } 186496b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 186596b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 186696b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 18679e961ae7SSagi Grimberg 18684c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx " 18699e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 18709e961ae7SSagi Grimberg mr_status.sig_err.err_type, 18719e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 18729e961ae7SSagi Grimberg mr_status.sig_err.expected, 18739e961ae7SSagi Grimberg mr_status.sig_err.actual); 18749e961ae7SSagi Grimberg ret = 1; 18759e961ae7SSagi Grimberg } 18769e961ae7SSagi Grimberg 18779e961ae7SSagi Grimberg fail_mr_status: 187896b7973eSSagi Grimberg return ret; 187996b7973eSSagi Grimberg } 188096b7973eSSagi Grimberg 188196b7973eSSagi Grimberg static void 188296b7973eSSagi Grimberg isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 188396b7973eSSagi Grimberg struct isert_cmd *isert_cmd) 188496b7973eSSagi Grimberg { 188596b7973eSSagi Grimberg struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 188696b7973eSSagi Grimberg struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 188796b7973eSSagi Grimberg struct se_cmd *se_cmd = &cmd->se_cmd; 188896b7973eSSagi Grimberg struct isert_conn *isert_conn = isert_cmd->conn; 1889dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 189096b7973eSSagi Grimberg int ret = 0; 189196b7973eSSagi Grimberg 189296b7973eSSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 189396b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 189496b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 189596b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 189696b7973eSSagi Grimberg } 189796b7973eSSagi Grimberg 1898f93f3a70SSagi Grimberg device->unreg_rdma_mem(isert_cmd, isert_conn); 1899e622f2f4SChristoph Hellwig wr->rdma_wr_num = 0; 19009e961ae7SSagi Grimberg if (ret) 19019e961ae7SSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 19029e961ae7SSagi Grimberg se_cmd->pi_err, 0); 19039e961ae7SSagi Grimberg else 1904f93f3a70SSagi Grimberg isert_put_response(isert_conn->conn, cmd); 1905f93f3a70SSagi Grimberg } 1906f93f3a70SSagi Grimberg 1907f93f3a70SSagi Grimberg static void 1908b8d26b3bSNicholas Bellinger isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1909b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd) 1910b8d26b3bSNicholas Bellinger { 1911b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1912d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1913b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 191490ecc6e2SVu Pham struct isert_conn *isert_conn = isert_cmd->conn; 1915dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 19165bac4b1aSSagi Grimberg int ret = 0; 1917b8d26b3bSNicholas Bellinger 19189e961ae7SSagi Grimberg if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 191996b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 192096b7973eSSagi Grimberg wr->fr_desc->pi_ctx->sig_mr); 192196b7973eSSagi Grimberg wr->fr_desc->ind &= ~ISERT_PROTECTED; 19229e961ae7SSagi Grimberg } 19239e961ae7SSagi Grimberg 1924b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1925d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1926e3d7e4c3SSagi Grimberg cmd->write_data_done = wr->data.len; 1927e622f2f4SChristoph Hellwig wr->rdma_wr_num = 0; 1928b8d26b3bSNicholas Bellinger 192924f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1930b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1931b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1932b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1933b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1934b8d26b3bSNicholas Bellinger 1935364189f0SSagi Grimberg if (ret) { 1936afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 19375bac4b1aSSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 19385bac4b1aSSagi Grimberg se_cmd->pi_err, 0); 1939364189f0SSagi Grimberg } else { 1940b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1941b8d26b3bSNicholas Bellinger } 1942364189f0SSagi Grimberg } 1943b8d26b3bSNicholas Bellinger 1944b8d26b3bSNicholas Bellinger static void 1945b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1946b8d26b3bSNicholas Bellinger { 1947b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1948b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1949b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1950dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1951d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1952b8d26b3bSNicholas Bellinger 19534c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 19544c22e07fSSagi Grimberg 1955b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1956b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1957b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 195810633c37SSagi Grimberg case ISTATE_SEND_REJECT: /* FALLTHRU */ 195910633c37SSagi Grimberg case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1960b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 19614c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 19624c22e07fSSagi Grimberg ib_dev, false); 19633df8f68aSNicholas Bellinger break; 1964b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1965b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1966b8d26b3bSNicholas Bellinger break; 1967b8d26b3bSNicholas Bellinger default: 19684c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state); 1969b8d26b3bSNicholas Bellinger dump_stack(); 1970b8d26b3bSNicholas Bellinger break; 1971b8d26b3bSNicholas Bellinger } 1972b8d26b3bSNicholas Bellinger } 1973b8d26b3bSNicholas Bellinger 1974b8d26b3bSNicholas Bellinger static void 1975b8d26b3bSNicholas Bellinger isert_response_completion(struct iser_tx_desc *tx_desc, 1976b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1977b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn, 1978b8d26b3bSNicholas Bellinger struct ib_device *ib_dev) 1979b8d26b3bSNicholas Bellinger { 1980d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1981b8d26b3bSNicholas Bellinger 1982b8d26b3bSNicholas Bellinger if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 19833df8f68aSNicholas Bellinger cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1984adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_REJECT || 1985adb54c29SNicholas Bellinger cmd->i_state == ISTATE_SEND_TEXTRSP) { 1986b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1987b8d26b3bSNicholas Bellinger 1988b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1989b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1990b8d26b3bSNicholas Bellinger return; 1991b8d26b3bSNicholas Bellinger } 1992897bb2c9SSagi Grimberg 1993b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 199403e7848aSNicholas Bellinger isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1995b8d26b3bSNicholas Bellinger } 1996b8d26b3bSNicholas Bellinger 1997b8d26b3bSNicholas Bellinger static void 19987748681bSSagi Grimberg isert_snd_completion(struct iser_tx_desc *tx_desc, 1999b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 2000b8d26b3bSNicholas Bellinger { 2001dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 2002b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2003b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr; 2004b8d26b3bSNicholas Bellinger 2005b8d26b3bSNicholas Bellinger if (!isert_cmd) { 2006b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 2007b8d26b3bSNicholas Bellinger return; 2008b8d26b3bSNicholas Bellinger } 2009b8d26b3bSNicholas Bellinger wr = &isert_cmd->rdma_wr; 2010b8d26b3bSNicholas Bellinger 20114c22e07fSSagi Grimberg isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 20124c22e07fSSagi Grimberg 2013b8d26b3bSNicholas Bellinger switch (wr->iser_ib_op) { 2014b8d26b3bSNicholas Bellinger case ISER_IB_SEND: 2015b8d26b3bSNicholas Bellinger isert_response_completion(tx_desc, isert_cmd, 2016b8d26b3bSNicholas Bellinger isert_conn, ib_dev); 2017b8d26b3bSNicholas Bellinger break; 2018b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_WRITE: 2019f93f3a70SSagi Grimberg isert_completion_rdma_write(tx_desc, isert_cmd); 2020b8d26b3bSNicholas Bellinger break; 2021b8d26b3bSNicholas Bellinger case ISER_IB_RDMA_READ: 2022b8d26b3bSNicholas Bellinger isert_completion_rdma_read(tx_desc, isert_cmd); 2023b8d26b3bSNicholas Bellinger break; 2024b8d26b3bSNicholas Bellinger default: 20254c22e07fSSagi Grimberg isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 2026b8d26b3bSNicholas Bellinger dump_stack(); 2027b8d26b3bSNicholas Bellinger break; 2028b8d26b3bSNicholas Bellinger } 2029b8d26b3bSNicholas Bellinger } 2030b8d26b3bSNicholas Bellinger 20316f0fae3dSSagi Grimberg /** 20326f0fae3dSSagi Grimberg * is_isert_tx_desc() - Indicate if the completion wr_id 20336f0fae3dSSagi Grimberg * is a TX descriptor or not. 20346f0fae3dSSagi Grimberg * @isert_conn: iser connection 20356f0fae3dSSagi Grimberg * @wr_id: completion WR identifier 20366f0fae3dSSagi Grimberg * 20376f0fae3dSSagi Grimberg * Since we cannot rely on wc opcode in FLUSH errors 20386f0fae3dSSagi Grimberg * we must work around it by checking if the wr_id address 20396f0fae3dSSagi Grimberg * falls in the iser connection rx_descs buffer. If so 20406f0fae3dSSagi Grimberg * it is an RX descriptor, otherwize it is a TX. 20416f0fae3dSSagi Grimberg */ 20426f0fae3dSSagi Grimberg static inline bool 20436f0fae3dSSagi Grimberg is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 2044b8d26b3bSNicholas Bellinger { 2045dac6ab30SSagi Grimberg void *start = isert_conn->rx_descs; 2046dac6ab30SSagi Grimberg int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); 20476f0fae3dSSagi Grimberg 20486f0fae3dSSagi Grimberg if (wr_id >= start && wr_id < start + len) 20496f0fae3dSSagi Grimberg return false; 20506f0fae3dSSagi Grimberg 20516f0fae3dSSagi Grimberg return true; 20526f0fae3dSSagi Grimberg } 20536f0fae3dSSagi Grimberg 20546f0fae3dSSagi Grimberg static void 20556f0fae3dSSagi Grimberg isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 20566f0fae3dSSagi Grimberg { 2057bdf20e72SSagi Grimberg if (wc->wr_id == ISER_BEACON_WRID) { 2058dac6ab30SSagi Grimberg isert_info("conn %p completing wait_comp_err\n", 2059bdf20e72SSagi Grimberg isert_conn); 2060dac6ab30SSagi Grimberg complete(&isert_conn->wait_comp_err); 2061ed4520aeSNicholas Bellinger } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 2062dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 2063df43debdSSagi Grimberg struct isert_cmd *isert_cmd; 20646f0fae3dSSagi Grimberg struct iser_tx_desc *desc; 2065b8d26b3bSNicholas Bellinger 20666f0fae3dSSagi Grimberg desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 20676f0fae3dSSagi Grimberg isert_cmd = desc->isert_cmd; 2068b8d26b3bSNicholas Bellinger if (!isert_cmd) 2069df43debdSSagi Grimberg isert_unmap_tx_desc(desc, ib_dev); 2070b8d26b3bSNicholas Bellinger else 2071df43debdSSagi Grimberg isert_completion_put(desc, isert_cmd, ib_dev, true); 2072df43debdSSagi Grimberg } else { 2073df43debdSSagi Grimberg isert_conn->post_recv_buf_count--; 2074bdf20e72SSagi Grimberg if (!isert_conn->post_recv_buf_count) 20759d49f5e2SSagi Grimberg iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2076b8d26b3bSNicholas Bellinger } 2077df43debdSSagi Grimberg } 2078b8d26b3bSNicholas Bellinger 2079b8d26b3bSNicholas Bellinger static void 20806f0fae3dSSagi Grimberg isert_handle_wc(struct ib_wc *wc) 2081b8d26b3bSNicholas Bellinger { 2082b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2083b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc; 20846f0fae3dSSagi Grimberg struct iser_rx_desc *rx_desc; 2085b8d26b3bSNicholas Bellinger 20866f0fae3dSSagi Grimberg isert_conn = wc->qp->qp_context; 20876f0fae3dSSagi Grimberg if (likely(wc->status == IB_WC_SUCCESS)) { 20886f0fae3dSSagi Grimberg if (wc->opcode == IB_WC_RECV) { 20896f0fae3dSSagi Grimberg rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 20907748681bSSagi Grimberg isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); 2091b8d26b3bSNicholas Bellinger } else { 20926f0fae3dSSagi Grimberg tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 20937748681bSSagi Grimberg isert_snd_completion(tx_desc, isert_conn); 2094b8d26b3bSNicholas Bellinger } 20956f0fae3dSSagi Grimberg } else { 20966f0fae3dSSagi Grimberg if (wc->status != IB_WC_WR_FLUSH_ERR) 2097ea8a1616SSagi Grimberg isert_err("%s (%d): wr id %llx vend_err %x\n", 2098ea8a1616SSagi Grimberg ib_wc_status_msg(wc->status), wc->status, 2099ea8a1616SSagi Grimberg wc->wr_id, wc->vendor_err); 21006f0fae3dSSagi Grimberg else 2101ea8a1616SSagi Grimberg isert_dbg("%s (%d): wr id %llx\n", 2102ea8a1616SSagi Grimberg ib_wc_status_msg(wc->status), wc->status, 2103ea8a1616SSagi Grimberg wc->wr_id); 2104b8d26b3bSNicholas Bellinger 21056f0fae3dSSagi Grimberg if (wc->wr_id != ISER_FASTREG_LI_WRID) 21066f0fae3dSSagi Grimberg isert_cq_comp_err(isert_conn, wc); 21076f0fae3dSSagi Grimberg } 2108b8d26b3bSNicholas Bellinger } 2109b8d26b3bSNicholas Bellinger 2110b8d26b3bSNicholas Bellinger static void 21116f0fae3dSSagi Grimberg isert_cq_work(struct work_struct *work) 2112b8d26b3bSNicholas Bellinger { 211337d9fe80SSagi Grimberg enum { isert_poll_budget = 65536 }; 21144a295baeSSagi Grimberg struct isert_comp *comp = container_of(work, struct isert_comp, 21156f0fae3dSSagi Grimberg work); 211636ea63b5SSagi Grimberg struct ib_wc *const wcs = comp->wcs; 211736ea63b5SSagi Grimberg int i, n, completed = 0; 2118b8d26b3bSNicholas Bellinger 211936ea63b5SSagi Grimberg while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 212036ea63b5SSagi Grimberg for (i = 0; i < n; i++) 212136ea63b5SSagi Grimberg isert_handle_wc(&wcs[i]); 2122b8d26b3bSNicholas Bellinger 212336ea63b5SSagi Grimberg completed += n; 212436ea63b5SSagi Grimberg if (completed >= isert_poll_budget) 212537d9fe80SSagi Grimberg break; 212637d9fe80SSagi Grimberg } 212737d9fe80SSagi Grimberg 21286f0fae3dSSagi Grimberg ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2129b8d26b3bSNicholas Bellinger } 2130b8d26b3bSNicholas Bellinger 2131b8d26b3bSNicholas Bellinger static void 21326f0fae3dSSagi Grimberg isert_cq_callback(struct ib_cq *cq, void *context) 2133b8d26b3bSNicholas Bellinger { 21344a295baeSSagi Grimberg struct isert_comp *comp = context; 2135b8d26b3bSNicholas Bellinger 21366f0fae3dSSagi Grimberg queue_work(isert_comp_wq, &comp->work); 2137b8d26b3bSNicholas Bellinger } 2138b8d26b3bSNicholas Bellinger 2139b8d26b3bSNicholas Bellinger static int 2140b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2141b8d26b3bSNicholas Bellinger { 2142b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 2143b8d26b3bSNicholas Bellinger int ret; 2144b8d26b3bSNicholas Bellinger 21454366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 21464366b19cSJenny Derzhavetz if (ret) { 21474366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", ret); 21484366b19cSJenny Derzhavetz return ret; 21494366b19cSJenny Derzhavetz } 21504366b19cSJenny Derzhavetz 2151dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2152b8d26b3bSNicholas Bellinger &wr_failed); 2153b8d26b3bSNicholas Bellinger if (ret) { 215424f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 2155b8d26b3bSNicholas Bellinger return ret; 2156b8d26b3bSNicholas Bellinger } 2157b8d26b3bSNicholas Bellinger return ret; 2158b8d26b3bSNicholas Bellinger } 2159b8d26b3bSNicholas Bellinger 2160b8d26b3bSNicholas Bellinger static int 2161b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2162b8d26b3bSNicholas Bellinger { 2163d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21646700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2165b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2166b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2167b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 2168b8d26b3bSNicholas Bellinger 2169b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2170b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2171b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2172b8d26b3bSNicholas Bellinger /* 2173b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 2174b8d26b3bSNicholas Bellinger */ 2175b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 2176b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2177b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2178dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 217967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2180b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2181dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 2182b8d26b3bSNicholas Bellinger 2183b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2184b8d26b3bSNicholas Bellinger cmd->sense_buffer); 2185b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2186b8d26b3bSNicholas Bellinger 2187b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2188b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2189dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2190b8d26b3bSNicholas Bellinger 2191dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2192dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 2193b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 2194b8d26b3bSNicholas Bellinger 2195dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 2196dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2197dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 219834efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2199b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2200b8d26b3bSNicholas Bellinger } 2201b8d26b3bSNicholas Bellinger 220268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2203b8d26b3bSNicholas Bellinger 22044c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n"); 2205b8d26b3bSNicholas Bellinger 2206b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2207b8d26b3bSNicholas Bellinger } 2208b8d26b3bSNicholas Bellinger 2209131e6abcSNicholas Bellinger static void 2210131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2211131e6abcSNicholas Bellinger { 2212131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 22136700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2214dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 2215131e6abcSNicholas Bellinger 2216131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 2217131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 2218131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 2219131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 2220131e6abcSNicholas Bellinger 2221131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 2222131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 2223131e6abcSNicholas Bellinger 2224131e6abcSNicholas Bellinger device->unreg_rdma_mem(isert_cmd, isert_conn); 2225131e6abcSNicholas Bellinger } 2226131e6abcSNicholas Bellinger 2227e70beee7SNicholas Bellinger static enum target_prot_op 2228e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 2229e70beee7SNicholas Bellinger { 22306700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2231dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 2232e70beee7SNicholas Bellinger 223323a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 223423a548eeSSagi Grimberg if (device->pi_capable) { 223524f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 223623a548eeSSagi Grimberg isert_conn->pi_support = true; 2237e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 223823a548eeSSagi Grimberg } 223923a548eeSSagi Grimberg } 224023a548eeSSagi Grimberg 224124f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 224223a548eeSSagi Grimberg isert_conn->pi_support = false; 2243e70beee7SNicholas Bellinger 2244e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 2245e70beee7SNicholas Bellinger } 2246e70beee7SNicholas Bellinger 2247b8d26b3bSNicholas Bellinger static int 2248b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2249b8d26b3bSNicholas Bellinger bool nopout_response) 2250b8d26b3bSNicholas Bellinger { 2251d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 22526700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2253b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2254b8d26b3bSNicholas Bellinger 2255b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2256b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2257b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 2258b8d26b3bSNicholas Bellinger nopout_response); 2259b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 226068a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2261b8d26b3bSNicholas Bellinger 22624c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2263b8d26b3bSNicholas Bellinger 2264b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2265b8d26b3bSNicholas Bellinger } 2266b8d26b3bSNicholas Bellinger 2267b8d26b3bSNicholas Bellinger static int 2268b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2269b8d26b3bSNicholas Bellinger { 2270d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 22716700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2272b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2273b8d26b3bSNicholas Bellinger 2274b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2275b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2276b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2277b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 227868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2279b8d26b3bSNicholas Bellinger 22804c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2281b8d26b3bSNicholas Bellinger 2282b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2283b8d26b3bSNicholas Bellinger } 2284b8d26b3bSNicholas Bellinger 2285b8d26b3bSNicholas Bellinger static int 2286b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2287b8d26b3bSNicholas Bellinger { 2288d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 22896700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2290b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2291b8d26b3bSNicholas Bellinger 2292b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2293b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2294b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2295b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 229668a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2297b8d26b3bSNicholas Bellinger 22984c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2299b8d26b3bSNicholas Bellinger 2300b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2301b8d26b3bSNicholas Bellinger } 2302b8d26b3bSNicholas Bellinger 2303b8d26b3bSNicholas Bellinger static int 2304b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2305b8d26b3bSNicholas Bellinger { 2306d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 23076700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2308b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2309dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 231067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 23113df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 23123df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 23133df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2314b8d26b3bSNicholas Bellinger 2315b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 23163df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 2317b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 23183df8f68aSNicholas Bellinger 23193df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2320dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 23213df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 23223df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2323dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2324dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 23253df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 232634efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 23273df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 23283df8f68aSNicholas Bellinger 232968a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2330b8d26b3bSNicholas Bellinger 23314c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn); 2332b8d26b3bSNicholas Bellinger 2333b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2334b8d26b3bSNicholas Bellinger } 2335b8d26b3bSNicholas Bellinger 2336b8d26b3bSNicholas Bellinger static int 2337adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2338adb54c29SNicholas Bellinger { 2339d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 23406700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2341adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2342adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2343adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2344adb54c29SNicholas Bellinger u32 txt_rsp_len; 2345adb54c29SNicholas Bellinger int rc; 2346adb54c29SNicholas Bellinger 2347adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 234822c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2349adb54c29SNicholas Bellinger if (rc < 0) 2350adb54c29SNicholas Bellinger return rc; 2351adb54c29SNicholas Bellinger 2352adb54c29SNicholas Bellinger txt_rsp_len = rc; 2353adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2354adb54c29SNicholas Bellinger 2355adb54c29SNicholas Bellinger if (txt_rsp_len) { 2356dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 235767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2358adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2359adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2360adb54c29SNicholas Bellinger 2361adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2362adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2363adb54c29SNicholas Bellinger 2364adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2365adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2366adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 236734efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2368adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2369adb54c29SNicholas Bellinger } 237068a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2371adb54c29SNicholas Bellinger 2372f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn); 2373adb54c29SNicholas Bellinger 2374adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2375adb54c29SNicholas Bellinger } 2376adb54c29SNicholas Bellinger 2377adb54c29SNicholas Bellinger static int 2378b8d26b3bSNicholas Bellinger isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2379e622f2f4SChristoph Hellwig struct ib_sge *ib_sge, struct ib_rdma_wr *rdma_wr, 2380b8d26b3bSNicholas Bellinger u32 data_left, u32 offset) 2381b8d26b3bSNicholas Bellinger { 2382d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2383b8d26b3bSNicholas Bellinger struct scatterlist *sg_start, *tmp_sg; 2384dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 238567cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2386b8d26b3bSNicholas Bellinger u32 sg_off, page_off; 2387b8d26b3bSNicholas Bellinger int i = 0, sg_nents; 2388b8d26b3bSNicholas Bellinger 2389b8d26b3bSNicholas Bellinger sg_off = offset / PAGE_SIZE; 2390b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2391b8d26b3bSNicholas Bellinger sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2392b8d26b3bSNicholas Bellinger page_off = offset % PAGE_SIZE; 2393b8d26b3bSNicholas Bellinger 2394e622f2f4SChristoph Hellwig rdma_wr->wr.sg_list = ib_sge; 2395e622f2f4SChristoph Hellwig rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; 2396b8d26b3bSNicholas Bellinger /* 2397b8d26b3bSNicholas Bellinger * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2398b8d26b3bSNicholas Bellinger */ 2399b8d26b3bSNicholas Bellinger for_each_sg(sg_start, tmp_sg, sg_nents, i) { 24004c22e07fSSagi Grimberg isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 24014c22e07fSSagi Grimberg "page_off: %u\n", 2402b8d26b3bSNicholas Bellinger (unsigned long long)tmp_sg->dma_address, 2403b8d26b3bSNicholas Bellinger tmp_sg->length, page_off); 2404b8d26b3bSNicholas Bellinger 2405b8d26b3bSNicholas Bellinger ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2406b8d26b3bSNicholas Bellinger ib_sge->length = min_t(u32, data_left, 2407b8d26b3bSNicholas Bellinger ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 240834efc7dfSJason Gunthorpe ib_sge->lkey = device->pd->local_dma_lkey; 2409b8d26b3bSNicholas Bellinger 24104c22e07fSSagi Grimberg isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 241190ecc6e2SVu Pham ib_sge->addr, ib_sge->length, ib_sge->lkey); 2412b8d26b3bSNicholas Bellinger page_off = 0; 2413b8d26b3bSNicholas Bellinger data_left -= ib_sge->length; 24149253e667SSagi Grimberg if (!data_left) 24159253e667SSagi Grimberg break; 2416b8d26b3bSNicholas Bellinger ib_sge++; 241724f412ddSSagi Grimberg isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2418b8d26b3bSNicholas Bellinger } 2419b8d26b3bSNicholas Bellinger 2420e622f2f4SChristoph Hellwig rdma_wr->wr.num_sge = ++i; 242124f412ddSSagi Grimberg isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2422e622f2f4SChristoph Hellwig rdma_wr->wr.sg_list, rdma_wr->wr.num_sge); 2423b8d26b3bSNicholas Bellinger 2424e622f2f4SChristoph Hellwig return rdma_wr->wr.num_sge; 2425b8d26b3bSNicholas Bellinger } 2426b8d26b3bSNicholas Bellinger 2427b8d26b3bSNicholas Bellinger static int 242890ecc6e2SVu Pham isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 242990ecc6e2SVu Pham struct isert_rdma_wr *wr) 2430b8d26b3bSNicholas Bellinger { 2431b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2432d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 24336700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2434e3d7e4c3SSagi Grimberg struct isert_data_buf *data = &wr->data; 2435e622f2f4SChristoph Hellwig struct ib_rdma_wr *rdma_wr; 2436b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge; 2437e3d7e4c3SSagi Grimberg u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2438e3d7e4c3SSagi Grimberg int ret = 0, i, ib_sge_cnt; 2439b8d26b3bSNicholas Bellinger 244090ecc6e2SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2441b8d26b3bSNicholas Bellinger 2442e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2443e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2444e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2445e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2446e3d7e4c3SSagi Grimberg if (ret) 2447e3d7e4c3SSagi Grimberg return ret; 2448b8d26b3bSNicholas Bellinger 2449e3d7e4c3SSagi Grimberg data_left = data->len; 2450e3d7e4c3SSagi Grimberg offset = data->offset; 2451b8d26b3bSNicholas Bellinger 2452e3d7e4c3SSagi Grimberg ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2453b8d26b3bSNicholas Bellinger if (!ib_sge) { 245424f412ddSSagi Grimberg isert_warn("Unable to allocate ib_sge\n"); 2455b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2456e3d7e4c3SSagi Grimberg goto unmap_cmd; 2457b8d26b3bSNicholas Bellinger } 245890ecc6e2SVu Pham wr->ib_sge = ib_sge; 2459b8d26b3bSNicholas Bellinger 2460e622f2f4SChristoph Hellwig wr->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2461e622f2f4SChristoph Hellwig wr->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * wr->rdma_wr_num, 2462b8d26b3bSNicholas Bellinger GFP_KERNEL); 2463e622f2f4SChristoph Hellwig if (!wr->rdma_wr) { 2464e622f2f4SChristoph Hellwig isert_dbg("Unable to allocate wr->rdma_wr\n"); 2465b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2466e3d7e4c3SSagi Grimberg goto unmap_cmd; 2467b8d26b3bSNicholas Bellinger } 2468b8d26b3bSNicholas Bellinger 2469b8d26b3bSNicholas Bellinger wr->isert_cmd = isert_cmd; 2470b8d26b3bSNicholas Bellinger rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2471b8d26b3bSNicholas Bellinger 2472e622f2f4SChristoph Hellwig for (i = 0; i < wr->rdma_wr_num; i++) { 2473e622f2f4SChristoph Hellwig rdma_wr = &isert_cmd->rdma_wr.rdma_wr[i]; 2474b8d26b3bSNicholas Bellinger data_len = min(data_left, rdma_write_max); 2475b8d26b3bSNicholas Bellinger 2476e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = 0; 247790ecc6e2SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2478e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2479e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->read_va + offset; 2480e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->read_stag; 2481e622f2f4SChristoph Hellwig if (i + 1 == wr->rdma_wr_num) 2482e622f2f4SChristoph Hellwig rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr; 248390ecc6e2SVu Pham else 2484e622f2f4SChristoph Hellwig rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; 248590ecc6e2SVu Pham } else { 2486e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2487e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->write_va + va_offset; 2488e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->write_stag; 2489e622f2f4SChristoph Hellwig if (i + 1 == wr->rdma_wr_num) 2490e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 249190ecc6e2SVu Pham else 2492e622f2f4SChristoph Hellwig rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; 249390ecc6e2SVu Pham } 2494b8d26b3bSNicholas Bellinger 2495b8d26b3bSNicholas Bellinger ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2496e622f2f4SChristoph Hellwig rdma_wr, data_len, offset); 2497b8d26b3bSNicholas Bellinger ib_sge += ib_sge_cnt; 2498b8d26b3bSNicholas Bellinger 2499b8d26b3bSNicholas Bellinger offset += data_len; 250090ecc6e2SVu Pham va_offset += data_len; 2501b8d26b3bSNicholas Bellinger data_left -= data_len; 2502b8d26b3bSNicholas Bellinger } 250390ecc6e2SVu Pham 250490ecc6e2SVu Pham return 0; 2505e3d7e4c3SSagi Grimberg unmap_cmd: 2506e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, data); 2507e3d7e4c3SSagi Grimberg 250890ecc6e2SVu Pham return ret; 250990ecc6e2SVu Pham } 251090ecc6e2SVu Pham 251110633c37SSagi Grimberg static inline void 251210633c37SSagi Grimberg isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 251310633c37SSagi Grimberg { 251410633c37SSagi Grimberg u32 rkey; 251510633c37SSagi Grimberg 251610633c37SSagi Grimberg memset(inv_wr, 0, sizeof(*inv_wr)); 251710633c37SSagi Grimberg inv_wr->wr_id = ISER_FASTREG_LI_WRID; 251810633c37SSagi Grimberg inv_wr->opcode = IB_WR_LOCAL_INV; 251910633c37SSagi Grimberg inv_wr->ex.invalidate_rkey = mr->rkey; 252010633c37SSagi Grimberg 252110633c37SSagi Grimberg /* Bump the key */ 252210633c37SSagi Grimberg rkey = ib_inc_rkey(mr->rkey); 252310633c37SSagi Grimberg ib_update_fast_reg_key(mr, rkey); 252410633c37SSagi Grimberg } 252510633c37SSagi Grimberg 252659464ef4SVu Pham static int 2527e3d7e4c3SSagi Grimberg isert_fast_reg_mr(struct isert_conn *isert_conn, 2528e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc, 2529e3d7e4c3SSagi Grimberg struct isert_data_buf *mem, 25309e961ae7SSagi Grimberg enum isert_indicator ind, 2531e3d7e4c3SSagi Grimberg struct ib_sge *sge) 253259464ef4SVu Pham { 2533dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 253467cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 25359e961ae7SSagi Grimberg struct ib_mr *mr; 253616c2d702SSagi Grimberg struct ib_reg_wr reg_wr; 2537e622f2f4SChristoph Hellwig struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 253816c2d702SSagi Grimberg int ret, n; 253959464ef4SVu Pham 2540e3d7e4c3SSagi Grimberg if (mem->dma_nents == 1) { 254134efc7dfSJason Gunthorpe sge->lkey = device->pd->local_dma_lkey; 2542e3d7e4c3SSagi Grimberg sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2543e3d7e4c3SSagi Grimberg sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 25444c22e07fSSagi Grimberg isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 25454c22e07fSSagi Grimberg sge->addr, sge->length, sge->lkey); 2546e3d7e4c3SSagi Grimberg return 0; 2547e3d7e4c3SSagi Grimberg } 2548e3d7e4c3SSagi Grimberg 254916c2d702SSagi Grimberg if (ind == ISERT_DATA_KEY_VALID) 25509e961ae7SSagi Grimberg /* Registering data buffer */ 25519e961ae7SSagi Grimberg mr = fr_desc->data_mr; 255216c2d702SSagi Grimberg else 25539e961ae7SSagi Grimberg /* Registering protection buffer */ 25549e961ae7SSagi Grimberg mr = fr_desc->pi_ctx->prot_mr; 255559464ef4SVu Pham 255610633c37SSagi Grimberg if (!(fr_desc->ind & ind)) { 255710633c37SSagi Grimberg isert_inv_rkey(&inv_wr, mr); 255859464ef4SVu Pham wr = &inv_wr; 255959464ef4SVu Pham } 256059464ef4SVu Pham 256116c2d702SSagi Grimberg n = ib_map_mr_sg(mr, mem->sg, mem->nents, PAGE_SIZE); 256216c2d702SSagi Grimberg if (unlikely(n != mem->nents)) { 256316c2d702SSagi Grimberg isert_err("failed to map mr sg (%d/%d)\n", 256416c2d702SSagi Grimberg n, mem->nents); 256516c2d702SSagi Grimberg return n < 0 ? n : -EINVAL; 256616c2d702SSagi Grimberg } 256716c2d702SSagi Grimberg 256816c2d702SSagi Grimberg isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 256916c2d702SSagi Grimberg fr_desc, mem->nents, mem->offset); 257016c2d702SSagi Grimberg 257116c2d702SSagi Grimberg reg_wr.wr.next = NULL; 257216c2d702SSagi Grimberg reg_wr.wr.opcode = IB_WR_REG_MR; 257316c2d702SSagi Grimberg reg_wr.wr.wr_id = ISER_FASTREG_LI_WRID; 257416c2d702SSagi Grimberg reg_wr.wr.send_flags = 0; 257516c2d702SSagi Grimberg reg_wr.wr.num_sge = 0; 257616c2d702SSagi Grimberg reg_wr.mr = mr; 257716c2d702SSagi Grimberg reg_wr.key = mr->lkey; 257816c2d702SSagi Grimberg reg_wr.access = IB_ACCESS_LOCAL_WRITE; 257959464ef4SVu Pham 258059464ef4SVu Pham if (!wr) 258116c2d702SSagi Grimberg wr = ®_wr.wr; 258259464ef4SVu Pham else 258316c2d702SSagi Grimberg wr->next = ®_wr.wr; 258459464ef4SVu Pham 2585dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 258659464ef4SVu Pham if (ret) { 258724f412ddSSagi Grimberg isert_err("fast registration failed, ret:%d\n", ret); 258859464ef4SVu Pham return ret; 258959464ef4SVu Pham } 25909e961ae7SSagi Grimberg fr_desc->ind &= ~ind; 259159464ef4SVu Pham 25929e961ae7SSagi Grimberg sge->lkey = mr->lkey; 259316c2d702SSagi Grimberg sge->addr = mr->iova; 259416c2d702SSagi Grimberg sge->length = mr->length; 259559464ef4SVu Pham 25964c22e07fSSagi Grimberg isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 25974c22e07fSSagi Grimberg sge->addr, sge->length, sge->lkey); 259859464ef4SVu Pham 259959464ef4SVu Pham return ret; 260059464ef4SVu Pham } 260159464ef4SVu Pham 26023d73cf1aSSagi Grimberg static inline void 26033d73cf1aSSagi Grimberg isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 26043d73cf1aSSagi Grimberg struct ib_sig_domain *domain) 26059e961ae7SSagi Grimberg { 260678eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 26073d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 26083d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 26093d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 261078eda2bbSSagi Grimberg /* 261178eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 261278eda2bbSSagi Grimberg * the target core would like to use it, we will take it 261378eda2bbSSagi Grimberg * from se_cmd. 261478eda2bbSSagi Grimberg */ 261578eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 261678eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 261778eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 261878eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 261978eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 262078eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 26213d73cf1aSSagi Grimberg }; 26223d73cf1aSSagi Grimberg 26233d73cf1aSSagi Grimberg static int 26243d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 26253d73cf1aSSagi Grimberg { 26269e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 26279e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 26289e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 262978eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 26303d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 26319e961ae7SSagi Grimberg break; 26329e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 26339e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 263478eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 26353d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 26369e961ae7SSagi Grimberg break; 26379e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 26389e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 26393d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 26403d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 26419e961ae7SSagi Grimberg break; 26429e961ae7SSagi Grimberg default: 264324f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 26449e961ae7SSagi Grimberg return -EINVAL; 26459e961ae7SSagi Grimberg } 26469e961ae7SSagi Grimberg 26479e961ae7SSagi Grimberg return 0; 26489e961ae7SSagi Grimberg } 26499e961ae7SSagi Grimberg 26509e961ae7SSagi Grimberg static inline u8 26519e961ae7SSagi Grimberg isert_set_prot_checks(u8 prot_checks) 26529e961ae7SSagi Grimberg { 26539e961ae7SSagi Grimberg return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 26549e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 26559e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 26569e961ae7SSagi Grimberg } 26579e961ae7SSagi Grimberg 26589e961ae7SSagi Grimberg static int 2659570db170SSagi Grimberg isert_reg_sig_mr(struct isert_conn *isert_conn, 2660570db170SSagi Grimberg struct se_cmd *se_cmd, 2661570db170SSagi Grimberg struct isert_rdma_wr *rdma_wr, 2662570db170SSagi Grimberg struct fast_reg_descriptor *fr_desc) 26639e961ae7SSagi Grimberg { 2664e622f2f4SChristoph Hellwig struct ib_sig_handover_wr sig_wr; 2665e622f2f4SChristoph Hellwig struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 26669e961ae7SSagi Grimberg struct pi_context *pi_ctx = fr_desc->pi_ctx; 26679e961ae7SSagi Grimberg struct ib_sig_attrs sig_attrs; 26689e961ae7SSagi Grimberg int ret; 26699e961ae7SSagi Grimberg 26709e961ae7SSagi Grimberg memset(&sig_attrs, 0, sizeof(sig_attrs)); 26719e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 26729e961ae7SSagi Grimberg if (ret) 26739e961ae7SSagi Grimberg goto err; 26749e961ae7SSagi Grimberg 26759e961ae7SSagi Grimberg sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 26769e961ae7SSagi Grimberg 26779e961ae7SSagi Grimberg if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 267810633c37SSagi Grimberg isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 26799e961ae7SSagi Grimberg wr = &inv_wr; 26809e961ae7SSagi Grimberg } 26819e961ae7SSagi Grimberg 26829e961ae7SSagi Grimberg memset(&sig_wr, 0, sizeof(sig_wr)); 2683e622f2f4SChristoph Hellwig sig_wr.wr.opcode = IB_WR_REG_SIG_MR; 2684e622f2f4SChristoph Hellwig sig_wr.wr.wr_id = ISER_FASTREG_LI_WRID; 2685e622f2f4SChristoph Hellwig sig_wr.wr.sg_list = &rdma_wr->ib_sg[DATA]; 2686e622f2f4SChristoph Hellwig sig_wr.wr.num_sge = 1; 2687e622f2f4SChristoph Hellwig sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; 2688e622f2f4SChristoph Hellwig sig_wr.sig_attrs = &sig_attrs; 2689e622f2f4SChristoph Hellwig sig_wr.sig_mr = pi_ctx->sig_mr; 26909e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 2691e622f2f4SChristoph Hellwig sig_wr.prot = &rdma_wr->ib_sg[PROT]; 26929e961ae7SSagi Grimberg 26939e961ae7SSagi Grimberg if (!wr) 2694e622f2f4SChristoph Hellwig wr = &sig_wr.wr; 26959e961ae7SSagi Grimberg else 2696e622f2f4SChristoph Hellwig wr->next = &sig_wr.wr; 26979e961ae7SSagi Grimberg 2698dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 26999e961ae7SSagi Grimberg if (ret) { 270024f412ddSSagi Grimberg isert_err("fast registration failed, ret:%d\n", ret); 27019e961ae7SSagi Grimberg goto err; 27029e961ae7SSagi Grimberg } 27039e961ae7SSagi Grimberg fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 27049e961ae7SSagi Grimberg 2705570db170SSagi Grimberg rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2706570db170SSagi Grimberg rdma_wr->ib_sg[SIG].addr = 0; 2707570db170SSagi Grimberg rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 27089e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 27099e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 27109e961ae7SSagi Grimberg /* 27119e961ae7SSagi Grimberg * We have protection guards on the wire 27129e961ae7SSagi Grimberg * so we need to set a larget transfer 27139e961ae7SSagi Grimberg */ 2714570db170SSagi Grimberg rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 27159e961ae7SSagi Grimberg 271624f412ddSSagi Grimberg isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2717570db170SSagi Grimberg rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2718570db170SSagi Grimberg rdma_wr->ib_sg[SIG].lkey); 27199e961ae7SSagi Grimberg err: 27209e961ae7SSagi Grimberg return ret; 27219e961ae7SSagi Grimberg } 27229e961ae7SSagi Grimberg 272359464ef4SVu Pham static int 2724570db170SSagi Grimberg isert_handle_prot_cmd(struct isert_conn *isert_conn, 2725570db170SSagi Grimberg struct isert_cmd *isert_cmd, 2726570db170SSagi Grimberg struct isert_rdma_wr *wr) 2727570db170SSagi Grimberg { 2728dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 2729570db170SSagi Grimberg struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2730570db170SSagi Grimberg int ret; 2731570db170SSagi Grimberg 2732570db170SSagi Grimberg if (!wr->fr_desc->pi_ctx) { 2733570db170SSagi Grimberg ret = isert_create_pi_ctx(wr->fr_desc, 2734570db170SSagi Grimberg device->ib_device, 273567cb3949SSagi Grimberg device->pd); 2736570db170SSagi Grimberg if (ret) { 273724f412ddSSagi Grimberg isert_err("conn %p failed to allocate pi_ctx\n", 2738570db170SSagi Grimberg isert_conn); 2739570db170SSagi Grimberg return ret; 2740570db170SSagi Grimberg } 2741570db170SSagi Grimberg } 2742570db170SSagi Grimberg 2743570db170SSagi Grimberg if (se_cmd->t_prot_sg) { 2744570db170SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, 2745570db170SSagi Grimberg se_cmd->t_prot_sg, 2746570db170SSagi Grimberg se_cmd->t_prot_nents, 2747570db170SSagi Grimberg se_cmd->prot_length, 2748570db170SSagi Grimberg 0, wr->iser_ib_op, &wr->prot); 2749570db170SSagi Grimberg if (ret) { 275024f412ddSSagi Grimberg isert_err("conn %p failed to map protection buffer\n", 2751570db170SSagi Grimberg isert_conn); 2752570db170SSagi Grimberg return ret; 2753570db170SSagi Grimberg } 2754570db170SSagi Grimberg 2755570db170SSagi Grimberg memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2756570db170SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2757570db170SSagi Grimberg ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2758570db170SSagi Grimberg if (ret) { 275924f412ddSSagi Grimberg isert_err("conn %p failed to fast reg mr\n", 2760570db170SSagi Grimberg isert_conn); 2761570db170SSagi Grimberg goto unmap_prot_cmd; 2762570db170SSagi Grimberg } 2763570db170SSagi Grimberg } 2764570db170SSagi Grimberg 2765570db170SSagi Grimberg ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2766570db170SSagi Grimberg if (ret) { 276724f412ddSSagi Grimberg isert_err("conn %p failed to fast reg mr\n", 2768570db170SSagi Grimberg isert_conn); 2769570db170SSagi Grimberg goto unmap_prot_cmd; 2770570db170SSagi Grimberg } 2771570db170SSagi Grimberg wr->fr_desc->ind |= ISERT_PROTECTED; 2772570db170SSagi Grimberg 2773570db170SSagi Grimberg return 0; 2774570db170SSagi Grimberg 2775570db170SSagi Grimberg unmap_prot_cmd: 2776570db170SSagi Grimberg if (se_cmd->t_prot_sg) 2777570db170SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->prot); 2778570db170SSagi Grimberg 2779570db170SSagi Grimberg return ret; 2780570db170SSagi Grimberg } 2781570db170SSagi Grimberg 2782570db170SSagi Grimberg static int 2783a3a5a826SSagi Grimberg isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 278459464ef4SVu Pham struct isert_rdma_wr *wr) 278559464ef4SVu Pham { 278659464ef4SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 278759464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2788e3d7e4c3SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2789e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc = NULL; 2790e622f2f4SChristoph Hellwig struct ib_rdma_wr *rdma_wr; 2791570db170SSagi Grimberg struct ib_sge *ib_sg; 2792e3d7e4c3SSagi Grimberg u32 offset; 2793e3d7e4c3SSagi Grimberg int ret = 0; 279459464ef4SVu Pham unsigned long flags; 279559464ef4SVu Pham 279659464ef4SVu Pham isert_cmd->tx_desc.isert_cmd = isert_cmd; 2797e3d7e4c3SSagi Grimberg 2798e3d7e4c3SSagi Grimberg offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2799e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2800e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2801e3d7e4c3SSagi Grimberg offset, wr->iser_ib_op, &wr->data); 2802e3d7e4c3SSagi Grimberg if (ret) 2803e3d7e4c3SSagi Grimberg return ret; 2804e3d7e4c3SSagi Grimberg 2805302cc7c3SSagi Grimberg if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2806dac6ab30SSagi Grimberg spin_lock_irqsave(&isert_conn->pool_lock, flags); 2807dac6ab30SSagi Grimberg fr_desc = list_first_entry(&isert_conn->fr_pool, 2808e3d7e4c3SSagi Grimberg struct fast_reg_descriptor, list); 2809e3d7e4c3SSagi Grimberg list_del(&fr_desc->list); 2810dac6ab30SSagi Grimberg spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2811e3d7e4c3SSagi Grimberg wr->fr_desc = fr_desc; 281259464ef4SVu Pham } 281359464ef4SVu Pham 28149e961ae7SSagi Grimberg ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2815570db170SSagi Grimberg ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2816e3d7e4c3SSagi Grimberg if (ret) 2817e3d7e4c3SSagi Grimberg goto unmap_cmd; 281859464ef4SVu Pham 2819302cc7c3SSagi Grimberg if (isert_prot_cmd(isert_conn, se_cmd)) { 2820570db170SSagi Grimberg ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 28219e961ae7SSagi Grimberg if (ret) 28229e961ae7SSagi Grimberg goto unmap_cmd; 28239e961ae7SSagi Grimberg 2824570db170SSagi Grimberg ib_sg = &wr->ib_sg[SIG]; 2825570db170SSagi Grimberg } else { 2826570db170SSagi Grimberg ib_sg = &wr->ib_sg[DATA]; 28279e961ae7SSagi Grimberg } 28289e961ae7SSagi Grimberg 2829570db170SSagi Grimberg memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2830e3d7e4c3SSagi Grimberg wr->ib_sge = &wr->s_ib_sge; 2831e622f2f4SChristoph Hellwig wr->rdma_wr_num = 1; 2832e622f2f4SChristoph Hellwig memset(&wr->s_rdma_wr, 0, sizeof(wr->s_rdma_wr)); 2833e622f2f4SChristoph Hellwig wr->rdma_wr = &wr->s_rdma_wr; 283459464ef4SVu Pham wr->isert_cmd = isert_cmd; 283559464ef4SVu Pham 2836e622f2f4SChristoph Hellwig rdma_wr = &isert_cmd->rdma_wr.s_rdma_wr; 2837e622f2f4SChristoph Hellwig rdma_wr->wr.sg_list = &wr->s_ib_sge; 2838e622f2f4SChristoph Hellwig rdma_wr->wr.num_sge = 1; 2839e622f2f4SChristoph Hellwig rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; 284059464ef4SVu Pham if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2841e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2842e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->read_va; 2843e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->read_stag; 2844e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 28459e961ae7SSagi Grimberg 0 : IB_SEND_SIGNALED; 284659464ef4SVu Pham } else { 2847e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2848e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->write_va; 2849e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->write_stag; 2850e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 285159464ef4SVu Pham } 285259464ef4SVu Pham 285359464ef4SVu Pham return 0; 2854570db170SSagi Grimberg 2855e3d7e4c3SSagi Grimberg unmap_cmd: 2856e3d7e4c3SSagi Grimberg if (fr_desc) { 2857dac6ab30SSagi Grimberg spin_lock_irqsave(&isert_conn->pool_lock, flags); 2858dac6ab30SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2859dac6ab30SSagi Grimberg spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2860e3d7e4c3SSagi Grimberg } 2861e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, &wr->data); 286259464ef4SVu Pham 286359464ef4SVu Pham return ret; 286459464ef4SVu Pham } 286559464ef4SVu Pham 286659464ef4SVu Pham static int 286790ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 286890ecc6e2SVu Pham { 286990ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 287059464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 287190ecc6e2SVu Pham struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 28726700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2873dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 287490ecc6e2SVu Pham struct ib_send_wr *wr_failed; 287590ecc6e2SVu Pham int rc; 287690ecc6e2SVu Pham 287724f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 287890ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 28794c22e07fSSagi Grimberg 288090ecc6e2SVu Pham wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2881d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 288290ecc6e2SVu Pham if (rc) { 288324f412ddSSagi Grimberg isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 288490ecc6e2SVu Pham return rc; 288590ecc6e2SVu Pham } 288690ecc6e2SVu Pham 2887302cc7c3SSagi Grimberg if (!isert_prot_cmd(isert_conn, se_cmd)) { 2888b8d26b3bSNicholas Bellinger /* 2889b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2890b8d26b3bSNicholas Bellinger */ 28919e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 28929e961ae7SSagi Grimberg &isert_cmd->tx_desc); 289304d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2894b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2895b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 289695b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 289768a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 2898e622f2f4SChristoph Hellwig isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr; 2899e622f2f4SChristoph Hellwig wr->rdma_wr_num += 1; 29004366b19cSJenny Derzhavetz 29014366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 29024366b19cSJenny Derzhavetz if (rc) { 29034366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 29044366b19cSJenny Derzhavetz return rc; 29054366b19cSJenny Derzhavetz } 29069e961ae7SSagi Grimberg } 2907b8d26b3bSNicholas Bellinger 2908e622f2f4SChristoph Hellwig rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); 2909bdf20e72SSagi Grimberg if (rc) 291024f412ddSSagi Grimberg isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 29119e961ae7SSagi Grimberg 2912302cc7c3SSagi Grimberg if (!isert_prot_cmd(isert_conn, se_cmd)) 291324f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 29149e961ae7SSagi Grimberg "READ\n", isert_cmd); 29159e961ae7SSagi Grimberg else 291624f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 291790ecc6e2SVu Pham isert_cmd); 2918b8d26b3bSNicholas Bellinger 291990ecc6e2SVu Pham return 1; 2920b8d26b3bSNicholas Bellinger } 2921b8d26b3bSNicholas Bellinger 2922b8d26b3bSNicholas Bellinger static int 2923b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2924b8d26b3bSNicholas Bellinger { 2925b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2926d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2927b8d26b3bSNicholas Bellinger struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 29286700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2929dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 293090ecc6e2SVu Pham struct ib_send_wr *wr_failed; 293190ecc6e2SVu Pham int rc; 2932b8d26b3bSNicholas Bellinger 293324f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 293490ecc6e2SVu Pham isert_cmd, se_cmd->data_length, cmd->write_data_done); 2935b8d26b3bSNicholas Bellinger wr->iser_ib_op = ISER_IB_RDMA_READ; 2936d40945d8SVu Pham rc = device->reg_rdma_mem(conn, cmd, wr); 293790ecc6e2SVu Pham if (rc) { 293824f412ddSSagi Grimberg isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 293990ecc6e2SVu Pham return rc; 2940b8d26b3bSNicholas Bellinger } 2941b8d26b3bSNicholas Bellinger 2942e622f2f4SChristoph Hellwig rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); 2943bdf20e72SSagi Grimberg if (rc) 294424f412ddSSagi Grimberg isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2945bdf20e72SSagi Grimberg 294624f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 294790ecc6e2SVu Pham isert_cmd); 2948b8d26b3bSNicholas Bellinger 294990ecc6e2SVu Pham return 0; 2950b8d26b3bSNicholas Bellinger } 2951b8d26b3bSNicholas Bellinger 2952b8d26b3bSNicholas Bellinger static int 2953b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2954b8d26b3bSNicholas Bellinger { 2955a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2956a4c15cd9SJenny Derzhavetz int ret = 0; 2957b8d26b3bSNicholas Bellinger 2958b8d26b3bSNicholas Bellinger switch (state) { 2959a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE: 2960a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 2961a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 2962a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 2963a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 2964a4c15cd9SJenny Derzhavetz break; 2965b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2966b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2967b8d26b3bSNicholas Bellinger break; 2968b8d26b3bSNicholas Bellinger default: 296924f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2970b8d26b3bSNicholas Bellinger ret = -EINVAL; 2971b8d26b3bSNicholas Bellinger break; 2972b8d26b3bSNicholas Bellinger } 2973b8d26b3bSNicholas Bellinger 2974b8d26b3bSNicholas Bellinger return ret; 2975b8d26b3bSNicholas Bellinger } 2976b8d26b3bSNicholas Bellinger 2977b8d26b3bSNicholas Bellinger static int 2978b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2979b8d26b3bSNicholas Bellinger { 2980991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2981b8d26b3bSNicholas Bellinger int ret; 2982b8d26b3bSNicholas Bellinger 2983b8d26b3bSNicholas Bellinger switch (state) { 2984b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2985b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2986991bb764SSagi Grimberg if (!ret) 2987991bb764SSagi Grimberg isert_conn->logout_posted = true; 2988b8d26b3bSNicholas Bellinger break; 2989b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2990b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2991b8d26b3bSNicholas Bellinger break; 2992b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2993b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2994b8d26b3bSNicholas Bellinger break; 2995b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2996b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2997b8d26b3bSNicholas Bellinger break; 2998adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2999adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 3000adb54c29SNicholas Bellinger break; 3001b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 3002b8d26b3bSNicholas Bellinger /* 3003b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 3004b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 3005b8d26b3bSNicholas Bellinger */ 3006b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 3007b8d26b3bSNicholas Bellinger break; 3008b8d26b3bSNicholas Bellinger default: 300924f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 3010b8d26b3bSNicholas Bellinger ret = -EINVAL; 3011b8d26b3bSNicholas Bellinger break; 3012b8d26b3bSNicholas Bellinger } 3013b8d26b3bSNicholas Bellinger 3014b8d26b3bSNicholas Bellinger return ret; 3015b8d26b3bSNicholas Bellinger } 3016b8d26b3bSNicholas Bellinger 3017ca6c1d82SSagi Grimberg struct rdma_cm_id * 3018ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 3019ca6c1d82SSagi Grimberg { 3020ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 3021ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 3022ca6c1d82SSagi Grimberg struct sockaddr *sa; 3023ca6c1d82SSagi Grimberg int ret; 3024ca6c1d82SSagi Grimberg 3025ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 302624f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3027ca6c1d82SSagi Grimberg 3028fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 3029ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 3030ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 303124f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3032ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 3033ca6c1d82SSagi Grimberg goto out; 3034ca6c1d82SSagi Grimberg } 303524f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 3036ca6c1d82SSagi Grimberg 3037ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 3038ca6c1d82SSagi Grimberg if (ret) { 303924f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 3040ca6c1d82SSagi Grimberg goto out_id; 3041ca6c1d82SSagi Grimberg } 3042ca6c1d82SSagi Grimberg 3043992607e8SSagi Grimberg ret = rdma_listen(id, 0); 3044ca6c1d82SSagi Grimberg if (ret) { 304524f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 3046ca6c1d82SSagi Grimberg goto out_id; 3047ca6c1d82SSagi Grimberg } 3048ca6c1d82SSagi Grimberg 3049ca6c1d82SSagi Grimberg return id; 3050ca6c1d82SSagi Grimberg out_id: 3051ca6c1d82SSagi Grimberg rdma_destroy_id(id); 3052ca6c1d82SSagi Grimberg out: 3053ca6c1d82SSagi Grimberg return ERR_PTR(ret); 3054ca6c1d82SSagi Grimberg } 3055ca6c1d82SSagi Grimberg 3056b8d26b3bSNicholas Bellinger static int 3057b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 305813a3cf08SAndy Grover struct sockaddr_storage *ksockaddr) 3059b8d26b3bSNicholas Bellinger { 3060b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 3061b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 3062b8d26b3bSNicholas Bellinger int ret; 3063b8d26b3bSNicholas Bellinger 3064b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3065b8d26b3bSNicholas Bellinger if (!isert_np) { 306624f412ddSSagi Grimberg isert_err("Unable to allocate struct isert_np\n"); 3067b8d26b3bSNicholas Bellinger return -ENOMEM; 3068b8d26b3bSNicholas Bellinger } 3069ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0); 3070ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex); 3071bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted); 3072bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending); 3073ca6c1d82SSagi Grimberg isert_np->np = np; 3074b8d26b3bSNicholas Bellinger 3075b8d26b3bSNicholas Bellinger /* 3076b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 3077b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 3078b8d26b3bSNicholas Bellinger */ 3079b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 308013a3cf08SAndy Grover sizeof(struct sockaddr_storage)); 3081b8d26b3bSNicholas Bellinger 3082ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 3083b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 3084b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 3085b8d26b3bSNicholas Bellinger goto out; 3086b8d26b3bSNicholas Bellinger } 3087b8d26b3bSNicholas Bellinger 3088ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid; 3089b8d26b3bSNicholas Bellinger np->np_context = isert_np; 3090b8d26b3bSNicholas Bellinger 3091b8d26b3bSNicholas Bellinger return 0; 3092b8d26b3bSNicholas Bellinger 3093b8d26b3bSNicholas Bellinger out: 3094b8d26b3bSNicholas Bellinger kfree(isert_np); 3095ca6c1d82SSagi Grimberg 3096b8d26b3bSNicholas Bellinger return ret; 3097b8d26b3bSNicholas Bellinger } 3098b8d26b3bSNicholas Bellinger 3099b8d26b3bSNicholas Bellinger static int 3100b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 3101b8d26b3bSNicholas Bellinger { 3102dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 3103b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 3104b8d26b3bSNicholas Bellinger int ret; 310513bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr; 3106b8d26b3bSNicholas Bellinger 3107b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 3108b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 3109b8d26b3bSNicholas Bellinger cp.retry_count = 7; 3110b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 3111b8d26b3bSNicholas Bellinger 311213bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 3113*422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 3114*422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv) 3115*422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 311613bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr; 311713bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr); 311813bce482SJenny Derzhavetz 3119b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 3120b8d26b3bSNicholas Bellinger if (ret) { 312124f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 3122b8d26b3bSNicholas Bellinger return ret; 3123b8d26b3bSNicholas Bellinger } 3124b8d26b3bSNicholas Bellinger 3125b8d26b3bSNicholas Bellinger return 0; 3126b8d26b3bSNicholas Bellinger } 3127b8d26b3bSNicholas Bellinger 3128b8d26b3bSNicholas Bellinger static int 3129b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3130b8d26b3bSNicholas Bellinger { 31316700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 3132b8d26b3bSNicholas Bellinger int ret; 3133b8d26b3bSNicholas Bellinger 313424f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 31352371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 31362371e5daSSagi Grimberg if (ret) { 313724f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 31382371e5daSSagi Grimberg isert_conn); 31392371e5daSSagi Grimberg return ret; 31402371e5daSSagi Grimberg } 31412371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 31422371e5daSSagi Grimberg 31436faaa85fSNicholas Bellinger /* 31446faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 31456faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 31466faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 31476faaa85fSNicholas Bellinger * into a NOP. 31486faaa85fSNicholas Bellinger */ 31496faaa85fSNicholas Bellinger if (!login->first_request) 31506faaa85fSNicholas Bellinger return 0; 3151b8d26b3bSNicholas Bellinger 31522371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 31532371e5daSSagi Grimberg 3154dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn); 3155dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3156b8d26b3bSNicholas Bellinger if (ret) 3157b8d26b3bSNicholas Bellinger return ret; 3158b8d26b3bSNicholas Bellinger 315924f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 31602371e5daSSagi Grimberg 3161b8d26b3bSNicholas Bellinger return 0; 3162b8d26b3bSNicholas Bellinger } 3163b8d26b3bSNicholas Bellinger 3164b8d26b3bSNicholas Bellinger static void 3165b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3166b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 3167b8d26b3bSNicholas Bellinger { 3168dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 3169b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 3170b8d26b3bSNicholas Bellinger 3171b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 3172b8d26b3bSNicholas Bellinger 3173dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr; 3174dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr; 3175b8d26b3bSNicholas Bellinger } 3176b8d26b3bSNicholas Bellinger 3177b8d26b3bSNicholas Bellinger static int 3178b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3179b8d26b3bSNicholas Bellinger { 31806700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 3181b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 3182c6b8e918SSagi Grimberg int ret; 3183b8d26b3bSNicholas Bellinger 3184b8d26b3bSNicholas Bellinger accept_wait: 3185ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem); 3186c6b8e918SSagi Grimberg if (ret) 3187b8d26b3bSNicholas Bellinger return -ENODEV; 3188b8d26b3bSNicholas Bellinger 3189b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 3190e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3191b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 319211378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n", 3193e346ab34SSagi Grimberg np->np_thread_state); 3194e346ab34SSagi Grimberg /** 3195e346ab34SSagi Grimberg * No point in stalling here when np_thread 3196e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 3197e346ab34SSagi Grimberg **/ 3198b8d26b3bSNicholas Bellinger return -ENODEV; 3199b8d26b3bSNicholas Bellinger } 3200b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3201b8d26b3bSNicholas Bellinger 3202ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 3203bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) { 3204ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 3205b8d26b3bSNicholas Bellinger goto accept_wait; 3206b8d26b3bSNicholas Bellinger } 3207bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending, 3208bd379220SJenny Derzhavetz struct isert_conn, node); 3209bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 3210ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 3211b8d26b3bSNicholas Bellinger 3212b8d26b3bSNicholas Bellinger conn->context = isert_conn; 3213b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 3214b8d26b3bSNicholas Bellinger 3215b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 3216b8d26b3bSNicholas Bellinger 321724f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 32182371e5daSSagi Grimberg 3219b8d26b3bSNicholas Bellinger return 0; 3220b8d26b3bSNicholas Bellinger } 3221b8d26b3bSNicholas Bellinger 3222b8d26b3bSNicholas Bellinger static void 3223b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 3224b8d26b3bSNicholas Bellinger { 32256700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 3226268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 3227b8d26b3bSNicholas Bellinger 3228ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id) 3229ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id); 3230b8d26b3bSNicholas Bellinger 3231268e6811SSagi Grimberg /* 3232268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 3233268e6811SSagi Grimberg * that at this point we don't have hanging connections that 3234268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 3235268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 3236bd379220SJenny Derzhavetz * up in accepted and pending lists. 3237268e6811SSagi Grimberg */ 3238ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 3239bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) { 3240bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n"); 3241268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 3242bd379220SJenny Derzhavetz &isert_np->pending, 3243bd379220SJenny Derzhavetz node) { 3244bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n", 3245bd379220SJenny Derzhavetz isert_conn, isert_conn->state); 3246bd379220SJenny Derzhavetz isert_connect_release(isert_conn); 3247bd379220SJenny Derzhavetz } 3248bd379220SJenny Derzhavetz } 3249bd379220SJenny Derzhavetz 3250bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) { 3251bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n"); 3252bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n, 3253bd379220SJenny Derzhavetz &isert_np->accepted, 3254bd379220SJenny Derzhavetz node) { 325524f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 3256268e6811SSagi Grimberg isert_conn, isert_conn->state); 3257268e6811SSagi Grimberg isert_connect_release(isert_conn); 3258268e6811SSagi Grimberg } 3259268e6811SSagi Grimberg } 3260ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 3261268e6811SSagi Grimberg 3262b8d26b3bSNicholas Bellinger np->np_context = NULL; 3263b8d26b3bSNicholas Bellinger kfree(isert_np); 3264b8d26b3bSNicholas Bellinger } 3265b8d26b3bSNicholas Bellinger 3266b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 3267b02efbfcSSagi Grimberg { 3268b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 3269b02efbfcSSagi Grimberg struct isert_conn, 3270b02efbfcSSagi Grimberg release_work); 3271b02efbfcSSagi Grimberg 327224f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 3273b02efbfcSSagi Grimberg 3274dac6ab30SSagi Grimberg wait_for_completion(&isert_conn->wait); 3275b02efbfcSSagi Grimberg 3276dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 3277b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 3278dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 3279b02efbfcSSagi Grimberg 328024f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 3281b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 3282b02efbfcSSagi Grimberg } 3283b02efbfcSSagi Grimberg 3284bdf20e72SSagi Grimberg static void 3285991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 3286991bb764SSagi Grimberg { 3287991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 3288991bb764SSagi Grimberg 32894c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 32904c22e07fSSagi Grimberg 3291991bb764SSagi Grimberg if (isert_conn->logout_posted) { 329224f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3293991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 3294991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 3295991bb764SSagi Grimberg } 3296991bb764SSagi Grimberg } 3297991bb764SSagi Grimberg 3298991bb764SSagi Grimberg static void 3299c7e160eeSSagi Grimberg isert_wait4cmds(struct iscsi_conn *conn) 3300c7e160eeSSagi Grimberg { 33014c22e07fSSagi Grimberg isert_info("iscsi_conn %p\n", conn); 33024c22e07fSSagi Grimberg 3303c7e160eeSSagi Grimberg if (conn->sess) { 3304c7e160eeSSagi Grimberg target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3305c7e160eeSSagi Grimberg target_wait_for_sess_cmds(conn->sess->se_sess); 3306c7e160eeSSagi Grimberg } 3307c7e160eeSSagi Grimberg } 3308c7e160eeSSagi Grimberg 3309c7e160eeSSagi Grimberg static void 3310bdf20e72SSagi Grimberg isert_wait4flush(struct isert_conn *isert_conn) 3311bdf20e72SSagi Grimberg { 3312bdf20e72SSagi Grimberg struct ib_recv_wr *bad_wr; 3313bdf20e72SSagi Grimberg 33144c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 33154c22e07fSSagi Grimberg 3316dac6ab30SSagi Grimberg init_completion(&isert_conn->wait_comp_err); 3317bdf20e72SSagi Grimberg isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3318bdf20e72SSagi Grimberg /* post an indication that all flush errors were consumed */ 3319dac6ab30SSagi Grimberg if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { 332024f412ddSSagi Grimberg isert_err("conn %p failed to post beacon", isert_conn); 3321bdf20e72SSagi Grimberg return; 3322bdf20e72SSagi Grimberg } 3323bdf20e72SSagi Grimberg 3324dac6ab30SSagi Grimberg wait_for_completion(&isert_conn->wait_comp_err); 3325bdf20e72SSagi Grimberg } 3326bdf20e72SSagi Grimberg 33273e03c4b0SJenny Derzhavetz /** 33283e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for 33293e03c4b0SJenny Derzhavetz * unsolicitate dataout 33303e03c4b0SJenny Derzhavetz * @conn: iscsi connection 33313e03c4b0SJenny Derzhavetz * 33323e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited 33333e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those 33343e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds 33353e03c4b0SJenny Derzhavetz */ 33363e03c4b0SJenny Derzhavetz static void 33373e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 33383e03c4b0SJenny Derzhavetz { 33393e03c4b0SJenny Derzhavetz struct iscsi_cmd *cmd, *tmp; 33403e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list); 33413e03c4b0SJenny Derzhavetz 33423e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 33433e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 33443e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 33453e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 33463e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length)) 33473e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 33483e03c4b0SJenny Derzhavetz } 33493e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 33503e03c4b0SJenny Derzhavetz 33513e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 33523e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 33533e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) { 33543e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 33553e03c4b0SJenny Derzhavetz 33563e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd); 33573e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 33583e03c4b0SJenny Derzhavetz } 33593e03c4b0SJenny Derzhavetz } 33603e03c4b0SJenny Derzhavetz } 33613e03c4b0SJenny Derzhavetz 3362defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 3363b8d26b3bSNicholas Bellinger { 3364b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3365b8d26b3bSNicholas Bellinger 33664c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn); 3367b8d26b3bSNicholas Bellinger 3368dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 3369b8d26b3bSNicholas Bellinger /* 3370dac6ab30SSagi Grimberg * Only wait for wait_comp_err if the isert_conn made it 3371b8d26b3bSNicholas Bellinger * into full feature phase.. 3372b8d26b3bSNicholas Bellinger */ 3373b2cb9649SNicholas Bellinger if (isert_conn->state == ISER_CONN_INIT) { 3374dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 3375b2cb9649SNicholas Bellinger return; 3376b2cb9649SNicholas Bellinger } 3377954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 3378dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 3379b2cb9649SNicholas Bellinger 3380bdf20e72SSagi Grimberg isert_wait4flush(isert_conn); 33813e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn); 33823e03c4b0SJenny Derzhavetz isert_wait4cmds(conn); 3383991bb764SSagi Grimberg isert_wait4logout(isert_conn); 3384954f2372SSagi Grimberg 3385b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 3386defd8848SNicholas Bellinger } 3387defd8848SNicholas Bellinger 3388defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 3389defd8848SNicholas Bellinger { 3390defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3391b8d26b3bSNicholas Bellinger 3392524630d5SSagi Grimberg isert_wait4flush(isert_conn); 3393b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 3394b8d26b3bSNicholas Bellinger } 3395b8d26b3bSNicholas Bellinger 3396b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 3397b8d26b3bSNicholas Bellinger .name = "IB/iSER", 3398b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 3399d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 3400b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 3401b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 3402b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 3403b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 3404defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 3405b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 3406b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 3407b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 3408b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 3409b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 3410b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 3411b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 3412b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 3413131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 3414e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3415b8d26b3bSNicholas Bellinger }; 3416b8d26b3bSNicholas Bellinger 3417b8d26b3bSNicholas Bellinger static int __init isert_init(void) 3418b8d26b3bSNicholas Bellinger { 3419b8d26b3bSNicholas Bellinger int ret; 3420b8d26b3bSNicholas Bellinger 3421631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq", 3422631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0); 3423b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 342424f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 3425b8d26b3bSNicholas Bellinger ret = -ENOMEM; 34266f0fae3dSSagi Grimberg return -ENOMEM; 3427b8d26b3bSNicholas Bellinger } 3428b8d26b3bSNicholas Bellinger 3429b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3430b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 3431b02efbfcSSagi Grimberg if (!isert_release_wq) { 343224f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 3433b02efbfcSSagi Grimberg ret = -ENOMEM; 3434b02efbfcSSagi Grimberg goto destroy_comp_wq; 3435b02efbfcSSagi Grimberg } 3436b02efbfcSSagi Grimberg 3437b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 343824f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3439b02efbfcSSagi Grimberg 3440b8d26b3bSNicholas Bellinger return 0; 3441b8d26b3bSNicholas Bellinger 3442b02efbfcSSagi Grimberg destroy_comp_wq: 3443b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 34446f0fae3dSSagi Grimberg 3445b8d26b3bSNicholas Bellinger return ret; 3446b8d26b3bSNicholas Bellinger } 3447b8d26b3bSNicholas Bellinger 3448b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 3449b8d26b3bSNicholas Bellinger { 3450f5ebec96SSagi Grimberg flush_scheduled_work(); 3451b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 3452b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 3453b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 34544c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3455b8d26b3bSNicholas Bellinger } 3456b8d26b3bSNicholas Bellinger 3457b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 34589e35eff4SSagi Grimberg MODULE_VERSION("1.0"); 3459b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3460b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 3461b8d26b3bSNicholas Bellinger 3462b8d26b3bSNicholas Bellinger module_init(isert_init); 3463b8d26b3bSNicholas Bellinger module_exit(isert_exit); 3464