1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 44c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 25b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 26b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 27b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 29b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 30531b7bf4SSagi Grimberg #include <linux/semaphore.h> 31b8d26b3bSNicholas Bellinger 32b8d26b3bSNicholas Bellinger #include "ib_isert.h" 33b8d26b3bSNicholas Bellinger 34b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 35b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36b8d26b3bSNicholas Bellinger #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 37bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 38bdf20e72SSagi Grimberg ISERT_MAX_CONN) 39b8d26b3bSNicholas Bellinger 4045678b6bSSagi Grimberg static int isert_debug_level; 4124f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 4224f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 4324f412ddSSagi Grimberg 44b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 45b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 46b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 47b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 48b8d26b3bSNicholas Bellinger 49b8d26b3bSNicholas Bellinger static void 50d40945d8SVu Pham isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51d40945d8SVu Pham static int 52d1ca2ed7SChristoph Hellwig isert_map_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn); 5359464ef4SVu Pham static void 54a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 5559464ef4SVu Pham static int 56d1ca2ed7SChristoph Hellwig isert_reg_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn); 57f93f3a70SSagi Grimberg static int 58f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 592371e5daSSagi Grimberg static int 609679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn); 612371e5daSSagi Grimberg static int 622371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 63ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 64d40945d8SVu Pham 652f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work); 669679cc51SChristoph Hellwig static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 679679cc51SChristoph Hellwig static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 689679cc51SChristoph Hellwig static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 699679cc51SChristoph Hellwig static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 702f1b6b7dSSagi Grimberg 71302cc7c3SSagi Grimberg static inline bool 72302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 73302cc7c3SSagi Grimberg { 7423a548eeSSagi Grimberg return (conn->pi_support && 75302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 76302cc7c3SSagi Grimberg } 77302cc7c3SSagi Grimberg 78302cc7c3SSagi Grimberg 79d40945d8SVu Pham static void 80b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 81b8d26b3bSNicholas Bellinger { 826700425eSSagi Grimberg struct isert_conn *isert_conn = context; 83b8d26b3bSNicholas Bellinger 84ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n", 85ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn); 86ea8a1616SSagi Grimberg 87b8d26b3bSNicholas Bellinger switch (e->event) { 88b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 89dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 90b8d26b3bSNicholas Bellinger break; 91b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 924c22e07fSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 93b8d26b3bSNicholas Bellinger break; 94b8d26b3bSNicholas Bellinger default: 95b8d26b3bSNicholas Bellinger break; 96b8d26b3bSNicholas Bellinger } 97b8d26b3bSNicholas Bellinger } 98b8d26b3bSNicholas Bellinger 9940fc069aSSagi Grimberg static struct isert_comp * 10040fc069aSSagi Grimberg isert_comp_get(struct isert_conn *isert_conn) 101b8d26b3bSNicholas Bellinger { 102dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1034a295baeSSagi Grimberg struct isert_comp *comp; 10440fc069aSSagi Grimberg int i, min = 0; 105b8d26b3bSNicholas Bellinger 106b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 1074a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 1084a295baeSSagi Grimberg if (device->comps[i].active_qps < 1094a295baeSSagi Grimberg device->comps[min].active_qps) 1104a295baeSSagi Grimberg min = i; 1114a295baeSSagi Grimberg comp = &device->comps[min]; 1124a295baeSSagi Grimberg comp->active_qps++; 11340fc069aSSagi Grimberg mutex_unlock(&device_list_mutex); 11440fc069aSSagi Grimberg 11524f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1164a295baeSSagi Grimberg isert_conn, comp, min); 11740fc069aSSagi Grimberg 11840fc069aSSagi Grimberg return comp; 11940fc069aSSagi Grimberg } 12040fc069aSSagi Grimberg 12140fc069aSSagi Grimberg static void 12240fc069aSSagi Grimberg isert_comp_put(struct isert_comp *comp) 12340fc069aSSagi Grimberg { 12440fc069aSSagi Grimberg mutex_lock(&device_list_mutex); 12540fc069aSSagi Grimberg comp->active_qps--; 126b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 12740fc069aSSagi Grimberg } 12840fc069aSSagi Grimberg 12940fc069aSSagi Grimberg static struct ib_qp * 13040fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn, 13140fc069aSSagi Grimberg struct isert_comp *comp, 13240fc069aSSagi Grimberg struct rdma_cm_id *cma_id) 13340fc069aSSagi Grimberg { 134dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 13540fc069aSSagi Grimberg struct ib_qp_init_attr attr; 13640fc069aSSagi Grimberg int ret; 137b8d26b3bSNicholas Bellinger 138b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 139b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 140b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1416f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1426f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 143572a1434SSagi Grimberg attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 144bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 1454a061b28SOr Gerlitz attr.cap.max_send_sge = device->ib_device->attrs.max_sge; 1464a061b28SOr Gerlitz isert_conn->max_sge = min(device->ib_device->attrs.max_sge, 1474a061b28SOr Gerlitz device->ib_device->attrs.max_sge_rd); 148b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 149b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 150b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 151570db170SSagi Grimberg if (device->pi_capable) 152d3e125daSSagi Grimberg attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 153b8d26b3bSNicholas Bellinger 15467cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr); 155b8d26b3bSNicholas Bellinger if (ret) { 15624f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 15740fc069aSSagi Grimberg return ERR_PTR(ret); 15840fc069aSSagi Grimberg } 15940fc069aSSagi Grimberg 16040fc069aSSagi Grimberg return cma_id->qp; 16140fc069aSSagi Grimberg } 16240fc069aSSagi Grimberg 16340fc069aSSagi Grimberg static int 16440fc069aSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 16540fc069aSSagi Grimberg { 16640fc069aSSagi Grimberg struct isert_comp *comp; 16740fc069aSSagi Grimberg int ret; 16840fc069aSSagi Grimberg 16940fc069aSSagi Grimberg comp = isert_comp_get(isert_conn); 170dac6ab30SSagi Grimberg isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 171dac6ab30SSagi Grimberg if (IS_ERR(isert_conn->qp)) { 172dac6ab30SSagi Grimberg ret = PTR_ERR(isert_conn->qp); 17319e2090fSSagi Grimberg goto err; 174b8d26b3bSNicholas Bellinger } 175b8d26b3bSNicholas Bellinger 176b8d26b3bSNicholas Bellinger return 0; 17719e2090fSSagi Grimberg err: 17840fc069aSSagi Grimberg isert_comp_put(comp); 17919e2090fSSagi Grimberg return ret; 180b8d26b3bSNicholas Bellinger } 181b8d26b3bSNicholas Bellinger 182b8d26b3bSNicholas Bellinger static int 183b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 184b8d26b3bSNicholas Bellinger { 185dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 18667cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 187b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 188b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 189b8d26b3bSNicholas Bellinger u64 dma_addr; 190b8d26b3bSNicholas Bellinger int i, j; 191b8d26b3bSNicholas Bellinger 192dac6ab30SSagi Grimberg isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 193b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 194dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 195b8d26b3bSNicholas Bellinger goto fail; 196b8d26b3bSNicholas Bellinger 197dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 198b8d26b3bSNicholas Bellinger 199b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 200b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 201b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 202b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 203b8d26b3bSNicholas Bellinger goto dma_map_fail; 204b8d26b3bSNicholas Bellinger 205b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 206b8d26b3bSNicholas Bellinger 207b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 208b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 209b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 21034efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey; 2119679cc51SChristoph Hellwig rx_desc->rx_cqe.done = isert_recv_done; 212b8d26b3bSNicholas Bellinger } 213b8d26b3bSNicholas Bellinger 214b8d26b3bSNicholas Bellinger return 0; 215b8d26b3bSNicholas Bellinger 216b8d26b3bSNicholas Bellinger dma_map_fail: 217dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 218b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 219b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 220b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 221b8d26b3bSNicholas Bellinger } 222dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 223dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 224b8d26b3bSNicholas Bellinger fail: 2254c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 2264c22e07fSSagi Grimberg 227b8d26b3bSNicholas Bellinger return -ENOMEM; 228b8d26b3bSNicholas Bellinger } 229b8d26b3bSNicholas Bellinger 230b8d26b3bSNicholas Bellinger static void 231b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 232b8d26b3bSNicholas Bellinger { 233dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 234b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 235b8d26b3bSNicholas Bellinger int i; 236b8d26b3bSNicholas Bellinger 237dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 238b8d26b3bSNicholas Bellinger return; 239b8d26b3bSNicholas Bellinger 240dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 241b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 242b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 243b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 244b8d26b3bSNicholas Bellinger } 245b8d26b3bSNicholas Bellinger 246dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 247dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 248b8d26b3bSNicholas Bellinger } 249b8d26b3bSNicholas Bellinger 250172369c5SSagi Grimberg static void 251172369c5SSagi Grimberg isert_free_comps(struct isert_device *device) 252b8d26b3bSNicholas Bellinger { 253172369c5SSagi Grimberg int i; 254b8d26b3bSNicholas Bellinger 255172369c5SSagi Grimberg for (i = 0; i < device->comps_used; i++) { 256172369c5SSagi Grimberg struct isert_comp *comp = &device->comps[i]; 25759464ef4SVu Pham 2589679cc51SChristoph Hellwig if (comp->cq) 2599679cc51SChristoph Hellwig ib_free_cq(comp->cq); 260172369c5SSagi Grimberg } 261172369c5SSagi Grimberg kfree(device->comps); 26259464ef4SVu Pham } 263d40945d8SVu Pham 264172369c5SSagi Grimberg static int 2654a061b28SOr Gerlitz isert_alloc_comps(struct isert_device *device) 266172369c5SSagi Grimberg { 267172369c5SSagi Grimberg int i, max_cqe, ret = 0; 268d3e125daSSagi Grimberg 2694a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2704a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 271172369c5SSagi Grimberg 27224f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 273d3e125daSSagi Grimberg "Fast registration %d pi_capable %d\n", 2744a295baeSSagi Grimberg device->comps_used, device->ib_device->name, 275d3e125daSSagi Grimberg device->ib_device->num_comp_vectors, device->use_fastreg, 276d3e125daSSagi Grimberg device->pi_capable); 2774a295baeSSagi Grimberg 2784a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2794a295baeSSagi Grimberg GFP_KERNEL); 2804a295baeSSagi Grimberg if (!device->comps) { 28124f412ddSSagi Grimberg isert_err("Unable to allocate completion contexts\n"); 282b8d26b3bSNicholas Bellinger return -ENOMEM; 283b8d26b3bSNicholas Bellinger } 284b8d26b3bSNicholas Bellinger 2854a061b28SOr Gerlitz max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 286172369c5SSagi Grimberg 2874a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2884a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 289b8d26b3bSNicholas Bellinger 2904a295baeSSagi Grimberg comp->device = device; 2919679cc51SChristoph Hellwig comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 2929679cc51SChristoph Hellwig IB_POLL_WORKQUEUE); 2936f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 294172369c5SSagi Grimberg isert_err("Unable to allocate cq\n"); 2956f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 2966f0fae3dSSagi Grimberg comp->cq = NULL; 297b8d26b3bSNicholas Bellinger goto out_cq; 29894a71110SWei Yongjun } 299b8d26b3bSNicholas Bellinger } 300b8d26b3bSNicholas Bellinger 301172369c5SSagi Grimberg return 0; 302172369c5SSagi Grimberg out_cq: 303172369c5SSagi Grimberg isert_free_comps(device); 304172369c5SSagi Grimberg return ret; 305172369c5SSagi Grimberg } 306172369c5SSagi Grimberg 307172369c5SSagi Grimberg static int 308172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device) 309172369c5SSagi Grimberg { 3104a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device; 311fd8205e8SSagi Grimberg int ret; 312172369c5SSagi Grimberg 3134a061b28SOr Gerlitz isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge); 3144a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 315172369c5SSagi Grimberg 316172369c5SSagi Grimberg /* asign function handlers */ 3174a061b28SOr Gerlitz if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 3184a061b28SOr Gerlitz ib_dev->attrs.device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 319172369c5SSagi Grimberg device->use_fastreg = 1; 320172369c5SSagi Grimberg device->reg_rdma_mem = isert_reg_rdma; 321172369c5SSagi Grimberg device->unreg_rdma_mem = isert_unreg_rdma; 322172369c5SSagi Grimberg } else { 323172369c5SSagi Grimberg device->use_fastreg = 0; 324172369c5SSagi Grimberg device->reg_rdma_mem = isert_map_rdma; 325172369c5SSagi Grimberg device->unreg_rdma_mem = isert_unmap_cmd; 326172369c5SSagi Grimberg } 327172369c5SSagi Grimberg 3284a061b28SOr Gerlitz ret = isert_alloc_comps(device); 329172369c5SSagi Grimberg if (ret) 330373a4cd7SNicholas Bellinger goto out; 331172369c5SSagi Grimberg 3324a061b28SOr Gerlitz device->pd = ib_alloc_pd(ib_dev); 33367cb3949SSagi Grimberg if (IS_ERR(device->pd)) { 33467cb3949SSagi Grimberg ret = PTR_ERR(device->pd); 33567cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n", 33667cb3949SSagi Grimberg device, ret); 33767cb3949SSagi Grimberg goto out_cq; 33867cb3949SSagi Grimberg } 33967cb3949SSagi Grimberg 340172369c5SSagi Grimberg /* Check signature cap */ 3414a061b28SOr Gerlitz device->pi_capable = ib_dev->attrs.device_cap_flags & 342172369c5SSagi Grimberg IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 34367cb3949SSagi Grimberg 344b8d26b3bSNicholas Bellinger return 0; 345b8d26b3bSNicholas Bellinger 346b8d26b3bSNicholas Bellinger out_cq: 347172369c5SSagi Grimberg isert_free_comps(device); 348373a4cd7SNicholas Bellinger out: 349373a4cd7SNicholas Bellinger if (ret > 0) 350373a4cd7SNicholas Bellinger ret = -EINVAL; 351b8d26b3bSNicholas Bellinger return ret; 352b8d26b3bSNicholas Bellinger } 353b8d26b3bSNicholas Bellinger 354b8d26b3bSNicholas Bellinger static void 355b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 356b8d26b3bSNicholas Bellinger { 35724f412ddSSagi Grimberg isert_info("device %p\n", device); 358b8d26b3bSNicholas Bellinger 35967cb3949SSagi Grimberg ib_dealloc_pd(device->pd); 360172369c5SSagi Grimberg isert_free_comps(device); 361b8d26b3bSNicholas Bellinger } 362b8d26b3bSNicholas Bellinger 363b8d26b3bSNicholas Bellinger static void 364cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device) 365b8d26b3bSNicholas Bellinger { 366b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 367b8d26b3bSNicholas Bellinger device->refcount--; 3684c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount); 369b8d26b3bSNicholas Bellinger if (!device->refcount) { 370b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 371b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 372b8d26b3bSNicholas Bellinger kfree(device); 373b8d26b3bSNicholas Bellinger } 374b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 375b8d26b3bSNicholas Bellinger } 376b8d26b3bSNicholas Bellinger 377b8d26b3bSNicholas Bellinger static struct isert_device * 378cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id) 379b8d26b3bSNicholas Bellinger { 380b8d26b3bSNicholas Bellinger struct isert_device *device; 381b8d26b3bSNicholas Bellinger int ret; 382b8d26b3bSNicholas Bellinger 383b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 384b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 385b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 386b8d26b3bSNicholas Bellinger device->refcount++; 3874c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n", 3884c22e07fSSagi Grimberg device, device->refcount); 389b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 390b8d26b3bSNicholas Bellinger return device; 391b8d26b3bSNicholas Bellinger } 392b8d26b3bSNicholas Bellinger } 393b8d26b3bSNicholas Bellinger 394b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 395b8d26b3bSNicholas Bellinger if (!device) { 396b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 397b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 398b8d26b3bSNicholas Bellinger } 399b8d26b3bSNicholas Bellinger 400b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 401b8d26b3bSNicholas Bellinger 402b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 403b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 404b8d26b3bSNicholas Bellinger if (ret) { 405b8d26b3bSNicholas Bellinger kfree(device); 406b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 407b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 408b8d26b3bSNicholas Bellinger } 409b8d26b3bSNicholas Bellinger 410b8d26b3bSNicholas Bellinger device->refcount++; 411b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 4124c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n", 4134c22e07fSSagi Grimberg device, device->refcount); 414b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 415b8d26b3bSNicholas Bellinger 416b8d26b3bSNicholas Bellinger return device; 417b8d26b3bSNicholas Bellinger } 418b8d26b3bSNicholas Bellinger 41959464ef4SVu Pham static void 420a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 42159464ef4SVu Pham { 42259464ef4SVu Pham struct fast_reg_descriptor *fr_desc, *tmp; 42359464ef4SVu Pham int i = 0; 42459464ef4SVu Pham 425dac6ab30SSagi Grimberg if (list_empty(&isert_conn->fr_pool)) 42659464ef4SVu Pham return; 42759464ef4SVu Pham 4284c22e07fSSagi Grimberg isert_info("Freeing conn %p fastreg pool", isert_conn); 42959464ef4SVu Pham 43059464ef4SVu Pham list_for_each_entry_safe(fr_desc, tmp, 431dac6ab30SSagi Grimberg &isert_conn->fr_pool, list) { 43259464ef4SVu Pham list_del(&fr_desc->list); 43359464ef4SVu Pham ib_dereg_mr(fr_desc->data_mr); 434d3e125daSSagi Grimberg if (fr_desc->pi_ctx) { 435d3e125daSSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 4368b91ffc1SSagi Grimberg ib_dereg_mr(fr_desc->pi_ctx->sig_mr); 437d3e125daSSagi Grimberg kfree(fr_desc->pi_ctx); 438d3e125daSSagi Grimberg } 43959464ef4SVu Pham kfree(fr_desc); 44059464ef4SVu Pham ++i; 44159464ef4SVu Pham } 44259464ef4SVu Pham 443dac6ab30SSagi Grimberg if (i < isert_conn->fr_pool_size) 44424f412ddSSagi Grimberg isert_warn("Pool still has %d regions registered\n", 445dac6ab30SSagi Grimberg isert_conn->fr_pool_size - i); 44659464ef4SVu Pham } 44759464ef4SVu Pham 44859464ef4SVu Pham static int 449570db170SSagi Grimberg isert_create_pi_ctx(struct fast_reg_descriptor *desc, 450570db170SSagi Grimberg struct ib_device *device, 451570db170SSagi Grimberg struct ib_pd *pd) 452570db170SSagi Grimberg { 453570db170SSagi Grimberg struct pi_context *pi_ctx; 454570db170SSagi Grimberg int ret; 455570db170SSagi Grimberg 456570db170SSagi Grimberg pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 457570db170SSagi Grimberg if (!pi_ctx) { 45824f412ddSSagi Grimberg isert_err("Failed to allocate pi context\n"); 459570db170SSagi Grimberg return -ENOMEM; 460570db170SSagi Grimberg } 461570db170SSagi Grimberg 462a89be2ccSSagi Grimberg pi_ctx->prot_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 463a89be2ccSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 464570db170SSagi Grimberg if (IS_ERR(pi_ctx->prot_mr)) { 46524f412ddSSagi Grimberg isert_err("Failed to allocate prot frmr err=%ld\n", 466570db170SSagi Grimberg PTR_ERR(pi_ctx->prot_mr)); 467570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->prot_mr); 46816c2d702SSagi Grimberg goto err_pi_ctx; 469570db170SSagi Grimberg } 470570db170SSagi Grimberg desc->ind |= ISERT_PROT_KEY_VALID; 471570db170SSagi Grimberg 4729bee178bSSagi Grimberg pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2); 473570db170SSagi Grimberg if (IS_ERR(pi_ctx->sig_mr)) { 47424f412ddSSagi Grimberg isert_err("Failed to allocate signature enabled mr err=%ld\n", 475570db170SSagi Grimberg PTR_ERR(pi_ctx->sig_mr)); 476570db170SSagi Grimberg ret = PTR_ERR(pi_ctx->sig_mr); 477570db170SSagi Grimberg goto err_prot_mr; 478570db170SSagi Grimberg } 479570db170SSagi Grimberg 480570db170SSagi Grimberg desc->pi_ctx = pi_ctx; 481570db170SSagi Grimberg desc->ind |= ISERT_SIG_KEY_VALID; 482570db170SSagi Grimberg desc->ind &= ~ISERT_PROTECTED; 483570db170SSagi Grimberg 484570db170SSagi Grimberg return 0; 485570db170SSagi Grimberg 486570db170SSagi Grimberg err_prot_mr: 487b2feda4fSRoland Dreier ib_dereg_mr(pi_ctx->prot_mr); 488570db170SSagi Grimberg err_pi_ctx: 489b2feda4fSRoland Dreier kfree(pi_ctx); 490570db170SSagi Grimberg 491570db170SSagi Grimberg return ret; 492570db170SSagi Grimberg } 493570db170SSagi Grimberg 494570db170SSagi Grimberg static int 495dc87a90fSSagi Grimberg isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 496570db170SSagi Grimberg struct fast_reg_descriptor *fr_desc) 497dc87a90fSSagi Grimberg { 498a89be2ccSSagi Grimberg fr_desc->data_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 499a89be2ccSSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 500dc87a90fSSagi Grimberg if (IS_ERR(fr_desc->data_mr)) { 50124f412ddSSagi Grimberg isert_err("Failed to allocate data frmr err=%ld\n", 502dc87a90fSSagi Grimberg PTR_ERR(fr_desc->data_mr)); 50316c2d702SSagi Grimberg return PTR_ERR(fr_desc->data_mr); 504dc87a90fSSagi Grimberg } 505d3e125daSSagi Grimberg fr_desc->ind |= ISERT_DATA_KEY_VALID; 506dc87a90fSSagi Grimberg 50724f412ddSSagi Grimberg isert_dbg("Created fr_desc %p\n", fr_desc); 508dc87a90fSSagi Grimberg 509dc87a90fSSagi Grimberg return 0; 510dc87a90fSSagi Grimberg } 511dc87a90fSSagi Grimberg 512dc87a90fSSagi Grimberg static int 513570db170SSagi Grimberg isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 51459464ef4SVu Pham { 51559464ef4SVu Pham struct fast_reg_descriptor *fr_desc; 516dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 517f46d6a8aSNicholas Bellinger struct se_session *se_sess = isert_conn->conn->sess->se_sess; 518f46d6a8aSNicholas Bellinger struct se_node_acl *se_nacl = se_sess->se_node_acl; 519f46d6a8aSNicholas Bellinger int i, ret, tag_num; 520f46d6a8aSNicholas Bellinger /* 521f46d6a8aSNicholas Bellinger * Setup the number of FRMRs based upon the number of tags 522f46d6a8aSNicholas Bellinger * available to session in iscsi_target_locate_portal(). 523f46d6a8aSNicholas Bellinger */ 524f46d6a8aSNicholas Bellinger tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 525f46d6a8aSNicholas Bellinger tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 52659464ef4SVu Pham 527dac6ab30SSagi Grimberg isert_conn->fr_pool_size = 0; 528f46d6a8aSNicholas Bellinger for (i = 0; i < tag_num; i++) { 52959464ef4SVu Pham fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 53059464ef4SVu Pham if (!fr_desc) { 53124f412ddSSagi Grimberg isert_err("Failed to allocate fast_reg descriptor\n"); 53259464ef4SVu Pham ret = -ENOMEM; 53359464ef4SVu Pham goto err; 53459464ef4SVu Pham } 53559464ef4SVu Pham 536dc87a90fSSagi Grimberg ret = isert_create_fr_desc(device->ib_device, 53767cb3949SSagi Grimberg device->pd, fr_desc); 538dc87a90fSSagi Grimberg if (ret) { 53924f412ddSSagi Grimberg isert_err("Failed to create fastreg descriptor err=%d\n", 540dc87a90fSSagi Grimberg ret); 541a80e21b3SNicholas Bellinger kfree(fr_desc); 54259464ef4SVu Pham goto err; 54359464ef4SVu Pham } 54459464ef4SVu Pham 545dac6ab30SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 546dac6ab30SSagi Grimberg isert_conn->fr_pool_size++; 54759464ef4SVu Pham } 54859464ef4SVu Pham 54924f412ddSSagi Grimberg isert_dbg("Creating conn %p fastreg pool size=%d", 550dac6ab30SSagi Grimberg isert_conn, isert_conn->fr_pool_size); 55159464ef4SVu Pham 55259464ef4SVu Pham return 0; 55359464ef4SVu Pham 55459464ef4SVu Pham err: 555a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 55659464ef4SVu Pham return ret; 55759464ef4SVu Pham } 55859464ef4SVu Pham 559ae9ea9edSSagi Grimberg static void 560ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn) 561ae9ea9edSSagi Grimberg { 562ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT; 563bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node); 564dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp); 565ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp); 566dac6ab30SSagi Grimberg kref_init(&isert_conn->kref); 567dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex); 568dac6ab30SSagi Grimberg spin_lock_init(&isert_conn->pool_lock); 569dac6ab30SSagi Grimberg INIT_LIST_HEAD(&isert_conn->fr_pool); 5702f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 571ae9ea9edSSagi Grimberg } 572ae9ea9edSSagi Grimberg 573ae9ea9edSSagi Grimberg static void 574ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn) 575ae9ea9edSSagi Grimberg { 576dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 577ae9ea9edSSagi Grimberg 578ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 579ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 5805adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 5815adabdd1SChristoph Hellwig 582ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 5835adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, 584ae9ea9edSSagi Grimberg DMA_FROM_DEVICE); 5855adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 586ae9ea9edSSagi Grimberg } 587ae9ea9edSSagi Grimberg 588ae9ea9edSSagi Grimberg static int 589ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn, 590ae9ea9edSSagi Grimberg struct ib_device *ib_dev) 591ae9ea9edSSagi Grimberg { 592ae9ea9edSSagi Grimberg int ret; 593ae9ea9edSSagi Grimberg 5945adabdd1SChristoph Hellwig isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 5955adabdd1SChristoph Hellwig GFP_KERNEL); 5965adabdd1SChristoph Hellwig if (!isert_conn->login_req_buf) { 597ae9ea9edSSagi Grimberg isert_err("Unable to allocate isert_conn->login_buf\n"); 598ae9ea9edSSagi Grimberg return -ENOMEM; 599ae9ea9edSSagi Grimberg } 600ae9ea9edSSagi Grimberg 601ae9ea9edSSagi Grimberg isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 6025adabdd1SChristoph Hellwig isert_conn->login_req_buf, 6035adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 604ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 605ae9ea9edSSagi Grimberg if (ret) { 606ae9ea9edSSagi Grimberg isert_err("login_req_dma mapping error: %d\n", ret); 607ae9ea9edSSagi Grimberg isert_conn->login_req_dma = 0; 6085adabdd1SChristoph Hellwig goto out_free_login_req_buf; 6095adabdd1SChristoph Hellwig } 6105adabdd1SChristoph Hellwig 6115adabdd1SChristoph Hellwig isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 6125adabdd1SChristoph Hellwig if (!isert_conn->login_rsp_buf) { 6135adabdd1SChristoph Hellwig isert_err("Unable to allocate isert_conn->login_rspbuf\n"); 6145adabdd1SChristoph Hellwig goto out_unmap_login_req_buf; 615ae9ea9edSSagi Grimberg } 616ae9ea9edSSagi Grimberg 617ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 6185adabdd1SChristoph Hellwig isert_conn->login_rsp_buf, 619ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 620ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 621ae9ea9edSSagi Grimberg if (ret) { 622ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret); 623ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0; 6245adabdd1SChristoph Hellwig goto out_free_login_rsp_buf; 625ae9ea9edSSagi Grimberg } 626ae9ea9edSSagi Grimberg 627ae9ea9edSSagi Grimberg return 0; 628ae9ea9edSSagi Grimberg 6295adabdd1SChristoph Hellwig out_free_login_rsp_buf: 6305adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 6315adabdd1SChristoph Hellwig out_unmap_login_req_buf: 632ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 6335adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 6345adabdd1SChristoph Hellwig out_free_login_req_buf: 6355adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 636ae9ea9edSSagi Grimberg return ret; 637ae9ea9edSSagi Grimberg } 638ae9ea9edSSagi Grimberg 639422bd0acSJenny Derzhavetz static void 640422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn, 641422bd0acSJenny Derzhavetz struct rdma_conn_param *param) 642422bd0acSJenny Derzhavetz { 643422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 644422bd0acSJenny Derzhavetz 645422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */ 646422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 647422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom); 648422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 649422bd0acSJenny Derzhavetz 650422bd0acSJenny Derzhavetz if (param->private_data) { 651422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data; 652422bd0acSJenny Derzhavetz 653422bd0acSJenny Derzhavetz /* 654422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator 655422bd0acSJenny Derzhavetz * and the HCA support it 656422bd0acSJenny Derzhavetz */ 657422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 658422bd0acSJenny Derzhavetz (attr->device_cap_flags & 659422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS); 660422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv) 661422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n"); 662422bd0acSJenny Derzhavetz } 663422bd0acSJenny Derzhavetz } 664422bd0acSJenny Derzhavetz 665b8d26b3bSNicholas Bellinger static int 666b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 667b8d26b3bSNicholas Bellinger { 668ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 669ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 670b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 671b8d26b3bSNicholas Bellinger struct isert_device *device; 672b8d26b3bSNicholas Bellinger int ret = 0; 67314f4b54fSSagi Grimberg 67414f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 67514f4b54fSSagi Grimberg if (!np->enabled) { 67614f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 67724f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 67814f4b54fSSagi Grimberg return rdma_reject(cma_id, NULL, 0); 67914f4b54fSSagi Grimberg } 68014f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 681b8d26b3bSNicholas Bellinger 6824c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n", 683b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 684b8d26b3bSNicholas Bellinger 685b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 686ae9ea9edSSagi Grimberg if (!isert_conn) 687b8d26b3bSNicholas Bellinger return -ENOMEM; 688b8d26b3bSNicholas Bellinger 689ae9ea9edSSagi Grimberg isert_init_conn(isert_conn); 690dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id; 691b8d26b3bSNicholas Bellinger 692ae9ea9edSSagi Grimberg ret = isert_alloc_login_buf(isert_conn, cma_id->device); 693ae9ea9edSSagi Grimberg if (ret) 694b8d26b3bSNicholas Bellinger goto out; 695b8d26b3bSNicholas Bellinger 696cf8ae958SSagi Grimberg device = isert_device_get(cma_id); 697b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 698b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 699b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 700b8d26b3bSNicholas Bellinger } 701dac6ab30SSagi Grimberg isert_conn->device = device; 702b8d26b3bSNicholas Bellinger 703422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn); 7041a92e17eSSagi Grimberg 705570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 706b8d26b3bSNicholas Bellinger if (ret) 707b8d26b3bSNicholas Bellinger goto out_conn_dev; 708b8d26b3bSNicholas Bellinger 7099679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 7102371e5daSSagi Grimberg if (ret) 7112371e5daSSagi Grimberg goto out_conn_dev; 7122371e5daSSagi Grimberg 7132371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 7142371e5daSSagi Grimberg if (ret) 7152371e5daSSagi Grimberg goto out_conn_dev; 7162371e5daSSagi Grimberg 717ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 718bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted); 719ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 720b8d26b3bSNicholas Bellinger 721b8d26b3bSNicholas Bellinger return 0; 722b8d26b3bSNicholas Bellinger 723b8d26b3bSNicholas Bellinger out_conn_dev: 724cf8ae958SSagi Grimberg isert_device_put(device); 725b8d26b3bSNicholas Bellinger out_rsp_dma_map: 726ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 727b8d26b3bSNicholas Bellinger out: 728b8d26b3bSNicholas Bellinger kfree(isert_conn); 7292371e5daSSagi Grimberg rdma_reject(cma_id, NULL, 0); 730b8d26b3bSNicholas Bellinger return ret; 731b8d26b3bSNicholas Bellinger } 732b8d26b3bSNicholas Bellinger 733b8d26b3bSNicholas Bellinger static void 734b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 735b8d26b3bSNicholas Bellinger { 736dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 737b8d26b3bSNicholas Bellinger 7384c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn); 739b8d26b3bSNicholas Bellinger 74057df81e3SSagi Grimberg BUG_ON(!device); 74157df81e3SSagi Grimberg 74257df81e3SSagi Grimberg if (device->use_fastreg) 743a3a5a826SSagi Grimberg isert_conn_free_fastreg_pool(isert_conn); 74459464ef4SVu Pham 74519e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 746dac6ab30SSagi Grimberg if (isert_conn->cm_id) 747dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id); 74819e2090fSSagi Grimberg 749dac6ab30SSagi Grimberg if (isert_conn->qp) { 750dac6ab30SSagi Grimberg struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 7514a295baeSSagi Grimberg 75240fc069aSSagi Grimberg isert_comp_put(comp); 753dac6ab30SSagi Grimberg ib_destroy_qp(isert_conn->qp); 754b8d26b3bSNicholas Bellinger } 755b8d26b3bSNicholas Bellinger 7565adabdd1SChristoph Hellwig if (isert_conn->login_req_buf) 757ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 758ae9ea9edSSagi Grimberg 759cf8ae958SSagi Grimberg isert_device_put(device); 76057df81e3SSagi Grimberg 76157df81e3SSagi Grimberg kfree(isert_conn); 762b8d26b3bSNicholas Bellinger } 763b8d26b3bSNicholas Bellinger 764b8d26b3bSNicholas Bellinger static void 765b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 766b8d26b3bSNicholas Bellinger { 76719e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 768bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 769c2f88b17SSagi Grimberg 77024f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 771128e9cc8SSagi Grimberg 772dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 773128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 774bd379220SJenny Derzhavetz kref_get(&isert_conn->kref); 775dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 776bd379220SJenny Derzhavetz 777bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex); 778bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending); 779bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 780bd379220SJenny Derzhavetz 781bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np); 782bd379220SJenny Derzhavetz up(&isert_np->sem); 783b8d26b3bSNicholas Bellinger } 784b8d26b3bSNicholas Bellinger 785b8d26b3bSNicholas Bellinger static void 786dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref) 787b8d26b3bSNicholas Bellinger { 788b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 789dac6ab30SSagi Grimberg struct isert_conn, kref); 790b8d26b3bSNicholas Bellinger 7914c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 7924c22e07fSSagi Grimberg current->pid); 793b8d26b3bSNicholas Bellinger 794b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 795b8d26b3bSNicholas Bellinger } 796b8d26b3bSNicholas Bellinger 797b8d26b3bSNicholas Bellinger static void 798b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 799b8d26b3bSNicholas Bellinger { 800dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref); 801b8d26b3bSNicholas Bellinger } 802b8d26b3bSNicholas Bellinger 8036d1fba0cSJenny Derzhavetz static void 8046d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(struct isert_conn *isert_conn) 8056d1fba0cSJenny Derzhavetz { 8066d1fba0cSJenny Derzhavetz struct isert_np *isert_np = isert_conn->cm_id->context; 8076d1fba0cSJenny Derzhavetz 8086d1fba0cSJenny Derzhavetz mutex_lock(&isert_np->mutex); 8096d1fba0cSJenny Derzhavetz if (!list_empty(&isert_conn->node)) { 8106d1fba0cSJenny Derzhavetz /* 8116d1fba0cSJenny Derzhavetz * This means iscsi doesn't know this connection 8126d1fba0cSJenny Derzhavetz * so schedule a cleanup ourselves 8136d1fba0cSJenny Derzhavetz */ 8146d1fba0cSJenny Derzhavetz list_del_init(&isert_conn->node); 8156d1fba0cSJenny Derzhavetz isert_put_conn(isert_conn); 8166d1fba0cSJenny Derzhavetz queue_work(isert_release_wq, &isert_conn->release_work); 8176d1fba0cSJenny Derzhavetz } 8186d1fba0cSJenny Derzhavetz mutex_unlock(&isert_np->mutex); 8196d1fba0cSJenny Derzhavetz } 8206d1fba0cSJenny Derzhavetz 821954f2372SSagi Grimberg /** 822954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 823954f2372SSagi Grimberg * @isert_conn: isert connection struct 824954f2372SSagi Grimberg * 825954f2372SSagi Grimberg * Notes: 826aea92980SJenny Derzhavetz * In case the connection state is BOUND, move state 827954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 828128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 829954f2372SSagi Grimberg * 830dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is 831954f2372SSagi Grimberg * safe to call multiple times. 832954f2372SSagi Grimberg */ 833954f2372SSagi Grimberg static void 834954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 835954f2372SSagi Grimberg { 836954f2372SSagi Grimberg int err; 837954f2372SSagi Grimberg 8386d1fba0cSJenny Derzhavetz if (isert_conn->state >= ISER_CONN_TERMINATING) 8396d1fba0cSJenny Derzhavetz return; 8406d1fba0cSJenny Derzhavetz 84124f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 842954f2372SSagi Grimberg isert_conn, isert_conn->state); 843128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 844dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id); 845954f2372SSagi Grimberg if (err) 84624f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 847954f2372SSagi Grimberg isert_conn); 848954f2372SSagi Grimberg } 849954f2372SSagi Grimberg 8503b726ae2SSagi Grimberg static int 851ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 852ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 853b8d26b3bSNicholas Bellinger { 854ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n", 855ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np); 8563b726ae2SSagi Grimberg 857ca6c1d82SSagi Grimberg switch (event) { 858ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 859ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 860ca6c1d82SSagi Grimberg break; 861ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 862ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np); 863ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) { 86424f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 865ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id)); 866ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 867ca6c1d82SSagi Grimberg } 868ca6c1d82SSagi Grimberg break; 869ca6c1d82SSagi Grimberg default: 87024f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 871ca6c1d82SSagi Grimberg isert_np, event); 872ca6c1d82SSagi Grimberg } 873ca6c1d82SSagi Grimberg 8743b726ae2SSagi Grimberg return -1; 8753b726ae2SSagi Grimberg } 8763b726ae2SSagi Grimberg 877ca6c1d82SSagi Grimberg static int 878ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 879ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 880ca6c1d82SSagi Grimberg { 881f81bf458SJenny Derzhavetz struct isert_conn *isert_conn = cma_id->qp->qp_context; 882ca6c1d82SSagi Grimberg 883dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 8846d1fba0cSJenny Derzhavetz switch (isert_conn->state) { 8856d1fba0cSJenny Derzhavetz case ISER_CONN_TERMINATING: 8866d1fba0cSJenny Derzhavetz break; 8876d1fba0cSJenny Derzhavetz case ISER_CONN_UP: 888128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 889572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 8906d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(isert_conn); 8916d1fba0cSJenny Derzhavetz break; 8926d1fba0cSJenny Derzhavetz case ISER_CONN_BOUND: 8936d1fba0cSJenny Derzhavetz case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 8946d1fba0cSJenny Derzhavetz iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 8956d1fba0cSJenny Derzhavetz break; 8966d1fba0cSJenny Derzhavetz default: 8976d1fba0cSJenny Derzhavetz isert_warn("conn %p teminating in state %d\n", 8986d1fba0cSJenny Derzhavetz isert_conn, isert_conn->state); 8996d1fba0cSJenny Derzhavetz } 900dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 901128e9cc8SSagi Grimberg 9023b726ae2SSagi Grimberg return 0; 903b8d26b3bSNicholas Bellinger } 904b8d26b3bSNicholas Bellinger 9054a579da2SSagi Grimberg static int 906954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 907954f2372SSagi Grimberg { 90819e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 909954f2372SSagi Grimberg 910bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 911dac6ab30SSagi Grimberg isert_conn->cm_id = NULL; 912954f2372SSagi Grimberg isert_put_conn(isert_conn); 9134a579da2SSagi Grimberg 9144a579da2SSagi Grimberg return -1; 915954f2372SSagi Grimberg } 916954f2372SSagi Grimberg 917b8d26b3bSNicholas Bellinger static int 918b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 919b8d26b3bSNicholas Bellinger { 920f81bf458SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 921b8d26b3bSNicholas Bellinger int ret = 0; 922b8d26b3bSNicholas Bellinger 923ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n", 924ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event, 9254c22e07fSSagi Grimberg event->status, cma_id, cma_id->context); 926b8d26b3bSNicholas Bellinger 927f81bf458SJenny Derzhavetz if (isert_np->cm_id == cma_id) 928f81bf458SJenny Derzhavetz return isert_np_cma_handler(cma_id->context, event->event); 929f81bf458SJenny Derzhavetz 930b8d26b3bSNicholas Bellinger switch (event->event) { 931b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 932b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 9333b726ae2SSagi Grimberg if (ret) 9344c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret); 935b8d26b3bSNicholas Bellinger break; 936b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 937b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 938b8d26b3bSNicholas Bellinger break; 93988c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 94088c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 94188c4015fSSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 94288c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 943ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 944b8d26b3bSNicholas Bellinger break; 945954f2372SSagi Grimberg case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 946954f2372SSagi Grimberg case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 947b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 9484a579da2SSagi Grimberg ret = isert_connect_error(cma_id); 949954f2372SSagi Grimberg break; 950b8d26b3bSNicholas Bellinger default: 95124f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 952b8d26b3bSNicholas Bellinger break; 953b8d26b3bSNicholas Bellinger } 954b8d26b3bSNicholas Bellinger 955b8d26b3bSNicholas Bellinger return ret; 956b8d26b3bSNicholas Bellinger } 957b8d26b3bSNicholas Bellinger 958b8d26b3bSNicholas Bellinger static int 9594366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count) 960b8d26b3bSNicholas Bellinger { 961b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 962b8d26b3bSNicholas Bellinger int i, ret; 963b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 964b8d26b3bSNicholas Bellinger 965dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 9664366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i]; 9679679cc51SChristoph Hellwig 9689679cc51SChristoph Hellwig rx_wr->wr_cqe = &rx_desc->rx_cqe; 969b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 970b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 971b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 972b8d26b3bSNicholas Bellinger } 973b8d26b3bSNicholas Bellinger rx_wr--; 974b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 975b8d26b3bSNicholas Bellinger 976dac6ab30SSagi Grimberg ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 977b8d26b3bSNicholas Bellinger &rx_wr_failed); 9786d1fba0cSJenny Derzhavetz if (ret) 97924f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 9804366b19cSJenny Derzhavetz 9814366b19cSJenny Derzhavetz return ret; 9824366b19cSJenny Derzhavetz } 9834366b19cSJenny Derzhavetz 9844366b19cSJenny Derzhavetz static int 9854366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 9864366b19cSJenny Derzhavetz { 9874366b19cSJenny Derzhavetz struct ib_recv_wr *rx_wr_failed, rx_wr; 9884366b19cSJenny Derzhavetz int ret; 9894366b19cSJenny Derzhavetz 9909679cc51SChristoph Hellwig rx_wr.wr_cqe = &rx_desc->rx_cqe; 9914366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg; 9924366b19cSJenny Derzhavetz rx_wr.num_sge = 1; 9934366b19cSJenny Derzhavetz rx_wr.next = NULL; 9944366b19cSJenny Derzhavetz 9954366b19cSJenny Derzhavetz ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 9966d1fba0cSJenny Derzhavetz if (ret) 9974366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret); 9984366b19cSJenny Derzhavetz 999b8d26b3bSNicholas Bellinger return ret; 1000b8d26b3bSNicholas Bellinger } 1001b8d26b3bSNicholas Bellinger 1002b8d26b3bSNicholas Bellinger static int 10039679cc51SChristoph Hellwig isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1004b8d26b3bSNicholas Bellinger { 1005dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1006b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 1007b8d26b3bSNicholas Bellinger int ret; 1008b8d26b3bSNicholas Bellinger 1009b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1010b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1011b8d26b3bSNicholas Bellinger 10129679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_login_send_done; 10139679cc51SChristoph Hellwig 1014b8d26b3bSNicholas Bellinger send_wr.next = NULL; 10159679cc51SChristoph Hellwig send_wr.wr_cqe = &tx_desc->tx_cqe; 1016b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 1017b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 1018b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 1019b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 1020b8d26b3bSNicholas Bellinger 1021dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1022bdf20e72SSagi Grimberg if (ret) 102324f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 1024b8d26b3bSNicholas Bellinger 1025b8d26b3bSNicholas Bellinger return ret; 1026b8d26b3bSNicholas Bellinger } 1027b8d26b3bSNicholas Bellinger 1028b8d26b3bSNicholas Bellinger static void 1029b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 1030b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 1031b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1032b8d26b3bSNicholas Bellinger { 1033dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 103467cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1035b8d26b3bSNicholas Bellinger 1036b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1037b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1038b8d26b3bSNicholas Bellinger 1039d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 1040d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL; 1041b8d26b3bSNicholas Bellinger 1042b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 1043b8d26b3bSNicholas Bellinger 104434efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 104534efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 104624f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1047b8d26b3bSNicholas Bellinger } 1048b8d26b3bSNicholas Bellinger } 1049b8d26b3bSNicholas Bellinger 1050b8d26b3bSNicholas Bellinger static int 1051b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 1052b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 1053b8d26b3bSNicholas Bellinger { 1054dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 105567cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1056b8d26b3bSNicholas Bellinger u64 dma_addr; 1057b8d26b3bSNicholas Bellinger 1058b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1059b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1060b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 106124f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 1062b8d26b3bSNicholas Bellinger return -ENOMEM; 1063b8d26b3bSNicholas Bellinger } 1064b8d26b3bSNicholas Bellinger 1065b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 1066b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1067b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 106834efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1069b8d26b3bSNicholas Bellinger 10704c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 10714c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 10724c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey); 1073b8d26b3bSNicholas Bellinger 1074b8d26b3bSNicholas Bellinger return 0; 1075b8d26b3bSNicholas Bellinger } 1076b8d26b3bSNicholas Bellinger 1077b8d26b3bSNicholas Bellinger static void 107895b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 107968a86deeSSagi Grimberg struct ib_send_wr *send_wr) 1080b8d26b3bSNicholas Bellinger { 108195b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 108295b60f07SNicholas Bellinger 1083d1ca2ed7SChristoph Hellwig isert_cmd->iser_ib_op = ISER_IB_SEND; 10849679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_send_done; 10859679cc51SChristoph Hellwig send_wr->wr_cqe = &tx_desc->tx_cqe; 1086422bd0acSJenny Derzhavetz 1087422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 1088422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV; 1089422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 1090422bd0acSJenny Derzhavetz } else { 1091b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 1092422bd0acSJenny Derzhavetz } 1093422bd0acSJenny Derzhavetz 109495b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 1095b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 109695b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 1097b8d26b3bSNicholas Bellinger } 1098b8d26b3bSNicholas Bellinger 1099b8d26b3bSNicholas Bellinger static int 11009679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn) 1101b8d26b3bSNicholas Bellinger { 1102b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 1103b8d26b3bSNicholas Bellinger struct ib_sge sge; 1104b8d26b3bSNicholas Bellinger int ret; 1105b8d26b3bSNicholas Bellinger 1106b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 1107b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 1108ed1083b2SChristoph Hellwig sge.length = ISER_RX_PAYLOAD_SIZE; 110934efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey; 1110b8d26b3bSNicholas Bellinger 111124f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1112b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 1113b8d26b3bSNicholas Bellinger 11149679cc51SChristoph Hellwig isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 11159679cc51SChristoph Hellwig 1116b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 11179679cc51SChristoph Hellwig rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 1118b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 1119b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 1120b8d26b3bSNicholas Bellinger 1121dac6ab30SSagi Grimberg ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 11226d1fba0cSJenny Derzhavetz if (ret) 112324f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 1124b8d26b3bSNicholas Bellinger 1125b8d26b3bSNicholas Bellinger return ret; 1126b8d26b3bSNicholas Bellinger } 1127b8d26b3bSNicholas Bellinger 1128b8d26b3bSNicholas Bellinger static int 1129b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1130b8d26b3bSNicholas Bellinger u32 length) 1131b8d26b3bSNicholas Bellinger { 1132b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 1133dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 113467cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1135dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1136b8d26b3bSNicholas Bellinger int ret; 1137b8d26b3bSNicholas Bellinger 1138b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 1139b8d26b3bSNicholas Bellinger 1140b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1141b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1142b8d26b3bSNicholas Bellinger 1143b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1144b8d26b3bSNicholas Bellinger 1145b8d26b3bSNicholas Bellinger if (length > 0) { 1146b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1147b8d26b3bSNicholas Bellinger 1148b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1149b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1150b8d26b3bSNicholas Bellinger 1151b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1152b8d26b3bSNicholas Bellinger 1153b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1154b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1155b8d26b3bSNicholas Bellinger 1156b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1157b8d26b3bSNicholas Bellinger tx_dsg->length = length; 115834efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1159b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1160b8d26b3bSNicholas Bellinger } 1161b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1162b8d26b3bSNicholas Bellinger if (login->login_complete) { 1163e0546fc1SSagi Grimberg if (!conn->sess->sess_ops->SessionType && 1164dac6ab30SSagi Grimberg isert_conn->device->use_fastreg) { 1165570db170SSagi Grimberg ret = isert_conn_create_fastreg_pool(isert_conn); 1166f46d6a8aSNicholas Bellinger if (ret) { 116724f412ddSSagi Grimberg isert_err("Conn: %p failed to create" 1168f46d6a8aSNicholas Bellinger " fastreg pool\n", isert_conn); 1169f46d6a8aSNicholas Bellinger return ret; 1170f46d6a8aSNicholas Bellinger } 1171f46d6a8aSNicholas Bellinger } 1172f46d6a8aSNicholas Bellinger 1173b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1174b8d26b3bSNicholas Bellinger if (ret) 1175b8d26b3bSNicholas Bellinger return ret; 1176b8d26b3bSNicholas Bellinger 11774366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn, 11784366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS); 1179b8d26b3bSNicholas Bellinger if (ret) 1180b8d26b3bSNicholas Bellinger return ret; 1181b8d26b3bSNicholas Bellinger 1182128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 1183dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 1184128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 1185dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1186b8d26b3bSNicholas Bellinger goto post_send; 1187b8d26b3bSNicholas Bellinger } 1188b8d26b3bSNicholas Bellinger 11899679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 1190b8d26b3bSNicholas Bellinger if (ret) 1191b8d26b3bSNicholas Bellinger return ret; 1192b8d26b3bSNicholas Bellinger } 1193b8d26b3bSNicholas Bellinger post_send: 11949679cc51SChristoph Hellwig ret = isert_login_post_send(isert_conn, tx_desc); 1195b8d26b3bSNicholas Bellinger if (ret) 1196b8d26b3bSNicholas Bellinger return ret; 1197b8d26b3bSNicholas Bellinger 1198b8d26b3bSNicholas Bellinger return 0; 1199b8d26b3bSNicholas Bellinger } 1200b8d26b3bSNicholas Bellinger 1201b8d26b3bSNicholas Bellinger static void 12022371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1203b8d26b3bSNicholas Bellinger { 12045adabdd1SChristoph Hellwig struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 12052371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1206b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1207b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1208b8d26b3bSNicholas Bellinger int size; 1209b8d26b3bSNicholas Bellinger 121024f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 12112371e5daSSagi Grimberg 12122371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1213b8d26b3bSNicholas Bellinger 1214b8d26b3bSNicholas Bellinger if (login->first_request) { 1215b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1216b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1217b8d26b3bSNicholas Bellinger /* 1218b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1219b8d26b3bSNicholas Bellinger * login request PDU. 1220b8d26b3bSNicholas Bellinger */ 1221b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1222b8d26b3bSNicholas Bellinger login->current_stage = 1223b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1224b8d26b3bSNicholas Bellinger >> 2; 1225b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1226b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1227b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1228b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1229b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1230b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1231b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1232b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1233b8d26b3bSNicholas Bellinger } 1234b8d26b3bSNicholas Bellinger 1235b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1236b8d26b3bSNicholas Bellinger 1237b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 12384c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d " 12394c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 12404c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS); 1241b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1242b8d26b3bSNicholas Bellinger 12436faaa85fSNicholas Bellinger if (login->first_request) { 1244dac6ab30SSagi Grimberg complete(&isert_conn->login_comp); 12456faaa85fSNicholas Bellinger return; 12466faaa85fSNicholas Bellinger } 12476faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1248b8d26b3bSNicholas Bellinger } 1249b8d26b3bSNicholas Bellinger 1250b8d26b3bSNicholas Bellinger static struct iscsi_cmd 12514366b19cSJenny Derzhavetz *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1252b8d26b3bSNicholas Bellinger { 12536700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1254b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1255d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1256b8d26b3bSNicholas Bellinger 1257676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1258d703ce2fSNicholas Bellinger if (!cmd) { 125924f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1260b8d26b3bSNicholas Bellinger return NULL; 1261b8d26b3bSNicholas Bellinger } 1262d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1263b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1264d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 12654366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc; 1266b8d26b3bSNicholas Bellinger 1267d703ce2fSNicholas Bellinger return cmd; 1268b8d26b3bSNicholas Bellinger } 1269b8d26b3bSNicholas Bellinger 1270b8d26b3bSNicholas Bellinger static int 1271b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1272d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1273d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1274b8d26b3bSNicholas Bellinger { 1275b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1276b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1277b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1278b8d26b3bSNicholas Bellinger bool dump_payload = false; 12799fd60088SJenny Derzhavetz unsigned int data_len; 1280b8d26b3bSNicholas Bellinger 1281b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1282b8d26b3bSNicholas Bellinger if (rc < 0) 1283b8d26b3bSNicholas Bellinger return rc; 1284b8d26b3bSNicholas Bellinger 1285b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1286b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1287b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 12889fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length; 1289b8d26b3bSNicholas Bellinger 12909fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len) 12919fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1292b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1293b8d26b3bSNicholas Bellinger if (rc < 0) { 1294b8d26b3bSNicholas Bellinger return 0; 1295b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1296b8d26b3bSNicholas Bellinger dump_payload = true; 1297b8d26b3bSNicholas Bellinger goto sequence_cmd; 1298b8d26b3bSNicholas Bellinger } 1299b8d26b3bSNicholas Bellinger 1300b8d26b3bSNicholas Bellinger if (!imm_data) 1301b8d26b3bSNicholas Bellinger return 0; 1302b8d26b3bSNicholas Bellinger 13039fd60088SJenny Derzhavetz if (imm_data_len != data_len) { 1304b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 13059fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 13069fd60088SJenny Derzhavetz &rx_desc->data[0], imm_data_len); 13079fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 13089fd60088SJenny Derzhavetz sg_nents, imm_data_len); 13099fd60088SJenny Derzhavetz } else { 13109fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1); 13119fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg; 13129fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1; 13139fd60088SJenny Derzhavetz sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 13149fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n", 13159fd60088SJenny Derzhavetz imm_data_len); 13169fd60088SJenny Derzhavetz } 1317b8d26b3bSNicholas Bellinger 1318b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1319b8d26b3bSNicholas Bellinger 1320b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1321b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1322b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1323b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1324b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1325b8d26b3bSNicholas Bellinger } 1326b8d26b3bSNicholas Bellinger 1327b8d26b3bSNicholas Bellinger sequence_cmd: 1328561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1329b8d26b3bSNicholas Bellinger 1330b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 1331b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 13326cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 1333afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd); 1334b8d26b3bSNicholas Bellinger 1335b8d26b3bSNicholas Bellinger return 0; 1336b8d26b3bSNicholas Bellinger } 1337b8d26b3bSNicholas Bellinger 1338b8d26b3bSNicholas Bellinger static int 1339b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1340b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1341b8d26b3bSNicholas Bellinger { 1342b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1343b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1344b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1345b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1346b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1347b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1348b8d26b3bSNicholas Bellinger 1349b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1350b8d26b3bSNicholas Bellinger if (rc < 0) 1351b8d26b3bSNicholas Bellinger return rc; 1352b8d26b3bSNicholas Bellinger else if (!cmd) 1353b8d26b3bSNicholas Bellinger return 0; 1354b8d26b3bSNicholas Bellinger /* 1355b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1356b8d26b3bSNicholas Bellinger */ 1357b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 135824f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1359b8d26b3bSNicholas Bellinger dump_stack(); 1360b8d26b3bSNicholas Bellinger return -1; 1361b8d26b3bSNicholas Bellinger } 1362b8d26b3bSNicholas Bellinger 13634c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 13644c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n", 13654c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done, 13664c22e07fSSagi Grimberg cmd->se_cmd.data_length); 1367b8d26b3bSNicholas Bellinger 1368b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1369b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1370b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1371b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1372b8d26b3bSNicholas Bellinger /* 1373b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1374b8d26b3bSNicholas Bellinger */ 1375b8d26b3bSNicholas Bellinger if (page_off) { 13764c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n"); 1377b8d26b3bSNicholas Bellinger dump_stack(); 1378b8d26b3bSNicholas Bellinger return -1; 1379b8d26b3bSNicholas Bellinger } 13804c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 13814c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off, 13824c22e07fSSagi Grimberg sg_nents, &rx_desc->data[0], unsol_data_len); 1383b8d26b3bSNicholas Bellinger 1384b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1385b8d26b3bSNicholas Bellinger unsol_data_len); 1386b8d26b3bSNicholas Bellinger 1387b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1388b8d26b3bSNicholas Bellinger if (rc < 0) 1389b8d26b3bSNicholas Bellinger return rc; 1390b8d26b3bSNicholas Bellinger 13914366b19cSJenny Derzhavetz /* 13924366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive - 13934366b19cSJenny Derzhavetz * so post the buffer before hand 13944366b19cSJenny Derzhavetz */ 13954366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, rx_desc); 13964366b19cSJenny Derzhavetz if (rc) { 13974366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 13984366b19cSJenny Derzhavetz return rc; 13994366b19cSJenny Derzhavetz } 1400b8d26b3bSNicholas Bellinger return 0; 1401b8d26b3bSNicholas Bellinger } 1402b8d26b3bSNicholas Bellinger 1403b8d26b3bSNicholas Bellinger static int 1404778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1405d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1406d703ce2fSNicholas Bellinger unsigned char *buf) 1407778de368SNicholas Bellinger { 1408778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1409778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1410778de368SNicholas Bellinger int rc; 1411778de368SNicholas Bellinger 1412778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1413778de368SNicholas Bellinger if (rc < 0) 1414778de368SNicholas Bellinger return rc; 1415778de368SNicholas Bellinger /* 1416778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1417778de368SNicholas Bellinger */ 1418778de368SNicholas Bellinger 1419778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1420778de368SNicholas Bellinger } 1421778de368SNicholas Bellinger 1422778de368SNicholas Bellinger static int 1423adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1424d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1425d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1426adb54c29SNicholas Bellinger { 1427adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1428adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1429adb54c29SNicholas Bellinger int rc; 1430b44a2b67SSagi Grimberg unsigned char *text_in = NULL; 1431adb54c29SNicholas Bellinger 1432adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1433adb54c29SNicholas Bellinger if (rc < 0) 1434adb54c29SNicholas Bellinger return rc; 1435adb54c29SNicholas Bellinger 1436b44a2b67SSagi Grimberg if (payload_length) { 1437adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 1438adb54c29SNicholas Bellinger if (!text_in) { 143924f412ddSSagi Grimberg isert_err("Unable to allocate text_in of payload_length: %u\n", 1440adb54c29SNicholas Bellinger payload_length); 1441adb54c29SNicholas Bellinger return -ENOMEM; 1442adb54c29SNicholas Bellinger } 1443b44a2b67SSagi Grimberg } 1444adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1445adb54c29SNicholas Bellinger 1446adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1447adb54c29SNicholas Bellinger 1448adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1449adb54c29SNicholas Bellinger } 1450adb54c29SNicholas Bellinger 1451adb54c29SNicholas Bellinger static int 1452b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1453b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1454b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1455b8d26b3bSNicholas Bellinger { 1456b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1457b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1458b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1459b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1460b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1461b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1462b8d26b3bSNicholas Bellinger 1463fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType && 1464ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 146524f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1466ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1467ca40d24eSNicholas Bellinger return 0; 1468ca40d24eSNicholas Bellinger } 1469ca40d24eSNicholas Bellinger 1470b8d26b3bSNicholas Bellinger switch (opcode) { 1471b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 14724366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1473b8d26b3bSNicholas Bellinger if (!cmd) 1474b8d26b3bSNicholas Bellinger break; 1475b8d26b3bSNicholas Bellinger 1476d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1477b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1478b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1479b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1480b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1481422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1482b8d26b3bSNicholas Bellinger 1483d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1484b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1485b8d26b3bSNicholas Bellinger break; 1486b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 14874366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1488b8d26b3bSNicholas Bellinger if (!cmd) 1489b8d26b3bSNicholas Bellinger break; 1490b8d26b3bSNicholas Bellinger 1491d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1492d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1493778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1494b8d26b3bSNicholas Bellinger break; 1495b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1496b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1497b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1498b8d26b3bSNicholas Bellinger break; 1499b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 15004366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1501b8d26b3bSNicholas Bellinger if (!cmd) 1502b8d26b3bSNicholas Bellinger break; 1503b8d26b3bSNicholas Bellinger 1504b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1505b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1506b8d26b3bSNicholas Bellinger break; 1507b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 15084366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1509b8d26b3bSNicholas Bellinger if (!cmd) 1510b8d26b3bSNicholas Bellinger break; 1511b8d26b3bSNicholas Bellinger 1512b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1513b8d26b3bSNicholas Bellinger break; 1514adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 15154366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1516e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 15174366b19cSJenny Derzhavetz else 15184366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 15194366b19cSJenny Derzhavetz 1520e4f4e801SSagi Grimberg if (!cmd) 1521e4f4e801SSagi Grimberg break; 1522adb54c29SNicholas Bellinger 1523d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1524d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1525adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1526adb54c29SNicholas Bellinger break; 1527b8d26b3bSNicholas Bellinger default: 152824f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1529b8d26b3bSNicholas Bellinger dump_stack(); 1530b8d26b3bSNicholas Bellinger break; 1531b8d26b3bSNicholas Bellinger } 1532b8d26b3bSNicholas Bellinger 1533b8d26b3bSNicholas Bellinger return ret; 1534b8d26b3bSNicholas Bellinger } 1535b8d26b3bSNicholas Bellinger 1536b8d26b3bSNicholas Bellinger static void 15379679cc51SChristoph Hellwig isert_print_wc(struct ib_wc *wc, const char *type) 1538b8d26b3bSNicholas Bellinger { 15399679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 15409679cc51SChristoph Hellwig isert_err("%s failure: %s (%d) vend_err %x\n", type, 15419679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status, 15429679cc51SChristoph Hellwig wc->vendor_err); 15439679cc51SChristoph Hellwig else 15449679cc51SChristoph Hellwig isert_dbg("%s failure: %s (%d)\n", type, 15459679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status); 15469679cc51SChristoph Hellwig } 15479679cc51SChristoph Hellwig 15489679cc51SChristoph Hellwig static void 15499679cc51SChristoph Hellwig isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 15509679cc51SChristoph Hellwig { 15519679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 15529679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 15539679cc51SChristoph Hellwig struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 15549679cc51SChristoph Hellwig struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1555d3cf81f9SSagi Grimberg struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1556b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1557b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1558b8d26b3bSNicholas Bellinger 15599679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 15609679cc51SChristoph Hellwig isert_print_wc(wc, "recv"); 15619679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 15629679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 15639679cc51SChristoph Hellwig return; 15649679cc51SChristoph Hellwig } 15659679cc51SChristoph Hellwig 15669679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 15679679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 15689679cc51SChristoph Hellwig 15699679cc51SChristoph Hellwig isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 15709679cc51SChristoph Hellwig rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 15719679cc51SChristoph Hellwig (int)(wc->byte_len - ISER_HEADERS_LEN)); 15729679cc51SChristoph Hellwig 1573d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) { 1574b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1575d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) { 1576d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag); 1577d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va); 15784c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1579b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1580b8d26b3bSNicholas Bellinger } 1581d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) { 1582d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag); 1583d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va); 15844c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1585b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1586b8d26b3bSNicholas Bellinger } 1587b8d26b3bSNicholas Bellinger 158824f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1589b8d26b3bSNicholas Bellinger break; 1590b8d26b3bSNicholas Bellinger case ISER_HELLO: 159124f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1592b8d26b3bSNicholas Bellinger break; 1593b8d26b3bSNicholas Bellinger default: 1594d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1595b8d26b3bSNicholas Bellinger break; 1596b8d26b3bSNicholas Bellinger } 1597b8d26b3bSNicholas Bellinger 15986c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc, 1599b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 16009679cc51SChristoph Hellwig 16019679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 16029679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1603b8d26b3bSNicholas Bellinger } 1604b8d26b3bSNicholas Bellinger 1605b8d26b3bSNicholas Bellinger static void 16069679cc51SChristoph Hellwig isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1607b8d26b3bSNicholas Bellinger { 16089679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1609dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1610b8d26b3bSNicholas Bellinger 16119679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 16129679cc51SChristoph Hellwig isert_print_wc(wc, "login recv"); 16139679cc51SChristoph Hellwig return; 1614b8d26b3bSNicholas Bellinger } 1615b8d26b3bSNicholas Bellinger 16169679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 16179679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1618b8d26b3bSNicholas Bellinger 16199679cc51SChristoph Hellwig isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1620b8d26b3bSNicholas Bellinger 16212371e5daSSagi Grimberg if (isert_conn->conn) { 16222371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 16232371e5daSSagi Grimberg 16242371e5daSSagi Grimberg if (login && !login->first_request) 16252371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 16262371e5daSSagi Grimberg } 16279679cc51SChristoph Hellwig 1628dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 16292371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 1630dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1631b8d26b3bSNicholas Bellinger 16329679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 16339679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1634b8d26b3bSNicholas Bellinger } 1635b8d26b3bSNicholas Bellinger 1636e3d7e4c3SSagi Grimberg static int 1637e3d7e4c3SSagi Grimberg isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1638e3d7e4c3SSagi Grimberg struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1639e3d7e4c3SSagi Grimberg enum iser_ib_op_code op, struct isert_data_buf *data) 1640e3d7e4c3SSagi Grimberg { 1641dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1642e3d7e4c3SSagi Grimberg 1643e3d7e4c3SSagi Grimberg data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1644e3d7e4c3SSagi Grimberg DMA_TO_DEVICE : DMA_FROM_DEVICE; 1645e3d7e4c3SSagi Grimberg 1646e3d7e4c3SSagi Grimberg data->len = length - offset; 1647e3d7e4c3SSagi Grimberg data->offset = offset; 1648e3d7e4c3SSagi Grimberg data->sg_off = data->offset / PAGE_SIZE; 1649e3d7e4c3SSagi Grimberg 1650e3d7e4c3SSagi Grimberg data->sg = &sg[data->sg_off]; 1651e3d7e4c3SSagi Grimberg data->nents = min_t(unsigned int, nents - data->sg_off, 1652e3d7e4c3SSagi Grimberg ISCSI_ISER_SG_TABLESIZE); 1653e3d7e4c3SSagi Grimberg data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1654e3d7e4c3SSagi Grimberg PAGE_SIZE); 1655e3d7e4c3SSagi Grimberg 1656e3d7e4c3SSagi Grimberg data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1657e3d7e4c3SSagi Grimberg data->dma_dir); 1658e3d7e4c3SSagi Grimberg if (unlikely(!data->dma_nents)) { 165924f412ddSSagi Grimberg isert_err("Cmd: unable to dma map SGs %p\n", sg); 1660e3d7e4c3SSagi Grimberg return -EINVAL; 1661e3d7e4c3SSagi Grimberg } 1662e3d7e4c3SSagi Grimberg 166324f412ddSSagi Grimberg isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1664e3d7e4c3SSagi Grimberg isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1665e3d7e4c3SSagi Grimberg 1666e3d7e4c3SSagi Grimberg return 0; 1667e3d7e4c3SSagi Grimberg } 1668e3d7e4c3SSagi Grimberg 1669e3d7e4c3SSagi Grimberg static void 1670e3d7e4c3SSagi Grimberg isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1671e3d7e4c3SSagi Grimberg { 1672dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1673e3d7e4c3SSagi Grimberg 1674e3d7e4c3SSagi Grimberg ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1675e3d7e4c3SSagi Grimberg memset(data, 0, sizeof(*data)); 1676e3d7e4c3SSagi Grimberg } 1677e3d7e4c3SSagi Grimberg 1678e3d7e4c3SSagi Grimberg 1679e3d7e4c3SSagi Grimberg 1680b8d26b3bSNicholas Bellinger static void 1681b8d26b3bSNicholas Bellinger isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1682b8d26b3bSNicholas Bellinger { 16834c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1684e3d7e4c3SSagi Grimberg 1685d1ca2ed7SChristoph Hellwig if (isert_cmd->data.sg) { 16864c22e07fSSagi Grimberg isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1687d1ca2ed7SChristoph Hellwig isert_unmap_data_buf(isert_conn, &isert_cmd->data); 1688b8d26b3bSNicholas Bellinger } 1689b8d26b3bSNicholas Bellinger 1690d1ca2ed7SChristoph Hellwig if (isert_cmd->rdma_wr) { 16914c22e07fSSagi Grimberg isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1692d1ca2ed7SChristoph Hellwig kfree(isert_cmd->rdma_wr); 1693d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr = NULL; 169490ecc6e2SVu Pham } 1695b8d26b3bSNicholas Bellinger 1696d1ca2ed7SChristoph Hellwig if (isert_cmd->ib_sge) { 16974c22e07fSSagi Grimberg isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1698d1ca2ed7SChristoph Hellwig kfree(isert_cmd->ib_sge); 1699d1ca2ed7SChristoph Hellwig isert_cmd->ib_sge = NULL; 170090ecc6e2SVu Pham } 1701b8d26b3bSNicholas Bellinger } 1702b8d26b3bSNicholas Bellinger 1703b8d26b3bSNicholas Bellinger static void 1704a3a5a826SSagi Grimberg isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 170559464ef4SVu Pham { 17064c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 170759464ef4SVu Pham 1708d1ca2ed7SChristoph Hellwig if (isert_cmd->fr_desc) { 1709d1ca2ed7SChristoph Hellwig isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, isert_cmd->fr_desc); 1710d1ca2ed7SChristoph Hellwig if (isert_cmd->fr_desc->ind & ISERT_PROTECTED) { 1711d1ca2ed7SChristoph Hellwig isert_unmap_data_buf(isert_conn, &isert_cmd->prot); 1712d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc->ind &= ~ISERT_PROTECTED; 17139e961ae7SSagi Grimberg } 1714dac6ab30SSagi Grimberg spin_lock_bh(&isert_conn->pool_lock); 1715d1ca2ed7SChristoph Hellwig list_add_tail(&isert_cmd->fr_desc->list, &isert_conn->fr_pool); 1716dac6ab30SSagi Grimberg spin_unlock_bh(&isert_conn->pool_lock); 1717d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc = NULL; 171859464ef4SVu Pham } 171959464ef4SVu Pham 1720d1ca2ed7SChristoph Hellwig if (isert_cmd->data.sg) { 17214c22e07fSSagi Grimberg isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1722d1ca2ed7SChristoph Hellwig isert_unmap_data_buf(isert_conn, &isert_cmd->data); 172359464ef4SVu Pham } 172459464ef4SVu Pham 1725d1ca2ed7SChristoph Hellwig isert_cmd->ib_sge = NULL; 1726d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr = NULL; 172759464ef4SVu Pham } 172859464ef4SVu Pham 172959464ef4SVu Pham static void 173003e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1731b8d26b3bSNicholas Bellinger { 1732d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1733b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1734186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1735dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1736e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr; 1737b8d26b3bSNicholas Bellinger 17384c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1739b8d26b3bSNicholas Bellinger 1740b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1741b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1742b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1743b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17445159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1745b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1746b8d26b3bSNicholas Bellinger 174703e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1748b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 174903e7848aSNicholas Bellinger /* 175003e7848aSNicholas Bellinger * Check for special case during comp_err where 175103e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 175203e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 175303e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 175403e7848aSNicholas Bellinger */ 175503e7848aSNicholas Bellinger if (comp_err && 175603e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 175703e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 175803e7848aSNicholas Bellinger 1759afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 176003e7848aSNicholas Bellinger } 176103e7848aSNicholas Bellinger } 1762b8d26b3bSNicholas Bellinger 1763d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1764186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1765186a9647SNicholas Bellinger break; 1766b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1767186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1768186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17695159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1770186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1771186a9647SNicholas Bellinger 1772b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1773b8d26b3bSNicholas Bellinger break; 1774b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1775b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1776adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1777e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1778e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */ 1779e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1780e4f4e801SSagi Grimberg break; 1781e4f4e801SSagi Grimberg 1782b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1783b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 17845159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1785b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1786b8d26b3bSNicholas Bellinger 1787b8d26b3bSNicholas Bellinger /* 1788b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1789b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1790b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1791b8d26b3bSNicholas Bellinger */ 1792b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 179311378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 17943df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1795b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1796b8d26b3bSNicholas Bellinger break; 1797b8d26b3bSNicholas Bellinger } 1798b8d26b3bSNicholas Bellinger /* 1799b8d26b3bSNicholas Bellinger * Fall-through 1800b8d26b3bSNicholas Bellinger */ 1801b8d26b3bSNicholas Bellinger default: 1802d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1803b8d26b3bSNicholas Bellinger break; 1804b8d26b3bSNicholas Bellinger } 1805b8d26b3bSNicholas Bellinger } 1806b8d26b3bSNicholas Bellinger 1807b8d26b3bSNicholas Bellinger static void 1808b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1809b8d26b3bSNicholas Bellinger { 1810b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 18114c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n"); 1812b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1813b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1814b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1815b8d26b3bSNicholas Bellinger } 1816b8d26b3bSNicholas Bellinger } 1817b8d26b3bSNicholas Bellinger 1818b8d26b3bSNicholas Bellinger static void 1819b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 182003e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1821b8d26b3bSNicholas Bellinger { 1822dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 18234c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1824dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1825dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1826dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1827b8d26b3bSNicholas Bellinger } 1828b8d26b3bSNicholas Bellinger 1829b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 183003e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1831b8d26b3bSNicholas Bellinger } 1832b8d26b3bSNicholas Bellinger 183396b7973eSSagi Grimberg static int 183496b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1835f93f3a70SSagi Grimberg { 18369e961ae7SSagi Grimberg struct ib_mr_status mr_status; 183796b7973eSSagi Grimberg int ret; 1838f93f3a70SSagi Grimberg 183996b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 18409e961ae7SSagi Grimberg if (ret) { 184124f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 18429e961ae7SSagi Grimberg goto fail_mr_status; 18439e961ae7SSagi Grimberg } 184496b7973eSSagi Grimberg 18459e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 184696b7973eSSagi Grimberg u64 sec_offset_err; 18479e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 18489e961ae7SSagi Grimberg 18499e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 18509e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 18519e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 18529e961ae7SSagi Grimberg break; 18539e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 18549e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 18559e961ae7SSagi Grimberg break; 18569e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 18579e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 18589e961ae7SSagi Grimberg break; 18599e961ae7SSagi Grimberg } 186096b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 186196b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 186296b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 18639e961ae7SSagi Grimberg 18644c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx " 18659e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 18669e961ae7SSagi Grimberg mr_status.sig_err.err_type, 18679e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 18689e961ae7SSagi Grimberg mr_status.sig_err.expected, 18699e961ae7SSagi Grimberg mr_status.sig_err.actual); 18709e961ae7SSagi Grimberg ret = 1; 18719e961ae7SSagi Grimberg } 18729e961ae7SSagi Grimberg 18739e961ae7SSagi Grimberg fail_mr_status: 187496b7973eSSagi Grimberg return ret; 187596b7973eSSagi Grimberg } 187696b7973eSSagi Grimberg 187796b7973eSSagi Grimberg static void 18789679cc51SChristoph Hellwig isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 187996b7973eSSagi Grimberg { 18809679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1881dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 18829679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1883e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 18849679cc51SChristoph Hellwig struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 188596b7973eSSagi Grimberg int ret = 0; 188696b7973eSSagi Grimberg 18879679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 18889679cc51SChristoph Hellwig isert_print_wc(wc, "rdma write"); 18899679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 18909679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 18919679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 18929679cc51SChristoph Hellwig return; 18939679cc51SChristoph Hellwig } 18949679cc51SChristoph Hellwig 18959679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 18969679cc51SChristoph Hellwig 1897d1ca2ed7SChristoph Hellwig if (isert_cmd->fr_desc && isert_cmd->fr_desc->ind & ISERT_PROTECTED) { 1898d1ca2ed7SChristoph Hellwig ret = isert_check_pi_status(cmd, 1899d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc->pi_ctx->sig_mr); 1900d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc->ind &= ~ISERT_PROTECTED; 190196b7973eSSagi Grimberg } 190296b7973eSSagi Grimberg 1903f93f3a70SSagi Grimberg device->unreg_rdma_mem(isert_cmd, isert_conn); 1904d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr_num = 0; 19059e961ae7SSagi Grimberg if (ret) 19069679cc51SChristoph Hellwig transport_send_check_condition_and_sense(cmd, cmd->pi_err, 0); 19079e961ae7SSagi Grimberg else 19089679cc51SChristoph Hellwig isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 1909f93f3a70SSagi Grimberg } 1910f93f3a70SSagi Grimberg 1911f93f3a70SSagi Grimberg static void 19129679cc51SChristoph Hellwig isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1913b8d26b3bSNicholas Bellinger { 19149679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 19159679cc51SChristoph Hellwig struct isert_device *device = isert_conn->device; 19169679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1917e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1918d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1919b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 19205bac4b1aSSagi Grimberg int ret = 0; 1921b8d26b3bSNicholas Bellinger 19229679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 19239679cc51SChristoph Hellwig isert_print_wc(wc, "rdma read"); 19249679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 19259679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 19269679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 19279679cc51SChristoph Hellwig return; 19289679cc51SChristoph Hellwig } 19299679cc51SChristoph Hellwig 19309679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 19319679cc51SChristoph Hellwig 1932d1ca2ed7SChristoph Hellwig if (isert_cmd->fr_desc && isert_cmd->fr_desc->ind & ISERT_PROTECTED) { 193396b7973eSSagi Grimberg ret = isert_check_pi_status(se_cmd, 1934d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc->pi_ctx->sig_mr); 1935d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc->ind &= ~ISERT_PROTECTED; 19369e961ae7SSagi Grimberg } 19379e961ae7SSagi Grimberg 1938b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 1939d40945d8SVu Pham device->unreg_rdma_mem(isert_cmd, isert_conn); 1940d1ca2ed7SChristoph Hellwig cmd->write_data_done = isert_cmd->data.len; 1941d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr_num = 0; 1942b8d26b3bSNicholas Bellinger 194324f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1944b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1945b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1946b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1947b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1948b8d26b3bSNicholas Bellinger 1949364189f0SSagi Grimberg if (ret) { 1950afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 19515bac4b1aSSagi Grimberg transport_send_check_condition_and_sense(se_cmd, 19525bac4b1aSSagi Grimberg se_cmd->pi_err, 0); 1953364189f0SSagi Grimberg } else { 1954b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1955b8d26b3bSNicholas Bellinger } 1956364189f0SSagi Grimberg } 1957b8d26b3bSNicholas Bellinger 1958b8d26b3bSNicholas Bellinger static void 1959b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1960b8d26b3bSNicholas Bellinger { 1961b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1962b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1963b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1964dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1965d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1966b8d26b3bSNicholas Bellinger 19674c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 19684c22e07fSSagi Grimberg 1969b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1970b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1971b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 197210633c37SSagi Grimberg case ISTATE_SEND_REJECT: /* FALLTHRU */ 197310633c37SSagi Grimberg case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1974b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 19754c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 19764c22e07fSSagi Grimberg ib_dev, false); 19773df8f68aSNicholas Bellinger break; 1978b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1979b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1980b8d26b3bSNicholas Bellinger break; 1981b8d26b3bSNicholas Bellinger default: 19824c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state); 1983b8d26b3bSNicholas Bellinger dump_stack(); 1984b8d26b3bSNicholas Bellinger break; 1985b8d26b3bSNicholas Bellinger } 1986b8d26b3bSNicholas Bellinger } 1987b8d26b3bSNicholas Bellinger 1988b8d26b3bSNicholas Bellinger static void 19899679cc51SChristoph Hellwig isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1990b8d26b3bSNicholas Bellinger { 19919679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 19929679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 19939679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1994b8d26b3bSNicholas Bellinger 19959679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 19969679cc51SChristoph Hellwig isert_print_wc(wc, "login send"); 19979679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 19989679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 19999679cc51SChristoph Hellwig } 20009679cc51SChristoph Hellwig 20019679cc51SChristoph Hellwig isert_unmap_tx_desc(tx_desc, ib_dev); 20029679cc51SChristoph Hellwig } 20039679cc51SChristoph Hellwig 20049679cc51SChristoph Hellwig static void 20059679cc51SChristoph Hellwig isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 20069679cc51SChristoph Hellwig { 20079679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 20089679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 20099679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 2010e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 20119679cc51SChristoph Hellwig 20129679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 20139679cc51SChristoph Hellwig isert_print_wc(wc, "send"); 20149679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 20159679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 20169679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 20179679cc51SChristoph Hellwig return; 20189679cc51SChristoph Hellwig } 20199679cc51SChristoph Hellwig 20209679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 20219679cc51SChristoph Hellwig 20229679cc51SChristoph Hellwig switch (isert_cmd->iscsi_cmd->i_state) { 20239679cc51SChristoph Hellwig case ISTATE_SEND_TASKMGTRSP: 20249679cc51SChristoph Hellwig case ISTATE_SEND_LOGOUTRSP: 20259679cc51SChristoph Hellwig case ISTATE_SEND_REJECT: 20269679cc51SChristoph Hellwig case ISTATE_SEND_TEXTRSP: 2027b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 2028b8d26b3bSNicholas Bellinger 2029b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 2030b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 2031b8d26b3bSNicholas Bellinger return; 2032b8d26b3bSNicholas Bellinger default: 20339679cc51SChristoph Hellwig isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 20349679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 2035b8d26b3bSNicholas Bellinger break; 2036b8d26b3bSNicholas Bellinger } 2037b8d26b3bSNicholas Bellinger } 2038b8d26b3bSNicholas Bellinger 2039b8d26b3bSNicholas Bellinger static int 2040b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2041b8d26b3bSNicholas Bellinger { 2042b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 2043b8d26b3bSNicholas Bellinger int ret; 2044b8d26b3bSNicholas Bellinger 20454366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 20464366b19cSJenny Derzhavetz if (ret) { 20474366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", ret); 20484366b19cSJenny Derzhavetz return ret; 20494366b19cSJenny Derzhavetz } 20504366b19cSJenny Derzhavetz 2051dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2052b8d26b3bSNicholas Bellinger &wr_failed); 2053b8d26b3bSNicholas Bellinger if (ret) { 205424f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 2055b8d26b3bSNicholas Bellinger return ret; 2056b8d26b3bSNicholas Bellinger } 2057b8d26b3bSNicholas Bellinger return ret; 2058b8d26b3bSNicholas Bellinger } 2059b8d26b3bSNicholas Bellinger 2060b8d26b3bSNicholas Bellinger static int 2061b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2062b8d26b3bSNicholas Bellinger { 2063d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 20646700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2065b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2066b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2067b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 2068b8d26b3bSNicholas Bellinger 2069b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2070b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2071b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2072b8d26b3bSNicholas Bellinger /* 2073b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 2074b8d26b3bSNicholas Bellinger */ 2075b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 2076b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2077b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2078dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 207967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2080b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2081dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 2082b8d26b3bSNicholas Bellinger 2083b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2084b8d26b3bSNicholas Bellinger cmd->sense_buffer); 2085b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2086b8d26b3bSNicholas Bellinger 2087b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2088b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2089dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2090b8d26b3bSNicholas Bellinger 2091dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2092dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 2093b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 2094b8d26b3bSNicholas Bellinger 2095dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 2096dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2097dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 209834efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2099b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2100b8d26b3bSNicholas Bellinger } 2101b8d26b3bSNicholas Bellinger 210268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2103b8d26b3bSNicholas Bellinger 21044c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n"); 2105b8d26b3bSNicholas Bellinger 2106b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2107b8d26b3bSNicholas Bellinger } 2108b8d26b3bSNicholas Bellinger 2109131e6abcSNicholas Bellinger static void 2110131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2111131e6abcSNicholas Bellinger { 2112131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21136700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2114dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 2115131e6abcSNicholas Bellinger 2116131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 2117131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 2118131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 2119131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 2120131e6abcSNicholas Bellinger 2121131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 2122131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 2123131e6abcSNicholas Bellinger 2124131e6abcSNicholas Bellinger device->unreg_rdma_mem(isert_cmd, isert_conn); 2125131e6abcSNicholas Bellinger } 2126131e6abcSNicholas Bellinger 2127e70beee7SNicholas Bellinger static enum target_prot_op 2128e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 2129e70beee7SNicholas Bellinger { 21306700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2131dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 2132e70beee7SNicholas Bellinger 213323a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 213423a548eeSSagi Grimberg if (device->pi_capable) { 213524f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 213623a548eeSSagi Grimberg isert_conn->pi_support = true; 2137e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 213823a548eeSSagi Grimberg } 213923a548eeSSagi Grimberg } 214023a548eeSSagi Grimberg 214124f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 214223a548eeSSagi Grimberg isert_conn->pi_support = false; 2143e70beee7SNicholas Bellinger 2144e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 2145e70beee7SNicholas Bellinger } 2146e70beee7SNicholas Bellinger 2147b8d26b3bSNicholas Bellinger static int 2148b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2149b8d26b3bSNicholas Bellinger bool nopout_response) 2150b8d26b3bSNicholas Bellinger { 2151d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21526700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2153b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2154b8d26b3bSNicholas Bellinger 2155b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2156b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2157b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 2158b8d26b3bSNicholas Bellinger nopout_response); 2159b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 216068a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2161b8d26b3bSNicholas Bellinger 21624c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2163b8d26b3bSNicholas Bellinger 2164b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2165b8d26b3bSNicholas Bellinger } 2166b8d26b3bSNicholas Bellinger 2167b8d26b3bSNicholas Bellinger static int 2168b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2169b8d26b3bSNicholas Bellinger { 2170d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21716700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2172b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2173b8d26b3bSNicholas Bellinger 2174b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2175b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2176b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2177b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 217868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2179b8d26b3bSNicholas Bellinger 21804c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2181b8d26b3bSNicholas Bellinger 2182b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2183b8d26b3bSNicholas Bellinger } 2184b8d26b3bSNicholas Bellinger 2185b8d26b3bSNicholas Bellinger static int 2186b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2187b8d26b3bSNicholas Bellinger { 2188d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21896700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2190b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2191b8d26b3bSNicholas Bellinger 2192b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2193b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2194b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2195b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 219668a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2197b8d26b3bSNicholas Bellinger 21984c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2199b8d26b3bSNicholas Bellinger 2200b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2201b8d26b3bSNicholas Bellinger } 2202b8d26b3bSNicholas Bellinger 2203b8d26b3bSNicholas Bellinger static int 2204b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2205b8d26b3bSNicholas Bellinger { 2206d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 22076700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2208b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2209dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 221067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 22113df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 22123df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 22133df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2214b8d26b3bSNicholas Bellinger 2215b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 22163df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 2217b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 22183df8f68aSNicholas Bellinger 22193df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2220dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 22213df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 22223df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2223dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2224dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 22253df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 222634efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 22273df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 22283df8f68aSNicholas Bellinger 222968a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2230b8d26b3bSNicholas Bellinger 22314c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn); 2232b8d26b3bSNicholas Bellinger 2233b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2234b8d26b3bSNicholas Bellinger } 2235b8d26b3bSNicholas Bellinger 2236b8d26b3bSNicholas Bellinger static int 2237adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2238adb54c29SNicholas Bellinger { 2239d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 22406700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2241adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2242adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2243adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2244adb54c29SNicholas Bellinger u32 txt_rsp_len; 2245adb54c29SNicholas Bellinger int rc; 2246adb54c29SNicholas Bellinger 2247adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 224822c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2249adb54c29SNicholas Bellinger if (rc < 0) 2250adb54c29SNicholas Bellinger return rc; 2251adb54c29SNicholas Bellinger 2252adb54c29SNicholas Bellinger txt_rsp_len = rc; 2253adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2254adb54c29SNicholas Bellinger 2255adb54c29SNicholas Bellinger if (txt_rsp_len) { 2256dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 225767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2258adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2259adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2260adb54c29SNicholas Bellinger 2261adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2262adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2263adb54c29SNicholas Bellinger 2264adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2265adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2266adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 226734efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2268adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2269adb54c29SNicholas Bellinger } 227068a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2271adb54c29SNicholas Bellinger 2272f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn); 2273adb54c29SNicholas Bellinger 2274adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2275adb54c29SNicholas Bellinger } 2276adb54c29SNicholas Bellinger 2277adb54c29SNicholas Bellinger static int 2278b8d26b3bSNicholas Bellinger isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2279e622f2f4SChristoph Hellwig struct ib_sge *ib_sge, struct ib_rdma_wr *rdma_wr, 2280b8d26b3bSNicholas Bellinger u32 data_left, u32 offset) 2281b8d26b3bSNicholas Bellinger { 2282d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2283b8d26b3bSNicholas Bellinger struct scatterlist *sg_start, *tmp_sg; 2284dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 228567cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2286b8d26b3bSNicholas Bellinger u32 sg_off, page_off; 2287b8d26b3bSNicholas Bellinger int i = 0, sg_nents; 2288b8d26b3bSNicholas Bellinger 2289b8d26b3bSNicholas Bellinger sg_off = offset / PAGE_SIZE; 2290b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2291b8d26b3bSNicholas Bellinger sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2292b8d26b3bSNicholas Bellinger page_off = offset % PAGE_SIZE; 2293b8d26b3bSNicholas Bellinger 2294e622f2f4SChristoph Hellwig rdma_wr->wr.sg_list = ib_sge; 22959679cc51SChristoph Hellwig rdma_wr->wr.wr_cqe = &isert_cmd->tx_desc.tx_cqe; 22969679cc51SChristoph Hellwig 2297b8d26b3bSNicholas Bellinger /* 2298b8d26b3bSNicholas Bellinger * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2299b8d26b3bSNicholas Bellinger */ 2300b8d26b3bSNicholas Bellinger for_each_sg(sg_start, tmp_sg, sg_nents, i) { 23014c22e07fSSagi Grimberg isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 23024c22e07fSSagi Grimberg "page_off: %u\n", 2303b8d26b3bSNicholas Bellinger (unsigned long long)tmp_sg->dma_address, 2304b8d26b3bSNicholas Bellinger tmp_sg->length, page_off); 2305b8d26b3bSNicholas Bellinger 2306b8d26b3bSNicholas Bellinger ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2307b8d26b3bSNicholas Bellinger ib_sge->length = min_t(u32, data_left, 2308b8d26b3bSNicholas Bellinger ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 230934efc7dfSJason Gunthorpe ib_sge->lkey = device->pd->local_dma_lkey; 2310b8d26b3bSNicholas Bellinger 23114c22e07fSSagi Grimberg isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 231290ecc6e2SVu Pham ib_sge->addr, ib_sge->length, ib_sge->lkey); 2313b8d26b3bSNicholas Bellinger page_off = 0; 2314b8d26b3bSNicholas Bellinger data_left -= ib_sge->length; 23159253e667SSagi Grimberg if (!data_left) 23169253e667SSagi Grimberg break; 2317b8d26b3bSNicholas Bellinger ib_sge++; 231824f412ddSSagi Grimberg isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2319b8d26b3bSNicholas Bellinger } 2320b8d26b3bSNicholas Bellinger 2321e622f2f4SChristoph Hellwig rdma_wr->wr.num_sge = ++i; 232224f412ddSSagi Grimberg isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2323e622f2f4SChristoph Hellwig rdma_wr->wr.sg_list, rdma_wr->wr.num_sge); 2324b8d26b3bSNicholas Bellinger 2325e622f2f4SChristoph Hellwig return rdma_wr->wr.num_sge; 2326b8d26b3bSNicholas Bellinger } 2327b8d26b3bSNicholas Bellinger 2328b8d26b3bSNicholas Bellinger static int 2329d1ca2ed7SChristoph Hellwig isert_map_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn) 2330b8d26b3bSNicholas Bellinger { 2331d1ca2ed7SChristoph Hellwig struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2332b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 23336700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2334d1ca2ed7SChristoph Hellwig struct isert_data_buf *data = &isert_cmd->data; 2335e622f2f4SChristoph Hellwig struct ib_rdma_wr *rdma_wr; 2336b8d26b3bSNicholas Bellinger struct ib_sge *ib_sge; 2337e3d7e4c3SSagi Grimberg u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2338e3d7e4c3SSagi Grimberg int ret = 0, i, ib_sge_cnt; 2339b8d26b3bSNicholas Bellinger 2340d1ca2ed7SChristoph Hellwig offset = isert_cmd->iser_ib_op == ISER_IB_RDMA_READ ? 2341d1ca2ed7SChristoph Hellwig cmd->write_data_done : 0; 2342e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2343e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2344d1ca2ed7SChristoph Hellwig offset, isert_cmd->iser_ib_op, 2345d1ca2ed7SChristoph Hellwig &isert_cmd->data); 2346e3d7e4c3SSagi Grimberg if (ret) 2347e3d7e4c3SSagi Grimberg return ret; 2348b8d26b3bSNicholas Bellinger 2349e3d7e4c3SSagi Grimberg data_left = data->len; 2350e3d7e4c3SSagi Grimberg offset = data->offset; 2351b8d26b3bSNicholas Bellinger 2352e3d7e4c3SSagi Grimberg ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2353b8d26b3bSNicholas Bellinger if (!ib_sge) { 235424f412ddSSagi Grimberg isert_warn("Unable to allocate ib_sge\n"); 2355b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2356e3d7e4c3SSagi Grimberg goto unmap_cmd; 2357b8d26b3bSNicholas Bellinger } 2358d1ca2ed7SChristoph Hellwig isert_cmd->ib_sge = ib_sge; 2359b8d26b3bSNicholas Bellinger 2360d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2361d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * 2362d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr_num, GFP_KERNEL); 2363d1ca2ed7SChristoph Hellwig if (!isert_cmd->rdma_wr) { 2364d1ca2ed7SChristoph Hellwig isert_dbg("Unable to allocate isert_cmd->rdma_wr\n"); 2365b8d26b3bSNicholas Bellinger ret = -ENOMEM; 2366e3d7e4c3SSagi Grimberg goto unmap_cmd; 2367b8d26b3bSNicholas Bellinger } 2368b8d26b3bSNicholas Bellinger 2369b8d26b3bSNicholas Bellinger rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2370b8d26b3bSNicholas Bellinger 2371d1ca2ed7SChristoph Hellwig for (i = 0; i < isert_cmd->rdma_wr_num; i++) { 2372d1ca2ed7SChristoph Hellwig rdma_wr = &isert_cmd->rdma_wr[i]; 2373b8d26b3bSNicholas Bellinger data_len = min(data_left, rdma_write_max); 2374b8d26b3bSNicholas Bellinger 2375e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = 0; 2376d1ca2ed7SChristoph Hellwig if (isert_cmd->iser_ib_op == ISER_IB_RDMA_WRITE) { 23779679cc51SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 23789679cc51SChristoph Hellwig 2379e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2380e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->read_va + offset; 2381e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->read_stag; 2382d1ca2ed7SChristoph Hellwig if (i + 1 == isert_cmd->rdma_wr_num) 2383e622f2f4SChristoph Hellwig rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr; 238490ecc6e2SVu Pham else 2385d1ca2ed7SChristoph Hellwig rdma_wr->wr.next = &isert_cmd->rdma_wr[i + 1].wr; 238690ecc6e2SVu Pham } else { 23879679cc51SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 23889679cc51SChristoph Hellwig 2389e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2390e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->write_va + va_offset; 2391e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->write_stag; 2392d1ca2ed7SChristoph Hellwig if (i + 1 == isert_cmd->rdma_wr_num) 2393e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 239490ecc6e2SVu Pham else 2395d1ca2ed7SChristoph Hellwig rdma_wr->wr.next = &isert_cmd->rdma_wr[i + 1].wr; 239690ecc6e2SVu Pham } 2397b8d26b3bSNicholas Bellinger 2398b8d26b3bSNicholas Bellinger ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2399e622f2f4SChristoph Hellwig rdma_wr, data_len, offset); 2400b8d26b3bSNicholas Bellinger ib_sge += ib_sge_cnt; 2401b8d26b3bSNicholas Bellinger 2402b8d26b3bSNicholas Bellinger offset += data_len; 240390ecc6e2SVu Pham va_offset += data_len; 2404b8d26b3bSNicholas Bellinger data_left -= data_len; 2405b8d26b3bSNicholas Bellinger } 240690ecc6e2SVu Pham 240790ecc6e2SVu Pham return 0; 2408e3d7e4c3SSagi Grimberg unmap_cmd: 2409e3d7e4c3SSagi Grimberg isert_unmap_data_buf(isert_conn, data); 2410e3d7e4c3SSagi Grimberg 241190ecc6e2SVu Pham return ret; 241290ecc6e2SVu Pham } 241390ecc6e2SVu Pham 241410633c37SSagi Grimberg static inline void 241510633c37SSagi Grimberg isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 241610633c37SSagi Grimberg { 241710633c37SSagi Grimberg u32 rkey; 241810633c37SSagi Grimberg 241910633c37SSagi Grimberg memset(inv_wr, 0, sizeof(*inv_wr)); 24209679cc51SChristoph Hellwig inv_wr->wr_cqe = NULL; 242110633c37SSagi Grimberg inv_wr->opcode = IB_WR_LOCAL_INV; 242210633c37SSagi Grimberg inv_wr->ex.invalidate_rkey = mr->rkey; 242310633c37SSagi Grimberg 242410633c37SSagi Grimberg /* Bump the key */ 242510633c37SSagi Grimberg rkey = ib_inc_rkey(mr->rkey); 242610633c37SSagi Grimberg ib_update_fast_reg_key(mr, rkey); 242710633c37SSagi Grimberg } 242810633c37SSagi Grimberg 242959464ef4SVu Pham static int 2430e3d7e4c3SSagi Grimberg isert_fast_reg_mr(struct isert_conn *isert_conn, 2431e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc, 2432e3d7e4c3SSagi Grimberg struct isert_data_buf *mem, 24339e961ae7SSagi Grimberg enum isert_indicator ind, 2434e3d7e4c3SSagi Grimberg struct ib_sge *sge) 243559464ef4SVu Pham { 2436dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 243767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 24389e961ae7SSagi Grimberg struct ib_mr *mr; 243916c2d702SSagi Grimberg struct ib_reg_wr reg_wr; 2440e622f2f4SChristoph Hellwig struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 244116c2d702SSagi Grimberg int ret, n; 244259464ef4SVu Pham 2443e3d7e4c3SSagi Grimberg if (mem->dma_nents == 1) { 244434efc7dfSJason Gunthorpe sge->lkey = device->pd->local_dma_lkey; 2445e3d7e4c3SSagi Grimberg sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2446e3d7e4c3SSagi Grimberg sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 24474c22e07fSSagi Grimberg isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 24484c22e07fSSagi Grimberg sge->addr, sge->length, sge->lkey); 2449e3d7e4c3SSagi Grimberg return 0; 2450e3d7e4c3SSagi Grimberg } 2451e3d7e4c3SSagi Grimberg 245216c2d702SSagi Grimberg if (ind == ISERT_DATA_KEY_VALID) 24539e961ae7SSagi Grimberg /* Registering data buffer */ 24549e961ae7SSagi Grimberg mr = fr_desc->data_mr; 245516c2d702SSagi Grimberg else 24569e961ae7SSagi Grimberg /* Registering protection buffer */ 24579e961ae7SSagi Grimberg mr = fr_desc->pi_ctx->prot_mr; 245859464ef4SVu Pham 245910633c37SSagi Grimberg if (!(fr_desc->ind & ind)) { 246010633c37SSagi Grimberg isert_inv_rkey(&inv_wr, mr); 246159464ef4SVu Pham wr = &inv_wr; 246259464ef4SVu Pham } 246359464ef4SVu Pham 246416c2d702SSagi Grimberg n = ib_map_mr_sg(mr, mem->sg, mem->nents, PAGE_SIZE); 246516c2d702SSagi Grimberg if (unlikely(n != mem->nents)) { 246616c2d702SSagi Grimberg isert_err("failed to map mr sg (%d/%d)\n", 246716c2d702SSagi Grimberg n, mem->nents); 246816c2d702SSagi Grimberg return n < 0 ? n : -EINVAL; 246916c2d702SSagi Grimberg } 247016c2d702SSagi Grimberg 247116c2d702SSagi Grimberg isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 247216c2d702SSagi Grimberg fr_desc, mem->nents, mem->offset); 247316c2d702SSagi Grimberg 247416c2d702SSagi Grimberg reg_wr.wr.next = NULL; 247516c2d702SSagi Grimberg reg_wr.wr.opcode = IB_WR_REG_MR; 24769679cc51SChristoph Hellwig reg_wr.wr.wr_cqe = NULL; 247716c2d702SSagi Grimberg reg_wr.wr.send_flags = 0; 247816c2d702SSagi Grimberg reg_wr.wr.num_sge = 0; 247916c2d702SSagi Grimberg reg_wr.mr = mr; 248016c2d702SSagi Grimberg reg_wr.key = mr->lkey; 248116c2d702SSagi Grimberg reg_wr.access = IB_ACCESS_LOCAL_WRITE; 248259464ef4SVu Pham 248359464ef4SVu Pham if (!wr) 248416c2d702SSagi Grimberg wr = ®_wr.wr; 248559464ef4SVu Pham else 248616c2d702SSagi Grimberg wr->next = ®_wr.wr; 248759464ef4SVu Pham 2488dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 248959464ef4SVu Pham if (ret) { 249024f412ddSSagi Grimberg isert_err("fast registration failed, ret:%d\n", ret); 249159464ef4SVu Pham return ret; 249259464ef4SVu Pham } 24939e961ae7SSagi Grimberg fr_desc->ind &= ~ind; 249459464ef4SVu Pham 24959e961ae7SSagi Grimberg sge->lkey = mr->lkey; 249616c2d702SSagi Grimberg sge->addr = mr->iova; 249716c2d702SSagi Grimberg sge->length = mr->length; 249859464ef4SVu Pham 24994c22e07fSSagi Grimberg isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 25004c22e07fSSagi Grimberg sge->addr, sge->length, sge->lkey); 250159464ef4SVu Pham 250259464ef4SVu Pham return ret; 250359464ef4SVu Pham } 250459464ef4SVu Pham 25053d73cf1aSSagi Grimberg static inline void 25063d73cf1aSSagi Grimberg isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 25073d73cf1aSSagi Grimberg struct ib_sig_domain *domain) 25089e961ae7SSagi Grimberg { 250978eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 25103d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 25113d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 25123d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 251378eda2bbSSagi Grimberg /* 251478eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 251578eda2bbSSagi Grimberg * the target core would like to use it, we will take it 251678eda2bbSSagi Grimberg * from se_cmd. 251778eda2bbSSagi Grimberg */ 251878eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 251978eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 252078eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 252178eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 252278eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 252378eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 25243d73cf1aSSagi Grimberg }; 25253d73cf1aSSagi Grimberg 25263d73cf1aSSagi Grimberg static int 25273d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 25283d73cf1aSSagi Grimberg { 25299e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 25309e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 25319e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 253278eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 25333d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 25349e961ae7SSagi Grimberg break; 25359e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 25369e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 253778eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 25383d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 25399e961ae7SSagi Grimberg break; 25409e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 25419e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 25423d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 25433d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 25449e961ae7SSagi Grimberg break; 25459e961ae7SSagi Grimberg default: 254624f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 25479e961ae7SSagi Grimberg return -EINVAL; 25489e961ae7SSagi Grimberg } 25499e961ae7SSagi Grimberg 25509e961ae7SSagi Grimberg return 0; 25519e961ae7SSagi Grimberg } 25529e961ae7SSagi Grimberg 25539e961ae7SSagi Grimberg static inline u8 25549e961ae7SSagi Grimberg isert_set_prot_checks(u8 prot_checks) 25559e961ae7SSagi Grimberg { 25569e961ae7SSagi Grimberg return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 25579e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 25589e961ae7SSagi Grimberg (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 25599e961ae7SSagi Grimberg } 25609e961ae7SSagi Grimberg 25619e961ae7SSagi Grimberg static int 2562570db170SSagi Grimberg isert_reg_sig_mr(struct isert_conn *isert_conn, 2563d1ca2ed7SChristoph Hellwig struct isert_cmd *isert_cmd, 2564570db170SSagi Grimberg struct fast_reg_descriptor *fr_desc) 25659e961ae7SSagi Grimberg { 2566d1ca2ed7SChristoph Hellwig struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2567e622f2f4SChristoph Hellwig struct ib_sig_handover_wr sig_wr; 2568e622f2f4SChristoph Hellwig struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 25699e961ae7SSagi Grimberg struct pi_context *pi_ctx = fr_desc->pi_ctx; 25709e961ae7SSagi Grimberg struct ib_sig_attrs sig_attrs; 25719e961ae7SSagi Grimberg int ret; 25729e961ae7SSagi Grimberg 25739e961ae7SSagi Grimberg memset(&sig_attrs, 0, sizeof(sig_attrs)); 25749e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 25759e961ae7SSagi Grimberg if (ret) 25769e961ae7SSagi Grimberg goto err; 25779e961ae7SSagi Grimberg 25789e961ae7SSagi Grimberg sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 25799e961ae7SSagi Grimberg 25809e961ae7SSagi Grimberg if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 258110633c37SSagi Grimberg isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 25829e961ae7SSagi Grimberg wr = &inv_wr; 25839e961ae7SSagi Grimberg } 25849e961ae7SSagi Grimberg 25859e961ae7SSagi Grimberg memset(&sig_wr, 0, sizeof(sig_wr)); 2586e622f2f4SChristoph Hellwig sig_wr.wr.opcode = IB_WR_REG_SIG_MR; 25879679cc51SChristoph Hellwig sig_wr.wr.wr_cqe = NULL; 2588d1ca2ed7SChristoph Hellwig sig_wr.wr.sg_list = &isert_cmd->ib_sg[DATA]; 2589e622f2f4SChristoph Hellwig sig_wr.wr.num_sge = 1; 2590e622f2f4SChristoph Hellwig sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; 2591e622f2f4SChristoph Hellwig sig_wr.sig_attrs = &sig_attrs; 2592e622f2f4SChristoph Hellwig sig_wr.sig_mr = pi_ctx->sig_mr; 25939e961ae7SSagi Grimberg if (se_cmd->t_prot_sg) 2594d1ca2ed7SChristoph Hellwig sig_wr.prot = &isert_cmd->ib_sg[PROT]; 25959e961ae7SSagi Grimberg 25969e961ae7SSagi Grimberg if (!wr) 2597e622f2f4SChristoph Hellwig wr = &sig_wr.wr; 25989e961ae7SSagi Grimberg else 2599e622f2f4SChristoph Hellwig wr->next = &sig_wr.wr; 26009e961ae7SSagi Grimberg 2601dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 26029e961ae7SSagi Grimberg if (ret) { 260324f412ddSSagi Grimberg isert_err("fast registration failed, ret:%d\n", ret); 26049e961ae7SSagi Grimberg goto err; 26059e961ae7SSagi Grimberg } 26069e961ae7SSagi Grimberg fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 26079e961ae7SSagi Grimberg 2608d1ca2ed7SChristoph Hellwig isert_cmd->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2609d1ca2ed7SChristoph Hellwig isert_cmd->ib_sg[SIG].addr = 0; 2610d1ca2ed7SChristoph Hellwig isert_cmd->ib_sg[SIG].length = se_cmd->data_length; 26119e961ae7SSagi Grimberg if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 26129e961ae7SSagi Grimberg se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 26139e961ae7SSagi Grimberg /* 26149e961ae7SSagi Grimberg * We have protection guards on the wire 26159e961ae7SSagi Grimberg * so we need to set a larget transfer 26169e961ae7SSagi Grimberg */ 2617d1ca2ed7SChristoph Hellwig isert_cmd->ib_sg[SIG].length += se_cmd->prot_length; 26189e961ae7SSagi Grimberg 261924f412ddSSagi Grimberg isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2620d1ca2ed7SChristoph Hellwig isert_cmd->ib_sg[SIG].addr, isert_cmd->ib_sg[SIG].length, 2621d1ca2ed7SChristoph Hellwig isert_cmd->ib_sg[SIG].lkey); 26229e961ae7SSagi Grimberg err: 26239e961ae7SSagi Grimberg return ret; 26249e961ae7SSagi Grimberg } 26259e961ae7SSagi Grimberg 262659464ef4SVu Pham static int 2627570db170SSagi Grimberg isert_handle_prot_cmd(struct isert_conn *isert_conn, 2628d1ca2ed7SChristoph Hellwig struct isert_cmd *isert_cmd) 2629570db170SSagi Grimberg { 2630dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 2631570db170SSagi Grimberg struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2632570db170SSagi Grimberg int ret; 2633570db170SSagi Grimberg 2634d1ca2ed7SChristoph Hellwig if (!isert_cmd->fr_desc->pi_ctx) { 2635d1ca2ed7SChristoph Hellwig ret = isert_create_pi_ctx(isert_cmd->fr_desc, 2636570db170SSagi Grimberg device->ib_device, 263767cb3949SSagi Grimberg device->pd); 2638570db170SSagi Grimberg if (ret) { 263924f412ddSSagi Grimberg isert_err("conn %p failed to allocate pi_ctx\n", 2640570db170SSagi Grimberg isert_conn); 2641570db170SSagi Grimberg return ret; 2642570db170SSagi Grimberg } 2643570db170SSagi Grimberg } 2644570db170SSagi Grimberg 2645570db170SSagi Grimberg if (se_cmd->t_prot_sg) { 2646570db170SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, 2647570db170SSagi Grimberg se_cmd->t_prot_sg, 2648570db170SSagi Grimberg se_cmd->t_prot_nents, 2649570db170SSagi Grimberg se_cmd->prot_length, 2650d1ca2ed7SChristoph Hellwig 0, 2651d1ca2ed7SChristoph Hellwig isert_cmd->iser_ib_op, 2652d1ca2ed7SChristoph Hellwig &isert_cmd->prot); 2653570db170SSagi Grimberg if (ret) { 265424f412ddSSagi Grimberg isert_err("conn %p failed to map protection buffer\n", 2655570db170SSagi Grimberg isert_conn); 2656570db170SSagi Grimberg return ret; 2657570db170SSagi Grimberg } 2658570db170SSagi Grimberg 2659d1ca2ed7SChristoph Hellwig memset(&isert_cmd->ib_sg[PROT], 0, sizeof(isert_cmd->ib_sg[PROT])); 2660d1ca2ed7SChristoph Hellwig ret = isert_fast_reg_mr(isert_conn, isert_cmd->fr_desc, 2661d1ca2ed7SChristoph Hellwig &isert_cmd->prot, 2662d1ca2ed7SChristoph Hellwig ISERT_PROT_KEY_VALID, 2663d1ca2ed7SChristoph Hellwig &isert_cmd->ib_sg[PROT]); 2664570db170SSagi Grimberg if (ret) { 266524f412ddSSagi Grimberg isert_err("conn %p failed to fast reg mr\n", 2666570db170SSagi Grimberg isert_conn); 2667570db170SSagi Grimberg goto unmap_prot_cmd; 2668570db170SSagi Grimberg } 2669570db170SSagi Grimberg } 2670570db170SSagi Grimberg 2671d1ca2ed7SChristoph Hellwig ret = isert_reg_sig_mr(isert_conn, isert_cmd, isert_cmd->fr_desc); 2672570db170SSagi Grimberg if (ret) { 267324f412ddSSagi Grimberg isert_err("conn %p failed to fast reg mr\n", 2674570db170SSagi Grimberg isert_conn); 2675570db170SSagi Grimberg goto unmap_prot_cmd; 2676570db170SSagi Grimberg } 2677d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc->ind |= ISERT_PROTECTED; 2678570db170SSagi Grimberg 2679570db170SSagi Grimberg return 0; 2680570db170SSagi Grimberg 2681570db170SSagi Grimberg unmap_prot_cmd: 2682570db170SSagi Grimberg if (se_cmd->t_prot_sg) 2683d1ca2ed7SChristoph Hellwig isert_unmap_data_buf(isert_conn, &isert_cmd->prot); 2684570db170SSagi Grimberg 2685570db170SSagi Grimberg return ret; 2686570db170SSagi Grimberg } 2687570db170SSagi Grimberg 2688570db170SSagi Grimberg static int 2689d1ca2ed7SChristoph Hellwig isert_reg_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn) 269059464ef4SVu Pham { 2691d1ca2ed7SChristoph Hellwig struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 269259464ef4SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 2693e3d7e4c3SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2694e3d7e4c3SSagi Grimberg struct fast_reg_descriptor *fr_desc = NULL; 2695e622f2f4SChristoph Hellwig struct ib_rdma_wr *rdma_wr; 2696570db170SSagi Grimberg struct ib_sge *ib_sg; 2697e3d7e4c3SSagi Grimberg u32 offset; 2698e3d7e4c3SSagi Grimberg int ret = 0; 269959464ef4SVu Pham unsigned long flags; 270059464ef4SVu Pham 2701d1ca2ed7SChristoph Hellwig offset = isert_cmd->iser_ib_op == ISER_IB_RDMA_READ ? 2702d1ca2ed7SChristoph Hellwig cmd->write_data_done : 0; 2703e3d7e4c3SSagi Grimberg ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2704e3d7e4c3SSagi Grimberg se_cmd->t_data_nents, se_cmd->data_length, 2705d1ca2ed7SChristoph Hellwig offset, isert_cmd->iser_ib_op, 2706d1ca2ed7SChristoph Hellwig &isert_cmd->data); 2707e3d7e4c3SSagi Grimberg if (ret) 2708e3d7e4c3SSagi Grimberg return ret; 2709e3d7e4c3SSagi Grimberg 2710d1ca2ed7SChristoph Hellwig if (isert_cmd->data.dma_nents != 1 || 2711d1ca2ed7SChristoph Hellwig isert_prot_cmd(isert_conn, se_cmd)) { 2712dac6ab30SSagi Grimberg spin_lock_irqsave(&isert_conn->pool_lock, flags); 2713dac6ab30SSagi Grimberg fr_desc = list_first_entry(&isert_conn->fr_pool, 2714e3d7e4c3SSagi Grimberg struct fast_reg_descriptor, list); 2715e3d7e4c3SSagi Grimberg list_del(&fr_desc->list); 2716dac6ab30SSagi Grimberg spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2717d1ca2ed7SChristoph Hellwig isert_cmd->fr_desc = fr_desc; 271859464ef4SVu Pham } 271959464ef4SVu Pham 2720d1ca2ed7SChristoph Hellwig ret = isert_fast_reg_mr(isert_conn, fr_desc, &isert_cmd->data, 2721d1ca2ed7SChristoph Hellwig ISERT_DATA_KEY_VALID, &isert_cmd->ib_sg[DATA]); 2722e3d7e4c3SSagi Grimberg if (ret) 2723e3d7e4c3SSagi Grimberg goto unmap_cmd; 272459464ef4SVu Pham 2725302cc7c3SSagi Grimberg if (isert_prot_cmd(isert_conn, se_cmd)) { 2726d1ca2ed7SChristoph Hellwig ret = isert_handle_prot_cmd(isert_conn, isert_cmd); 27279e961ae7SSagi Grimberg if (ret) 27289e961ae7SSagi Grimberg goto unmap_cmd; 27299e961ae7SSagi Grimberg 2730d1ca2ed7SChristoph Hellwig ib_sg = &isert_cmd->ib_sg[SIG]; 2731570db170SSagi Grimberg } else { 2732d1ca2ed7SChristoph Hellwig ib_sg = &isert_cmd->ib_sg[DATA]; 27339e961ae7SSagi Grimberg } 27349e961ae7SSagi Grimberg 2735d1ca2ed7SChristoph Hellwig memcpy(&isert_cmd->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2736d1ca2ed7SChristoph Hellwig isert_cmd->ib_sge = &isert_cmd->s_ib_sge; 2737d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr_num = 1; 2738d1ca2ed7SChristoph Hellwig memset(&isert_cmd->s_rdma_wr, 0, sizeof(isert_cmd->s_rdma_wr)); 2739d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr = &isert_cmd->s_rdma_wr; 274059464ef4SVu Pham 2741d1ca2ed7SChristoph Hellwig rdma_wr = &isert_cmd->s_rdma_wr; 2742d1ca2ed7SChristoph Hellwig rdma_wr->wr.sg_list = &isert_cmd->s_ib_sge; 2743e622f2f4SChristoph Hellwig rdma_wr->wr.num_sge = 1; 27449679cc51SChristoph Hellwig rdma_wr->wr.wr_cqe = &isert_cmd->tx_desc.tx_cqe; 2745d1ca2ed7SChristoph Hellwig if (isert_cmd->iser_ib_op == ISER_IB_RDMA_WRITE) { 27469679cc51SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 27479679cc51SChristoph Hellwig 2748e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2749e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->read_va; 2750e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->read_stag; 2751e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 27529e961ae7SSagi Grimberg 0 : IB_SEND_SIGNALED; 275359464ef4SVu Pham } else { 27549679cc51SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 27559679cc51SChristoph Hellwig 2756e622f2f4SChristoph Hellwig rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2757e622f2f4SChristoph Hellwig rdma_wr->remote_addr = isert_cmd->write_va; 2758e622f2f4SChristoph Hellwig rdma_wr->rkey = isert_cmd->write_stag; 2759e622f2f4SChristoph Hellwig rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 276059464ef4SVu Pham } 276159464ef4SVu Pham 276259464ef4SVu Pham return 0; 2763570db170SSagi Grimberg 2764e3d7e4c3SSagi Grimberg unmap_cmd: 2765e3d7e4c3SSagi Grimberg if (fr_desc) { 2766dac6ab30SSagi Grimberg spin_lock_irqsave(&isert_conn->pool_lock, flags); 2767dac6ab30SSagi Grimberg list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2768dac6ab30SSagi Grimberg spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2769e3d7e4c3SSagi Grimberg } 2770d1ca2ed7SChristoph Hellwig isert_unmap_data_buf(isert_conn, &isert_cmd->data); 277159464ef4SVu Pham 277259464ef4SVu Pham return ret; 277359464ef4SVu Pham } 277459464ef4SVu Pham 277559464ef4SVu Pham static int 277690ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 277790ecc6e2SVu Pham { 277890ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 277959464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 27806700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2781dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 278290ecc6e2SVu Pham struct ib_send_wr *wr_failed; 278390ecc6e2SVu Pham int rc; 278490ecc6e2SVu Pham 278524f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 278690ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 27874c22e07fSSagi Grimberg 2788d1ca2ed7SChristoph Hellwig isert_cmd->iser_ib_op = ISER_IB_RDMA_WRITE; 2789d1ca2ed7SChristoph Hellwig rc = device->reg_rdma_mem(isert_cmd, conn); 279090ecc6e2SVu Pham if (rc) { 279124f412ddSSagi Grimberg isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 279290ecc6e2SVu Pham return rc; 279390ecc6e2SVu Pham } 279490ecc6e2SVu Pham 2795302cc7c3SSagi Grimberg if (!isert_prot_cmd(isert_conn, se_cmd)) { 2796b8d26b3bSNicholas Bellinger /* 2797b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2798b8d26b3bSNicholas Bellinger */ 27999e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 28009e961ae7SSagi Grimberg &isert_cmd->tx_desc); 280104d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2802b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2803b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 280495b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 280568a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 2806d1ca2ed7SChristoph Hellwig isert_cmd->s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr; 2807d1ca2ed7SChristoph Hellwig isert_cmd->rdma_wr_num += 1; 28084366b19cSJenny Derzhavetz 28094366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 28104366b19cSJenny Derzhavetz if (rc) { 28114366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 28124366b19cSJenny Derzhavetz return rc; 28134366b19cSJenny Derzhavetz } 28149e961ae7SSagi Grimberg } 2815b8d26b3bSNicholas Bellinger 2816d1ca2ed7SChristoph Hellwig rc = ib_post_send(isert_conn->qp, &isert_cmd->rdma_wr->wr, &wr_failed); 2817bdf20e72SSagi Grimberg if (rc) 281824f412ddSSagi Grimberg isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 28199e961ae7SSagi Grimberg 2820302cc7c3SSagi Grimberg if (!isert_prot_cmd(isert_conn, se_cmd)) 282124f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 28229e961ae7SSagi Grimberg "READ\n", isert_cmd); 28239e961ae7SSagi Grimberg else 282424f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 282590ecc6e2SVu Pham isert_cmd); 2826b8d26b3bSNicholas Bellinger 282790ecc6e2SVu Pham return 1; 2828b8d26b3bSNicholas Bellinger } 2829b8d26b3bSNicholas Bellinger 2830b8d26b3bSNicholas Bellinger static int 2831b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2832b8d26b3bSNicholas Bellinger { 2833b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 2834d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 28356700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2836dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 283790ecc6e2SVu Pham struct ib_send_wr *wr_failed; 283890ecc6e2SVu Pham int rc; 2839b8d26b3bSNicholas Bellinger 284024f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 284190ecc6e2SVu Pham isert_cmd, se_cmd->data_length, cmd->write_data_done); 2842d1ca2ed7SChristoph Hellwig isert_cmd->iser_ib_op = ISER_IB_RDMA_READ; 2843d1ca2ed7SChristoph Hellwig rc = device->reg_rdma_mem(isert_cmd, conn); 284490ecc6e2SVu Pham if (rc) { 284524f412ddSSagi Grimberg isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 284690ecc6e2SVu Pham return rc; 2847b8d26b3bSNicholas Bellinger } 2848b8d26b3bSNicholas Bellinger 2849d1ca2ed7SChristoph Hellwig rc = ib_post_send(isert_conn->qp, &isert_cmd->rdma_wr->wr, &wr_failed); 2850bdf20e72SSagi Grimberg if (rc) 285124f412ddSSagi Grimberg isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2852bdf20e72SSagi Grimberg 285324f412ddSSagi Grimberg isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 285490ecc6e2SVu Pham isert_cmd); 2855b8d26b3bSNicholas Bellinger 285690ecc6e2SVu Pham return 0; 2857b8d26b3bSNicholas Bellinger } 2858b8d26b3bSNicholas Bellinger 2859b8d26b3bSNicholas Bellinger static int 2860b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2861b8d26b3bSNicholas Bellinger { 2862a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2863a4c15cd9SJenny Derzhavetz int ret = 0; 2864b8d26b3bSNicholas Bellinger 2865b8d26b3bSNicholas Bellinger switch (state) { 2866a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE: 2867a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 2868a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 2869a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 2870a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 2871a4c15cd9SJenny Derzhavetz break; 2872b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2873b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2874b8d26b3bSNicholas Bellinger break; 2875b8d26b3bSNicholas Bellinger default: 287624f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2877b8d26b3bSNicholas Bellinger ret = -EINVAL; 2878b8d26b3bSNicholas Bellinger break; 2879b8d26b3bSNicholas Bellinger } 2880b8d26b3bSNicholas Bellinger 2881b8d26b3bSNicholas Bellinger return ret; 2882b8d26b3bSNicholas Bellinger } 2883b8d26b3bSNicholas Bellinger 2884b8d26b3bSNicholas Bellinger static int 2885b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2886b8d26b3bSNicholas Bellinger { 2887991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2888b8d26b3bSNicholas Bellinger int ret; 2889b8d26b3bSNicholas Bellinger 2890b8d26b3bSNicholas Bellinger switch (state) { 2891b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2892b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2893991bb764SSagi Grimberg if (!ret) 2894991bb764SSagi Grimberg isert_conn->logout_posted = true; 2895b8d26b3bSNicholas Bellinger break; 2896b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2897b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2898b8d26b3bSNicholas Bellinger break; 2899b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2900b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2901b8d26b3bSNicholas Bellinger break; 2902b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2903b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2904b8d26b3bSNicholas Bellinger break; 2905adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2906adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2907adb54c29SNicholas Bellinger break; 2908b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2909b8d26b3bSNicholas Bellinger /* 2910b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2911b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2912b8d26b3bSNicholas Bellinger */ 2913b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2914b8d26b3bSNicholas Bellinger break; 2915b8d26b3bSNicholas Bellinger default: 291624f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 2917b8d26b3bSNicholas Bellinger ret = -EINVAL; 2918b8d26b3bSNicholas Bellinger break; 2919b8d26b3bSNicholas Bellinger } 2920b8d26b3bSNicholas Bellinger 2921b8d26b3bSNicholas Bellinger return ret; 2922b8d26b3bSNicholas Bellinger } 2923b8d26b3bSNicholas Bellinger 2924ca6c1d82SSagi Grimberg struct rdma_cm_id * 2925ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 2926ca6c1d82SSagi Grimberg { 2927ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 2928ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 2929ca6c1d82SSagi Grimberg struct sockaddr *sa; 2930ca6c1d82SSagi Grimberg int ret; 2931ca6c1d82SSagi Grimberg 2932ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 293324f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2934ca6c1d82SSagi Grimberg 2935fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2936ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 2937ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 293824f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2939ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 2940ca6c1d82SSagi Grimberg goto out; 2941ca6c1d82SSagi Grimberg } 294224f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 2943ca6c1d82SSagi Grimberg 2944ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 2945ca6c1d82SSagi Grimberg if (ret) { 294624f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 2947ca6c1d82SSagi Grimberg goto out_id; 2948ca6c1d82SSagi Grimberg } 2949ca6c1d82SSagi Grimberg 2950992607e8SSagi Grimberg ret = rdma_listen(id, 0); 2951ca6c1d82SSagi Grimberg if (ret) { 295224f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 2953ca6c1d82SSagi Grimberg goto out_id; 2954ca6c1d82SSagi Grimberg } 2955ca6c1d82SSagi Grimberg 2956ca6c1d82SSagi Grimberg return id; 2957ca6c1d82SSagi Grimberg out_id: 2958ca6c1d82SSagi Grimberg rdma_destroy_id(id); 2959ca6c1d82SSagi Grimberg out: 2960ca6c1d82SSagi Grimberg return ERR_PTR(ret); 2961ca6c1d82SSagi Grimberg } 2962ca6c1d82SSagi Grimberg 2963b8d26b3bSNicholas Bellinger static int 2964b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 296513a3cf08SAndy Grover struct sockaddr_storage *ksockaddr) 2966b8d26b3bSNicholas Bellinger { 2967b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2968b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2969b8d26b3bSNicholas Bellinger int ret; 2970b8d26b3bSNicholas Bellinger 2971b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2972b8d26b3bSNicholas Bellinger if (!isert_np) { 297324f412ddSSagi Grimberg isert_err("Unable to allocate struct isert_np\n"); 2974b8d26b3bSNicholas Bellinger return -ENOMEM; 2975b8d26b3bSNicholas Bellinger } 2976ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0); 2977ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex); 2978bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted); 2979bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending); 2980ca6c1d82SSagi Grimberg isert_np->np = np; 2981b8d26b3bSNicholas Bellinger 2982b8d26b3bSNicholas Bellinger /* 2983b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 2984b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 2985b8d26b3bSNicholas Bellinger */ 2986b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 298713a3cf08SAndy Grover sizeof(struct sockaddr_storage)); 2988b8d26b3bSNicholas Bellinger 2989ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 2990b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 2991b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 2992b8d26b3bSNicholas Bellinger goto out; 2993b8d26b3bSNicholas Bellinger } 2994b8d26b3bSNicholas Bellinger 2995ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid; 2996b8d26b3bSNicholas Bellinger np->np_context = isert_np; 2997b8d26b3bSNicholas Bellinger 2998b8d26b3bSNicholas Bellinger return 0; 2999b8d26b3bSNicholas Bellinger 3000b8d26b3bSNicholas Bellinger out: 3001b8d26b3bSNicholas Bellinger kfree(isert_np); 3002ca6c1d82SSagi Grimberg 3003b8d26b3bSNicholas Bellinger return ret; 3004b8d26b3bSNicholas Bellinger } 3005b8d26b3bSNicholas Bellinger 3006b8d26b3bSNicholas Bellinger static int 3007b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 3008b8d26b3bSNicholas Bellinger { 3009dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 3010b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 3011b8d26b3bSNicholas Bellinger int ret; 301213bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr; 3013b8d26b3bSNicholas Bellinger 3014b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 3015b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 3016b8d26b3bSNicholas Bellinger cp.retry_count = 7; 3017b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 3018b8d26b3bSNicholas Bellinger 301913bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 3020422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 3021422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv) 3022422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 302313bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr; 302413bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr); 302513bce482SJenny Derzhavetz 3026b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 3027b8d26b3bSNicholas Bellinger if (ret) { 302824f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 3029b8d26b3bSNicholas Bellinger return ret; 3030b8d26b3bSNicholas Bellinger } 3031b8d26b3bSNicholas Bellinger 3032b8d26b3bSNicholas Bellinger return 0; 3033b8d26b3bSNicholas Bellinger } 3034b8d26b3bSNicholas Bellinger 3035b8d26b3bSNicholas Bellinger static int 3036b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3037b8d26b3bSNicholas Bellinger { 30386700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 3039b8d26b3bSNicholas Bellinger int ret; 3040b8d26b3bSNicholas Bellinger 304124f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 30422371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 30432371e5daSSagi Grimberg if (ret) { 304424f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 30452371e5daSSagi Grimberg isert_conn); 30462371e5daSSagi Grimberg return ret; 30472371e5daSSagi Grimberg } 30482371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 30492371e5daSSagi Grimberg 30506faaa85fSNicholas Bellinger /* 30516faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 30526faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 30536faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 30546faaa85fSNicholas Bellinger * into a NOP. 30556faaa85fSNicholas Bellinger */ 30566faaa85fSNicholas Bellinger if (!login->first_request) 30576faaa85fSNicholas Bellinger return 0; 3058b8d26b3bSNicholas Bellinger 30592371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 30602371e5daSSagi Grimberg 3061dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn); 3062dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3063b8d26b3bSNicholas Bellinger if (ret) 3064b8d26b3bSNicholas Bellinger return ret; 3065b8d26b3bSNicholas Bellinger 306624f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 30672371e5daSSagi Grimberg 3068b8d26b3bSNicholas Bellinger return 0; 3069b8d26b3bSNicholas Bellinger } 3070b8d26b3bSNicholas Bellinger 3071b8d26b3bSNicholas Bellinger static void 3072b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3073b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 3074b8d26b3bSNicholas Bellinger { 3075dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 3076b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 3077b8d26b3bSNicholas Bellinger 3078b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 3079b8d26b3bSNicholas Bellinger 3080dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr; 3081dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr; 3082b8d26b3bSNicholas Bellinger } 3083b8d26b3bSNicholas Bellinger 3084b8d26b3bSNicholas Bellinger static int 3085b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3086b8d26b3bSNicholas Bellinger { 30876700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 3088b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 3089c6b8e918SSagi Grimberg int ret; 3090b8d26b3bSNicholas Bellinger 3091b8d26b3bSNicholas Bellinger accept_wait: 3092ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem); 3093c6b8e918SSagi Grimberg if (ret) 3094b8d26b3bSNicholas Bellinger return -ENODEV; 3095b8d26b3bSNicholas Bellinger 3096b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 3097e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3098b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 309911378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n", 3100e346ab34SSagi Grimberg np->np_thread_state); 3101e346ab34SSagi Grimberg /** 3102e346ab34SSagi Grimberg * No point in stalling here when np_thread 3103e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 3104e346ab34SSagi Grimberg **/ 3105b8d26b3bSNicholas Bellinger return -ENODEV; 3106b8d26b3bSNicholas Bellinger } 3107b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 3108b8d26b3bSNicholas Bellinger 3109ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 3110bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) { 3111ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 3112b8d26b3bSNicholas Bellinger goto accept_wait; 3113b8d26b3bSNicholas Bellinger } 3114bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending, 3115bd379220SJenny Derzhavetz struct isert_conn, node); 3116bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 3117ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 3118b8d26b3bSNicholas Bellinger 3119b8d26b3bSNicholas Bellinger conn->context = isert_conn; 3120b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 3121aea92980SJenny Derzhavetz isert_conn->state = ISER_CONN_BOUND; 3122b8d26b3bSNicholas Bellinger 3123b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 3124b8d26b3bSNicholas Bellinger 312524f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 31262371e5daSSagi Grimberg 3127b8d26b3bSNicholas Bellinger return 0; 3128b8d26b3bSNicholas Bellinger } 3129b8d26b3bSNicholas Bellinger 3130b8d26b3bSNicholas Bellinger static void 3131b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 3132b8d26b3bSNicholas Bellinger { 31336700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 3134268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 3135b8d26b3bSNicholas Bellinger 3136ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id) 3137ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id); 3138b8d26b3bSNicholas Bellinger 3139268e6811SSagi Grimberg /* 3140268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 3141268e6811SSagi Grimberg * that at this point we don't have hanging connections that 3142268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 3143268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 3144bd379220SJenny Derzhavetz * up in accepted and pending lists. 3145268e6811SSagi Grimberg */ 3146ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 3147bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) { 3148bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n"); 3149268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 3150bd379220SJenny Derzhavetz &isert_np->pending, 3151bd379220SJenny Derzhavetz node) { 3152bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n", 3153bd379220SJenny Derzhavetz isert_conn, isert_conn->state); 3154bd379220SJenny Derzhavetz isert_connect_release(isert_conn); 3155bd379220SJenny Derzhavetz } 3156bd379220SJenny Derzhavetz } 3157bd379220SJenny Derzhavetz 3158bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) { 3159bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n"); 3160bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n, 3161bd379220SJenny Derzhavetz &isert_np->accepted, 3162bd379220SJenny Derzhavetz node) { 316324f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 3164268e6811SSagi Grimberg isert_conn, isert_conn->state); 3165268e6811SSagi Grimberg isert_connect_release(isert_conn); 3166268e6811SSagi Grimberg } 3167268e6811SSagi Grimberg } 3168ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 3169268e6811SSagi Grimberg 3170b8d26b3bSNicholas Bellinger np->np_context = NULL; 3171b8d26b3bSNicholas Bellinger kfree(isert_np); 3172b8d26b3bSNicholas Bellinger } 3173b8d26b3bSNicholas Bellinger 3174b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 3175b02efbfcSSagi Grimberg { 3176b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 3177b02efbfcSSagi Grimberg struct isert_conn, 3178b02efbfcSSagi Grimberg release_work); 3179b02efbfcSSagi Grimberg 318024f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 3181b02efbfcSSagi Grimberg 3182dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 3183b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 3184dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 3185b02efbfcSSagi Grimberg 318624f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 3187b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 3188b02efbfcSSagi Grimberg } 3189b02efbfcSSagi Grimberg 3190bdf20e72SSagi Grimberg static void 3191991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 3192991bb764SSagi Grimberg { 3193991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 3194991bb764SSagi Grimberg 31954c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 31964c22e07fSSagi Grimberg 3197991bb764SSagi Grimberg if (isert_conn->logout_posted) { 319824f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3199991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 3200991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 3201991bb764SSagi Grimberg } 3202991bb764SSagi Grimberg } 3203991bb764SSagi Grimberg 3204991bb764SSagi Grimberg static void 3205c7e160eeSSagi Grimberg isert_wait4cmds(struct iscsi_conn *conn) 3206c7e160eeSSagi Grimberg { 32074c22e07fSSagi Grimberg isert_info("iscsi_conn %p\n", conn); 32084c22e07fSSagi Grimberg 3209c7e160eeSSagi Grimberg if (conn->sess) { 3210c7e160eeSSagi Grimberg target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3211c7e160eeSSagi Grimberg target_wait_for_sess_cmds(conn->sess->se_sess); 3212c7e160eeSSagi Grimberg } 3213c7e160eeSSagi Grimberg } 3214c7e160eeSSagi Grimberg 32153e03c4b0SJenny Derzhavetz /** 32163e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for 32173e03c4b0SJenny Derzhavetz * unsolicitate dataout 32183e03c4b0SJenny Derzhavetz * @conn: iscsi connection 32193e03c4b0SJenny Derzhavetz * 32203e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited 32213e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those 32223e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds 32233e03c4b0SJenny Derzhavetz */ 32243e03c4b0SJenny Derzhavetz static void 32253e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 32263e03c4b0SJenny Derzhavetz { 32273e03c4b0SJenny Derzhavetz struct iscsi_cmd *cmd, *tmp; 32283e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list); 32293e03c4b0SJenny Derzhavetz 32303e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 32313e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 32323e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 32333e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 32343e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length)) 32353e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 32363e03c4b0SJenny Derzhavetz } 32373e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 32383e03c4b0SJenny Derzhavetz 32393e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 32403e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 32413e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) { 32423e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 32433e03c4b0SJenny Derzhavetz 32443e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd); 32453e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 32463e03c4b0SJenny Derzhavetz } 32473e03c4b0SJenny Derzhavetz } 32483e03c4b0SJenny Derzhavetz } 32493e03c4b0SJenny Derzhavetz 3250defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 3251b8d26b3bSNicholas Bellinger { 3252b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3253b8d26b3bSNicholas Bellinger 32544c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn); 3255b8d26b3bSNicholas Bellinger 3256dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 3257954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 3258dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 3259b2cb9649SNicholas Bellinger 3260572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 32613e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn); 32623e03c4b0SJenny Derzhavetz isert_wait4cmds(conn); 3263991bb764SSagi Grimberg isert_wait4logout(isert_conn); 3264954f2372SSagi Grimberg 3265b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 3266defd8848SNicholas Bellinger } 3267defd8848SNicholas Bellinger 3268defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 3269defd8848SNicholas Bellinger { 3270defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 3271b8d26b3bSNicholas Bellinger 3272572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 3273b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 3274b8d26b3bSNicholas Bellinger } 3275b8d26b3bSNicholas Bellinger 3276187e84f7SVarun Prakash static void isert_get_rx_pdu(struct iscsi_conn *conn) 3277187e84f7SVarun Prakash { 3278187e84f7SVarun Prakash struct completion comp; 3279187e84f7SVarun Prakash 3280187e84f7SVarun Prakash init_completion(&comp); 3281187e84f7SVarun Prakash 3282187e84f7SVarun Prakash wait_for_completion_interruptible(&comp); 3283187e84f7SVarun Prakash } 3284187e84f7SVarun Prakash 3285b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 3286b8d26b3bSNicholas Bellinger .name = "IB/iSER", 3287b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 3288*bd027d85SNicholas Bellinger .rdma_shutdown = true, 3289d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 3290b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 3291b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 3292b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 3293b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 3294defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 3295b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 3296b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 3297b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 3298b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 3299b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 3300b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 3301b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 3302b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 3303131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 3304187e84f7SVarun Prakash .iscsit_get_rx_pdu = isert_get_rx_pdu, 3305e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3306b8d26b3bSNicholas Bellinger }; 3307b8d26b3bSNicholas Bellinger 3308b8d26b3bSNicholas Bellinger static int __init isert_init(void) 3309b8d26b3bSNicholas Bellinger { 3310b8d26b3bSNicholas Bellinger int ret; 3311b8d26b3bSNicholas Bellinger 3312631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq", 3313631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0); 3314b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 331524f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 3316b8d26b3bSNicholas Bellinger ret = -ENOMEM; 33176f0fae3dSSagi Grimberg return -ENOMEM; 3318b8d26b3bSNicholas Bellinger } 3319b8d26b3bSNicholas Bellinger 3320b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3321b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 3322b02efbfcSSagi Grimberg if (!isert_release_wq) { 332324f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 3324b02efbfcSSagi Grimberg ret = -ENOMEM; 3325b02efbfcSSagi Grimberg goto destroy_comp_wq; 3326b02efbfcSSagi Grimberg } 3327b02efbfcSSagi Grimberg 3328b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 332924f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3330b02efbfcSSagi Grimberg 3331b8d26b3bSNicholas Bellinger return 0; 3332b8d26b3bSNicholas Bellinger 3333b02efbfcSSagi Grimberg destroy_comp_wq: 3334b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 33356f0fae3dSSagi Grimberg 3336b8d26b3bSNicholas Bellinger return ret; 3337b8d26b3bSNicholas Bellinger } 3338b8d26b3bSNicholas Bellinger 3339b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 3340b8d26b3bSNicholas Bellinger { 3341f5ebec96SSagi Grimberg flush_scheduled_work(); 3342b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 3343b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 3344b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 33454c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3346b8d26b3bSNicholas Bellinger } 3347b8d26b3bSNicholas Bellinger 3348b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 33499e35eff4SSagi Grimberg MODULE_VERSION("1.0"); 3350b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3351b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 3352b8d26b3bSNicholas Bellinger 3353b8d26b3bSNicholas Bellinger module_init(isert_init); 3354b8d26b3bSNicholas Bellinger module_exit(isert_exit); 3355