1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2b8d26b3bSNicholas Bellinger /******************************************************************************* 3b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 4b8d26b3bSNicholas Bellinger * 54c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 6b8d26b3bSNicholas Bellinger * 7b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 8b8d26b3bSNicholas Bellinger * 9b8d26b3bSNicholas Bellinger ****************************************************************************/ 10b8d26b3bSNicholas Bellinger 11b8d26b3bSNicholas Bellinger #include <linux/string.h> 12b8d26b3bSNicholas Bellinger #include <linux/module.h> 13b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 14b8d26b3bSNicholas Bellinger #include <linux/socket.h> 15b8d26b3bSNicholas Bellinger #include <linux/in.h> 16b8d26b3bSNicholas Bellinger #include <linux/in6.h> 17b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 188094ba0aSLeon Romanovsky #include <rdma/ib_cm.h> 19b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 20b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 21b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 22b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 23531b7bf4SSagi Grimberg #include <linux/semaphore.h> 24b8d26b3bSNicholas Bellinger 25b8d26b3bSNicholas Bellinger #include "ib_isert.h" 26b8d26b3bSNicholas Bellinger 27b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 28b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 2938a2d0d4SChristoph Hellwig #define ISER_MAX_TX_CQ_LEN \ 3038a2d0d4SChristoph Hellwig ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) 31bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 32bdf20e72SSagi Grimberg ISERT_MAX_CONN) 33b8d26b3bSNicholas Bellinger 3445678b6bSSagi Grimberg static int isert_debug_level; 3524f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 3624f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 3724f412ddSSagi Grimberg 38b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 39b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 40b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 41b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 42b8d26b3bSNicholas Bellinger 43f93f3a70SSagi Grimberg static int 44f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 452371e5daSSagi Grimberg static int 469679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn); 472371e5daSSagi Grimberg static int 482371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 49ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 50d40945d8SVu Pham 512f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work); 529679cc51SChristoph Hellwig static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 539679cc51SChristoph Hellwig static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 549679cc51SChristoph Hellwig static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 559679cc51SChristoph Hellwig static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 562f1b6b7dSSagi Grimberg 57302cc7c3SSagi Grimberg static inline bool 58302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 59302cc7c3SSagi Grimberg { 6023a548eeSSagi Grimberg return (conn->pi_support && 61302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 62302cc7c3SSagi Grimberg } 63302cc7c3SSagi Grimberg 64302cc7c3SSagi Grimberg 65d40945d8SVu Pham static void 66b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 67b8d26b3bSNicholas Bellinger { 686700425eSSagi Grimberg struct isert_conn *isert_conn = context; 69b8d26b3bSNicholas Bellinger 70ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n", 71ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn); 72ea8a1616SSagi Grimberg 73b8d26b3bSNicholas Bellinger switch (e->event) { 74b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 75dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 76b8d26b3bSNicholas Bellinger break; 77b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 784c22e07fSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 79b8d26b3bSNicholas Bellinger break; 80b8d26b3bSNicholas Bellinger default: 81b8d26b3bSNicholas Bellinger break; 82b8d26b3bSNicholas Bellinger } 83b8d26b3bSNicholas Bellinger } 84b8d26b3bSNicholas Bellinger 8540fc069aSSagi Grimberg static struct isert_comp * 8640fc069aSSagi Grimberg isert_comp_get(struct isert_conn *isert_conn) 87b8d26b3bSNicholas Bellinger { 88dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 894a295baeSSagi Grimberg struct isert_comp *comp; 9040fc069aSSagi Grimberg int i, min = 0; 91b8d26b3bSNicholas Bellinger 92b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 934a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 944a295baeSSagi Grimberg if (device->comps[i].active_qps < 954a295baeSSagi Grimberg device->comps[min].active_qps) 964a295baeSSagi Grimberg min = i; 974a295baeSSagi Grimberg comp = &device->comps[min]; 984a295baeSSagi Grimberg comp->active_qps++; 9940fc069aSSagi Grimberg mutex_unlock(&device_list_mutex); 10040fc069aSSagi Grimberg 10124f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1024a295baeSSagi Grimberg isert_conn, comp, min); 10340fc069aSSagi Grimberg 10440fc069aSSagi Grimberg return comp; 10540fc069aSSagi Grimberg } 10640fc069aSSagi Grimberg 10740fc069aSSagi Grimberg static void 10840fc069aSSagi Grimberg isert_comp_put(struct isert_comp *comp) 10940fc069aSSagi Grimberg { 11040fc069aSSagi Grimberg mutex_lock(&device_list_mutex); 11140fc069aSSagi Grimberg comp->active_qps--; 112b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 11340fc069aSSagi Grimberg } 11440fc069aSSagi Grimberg 11540fc069aSSagi Grimberg static struct ib_qp * 11640fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn, 11740fc069aSSagi Grimberg struct isert_comp *comp, 11840fc069aSSagi Grimberg struct rdma_cm_id *cma_id) 11940fc069aSSagi Grimberg { 120dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 12140fc069aSSagi Grimberg struct ib_qp_init_attr attr; 122*317000b9SMax Gurtovoy int ret, factor; 123b8d26b3bSNicholas Bellinger 124b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 125b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 126b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1276f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1286f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 129572a1434SSagi Grimberg attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 130bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 131*317000b9SMax Gurtovoy factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 132*317000b9SMax Gurtovoy ISCSI_ISER_MAX_SG_TABLESIZE); 133*317000b9SMax Gurtovoy attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 13433023fb8SSteve Wise attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 135b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 136b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 137b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 138570db170SSagi Grimberg if (device->pi_capable) 139c0a6cbb9SIsrael Rukshin attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 140b8d26b3bSNicholas Bellinger 14167cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr); 142b8d26b3bSNicholas Bellinger if (ret) { 14324f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 14440fc069aSSagi Grimberg return ERR_PTR(ret); 14540fc069aSSagi Grimberg } 14640fc069aSSagi Grimberg 14740fc069aSSagi Grimberg return cma_id->qp; 14840fc069aSSagi Grimberg } 14940fc069aSSagi Grimberg 15040fc069aSSagi Grimberg static int 15140fc069aSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 15240fc069aSSagi Grimberg { 15340fc069aSSagi Grimberg struct isert_comp *comp; 15440fc069aSSagi Grimberg int ret; 15540fc069aSSagi Grimberg 15640fc069aSSagi Grimberg comp = isert_comp_get(isert_conn); 157dac6ab30SSagi Grimberg isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 158dac6ab30SSagi Grimberg if (IS_ERR(isert_conn->qp)) { 159dac6ab30SSagi Grimberg ret = PTR_ERR(isert_conn->qp); 16019e2090fSSagi Grimberg goto err; 161b8d26b3bSNicholas Bellinger } 162b8d26b3bSNicholas Bellinger 163b8d26b3bSNicholas Bellinger return 0; 16419e2090fSSagi Grimberg err: 16540fc069aSSagi Grimberg isert_comp_put(comp); 16619e2090fSSagi Grimberg return ret; 167b8d26b3bSNicholas Bellinger } 168b8d26b3bSNicholas Bellinger 169b8d26b3bSNicholas Bellinger static int 170b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 171b8d26b3bSNicholas Bellinger { 172dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 17367cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 174b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 175b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 176b8d26b3bSNicholas Bellinger u64 dma_addr; 177b8d26b3bSNicholas Bellinger int i, j; 178b8d26b3bSNicholas Bellinger 1796396bb22SKees Cook isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 1806396bb22SKees Cook sizeof(struct iser_rx_desc), 1816396bb22SKees Cook GFP_KERNEL); 182dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 18393b80f29SLeon Romanovsky return -ENOMEM; 184b8d26b3bSNicholas Bellinger 185dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 186b8d26b3bSNicholas Bellinger 187b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 188b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 189b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 190b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 191b8d26b3bSNicholas Bellinger goto dma_map_fail; 192b8d26b3bSNicholas Bellinger 193b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 194b8d26b3bSNicholas Bellinger 195b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 196b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 197b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 19834efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey; 1999679cc51SChristoph Hellwig rx_desc->rx_cqe.done = isert_recv_done; 200b8d26b3bSNicholas Bellinger } 201b8d26b3bSNicholas Bellinger 202b8d26b3bSNicholas Bellinger return 0; 203b8d26b3bSNicholas Bellinger 204b8d26b3bSNicholas Bellinger dma_map_fail: 205dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 206b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 207b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 208b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 209b8d26b3bSNicholas Bellinger } 210dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 211dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 2124c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 213b8d26b3bSNicholas Bellinger return -ENOMEM; 214b8d26b3bSNicholas Bellinger } 215b8d26b3bSNicholas Bellinger 216b8d26b3bSNicholas Bellinger static void 217b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 218b8d26b3bSNicholas Bellinger { 219dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 220b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 221b8d26b3bSNicholas Bellinger int i; 222b8d26b3bSNicholas Bellinger 223dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 224b8d26b3bSNicholas Bellinger return; 225b8d26b3bSNicholas Bellinger 226dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 227b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 228b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 229b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 230b8d26b3bSNicholas Bellinger } 231b8d26b3bSNicholas Bellinger 232dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 233dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 234b8d26b3bSNicholas Bellinger } 235b8d26b3bSNicholas Bellinger 236172369c5SSagi Grimberg static void 237172369c5SSagi Grimberg isert_free_comps(struct isert_device *device) 238b8d26b3bSNicholas Bellinger { 239172369c5SSagi Grimberg int i; 240b8d26b3bSNicholas Bellinger 241172369c5SSagi Grimberg for (i = 0; i < device->comps_used; i++) { 242172369c5SSagi Grimberg struct isert_comp *comp = &device->comps[i]; 24359464ef4SVu Pham 2449679cc51SChristoph Hellwig if (comp->cq) 2459679cc51SChristoph Hellwig ib_free_cq(comp->cq); 246172369c5SSagi Grimberg } 247172369c5SSagi Grimberg kfree(device->comps); 24859464ef4SVu Pham } 249d40945d8SVu Pham 250172369c5SSagi Grimberg static int 2514a061b28SOr Gerlitz isert_alloc_comps(struct isert_device *device) 252172369c5SSagi Grimberg { 253172369c5SSagi Grimberg int i, max_cqe, ret = 0; 254d3e125daSSagi Grimberg 2554a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2564a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 257172369c5SSagi Grimberg 25824f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 25938a2d0d4SChristoph Hellwig "pi_capable %d\n", 2606c854111SJason Gunthorpe device->comps_used, dev_name(&device->ib_device->dev), 26138a2d0d4SChristoph Hellwig device->ib_device->num_comp_vectors, 262d3e125daSSagi Grimberg device->pi_capable); 2634a295baeSSagi Grimberg 2644a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2654a295baeSSagi Grimberg GFP_KERNEL); 26693b80f29SLeon Romanovsky if (!device->comps) 267b8d26b3bSNicholas Bellinger return -ENOMEM; 268b8d26b3bSNicholas Bellinger 2694a061b28SOr Gerlitz max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 270172369c5SSagi Grimberg 2714a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2724a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 273b8d26b3bSNicholas Bellinger 2744a295baeSSagi Grimberg comp->device = device; 2759679cc51SChristoph Hellwig comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 2769679cc51SChristoph Hellwig IB_POLL_WORKQUEUE); 2776f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 278172369c5SSagi Grimberg isert_err("Unable to allocate cq\n"); 2796f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 2806f0fae3dSSagi Grimberg comp->cq = NULL; 281b8d26b3bSNicholas Bellinger goto out_cq; 28294a71110SWei Yongjun } 283b8d26b3bSNicholas Bellinger } 284b8d26b3bSNicholas Bellinger 285172369c5SSagi Grimberg return 0; 286172369c5SSagi Grimberg out_cq: 287172369c5SSagi Grimberg isert_free_comps(device); 288172369c5SSagi Grimberg return ret; 289172369c5SSagi Grimberg } 290172369c5SSagi Grimberg 291172369c5SSagi Grimberg static int 292172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device) 293172369c5SSagi Grimberg { 2944a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device; 295fd8205e8SSagi Grimberg int ret; 296172369c5SSagi Grimberg 29733023fb8SSteve Wise isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 29833023fb8SSteve Wise ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 2994a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 300172369c5SSagi Grimberg 3014a061b28SOr Gerlitz ret = isert_alloc_comps(device); 302172369c5SSagi Grimberg if (ret) 303373a4cd7SNicholas Bellinger goto out; 304172369c5SSagi Grimberg 305ed082d36SChristoph Hellwig device->pd = ib_alloc_pd(ib_dev, 0); 30667cb3949SSagi Grimberg if (IS_ERR(device->pd)) { 30767cb3949SSagi Grimberg ret = PTR_ERR(device->pd); 30867cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n", 30967cb3949SSagi Grimberg device, ret); 31067cb3949SSagi Grimberg goto out_cq; 31167cb3949SSagi Grimberg } 31267cb3949SSagi Grimberg 313172369c5SSagi Grimberg /* Check signature cap */ 3144a061b28SOr Gerlitz device->pi_capable = ib_dev->attrs.device_cap_flags & 315c0a6cbb9SIsrael Rukshin IB_DEVICE_INTEGRITY_HANDOVER ? true : false; 31667cb3949SSagi Grimberg 317b8d26b3bSNicholas Bellinger return 0; 318b8d26b3bSNicholas Bellinger 319b8d26b3bSNicholas Bellinger out_cq: 320172369c5SSagi Grimberg isert_free_comps(device); 321373a4cd7SNicholas Bellinger out: 322373a4cd7SNicholas Bellinger if (ret > 0) 323373a4cd7SNicholas Bellinger ret = -EINVAL; 324b8d26b3bSNicholas Bellinger return ret; 325b8d26b3bSNicholas Bellinger } 326b8d26b3bSNicholas Bellinger 327b8d26b3bSNicholas Bellinger static void 328b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 329b8d26b3bSNicholas Bellinger { 33024f412ddSSagi Grimberg isert_info("device %p\n", device); 331b8d26b3bSNicholas Bellinger 33267cb3949SSagi Grimberg ib_dealloc_pd(device->pd); 333172369c5SSagi Grimberg isert_free_comps(device); 334b8d26b3bSNicholas Bellinger } 335b8d26b3bSNicholas Bellinger 336b8d26b3bSNicholas Bellinger static void 337cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device) 338b8d26b3bSNicholas Bellinger { 339b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 340b8d26b3bSNicholas Bellinger device->refcount--; 3414c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount); 342b8d26b3bSNicholas Bellinger if (!device->refcount) { 343b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 344b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 345b8d26b3bSNicholas Bellinger kfree(device); 346b8d26b3bSNicholas Bellinger } 347b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 348b8d26b3bSNicholas Bellinger } 349b8d26b3bSNicholas Bellinger 350b8d26b3bSNicholas Bellinger static struct isert_device * 351cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id) 352b8d26b3bSNicholas Bellinger { 353b8d26b3bSNicholas Bellinger struct isert_device *device; 354b8d26b3bSNicholas Bellinger int ret; 355b8d26b3bSNicholas Bellinger 356b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 357b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 358b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 359b8d26b3bSNicholas Bellinger device->refcount++; 3604c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n", 3614c22e07fSSagi Grimberg device, device->refcount); 362b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 363b8d26b3bSNicholas Bellinger return device; 364b8d26b3bSNicholas Bellinger } 365b8d26b3bSNicholas Bellinger } 366b8d26b3bSNicholas Bellinger 367b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 368b8d26b3bSNicholas Bellinger if (!device) { 369b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 370b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 371b8d26b3bSNicholas Bellinger } 372b8d26b3bSNicholas Bellinger 373b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 374b8d26b3bSNicholas Bellinger 375b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 376b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 377b8d26b3bSNicholas Bellinger if (ret) { 378b8d26b3bSNicholas Bellinger kfree(device); 379b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 380b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 381b8d26b3bSNicholas Bellinger } 382b8d26b3bSNicholas Bellinger 383b8d26b3bSNicholas Bellinger device->refcount++; 384b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 3854c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n", 3864c22e07fSSagi Grimberg device, device->refcount); 387b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 388b8d26b3bSNicholas Bellinger 389b8d26b3bSNicholas Bellinger return device; 390b8d26b3bSNicholas Bellinger } 391b8d26b3bSNicholas Bellinger 39259464ef4SVu Pham static void 393ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn) 394ae9ea9edSSagi Grimberg { 395ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT; 396bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node); 397dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp); 398ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp); 39963b268d2SRaju Rangoju init_waitqueue_head(&isert_conn->rem_wait); 400dac6ab30SSagi Grimberg kref_init(&isert_conn->kref); 401dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex); 4022f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 403ae9ea9edSSagi Grimberg } 404ae9ea9edSSagi Grimberg 405ae9ea9edSSagi Grimberg static void 406ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn) 407ae9ea9edSSagi Grimberg { 408dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 409ae9ea9edSSagi Grimberg 410ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 411ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 4125adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4135adabdd1SChristoph Hellwig 414ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4155adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, 416ae9ea9edSSagi Grimberg DMA_FROM_DEVICE); 4175adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 418ae9ea9edSSagi Grimberg } 419ae9ea9edSSagi Grimberg 420ae9ea9edSSagi Grimberg static int 421ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn, 422ae9ea9edSSagi Grimberg struct ib_device *ib_dev) 423ae9ea9edSSagi Grimberg { 424ae9ea9edSSagi Grimberg int ret; 425ae9ea9edSSagi Grimberg 4265adabdd1SChristoph Hellwig isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 4275adabdd1SChristoph Hellwig GFP_KERNEL); 42893b80f29SLeon Romanovsky if (!isert_conn->login_req_buf) 429ae9ea9edSSagi Grimberg return -ENOMEM; 430ae9ea9edSSagi Grimberg 431ae9ea9edSSagi Grimberg isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 4325adabdd1SChristoph Hellwig isert_conn->login_req_buf, 4335adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 434ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 435ae9ea9edSSagi Grimberg if (ret) { 436ae9ea9edSSagi Grimberg isert_err("login_req_dma mapping error: %d\n", ret); 437ae9ea9edSSagi Grimberg isert_conn->login_req_dma = 0; 4385adabdd1SChristoph Hellwig goto out_free_login_req_buf; 4395adabdd1SChristoph Hellwig } 4405adabdd1SChristoph Hellwig 4415adabdd1SChristoph Hellwig isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 4425adabdd1SChristoph Hellwig if (!isert_conn->login_rsp_buf) { 4431d5840c9SWei Yongjun ret = -ENOMEM; 4445adabdd1SChristoph Hellwig goto out_unmap_login_req_buf; 445ae9ea9edSSagi Grimberg } 446ae9ea9edSSagi Grimberg 447ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 4485adabdd1SChristoph Hellwig isert_conn->login_rsp_buf, 449ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 450ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 451ae9ea9edSSagi Grimberg if (ret) { 452ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret); 453ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0; 4545adabdd1SChristoph Hellwig goto out_free_login_rsp_buf; 455ae9ea9edSSagi Grimberg } 456ae9ea9edSSagi Grimberg 457ae9ea9edSSagi Grimberg return 0; 458ae9ea9edSSagi Grimberg 4595adabdd1SChristoph Hellwig out_free_login_rsp_buf: 4605adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4615adabdd1SChristoph Hellwig out_unmap_login_req_buf: 462ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4635adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 4645adabdd1SChristoph Hellwig out_free_login_req_buf: 4655adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 466ae9ea9edSSagi Grimberg return ret; 467ae9ea9edSSagi Grimberg } 468ae9ea9edSSagi Grimberg 469422bd0acSJenny Derzhavetz static void 470422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn, 471422bd0acSJenny Derzhavetz struct rdma_conn_param *param) 472422bd0acSJenny Derzhavetz { 473422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 474422bd0acSJenny Derzhavetz 475422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */ 476422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 477422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom); 478422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 479422bd0acSJenny Derzhavetz 480422bd0acSJenny Derzhavetz if (param->private_data) { 481422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data; 482422bd0acSJenny Derzhavetz 483422bd0acSJenny Derzhavetz /* 484422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator 485422bd0acSJenny Derzhavetz * and the HCA support it 486422bd0acSJenny Derzhavetz */ 487422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 488422bd0acSJenny Derzhavetz (attr->device_cap_flags & 489422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS); 490422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv) 491422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n"); 492422bd0acSJenny Derzhavetz } 493422bd0acSJenny Derzhavetz } 494422bd0acSJenny Derzhavetz 495b8d26b3bSNicholas Bellinger static int 496b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 497b8d26b3bSNicholas Bellinger { 498ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 499ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 500b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 501b8d26b3bSNicholas Bellinger struct isert_device *device; 502b8d26b3bSNicholas Bellinger int ret = 0; 50314f4b54fSSagi Grimberg 50414f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 50514f4b54fSSagi Grimberg if (!np->enabled) { 50614f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 50724f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 5088094ba0aSLeon Romanovsky return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 50914f4b54fSSagi Grimberg } 51014f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 511b8d26b3bSNicholas Bellinger 5124c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n", 513b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 514b8d26b3bSNicholas Bellinger 515b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 516ae9ea9edSSagi Grimberg if (!isert_conn) 517b8d26b3bSNicholas Bellinger return -ENOMEM; 518b8d26b3bSNicholas Bellinger 519ae9ea9edSSagi Grimberg isert_init_conn(isert_conn); 520dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id; 521b8d26b3bSNicholas Bellinger 522ae9ea9edSSagi Grimberg ret = isert_alloc_login_buf(isert_conn, cma_id->device); 523ae9ea9edSSagi Grimberg if (ret) 524b8d26b3bSNicholas Bellinger goto out; 525b8d26b3bSNicholas Bellinger 526cf8ae958SSagi Grimberg device = isert_device_get(cma_id); 527b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 528b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 529b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 530b8d26b3bSNicholas Bellinger } 531dac6ab30SSagi Grimberg isert_conn->device = device; 532b8d26b3bSNicholas Bellinger 533422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn); 5341a92e17eSSagi Grimberg 535570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 536b8d26b3bSNicholas Bellinger if (ret) 537b8d26b3bSNicholas Bellinger goto out_conn_dev; 538b8d26b3bSNicholas Bellinger 5399679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 5402371e5daSSagi Grimberg if (ret) 5412371e5daSSagi Grimberg goto out_conn_dev; 5422371e5daSSagi Grimberg 5432371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 5442371e5daSSagi Grimberg if (ret) 5452371e5daSSagi Grimberg goto out_conn_dev; 5462371e5daSSagi Grimberg 547ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 548bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted); 549ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 550b8d26b3bSNicholas Bellinger 551b8d26b3bSNicholas Bellinger return 0; 552b8d26b3bSNicholas Bellinger 553b8d26b3bSNicholas Bellinger out_conn_dev: 554cf8ae958SSagi Grimberg isert_device_put(device); 555b8d26b3bSNicholas Bellinger out_rsp_dma_map: 556ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 557b8d26b3bSNicholas Bellinger out: 558b8d26b3bSNicholas Bellinger kfree(isert_conn); 5598094ba0aSLeon Romanovsky rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 560b8d26b3bSNicholas Bellinger return ret; 561b8d26b3bSNicholas Bellinger } 562b8d26b3bSNicholas Bellinger 563b8d26b3bSNicholas Bellinger static void 564b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 565b8d26b3bSNicholas Bellinger { 566dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 567b8d26b3bSNicholas Bellinger 5684c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn); 569b8d26b3bSNicholas Bellinger 57057df81e3SSagi Grimberg BUG_ON(!device); 57157df81e3SSagi Grimberg 57219e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 57363b268d2SRaju Rangoju if (isert_conn->cm_id && 57463b268d2SRaju Rangoju !isert_conn->dev_removed) 575dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id); 57619e2090fSSagi Grimberg 577dac6ab30SSagi Grimberg if (isert_conn->qp) { 578dac6ab30SSagi Grimberg struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 5794a295baeSSagi Grimberg 58040fc069aSSagi Grimberg isert_comp_put(comp); 581dac6ab30SSagi Grimberg ib_destroy_qp(isert_conn->qp); 582b8d26b3bSNicholas Bellinger } 583b8d26b3bSNicholas Bellinger 5845adabdd1SChristoph Hellwig if (isert_conn->login_req_buf) 585ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 586ae9ea9edSSagi Grimberg 587cf8ae958SSagi Grimberg isert_device_put(device); 58857df81e3SSagi Grimberg 58963b268d2SRaju Rangoju if (isert_conn->dev_removed) 59063b268d2SRaju Rangoju wake_up_interruptible(&isert_conn->rem_wait); 59163b268d2SRaju Rangoju else 59257df81e3SSagi Grimberg kfree(isert_conn); 593b8d26b3bSNicholas Bellinger } 594b8d26b3bSNicholas Bellinger 595b8d26b3bSNicholas Bellinger static void 596b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 597b8d26b3bSNicholas Bellinger { 59819e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 599bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 600c2f88b17SSagi Grimberg 60124f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 602128e9cc8SSagi Grimberg 603dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 604128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 605bd379220SJenny Derzhavetz kref_get(&isert_conn->kref); 606dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 607bd379220SJenny Derzhavetz 608bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex); 609bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending); 610bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 611bd379220SJenny Derzhavetz 612bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np); 613bd379220SJenny Derzhavetz up(&isert_np->sem); 614b8d26b3bSNicholas Bellinger } 615b8d26b3bSNicholas Bellinger 616b8d26b3bSNicholas Bellinger static void 617dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref) 618b8d26b3bSNicholas Bellinger { 619b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 620dac6ab30SSagi Grimberg struct isert_conn, kref); 621b8d26b3bSNicholas Bellinger 6224c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 6234c22e07fSSagi Grimberg current->pid); 624b8d26b3bSNicholas Bellinger 625b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 626b8d26b3bSNicholas Bellinger } 627b8d26b3bSNicholas Bellinger 628b8d26b3bSNicholas Bellinger static void 629b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 630b8d26b3bSNicholas Bellinger { 631dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref); 632b8d26b3bSNicholas Bellinger } 633b8d26b3bSNicholas Bellinger 6346d1fba0cSJenny Derzhavetz static void 6356d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(struct isert_conn *isert_conn) 6366d1fba0cSJenny Derzhavetz { 6376d1fba0cSJenny Derzhavetz struct isert_np *isert_np = isert_conn->cm_id->context; 6386d1fba0cSJenny Derzhavetz 6396d1fba0cSJenny Derzhavetz mutex_lock(&isert_np->mutex); 6406d1fba0cSJenny Derzhavetz if (!list_empty(&isert_conn->node)) { 6416d1fba0cSJenny Derzhavetz /* 6426d1fba0cSJenny Derzhavetz * This means iscsi doesn't know this connection 6436d1fba0cSJenny Derzhavetz * so schedule a cleanup ourselves 6446d1fba0cSJenny Derzhavetz */ 6456d1fba0cSJenny Derzhavetz list_del_init(&isert_conn->node); 6466d1fba0cSJenny Derzhavetz isert_put_conn(isert_conn); 6476d1fba0cSJenny Derzhavetz queue_work(isert_release_wq, &isert_conn->release_work); 6486d1fba0cSJenny Derzhavetz } 6496d1fba0cSJenny Derzhavetz mutex_unlock(&isert_np->mutex); 6506d1fba0cSJenny Derzhavetz } 6516d1fba0cSJenny Derzhavetz 652954f2372SSagi Grimberg /** 653954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 654954f2372SSagi Grimberg * @isert_conn: isert connection struct 655954f2372SSagi Grimberg * 656954f2372SSagi Grimberg * Notes: 657aea92980SJenny Derzhavetz * In case the connection state is BOUND, move state 658954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 659128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 660954f2372SSagi Grimberg * 661dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is 662954f2372SSagi Grimberg * safe to call multiple times. 663954f2372SSagi Grimberg */ 664954f2372SSagi Grimberg static void 665954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 666954f2372SSagi Grimberg { 667954f2372SSagi Grimberg int err; 668954f2372SSagi Grimberg 6696d1fba0cSJenny Derzhavetz if (isert_conn->state >= ISER_CONN_TERMINATING) 6706d1fba0cSJenny Derzhavetz return; 6716d1fba0cSJenny Derzhavetz 67224f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 673954f2372SSagi Grimberg isert_conn, isert_conn->state); 674128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 675dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id); 676954f2372SSagi Grimberg if (err) 67724f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 678954f2372SSagi Grimberg isert_conn); 679954f2372SSagi Grimberg } 680954f2372SSagi Grimberg 6813b726ae2SSagi Grimberg static int 682ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 683ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 684b8d26b3bSNicholas Bellinger { 685ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n", 686ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np); 6873b726ae2SSagi Grimberg 688ca6c1d82SSagi Grimberg switch (event) { 689ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 690ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 691ca6c1d82SSagi Grimberg break; 692ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 693ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np); 694ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) { 69524f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 696ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id)); 697ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 698ca6c1d82SSagi Grimberg } 699ca6c1d82SSagi Grimberg break; 700ca6c1d82SSagi Grimberg default: 70124f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 702ca6c1d82SSagi Grimberg isert_np, event); 703ca6c1d82SSagi Grimberg } 704ca6c1d82SSagi Grimberg 7053b726ae2SSagi Grimberg return -1; 7063b726ae2SSagi Grimberg } 7073b726ae2SSagi Grimberg 708ca6c1d82SSagi Grimberg static int 709ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 710ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 711ca6c1d82SSagi Grimberg { 712f81bf458SJenny Derzhavetz struct isert_conn *isert_conn = cma_id->qp->qp_context; 713ca6c1d82SSagi Grimberg 714dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 7156d1fba0cSJenny Derzhavetz switch (isert_conn->state) { 7166d1fba0cSJenny Derzhavetz case ISER_CONN_TERMINATING: 7176d1fba0cSJenny Derzhavetz break; 7186d1fba0cSJenny Derzhavetz case ISER_CONN_UP: 719128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 720572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 7216d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(isert_conn); 7226d1fba0cSJenny Derzhavetz break; 7236d1fba0cSJenny Derzhavetz case ISER_CONN_BOUND: 7246d1fba0cSJenny Derzhavetz case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 7256d1fba0cSJenny Derzhavetz iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 7266d1fba0cSJenny Derzhavetz break; 7276d1fba0cSJenny Derzhavetz default: 728506f71d1SColin Ian King isert_warn("conn %p terminating in state %d\n", 7296d1fba0cSJenny Derzhavetz isert_conn, isert_conn->state); 7306d1fba0cSJenny Derzhavetz } 731dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 732128e9cc8SSagi Grimberg 7333b726ae2SSagi Grimberg return 0; 734b8d26b3bSNicholas Bellinger } 735b8d26b3bSNicholas Bellinger 7364a579da2SSagi Grimberg static int 737954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 738954f2372SSagi Grimberg { 73919e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 740954f2372SSagi Grimberg 741cd52cb26SSagi Grimberg ib_drain_qp(isert_conn->qp); 742bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 743dac6ab30SSagi Grimberg isert_conn->cm_id = NULL; 744954f2372SSagi Grimberg isert_put_conn(isert_conn); 7454a579da2SSagi Grimberg 7464a579da2SSagi Grimberg return -1; 747954f2372SSagi Grimberg } 748954f2372SSagi Grimberg 749b8d26b3bSNicholas Bellinger static int 750b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 751b8d26b3bSNicholas Bellinger { 752f81bf458SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 75363b268d2SRaju Rangoju struct isert_conn *isert_conn; 754b8d26b3bSNicholas Bellinger int ret = 0; 755b8d26b3bSNicholas Bellinger 756ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n", 757ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event, 7584c22e07fSSagi Grimberg event->status, cma_id, cma_id->context); 759b8d26b3bSNicholas Bellinger 760f81bf458SJenny Derzhavetz if (isert_np->cm_id == cma_id) 761f81bf458SJenny Derzhavetz return isert_np_cma_handler(cma_id->context, event->event); 762f81bf458SJenny Derzhavetz 763b8d26b3bSNicholas Bellinger switch (event->event) { 764b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 765b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 7663b726ae2SSagi Grimberg if (ret) 7674c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret); 768b8d26b3bSNicholas Bellinger break; 769b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 770b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 771b8d26b3bSNicholas Bellinger break; 77288c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 77388c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 77488c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 775ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 776b8d26b3bSNicholas Bellinger break; 77763b268d2SRaju Rangoju case RDMA_CM_EVENT_DEVICE_REMOVAL: 77863b268d2SRaju Rangoju isert_conn = cma_id->qp->qp_context; 77963b268d2SRaju Rangoju isert_conn->dev_removed = true; 78063b268d2SRaju Rangoju isert_disconnected_handler(cma_id, event->event); 78163b268d2SRaju Rangoju wait_event_interruptible(isert_conn->rem_wait, 78263b268d2SRaju Rangoju isert_conn->state == ISER_CONN_DOWN); 78363b268d2SRaju Rangoju kfree(isert_conn); 78463b268d2SRaju Rangoju /* 78563b268d2SRaju Rangoju * return non-zero from the callback to destroy 78663b268d2SRaju Rangoju * the rdma cm id 78763b268d2SRaju Rangoju */ 78863b268d2SRaju Rangoju return 1; 789d2271bd0SBart Van Assche case RDMA_CM_EVENT_REJECTED: 7901e38a366SSteve Wise isert_info("Connection rejected: %s\n", 7911e38a366SSteve Wise rdma_reject_msg(cma_id, event->status)); 792d2271bd0SBart Van Assche /* fall through */ 793d2271bd0SBart Van Assche case RDMA_CM_EVENT_UNREACHABLE: 794b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 7954a579da2SSagi Grimberg ret = isert_connect_error(cma_id); 796954f2372SSagi Grimberg break; 797b8d26b3bSNicholas Bellinger default: 79824f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 799b8d26b3bSNicholas Bellinger break; 800b8d26b3bSNicholas Bellinger } 801b8d26b3bSNicholas Bellinger 802b8d26b3bSNicholas Bellinger return ret; 803b8d26b3bSNicholas Bellinger } 804b8d26b3bSNicholas Bellinger 805b8d26b3bSNicholas Bellinger static int 8064366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count) 807b8d26b3bSNicholas Bellinger { 808e01a7674SBart Van Assche struct ib_recv_wr *rx_wr; 809b8d26b3bSNicholas Bellinger int i, ret; 810b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 811b8d26b3bSNicholas Bellinger 812dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 8134366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i]; 8149679cc51SChristoph Hellwig 8159679cc51SChristoph Hellwig rx_wr->wr_cqe = &rx_desc->rx_cqe; 816b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 817b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 818b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 8197a56dc88SSagi Grimberg rx_desc->in_use = false; 820b8d26b3bSNicholas Bellinger } 821b8d26b3bSNicholas Bellinger rx_wr--; 822b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 823b8d26b3bSNicholas Bellinger 824e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 8256d1fba0cSJenny Derzhavetz if (ret) 82624f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 8274366b19cSJenny Derzhavetz 8284366b19cSJenny Derzhavetz return ret; 8294366b19cSJenny Derzhavetz } 8304366b19cSJenny Derzhavetz 8314366b19cSJenny Derzhavetz static int 8324366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 8334366b19cSJenny Derzhavetz { 834e01a7674SBart Van Assche struct ib_recv_wr rx_wr; 8354366b19cSJenny Derzhavetz int ret; 8364366b19cSJenny Derzhavetz 8377a56dc88SSagi Grimberg if (!rx_desc->in_use) { 8387a56dc88SSagi Grimberg /* 8397a56dc88SSagi Grimberg * if the descriptor is not in-use we already reposted it 8407a56dc88SSagi Grimberg * for recv, so just silently return 8417a56dc88SSagi Grimberg */ 8427a56dc88SSagi Grimberg return 0; 8437a56dc88SSagi Grimberg } 8447a56dc88SSagi Grimberg 8457a56dc88SSagi Grimberg rx_desc->in_use = false; 8469679cc51SChristoph Hellwig rx_wr.wr_cqe = &rx_desc->rx_cqe; 8474366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg; 8484366b19cSJenny Derzhavetz rx_wr.num_sge = 1; 8494366b19cSJenny Derzhavetz rx_wr.next = NULL; 8504366b19cSJenny Derzhavetz 851e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 8526d1fba0cSJenny Derzhavetz if (ret) 8534366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret); 8544366b19cSJenny Derzhavetz 855b8d26b3bSNicholas Bellinger return ret; 856b8d26b3bSNicholas Bellinger } 857b8d26b3bSNicholas Bellinger 858b8d26b3bSNicholas Bellinger static int 8599679cc51SChristoph Hellwig isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 860b8d26b3bSNicholas Bellinger { 861dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 862e01a7674SBart Van Assche struct ib_send_wr send_wr; 863b8d26b3bSNicholas Bellinger int ret; 864b8d26b3bSNicholas Bellinger 865b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 866b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 867b8d26b3bSNicholas Bellinger 8689679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_login_send_done; 8699679cc51SChristoph Hellwig 870b8d26b3bSNicholas Bellinger send_wr.next = NULL; 8719679cc51SChristoph Hellwig send_wr.wr_cqe = &tx_desc->tx_cqe; 872b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 873b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 874b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 875b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 876b8d26b3bSNicholas Bellinger 877e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 878bdf20e72SSagi Grimberg if (ret) 87924f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 880b8d26b3bSNicholas Bellinger 881b8d26b3bSNicholas Bellinger return ret; 882b8d26b3bSNicholas Bellinger } 883b8d26b3bSNicholas Bellinger 884b8d26b3bSNicholas Bellinger static void 885763b6965SAlex Estrin __isert_create_send_desc(struct isert_device *device, 886b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 887b8d26b3bSNicholas Bellinger { 888b8d26b3bSNicholas Bellinger 889d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 890d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL; 891b8d26b3bSNicholas Bellinger 892b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 893b8d26b3bSNicholas Bellinger 89434efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 89534efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 89624f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 897b8d26b3bSNicholas Bellinger } 898b8d26b3bSNicholas Bellinger } 899b8d26b3bSNicholas Bellinger 900763b6965SAlex Estrin static void 901763b6965SAlex Estrin isert_create_send_desc(struct isert_conn *isert_conn, 902763b6965SAlex Estrin struct isert_cmd *isert_cmd, 903763b6965SAlex Estrin struct iser_tx_desc *tx_desc) 904763b6965SAlex Estrin { 905763b6965SAlex Estrin struct isert_device *device = isert_conn->device; 906763b6965SAlex Estrin struct ib_device *ib_dev = device->ib_device; 907763b6965SAlex Estrin 908763b6965SAlex Estrin ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 909763b6965SAlex Estrin ISER_HEADERS_LEN, DMA_TO_DEVICE); 910763b6965SAlex Estrin 911763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc); 912763b6965SAlex Estrin } 913763b6965SAlex Estrin 914b8d26b3bSNicholas Bellinger static int 915b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 916b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 917b8d26b3bSNicholas Bellinger { 918dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 91967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 920b8d26b3bSNicholas Bellinger u64 dma_addr; 921b8d26b3bSNicholas Bellinger 922b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 923b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 924b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 92524f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 926b8d26b3bSNicholas Bellinger return -ENOMEM; 927b8d26b3bSNicholas Bellinger } 928b8d26b3bSNicholas Bellinger 929b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 930b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 931b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 93234efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 933b8d26b3bSNicholas Bellinger 9344c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 9354c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 9364c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey); 937b8d26b3bSNicholas Bellinger 938b8d26b3bSNicholas Bellinger return 0; 939b8d26b3bSNicholas Bellinger } 940b8d26b3bSNicholas Bellinger 941b8d26b3bSNicholas Bellinger static void 94295b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 94368a86deeSSagi Grimberg struct ib_send_wr *send_wr) 944b8d26b3bSNicholas Bellinger { 94595b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 94695b60f07SNicholas Bellinger 9479679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_send_done; 9489679cc51SChristoph Hellwig send_wr->wr_cqe = &tx_desc->tx_cqe; 949422bd0acSJenny Derzhavetz 950422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 951422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV; 952422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 953422bd0acSJenny Derzhavetz } else { 954b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 955422bd0acSJenny Derzhavetz } 956422bd0acSJenny Derzhavetz 95795b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 958b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 95995b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 960b8d26b3bSNicholas Bellinger } 961b8d26b3bSNicholas Bellinger 962b8d26b3bSNicholas Bellinger static int 9639679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn) 964b8d26b3bSNicholas Bellinger { 965e01a7674SBart Van Assche struct ib_recv_wr rx_wr; 966b8d26b3bSNicholas Bellinger struct ib_sge sge; 967b8d26b3bSNicholas Bellinger int ret; 968b8d26b3bSNicholas Bellinger 969b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 970b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 971ed1083b2SChristoph Hellwig sge.length = ISER_RX_PAYLOAD_SIZE; 97234efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey; 973b8d26b3bSNicholas Bellinger 97424f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 975b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 976b8d26b3bSNicholas Bellinger 9779679cc51SChristoph Hellwig isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 9789679cc51SChristoph Hellwig 979b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 9809679cc51SChristoph Hellwig rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 981b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 982b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 983b8d26b3bSNicholas Bellinger 984e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 9856d1fba0cSJenny Derzhavetz if (ret) 98624f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 987b8d26b3bSNicholas Bellinger 988b8d26b3bSNicholas Bellinger return ret; 989b8d26b3bSNicholas Bellinger } 990b8d26b3bSNicholas Bellinger 991b8d26b3bSNicholas Bellinger static int 992b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 993b8d26b3bSNicholas Bellinger u32 length) 994b8d26b3bSNicholas Bellinger { 995b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 996dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 99767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 998dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 999b8d26b3bSNicholas Bellinger int ret; 1000b8d26b3bSNicholas Bellinger 1001763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc); 1002b8d26b3bSNicholas Bellinger 1003b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1004b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1005b8d26b3bSNicholas Bellinger 1006b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1007b8d26b3bSNicholas Bellinger 1008b8d26b3bSNicholas Bellinger if (length > 0) { 1009b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1010b8d26b3bSNicholas Bellinger 1011b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1012b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1013b8d26b3bSNicholas Bellinger 1014b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1015b8d26b3bSNicholas Bellinger 1016b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1017b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1018b8d26b3bSNicholas Bellinger 1019b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1020b8d26b3bSNicholas Bellinger tx_dsg->length = length; 102134efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1022b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1023b8d26b3bSNicholas Bellinger } 1024b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1025b8d26b3bSNicholas Bellinger if (login->login_complete) { 1026b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1027b8d26b3bSNicholas Bellinger if (ret) 1028b8d26b3bSNicholas Bellinger return ret; 1029b8d26b3bSNicholas Bellinger 10304366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn, 10314366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS); 1032b8d26b3bSNicholas Bellinger if (ret) 1033b8d26b3bSNicholas Bellinger return ret; 1034b8d26b3bSNicholas Bellinger 1035128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 1036dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 1037128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 1038dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1039b8d26b3bSNicholas Bellinger goto post_send; 1040b8d26b3bSNicholas Bellinger } 1041b8d26b3bSNicholas Bellinger 10429679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 1043b8d26b3bSNicholas Bellinger if (ret) 1044b8d26b3bSNicholas Bellinger return ret; 1045b8d26b3bSNicholas Bellinger } 1046b8d26b3bSNicholas Bellinger post_send: 10479679cc51SChristoph Hellwig ret = isert_login_post_send(isert_conn, tx_desc); 1048b8d26b3bSNicholas Bellinger if (ret) 1049b8d26b3bSNicholas Bellinger return ret; 1050b8d26b3bSNicholas Bellinger 1051b8d26b3bSNicholas Bellinger return 0; 1052b8d26b3bSNicholas Bellinger } 1053b8d26b3bSNicholas Bellinger 1054b8d26b3bSNicholas Bellinger static void 10552371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1056b8d26b3bSNicholas Bellinger { 10575adabdd1SChristoph Hellwig struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 10582371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1059b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1060b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1061b8d26b3bSNicholas Bellinger int size; 1062b8d26b3bSNicholas Bellinger 106324f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 10642371e5daSSagi Grimberg 10652371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1066b8d26b3bSNicholas Bellinger 1067b8d26b3bSNicholas Bellinger if (login->first_request) { 1068b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1069b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1070b8d26b3bSNicholas Bellinger /* 1071b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1072b8d26b3bSNicholas Bellinger * login request PDU. 1073b8d26b3bSNicholas Bellinger */ 1074b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1075b8d26b3bSNicholas Bellinger login->current_stage = 1076b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1077b8d26b3bSNicholas Bellinger >> 2; 1078b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1079b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1080b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1081b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1082b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1083b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1084b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1085b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1086b8d26b3bSNicholas Bellinger } 1087b8d26b3bSNicholas Bellinger 1088b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1089b8d26b3bSNicholas Bellinger 1090b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 10914c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d " 10924c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 10934c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS); 1094b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1095b8d26b3bSNicholas Bellinger 10966faaa85fSNicholas Bellinger if (login->first_request) { 1097dac6ab30SSagi Grimberg complete(&isert_conn->login_comp); 10986faaa85fSNicholas Bellinger return; 10996faaa85fSNicholas Bellinger } 11006faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1101b8d26b3bSNicholas Bellinger } 1102b8d26b3bSNicholas Bellinger 1103b8d26b3bSNicholas Bellinger static struct iscsi_cmd 11044366b19cSJenny Derzhavetz *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1105b8d26b3bSNicholas Bellinger { 11066700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1107b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1108d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1109b8d26b3bSNicholas Bellinger 1110676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1111d703ce2fSNicholas Bellinger if (!cmd) { 111224f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1113b8d26b3bSNicholas Bellinger return NULL; 1114b8d26b3bSNicholas Bellinger } 1115d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1116b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1117d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 11184366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc; 1119b8d26b3bSNicholas Bellinger 1120d703ce2fSNicholas Bellinger return cmd; 1121b8d26b3bSNicholas Bellinger } 1122b8d26b3bSNicholas Bellinger 1123b8d26b3bSNicholas Bellinger static int 1124b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1125d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1126d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1127b8d26b3bSNicholas Bellinger { 1128b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1129b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1130b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1131b8d26b3bSNicholas Bellinger bool dump_payload = false; 11329fd60088SJenny Derzhavetz unsigned int data_len; 1133b8d26b3bSNicholas Bellinger 1134b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1135b8d26b3bSNicholas Bellinger if (rc < 0) 1136b8d26b3bSNicholas Bellinger return rc; 1137b8d26b3bSNicholas Bellinger 1138b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1139b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1140b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 11419fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length; 1142b8d26b3bSNicholas Bellinger 11439fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len) 11449fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1145b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1146b8d26b3bSNicholas Bellinger if (rc < 0) { 1147b8d26b3bSNicholas Bellinger return 0; 1148b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1149b8d26b3bSNicholas Bellinger dump_payload = true; 1150b8d26b3bSNicholas Bellinger goto sequence_cmd; 1151b8d26b3bSNicholas Bellinger } 1152b8d26b3bSNicholas Bellinger 1153b8d26b3bSNicholas Bellinger if (!imm_data) 1154b8d26b3bSNicholas Bellinger return 0; 1155b8d26b3bSNicholas Bellinger 11569fd60088SJenny Derzhavetz if (imm_data_len != data_len) { 1157b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 11589fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 11599fd60088SJenny Derzhavetz &rx_desc->data[0], imm_data_len); 11609fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 11619fd60088SJenny Derzhavetz sg_nents, imm_data_len); 11629fd60088SJenny Derzhavetz } else { 11639fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1); 11649fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg; 11659fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1; 11669fd60088SJenny Derzhavetz sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 11679fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n", 11689fd60088SJenny Derzhavetz imm_data_len); 11699fd60088SJenny Derzhavetz } 1170b8d26b3bSNicholas Bellinger 1171b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1172b8d26b3bSNicholas Bellinger 1173b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1174b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1175b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1176b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1177b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1178b8d26b3bSNicholas Bellinger } 1179b8d26b3bSNicholas Bellinger 1180b8d26b3bSNicholas Bellinger sequence_cmd: 1181561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1182b8d26b3bSNicholas Bellinger 1183b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 11840300b114SBart Van Assche iscsit_set_unsolicited_dataout(cmd); 11856cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 1186afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd); 1187b8d26b3bSNicholas Bellinger 1188b8d26b3bSNicholas Bellinger return 0; 1189b8d26b3bSNicholas Bellinger } 1190b8d26b3bSNicholas Bellinger 1191b8d26b3bSNicholas Bellinger static int 1192b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1193b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1194b8d26b3bSNicholas Bellinger { 1195b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1196b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1197b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1198b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1199b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1200b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1201b8d26b3bSNicholas Bellinger 1202b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1203b8d26b3bSNicholas Bellinger if (rc < 0) 1204b8d26b3bSNicholas Bellinger return rc; 1205b8d26b3bSNicholas Bellinger else if (!cmd) 1206b8d26b3bSNicholas Bellinger return 0; 1207b8d26b3bSNicholas Bellinger /* 1208b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1209b8d26b3bSNicholas Bellinger */ 1210b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 121124f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1212b8d26b3bSNicholas Bellinger dump_stack(); 1213b8d26b3bSNicholas Bellinger return -1; 1214b8d26b3bSNicholas Bellinger } 1215b8d26b3bSNicholas Bellinger 12164c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 12174c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n", 12184c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done, 12194c22e07fSSagi Grimberg cmd->se_cmd.data_length); 1220b8d26b3bSNicholas Bellinger 1221b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1222b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1223b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1224b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1225b8d26b3bSNicholas Bellinger /* 1226b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1227b8d26b3bSNicholas Bellinger */ 1228b8d26b3bSNicholas Bellinger if (page_off) { 12294c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n"); 1230b8d26b3bSNicholas Bellinger dump_stack(); 1231b8d26b3bSNicholas Bellinger return -1; 1232b8d26b3bSNicholas Bellinger } 12334c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 12344c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off, 12354c22e07fSSagi Grimberg sg_nents, &rx_desc->data[0], unsol_data_len); 1236b8d26b3bSNicholas Bellinger 1237b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1238b8d26b3bSNicholas Bellinger unsol_data_len); 1239b8d26b3bSNicholas Bellinger 1240b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1241b8d26b3bSNicholas Bellinger if (rc < 0) 1242b8d26b3bSNicholas Bellinger return rc; 1243b8d26b3bSNicholas Bellinger 12444366b19cSJenny Derzhavetz /* 12454366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive - 12464366b19cSJenny Derzhavetz * so post the buffer before hand 12474366b19cSJenny Derzhavetz */ 12484366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, rx_desc); 12494366b19cSJenny Derzhavetz if (rc) { 12504366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 12514366b19cSJenny Derzhavetz return rc; 12524366b19cSJenny Derzhavetz } 1253b8d26b3bSNicholas Bellinger return 0; 1254b8d26b3bSNicholas Bellinger } 1255b8d26b3bSNicholas Bellinger 1256b8d26b3bSNicholas Bellinger static int 1257778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1258d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1259d703ce2fSNicholas Bellinger unsigned char *buf) 1260778de368SNicholas Bellinger { 1261778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1262778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1263778de368SNicholas Bellinger int rc; 1264778de368SNicholas Bellinger 1265778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1266778de368SNicholas Bellinger if (rc < 0) 1267778de368SNicholas Bellinger return rc; 1268778de368SNicholas Bellinger /* 1269778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1270778de368SNicholas Bellinger */ 1271778de368SNicholas Bellinger 1272778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1273778de368SNicholas Bellinger } 1274778de368SNicholas Bellinger 1275778de368SNicholas Bellinger static int 1276adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1277d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1278d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1279adb54c29SNicholas Bellinger { 1280adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1281adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1282adb54c29SNicholas Bellinger int rc; 1283b44a2b67SSagi Grimberg unsigned char *text_in = NULL; 1284adb54c29SNicholas Bellinger 1285adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1286adb54c29SNicholas Bellinger if (rc < 0) 1287adb54c29SNicholas Bellinger return rc; 1288adb54c29SNicholas Bellinger 1289b44a2b67SSagi Grimberg if (payload_length) { 1290adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 129193b80f29SLeon Romanovsky if (!text_in) 1292adb54c29SNicholas Bellinger return -ENOMEM; 1293adb54c29SNicholas Bellinger } 1294adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1295adb54c29SNicholas Bellinger 1296adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1297adb54c29SNicholas Bellinger 1298adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1299adb54c29SNicholas Bellinger } 1300adb54c29SNicholas Bellinger 1301adb54c29SNicholas Bellinger static int 1302b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1303b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1304b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1305b8d26b3bSNicholas Bellinger { 1306b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1307b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1308b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1309b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1310b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1311b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1312b8d26b3bSNicholas Bellinger 1313fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType && 1314ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 131524f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1316ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1317ca40d24eSNicholas Bellinger return 0; 1318ca40d24eSNicholas Bellinger } 1319ca40d24eSNicholas Bellinger 1320b8d26b3bSNicholas Bellinger switch (opcode) { 1321b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 13224366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1323b8d26b3bSNicholas Bellinger if (!cmd) 1324b8d26b3bSNicholas Bellinger break; 1325b8d26b3bSNicholas Bellinger 1326d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1327b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1328b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1329b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1330b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1331422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1332b8d26b3bSNicholas Bellinger 1333d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1334b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1335b8d26b3bSNicholas Bellinger break; 1336b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 13374366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1338b8d26b3bSNicholas Bellinger if (!cmd) 1339b8d26b3bSNicholas Bellinger break; 1340b8d26b3bSNicholas Bellinger 1341d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1342d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1343778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1344b8d26b3bSNicholas Bellinger break; 1345b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1346b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1347b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1348b8d26b3bSNicholas Bellinger break; 1349b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 13504366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1351b8d26b3bSNicholas Bellinger if (!cmd) 1352b8d26b3bSNicholas Bellinger break; 1353b8d26b3bSNicholas Bellinger 1354b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1355b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1356b8d26b3bSNicholas Bellinger break; 1357b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 13584366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1359b8d26b3bSNicholas Bellinger if (!cmd) 1360b8d26b3bSNicholas Bellinger break; 1361b8d26b3bSNicholas Bellinger 1362b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1363b8d26b3bSNicholas Bellinger break; 1364adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 13654366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1366e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 13674366b19cSJenny Derzhavetz else 13684366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 13694366b19cSJenny Derzhavetz 1370e4f4e801SSagi Grimberg if (!cmd) 1371e4f4e801SSagi Grimberg break; 1372adb54c29SNicholas Bellinger 1373d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1374d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1375adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1376adb54c29SNicholas Bellinger break; 1377b8d26b3bSNicholas Bellinger default: 137824f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1379b8d26b3bSNicholas Bellinger dump_stack(); 1380b8d26b3bSNicholas Bellinger break; 1381b8d26b3bSNicholas Bellinger } 1382b8d26b3bSNicholas Bellinger 1383b8d26b3bSNicholas Bellinger return ret; 1384b8d26b3bSNicholas Bellinger } 1385b8d26b3bSNicholas Bellinger 1386b8d26b3bSNicholas Bellinger static void 13879679cc51SChristoph Hellwig isert_print_wc(struct ib_wc *wc, const char *type) 1388b8d26b3bSNicholas Bellinger { 13899679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 13909679cc51SChristoph Hellwig isert_err("%s failure: %s (%d) vend_err %x\n", type, 13919679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status, 13929679cc51SChristoph Hellwig wc->vendor_err); 13939679cc51SChristoph Hellwig else 13949679cc51SChristoph Hellwig isert_dbg("%s failure: %s (%d)\n", type, 13959679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status); 13969679cc51SChristoph Hellwig } 13979679cc51SChristoph Hellwig 13989679cc51SChristoph Hellwig static void 13999679cc51SChristoph Hellwig isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 14009679cc51SChristoph Hellwig { 14019679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 14029679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 14039679cc51SChristoph Hellwig struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 14049679cc51SChristoph Hellwig struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1405d3cf81f9SSagi Grimberg struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1406b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1407b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1408b8d26b3bSNicholas Bellinger 14099679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14109679cc51SChristoph Hellwig isert_print_wc(wc, "recv"); 14119679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 14129679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 14139679cc51SChristoph Hellwig return; 14149679cc51SChristoph Hellwig } 14159679cc51SChristoph Hellwig 14167a56dc88SSagi Grimberg rx_desc->in_use = true; 14177a56dc88SSagi Grimberg 14189679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 14199679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 14209679cc51SChristoph Hellwig 14219679cc51SChristoph Hellwig isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 14229679cc51SChristoph Hellwig rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 14239679cc51SChristoph Hellwig (int)(wc->byte_len - ISER_HEADERS_LEN)); 14249679cc51SChristoph Hellwig 1425d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) { 1426b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1427d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) { 1428d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag); 1429d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va); 14304c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1431b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1432b8d26b3bSNicholas Bellinger } 1433d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) { 1434d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag); 1435d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va); 14364c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1437b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1438b8d26b3bSNicholas Bellinger } 1439b8d26b3bSNicholas Bellinger 144024f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1441b8d26b3bSNicholas Bellinger break; 1442b8d26b3bSNicholas Bellinger case ISER_HELLO: 144324f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1444b8d26b3bSNicholas Bellinger break; 1445b8d26b3bSNicholas Bellinger default: 1446d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1447b8d26b3bSNicholas Bellinger break; 1448b8d26b3bSNicholas Bellinger } 1449b8d26b3bSNicholas Bellinger 14506c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc, 1451b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 14529679cc51SChristoph Hellwig 14539679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 14549679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1455b8d26b3bSNicholas Bellinger } 1456b8d26b3bSNicholas Bellinger 1457b8d26b3bSNicholas Bellinger static void 14589679cc51SChristoph Hellwig isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1459b8d26b3bSNicholas Bellinger { 14609679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1461fce50a2fSNicholas Bellinger struct ib_device *ib_dev = isert_conn->device->ib_device; 1462b8d26b3bSNicholas Bellinger 14639679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14649679cc51SChristoph Hellwig isert_print_wc(wc, "login recv"); 14659679cc51SChristoph Hellwig return; 1466b8d26b3bSNicholas Bellinger } 1467b8d26b3bSNicholas Bellinger 14689679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 14699679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1470b8d26b3bSNicholas Bellinger 14719679cc51SChristoph Hellwig isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1472b8d26b3bSNicholas Bellinger 14732371e5daSSagi Grimberg if (isert_conn->conn) { 14742371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 14752371e5daSSagi Grimberg 14762371e5daSSagi Grimberg if (login && !login->first_request) 14772371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 14782371e5daSSagi Grimberg } 14799679cc51SChristoph Hellwig 1480dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 14812371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 1482dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1483b8d26b3bSNicholas Bellinger 14849679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 14859679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1486b8d26b3bSNicholas Bellinger } 1487b8d26b3bSNicholas Bellinger 1488e3d7e4c3SSagi Grimberg static void 148938a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1490e3d7e4c3SSagi Grimberg { 149138a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 149238a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1493e3d7e4c3SSagi Grimberg 149438a2d0d4SChristoph Hellwig if (!cmd->rw.nr_ops) 149538a2d0d4SChristoph Hellwig return; 149638a2d0d4SChristoph Hellwig 149738a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 149838a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 149938a2d0d4SChristoph Hellwig conn->cm_id->port_num, se_cmd->t_data_sg, 150038a2d0d4SChristoph Hellwig se_cmd->t_data_nents, se_cmd->t_prot_sg, 150138a2d0d4SChristoph Hellwig se_cmd->t_prot_nents, dir); 150238a2d0d4SChristoph Hellwig } else { 150338a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 150438a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1505e3d7e4c3SSagi Grimberg } 1506e3d7e4c3SSagi Grimberg 150738a2d0d4SChristoph Hellwig cmd->rw.nr_ops = 0; 150859464ef4SVu Pham } 150959464ef4SVu Pham 151059464ef4SVu Pham static void 151103e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1512b8d26b3bSNicholas Bellinger { 1513d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1514b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1515186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1516e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr; 1517b8d26b3bSNicholas Bellinger 15184c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1519b8d26b3bSNicholas Bellinger 1520b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1521b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1522b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1523b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15245159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1525b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1526b8d26b3bSNicholas Bellinger 152703e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1528b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 152903e7848aSNicholas Bellinger /* 153003e7848aSNicholas Bellinger * Check for special case during comp_err where 153103e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 153203e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 153303e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 153403e7848aSNicholas Bellinger */ 153503e7848aSNicholas Bellinger if (comp_err && 153603e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 153703e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 153803e7848aSNicholas Bellinger 1539afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 154003e7848aSNicholas Bellinger } 154103e7848aSNicholas Bellinger } 1542b8d26b3bSNicholas Bellinger 154338a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1544186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1545186a9647SNicholas Bellinger break; 1546b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1547186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1548186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15495159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1550186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1551186a9647SNicholas Bellinger 1552b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1553b8d26b3bSNicholas Bellinger break; 1554b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1555b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1556adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1557e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1558e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */ 1559e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1560e4f4e801SSagi Grimberg break; 1561e4f4e801SSagi Grimberg 1562b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1563b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15645159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1565b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1566b8d26b3bSNicholas Bellinger 1567b8d26b3bSNicholas Bellinger /* 1568b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1569b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1570b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1571b8d26b3bSNicholas Bellinger */ 1572b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 157311378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 15743df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1575b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1576b8d26b3bSNicholas Bellinger break; 1577b8d26b3bSNicholas Bellinger } 1578d2271bd0SBart Van Assche /* fall through */ 1579b8d26b3bSNicholas Bellinger default: 1580d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1581b8d26b3bSNicholas Bellinger break; 1582b8d26b3bSNicholas Bellinger } 1583b8d26b3bSNicholas Bellinger } 1584b8d26b3bSNicholas Bellinger 1585b8d26b3bSNicholas Bellinger static void 1586b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1587b8d26b3bSNicholas Bellinger { 1588b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 15894c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n"); 1590b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1591b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1592b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1593b8d26b3bSNicholas Bellinger } 1594b8d26b3bSNicholas Bellinger } 1595b8d26b3bSNicholas Bellinger 1596b8d26b3bSNicholas Bellinger static void 1597b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 159803e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1599b8d26b3bSNicholas Bellinger { 1600dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 16014c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1602dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1603dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1604dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1605b8d26b3bSNicholas Bellinger } 1606b8d26b3bSNicholas Bellinger 1607b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 160803e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1609b8d26b3bSNicholas Bellinger } 1610b8d26b3bSNicholas Bellinger 161196b7973eSSagi Grimberg static int 161296b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1613f93f3a70SSagi Grimberg { 16149e961ae7SSagi Grimberg struct ib_mr_status mr_status; 161596b7973eSSagi Grimberg int ret; 1616f93f3a70SSagi Grimberg 161796b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 16189e961ae7SSagi Grimberg if (ret) { 161924f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 16209e961ae7SSagi Grimberg goto fail_mr_status; 16219e961ae7SSagi Grimberg } 162296b7973eSSagi Grimberg 16239e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 162496b7973eSSagi Grimberg u64 sec_offset_err; 16259e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 16269e961ae7SSagi Grimberg 16279e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 16289e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 16299e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 16309e961ae7SSagi Grimberg break; 16319e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 16329e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 16339e961ae7SSagi Grimberg break; 16349e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 16359e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 16369e961ae7SSagi Grimberg break; 16379e961ae7SSagi Grimberg } 163896b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 163996b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 164096b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 16419e961ae7SSagi Grimberg 16424c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx " 16439e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 16449e961ae7SSagi Grimberg mr_status.sig_err.err_type, 16459e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 16469e961ae7SSagi Grimberg mr_status.sig_err.expected, 16479e961ae7SSagi Grimberg mr_status.sig_err.actual); 16489e961ae7SSagi Grimberg ret = 1; 16499e961ae7SSagi Grimberg } 16509e961ae7SSagi Grimberg 16519e961ae7SSagi Grimberg fail_mr_status: 165296b7973eSSagi Grimberg return ret; 165396b7973eSSagi Grimberg } 165496b7973eSSagi Grimberg 165596b7973eSSagi Grimberg static void 16569679cc51SChristoph Hellwig isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 165796b7973eSSagi Grimberg { 16589679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1659dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 16609679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1661e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 16629679cc51SChristoph Hellwig struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 166396b7973eSSagi Grimberg int ret = 0; 166496b7973eSSagi Grimberg 16659679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 16669679cc51SChristoph Hellwig isert_print_wc(wc, "rdma write"); 16679679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 16689679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 16699679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 16709679cc51SChristoph Hellwig return; 16719679cc51SChristoph Hellwig } 16729679cc51SChristoph Hellwig 16739679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 16749679cc51SChristoph Hellwig 1675e9a53e73SIsrael Rukshin ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 167638a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 167796b7973eSSagi Grimberg 1678555a65f6SNicholas Bellinger if (ret) { 1679555a65f6SNicholas Bellinger /* 1680555a65f6SNicholas Bellinger * transport_generic_request_failure() expects to have 1681555a65f6SNicholas Bellinger * plus two references to handle queue-full, so re-add 1682555a65f6SNicholas Bellinger * one here as target-core will have already dropped 1683555a65f6SNicholas Bellinger * it after the first isert_put_datain() callback. 1684555a65f6SNicholas Bellinger */ 1685555a65f6SNicholas Bellinger kref_get(&cmd->cmd_kref); 1686555a65f6SNicholas Bellinger transport_generic_request_failure(cmd, cmd->pi_err); 1687555a65f6SNicholas Bellinger } else { 1688555a65f6SNicholas Bellinger /* 1689555a65f6SNicholas Bellinger * XXX: isert_put_response() failure is not retried. 1690555a65f6SNicholas Bellinger */ 1691555a65f6SNicholas Bellinger ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 16929e961ae7SSagi Grimberg if (ret) 1693555a65f6SNicholas Bellinger pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1694555a65f6SNicholas Bellinger } 1695f93f3a70SSagi Grimberg } 1696f93f3a70SSagi Grimberg 1697f93f3a70SSagi Grimberg static void 16989679cc51SChristoph Hellwig isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1699b8d26b3bSNicholas Bellinger { 17009679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17019679cc51SChristoph Hellwig struct isert_device *device = isert_conn->device; 17029679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1703e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1704d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1705b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 17065bac4b1aSSagi Grimberg int ret = 0; 1707b8d26b3bSNicholas Bellinger 17089679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17099679cc51SChristoph Hellwig isert_print_wc(wc, "rdma read"); 17109679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17119679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17129679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 17139679cc51SChristoph Hellwig return; 17149679cc51SChristoph Hellwig } 17159679cc51SChristoph Hellwig 17169679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 17179679cc51SChristoph Hellwig 1718b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 171938a2d0d4SChristoph Hellwig 172038a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) 1721e9a53e73SIsrael Rukshin ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 172238a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 172338a2d0d4SChristoph Hellwig cmd->write_data_done = 0; 1724b8d26b3bSNicholas Bellinger 172524f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1726b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1727b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1728b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1729b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1730b8d26b3bSNicholas Bellinger 1731555a65f6SNicholas Bellinger /* 1732555a65f6SNicholas Bellinger * transport_generic_request_failure() will drop the extra 1733555a65f6SNicholas Bellinger * se_cmd->cmd_kref reference after T10-PI error, and handle 1734555a65f6SNicholas Bellinger * any non-zero ->queue_status() callback error retries. 1735555a65f6SNicholas Bellinger */ 1736555a65f6SNicholas Bellinger if (ret) 1737555a65f6SNicholas Bellinger transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1738555a65f6SNicholas Bellinger else 1739b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1740b8d26b3bSNicholas Bellinger } 1741b8d26b3bSNicholas Bellinger 1742b8d26b3bSNicholas Bellinger static void 1743b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1744b8d26b3bSNicholas Bellinger { 1745b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1746b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1747b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1748dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1749d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1750b8d26b3bSNicholas Bellinger 17514c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 17524c22e07fSSagi Grimberg 1753b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1754b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1755b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1756d2271bd0SBart Van Assche /* fall through */ 1757d2271bd0SBart Van Assche case ISTATE_SEND_REJECT: 1758d2271bd0SBart Van Assche case ISTATE_SEND_TEXTRSP: 1759b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 17604c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 17614c22e07fSSagi Grimberg ib_dev, false); 17623df8f68aSNicholas Bellinger break; 1763b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1764b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1765b8d26b3bSNicholas Bellinger break; 1766b8d26b3bSNicholas Bellinger default: 17674c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state); 1768b8d26b3bSNicholas Bellinger dump_stack(); 1769b8d26b3bSNicholas Bellinger break; 1770b8d26b3bSNicholas Bellinger } 1771b8d26b3bSNicholas Bellinger } 1772b8d26b3bSNicholas Bellinger 1773b8d26b3bSNicholas Bellinger static void 17749679cc51SChristoph Hellwig isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1775b8d26b3bSNicholas Bellinger { 17769679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17779679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17789679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1779b8d26b3bSNicholas Bellinger 17809679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17819679cc51SChristoph Hellwig isert_print_wc(wc, "login send"); 17829679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17839679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17849679cc51SChristoph Hellwig } 17859679cc51SChristoph Hellwig 17869679cc51SChristoph Hellwig isert_unmap_tx_desc(tx_desc, ib_dev); 17879679cc51SChristoph Hellwig } 17889679cc51SChristoph Hellwig 17899679cc51SChristoph Hellwig static void 17909679cc51SChristoph Hellwig isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 17919679cc51SChristoph Hellwig { 17929679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17939679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17949679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1795e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 17969679cc51SChristoph Hellwig 17979679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17989679cc51SChristoph Hellwig isert_print_wc(wc, "send"); 17999679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 18009679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 18019679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 18029679cc51SChristoph Hellwig return; 18039679cc51SChristoph Hellwig } 18049679cc51SChristoph Hellwig 18059679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 18069679cc51SChristoph Hellwig 18079679cc51SChristoph Hellwig switch (isert_cmd->iscsi_cmd->i_state) { 18089679cc51SChristoph Hellwig case ISTATE_SEND_TASKMGTRSP: 18099679cc51SChristoph Hellwig case ISTATE_SEND_LOGOUTRSP: 18109679cc51SChristoph Hellwig case ISTATE_SEND_REJECT: 18119679cc51SChristoph Hellwig case ISTATE_SEND_TEXTRSP: 1812b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1813b8d26b3bSNicholas Bellinger 1814b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1815b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1816b8d26b3bSNicholas Bellinger return; 1817b8d26b3bSNicholas Bellinger default: 18189679cc51SChristoph Hellwig isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 18199679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1820b8d26b3bSNicholas Bellinger break; 1821b8d26b3bSNicholas Bellinger } 1822b8d26b3bSNicholas Bellinger } 1823b8d26b3bSNicholas Bellinger 1824b8d26b3bSNicholas Bellinger static int 1825b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1826b8d26b3bSNicholas Bellinger { 1827b8d26b3bSNicholas Bellinger int ret; 1828b8d26b3bSNicholas Bellinger 18294366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 18304366b19cSJenny Derzhavetz if (ret) { 18314366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", ret); 18324366b19cSJenny Derzhavetz return ret; 18334366b19cSJenny Derzhavetz } 18344366b19cSJenny Derzhavetz 1835e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1836b8d26b3bSNicholas Bellinger if (ret) { 183724f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 1838b8d26b3bSNicholas Bellinger return ret; 1839b8d26b3bSNicholas Bellinger } 1840b8d26b3bSNicholas Bellinger return ret; 1841b8d26b3bSNicholas Bellinger } 1842b8d26b3bSNicholas Bellinger 1843b8d26b3bSNicholas Bellinger static int 1844b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1845b8d26b3bSNicholas Bellinger { 1846d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18476700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1848b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1849b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1850b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 1851b8d26b3bSNicholas Bellinger 1852b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1853b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1854b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1855b8d26b3bSNicholas Bellinger /* 1856b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 1857b8d26b3bSNicholas Bellinger */ 1858b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 1859b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1860b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1861dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 186267cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1863b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1864dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 1865b8d26b3bSNicholas Bellinger 1866b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1867b8d26b3bSNicholas Bellinger cmd->sense_buffer); 1868b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1869b8d26b3bSNicholas Bellinger 1870b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1871b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1872dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1873b8d26b3bSNicholas Bellinger 1874dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1875dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 1876b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 1877def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1878def4a6ffSAlexey Khoroshilov return -ENOMEM; 1879b8d26b3bSNicholas Bellinger 1880dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 1881dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 1882dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 188334efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 1884b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 1885b8d26b3bSNicholas Bellinger } 1886b8d26b3bSNicholas Bellinger 188768a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1888b8d26b3bSNicholas Bellinger 18894c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n"); 1890b8d26b3bSNicholas Bellinger 1891b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1892b8d26b3bSNicholas Bellinger } 1893b8d26b3bSNicholas Bellinger 1894131e6abcSNicholas Bellinger static void 1895131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1896131e6abcSNicholas Bellinger { 1897131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18986700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1899131e6abcSNicholas Bellinger 1900131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1901131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1902131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 1903131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1904131e6abcSNicholas Bellinger 1905131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 1906131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 190738a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1908131e6abcSNicholas Bellinger } 1909131e6abcSNicholas Bellinger 1910e70beee7SNicholas Bellinger static enum target_prot_op 1911e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 1912e70beee7SNicholas Bellinger { 19136700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1914dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1915e70beee7SNicholas Bellinger 191623a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 191723a548eeSSagi Grimberg if (device->pi_capable) { 191824f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 191923a548eeSSagi Grimberg isert_conn->pi_support = true; 1920e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 192123a548eeSSagi Grimberg } 192223a548eeSSagi Grimberg } 192323a548eeSSagi Grimberg 192424f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 192523a548eeSSagi Grimberg isert_conn->pi_support = false; 1926e70beee7SNicholas Bellinger 1927e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 1928e70beee7SNicholas Bellinger } 1929e70beee7SNicholas Bellinger 1930b8d26b3bSNicholas Bellinger static int 1931b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1932b8d26b3bSNicholas Bellinger bool nopout_response) 1933b8d26b3bSNicholas Bellinger { 1934d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19356700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1936b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1937b8d26b3bSNicholas Bellinger 1938b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1939b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1940b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 1941b8d26b3bSNicholas Bellinger nopout_response); 1942b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 194368a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1944b8d26b3bSNicholas Bellinger 19454c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1946b8d26b3bSNicholas Bellinger 1947b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1948b8d26b3bSNicholas Bellinger } 1949b8d26b3bSNicholas Bellinger 1950b8d26b3bSNicholas Bellinger static int 1951b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1952b8d26b3bSNicholas Bellinger { 1953d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19546700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1955b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1956b8d26b3bSNicholas Bellinger 1957b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1958b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1959b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1960b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 196168a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1962b8d26b3bSNicholas Bellinger 19634c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1964b8d26b3bSNicholas Bellinger 1965b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1966b8d26b3bSNicholas Bellinger } 1967b8d26b3bSNicholas Bellinger 1968b8d26b3bSNicholas Bellinger static int 1969b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1970b8d26b3bSNicholas Bellinger { 1971d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19726700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1973b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1974b8d26b3bSNicholas Bellinger 1975b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1976b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1977b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1978b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 197968a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1980b8d26b3bSNicholas Bellinger 19814c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1982b8d26b3bSNicholas Bellinger 1983b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1984b8d26b3bSNicholas Bellinger } 1985b8d26b3bSNicholas Bellinger 1986b8d26b3bSNicholas Bellinger static int 1987b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1988b8d26b3bSNicholas Bellinger { 1989d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19906700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1991b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1992dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 199367cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 19943df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 19953df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 19963df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1997b8d26b3bSNicholas Bellinger 1998b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 19993df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 2000b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 20013df8f68aSNicholas Bellinger 20023df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2003dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 20043df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 20053df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2006def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2007def4a6ffSAlexey Khoroshilov return -ENOMEM; 2008dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2009dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 20103df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 201134efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 20123df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 20133df8f68aSNicholas Bellinger 201468a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2015b8d26b3bSNicholas Bellinger 20164c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn); 2017b8d26b3bSNicholas Bellinger 2018b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2019b8d26b3bSNicholas Bellinger } 2020b8d26b3bSNicholas Bellinger 2021b8d26b3bSNicholas Bellinger static int 2022adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2023adb54c29SNicholas Bellinger { 2024d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 20256700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2026adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2027adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2028adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2029adb54c29SNicholas Bellinger u32 txt_rsp_len; 2030adb54c29SNicholas Bellinger int rc; 2031adb54c29SNicholas Bellinger 2032adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 203322c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2034adb54c29SNicholas Bellinger if (rc < 0) 2035adb54c29SNicholas Bellinger return rc; 2036adb54c29SNicholas Bellinger 2037adb54c29SNicholas Bellinger txt_rsp_len = rc; 2038adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2039adb54c29SNicholas Bellinger 2040adb54c29SNicholas Bellinger if (txt_rsp_len) { 2041dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 204267cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2043adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2044adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2045adb54c29SNicholas Bellinger 2046adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2047adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2048def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2049def4a6ffSAlexey Khoroshilov return -ENOMEM; 2050adb54c29SNicholas Bellinger 2051adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2052adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2053adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 205434efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2055adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2056adb54c29SNicholas Bellinger } 205768a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2058adb54c29SNicholas Bellinger 2059f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn); 2060adb54c29SNicholas Bellinger 2061adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2062adb54c29SNicholas Bellinger } 2063adb54c29SNicholas Bellinger 20643d73cf1aSSagi Grimberg static inline void 2065dae6345aSIsrael Rukshin isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 20669e961ae7SSagi Grimberg { 206778eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 20683d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 20693d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 20703d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 207178eda2bbSSagi Grimberg /* 207278eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 207378eda2bbSSagi Grimberg * the target core would like to use it, we will take it 207478eda2bbSSagi Grimberg * from se_cmd. 207578eda2bbSSagi Grimberg */ 207678eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 207778eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 207878eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 207978eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 208078eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 208178eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 20823d73cf1aSSagi Grimberg }; 20833d73cf1aSSagi Grimberg 20843d73cf1aSSagi Grimberg static int 20853d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 20863d73cf1aSSagi Grimberg { 208738a2d0d4SChristoph Hellwig memset(sig_attrs, 0, sizeof(*sig_attrs)); 208838a2d0d4SChristoph Hellwig 20899e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 20909e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 20919e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 209278eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2093dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire); 20949e961ae7SSagi Grimberg break; 20959e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 20969e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 209778eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2098dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem); 20999e961ae7SSagi Grimberg break; 21009e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 21019e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 2102dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2103dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem); 21049e961ae7SSagi Grimberg break; 21059e961ae7SSagi Grimberg default: 210624f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 21079e961ae7SSagi Grimberg return -EINVAL; 21089e961ae7SSagi Grimberg } 21099e961ae7SSagi Grimberg 211026232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 211126232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 211226232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 211326232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 211426232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 211526232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 211626232872SMax Gurtovoy 21179e961ae7SSagi Grimberg return 0; 21189e961ae7SSagi Grimberg } 21199e961ae7SSagi Grimberg 21209e961ae7SSagi Grimberg static int 212138a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 212238a2d0d4SChristoph Hellwig struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 21239e961ae7SSagi Grimberg { 212438a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 212538a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 212638a2d0d4SChristoph Hellwig u8 port_num = conn->cm_id->port_num; 212738a2d0d4SChristoph Hellwig u64 addr; 212838a2d0d4SChristoph Hellwig u32 rkey, offset; 21299e961ae7SSagi Grimberg int ret; 21309e961ae7SSagi Grimberg 213166f53e6fSBharat Potnuri if (cmd->ctx_init_done) 213266f53e6fSBharat Potnuri goto rdma_ctx_post; 213366f53e6fSBharat Potnuri 213438a2d0d4SChristoph Hellwig if (dir == DMA_FROM_DEVICE) { 213538a2d0d4SChristoph Hellwig addr = cmd->write_va; 213638a2d0d4SChristoph Hellwig rkey = cmd->write_stag; 213738a2d0d4SChristoph Hellwig offset = cmd->iscsi_cmd->write_data_done; 213838a2d0d4SChristoph Hellwig } else { 213938a2d0d4SChristoph Hellwig addr = cmd->read_va; 214038a2d0d4SChristoph Hellwig rkey = cmd->read_stag; 214138a2d0d4SChristoph Hellwig offset = 0; 214238a2d0d4SChristoph Hellwig } 214338a2d0d4SChristoph Hellwig 214438a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 214538a2d0d4SChristoph Hellwig struct ib_sig_attrs sig_attrs; 214638a2d0d4SChristoph Hellwig 21479e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 21489e961ae7SSagi Grimberg if (ret) 2149e3d7e4c3SSagi Grimberg return ret; 2150e3d7e4c3SSagi Grimberg 215138a2d0d4SChristoph Hellwig WARN_ON_ONCE(offset); 215238a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 215338a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 215438a2d0d4SChristoph Hellwig se_cmd->t_prot_sg, se_cmd->t_prot_nents, 215538a2d0d4SChristoph Hellwig &sig_attrs, addr, rkey, dir); 2156570db170SSagi Grimberg } else { 215738a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 215838a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 215938a2d0d4SChristoph Hellwig offset, addr, rkey, dir); 216038a2d0d4SChristoph Hellwig } 216166f53e6fSBharat Potnuri 216238a2d0d4SChristoph Hellwig if (ret < 0) { 216338a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 216438a2d0d4SChristoph Hellwig return ret; 21659e961ae7SSagi Grimberg } 21669e961ae7SSagi Grimberg 216766f53e6fSBharat Potnuri cmd->ctx_init_done = true; 216866f53e6fSBharat Potnuri 216966f53e6fSBharat Potnuri rdma_ctx_post: 217038a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 217138a2d0d4SChristoph Hellwig if (ret < 0) 217238a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to post RDMA res\n", cmd); 217359464ef4SVu Pham return ret; 217459464ef4SVu Pham } 217559464ef4SVu Pham 217659464ef4SVu Pham static int 217790ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 217890ecc6e2SVu Pham { 217990ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 218059464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21816700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 218238a2d0d4SChristoph Hellwig struct ib_cqe *cqe = NULL; 218338a2d0d4SChristoph Hellwig struct ib_send_wr *chain_wr = NULL; 218490ecc6e2SVu Pham int rc; 218590ecc6e2SVu Pham 218624f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 218790ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 21884c22e07fSSagi Grimberg 218938a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) { 219038a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 219138a2d0d4SChristoph Hellwig cqe = &isert_cmd->tx_desc.tx_cqe; 219238a2d0d4SChristoph Hellwig } else { 2193b8d26b3bSNicholas Bellinger /* 2194b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2195b8d26b3bSNicholas Bellinger */ 21969e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 21979e961ae7SSagi Grimberg &isert_cmd->tx_desc); 219804d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2199b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2200b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 220195b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 220268a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 22034366b19cSJenny Derzhavetz 22044366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 22054366b19cSJenny Derzhavetz if (rc) { 22064366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 22074366b19cSJenny Derzhavetz return rc; 22084366b19cSJenny Derzhavetz } 220938a2d0d4SChristoph Hellwig 221038a2d0d4SChristoph Hellwig chain_wr = &isert_cmd->tx_desc.send_wr; 22119e961ae7SSagi Grimberg } 2212b8d26b3bSNicholas Bellinger 2213555a65f6SNicholas Bellinger rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2214555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2215555a65f6SNicholas Bellinger isert_cmd, rc); 2216555a65f6SNicholas Bellinger return rc; 2217b8d26b3bSNicholas Bellinger } 2218b8d26b3bSNicholas Bellinger 2219b8d26b3bSNicholas Bellinger static int 2220b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2221b8d26b3bSNicholas Bellinger { 2222d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2223555a65f6SNicholas Bellinger int ret; 2224b8d26b3bSNicholas Bellinger 222524f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 222638a2d0d4SChristoph Hellwig isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2227b8d26b3bSNicholas Bellinger 222838a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2229555a65f6SNicholas Bellinger ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 223038a2d0d4SChristoph Hellwig &isert_cmd->tx_desc.tx_cqe, NULL); 2231bdf20e72SSagi Grimberg 2232555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2233555a65f6SNicholas Bellinger isert_cmd, ret); 2234555a65f6SNicholas Bellinger return ret; 2235b8d26b3bSNicholas Bellinger } 2236b8d26b3bSNicholas Bellinger 2237b8d26b3bSNicholas Bellinger static int 2238b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2239b8d26b3bSNicholas Bellinger { 2240a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2241a4c15cd9SJenny Derzhavetz int ret = 0; 2242b8d26b3bSNicholas Bellinger 2243b8d26b3bSNicholas Bellinger switch (state) { 2244a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE: 2245a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 2246a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 2247a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 2248a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 2249a4c15cd9SJenny Derzhavetz break; 2250b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2251b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2252b8d26b3bSNicholas Bellinger break; 2253b8d26b3bSNicholas Bellinger default: 225424f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2255b8d26b3bSNicholas Bellinger ret = -EINVAL; 2256b8d26b3bSNicholas Bellinger break; 2257b8d26b3bSNicholas Bellinger } 2258b8d26b3bSNicholas Bellinger 2259b8d26b3bSNicholas Bellinger return ret; 2260b8d26b3bSNicholas Bellinger } 2261b8d26b3bSNicholas Bellinger 2262b8d26b3bSNicholas Bellinger static int 2263b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2264b8d26b3bSNicholas Bellinger { 2265991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2266b8d26b3bSNicholas Bellinger int ret; 2267b8d26b3bSNicholas Bellinger 2268b8d26b3bSNicholas Bellinger switch (state) { 2269b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2270b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2271991bb764SSagi Grimberg if (!ret) 2272991bb764SSagi Grimberg isert_conn->logout_posted = true; 2273b8d26b3bSNicholas Bellinger break; 2274b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2275b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2276b8d26b3bSNicholas Bellinger break; 2277b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2278b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2279b8d26b3bSNicholas Bellinger break; 2280b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2281b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2282b8d26b3bSNicholas Bellinger break; 2283adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2284adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2285adb54c29SNicholas Bellinger break; 2286b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2287b8d26b3bSNicholas Bellinger /* 2288b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2289b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2290b8d26b3bSNicholas Bellinger */ 2291b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2292b8d26b3bSNicholas Bellinger break; 2293b8d26b3bSNicholas Bellinger default: 229424f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 2295b8d26b3bSNicholas Bellinger ret = -EINVAL; 2296b8d26b3bSNicholas Bellinger break; 2297b8d26b3bSNicholas Bellinger } 2298b8d26b3bSNicholas Bellinger 2299b8d26b3bSNicholas Bellinger return ret; 2300b8d26b3bSNicholas Bellinger } 2301b8d26b3bSNicholas Bellinger 2302ca6c1d82SSagi Grimberg struct rdma_cm_id * 2303ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 2304ca6c1d82SSagi Grimberg { 2305ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 2306ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 2307ca6c1d82SSagi Grimberg struct sockaddr *sa; 2308ca6c1d82SSagi Grimberg int ret; 2309ca6c1d82SSagi Grimberg 2310ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 231124f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2312ca6c1d82SSagi Grimberg 2313fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2314ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 2315ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 231624f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2317ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 2318ca6c1d82SSagi Grimberg goto out; 2319ca6c1d82SSagi Grimberg } 232024f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 2321ca6c1d82SSagi Grimberg 2322ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 2323ca6c1d82SSagi Grimberg if (ret) { 232424f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 2325ca6c1d82SSagi Grimberg goto out_id; 2326ca6c1d82SSagi Grimberg } 2327ca6c1d82SSagi Grimberg 2328992607e8SSagi Grimberg ret = rdma_listen(id, 0); 2329ca6c1d82SSagi Grimberg if (ret) { 233024f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 2331ca6c1d82SSagi Grimberg goto out_id; 2332ca6c1d82SSagi Grimberg } 2333ca6c1d82SSagi Grimberg 2334ca6c1d82SSagi Grimberg return id; 2335ca6c1d82SSagi Grimberg out_id: 2336ca6c1d82SSagi Grimberg rdma_destroy_id(id); 2337ca6c1d82SSagi Grimberg out: 2338ca6c1d82SSagi Grimberg return ERR_PTR(ret); 2339ca6c1d82SSagi Grimberg } 2340ca6c1d82SSagi Grimberg 2341b8d26b3bSNicholas Bellinger static int 2342b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 234313a3cf08SAndy Grover struct sockaddr_storage *ksockaddr) 2344b8d26b3bSNicholas Bellinger { 2345b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2346b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2347b8d26b3bSNicholas Bellinger int ret; 2348b8d26b3bSNicholas Bellinger 2349b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 235093b80f29SLeon Romanovsky if (!isert_np) 2351b8d26b3bSNicholas Bellinger return -ENOMEM; 235293b80f29SLeon Romanovsky 2353ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0); 2354ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex); 2355bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted); 2356bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending); 2357ca6c1d82SSagi Grimberg isert_np->np = np; 2358b8d26b3bSNicholas Bellinger 2359b8d26b3bSNicholas Bellinger /* 2360b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 2361b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 2362b8d26b3bSNicholas Bellinger */ 2363b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 236413a3cf08SAndy Grover sizeof(struct sockaddr_storage)); 2365b8d26b3bSNicholas Bellinger 2366ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 2367b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 2368b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 2369b8d26b3bSNicholas Bellinger goto out; 2370b8d26b3bSNicholas Bellinger } 2371b8d26b3bSNicholas Bellinger 2372ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid; 2373b8d26b3bSNicholas Bellinger np->np_context = isert_np; 2374b8d26b3bSNicholas Bellinger 2375b8d26b3bSNicholas Bellinger return 0; 2376b8d26b3bSNicholas Bellinger 2377b8d26b3bSNicholas Bellinger out: 2378b8d26b3bSNicholas Bellinger kfree(isert_np); 2379ca6c1d82SSagi Grimberg 2380b8d26b3bSNicholas Bellinger return ret; 2381b8d26b3bSNicholas Bellinger } 2382b8d26b3bSNicholas Bellinger 2383b8d26b3bSNicholas Bellinger static int 2384b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 2385b8d26b3bSNicholas Bellinger { 2386dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2387b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 2388b8d26b3bSNicholas Bellinger int ret; 238913bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr; 2390b8d26b3bSNicholas Bellinger 2391b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 2392b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 2393b8d26b3bSNicholas Bellinger cp.retry_count = 7; 2394b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 2395b8d26b3bSNicholas Bellinger 239613bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2397422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2398422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv) 2399422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 240013bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr; 240113bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr); 240213bce482SJenny Derzhavetz 2403b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 2404b8d26b3bSNicholas Bellinger if (ret) { 240524f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 2406b8d26b3bSNicholas Bellinger return ret; 2407b8d26b3bSNicholas Bellinger } 2408b8d26b3bSNicholas Bellinger 2409b8d26b3bSNicholas Bellinger return 0; 2410b8d26b3bSNicholas Bellinger } 2411b8d26b3bSNicholas Bellinger 2412b8d26b3bSNicholas Bellinger static int 2413b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2414b8d26b3bSNicholas Bellinger { 24156700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2416b8d26b3bSNicholas Bellinger int ret; 2417b8d26b3bSNicholas Bellinger 241824f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 24192371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 24202371e5daSSagi Grimberg if (ret) { 242124f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 24222371e5daSSagi Grimberg isert_conn); 24232371e5daSSagi Grimberg return ret; 24242371e5daSSagi Grimberg } 24252371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 24262371e5daSSagi Grimberg 24276faaa85fSNicholas Bellinger /* 24286faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 24296faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 24306faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 24316faaa85fSNicholas Bellinger * into a NOP. 24326faaa85fSNicholas Bellinger */ 24336faaa85fSNicholas Bellinger if (!login->first_request) 24346faaa85fSNicholas Bellinger return 0; 2435b8d26b3bSNicholas Bellinger 24362371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 24372371e5daSSagi Grimberg 2438dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn); 2439dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2440b8d26b3bSNicholas Bellinger if (ret) 2441b8d26b3bSNicholas Bellinger return ret; 2442b8d26b3bSNicholas Bellinger 244324f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 24442371e5daSSagi Grimberg 2445b8d26b3bSNicholas Bellinger return 0; 2446b8d26b3bSNicholas Bellinger } 2447b8d26b3bSNicholas Bellinger 2448b8d26b3bSNicholas Bellinger static void 2449b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2450b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 2451b8d26b3bSNicholas Bellinger { 2452dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2453b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 2454b8d26b3bSNicholas Bellinger 2455b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 2456b8d26b3bSNicholas Bellinger 2457dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr; 2458dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr; 2459b8d26b3bSNicholas Bellinger } 2460b8d26b3bSNicholas Bellinger 2461b8d26b3bSNicholas Bellinger static int 2462b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2463b8d26b3bSNicholas Bellinger { 24646700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2465b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2466c6b8e918SSagi Grimberg int ret; 2467b8d26b3bSNicholas Bellinger 2468b8d26b3bSNicholas Bellinger accept_wait: 2469ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem); 2470c6b8e918SSagi Grimberg if (ret) 2471b8d26b3bSNicholas Bellinger return -ENODEV; 2472b8d26b3bSNicholas Bellinger 2473b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 2474e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2475b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 247611378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n", 2477e346ab34SSagi Grimberg np->np_thread_state); 2478e346ab34SSagi Grimberg /** 2479e346ab34SSagi Grimberg * No point in stalling here when np_thread 2480e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 2481e346ab34SSagi Grimberg **/ 2482b8d26b3bSNicholas Bellinger return -ENODEV; 2483b8d26b3bSNicholas Bellinger } 2484b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 2485b8d26b3bSNicholas Bellinger 2486ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2487bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) { 2488ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2489b8d26b3bSNicholas Bellinger goto accept_wait; 2490b8d26b3bSNicholas Bellinger } 2491bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending, 2492bd379220SJenny Derzhavetz struct isert_conn, node); 2493bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 2494ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2495b8d26b3bSNicholas Bellinger 2496b8d26b3bSNicholas Bellinger conn->context = isert_conn; 2497b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 2498aea92980SJenny Derzhavetz isert_conn->state = ISER_CONN_BOUND; 2499b8d26b3bSNicholas Bellinger 2500b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 2501b8d26b3bSNicholas Bellinger 250224f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 25032371e5daSSagi Grimberg 2504b8d26b3bSNicholas Bellinger return 0; 2505b8d26b3bSNicholas Bellinger } 2506b8d26b3bSNicholas Bellinger 2507b8d26b3bSNicholas Bellinger static void 2508b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 2509b8d26b3bSNicholas Bellinger { 25106700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2511268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 2512b8d26b3bSNicholas Bellinger 2513ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id) 2514ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id); 2515b8d26b3bSNicholas Bellinger 2516268e6811SSagi Grimberg /* 2517268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 2518268e6811SSagi Grimberg * that at this point we don't have hanging connections that 2519268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 2520268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 2521bd379220SJenny Derzhavetz * up in accepted and pending lists. 2522268e6811SSagi Grimberg */ 2523ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2524bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) { 2525bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n"); 2526268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 2527bd379220SJenny Derzhavetz &isert_np->pending, 2528bd379220SJenny Derzhavetz node) { 2529bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n", 2530bd379220SJenny Derzhavetz isert_conn, isert_conn->state); 2531bd379220SJenny Derzhavetz isert_connect_release(isert_conn); 2532bd379220SJenny Derzhavetz } 2533bd379220SJenny Derzhavetz } 2534bd379220SJenny Derzhavetz 2535bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) { 2536bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n"); 2537bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n, 2538bd379220SJenny Derzhavetz &isert_np->accepted, 2539bd379220SJenny Derzhavetz node) { 254024f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 2541268e6811SSagi Grimberg isert_conn, isert_conn->state); 2542268e6811SSagi Grimberg isert_connect_release(isert_conn); 2543268e6811SSagi Grimberg } 2544268e6811SSagi Grimberg } 2545ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2546268e6811SSagi Grimberg 2547b8d26b3bSNicholas Bellinger np->np_context = NULL; 2548b8d26b3bSNicholas Bellinger kfree(isert_np); 2549b8d26b3bSNicholas Bellinger } 2550b8d26b3bSNicholas Bellinger 2551b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 2552b02efbfcSSagi Grimberg { 2553b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 2554b02efbfcSSagi Grimberg struct isert_conn, 2555b02efbfcSSagi Grimberg release_work); 2556b02efbfcSSagi Grimberg 255724f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 2558b02efbfcSSagi Grimberg 2559dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2560b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 2561dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2562b02efbfcSSagi Grimberg 256324f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 2564b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 2565b02efbfcSSagi Grimberg } 2566b02efbfcSSagi Grimberg 2567bdf20e72SSagi Grimberg static void 2568991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 2569991bb764SSagi Grimberg { 2570991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 2571991bb764SSagi Grimberg 25724c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 25734c22e07fSSagi Grimberg 2574991bb764SSagi Grimberg if (isert_conn->logout_posted) { 257524f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2576991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 2577991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 2578991bb764SSagi Grimberg } 2579991bb764SSagi Grimberg } 2580991bb764SSagi Grimberg 258176261adaSBart Van Assche static void 258276261adaSBart Van Assche isert_wait4cmds(struct iscsi_conn *conn) 258376261adaSBart Van Assche { 258476261adaSBart Van Assche isert_info("iscsi_conn %p\n", conn); 258576261adaSBart Van Assche 258676261adaSBart Van Assche if (conn->sess) { 258776261adaSBart Van Assche target_sess_cmd_list_set_waiting(conn->sess->se_sess); 258876261adaSBart Van Assche target_wait_for_sess_cmds(conn->sess->se_sess); 258976261adaSBart Van Assche } 259076261adaSBart Van Assche } 259176261adaSBart Van Assche 25923e03c4b0SJenny Derzhavetz /** 25933e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for 25943e03c4b0SJenny Derzhavetz * unsolicitate dataout 25953e03c4b0SJenny Derzhavetz * @conn: iscsi connection 25963e03c4b0SJenny Derzhavetz * 25973e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited 25983e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those 25993e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds 26003e03c4b0SJenny Derzhavetz */ 26013e03c4b0SJenny Derzhavetz static void 26023e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 26033e03c4b0SJenny Derzhavetz { 26043e03c4b0SJenny Derzhavetz struct iscsi_cmd *cmd, *tmp; 26053e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list); 26063e03c4b0SJenny Derzhavetz 26073e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 26083e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 26093e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 26103e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 26113e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length)) 26123e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 26133e03c4b0SJenny Derzhavetz } 26143e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 26153e03c4b0SJenny Derzhavetz 26163e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 26173e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 26183e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) { 26193e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 26203e03c4b0SJenny Derzhavetz 26213e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd); 26223e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 26233e03c4b0SJenny Derzhavetz } 26243e03c4b0SJenny Derzhavetz } 26253e03c4b0SJenny Derzhavetz } 26263e03c4b0SJenny Derzhavetz 2627defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 2628b8d26b3bSNicholas Bellinger { 2629b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2630b8d26b3bSNicholas Bellinger 26314c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn); 2632b8d26b3bSNicholas Bellinger 2633dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2634954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 2635dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2636b2cb9649SNicholas Bellinger 2637572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 26383e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn); 263976261adaSBart Van Assche isert_wait4cmds(conn); 2640991bb764SSagi Grimberg isert_wait4logout(isert_conn); 2641954f2372SSagi Grimberg 2642b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 2643defd8848SNicholas Bellinger } 2644defd8848SNicholas Bellinger 2645defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 2646defd8848SNicholas Bellinger { 2647defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2648b8d26b3bSNicholas Bellinger 2649572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 2650b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 2651b8d26b3bSNicholas Bellinger } 2652b8d26b3bSNicholas Bellinger 2653187e84f7SVarun Prakash static void isert_get_rx_pdu(struct iscsi_conn *conn) 2654187e84f7SVarun Prakash { 2655187e84f7SVarun Prakash struct completion comp; 2656187e84f7SVarun Prakash 2657187e84f7SVarun Prakash init_completion(&comp); 2658187e84f7SVarun Prakash 2659187e84f7SVarun Prakash wait_for_completion_interruptible(&comp); 2660187e84f7SVarun Prakash } 2661187e84f7SVarun Prakash 2662b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 2663b8d26b3bSNicholas Bellinger .name = "IB/iSER", 2664b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 2665bd027d85SNicholas Bellinger .rdma_shutdown = true, 2666d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 2667b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 2668b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 2669b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 2670b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 2671defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 2672b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 2673b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 2674b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 2675b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 2676b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 2677b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 2678b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 2679b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 2680131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 2681187e84f7SVarun Prakash .iscsit_get_rx_pdu = isert_get_rx_pdu, 2682e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2683b8d26b3bSNicholas Bellinger }; 2684b8d26b3bSNicholas Bellinger 2685b8d26b3bSNicholas Bellinger static int __init isert_init(void) 2686b8d26b3bSNicholas Bellinger { 2687b8d26b3bSNicholas Bellinger int ret; 2688b8d26b3bSNicholas Bellinger 2689631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq", 2690631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0); 2691b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 269224f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 26936f0fae3dSSagi Grimberg return -ENOMEM; 2694b8d26b3bSNicholas Bellinger } 2695b8d26b3bSNicholas Bellinger 2696b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2697b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 2698b02efbfcSSagi Grimberg if (!isert_release_wq) { 269924f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 2700b02efbfcSSagi Grimberg ret = -ENOMEM; 2701b02efbfcSSagi Grimberg goto destroy_comp_wq; 2702b02efbfcSSagi Grimberg } 2703b02efbfcSSagi Grimberg 2704b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 270524f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2706b02efbfcSSagi Grimberg 2707b8d26b3bSNicholas Bellinger return 0; 2708b8d26b3bSNicholas Bellinger 2709b02efbfcSSagi Grimberg destroy_comp_wq: 2710b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 27116f0fae3dSSagi Grimberg 2712b8d26b3bSNicholas Bellinger return ret; 2713b8d26b3bSNicholas Bellinger } 2714b8d26b3bSNicholas Bellinger 2715b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 2716b8d26b3bSNicholas Bellinger { 2717f5ebec96SSagi Grimberg flush_scheduled_work(); 2718b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 2719b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 2720b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 27214c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2722b8d26b3bSNicholas Bellinger } 2723b8d26b3bSNicholas Bellinger 2724b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2725b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2726b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 2727b8d26b3bSNicholas Bellinger 2728b8d26b3bSNicholas Bellinger module_init(isert_init); 2729b8d26b3bSNicholas Bellinger module_exit(isert_exit); 2730