1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2b8d26b3bSNicholas Bellinger /******************************************************************************* 3b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 4b8d26b3bSNicholas Bellinger * 54c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 6b8d26b3bSNicholas Bellinger * 7b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 8b8d26b3bSNicholas Bellinger * 9b8d26b3bSNicholas Bellinger ****************************************************************************/ 10b8d26b3bSNicholas Bellinger 11b8d26b3bSNicholas Bellinger #include <linux/string.h> 12b8d26b3bSNicholas Bellinger #include <linux/module.h> 13b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 14b8d26b3bSNicholas Bellinger #include <linux/socket.h> 15b8d26b3bSNicholas Bellinger #include <linux/in.h> 16b8d26b3bSNicholas Bellinger #include <linux/in6.h> 17b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 18*8094ba0aSLeon Romanovsky #include <rdma/ib_cm.h> 19b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 20b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 21b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 22b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 23531b7bf4SSagi Grimberg #include <linux/semaphore.h> 24b8d26b3bSNicholas Bellinger 25b8d26b3bSNicholas Bellinger #include "ib_isert.h" 26b8d26b3bSNicholas Bellinger 27b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 28b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 2938a2d0d4SChristoph Hellwig #define ISER_MAX_TX_CQ_LEN \ 3038a2d0d4SChristoph Hellwig ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) 31bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 32bdf20e72SSagi Grimberg ISERT_MAX_CONN) 33b8d26b3bSNicholas Bellinger 3445678b6bSSagi Grimberg static int isert_debug_level; 3524f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 3624f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 3724f412ddSSagi Grimberg 38b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 39b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 40b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 41b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 42b8d26b3bSNicholas Bellinger 43f93f3a70SSagi Grimberg static int 44f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 452371e5daSSagi Grimberg static int 469679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn); 472371e5daSSagi Grimberg static int 482371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 49ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 50d40945d8SVu Pham 512f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work); 529679cc51SChristoph Hellwig static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 539679cc51SChristoph Hellwig static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 549679cc51SChristoph Hellwig static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 559679cc51SChristoph Hellwig static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 562f1b6b7dSSagi Grimberg 57302cc7c3SSagi Grimberg static inline bool 58302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 59302cc7c3SSagi Grimberg { 6023a548eeSSagi Grimberg return (conn->pi_support && 61302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 62302cc7c3SSagi Grimberg } 63302cc7c3SSagi Grimberg 64302cc7c3SSagi Grimberg 65d40945d8SVu Pham static void 66b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 67b8d26b3bSNicholas Bellinger { 686700425eSSagi Grimberg struct isert_conn *isert_conn = context; 69b8d26b3bSNicholas Bellinger 70ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n", 71ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn); 72ea8a1616SSagi Grimberg 73b8d26b3bSNicholas Bellinger switch (e->event) { 74b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 75dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 76b8d26b3bSNicholas Bellinger break; 77b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 784c22e07fSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 79b8d26b3bSNicholas Bellinger break; 80b8d26b3bSNicholas Bellinger default: 81b8d26b3bSNicholas Bellinger break; 82b8d26b3bSNicholas Bellinger } 83b8d26b3bSNicholas Bellinger } 84b8d26b3bSNicholas Bellinger 8540fc069aSSagi Grimberg static struct isert_comp * 8640fc069aSSagi Grimberg isert_comp_get(struct isert_conn *isert_conn) 87b8d26b3bSNicholas Bellinger { 88dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 894a295baeSSagi Grimberg struct isert_comp *comp; 9040fc069aSSagi Grimberg int i, min = 0; 91b8d26b3bSNicholas Bellinger 92b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 934a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 944a295baeSSagi Grimberg if (device->comps[i].active_qps < 954a295baeSSagi Grimberg device->comps[min].active_qps) 964a295baeSSagi Grimberg min = i; 974a295baeSSagi Grimberg comp = &device->comps[min]; 984a295baeSSagi Grimberg comp->active_qps++; 9940fc069aSSagi Grimberg mutex_unlock(&device_list_mutex); 10040fc069aSSagi Grimberg 10124f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1024a295baeSSagi Grimberg isert_conn, comp, min); 10340fc069aSSagi Grimberg 10440fc069aSSagi Grimberg return comp; 10540fc069aSSagi Grimberg } 10640fc069aSSagi Grimberg 10740fc069aSSagi Grimberg static void 10840fc069aSSagi Grimberg isert_comp_put(struct isert_comp *comp) 10940fc069aSSagi Grimberg { 11040fc069aSSagi Grimberg mutex_lock(&device_list_mutex); 11140fc069aSSagi Grimberg comp->active_qps--; 112b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 11340fc069aSSagi Grimberg } 11440fc069aSSagi Grimberg 11540fc069aSSagi Grimberg static struct ib_qp * 11640fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn, 11740fc069aSSagi Grimberg struct isert_comp *comp, 11840fc069aSSagi Grimberg struct rdma_cm_id *cma_id) 11940fc069aSSagi Grimberg { 120dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 12140fc069aSSagi Grimberg struct ib_qp_init_attr attr; 12240fc069aSSagi Grimberg int ret; 123b8d26b3bSNicholas Bellinger 124b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 125b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 126b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1276f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1286f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 129572a1434SSagi Grimberg attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 130bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 13138a2d0d4SChristoph Hellwig attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX; 13233023fb8SSteve Wise attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 133b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 134b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 135b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 136570db170SSagi Grimberg if (device->pi_capable) 137c0a6cbb9SIsrael Rukshin attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 138b8d26b3bSNicholas Bellinger 13967cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr); 140b8d26b3bSNicholas Bellinger if (ret) { 14124f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 14240fc069aSSagi Grimberg return ERR_PTR(ret); 14340fc069aSSagi Grimberg } 14440fc069aSSagi Grimberg 14540fc069aSSagi Grimberg return cma_id->qp; 14640fc069aSSagi Grimberg } 14740fc069aSSagi Grimberg 14840fc069aSSagi Grimberg static int 14940fc069aSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 15040fc069aSSagi Grimberg { 15140fc069aSSagi Grimberg struct isert_comp *comp; 15240fc069aSSagi Grimberg int ret; 15340fc069aSSagi Grimberg 15440fc069aSSagi Grimberg comp = isert_comp_get(isert_conn); 155dac6ab30SSagi Grimberg isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 156dac6ab30SSagi Grimberg if (IS_ERR(isert_conn->qp)) { 157dac6ab30SSagi Grimberg ret = PTR_ERR(isert_conn->qp); 15819e2090fSSagi Grimberg goto err; 159b8d26b3bSNicholas Bellinger } 160b8d26b3bSNicholas Bellinger 161b8d26b3bSNicholas Bellinger return 0; 16219e2090fSSagi Grimberg err: 16340fc069aSSagi Grimberg isert_comp_put(comp); 16419e2090fSSagi Grimberg return ret; 165b8d26b3bSNicholas Bellinger } 166b8d26b3bSNicholas Bellinger 167b8d26b3bSNicholas Bellinger static int 168b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 169b8d26b3bSNicholas Bellinger { 170dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 17167cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 172b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 173b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 174b8d26b3bSNicholas Bellinger u64 dma_addr; 175b8d26b3bSNicholas Bellinger int i, j; 176b8d26b3bSNicholas Bellinger 1776396bb22SKees Cook isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 1786396bb22SKees Cook sizeof(struct iser_rx_desc), 1796396bb22SKees Cook GFP_KERNEL); 180dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 18193b80f29SLeon Romanovsky return -ENOMEM; 182b8d26b3bSNicholas Bellinger 183dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 184b8d26b3bSNicholas Bellinger 185b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 186b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 187b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 188b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 189b8d26b3bSNicholas Bellinger goto dma_map_fail; 190b8d26b3bSNicholas Bellinger 191b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 192b8d26b3bSNicholas Bellinger 193b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 194b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 195b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 19634efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey; 1979679cc51SChristoph Hellwig rx_desc->rx_cqe.done = isert_recv_done; 198b8d26b3bSNicholas Bellinger } 199b8d26b3bSNicholas Bellinger 200b8d26b3bSNicholas Bellinger return 0; 201b8d26b3bSNicholas Bellinger 202b8d26b3bSNicholas Bellinger dma_map_fail: 203dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 204b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 205b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 206b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 207b8d26b3bSNicholas Bellinger } 208dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 209dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 2104c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 211b8d26b3bSNicholas Bellinger return -ENOMEM; 212b8d26b3bSNicholas Bellinger } 213b8d26b3bSNicholas Bellinger 214b8d26b3bSNicholas Bellinger static void 215b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 216b8d26b3bSNicholas Bellinger { 217dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 218b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 219b8d26b3bSNicholas Bellinger int i; 220b8d26b3bSNicholas Bellinger 221dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 222b8d26b3bSNicholas Bellinger return; 223b8d26b3bSNicholas Bellinger 224dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 225b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 226b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 227b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 228b8d26b3bSNicholas Bellinger } 229b8d26b3bSNicholas Bellinger 230dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 231dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 232b8d26b3bSNicholas Bellinger } 233b8d26b3bSNicholas Bellinger 234172369c5SSagi Grimberg static void 235172369c5SSagi Grimberg isert_free_comps(struct isert_device *device) 236b8d26b3bSNicholas Bellinger { 237172369c5SSagi Grimberg int i; 238b8d26b3bSNicholas Bellinger 239172369c5SSagi Grimberg for (i = 0; i < device->comps_used; i++) { 240172369c5SSagi Grimberg struct isert_comp *comp = &device->comps[i]; 24159464ef4SVu Pham 2429679cc51SChristoph Hellwig if (comp->cq) 2439679cc51SChristoph Hellwig ib_free_cq(comp->cq); 244172369c5SSagi Grimberg } 245172369c5SSagi Grimberg kfree(device->comps); 24659464ef4SVu Pham } 247d40945d8SVu Pham 248172369c5SSagi Grimberg static int 2494a061b28SOr Gerlitz isert_alloc_comps(struct isert_device *device) 250172369c5SSagi Grimberg { 251172369c5SSagi Grimberg int i, max_cqe, ret = 0; 252d3e125daSSagi Grimberg 2534a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2544a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 255172369c5SSagi Grimberg 25624f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 25738a2d0d4SChristoph Hellwig "pi_capable %d\n", 2586c854111SJason Gunthorpe device->comps_used, dev_name(&device->ib_device->dev), 25938a2d0d4SChristoph Hellwig device->ib_device->num_comp_vectors, 260d3e125daSSagi Grimberg device->pi_capable); 2614a295baeSSagi Grimberg 2624a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2634a295baeSSagi Grimberg GFP_KERNEL); 26493b80f29SLeon Romanovsky if (!device->comps) 265b8d26b3bSNicholas Bellinger return -ENOMEM; 266b8d26b3bSNicholas Bellinger 2674a061b28SOr Gerlitz max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 268172369c5SSagi Grimberg 2694a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2704a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 271b8d26b3bSNicholas Bellinger 2724a295baeSSagi Grimberg comp->device = device; 2739679cc51SChristoph Hellwig comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 2749679cc51SChristoph Hellwig IB_POLL_WORKQUEUE); 2756f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 276172369c5SSagi Grimberg isert_err("Unable to allocate cq\n"); 2776f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 2786f0fae3dSSagi Grimberg comp->cq = NULL; 279b8d26b3bSNicholas Bellinger goto out_cq; 28094a71110SWei Yongjun } 281b8d26b3bSNicholas Bellinger } 282b8d26b3bSNicholas Bellinger 283172369c5SSagi Grimberg return 0; 284172369c5SSagi Grimberg out_cq: 285172369c5SSagi Grimberg isert_free_comps(device); 286172369c5SSagi Grimberg return ret; 287172369c5SSagi Grimberg } 288172369c5SSagi Grimberg 289172369c5SSagi Grimberg static int 290172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device) 291172369c5SSagi Grimberg { 2924a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device; 293fd8205e8SSagi Grimberg int ret; 294172369c5SSagi Grimberg 29533023fb8SSteve Wise isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 29633023fb8SSteve Wise ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 2974a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 298172369c5SSagi Grimberg 2994a061b28SOr Gerlitz ret = isert_alloc_comps(device); 300172369c5SSagi Grimberg if (ret) 301373a4cd7SNicholas Bellinger goto out; 302172369c5SSagi Grimberg 303ed082d36SChristoph Hellwig device->pd = ib_alloc_pd(ib_dev, 0); 30467cb3949SSagi Grimberg if (IS_ERR(device->pd)) { 30567cb3949SSagi Grimberg ret = PTR_ERR(device->pd); 30667cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n", 30767cb3949SSagi Grimberg device, ret); 30867cb3949SSagi Grimberg goto out_cq; 30967cb3949SSagi Grimberg } 31067cb3949SSagi Grimberg 311172369c5SSagi Grimberg /* Check signature cap */ 3124a061b28SOr Gerlitz device->pi_capable = ib_dev->attrs.device_cap_flags & 313c0a6cbb9SIsrael Rukshin IB_DEVICE_INTEGRITY_HANDOVER ? true : false; 31467cb3949SSagi Grimberg 315b8d26b3bSNicholas Bellinger return 0; 316b8d26b3bSNicholas Bellinger 317b8d26b3bSNicholas Bellinger out_cq: 318172369c5SSagi Grimberg isert_free_comps(device); 319373a4cd7SNicholas Bellinger out: 320373a4cd7SNicholas Bellinger if (ret > 0) 321373a4cd7SNicholas Bellinger ret = -EINVAL; 322b8d26b3bSNicholas Bellinger return ret; 323b8d26b3bSNicholas Bellinger } 324b8d26b3bSNicholas Bellinger 325b8d26b3bSNicholas Bellinger static void 326b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 327b8d26b3bSNicholas Bellinger { 32824f412ddSSagi Grimberg isert_info("device %p\n", device); 329b8d26b3bSNicholas Bellinger 33067cb3949SSagi Grimberg ib_dealloc_pd(device->pd); 331172369c5SSagi Grimberg isert_free_comps(device); 332b8d26b3bSNicholas Bellinger } 333b8d26b3bSNicholas Bellinger 334b8d26b3bSNicholas Bellinger static void 335cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device) 336b8d26b3bSNicholas Bellinger { 337b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 338b8d26b3bSNicholas Bellinger device->refcount--; 3394c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount); 340b8d26b3bSNicholas Bellinger if (!device->refcount) { 341b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 342b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 343b8d26b3bSNicholas Bellinger kfree(device); 344b8d26b3bSNicholas Bellinger } 345b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 346b8d26b3bSNicholas Bellinger } 347b8d26b3bSNicholas Bellinger 348b8d26b3bSNicholas Bellinger static struct isert_device * 349cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id) 350b8d26b3bSNicholas Bellinger { 351b8d26b3bSNicholas Bellinger struct isert_device *device; 352b8d26b3bSNicholas Bellinger int ret; 353b8d26b3bSNicholas Bellinger 354b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 355b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 356b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 357b8d26b3bSNicholas Bellinger device->refcount++; 3584c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n", 3594c22e07fSSagi Grimberg device, device->refcount); 360b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 361b8d26b3bSNicholas Bellinger return device; 362b8d26b3bSNicholas Bellinger } 363b8d26b3bSNicholas Bellinger } 364b8d26b3bSNicholas Bellinger 365b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 366b8d26b3bSNicholas Bellinger if (!device) { 367b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 368b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 369b8d26b3bSNicholas Bellinger } 370b8d26b3bSNicholas Bellinger 371b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 372b8d26b3bSNicholas Bellinger 373b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 374b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 375b8d26b3bSNicholas Bellinger if (ret) { 376b8d26b3bSNicholas Bellinger kfree(device); 377b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 378b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 379b8d26b3bSNicholas Bellinger } 380b8d26b3bSNicholas Bellinger 381b8d26b3bSNicholas Bellinger device->refcount++; 382b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 3834c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n", 3844c22e07fSSagi Grimberg device, device->refcount); 385b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 386b8d26b3bSNicholas Bellinger 387b8d26b3bSNicholas Bellinger return device; 388b8d26b3bSNicholas Bellinger } 389b8d26b3bSNicholas Bellinger 39059464ef4SVu Pham static void 391ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn) 392ae9ea9edSSagi Grimberg { 393ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT; 394bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node); 395dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp); 396ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp); 39763b268d2SRaju Rangoju init_waitqueue_head(&isert_conn->rem_wait); 398dac6ab30SSagi Grimberg kref_init(&isert_conn->kref); 399dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex); 4002f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 401ae9ea9edSSagi Grimberg } 402ae9ea9edSSagi Grimberg 403ae9ea9edSSagi Grimberg static void 404ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn) 405ae9ea9edSSagi Grimberg { 406dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 407ae9ea9edSSagi Grimberg 408ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 409ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 4105adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4115adabdd1SChristoph Hellwig 412ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4135adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, 414ae9ea9edSSagi Grimberg DMA_FROM_DEVICE); 4155adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 416ae9ea9edSSagi Grimberg } 417ae9ea9edSSagi Grimberg 418ae9ea9edSSagi Grimberg static int 419ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn, 420ae9ea9edSSagi Grimberg struct ib_device *ib_dev) 421ae9ea9edSSagi Grimberg { 422ae9ea9edSSagi Grimberg int ret; 423ae9ea9edSSagi Grimberg 4245adabdd1SChristoph Hellwig isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 4255adabdd1SChristoph Hellwig GFP_KERNEL); 42693b80f29SLeon Romanovsky if (!isert_conn->login_req_buf) 427ae9ea9edSSagi Grimberg return -ENOMEM; 428ae9ea9edSSagi Grimberg 429ae9ea9edSSagi Grimberg isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 4305adabdd1SChristoph Hellwig isert_conn->login_req_buf, 4315adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 432ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 433ae9ea9edSSagi Grimberg if (ret) { 434ae9ea9edSSagi Grimberg isert_err("login_req_dma mapping error: %d\n", ret); 435ae9ea9edSSagi Grimberg isert_conn->login_req_dma = 0; 4365adabdd1SChristoph Hellwig goto out_free_login_req_buf; 4375adabdd1SChristoph Hellwig } 4385adabdd1SChristoph Hellwig 4395adabdd1SChristoph Hellwig isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 4405adabdd1SChristoph Hellwig if (!isert_conn->login_rsp_buf) { 4411d5840c9SWei Yongjun ret = -ENOMEM; 4425adabdd1SChristoph Hellwig goto out_unmap_login_req_buf; 443ae9ea9edSSagi Grimberg } 444ae9ea9edSSagi Grimberg 445ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 4465adabdd1SChristoph Hellwig isert_conn->login_rsp_buf, 447ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 448ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 449ae9ea9edSSagi Grimberg if (ret) { 450ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret); 451ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0; 4525adabdd1SChristoph Hellwig goto out_free_login_rsp_buf; 453ae9ea9edSSagi Grimberg } 454ae9ea9edSSagi Grimberg 455ae9ea9edSSagi Grimberg return 0; 456ae9ea9edSSagi Grimberg 4575adabdd1SChristoph Hellwig out_free_login_rsp_buf: 4585adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4595adabdd1SChristoph Hellwig out_unmap_login_req_buf: 460ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4615adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 4625adabdd1SChristoph Hellwig out_free_login_req_buf: 4635adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 464ae9ea9edSSagi Grimberg return ret; 465ae9ea9edSSagi Grimberg } 466ae9ea9edSSagi Grimberg 467422bd0acSJenny Derzhavetz static void 468422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn, 469422bd0acSJenny Derzhavetz struct rdma_conn_param *param) 470422bd0acSJenny Derzhavetz { 471422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 472422bd0acSJenny Derzhavetz 473422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */ 474422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 475422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom); 476422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 477422bd0acSJenny Derzhavetz 478422bd0acSJenny Derzhavetz if (param->private_data) { 479422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data; 480422bd0acSJenny Derzhavetz 481422bd0acSJenny Derzhavetz /* 482422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator 483422bd0acSJenny Derzhavetz * and the HCA support it 484422bd0acSJenny Derzhavetz */ 485422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 486422bd0acSJenny Derzhavetz (attr->device_cap_flags & 487422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS); 488422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv) 489422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n"); 490422bd0acSJenny Derzhavetz } 491422bd0acSJenny Derzhavetz } 492422bd0acSJenny Derzhavetz 493b8d26b3bSNicholas Bellinger static int 494b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 495b8d26b3bSNicholas Bellinger { 496ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 497ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 498b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 499b8d26b3bSNicholas Bellinger struct isert_device *device; 500b8d26b3bSNicholas Bellinger int ret = 0; 50114f4b54fSSagi Grimberg 50214f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 50314f4b54fSSagi Grimberg if (!np->enabled) { 50414f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 50524f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 506*8094ba0aSLeon Romanovsky return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 50714f4b54fSSagi Grimberg } 50814f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 509b8d26b3bSNicholas Bellinger 5104c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n", 511b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 512b8d26b3bSNicholas Bellinger 513b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 514ae9ea9edSSagi Grimberg if (!isert_conn) 515b8d26b3bSNicholas Bellinger return -ENOMEM; 516b8d26b3bSNicholas Bellinger 517ae9ea9edSSagi Grimberg isert_init_conn(isert_conn); 518dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id; 519b8d26b3bSNicholas Bellinger 520ae9ea9edSSagi Grimberg ret = isert_alloc_login_buf(isert_conn, cma_id->device); 521ae9ea9edSSagi Grimberg if (ret) 522b8d26b3bSNicholas Bellinger goto out; 523b8d26b3bSNicholas Bellinger 524cf8ae958SSagi Grimberg device = isert_device_get(cma_id); 525b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 526b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 527b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 528b8d26b3bSNicholas Bellinger } 529dac6ab30SSagi Grimberg isert_conn->device = device; 530b8d26b3bSNicholas Bellinger 531422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn); 5321a92e17eSSagi Grimberg 533570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 534b8d26b3bSNicholas Bellinger if (ret) 535b8d26b3bSNicholas Bellinger goto out_conn_dev; 536b8d26b3bSNicholas Bellinger 5379679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 5382371e5daSSagi Grimberg if (ret) 5392371e5daSSagi Grimberg goto out_conn_dev; 5402371e5daSSagi Grimberg 5412371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 5422371e5daSSagi Grimberg if (ret) 5432371e5daSSagi Grimberg goto out_conn_dev; 5442371e5daSSagi Grimberg 545ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 546bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted); 547ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 548b8d26b3bSNicholas Bellinger 549b8d26b3bSNicholas Bellinger return 0; 550b8d26b3bSNicholas Bellinger 551b8d26b3bSNicholas Bellinger out_conn_dev: 552cf8ae958SSagi Grimberg isert_device_put(device); 553b8d26b3bSNicholas Bellinger out_rsp_dma_map: 554ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 555b8d26b3bSNicholas Bellinger out: 556b8d26b3bSNicholas Bellinger kfree(isert_conn); 557*8094ba0aSLeon Romanovsky rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 558b8d26b3bSNicholas Bellinger return ret; 559b8d26b3bSNicholas Bellinger } 560b8d26b3bSNicholas Bellinger 561b8d26b3bSNicholas Bellinger static void 562b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 563b8d26b3bSNicholas Bellinger { 564dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 565b8d26b3bSNicholas Bellinger 5664c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn); 567b8d26b3bSNicholas Bellinger 56857df81e3SSagi Grimberg BUG_ON(!device); 56957df81e3SSagi Grimberg 57019e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 57163b268d2SRaju Rangoju if (isert_conn->cm_id && 57263b268d2SRaju Rangoju !isert_conn->dev_removed) 573dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id); 57419e2090fSSagi Grimberg 575dac6ab30SSagi Grimberg if (isert_conn->qp) { 576dac6ab30SSagi Grimberg struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 5774a295baeSSagi Grimberg 57840fc069aSSagi Grimberg isert_comp_put(comp); 579dac6ab30SSagi Grimberg ib_destroy_qp(isert_conn->qp); 580b8d26b3bSNicholas Bellinger } 581b8d26b3bSNicholas Bellinger 5825adabdd1SChristoph Hellwig if (isert_conn->login_req_buf) 583ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 584ae9ea9edSSagi Grimberg 585cf8ae958SSagi Grimberg isert_device_put(device); 58657df81e3SSagi Grimberg 58763b268d2SRaju Rangoju if (isert_conn->dev_removed) 58863b268d2SRaju Rangoju wake_up_interruptible(&isert_conn->rem_wait); 58963b268d2SRaju Rangoju else 59057df81e3SSagi Grimberg kfree(isert_conn); 591b8d26b3bSNicholas Bellinger } 592b8d26b3bSNicholas Bellinger 593b8d26b3bSNicholas Bellinger static void 594b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 595b8d26b3bSNicholas Bellinger { 59619e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 597bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 598c2f88b17SSagi Grimberg 59924f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 600128e9cc8SSagi Grimberg 601dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 602128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 603bd379220SJenny Derzhavetz kref_get(&isert_conn->kref); 604dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 605bd379220SJenny Derzhavetz 606bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex); 607bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending); 608bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 609bd379220SJenny Derzhavetz 610bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np); 611bd379220SJenny Derzhavetz up(&isert_np->sem); 612b8d26b3bSNicholas Bellinger } 613b8d26b3bSNicholas Bellinger 614b8d26b3bSNicholas Bellinger static void 615dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref) 616b8d26b3bSNicholas Bellinger { 617b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 618dac6ab30SSagi Grimberg struct isert_conn, kref); 619b8d26b3bSNicholas Bellinger 6204c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 6214c22e07fSSagi Grimberg current->pid); 622b8d26b3bSNicholas Bellinger 623b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 624b8d26b3bSNicholas Bellinger } 625b8d26b3bSNicholas Bellinger 626b8d26b3bSNicholas Bellinger static void 627b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 628b8d26b3bSNicholas Bellinger { 629dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref); 630b8d26b3bSNicholas Bellinger } 631b8d26b3bSNicholas Bellinger 6326d1fba0cSJenny Derzhavetz static void 6336d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(struct isert_conn *isert_conn) 6346d1fba0cSJenny Derzhavetz { 6356d1fba0cSJenny Derzhavetz struct isert_np *isert_np = isert_conn->cm_id->context; 6366d1fba0cSJenny Derzhavetz 6376d1fba0cSJenny Derzhavetz mutex_lock(&isert_np->mutex); 6386d1fba0cSJenny Derzhavetz if (!list_empty(&isert_conn->node)) { 6396d1fba0cSJenny Derzhavetz /* 6406d1fba0cSJenny Derzhavetz * This means iscsi doesn't know this connection 6416d1fba0cSJenny Derzhavetz * so schedule a cleanup ourselves 6426d1fba0cSJenny Derzhavetz */ 6436d1fba0cSJenny Derzhavetz list_del_init(&isert_conn->node); 6446d1fba0cSJenny Derzhavetz isert_put_conn(isert_conn); 6456d1fba0cSJenny Derzhavetz queue_work(isert_release_wq, &isert_conn->release_work); 6466d1fba0cSJenny Derzhavetz } 6476d1fba0cSJenny Derzhavetz mutex_unlock(&isert_np->mutex); 6486d1fba0cSJenny Derzhavetz } 6496d1fba0cSJenny Derzhavetz 650954f2372SSagi Grimberg /** 651954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 652954f2372SSagi Grimberg * @isert_conn: isert connection struct 653954f2372SSagi Grimberg * 654954f2372SSagi Grimberg * Notes: 655aea92980SJenny Derzhavetz * In case the connection state is BOUND, move state 656954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 657128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 658954f2372SSagi Grimberg * 659dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is 660954f2372SSagi Grimberg * safe to call multiple times. 661954f2372SSagi Grimberg */ 662954f2372SSagi Grimberg static void 663954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 664954f2372SSagi Grimberg { 665954f2372SSagi Grimberg int err; 666954f2372SSagi Grimberg 6676d1fba0cSJenny Derzhavetz if (isert_conn->state >= ISER_CONN_TERMINATING) 6686d1fba0cSJenny Derzhavetz return; 6696d1fba0cSJenny Derzhavetz 67024f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 671954f2372SSagi Grimberg isert_conn, isert_conn->state); 672128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 673dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id); 674954f2372SSagi Grimberg if (err) 67524f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 676954f2372SSagi Grimberg isert_conn); 677954f2372SSagi Grimberg } 678954f2372SSagi Grimberg 6793b726ae2SSagi Grimberg static int 680ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 681ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 682b8d26b3bSNicholas Bellinger { 683ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n", 684ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np); 6853b726ae2SSagi Grimberg 686ca6c1d82SSagi Grimberg switch (event) { 687ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 688ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 689ca6c1d82SSagi Grimberg break; 690ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 691ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np); 692ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) { 69324f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 694ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id)); 695ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 696ca6c1d82SSagi Grimberg } 697ca6c1d82SSagi Grimberg break; 698ca6c1d82SSagi Grimberg default: 69924f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 700ca6c1d82SSagi Grimberg isert_np, event); 701ca6c1d82SSagi Grimberg } 702ca6c1d82SSagi Grimberg 7033b726ae2SSagi Grimberg return -1; 7043b726ae2SSagi Grimberg } 7053b726ae2SSagi Grimberg 706ca6c1d82SSagi Grimberg static int 707ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 708ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 709ca6c1d82SSagi Grimberg { 710f81bf458SJenny Derzhavetz struct isert_conn *isert_conn = cma_id->qp->qp_context; 711ca6c1d82SSagi Grimberg 712dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 7136d1fba0cSJenny Derzhavetz switch (isert_conn->state) { 7146d1fba0cSJenny Derzhavetz case ISER_CONN_TERMINATING: 7156d1fba0cSJenny Derzhavetz break; 7166d1fba0cSJenny Derzhavetz case ISER_CONN_UP: 717128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 718572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 7196d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(isert_conn); 7206d1fba0cSJenny Derzhavetz break; 7216d1fba0cSJenny Derzhavetz case ISER_CONN_BOUND: 7226d1fba0cSJenny Derzhavetz case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 7236d1fba0cSJenny Derzhavetz iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 7246d1fba0cSJenny Derzhavetz break; 7256d1fba0cSJenny Derzhavetz default: 726506f71d1SColin Ian King isert_warn("conn %p terminating in state %d\n", 7276d1fba0cSJenny Derzhavetz isert_conn, isert_conn->state); 7286d1fba0cSJenny Derzhavetz } 729dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 730128e9cc8SSagi Grimberg 7313b726ae2SSagi Grimberg return 0; 732b8d26b3bSNicholas Bellinger } 733b8d26b3bSNicholas Bellinger 7344a579da2SSagi Grimberg static int 735954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 736954f2372SSagi Grimberg { 73719e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 738954f2372SSagi Grimberg 739cd52cb26SSagi Grimberg ib_drain_qp(isert_conn->qp); 740bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 741dac6ab30SSagi Grimberg isert_conn->cm_id = NULL; 742954f2372SSagi Grimberg isert_put_conn(isert_conn); 7434a579da2SSagi Grimberg 7444a579da2SSagi Grimberg return -1; 745954f2372SSagi Grimberg } 746954f2372SSagi Grimberg 747b8d26b3bSNicholas Bellinger static int 748b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 749b8d26b3bSNicholas Bellinger { 750f81bf458SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 75163b268d2SRaju Rangoju struct isert_conn *isert_conn; 752b8d26b3bSNicholas Bellinger int ret = 0; 753b8d26b3bSNicholas Bellinger 754ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n", 755ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event, 7564c22e07fSSagi Grimberg event->status, cma_id, cma_id->context); 757b8d26b3bSNicholas Bellinger 758f81bf458SJenny Derzhavetz if (isert_np->cm_id == cma_id) 759f81bf458SJenny Derzhavetz return isert_np_cma_handler(cma_id->context, event->event); 760f81bf458SJenny Derzhavetz 761b8d26b3bSNicholas Bellinger switch (event->event) { 762b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 763b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 7643b726ae2SSagi Grimberg if (ret) 7654c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret); 766b8d26b3bSNicholas Bellinger break; 767b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 768b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 769b8d26b3bSNicholas Bellinger break; 77088c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 77188c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 77288c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 773ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 774b8d26b3bSNicholas Bellinger break; 77563b268d2SRaju Rangoju case RDMA_CM_EVENT_DEVICE_REMOVAL: 77663b268d2SRaju Rangoju isert_conn = cma_id->qp->qp_context; 77763b268d2SRaju Rangoju isert_conn->dev_removed = true; 77863b268d2SRaju Rangoju isert_disconnected_handler(cma_id, event->event); 77963b268d2SRaju Rangoju wait_event_interruptible(isert_conn->rem_wait, 78063b268d2SRaju Rangoju isert_conn->state == ISER_CONN_DOWN); 78163b268d2SRaju Rangoju kfree(isert_conn); 78263b268d2SRaju Rangoju /* 78363b268d2SRaju Rangoju * return non-zero from the callback to destroy 78463b268d2SRaju Rangoju * the rdma cm id 78563b268d2SRaju Rangoju */ 78663b268d2SRaju Rangoju return 1; 787d2271bd0SBart Van Assche case RDMA_CM_EVENT_REJECTED: 7881e38a366SSteve Wise isert_info("Connection rejected: %s\n", 7891e38a366SSteve Wise rdma_reject_msg(cma_id, event->status)); 790d2271bd0SBart Van Assche /* fall through */ 791d2271bd0SBart Van Assche case RDMA_CM_EVENT_UNREACHABLE: 792b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 7934a579da2SSagi Grimberg ret = isert_connect_error(cma_id); 794954f2372SSagi Grimberg break; 795b8d26b3bSNicholas Bellinger default: 79624f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 797b8d26b3bSNicholas Bellinger break; 798b8d26b3bSNicholas Bellinger } 799b8d26b3bSNicholas Bellinger 800b8d26b3bSNicholas Bellinger return ret; 801b8d26b3bSNicholas Bellinger } 802b8d26b3bSNicholas Bellinger 803b8d26b3bSNicholas Bellinger static int 8044366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count) 805b8d26b3bSNicholas Bellinger { 806e01a7674SBart Van Assche struct ib_recv_wr *rx_wr; 807b8d26b3bSNicholas Bellinger int i, ret; 808b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 809b8d26b3bSNicholas Bellinger 810dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 8114366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i]; 8129679cc51SChristoph Hellwig 8139679cc51SChristoph Hellwig rx_wr->wr_cqe = &rx_desc->rx_cqe; 814b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 815b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 816b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 8177a56dc88SSagi Grimberg rx_desc->in_use = false; 818b8d26b3bSNicholas Bellinger } 819b8d26b3bSNicholas Bellinger rx_wr--; 820b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 821b8d26b3bSNicholas Bellinger 822e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 8236d1fba0cSJenny Derzhavetz if (ret) 82424f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 8254366b19cSJenny Derzhavetz 8264366b19cSJenny Derzhavetz return ret; 8274366b19cSJenny Derzhavetz } 8284366b19cSJenny Derzhavetz 8294366b19cSJenny Derzhavetz static int 8304366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 8314366b19cSJenny Derzhavetz { 832e01a7674SBart Van Assche struct ib_recv_wr rx_wr; 8334366b19cSJenny Derzhavetz int ret; 8344366b19cSJenny Derzhavetz 8357a56dc88SSagi Grimberg if (!rx_desc->in_use) { 8367a56dc88SSagi Grimberg /* 8377a56dc88SSagi Grimberg * if the descriptor is not in-use we already reposted it 8387a56dc88SSagi Grimberg * for recv, so just silently return 8397a56dc88SSagi Grimberg */ 8407a56dc88SSagi Grimberg return 0; 8417a56dc88SSagi Grimberg } 8427a56dc88SSagi Grimberg 8437a56dc88SSagi Grimberg rx_desc->in_use = false; 8449679cc51SChristoph Hellwig rx_wr.wr_cqe = &rx_desc->rx_cqe; 8454366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg; 8464366b19cSJenny Derzhavetz rx_wr.num_sge = 1; 8474366b19cSJenny Derzhavetz rx_wr.next = NULL; 8484366b19cSJenny Derzhavetz 849e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 8506d1fba0cSJenny Derzhavetz if (ret) 8514366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret); 8524366b19cSJenny Derzhavetz 853b8d26b3bSNicholas Bellinger return ret; 854b8d26b3bSNicholas Bellinger } 855b8d26b3bSNicholas Bellinger 856b8d26b3bSNicholas Bellinger static int 8579679cc51SChristoph Hellwig isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 858b8d26b3bSNicholas Bellinger { 859dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 860e01a7674SBart Van Assche struct ib_send_wr send_wr; 861b8d26b3bSNicholas Bellinger int ret; 862b8d26b3bSNicholas Bellinger 863b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 864b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 865b8d26b3bSNicholas Bellinger 8669679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_login_send_done; 8679679cc51SChristoph Hellwig 868b8d26b3bSNicholas Bellinger send_wr.next = NULL; 8699679cc51SChristoph Hellwig send_wr.wr_cqe = &tx_desc->tx_cqe; 870b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 871b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 872b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 873b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 874b8d26b3bSNicholas Bellinger 875e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 876bdf20e72SSagi Grimberg if (ret) 87724f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 878b8d26b3bSNicholas Bellinger 879b8d26b3bSNicholas Bellinger return ret; 880b8d26b3bSNicholas Bellinger } 881b8d26b3bSNicholas Bellinger 882b8d26b3bSNicholas Bellinger static void 883763b6965SAlex Estrin __isert_create_send_desc(struct isert_device *device, 884b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 885b8d26b3bSNicholas Bellinger { 886b8d26b3bSNicholas Bellinger 887d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 888d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL; 889b8d26b3bSNicholas Bellinger 890b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 891b8d26b3bSNicholas Bellinger 89234efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 89334efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 89424f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 895b8d26b3bSNicholas Bellinger } 896b8d26b3bSNicholas Bellinger } 897b8d26b3bSNicholas Bellinger 898763b6965SAlex Estrin static void 899763b6965SAlex Estrin isert_create_send_desc(struct isert_conn *isert_conn, 900763b6965SAlex Estrin struct isert_cmd *isert_cmd, 901763b6965SAlex Estrin struct iser_tx_desc *tx_desc) 902763b6965SAlex Estrin { 903763b6965SAlex Estrin struct isert_device *device = isert_conn->device; 904763b6965SAlex Estrin struct ib_device *ib_dev = device->ib_device; 905763b6965SAlex Estrin 906763b6965SAlex Estrin ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 907763b6965SAlex Estrin ISER_HEADERS_LEN, DMA_TO_DEVICE); 908763b6965SAlex Estrin 909763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc); 910763b6965SAlex Estrin } 911763b6965SAlex Estrin 912b8d26b3bSNicholas Bellinger static int 913b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 914b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 915b8d26b3bSNicholas Bellinger { 916dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 91767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 918b8d26b3bSNicholas Bellinger u64 dma_addr; 919b8d26b3bSNicholas Bellinger 920b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 921b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 922b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 92324f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 924b8d26b3bSNicholas Bellinger return -ENOMEM; 925b8d26b3bSNicholas Bellinger } 926b8d26b3bSNicholas Bellinger 927b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 928b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 929b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 93034efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 931b8d26b3bSNicholas Bellinger 9324c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 9334c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 9344c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey); 935b8d26b3bSNicholas Bellinger 936b8d26b3bSNicholas Bellinger return 0; 937b8d26b3bSNicholas Bellinger } 938b8d26b3bSNicholas Bellinger 939b8d26b3bSNicholas Bellinger static void 94095b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 94168a86deeSSagi Grimberg struct ib_send_wr *send_wr) 942b8d26b3bSNicholas Bellinger { 94395b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 94495b60f07SNicholas Bellinger 9459679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_send_done; 9469679cc51SChristoph Hellwig send_wr->wr_cqe = &tx_desc->tx_cqe; 947422bd0acSJenny Derzhavetz 948422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 949422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV; 950422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 951422bd0acSJenny Derzhavetz } else { 952b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 953422bd0acSJenny Derzhavetz } 954422bd0acSJenny Derzhavetz 95595b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 956b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 95795b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 958b8d26b3bSNicholas Bellinger } 959b8d26b3bSNicholas Bellinger 960b8d26b3bSNicholas Bellinger static int 9619679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn) 962b8d26b3bSNicholas Bellinger { 963e01a7674SBart Van Assche struct ib_recv_wr rx_wr; 964b8d26b3bSNicholas Bellinger struct ib_sge sge; 965b8d26b3bSNicholas Bellinger int ret; 966b8d26b3bSNicholas Bellinger 967b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 968b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 969ed1083b2SChristoph Hellwig sge.length = ISER_RX_PAYLOAD_SIZE; 97034efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey; 971b8d26b3bSNicholas Bellinger 97224f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 973b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 974b8d26b3bSNicholas Bellinger 9759679cc51SChristoph Hellwig isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 9769679cc51SChristoph Hellwig 977b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 9789679cc51SChristoph Hellwig rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 979b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 980b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 981b8d26b3bSNicholas Bellinger 982e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 9836d1fba0cSJenny Derzhavetz if (ret) 98424f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 985b8d26b3bSNicholas Bellinger 986b8d26b3bSNicholas Bellinger return ret; 987b8d26b3bSNicholas Bellinger } 988b8d26b3bSNicholas Bellinger 989b8d26b3bSNicholas Bellinger static int 990b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 991b8d26b3bSNicholas Bellinger u32 length) 992b8d26b3bSNicholas Bellinger { 993b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 994dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 99567cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 996dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 997b8d26b3bSNicholas Bellinger int ret; 998b8d26b3bSNicholas Bellinger 999763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc); 1000b8d26b3bSNicholas Bellinger 1001b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1002b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1003b8d26b3bSNicholas Bellinger 1004b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1005b8d26b3bSNicholas Bellinger 1006b8d26b3bSNicholas Bellinger if (length > 0) { 1007b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1008b8d26b3bSNicholas Bellinger 1009b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1010b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1011b8d26b3bSNicholas Bellinger 1012b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1013b8d26b3bSNicholas Bellinger 1014b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1015b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1016b8d26b3bSNicholas Bellinger 1017b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1018b8d26b3bSNicholas Bellinger tx_dsg->length = length; 101934efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1020b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1021b8d26b3bSNicholas Bellinger } 1022b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1023b8d26b3bSNicholas Bellinger if (login->login_complete) { 1024b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1025b8d26b3bSNicholas Bellinger if (ret) 1026b8d26b3bSNicholas Bellinger return ret; 1027b8d26b3bSNicholas Bellinger 10284366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn, 10294366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS); 1030b8d26b3bSNicholas Bellinger if (ret) 1031b8d26b3bSNicholas Bellinger return ret; 1032b8d26b3bSNicholas Bellinger 1033128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 1034dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 1035128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 1036dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1037b8d26b3bSNicholas Bellinger goto post_send; 1038b8d26b3bSNicholas Bellinger } 1039b8d26b3bSNicholas Bellinger 10409679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 1041b8d26b3bSNicholas Bellinger if (ret) 1042b8d26b3bSNicholas Bellinger return ret; 1043b8d26b3bSNicholas Bellinger } 1044b8d26b3bSNicholas Bellinger post_send: 10459679cc51SChristoph Hellwig ret = isert_login_post_send(isert_conn, tx_desc); 1046b8d26b3bSNicholas Bellinger if (ret) 1047b8d26b3bSNicholas Bellinger return ret; 1048b8d26b3bSNicholas Bellinger 1049b8d26b3bSNicholas Bellinger return 0; 1050b8d26b3bSNicholas Bellinger } 1051b8d26b3bSNicholas Bellinger 1052b8d26b3bSNicholas Bellinger static void 10532371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1054b8d26b3bSNicholas Bellinger { 10555adabdd1SChristoph Hellwig struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 10562371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1057b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1058b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1059b8d26b3bSNicholas Bellinger int size; 1060b8d26b3bSNicholas Bellinger 106124f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 10622371e5daSSagi Grimberg 10632371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1064b8d26b3bSNicholas Bellinger 1065b8d26b3bSNicholas Bellinger if (login->first_request) { 1066b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1067b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1068b8d26b3bSNicholas Bellinger /* 1069b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1070b8d26b3bSNicholas Bellinger * login request PDU. 1071b8d26b3bSNicholas Bellinger */ 1072b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1073b8d26b3bSNicholas Bellinger login->current_stage = 1074b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1075b8d26b3bSNicholas Bellinger >> 2; 1076b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1077b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1078b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1079b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1080b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1081b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1082b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1083b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1084b8d26b3bSNicholas Bellinger } 1085b8d26b3bSNicholas Bellinger 1086b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1087b8d26b3bSNicholas Bellinger 1088b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 10894c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d " 10904c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 10914c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS); 1092b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1093b8d26b3bSNicholas Bellinger 10946faaa85fSNicholas Bellinger if (login->first_request) { 1095dac6ab30SSagi Grimberg complete(&isert_conn->login_comp); 10966faaa85fSNicholas Bellinger return; 10976faaa85fSNicholas Bellinger } 10986faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1099b8d26b3bSNicholas Bellinger } 1100b8d26b3bSNicholas Bellinger 1101b8d26b3bSNicholas Bellinger static struct iscsi_cmd 11024366b19cSJenny Derzhavetz *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1103b8d26b3bSNicholas Bellinger { 11046700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1105b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1106d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1107b8d26b3bSNicholas Bellinger 1108676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1109d703ce2fSNicholas Bellinger if (!cmd) { 111024f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1111b8d26b3bSNicholas Bellinger return NULL; 1112b8d26b3bSNicholas Bellinger } 1113d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1114b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1115d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 11164366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc; 1117b8d26b3bSNicholas Bellinger 1118d703ce2fSNicholas Bellinger return cmd; 1119b8d26b3bSNicholas Bellinger } 1120b8d26b3bSNicholas Bellinger 1121b8d26b3bSNicholas Bellinger static int 1122b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1123d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1124d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1125b8d26b3bSNicholas Bellinger { 1126b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1127b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1128b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1129b8d26b3bSNicholas Bellinger bool dump_payload = false; 11309fd60088SJenny Derzhavetz unsigned int data_len; 1131b8d26b3bSNicholas Bellinger 1132b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1133b8d26b3bSNicholas Bellinger if (rc < 0) 1134b8d26b3bSNicholas Bellinger return rc; 1135b8d26b3bSNicholas Bellinger 1136b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1137b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1138b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 11399fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length; 1140b8d26b3bSNicholas Bellinger 11419fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len) 11429fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1143b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1144b8d26b3bSNicholas Bellinger if (rc < 0) { 1145b8d26b3bSNicholas Bellinger return 0; 1146b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1147b8d26b3bSNicholas Bellinger dump_payload = true; 1148b8d26b3bSNicholas Bellinger goto sequence_cmd; 1149b8d26b3bSNicholas Bellinger } 1150b8d26b3bSNicholas Bellinger 1151b8d26b3bSNicholas Bellinger if (!imm_data) 1152b8d26b3bSNicholas Bellinger return 0; 1153b8d26b3bSNicholas Bellinger 11549fd60088SJenny Derzhavetz if (imm_data_len != data_len) { 1155b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 11569fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 11579fd60088SJenny Derzhavetz &rx_desc->data[0], imm_data_len); 11589fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 11599fd60088SJenny Derzhavetz sg_nents, imm_data_len); 11609fd60088SJenny Derzhavetz } else { 11619fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1); 11629fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg; 11639fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1; 11649fd60088SJenny Derzhavetz sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 11659fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n", 11669fd60088SJenny Derzhavetz imm_data_len); 11679fd60088SJenny Derzhavetz } 1168b8d26b3bSNicholas Bellinger 1169b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1170b8d26b3bSNicholas Bellinger 1171b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1172b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1173b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1174b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1175b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1176b8d26b3bSNicholas Bellinger } 1177b8d26b3bSNicholas Bellinger 1178b8d26b3bSNicholas Bellinger sequence_cmd: 1179561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1180b8d26b3bSNicholas Bellinger 1181b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 11820300b114SBart Van Assche iscsit_set_unsolicited_dataout(cmd); 11836cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 1184afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd); 1185b8d26b3bSNicholas Bellinger 1186b8d26b3bSNicholas Bellinger return 0; 1187b8d26b3bSNicholas Bellinger } 1188b8d26b3bSNicholas Bellinger 1189b8d26b3bSNicholas Bellinger static int 1190b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1191b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1192b8d26b3bSNicholas Bellinger { 1193b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1194b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1195b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1196b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1197b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1198b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1199b8d26b3bSNicholas Bellinger 1200b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1201b8d26b3bSNicholas Bellinger if (rc < 0) 1202b8d26b3bSNicholas Bellinger return rc; 1203b8d26b3bSNicholas Bellinger else if (!cmd) 1204b8d26b3bSNicholas Bellinger return 0; 1205b8d26b3bSNicholas Bellinger /* 1206b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1207b8d26b3bSNicholas Bellinger */ 1208b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 120924f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1210b8d26b3bSNicholas Bellinger dump_stack(); 1211b8d26b3bSNicholas Bellinger return -1; 1212b8d26b3bSNicholas Bellinger } 1213b8d26b3bSNicholas Bellinger 12144c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 12154c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n", 12164c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done, 12174c22e07fSSagi Grimberg cmd->se_cmd.data_length); 1218b8d26b3bSNicholas Bellinger 1219b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1220b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1221b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1222b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1223b8d26b3bSNicholas Bellinger /* 1224b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1225b8d26b3bSNicholas Bellinger */ 1226b8d26b3bSNicholas Bellinger if (page_off) { 12274c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n"); 1228b8d26b3bSNicholas Bellinger dump_stack(); 1229b8d26b3bSNicholas Bellinger return -1; 1230b8d26b3bSNicholas Bellinger } 12314c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 12324c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off, 12334c22e07fSSagi Grimberg sg_nents, &rx_desc->data[0], unsol_data_len); 1234b8d26b3bSNicholas Bellinger 1235b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1236b8d26b3bSNicholas Bellinger unsol_data_len); 1237b8d26b3bSNicholas Bellinger 1238b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1239b8d26b3bSNicholas Bellinger if (rc < 0) 1240b8d26b3bSNicholas Bellinger return rc; 1241b8d26b3bSNicholas Bellinger 12424366b19cSJenny Derzhavetz /* 12434366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive - 12444366b19cSJenny Derzhavetz * so post the buffer before hand 12454366b19cSJenny Derzhavetz */ 12464366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, rx_desc); 12474366b19cSJenny Derzhavetz if (rc) { 12484366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 12494366b19cSJenny Derzhavetz return rc; 12504366b19cSJenny Derzhavetz } 1251b8d26b3bSNicholas Bellinger return 0; 1252b8d26b3bSNicholas Bellinger } 1253b8d26b3bSNicholas Bellinger 1254b8d26b3bSNicholas Bellinger static int 1255778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1256d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1257d703ce2fSNicholas Bellinger unsigned char *buf) 1258778de368SNicholas Bellinger { 1259778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1260778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1261778de368SNicholas Bellinger int rc; 1262778de368SNicholas Bellinger 1263778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1264778de368SNicholas Bellinger if (rc < 0) 1265778de368SNicholas Bellinger return rc; 1266778de368SNicholas Bellinger /* 1267778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1268778de368SNicholas Bellinger */ 1269778de368SNicholas Bellinger 1270778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1271778de368SNicholas Bellinger } 1272778de368SNicholas Bellinger 1273778de368SNicholas Bellinger static int 1274adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1275d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1276d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1277adb54c29SNicholas Bellinger { 1278adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1279adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1280adb54c29SNicholas Bellinger int rc; 1281b44a2b67SSagi Grimberg unsigned char *text_in = NULL; 1282adb54c29SNicholas Bellinger 1283adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1284adb54c29SNicholas Bellinger if (rc < 0) 1285adb54c29SNicholas Bellinger return rc; 1286adb54c29SNicholas Bellinger 1287b44a2b67SSagi Grimberg if (payload_length) { 1288adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 128993b80f29SLeon Romanovsky if (!text_in) 1290adb54c29SNicholas Bellinger return -ENOMEM; 1291adb54c29SNicholas Bellinger } 1292adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1293adb54c29SNicholas Bellinger 1294adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1295adb54c29SNicholas Bellinger 1296adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1297adb54c29SNicholas Bellinger } 1298adb54c29SNicholas Bellinger 1299adb54c29SNicholas Bellinger static int 1300b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1301b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1302b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1303b8d26b3bSNicholas Bellinger { 1304b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1305b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1306b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1307b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1308b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1309b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1310b8d26b3bSNicholas Bellinger 1311fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType && 1312ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 131324f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1314ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1315ca40d24eSNicholas Bellinger return 0; 1316ca40d24eSNicholas Bellinger } 1317ca40d24eSNicholas Bellinger 1318b8d26b3bSNicholas Bellinger switch (opcode) { 1319b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 13204366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1321b8d26b3bSNicholas Bellinger if (!cmd) 1322b8d26b3bSNicholas Bellinger break; 1323b8d26b3bSNicholas Bellinger 1324d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1325b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1326b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1327b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1328b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1329422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1330b8d26b3bSNicholas Bellinger 1331d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1332b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1333b8d26b3bSNicholas Bellinger break; 1334b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 13354366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1336b8d26b3bSNicholas Bellinger if (!cmd) 1337b8d26b3bSNicholas Bellinger break; 1338b8d26b3bSNicholas Bellinger 1339d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1340d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1341778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1342b8d26b3bSNicholas Bellinger break; 1343b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1344b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1345b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1346b8d26b3bSNicholas Bellinger break; 1347b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 13484366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1349b8d26b3bSNicholas Bellinger if (!cmd) 1350b8d26b3bSNicholas Bellinger break; 1351b8d26b3bSNicholas Bellinger 1352b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1353b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1354b8d26b3bSNicholas Bellinger break; 1355b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 13564366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1357b8d26b3bSNicholas Bellinger if (!cmd) 1358b8d26b3bSNicholas Bellinger break; 1359b8d26b3bSNicholas Bellinger 1360b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1361b8d26b3bSNicholas Bellinger break; 1362adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 13634366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1364e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 13654366b19cSJenny Derzhavetz else 13664366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 13674366b19cSJenny Derzhavetz 1368e4f4e801SSagi Grimberg if (!cmd) 1369e4f4e801SSagi Grimberg break; 1370adb54c29SNicholas Bellinger 1371d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1372d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1373adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1374adb54c29SNicholas Bellinger break; 1375b8d26b3bSNicholas Bellinger default: 137624f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1377b8d26b3bSNicholas Bellinger dump_stack(); 1378b8d26b3bSNicholas Bellinger break; 1379b8d26b3bSNicholas Bellinger } 1380b8d26b3bSNicholas Bellinger 1381b8d26b3bSNicholas Bellinger return ret; 1382b8d26b3bSNicholas Bellinger } 1383b8d26b3bSNicholas Bellinger 1384b8d26b3bSNicholas Bellinger static void 13859679cc51SChristoph Hellwig isert_print_wc(struct ib_wc *wc, const char *type) 1386b8d26b3bSNicholas Bellinger { 13879679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 13889679cc51SChristoph Hellwig isert_err("%s failure: %s (%d) vend_err %x\n", type, 13899679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status, 13909679cc51SChristoph Hellwig wc->vendor_err); 13919679cc51SChristoph Hellwig else 13929679cc51SChristoph Hellwig isert_dbg("%s failure: %s (%d)\n", type, 13939679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status); 13949679cc51SChristoph Hellwig } 13959679cc51SChristoph Hellwig 13969679cc51SChristoph Hellwig static void 13979679cc51SChristoph Hellwig isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 13989679cc51SChristoph Hellwig { 13999679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 14009679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 14019679cc51SChristoph Hellwig struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 14029679cc51SChristoph Hellwig struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1403d3cf81f9SSagi Grimberg struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1404b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1405b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1406b8d26b3bSNicholas Bellinger 14079679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14089679cc51SChristoph Hellwig isert_print_wc(wc, "recv"); 14099679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 14109679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 14119679cc51SChristoph Hellwig return; 14129679cc51SChristoph Hellwig } 14139679cc51SChristoph Hellwig 14147a56dc88SSagi Grimberg rx_desc->in_use = true; 14157a56dc88SSagi Grimberg 14169679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 14179679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 14189679cc51SChristoph Hellwig 14199679cc51SChristoph Hellwig isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 14209679cc51SChristoph Hellwig rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 14219679cc51SChristoph Hellwig (int)(wc->byte_len - ISER_HEADERS_LEN)); 14229679cc51SChristoph Hellwig 1423d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) { 1424b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1425d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) { 1426d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag); 1427d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va); 14284c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1429b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1430b8d26b3bSNicholas Bellinger } 1431d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) { 1432d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag); 1433d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va); 14344c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1435b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1436b8d26b3bSNicholas Bellinger } 1437b8d26b3bSNicholas Bellinger 143824f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1439b8d26b3bSNicholas Bellinger break; 1440b8d26b3bSNicholas Bellinger case ISER_HELLO: 144124f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1442b8d26b3bSNicholas Bellinger break; 1443b8d26b3bSNicholas Bellinger default: 1444d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1445b8d26b3bSNicholas Bellinger break; 1446b8d26b3bSNicholas Bellinger } 1447b8d26b3bSNicholas Bellinger 14486c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc, 1449b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 14509679cc51SChristoph Hellwig 14519679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 14529679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1453b8d26b3bSNicholas Bellinger } 1454b8d26b3bSNicholas Bellinger 1455b8d26b3bSNicholas Bellinger static void 14569679cc51SChristoph Hellwig isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1457b8d26b3bSNicholas Bellinger { 14589679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1459fce50a2fSNicholas Bellinger struct ib_device *ib_dev = isert_conn->device->ib_device; 1460b8d26b3bSNicholas Bellinger 14619679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14629679cc51SChristoph Hellwig isert_print_wc(wc, "login recv"); 14639679cc51SChristoph Hellwig return; 1464b8d26b3bSNicholas Bellinger } 1465b8d26b3bSNicholas Bellinger 14669679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 14679679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1468b8d26b3bSNicholas Bellinger 14699679cc51SChristoph Hellwig isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1470b8d26b3bSNicholas Bellinger 14712371e5daSSagi Grimberg if (isert_conn->conn) { 14722371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 14732371e5daSSagi Grimberg 14742371e5daSSagi Grimberg if (login && !login->first_request) 14752371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 14762371e5daSSagi Grimberg } 14779679cc51SChristoph Hellwig 1478dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 14792371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 1480dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1481b8d26b3bSNicholas Bellinger 14829679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 14839679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1484b8d26b3bSNicholas Bellinger } 1485b8d26b3bSNicholas Bellinger 1486e3d7e4c3SSagi Grimberg static void 148738a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1488e3d7e4c3SSagi Grimberg { 148938a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 149038a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1491e3d7e4c3SSagi Grimberg 149238a2d0d4SChristoph Hellwig if (!cmd->rw.nr_ops) 149338a2d0d4SChristoph Hellwig return; 149438a2d0d4SChristoph Hellwig 149538a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 149638a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 149738a2d0d4SChristoph Hellwig conn->cm_id->port_num, se_cmd->t_data_sg, 149838a2d0d4SChristoph Hellwig se_cmd->t_data_nents, se_cmd->t_prot_sg, 149938a2d0d4SChristoph Hellwig se_cmd->t_prot_nents, dir); 150038a2d0d4SChristoph Hellwig } else { 150138a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 150238a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1503e3d7e4c3SSagi Grimberg } 1504e3d7e4c3SSagi Grimberg 150538a2d0d4SChristoph Hellwig cmd->rw.nr_ops = 0; 150659464ef4SVu Pham } 150759464ef4SVu Pham 150859464ef4SVu Pham static void 150903e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1510b8d26b3bSNicholas Bellinger { 1511d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1512b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1513186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1514e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr; 1515b8d26b3bSNicholas Bellinger 15164c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1517b8d26b3bSNicholas Bellinger 1518b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1519b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1520b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1521b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15225159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1523b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1524b8d26b3bSNicholas Bellinger 152503e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1526b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 152703e7848aSNicholas Bellinger /* 152803e7848aSNicholas Bellinger * Check for special case during comp_err where 152903e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 153003e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 153103e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 153203e7848aSNicholas Bellinger */ 153303e7848aSNicholas Bellinger if (comp_err && 153403e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 153503e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 153603e7848aSNicholas Bellinger 1537afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 153803e7848aSNicholas Bellinger } 153903e7848aSNicholas Bellinger } 1540b8d26b3bSNicholas Bellinger 154138a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1542186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1543186a9647SNicholas Bellinger break; 1544b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1545186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1546186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15475159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1548186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1549186a9647SNicholas Bellinger 1550b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1551b8d26b3bSNicholas Bellinger break; 1552b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1553b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1554adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1555e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1556e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */ 1557e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1558e4f4e801SSagi Grimberg break; 1559e4f4e801SSagi Grimberg 1560b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1561b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15625159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1563b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1564b8d26b3bSNicholas Bellinger 1565b8d26b3bSNicholas Bellinger /* 1566b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1567b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1568b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1569b8d26b3bSNicholas Bellinger */ 1570b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 157111378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 15723df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1573b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1574b8d26b3bSNicholas Bellinger break; 1575b8d26b3bSNicholas Bellinger } 1576d2271bd0SBart Van Assche /* fall through */ 1577b8d26b3bSNicholas Bellinger default: 1578d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1579b8d26b3bSNicholas Bellinger break; 1580b8d26b3bSNicholas Bellinger } 1581b8d26b3bSNicholas Bellinger } 1582b8d26b3bSNicholas Bellinger 1583b8d26b3bSNicholas Bellinger static void 1584b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1585b8d26b3bSNicholas Bellinger { 1586b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 15874c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n"); 1588b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1589b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1590b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1591b8d26b3bSNicholas Bellinger } 1592b8d26b3bSNicholas Bellinger } 1593b8d26b3bSNicholas Bellinger 1594b8d26b3bSNicholas Bellinger static void 1595b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 159603e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1597b8d26b3bSNicholas Bellinger { 1598dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 15994c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1600dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1601dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1602dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1603b8d26b3bSNicholas Bellinger } 1604b8d26b3bSNicholas Bellinger 1605b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 160603e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1607b8d26b3bSNicholas Bellinger } 1608b8d26b3bSNicholas Bellinger 160996b7973eSSagi Grimberg static int 161096b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1611f93f3a70SSagi Grimberg { 16129e961ae7SSagi Grimberg struct ib_mr_status mr_status; 161396b7973eSSagi Grimberg int ret; 1614f93f3a70SSagi Grimberg 161596b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 16169e961ae7SSagi Grimberg if (ret) { 161724f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 16189e961ae7SSagi Grimberg goto fail_mr_status; 16199e961ae7SSagi Grimberg } 162096b7973eSSagi Grimberg 16219e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 162296b7973eSSagi Grimberg u64 sec_offset_err; 16239e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 16249e961ae7SSagi Grimberg 16259e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 16269e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 16279e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 16289e961ae7SSagi Grimberg break; 16299e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 16309e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 16319e961ae7SSagi Grimberg break; 16329e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 16339e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 16349e961ae7SSagi Grimberg break; 16359e961ae7SSagi Grimberg } 163696b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 163796b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 163896b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 16399e961ae7SSagi Grimberg 16404c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx " 16419e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 16429e961ae7SSagi Grimberg mr_status.sig_err.err_type, 16439e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 16449e961ae7SSagi Grimberg mr_status.sig_err.expected, 16459e961ae7SSagi Grimberg mr_status.sig_err.actual); 16469e961ae7SSagi Grimberg ret = 1; 16479e961ae7SSagi Grimberg } 16489e961ae7SSagi Grimberg 16499e961ae7SSagi Grimberg fail_mr_status: 165096b7973eSSagi Grimberg return ret; 165196b7973eSSagi Grimberg } 165296b7973eSSagi Grimberg 165396b7973eSSagi Grimberg static void 16549679cc51SChristoph Hellwig isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 165596b7973eSSagi Grimberg { 16569679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1657dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 16589679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1659e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 16609679cc51SChristoph Hellwig struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 166196b7973eSSagi Grimberg int ret = 0; 166296b7973eSSagi Grimberg 16639679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 16649679cc51SChristoph Hellwig isert_print_wc(wc, "rdma write"); 16659679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 16669679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 16679679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 16689679cc51SChristoph Hellwig return; 16699679cc51SChristoph Hellwig } 16709679cc51SChristoph Hellwig 16719679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 16729679cc51SChristoph Hellwig 1673e9a53e73SIsrael Rukshin ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 167438a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 167596b7973eSSagi Grimberg 1676555a65f6SNicholas Bellinger if (ret) { 1677555a65f6SNicholas Bellinger /* 1678555a65f6SNicholas Bellinger * transport_generic_request_failure() expects to have 1679555a65f6SNicholas Bellinger * plus two references to handle queue-full, so re-add 1680555a65f6SNicholas Bellinger * one here as target-core will have already dropped 1681555a65f6SNicholas Bellinger * it after the first isert_put_datain() callback. 1682555a65f6SNicholas Bellinger */ 1683555a65f6SNicholas Bellinger kref_get(&cmd->cmd_kref); 1684555a65f6SNicholas Bellinger transport_generic_request_failure(cmd, cmd->pi_err); 1685555a65f6SNicholas Bellinger } else { 1686555a65f6SNicholas Bellinger /* 1687555a65f6SNicholas Bellinger * XXX: isert_put_response() failure is not retried. 1688555a65f6SNicholas Bellinger */ 1689555a65f6SNicholas Bellinger ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 16909e961ae7SSagi Grimberg if (ret) 1691555a65f6SNicholas Bellinger pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1692555a65f6SNicholas Bellinger } 1693f93f3a70SSagi Grimberg } 1694f93f3a70SSagi Grimberg 1695f93f3a70SSagi Grimberg static void 16969679cc51SChristoph Hellwig isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1697b8d26b3bSNicholas Bellinger { 16989679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 16999679cc51SChristoph Hellwig struct isert_device *device = isert_conn->device; 17009679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1701e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1702d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1703b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 17045bac4b1aSSagi Grimberg int ret = 0; 1705b8d26b3bSNicholas Bellinger 17069679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17079679cc51SChristoph Hellwig isert_print_wc(wc, "rdma read"); 17089679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17099679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17109679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 17119679cc51SChristoph Hellwig return; 17129679cc51SChristoph Hellwig } 17139679cc51SChristoph Hellwig 17149679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 17159679cc51SChristoph Hellwig 1716b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 171738a2d0d4SChristoph Hellwig 171838a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) 1719e9a53e73SIsrael Rukshin ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 172038a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 172138a2d0d4SChristoph Hellwig cmd->write_data_done = 0; 1722b8d26b3bSNicholas Bellinger 172324f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1724b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1725b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1726b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1727b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1728b8d26b3bSNicholas Bellinger 1729555a65f6SNicholas Bellinger /* 1730555a65f6SNicholas Bellinger * transport_generic_request_failure() will drop the extra 1731555a65f6SNicholas Bellinger * se_cmd->cmd_kref reference after T10-PI error, and handle 1732555a65f6SNicholas Bellinger * any non-zero ->queue_status() callback error retries. 1733555a65f6SNicholas Bellinger */ 1734555a65f6SNicholas Bellinger if (ret) 1735555a65f6SNicholas Bellinger transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1736555a65f6SNicholas Bellinger else 1737b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1738b8d26b3bSNicholas Bellinger } 1739b8d26b3bSNicholas Bellinger 1740b8d26b3bSNicholas Bellinger static void 1741b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1742b8d26b3bSNicholas Bellinger { 1743b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1744b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1745b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1746dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1747d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1748b8d26b3bSNicholas Bellinger 17494c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 17504c22e07fSSagi Grimberg 1751b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1752b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1753b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1754d2271bd0SBart Van Assche /* fall through */ 1755d2271bd0SBart Van Assche case ISTATE_SEND_REJECT: 1756d2271bd0SBart Van Assche case ISTATE_SEND_TEXTRSP: 1757b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 17584c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 17594c22e07fSSagi Grimberg ib_dev, false); 17603df8f68aSNicholas Bellinger break; 1761b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1762b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1763b8d26b3bSNicholas Bellinger break; 1764b8d26b3bSNicholas Bellinger default: 17654c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state); 1766b8d26b3bSNicholas Bellinger dump_stack(); 1767b8d26b3bSNicholas Bellinger break; 1768b8d26b3bSNicholas Bellinger } 1769b8d26b3bSNicholas Bellinger } 1770b8d26b3bSNicholas Bellinger 1771b8d26b3bSNicholas Bellinger static void 17729679cc51SChristoph Hellwig isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1773b8d26b3bSNicholas Bellinger { 17749679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17759679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17769679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1777b8d26b3bSNicholas Bellinger 17789679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17799679cc51SChristoph Hellwig isert_print_wc(wc, "login send"); 17809679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17819679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17829679cc51SChristoph Hellwig } 17839679cc51SChristoph Hellwig 17849679cc51SChristoph Hellwig isert_unmap_tx_desc(tx_desc, ib_dev); 17859679cc51SChristoph Hellwig } 17869679cc51SChristoph Hellwig 17879679cc51SChristoph Hellwig static void 17889679cc51SChristoph Hellwig isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 17899679cc51SChristoph Hellwig { 17909679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17919679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17929679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1793e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 17949679cc51SChristoph Hellwig 17959679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17969679cc51SChristoph Hellwig isert_print_wc(wc, "send"); 17979679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17989679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17999679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 18009679cc51SChristoph Hellwig return; 18019679cc51SChristoph Hellwig } 18029679cc51SChristoph Hellwig 18039679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 18049679cc51SChristoph Hellwig 18059679cc51SChristoph Hellwig switch (isert_cmd->iscsi_cmd->i_state) { 18069679cc51SChristoph Hellwig case ISTATE_SEND_TASKMGTRSP: 18079679cc51SChristoph Hellwig case ISTATE_SEND_LOGOUTRSP: 18089679cc51SChristoph Hellwig case ISTATE_SEND_REJECT: 18099679cc51SChristoph Hellwig case ISTATE_SEND_TEXTRSP: 1810b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1811b8d26b3bSNicholas Bellinger 1812b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1813b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1814b8d26b3bSNicholas Bellinger return; 1815b8d26b3bSNicholas Bellinger default: 18169679cc51SChristoph Hellwig isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 18179679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1818b8d26b3bSNicholas Bellinger break; 1819b8d26b3bSNicholas Bellinger } 1820b8d26b3bSNicholas Bellinger } 1821b8d26b3bSNicholas Bellinger 1822b8d26b3bSNicholas Bellinger static int 1823b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1824b8d26b3bSNicholas Bellinger { 1825b8d26b3bSNicholas Bellinger int ret; 1826b8d26b3bSNicholas Bellinger 18274366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 18284366b19cSJenny Derzhavetz if (ret) { 18294366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", ret); 18304366b19cSJenny Derzhavetz return ret; 18314366b19cSJenny Derzhavetz } 18324366b19cSJenny Derzhavetz 1833e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1834b8d26b3bSNicholas Bellinger if (ret) { 183524f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 1836b8d26b3bSNicholas Bellinger return ret; 1837b8d26b3bSNicholas Bellinger } 1838b8d26b3bSNicholas Bellinger return ret; 1839b8d26b3bSNicholas Bellinger } 1840b8d26b3bSNicholas Bellinger 1841b8d26b3bSNicholas Bellinger static int 1842b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1843b8d26b3bSNicholas Bellinger { 1844d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18456700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1846b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1847b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1848b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 1849b8d26b3bSNicholas Bellinger 1850b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1851b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1852b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1853b8d26b3bSNicholas Bellinger /* 1854b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 1855b8d26b3bSNicholas Bellinger */ 1856b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 1857b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1858b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1859dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 186067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1861b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1862dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 1863b8d26b3bSNicholas Bellinger 1864b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1865b8d26b3bSNicholas Bellinger cmd->sense_buffer); 1866b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1867b8d26b3bSNicholas Bellinger 1868b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1869b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1870dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1871b8d26b3bSNicholas Bellinger 1872dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1873dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 1874b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 1875def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1876def4a6ffSAlexey Khoroshilov return -ENOMEM; 1877b8d26b3bSNicholas Bellinger 1878dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 1879dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 1880dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 188134efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 1882b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 1883b8d26b3bSNicholas Bellinger } 1884b8d26b3bSNicholas Bellinger 188568a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1886b8d26b3bSNicholas Bellinger 18874c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n"); 1888b8d26b3bSNicholas Bellinger 1889b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1890b8d26b3bSNicholas Bellinger } 1891b8d26b3bSNicholas Bellinger 1892131e6abcSNicholas Bellinger static void 1893131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1894131e6abcSNicholas Bellinger { 1895131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18966700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1897131e6abcSNicholas Bellinger 1898131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1899131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1900131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 1901131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1902131e6abcSNicholas Bellinger 1903131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 1904131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 190538a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1906131e6abcSNicholas Bellinger } 1907131e6abcSNicholas Bellinger 1908e70beee7SNicholas Bellinger static enum target_prot_op 1909e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 1910e70beee7SNicholas Bellinger { 19116700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1912dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1913e70beee7SNicholas Bellinger 191423a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 191523a548eeSSagi Grimberg if (device->pi_capable) { 191624f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 191723a548eeSSagi Grimberg isert_conn->pi_support = true; 1918e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 191923a548eeSSagi Grimberg } 192023a548eeSSagi Grimberg } 192123a548eeSSagi Grimberg 192224f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 192323a548eeSSagi Grimberg isert_conn->pi_support = false; 1924e70beee7SNicholas Bellinger 1925e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 1926e70beee7SNicholas Bellinger } 1927e70beee7SNicholas Bellinger 1928b8d26b3bSNicholas Bellinger static int 1929b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1930b8d26b3bSNicholas Bellinger bool nopout_response) 1931b8d26b3bSNicholas Bellinger { 1932d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19336700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1934b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1935b8d26b3bSNicholas Bellinger 1936b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1937b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1938b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 1939b8d26b3bSNicholas Bellinger nopout_response); 1940b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 194168a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1942b8d26b3bSNicholas Bellinger 19434c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1944b8d26b3bSNicholas Bellinger 1945b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1946b8d26b3bSNicholas Bellinger } 1947b8d26b3bSNicholas Bellinger 1948b8d26b3bSNicholas Bellinger static int 1949b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1950b8d26b3bSNicholas Bellinger { 1951d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19526700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1953b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1954b8d26b3bSNicholas Bellinger 1955b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1956b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1957b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1958b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 195968a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1960b8d26b3bSNicholas Bellinger 19614c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1962b8d26b3bSNicholas Bellinger 1963b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1964b8d26b3bSNicholas Bellinger } 1965b8d26b3bSNicholas Bellinger 1966b8d26b3bSNicholas Bellinger static int 1967b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1968b8d26b3bSNicholas Bellinger { 1969d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19706700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1971b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1972b8d26b3bSNicholas Bellinger 1973b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1974b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1975b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1976b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 197768a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1978b8d26b3bSNicholas Bellinger 19794c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1980b8d26b3bSNicholas Bellinger 1981b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1982b8d26b3bSNicholas Bellinger } 1983b8d26b3bSNicholas Bellinger 1984b8d26b3bSNicholas Bellinger static int 1985b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1986b8d26b3bSNicholas Bellinger { 1987d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19886700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1989b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1990dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 199167cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 19923df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 19933df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 19943df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1995b8d26b3bSNicholas Bellinger 1996b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 19973df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 1998b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 19993df8f68aSNicholas Bellinger 20003df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2001dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 20023df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 20033df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2004def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2005def4a6ffSAlexey Khoroshilov return -ENOMEM; 2006dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2007dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 20083df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 200934efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 20103df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 20113df8f68aSNicholas Bellinger 201268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2013b8d26b3bSNicholas Bellinger 20144c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn); 2015b8d26b3bSNicholas Bellinger 2016b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2017b8d26b3bSNicholas Bellinger } 2018b8d26b3bSNicholas Bellinger 2019b8d26b3bSNicholas Bellinger static int 2020adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2021adb54c29SNicholas Bellinger { 2022d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 20236700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2024adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2025adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2026adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2027adb54c29SNicholas Bellinger u32 txt_rsp_len; 2028adb54c29SNicholas Bellinger int rc; 2029adb54c29SNicholas Bellinger 2030adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 203122c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2032adb54c29SNicholas Bellinger if (rc < 0) 2033adb54c29SNicholas Bellinger return rc; 2034adb54c29SNicholas Bellinger 2035adb54c29SNicholas Bellinger txt_rsp_len = rc; 2036adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2037adb54c29SNicholas Bellinger 2038adb54c29SNicholas Bellinger if (txt_rsp_len) { 2039dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 204067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2041adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2042adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2043adb54c29SNicholas Bellinger 2044adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2045adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2046def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2047def4a6ffSAlexey Khoroshilov return -ENOMEM; 2048adb54c29SNicholas Bellinger 2049adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2050adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2051adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 205234efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2053adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2054adb54c29SNicholas Bellinger } 205568a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2056adb54c29SNicholas Bellinger 2057f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn); 2058adb54c29SNicholas Bellinger 2059adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2060adb54c29SNicholas Bellinger } 2061adb54c29SNicholas Bellinger 20623d73cf1aSSagi Grimberg static inline void 2063dae6345aSIsrael Rukshin isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 20649e961ae7SSagi Grimberg { 206578eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 20663d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 20673d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 20683d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 206978eda2bbSSagi Grimberg /* 207078eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 207178eda2bbSSagi Grimberg * the target core would like to use it, we will take it 207278eda2bbSSagi Grimberg * from se_cmd. 207378eda2bbSSagi Grimberg */ 207478eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 207578eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 207678eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 207778eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 207878eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 207978eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 20803d73cf1aSSagi Grimberg }; 20813d73cf1aSSagi Grimberg 20823d73cf1aSSagi Grimberg static int 20833d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 20843d73cf1aSSagi Grimberg { 208538a2d0d4SChristoph Hellwig memset(sig_attrs, 0, sizeof(*sig_attrs)); 208638a2d0d4SChristoph Hellwig 20879e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 20889e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 20899e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 209078eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2091dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire); 20929e961ae7SSagi Grimberg break; 20939e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 20949e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 209578eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2096dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem); 20979e961ae7SSagi Grimberg break; 20989e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 20999e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 2100dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2101dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem); 21029e961ae7SSagi Grimberg break; 21039e961ae7SSagi Grimberg default: 210424f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 21059e961ae7SSagi Grimberg return -EINVAL; 21069e961ae7SSagi Grimberg } 21079e961ae7SSagi Grimberg 210826232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 210926232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 211026232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 211126232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 211226232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 211326232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 211426232872SMax Gurtovoy 21159e961ae7SSagi Grimberg return 0; 21169e961ae7SSagi Grimberg } 21179e961ae7SSagi Grimberg 21189e961ae7SSagi Grimberg static int 211938a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 212038a2d0d4SChristoph Hellwig struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 21219e961ae7SSagi Grimberg { 212238a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 212338a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 212438a2d0d4SChristoph Hellwig u8 port_num = conn->cm_id->port_num; 212538a2d0d4SChristoph Hellwig u64 addr; 212638a2d0d4SChristoph Hellwig u32 rkey, offset; 21279e961ae7SSagi Grimberg int ret; 21289e961ae7SSagi Grimberg 212966f53e6fSBharat Potnuri if (cmd->ctx_init_done) 213066f53e6fSBharat Potnuri goto rdma_ctx_post; 213166f53e6fSBharat Potnuri 213238a2d0d4SChristoph Hellwig if (dir == DMA_FROM_DEVICE) { 213338a2d0d4SChristoph Hellwig addr = cmd->write_va; 213438a2d0d4SChristoph Hellwig rkey = cmd->write_stag; 213538a2d0d4SChristoph Hellwig offset = cmd->iscsi_cmd->write_data_done; 213638a2d0d4SChristoph Hellwig } else { 213738a2d0d4SChristoph Hellwig addr = cmd->read_va; 213838a2d0d4SChristoph Hellwig rkey = cmd->read_stag; 213938a2d0d4SChristoph Hellwig offset = 0; 214038a2d0d4SChristoph Hellwig } 214138a2d0d4SChristoph Hellwig 214238a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 214338a2d0d4SChristoph Hellwig struct ib_sig_attrs sig_attrs; 214438a2d0d4SChristoph Hellwig 21459e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 21469e961ae7SSagi Grimberg if (ret) 2147e3d7e4c3SSagi Grimberg return ret; 2148e3d7e4c3SSagi Grimberg 214938a2d0d4SChristoph Hellwig WARN_ON_ONCE(offset); 215038a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 215138a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 215238a2d0d4SChristoph Hellwig se_cmd->t_prot_sg, se_cmd->t_prot_nents, 215338a2d0d4SChristoph Hellwig &sig_attrs, addr, rkey, dir); 2154570db170SSagi Grimberg } else { 215538a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 215638a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 215738a2d0d4SChristoph Hellwig offset, addr, rkey, dir); 215838a2d0d4SChristoph Hellwig } 215966f53e6fSBharat Potnuri 216038a2d0d4SChristoph Hellwig if (ret < 0) { 216138a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 216238a2d0d4SChristoph Hellwig return ret; 21639e961ae7SSagi Grimberg } 21649e961ae7SSagi Grimberg 216566f53e6fSBharat Potnuri cmd->ctx_init_done = true; 216666f53e6fSBharat Potnuri 216766f53e6fSBharat Potnuri rdma_ctx_post: 216838a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 216938a2d0d4SChristoph Hellwig if (ret < 0) 217038a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to post RDMA res\n", cmd); 217159464ef4SVu Pham return ret; 217259464ef4SVu Pham } 217359464ef4SVu Pham 217459464ef4SVu Pham static int 217590ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 217690ecc6e2SVu Pham { 217790ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 217859464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21796700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 218038a2d0d4SChristoph Hellwig struct ib_cqe *cqe = NULL; 218138a2d0d4SChristoph Hellwig struct ib_send_wr *chain_wr = NULL; 218290ecc6e2SVu Pham int rc; 218390ecc6e2SVu Pham 218424f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 218590ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 21864c22e07fSSagi Grimberg 218738a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) { 218838a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 218938a2d0d4SChristoph Hellwig cqe = &isert_cmd->tx_desc.tx_cqe; 219038a2d0d4SChristoph Hellwig } else { 2191b8d26b3bSNicholas Bellinger /* 2192b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2193b8d26b3bSNicholas Bellinger */ 21949e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 21959e961ae7SSagi Grimberg &isert_cmd->tx_desc); 219604d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2197b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2198b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 219995b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 220068a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 22014366b19cSJenny Derzhavetz 22024366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 22034366b19cSJenny Derzhavetz if (rc) { 22044366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 22054366b19cSJenny Derzhavetz return rc; 22064366b19cSJenny Derzhavetz } 220738a2d0d4SChristoph Hellwig 220838a2d0d4SChristoph Hellwig chain_wr = &isert_cmd->tx_desc.send_wr; 22099e961ae7SSagi Grimberg } 2210b8d26b3bSNicholas Bellinger 2211555a65f6SNicholas Bellinger rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2212555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2213555a65f6SNicholas Bellinger isert_cmd, rc); 2214555a65f6SNicholas Bellinger return rc; 2215b8d26b3bSNicholas Bellinger } 2216b8d26b3bSNicholas Bellinger 2217b8d26b3bSNicholas Bellinger static int 2218b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2219b8d26b3bSNicholas Bellinger { 2220d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2221555a65f6SNicholas Bellinger int ret; 2222b8d26b3bSNicholas Bellinger 222324f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 222438a2d0d4SChristoph Hellwig isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2225b8d26b3bSNicholas Bellinger 222638a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2227555a65f6SNicholas Bellinger ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 222838a2d0d4SChristoph Hellwig &isert_cmd->tx_desc.tx_cqe, NULL); 2229bdf20e72SSagi Grimberg 2230555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2231555a65f6SNicholas Bellinger isert_cmd, ret); 2232555a65f6SNicholas Bellinger return ret; 2233b8d26b3bSNicholas Bellinger } 2234b8d26b3bSNicholas Bellinger 2235b8d26b3bSNicholas Bellinger static int 2236b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2237b8d26b3bSNicholas Bellinger { 2238a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2239a4c15cd9SJenny Derzhavetz int ret = 0; 2240b8d26b3bSNicholas Bellinger 2241b8d26b3bSNicholas Bellinger switch (state) { 2242a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE: 2243a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 2244a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 2245a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 2246a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 2247a4c15cd9SJenny Derzhavetz break; 2248b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2249b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2250b8d26b3bSNicholas Bellinger break; 2251b8d26b3bSNicholas Bellinger default: 225224f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2253b8d26b3bSNicholas Bellinger ret = -EINVAL; 2254b8d26b3bSNicholas Bellinger break; 2255b8d26b3bSNicholas Bellinger } 2256b8d26b3bSNicholas Bellinger 2257b8d26b3bSNicholas Bellinger return ret; 2258b8d26b3bSNicholas Bellinger } 2259b8d26b3bSNicholas Bellinger 2260b8d26b3bSNicholas Bellinger static int 2261b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2262b8d26b3bSNicholas Bellinger { 2263991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2264b8d26b3bSNicholas Bellinger int ret; 2265b8d26b3bSNicholas Bellinger 2266b8d26b3bSNicholas Bellinger switch (state) { 2267b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2268b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2269991bb764SSagi Grimberg if (!ret) 2270991bb764SSagi Grimberg isert_conn->logout_posted = true; 2271b8d26b3bSNicholas Bellinger break; 2272b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2273b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2274b8d26b3bSNicholas Bellinger break; 2275b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2276b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2277b8d26b3bSNicholas Bellinger break; 2278b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2279b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2280b8d26b3bSNicholas Bellinger break; 2281adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2282adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2283adb54c29SNicholas Bellinger break; 2284b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2285b8d26b3bSNicholas Bellinger /* 2286b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2287b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2288b8d26b3bSNicholas Bellinger */ 2289b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2290b8d26b3bSNicholas Bellinger break; 2291b8d26b3bSNicholas Bellinger default: 229224f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 2293b8d26b3bSNicholas Bellinger ret = -EINVAL; 2294b8d26b3bSNicholas Bellinger break; 2295b8d26b3bSNicholas Bellinger } 2296b8d26b3bSNicholas Bellinger 2297b8d26b3bSNicholas Bellinger return ret; 2298b8d26b3bSNicholas Bellinger } 2299b8d26b3bSNicholas Bellinger 2300ca6c1d82SSagi Grimberg struct rdma_cm_id * 2301ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 2302ca6c1d82SSagi Grimberg { 2303ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 2304ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 2305ca6c1d82SSagi Grimberg struct sockaddr *sa; 2306ca6c1d82SSagi Grimberg int ret; 2307ca6c1d82SSagi Grimberg 2308ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 230924f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2310ca6c1d82SSagi Grimberg 2311fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2312ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 2313ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 231424f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2315ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 2316ca6c1d82SSagi Grimberg goto out; 2317ca6c1d82SSagi Grimberg } 231824f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 2319ca6c1d82SSagi Grimberg 2320ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 2321ca6c1d82SSagi Grimberg if (ret) { 232224f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 2323ca6c1d82SSagi Grimberg goto out_id; 2324ca6c1d82SSagi Grimberg } 2325ca6c1d82SSagi Grimberg 2326992607e8SSagi Grimberg ret = rdma_listen(id, 0); 2327ca6c1d82SSagi Grimberg if (ret) { 232824f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 2329ca6c1d82SSagi Grimberg goto out_id; 2330ca6c1d82SSagi Grimberg } 2331ca6c1d82SSagi Grimberg 2332ca6c1d82SSagi Grimberg return id; 2333ca6c1d82SSagi Grimberg out_id: 2334ca6c1d82SSagi Grimberg rdma_destroy_id(id); 2335ca6c1d82SSagi Grimberg out: 2336ca6c1d82SSagi Grimberg return ERR_PTR(ret); 2337ca6c1d82SSagi Grimberg } 2338ca6c1d82SSagi Grimberg 2339b8d26b3bSNicholas Bellinger static int 2340b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 234113a3cf08SAndy Grover struct sockaddr_storage *ksockaddr) 2342b8d26b3bSNicholas Bellinger { 2343b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2344b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2345b8d26b3bSNicholas Bellinger int ret; 2346b8d26b3bSNicholas Bellinger 2347b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 234893b80f29SLeon Romanovsky if (!isert_np) 2349b8d26b3bSNicholas Bellinger return -ENOMEM; 235093b80f29SLeon Romanovsky 2351ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0); 2352ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex); 2353bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted); 2354bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending); 2355ca6c1d82SSagi Grimberg isert_np->np = np; 2356b8d26b3bSNicholas Bellinger 2357b8d26b3bSNicholas Bellinger /* 2358b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 2359b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 2360b8d26b3bSNicholas Bellinger */ 2361b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 236213a3cf08SAndy Grover sizeof(struct sockaddr_storage)); 2363b8d26b3bSNicholas Bellinger 2364ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 2365b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 2366b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 2367b8d26b3bSNicholas Bellinger goto out; 2368b8d26b3bSNicholas Bellinger } 2369b8d26b3bSNicholas Bellinger 2370ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid; 2371b8d26b3bSNicholas Bellinger np->np_context = isert_np; 2372b8d26b3bSNicholas Bellinger 2373b8d26b3bSNicholas Bellinger return 0; 2374b8d26b3bSNicholas Bellinger 2375b8d26b3bSNicholas Bellinger out: 2376b8d26b3bSNicholas Bellinger kfree(isert_np); 2377ca6c1d82SSagi Grimberg 2378b8d26b3bSNicholas Bellinger return ret; 2379b8d26b3bSNicholas Bellinger } 2380b8d26b3bSNicholas Bellinger 2381b8d26b3bSNicholas Bellinger static int 2382b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 2383b8d26b3bSNicholas Bellinger { 2384dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2385b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 2386b8d26b3bSNicholas Bellinger int ret; 238713bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr; 2388b8d26b3bSNicholas Bellinger 2389b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 2390b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 2391b8d26b3bSNicholas Bellinger cp.retry_count = 7; 2392b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 2393b8d26b3bSNicholas Bellinger 239413bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2395422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2396422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv) 2397422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 239813bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr; 239913bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr); 240013bce482SJenny Derzhavetz 2401b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 2402b8d26b3bSNicholas Bellinger if (ret) { 240324f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 2404b8d26b3bSNicholas Bellinger return ret; 2405b8d26b3bSNicholas Bellinger } 2406b8d26b3bSNicholas Bellinger 2407b8d26b3bSNicholas Bellinger return 0; 2408b8d26b3bSNicholas Bellinger } 2409b8d26b3bSNicholas Bellinger 2410b8d26b3bSNicholas Bellinger static int 2411b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2412b8d26b3bSNicholas Bellinger { 24136700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2414b8d26b3bSNicholas Bellinger int ret; 2415b8d26b3bSNicholas Bellinger 241624f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 24172371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 24182371e5daSSagi Grimberg if (ret) { 241924f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 24202371e5daSSagi Grimberg isert_conn); 24212371e5daSSagi Grimberg return ret; 24222371e5daSSagi Grimberg } 24232371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 24242371e5daSSagi Grimberg 24256faaa85fSNicholas Bellinger /* 24266faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 24276faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 24286faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 24296faaa85fSNicholas Bellinger * into a NOP. 24306faaa85fSNicholas Bellinger */ 24316faaa85fSNicholas Bellinger if (!login->first_request) 24326faaa85fSNicholas Bellinger return 0; 2433b8d26b3bSNicholas Bellinger 24342371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 24352371e5daSSagi Grimberg 2436dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn); 2437dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2438b8d26b3bSNicholas Bellinger if (ret) 2439b8d26b3bSNicholas Bellinger return ret; 2440b8d26b3bSNicholas Bellinger 244124f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 24422371e5daSSagi Grimberg 2443b8d26b3bSNicholas Bellinger return 0; 2444b8d26b3bSNicholas Bellinger } 2445b8d26b3bSNicholas Bellinger 2446b8d26b3bSNicholas Bellinger static void 2447b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2448b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 2449b8d26b3bSNicholas Bellinger { 2450dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2451b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 2452b8d26b3bSNicholas Bellinger 2453b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 2454b8d26b3bSNicholas Bellinger 2455dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr; 2456dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr; 2457b8d26b3bSNicholas Bellinger } 2458b8d26b3bSNicholas Bellinger 2459b8d26b3bSNicholas Bellinger static int 2460b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2461b8d26b3bSNicholas Bellinger { 24626700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2463b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2464c6b8e918SSagi Grimberg int ret; 2465b8d26b3bSNicholas Bellinger 2466b8d26b3bSNicholas Bellinger accept_wait: 2467ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem); 2468c6b8e918SSagi Grimberg if (ret) 2469b8d26b3bSNicholas Bellinger return -ENODEV; 2470b8d26b3bSNicholas Bellinger 2471b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 2472e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2473b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 247411378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n", 2475e346ab34SSagi Grimberg np->np_thread_state); 2476e346ab34SSagi Grimberg /** 2477e346ab34SSagi Grimberg * No point in stalling here when np_thread 2478e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 2479e346ab34SSagi Grimberg **/ 2480b8d26b3bSNicholas Bellinger return -ENODEV; 2481b8d26b3bSNicholas Bellinger } 2482b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 2483b8d26b3bSNicholas Bellinger 2484ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2485bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) { 2486ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2487b8d26b3bSNicholas Bellinger goto accept_wait; 2488b8d26b3bSNicholas Bellinger } 2489bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending, 2490bd379220SJenny Derzhavetz struct isert_conn, node); 2491bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 2492ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2493b8d26b3bSNicholas Bellinger 2494b8d26b3bSNicholas Bellinger conn->context = isert_conn; 2495b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 2496aea92980SJenny Derzhavetz isert_conn->state = ISER_CONN_BOUND; 2497b8d26b3bSNicholas Bellinger 2498b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 2499b8d26b3bSNicholas Bellinger 250024f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 25012371e5daSSagi Grimberg 2502b8d26b3bSNicholas Bellinger return 0; 2503b8d26b3bSNicholas Bellinger } 2504b8d26b3bSNicholas Bellinger 2505b8d26b3bSNicholas Bellinger static void 2506b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 2507b8d26b3bSNicholas Bellinger { 25086700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2509268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 2510b8d26b3bSNicholas Bellinger 2511ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id) 2512ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id); 2513b8d26b3bSNicholas Bellinger 2514268e6811SSagi Grimberg /* 2515268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 2516268e6811SSagi Grimberg * that at this point we don't have hanging connections that 2517268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 2518268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 2519bd379220SJenny Derzhavetz * up in accepted and pending lists. 2520268e6811SSagi Grimberg */ 2521ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2522bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) { 2523bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n"); 2524268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 2525bd379220SJenny Derzhavetz &isert_np->pending, 2526bd379220SJenny Derzhavetz node) { 2527bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n", 2528bd379220SJenny Derzhavetz isert_conn, isert_conn->state); 2529bd379220SJenny Derzhavetz isert_connect_release(isert_conn); 2530bd379220SJenny Derzhavetz } 2531bd379220SJenny Derzhavetz } 2532bd379220SJenny Derzhavetz 2533bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) { 2534bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n"); 2535bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n, 2536bd379220SJenny Derzhavetz &isert_np->accepted, 2537bd379220SJenny Derzhavetz node) { 253824f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 2539268e6811SSagi Grimberg isert_conn, isert_conn->state); 2540268e6811SSagi Grimberg isert_connect_release(isert_conn); 2541268e6811SSagi Grimberg } 2542268e6811SSagi Grimberg } 2543ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2544268e6811SSagi Grimberg 2545b8d26b3bSNicholas Bellinger np->np_context = NULL; 2546b8d26b3bSNicholas Bellinger kfree(isert_np); 2547b8d26b3bSNicholas Bellinger } 2548b8d26b3bSNicholas Bellinger 2549b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 2550b02efbfcSSagi Grimberg { 2551b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 2552b02efbfcSSagi Grimberg struct isert_conn, 2553b02efbfcSSagi Grimberg release_work); 2554b02efbfcSSagi Grimberg 255524f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 2556b02efbfcSSagi Grimberg 2557dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2558b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 2559dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2560b02efbfcSSagi Grimberg 256124f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 2562b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 2563b02efbfcSSagi Grimberg } 2564b02efbfcSSagi Grimberg 2565bdf20e72SSagi Grimberg static void 2566991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 2567991bb764SSagi Grimberg { 2568991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 2569991bb764SSagi Grimberg 25704c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 25714c22e07fSSagi Grimberg 2572991bb764SSagi Grimberg if (isert_conn->logout_posted) { 257324f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2574991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 2575991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 2576991bb764SSagi Grimberg } 2577991bb764SSagi Grimberg } 2578991bb764SSagi Grimberg 257976261adaSBart Van Assche static void 258076261adaSBart Van Assche isert_wait4cmds(struct iscsi_conn *conn) 258176261adaSBart Van Assche { 258276261adaSBart Van Assche isert_info("iscsi_conn %p\n", conn); 258376261adaSBart Van Assche 258476261adaSBart Van Assche if (conn->sess) { 258576261adaSBart Van Assche target_sess_cmd_list_set_waiting(conn->sess->se_sess); 258676261adaSBart Van Assche target_wait_for_sess_cmds(conn->sess->se_sess); 258776261adaSBart Van Assche } 258876261adaSBart Van Assche } 258976261adaSBart Van Assche 25903e03c4b0SJenny Derzhavetz /** 25913e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for 25923e03c4b0SJenny Derzhavetz * unsolicitate dataout 25933e03c4b0SJenny Derzhavetz * @conn: iscsi connection 25943e03c4b0SJenny Derzhavetz * 25953e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited 25963e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those 25973e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds 25983e03c4b0SJenny Derzhavetz */ 25993e03c4b0SJenny Derzhavetz static void 26003e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 26013e03c4b0SJenny Derzhavetz { 26023e03c4b0SJenny Derzhavetz struct iscsi_cmd *cmd, *tmp; 26033e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list); 26043e03c4b0SJenny Derzhavetz 26053e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 26063e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 26073e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 26083e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 26093e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length)) 26103e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 26113e03c4b0SJenny Derzhavetz } 26123e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 26133e03c4b0SJenny Derzhavetz 26143e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 26153e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 26163e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) { 26173e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 26183e03c4b0SJenny Derzhavetz 26193e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd); 26203e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 26213e03c4b0SJenny Derzhavetz } 26223e03c4b0SJenny Derzhavetz } 26233e03c4b0SJenny Derzhavetz } 26243e03c4b0SJenny Derzhavetz 2625defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 2626b8d26b3bSNicholas Bellinger { 2627b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2628b8d26b3bSNicholas Bellinger 26294c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn); 2630b8d26b3bSNicholas Bellinger 2631dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2632954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 2633dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2634b2cb9649SNicholas Bellinger 2635572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 26363e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn); 263776261adaSBart Van Assche isert_wait4cmds(conn); 2638991bb764SSagi Grimberg isert_wait4logout(isert_conn); 2639954f2372SSagi Grimberg 2640b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 2641defd8848SNicholas Bellinger } 2642defd8848SNicholas Bellinger 2643defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 2644defd8848SNicholas Bellinger { 2645defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2646b8d26b3bSNicholas Bellinger 2647572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 2648b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 2649b8d26b3bSNicholas Bellinger } 2650b8d26b3bSNicholas Bellinger 2651187e84f7SVarun Prakash static void isert_get_rx_pdu(struct iscsi_conn *conn) 2652187e84f7SVarun Prakash { 2653187e84f7SVarun Prakash struct completion comp; 2654187e84f7SVarun Prakash 2655187e84f7SVarun Prakash init_completion(&comp); 2656187e84f7SVarun Prakash 2657187e84f7SVarun Prakash wait_for_completion_interruptible(&comp); 2658187e84f7SVarun Prakash } 2659187e84f7SVarun Prakash 2660b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 2661b8d26b3bSNicholas Bellinger .name = "IB/iSER", 2662b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 2663bd027d85SNicholas Bellinger .rdma_shutdown = true, 2664d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 2665b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 2666b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 2667b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 2668b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 2669defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 2670b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 2671b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 2672b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 2673b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 2674b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 2675b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 2676b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 2677b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 2678131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 2679187e84f7SVarun Prakash .iscsit_get_rx_pdu = isert_get_rx_pdu, 2680e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2681b8d26b3bSNicholas Bellinger }; 2682b8d26b3bSNicholas Bellinger 2683b8d26b3bSNicholas Bellinger static int __init isert_init(void) 2684b8d26b3bSNicholas Bellinger { 2685b8d26b3bSNicholas Bellinger int ret; 2686b8d26b3bSNicholas Bellinger 2687631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq", 2688631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0); 2689b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 269024f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 26916f0fae3dSSagi Grimberg return -ENOMEM; 2692b8d26b3bSNicholas Bellinger } 2693b8d26b3bSNicholas Bellinger 2694b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2695b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 2696b02efbfcSSagi Grimberg if (!isert_release_wq) { 269724f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 2698b02efbfcSSagi Grimberg ret = -ENOMEM; 2699b02efbfcSSagi Grimberg goto destroy_comp_wq; 2700b02efbfcSSagi Grimberg } 2701b02efbfcSSagi Grimberg 2702b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 270324f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2704b02efbfcSSagi Grimberg 2705b8d26b3bSNicholas Bellinger return 0; 2706b8d26b3bSNicholas Bellinger 2707b02efbfcSSagi Grimberg destroy_comp_wq: 2708b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 27096f0fae3dSSagi Grimberg 2710b8d26b3bSNicholas Bellinger return ret; 2711b8d26b3bSNicholas Bellinger } 2712b8d26b3bSNicholas Bellinger 2713b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 2714b8d26b3bSNicholas Bellinger { 2715f5ebec96SSagi Grimberg flush_scheduled_work(); 2716b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 2717b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 2718b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 27194c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2720b8d26b3bSNicholas Bellinger } 2721b8d26b3bSNicholas Bellinger 2722b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2723b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2724b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 2725b8d26b3bSNicholas Bellinger 2726b8d26b3bSNicholas Bellinger module_init(isert_init); 2727b8d26b3bSNicholas Bellinger module_exit(isert_exit); 2728