1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2b8d26b3bSNicholas Bellinger /******************************************************************************* 3b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 4b8d26b3bSNicholas Bellinger * 54c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 6b8d26b3bSNicholas Bellinger * 7b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 8b8d26b3bSNicholas Bellinger * 9b8d26b3bSNicholas Bellinger ****************************************************************************/ 10b8d26b3bSNicholas Bellinger 11b8d26b3bSNicholas Bellinger #include <linux/string.h> 12b8d26b3bSNicholas Bellinger #include <linux/module.h> 13b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 14b8d26b3bSNicholas Bellinger #include <linux/socket.h> 15b8d26b3bSNicholas Bellinger #include <linux/in.h> 16b8d26b3bSNicholas Bellinger #include <linux/in6.h> 17b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 18b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 19b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 20b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 21b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 22531b7bf4SSagi Grimberg #include <linux/semaphore.h> 23b8d26b3bSNicholas Bellinger 24b8d26b3bSNicholas Bellinger #include "ib_isert.h" 25b8d26b3bSNicholas Bellinger 26b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 27b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 2838a2d0d4SChristoph Hellwig #define ISER_MAX_TX_CQ_LEN \ 2938a2d0d4SChristoph Hellwig ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) 30bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 31bdf20e72SSagi Grimberg ISERT_MAX_CONN) 32b8d26b3bSNicholas Bellinger 3345678b6bSSagi Grimberg static int isert_debug_level; 3424f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 3524f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 3624f412ddSSagi Grimberg 37b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 38b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 39b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 40b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 41b8d26b3bSNicholas Bellinger 42f93f3a70SSagi Grimberg static int 43f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 442371e5daSSagi Grimberg static int 459679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn); 462371e5daSSagi Grimberg static int 472371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 48ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 49d40945d8SVu Pham 502f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work); 519679cc51SChristoph Hellwig static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 529679cc51SChristoph Hellwig static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 539679cc51SChristoph Hellwig static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 549679cc51SChristoph Hellwig static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 552f1b6b7dSSagi Grimberg 56302cc7c3SSagi Grimberg static inline bool 57302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 58302cc7c3SSagi Grimberg { 5923a548eeSSagi Grimberg return (conn->pi_support && 60302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 61302cc7c3SSagi Grimberg } 62302cc7c3SSagi Grimberg 63302cc7c3SSagi Grimberg 64d40945d8SVu Pham static void 65b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 66b8d26b3bSNicholas Bellinger { 676700425eSSagi Grimberg struct isert_conn *isert_conn = context; 68b8d26b3bSNicholas Bellinger 69ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n", 70ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn); 71ea8a1616SSagi Grimberg 72b8d26b3bSNicholas Bellinger switch (e->event) { 73b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 74dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 75b8d26b3bSNicholas Bellinger break; 76b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 774c22e07fSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 78b8d26b3bSNicholas Bellinger break; 79b8d26b3bSNicholas Bellinger default: 80b8d26b3bSNicholas Bellinger break; 81b8d26b3bSNicholas Bellinger } 82b8d26b3bSNicholas Bellinger } 83b8d26b3bSNicholas Bellinger 8440fc069aSSagi Grimberg static struct isert_comp * 8540fc069aSSagi Grimberg isert_comp_get(struct isert_conn *isert_conn) 86b8d26b3bSNicholas Bellinger { 87dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 884a295baeSSagi Grimberg struct isert_comp *comp; 8940fc069aSSagi Grimberg int i, min = 0; 90b8d26b3bSNicholas Bellinger 91b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 924a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 934a295baeSSagi Grimberg if (device->comps[i].active_qps < 944a295baeSSagi Grimberg device->comps[min].active_qps) 954a295baeSSagi Grimberg min = i; 964a295baeSSagi Grimberg comp = &device->comps[min]; 974a295baeSSagi Grimberg comp->active_qps++; 9840fc069aSSagi Grimberg mutex_unlock(&device_list_mutex); 9940fc069aSSagi Grimberg 10024f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1014a295baeSSagi Grimberg isert_conn, comp, min); 10240fc069aSSagi Grimberg 10340fc069aSSagi Grimberg return comp; 10440fc069aSSagi Grimberg } 10540fc069aSSagi Grimberg 10640fc069aSSagi Grimberg static void 10740fc069aSSagi Grimberg isert_comp_put(struct isert_comp *comp) 10840fc069aSSagi Grimberg { 10940fc069aSSagi Grimberg mutex_lock(&device_list_mutex); 11040fc069aSSagi Grimberg comp->active_qps--; 111b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 11240fc069aSSagi Grimberg } 11340fc069aSSagi Grimberg 11440fc069aSSagi Grimberg static struct ib_qp * 11540fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn, 11640fc069aSSagi Grimberg struct isert_comp *comp, 11740fc069aSSagi Grimberg struct rdma_cm_id *cma_id) 11840fc069aSSagi Grimberg { 119dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 12040fc069aSSagi Grimberg struct ib_qp_init_attr attr; 12140fc069aSSagi Grimberg int ret; 122b8d26b3bSNicholas Bellinger 123b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 124b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 125b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1266f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1276f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 128572a1434SSagi Grimberg attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 129bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 13038a2d0d4SChristoph Hellwig attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX; 13133023fb8SSteve Wise attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 132b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 133b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 134b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 135570db170SSagi Grimberg if (device->pi_capable) 136c0a6cbb9SIsrael Rukshin attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 137b8d26b3bSNicholas Bellinger 13867cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr); 139b8d26b3bSNicholas Bellinger if (ret) { 14024f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 14140fc069aSSagi Grimberg return ERR_PTR(ret); 14240fc069aSSagi Grimberg } 14340fc069aSSagi Grimberg 14440fc069aSSagi Grimberg return cma_id->qp; 14540fc069aSSagi Grimberg } 14640fc069aSSagi Grimberg 14740fc069aSSagi Grimberg static int 14840fc069aSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 14940fc069aSSagi Grimberg { 15040fc069aSSagi Grimberg struct isert_comp *comp; 15140fc069aSSagi Grimberg int ret; 15240fc069aSSagi Grimberg 15340fc069aSSagi Grimberg comp = isert_comp_get(isert_conn); 154dac6ab30SSagi Grimberg isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 155dac6ab30SSagi Grimberg if (IS_ERR(isert_conn->qp)) { 156dac6ab30SSagi Grimberg ret = PTR_ERR(isert_conn->qp); 15719e2090fSSagi Grimberg goto err; 158b8d26b3bSNicholas Bellinger } 159b8d26b3bSNicholas Bellinger 160b8d26b3bSNicholas Bellinger return 0; 16119e2090fSSagi Grimberg err: 16240fc069aSSagi Grimberg isert_comp_put(comp); 16319e2090fSSagi Grimberg return ret; 164b8d26b3bSNicholas Bellinger } 165b8d26b3bSNicholas Bellinger 166b8d26b3bSNicholas Bellinger static int 167b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 168b8d26b3bSNicholas Bellinger { 169dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 17067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 171b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 172b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 173b8d26b3bSNicholas Bellinger u64 dma_addr; 174b8d26b3bSNicholas Bellinger int i, j; 175b8d26b3bSNicholas Bellinger 1766396bb22SKees Cook isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 1776396bb22SKees Cook sizeof(struct iser_rx_desc), 1786396bb22SKees Cook GFP_KERNEL); 179dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 18093b80f29SLeon Romanovsky return -ENOMEM; 181b8d26b3bSNicholas Bellinger 182dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 183b8d26b3bSNicholas Bellinger 184b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 185b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 186b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 187b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 188b8d26b3bSNicholas Bellinger goto dma_map_fail; 189b8d26b3bSNicholas Bellinger 190b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 191b8d26b3bSNicholas Bellinger 192b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 193b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 194b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 19534efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey; 1969679cc51SChristoph Hellwig rx_desc->rx_cqe.done = isert_recv_done; 197b8d26b3bSNicholas Bellinger } 198b8d26b3bSNicholas Bellinger 199b8d26b3bSNicholas Bellinger return 0; 200b8d26b3bSNicholas Bellinger 201b8d26b3bSNicholas Bellinger dma_map_fail: 202dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 203b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 204b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 205b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 206b8d26b3bSNicholas Bellinger } 207dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 208dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 2094c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 210b8d26b3bSNicholas Bellinger return -ENOMEM; 211b8d26b3bSNicholas Bellinger } 212b8d26b3bSNicholas Bellinger 213b8d26b3bSNicholas Bellinger static void 214b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 215b8d26b3bSNicholas Bellinger { 216dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 217b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 218b8d26b3bSNicholas Bellinger int i; 219b8d26b3bSNicholas Bellinger 220dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 221b8d26b3bSNicholas Bellinger return; 222b8d26b3bSNicholas Bellinger 223dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 224b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 225b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 226b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 227b8d26b3bSNicholas Bellinger } 228b8d26b3bSNicholas Bellinger 229dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 230dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 231b8d26b3bSNicholas Bellinger } 232b8d26b3bSNicholas Bellinger 233172369c5SSagi Grimberg static void 234172369c5SSagi Grimberg isert_free_comps(struct isert_device *device) 235b8d26b3bSNicholas Bellinger { 236172369c5SSagi Grimberg int i; 237b8d26b3bSNicholas Bellinger 238172369c5SSagi Grimberg for (i = 0; i < device->comps_used; i++) { 239172369c5SSagi Grimberg struct isert_comp *comp = &device->comps[i]; 24059464ef4SVu Pham 2419679cc51SChristoph Hellwig if (comp->cq) 2429679cc51SChristoph Hellwig ib_free_cq(comp->cq); 243172369c5SSagi Grimberg } 244172369c5SSagi Grimberg kfree(device->comps); 24559464ef4SVu Pham } 246d40945d8SVu Pham 247172369c5SSagi Grimberg static int 2484a061b28SOr Gerlitz isert_alloc_comps(struct isert_device *device) 249172369c5SSagi Grimberg { 250172369c5SSagi Grimberg int i, max_cqe, ret = 0; 251d3e125daSSagi Grimberg 2524a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2534a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 254172369c5SSagi Grimberg 25524f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 25638a2d0d4SChristoph Hellwig "pi_capable %d\n", 2576c854111SJason Gunthorpe device->comps_used, dev_name(&device->ib_device->dev), 25838a2d0d4SChristoph Hellwig device->ib_device->num_comp_vectors, 259d3e125daSSagi Grimberg device->pi_capable); 2604a295baeSSagi Grimberg 2614a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2624a295baeSSagi Grimberg GFP_KERNEL); 26393b80f29SLeon Romanovsky if (!device->comps) 264b8d26b3bSNicholas Bellinger return -ENOMEM; 265b8d26b3bSNicholas Bellinger 2664a061b28SOr Gerlitz max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 267172369c5SSagi Grimberg 2684a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2694a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 270b8d26b3bSNicholas Bellinger 2714a295baeSSagi Grimberg comp->device = device; 2729679cc51SChristoph Hellwig comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 2739679cc51SChristoph Hellwig IB_POLL_WORKQUEUE); 2746f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 275172369c5SSagi Grimberg isert_err("Unable to allocate cq\n"); 2766f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 2776f0fae3dSSagi Grimberg comp->cq = NULL; 278b8d26b3bSNicholas Bellinger goto out_cq; 27994a71110SWei Yongjun } 280b8d26b3bSNicholas Bellinger } 281b8d26b3bSNicholas Bellinger 282172369c5SSagi Grimberg return 0; 283172369c5SSagi Grimberg out_cq: 284172369c5SSagi Grimberg isert_free_comps(device); 285172369c5SSagi Grimberg return ret; 286172369c5SSagi Grimberg } 287172369c5SSagi Grimberg 288172369c5SSagi Grimberg static int 289172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device) 290172369c5SSagi Grimberg { 2914a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device; 292fd8205e8SSagi Grimberg int ret; 293172369c5SSagi Grimberg 29433023fb8SSteve Wise isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 29533023fb8SSteve Wise ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 2964a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 297172369c5SSagi Grimberg 2984a061b28SOr Gerlitz ret = isert_alloc_comps(device); 299172369c5SSagi Grimberg if (ret) 300373a4cd7SNicholas Bellinger goto out; 301172369c5SSagi Grimberg 302ed082d36SChristoph Hellwig device->pd = ib_alloc_pd(ib_dev, 0); 30367cb3949SSagi Grimberg if (IS_ERR(device->pd)) { 30467cb3949SSagi Grimberg ret = PTR_ERR(device->pd); 30567cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n", 30667cb3949SSagi Grimberg device, ret); 30767cb3949SSagi Grimberg goto out_cq; 30867cb3949SSagi Grimberg } 30967cb3949SSagi Grimberg 310172369c5SSagi Grimberg /* Check signature cap */ 3114a061b28SOr Gerlitz device->pi_capable = ib_dev->attrs.device_cap_flags & 312c0a6cbb9SIsrael Rukshin IB_DEVICE_INTEGRITY_HANDOVER ? true : false; 31367cb3949SSagi Grimberg 314b8d26b3bSNicholas Bellinger return 0; 315b8d26b3bSNicholas Bellinger 316b8d26b3bSNicholas Bellinger out_cq: 317172369c5SSagi Grimberg isert_free_comps(device); 318373a4cd7SNicholas Bellinger out: 319373a4cd7SNicholas Bellinger if (ret > 0) 320373a4cd7SNicholas Bellinger ret = -EINVAL; 321b8d26b3bSNicholas Bellinger return ret; 322b8d26b3bSNicholas Bellinger } 323b8d26b3bSNicholas Bellinger 324b8d26b3bSNicholas Bellinger static void 325b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 326b8d26b3bSNicholas Bellinger { 32724f412ddSSagi Grimberg isert_info("device %p\n", device); 328b8d26b3bSNicholas Bellinger 32967cb3949SSagi Grimberg ib_dealloc_pd(device->pd); 330172369c5SSagi Grimberg isert_free_comps(device); 331b8d26b3bSNicholas Bellinger } 332b8d26b3bSNicholas Bellinger 333b8d26b3bSNicholas Bellinger static void 334cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device) 335b8d26b3bSNicholas Bellinger { 336b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 337b8d26b3bSNicholas Bellinger device->refcount--; 3384c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount); 339b8d26b3bSNicholas Bellinger if (!device->refcount) { 340b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 341b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 342b8d26b3bSNicholas Bellinger kfree(device); 343b8d26b3bSNicholas Bellinger } 344b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 345b8d26b3bSNicholas Bellinger } 346b8d26b3bSNicholas Bellinger 347b8d26b3bSNicholas Bellinger static struct isert_device * 348cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id) 349b8d26b3bSNicholas Bellinger { 350b8d26b3bSNicholas Bellinger struct isert_device *device; 351b8d26b3bSNicholas Bellinger int ret; 352b8d26b3bSNicholas Bellinger 353b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 354b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 355b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 356b8d26b3bSNicholas Bellinger device->refcount++; 3574c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n", 3584c22e07fSSagi Grimberg device, device->refcount); 359b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 360b8d26b3bSNicholas Bellinger return device; 361b8d26b3bSNicholas Bellinger } 362b8d26b3bSNicholas Bellinger } 363b8d26b3bSNicholas Bellinger 364b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 365b8d26b3bSNicholas Bellinger if (!device) { 366b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 367b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 368b8d26b3bSNicholas Bellinger } 369b8d26b3bSNicholas Bellinger 370b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 371b8d26b3bSNicholas Bellinger 372b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 373b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 374b8d26b3bSNicholas Bellinger if (ret) { 375b8d26b3bSNicholas Bellinger kfree(device); 376b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 377b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 378b8d26b3bSNicholas Bellinger } 379b8d26b3bSNicholas Bellinger 380b8d26b3bSNicholas Bellinger device->refcount++; 381b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 3824c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n", 3834c22e07fSSagi Grimberg device, device->refcount); 384b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 385b8d26b3bSNicholas Bellinger 386b8d26b3bSNicholas Bellinger return device; 387b8d26b3bSNicholas Bellinger } 388b8d26b3bSNicholas Bellinger 38959464ef4SVu Pham static void 390ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn) 391ae9ea9edSSagi Grimberg { 392ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT; 393bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node); 394dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp); 395ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp); 39663b268d2SRaju Rangoju init_waitqueue_head(&isert_conn->rem_wait); 397dac6ab30SSagi Grimberg kref_init(&isert_conn->kref); 398dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex); 3992f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 400ae9ea9edSSagi Grimberg } 401ae9ea9edSSagi Grimberg 402ae9ea9edSSagi Grimberg static void 403ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn) 404ae9ea9edSSagi Grimberg { 405dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 406ae9ea9edSSagi Grimberg 407ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 408ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 4095adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4105adabdd1SChristoph Hellwig 411ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4125adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, 413ae9ea9edSSagi Grimberg DMA_FROM_DEVICE); 4145adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 415ae9ea9edSSagi Grimberg } 416ae9ea9edSSagi Grimberg 417ae9ea9edSSagi Grimberg static int 418ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn, 419ae9ea9edSSagi Grimberg struct ib_device *ib_dev) 420ae9ea9edSSagi Grimberg { 421ae9ea9edSSagi Grimberg int ret; 422ae9ea9edSSagi Grimberg 4235adabdd1SChristoph Hellwig isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 4245adabdd1SChristoph Hellwig GFP_KERNEL); 42593b80f29SLeon Romanovsky if (!isert_conn->login_req_buf) 426ae9ea9edSSagi Grimberg return -ENOMEM; 427ae9ea9edSSagi Grimberg 428ae9ea9edSSagi Grimberg isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 4295adabdd1SChristoph Hellwig isert_conn->login_req_buf, 4305adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 431ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 432ae9ea9edSSagi Grimberg if (ret) { 433ae9ea9edSSagi Grimberg isert_err("login_req_dma mapping error: %d\n", ret); 434ae9ea9edSSagi Grimberg isert_conn->login_req_dma = 0; 4355adabdd1SChristoph Hellwig goto out_free_login_req_buf; 4365adabdd1SChristoph Hellwig } 4375adabdd1SChristoph Hellwig 4385adabdd1SChristoph Hellwig isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 4395adabdd1SChristoph Hellwig if (!isert_conn->login_rsp_buf) { 4401d5840c9SWei Yongjun ret = -ENOMEM; 4415adabdd1SChristoph Hellwig goto out_unmap_login_req_buf; 442ae9ea9edSSagi Grimberg } 443ae9ea9edSSagi Grimberg 444ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 4455adabdd1SChristoph Hellwig isert_conn->login_rsp_buf, 446ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 447ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 448ae9ea9edSSagi Grimberg if (ret) { 449ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret); 450ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0; 4515adabdd1SChristoph Hellwig goto out_free_login_rsp_buf; 452ae9ea9edSSagi Grimberg } 453ae9ea9edSSagi Grimberg 454ae9ea9edSSagi Grimberg return 0; 455ae9ea9edSSagi Grimberg 4565adabdd1SChristoph Hellwig out_free_login_rsp_buf: 4575adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4585adabdd1SChristoph Hellwig out_unmap_login_req_buf: 459ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4605adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 4615adabdd1SChristoph Hellwig out_free_login_req_buf: 4625adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 463ae9ea9edSSagi Grimberg return ret; 464ae9ea9edSSagi Grimberg } 465ae9ea9edSSagi Grimberg 466422bd0acSJenny Derzhavetz static void 467422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn, 468422bd0acSJenny Derzhavetz struct rdma_conn_param *param) 469422bd0acSJenny Derzhavetz { 470422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 471422bd0acSJenny Derzhavetz 472422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */ 473422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 474422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom); 475422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 476422bd0acSJenny Derzhavetz 477422bd0acSJenny Derzhavetz if (param->private_data) { 478422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data; 479422bd0acSJenny Derzhavetz 480422bd0acSJenny Derzhavetz /* 481422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator 482422bd0acSJenny Derzhavetz * and the HCA support it 483422bd0acSJenny Derzhavetz */ 484422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 485422bd0acSJenny Derzhavetz (attr->device_cap_flags & 486422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS); 487422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv) 488422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n"); 489422bd0acSJenny Derzhavetz } 490422bd0acSJenny Derzhavetz } 491422bd0acSJenny Derzhavetz 492b8d26b3bSNicholas Bellinger static int 493b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 494b8d26b3bSNicholas Bellinger { 495ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 496ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 497b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 498b8d26b3bSNicholas Bellinger struct isert_device *device; 499b8d26b3bSNicholas Bellinger int ret = 0; 50014f4b54fSSagi Grimberg 50114f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 50214f4b54fSSagi Grimberg if (!np->enabled) { 50314f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 50424f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 50514f4b54fSSagi Grimberg return rdma_reject(cma_id, NULL, 0); 50614f4b54fSSagi Grimberg } 50714f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 508b8d26b3bSNicholas Bellinger 5094c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n", 510b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 511b8d26b3bSNicholas Bellinger 512b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 513ae9ea9edSSagi Grimberg if (!isert_conn) 514b8d26b3bSNicholas Bellinger return -ENOMEM; 515b8d26b3bSNicholas Bellinger 516ae9ea9edSSagi Grimberg isert_init_conn(isert_conn); 517dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id; 518b8d26b3bSNicholas Bellinger 519ae9ea9edSSagi Grimberg ret = isert_alloc_login_buf(isert_conn, cma_id->device); 520ae9ea9edSSagi Grimberg if (ret) 521b8d26b3bSNicholas Bellinger goto out; 522b8d26b3bSNicholas Bellinger 523cf8ae958SSagi Grimberg device = isert_device_get(cma_id); 524b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 525b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 526b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 527b8d26b3bSNicholas Bellinger } 528dac6ab30SSagi Grimberg isert_conn->device = device; 529b8d26b3bSNicholas Bellinger 530422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn); 5311a92e17eSSagi Grimberg 532570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 533b8d26b3bSNicholas Bellinger if (ret) 534b8d26b3bSNicholas Bellinger goto out_conn_dev; 535b8d26b3bSNicholas Bellinger 5369679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 5372371e5daSSagi Grimberg if (ret) 5382371e5daSSagi Grimberg goto out_conn_dev; 5392371e5daSSagi Grimberg 5402371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 5412371e5daSSagi Grimberg if (ret) 5422371e5daSSagi Grimberg goto out_conn_dev; 5432371e5daSSagi Grimberg 544ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 545bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted); 546ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 547b8d26b3bSNicholas Bellinger 548b8d26b3bSNicholas Bellinger return 0; 549b8d26b3bSNicholas Bellinger 550b8d26b3bSNicholas Bellinger out_conn_dev: 551cf8ae958SSagi Grimberg isert_device_put(device); 552b8d26b3bSNicholas Bellinger out_rsp_dma_map: 553ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 554b8d26b3bSNicholas Bellinger out: 555b8d26b3bSNicholas Bellinger kfree(isert_conn); 5562371e5daSSagi Grimberg rdma_reject(cma_id, NULL, 0); 557b8d26b3bSNicholas Bellinger return ret; 558b8d26b3bSNicholas Bellinger } 559b8d26b3bSNicholas Bellinger 560b8d26b3bSNicholas Bellinger static void 561b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 562b8d26b3bSNicholas Bellinger { 563dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 564b8d26b3bSNicholas Bellinger 5654c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn); 566b8d26b3bSNicholas Bellinger 56757df81e3SSagi Grimberg BUG_ON(!device); 56857df81e3SSagi Grimberg 56919e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 57063b268d2SRaju Rangoju if (isert_conn->cm_id && 57163b268d2SRaju Rangoju !isert_conn->dev_removed) 572dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id); 57319e2090fSSagi Grimberg 574dac6ab30SSagi Grimberg if (isert_conn->qp) { 575dac6ab30SSagi Grimberg struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 5764a295baeSSagi Grimberg 57740fc069aSSagi Grimberg isert_comp_put(comp); 578dac6ab30SSagi Grimberg ib_destroy_qp(isert_conn->qp); 579b8d26b3bSNicholas Bellinger } 580b8d26b3bSNicholas Bellinger 5815adabdd1SChristoph Hellwig if (isert_conn->login_req_buf) 582ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 583ae9ea9edSSagi Grimberg 584cf8ae958SSagi Grimberg isert_device_put(device); 58557df81e3SSagi Grimberg 58663b268d2SRaju Rangoju if (isert_conn->dev_removed) 58763b268d2SRaju Rangoju wake_up_interruptible(&isert_conn->rem_wait); 58863b268d2SRaju Rangoju else 58957df81e3SSagi Grimberg kfree(isert_conn); 590b8d26b3bSNicholas Bellinger } 591b8d26b3bSNicholas Bellinger 592b8d26b3bSNicholas Bellinger static void 593b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 594b8d26b3bSNicholas Bellinger { 59519e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 596bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 597c2f88b17SSagi Grimberg 59824f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 599128e9cc8SSagi Grimberg 600dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 601128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 602bd379220SJenny Derzhavetz kref_get(&isert_conn->kref); 603dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 604bd379220SJenny Derzhavetz 605bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex); 606bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending); 607bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 608bd379220SJenny Derzhavetz 609bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np); 610bd379220SJenny Derzhavetz up(&isert_np->sem); 611b8d26b3bSNicholas Bellinger } 612b8d26b3bSNicholas Bellinger 613b8d26b3bSNicholas Bellinger static void 614dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref) 615b8d26b3bSNicholas Bellinger { 616b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 617dac6ab30SSagi Grimberg struct isert_conn, kref); 618b8d26b3bSNicholas Bellinger 6194c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 6204c22e07fSSagi Grimberg current->pid); 621b8d26b3bSNicholas Bellinger 622b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 623b8d26b3bSNicholas Bellinger } 624b8d26b3bSNicholas Bellinger 625b8d26b3bSNicholas Bellinger static void 626b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 627b8d26b3bSNicholas Bellinger { 628dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref); 629b8d26b3bSNicholas Bellinger } 630b8d26b3bSNicholas Bellinger 6316d1fba0cSJenny Derzhavetz static void 6326d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(struct isert_conn *isert_conn) 6336d1fba0cSJenny Derzhavetz { 6346d1fba0cSJenny Derzhavetz struct isert_np *isert_np = isert_conn->cm_id->context; 6356d1fba0cSJenny Derzhavetz 6366d1fba0cSJenny Derzhavetz mutex_lock(&isert_np->mutex); 6376d1fba0cSJenny Derzhavetz if (!list_empty(&isert_conn->node)) { 6386d1fba0cSJenny Derzhavetz /* 6396d1fba0cSJenny Derzhavetz * This means iscsi doesn't know this connection 6406d1fba0cSJenny Derzhavetz * so schedule a cleanup ourselves 6416d1fba0cSJenny Derzhavetz */ 6426d1fba0cSJenny Derzhavetz list_del_init(&isert_conn->node); 6436d1fba0cSJenny Derzhavetz isert_put_conn(isert_conn); 6446d1fba0cSJenny Derzhavetz queue_work(isert_release_wq, &isert_conn->release_work); 6456d1fba0cSJenny Derzhavetz } 6466d1fba0cSJenny Derzhavetz mutex_unlock(&isert_np->mutex); 6476d1fba0cSJenny Derzhavetz } 6486d1fba0cSJenny Derzhavetz 649954f2372SSagi Grimberg /** 650954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 651954f2372SSagi Grimberg * @isert_conn: isert connection struct 652954f2372SSagi Grimberg * 653954f2372SSagi Grimberg * Notes: 654aea92980SJenny Derzhavetz * In case the connection state is BOUND, move state 655954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 656128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 657954f2372SSagi Grimberg * 658dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is 659954f2372SSagi Grimberg * safe to call multiple times. 660954f2372SSagi Grimberg */ 661954f2372SSagi Grimberg static void 662954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 663954f2372SSagi Grimberg { 664954f2372SSagi Grimberg int err; 665954f2372SSagi Grimberg 6666d1fba0cSJenny Derzhavetz if (isert_conn->state >= ISER_CONN_TERMINATING) 6676d1fba0cSJenny Derzhavetz return; 6686d1fba0cSJenny Derzhavetz 66924f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 670954f2372SSagi Grimberg isert_conn, isert_conn->state); 671128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 672dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id); 673954f2372SSagi Grimberg if (err) 67424f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 675954f2372SSagi Grimberg isert_conn); 676954f2372SSagi Grimberg } 677954f2372SSagi Grimberg 6783b726ae2SSagi Grimberg static int 679ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 680ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 681b8d26b3bSNicholas Bellinger { 682ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n", 683ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np); 6843b726ae2SSagi Grimberg 685ca6c1d82SSagi Grimberg switch (event) { 686ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 687ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 688ca6c1d82SSagi Grimberg break; 689ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 690ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np); 691ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) { 69224f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 693ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id)); 694ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 695ca6c1d82SSagi Grimberg } 696ca6c1d82SSagi Grimberg break; 697ca6c1d82SSagi Grimberg default: 69824f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 699ca6c1d82SSagi Grimberg isert_np, event); 700ca6c1d82SSagi Grimberg } 701ca6c1d82SSagi Grimberg 7023b726ae2SSagi Grimberg return -1; 7033b726ae2SSagi Grimberg } 7043b726ae2SSagi Grimberg 705ca6c1d82SSagi Grimberg static int 706ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 707ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 708ca6c1d82SSagi Grimberg { 709f81bf458SJenny Derzhavetz struct isert_conn *isert_conn = cma_id->qp->qp_context; 710ca6c1d82SSagi Grimberg 711dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 7126d1fba0cSJenny Derzhavetz switch (isert_conn->state) { 7136d1fba0cSJenny Derzhavetz case ISER_CONN_TERMINATING: 7146d1fba0cSJenny Derzhavetz break; 7156d1fba0cSJenny Derzhavetz case ISER_CONN_UP: 716128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 717572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 7186d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(isert_conn); 7196d1fba0cSJenny Derzhavetz break; 7206d1fba0cSJenny Derzhavetz case ISER_CONN_BOUND: 7216d1fba0cSJenny Derzhavetz case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 7226d1fba0cSJenny Derzhavetz iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 7236d1fba0cSJenny Derzhavetz break; 7246d1fba0cSJenny Derzhavetz default: 725506f71d1SColin Ian King isert_warn("conn %p terminating in state %d\n", 7266d1fba0cSJenny Derzhavetz isert_conn, isert_conn->state); 7276d1fba0cSJenny Derzhavetz } 728dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 729128e9cc8SSagi Grimberg 7303b726ae2SSagi Grimberg return 0; 731b8d26b3bSNicholas Bellinger } 732b8d26b3bSNicholas Bellinger 7334a579da2SSagi Grimberg static int 734954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 735954f2372SSagi Grimberg { 73619e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 737954f2372SSagi Grimberg 738cd52cb26SSagi Grimberg ib_drain_qp(isert_conn->qp); 739bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 740dac6ab30SSagi Grimberg isert_conn->cm_id = NULL; 741954f2372SSagi Grimberg isert_put_conn(isert_conn); 7424a579da2SSagi Grimberg 7434a579da2SSagi Grimberg return -1; 744954f2372SSagi Grimberg } 745954f2372SSagi Grimberg 746b8d26b3bSNicholas Bellinger static int 747b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 748b8d26b3bSNicholas Bellinger { 749f81bf458SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 75063b268d2SRaju Rangoju struct isert_conn *isert_conn; 751b8d26b3bSNicholas Bellinger int ret = 0; 752b8d26b3bSNicholas Bellinger 753ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n", 754ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event, 7554c22e07fSSagi Grimberg event->status, cma_id, cma_id->context); 756b8d26b3bSNicholas Bellinger 757f81bf458SJenny Derzhavetz if (isert_np->cm_id == cma_id) 758f81bf458SJenny Derzhavetz return isert_np_cma_handler(cma_id->context, event->event); 759f81bf458SJenny Derzhavetz 760b8d26b3bSNicholas Bellinger switch (event->event) { 761b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 762b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 7633b726ae2SSagi Grimberg if (ret) 7644c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret); 765b8d26b3bSNicholas Bellinger break; 766b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 767b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 768b8d26b3bSNicholas Bellinger break; 76988c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 77088c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 77188c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 772ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 773b8d26b3bSNicholas Bellinger break; 77463b268d2SRaju Rangoju case RDMA_CM_EVENT_DEVICE_REMOVAL: 77563b268d2SRaju Rangoju isert_conn = cma_id->qp->qp_context; 77663b268d2SRaju Rangoju isert_conn->dev_removed = true; 77763b268d2SRaju Rangoju isert_disconnected_handler(cma_id, event->event); 77863b268d2SRaju Rangoju wait_event_interruptible(isert_conn->rem_wait, 77963b268d2SRaju Rangoju isert_conn->state == ISER_CONN_DOWN); 78063b268d2SRaju Rangoju kfree(isert_conn); 78163b268d2SRaju Rangoju /* 78263b268d2SRaju Rangoju * return non-zero from the callback to destroy 78363b268d2SRaju Rangoju * the rdma cm id 78463b268d2SRaju Rangoju */ 78563b268d2SRaju Rangoju return 1; 786d2271bd0SBart Van Assche case RDMA_CM_EVENT_REJECTED: 7871e38a366SSteve Wise isert_info("Connection rejected: %s\n", 7881e38a366SSteve Wise rdma_reject_msg(cma_id, event->status)); 789d2271bd0SBart Van Assche /* fall through */ 790d2271bd0SBart Van Assche case RDMA_CM_EVENT_UNREACHABLE: 791b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 7924a579da2SSagi Grimberg ret = isert_connect_error(cma_id); 793954f2372SSagi Grimberg break; 794b8d26b3bSNicholas Bellinger default: 79524f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 796b8d26b3bSNicholas Bellinger break; 797b8d26b3bSNicholas Bellinger } 798b8d26b3bSNicholas Bellinger 799b8d26b3bSNicholas Bellinger return ret; 800b8d26b3bSNicholas Bellinger } 801b8d26b3bSNicholas Bellinger 802b8d26b3bSNicholas Bellinger static int 8034366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count) 804b8d26b3bSNicholas Bellinger { 805e01a7674SBart Van Assche struct ib_recv_wr *rx_wr; 806b8d26b3bSNicholas Bellinger int i, ret; 807b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 808b8d26b3bSNicholas Bellinger 809dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 8104366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i]; 8119679cc51SChristoph Hellwig 8129679cc51SChristoph Hellwig rx_wr->wr_cqe = &rx_desc->rx_cqe; 813b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 814b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 815b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 8167a56dc88SSagi Grimberg rx_desc->in_use = false; 817b8d26b3bSNicholas Bellinger } 818b8d26b3bSNicholas Bellinger rx_wr--; 819b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 820b8d26b3bSNicholas Bellinger 821e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 8226d1fba0cSJenny Derzhavetz if (ret) 82324f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 8244366b19cSJenny Derzhavetz 8254366b19cSJenny Derzhavetz return ret; 8264366b19cSJenny Derzhavetz } 8274366b19cSJenny Derzhavetz 8284366b19cSJenny Derzhavetz static int 8294366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 8304366b19cSJenny Derzhavetz { 831e01a7674SBart Van Assche struct ib_recv_wr rx_wr; 8324366b19cSJenny Derzhavetz int ret; 8334366b19cSJenny Derzhavetz 8347a56dc88SSagi Grimberg if (!rx_desc->in_use) { 8357a56dc88SSagi Grimberg /* 8367a56dc88SSagi Grimberg * if the descriptor is not in-use we already reposted it 8377a56dc88SSagi Grimberg * for recv, so just silently return 8387a56dc88SSagi Grimberg */ 8397a56dc88SSagi Grimberg return 0; 8407a56dc88SSagi Grimberg } 8417a56dc88SSagi Grimberg 8427a56dc88SSagi Grimberg rx_desc->in_use = false; 8439679cc51SChristoph Hellwig rx_wr.wr_cqe = &rx_desc->rx_cqe; 8444366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg; 8454366b19cSJenny Derzhavetz rx_wr.num_sge = 1; 8464366b19cSJenny Derzhavetz rx_wr.next = NULL; 8474366b19cSJenny Derzhavetz 848e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 8496d1fba0cSJenny Derzhavetz if (ret) 8504366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret); 8514366b19cSJenny Derzhavetz 852b8d26b3bSNicholas Bellinger return ret; 853b8d26b3bSNicholas Bellinger } 854b8d26b3bSNicholas Bellinger 855b8d26b3bSNicholas Bellinger static int 8569679cc51SChristoph Hellwig isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 857b8d26b3bSNicholas Bellinger { 858dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 859e01a7674SBart Van Assche struct ib_send_wr send_wr; 860b8d26b3bSNicholas Bellinger int ret; 861b8d26b3bSNicholas Bellinger 862b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 863b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 864b8d26b3bSNicholas Bellinger 8659679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_login_send_done; 8669679cc51SChristoph Hellwig 867b8d26b3bSNicholas Bellinger send_wr.next = NULL; 8689679cc51SChristoph Hellwig send_wr.wr_cqe = &tx_desc->tx_cqe; 869b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 870b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 871b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 872b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 873b8d26b3bSNicholas Bellinger 874e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 875bdf20e72SSagi Grimberg if (ret) 87624f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 877b8d26b3bSNicholas Bellinger 878b8d26b3bSNicholas Bellinger return ret; 879b8d26b3bSNicholas Bellinger } 880b8d26b3bSNicholas Bellinger 881b8d26b3bSNicholas Bellinger static void 882763b6965SAlex Estrin __isert_create_send_desc(struct isert_device *device, 883b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 884b8d26b3bSNicholas Bellinger { 885b8d26b3bSNicholas Bellinger 886d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 887d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL; 888b8d26b3bSNicholas Bellinger 889b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 890b8d26b3bSNicholas Bellinger 89134efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 89234efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 89324f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 894b8d26b3bSNicholas Bellinger } 895b8d26b3bSNicholas Bellinger } 896b8d26b3bSNicholas Bellinger 897763b6965SAlex Estrin static void 898763b6965SAlex Estrin isert_create_send_desc(struct isert_conn *isert_conn, 899763b6965SAlex Estrin struct isert_cmd *isert_cmd, 900763b6965SAlex Estrin struct iser_tx_desc *tx_desc) 901763b6965SAlex Estrin { 902763b6965SAlex Estrin struct isert_device *device = isert_conn->device; 903763b6965SAlex Estrin struct ib_device *ib_dev = device->ib_device; 904763b6965SAlex Estrin 905763b6965SAlex Estrin ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 906763b6965SAlex Estrin ISER_HEADERS_LEN, DMA_TO_DEVICE); 907763b6965SAlex Estrin 908763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc); 909763b6965SAlex Estrin } 910763b6965SAlex Estrin 911b8d26b3bSNicholas Bellinger static int 912b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 913b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 914b8d26b3bSNicholas Bellinger { 915dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 91667cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 917b8d26b3bSNicholas Bellinger u64 dma_addr; 918b8d26b3bSNicholas Bellinger 919b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 920b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 921b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 92224f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 923b8d26b3bSNicholas Bellinger return -ENOMEM; 924b8d26b3bSNicholas Bellinger } 925b8d26b3bSNicholas Bellinger 926b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 927b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 928b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 92934efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 930b8d26b3bSNicholas Bellinger 9314c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 9324c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 9334c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey); 934b8d26b3bSNicholas Bellinger 935b8d26b3bSNicholas Bellinger return 0; 936b8d26b3bSNicholas Bellinger } 937b8d26b3bSNicholas Bellinger 938b8d26b3bSNicholas Bellinger static void 93995b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 94068a86deeSSagi Grimberg struct ib_send_wr *send_wr) 941b8d26b3bSNicholas Bellinger { 94295b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 94395b60f07SNicholas Bellinger 9449679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_send_done; 9459679cc51SChristoph Hellwig send_wr->wr_cqe = &tx_desc->tx_cqe; 946422bd0acSJenny Derzhavetz 947422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 948422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV; 949422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 950422bd0acSJenny Derzhavetz } else { 951b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 952422bd0acSJenny Derzhavetz } 953422bd0acSJenny Derzhavetz 95495b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 955b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 95695b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 957b8d26b3bSNicholas Bellinger } 958b8d26b3bSNicholas Bellinger 959b8d26b3bSNicholas Bellinger static int 9609679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn) 961b8d26b3bSNicholas Bellinger { 962e01a7674SBart Van Assche struct ib_recv_wr rx_wr; 963b8d26b3bSNicholas Bellinger struct ib_sge sge; 964b8d26b3bSNicholas Bellinger int ret; 965b8d26b3bSNicholas Bellinger 966b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 967b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 968ed1083b2SChristoph Hellwig sge.length = ISER_RX_PAYLOAD_SIZE; 96934efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey; 970b8d26b3bSNicholas Bellinger 97124f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 972b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 973b8d26b3bSNicholas Bellinger 9749679cc51SChristoph Hellwig isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 9759679cc51SChristoph Hellwig 976b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 9779679cc51SChristoph Hellwig rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 978b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 979b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 980b8d26b3bSNicholas Bellinger 981e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 9826d1fba0cSJenny Derzhavetz if (ret) 98324f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 984b8d26b3bSNicholas Bellinger 985b8d26b3bSNicholas Bellinger return ret; 986b8d26b3bSNicholas Bellinger } 987b8d26b3bSNicholas Bellinger 988b8d26b3bSNicholas Bellinger static int 989b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 990b8d26b3bSNicholas Bellinger u32 length) 991b8d26b3bSNicholas Bellinger { 992b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 993dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 99467cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 995dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 996b8d26b3bSNicholas Bellinger int ret; 997b8d26b3bSNicholas Bellinger 998763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc); 999b8d26b3bSNicholas Bellinger 1000b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1001b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 1002b8d26b3bSNicholas Bellinger 1003b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 1004b8d26b3bSNicholas Bellinger 1005b8d26b3bSNicholas Bellinger if (length > 0) { 1006b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1007b8d26b3bSNicholas Bellinger 1008b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1009b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1010b8d26b3bSNicholas Bellinger 1011b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1012b8d26b3bSNicholas Bellinger 1013b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1014b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1015b8d26b3bSNicholas Bellinger 1016b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1017b8d26b3bSNicholas Bellinger tx_dsg->length = length; 101834efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1019b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1020b8d26b3bSNicholas Bellinger } 1021b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1022b8d26b3bSNicholas Bellinger if (login->login_complete) { 1023b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1024b8d26b3bSNicholas Bellinger if (ret) 1025b8d26b3bSNicholas Bellinger return ret; 1026b8d26b3bSNicholas Bellinger 10274366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn, 10284366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS); 1029b8d26b3bSNicholas Bellinger if (ret) 1030b8d26b3bSNicholas Bellinger return ret; 1031b8d26b3bSNicholas Bellinger 1032128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 1033dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 1034128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 1035dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1036b8d26b3bSNicholas Bellinger goto post_send; 1037b8d26b3bSNicholas Bellinger } 1038b8d26b3bSNicholas Bellinger 10399679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 1040b8d26b3bSNicholas Bellinger if (ret) 1041b8d26b3bSNicholas Bellinger return ret; 1042b8d26b3bSNicholas Bellinger } 1043b8d26b3bSNicholas Bellinger post_send: 10449679cc51SChristoph Hellwig ret = isert_login_post_send(isert_conn, tx_desc); 1045b8d26b3bSNicholas Bellinger if (ret) 1046b8d26b3bSNicholas Bellinger return ret; 1047b8d26b3bSNicholas Bellinger 1048b8d26b3bSNicholas Bellinger return 0; 1049b8d26b3bSNicholas Bellinger } 1050b8d26b3bSNicholas Bellinger 1051b8d26b3bSNicholas Bellinger static void 10522371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1053b8d26b3bSNicholas Bellinger { 10545adabdd1SChristoph Hellwig struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 10552371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1056b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1057b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1058b8d26b3bSNicholas Bellinger int size; 1059b8d26b3bSNicholas Bellinger 106024f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 10612371e5daSSagi Grimberg 10622371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1063b8d26b3bSNicholas Bellinger 1064b8d26b3bSNicholas Bellinger if (login->first_request) { 1065b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1066b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1067b8d26b3bSNicholas Bellinger /* 1068b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1069b8d26b3bSNicholas Bellinger * login request PDU. 1070b8d26b3bSNicholas Bellinger */ 1071b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1072b8d26b3bSNicholas Bellinger login->current_stage = 1073b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1074b8d26b3bSNicholas Bellinger >> 2; 1075b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1076b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1077b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1078b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1079b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1080b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1081b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1082b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1083b8d26b3bSNicholas Bellinger } 1084b8d26b3bSNicholas Bellinger 1085b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1086b8d26b3bSNicholas Bellinger 1087b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 10884c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d " 10894c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 10904c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS); 1091b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1092b8d26b3bSNicholas Bellinger 10936faaa85fSNicholas Bellinger if (login->first_request) { 1094dac6ab30SSagi Grimberg complete(&isert_conn->login_comp); 10956faaa85fSNicholas Bellinger return; 10966faaa85fSNicholas Bellinger } 10976faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1098b8d26b3bSNicholas Bellinger } 1099b8d26b3bSNicholas Bellinger 1100b8d26b3bSNicholas Bellinger static struct iscsi_cmd 11014366b19cSJenny Derzhavetz *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1102b8d26b3bSNicholas Bellinger { 11036700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1104b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1105d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1106b8d26b3bSNicholas Bellinger 1107676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1108d703ce2fSNicholas Bellinger if (!cmd) { 110924f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1110b8d26b3bSNicholas Bellinger return NULL; 1111b8d26b3bSNicholas Bellinger } 1112d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1113b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1114d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 11154366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc; 1116b8d26b3bSNicholas Bellinger 1117d703ce2fSNicholas Bellinger return cmd; 1118b8d26b3bSNicholas Bellinger } 1119b8d26b3bSNicholas Bellinger 1120b8d26b3bSNicholas Bellinger static int 1121b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1122d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1123d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1124b8d26b3bSNicholas Bellinger { 1125b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1126b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1127b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1128b8d26b3bSNicholas Bellinger bool dump_payload = false; 11299fd60088SJenny Derzhavetz unsigned int data_len; 1130b8d26b3bSNicholas Bellinger 1131b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1132b8d26b3bSNicholas Bellinger if (rc < 0) 1133b8d26b3bSNicholas Bellinger return rc; 1134b8d26b3bSNicholas Bellinger 1135b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1136b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1137b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 11389fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length; 1139b8d26b3bSNicholas Bellinger 11409fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len) 11419fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1142b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1143b8d26b3bSNicholas Bellinger if (rc < 0) { 1144b8d26b3bSNicholas Bellinger return 0; 1145b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1146b8d26b3bSNicholas Bellinger dump_payload = true; 1147b8d26b3bSNicholas Bellinger goto sequence_cmd; 1148b8d26b3bSNicholas Bellinger } 1149b8d26b3bSNicholas Bellinger 1150b8d26b3bSNicholas Bellinger if (!imm_data) 1151b8d26b3bSNicholas Bellinger return 0; 1152b8d26b3bSNicholas Bellinger 11539fd60088SJenny Derzhavetz if (imm_data_len != data_len) { 1154b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 11559fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 11569fd60088SJenny Derzhavetz &rx_desc->data[0], imm_data_len); 11579fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 11589fd60088SJenny Derzhavetz sg_nents, imm_data_len); 11599fd60088SJenny Derzhavetz } else { 11609fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1); 11619fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg; 11629fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1; 11639fd60088SJenny Derzhavetz sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 11649fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n", 11659fd60088SJenny Derzhavetz imm_data_len); 11669fd60088SJenny Derzhavetz } 1167b8d26b3bSNicholas Bellinger 1168b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1169b8d26b3bSNicholas Bellinger 1170b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1171b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1172b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1173b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1174b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1175b8d26b3bSNicholas Bellinger } 1176b8d26b3bSNicholas Bellinger 1177b8d26b3bSNicholas Bellinger sequence_cmd: 1178561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1179b8d26b3bSNicholas Bellinger 1180b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 11810300b114SBart Van Assche iscsit_set_unsolicited_dataout(cmd); 11826cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 1183afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd); 1184b8d26b3bSNicholas Bellinger 1185b8d26b3bSNicholas Bellinger return 0; 1186b8d26b3bSNicholas Bellinger } 1187b8d26b3bSNicholas Bellinger 1188b8d26b3bSNicholas Bellinger static int 1189b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1190b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1191b8d26b3bSNicholas Bellinger { 1192b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1193b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1194b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1195b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1196b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1197b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1198b8d26b3bSNicholas Bellinger 1199b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1200b8d26b3bSNicholas Bellinger if (rc < 0) 1201b8d26b3bSNicholas Bellinger return rc; 1202b8d26b3bSNicholas Bellinger else if (!cmd) 1203b8d26b3bSNicholas Bellinger return 0; 1204b8d26b3bSNicholas Bellinger /* 1205b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1206b8d26b3bSNicholas Bellinger */ 1207b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 120824f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1209b8d26b3bSNicholas Bellinger dump_stack(); 1210b8d26b3bSNicholas Bellinger return -1; 1211b8d26b3bSNicholas Bellinger } 1212b8d26b3bSNicholas Bellinger 12134c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 12144c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n", 12154c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done, 12164c22e07fSSagi Grimberg cmd->se_cmd.data_length); 1217b8d26b3bSNicholas Bellinger 1218b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1219b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1220b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1221b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1222b8d26b3bSNicholas Bellinger /* 1223b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1224b8d26b3bSNicholas Bellinger */ 1225b8d26b3bSNicholas Bellinger if (page_off) { 12264c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n"); 1227b8d26b3bSNicholas Bellinger dump_stack(); 1228b8d26b3bSNicholas Bellinger return -1; 1229b8d26b3bSNicholas Bellinger } 12304c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 12314c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off, 12324c22e07fSSagi Grimberg sg_nents, &rx_desc->data[0], unsol_data_len); 1233b8d26b3bSNicholas Bellinger 1234b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1235b8d26b3bSNicholas Bellinger unsol_data_len); 1236b8d26b3bSNicholas Bellinger 1237b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1238b8d26b3bSNicholas Bellinger if (rc < 0) 1239b8d26b3bSNicholas Bellinger return rc; 1240b8d26b3bSNicholas Bellinger 12414366b19cSJenny Derzhavetz /* 12424366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive - 12434366b19cSJenny Derzhavetz * so post the buffer before hand 12444366b19cSJenny Derzhavetz */ 12454366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, rx_desc); 12464366b19cSJenny Derzhavetz if (rc) { 12474366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 12484366b19cSJenny Derzhavetz return rc; 12494366b19cSJenny Derzhavetz } 1250b8d26b3bSNicholas Bellinger return 0; 1251b8d26b3bSNicholas Bellinger } 1252b8d26b3bSNicholas Bellinger 1253b8d26b3bSNicholas Bellinger static int 1254778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1255d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1256d703ce2fSNicholas Bellinger unsigned char *buf) 1257778de368SNicholas Bellinger { 1258778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1259778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1260778de368SNicholas Bellinger int rc; 1261778de368SNicholas Bellinger 1262778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1263778de368SNicholas Bellinger if (rc < 0) 1264778de368SNicholas Bellinger return rc; 1265778de368SNicholas Bellinger /* 1266778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1267778de368SNicholas Bellinger */ 1268778de368SNicholas Bellinger 1269778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1270778de368SNicholas Bellinger } 1271778de368SNicholas Bellinger 1272778de368SNicholas Bellinger static int 1273adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1274d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1275d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1276adb54c29SNicholas Bellinger { 1277adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1278adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1279adb54c29SNicholas Bellinger int rc; 1280b44a2b67SSagi Grimberg unsigned char *text_in = NULL; 1281adb54c29SNicholas Bellinger 1282adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1283adb54c29SNicholas Bellinger if (rc < 0) 1284adb54c29SNicholas Bellinger return rc; 1285adb54c29SNicholas Bellinger 1286b44a2b67SSagi Grimberg if (payload_length) { 1287adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 128893b80f29SLeon Romanovsky if (!text_in) 1289adb54c29SNicholas Bellinger return -ENOMEM; 1290adb54c29SNicholas Bellinger } 1291adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1292adb54c29SNicholas Bellinger 1293adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1294adb54c29SNicholas Bellinger 1295adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1296adb54c29SNicholas Bellinger } 1297adb54c29SNicholas Bellinger 1298adb54c29SNicholas Bellinger static int 1299b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1300b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1301b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1302b8d26b3bSNicholas Bellinger { 1303b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1304b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1305b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1306b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1307b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1308b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1309b8d26b3bSNicholas Bellinger 1310fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType && 1311ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 131224f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1313ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1314ca40d24eSNicholas Bellinger return 0; 1315ca40d24eSNicholas Bellinger } 1316ca40d24eSNicholas Bellinger 1317b8d26b3bSNicholas Bellinger switch (opcode) { 1318b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 13194366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1320b8d26b3bSNicholas Bellinger if (!cmd) 1321b8d26b3bSNicholas Bellinger break; 1322b8d26b3bSNicholas Bellinger 1323d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1324b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1325b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1326b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1327b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1328422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1329b8d26b3bSNicholas Bellinger 1330d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1331b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1332b8d26b3bSNicholas Bellinger break; 1333b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 13344366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1335b8d26b3bSNicholas Bellinger if (!cmd) 1336b8d26b3bSNicholas Bellinger break; 1337b8d26b3bSNicholas Bellinger 1338d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1339d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1340778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1341b8d26b3bSNicholas Bellinger break; 1342b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1343b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1344b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1345b8d26b3bSNicholas Bellinger break; 1346b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 13474366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1348b8d26b3bSNicholas Bellinger if (!cmd) 1349b8d26b3bSNicholas Bellinger break; 1350b8d26b3bSNicholas Bellinger 1351b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1352b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1353b8d26b3bSNicholas Bellinger break; 1354b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 13554366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1356b8d26b3bSNicholas Bellinger if (!cmd) 1357b8d26b3bSNicholas Bellinger break; 1358b8d26b3bSNicholas Bellinger 1359b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1360b8d26b3bSNicholas Bellinger break; 1361adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 13624366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1363e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 13644366b19cSJenny Derzhavetz else 13654366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 13664366b19cSJenny Derzhavetz 1367e4f4e801SSagi Grimberg if (!cmd) 1368e4f4e801SSagi Grimberg break; 1369adb54c29SNicholas Bellinger 1370d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1371d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1372adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1373adb54c29SNicholas Bellinger break; 1374b8d26b3bSNicholas Bellinger default: 137524f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1376b8d26b3bSNicholas Bellinger dump_stack(); 1377b8d26b3bSNicholas Bellinger break; 1378b8d26b3bSNicholas Bellinger } 1379b8d26b3bSNicholas Bellinger 1380b8d26b3bSNicholas Bellinger return ret; 1381b8d26b3bSNicholas Bellinger } 1382b8d26b3bSNicholas Bellinger 1383b8d26b3bSNicholas Bellinger static void 13849679cc51SChristoph Hellwig isert_print_wc(struct ib_wc *wc, const char *type) 1385b8d26b3bSNicholas Bellinger { 13869679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 13879679cc51SChristoph Hellwig isert_err("%s failure: %s (%d) vend_err %x\n", type, 13889679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status, 13899679cc51SChristoph Hellwig wc->vendor_err); 13909679cc51SChristoph Hellwig else 13919679cc51SChristoph Hellwig isert_dbg("%s failure: %s (%d)\n", type, 13929679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status); 13939679cc51SChristoph Hellwig } 13949679cc51SChristoph Hellwig 13959679cc51SChristoph Hellwig static void 13969679cc51SChristoph Hellwig isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 13979679cc51SChristoph Hellwig { 13989679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 13999679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 14009679cc51SChristoph Hellwig struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 14019679cc51SChristoph Hellwig struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1402d3cf81f9SSagi Grimberg struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1403b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1404b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1405b8d26b3bSNicholas Bellinger 14069679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14079679cc51SChristoph Hellwig isert_print_wc(wc, "recv"); 14089679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 14099679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 14109679cc51SChristoph Hellwig return; 14119679cc51SChristoph Hellwig } 14129679cc51SChristoph Hellwig 14137a56dc88SSagi Grimberg rx_desc->in_use = true; 14147a56dc88SSagi Grimberg 14159679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 14169679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 14179679cc51SChristoph Hellwig 14189679cc51SChristoph Hellwig isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 14199679cc51SChristoph Hellwig rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 14209679cc51SChristoph Hellwig (int)(wc->byte_len - ISER_HEADERS_LEN)); 14219679cc51SChristoph Hellwig 1422d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) { 1423b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1424d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) { 1425d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag); 1426d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va); 14274c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1428b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1429b8d26b3bSNicholas Bellinger } 1430d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) { 1431d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag); 1432d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va); 14334c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1434b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1435b8d26b3bSNicholas Bellinger } 1436b8d26b3bSNicholas Bellinger 143724f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1438b8d26b3bSNicholas Bellinger break; 1439b8d26b3bSNicholas Bellinger case ISER_HELLO: 144024f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1441b8d26b3bSNicholas Bellinger break; 1442b8d26b3bSNicholas Bellinger default: 1443d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1444b8d26b3bSNicholas Bellinger break; 1445b8d26b3bSNicholas Bellinger } 1446b8d26b3bSNicholas Bellinger 14476c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc, 1448b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 14499679cc51SChristoph Hellwig 14509679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 14519679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1452b8d26b3bSNicholas Bellinger } 1453b8d26b3bSNicholas Bellinger 1454b8d26b3bSNicholas Bellinger static void 14559679cc51SChristoph Hellwig isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1456b8d26b3bSNicholas Bellinger { 14579679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1458fce50a2fSNicholas Bellinger struct ib_device *ib_dev = isert_conn->device->ib_device; 1459b8d26b3bSNicholas Bellinger 14609679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14619679cc51SChristoph Hellwig isert_print_wc(wc, "login recv"); 14629679cc51SChristoph Hellwig return; 1463b8d26b3bSNicholas Bellinger } 1464b8d26b3bSNicholas Bellinger 14659679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 14669679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1467b8d26b3bSNicholas Bellinger 14689679cc51SChristoph Hellwig isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1469b8d26b3bSNicholas Bellinger 14702371e5daSSagi Grimberg if (isert_conn->conn) { 14712371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 14722371e5daSSagi Grimberg 14732371e5daSSagi Grimberg if (login && !login->first_request) 14742371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 14752371e5daSSagi Grimberg } 14769679cc51SChristoph Hellwig 1477dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 14782371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 1479dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1480b8d26b3bSNicholas Bellinger 14819679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 14829679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1483b8d26b3bSNicholas Bellinger } 1484b8d26b3bSNicholas Bellinger 1485e3d7e4c3SSagi Grimberg static void 148638a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1487e3d7e4c3SSagi Grimberg { 148838a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 148938a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1490e3d7e4c3SSagi Grimberg 149138a2d0d4SChristoph Hellwig if (!cmd->rw.nr_ops) 149238a2d0d4SChristoph Hellwig return; 149338a2d0d4SChristoph Hellwig 149438a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 149538a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 149638a2d0d4SChristoph Hellwig conn->cm_id->port_num, se_cmd->t_data_sg, 149738a2d0d4SChristoph Hellwig se_cmd->t_data_nents, se_cmd->t_prot_sg, 149838a2d0d4SChristoph Hellwig se_cmd->t_prot_nents, dir); 149938a2d0d4SChristoph Hellwig } else { 150038a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 150138a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1502e3d7e4c3SSagi Grimberg } 1503e3d7e4c3SSagi Grimberg 150438a2d0d4SChristoph Hellwig cmd->rw.nr_ops = 0; 150559464ef4SVu Pham } 150659464ef4SVu Pham 150759464ef4SVu Pham static void 150803e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1509b8d26b3bSNicholas Bellinger { 1510d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1511b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1512186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1513e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr; 1514b8d26b3bSNicholas Bellinger 15154c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1516b8d26b3bSNicholas Bellinger 1517b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1518b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1519b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1520b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15215159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1522b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1523b8d26b3bSNicholas Bellinger 152403e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1525b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 152603e7848aSNicholas Bellinger /* 152703e7848aSNicholas Bellinger * Check for special case during comp_err where 152803e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 152903e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 153003e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 153103e7848aSNicholas Bellinger */ 153203e7848aSNicholas Bellinger if (comp_err && 153303e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 153403e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 153503e7848aSNicholas Bellinger 1536afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 153703e7848aSNicholas Bellinger } 153803e7848aSNicholas Bellinger } 1539b8d26b3bSNicholas Bellinger 154038a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1541186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1542186a9647SNicholas Bellinger break; 1543b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1544186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1545186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15465159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1547186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1548186a9647SNicholas Bellinger 1549b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1550b8d26b3bSNicholas Bellinger break; 1551b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1552b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1553adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1554e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1555e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */ 1556e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1557e4f4e801SSagi Grimberg break; 1558e4f4e801SSagi Grimberg 1559b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1560b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15615159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1562b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1563b8d26b3bSNicholas Bellinger 1564b8d26b3bSNicholas Bellinger /* 1565b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1566b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1567b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1568b8d26b3bSNicholas Bellinger */ 1569b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 157011378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 15713df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1572b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1573b8d26b3bSNicholas Bellinger break; 1574b8d26b3bSNicholas Bellinger } 1575d2271bd0SBart Van Assche /* fall through */ 1576b8d26b3bSNicholas Bellinger default: 1577d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1578b8d26b3bSNicholas Bellinger break; 1579b8d26b3bSNicholas Bellinger } 1580b8d26b3bSNicholas Bellinger } 1581b8d26b3bSNicholas Bellinger 1582b8d26b3bSNicholas Bellinger static void 1583b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1584b8d26b3bSNicholas Bellinger { 1585b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 15864c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n"); 1587b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1588b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1589b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1590b8d26b3bSNicholas Bellinger } 1591b8d26b3bSNicholas Bellinger } 1592b8d26b3bSNicholas Bellinger 1593b8d26b3bSNicholas Bellinger static void 1594b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 159503e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1596b8d26b3bSNicholas Bellinger { 1597dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 15984c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1599dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1600dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1601dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1602b8d26b3bSNicholas Bellinger } 1603b8d26b3bSNicholas Bellinger 1604b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 160503e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1606b8d26b3bSNicholas Bellinger } 1607b8d26b3bSNicholas Bellinger 160896b7973eSSagi Grimberg static int 160996b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1610f93f3a70SSagi Grimberg { 16119e961ae7SSagi Grimberg struct ib_mr_status mr_status; 161296b7973eSSagi Grimberg int ret; 1613f93f3a70SSagi Grimberg 161496b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 16159e961ae7SSagi Grimberg if (ret) { 161624f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 16179e961ae7SSagi Grimberg goto fail_mr_status; 16189e961ae7SSagi Grimberg } 161996b7973eSSagi Grimberg 16209e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 162196b7973eSSagi Grimberg u64 sec_offset_err; 16229e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 16239e961ae7SSagi Grimberg 16249e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 16259e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 16269e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 16279e961ae7SSagi Grimberg break; 16289e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 16299e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 16309e961ae7SSagi Grimberg break; 16319e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 16329e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 16339e961ae7SSagi Grimberg break; 16349e961ae7SSagi Grimberg } 163596b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 163696b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 163796b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 16389e961ae7SSagi Grimberg 16394c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx " 16409e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 16419e961ae7SSagi Grimberg mr_status.sig_err.err_type, 16429e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 16439e961ae7SSagi Grimberg mr_status.sig_err.expected, 16449e961ae7SSagi Grimberg mr_status.sig_err.actual); 16459e961ae7SSagi Grimberg ret = 1; 16469e961ae7SSagi Grimberg } 16479e961ae7SSagi Grimberg 16489e961ae7SSagi Grimberg fail_mr_status: 164996b7973eSSagi Grimberg return ret; 165096b7973eSSagi Grimberg } 165196b7973eSSagi Grimberg 165296b7973eSSagi Grimberg static void 16539679cc51SChristoph Hellwig isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 165496b7973eSSagi Grimberg { 16559679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1656dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 16579679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1658e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 16599679cc51SChristoph Hellwig struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 166096b7973eSSagi Grimberg int ret = 0; 166196b7973eSSagi Grimberg 16629679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 16639679cc51SChristoph Hellwig isert_print_wc(wc, "rdma write"); 16649679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 16659679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 16669679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 16679679cc51SChristoph Hellwig return; 16689679cc51SChristoph Hellwig } 16699679cc51SChristoph Hellwig 16709679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 16719679cc51SChristoph Hellwig 1672e9a53e73SIsrael Rukshin ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 167338a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 167496b7973eSSagi Grimberg 1675555a65f6SNicholas Bellinger if (ret) { 1676555a65f6SNicholas Bellinger /* 1677555a65f6SNicholas Bellinger * transport_generic_request_failure() expects to have 1678555a65f6SNicholas Bellinger * plus two references to handle queue-full, so re-add 1679555a65f6SNicholas Bellinger * one here as target-core will have already dropped 1680555a65f6SNicholas Bellinger * it after the first isert_put_datain() callback. 1681555a65f6SNicholas Bellinger */ 1682555a65f6SNicholas Bellinger kref_get(&cmd->cmd_kref); 1683555a65f6SNicholas Bellinger transport_generic_request_failure(cmd, cmd->pi_err); 1684555a65f6SNicholas Bellinger } else { 1685555a65f6SNicholas Bellinger /* 1686555a65f6SNicholas Bellinger * XXX: isert_put_response() failure is not retried. 1687555a65f6SNicholas Bellinger */ 1688555a65f6SNicholas Bellinger ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 16899e961ae7SSagi Grimberg if (ret) 1690555a65f6SNicholas Bellinger pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1691555a65f6SNicholas Bellinger } 1692f93f3a70SSagi Grimberg } 1693f93f3a70SSagi Grimberg 1694f93f3a70SSagi Grimberg static void 16959679cc51SChristoph Hellwig isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1696b8d26b3bSNicholas Bellinger { 16979679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 16989679cc51SChristoph Hellwig struct isert_device *device = isert_conn->device; 16999679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1700e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1701d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1702b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 17035bac4b1aSSagi Grimberg int ret = 0; 1704b8d26b3bSNicholas Bellinger 17059679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17069679cc51SChristoph Hellwig isert_print_wc(wc, "rdma read"); 17079679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17089679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17099679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 17109679cc51SChristoph Hellwig return; 17119679cc51SChristoph Hellwig } 17129679cc51SChristoph Hellwig 17139679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 17149679cc51SChristoph Hellwig 1715b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 171638a2d0d4SChristoph Hellwig 171738a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) 1718e9a53e73SIsrael Rukshin ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 171938a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 172038a2d0d4SChristoph Hellwig cmd->write_data_done = 0; 1721b8d26b3bSNicholas Bellinger 172224f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1723b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1724b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1725b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1726b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1727b8d26b3bSNicholas Bellinger 1728555a65f6SNicholas Bellinger /* 1729555a65f6SNicholas Bellinger * transport_generic_request_failure() will drop the extra 1730555a65f6SNicholas Bellinger * se_cmd->cmd_kref reference after T10-PI error, and handle 1731555a65f6SNicholas Bellinger * any non-zero ->queue_status() callback error retries. 1732555a65f6SNicholas Bellinger */ 1733555a65f6SNicholas Bellinger if (ret) 1734555a65f6SNicholas Bellinger transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1735555a65f6SNicholas Bellinger else 1736b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1737b8d26b3bSNicholas Bellinger } 1738b8d26b3bSNicholas Bellinger 1739b8d26b3bSNicholas Bellinger static void 1740b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1741b8d26b3bSNicholas Bellinger { 1742b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1743b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1744b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1745dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1746d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1747b8d26b3bSNicholas Bellinger 17484c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 17494c22e07fSSagi Grimberg 1750b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1751b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1752b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 1753d2271bd0SBart Van Assche /* fall through */ 1754d2271bd0SBart Van Assche case ISTATE_SEND_REJECT: 1755d2271bd0SBart Van Assche case ISTATE_SEND_TEXTRSP: 1756b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 17574c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 17584c22e07fSSagi Grimberg ib_dev, false); 17593df8f68aSNicholas Bellinger break; 1760b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1761b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1762b8d26b3bSNicholas Bellinger break; 1763b8d26b3bSNicholas Bellinger default: 17644c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state); 1765b8d26b3bSNicholas Bellinger dump_stack(); 1766b8d26b3bSNicholas Bellinger break; 1767b8d26b3bSNicholas Bellinger } 1768b8d26b3bSNicholas Bellinger } 1769b8d26b3bSNicholas Bellinger 1770b8d26b3bSNicholas Bellinger static void 17719679cc51SChristoph Hellwig isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1772b8d26b3bSNicholas Bellinger { 17739679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17749679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17759679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1776b8d26b3bSNicholas Bellinger 17779679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17789679cc51SChristoph Hellwig isert_print_wc(wc, "login send"); 17799679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17809679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17819679cc51SChristoph Hellwig } 17829679cc51SChristoph Hellwig 17839679cc51SChristoph Hellwig isert_unmap_tx_desc(tx_desc, ib_dev); 17849679cc51SChristoph Hellwig } 17859679cc51SChristoph Hellwig 17869679cc51SChristoph Hellwig static void 17879679cc51SChristoph Hellwig isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 17889679cc51SChristoph Hellwig { 17899679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17909679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17919679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1792e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 17939679cc51SChristoph Hellwig 17949679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17959679cc51SChristoph Hellwig isert_print_wc(wc, "send"); 17969679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17979679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17989679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 17999679cc51SChristoph Hellwig return; 18009679cc51SChristoph Hellwig } 18019679cc51SChristoph Hellwig 18029679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 18039679cc51SChristoph Hellwig 18049679cc51SChristoph Hellwig switch (isert_cmd->iscsi_cmd->i_state) { 18059679cc51SChristoph Hellwig case ISTATE_SEND_TASKMGTRSP: 18069679cc51SChristoph Hellwig case ISTATE_SEND_LOGOUTRSP: 18079679cc51SChristoph Hellwig case ISTATE_SEND_REJECT: 18089679cc51SChristoph Hellwig case ISTATE_SEND_TEXTRSP: 1809b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1810b8d26b3bSNicholas Bellinger 1811b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1812b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1813b8d26b3bSNicholas Bellinger return; 1814b8d26b3bSNicholas Bellinger default: 18159679cc51SChristoph Hellwig isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 18169679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1817b8d26b3bSNicholas Bellinger break; 1818b8d26b3bSNicholas Bellinger } 1819b8d26b3bSNicholas Bellinger } 1820b8d26b3bSNicholas Bellinger 1821b8d26b3bSNicholas Bellinger static int 1822b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1823b8d26b3bSNicholas Bellinger { 1824b8d26b3bSNicholas Bellinger int ret; 1825b8d26b3bSNicholas Bellinger 18264366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 18274366b19cSJenny Derzhavetz if (ret) { 18284366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", ret); 18294366b19cSJenny Derzhavetz return ret; 18304366b19cSJenny Derzhavetz } 18314366b19cSJenny Derzhavetz 1832e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1833b8d26b3bSNicholas Bellinger if (ret) { 183424f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 1835b8d26b3bSNicholas Bellinger return ret; 1836b8d26b3bSNicholas Bellinger } 1837b8d26b3bSNicholas Bellinger return ret; 1838b8d26b3bSNicholas Bellinger } 1839b8d26b3bSNicholas Bellinger 1840b8d26b3bSNicholas Bellinger static int 1841b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1842b8d26b3bSNicholas Bellinger { 1843d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18446700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1845b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1846b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1847b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 1848b8d26b3bSNicholas Bellinger 1849b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1850b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1851b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1852b8d26b3bSNicholas Bellinger /* 1853b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 1854b8d26b3bSNicholas Bellinger */ 1855b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 1856b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1857b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1858dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 185967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1860b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1861dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 1862b8d26b3bSNicholas Bellinger 1863b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1864b8d26b3bSNicholas Bellinger cmd->sense_buffer); 1865b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1866b8d26b3bSNicholas Bellinger 1867b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1868b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1869dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1870b8d26b3bSNicholas Bellinger 1871dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1872dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 1873b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 1874def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1875def4a6ffSAlexey Khoroshilov return -ENOMEM; 1876b8d26b3bSNicholas Bellinger 1877dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 1878dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 1879dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 188034efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 1881b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 1882b8d26b3bSNicholas Bellinger } 1883b8d26b3bSNicholas Bellinger 188468a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1885b8d26b3bSNicholas Bellinger 18864c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n"); 1887b8d26b3bSNicholas Bellinger 1888b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1889b8d26b3bSNicholas Bellinger } 1890b8d26b3bSNicholas Bellinger 1891131e6abcSNicholas Bellinger static void 1892131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1893131e6abcSNicholas Bellinger { 1894131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18956700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1896131e6abcSNicholas Bellinger 1897131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1898131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1899131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 1900131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1901131e6abcSNicholas Bellinger 1902131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 1903131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 190438a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1905131e6abcSNicholas Bellinger } 1906131e6abcSNicholas Bellinger 1907e70beee7SNicholas Bellinger static enum target_prot_op 1908e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 1909e70beee7SNicholas Bellinger { 19106700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1911dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1912e70beee7SNicholas Bellinger 191323a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 191423a548eeSSagi Grimberg if (device->pi_capable) { 191524f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 191623a548eeSSagi Grimberg isert_conn->pi_support = true; 1917e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 191823a548eeSSagi Grimberg } 191923a548eeSSagi Grimberg } 192023a548eeSSagi Grimberg 192124f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 192223a548eeSSagi Grimberg isert_conn->pi_support = false; 1923e70beee7SNicholas Bellinger 1924e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 1925e70beee7SNicholas Bellinger } 1926e70beee7SNicholas Bellinger 1927b8d26b3bSNicholas Bellinger static int 1928b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1929b8d26b3bSNicholas Bellinger bool nopout_response) 1930b8d26b3bSNicholas Bellinger { 1931d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19326700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1933b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1934b8d26b3bSNicholas Bellinger 1935b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1936b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1937b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 1938b8d26b3bSNicholas Bellinger nopout_response); 1939b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 194068a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1941b8d26b3bSNicholas Bellinger 19424c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1943b8d26b3bSNicholas Bellinger 1944b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1945b8d26b3bSNicholas Bellinger } 1946b8d26b3bSNicholas Bellinger 1947b8d26b3bSNicholas Bellinger static int 1948b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1949b8d26b3bSNicholas Bellinger { 1950d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19516700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1952b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1953b8d26b3bSNicholas Bellinger 1954b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1955b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1956b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1957b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 195868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1959b8d26b3bSNicholas Bellinger 19604c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1961b8d26b3bSNicholas Bellinger 1962b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1963b8d26b3bSNicholas Bellinger } 1964b8d26b3bSNicholas Bellinger 1965b8d26b3bSNicholas Bellinger static int 1966b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1967b8d26b3bSNicholas Bellinger { 1968d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19696700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1970b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1971b8d26b3bSNicholas Bellinger 1972b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1973b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1974b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1975b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 197668a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1977b8d26b3bSNicholas Bellinger 19784c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1979b8d26b3bSNicholas Bellinger 1980b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1981b8d26b3bSNicholas Bellinger } 1982b8d26b3bSNicholas Bellinger 1983b8d26b3bSNicholas Bellinger static int 1984b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1985b8d26b3bSNicholas Bellinger { 1986d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19876700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1988b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1989dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 199067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 19913df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 19923df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 19933df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1994b8d26b3bSNicholas Bellinger 1995b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 19963df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 1997b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 19983df8f68aSNicholas Bellinger 19993df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 2000dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 20013df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 20023df8f68aSNicholas Bellinger DMA_TO_DEVICE); 2003def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2004def4a6ffSAlexey Khoroshilov return -ENOMEM; 2005dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2006dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 20073df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 200834efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 20093df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 20103df8f68aSNicholas Bellinger 201168a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2012b8d26b3bSNicholas Bellinger 20134c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn); 2014b8d26b3bSNicholas Bellinger 2015b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2016b8d26b3bSNicholas Bellinger } 2017b8d26b3bSNicholas Bellinger 2018b8d26b3bSNicholas Bellinger static int 2019adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2020adb54c29SNicholas Bellinger { 2021d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 20226700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2023adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2024adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2025adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2026adb54c29SNicholas Bellinger u32 txt_rsp_len; 2027adb54c29SNicholas Bellinger int rc; 2028adb54c29SNicholas Bellinger 2029adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 203022c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2031adb54c29SNicholas Bellinger if (rc < 0) 2032adb54c29SNicholas Bellinger return rc; 2033adb54c29SNicholas Bellinger 2034adb54c29SNicholas Bellinger txt_rsp_len = rc; 2035adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2036adb54c29SNicholas Bellinger 2037adb54c29SNicholas Bellinger if (txt_rsp_len) { 2038dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 203967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2040adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2041adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2042adb54c29SNicholas Bellinger 2043adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2044adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2045def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2046def4a6ffSAlexey Khoroshilov return -ENOMEM; 2047adb54c29SNicholas Bellinger 2048adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2049adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2050adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 205134efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2052adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2053adb54c29SNicholas Bellinger } 205468a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2055adb54c29SNicholas Bellinger 2056f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn); 2057adb54c29SNicholas Bellinger 2058adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2059adb54c29SNicholas Bellinger } 2060adb54c29SNicholas Bellinger 20613d73cf1aSSagi Grimberg static inline void 2062dae6345aSIsrael Rukshin isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 20639e961ae7SSagi Grimberg { 206478eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 20653d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 20663d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 20673d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 206878eda2bbSSagi Grimberg /* 206978eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 207078eda2bbSSagi Grimberg * the target core would like to use it, we will take it 207178eda2bbSSagi Grimberg * from se_cmd. 207278eda2bbSSagi Grimberg */ 207378eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 207478eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 207578eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 207678eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 207778eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 207878eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 20793d73cf1aSSagi Grimberg }; 20803d73cf1aSSagi Grimberg 20813d73cf1aSSagi Grimberg static int 20823d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 20833d73cf1aSSagi Grimberg { 208438a2d0d4SChristoph Hellwig memset(sig_attrs, 0, sizeof(*sig_attrs)); 208538a2d0d4SChristoph Hellwig 20869e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 20879e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 20889e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 208978eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2090dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire); 20919e961ae7SSagi Grimberg break; 20929e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 20939e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 209478eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2095dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem); 20969e961ae7SSagi Grimberg break; 20979e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 20989e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 2099dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2100dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem); 21019e961ae7SSagi Grimberg break; 21029e961ae7SSagi Grimberg default: 210324f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 21049e961ae7SSagi Grimberg return -EINVAL; 21059e961ae7SSagi Grimberg } 21069e961ae7SSagi Grimberg 210726232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 210826232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 210926232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 211026232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 211126232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 211226232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 211326232872SMax Gurtovoy 21149e961ae7SSagi Grimberg return 0; 21159e961ae7SSagi Grimberg } 21169e961ae7SSagi Grimberg 21179e961ae7SSagi Grimberg static int 211838a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 211938a2d0d4SChristoph Hellwig struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 21209e961ae7SSagi Grimberg { 212138a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 212238a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 212338a2d0d4SChristoph Hellwig u8 port_num = conn->cm_id->port_num; 212438a2d0d4SChristoph Hellwig u64 addr; 212538a2d0d4SChristoph Hellwig u32 rkey, offset; 21269e961ae7SSagi Grimberg int ret; 21279e961ae7SSagi Grimberg 212866f53e6fSBharat Potnuri if (cmd->ctx_init_done) 212966f53e6fSBharat Potnuri goto rdma_ctx_post; 213066f53e6fSBharat Potnuri 213138a2d0d4SChristoph Hellwig if (dir == DMA_FROM_DEVICE) { 213238a2d0d4SChristoph Hellwig addr = cmd->write_va; 213338a2d0d4SChristoph Hellwig rkey = cmd->write_stag; 213438a2d0d4SChristoph Hellwig offset = cmd->iscsi_cmd->write_data_done; 213538a2d0d4SChristoph Hellwig } else { 213638a2d0d4SChristoph Hellwig addr = cmd->read_va; 213738a2d0d4SChristoph Hellwig rkey = cmd->read_stag; 213838a2d0d4SChristoph Hellwig offset = 0; 213938a2d0d4SChristoph Hellwig } 214038a2d0d4SChristoph Hellwig 214138a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 214238a2d0d4SChristoph Hellwig struct ib_sig_attrs sig_attrs; 214338a2d0d4SChristoph Hellwig 21449e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 21459e961ae7SSagi Grimberg if (ret) 2146e3d7e4c3SSagi Grimberg return ret; 2147e3d7e4c3SSagi Grimberg 214838a2d0d4SChristoph Hellwig WARN_ON_ONCE(offset); 214938a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 215038a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 215138a2d0d4SChristoph Hellwig se_cmd->t_prot_sg, se_cmd->t_prot_nents, 215238a2d0d4SChristoph Hellwig &sig_attrs, addr, rkey, dir); 2153570db170SSagi Grimberg } else { 215438a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 215538a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 215638a2d0d4SChristoph Hellwig offset, addr, rkey, dir); 215738a2d0d4SChristoph Hellwig } 215866f53e6fSBharat Potnuri 215938a2d0d4SChristoph Hellwig if (ret < 0) { 216038a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 216138a2d0d4SChristoph Hellwig return ret; 21629e961ae7SSagi Grimberg } 21639e961ae7SSagi Grimberg 216466f53e6fSBharat Potnuri cmd->ctx_init_done = true; 216566f53e6fSBharat Potnuri 216666f53e6fSBharat Potnuri rdma_ctx_post: 216738a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 216838a2d0d4SChristoph Hellwig if (ret < 0) 216938a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to post RDMA res\n", cmd); 217059464ef4SVu Pham return ret; 217159464ef4SVu Pham } 217259464ef4SVu Pham 217359464ef4SVu Pham static int 217490ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 217590ecc6e2SVu Pham { 217690ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 217759464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21786700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 217938a2d0d4SChristoph Hellwig struct ib_cqe *cqe = NULL; 218038a2d0d4SChristoph Hellwig struct ib_send_wr *chain_wr = NULL; 218190ecc6e2SVu Pham int rc; 218290ecc6e2SVu Pham 218324f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 218490ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 21854c22e07fSSagi Grimberg 218638a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) { 218738a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 218838a2d0d4SChristoph Hellwig cqe = &isert_cmd->tx_desc.tx_cqe; 218938a2d0d4SChristoph Hellwig } else { 2190b8d26b3bSNicholas Bellinger /* 2191b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2192b8d26b3bSNicholas Bellinger */ 21939e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 21949e961ae7SSagi Grimberg &isert_cmd->tx_desc); 219504d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2196b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2197b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 219895b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 219968a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 22004366b19cSJenny Derzhavetz 22014366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 22024366b19cSJenny Derzhavetz if (rc) { 22034366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 22044366b19cSJenny Derzhavetz return rc; 22054366b19cSJenny Derzhavetz } 220638a2d0d4SChristoph Hellwig 220738a2d0d4SChristoph Hellwig chain_wr = &isert_cmd->tx_desc.send_wr; 22089e961ae7SSagi Grimberg } 2209b8d26b3bSNicholas Bellinger 2210555a65f6SNicholas Bellinger rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2211555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2212555a65f6SNicholas Bellinger isert_cmd, rc); 2213555a65f6SNicholas Bellinger return rc; 2214b8d26b3bSNicholas Bellinger } 2215b8d26b3bSNicholas Bellinger 2216b8d26b3bSNicholas Bellinger static int 2217b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2218b8d26b3bSNicholas Bellinger { 2219d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2220555a65f6SNicholas Bellinger int ret; 2221b8d26b3bSNicholas Bellinger 222224f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 222338a2d0d4SChristoph Hellwig isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2224b8d26b3bSNicholas Bellinger 222538a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2226555a65f6SNicholas Bellinger ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 222738a2d0d4SChristoph Hellwig &isert_cmd->tx_desc.tx_cqe, NULL); 2228bdf20e72SSagi Grimberg 2229555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2230555a65f6SNicholas Bellinger isert_cmd, ret); 2231555a65f6SNicholas Bellinger return ret; 2232b8d26b3bSNicholas Bellinger } 2233b8d26b3bSNicholas Bellinger 2234b8d26b3bSNicholas Bellinger static int 2235b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2236b8d26b3bSNicholas Bellinger { 2237a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2238a4c15cd9SJenny Derzhavetz int ret = 0; 2239b8d26b3bSNicholas Bellinger 2240b8d26b3bSNicholas Bellinger switch (state) { 2241a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE: 2242a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 2243a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 2244a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 2245a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 2246a4c15cd9SJenny Derzhavetz break; 2247b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2248b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2249b8d26b3bSNicholas Bellinger break; 2250b8d26b3bSNicholas Bellinger default: 225124f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2252b8d26b3bSNicholas Bellinger ret = -EINVAL; 2253b8d26b3bSNicholas Bellinger break; 2254b8d26b3bSNicholas Bellinger } 2255b8d26b3bSNicholas Bellinger 2256b8d26b3bSNicholas Bellinger return ret; 2257b8d26b3bSNicholas Bellinger } 2258b8d26b3bSNicholas Bellinger 2259b8d26b3bSNicholas Bellinger static int 2260b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2261b8d26b3bSNicholas Bellinger { 2262991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2263b8d26b3bSNicholas Bellinger int ret; 2264b8d26b3bSNicholas Bellinger 2265b8d26b3bSNicholas Bellinger switch (state) { 2266b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2267b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2268991bb764SSagi Grimberg if (!ret) 2269991bb764SSagi Grimberg isert_conn->logout_posted = true; 2270b8d26b3bSNicholas Bellinger break; 2271b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2272b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2273b8d26b3bSNicholas Bellinger break; 2274b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2275b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2276b8d26b3bSNicholas Bellinger break; 2277b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2278b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2279b8d26b3bSNicholas Bellinger break; 2280adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2281adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2282adb54c29SNicholas Bellinger break; 2283b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2284b8d26b3bSNicholas Bellinger /* 2285b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2286b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2287b8d26b3bSNicholas Bellinger */ 2288b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2289b8d26b3bSNicholas Bellinger break; 2290b8d26b3bSNicholas Bellinger default: 229124f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 2292b8d26b3bSNicholas Bellinger ret = -EINVAL; 2293b8d26b3bSNicholas Bellinger break; 2294b8d26b3bSNicholas Bellinger } 2295b8d26b3bSNicholas Bellinger 2296b8d26b3bSNicholas Bellinger return ret; 2297b8d26b3bSNicholas Bellinger } 2298b8d26b3bSNicholas Bellinger 2299ca6c1d82SSagi Grimberg struct rdma_cm_id * 2300ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 2301ca6c1d82SSagi Grimberg { 2302ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 2303ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 2304ca6c1d82SSagi Grimberg struct sockaddr *sa; 2305ca6c1d82SSagi Grimberg int ret; 2306ca6c1d82SSagi Grimberg 2307ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 230824f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2309ca6c1d82SSagi Grimberg 2310fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2311ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 2312ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 231324f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2314ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 2315ca6c1d82SSagi Grimberg goto out; 2316ca6c1d82SSagi Grimberg } 231724f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 2318ca6c1d82SSagi Grimberg 2319ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 2320ca6c1d82SSagi Grimberg if (ret) { 232124f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 2322ca6c1d82SSagi Grimberg goto out_id; 2323ca6c1d82SSagi Grimberg } 2324ca6c1d82SSagi Grimberg 2325992607e8SSagi Grimberg ret = rdma_listen(id, 0); 2326ca6c1d82SSagi Grimberg if (ret) { 232724f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 2328ca6c1d82SSagi Grimberg goto out_id; 2329ca6c1d82SSagi Grimberg } 2330ca6c1d82SSagi Grimberg 2331ca6c1d82SSagi Grimberg return id; 2332ca6c1d82SSagi Grimberg out_id: 2333ca6c1d82SSagi Grimberg rdma_destroy_id(id); 2334ca6c1d82SSagi Grimberg out: 2335ca6c1d82SSagi Grimberg return ERR_PTR(ret); 2336ca6c1d82SSagi Grimberg } 2337ca6c1d82SSagi Grimberg 2338b8d26b3bSNicholas Bellinger static int 2339b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 234013a3cf08SAndy Grover struct sockaddr_storage *ksockaddr) 2341b8d26b3bSNicholas Bellinger { 2342b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2343b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2344b8d26b3bSNicholas Bellinger int ret; 2345b8d26b3bSNicholas Bellinger 2346b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 234793b80f29SLeon Romanovsky if (!isert_np) 2348b8d26b3bSNicholas Bellinger return -ENOMEM; 234993b80f29SLeon Romanovsky 2350ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0); 2351ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex); 2352bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted); 2353bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending); 2354ca6c1d82SSagi Grimberg isert_np->np = np; 2355b8d26b3bSNicholas Bellinger 2356b8d26b3bSNicholas Bellinger /* 2357b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 2358b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 2359b8d26b3bSNicholas Bellinger */ 2360b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 236113a3cf08SAndy Grover sizeof(struct sockaddr_storage)); 2362b8d26b3bSNicholas Bellinger 2363ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 2364b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 2365b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 2366b8d26b3bSNicholas Bellinger goto out; 2367b8d26b3bSNicholas Bellinger } 2368b8d26b3bSNicholas Bellinger 2369ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid; 2370b8d26b3bSNicholas Bellinger np->np_context = isert_np; 2371b8d26b3bSNicholas Bellinger 2372b8d26b3bSNicholas Bellinger return 0; 2373b8d26b3bSNicholas Bellinger 2374b8d26b3bSNicholas Bellinger out: 2375b8d26b3bSNicholas Bellinger kfree(isert_np); 2376ca6c1d82SSagi Grimberg 2377b8d26b3bSNicholas Bellinger return ret; 2378b8d26b3bSNicholas Bellinger } 2379b8d26b3bSNicholas Bellinger 2380b8d26b3bSNicholas Bellinger static int 2381b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 2382b8d26b3bSNicholas Bellinger { 2383dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2384b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 2385b8d26b3bSNicholas Bellinger int ret; 238613bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr; 2387b8d26b3bSNicholas Bellinger 2388b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 2389b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 2390b8d26b3bSNicholas Bellinger cp.retry_count = 7; 2391b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 2392b8d26b3bSNicholas Bellinger 239313bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2394422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2395422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv) 2396422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 239713bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr; 239813bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr); 239913bce482SJenny Derzhavetz 2400b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 2401b8d26b3bSNicholas Bellinger if (ret) { 240224f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 2403b8d26b3bSNicholas Bellinger return ret; 2404b8d26b3bSNicholas Bellinger } 2405b8d26b3bSNicholas Bellinger 2406b8d26b3bSNicholas Bellinger return 0; 2407b8d26b3bSNicholas Bellinger } 2408b8d26b3bSNicholas Bellinger 2409b8d26b3bSNicholas Bellinger static int 2410b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2411b8d26b3bSNicholas Bellinger { 24126700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2413b8d26b3bSNicholas Bellinger int ret; 2414b8d26b3bSNicholas Bellinger 241524f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 24162371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 24172371e5daSSagi Grimberg if (ret) { 241824f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 24192371e5daSSagi Grimberg isert_conn); 24202371e5daSSagi Grimberg return ret; 24212371e5daSSagi Grimberg } 24222371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 24232371e5daSSagi Grimberg 24246faaa85fSNicholas Bellinger /* 24256faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 24266faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 24276faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 24286faaa85fSNicholas Bellinger * into a NOP. 24296faaa85fSNicholas Bellinger */ 24306faaa85fSNicholas Bellinger if (!login->first_request) 24316faaa85fSNicholas Bellinger return 0; 2432b8d26b3bSNicholas Bellinger 24332371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 24342371e5daSSagi Grimberg 2435dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn); 2436dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2437b8d26b3bSNicholas Bellinger if (ret) 2438b8d26b3bSNicholas Bellinger return ret; 2439b8d26b3bSNicholas Bellinger 244024f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 24412371e5daSSagi Grimberg 2442b8d26b3bSNicholas Bellinger return 0; 2443b8d26b3bSNicholas Bellinger } 2444b8d26b3bSNicholas Bellinger 2445b8d26b3bSNicholas Bellinger static void 2446b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2447b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 2448b8d26b3bSNicholas Bellinger { 2449dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2450b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 2451b8d26b3bSNicholas Bellinger 2452b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 2453b8d26b3bSNicholas Bellinger 2454dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr; 2455dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr; 2456b8d26b3bSNicholas Bellinger } 2457b8d26b3bSNicholas Bellinger 2458b8d26b3bSNicholas Bellinger static int 2459b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2460b8d26b3bSNicholas Bellinger { 24616700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2462b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2463c6b8e918SSagi Grimberg int ret; 2464b8d26b3bSNicholas Bellinger 2465b8d26b3bSNicholas Bellinger accept_wait: 2466ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem); 2467c6b8e918SSagi Grimberg if (ret) 2468b8d26b3bSNicholas Bellinger return -ENODEV; 2469b8d26b3bSNicholas Bellinger 2470b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 2471e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2472b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 247311378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n", 2474e346ab34SSagi Grimberg np->np_thread_state); 2475e346ab34SSagi Grimberg /** 2476e346ab34SSagi Grimberg * No point in stalling here when np_thread 2477e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 2478e346ab34SSagi Grimberg **/ 2479b8d26b3bSNicholas Bellinger return -ENODEV; 2480b8d26b3bSNicholas Bellinger } 2481b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 2482b8d26b3bSNicholas Bellinger 2483ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2484bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) { 2485ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2486b8d26b3bSNicholas Bellinger goto accept_wait; 2487b8d26b3bSNicholas Bellinger } 2488bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending, 2489bd379220SJenny Derzhavetz struct isert_conn, node); 2490bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 2491ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2492b8d26b3bSNicholas Bellinger 2493b8d26b3bSNicholas Bellinger conn->context = isert_conn; 2494b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 2495aea92980SJenny Derzhavetz isert_conn->state = ISER_CONN_BOUND; 2496b8d26b3bSNicholas Bellinger 2497b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 2498b8d26b3bSNicholas Bellinger 249924f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 25002371e5daSSagi Grimberg 2501b8d26b3bSNicholas Bellinger return 0; 2502b8d26b3bSNicholas Bellinger } 2503b8d26b3bSNicholas Bellinger 2504b8d26b3bSNicholas Bellinger static void 2505b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 2506b8d26b3bSNicholas Bellinger { 25076700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2508268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 2509b8d26b3bSNicholas Bellinger 2510ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id) 2511ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id); 2512b8d26b3bSNicholas Bellinger 2513268e6811SSagi Grimberg /* 2514268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 2515268e6811SSagi Grimberg * that at this point we don't have hanging connections that 2516268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 2517268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 2518bd379220SJenny Derzhavetz * up in accepted and pending lists. 2519268e6811SSagi Grimberg */ 2520ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2521bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) { 2522bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n"); 2523268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 2524bd379220SJenny Derzhavetz &isert_np->pending, 2525bd379220SJenny Derzhavetz node) { 2526bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n", 2527bd379220SJenny Derzhavetz isert_conn, isert_conn->state); 2528bd379220SJenny Derzhavetz isert_connect_release(isert_conn); 2529bd379220SJenny Derzhavetz } 2530bd379220SJenny Derzhavetz } 2531bd379220SJenny Derzhavetz 2532bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) { 2533bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n"); 2534bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n, 2535bd379220SJenny Derzhavetz &isert_np->accepted, 2536bd379220SJenny Derzhavetz node) { 253724f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 2538268e6811SSagi Grimberg isert_conn, isert_conn->state); 2539268e6811SSagi Grimberg isert_connect_release(isert_conn); 2540268e6811SSagi Grimberg } 2541268e6811SSagi Grimberg } 2542ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2543268e6811SSagi Grimberg 2544b8d26b3bSNicholas Bellinger np->np_context = NULL; 2545b8d26b3bSNicholas Bellinger kfree(isert_np); 2546b8d26b3bSNicholas Bellinger } 2547b8d26b3bSNicholas Bellinger 2548b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 2549b02efbfcSSagi Grimberg { 2550b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 2551b02efbfcSSagi Grimberg struct isert_conn, 2552b02efbfcSSagi Grimberg release_work); 2553b02efbfcSSagi Grimberg 255424f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 2555b02efbfcSSagi Grimberg 2556dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2557b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 2558dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2559b02efbfcSSagi Grimberg 256024f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 2561b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 2562b02efbfcSSagi Grimberg } 2563b02efbfcSSagi Grimberg 2564bdf20e72SSagi Grimberg static void 2565991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 2566991bb764SSagi Grimberg { 2567991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 2568991bb764SSagi Grimberg 25694c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 25704c22e07fSSagi Grimberg 2571991bb764SSagi Grimberg if (isert_conn->logout_posted) { 257224f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2573991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 2574991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 2575991bb764SSagi Grimberg } 2576991bb764SSagi Grimberg } 2577991bb764SSagi Grimberg 2578*76261adaSBart Van Assche static void 2579*76261adaSBart Van Assche isert_wait4cmds(struct iscsi_conn *conn) 2580*76261adaSBart Van Assche { 2581*76261adaSBart Van Assche isert_info("iscsi_conn %p\n", conn); 2582*76261adaSBart Van Assche 2583*76261adaSBart Van Assche if (conn->sess) { 2584*76261adaSBart Van Assche target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2585*76261adaSBart Van Assche target_wait_for_sess_cmds(conn->sess->se_sess); 2586*76261adaSBart Van Assche } 2587*76261adaSBart Van Assche } 2588*76261adaSBart Van Assche 25893e03c4b0SJenny Derzhavetz /** 25903e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for 25913e03c4b0SJenny Derzhavetz * unsolicitate dataout 25923e03c4b0SJenny Derzhavetz * @conn: iscsi connection 25933e03c4b0SJenny Derzhavetz * 25943e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited 25953e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those 25963e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds 25973e03c4b0SJenny Derzhavetz */ 25983e03c4b0SJenny Derzhavetz static void 25993e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 26003e03c4b0SJenny Derzhavetz { 26013e03c4b0SJenny Derzhavetz struct iscsi_cmd *cmd, *tmp; 26023e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list); 26033e03c4b0SJenny Derzhavetz 26043e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 26053e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 26063e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 26073e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 26083e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length)) 26093e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 26103e03c4b0SJenny Derzhavetz } 26113e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 26123e03c4b0SJenny Derzhavetz 26133e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 26143e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 26153e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) { 26163e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 26173e03c4b0SJenny Derzhavetz 26183e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd); 26193e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 26203e03c4b0SJenny Derzhavetz } 26213e03c4b0SJenny Derzhavetz } 26223e03c4b0SJenny Derzhavetz } 26233e03c4b0SJenny Derzhavetz 2624defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 2625b8d26b3bSNicholas Bellinger { 2626b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2627b8d26b3bSNicholas Bellinger 26284c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn); 2629b8d26b3bSNicholas Bellinger 2630dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2631954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 2632dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2633b2cb9649SNicholas Bellinger 2634572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 26353e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn); 2636*76261adaSBart Van Assche isert_wait4cmds(conn); 2637991bb764SSagi Grimberg isert_wait4logout(isert_conn); 2638954f2372SSagi Grimberg 2639b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 2640defd8848SNicholas Bellinger } 2641defd8848SNicholas Bellinger 2642defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 2643defd8848SNicholas Bellinger { 2644defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2645b8d26b3bSNicholas Bellinger 2646572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 2647b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 2648b8d26b3bSNicholas Bellinger } 2649b8d26b3bSNicholas Bellinger 2650187e84f7SVarun Prakash static void isert_get_rx_pdu(struct iscsi_conn *conn) 2651187e84f7SVarun Prakash { 2652187e84f7SVarun Prakash struct completion comp; 2653187e84f7SVarun Prakash 2654187e84f7SVarun Prakash init_completion(&comp); 2655187e84f7SVarun Prakash 2656187e84f7SVarun Prakash wait_for_completion_interruptible(&comp); 2657187e84f7SVarun Prakash } 2658187e84f7SVarun Prakash 2659b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 2660b8d26b3bSNicholas Bellinger .name = "IB/iSER", 2661b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 2662bd027d85SNicholas Bellinger .rdma_shutdown = true, 2663d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 2664b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 2665b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 2666b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 2667b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 2668defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 2669b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 2670b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 2671b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 2672b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 2673b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 2674b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 2675b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 2676b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 2677131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 2678187e84f7SVarun Prakash .iscsit_get_rx_pdu = isert_get_rx_pdu, 2679e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2680b8d26b3bSNicholas Bellinger }; 2681b8d26b3bSNicholas Bellinger 2682b8d26b3bSNicholas Bellinger static int __init isert_init(void) 2683b8d26b3bSNicholas Bellinger { 2684b8d26b3bSNicholas Bellinger int ret; 2685b8d26b3bSNicholas Bellinger 2686631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq", 2687631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0); 2688b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 268924f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 26906f0fae3dSSagi Grimberg return -ENOMEM; 2691b8d26b3bSNicholas Bellinger } 2692b8d26b3bSNicholas Bellinger 2693b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2694b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 2695b02efbfcSSagi Grimberg if (!isert_release_wq) { 269624f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 2697b02efbfcSSagi Grimberg ret = -ENOMEM; 2698b02efbfcSSagi Grimberg goto destroy_comp_wq; 2699b02efbfcSSagi Grimberg } 2700b02efbfcSSagi Grimberg 2701b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 270224f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2703b02efbfcSSagi Grimberg 2704b8d26b3bSNicholas Bellinger return 0; 2705b8d26b3bSNicholas Bellinger 2706b02efbfcSSagi Grimberg destroy_comp_wq: 2707b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 27086f0fae3dSSagi Grimberg 2709b8d26b3bSNicholas Bellinger return ret; 2710b8d26b3bSNicholas Bellinger } 2711b8d26b3bSNicholas Bellinger 2712b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 2713b8d26b3bSNicholas Bellinger { 2714f5ebec96SSagi Grimberg flush_scheduled_work(); 2715b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 2716b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 2717b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 27184c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2719b8d26b3bSNicholas Bellinger } 2720b8d26b3bSNicholas Bellinger 2721b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2722b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2723b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 2724b8d26b3bSNicholas Bellinger 2725b8d26b3bSNicholas Bellinger module_init(isert_init); 2726b8d26b3bSNicholas Bellinger module_exit(isert_exit); 2727