1b8d26b3bSNicholas Bellinger /******************************************************************************* 2b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs 3b8d26b3bSNicholas Bellinger * 44c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc. 5b8d26b3bSNicholas Bellinger * 6b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org> 7b8d26b3bSNicholas Bellinger * 8b8d26b3bSNicholas Bellinger * This program is free software; you can redistribute it and/or modify 9b8d26b3bSNicholas Bellinger * it under the terms of the GNU General Public License as published by 10b8d26b3bSNicholas Bellinger * the Free Software Foundation; either version 2 of the License, or 11b8d26b3bSNicholas Bellinger * (at your option) any later version. 12b8d26b3bSNicholas Bellinger * 13b8d26b3bSNicholas Bellinger * This program is distributed in the hope that it will be useful, 14b8d26b3bSNicholas Bellinger * but WITHOUT ANY WARRANTY; without even the implied warranty of 15b8d26b3bSNicholas Bellinger * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16b8d26b3bSNicholas Bellinger * GNU General Public License for more details. 17b8d26b3bSNicholas Bellinger ****************************************************************************/ 18b8d26b3bSNicholas Bellinger 19b8d26b3bSNicholas Bellinger #include <linux/string.h> 20b8d26b3bSNicholas Bellinger #include <linux/module.h> 21b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h> 22b8d26b3bSNicholas Bellinger #include <linux/socket.h> 23b8d26b3bSNicholas Bellinger #include <linux/in.h> 24b8d26b3bSNicholas Bellinger #include <linux/in6.h> 25b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h> 26b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h> 27b8d26b3bSNicholas Bellinger #include <target/target_core_base.h> 28b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h> 29b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h> 30531b7bf4SSagi Grimberg #include <linux/semaphore.h> 31b8d26b3bSNicholas Bellinger 32b8d26b3bSNicholas Bellinger #include "ib_isert.h" 33b8d26b3bSNicholas Bellinger 34b8d26b3bSNicholas Bellinger #define ISERT_MAX_CONN 8 35b8d26b3bSNicholas Bellinger #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 3638a2d0d4SChristoph Hellwig #define ISER_MAX_TX_CQ_LEN \ 3738a2d0d4SChristoph Hellwig ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) 38bdf20e72SSagi Grimberg #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39bdf20e72SSagi Grimberg ISERT_MAX_CONN) 40b8d26b3bSNicholas Bellinger 4145678b6bSSagi Grimberg static int isert_debug_level; 4224f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644); 4324f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 4424f412ddSSagi Grimberg 45b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex); 46b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list); 47b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq; 48b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq; 49b8d26b3bSNicholas Bellinger 50f93f3a70SSagi Grimberg static int 51f93f3a70SSagi Grimberg isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 522371e5daSSagi Grimberg static int 539679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn); 542371e5daSSagi Grimberg static int 552371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn); 56ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 57d40945d8SVu Pham 582f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work); 599679cc51SChristoph Hellwig static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 609679cc51SChristoph Hellwig static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 619679cc51SChristoph Hellwig static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 629679cc51SChristoph Hellwig static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 632f1b6b7dSSagi Grimberg 64302cc7c3SSagi Grimberg static inline bool 65302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 66302cc7c3SSagi Grimberg { 6723a548eeSSagi Grimberg return (conn->pi_support && 68302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL); 69302cc7c3SSagi Grimberg } 70302cc7c3SSagi Grimberg 71302cc7c3SSagi Grimberg 72d40945d8SVu Pham static void 73b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context) 74b8d26b3bSNicholas Bellinger { 756700425eSSagi Grimberg struct isert_conn *isert_conn = context; 76b8d26b3bSNicholas Bellinger 77ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n", 78ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn); 79ea8a1616SSagi Grimberg 80b8d26b3bSNicholas Bellinger switch (e->event) { 81b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST: 82dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 83b8d26b3bSNicholas Bellinger break; 84b8d26b3bSNicholas Bellinger case IB_EVENT_QP_LAST_WQE_REACHED: 854c22e07fSSagi Grimberg isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 86b8d26b3bSNicholas Bellinger break; 87b8d26b3bSNicholas Bellinger default: 88b8d26b3bSNicholas Bellinger break; 89b8d26b3bSNicholas Bellinger } 90b8d26b3bSNicholas Bellinger } 91b8d26b3bSNicholas Bellinger 9240fc069aSSagi Grimberg static struct isert_comp * 9340fc069aSSagi Grimberg isert_comp_get(struct isert_conn *isert_conn) 94b8d26b3bSNicholas Bellinger { 95dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 964a295baeSSagi Grimberg struct isert_comp *comp; 9740fc069aSSagi Grimberg int i, min = 0; 98b8d26b3bSNicholas Bellinger 99b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 1004a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) 1014a295baeSSagi Grimberg if (device->comps[i].active_qps < 1024a295baeSSagi Grimberg device->comps[min].active_qps) 1034a295baeSSagi Grimberg min = i; 1044a295baeSSagi Grimberg comp = &device->comps[min]; 1054a295baeSSagi Grimberg comp->active_qps++; 10640fc069aSSagi Grimberg mutex_unlock(&device_list_mutex); 10740fc069aSSagi Grimberg 10824f412ddSSagi Grimberg isert_info("conn %p, using comp %p min_index: %d\n", 1094a295baeSSagi Grimberg isert_conn, comp, min); 11040fc069aSSagi Grimberg 11140fc069aSSagi Grimberg return comp; 11240fc069aSSagi Grimberg } 11340fc069aSSagi Grimberg 11440fc069aSSagi Grimberg static void 11540fc069aSSagi Grimberg isert_comp_put(struct isert_comp *comp) 11640fc069aSSagi Grimberg { 11740fc069aSSagi Grimberg mutex_lock(&device_list_mutex); 11840fc069aSSagi Grimberg comp->active_qps--; 119b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 12040fc069aSSagi Grimberg } 12140fc069aSSagi Grimberg 12240fc069aSSagi Grimberg static struct ib_qp * 12340fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn, 12440fc069aSSagi Grimberg struct isert_comp *comp, 12540fc069aSSagi Grimberg struct rdma_cm_id *cma_id) 12640fc069aSSagi Grimberg { 127dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 12840fc069aSSagi Grimberg struct ib_qp_init_attr attr; 12940fc069aSSagi Grimberg int ret; 130b8d26b3bSNicholas Bellinger 131b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 132b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback; 133b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn; 1346f0fae3dSSagi Grimberg attr.send_cq = comp->cq; 1356f0fae3dSSagi Grimberg attr.recv_cq = comp->cq; 136572a1434SSagi Grimberg attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 137bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 13838a2d0d4SChristoph Hellwig attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX; 1394a061b28SOr Gerlitz attr.cap.max_send_sge = device->ib_device->attrs.max_sge; 140b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1; 141b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR; 142b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC; 143570db170SSagi Grimberg if (device->pi_capable) 144d3e125daSSagi Grimberg attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 145b8d26b3bSNicholas Bellinger 14667cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr); 147b8d26b3bSNicholas Bellinger if (ret) { 14824f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret); 14940fc069aSSagi Grimberg return ERR_PTR(ret); 15040fc069aSSagi Grimberg } 15140fc069aSSagi Grimberg 15240fc069aSSagi Grimberg return cma_id->qp; 15340fc069aSSagi Grimberg } 15440fc069aSSagi Grimberg 15540fc069aSSagi Grimberg static int 15640fc069aSSagi Grimberg isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 15740fc069aSSagi Grimberg { 15840fc069aSSagi Grimberg struct isert_comp *comp; 15940fc069aSSagi Grimberg int ret; 16040fc069aSSagi Grimberg 16140fc069aSSagi Grimberg comp = isert_comp_get(isert_conn); 162dac6ab30SSagi Grimberg isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 163dac6ab30SSagi Grimberg if (IS_ERR(isert_conn->qp)) { 164dac6ab30SSagi Grimberg ret = PTR_ERR(isert_conn->qp); 16519e2090fSSagi Grimberg goto err; 166b8d26b3bSNicholas Bellinger } 167b8d26b3bSNicholas Bellinger 168b8d26b3bSNicholas Bellinger return 0; 16919e2090fSSagi Grimberg err: 17040fc069aSSagi Grimberg isert_comp_put(comp); 17119e2090fSSagi Grimberg return ret; 172b8d26b3bSNicholas Bellinger } 173b8d26b3bSNicholas Bellinger 174b8d26b3bSNicholas Bellinger static int 175b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 176b8d26b3bSNicholas Bellinger { 177dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 17867cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 179b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 180b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg; 181b8d26b3bSNicholas Bellinger u64 dma_addr; 182b8d26b3bSNicholas Bellinger int i, j; 183b8d26b3bSNicholas Bellinger 184dac6ab30SSagi Grimberg isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 185b8d26b3bSNicholas Bellinger sizeof(struct iser_rx_desc), GFP_KERNEL); 186dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 18793b80f29SLeon Romanovsky return -ENOMEM; 188b8d26b3bSNicholas Bellinger 189dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 190b8d26b3bSNicholas Bellinger 191b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 192b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 193b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 194b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) 195b8d26b3bSNicholas Bellinger goto dma_map_fail; 196b8d26b3bSNicholas Bellinger 197b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr; 198b8d26b3bSNicholas Bellinger 199b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg; 200b8d26b3bSNicholas Bellinger rx_sg->addr = rx_desc->dma_addr; 201b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE; 20234efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey; 2039679cc51SChristoph Hellwig rx_desc->rx_cqe.done = isert_recv_done; 204b8d26b3bSNicholas Bellinger } 205b8d26b3bSNicholas Bellinger 206b8d26b3bSNicholas Bellinger return 0; 207b8d26b3bSNicholas Bellinger 208b8d26b3bSNicholas Bellinger dma_map_fail: 209dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 210b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) { 211b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 212b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 213b8d26b3bSNicholas Bellinger } 214dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 215dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 2164c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 217b8d26b3bSNicholas Bellinger return -ENOMEM; 218b8d26b3bSNicholas Bellinger } 219b8d26b3bSNicholas Bellinger 220b8d26b3bSNicholas Bellinger static void 221b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn) 222b8d26b3bSNicholas Bellinger { 223dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 224b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 225b8d26b3bSNicholas Bellinger int i; 226b8d26b3bSNicholas Bellinger 227dac6ab30SSagi Grimberg if (!isert_conn->rx_descs) 228b8d26b3bSNicholas Bellinger return; 229b8d26b3bSNicholas Bellinger 230dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs; 231b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 232b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 233b8d26b3bSNicholas Bellinger ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 234b8d26b3bSNicholas Bellinger } 235b8d26b3bSNicholas Bellinger 236dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs); 237dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL; 238b8d26b3bSNicholas Bellinger } 239b8d26b3bSNicholas Bellinger 240172369c5SSagi Grimberg static void 241172369c5SSagi Grimberg isert_free_comps(struct isert_device *device) 242b8d26b3bSNicholas Bellinger { 243172369c5SSagi Grimberg int i; 244b8d26b3bSNicholas Bellinger 245172369c5SSagi Grimberg for (i = 0; i < device->comps_used; i++) { 246172369c5SSagi Grimberg struct isert_comp *comp = &device->comps[i]; 24759464ef4SVu Pham 2489679cc51SChristoph Hellwig if (comp->cq) 2499679cc51SChristoph Hellwig ib_free_cq(comp->cq); 250172369c5SSagi Grimberg } 251172369c5SSagi Grimberg kfree(device->comps); 25259464ef4SVu Pham } 253d40945d8SVu Pham 254172369c5SSagi Grimberg static int 2554a061b28SOr Gerlitz isert_alloc_comps(struct isert_device *device) 256172369c5SSagi Grimberg { 257172369c5SSagi Grimberg int i, max_cqe, ret = 0; 258d3e125daSSagi Grimberg 2594a295baeSSagi Grimberg device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 2604a295baeSSagi Grimberg device->ib_device->num_comp_vectors)); 261172369c5SSagi Grimberg 26224f412ddSSagi Grimberg isert_info("Using %d CQs, %s supports %d vectors support " 26338a2d0d4SChristoph Hellwig "pi_capable %d\n", 2644a295baeSSagi Grimberg device->comps_used, device->ib_device->name, 26538a2d0d4SChristoph Hellwig device->ib_device->num_comp_vectors, 266d3e125daSSagi Grimberg device->pi_capable); 2674a295baeSSagi Grimberg 2684a295baeSSagi Grimberg device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 2694a295baeSSagi Grimberg GFP_KERNEL); 27093b80f29SLeon Romanovsky if (!device->comps) 271b8d26b3bSNicholas Bellinger return -ENOMEM; 272b8d26b3bSNicholas Bellinger 2734a061b28SOr Gerlitz max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 274172369c5SSagi Grimberg 2754a295baeSSagi Grimberg for (i = 0; i < device->comps_used; i++) { 2764a295baeSSagi Grimberg struct isert_comp *comp = &device->comps[i]; 277b8d26b3bSNicholas Bellinger 2784a295baeSSagi Grimberg comp->device = device; 2799679cc51SChristoph Hellwig comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 2809679cc51SChristoph Hellwig IB_POLL_WORKQUEUE); 2816f0fae3dSSagi Grimberg if (IS_ERR(comp->cq)) { 282172369c5SSagi Grimberg isert_err("Unable to allocate cq\n"); 2836f0fae3dSSagi Grimberg ret = PTR_ERR(comp->cq); 2846f0fae3dSSagi Grimberg comp->cq = NULL; 285b8d26b3bSNicholas Bellinger goto out_cq; 28694a71110SWei Yongjun } 287b8d26b3bSNicholas Bellinger } 288b8d26b3bSNicholas Bellinger 289172369c5SSagi Grimberg return 0; 290172369c5SSagi Grimberg out_cq: 291172369c5SSagi Grimberg isert_free_comps(device); 292172369c5SSagi Grimberg return ret; 293172369c5SSagi Grimberg } 294172369c5SSagi Grimberg 295172369c5SSagi Grimberg static int 296172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device) 297172369c5SSagi Grimberg { 2984a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device; 299fd8205e8SSagi Grimberg int ret; 300172369c5SSagi Grimberg 3014a061b28SOr Gerlitz isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge); 3024a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 303172369c5SSagi Grimberg 3044a061b28SOr Gerlitz ret = isert_alloc_comps(device); 305172369c5SSagi Grimberg if (ret) 306373a4cd7SNicholas Bellinger goto out; 307172369c5SSagi Grimberg 308ed082d36SChristoph Hellwig device->pd = ib_alloc_pd(ib_dev, 0); 30967cb3949SSagi Grimberg if (IS_ERR(device->pd)) { 31067cb3949SSagi Grimberg ret = PTR_ERR(device->pd); 31167cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n", 31267cb3949SSagi Grimberg device, ret); 31367cb3949SSagi Grimberg goto out_cq; 31467cb3949SSagi Grimberg } 31567cb3949SSagi Grimberg 316172369c5SSagi Grimberg /* Check signature cap */ 3174a061b28SOr Gerlitz device->pi_capable = ib_dev->attrs.device_cap_flags & 318172369c5SSagi Grimberg IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 31967cb3949SSagi Grimberg 320b8d26b3bSNicholas Bellinger return 0; 321b8d26b3bSNicholas Bellinger 322b8d26b3bSNicholas Bellinger out_cq: 323172369c5SSagi Grimberg isert_free_comps(device); 324373a4cd7SNicholas Bellinger out: 325373a4cd7SNicholas Bellinger if (ret > 0) 326373a4cd7SNicholas Bellinger ret = -EINVAL; 327b8d26b3bSNicholas Bellinger return ret; 328b8d26b3bSNicholas Bellinger } 329b8d26b3bSNicholas Bellinger 330b8d26b3bSNicholas Bellinger static void 331b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device) 332b8d26b3bSNicholas Bellinger { 33324f412ddSSagi Grimberg isert_info("device %p\n", device); 334b8d26b3bSNicholas Bellinger 33567cb3949SSagi Grimberg ib_dealloc_pd(device->pd); 336172369c5SSagi Grimberg isert_free_comps(device); 337b8d26b3bSNicholas Bellinger } 338b8d26b3bSNicholas Bellinger 339b8d26b3bSNicholas Bellinger static void 340cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device) 341b8d26b3bSNicholas Bellinger { 342b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 343b8d26b3bSNicholas Bellinger device->refcount--; 3444c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount); 345b8d26b3bSNicholas Bellinger if (!device->refcount) { 346b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device); 347b8d26b3bSNicholas Bellinger list_del(&device->dev_node); 348b8d26b3bSNicholas Bellinger kfree(device); 349b8d26b3bSNicholas Bellinger } 350b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 351b8d26b3bSNicholas Bellinger } 352b8d26b3bSNicholas Bellinger 353b8d26b3bSNicholas Bellinger static struct isert_device * 354cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id) 355b8d26b3bSNicholas Bellinger { 356b8d26b3bSNicholas Bellinger struct isert_device *device; 357b8d26b3bSNicholas Bellinger int ret; 358b8d26b3bSNicholas Bellinger 359b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex); 360b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) { 361b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) { 362b8d26b3bSNicholas Bellinger device->refcount++; 3634c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n", 3644c22e07fSSagi Grimberg device, device->refcount); 365b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 366b8d26b3bSNicholas Bellinger return device; 367b8d26b3bSNicholas Bellinger } 368b8d26b3bSNicholas Bellinger } 369b8d26b3bSNicholas Bellinger 370b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 371b8d26b3bSNicholas Bellinger if (!device) { 372b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 373b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM); 374b8d26b3bSNicholas Bellinger } 375b8d26b3bSNicholas Bellinger 376b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node); 377b8d26b3bSNicholas Bellinger 378b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device; 379b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device); 380b8d26b3bSNicholas Bellinger if (ret) { 381b8d26b3bSNicholas Bellinger kfree(device); 382b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 383b8d26b3bSNicholas Bellinger return ERR_PTR(ret); 384b8d26b3bSNicholas Bellinger } 385b8d26b3bSNicholas Bellinger 386b8d26b3bSNicholas Bellinger device->refcount++; 387b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list); 3884c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n", 3894c22e07fSSagi Grimberg device, device->refcount); 390b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex); 391b8d26b3bSNicholas Bellinger 392b8d26b3bSNicholas Bellinger return device; 393b8d26b3bSNicholas Bellinger } 394b8d26b3bSNicholas Bellinger 39559464ef4SVu Pham static void 396ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn) 397ae9ea9edSSagi Grimberg { 398ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT; 399bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node); 400dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp); 401ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp); 40263b268d2SRaju Rangoju init_waitqueue_head(&isert_conn->rem_wait); 403dac6ab30SSagi Grimberg kref_init(&isert_conn->kref); 404dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex); 4052f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work); 406ae9ea9edSSagi Grimberg } 407ae9ea9edSSagi Grimberg 408ae9ea9edSSagi Grimberg static void 409ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn) 410ae9ea9edSSagi Grimberg { 411dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device; 412ae9ea9edSSagi Grimberg 413ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 414ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 4155adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4165adabdd1SChristoph Hellwig 417ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4185adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, 419ae9ea9edSSagi Grimberg DMA_FROM_DEVICE); 4205adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 421ae9ea9edSSagi Grimberg } 422ae9ea9edSSagi Grimberg 423ae9ea9edSSagi Grimberg static int 424ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn, 425ae9ea9edSSagi Grimberg struct ib_device *ib_dev) 426ae9ea9edSSagi Grimberg { 427ae9ea9edSSagi Grimberg int ret; 428ae9ea9edSSagi Grimberg 4295adabdd1SChristoph Hellwig isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 4305adabdd1SChristoph Hellwig GFP_KERNEL); 43193b80f29SLeon Romanovsky if (!isert_conn->login_req_buf) 432ae9ea9edSSagi Grimberg return -ENOMEM; 433ae9ea9edSSagi Grimberg 434ae9ea9edSSagi Grimberg isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 4355adabdd1SChristoph Hellwig isert_conn->login_req_buf, 4365adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 437ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 438ae9ea9edSSagi Grimberg if (ret) { 439ae9ea9edSSagi Grimberg isert_err("login_req_dma mapping error: %d\n", ret); 440ae9ea9edSSagi Grimberg isert_conn->login_req_dma = 0; 4415adabdd1SChristoph Hellwig goto out_free_login_req_buf; 4425adabdd1SChristoph Hellwig } 4435adabdd1SChristoph Hellwig 4445adabdd1SChristoph Hellwig isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 4455adabdd1SChristoph Hellwig if (!isert_conn->login_rsp_buf) { 4461d5840c9SWei Yongjun ret = -ENOMEM; 4475adabdd1SChristoph Hellwig goto out_unmap_login_req_buf; 448ae9ea9edSSagi Grimberg } 449ae9ea9edSSagi Grimberg 450ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 4515adabdd1SChristoph Hellwig isert_conn->login_rsp_buf, 452ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 453ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 454ae9ea9edSSagi Grimberg if (ret) { 455ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret); 456ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0; 4575adabdd1SChristoph Hellwig goto out_free_login_rsp_buf; 458ae9ea9edSSagi Grimberg } 459ae9ea9edSSagi Grimberg 460ae9ea9edSSagi Grimberg return 0; 461ae9ea9edSSagi Grimberg 4625adabdd1SChristoph Hellwig out_free_login_rsp_buf: 4635adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf); 4645adabdd1SChristoph Hellwig out_unmap_login_req_buf: 465ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 4665adabdd1SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 4675adabdd1SChristoph Hellwig out_free_login_req_buf: 4685adabdd1SChristoph Hellwig kfree(isert_conn->login_req_buf); 469ae9ea9edSSagi Grimberg return ret; 470ae9ea9edSSagi Grimberg } 471ae9ea9edSSagi Grimberg 472422bd0acSJenny Derzhavetz static void 473422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn, 474422bd0acSJenny Derzhavetz struct rdma_conn_param *param) 475422bd0acSJenny Derzhavetz { 476422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 477422bd0acSJenny Derzhavetz 478422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */ 479422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 480422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom); 481422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 482422bd0acSJenny Derzhavetz 483422bd0acSJenny Derzhavetz if (param->private_data) { 484422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data; 485422bd0acSJenny Derzhavetz 486422bd0acSJenny Derzhavetz /* 487422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator 488422bd0acSJenny Derzhavetz * and the HCA support it 489422bd0acSJenny Derzhavetz */ 490422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 491422bd0acSJenny Derzhavetz (attr->device_cap_flags & 492422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS); 493422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv) 494422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n"); 495422bd0acSJenny Derzhavetz } 496422bd0acSJenny Derzhavetz } 497422bd0acSJenny Derzhavetz 498b8d26b3bSNicholas Bellinger static int 499b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 500b8d26b3bSNicholas Bellinger { 501ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context; 502ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 503b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 504b8d26b3bSNicholas Bellinger struct isert_device *device; 505b8d26b3bSNicholas Bellinger int ret = 0; 50614f4b54fSSagi Grimberg 50714f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock); 50814f4b54fSSagi Grimberg if (!np->enabled) { 50914f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 51024f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n"); 51114f4b54fSSagi Grimberg return rdma_reject(cma_id, NULL, 0); 51214f4b54fSSagi Grimberg } 51314f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock); 514b8d26b3bSNicholas Bellinger 5154c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n", 516b8d26b3bSNicholas Bellinger cma_id, cma_id->context); 517b8d26b3bSNicholas Bellinger 518b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 519ae9ea9edSSagi Grimberg if (!isert_conn) 520b8d26b3bSNicholas Bellinger return -ENOMEM; 521b8d26b3bSNicholas Bellinger 522ae9ea9edSSagi Grimberg isert_init_conn(isert_conn); 523dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id; 524b8d26b3bSNicholas Bellinger 525ae9ea9edSSagi Grimberg ret = isert_alloc_login_buf(isert_conn, cma_id->device); 526ae9ea9edSSagi Grimberg if (ret) 527b8d26b3bSNicholas Bellinger goto out; 528b8d26b3bSNicholas Bellinger 529cf8ae958SSagi Grimberg device = isert_device_get(cma_id); 530b8d26b3bSNicholas Bellinger if (IS_ERR(device)) { 531b8d26b3bSNicholas Bellinger ret = PTR_ERR(device); 532b8d26b3bSNicholas Bellinger goto out_rsp_dma_map; 533b8d26b3bSNicholas Bellinger } 534dac6ab30SSagi Grimberg isert_conn->device = device; 535b8d26b3bSNicholas Bellinger 536422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn); 5371a92e17eSSagi Grimberg 538570db170SSagi Grimberg ret = isert_conn_setup_qp(isert_conn, cma_id); 539b8d26b3bSNicholas Bellinger if (ret) 540b8d26b3bSNicholas Bellinger goto out_conn_dev; 541b8d26b3bSNicholas Bellinger 5429679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 5432371e5daSSagi Grimberg if (ret) 5442371e5daSSagi Grimberg goto out_conn_dev; 5452371e5daSSagi Grimberg 5462371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn); 5472371e5daSSagi Grimberg if (ret) 5482371e5daSSagi Grimberg goto out_conn_dev; 5492371e5daSSagi Grimberg 550ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 551bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted); 552ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 553b8d26b3bSNicholas Bellinger 554b8d26b3bSNicholas Bellinger return 0; 555b8d26b3bSNicholas Bellinger 556b8d26b3bSNicholas Bellinger out_conn_dev: 557cf8ae958SSagi Grimberg isert_device_put(device); 558b8d26b3bSNicholas Bellinger out_rsp_dma_map: 559ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 560b8d26b3bSNicholas Bellinger out: 561b8d26b3bSNicholas Bellinger kfree(isert_conn); 5622371e5daSSagi Grimberg rdma_reject(cma_id, NULL, 0); 563b8d26b3bSNicholas Bellinger return ret; 564b8d26b3bSNicholas Bellinger } 565b8d26b3bSNicholas Bellinger 566b8d26b3bSNicholas Bellinger static void 567b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn) 568b8d26b3bSNicholas Bellinger { 569dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 570b8d26b3bSNicholas Bellinger 5714c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn); 572b8d26b3bSNicholas Bellinger 57357df81e3SSagi Grimberg BUG_ON(!device); 57457df81e3SSagi Grimberg 57519e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn); 57663b268d2SRaju Rangoju if (isert_conn->cm_id && 57763b268d2SRaju Rangoju !isert_conn->dev_removed) 578dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id); 57919e2090fSSagi Grimberg 580dac6ab30SSagi Grimberg if (isert_conn->qp) { 581dac6ab30SSagi Grimberg struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 5824a295baeSSagi Grimberg 58340fc069aSSagi Grimberg isert_comp_put(comp); 584dac6ab30SSagi Grimberg ib_destroy_qp(isert_conn->qp); 585b8d26b3bSNicholas Bellinger } 586b8d26b3bSNicholas Bellinger 5875adabdd1SChristoph Hellwig if (isert_conn->login_req_buf) 588ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn); 589ae9ea9edSSagi Grimberg 590cf8ae958SSagi Grimberg isert_device_put(device); 59157df81e3SSagi Grimberg 59263b268d2SRaju Rangoju if (isert_conn->dev_removed) 59363b268d2SRaju Rangoju wake_up_interruptible(&isert_conn->rem_wait); 59463b268d2SRaju Rangoju else 59557df81e3SSagi Grimberg kfree(isert_conn); 596b8d26b3bSNicholas Bellinger } 597b8d26b3bSNicholas Bellinger 598b8d26b3bSNicholas Bellinger static void 599b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id) 600b8d26b3bSNicholas Bellinger { 60119e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 602bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 603c2f88b17SSagi Grimberg 60424f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 605128e9cc8SSagi Grimberg 606dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 607128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP; 608bd379220SJenny Derzhavetz kref_get(&isert_conn->kref); 609dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 610bd379220SJenny Derzhavetz 611bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex); 612bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending); 613bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 614bd379220SJenny Derzhavetz 615bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np); 616bd379220SJenny Derzhavetz up(&isert_np->sem); 617b8d26b3bSNicholas Bellinger } 618b8d26b3bSNicholas Bellinger 619b8d26b3bSNicholas Bellinger static void 620dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref) 621b8d26b3bSNicholas Bellinger { 622b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref, 623dac6ab30SSagi Grimberg struct isert_conn, kref); 624b8d26b3bSNicholas Bellinger 6254c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 6264c22e07fSSagi Grimberg current->pid); 627b8d26b3bSNicholas Bellinger 628b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn); 629b8d26b3bSNicholas Bellinger } 630b8d26b3bSNicholas Bellinger 631b8d26b3bSNicholas Bellinger static void 632b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn) 633b8d26b3bSNicholas Bellinger { 634dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref); 635b8d26b3bSNicholas Bellinger } 636b8d26b3bSNicholas Bellinger 6376d1fba0cSJenny Derzhavetz static void 6386d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(struct isert_conn *isert_conn) 6396d1fba0cSJenny Derzhavetz { 6406d1fba0cSJenny Derzhavetz struct isert_np *isert_np = isert_conn->cm_id->context; 6416d1fba0cSJenny Derzhavetz 6426d1fba0cSJenny Derzhavetz mutex_lock(&isert_np->mutex); 6436d1fba0cSJenny Derzhavetz if (!list_empty(&isert_conn->node)) { 6446d1fba0cSJenny Derzhavetz /* 6456d1fba0cSJenny Derzhavetz * This means iscsi doesn't know this connection 6466d1fba0cSJenny Derzhavetz * so schedule a cleanup ourselves 6476d1fba0cSJenny Derzhavetz */ 6486d1fba0cSJenny Derzhavetz list_del_init(&isert_conn->node); 6496d1fba0cSJenny Derzhavetz isert_put_conn(isert_conn); 6506d1fba0cSJenny Derzhavetz queue_work(isert_release_wq, &isert_conn->release_work); 6516d1fba0cSJenny Derzhavetz } 6526d1fba0cSJenny Derzhavetz mutex_unlock(&isert_np->mutex); 6536d1fba0cSJenny Derzhavetz } 6546d1fba0cSJenny Derzhavetz 655954f2372SSagi Grimberg /** 656954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination 657954f2372SSagi Grimberg * @isert_conn: isert connection struct 658954f2372SSagi Grimberg * 659954f2372SSagi Grimberg * Notes: 660aea92980SJenny Derzhavetz * In case the connection state is BOUND, move state 661954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect). 662128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well. 663954f2372SSagi Grimberg * 664dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is 665954f2372SSagi Grimberg * safe to call multiple times. 666954f2372SSagi Grimberg */ 667954f2372SSagi Grimberg static void 668954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn) 669954f2372SSagi Grimberg { 670954f2372SSagi Grimberg int err; 671954f2372SSagi Grimberg 6726d1fba0cSJenny Derzhavetz if (isert_conn->state >= ISER_CONN_TERMINATING) 6736d1fba0cSJenny Derzhavetz return; 6746d1fba0cSJenny Derzhavetz 67524f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n", 676954f2372SSagi Grimberg isert_conn, isert_conn->state); 677128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING; 678dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id); 679954f2372SSagi Grimberg if (err) 68024f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n", 681954f2372SSagi Grimberg isert_conn); 682954f2372SSagi Grimberg } 683954f2372SSagi Grimberg 6843b726ae2SSagi Grimberg static int 685ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np, 686ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 687b8d26b3bSNicholas Bellinger { 688ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n", 689ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np); 6903b726ae2SSagi Grimberg 691ca6c1d82SSagi Grimberg switch (event) { 692ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL: 693ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 694ca6c1d82SSagi Grimberg break; 695ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: 696ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np); 697ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) { 69824f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n", 699ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id)); 700ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL; 701ca6c1d82SSagi Grimberg } 702ca6c1d82SSagi Grimberg break; 703ca6c1d82SSagi Grimberg default: 70424f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n", 705ca6c1d82SSagi Grimberg isert_np, event); 706ca6c1d82SSagi Grimberg } 707ca6c1d82SSagi Grimberg 7083b726ae2SSagi Grimberg return -1; 7093b726ae2SSagi Grimberg } 7103b726ae2SSagi Grimberg 711ca6c1d82SSagi Grimberg static int 712ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id, 713ca6c1d82SSagi Grimberg enum rdma_cm_event_type event) 714ca6c1d82SSagi Grimberg { 715f81bf458SJenny Derzhavetz struct isert_conn *isert_conn = cma_id->qp->qp_context; 716ca6c1d82SSagi Grimberg 717dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 7186d1fba0cSJenny Derzhavetz switch (isert_conn->state) { 7196d1fba0cSJenny Derzhavetz case ISER_CONN_TERMINATING: 7206d1fba0cSJenny Derzhavetz break; 7216d1fba0cSJenny Derzhavetz case ISER_CONN_UP: 722128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn); 723572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 7246d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(isert_conn); 7256d1fba0cSJenny Derzhavetz break; 7266d1fba0cSJenny Derzhavetz case ISER_CONN_BOUND: 7276d1fba0cSJenny Derzhavetz case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 7286d1fba0cSJenny Derzhavetz iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 7296d1fba0cSJenny Derzhavetz break; 7306d1fba0cSJenny Derzhavetz default: 731506f71d1SColin Ian King isert_warn("conn %p terminating in state %d\n", 7326d1fba0cSJenny Derzhavetz isert_conn, isert_conn->state); 7336d1fba0cSJenny Derzhavetz } 734dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 735128e9cc8SSagi Grimberg 7363b726ae2SSagi Grimberg return 0; 737b8d26b3bSNicholas Bellinger } 738b8d26b3bSNicholas Bellinger 7394a579da2SSagi Grimberg static int 740954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id) 741954f2372SSagi Grimberg { 74219e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context; 743954f2372SSagi Grimberg 744bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 745dac6ab30SSagi Grimberg isert_conn->cm_id = NULL; 746954f2372SSagi Grimberg isert_put_conn(isert_conn); 7474a579da2SSagi Grimberg 7484a579da2SSagi Grimberg return -1; 749954f2372SSagi Grimberg } 750954f2372SSagi Grimberg 751b8d26b3bSNicholas Bellinger static int 752b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 753b8d26b3bSNicholas Bellinger { 754f81bf458SJenny Derzhavetz struct isert_np *isert_np = cma_id->context; 75563b268d2SRaju Rangoju struct isert_conn *isert_conn; 756b8d26b3bSNicholas Bellinger int ret = 0; 757b8d26b3bSNicholas Bellinger 758ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n", 759ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event, 7604c22e07fSSagi Grimberg event->status, cma_id, cma_id->context); 761b8d26b3bSNicholas Bellinger 762f81bf458SJenny Derzhavetz if (isert_np->cm_id == cma_id) 763f81bf458SJenny Derzhavetz return isert_np_cma_handler(cma_id->context, event->event); 764f81bf458SJenny Derzhavetz 765b8d26b3bSNicholas Bellinger switch (event->event) { 766b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST: 767b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event); 7683b726ae2SSagi Grimberg if (ret) 7694c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret); 770b8d26b3bSNicholas Bellinger break; 771b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED: 772b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id); 773b8d26b3bSNicholas Bellinger break; 77488c4015fSSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 77588c4015fSSagi Grimberg case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 77688c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 777ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event); 778b8d26b3bSNicholas Bellinger break; 77963b268d2SRaju Rangoju case RDMA_CM_EVENT_DEVICE_REMOVAL: 78063b268d2SRaju Rangoju isert_conn = cma_id->qp->qp_context; 78163b268d2SRaju Rangoju isert_conn->dev_removed = true; 78263b268d2SRaju Rangoju isert_disconnected_handler(cma_id, event->event); 78363b268d2SRaju Rangoju wait_event_interruptible(isert_conn->rem_wait, 78463b268d2SRaju Rangoju isert_conn->state == ISER_CONN_DOWN); 78563b268d2SRaju Rangoju kfree(isert_conn); 78663b268d2SRaju Rangoju /* 78763b268d2SRaju Rangoju * return non-zero from the callback to destroy 78863b268d2SRaju Rangoju * the rdma cm id 78963b268d2SRaju Rangoju */ 79063b268d2SRaju Rangoju return 1; 791954f2372SSagi Grimberg case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 7921e38a366SSteve Wise isert_info("Connection rejected: %s\n", 7931e38a366SSteve Wise rdma_reject_msg(cma_id, event->status)); 794954f2372SSagi Grimberg case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 795b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR: 7964a579da2SSagi Grimberg ret = isert_connect_error(cma_id); 797954f2372SSagi Grimberg break; 798b8d26b3bSNicholas Bellinger default: 79924f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event); 800b8d26b3bSNicholas Bellinger break; 801b8d26b3bSNicholas Bellinger } 802b8d26b3bSNicholas Bellinger 803b8d26b3bSNicholas Bellinger return ret; 804b8d26b3bSNicholas Bellinger } 805b8d26b3bSNicholas Bellinger 806b8d26b3bSNicholas Bellinger static int 8074366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count) 808b8d26b3bSNicholas Bellinger { 809b8d26b3bSNicholas Bellinger struct ib_recv_wr *rx_wr, *rx_wr_failed; 810b8d26b3bSNicholas Bellinger int i, ret; 811b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc; 812b8d26b3bSNicholas Bellinger 813dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 8144366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i]; 8159679cc51SChristoph Hellwig 8169679cc51SChristoph Hellwig rx_wr->wr_cqe = &rx_desc->rx_cqe; 817b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg; 818b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1; 819b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1; 820b8d26b3bSNicholas Bellinger } 821b8d26b3bSNicholas Bellinger rx_wr--; 822b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */ 823b8d26b3bSNicholas Bellinger 824dac6ab30SSagi Grimberg ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 825b8d26b3bSNicholas Bellinger &rx_wr_failed); 8266d1fba0cSJenny Derzhavetz if (ret) 82724f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret); 8284366b19cSJenny Derzhavetz 8294366b19cSJenny Derzhavetz return ret; 8304366b19cSJenny Derzhavetz } 8314366b19cSJenny Derzhavetz 8324366b19cSJenny Derzhavetz static int 8334366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 8344366b19cSJenny Derzhavetz { 8354366b19cSJenny Derzhavetz struct ib_recv_wr *rx_wr_failed, rx_wr; 8364366b19cSJenny Derzhavetz int ret; 8374366b19cSJenny Derzhavetz 8389679cc51SChristoph Hellwig rx_wr.wr_cqe = &rx_desc->rx_cqe; 8394366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg; 8404366b19cSJenny Derzhavetz rx_wr.num_sge = 1; 8414366b19cSJenny Derzhavetz rx_wr.next = NULL; 8424366b19cSJenny Derzhavetz 8434366b19cSJenny Derzhavetz ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 8446d1fba0cSJenny Derzhavetz if (ret) 8454366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret); 8464366b19cSJenny Derzhavetz 847b8d26b3bSNicholas Bellinger return ret; 848b8d26b3bSNicholas Bellinger } 849b8d26b3bSNicholas Bellinger 850b8d26b3bSNicholas Bellinger static int 8519679cc51SChristoph Hellwig isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 852b8d26b3bSNicholas Bellinger { 853dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 854b8d26b3bSNicholas Bellinger struct ib_send_wr send_wr, *send_wr_failed; 855b8d26b3bSNicholas Bellinger int ret; 856b8d26b3bSNicholas Bellinger 857b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 858b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 859b8d26b3bSNicholas Bellinger 8609679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_login_send_done; 8619679cc51SChristoph Hellwig 862b8d26b3bSNicholas Bellinger send_wr.next = NULL; 8639679cc51SChristoph Hellwig send_wr.wr_cqe = &tx_desc->tx_cqe; 864b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg; 865b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge; 866b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND; 867b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED; 868b8d26b3bSNicholas Bellinger 869dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 870bdf20e72SSagi Grimberg if (ret) 87124f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret); 872b8d26b3bSNicholas Bellinger 873b8d26b3bSNicholas Bellinger return ret; 874b8d26b3bSNicholas Bellinger } 875b8d26b3bSNicholas Bellinger 876b8d26b3bSNicholas Bellinger static void 877b8d26b3bSNicholas Bellinger isert_create_send_desc(struct isert_conn *isert_conn, 878b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd, 879b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 880b8d26b3bSNicholas Bellinger { 881dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 88267cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 883b8d26b3bSNicholas Bellinger 884b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 885b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 886b8d26b3bSNicholas Bellinger 887d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 888d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL; 889b8d26b3bSNicholas Bellinger 890b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1; 891b8d26b3bSNicholas Bellinger 89234efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 89334efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 89424f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 895b8d26b3bSNicholas Bellinger } 896b8d26b3bSNicholas Bellinger } 897b8d26b3bSNicholas Bellinger 898b8d26b3bSNicholas Bellinger static int 899b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn, 900b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc) 901b8d26b3bSNicholas Bellinger { 902dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 90367cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 904b8d26b3bSNicholas Bellinger u64 dma_addr; 905b8d26b3bSNicholas Bellinger 906b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 907b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 908b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) { 90924f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n"); 910b8d26b3bSNicholas Bellinger return -ENOMEM; 911b8d26b3bSNicholas Bellinger } 912b8d26b3bSNicholas Bellinger 913b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr; 914b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 915b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 91634efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 917b8d26b3bSNicholas Bellinger 9184c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 9194c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 9204c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey); 921b8d26b3bSNicholas Bellinger 922b8d26b3bSNicholas Bellinger return 0; 923b8d26b3bSNicholas Bellinger } 924b8d26b3bSNicholas Bellinger 925b8d26b3bSNicholas Bellinger static void 92695b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 92768a86deeSSagi Grimberg struct ib_send_wr *send_wr) 928b8d26b3bSNicholas Bellinger { 92995b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 93095b60f07SNicholas Bellinger 9319679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_send_done; 9329679cc51SChristoph Hellwig send_wr->wr_cqe = &tx_desc->tx_cqe; 933422bd0acSJenny Derzhavetz 934422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 935422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV; 936422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 937422bd0acSJenny Derzhavetz } else { 938b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND; 939422bd0acSJenny Derzhavetz } 940422bd0acSJenny Derzhavetz 94195b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0]; 942b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge; 94395b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED; 944b8d26b3bSNicholas Bellinger } 945b8d26b3bSNicholas Bellinger 946b8d26b3bSNicholas Bellinger static int 9479679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn) 948b8d26b3bSNicholas Bellinger { 949b8d26b3bSNicholas Bellinger struct ib_recv_wr rx_wr, *rx_wr_fail; 950b8d26b3bSNicholas Bellinger struct ib_sge sge; 951b8d26b3bSNicholas Bellinger int ret; 952b8d26b3bSNicholas Bellinger 953b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge)); 954b8d26b3bSNicholas Bellinger sge.addr = isert_conn->login_req_dma; 955ed1083b2SChristoph Hellwig sge.length = ISER_RX_PAYLOAD_SIZE; 95634efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey; 957b8d26b3bSNicholas Bellinger 95824f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 959b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey); 960b8d26b3bSNicholas Bellinger 9619679cc51SChristoph Hellwig isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 9629679cc51SChristoph Hellwig 963b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 9649679cc51SChristoph Hellwig rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 965b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge; 966b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1; 967b8d26b3bSNicholas Bellinger 968dac6ab30SSagi Grimberg ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 9696d1fba0cSJenny Derzhavetz if (ret) 97024f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret); 971b8d26b3bSNicholas Bellinger 972b8d26b3bSNicholas Bellinger return ret; 973b8d26b3bSNicholas Bellinger } 974b8d26b3bSNicholas Bellinger 975b8d26b3bSNicholas Bellinger static int 976b8d26b3bSNicholas Bellinger isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 977b8d26b3bSNicholas Bellinger u32 length) 978b8d26b3bSNicholas Bellinger { 979b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 980dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 98167cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 982dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 983b8d26b3bSNicholas Bellinger int ret; 984b8d26b3bSNicholas Bellinger 985b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, NULL, tx_desc); 986b8d26b3bSNicholas Bellinger 987b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0], 988b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr)); 989b8d26b3bSNicholas Bellinger 990b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc); 991b8d26b3bSNicholas Bellinger 992b8d26b3bSNicholas Bellinger if (length > 0) { 993b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 994b8d26b3bSNicholas Bellinger 995b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 996b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 997b8d26b3bSNicholas Bellinger 998b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 999b8d26b3bSNicholas Bellinger 1000b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1001b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE); 1002b8d26b3bSNicholas Bellinger 1003b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma; 1004b8d26b3bSNicholas Bellinger tx_dsg->length = length; 100534efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1006b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2; 1007b8d26b3bSNicholas Bellinger } 1008b8d26b3bSNicholas Bellinger if (!login->login_failed) { 1009b8d26b3bSNicholas Bellinger if (login->login_complete) { 1010b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn); 1011b8d26b3bSNicholas Bellinger if (ret) 1012b8d26b3bSNicholas Bellinger return ret; 1013b8d26b3bSNicholas Bellinger 10144366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn, 10154366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS); 1016b8d26b3bSNicholas Bellinger if (ret) 1017b8d26b3bSNicholas Bellinger return ret; 1018b8d26b3bSNicholas Bellinger 1019128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */ 1020dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 1021128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE; 1022dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1023b8d26b3bSNicholas Bellinger goto post_send; 1024b8d26b3bSNicholas Bellinger } 1025b8d26b3bSNicholas Bellinger 10269679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn); 1027b8d26b3bSNicholas Bellinger if (ret) 1028b8d26b3bSNicholas Bellinger return ret; 1029b8d26b3bSNicholas Bellinger } 1030b8d26b3bSNicholas Bellinger post_send: 10319679cc51SChristoph Hellwig ret = isert_login_post_send(isert_conn, tx_desc); 1032b8d26b3bSNicholas Bellinger if (ret) 1033b8d26b3bSNicholas Bellinger return ret; 1034b8d26b3bSNicholas Bellinger 1035b8d26b3bSNicholas Bellinger return 0; 1036b8d26b3bSNicholas Bellinger } 1037b8d26b3bSNicholas Bellinger 1038b8d26b3bSNicholas Bellinger static void 10392371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn) 1040b8d26b3bSNicholas Bellinger { 10415adabdd1SChristoph Hellwig struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 10422371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len; 1043b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1044b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login; 1045b8d26b3bSNicholas Bellinger int size; 1046b8d26b3bSNicholas Bellinger 104724f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn); 10482371e5daSSagi Grimberg 10492371e5daSSagi Grimberg WARN_ON_ONCE(!login); 1050b8d26b3bSNicholas Bellinger 1051b8d26b3bSNicholas Bellinger if (login->first_request) { 1052b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req = 1053b8d26b3bSNicholas Bellinger (struct iscsi_login_req *)&rx_desc->iscsi_header; 1054b8d26b3bSNicholas Bellinger /* 1055b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading 1056b8d26b3bSNicholas Bellinger * login request PDU. 1057b8d26b3bSNicholas Bellinger */ 1058b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0; 1059b8d26b3bSNicholas Bellinger login->current_stage = 1060b8d26b3bSNicholas Bellinger (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1061b8d26b3bSNicholas Bellinger >> 2; 1062b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version; 1063b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version; 1064b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6); 1065b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1066b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt; 1067b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1068b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid); 1069b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih); 1070b8d26b3bSNicholas Bellinger } 1071b8d26b3bSNicholas Bellinger 1072b8d26b3bSNicholas Bellinger memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1073b8d26b3bSNicholas Bellinger 1074b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 10754c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d " 10764c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 10774c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS); 1078b8d26b3bSNicholas Bellinger memcpy(login->req_buf, &rx_desc->data[0], size); 1079b8d26b3bSNicholas Bellinger 10806faaa85fSNicholas Bellinger if (login->first_request) { 1081dac6ab30SSagi Grimberg complete(&isert_conn->login_comp); 10826faaa85fSNicholas Bellinger return; 10836faaa85fSNicholas Bellinger } 10846faaa85fSNicholas Bellinger schedule_delayed_work(&conn->login_work, 0); 1085b8d26b3bSNicholas Bellinger } 1086b8d26b3bSNicholas Bellinger 1087b8d26b3bSNicholas Bellinger static struct iscsi_cmd 10884366b19cSJenny Derzhavetz *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1089b8d26b3bSNicholas Bellinger { 10906700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1091b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1092d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd; 1093b8d26b3bSNicholas Bellinger 1094676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1095d703ce2fSNicholas Bellinger if (!cmd) { 109624f412ddSSagi Grimberg isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1097b8d26b3bSNicholas Bellinger return NULL; 1098b8d26b3bSNicholas Bellinger } 1099d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1100b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn; 1101d703ce2fSNicholas Bellinger isert_cmd->iscsi_cmd = cmd; 11024366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc; 1103b8d26b3bSNicholas Bellinger 1104d703ce2fSNicholas Bellinger return cmd; 1105b8d26b3bSNicholas Bellinger } 1106b8d26b3bSNicholas Bellinger 1107b8d26b3bSNicholas Bellinger static int 1108b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1109d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1110d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1111b8d26b3bSNicholas Bellinger { 1112b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1113b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1114b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1115b8d26b3bSNicholas Bellinger bool dump_payload = false; 11169fd60088SJenny Derzhavetz unsigned int data_len; 1117b8d26b3bSNicholas Bellinger 1118b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1119b8d26b3bSNicholas Bellinger if (rc < 0) 1120b8d26b3bSNicholas Bellinger return rc; 1121b8d26b3bSNicholas Bellinger 1122b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data; 1123b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len; 1124b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data; 11259fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length; 1126b8d26b3bSNicholas Bellinger 11279fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len) 11289fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1129b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1130b8d26b3bSNicholas Bellinger if (rc < 0) { 1131b8d26b3bSNicholas Bellinger return 0; 1132b8d26b3bSNicholas Bellinger } else if (rc > 0) { 1133b8d26b3bSNicholas Bellinger dump_payload = true; 1134b8d26b3bSNicholas Bellinger goto sequence_cmd; 1135b8d26b3bSNicholas Bellinger } 1136b8d26b3bSNicholas Bellinger 1137b8d26b3bSNicholas Bellinger if (!imm_data) 1138b8d26b3bSNicholas Bellinger return 0; 1139b8d26b3bSNicholas Bellinger 11409fd60088SJenny Derzhavetz if (imm_data_len != data_len) { 1141b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 11429fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 11439fd60088SJenny Derzhavetz &rx_desc->data[0], imm_data_len); 11449fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 11459fd60088SJenny Derzhavetz sg_nents, imm_data_len); 11469fd60088SJenny Derzhavetz } else { 11479fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1); 11489fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg; 11499fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1; 11509fd60088SJenny Derzhavetz sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 11519fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n", 11529fd60088SJenny Derzhavetz imm_data_len); 11539fd60088SJenny Derzhavetz } 1154b8d26b3bSNicholas Bellinger 1155b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len; 1156b8d26b3bSNicholas Bellinger 1157b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) { 1158b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1159b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1160b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1161b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1162b8d26b3bSNicholas Bellinger } 1163b8d26b3bSNicholas Bellinger 1164b8d26b3bSNicholas Bellinger sequence_cmd: 1165561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1166b8d26b3bSNicholas Bellinger 1167b8d26b3bSNicholas Bellinger if (!rc && dump_payload == false && unsol_data) 1168b8d26b3bSNicholas Bellinger iscsit_set_unsoliticed_dataout(cmd); 11696cc44a6fSNicholas Bellinger else if (dump_payload && imm_data) 1170afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd); 1171b8d26b3bSNicholas Bellinger 1172b8d26b3bSNicholas Bellinger return 0; 1173b8d26b3bSNicholas Bellinger } 1174b8d26b3bSNicholas Bellinger 1175b8d26b3bSNicholas Bellinger static int 1176b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1177b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf) 1178b8d26b3bSNicholas Bellinger { 1179b8d26b3bSNicholas Bellinger struct scatterlist *sg_start; 1180b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1181b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd = NULL; 1182b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf; 1183b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength); 1184b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off; 1185b8d26b3bSNicholas Bellinger 1186b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1187b8d26b3bSNicholas Bellinger if (rc < 0) 1188b8d26b3bSNicholas Bellinger return rc; 1189b8d26b3bSNicholas Bellinger else if (!cmd) 1190b8d26b3bSNicholas Bellinger return 0; 1191b8d26b3bSNicholas Bellinger /* 1192b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out 1193b8d26b3bSNicholas Bellinger */ 1194b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) { 119524f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n"); 1196b8d26b3bSNicholas Bellinger dump_stack(); 1197b8d26b3bSNicholas Bellinger return -1; 1198b8d26b3bSNicholas Bellinger } 1199b8d26b3bSNicholas Bellinger 12004c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 12014c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n", 12024c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done, 12034c22e07fSSagi Grimberg cmd->se_cmd.data_length); 1204b8d26b3bSNicholas Bellinger 1205b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE; 1206b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1207b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1208b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE; 1209b8d26b3bSNicholas Bellinger /* 1210b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out 1211b8d26b3bSNicholas Bellinger */ 1212b8d26b3bSNicholas Bellinger if (page_off) { 12134c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n"); 1214b8d26b3bSNicholas Bellinger dump_stack(); 1215b8d26b3bSNicholas Bellinger return -1; 1216b8d26b3bSNicholas Bellinger } 12174c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 12184c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off, 12194c22e07fSSagi Grimberg sg_nents, &rx_desc->data[0], unsol_data_len); 1220b8d26b3bSNicholas Bellinger 1221b8d26b3bSNicholas Bellinger sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1222b8d26b3bSNicholas Bellinger unsol_data_len); 1223b8d26b3bSNicholas Bellinger 1224b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false); 1225b8d26b3bSNicholas Bellinger if (rc < 0) 1226b8d26b3bSNicholas Bellinger return rc; 1227b8d26b3bSNicholas Bellinger 12284366b19cSJenny Derzhavetz /* 12294366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive - 12304366b19cSJenny Derzhavetz * so post the buffer before hand 12314366b19cSJenny Derzhavetz */ 12324366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, rx_desc); 12334366b19cSJenny Derzhavetz if (rc) { 12344366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 12354366b19cSJenny Derzhavetz return rc; 12364366b19cSJenny Derzhavetz } 1237b8d26b3bSNicholas Bellinger return 0; 1238b8d26b3bSNicholas Bellinger } 1239b8d26b3bSNicholas Bellinger 1240b8d26b3bSNicholas Bellinger static int 1241778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1242d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1243d703ce2fSNicholas Bellinger unsigned char *buf) 1244778de368SNicholas Bellinger { 1245778de368SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1246778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1247778de368SNicholas Bellinger int rc; 1248778de368SNicholas Bellinger 1249778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr); 1250778de368SNicholas Bellinger if (rc < 0) 1251778de368SNicholas Bellinger return rc; 1252778de368SNicholas Bellinger /* 1253778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1254778de368SNicholas Bellinger */ 1255778de368SNicholas Bellinger 1256778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr); 1257778de368SNicholas Bellinger } 1258778de368SNicholas Bellinger 1259778de368SNicholas Bellinger static int 1260adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1261d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1262d703ce2fSNicholas Bellinger struct iscsi_text *hdr) 1263adb54c29SNicholas Bellinger { 1264adb54c29SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1265adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength); 1266adb54c29SNicholas Bellinger int rc; 1267b44a2b67SSagi Grimberg unsigned char *text_in = NULL; 1268adb54c29SNicholas Bellinger 1269adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1270adb54c29SNicholas Bellinger if (rc < 0) 1271adb54c29SNicholas Bellinger return rc; 1272adb54c29SNicholas Bellinger 1273b44a2b67SSagi Grimberg if (payload_length) { 1274adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL); 127593b80f29SLeon Romanovsky if (!text_in) 1276adb54c29SNicholas Bellinger return -ENOMEM; 1277adb54c29SNicholas Bellinger } 1278adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in; 1279adb54c29SNicholas Bellinger 1280adb54c29SNicholas Bellinger memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1281adb54c29SNicholas Bellinger 1282adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr); 1283adb54c29SNicholas Bellinger } 1284adb54c29SNicholas Bellinger 1285adb54c29SNicholas Bellinger static int 1286b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1287b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va, 1288b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va) 1289b8d26b3bSNicholas Bellinger { 1290b8d26b3bSNicholas Bellinger struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1291b8d26b3bSNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1292b8d26b3bSNicholas Bellinger struct iscsi_cmd *cmd; 1293b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd; 1294b8d26b3bSNicholas Bellinger int ret = -EINVAL; 1295b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1296b8d26b3bSNicholas Bellinger 1297fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType && 1298ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 129924f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1300ca40d24eSNicholas Bellinger " ignoring\n", opcode); 1301ca40d24eSNicholas Bellinger return 0; 1302ca40d24eSNicholas Bellinger } 1303ca40d24eSNicholas Bellinger 1304b8d26b3bSNicholas Bellinger switch (opcode) { 1305b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 13064366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1307b8d26b3bSNicholas Bellinger if (!cmd) 1308b8d26b3bSNicholas Bellinger break; 1309b8d26b3bSNicholas Bellinger 1310d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1311b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag; 1312b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va; 1313b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag; 1314b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va; 1315422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1316b8d26b3bSNicholas Bellinger 1317d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1318b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr); 1319b8d26b3bSNicholas Bellinger break; 1320b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 13214366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1322b8d26b3bSNicholas Bellinger if (!cmd) 1323b8d26b3bSNicholas Bellinger break; 1324b8d26b3bSNicholas Bellinger 1325d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1326d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1327778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr); 1328b8d26b3bSNicholas Bellinger break; 1329b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT: 1330b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1331b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1332b8d26b3bSNicholas Bellinger break; 1333b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 13344366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1335b8d26b3bSNicholas Bellinger if (!cmd) 1336b8d26b3bSNicholas Bellinger break; 1337b8d26b3bSNicholas Bellinger 1338b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1339b8d26b3bSNicholas Bellinger (unsigned char *)hdr); 1340b8d26b3bSNicholas Bellinger break; 1341b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT: 13424366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 1343b8d26b3bSNicholas Bellinger if (!cmd) 1344b8d26b3bSNicholas Bellinger break; 1345b8d26b3bSNicholas Bellinger 1346b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1347b8d26b3bSNicholas Bellinger break; 1348adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 13494366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1350e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 13514366b19cSJenny Derzhavetz else 13524366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc); 13534366b19cSJenny Derzhavetz 1354e4f4e801SSagi Grimberg if (!cmd) 1355e4f4e801SSagi Grimberg break; 1356adb54c29SNicholas Bellinger 1357d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd); 1358d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1359adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr); 1360adb54c29SNicholas Bellinger break; 1361b8d26b3bSNicholas Bellinger default: 136224f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1363b8d26b3bSNicholas Bellinger dump_stack(); 1364b8d26b3bSNicholas Bellinger break; 1365b8d26b3bSNicholas Bellinger } 1366b8d26b3bSNicholas Bellinger 1367b8d26b3bSNicholas Bellinger return ret; 1368b8d26b3bSNicholas Bellinger } 1369b8d26b3bSNicholas Bellinger 1370b8d26b3bSNicholas Bellinger static void 13719679cc51SChristoph Hellwig isert_print_wc(struct ib_wc *wc, const char *type) 1372b8d26b3bSNicholas Bellinger { 13739679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 13749679cc51SChristoph Hellwig isert_err("%s failure: %s (%d) vend_err %x\n", type, 13759679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status, 13769679cc51SChristoph Hellwig wc->vendor_err); 13779679cc51SChristoph Hellwig else 13789679cc51SChristoph Hellwig isert_dbg("%s failure: %s (%d)\n", type, 13799679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status); 13809679cc51SChristoph Hellwig } 13819679cc51SChristoph Hellwig 13829679cc51SChristoph Hellwig static void 13839679cc51SChristoph Hellwig isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 13849679cc51SChristoph Hellwig { 13859679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 13869679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 13879679cc51SChristoph Hellwig struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 13889679cc51SChristoph Hellwig struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1389d3cf81f9SSagi Grimberg struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1390b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0; 1391b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0; 1392b8d26b3bSNicholas Bellinger 13939679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 13949679cc51SChristoph Hellwig isert_print_wc(wc, "recv"); 13959679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 13969679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 13979679cc51SChristoph Hellwig return; 13989679cc51SChristoph Hellwig } 13999679cc51SChristoph Hellwig 14009679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 14019679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 14029679cc51SChristoph Hellwig 14039679cc51SChristoph Hellwig isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 14049679cc51SChristoph Hellwig rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 14059679cc51SChristoph Hellwig (int)(wc->byte_len - ISER_HEADERS_LEN)); 14069679cc51SChristoph Hellwig 1407d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) { 1408b8d26b3bSNicholas Bellinger case ISCSI_CTRL: 1409d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) { 1410d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag); 1411d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va); 14124c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1413b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va); 1414b8d26b3bSNicholas Bellinger } 1415d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) { 1416d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag); 1417d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va); 14184c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1419b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va); 1420b8d26b3bSNicholas Bellinger } 1421b8d26b3bSNicholas Bellinger 142224f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n"); 1423b8d26b3bSNicholas Bellinger break; 1424b8d26b3bSNicholas Bellinger case ISER_HELLO: 142524f412ddSSagi Grimberg isert_err("iSER Hello message\n"); 1426b8d26b3bSNicholas Bellinger break; 1427b8d26b3bSNicholas Bellinger default: 1428d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1429b8d26b3bSNicholas Bellinger break; 1430b8d26b3bSNicholas Bellinger } 1431b8d26b3bSNicholas Bellinger 14326c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc, 1433b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va); 14349679cc51SChristoph Hellwig 14359679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 14369679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1437b8d26b3bSNicholas Bellinger } 1438b8d26b3bSNicholas Bellinger 1439b8d26b3bSNicholas Bellinger static void 14409679cc51SChristoph Hellwig isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1441b8d26b3bSNicholas Bellinger { 14429679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1443dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1444b8d26b3bSNicholas Bellinger 14459679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 14469679cc51SChristoph Hellwig isert_print_wc(wc, "login recv"); 14479679cc51SChristoph Hellwig return; 1448b8d26b3bSNicholas Bellinger } 1449b8d26b3bSNicholas Bellinger 14509679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 14519679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1452b8d26b3bSNicholas Bellinger 14539679cc51SChristoph Hellwig isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1454b8d26b3bSNicholas Bellinger 14552371e5daSSagi Grimberg if (isert_conn->conn) { 14562371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login; 14572371e5daSSagi Grimberg 14582371e5daSSagi Grimberg if (login && !login->first_request) 14592371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 14602371e5daSSagi Grimberg } 14619679cc51SChristoph Hellwig 1462dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 14632371e5daSSagi Grimberg complete(&isert_conn->login_req_comp); 1464dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 1465b8d26b3bSNicholas Bellinger 14669679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 14679679cc51SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1468b8d26b3bSNicholas Bellinger } 1469b8d26b3bSNicholas Bellinger 1470e3d7e4c3SSagi Grimberg static void 147138a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1472e3d7e4c3SSagi Grimberg { 147338a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 147438a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1475e3d7e4c3SSagi Grimberg 147638a2d0d4SChristoph Hellwig if (!cmd->rw.nr_ops) 147738a2d0d4SChristoph Hellwig return; 147838a2d0d4SChristoph Hellwig 147938a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 148038a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 148138a2d0d4SChristoph Hellwig conn->cm_id->port_num, se_cmd->t_data_sg, 148238a2d0d4SChristoph Hellwig se_cmd->t_data_nents, se_cmd->t_prot_sg, 148338a2d0d4SChristoph Hellwig se_cmd->t_prot_nents, dir); 148438a2d0d4SChristoph Hellwig } else { 148538a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 148638a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1487e3d7e4c3SSagi Grimberg } 1488e3d7e4c3SSagi Grimberg 148938a2d0d4SChristoph Hellwig cmd->rw.nr_ops = 0; 149059464ef4SVu Pham } 149159464ef4SVu Pham 149259464ef4SVu Pham static void 149303e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1494b8d26b3bSNicholas Bellinger { 1495d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1496b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1497186a9647SNicholas Bellinger struct iscsi_conn *conn = isert_conn->conn; 1498e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr; 1499b8d26b3bSNicholas Bellinger 15004c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd); 1501b8d26b3bSNicholas Bellinger 1502b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) { 1503b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD: 1504b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1505b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15065159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1507b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1508b8d26b3bSNicholas Bellinger 150903e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) { 1510b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 151103e7848aSNicholas Bellinger /* 151203e7848aSNicholas Bellinger * Check for special case during comp_err where 151303e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core, 151403e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd() 151503e7848aSNicholas Bellinger * before transport_generic_free_cmd() below. 151603e7848aSNicholas Bellinger */ 151703e7848aSNicholas Bellinger if (comp_err && 151803e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 151903e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 152003e7848aSNicholas Bellinger 1521afc16604SBart Van Assche target_put_sess_cmd(se_cmd); 152203e7848aSNicholas Bellinger } 152303e7848aSNicholas Bellinger } 1524b8d26b3bSNicholas Bellinger 152538a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1526186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1527186a9647SNicholas Bellinger break; 1528b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC: 1529186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1530186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15315159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1532186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1533186a9647SNicholas Bellinger 1534b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1535b8d26b3bSNicholas Bellinger break; 1536b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT: 1537b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT: 1538adb54c29SNicholas Bellinger case ISCSI_OP_TEXT: 1539e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1540e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */ 1541e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1542e4f4e801SSagi Grimberg break; 1543e4f4e801SSagi Grimberg 1544b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1545b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 15465159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node); 1547b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1548b8d26b3bSNicholas Bellinger 1549b8d26b3bSNicholas Bellinger /* 1550b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has 1551b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the 1552b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released. 1553b8d26b3bSNicholas Bellinger */ 1554b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) { 155511378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 15563df8f68aSNicholas Bellinger cmd->iscsi_opcode); 1557b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0); 1558b8d26b3bSNicholas Bellinger break; 1559b8d26b3bSNicholas Bellinger } 1560b8d26b3bSNicholas Bellinger /* 1561b8d26b3bSNicholas Bellinger * Fall-through 1562b8d26b3bSNicholas Bellinger */ 1563b8d26b3bSNicholas Bellinger default: 1564d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd); 1565b8d26b3bSNicholas Bellinger break; 1566b8d26b3bSNicholas Bellinger } 1567b8d26b3bSNicholas Bellinger } 1568b8d26b3bSNicholas Bellinger 1569b8d26b3bSNicholas Bellinger static void 1570b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1571b8d26b3bSNicholas Bellinger { 1572b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) { 15734c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n"); 1574b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1575b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE); 1576b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0; 1577b8d26b3bSNicholas Bellinger } 1578b8d26b3bSNicholas Bellinger } 1579b8d26b3bSNicholas Bellinger 1580b8d26b3bSNicholas Bellinger static void 1581b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 158203e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err) 1583b8d26b3bSNicholas Bellinger { 1584dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) { 15854c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1586dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1587dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1588dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0; 1589b8d26b3bSNicholas Bellinger } 1590b8d26b3bSNicholas Bellinger 1591b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 159203e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err); 1593b8d26b3bSNicholas Bellinger } 1594b8d26b3bSNicholas Bellinger 159596b7973eSSagi Grimberg static int 159696b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1597f93f3a70SSagi Grimberg { 15989e961ae7SSagi Grimberg struct ib_mr_status mr_status; 159996b7973eSSagi Grimberg int ret; 1600f93f3a70SSagi Grimberg 160196b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 16029e961ae7SSagi Grimberg if (ret) { 160324f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret); 16049e961ae7SSagi Grimberg goto fail_mr_status; 16059e961ae7SSagi Grimberg } 160696b7973eSSagi Grimberg 16079e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 160896b7973eSSagi Grimberg u64 sec_offset_err; 16099e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 16109e961ae7SSagi Grimberg 16119e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) { 16129e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD: 16139e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 16149e961ae7SSagi Grimberg break; 16159e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG: 16169e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 16179e961ae7SSagi Grimberg break; 16189e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG: 16199e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 16209e961ae7SSagi Grimberg break; 16219e961ae7SSagi Grimberg } 162296b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset; 162396b7973eSSagi Grimberg do_div(sec_offset_err, block_size); 162496b7973eSSagi Grimberg se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 16259e961ae7SSagi Grimberg 16264c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx " 16279e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n", 16289e961ae7SSagi Grimberg mr_status.sig_err.err_type, 16299e961ae7SSagi Grimberg (unsigned long long)se_cmd->bad_sector, 16309e961ae7SSagi Grimberg mr_status.sig_err.expected, 16319e961ae7SSagi Grimberg mr_status.sig_err.actual); 16329e961ae7SSagi Grimberg ret = 1; 16339e961ae7SSagi Grimberg } 16349e961ae7SSagi Grimberg 16359e961ae7SSagi Grimberg fail_mr_status: 163696b7973eSSagi Grimberg return ret; 163796b7973eSSagi Grimberg } 163896b7973eSSagi Grimberg 163996b7973eSSagi Grimberg static void 16409679cc51SChristoph Hellwig isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 164196b7973eSSagi Grimberg { 16429679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 1643dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 16449679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1645e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 16469679cc51SChristoph Hellwig struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 164796b7973eSSagi Grimberg int ret = 0; 164896b7973eSSagi Grimberg 16499679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 16509679cc51SChristoph Hellwig isert_print_wc(wc, "rdma write"); 16519679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 16529679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 16539679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 16549679cc51SChristoph Hellwig return; 16559679cc51SChristoph Hellwig } 16569679cc51SChristoph Hellwig 16579679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 16589679cc51SChristoph Hellwig 165938a2d0d4SChristoph Hellwig ret = isert_check_pi_status(cmd, isert_cmd->rw.sig->sig_mr); 166038a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 166196b7973eSSagi Grimberg 1662*555a65f6SNicholas Bellinger if (ret) { 1663*555a65f6SNicholas Bellinger /* 1664*555a65f6SNicholas Bellinger * transport_generic_request_failure() expects to have 1665*555a65f6SNicholas Bellinger * plus two references to handle queue-full, so re-add 1666*555a65f6SNicholas Bellinger * one here as target-core will have already dropped 1667*555a65f6SNicholas Bellinger * it after the first isert_put_datain() callback. 1668*555a65f6SNicholas Bellinger */ 1669*555a65f6SNicholas Bellinger kref_get(&cmd->cmd_kref); 1670*555a65f6SNicholas Bellinger transport_generic_request_failure(cmd, cmd->pi_err); 1671*555a65f6SNicholas Bellinger } else { 1672*555a65f6SNicholas Bellinger /* 1673*555a65f6SNicholas Bellinger * XXX: isert_put_response() failure is not retried. 1674*555a65f6SNicholas Bellinger */ 1675*555a65f6SNicholas Bellinger ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 16769e961ae7SSagi Grimberg if (ret) 1677*555a65f6SNicholas Bellinger pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1678*555a65f6SNicholas Bellinger } 1679f93f3a70SSagi Grimberg } 1680f93f3a70SSagi Grimberg 1681f93f3a70SSagi Grimberg static void 16829679cc51SChristoph Hellwig isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1683b8d26b3bSNicholas Bellinger { 16849679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 16859679cc51SChristoph Hellwig struct isert_device *device = isert_conn->device; 16869679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1687e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1688d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1689b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd; 16905bac4b1aSSagi Grimberg int ret = 0; 1691b8d26b3bSNicholas Bellinger 16929679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 16939679cc51SChristoph Hellwig isert_print_wc(wc, "rdma read"); 16949679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 16959679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 16969679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true); 16979679cc51SChristoph Hellwig return; 16989679cc51SChristoph Hellwig } 16999679cc51SChristoph Hellwig 17009679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 17019679cc51SChristoph Hellwig 1702b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 170338a2d0d4SChristoph Hellwig 170438a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) 170538a2d0d4SChristoph Hellwig ret = isert_check_pi_status(se_cmd, isert_cmd->rw.sig->sig_mr); 170638a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 170738a2d0d4SChristoph Hellwig cmd->write_data_done = 0; 1708b8d26b3bSNicholas Bellinger 170924f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1710b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock); 1711b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1712b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1713b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock); 1714b8d26b3bSNicholas Bellinger 1715*555a65f6SNicholas Bellinger /* 1716*555a65f6SNicholas Bellinger * transport_generic_request_failure() will drop the extra 1717*555a65f6SNicholas Bellinger * se_cmd->cmd_kref reference after T10-PI error, and handle 1718*555a65f6SNicholas Bellinger * any non-zero ->queue_status() callback error retries. 1719*555a65f6SNicholas Bellinger */ 1720*555a65f6SNicholas Bellinger if (ret) 1721*555a65f6SNicholas Bellinger transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1722*555a65f6SNicholas Bellinger else 1723b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd); 1724b8d26b3bSNicholas Bellinger } 1725b8d26b3bSNicholas Bellinger 1726b8d26b3bSNicholas Bellinger static void 1727b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work) 1728b8d26b3bSNicholas Bellinger { 1729b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work, 1730b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work); 1731b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn; 1732dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device; 1733d703ce2fSNicholas Bellinger struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1734b8d26b3bSNicholas Bellinger 17354c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 17364c22e07fSSagi Grimberg 1737b8d26b3bSNicholas Bellinger switch (cmd->i_state) { 1738b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 1739b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn); 174010633c37SSagi Grimberg case ISTATE_SEND_REJECT: /* FALLTHRU */ 174110633c37SSagi Grimberg case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1742b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS; 17434c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 17444c22e07fSSagi Grimberg ib_dev, false); 17453df8f68aSNicholas Bellinger break; 1746b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 1747b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn); 1748b8d26b3bSNicholas Bellinger break; 1749b8d26b3bSNicholas Bellinger default: 17504c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state); 1751b8d26b3bSNicholas Bellinger dump_stack(); 1752b8d26b3bSNicholas Bellinger break; 1753b8d26b3bSNicholas Bellinger } 1754b8d26b3bSNicholas Bellinger } 1755b8d26b3bSNicholas Bellinger 1756b8d26b3bSNicholas Bellinger static void 17579679cc51SChristoph Hellwig isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1758b8d26b3bSNicholas Bellinger { 17599679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17609679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17619679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1762b8d26b3bSNicholas Bellinger 17639679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17649679cc51SChristoph Hellwig isert_print_wc(wc, "login send"); 17659679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17669679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17679679cc51SChristoph Hellwig } 17689679cc51SChristoph Hellwig 17699679cc51SChristoph Hellwig isert_unmap_tx_desc(tx_desc, ib_dev); 17709679cc51SChristoph Hellwig } 17719679cc51SChristoph Hellwig 17729679cc51SChristoph Hellwig static void 17739679cc51SChristoph Hellwig isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 17749679cc51SChristoph Hellwig { 17759679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context; 17769679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device; 17779679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1778e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 17799679cc51SChristoph Hellwig 17809679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) { 17819679cc51SChristoph Hellwig isert_print_wc(wc, "send"); 17829679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR) 17839679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 17849679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 17859679cc51SChristoph Hellwig return; 17869679cc51SChristoph Hellwig } 17879679cc51SChristoph Hellwig 17889679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd); 17899679cc51SChristoph Hellwig 17909679cc51SChristoph Hellwig switch (isert_cmd->iscsi_cmd->i_state) { 17919679cc51SChristoph Hellwig case ISTATE_SEND_TASKMGTRSP: 17929679cc51SChristoph Hellwig case ISTATE_SEND_LOGOUTRSP: 17939679cc51SChristoph Hellwig case ISTATE_SEND_REJECT: 17949679cc51SChristoph Hellwig case ISTATE_SEND_TEXTRSP: 1795b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev); 1796b8d26b3bSNicholas Bellinger 1797b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1798b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work); 1799b8d26b3bSNicholas Bellinger return; 1800b8d26b3bSNicholas Bellinger default: 18019679cc51SChristoph Hellwig isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 18029679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1803b8d26b3bSNicholas Bellinger break; 1804b8d26b3bSNicholas Bellinger } 1805b8d26b3bSNicholas Bellinger } 1806b8d26b3bSNicholas Bellinger 1807b8d26b3bSNicholas Bellinger static int 1808b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1809b8d26b3bSNicholas Bellinger { 1810b8d26b3bSNicholas Bellinger struct ib_send_wr *wr_failed; 1811b8d26b3bSNicholas Bellinger int ret; 1812b8d26b3bSNicholas Bellinger 18134366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 18144366b19cSJenny Derzhavetz if (ret) { 18154366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", ret); 18164366b19cSJenny Derzhavetz return ret; 18174366b19cSJenny Derzhavetz } 18184366b19cSJenny Derzhavetz 1819dac6ab30SSagi Grimberg ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 1820b8d26b3bSNicholas Bellinger &wr_failed); 1821b8d26b3bSNicholas Bellinger if (ret) { 182224f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret); 1823b8d26b3bSNicholas Bellinger return ret; 1824b8d26b3bSNicholas Bellinger } 1825b8d26b3bSNicholas Bellinger return ret; 1826b8d26b3bSNicholas Bellinger } 1827b8d26b3bSNicholas Bellinger 1828b8d26b3bSNicholas Bellinger static int 1829b8d26b3bSNicholas Bellinger isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1830b8d26b3bSNicholas Bellinger { 1831d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18326700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1833b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1834b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1835b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header; 1836b8d26b3bSNicholas Bellinger 1837b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1838b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1839b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1840b8d26b3bSNicholas Bellinger /* 1841b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU 1842b8d26b3bSNicholas Bellinger */ 1843b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer && 1844b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1845b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1846dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 184767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 1848b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1849dbbc5d11SNicholas Bellinger u32 padding, pdu_len; 1850b8d26b3bSNicholas Bellinger 1851b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1852b8d26b3bSNicholas Bellinger cmd->sense_buffer); 1853b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1854b8d26b3bSNicholas Bellinger 1855b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1856b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1857dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1858b8d26b3bSNicholas Bellinger 1859dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1860dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len, 1861b8d26b3bSNicholas Bellinger DMA_TO_DEVICE); 1862def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1863def4a6ffSAlexey Khoroshilov return -ENOMEM; 1864b8d26b3bSNicholas Bellinger 1865dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len; 1866dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 1867dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len; 186834efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 1869b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 1870b8d26b3bSNicholas Bellinger } 1871b8d26b3bSNicholas Bellinger 187268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1873b8d26b3bSNicholas Bellinger 18744c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n"); 1875b8d26b3bSNicholas Bellinger 1876b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1877b8d26b3bSNicholas Bellinger } 1878b8d26b3bSNicholas Bellinger 1879131e6abcSNicholas Bellinger static void 1880131e6abcSNicholas Bellinger isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1881131e6abcSNicholas Bellinger { 1882131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 18836700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1884131e6abcSNicholas Bellinger 1885131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock); 1886131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node)) 1887131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node); 1888131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock); 1889131e6abcSNicholas Bellinger 1890131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) 1891131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd); 189238a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1893131e6abcSNicholas Bellinger } 1894131e6abcSNicholas Bellinger 1895e70beee7SNicholas Bellinger static enum target_prot_op 1896e70beee7SNicholas Bellinger isert_get_sup_prot_ops(struct iscsi_conn *conn) 1897e70beee7SNicholas Bellinger { 18986700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1899dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 1900e70beee7SNicholas Bellinger 190123a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) { 190223a548eeSSagi Grimberg if (device->pi_capable) { 190324f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn); 190423a548eeSSagi Grimberg isert_conn->pi_support = true; 1905e70beee7SNicholas Bellinger return TARGET_PROT_ALL; 190623a548eeSSagi Grimberg } 190723a548eeSSagi Grimberg } 190823a548eeSSagi Grimberg 190924f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn); 191023a548eeSSagi Grimberg isert_conn->pi_support = false; 1911e70beee7SNicholas Bellinger 1912e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL; 1913e70beee7SNicholas Bellinger } 1914e70beee7SNicholas Bellinger 1915b8d26b3bSNicholas Bellinger static int 1916b8d26b3bSNicholas Bellinger isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1917b8d26b3bSNicholas Bellinger bool nopout_response) 1918b8d26b3bSNicholas Bellinger { 1919d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19206700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1921b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1922b8d26b3bSNicholas Bellinger 1923b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1924b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1925b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header, 1926b8d26b3bSNicholas Bellinger nopout_response); 1927b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 192868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1929b8d26b3bSNicholas Bellinger 19304c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1931b8d26b3bSNicholas Bellinger 1932b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1933b8d26b3bSNicholas Bellinger } 1934b8d26b3bSNicholas Bellinger 1935b8d26b3bSNicholas Bellinger static int 1936b8d26b3bSNicholas Bellinger isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1937b8d26b3bSNicholas Bellinger { 1938d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19396700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1940b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1941b8d26b3bSNicholas Bellinger 1942b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1943b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1944b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1945b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 194668a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1947b8d26b3bSNicholas Bellinger 19484c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1949b8d26b3bSNicholas Bellinger 1950b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1951b8d26b3bSNicholas Bellinger } 1952b8d26b3bSNicholas Bellinger 1953b8d26b3bSNicholas Bellinger static int 1954b8d26b3bSNicholas Bellinger isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1955b8d26b3bSNicholas Bellinger { 1956d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19576700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1958b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1959b8d26b3bSNicholas Bellinger 1960b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1961b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1962b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 1963b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 196468a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1965b8d26b3bSNicholas Bellinger 19664c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1967b8d26b3bSNicholas Bellinger 1968b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 1969b8d26b3bSNicholas Bellinger } 1970b8d26b3bSNicholas Bellinger 1971b8d26b3bSNicholas Bellinger static int 1972b8d26b3bSNicholas Bellinger isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1973b8d26b3bSNicholas Bellinger { 1974d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 19756700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 1976b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1977dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 197867cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 19793df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 19803df8f68aSNicholas Bellinger struct iscsi_reject *hdr = 19813df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1982b8d26b3bSNicholas Bellinger 1983b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 19843df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr); 1985b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 19863df8f68aSNicholas Bellinger 19873df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN); 1988dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 19893df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 19903df8f68aSNicholas Bellinger DMA_TO_DEVICE); 1991def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1992def4a6ffSAlexey Khoroshilov return -ENOMEM; 1993dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1994dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 19953df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN; 199634efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 19973df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 19983df8f68aSNicholas Bellinger 199968a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2000b8d26b3bSNicholas Bellinger 20014c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn); 2002b8d26b3bSNicholas Bellinger 2003b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2004b8d26b3bSNicholas Bellinger } 2005b8d26b3bSNicholas Bellinger 2006b8d26b3bSNicholas Bellinger static int 2007adb54c29SNicholas Bellinger isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2008adb54c29SNicholas Bellinger { 2009d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 20106700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2011adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2012adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr = 2013adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2014adb54c29SNicholas Bellinger u32 txt_rsp_len; 2015adb54c29SNicholas Bellinger int rc; 2016adb54c29SNicholas Bellinger 2017adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 201822c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2019adb54c29SNicholas Bellinger if (rc < 0) 2020adb54c29SNicholas Bellinger return rc; 2021adb54c29SNicholas Bellinger 2022adb54c29SNicholas Bellinger txt_rsp_len = rc; 2023adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2024adb54c29SNicholas Bellinger 2025adb54c29SNicholas Bellinger if (txt_rsp_len) { 2026dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device; 202767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device; 2028adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2029adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr; 2030adb54c29SNicholas Bellinger 2031adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2032adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2033def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2034def4a6ffSAlexey Khoroshilov return -ENOMEM; 2035adb54c29SNicholas Bellinger 2036adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len; 2037adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma; 2038adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len; 203934efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey; 2040adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2; 2041adb54c29SNicholas Bellinger } 204268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2043adb54c29SNicholas Bellinger 2044f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn); 2045adb54c29SNicholas Bellinger 2046adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd); 2047adb54c29SNicholas Bellinger } 2048adb54c29SNicholas Bellinger 20493d73cf1aSSagi Grimberg static inline void 20503d73cf1aSSagi Grimberg isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 20513d73cf1aSSagi Grimberg struct ib_sig_domain *domain) 20529e961ae7SSagi Grimberg { 205378eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF; 20543d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC; 20553d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 20563d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed; 205778eda2bbSSagi Grimberg /* 205878eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future 205978eda2bbSSagi Grimberg * the target core would like to use it, we will take it 206078eda2bbSSagi Grimberg * from se_cmd. 206178eda2bbSSagi Grimberg */ 206278eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff; 206378eda2bbSSagi Grimberg domain->sig.dif.app_escape = true; 206478eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true; 206578eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 206678eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 206778eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true; 20683d73cf1aSSagi Grimberg }; 20693d73cf1aSSagi Grimberg 20703d73cf1aSSagi Grimberg static int 20713d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 20723d73cf1aSSagi Grimberg { 207338a2d0d4SChristoph Hellwig memset(sig_attrs, 0, sizeof(*sig_attrs)); 207438a2d0d4SChristoph Hellwig 20759e961ae7SSagi Grimberg switch (se_cmd->prot_op) { 20769e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT: 20779e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP: 207878eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 20793d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 20809e961ae7SSagi Grimberg break; 20819e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT: 20829e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP: 208378eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 20843d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 20859e961ae7SSagi Grimberg break; 20869e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS: 20879e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS: 20883d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 20893d73cf1aSSagi Grimberg isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 20909e961ae7SSagi Grimberg break; 20919e961ae7SSagi Grimberg default: 209224f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 20939e961ae7SSagi Grimberg return -EINVAL; 20949e961ae7SSagi Grimberg } 20959e961ae7SSagi Grimberg 209638a2d0d4SChristoph Hellwig sig_attrs->check_mask = 209738a2d0d4SChristoph Hellwig (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 209838a2d0d4SChristoph Hellwig (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 209938a2d0d4SChristoph Hellwig (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 21009e961ae7SSagi Grimberg return 0; 21019e961ae7SSagi Grimberg } 21029e961ae7SSagi Grimberg 21039e961ae7SSagi Grimberg static int 210438a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 210538a2d0d4SChristoph Hellwig struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 21069e961ae7SSagi Grimberg { 210738a2d0d4SChristoph Hellwig struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 210838a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 210938a2d0d4SChristoph Hellwig u8 port_num = conn->cm_id->port_num; 211038a2d0d4SChristoph Hellwig u64 addr; 211138a2d0d4SChristoph Hellwig u32 rkey, offset; 21129e961ae7SSagi Grimberg int ret; 21139e961ae7SSagi Grimberg 211438a2d0d4SChristoph Hellwig if (dir == DMA_FROM_DEVICE) { 211538a2d0d4SChristoph Hellwig addr = cmd->write_va; 211638a2d0d4SChristoph Hellwig rkey = cmd->write_stag; 211738a2d0d4SChristoph Hellwig offset = cmd->iscsi_cmd->write_data_done; 211838a2d0d4SChristoph Hellwig } else { 211938a2d0d4SChristoph Hellwig addr = cmd->read_va; 212038a2d0d4SChristoph Hellwig rkey = cmd->read_stag; 212138a2d0d4SChristoph Hellwig offset = 0; 212238a2d0d4SChristoph Hellwig } 212338a2d0d4SChristoph Hellwig 212438a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) { 212538a2d0d4SChristoph Hellwig struct ib_sig_attrs sig_attrs; 212638a2d0d4SChristoph Hellwig 21279e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 21289e961ae7SSagi Grimberg if (ret) 2129e3d7e4c3SSagi Grimberg return ret; 2130e3d7e4c3SSagi Grimberg 213138a2d0d4SChristoph Hellwig WARN_ON_ONCE(offset); 213238a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 213338a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 213438a2d0d4SChristoph Hellwig se_cmd->t_prot_sg, se_cmd->t_prot_nents, 213538a2d0d4SChristoph Hellwig &sig_attrs, addr, rkey, dir); 2136570db170SSagi Grimberg } else { 213738a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 213838a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, 213938a2d0d4SChristoph Hellwig offset, addr, rkey, dir); 214038a2d0d4SChristoph Hellwig } 214138a2d0d4SChristoph Hellwig if (ret < 0) { 214238a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 214338a2d0d4SChristoph Hellwig return ret; 21449e961ae7SSagi Grimberg } 21459e961ae7SSagi Grimberg 214638a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 214738a2d0d4SChristoph Hellwig if (ret < 0) 214838a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to post RDMA res\n", cmd); 214959464ef4SVu Pham return ret; 215059464ef4SVu Pham } 215159464ef4SVu Pham 215259464ef4SVu Pham static int 215390ecc6e2SVu Pham isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 215490ecc6e2SVu Pham { 215590ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd; 215659464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 21576700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 215838a2d0d4SChristoph Hellwig struct ib_cqe *cqe = NULL; 215938a2d0d4SChristoph Hellwig struct ib_send_wr *chain_wr = NULL; 216090ecc6e2SVu Pham int rc; 216190ecc6e2SVu Pham 216224f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 216390ecc6e2SVu Pham isert_cmd, se_cmd->data_length); 21644c22e07fSSagi Grimberg 216538a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) { 216638a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 216738a2d0d4SChristoph Hellwig cqe = &isert_cmd->tx_desc.tx_cqe; 216838a2d0d4SChristoph Hellwig } else { 2169b8d26b3bSNicholas Bellinger /* 2170b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach 2171b8d26b3bSNicholas Bellinger */ 21729e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd, 21739e961ae7SSagi Grimberg &isert_cmd->tx_desc); 217404d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2175b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header); 2176b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 217795b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd, 217868a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr); 21794366b19cSJenny Derzhavetz 21804366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 21814366b19cSJenny Derzhavetz if (rc) { 21824366b19cSJenny Derzhavetz isert_err("ib_post_recv failed with %d\n", rc); 21834366b19cSJenny Derzhavetz return rc; 21844366b19cSJenny Derzhavetz } 218538a2d0d4SChristoph Hellwig 218638a2d0d4SChristoph Hellwig chain_wr = &isert_cmd->tx_desc.send_wr; 21879e961ae7SSagi Grimberg } 2188b8d26b3bSNicholas Bellinger 2189*555a65f6SNicholas Bellinger rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2190*555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2191*555a65f6SNicholas Bellinger isert_cmd, rc); 2192*555a65f6SNicholas Bellinger return rc; 2193b8d26b3bSNicholas Bellinger } 2194b8d26b3bSNicholas Bellinger 2195b8d26b3bSNicholas Bellinger static int 2196b8d26b3bSNicholas Bellinger isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2197b8d26b3bSNicholas Bellinger { 2198d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2199*555a65f6SNicholas Bellinger int ret; 2200b8d26b3bSNicholas Bellinger 220124f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 220238a2d0d4SChristoph Hellwig isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2203b8d26b3bSNicholas Bellinger 220438a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2205*555a65f6SNicholas Bellinger ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 220638a2d0d4SChristoph Hellwig &isert_cmd->tx_desc.tx_cqe, NULL); 2207bdf20e72SSagi Grimberg 2208*555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2209*555a65f6SNicholas Bellinger isert_cmd, ret); 2210*555a65f6SNicholas Bellinger return ret; 2211b8d26b3bSNicholas Bellinger } 2212b8d26b3bSNicholas Bellinger 2213b8d26b3bSNicholas Bellinger static int 2214b8d26b3bSNicholas Bellinger isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2215b8d26b3bSNicholas Bellinger { 2216a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2217a4c15cd9SJenny Derzhavetz int ret = 0; 2218b8d26b3bSNicholas Bellinger 2219b8d26b3bSNicholas Bellinger switch (state) { 2220a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE: 2221a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 2222a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 2223a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 2224a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 2225a4c15cd9SJenny Derzhavetz break; 2226b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2227b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false); 2228b8d26b3bSNicholas Bellinger break; 2229b8d26b3bSNicholas Bellinger default: 223024f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state); 2231b8d26b3bSNicholas Bellinger ret = -EINVAL; 2232b8d26b3bSNicholas Bellinger break; 2233b8d26b3bSNicholas Bellinger } 2234b8d26b3bSNicholas Bellinger 2235b8d26b3bSNicholas Bellinger return ret; 2236b8d26b3bSNicholas Bellinger } 2237b8d26b3bSNicholas Bellinger 2238b8d26b3bSNicholas Bellinger static int 2239b8d26b3bSNicholas Bellinger isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2240b8d26b3bSNicholas Bellinger { 2241991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context; 2242b8d26b3bSNicholas Bellinger int ret; 2243b8d26b3bSNicholas Bellinger 2244b8d26b3bSNicholas Bellinger switch (state) { 2245b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP: 2246b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn); 2247991bb764SSagi Grimberg if (!ret) 2248991bb764SSagi Grimberg isert_conn->logout_posted = true; 2249b8d26b3bSNicholas Bellinger break; 2250b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN: 2251b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true); 2252b8d26b3bSNicholas Bellinger break; 2253b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP: 2254b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn); 2255b8d26b3bSNicholas Bellinger break; 2256b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT: 2257b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn); 2258b8d26b3bSNicholas Bellinger break; 2259adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP: 2260adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn); 2261adb54c29SNicholas Bellinger break; 2262b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS: 2263b8d26b3bSNicholas Bellinger /* 2264b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread 2265b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure. 2266b8d26b3bSNicholas Bellinger */ 2267b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd); 2268b8d26b3bSNicholas Bellinger break; 2269b8d26b3bSNicholas Bellinger default: 227024f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state); 2271b8d26b3bSNicholas Bellinger ret = -EINVAL; 2272b8d26b3bSNicholas Bellinger break; 2273b8d26b3bSNicholas Bellinger } 2274b8d26b3bSNicholas Bellinger 2275b8d26b3bSNicholas Bellinger return ret; 2276b8d26b3bSNicholas Bellinger } 2277b8d26b3bSNicholas Bellinger 2278ca6c1d82SSagi Grimberg struct rdma_cm_id * 2279ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np) 2280ca6c1d82SSagi Grimberg { 2281ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np; 2282ca6c1d82SSagi Grimberg struct rdma_cm_id *id; 2283ca6c1d82SSagi Grimberg struct sockaddr *sa; 2284ca6c1d82SSagi Grimberg int ret; 2285ca6c1d82SSagi Grimberg 2286ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr; 228724f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2288ca6c1d82SSagi Grimberg 2289fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2290ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC); 2291ca6c1d82SSagi Grimberg if (IS_ERR(id)) { 229224f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2293ca6c1d82SSagi Grimberg ret = PTR_ERR(id); 2294ca6c1d82SSagi Grimberg goto out; 2295ca6c1d82SSagi Grimberg } 229624f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context); 2297ca6c1d82SSagi Grimberg 2298ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa); 2299ca6c1d82SSagi Grimberg if (ret) { 230024f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret); 2301ca6c1d82SSagi Grimberg goto out_id; 2302ca6c1d82SSagi Grimberg } 2303ca6c1d82SSagi Grimberg 2304992607e8SSagi Grimberg ret = rdma_listen(id, 0); 2305ca6c1d82SSagi Grimberg if (ret) { 230624f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret); 2307ca6c1d82SSagi Grimberg goto out_id; 2308ca6c1d82SSagi Grimberg } 2309ca6c1d82SSagi Grimberg 2310ca6c1d82SSagi Grimberg return id; 2311ca6c1d82SSagi Grimberg out_id: 2312ca6c1d82SSagi Grimberg rdma_destroy_id(id); 2313ca6c1d82SSagi Grimberg out: 2314ca6c1d82SSagi Grimberg return ERR_PTR(ret); 2315ca6c1d82SSagi Grimberg } 2316ca6c1d82SSagi Grimberg 2317b8d26b3bSNicholas Bellinger static int 2318b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np, 231913a3cf08SAndy Grover struct sockaddr_storage *ksockaddr) 2320b8d26b3bSNicholas Bellinger { 2321b8d26b3bSNicholas Bellinger struct isert_np *isert_np; 2322b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid; 2323b8d26b3bSNicholas Bellinger int ret; 2324b8d26b3bSNicholas Bellinger 2325b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 232693b80f29SLeon Romanovsky if (!isert_np) 2327b8d26b3bSNicholas Bellinger return -ENOMEM; 232893b80f29SLeon Romanovsky 2329ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0); 2330ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex); 2331bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted); 2332bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending); 2333ca6c1d82SSagi Grimberg isert_np->np = np; 2334b8d26b3bSNicholas Bellinger 2335b8d26b3bSNicholas Bellinger /* 2336b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup 2337b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code.. 2338b8d26b3bSNicholas Bellinger */ 2339b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr, 234013a3cf08SAndy Grover sizeof(struct sockaddr_storage)); 2341b8d26b3bSNicholas Bellinger 2342ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np); 2343b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) { 2344b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid); 2345b8d26b3bSNicholas Bellinger goto out; 2346b8d26b3bSNicholas Bellinger } 2347b8d26b3bSNicholas Bellinger 2348ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid; 2349b8d26b3bSNicholas Bellinger np->np_context = isert_np; 2350b8d26b3bSNicholas Bellinger 2351b8d26b3bSNicholas Bellinger return 0; 2352b8d26b3bSNicholas Bellinger 2353b8d26b3bSNicholas Bellinger out: 2354b8d26b3bSNicholas Bellinger kfree(isert_np); 2355ca6c1d82SSagi Grimberg 2356b8d26b3bSNicholas Bellinger return ret; 2357b8d26b3bSNicholas Bellinger } 2358b8d26b3bSNicholas Bellinger 2359b8d26b3bSNicholas Bellinger static int 2360b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn) 2361b8d26b3bSNicholas Bellinger { 2362dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2363b8d26b3bSNicholas Bellinger struct rdma_conn_param cp; 2364b8d26b3bSNicholas Bellinger int ret; 236513bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr; 2366b8d26b3bSNicholas Bellinger 2367b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param)); 2368b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth; 2369b8d26b3bSNicholas Bellinger cp.retry_count = 7; 2370b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7; 2371b8d26b3bSNicholas Bellinger 237213bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2373422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2374422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv) 2375422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 237613bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr; 237713bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr); 237813bce482SJenny Derzhavetz 2379b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp); 2380b8d26b3bSNicholas Bellinger if (ret) { 238124f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret); 2382b8d26b3bSNicholas Bellinger return ret; 2383b8d26b3bSNicholas Bellinger } 2384b8d26b3bSNicholas Bellinger 2385b8d26b3bSNicholas Bellinger return 0; 2386b8d26b3bSNicholas Bellinger } 2387b8d26b3bSNicholas Bellinger 2388b8d26b3bSNicholas Bellinger static int 2389b8d26b3bSNicholas Bellinger isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2390b8d26b3bSNicholas Bellinger { 23916700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context; 2392b8d26b3bSNicholas Bellinger int ret; 2393b8d26b3bSNicholas Bellinger 239424f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn); 23952371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 23962371e5daSSagi Grimberg if (ret) { 239724f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n", 23982371e5daSSagi Grimberg isert_conn); 23992371e5daSSagi Grimberg return ret; 24002371e5daSSagi Grimberg } 24012371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp); 24022371e5daSSagi Grimberg 24036faaa85fSNicholas Bellinger /* 24046faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will 24056faaa85fSNicholas Bellinger * kick schedule_delayed_work(&conn->login_work) as the packet is 24066faaa85fSNicholas Bellinger * received, which turns this callback from iscsi_target_do_login_rx() 24076faaa85fSNicholas Bellinger * into a NOP. 24086faaa85fSNicholas Bellinger */ 24096faaa85fSNicholas Bellinger if (!login->first_request) 24106faaa85fSNicholas Bellinger return 0; 2411b8d26b3bSNicholas Bellinger 24122371e5daSSagi Grimberg isert_rx_login_req(isert_conn); 24132371e5daSSagi Grimberg 2414dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn); 2415dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2416b8d26b3bSNicholas Bellinger if (ret) 2417b8d26b3bSNicholas Bellinger return ret; 2418b8d26b3bSNicholas Bellinger 241924f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req); 24202371e5daSSagi Grimberg 2421b8d26b3bSNicholas Bellinger return 0; 2422b8d26b3bSNicholas Bellinger } 2423b8d26b3bSNicholas Bellinger 2424b8d26b3bSNicholas Bellinger static void 2425b8d26b3bSNicholas Bellinger isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2426b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn) 2427b8d26b3bSNicholas Bellinger { 2428dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id; 2429b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route; 2430b8d26b3bSNicholas Bellinger 2431b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family; 2432b8d26b3bSNicholas Bellinger 2433dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr; 2434dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr; 2435b8d26b3bSNicholas Bellinger } 2436b8d26b3bSNicholas Bellinger 2437b8d26b3bSNicholas Bellinger static int 2438b8d26b3bSNicholas Bellinger isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2439b8d26b3bSNicholas Bellinger { 24406700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2441b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn; 2442c6b8e918SSagi Grimberg int ret; 2443b8d26b3bSNicholas Bellinger 2444b8d26b3bSNicholas Bellinger accept_wait: 2445ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem); 2446c6b8e918SSagi Grimberg if (ret) 2447b8d26b3bSNicholas Bellinger return -ENODEV; 2448b8d26b3bSNicholas Bellinger 2449b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock); 2450e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2451b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 245211378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n", 2453e346ab34SSagi Grimberg np->np_thread_state); 2454e346ab34SSagi Grimberg /** 2455e346ab34SSagi Grimberg * No point in stalling here when np_thread 2456e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail 2457e346ab34SSagi Grimberg **/ 2458b8d26b3bSNicholas Bellinger return -ENODEV; 2459b8d26b3bSNicholas Bellinger } 2460b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock); 2461b8d26b3bSNicholas Bellinger 2462ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2463bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) { 2464ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2465b8d26b3bSNicholas Bellinger goto accept_wait; 2466b8d26b3bSNicholas Bellinger } 2467bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending, 2468bd379220SJenny Derzhavetz struct isert_conn, node); 2469bd379220SJenny Derzhavetz list_del_init(&isert_conn->node); 2470ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2471b8d26b3bSNicholas Bellinger 2472b8d26b3bSNicholas Bellinger conn->context = isert_conn; 2473b8d26b3bSNicholas Bellinger isert_conn->conn = conn; 2474aea92980SJenny Derzhavetz isert_conn->state = ISER_CONN_BOUND; 2475b8d26b3bSNicholas Bellinger 2476b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn); 2477b8d26b3bSNicholas Bellinger 247824f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn); 24792371e5daSSagi Grimberg 2480b8d26b3bSNicholas Bellinger return 0; 2481b8d26b3bSNicholas Bellinger } 2482b8d26b3bSNicholas Bellinger 2483b8d26b3bSNicholas Bellinger static void 2484b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np) 2485b8d26b3bSNicholas Bellinger { 24866700425eSSagi Grimberg struct isert_np *isert_np = np->np_context; 2487268e6811SSagi Grimberg struct isert_conn *isert_conn, *n; 2488b8d26b3bSNicholas Bellinger 2489ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id) 2490ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id); 2491b8d26b3bSNicholas Bellinger 2492268e6811SSagi Grimberg /* 2493268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure 2494268e6811SSagi Grimberg * that at this point we don't have hanging connections that 2495268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login 2496268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled 2497bd379220SJenny Derzhavetz * up in accepted and pending lists. 2498268e6811SSagi Grimberg */ 2499ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex); 2500bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) { 2501bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n"); 2502268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n, 2503bd379220SJenny Derzhavetz &isert_np->pending, 2504bd379220SJenny Derzhavetz node) { 2505bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n", 2506bd379220SJenny Derzhavetz isert_conn, isert_conn->state); 2507bd379220SJenny Derzhavetz isert_connect_release(isert_conn); 2508bd379220SJenny Derzhavetz } 2509bd379220SJenny Derzhavetz } 2510bd379220SJenny Derzhavetz 2511bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) { 2512bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n"); 2513bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n, 2514bd379220SJenny Derzhavetz &isert_np->accepted, 2515bd379220SJenny Derzhavetz node) { 251624f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n", 2517268e6811SSagi Grimberg isert_conn, isert_conn->state); 2518268e6811SSagi Grimberg isert_connect_release(isert_conn); 2519268e6811SSagi Grimberg } 2520268e6811SSagi Grimberg } 2521ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex); 2522268e6811SSagi Grimberg 2523b8d26b3bSNicholas Bellinger np->np_context = NULL; 2524b8d26b3bSNicholas Bellinger kfree(isert_np); 2525b8d26b3bSNicholas Bellinger } 2526b8d26b3bSNicholas Bellinger 2527b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work) 2528b02efbfcSSagi Grimberg { 2529b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work, 2530b02efbfcSSagi Grimberg struct isert_conn, 2531b02efbfcSSagi Grimberg release_work); 2532b02efbfcSSagi Grimberg 253324f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn); 2534b02efbfcSSagi Grimberg 2535dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2536b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN; 2537dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2538b02efbfcSSagi Grimberg 253924f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn); 2540b02efbfcSSagi Grimberg isert_put_conn(isert_conn); 2541b02efbfcSSagi Grimberg } 2542b02efbfcSSagi Grimberg 2543bdf20e72SSagi Grimberg static void 2544991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn) 2545991bb764SSagi Grimberg { 2546991bb764SSagi Grimberg struct iscsi_conn *conn = isert_conn->conn; 2547991bb764SSagi Grimberg 25484c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn); 25494c22e07fSSagi Grimberg 2550991bb764SSagi Grimberg if (isert_conn->logout_posted) { 255124f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2552991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp, 2553991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ); 2554991bb764SSagi Grimberg } 2555991bb764SSagi Grimberg } 2556991bb764SSagi Grimberg 2557991bb764SSagi Grimberg static void 2558c7e160eeSSagi Grimberg isert_wait4cmds(struct iscsi_conn *conn) 2559c7e160eeSSagi Grimberg { 25604c22e07fSSagi Grimberg isert_info("iscsi_conn %p\n", conn); 25614c22e07fSSagi Grimberg 2562c7e160eeSSagi Grimberg if (conn->sess) { 2563c7e160eeSSagi Grimberg target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2564c7e160eeSSagi Grimberg target_wait_for_sess_cmds(conn->sess->se_sess); 2565c7e160eeSSagi Grimberg } 2566c7e160eeSSagi Grimberg } 2567c7e160eeSSagi Grimberg 25683e03c4b0SJenny Derzhavetz /** 25693e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for 25703e03c4b0SJenny Derzhavetz * unsolicitate dataout 25713e03c4b0SJenny Derzhavetz * @conn: iscsi connection 25723e03c4b0SJenny Derzhavetz * 25733e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited 25743e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those 25753e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds 25763e03c4b0SJenny Derzhavetz */ 25773e03c4b0SJenny Derzhavetz static void 25783e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 25793e03c4b0SJenny Derzhavetz { 25803e03c4b0SJenny Derzhavetz struct iscsi_cmd *cmd, *tmp; 25813e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list); 25823e03c4b0SJenny Derzhavetz 25833e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock); 25843e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 25853e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 25863e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 25873e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length)) 25883e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 25893e03c4b0SJenny Derzhavetz } 25903e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock); 25913e03c4b0SJenny Derzhavetz 25923e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 25933e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node); 25943e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) { 25953e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 25963e03c4b0SJenny Derzhavetz 25973e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd); 25983e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true); 25993e03c4b0SJenny Derzhavetz } 26003e03c4b0SJenny Derzhavetz } 26013e03c4b0SJenny Derzhavetz } 26023e03c4b0SJenny Derzhavetz 2603defd8848SNicholas Bellinger static void isert_wait_conn(struct iscsi_conn *conn) 2604b8d26b3bSNicholas Bellinger { 2605b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2606b8d26b3bSNicholas Bellinger 26074c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn); 2608b8d26b3bSNicholas Bellinger 2609dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex); 2610954f2372SSagi Grimberg isert_conn_terminate(isert_conn); 2611dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex); 2612b2cb9649SNicholas Bellinger 2613572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 26143e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn); 26153e03c4b0SJenny Derzhavetz isert_wait4cmds(conn); 2616991bb764SSagi Grimberg isert_wait4logout(isert_conn); 2617954f2372SSagi Grimberg 2618b02efbfcSSagi Grimberg queue_work(isert_release_wq, &isert_conn->release_work); 2619defd8848SNicholas Bellinger } 2620defd8848SNicholas Bellinger 2621defd8848SNicholas Bellinger static void isert_free_conn(struct iscsi_conn *conn) 2622defd8848SNicholas Bellinger { 2623defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context; 2624b8d26b3bSNicholas Bellinger 2625572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp); 2626b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn); 2627b8d26b3bSNicholas Bellinger } 2628b8d26b3bSNicholas Bellinger 2629187e84f7SVarun Prakash static void isert_get_rx_pdu(struct iscsi_conn *conn) 2630187e84f7SVarun Prakash { 2631187e84f7SVarun Prakash struct completion comp; 2632187e84f7SVarun Prakash 2633187e84f7SVarun Prakash init_completion(&comp); 2634187e84f7SVarun Prakash 2635187e84f7SVarun Prakash wait_for_completion_interruptible(&comp); 2636187e84f7SVarun Prakash } 2637187e84f7SVarun Prakash 2638b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = { 2639b8d26b3bSNicholas Bellinger .name = "IB/iSER", 2640b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND, 2641bd027d85SNicholas Bellinger .rdma_shutdown = true, 2642d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd), 2643b8d26b3bSNicholas Bellinger .owner = THIS_MODULE, 2644b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np, 2645b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np, 2646b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np, 2647defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn, 2648b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn, 2649b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx, 2650b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx, 2651b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue, 2652b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue, 2653b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout, 2654b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain, 2655b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response, 2656131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task, 2657187e84f7SVarun Prakash .iscsit_get_rx_pdu = isert_get_rx_pdu, 2658e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2659b8d26b3bSNicholas Bellinger }; 2660b8d26b3bSNicholas Bellinger 2661b8d26b3bSNicholas Bellinger static int __init isert_init(void) 2662b8d26b3bSNicholas Bellinger { 2663b8d26b3bSNicholas Bellinger int ret; 2664b8d26b3bSNicholas Bellinger 2665631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq", 2666631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0); 2667b8d26b3bSNicholas Bellinger if (!isert_comp_wq) { 266824f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n"); 26696f0fae3dSSagi Grimberg return -ENOMEM; 2670b8d26b3bSNicholas Bellinger } 2671b8d26b3bSNicholas Bellinger 2672b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2673b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE); 2674b02efbfcSSagi Grimberg if (!isert_release_wq) { 267524f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n"); 2676b02efbfcSSagi Grimberg ret = -ENOMEM; 2677b02efbfcSSagi Grimberg goto destroy_comp_wq; 2678b02efbfcSSagi Grimberg } 2679b02efbfcSSagi Grimberg 2680b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport); 268124f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2682b02efbfcSSagi Grimberg 2683b8d26b3bSNicholas Bellinger return 0; 2684b8d26b3bSNicholas Bellinger 2685b02efbfcSSagi Grimberg destroy_comp_wq: 2686b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq); 26876f0fae3dSSagi Grimberg 2688b8d26b3bSNicholas Bellinger return ret; 2689b8d26b3bSNicholas Bellinger } 2690b8d26b3bSNicholas Bellinger 2691b8d26b3bSNicholas Bellinger static void __exit isert_exit(void) 2692b8d26b3bSNicholas Bellinger { 2693f5ebec96SSagi Grimberg flush_scheduled_work(); 2694b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq); 2695b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq); 2696b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport); 26974c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2698b8d26b3bSNicholas Bellinger } 2699b8d26b3bSNicholas Bellinger 2700b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 27019e35eff4SSagi Grimberg MODULE_VERSION("1.0"); 2702b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2703b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL"); 2704b8d26b3bSNicholas Bellinger 2705b8d26b3bSNicholas Bellinger module_init(isert_init); 2706b8d26b3bSNicholas Bellinger module_exit(isert_exit); 2707