1c942fddfSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2b8d26b3bSNicholas Bellinger /*******************************************************************************
3b8d26b3bSNicholas Bellinger * This file contains iSCSI extentions for RDMA (iSER) Verbs
4b8d26b3bSNicholas Bellinger *
54c76251eSNicholas Bellinger * (c) Copyright 2013 Datera, Inc.
6b8d26b3bSNicholas Bellinger *
7b8d26b3bSNicholas Bellinger * Nicholas A. Bellinger <nab@linux-iscsi.org>
8b8d26b3bSNicholas Bellinger *
9b8d26b3bSNicholas Bellinger ****************************************************************************/
10b8d26b3bSNicholas Bellinger
11b8d26b3bSNicholas Bellinger #include <linux/string.h>
12b8d26b3bSNicholas Bellinger #include <linux/module.h>
13b8d26b3bSNicholas Bellinger #include <linux/scatterlist.h>
14b8d26b3bSNicholas Bellinger #include <linux/socket.h>
15b8d26b3bSNicholas Bellinger #include <linux/in.h>
16b8d26b3bSNicholas Bellinger #include <linux/in6.h>
17b8d26b3bSNicholas Bellinger #include <rdma/ib_verbs.h>
188094ba0aSLeon Romanovsky #include <rdma/ib_cm.h>
19b8d26b3bSNicholas Bellinger #include <rdma/rdma_cm.h>
20b8d26b3bSNicholas Bellinger #include <target/target_core_base.h>
21b8d26b3bSNicholas Bellinger #include <target/target_core_fabric.h>
22b8d26b3bSNicholas Bellinger #include <target/iscsi/iscsi_transport.h>
23531b7bf4SSagi Grimberg #include <linux/semaphore.h>
24b8d26b3bSNicholas Bellinger
25b8d26b3bSNicholas Bellinger #include "ib_isert.h"
26b8d26b3bSNicholas Bellinger
2745678b6bSSagi Grimberg static int isert_debug_level;
2824f412ddSSagi Grimberg module_param_named(debug_level, isert_debug_level, int, 0644);
2924f412ddSSagi Grimberg MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
3024f412ddSSagi Grimberg
31dae7a75fSMax Gurtovoy static int isert_sg_tablesize_set(const char *val,
32dae7a75fSMax Gurtovoy const struct kernel_param *kp);
33dae7a75fSMax Gurtovoy static const struct kernel_param_ops sg_tablesize_ops = {
34dae7a75fSMax Gurtovoy .set = isert_sg_tablesize_set,
35dae7a75fSMax Gurtovoy .get = param_get_int,
36dae7a75fSMax Gurtovoy };
37dae7a75fSMax Gurtovoy
38109d19a5SMax Gurtovoy static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE;
39dae7a75fSMax Gurtovoy module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644);
40dae7a75fSMax Gurtovoy MODULE_PARM_DESC(sg_tablesize,
41109d19a5SMax Gurtovoy "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)");
42dae7a75fSMax Gurtovoy
43b8d26b3bSNicholas Bellinger static DEFINE_MUTEX(device_list_mutex);
44b8d26b3bSNicholas Bellinger static LIST_HEAD(device_list);
45549f39a5STetsuo Handa static struct workqueue_struct *isert_login_wq;
46b8d26b3bSNicholas Bellinger static struct workqueue_struct *isert_comp_wq;
47b02efbfcSSagi Grimberg static struct workqueue_struct *isert_release_wq;
48b8d26b3bSNicholas Bellinger
49f93f3a70SSagi Grimberg static int
50be36d683SMax Gurtovoy isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd);
512371e5daSSagi Grimberg static int
529679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn);
532371e5daSSagi Grimberg static int
542371e5daSSagi Grimberg isert_rdma_accept(struct isert_conn *isert_conn);
55ca6c1d82SSagi Grimberg struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
56d40945d8SVu Pham
572f1b6b7dSSagi Grimberg static void isert_release_work(struct work_struct *work);
589679cc51SChristoph Hellwig static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
599679cc51SChristoph Hellwig static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
609679cc51SChristoph Hellwig static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
619679cc51SChristoph Hellwig static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
622f1b6b7dSSagi Grimberg
isert_sg_tablesize_set(const char * val,const struct kernel_param * kp)63dae7a75fSMax Gurtovoy static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp)
64dae7a75fSMax Gurtovoy {
65dae7a75fSMax Gurtovoy int n = 0, ret;
66dae7a75fSMax Gurtovoy
67dae7a75fSMax Gurtovoy ret = kstrtoint(val, 10, &n);
68dae7a75fSMax Gurtovoy if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE ||
69dae7a75fSMax Gurtovoy n > ISCSI_ISER_MAX_SG_TABLESIZE)
70dae7a75fSMax Gurtovoy return -EINVAL;
71dae7a75fSMax Gurtovoy
72dae7a75fSMax Gurtovoy return param_set_int(val, kp);
73dae7a75fSMax Gurtovoy }
74dae7a75fSMax Gurtovoy
75302cc7c3SSagi Grimberg static inline bool
isert_prot_cmd(struct isert_conn * conn,struct se_cmd * cmd)76302cc7c3SSagi Grimberg isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
77302cc7c3SSagi Grimberg {
7823a548eeSSagi Grimberg return (conn->pi_support &&
79302cc7c3SSagi Grimberg cmd->prot_op != TARGET_PROT_NORMAL);
80302cc7c3SSagi Grimberg }
81302cc7c3SSagi Grimberg
82d40945d8SVu Pham static void
isert_qp_event_callback(struct ib_event * e,void * context)83b8d26b3bSNicholas Bellinger isert_qp_event_callback(struct ib_event *e, void *context)
84b8d26b3bSNicholas Bellinger {
856700425eSSagi Grimberg struct isert_conn *isert_conn = context;
86b8d26b3bSNicholas Bellinger
87ea8a1616SSagi Grimberg isert_err("%s (%d): conn %p\n",
88ea8a1616SSagi Grimberg ib_event_msg(e->event), e->event, isert_conn);
89ea8a1616SSagi Grimberg
90b8d26b3bSNicholas Bellinger switch (e->event) {
91b8d26b3bSNicholas Bellinger case IB_EVENT_COMM_EST:
92dac6ab30SSagi Grimberg rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
93b8d26b3bSNicholas Bellinger break;
94b8d26b3bSNicholas Bellinger default:
95b8d26b3bSNicholas Bellinger break;
96b8d26b3bSNicholas Bellinger }
97b8d26b3bSNicholas Bellinger }
98b8d26b3bSNicholas Bellinger
9940fc069aSSagi Grimberg static struct ib_qp *
isert_create_qp(struct isert_conn * isert_conn,struct rdma_cm_id * cma_id)10040fc069aSSagi Grimberg isert_create_qp(struct isert_conn *isert_conn,
10140fc069aSSagi Grimberg struct rdma_cm_id *cma_id)
10240fc069aSSagi Grimberg {
103c6e66307SYamin Friedman u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2;
104dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
105c6e66307SYamin Friedman struct ib_device *ib_dev = device->ib_device;
10640fc069aSSagi Grimberg struct ib_qp_init_attr attr;
107317000b9SMax Gurtovoy int ret, factor;
108b8d26b3bSNicholas Bellinger
109c6e66307SYamin Friedman isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE);
110c6e66307SYamin Friedman if (IS_ERR(isert_conn->cq)) {
111c6e66307SYamin Friedman isert_err("Unable to allocate cq\n");
112c6e66307SYamin Friedman ret = PTR_ERR(isert_conn->cq);
113c6e66307SYamin Friedman return ERR_PTR(ret);
114c6e66307SYamin Friedman }
115c6e66307SYamin Friedman isert_conn->cq_size = cq_size;
116c6e66307SYamin Friedman
117b8d26b3bSNicholas Bellinger memset(&attr, 0, sizeof(struct ib_qp_init_attr));
118b8d26b3bSNicholas Bellinger attr.event_handler = isert_qp_event_callback;
119b8d26b3bSNicholas Bellinger attr.qp_context = isert_conn;
120c6e66307SYamin Friedman attr.send_cq = isert_conn->cq;
121c6e66307SYamin Friedman attr.recv_cq = isert_conn->cq;
122572a1434SSagi Grimberg attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
123bdf20e72SSagi Grimberg attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
124317000b9SMax Gurtovoy factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num,
125dae7a75fSMax Gurtovoy isert_sg_tablesize);
126317000b9SMax Gurtovoy attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor;
12733023fb8SSteve Wise attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
128b8d26b3bSNicholas Bellinger attr.cap.max_recv_sge = 1;
129b8d26b3bSNicholas Bellinger attr.sq_sig_type = IB_SIGNAL_REQ_WR;
130b8d26b3bSNicholas Bellinger attr.qp_type = IB_QPT_RC;
131570db170SSagi Grimberg if (device->pi_capable)
132c0a6cbb9SIsrael Rukshin attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
133b8d26b3bSNicholas Bellinger
13467cb3949SSagi Grimberg ret = rdma_create_qp(cma_id, device->pd, &attr);
135b8d26b3bSNicholas Bellinger if (ret) {
13624f412ddSSagi Grimberg isert_err("rdma_create_qp failed for cma_id %d\n", ret);
137c6e66307SYamin Friedman ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
138c6e66307SYamin Friedman
13940fc069aSSagi Grimberg return ERR_PTR(ret);
14040fc069aSSagi Grimberg }
14140fc069aSSagi Grimberg
14240fc069aSSagi Grimberg return cma_id->qp;
14340fc069aSSagi Grimberg }
14440fc069aSSagi Grimberg
14540fc069aSSagi Grimberg static int
isert_alloc_rx_descriptors(struct isert_conn * isert_conn)146b8d26b3bSNicholas Bellinger isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
147b8d26b3bSNicholas Bellinger {
148dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
14967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device;
150b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc;
151b8d26b3bSNicholas Bellinger struct ib_sge *rx_sg;
152b8d26b3bSNicholas Bellinger u64 dma_addr;
153b8d26b3bSNicholas Bellinger int i, j;
154b8d26b3bSNicholas Bellinger
1556396bb22SKees Cook isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS,
1566396bb22SKees Cook sizeof(struct iser_rx_desc),
1576396bb22SKees Cook GFP_KERNEL);
158dac6ab30SSagi Grimberg if (!isert_conn->rx_descs)
15993b80f29SLeon Romanovsky return -ENOMEM;
160b8d26b3bSNicholas Bellinger
161dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs;
162b8d26b3bSNicholas Bellinger
163b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
1640b089c1eSSagi Grimberg dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf,
1650b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
166b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr))
167b8d26b3bSNicholas Bellinger goto dma_map_fail;
168b8d26b3bSNicholas Bellinger
169b8d26b3bSNicholas Bellinger rx_desc->dma_addr = dma_addr;
170b8d26b3bSNicholas Bellinger
171b8d26b3bSNicholas Bellinger rx_sg = &rx_desc->rx_sg;
1720b089c1eSSagi Grimberg rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc);
173b8d26b3bSNicholas Bellinger rx_sg->length = ISER_RX_PAYLOAD_SIZE;
17434efc7dfSJason Gunthorpe rx_sg->lkey = device->pd->local_dma_lkey;
1759679cc51SChristoph Hellwig rx_desc->rx_cqe.done = isert_recv_done;
176b8d26b3bSNicholas Bellinger }
177b8d26b3bSNicholas Bellinger
178b8d26b3bSNicholas Bellinger return 0;
179b8d26b3bSNicholas Bellinger
180b8d26b3bSNicholas Bellinger dma_map_fail:
181dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs;
182b8d26b3bSNicholas Bellinger for (j = 0; j < i; j++, rx_desc++) {
183b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
1840b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
185b8d26b3bSNicholas Bellinger }
186dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs);
187dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL;
1884c22e07fSSagi Grimberg isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
189b8d26b3bSNicholas Bellinger return -ENOMEM;
190b8d26b3bSNicholas Bellinger }
191b8d26b3bSNicholas Bellinger
192b8d26b3bSNicholas Bellinger static void
isert_free_rx_descriptors(struct isert_conn * isert_conn)193b8d26b3bSNicholas Bellinger isert_free_rx_descriptors(struct isert_conn *isert_conn)
194b8d26b3bSNicholas Bellinger {
195dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device;
196b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc;
197b8d26b3bSNicholas Bellinger int i;
198b8d26b3bSNicholas Bellinger
199dac6ab30SSagi Grimberg if (!isert_conn->rx_descs)
200b8d26b3bSNicholas Bellinger return;
201b8d26b3bSNicholas Bellinger
202dac6ab30SSagi Grimberg rx_desc = isert_conn->rx_descs;
203b8d26b3bSNicholas Bellinger for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
204b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
2050b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
206b8d26b3bSNicholas Bellinger }
207b8d26b3bSNicholas Bellinger
208dac6ab30SSagi Grimberg kfree(isert_conn->rx_descs);
209dac6ab30SSagi Grimberg isert_conn->rx_descs = NULL;
210b8d26b3bSNicholas Bellinger }
211b8d26b3bSNicholas Bellinger
212172369c5SSagi Grimberg static int
isert_create_device_ib_res(struct isert_device * device)213172369c5SSagi Grimberg isert_create_device_ib_res(struct isert_device *device)
214172369c5SSagi Grimberg {
2154a061b28SOr Gerlitz struct ib_device *ib_dev = device->ib_device;
216fd8205e8SSagi Grimberg int ret;
217172369c5SSagi Grimberg
21833023fb8SSteve Wise isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
21933023fb8SSteve Wise ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
2204a061b28SOr Gerlitz isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
221172369c5SSagi Grimberg
222ed082d36SChristoph Hellwig device->pd = ib_alloc_pd(ib_dev, 0);
22367cb3949SSagi Grimberg if (IS_ERR(device->pd)) {
22467cb3949SSagi Grimberg ret = PTR_ERR(device->pd);
22567cb3949SSagi Grimberg isert_err("failed to allocate pd, device %p, ret=%d\n",
22667cb3949SSagi Grimberg device, ret);
227c6e66307SYamin Friedman return ret;
22867cb3949SSagi Grimberg }
22967cb3949SSagi Grimberg
230172369c5SSagi Grimberg /* Check signature cap */
231e945c653SJason Gunthorpe if (ib_dev->attrs.kernel_cap_flags & IBK_INTEGRITY_HANDOVER)
232a6dc16b6SMax Gurtovoy device->pi_capable = true;
233a6dc16b6SMax Gurtovoy else
234a6dc16b6SMax Gurtovoy device->pi_capable = false;
23567cb3949SSagi Grimberg
236b8d26b3bSNicholas Bellinger return 0;
237b8d26b3bSNicholas Bellinger }
238b8d26b3bSNicholas Bellinger
239b8d26b3bSNicholas Bellinger static void
isert_free_device_ib_res(struct isert_device * device)240b8d26b3bSNicholas Bellinger isert_free_device_ib_res(struct isert_device *device)
241b8d26b3bSNicholas Bellinger {
24224f412ddSSagi Grimberg isert_info("device %p\n", device);
243b8d26b3bSNicholas Bellinger
24467cb3949SSagi Grimberg ib_dealloc_pd(device->pd);
245b8d26b3bSNicholas Bellinger }
246b8d26b3bSNicholas Bellinger
247b8d26b3bSNicholas Bellinger static void
isert_device_put(struct isert_device * device)248cf8ae958SSagi Grimberg isert_device_put(struct isert_device *device)
249b8d26b3bSNicholas Bellinger {
250b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex);
251b8d26b3bSNicholas Bellinger device->refcount--;
2524c22e07fSSagi Grimberg isert_info("device %p refcount %d\n", device, device->refcount);
253b8d26b3bSNicholas Bellinger if (!device->refcount) {
254b8d26b3bSNicholas Bellinger isert_free_device_ib_res(device);
255b8d26b3bSNicholas Bellinger list_del(&device->dev_node);
256b8d26b3bSNicholas Bellinger kfree(device);
257b8d26b3bSNicholas Bellinger }
258b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex);
259b8d26b3bSNicholas Bellinger }
260b8d26b3bSNicholas Bellinger
261b8d26b3bSNicholas Bellinger static struct isert_device *
isert_device_get(struct rdma_cm_id * cma_id)262cf8ae958SSagi Grimberg isert_device_get(struct rdma_cm_id *cma_id)
263b8d26b3bSNicholas Bellinger {
264b8d26b3bSNicholas Bellinger struct isert_device *device;
265b8d26b3bSNicholas Bellinger int ret;
266b8d26b3bSNicholas Bellinger
267b8d26b3bSNicholas Bellinger mutex_lock(&device_list_mutex);
268b8d26b3bSNicholas Bellinger list_for_each_entry(device, &device_list, dev_node) {
269b8d26b3bSNicholas Bellinger if (device->ib_device->node_guid == cma_id->device->node_guid) {
270b8d26b3bSNicholas Bellinger device->refcount++;
2714c22e07fSSagi Grimberg isert_info("Found iser device %p refcount %d\n",
2724c22e07fSSagi Grimberg device, device->refcount);
273b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex);
274b8d26b3bSNicholas Bellinger return device;
275b8d26b3bSNicholas Bellinger }
276b8d26b3bSNicholas Bellinger }
277b8d26b3bSNicholas Bellinger
278b8d26b3bSNicholas Bellinger device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
279b8d26b3bSNicholas Bellinger if (!device) {
280b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex);
281b8d26b3bSNicholas Bellinger return ERR_PTR(-ENOMEM);
282b8d26b3bSNicholas Bellinger }
283b8d26b3bSNicholas Bellinger
284b8d26b3bSNicholas Bellinger INIT_LIST_HEAD(&device->dev_node);
285b8d26b3bSNicholas Bellinger
286b8d26b3bSNicholas Bellinger device->ib_device = cma_id->device;
287b8d26b3bSNicholas Bellinger ret = isert_create_device_ib_res(device);
288b8d26b3bSNicholas Bellinger if (ret) {
289b8d26b3bSNicholas Bellinger kfree(device);
290b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex);
291b8d26b3bSNicholas Bellinger return ERR_PTR(ret);
292b8d26b3bSNicholas Bellinger }
293b8d26b3bSNicholas Bellinger
294b8d26b3bSNicholas Bellinger device->refcount++;
295b8d26b3bSNicholas Bellinger list_add_tail(&device->dev_node, &device_list);
2964c22e07fSSagi Grimberg isert_info("Created a new iser device %p refcount %d\n",
2974c22e07fSSagi Grimberg device, device->refcount);
298b8d26b3bSNicholas Bellinger mutex_unlock(&device_list_mutex);
299b8d26b3bSNicholas Bellinger
300b8d26b3bSNicholas Bellinger return device;
301b8d26b3bSNicholas Bellinger }
302b8d26b3bSNicholas Bellinger
30359464ef4SVu Pham static void
isert_init_conn(struct isert_conn * isert_conn)304ae9ea9edSSagi Grimberg isert_init_conn(struct isert_conn *isert_conn)
305ae9ea9edSSagi Grimberg {
306ae9ea9edSSagi Grimberg isert_conn->state = ISER_CONN_INIT;
307bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_conn->node);
308dac6ab30SSagi Grimberg init_completion(&isert_conn->login_comp);
309ae9ea9edSSagi Grimberg init_completion(&isert_conn->login_req_comp);
31063b268d2SRaju Rangoju init_waitqueue_head(&isert_conn->rem_wait);
311dac6ab30SSagi Grimberg kref_init(&isert_conn->kref);
312dac6ab30SSagi Grimberg mutex_init(&isert_conn->mutex);
3132f1b6b7dSSagi Grimberg INIT_WORK(&isert_conn->release_work, isert_release_work);
314ae9ea9edSSagi Grimberg }
315ae9ea9edSSagi Grimberg
316ae9ea9edSSagi Grimberg static void
isert_free_login_buf(struct isert_conn * isert_conn)317ae9ea9edSSagi Grimberg isert_free_login_buf(struct isert_conn *isert_conn)
318ae9ea9edSSagi Grimberg {
319dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->device->ib_device;
320ae9ea9edSSagi Grimberg
321ae9ea9edSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
322ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
3235adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf);
3245adabdd1SChristoph Hellwig
3250b089c1eSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr,
3260b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
3270b089c1eSSagi Grimberg kfree(isert_conn->login_desc);
328ae9ea9edSSagi Grimberg }
329ae9ea9edSSagi Grimberg
330ae9ea9edSSagi Grimberg static int
isert_alloc_login_buf(struct isert_conn * isert_conn,struct ib_device * ib_dev)331ae9ea9edSSagi Grimberg isert_alloc_login_buf(struct isert_conn *isert_conn,
332ae9ea9edSSagi Grimberg struct ib_device *ib_dev)
333ae9ea9edSSagi Grimberg {
334ae9ea9edSSagi Grimberg int ret;
335ae9ea9edSSagi Grimberg
3360b089c1eSSagi Grimberg isert_conn->login_desc = kzalloc(sizeof(*isert_conn->login_desc),
3375adabdd1SChristoph Hellwig GFP_KERNEL);
3380b089c1eSSagi Grimberg if (!isert_conn->login_desc)
339ae9ea9edSSagi Grimberg return -ENOMEM;
340ae9ea9edSSagi Grimberg
3410b089c1eSSagi Grimberg isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev,
3420b089c1eSSagi Grimberg isert_conn->login_desc->buf,
3430b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
3440b089c1eSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr);
345ae9ea9edSSagi Grimberg if (ret) {
3460b089c1eSSagi Grimberg isert_err("login_desc dma mapping error: %d\n", ret);
3470b089c1eSSagi Grimberg isert_conn->login_desc->dma_addr = 0;
3480b089c1eSSagi Grimberg goto out_free_login_desc;
3495adabdd1SChristoph Hellwig }
3505adabdd1SChristoph Hellwig
3515adabdd1SChristoph Hellwig isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
3525adabdd1SChristoph Hellwig if (!isert_conn->login_rsp_buf) {
3531d5840c9SWei Yongjun ret = -ENOMEM;
3540b089c1eSSagi Grimberg goto out_unmap_login_desc;
355ae9ea9edSSagi Grimberg }
356ae9ea9edSSagi Grimberg
357ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
3585adabdd1SChristoph Hellwig isert_conn->login_rsp_buf,
359ed1083b2SChristoph Hellwig ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
360ae9ea9edSSagi Grimberg ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
361ae9ea9edSSagi Grimberg if (ret) {
362ae9ea9edSSagi Grimberg isert_err("login_rsp_dma mapping error: %d\n", ret);
363ae9ea9edSSagi Grimberg isert_conn->login_rsp_dma = 0;
3645adabdd1SChristoph Hellwig goto out_free_login_rsp_buf;
365ae9ea9edSSagi Grimberg }
366ae9ea9edSSagi Grimberg
367ae9ea9edSSagi Grimberg return 0;
368ae9ea9edSSagi Grimberg
3695adabdd1SChristoph Hellwig out_free_login_rsp_buf:
3705adabdd1SChristoph Hellwig kfree(isert_conn->login_rsp_buf);
3710b089c1eSSagi Grimberg out_unmap_login_desc:
3720b089c1eSSagi Grimberg ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr,
3730b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
3740b089c1eSSagi Grimberg out_free_login_desc:
3750b089c1eSSagi Grimberg kfree(isert_conn->login_desc);
376ae9ea9edSSagi Grimberg return ret;
377ae9ea9edSSagi Grimberg }
378ae9ea9edSSagi Grimberg
379422bd0acSJenny Derzhavetz static void
isert_set_nego_params(struct isert_conn * isert_conn,struct rdma_conn_param * param)380422bd0acSJenny Derzhavetz isert_set_nego_params(struct isert_conn *isert_conn,
381422bd0acSJenny Derzhavetz struct rdma_conn_param *param)
382422bd0acSJenny Derzhavetz {
383422bd0acSJenny Derzhavetz struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;
384422bd0acSJenny Derzhavetz
385422bd0acSJenny Derzhavetz /* Set max inflight RDMA READ requests */
386422bd0acSJenny Derzhavetz isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
387422bd0acSJenny Derzhavetz attr->max_qp_init_rd_atom);
388422bd0acSJenny Derzhavetz isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
389422bd0acSJenny Derzhavetz
390422bd0acSJenny Derzhavetz if (param->private_data) {
391422bd0acSJenny Derzhavetz u8 flags = *(u8 *)param->private_data;
392422bd0acSJenny Derzhavetz
393422bd0acSJenny Derzhavetz /*
394422bd0acSJenny Derzhavetz * use remote invalidation if the both initiator
395422bd0acSJenny Derzhavetz * and the HCA support it
396422bd0acSJenny Derzhavetz */
397422bd0acSJenny Derzhavetz isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
398422bd0acSJenny Derzhavetz (attr->device_cap_flags &
399422bd0acSJenny Derzhavetz IB_DEVICE_MEM_MGT_EXTENSIONS);
400422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv)
401422bd0acSJenny Derzhavetz isert_info("Using remote invalidation\n");
402422bd0acSJenny Derzhavetz }
403422bd0acSJenny Derzhavetz }
404422bd0acSJenny Derzhavetz
405c6e66307SYamin Friedman static void
isert_destroy_qp(struct isert_conn * isert_conn)406c6e66307SYamin Friedman isert_destroy_qp(struct isert_conn *isert_conn)
407c6e66307SYamin Friedman {
408c6e66307SYamin Friedman ib_destroy_qp(isert_conn->qp);
409c6e66307SYamin Friedman ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size);
410c6e66307SYamin Friedman }
411c6e66307SYamin Friedman
412b8d26b3bSNicholas Bellinger static int
isert_connect_request(struct rdma_cm_id * cma_id,struct rdma_cm_event * event)413b8d26b3bSNicholas Bellinger isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
414b8d26b3bSNicholas Bellinger {
415ca6c1d82SSagi Grimberg struct isert_np *isert_np = cma_id->context;
416ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np;
417b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn;
418b8d26b3bSNicholas Bellinger struct isert_device *device;
419b8d26b3bSNicholas Bellinger int ret = 0;
42014f4b54fSSagi Grimberg
42114f4b54fSSagi Grimberg spin_lock_bh(&np->np_thread_lock);
42214f4b54fSSagi Grimberg if (!np->enabled) {
42314f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock);
42424f412ddSSagi Grimberg isert_dbg("iscsi_np is not enabled, reject connect request\n");
4258094ba0aSLeon Romanovsky return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
42614f4b54fSSagi Grimberg }
42714f4b54fSSagi Grimberg spin_unlock_bh(&np->np_thread_lock);
428b8d26b3bSNicholas Bellinger
4294c22e07fSSagi Grimberg isert_dbg("cma_id: %p, portal: %p\n",
430b8d26b3bSNicholas Bellinger cma_id, cma_id->context);
431b8d26b3bSNicholas Bellinger
432b8d26b3bSNicholas Bellinger isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
433ae9ea9edSSagi Grimberg if (!isert_conn)
434b8d26b3bSNicholas Bellinger return -ENOMEM;
435b8d26b3bSNicholas Bellinger
436ae9ea9edSSagi Grimberg isert_init_conn(isert_conn);
437dac6ab30SSagi Grimberg isert_conn->cm_id = cma_id;
438b8d26b3bSNicholas Bellinger
439cf8ae958SSagi Grimberg device = isert_device_get(cma_id);
440b8d26b3bSNicholas Bellinger if (IS_ERR(device)) {
441b8d26b3bSNicholas Bellinger ret = PTR_ERR(device);
442adb76a52SLv Yunlong goto out;
443b8d26b3bSNicholas Bellinger }
444dac6ab30SSagi Grimberg isert_conn->device = device;
445b8d26b3bSNicholas Bellinger
446adb76a52SLv Yunlong ret = isert_alloc_login_buf(isert_conn, cma_id->device);
447adb76a52SLv Yunlong if (ret)
448adb76a52SLv Yunlong goto out_conn_dev;
449adb76a52SLv Yunlong
450422bd0acSJenny Derzhavetz isert_set_nego_params(isert_conn, &event->param.conn);
4511a92e17eSSagi Grimberg
452c6e66307SYamin Friedman isert_conn->qp = isert_create_qp(isert_conn, cma_id);
453c6e66307SYamin Friedman if (IS_ERR(isert_conn->qp)) {
454c6e66307SYamin Friedman ret = PTR_ERR(isert_conn->qp);
455adb76a52SLv Yunlong goto out_rsp_dma_map;
456c6e66307SYamin Friedman }
457b8d26b3bSNicholas Bellinger
4589679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn);
4592371e5daSSagi Grimberg if (ret)
460c6e66307SYamin Friedman goto out_destroy_qp;
4612371e5daSSagi Grimberg
4622371e5daSSagi Grimberg ret = isert_rdma_accept(isert_conn);
4632371e5daSSagi Grimberg if (ret)
464c6e66307SYamin Friedman goto out_destroy_qp;
4652371e5daSSagi Grimberg
466ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex);
467bd379220SJenny Derzhavetz list_add_tail(&isert_conn->node, &isert_np->accepted);
468ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex);
469b8d26b3bSNicholas Bellinger
470b8d26b3bSNicholas Bellinger return 0;
471b8d26b3bSNicholas Bellinger
472c6e66307SYamin Friedman out_destroy_qp:
473c6e66307SYamin Friedman isert_destroy_qp(isert_conn);
474b8d26b3bSNicholas Bellinger out_rsp_dma_map:
475ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn);
476adb76a52SLv Yunlong out_conn_dev:
477adb76a52SLv Yunlong isert_device_put(device);
478b8d26b3bSNicholas Bellinger out:
479b8d26b3bSNicholas Bellinger kfree(isert_conn);
4808094ba0aSLeon Romanovsky rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
481b8d26b3bSNicholas Bellinger return ret;
482b8d26b3bSNicholas Bellinger }
483b8d26b3bSNicholas Bellinger
484b8d26b3bSNicholas Bellinger static void
isert_connect_release(struct isert_conn * isert_conn)485b8d26b3bSNicholas Bellinger isert_connect_release(struct isert_conn *isert_conn)
486b8d26b3bSNicholas Bellinger {
487dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
488b8d26b3bSNicholas Bellinger
4894c22e07fSSagi Grimberg isert_dbg("conn %p\n", isert_conn);
490b8d26b3bSNicholas Bellinger
49157df81e3SSagi Grimberg BUG_ON(!device);
49257df81e3SSagi Grimberg
49319e2090fSSagi Grimberg isert_free_rx_descriptors(isert_conn);
49463b268d2SRaju Rangoju if (isert_conn->cm_id &&
49563b268d2SRaju Rangoju !isert_conn->dev_removed)
496dac6ab30SSagi Grimberg rdma_destroy_id(isert_conn->cm_id);
49719e2090fSSagi Grimberg
498c6e66307SYamin Friedman if (isert_conn->qp)
499c6e66307SYamin Friedman isert_destroy_qp(isert_conn);
500b8d26b3bSNicholas Bellinger
5010b089c1eSSagi Grimberg if (isert_conn->login_desc)
502ae9ea9edSSagi Grimberg isert_free_login_buf(isert_conn);
503ae9ea9edSSagi Grimberg
504cf8ae958SSagi Grimberg isert_device_put(device);
50557df81e3SSagi Grimberg
50663b268d2SRaju Rangoju if (isert_conn->dev_removed)
50763b268d2SRaju Rangoju wake_up_interruptible(&isert_conn->rem_wait);
50863b268d2SRaju Rangoju else
50957df81e3SSagi Grimberg kfree(isert_conn);
510b8d26b3bSNicholas Bellinger }
511b8d26b3bSNicholas Bellinger
512b8d26b3bSNicholas Bellinger static void
isert_connected_handler(struct rdma_cm_id * cma_id)513b8d26b3bSNicholas Bellinger isert_connected_handler(struct rdma_cm_id *cma_id)
514b8d26b3bSNicholas Bellinger {
51519e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context;
516bd379220SJenny Derzhavetz struct isert_np *isert_np = cma_id->context;
517c2f88b17SSagi Grimberg
51824f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn);
519128e9cc8SSagi Grimberg
520dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex);
521128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_UP;
522bd379220SJenny Derzhavetz kref_get(&isert_conn->kref);
523dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex);
524bd379220SJenny Derzhavetz
525bd379220SJenny Derzhavetz mutex_lock(&isert_np->mutex);
526bd379220SJenny Derzhavetz list_move_tail(&isert_conn->node, &isert_np->pending);
527bd379220SJenny Derzhavetz mutex_unlock(&isert_np->mutex);
528bd379220SJenny Derzhavetz
529bd379220SJenny Derzhavetz isert_info("np %p: Allow accept_np to continue\n", isert_np);
530bd379220SJenny Derzhavetz up(&isert_np->sem);
531b8d26b3bSNicholas Bellinger }
532b8d26b3bSNicholas Bellinger
533b8d26b3bSNicholas Bellinger static void
isert_release_kref(struct kref * kref)534dac6ab30SSagi Grimberg isert_release_kref(struct kref *kref)
535b8d26b3bSNicholas Bellinger {
536b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = container_of(kref,
537dac6ab30SSagi Grimberg struct isert_conn, kref);
538b8d26b3bSNicholas Bellinger
5394c22e07fSSagi Grimberg isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
5404c22e07fSSagi Grimberg current->pid);
541b8d26b3bSNicholas Bellinger
542b8d26b3bSNicholas Bellinger isert_connect_release(isert_conn);
543b8d26b3bSNicholas Bellinger }
544b8d26b3bSNicholas Bellinger
545b8d26b3bSNicholas Bellinger static void
isert_put_conn(struct isert_conn * isert_conn)546b8d26b3bSNicholas Bellinger isert_put_conn(struct isert_conn *isert_conn)
547b8d26b3bSNicholas Bellinger {
548dac6ab30SSagi Grimberg kref_put(&isert_conn->kref, isert_release_kref);
549b8d26b3bSNicholas Bellinger }
550b8d26b3bSNicholas Bellinger
5516d1fba0cSJenny Derzhavetz static void
isert_handle_unbound_conn(struct isert_conn * isert_conn)5526d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(struct isert_conn *isert_conn)
5536d1fba0cSJenny Derzhavetz {
5546d1fba0cSJenny Derzhavetz struct isert_np *isert_np = isert_conn->cm_id->context;
5556d1fba0cSJenny Derzhavetz
5566d1fba0cSJenny Derzhavetz mutex_lock(&isert_np->mutex);
5576d1fba0cSJenny Derzhavetz if (!list_empty(&isert_conn->node)) {
5586d1fba0cSJenny Derzhavetz /*
5596d1fba0cSJenny Derzhavetz * This means iscsi doesn't know this connection
5606d1fba0cSJenny Derzhavetz * so schedule a cleanup ourselves
5616d1fba0cSJenny Derzhavetz */
5626d1fba0cSJenny Derzhavetz list_del_init(&isert_conn->node);
5636d1fba0cSJenny Derzhavetz isert_put_conn(isert_conn);
5646d1fba0cSJenny Derzhavetz queue_work(isert_release_wq, &isert_conn->release_work);
5656d1fba0cSJenny Derzhavetz }
5666d1fba0cSJenny Derzhavetz mutex_unlock(&isert_np->mutex);
5676d1fba0cSJenny Derzhavetz }
5686d1fba0cSJenny Derzhavetz
569954f2372SSagi Grimberg /**
570954f2372SSagi Grimberg * isert_conn_terminate() - Initiate connection termination
571954f2372SSagi Grimberg * @isert_conn: isert connection struct
572954f2372SSagi Grimberg *
573954f2372SSagi Grimberg * Notes:
574aea92980SJenny Derzhavetz * In case the connection state is BOUND, move state
575954f2372SSagi Grimberg * to TEMINATING and start teardown sequence (rdma_disconnect).
576128e9cc8SSagi Grimberg * In case the connection state is UP, complete flush as well.
577954f2372SSagi Grimberg *
578dac6ab30SSagi Grimberg * This routine must be called with mutex held. Thus it is
579954f2372SSagi Grimberg * safe to call multiple times.
580954f2372SSagi Grimberg */
581954f2372SSagi Grimberg static void
isert_conn_terminate(struct isert_conn * isert_conn)582954f2372SSagi Grimberg isert_conn_terminate(struct isert_conn *isert_conn)
583954f2372SSagi Grimberg {
584954f2372SSagi Grimberg int err;
585954f2372SSagi Grimberg
5866d1fba0cSJenny Derzhavetz if (isert_conn->state >= ISER_CONN_TERMINATING)
5876d1fba0cSJenny Derzhavetz return;
5886d1fba0cSJenny Derzhavetz
58924f412ddSSagi Grimberg isert_info("Terminating conn %p state %d\n",
590954f2372SSagi Grimberg isert_conn, isert_conn->state);
591128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_TERMINATING;
592dac6ab30SSagi Grimberg err = rdma_disconnect(isert_conn->cm_id);
593954f2372SSagi Grimberg if (err)
59424f412ddSSagi Grimberg isert_warn("Failed rdma_disconnect isert_conn %p\n",
595954f2372SSagi Grimberg isert_conn);
596954f2372SSagi Grimberg }
597954f2372SSagi Grimberg
5983b726ae2SSagi Grimberg static int
isert_np_cma_handler(struct isert_np * isert_np,enum rdma_cm_event_type event)599ca6c1d82SSagi Grimberg isert_np_cma_handler(struct isert_np *isert_np,
600ca6c1d82SSagi Grimberg enum rdma_cm_event_type event)
601b8d26b3bSNicholas Bellinger {
602ea8a1616SSagi Grimberg isert_dbg("%s (%d): isert np %p\n",
603ea8a1616SSagi Grimberg rdma_event_msg(event), event, isert_np);
6043b726ae2SSagi Grimberg
605ca6c1d82SSagi Grimberg switch (event) {
606ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_DEVICE_REMOVAL:
607ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL;
608ca6c1d82SSagi Grimberg break;
609ca6c1d82SSagi Grimberg case RDMA_CM_EVENT_ADDR_CHANGE:
610ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_setup_id(isert_np);
611ed8cb0a4SJenny Derzhavetz if (IS_ERR(isert_np->cm_id)) {
61224f412ddSSagi Grimberg isert_err("isert np %p setup id failed: %ld\n",
613ed8cb0a4SJenny Derzhavetz isert_np, PTR_ERR(isert_np->cm_id));
614ed8cb0a4SJenny Derzhavetz isert_np->cm_id = NULL;
615ca6c1d82SSagi Grimberg }
616ca6c1d82SSagi Grimberg break;
617ca6c1d82SSagi Grimberg default:
61824f412ddSSagi Grimberg isert_err("isert np %p Unexpected event %d\n",
619ca6c1d82SSagi Grimberg isert_np, event);
620ca6c1d82SSagi Grimberg }
621ca6c1d82SSagi Grimberg
6223b726ae2SSagi Grimberg return -1;
6233b726ae2SSagi Grimberg }
6243b726ae2SSagi Grimberg
625ca6c1d82SSagi Grimberg static int
isert_disconnected_handler(struct rdma_cm_id * cma_id,enum rdma_cm_event_type event)626ca6c1d82SSagi Grimberg isert_disconnected_handler(struct rdma_cm_id *cma_id,
627ca6c1d82SSagi Grimberg enum rdma_cm_event_type event)
628ca6c1d82SSagi Grimberg {
629f81bf458SJenny Derzhavetz struct isert_conn *isert_conn = cma_id->qp->qp_context;
630ca6c1d82SSagi Grimberg
631dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex);
6326d1fba0cSJenny Derzhavetz switch (isert_conn->state) {
6336d1fba0cSJenny Derzhavetz case ISER_CONN_TERMINATING:
6346d1fba0cSJenny Derzhavetz break;
6356d1fba0cSJenny Derzhavetz case ISER_CONN_UP:
636128e9cc8SSagi Grimberg isert_conn_terminate(isert_conn);
637572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp);
6386d1fba0cSJenny Derzhavetz isert_handle_unbound_conn(isert_conn);
6396d1fba0cSJenny Derzhavetz break;
6406d1fba0cSJenny Derzhavetz case ISER_CONN_BOUND:
6416d1fba0cSJenny Derzhavetz case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
6426d1fba0cSJenny Derzhavetz iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
6436d1fba0cSJenny Derzhavetz break;
6446d1fba0cSJenny Derzhavetz default:
645506f71d1SColin Ian King isert_warn("conn %p terminating in state %d\n",
6466d1fba0cSJenny Derzhavetz isert_conn, isert_conn->state);
6476d1fba0cSJenny Derzhavetz }
648dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex);
649128e9cc8SSagi Grimberg
6503b726ae2SSagi Grimberg return 0;
651b8d26b3bSNicholas Bellinger }
652b8d26b3bSNicholas Bellinger
6534a579da2SSagi Grimberg static int
isert_connect_error(struct rdma_cm_id * cma_id)654954f2372SSagi Grimberg isert_connect_error(struct rdma_cm_id *cma_id)
655954f2372SSagi Grimberg {
65619e2090fSSagi Grimberg struct isert_conn *isert_conn = cma_id->qp->qp_context;
6577651e2d6SSaravanan Vajravel struct isert_np *isert_np = cma_id->context;
658954f2372SSagi Grimberg
659cd52cb26SSagi Grimberg ib_drain_qp(isert_conn->qp);
6607651e2d6SSaravanan Vajravel
6617651e2d6SSaravanan Vajravel mutex_lock(&isert_np->mutex);
662bd379220SJenny Derzhavetz list_del_init(&isert_conn->node);
6637651e2d6SSaravanan Vajravel mutex_unlock(&isert_np->mutex);
664dac6ab30SSagi Grimberg isert_conn->cm_id = NULL;
665954f2372SSagi Grimberg isert_put_conn(isert_conn);
6664a579da2SSagi Grimberg
6674a579da2SSagi Grimberg return -1;
668954f2372SSagi Grimberg }
669954f2372SSagi Grimberg
670b8d26b3bSNicholas Bellinger static int
isert_cma_handler(struct rdma_cm_id * cma_id,struct rdma_cm_event * event)671b8d26b3bSNicholas Bellinger isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
672b8d26b3bSNicholas Bellinger {
673f81bf458SJenny Derzhavetz struct isert_np *isert_np = cma_id->context;
67463b268d2SRaju Rangoju struct isert_conn *isert_conn;
675b8d26b3bSNicholas Bellinger int ret = 0;
676b8d26b3bSNicholas Bellinger
677ea8a1616SSagi Grimberg isert_info("%s (%d): status %d id %p np %p\n",
678ea8a1616SSagi Grimberg rdma_event_msg(event->event), event->event,
6794c22e07fSSagi Grimberg event->status, cma_id, cma_id->context);
680b8d26b3bSNicholas Bellinger
681f81bf458SJenny Derzhavetz if (isert_np->cm_id == cma_id)
682f81bf458SJenny Derzhavetz return isert_np_cma_handler(cma_id->context, event->event);
683f81bf458SJenny Derzhavetz
684b8d26b3bSNicholas Bellinger switch (event->event) {
685b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_REQUEST:
686b8d26b3bSNicholas Bellinger ret = isert_connect_request(cma_id, event);
6873b726ae2SSagi Grimberg if (ret)
6884c22e07fSSagi Grimberg isert_err("failed handle connect request %d\n", ret);
689b8d26b3bSNicholas Bellinger break;
690b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_ESTABLISHED:
691b8d26b3bSNicholas Bellinger isert_connected_handler(cma_id);
692b8d26b3bSNicholas Bellinger break;
693df561f66SGustavo A. R. Silva case RDMA_CM_EVENT_ADDR_CHANGE:
694df561f66SGustavo A. R. Silva case RDMA_CM_EVENT_DISCONNECTED:
69588c4015fSSagi Grimberg case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */
696ca6c1d82SSagi Grimberg ret = isert_disconnected_handler(cma_id, event->event);
697b8d26b3bSNicholas Bellinger break;
69863b268d2SRaju Rangoju case RDMA_CM_EVENT_DEVICE_REMOVAL:
69963b268d2SRaju Rangoju isert_conn = cma_id->qp->qp_context;
70063b268d2SRaju Rangoju isert_conn->dev_removed = true;
70163b268d2SRaju Rangoju isert_disconnected_handler(cma_id, event->event);
70263b268d2SRaju Rangoju wait_event_interruptible(isert_conn->rem_wait,
70363b268d2SRaju Rangoju isert_conn->state == ISER_CONN_DOWN);
70463b268d2SRaju Rangoju kfree(isert_conn);
70563b268d2SRaju Rangoju /*
70663b268d2SRaju Rangoju * return non-zero from the callback to destroy
70763b268d2SRaju Rangoju * the rdma cm id
70863b268d2SRaju Rangoju */
70963b268d2SRaju Rangoju return 1;
710d2271bd0SBart Van Assche case RDMA_CM_EVENT_REJECTED:
7111e38a366SSteve Wise isert_info("Connection rejected: %s\n",
7121e38a366SSteve Wise rdma_reject_msg(cma_id, event->status));
713df561f66SGustavo A. R. Silva fallthrough;
714d2271bd0SBart Van Assche case RDMA_CM_EVENT_UNREACHABLE:
715b8d26b3bSNicholas Bellinger case RDMA_CM_EVENT_CONNECT_ERROR:
7164a579da2SSagi Grimberg ret = isert_connect_error(cma_id);
717954f2372SSagi Grimberg break;
718b8d26b3bSNicholas Bellinger default:
71924f412ddSSagi Grimberg isert_err("Unhandled RDMA CMA event: %d\n", event->event);
720b8d26b3bSNicholas Bellinger break;
721b8d26b3bSNicholas Bellinger }
722b8d26b3bSNicholas Bellinger
723b8d26b3bSNicholas Bellinger return ret;
724b8d26b3bSNicholas Bellinger }
725b8d26b3bSNicholas Bellinger
726b8d26b3bSNicholas Bellinger static int
isert_post_recvm(struct isert_conn * isert_conn,u32 count)7274366b19cSJenny Derzhavetz isert_post_recvm(struct isert_conn *isert_conn, u32 count)
728b8d26b3bSNicholas Bellinger {
729e01a7674SBart Van Assche struct ib_recv_wr *rx_wr;
730b8d26b3bSNicholas Bellinger int i, ret;
731b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc;
732b8d26b3bSNicholas Bellinger
733dac6ab30SSagi Grimberg for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
7344366b19cSJenny Derzhavetz rx_desc = &isert_conn->rx_descs[i];
7359679cc51SChristoph Hellwig
7369679cc51SChristoph Hellwig rx_wr->wr_cqe = &rx_desc->rx_cqe;
737b8d26b3bSNicholas Bellinger rx_wr->sg_list = &rx_desc->rx_sg;
738b8d26b3bSNicholas Bellinger rx_wr->num_sge = 1;
739b8d26b3bSNicholas Bellinger rx_wr->next = rx_wr + 1;
7407a56dc88SSagi Grimberg rx_desc->in_use = false;
741b8d26b3bSNicholas Bellinger }
742b8d26b3bSNicholas Bellinger rx_wr--;
743b8d26b3bSNicholas Bellinger rx_wr->next = NULL; /* mark end of work requests list */
744b8d26b3bSNicholas Bellinger
745e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL);
7466d1fba0cSJenny Derzhavetz if (ret)
74724f412ddSSagi Grimberg isert_err("ib_post_recv() failed with ret: %d\n", ret);
7484366b19cSJenny Derzhavetz
7494366b19cSJenny Derzhavetz return ret;
7504366b19cSJenny Derzhavetz }
7514366b19cSJenny Derzhavetz
7524366b19cSJenny Derzhavetz static int
isert_post_recv(struct isert_conn * isert_conn,struct iser_rx_desc * rx_desc)7534366b19cSJenny Derzhavetz isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
7544366b19cSJenny Derzhavetz {
755e01a7674SBart Van Assche struct ib_recv_wr rx_wr;
7564366b19cSJenny Derzhavetz int ret;
7574366b19cSJenny Derzhavetz
7587a56dc88SSagi Grimberg if (!rx_desc->in_use) {
7597a56dc88SSagi Grimberg /*
7607a56dc88SSagi Grimberg * if the descriptor is not in-use we already reposted it
7617a56dc88SSagi Grimberg * for recv, so just silently return
7627a56dc88SSagi Grimberg */
7637a56dc88SSagi Grimberg return 0;
7647a56dc88SSagi Grimberg }
7657a56dc88SSagi Grimberg
7667a56dc88SSagi Grimberg rx_desc->in_use = false;
7679679cc51SChristoph Hellwig rx_wr.wr_cqe = &rx_desc->rx_cqe;
7684366b19cSJenny Derzhavetz rx_wr.sg_list = &rx_desc->rx_sg;
7694366b19cSJenny Derzhavetz rx_wr.num_sge = 1;
7704366b19cSJenny Derzhavetz rx_wr.next = NULL;
7714366b19cSJenny Derzhavetz
772e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
7736d1fba0cSJenny Derzhavetz if (ret)
7744366b19cSJenny Derzhavetz isert_err("ib_post_recv() failed with ret: %d\n", ret);
7754366b19cSJenny Derzhavetz
776b8d26b3bSNicholas Bellinger return ret;
777b8d26b3bSNicholas Bellinger }
778b8d26b3bSNicholas Bellinger
779b8d26b3bSNicholas Bellinger static int
isert_login_post_send(struct isert_conn * isert_conn,struct iser_tx_desc * tx_desc)7809679cc51SChristoph Hellwig isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
781b8d26b3bSNicholas Bellinger {
782dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device;
783e01a7674SBart Van Assche struct ib_send_wr send_wr;
784b8d26b3bSNicholas Bellinger int ret;
785b8d26b3bSNicholas Bellinger
786b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
787b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE);
788b8d26b3bSNicholas Bellinger
7899679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_login_send_done;
7909679cc51SChristoph Hellwig
791b8d26b3bSNicholas Bellinger send_wr.next = NULL;
7929679cc51SChristoph Hellwig send_wr.wr_cqe = &tx_desc->tx_cqe;
793b8d26b3bSNicholas Bellinger send_wr.sg_list = tx_desc->tx_sg;
794b8d26b3bSNicholas Bellinger send_wr.num_sge = tx_desc->num_sge;
795b8d26b3bSNicholas Bellinger send_wr.opcode = IB_WR_SEND;
796b8d26b3bSNicholas Bellinger send_wr.send_flags = IB_SEND_SIGNALED;
797b8d26b3bSNicholas Bellinger
798e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &send_wr, NULL);
799bdf20e72SSagi Grimberg if (ret)
80024f412ddSSagi Grimberg isert_err("ib_post_send() failed, ret: %d\n", ret);
801b8d26b3bSNicholas Bellinger
802b8d26b3bSNicholas Bellinger return ret;
803b8d26b3bSNicholas Bellinger }
804b8d26b3bSNicholas Bellinger
805b8d26b3bSNicholas Bellinger static void
__isert_create_send_desc(struct isert_device * device,struct iser_tx_desc * tx_desc)806763b6965SAlex Estrin __isert_create_send_desc(struct isert_device *device,
807b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc)
808b8d26b3bSNicholas Bellinger {
809b8d26b3bSNicholas Bellinger
810d3cf81f9SSagi Grimberg memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
811d3cf81f9SSagi Grimberg tx_desc->iser_header.flags = ISCSI_CTRL;
812b8d26b3bSNicholas Bellinger
813b8d26b3bSNicholas Bellinger tx_desc->num_sge = 1;
814b8d26b3bSNicholas Bellinger
81534efc7dfSJason Gunthorpe if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
81634efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
81724f412ddSSagi Grimberg isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
818b8d26b3bSNicholas Bellinger }
819b8d26b3bSNicholas Bellinger }
820b8d26b3bSNicholas Bellinger
821763b6965SAlex Estrin static void
isert_create_send_desc(struct isert_conn * isert_conn,struct isert_cmd * isert_cmd,struct iser_tx_desc * tx_desc)822763b6965SAlex Estrin isert_create_send_desc(struct isert_conn *isert_conn,
823763b6965SAlex Estrin struct isert_cmd *isert_cmd,
824763b6965SAlex Estrin struct iser_tx_desc *tx_desc)
825763b6965SAlex Estrin {
826763b6965SAlex Estrin struct isert_device *device = isert_conn->device;
827763b6965SAlex Estrin struct ib_device *ib_dev = device->ib_device;
828763b6965SAlex Estrin
829763b6965SAlex Estrin ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
830763b6965SAlex Estrin ISER_HEADERS_LEN, DMA_TO_DEVICE);
831763b6965SAlex Estrin
832763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc);
833763b6965SAlex Estrin }
834763b6965SAlex Estrin
835b8d26b3bSNicholas Bellinger static int
isert_init_tx_hdrs(struct isert_conn * isert_conn,struct iser_tx_desc * tx_desc)836b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(struct isert_conn *isert_conn,
837b8d26b3bSNicholas Bellinger struct iser_tx_desc *tx_desc)
838b8d26b3bSNicholas Bellinger {
839dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
84067cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device;
841b8d26b3bSNicholas Bellinger u64 dma_addr;
842b8d26b3bSNicholas Bellinger
843b8d26b3bSNicholas Bellinger dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
844b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE);
845b8d26b3bSNicholas Bellinger if (ib_dma_mapping_error(ib_dev, dma_addr)) {
84624f412ddSSagi Grimberg isert_err("ib_dma_mapping_error() failed\n");
847b8d26b3bSNicholas Bellinger return -ENOMEM;
848b8d26b3bSNicholas Bellinger }
849b8d26b3bSNicholas Bellinger
850b8d26b3bSNicholas Bellinger tx_desc->dma_addr = dma_addr;
851b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
852b8d26b3bSNicholas Bellinger tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
85334efc7dfSJason Gunthorpe tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
854b8d26b3bSNicholas Bellinger
8554c22e07fSSagi Grimberg isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
8564c22e07fSSagi Grimberg tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
8574c22e07fSSagi Grimberg tx_desc->tx_sg[0].lkey);
858b8d26b3bSNicholas Bellinger
859b8d26b3bSNicholas Bellinger return 0;
860b8d26b3bSNicholas Bellinger }
861b8d26b3bSNicholas Bellinger
862b8d26b3bSNicholas Bellinger static void
isert_init_send_wr(struct isert_conn * isert_conn,struct isert_cmd * isert_cmd,struct ib_send_wr * send_wr)86395b60f07SNicholas Bellinger isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
86468a86deeSSagi Grimberg struct ib_send_wr *send_wr)
865b8d26b3bSNicholas Bellinger {
86695b60f07SNicholas Bellinger struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
86795b60f07SNicholas Bellinger
8689679cc51SChristoph Hellwig tx_desc->tx_cqe.done = isert_send_done;
8699679cc51SChristoph Hellwig send_wr->wr_cqe = &tx_desc->tx_cqe;
870422bd0acSJenny Derzhavetz
871422bd0acSJenny Derzhavetz if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
872422bd0acSJenny Derzhavetz send_wr->opcode = IB_WR_SEND_WITH_INV;
873422bd0acSJenny Derzhavetz send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
874422bd0acSJenny Derzhavetz } else {
875b8d26b3bSNicholas Bellinger send_wr->opcode = IB_WR_SEND;
876422bd0acSJenny Derzhavetz }
877422bd0acSJenny Derzhavetz
87895b60f07SNicholas Bellinger send_wr->sg_list = &tx_desc->tx_sg[0];
879b8d26b3bSNicholas Bellinger send_wr->num_sge = isert_cmd->tx_desc.num_sge;
88095b60f07SNicholas Bellinger send_wr->send_flags = IB_SEND_SIGNALED;
881b8d26b3bSNicholas Bellinger }
882b8d26b3bSNicholas Bellinger
883b8d26b3bSNicholas Bellinger static int
isert_login_post_recv(struct isert_conn * isert_conn)8849679cc51SChristoph Hellwig isert_login_post_recv(struct isert_conn *isert_conn)
885b8d26b3bSNicholas Bellinger {
886e01a7674SBart Van Assche struct ib_recv_wr rx_wr;
887b8d26b3bSNicholas Bellinger struct ib_sge sge;
888b8d26b3bSNicholas Bellinger int ret;
889b8d26b3bSNicholas Bellinger
890b8d26b3bSNicholas Bellinger memset(&sge, 0, sizeof(struct ib_sge));
8910b089c1eSSagi Grimberg sge.addr = isert_conn->login_desc->dma_addr +
8920b089c1eSSagi Grimberg isert_get_hdr_offset(isert_conn->login_desc);
893ed1083b2SChristoph Hellwig sge.length = ISER_RX_PAYLOAD_SIZE;
89434efc7dfSJason Gunthorpe sge.lkey = isert_conn->device->pd->local_dma_lkey;
895b8d26b3bSNicholas Bellinger
89624f412ddSSagi Grimberg isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
897b8d26b3bSNicholas Bellinger sge.addr, sge.length, sge.lkey);
898b8d26b3bSNicholas Bellinger
8990b089c1eSSagi Grimberg isert_conn->login_desc->rx_cqe.done = isert_login_recv_done;
9009679cc51SChristoph Hellwig
901b8d26b3bSNicholas Bellinger memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
9020b089c1eSSagi Grimberg rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe;
903b8d26b3bSNicholas Bellinger rx_wr.sg_list = &sge;
904b8d26b3bSNicholas Bellinger rx_wr.num_sge = 1;
905b8d26b3bSNicholas Bellinger
906e01a7674SBart Van Assche ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
9076d1fba0cSJenny Derzhavetz if (ret)
90824f412ddSSagi Grimberg isert_err("ib_post_recv() failed: %d\n", ret);
909b8d26b3bSNicholas Bellinger
910b8d26b3bSNicholas Bellinger return ret;
911b8d26b3bSNicholas Bellinger }
912b8d26b3bSNicholas Bellinger
913b8d26b3bSNicholas Bellinger static int
isert_put_login_tx(struct iscsit_conn * conn,struct iscsi_login * login,u32 length)914be36d683SMax Gurtovoy isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login,
915b8d26b3bSNicholas Bellinger u32 length)
916b8d26b3bSNicholas Bellinger {
917b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context;
918dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
91967cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device;
920dac6ab30SSagi Grimberg struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
921b8d26b3bSNicholas Bellinger int ret;
922b8d26b3bSNicholas Bellinger
923763b6965SAlex Estrin __isert_create_send_desc(device, tx_desc);
924b8d26b3bSNicholas Bellinger
925b8d26b3bSNicholas Bellinger memcpy(&tx_desc->iscsi_header, &login->rsp[0],
926b8d26b3bSNicholas Bellinger sizeof(struct iscsi_hdr));
927b8d26b3bSNicholas Bellinger
928b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, tx_desc);
929b8d26b3bSNicholas Bellinger
930b8d26b3bSNicholas Bellinger if (length > 0) {
931b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
932b8d26b3bSNicholas Bellinger
933b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
934b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE);
935b8d26b3bSNicholas Bellinger
936b8d26b3bSNicholas Bellinger memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
937b8d26b3bSNicholas Bellinger
938b8d26b3bSNicholas Bellinger ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
939b8d26b3bSNicholas Bellinger length, DMA_TO_DEVICE);
940b8d26b3bSNicholas Bellinger
941b8d26b3bSNicholas Bellinger tx_dsg->addr = isert_conn->login_rsp_dma;
942b8d26b3bSNicholas Bellinger tx_dsg->length = length;
94334efc7dfSJason Gunthorpe tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey;
944b8d26b3bSNicholas Bellinger tx_desc->num_sge = 2;
945b8d26b3bSNicholas Bellinger }
946b8d26b3bSNicholas Bellinger if (!login->login_failed) {
947b8d26b3bSNicholas Bellinger if (login->login_complete) {
948b8d26b3bSNicholas Bellinger ret = isert_alloc_rx_descriptors(isert_conn);
949b8d26b3bSNicholas Bellinger if (ret)
950b8d26b3bSNicholas Bellinger return ret;
951b8d26b3bSNicholas Bellinger
9524366b19cSJenny Derzhavetz ret = isert_post_recvm(isert_conn,
9534366b19cSJenny Derzhavetz ISERT_QP_MAX_RECV_DTOS);
954b8d26b3bSNicholas Bellinger if (ret)
955b8d26b3bSNicholas Bellinger return ret;
956b8d26b3bSNicholas Bellinger
957128e9cc8SSagi Grimberg /* Now we are in FULL_FEATURE phase */
958dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex);
959128e9cc8SSagi Grimberg isert_conn->state = ISER_CONN_FULL_FEATURE;
960dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex);
961b8d26b3bSNicholas Bellinger goto post_send;
962b8d26b3bSNicholas Bellinger }
963b8d26b3bSNicholas Bellinger
9649679cc51SChristoph Hellwig ret = isert_login_post_recv(isert_conn);
965b8d26b3bSNicholas Bellinger if (ret)
966b8d26b3bSNicholas Bellinger return ret;
967b8d26b3bSNicholas Bellinger }
968b8d26b3bSNicholas Bellinger post_send:
9699679cc51SChristoph Hellwig ret = isert_login_post_send(isert_conn, tx_desc);
970b8d26b3bSNicholas Bellinger if (ret)
971b8d26b3bSNicholas Bellinger return ret;
972b8d26b3bSNicholas Bellinger
973b8d26b3bSNicholas Bellinger return 0;
974b8d26b3bSNicholas Bellinger }
975b8d26b3bSNicholas Bellinger
976b8d26b3bSNicholas Bellinger static void
isert_rx_login_req(struct isert_conn * isert_conn)9772371e5daSSagi Grimberg isert_rx_login_req(struct isert_conn *isert_conn)
978b8d26b3bSNicholas Bellinger {
9790b089c1eSSagi Grimberg struct iser_rx_desc *rx_desc = isert_conn->login_desc;
9802371e5daSSagi Grimberg int rx_buflen = isert_conn->login_req_len;
981be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
982b8d26b3bSNicholas Bellinger struct iscsi_login *login = conn->conn_login;
983b8d26b3bSNicholas Bellinger int size;
984b8d26b3bSNicholas Bellinger
98524f412ddSSagi Grimberg isert_info("conn %p\n", isert_conn);
9862371e5daSSagi Grimberg
9872371e5daSSagi Grimberg WARN_ON_ONCE(!login);
988b8d26b3bSNicholas Bellinger
989b8d26b3bSNicholas Bellinger if (login->first_request) {
990b8d26b3bSNicholas Bellinger struct iscsi_login_req *login_req =
9910b089c1eSSagi Grimberg (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc);
992b8d26b3bSNicholas Bellinger /*
993b8d26b3bSNicholas Bellinger * Setup the initial iscsi_login values from the leading
994b8d26b3bSNicholas Bellinger * login request PDU.
995b8d26b3bSNicholas Bellinger */
996b8d26b3bSNicholas Bellinger login->leading_connection = (!login_req->tsih) ? 1 : 0;
9972a402120SMaurizio Lombardi login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
9982a402120SMaurizio Lombardi login_req->flags);
999b8d26b3bSNicholas Bellinger login->version_min = login_req->min_version;
1000b8d26b3bSNicholas Bellinger login->version_max = login_req->max_version;
1001b8d26b3bSNicholas Bellinger memcpy(login->isid, login_req->isid, 6);
1002b8d26b3bSNicholas Bellinger login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1003b8d26b3bSNicholas Bellinger login->init_task_tag = login_req->itt;
1004b8d26b3bSNicholas Bellinger login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1005b8d26b3bSNicholas Bellinger login->cid = be16_to_cpu(login_req->cid);
1006b8d26b3bSNicholas Bellinger login->tsih = be16_to_cpu(login_req->tsih);
1007b8d26b3bSNicholas Bellinger }
1008b8d26b3bSNicholas Bellinger
10090b089c1eSSagi Grimberg memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN);
1010b8d26b3bSNicholas Bellinger
1011b8d26b3bSNicholas Bellinger size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
10124c22e07fSSagi Grimberg isert_dbg("Using login payload size: %d, rx_buflen: %d "
10134c22e07fSSagi Grimberg "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen,
10144c22e07fSSagi Grimberg MAX_KEY_VALUE_PAIRS);
10150b089c1eSSagi Grimberg memcpy(login->req_buf, isert_get_data(rx_desc), size);
1016b8d26b3bSNicholas Bellinger
10176faaa85fSNicholas Bellinger if (login->first_request) {
1018dac6ab30SSagi Grimberg complete(&isert_conn->login_comp);
10196faaa85fSNicholas Bellinger return;
10206faaa85fSNicholas Bellinger }
1021549f39a5STetsuo Handa queue_delayed_work(isert_login_wq, &conn->login_work, 0);
1022b8d26b3bSNicholas Bellinger }
1023b8d26b3bSNicholas Bellinger
102466cd9d4eSMax Gurtovoy static struct iscsit_cmd
isert_allocate_cmd(struct iscsit_conn * conn,struct iser_rx_desc * rx_desc)1025be36d683SMax Gurtovoy *isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc)
1026b8d26b3bSNicholas Bellinger {
10276700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1028b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd;
102966cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd;
1030b8d26b3bSNicholas Bellinger
1031676687c6SNicholas Bellinger cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
1032d703ce2fSNicholas Bellinger if (!cmd) {
103366cd9d4eSMax Gurtovoy isert_err("Unable to allocate iscsit_cmd + isert_cmd\n");
1034b8d26b3bSNicholas Bellinger return NULL;
1035b8d26b3bSNicholas Bellinger }
1036d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd);
1037b8d26b3bSNicholas Bellinger isert_cmd->conn = isert_conn;
103866cd9d4eSMax Gurtovoy isert_cmd->iscsit_cmd = cmd;
10394366b19cSJenny Derzhavetz isert_cmd->rx_desc = rx_desc;
1040b8d26b3bSNicholas Bellinger
1041d703ce2fSNicholas Bellinger return cmd;
1042b8d26b3bSNicholas Bellinger }
1043b8d26b3bSNicholas Bellinger
1044b8d26b3bSNicholas Bellinger static int
isert_handle_scsi_cmd(struct isert_conn * isert_conn,struct isert_cmd * isert_cmd,struct iscsit_cmd * cmd,struct iser_rx_desc * rx_desc,unsigned char * buf)1045b8d26b3bSNicholas Bellinger isert_handle_scsi_cmd(struct isert_conn *isert_conn,
104666cd9d4eSMax Gurtovoy struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd,
1047d703ce2fSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf)
1048b8d26b3bSNicholas Bellinger {
1049be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
1050b8d26b3bSNicholas Bellinger struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1051b8d26b3bSNicholas Bellinger int imm_data, imm_data_len, unsol_data, sg_nents, rc;
1052b8d26b3bSNicholas Bellinger bool dump_payload = false;
10539fd60088SJenny Derzhavetz unsigned int data_len;
1054b8d26b3bSNicholas Bellinger
1055b8d26b3bSNicholas Bellinger rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1056b8d26b3bSNicholas Bellinger if (rc < 0)
1057b8d26b3bSNicholas Bellinger return rc;
1058b8d26b3bSNicholas Bellinger
1059b8d26b3bSNicholas Bellinger imm_data = cmd->immediate_data;
1060b8d26b3bSNicholas Bellinger imm_data_len = cmd->first_burst_len;
1061b8d26b3bSNicholas Bellinger unsol_data = cmd->unsolicited_data;
10629fd60088SJenny Derzhavetz data_len = cmd->se_cmd.data_length;
1063b8d26b3bSNicholas Bellinger
10649fd60088SJenny Derzhavetz if (imm_data && imm_data_len == data_len)
10659fd60088SJenny Derzhavetz cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1066b8d26b3bSNicholas Bellinger rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1067b8d26b3bSNicholas Bellinger if (rc < 0) {
1068b8d26b3bSNicholas Bellinger return 0;
1069b8d26b3bSNicholas Bellinger } else if (rc > 0) {
1070b8d26b3bSNicholas Bellinger dump_payload = true;
1071b8d26b3bSNicholas Bellinger goto sequence_cmd;
1072b8d26b3bSNicholas Bellinger }
1073b8d26b3bSNicholas Bellinger
1074b8d26b3bSNicholas Bellinger if (!imm_data)
1075b8d26b3bSNicholas Bellinger return 0;
1076b8d26b3bSNicholas Bellinger
10779fd60088SJenny Derzhavetz if (imm_data_len != data_len) {
1078b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
10799fd60088SJenny Derzhavetz sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
10800b089c1eSSagi Grimberg isert_get_data(rx_desc), imm_data_len);
10819fd60088SJenny Derzhavetz isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
10829fd60088SJenny Derzhavetz sg_nents, imm_data_len);
10839fd60088SJenny Derzhavetz } else {
10849fd60088SJenny Derzhavetz sg_init_table(&isert_cmd->sg, 1);
10859fd60088SJenny Derzhavetz cmd->se_cmd.t_data_sg = &isert_cmd->sg;
10869fd60088SJenny Derzhavetz cmd->se_cmd.t_data_nents = 1;
10870b089c1eSSagi Grimberg sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc),
10880b089c1eSSagi Grimberg imm_data_len);
10899fd60088SJenny Derzhavetz isert_dbg("Transfer Immediate imm_data_len: %d\n",
10909fd60088SJenny Derzhavetz imm_data_len);
10919fd60088SJenny Derzhavetz }
1092b8d26b3bSNicholas Bellinger
1093b8d26b3bSNicholas Bellinger cmd->write_data_done += imm_data_len;
1094b8d26b3bSNicholas Bellinger
1095b8d26b3bSNicholas Bellinger if (cmd->write_data_done == cmd->se_cmd.data_length) {
1096b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock);
1097b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1098b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1099b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock);
1100b8d26b3bSNicholas Bellinger }
1101b8d26b3bSNicholas Bellinger
1102b8d26b3bSNicholas Bellinger sequence_cmd:
1103561bf158SNicholas Bellinger rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
1104b8d26b3bSNicholas Bellinger
1105f7a95c90SZou Wei if (!rc && !dump_payload && unsol_data)
11060300b114SBart Van Assche iscsit_set_unsolicited_dataout(cmd);
11076cc44a6fSNicholas Bellinger else if (dump_payload && imm_data)
1108afc16604SBart Van Assche target_put_sess_cmd(&cmd->se_cmd);
1109b8d26b3bSNicholas Bellinger
1110b8d26b3bSNicholas Bellinger return 0;
1111b8d26b3bSNicholas Bellinger }
1112b8d26b3bSNicholas Bellinger
1113b8d26b3bSNicholas Bellinger static int
isert_handle_iscsi_dataout(struct isert_conn * isert_conn,struct iser_rx_desc * rx_desc,unsigned char * buf)1114b8d26b3bSNicholas Bellinger isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
1115b8d26b3bSNicholas Bellinger struct iser_rx_desc *rx_desc, unsigned char *buf)
1116b8d26b3bSNicholas Bellinger {
1117b8d26b3bSNicholas Bellinger struct scatterlist *sg_start;
1118be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
111966cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd = NULL;
1120b8d26b3bSNicholas Bellinger struct iscsi_data *hdr = (struct iscsi_data *)buf;
1121b8d26b3bSNicholas Bellinger u32 unsol_data_len = ntoh24(hdr->dlength);
1122b8d26b3bSNicholas Bellinger int rc, sg_nents, sg_off, page_off;
1123b8d26b3bSNicholas Bellinger
1124b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1125b8d26b3bSNicholas Bellinger if (rc < 0)
1126b8d26b3bSNicholas Bellinger return rc;
1127b8d26b3bSNicholas Bellinger else if (!cmd)
1128b8d26b3bSNicholas Bellinger return 0;
1129b8d26b3bSNicholas Bellinger /*
1130b8d26b3bSNicholas Bellinger * FIXME: Unexpected unsolicited_data out
1131b8d26b3bSNicholas Bellinger */
1132b8d26b3bSNicholas Bellinger if (!cmd->unsolicited_data) {
113324f412ddSSagi Grimberg isert_err("Received unexpected solicited data payload\n");
1134b8d26b3bSNicholas Bellinger dump_stack();
1135b8d26b3bSNicholas Bellinger return -1;
1136b8d26b3bSNicholas Bellinger }
1137b8d26b3bSNicholas Bellinger
11384c22e07fSSagi Grimberg isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
11394c22e07fSSagi Grimberg "write_data_done: %u, data_length: %u\n",
11404c22e07fSSagi Grimberg unsol_data_len, cmd->write_data_done,
11414c22e07fSSagi Grimberg cmd->se_cmd.data_length);
1142b8d26b3bSNicholas Bellinger
1143b8d26b3bSNicholas Bellinger sg_off = cmd->write_data_done / PAGE_SIZE;
1144b8d26b3bSNicholas Bellinger sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1145b8d26b3bSNicholas Bellinger sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1146b8d26b3bSNicholas Bellinger page_off = cmd->write_data_done % PAGE_SIZE;
1147b8d26b3bSNicholas Bellinger /*
1148b8d26b3bSNicholas Bellinger * FIXME: Non page-aligned unsolicited_data out
1149b8d26b3bSNicholas Bellinger */
1150b8d26b3bSNicholas Bellinger if (page_off) {
11514c22e07fSSagi Grimberg isert_err("unexpected non-page aligned data payload\n");
1152b8d26b3bSNicholas Bellinger dump_stack();
1153b8d26b3bSNicholas Bellinger return -1;
1154b8d26b3bSNicholas Bellinger }
11554c22e07fSSagi Grimberg isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
11564c22e07fSSagi Grimberg "sg_nents: %u from %p %u\n", sg_start, sg_off,
11570b089c1eSSagi Grimberg sg_nents, isert_get_data(rx_desc), unsol_data_len);
1158b8d26b3bSNicholas Bellinger
11590b089c1eSSagi Grimberg sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc),
1160b8d26b3bSNicholas Bellinger unsol_data_len);
1161b8d26b3bSNicholas Bellinger
1162b8d26b3bSNicholas Bellinger rc = iscsit_check_dataout_payload(cmd, hdr, false);
1163b8d26b3bSNicholas Bellinger if (rc < 0)
1164b8d26b3bSNicholas Bellinger return rc;
1165b8d26b3bSNicholas Bellinger
11664366b19cSJenny Derzhavetz /*
11674366b19cSJenny Derzhavetz * multiple data-outs on the same command can arrive -
11684366b19cSJenny Derzhavetz * so post the buffer before hand
11694366b19cSJenny Derzhavetz */
1170c97119b6SMax Gurtovoy return isert_post_recv(isert_conn, rx_desc);
1171b8d26b3bSNicholas Bellinger }
1172b8d26b3bSNicholas Bellinger
1173b8d26b3bSNicholas Bellinger static int
isert_handle_nop_out(struct isert_conn * isert_conn,struct isert_cmd * isert_cmd,struct iscsit_cmd * cmd,struct iser_rx_desc * rx_desc,unsigned char * buf)1174778de368SNicholas Bellinger isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
117566cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc,
1176d703ce2fSNicholas Bellinger unsigned char *buf)
1177778de368SNicholas Bellinger {
1178be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
1179778de368SNicholas Bellinger struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1180778de368SNicholas Bellinger int rc;
1181778de368SNicholas Bellinger
1182778de368SNicholas Bellinger rc = iscsit_setup_nop_out(conn, cmd, hdr);
1183778de368SNicholas Bellinger if (rc < 0)
1184778de368SNicholas Bellinger return rc;
1185778de368SNicholas Bellinger /*
1186778de368SNicholas Bellinger * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1187778de368SNicholas Bellinger */
1188778de368SNicholas Bellinger
1189778de368SNicholas Bellinger return iscsit_process_nop_out(conn, cmd, hdr);
1190778de368SNicholas Bellinger }
1191778de368SNicholas Bellinger
1192778de368SNicholas Bellinger static int
isert_handle_text_cmd(struct isert_conn * isert_conn,struct isert_cmd * isert_cmd,struct iscsit_cmd * cmd,struct iser_rx_desc * rx_desc,struct iscsi_text * hdr)1193adb54c29SNicholas Bellinger isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
119466cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc,
1195d703ce2fSNicholas Bellinger struct iscsi_text *hdr)
1196adb54c29SNicholas Bellinger {
1197be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
1198adb54c29SNicholas Bellinger u32 payload_length = ntoh24(hdr->dlength);
1199adb54c29SNicholas Bellinger int rc;
1200b44a2b67SSagi Grimberg unsigned char *text_in = NULL;
1201adb54c29SNicholas Bellinger
1202adb54c29SNicholas Bellinger rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1203adb54c29SNicholas Bellinger if (rc < 0)
1204adb54c29SNicholas Bellinger return rc;
1205adb54c29SNicholas Bellinger
1206b44a2b67SSagi Grimberg if (payload_length) {
1207adb54c29SNicholas Bellinger text_in = kzalloc(payload_length, GFP_KERNEL);
120893b80f29SLeon Romanovsky if (!text_in)
1209adb54c29SNicholas Bellinger return -ENOMEM;
1210adb54c29SNicholas Bellinger }
1211adb54c29SNicholas Bellinger cmd->text_in_ptr = text_in;
1212adb54c29SNicholas Bellinger
12130b089c1eSSagi Grimberg memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length);
1214adb54c29SNicholas Bellinger
1215adb54c29SNicholas Bellinger return iscsit_process_text_cmd(conn, cmd, hdr);
1216adb54c29SNicholas Bellinger }
1217adb54c29SNicholas Bellinger
1218adb54c29SNicholas Bellinger static int
isert_rx_opcode(struct isert_conn * isert_conn,struct iser_rx_desc * rx_desc,uint32_t read_stag,uint64_t read_va,uint32_t write_stag,uint64_t write_va)1219b8d26b3bSNicholas Bellinger isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1220b8d26b3bSNicholas Bellinger uint32_t read_stag, uint64_t read_va,
1221b8d26b3bSNicholas Bellinger uint32_t write_stag, uint64_t write_va)
1222b8d26b3bSNicholas Bellinger {
12230b089c1eSSagi Grimberg struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc);
1224be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
122566cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd;
1226b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd;
1227b8d26b3bSNicholas Bellinger int ret = -EINVAL;
1228b8d26b3bSNicholas Bellinger u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1229b8d26b3bSNicholas Bellinger
1230fb140271SSagi Grimberg if (conn->sess->sess_ops->SessionType &&
1231ca40d24eSNicholas Bellinger (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
123224f412ddSSagi Grimberg isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1233ca40d24eSNicholas Bellinger " ignoring\n", opcode);
1234ca40d24eSNicholas Bellinger return 0;
1235ca40d24eSNicholas Bellinger }
1236ca40d24eSNicholas Bellinger
1237b8d26b3bSNicholas Bellinger switch (opcode) {
1238b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD:
12394366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc);
1240b8d26b3bSNicholas Bellinger if (!cmd)
1241b8d26b3bSNicholas Bellinger break;
1242b8d26b3bSNicholas Bellinger
1243d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd);
1244b8d26b3bSNicholas Bellinger isert_cmd->read_stag = read_stag;
1245b8d26b3bSNicholas Bellinger isert_cmd->read_va = read_va;
1246b8d26b3bSNicholas Bellinger isert_cmd->write_stag = write_stag;
1247b8d26b3bSNicholas Bellinger isert_cmd->write_va = write_va;
1248422bd0acSJenny Derzhavetz isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
1249b8d26b3bSNicholas Bellinger
1250d703ce2fSNicholas Bellinger ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
1251b8d26b3bSNicholas Bellinger rx_desc, (unsigned char *)hdr);
1252b8d26b3bSNicholas Bellinger break;
1253b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT:
12544366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc);
1255b8d26b3bSNicholas Bellinger if (!cmd)
1256b8d26b3bSNicholas Bellinger break;
1257b8d26b3bSNicholas Bellinger
1258d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd);
1259d703ce2fSNicholas Bellinger ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
1260778de368SNicholas Bellinger rx_desc, (unsigned char *)hdr);
1261b8d26b3bSNicholas Bellinger break;
1262b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_DATA_OUT:
1263b8d26b3bSNicholas Bellinger ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1264b8d26b3bSNicholas Bellinger (unsigned char *)hdr);
1265b8d26b3bSNicholas Bellinger break;
1266b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC:
12674366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc);
1268b8d26b3bSNicholas Bellinger if (!cmd)
1269b8d26b3bSNicholas Bellinger break;
1270b8d26b3bSNicholas Bellinger
1271b8d26b3bSNicholas Bellinger ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1272b8d26b3bSNicholas Bellinger (unsigned char *)hdr);
1273b8d26b3bSNicholas Bellinger break;
1274b8d26b3bSNicholas Bellinger case ISCSI_OP_LOGOUT:
12754366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc);
1276b8d26b3bSNicholas Bellinger if (!cmd)
1277b8d26b3bSNicholas Bellinger break;
1278b8d26b3bSNicholas Bellinger
1279b8d26b3bSNicholas Bellinger ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1280b8d26b3bSNicholas Bellinger break;
1281adb54c29SNicholas Bellinger case ISCSI_OP_TEXT:
12824366b19cSJenny Derzhavetz if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
1283e4f4e801SSagi Grimberg cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
12844366b19cSJenny Derzhavetz else
12854366b19cSJenny Derzhavetz cmd = isert_allocate_cmd(conn, rx_desc);
12864366b19cSJenny Derzhavetz
1287e4f4e801SSagi Grimberg if (!cmd)
1288e4f4e801SSagi Grimberg break;
1289adb54c29SNicholas Bellinger
1290d703ce2fSNicholas Bellinger isert_cmd = iscsit_priv_cmd(cmd);
1291d703ce2fSNicholas Bellinger ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
1292adb54c29SNicholas Bellinger rx_desc, (struct iscsi_text *)hdr);
1293adb54c29SNicholas Bellinger break;
1294b8d26b3bSNicholas Bellinger default:
129524f412ddSSagi Grimberg isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1296b8d26b3bSNicholas Bellinger dump_stack();
1297b8d26b3bSNicholas Bellinger break;
1298b8d26b3bSNicholas Bellinger }
1299b8d26b3bSNicholas Bellinger
1300b8d26b3bSNicholas Bellinger return ret;
1301b8d26b3bSNicholas Bellinger }
1302b8d26b3bSNicholas Bellinger
1303b8d26b3bSNicholas Bellinger static void
isert_print_wc(struct ib_wc * wc,const char * type)13049679cc51SChristoph Hellwig isert_print_wc(struct ib_wc *wc, const char *type)
1305b8d26b3bSNicholas Bellinger {
13069679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR)
13079679cc51SChristoph Hellwig isert_err("%s failure: %s (%d) vend_err %x\n", type,
13089679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status,
13099679cc51SChristoph Hellwig wc->vendor_err);
13109679cc51SChristoph Hellwig else
13119679cc51SChristoph Hellwig isert_dbg("%s failure: %s (%d)\n", type,
13129679cc51SChristoph Hellwig ib_wc_status_msg(wc->status), wc->status);
13139679cc51SChristoph Hellwig }
13149679cc51SChristoph Hellwig
13159679cc51SChristoph Hellwig static void
isert_recv_done(struct ib_cq * cq,struct ib_wc * wc)13169679cc51SChristoph Hellwig isert_recv_done(struct ib_cq *cq, struct ib_wc *wc)
13179679cc51SChristoph Hellwig {
13189679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context;
13199679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device;
13209679cc51SChristoph Hellwig struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe);
13210b089c1eSSagi Grimberg struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc);
13220b089c1eSSagi Grimberg struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc);
1323b8d26b3bSNicholas Bellinger uint64_t read_va = 0, write_va = 0;
1324b8d26b3bSNicholas Bellinger uint32_t read_stag = 0, write_stag = 0;
1325b8d26b3bSNicholas Bellinger
13269679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) {
13279679cc51SChristoph Hellwig isert_print_wc(wc, "recv");
13289679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR)
13299679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
13309679cc51SChristoph Hellwig return;
13319679cc51SChristoph Hellwig }
13329679cc51SChristoph Hellwig
13337a56dc88SSagi Grimberg rx_desc->in_use = true;
13347a56dc88SSagi Grimberg
13359679cc51SChristoph Hellwig ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr,
13360b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
13379679cc51SChristoph Hellwig
13389679cc51SChristoph Hellwig isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
13399679cc51SChristoph Hellwig rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags,
13409679cc51SChristoph Hellwig (int)(wc->byte_len - ISER_HEADERS_LEN));
13419679cc51SChristoph Hellwig
1342d3cf81f9SSagi Grimberg switch (iser_ctrl->flags & 0xF0) {
1343b8d26b3bSNicholas Bellinger case ISCSI_CTRL:
1344d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_RSV) {
1345d3cf81f9SSagi Grimberg read_stag = be32_to_cpu(iser_ctrl->read_stag);
1346d3cf81f9SSagi Grimberg read_va = be64_to_cpu(iser_ctrl->read_va);
13474c22e07fSSagi Grimberg isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
1348b8d26b3bSNicholas Bellinger read_stag, (unsigned long long)read_va);
1349b8d26b3bSNicholas Bellinger }
1350d3cf81f9SSagi Grimberg if (iser_ctrl->flags & ISER_WSV) {
1351d3cf81f9SSagi Grimberg write_stag = be32_to_cpu(iser_ctrl->write_stag);
1352d3cf81f9SSagi Grimberg write_va = be64_to_cpu(iser_ctrl->write_va);
13534c22e07fSSagi Grimberg isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
1354b8d26b3bSNicholas Bellinger write_stag, (unsigned long long)write_va);
1355b8d26b3bSNicholas Bellinger }
1356b8d26b3bSNicholas Bellinger
135724f412ddSSagi Grimberg isert_dbg("ISER ISCSI_CTRL PDU\n");
1358b8d26b3bSNicholas Bellinger break;
1359b8d26b3bSNicholas Bellinger case ISER_HELLO:
136024f412ddSSagi Grimberg isert_err("iSER Hello message\n");
1361b8d26b3bSNicholas Bellinger break;
1362b8d26b3bSNicholas Bellinger default:
1363d3cf81f9SSagi Grimberg isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags);
1364b8d26b3bSNicholas Bellinger break;
1365b8d26b3bSNicholas Bellinger }
1366b8d26b3bSNicholas Bellinger
13676c760b3dSBart Van Assche isert_rx_opcode(isert_conn, rx_desc,
1368b8d26b3bSNicholas Bellinger read_stag, read_va, write_stag, write_va);
13699679cc51SChristoph Hellwig
13709679cc51SChristoph Hellwig ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr,
13710b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
1372b8d26b3bSNicholas Bellinger }
1373b8d26b3bSNicholas Bellinger
1374b8d26b3bSNicholas Bellinger static void
isert_login_recv_done(struct ib_cq * cq,struct ib_wc * wc)13759679cc51SChristoph Hellwig isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1376b8d26b3bSNicholas Bellinger {
13779679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context;
1378fce50a2fSNicholas Bellinger struct ib_device *ib_dev = isert_conn->device->ib_device;
1379b8d26b3bSNicholas Bellinger
13809679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) {
13819679cc51SChristoph Hellwig isert_print_wc(wc, "login recv");
13829679cc51SChristoph Hellwig return;
1383b8d26b3bSNicholas Bellinger }
1384b8d26b3bSNicholas Bellinger
13850b089c1eSSagi Grimberg ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr,
13860b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
1387b8d26b3bSNicholas Bellinger
13889679cc51SChristoph Hellwig isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN;
1389b8d26b3bSNicholas Bellinger
13902371e5daSSagi Grimberg if (isert_conn->conn) {
13912371e5daSSagi Grimberg struct iscsi_login *login = isert_conn->conn->conn_login;
13922371e5daSSagi Grimberg
13932371e5daSSagi Grimberg if (login && !login->first_request)
13942371e5daSSagi Grimberg isert_rx_login_req(isert_conn);
13952371e5daSSagi Grimberg }
13969679cc51SChristoph Hellwig
1397dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex);
13982371e5daSSagi Grimberg complete(&isert_conn->login_req_comp);
1399dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex);
1400b8d26b3bSNicholas Bellinger
14010b089c1eSSagi Grimberg ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr,
14020b089c1eSSagi Grimberg ISER_RX_SIZE, DMA_FROM_DEVICE);
1403b8d26b3bSNicholas Bellinger }
1404b8d26b3bSNicholas Bellinger
1405e3d7e4c3SSagi Grimberg static void
isert_rdma_rw_ctx_destroy(struct isert_cmd * cmd,struct isert_conn * conn)140638a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn)
1407e3d7e4c3SSagi Grimberg {
140866cd9d4eSMax Gurtovoy struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd;
140938a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
1410e3d7e4c3SSagi Grimberg
141138a2d0d4SChristoph Hellwig if (!cmd->rw.nr_ops)
141238a2d0d4SChristoph Hellwig return;
141338a2d0d4SChristoph Hellwig
141438a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) {
141538a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp,
141638a2d0d4SChristoph Hellwig conn->cm_id->port_num, se_cmd->t_data_sg,
141738a2d0d4SChristoph Hellwig se_cmd->t_data_nents, se_cmd->t_prot_sg,
141838a2d0d4SChristoph Hellwig se_cmd->t_prot_nents, dir);
141938a2d0d4SChristoph Hellwig } else {
142038a2d0d4SChristoph Hellwig rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num,
142138a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents, dir);
1422e3d7e4c3SSagi Grimberg }
1423e3d7e4c3SSagi Grimberg
142438a2d0d4SChristoph Hellwig cmd->rw.nr_ops = 0;
142559464ef4SVu Pham }
142659464ef4SVu Pham
142759464ef4SVu Pham static void
isert_put_cmd(struct isert_cmd * isert_cmd,bool comp_err)142803e7848aSNicholas Bellinger isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
1429b8d26b3bSNicholas Bellinger {
143066cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd;
1431b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn;
1432be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
1433e4f4e801SSagi Grimberg struct iscsi_text_rsp *hdr;
1434b8d26b3bSNicholas Bellinger
14354c22e07fSSagi Grimberg isert_dbg("Cmd %p\n", isert_cmd);
1436b8d26b3bSNicholas Bellinger
1437b8d26b3bSNicholas Bellinger switch (cmd->iscsi_opcode) {
1438b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_CMD:
1439b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock);
1440b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node))
14415159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node);
1442b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock);
1443b8d26b3bSNicholas Bellinger
144403e7848aSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE) {
1445b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd);
144603e7848aSNicholas Bellinger /*
144703e7848aSNicholas Bellinger * Check for special case during comp_err where
144803e7848aSNicholas Bellinger * WRITE_PENDING has been handed off from core,
144903e7848aSNicholas Bellinger * but requires an extra target_put_sess_cmd()
145003e7848aSNicholas Bellinger * before transport_generic_free_cmd() below.
145103e7848aSNicholas Bellinger */
145203e7848aSNicholas Bellinger if (comp_err &&
145303e7848aSNicholas Bellinger cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
145403e7848aSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd;
145503e7848aSNicholas Bellinger
1456afc16604SBart Van Assche target_put_sess_cmd(se_cmd);
145703e7848aSNicholas Bellinger }
145803e7848aSNicholas Bellinger }
1459b8d26b3bSNicholas Bellinger
146038a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1461186a9647SNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0);
1462186a9647SNicholas Bellinger break;
1463b8d26b3bSNicholas Bellinger case ISCSI_OP_SCSI_TMFUNC:
1464186a9647SNicholas Bellinger spin_lock_bh(&conn->cmd_lock);
1465186a9647SNicholas Bellinger if (!list_empty(&cmd->i_conn_node))
14665159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node);
1467186a9647SNicholas Bellinger spin_unlock_bh(&conn->cmd_lock);
1468186a9647SNicholas Bellinger
1469b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0);
1470b8d26b3bSNicholas Bellinger break;
1471b8d26b3bSNicholas Bellinger case ISCSI_OP_REJECT:
1472b8d26b3bSNicholas Bellinger case ISCSI_OP_NOOP_OUT:
1473adb54c29SNicholas Bellinger case ISCSI_OP_TEXT:
1474e4f4e801SSagi Grimberg hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
1475e4f4e801SSagi Grimberg /* If the continue bit is on, keep the command alive */
1476e4f4e801SSagi Grimberg if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)
1477e4f4e801SSagi Grimberg break;
1478e4f4e801SSagi Grimberg
1479b8d26b3bSNicholas Bellinger spin_lock_bh(&conn->cmd_lock);
1480b8d26b3bSNicholas Bellinger if (!list_empty(&cmd->i_conn_node))
14815159d763SNicholas Bellinger list_del_init(&cmd->i_conn_node);
1482b8d26b3bSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock);
1483b8d26b3bSNicholas Bellinger
1484b8d26b3bSNicholas Bellinger /*
1485b8d26b3bSNicholas Bellinger * Handle special case for REJECT when iscsi_add_reject*() has
1486b8d26b3bSNicholas Bellinger * overwritten the original iscsi_opcode assignment, and the
1487b8d26b3bSNicholas Bellinger * associated cmd->se_cmd needs to be released.
1488b8d26b3bSNicholas Bellinger */
1489b8d26b3bSNicholas Bellinger if (cmd->se_cmd.se_tfo != NULL) {
149011378cdbSRasmus Villemoes isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
14913df8f68aSNicholas Bellinger cmd->iscsi_opcode);
1492b8d26b3bSNicholas Bellinger transport_generic_free_cmd(&cmd->se_cmd, 0);
1493b8d26b3bSNicholas Bellinger break;
1494b8d26b3bSNicholas Bellinger }
1495df561f66SGustavo A. R. Silva fallthrough;
1496b8d26b3bSNicholas Bellinger default:
1497d703ce2fSNicholas Bellinger iscsit_release_cmd(cmd);
1498b8d26b3bSNicholas Bellinger break;
1499b8d26b3bSNicholas Bellinger }
1500b8d26b3bSNicholas Bellinger }
1501b8d26b3bSNicholas Bellinger
1502b8d26b3bSNicholas Bellinger static void
isert_unmap_tx_desc(struct iser_tx_desc * tx_desc,struct ib_device * ib_dev)1503b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1504b8d26b3bSNicholas Bellinger {
1505b8d26b3bSNicholas Bellinger if (tx_desc->dma_addr != 0) {
15064c22e07fSSagi Grimberg isert_dbg("unmap single for tx_desc->dma_addr\n");
1507b8d26b3bSNicholas Bellinger ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1508b8d26b3bSNicholas Bellinger ISER_HEADERS_LEN, DMA_TO_DEVICE);
1509b8d26b3bSNicholas Bellinger tx_desc->dma_addr = 0;
1510b8d26b3bSNicholas Bellinger }
1511b8d26b3bSNicholas Bellinger }
1512b8d26b3bSNicholas Bellinger
1513b8d26b3bSNicholas Bellinger static void
isert_completion_put(struct iser_tx_desc * tx_desc,struct isert_cmd * isert_cmd,struct ib_device * ib_dev,bool comp_err)1514b8d26b3bSNicholas Bellinger isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
151503e7848aSNicholas Bellinger struct ib_device *ib_dev, bool comp_err)
1516b8d26b3bSNicholas Bellinger {
1517dbbc5d11SNicholas Bellinger if (isert_cmd->pdu_buf_dma != 0) {
15184c22e07fSSagi Grimberg isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
1519dbbc5d11SNicholas Bellinger ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
1520dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
1521dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = 0;
1522b8d26b3bSNicholas Bellinger }
1523b8d26b3bSNicholas Bellinger
1524b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev);
152503e7848aSNicholas Bellinger isert_put_cmd(isert_cmd, comp_err);
1526b8d26b3bSNicholas Bellinger }
1527b8d26b3bSNicholas Bellinger
152896b7973eSSagi Grimberg static int
isert_check_pi_status(struct se_cmd * se_cmd,struct ib_mr * sig_mr)152996b7973eSSagi Grimberg isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
1530f93f3a70SSagi Grimberg {
15319e961ae7SSagi Grimberg struct ib_mr_status mr_status;
153296b7973eSSagi Grimberg int ret;
1533f93f3a70SSagi Grimberg
153496b7973eSSagi Grimberg ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
15359e961ae7SSagi Grimberg if (ret) {
153624f412ddSSagi Grimberg isert_err("ib_check_mr_status failed, ret %d\n", ret);
15379e961ae7SSagi Grimberg goto fail_mr_status;
15389e961ae7SSagi Grimberg }
153996b7973eSSagi Grimberg
15409e961ae7SSagi Grimberg if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
154196b7973eSSagi Grimberg u64 sec_offset_err;
15429e961ae7SSagi Grimberg u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;
15439e961ae7SSagi Grimberg
15449e961ae7SSagi Grimberg switch (mr_status.sig_err.err_type) {
15459e961ae7SSagi Grimberg case IB_SIG_BAD_GUARD:
15469e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
15479e961ae7SSagi Grimberg break;
15489e961ae7SSagi Grimberg case IB_SIG_BAD_REFTAG:
15499e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
15509e961ae7SSagi Grimberg break;
15519e961ae7SSagi Grimberg case IB_SIG_BAD_APPTAG:
15529e961ae7SSagi Grimberg se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
15539e961ae7SSagi Grimberg break;
15549e961ae7SSagi Grimberg }
155596b7973eSSagi Grimberg sec_offset_err = mr_status.sig_err.sig_err_offset;
155696b7973eSSagi Grimberg do_div(sec_offset_err, block_size);
15578dd992fbSDavid Disseldorp se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba;
15589e961ae7SSagi Grimberg
15594c22e07fSSagi Grimberg isert_err("PI error found type %d at sector 0x%llx "
15609e961ae7SSagi Grimberg "expected 0x%x vs actual 0x%x\n",
15619e961ae7SSagi Grimberg mr_status.sig_err.err_type,
15628dd992fbSDavid Disseldorp (unsigned long long)se_cmd->sense_info,
15639e961ae7SSagi Grimberg mr_status.sig_err.expected,
15649e961ae7SSagi Grimberg mr_status.sig_err.actual);
15659e961ae7SSagi Grimberg ret = 1;
15669e961ae7SSagi Grimberg }
15679e961ae7SSagi Grimberg
15689e961ae7SSagi Grimberg fail_mr_status:
156996b7973eSSagi Grimberg return ret;
157096b7973eSSagi Grimberg }
157196b7973eSSagi Grimberg
157296b7973eSSagi Grimberg static void
isert_rdma_write_done(struct ib_cq * cq,struct ib_wc * wc)15739679cc51SChristoph Hellwig isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
157496b7973eSSagi Grimberg {
15759679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context;
1576dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
15779679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1578e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
157966cd9d4eSMax Gurtovoy struct se_cmd *cmd = &isert_cmd->iscsit_cmd->se_cmd;
158096b7973eSSagi Grimberg int ret = 0;
158196b7973eSSagi Grimberg
15829679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) {
15839679cc51SChristoph Hellwig isert_print_wc(wc, "rdma write");
15849679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR)
15859679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
15869679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true);
15879679cc51SChristoph Hellwig return;
15889679cc51SChristoph Hellwig }
15899679cc51SChristoph Hellwig
15909679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd);
15919679cc51SChristoph Hellwig
1592e9a53e73SIsrael Rukshin ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr);
159338a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
159496b7973eSSagi Grimberg
1595555a65f6SNicholas Bellinger if (ret) {
1596555a65f6SNicholas Bellinger /*
1597555a65f6SNicholas Bellinger * transport_generic_request_failure() expects to have
1598555a65f6SNicholas Bellinger * plus two references to handle queue-full, so re-add
1599555a65f6SNicholas Bellinger * one here as target-core will have already dropped
1600555a65f6SNicholas Bellinger * it after the first isert_put_datain() callback.
1601555a65f6SNicholas Bellinger */
1602555a65f6SNicholas Bellinger kref_get(&cmd->cmd_kref);
1603555a65f6SNicholas Bellinger transport_generic_request_failure(cmd, cmd->pi_err);
1604555a65f6SNicholas Bellinger } else {
1605555a65f6SNicholas Bellinger /*
1606555a65f6SNicholas Bellinger * XXX: isert_put_response() failure is not retried.
1607555a65f6SNicholas Bellinger */
160866cd9d4eSMax Gurtovoy ret = isert_put_response(isert_conn->conn, isert_cmd->iscsit_cmd);
16099e961ae7SSagi Grimberg if (ret)
1610555a65f6SNicholas Bellinger pr_warn_ratelimited("isert_put_response() ret: %d\n", ret);
1611555a65f6SNicholas Bellinger }
1612f93f3a70SSagi Grimberg }
1613f93f3a70SSagi Grimberg
1614f93f3a70SSagi Grimberg static void
isert_rdma_read_done(struct ib_cq * cq,struct ib_wc * wc)16159679cc51SChristoph Hellwig isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1616b8d26b3bSNicholas Bellinger {
16179679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context;
16189679cc51SChristoph Hellwig struct isert_device *device = isert_conn->device;
16199679cc51SChristoph Hellwig struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1620e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
162166cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd;
1622b8d26b3bSNicholas Bellinger struct se_cmd *se_cmd = &cmd->se_cmd;
16235bac4b1aSSagi Grimberg int ret = 0;
1624b8d26b3bSNicholas Bellinger
16259679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) {
16269679cc51SChristoph Hellwig isert_print_wc(wc, "rdma read");
16279679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR)
16289679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
16299679cc51SChristoph Hellwig isert_completion_put(desc, isert_cmd, device->ib_device, true);
16309679cc51SChristoph Hellwig return;
16319679cc51SChristoph Hellwig }
16329679cc51SChristoph Hellwig
16339679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd);
16349679cc51SChristoph Hellwig
1635b8d26b3bSNicholas Bellinger iscsit_stop_dataout_timer(cmd);
163638a2d0d4SChristoph Hellwig
163738a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd))
1638e9a53e73SIsrael Rukshin ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr);
163938a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
164038a2d0d4SChristoph Hellwig cmd->write_data_done = 0;
1641b8d26b3bSNicholas Bellinger
164224f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
1643b8d26b3bSNicholas Bellinger spin_lock_bh(&cmd->istate_lock);
1644b8d26b3bSNicholas Bellinger cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1645b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1646b8d26b3bSNicholas Bellinger spin_unlock_bh(&cmd->istate_lock);
1647b8d26b3bSNicholas Bellinger
1648555a65f6SNicholas Bellinger /*
1649555a65f6SNicholas Bellinger * transport_generic_request_failure() will drop the extra
1650555a65f6SNicholas Bellinger * se_cmd->cmd_kref reference after T10-PI error, and handle
1651555a65f6SNicholas Bellinger * any non-zero ->queue_status() callback error retries.
1652555a65f6SNicholas Bellinger */
1653555a65f6SNicholas Bellinger if (ret)
1654555a65f6SNicholas Bellinger transport_generic_request_failure(se_cmd, se_cmd->pi_err);
1655555a65f6SNicholas Bellinger else
1656b8d26b3bSNicholas Bellinger target_execute_cmd(se_cmd);
1657b8d26b3bSNicholas Bellinger }
1658b8d26b3bSNicholas Bellinger
1659b8d26b3bSNicholas Bellinger static void
isert_do_control_comp(struct work_struct * work)1660b8d26b3bSNicholas Bellinger isert_do_control_comp(struct work_struct *work)
1661b8d26b3bSNicholas Bellinger {
1662b8d26b3bSNicholas Bellinger struct isert_cmd *isert_cmd = container_of(work,
1663b8d26b3bSNicholas Bellinger struct isert_cmd, comp_work);
1664b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = isert_cmd->conn;
1665dac6ab30SSagi Grimberg struct ib_device *ib_dev = isert_conn->cm_id->device;
166666cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd;
1667b8d26b3bSNicholas Bellinger
16684c22e07fSSagi Grimberg isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);
16694c22e07fSSagi Grimberg
1670b8d26b3bSNicholas Bellinger switch (cmd->i_state) {
1671b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP:
1672b8d26b3bSNicholas Bellinger iscsit_tmr_post_handler(cmd, cmd->conn);
1673df561f66SGustavo A. R. Silva fallthrough;
1674d2271bd0SBart Van Assche case ISTATE_SEND_REJECT:
1675d2271bd0SBart Van Assche case ISTATE_SEND_TEXTRSP:
1676b8d26b3bSNicholas Bellinger cmd->i_state = ISTATE_SENT_STATUS;
16774c22e07fSSagi Grimberg isert_completion_put(&isert_cmd->tx_desc, isert_cmd,
16784c22e07fSSagi Grimberg ib_dev, false);
16793df8f68aSNicholas Bellinger break;
1680b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP:
1681b8d26b3bSNicholas Bellinger iscsit_logout_post_handler(cmd, cmd->conn);
1682b8d26b3bSNicholas Bellinger break;
1683b8d26b3bSNicholas Bellinger default:
16844c22e07fSSagi Grimberg isert_err("Unknown i_state %d\n", cmd->i_state);
1685b8d26b3bSNicholas Bellinger dump_stack();
1686b8d26b3bSNicholas Bellinger break;
1687b8d26b3bSNicholas Bellinger }
1688b8d26b3bSNicholas Bellinger }
1689b8d26b3bSNicholas Bellinger
1690b8d26b3bSNicholas Bellinger static void
isert_login_send_done(struct ib_cq * cq,struct ib_wc * wc)16919679cc51SChristoph Hellwig isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc)
1692b8d26b3bSNicholas Bellinger {
16939679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context;
16949679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device;
16959679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1696b8d26b3bSNicholas Bellinger
16979679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) {
16989679cc51SChristoph Hellwig isert_print_wc(wc, "login send");
16999679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR)
17009679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
17019679cc51SChristoph Hellwig }
17029679cc51SChristoph Hellwig
17039679cc51SChristoph Hellwig isert_unmap_tx_desc(tx_desc, ib_dev);
17049679cc51SChristoph Hellwig }
17059679cc51SChristoph Hellwig
17069679cc51SChristoph Hellwig static void
isert_send_done(struct ib_cq * cq,struct ib_wc * wc)17079679cc51SChristoph Hellwig isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
17089679cc51SChristoph Hellwig {
17099679cc51SChristoph Hellwig struct isert_conn *isert_conn = wc->qp->qp_context;
17109679cc51SChristoph Hellwig struct ib_device *ib_dev = isert_conn->cm_id->device;
17119679cc51SChristoph Hellwig struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1712e3416ab2SChristoph Hellwig struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc);
17139679cc51SChristoph Hellwig
17149679cc51SChristoph Hellwig if (unlikely(wc->status != IB_WC_SUCCESS)) {
17159679cc51SChristoph Hellwig isert_print_wc(wc, "send");
17169679cc51SChristoph Hellwig if (wc->status != IB_WC_WR_FLUSH_ERR)
17179679cc51SChristoph Hellwig iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
17189679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
17199679cc51SChristoph Hellwig return;
17209679cc51SChristoph Hellwig }
17219679cc51SChristoph Hellwig
17229679cc51SChristoph Hellwig isert_dbg("Cmd %p\n", isert_cmd);
17239679cc51SChristoph Hellwig
172466cd9d4eSMax Gurtovoy switch (isert_cmd->iscsit_cmd->i_state) {
17259679cc51SChristoph Hellwig case ISTATE_SEND_TASKMGTRSP:
17269679cc51SChristoph Hellwig case ISTATE_SEND_LOGOUTRSP:
17279679cc51SChristoph Hellwig case ISTATE_SEND_REJECT:
17289679cc51SChristoph Hellwig case ISTATE_SEND_TEXTRSP:
1729b8d26b3bSNicholas Bellinger isert_unmap_tx_desc(tx_desc, ib_dev);
1730b8d26b3bSNicholas Bellinger
1731b8d26b3bSNicholas Bellinger INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1732b8d26b3bSNicholas Bellinger queue_work(isert_comp_wq, &isert_cmd->comp_work);
1733b8d26b3bSNicholas Bellinger return;
1734b8d26b3bSNicholas Bellinger default:
173566cd9d4eSMax Gurtovoy isert_cmd->iscsit_cmd->i_state = ISTATE_SENT_STATUS;
17369679cc51SChristoph Hellwig isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
1737b8d26b3bSNicholas Bellinger break;
1738b8d26b3bSNicholas Bellinger }
1739b8d26b3bSNicholas Bellinger }
1740b8d26b3bSNicholas Bellinger
1741b8d26b3bSNicholas Bellinger static int
isert_post_response(struct isert_conn * isert_conn,struct isert_cmd * isert_cmd)1742b8d26b3bSNicholas Bellinger isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1743b8d26b3bSNicholas Bellinger {
1744b8d26b3bSNicholas Bellinger int ret;
1745b8d26b3bSNicholas Bellinger
17464366b19cSJenny Derzhavetz ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
1747c97119b6SMax Gurtovoy if (ret)
17484366b19cSJenny Derzhavetz return ret;
17494366b19cSJenny Derzhavetz
1750e01a7674SBart Van Assche ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL);
1751b8d26b3bSNicholas Bellinger if (ret) {
175224f412ddSSagi Grimberg isert_err("ib_post_send failed with %d\n", ret);
1753b8d26b3bSNicholas Bellinger return ret;
1754b8d26b3bSNicholas Bellinger }
1755b8d26b3bSNicholas Bellinger return ret;
1756b8d26b3bSNicholas Bellinger }
1757b8d26b3bSNicholas Bellinger
1758b8d26b3bSNicholas Bellinger static int
isert_put_response(struct iscsit_conn * conn,struct iscsit_cmd * cmd)1759be36d683SMax Gurtovoy isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
1760b8d26b3bSNicholas Bellinger {
1761d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
17626700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1763b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1764b8d26b3bSNicholas Bellinger struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1765b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header;
1766b8d26b3bSNicholas Bellinger
1767b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1768b8d26b3bSNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1769b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1770b8d26b3bSNicholas Bellinger /*
1771b8d26b3bSNicholas Bellinger * Attach SENSE DATA payload to iSCSI Response PDU
1772b8d26b3bSNicholas Bellinger */
1773b8d26b3bSNicholas Bellinger if (cmd->se_cmd.sense_buffer &&
1774b8d26b3bSNicholas Bellinger ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1775b8d26b3bSNicholas Bellinger (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1776dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
177767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device;
1778b8d26b3bSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1779dbbc5d11SNicholas Bellinger u32 padding, pdu_len;
1780b8d26b3bSNicholas Bellinger
1781b8d26b3bSNicholas Bellinger put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1782b8d26b3bSNicholas Bellinger cmd->sense_buffer);
1783b8d26b3bSNicholas Bellinger cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1784b8d26b3bSNicholas Bellinger
1785b8d26b3bSNicholas Bellinger padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1786b8d26b3bSNicholas Bellinger hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1787dbbc5d11SNicholas Bellinger pdu_len = cmd->se_cmd.scsi_sense_length + padding;
1788b8d26b3bSNicholas Bellinger
1789dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
1790dbbc5d11SNicholas Bellinger (void *)cmd->sense_buffer, pdu_len,
1791b8d26b3bSNicholas Bellinger DMA_TO_DEVICE);
1792def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1793def4a6ffSAlexey Khoroshilov return -ENOMEM;
1794b8d26b3bSNicholas Bellinger
1795dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = pdu_len;
1796dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma;
1797dbbc5d11SNicholas Bellinger tx_dsg->length = pdu_len;
179834efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey;
1799b8d26b3bSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2;
1800b8d26b3bSNicholas Bellinger }
1801b8d26b3bSNicholas Bellinger
180268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1803b8d26b3bSNicholas Bellinger
18044c22e07fSSagi Grimberg isert_dbg("Posting SCSI Response\n");
1805b8d26b3bSNicholas Bellinger
1806b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd);
1807b8d26b3bSNicholas Bellinger }
1808b8d26b3bSNicholas Bellinger
1809131e6abcSNicholas Bellinger static void
isert_aborted_task(struct iscsit_conn * conn,struct iscsit_cmd * cmd)1810be36d683SMax Gurtovoy isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
1811131e6abcSNicholas Bellinger {
1812131e6abcSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
18136700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1814131e6abcSNicholas Bellinger
1815131e6abcSNicholas Bellinger spin_lock_bh(&conn->cmd_lock);
1816131e6abcSNicholas Bellinger if (!list_empty(&cmd->i_conn_node))
1817131e6abcSNicholas Bellinger list_del_init(&cmd->i_conn_node);
1818131e6abcSNicholas Bellinger spin_unlock_bh(&conn->cmd_lock);
1819131e6abcSNicholas Bellinger
1820131e6abcSNicholas Bellinger if (cmd->data_direction == DMA_TO_DEVICE)
1821131e6abcSNicholas Bellinger iscsit_stop_dataout_timer(cmd);
182238a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1823131e6abcSNicholas Bellinger }
1824131e6abcSNicholas Bellinger
1825e70beee7SNicholas Bellinger static enum target_prot_op
isert_get_sup_prot_ops(struct iscsit_conn * conn)1826be36d683SMax Gurtovoy isert_get_sup_prot_ops(struct iscsit_conn *conn)
1827e70beee7SNicholas Bellinger {
18286700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1829dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
1830e70beee7SNicholas Bellinger
183123a548eeSSagi Grimberg if (conn->tpg->tpg_attrib.t10_pi) {
183223a548eeSSagi Grimberg if (device->pi_capable) {
183324f412ddSSagi Grimberg isert_info("conn %p PI offload enabled\n", isert_conn);
183423a548eeSSagi Grimberg isert_conn->pi_support = true;
1835e70beee7SNicholas Bellinger return TARGET_PROT_ALL;
183623a548eeSSagi Grimberg }
183723a548eeSSagi Grimberg }
183823a548eeSSagi Grimberg
183924f412ddSSagi Grimberg isert_info("conn %p PI offload disabled\n", isert_conn);
184023a548eeSSagi Grimberg isert_conn->pi_support = false;
1841e70beee7SNicholas Bellinger
1842e70beee7SNicholas Bellinger return TARGET_PROT_NORMAL;
1843e70beee7SNicholas Bellinger }
1844e70beee7SNicholas Bellinger
1845b8d26b3bSNicholas Bellinger static int
isert_put_nopin(struct iscsit_cmd * cmd,struct iscsit_conn * conn,bool nopout_response)1846be36d683SMax Gurtovoy isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn,
1847b8d26b3bSNicholas Bellinger bool nopout_response)
1848b8d26b3bSNicholas Bellinger {
1849d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
18506700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1851b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1852b8d26b3bSNicholas Bellinger
1853b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1854b8d26b3bSNicholas Bellinger iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1855b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header,
1856b8d26b3bSNicholas Bellinger nopout_response);
1857b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
185868a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1859b8d26b3bSNicholas Bellinger
18604c22e07fSSagi Grimberg isert_dbg("conn %p Posting NOPIN Response\n", isert_conn);
1861b8d26b3bSNicholas Bellinger
1862b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd);
1863b8d26b3bSNicholas Bellinger }
1864b8d26b3bSNicholas Bellinger
1865b8d26b3bSNicholas Bellinger static int
isert_put_logout_rsp(struct iscsit_cmd * cmd,struct iscsit_conn * conn)1866be36d683SMax Gurtovoy isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
1867b8d26b3bSNicholas Bellinger {
1868d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
18696700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1870b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1871b8d26b3bSNicholas Bellinger
1872b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1873b8d26b3bSNicholas Bellinger iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1874b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header);
1875b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
187668a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1877b8d26b3bSNicholas Bellinger
18784c22e07fSSagi Grimberg isert_dbg("conn %p Posting Logout Response\n", isert_conn);
1879b8d26b3bSNicholas Bellinger
1880b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd);
1881b8d26b3bSNicholas Bellinger }
1882b8d26b3bSNicholas Bellinger
1883b8d26b3bSNicholas Bellinger static int
isert_put_tm_rsp(struct iscsit_cmd * cmd,struct iscsit_conn * conn)1884be36d683SMax Gurtovoy isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
1885b8d26b3bSNicholas Bellinger {
1886d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
18876700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1888b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1889b8d26b3bSNicholas Bellinger
1890b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1891b8d26b3bSNicholas Bellinger iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1892b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header);
1893b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
189468a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1895b8d26b3bSNicholas Bellinger
18964c22e07fSSagi Grimberg isert_dbg("conn %p Posting Task Management Response\n", isert_conn);
1897b8d26b3bSNicholas Bellinger
1898b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd);
1899b8d26b3bSNicholas Bellinger }
1900b8d26b3bSNicholas Bellinger
1901b8d26b3bSNicholas Bellinger static int
isert_put_reject(struct iscsit_cmd * cmd,struct iscsit_conn * conn)1902be36d683SMax Gurtovoy isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
1903b8d26b3bSNicholas Bellinger {
1904d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
19056700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1906b8d26b3bSNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1907dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
190867cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device;
19093df8f68aSNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
19103df8f68aSNicholas Bellinger struct iscsi_reject *hdr =
19113df8f68aSNicholas Bellinger (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
1912b8d26b3bSNicholas Bellinger
1913b8d26b3bSNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
19143df8f68aSNicholas Bellinger iscsit_build_reject(cmd, conn, hdr);
1915b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
19163df8f68aSNicholas Bellinger
19173df8f68aSNicholas Bellinger hton24(hdr->dlength, ISCSI_HDR_LEN);
1918dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
19193df8f68aSNicholas Bellinger (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
19203df8f68aSNicholas Bellinger DMA_TO_DEVICE);
1921def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1922def4a6ffSAlexey Khoroshilov return -ENOMEM;
1923dbbc5d11SNicholas Bellinger isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
1924dbbc5d11SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma;
19253df8f68aSNicholas Bellinger tx_dsg->length = ISCSI_HDR_LEN;
192634efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey;
19273df8f68aSNicholas Bellinger isert_cmd->tx_desc.num_sge = 2;
19283df8f68aSNicholas Bellinger
192968a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1930b8d26b3bSNicholas Bellinger
19314c22e07fSSagi Grimberg isert_dbg("conn %p Posting Reject\n", isert_conn);
1932b8d26b3bSNicholas Bellinger
1933b8d26b3bSNicholas Bellinger return isert_post_response(isert_conn, isert_cmd);
1934b8d26b3bSNicholas Bellinger }
1935b8d26b3bSNicholas Bellinger
1936b8d26b3bSNicholas Bellinger static int
isert_put_text_rsp(struct iscsit_cmd * cmd,struct iscsit_conn * conn)1937be36d683SMax Gurtovoy isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn)
1938adb54c29SNicholas Bellinger {
1939d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
19406700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
1941adb54c29SNicholas Bellinger struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1942adb54c29SNicholas Bellinger struct iscsi_text_rsp *hdr =
1943adb54c29SNicholas Bellinger (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
1944adb54c29SNicholas Bellinger u32 txt_rsp_len;
1945adb54c29SNicholas Bellinger int rc;
1946adb54c29SNicholas Bellinger
1947adb54c29SNicholas Bellinger isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
194822c7aaa5SSagi Grimberg rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
1949adb54c29SNicholas Bellinger if (rc < 0)
1950adb54c29SNicholas Bellinger return rc;
1951adb54c29SNicholas Bellinger
1952adb54c29SNicholas Bellinger txt_rsp_len = rc;
1953adb54c29SNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1954adb54c29SNicholas Bellinger
1955adb54c29SNicholas Bellinger if (txt_rsp_len) {
1956dac6ab30SSagi Grimberg struct isert_device *device = isert_conn->device;
195767cb3949SSagi Grimberg struct ib_device *ib_dev = device->ib_device;
1958adb54c29SNicholas Bellinger struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1959adb54c29SNicholas Bellinger void *txt_rsp_buf = cmd->buf_ptr;
1960adb54c29SNicholas Bellinger
1961adb54c29SNicholas Bellinger isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
1962adb54c29SNicholas Bellinger txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
1963def4a6ffSAlexey Khoroshilov if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1964def4a6ffSAlexey Khoroshilov return -ENOMEM;
1965adb54c29SNicholas Bellinger
1966adb54c29SNicholas Bellinger isert_cmd->pdu_buf_len = txt_rsp_len;
1967adb54c29SNicholas Bellinger tx_dsg->addr = isert_cmd->pdu_buf_dma;
1968adb54c29SNicholas Bellinger tx_dsg->length = txt_rsp_len;
196934efc7dfSJason Gunthorpe tx_dsg->lkey = device->pd->local_dma_lkey;
1970adb54c29SNicholas Bellinger isert_cmd->tx_desc.num_sge = 2;
1971adb54c29SNicholas Bellinger }
197268a86deeSSagi Grimberg isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1973adb54c29SNicholas Bellinger
1974f64d2792SSagi Grimberg isert_dbg("conn %p Text Response\n", isert_conn);
1975adb54c29SNicholas Bellinger
1976adb54c29SNicholas Bellinger return isert_post_response(isert_conn, isert_cmd);
1977adb54c29SNicholas Bellinger }
1978adb54c29SNicholas Bellinger
19793d73cf1aSSagi Grimberg static inline void
isert_set_dif_domain(struct se_cmd * se_cmd,struct ib_sig_domain * domain)1980dae6345aSIsrael Rukshin isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain)
19819e961ae7SSagi Grimberg {
198278eda2bbSSagi Grimberg domain->sig_type = IB_SIG_TYPE_T10_DIF;
19833d73cf1aSSagi Grimberg domain->sig.dif.bg_type = IB_T10DIF_CRC;
19843d73cf1aSSagi Grimberg domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
19853d73cf1aSSagi Grimberg domain->sig.dif.ref_tag = se_cmd->reftag_seed;
198678eda2bbSSagi Grimberg /*
198778eda2bbSSagi Grimberg * At the moment we hard code those, but if in the future
198878eda2bbSSagi Grimberg * the target core would like to use it, we will take it
198978eda2bbSSagi Grimberg * from se_cmd.
199078eda2bbSSagi Grimberg */
199178eda2bbSSagi Grimberg domain->sig.dif.apptag_check_mask = 0xffff;
199278eda2bbSSagi Grimberg domain->sig.dif.app_escape = true;
199378eda2bbSSagi Grimberg domain->sig.dif.ref_escape = true;
199478eda2bbSSagi Grimberg if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
199578eda2bbSSagi Grimberg se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
199678eda2bbSSagi Grimberg domain->sig.dif.ref_remap = true;
1997ec53a2a6SMax Gurtovoy }
19983d73cf1aSSagi Grimberg
19993d73cf1aSSagi Grimberg static int
isert_set_sig_attrs(struct se_cmd * se_cmd,struct ib_sig_attrs * sig_attrs)20003d73cf1aSSagi Grimberg isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
20013d73cf1aSSagi Grimberg {
200238a2d0d4SChristoph Hellwig memset(sig_attrs, 0, sizeof(*sig_attrs));
200338a2d0d4SChristoph Hellwig
20049e961ae7SSagi Grimberg switch (se_cmd->prot_op) {
20059e961ae7SSagi Grimberg case TARGET_PROT_DIN_INSERT:
20069e961ae7SSagi Grimberg case TARGET_PROT_DOUT_STRIP:
200778eda2bbSSagi Grimberg sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
2008dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire);
20099e961ae7SSagi Grimberg break;
20109e961ae7SSagi Grimberg case TARGET_PROT_DOUT_INSERT:
20119e961ae7SSagi Grimberg case TARGET_PROT_DIN_STRIP:
201278eda2bbSSagi Grimberg sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
2013dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem);
20149e961ae7SSagi Grimberg break;
20159e961ae7SSagi Grimberg case TARGET_PROT_DIN_PASS:
20169e961ae7SSagi Grimberg case TARGET_PROT_DOUT_PASS:
2017dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->wire);
2018dae6345aSIsrael Rukshin isert_set_dif_domain(se_cmd, &sig_attrs->mem);
20199e961ae7SSagi Grimberg break;
20209e961ae7SSagi Grimberg default:
202124f412ddSSagi Grimberg isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
20229e961ae7SSagi Grimberg return -EINVAL;
20239e961ae7SSagi Grimberg }
20249e961ae7SSagi Grimberg
202526232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD)
202626232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
202726232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG)
202826232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
202926232872SMax Gurtovoy if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG)
203026232872SMax Gurtovoy sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
203126232872SMax Gurtovoy
20329e961ae7SSagi Grimberg return 0;
20339e961ae7SSagi Grimberg }
20349e961ae7SSagi Grimberg
20359e961ae7SSagi Grimberg static int
isert_rdma_rw_ctx_post(struct isert_cmd * cmd,struct isert_conn * conn,struct ib_cqe * cqe,struct ib_send_wr * chain_wr)203638a2d0d4SChristoph Hellwig isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
203738a2d0d4SChristoph Hellwig struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
20389e961ae7SSagi Grimberg {
203966cd9d4eSMax Gurtovoy struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd;
204038a2d0d4SChristoph Hellwig enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
204138a2d0d4SChristoph Hellwig u8 port_num = conn->cm_id->port_num;
204238a2d0d4SChristoph Hellwig u64 addr;
204338a2d0d4SChristoph Hellwig u32 rkey, offset;
20449e961ae7SSagi Grimberg int ret;
20459e961ae7SSagi Grimberg
204666f53e6fSBharat Potnuri if (cmd->ctx_init_done)
204766f53e6fSBharat Potnuri goto rdma_ctx_post;
204866f53e6fSBharat Potnuri
204938a2d0d4SChristoph Hellwig if (dir == DMA_FROM_DEVICE) {
205038a2d0d4SChristoph Hellwig addr = cmd->write_va;
205138a2d0d4SChristoph Hellwig rkey = cmd->write_stag;
205266cd9d4eSMax Gurtovoy offset = cmd->iscsit_cmd->write_data_done;
205338a2d0d4SChristoph Hellwig } else {
205438a2d0d4SChristoph Hellwig addr = cmd->read_va;
205538a2d0d4SChristoph Hellwig rkey = cmd->read_stag;
205638a2d0d4SChristoph Hellwig offset = 0;
205738a2d0d4SChristoph Hellwig }
205838a2d0d4SChristoph Hellwig
205938a2d0d4SChristoph Hellwig if (isert_prot_cmd(conn, se_cmd)) {
206038a2d0d4SChristoph Hellwig struct ib_sig_attrs sig_attrs;
206138a2d0d4SChristoph Hellwig
20629e961ae7SSagi Grimberg ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
20639e961ae7SSagi Grimberg if (ret)
2064e3d7e4c3SSagi Grimberg return ret;
2065e3d7e4c3SSagi Grimberg
206638a2d0d4SChristoph Hellwig WARN_ON_ONCE(offset);
206738a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num,
206838a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents,
206938a2d0d4SChristoph Hellwig se_cmd->t_prot_sg, se_cmd->t_prot_nents,
207038a2d0d4SChristoph Hellwig &sig_attrs, addr, rkey, dir);
2071570db170SSagi Grimberg } else {
207238a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num,
207338a2d0d4SChristoph Hellwig se_cmd->t_data_sg, se_cmd->t_data_nents,
207438a2d0d4SChristoph Hellwig offset, addr, rkey, dir);
207538a2d0d4SChristoph Hellwig }
207666f53e6fSBharat Potnuri
207738a2d0d4SChristoph Hellwig if (ret < 0) {
207838a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to prepare RDMA res\n", cmd);
207938a2d0d4SChristoph Hellwig return ret;
20809e961ae7SSagi Grimberg }
20819e961ae7SSagi Grimberg
208266f53e6fSBharat Potnuri cmd->ctx_init_done = true;
208366f53e6fSBharat Potnuri
208466f53e6fSBharat Potnuri rdma_ctx_post:
208538a2d0d4SChristoph Hellwig ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr);
208638a2d0d4SChristoph Hellwig if (ret < 0)
208738a2d0d4SChristoph Hellwig isert_err("Cmd: %p failed to post RDMA res\n", cmd);
208859464ef4SVu Pham return ret;
208959464ef4SVu Pham }
209059464ef4SVu Pham
209159464ef4SVu Pham static int
isert_put_datain(struct iscsit_conn * conn,struct iscsit_cmd * cmd)2092be36d683SMax Gurtovoy isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
209390ecc6e2SVu Pham {
209490ecc6e2SVu Pham struct se_cmd *se_cmd = &cmd->se_cmd;
209559464ef4SVu Pham struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
20966700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
209738a2d0d4SChristoph Hellwig struct ib_cqe *cqe = NULL;
209838a2d0d4SChristoph Hellwig struct ib_send_wr *chain_wr = NULL;
209990ecc6e2SVu Pham int rc;
210090ecc6e2SVu Pham
210124f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
210290ecc6e2SVu Pham isert_cmd, se_cmd->data_length);
21034c22e07fSSagi Grimberg
210438a2d0d4SChristoph Hellwig if (isert_prot_cmd(isert_conn, se_cmd)) {
210538a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done;
210638a2d0d4SChristoph Hellwig cqe = &isert_cmd->tx_desc.tx_cqe;
210738a2d0d4SChristoph Hellwig } else {
2108b8d26b3bSNicholas Bellinger /*
2109b8d26b3bSNicholas Bellinger * Build isert_conn->tx_desc for iSCSI response PDU and attach
2110b8d26b3bSNicholas Bellinger */
21119e961ae7SSagi Grimberg isert_create_send_desc(isert_conn, isert_cmd,
21129e961ae7SSagi Grimberg &isert_cmd->tx_desc);
211304d9cd12SNicholas Bellinger iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
2114b8d26b3bSNicholas Bellinger &isert_cmd->tx_desc.iscsi_header);
2115b8d26b3bSNicholas Bellinger isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
211695b60f07SNicholas Bellinger isert_init_send_wr(isert_conn, isert_cmd,
211768a86deeSSagi Grimberg &isert_cmd->tx_desc.send_wr);
21184366b19cSJenny Derzhavetz
21194366b19cSJenny Derzhavetz rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
2120c97119b6SMax Gurtovoy if (rc)
21214366b19cSJenny Derzhavetz return rc;
212238a2d0d4SChristoph Hellwig
212338a2d0d4SChristoph Hellwig chain_wr = &isert_cmd->tx_desc.send_wr;
21249e961ae7SSagi Grimberg }
2125b8d26b3bSNicholas Bellinger
2126555a65f6SNicholas Bellinger rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr);
2127555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n",
2128555a65f6SNicholas Bellinger isert_cmd, rc);
2129555a65f6SNicholas Bellinger return rc;
2130b8d26b3bSNicholas Bellinger }
2131b8d26b3bSNicholas Bellinger
2132b8d26b3bSNicholas Bellinger static int
isert_get_dataout(struct iscsit_conn * conn,struct iscsit_cmd * cmd,bool recovery)2133be36d683SMax Gurtovoy isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery)
2134b8d26b3bSNicholas Bellinger {
2135d703ce2fSNicholas Bellinger struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2136555a65f6SNicholas Bellinger int ret;
2137b8d26b3bSNicholas Bellinger
213824f412ddSSagi Grimberg isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
213938a2d0d4SChristoph Hellwig isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done);
2140b8d26b3bSNicholas Bellinger
214138a2d0d4SChristoph Hellwig isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done;
2142555a65f6SNicholas Bellinger ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context,
214338a2d0d4SChristoph Hellwig &isert_cmd->tx_desc.tx_cqe, NULL);
2144bdf20e72SSagi Grimberg
2145555a65f6SNicholas Bellinger isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n",
2146555a65f6SNicholas Bellinger isert_cmd, ret);
2147555a65f6SNicholas Bellinger return ret;
2148b8d26b3bSNicholas Bellinger }
2149b8d26b3bSNicholas Bellinger
2150b8d26b3bSNicholas Bellinger static int
isert_immediate_queue(struct iscsit_conn * conn,struct iscsit_cmd * cmd,int state)2151be36d683SMax Gurtovoy isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state)
2152b8d26b3bSNicholas Bellinger {
2153a4c15cd9SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2154a4c15cd9SJenny Derzhavetz int ret = 0;
2155b8d26b3bSNicholas Bellinger
2156b8d26b3bSNicholas Bellinger switch (state) {
2157a4c15cd9SJenny Derzhavetz case ISTATE_REMOVE:
2158a4c15cd9SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock);
2159a4c15cd9SJenny Derzhavetz list_del_init(&cmd->i_conn_node);
2160a4c15cd9SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock);
2161a4c15cd9SJenny Derzhavetz isert_put_cmd(isert_cmd, true);
2162a4c15cd9SJenny Derzhavetz break;
2163b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN_WANT_RESPONSE:
2164b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, false);
2165b8d26b3bSNicholas Bellinger break;
2166b8d26b3bSNicholas Bellinger default:
216724f412ddSSagi Grimberg isert_err("Unknown immediate state: 0x%02x\n", state);
2168b8d26b3bSNicholas Bellinger ret = -EINVAL;
2169b8d26b3bSNicholas Bellinger break;
2170b8d26b3bSNicholas Bellinger }
2171b8d26b3bSNicholas Bellinger
2172b8d26b3bSNicholas Bellinger return ret;
2173b8d26b3bSNicholas Bellinger }
2174b8d26b3bSNicholas Bellinger
2175b8d26b3bSNicholas Bellinger static int
isert_response_queue(struct iscsit_conn * conn,struct iscsit_cmd * cmd,int state)2176be36d683SMax Gurtovoy isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state)
2177b8d26b3bSNicholas Bellinger {
2178991bb764SSagi Grimberg struct isert_conn *isert_conn = conn->context;
2179b8d26b3bSNicholas Bellinger int ret;
2180b8d26b3bSNicholas Bellinger
2181b8d26b3bSNicholas Bellinger switch (state) {
2182b8d26b3bSNicholas Bellinger case ISTATE_SEND_LOGOUTRSP:
2183b8d26b3bSNicholas Bellinger ret = isert_put_logout_rsp(cmd, conn);
2184991bb764SSagi Grimberg if (!ret)
2185991bb764SSagi Grimberg isert_conn->logout_posted = true;
2186b8d26b3bSNicholas Bellinger break;
2187b8d26b3bSNicholas Bellinger case ISTATE_SEND_NOPIN:
2188b8d26b3bSNicholas Bellinger ret = isert_put_nopin(cmd, conn, true);
2189b8d26b3bSNicholas Bellinger break;
2190b8d26b3bSNicholas Bellinger case ISTATE_SEND_TASKMGTRSP:
2191b8d26b3bSNicholas Bellinger ret = isert_put_tm_rsp(cmd, conn);
2192b8d26b3bSNicholas Bellinger break;
2193b8d26b3bSNicholas Bellinger case ISTATE_SEND_REJECT:
2194b8d26b3bSNicholas Bellinger ret = isert_put_reject(cmd, conn);
2195b8d26b3bSNicholas Bellinger break;
2196adb54c29SNicholas Bellinger case ISTATE_SEND_TEXTRSP:
2197adb54c29SNicholas Bellinger ret = isert_put_text_rsp(cmd, conn);
2198adb54c29SNicholas Bellinger break;
2199b8d26b3bSNicholas Bellinger case ISTATE_SEND_STATUS:
2200b8d26b3bSNicholas Bellinger /*
2201b8d26b3bSNicholas Bellinger * Special case for sending non GOOD SCSI status from TX thread
2202b8d26b3bSNicholas Bellinger * context during pre se_cmd excecution failure.
2203b8d26b3bSNicholas Bellinger */
2204b8d26b3bSNicholas Bellinger ret = isert_put_response(conn, cmd);
2205b8d26b3bSNicholas Bellinger break;
2206b8d26b3bSNicholas Bellinger default:
220724f412ddSSagi Grimberg isert_err("Unknown response state: 0x%02x\n", state);
2208b8d26b3bSNicholas Bellinger ret = -EINVAL;
2209b8d26b3bSNicholas Bellinger break;
2210b8d26b3bSNicholas Bellinger }
2211b8d26b3bSNicholas Bellinger
2212b8d26b3bSNicholas Bellinger return ret;
2213b8d26b3bSNicholas Bellinger }
2214b8d26b3bSNicholas Bellinger
2215ca6c1d82SSagi Grimberg struct rdma_cm_id *
isert_setup_id(struct isert_np * isert_np)2216ca6c1d82SSagi Grimberg isert_setup_id(struct isert_np *isert_np)
2217ca6c1d82SSagi Grimberg {
2218ca6c1d82SSagi Grimberg struct iscsi_np *np = isert_np->np;
2219ca6c1d82SSagi Grimberg struct rdma_cm_id *id;
2220ca6c1d82SSagi Grimberg struct sockaddr *sa;
2221ca6c1d82SSagi Grimberg int ret;
2222ca6c1d82SSagi Grimberg
2223ca6c1d82SSagi Grimberg sa = (struct sockaddr *)&np->np_sockaddr;
222424f412ddSSagi Grimberg isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
2225ca6c1d82SSagi Grimberg
2226fa20105eSGuy Shapiro id = rdma_create_id(&init_net, isert_cma_handler, isert_np,
2227ca6c1d82SSagi Grimberg RDMA_PS_TCP, IB_QPT_RC);
2228ca6c1d82SSagi Grimberg if (IS_ERR(id)) {
222924f412ddSSagi Grimberg isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
2230ca6c1d82SSagi Grimberg ret = PTR_ERR(id);
2231ca6c1d82SSagi Grimberg goto out;
2232ca6c1d82SSagi Grimberg }
223324f412ddSSagi Grimberg isert_dbg("id %p context %p\n", id, id->context);
2234ca6c1d82SSagi Grimberg
2235221442eaSMax Gurtovoy /*
2236221442eaSMax Gurtovoy * Allow both IPv4 and IPv6 sockets to bind a single port
2237221442eaSMax Gurtovoy * at the same time.
2238221442eaSMax Gurtovoy */
2239221442eaSMax Gurtovoy ret = rdma_set_afonly(id, 1);
2240221442eaSMax Gurtovoy if (ret) {
2241221442eaSMax Gurtovoy isert_err("rdma_set_afonly() failed: %d\n", ret);
2242221442eaSMax Gurtovoy goto out_id;
2243221442eaSMax Gurtovoy }
2244221442eaSMax Gurtovoy
2245ca6c1d82SSagi Grimberg ret = rdma_bind_addr(id, sa);
2246ca6c1d82SSagi Grimberg if (ret) {
224724f412ddSSagi Grimberg isert_err("rdma_bind_addr() failed: %d\n", ret);
2248ca6c1d82SSagi Grimberg goto out_id;
2249ca6c1d82SSagi Grimberg }
2250ca6c1d82SSagi Grimberg
2251992607e8SSagi Grimberg ret = rdma_listen(id, 0);
2252ca6c1d82SSagi Grimberg if (ret) {
225324f412ddSSagi Grimberg isert_err("rdma_listen() failed: %d\n", ret);
2254ca6c1d82SSagi Grimberg goto out_id;
2255ca6c1d82SSagi Grimberg }
2256ca6c1d82SSagi Grimberg
2257ca6c1d82SSagi Grimberg return id;
2258ca6c1d82SSagi Grimberg out_id:
2259ca6c1d82SSagi Grimberg rdma_destroy_id(id);
2260ca6c1d82SSagi Grimberg out:
2261ca6c1d82SSagi Grimberg return ERR_PTR(ret);
2262ca6c1d82SSagi Grimberg }
2263ca6c1d82SSagi Grimberg
2264b8d26b3bSNicholas Bellinger static int
isert_setup_np(struct iscsi_np * np,struct sockaddr_storage * ksockaddr)2265b8d26b3bSNicholas Bellinger isert_setup_np(struct iscsi_np *np,
226613a3cf08SAndy Grover struct sockaddr_storage *ksockaddr)
2267b8d26b3bSNicholas Bellinger {
2268b8d26b3bSNicholas Bellinger struct isert_np *isert_np;
2269b8d26b3bSNicholas Bellinger struct rdma_cm_id *isert_lid;
2270b8d26b3bSNicholas Bellinger int ret;
2271b8d26b3bSNicholas Bellinger
2272b8d26b3bSNicholas Bellinger isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
227393b80f29SLeon Romanovsky if (!isert_np)
2274b8d26b3bSNicholas Bellinger return -ENOMEM;
227593b80f29SLeon Romanovsky
2276ed8cb0a4SJenny Derzhavetz sema_init(&isert_np->sem, 0);
2277ed8cb0a4SJenny Derzhavetz mutex_init(&isert_np->mutex);
2278bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->accepted);
2279bd379220SJenny Derzhavetz INIT_LIST_HEAD(&isert_np->pending);
2280ca6c1d82SSagi Grimberg isert_np->np = np;
2281b8d26b3bSNicholas Bellinger
2282b8d26b3bSNicholas Bellinger /*
2283b8d26b3bSNicholas Bellinger * Setup the np->np_sockaddr from the passed sockaddr setup
2284b8d26b3bSNicholas Bellinger * in iscsi_target_configfs.c code..
2285b8d26b3bSNicholas Bellinger */
2286b8d26b3bSNicholas Bellinger memcpy(&np->np_sockaddr, ksockaddr,
228713a3cf08SAndy Grover sizeof(struct sockaddr_storage));
2288b8d26b3bSNicholas Bellinger
2289ca6c1d82SSagi Grimberg isert_lid = isert_setup_id(isert_np);
2290b8d26b3bSNicholas Bellinger if (IS_ERR(isert_lid)) {
2291b8d26b3bSNicholas Bellinger ret = PTR_ERR(isert_lid);
2292b8d26b3bSNicholas Bellinger goto out;
2293b8d26b3bSNicholas Bellinger }
2294b8d26b3bSNicholas Bellinger
2295ed8cb0a4SJenny Derzhavetz isert_np->cm_id = isert_lid;
2296b8d26b3bSNicholas Bellinger np->np_context = isert_np;
2297b8d26b3bSNicholas Bellinger
2298b8d26b3bSNicholas Bellinger return 0;
2299b8d26b3bSNicholas Bellinger
2300b8d26b3bSNicholas Bellinger out:
2301b8d26b3bSNicholas Bellinger kfree(isert_np);
2302ca6c1d82SSagi Grimberg
2303b8d26b3bSNicholas Bellinger return ret;
2304b8d26b3bSNicholas Bellinger }
2305b8d26b3bSNicholas Bellinger
2306b8d26b3bSNicholas Bellinger static int
isert_rdma_accept(struct isert_conn * isert_conn)2307b8d26b3bSNicholas Bellinger isert_rdma_accept(struct isert_conn *isert_conn)
2308b8d26b3bSNicholas Bellinger {
2309dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id;
2310b8d26b3bSNicholas Bellinger struct rdma_conn_param cp;
2311b8d26b3bSNicholas Bellinger int ret;
231213bce482SJenny Derzhavetz struct iser_cm_hdr rsp_hdr;
2313b8d26b3bSNicholas Bellinger
2314b8d26b3bSNicholas Bellinger memset(&cp, 0, sizeof(struct rdma_conn_param));
2315b8d26b3bSNicholas Bellinger cp.initiator_depth = isert_conn->initiator_depth;
2316b8d26b3bSNicholas Bellinger cp.retry_count = 7;
2317b8d26b3bSNicholas Bellinger cp.rnr_retry_count = 7;
2318b8d26b3bSNicholas Bellinger
231913bce482SJenny Derzhavetz memset(&rsp_hdr, 0, sizeof(rsp_hdr));
2320422bd0acSJenny Derzhavetz rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
2321422bd0acSJenny Derzhavetz if (!isert_conn->snd_w_inv)
2322422bd0acSJenny Derzhavetz rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
232313bce482SJenny Derzhavetz cp.private_data = (void *)&rsp_hdr;
232413bce482SJenny Derzhavetz cp.private_data_len = sizeof(rsp_hdr);
232513bce482SJenny Derzhavetz
2326b8d26b3bSNicholas Bellinger ret = rdma_accept(cm_id, &cp);
2327b8d26b3bSNicholas Bellinger if (ret) {
232824f412ddSSagi Grimberg isert_err("rdma_accept() failed with: %d\n", ret);
2329b8d26b3bSNicholas Bellinger return ret;
2330b8d26b3bSNicholas Bellinger }
2331b8d26b3bSNicholas Bellinger
2332b8d26b3bSNicholas Bellinger return 0;
2333b8d26b3bSNicholas Bellinger }
2334b8d26b3bSNicholas Bellinger
2335b8d26b3bSNicholas Bellinger static int
isert_get_login_rx(struct iscsit_conn * conn,struct iscsi_login * login)2336be36d683SMax Gurtovoy isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login)
2337b8d26b3bSNicholas Bellinger {
23386700425eSSagi Grimberg struct isert_conn *isert_conn = conn->context;
2339b8d26b3bSNicholas Bellinger int ret;
2340b8d26b3bSNicholas Bellinger
234124f412ddSSagi Grimberg isert_info("before login_req comp conn: %p\n", isert_conn);
23422371e5daSSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
23432371e5daSSagi Grimberg if (ret) {
234424f412ddSSagi Grimberg isert_err("isert_conn %p interrupted before got login req\n",
23452371e5daSSagi Grimberg isert_conn);
23462371e5daSSagi Grimberg return ret;
23472371e5daSSagi Grimberg }
23482371e5daSSagi Grimberg reinit_completion(&isert_conn->login_req_comp);
23492371e5daSSagi Grimberg
23506faaa85fSNicholas Bellinger /*
23516faaa85fSNicholas Bellinger * For login requests after the first PDU, isert_rx_login_req() will
2352549f39a5STetsuo Handa * kick queue_delayed_work(isert_login_wq, &conn->login_work) as
2353549f39a5STetsuo Handa * the packet is received, which turns this callback from
2354549f39a5STetsuo Handa * iscsi_target_do_login_rx() into a NOP.
23556faaa85fSNicholas Bellinger */
23566faaa85fSNicholas Bellinger if (!login->first_request)
23576faaa85fSNicholas Bellinger return 0;
2358b8d26b3bSNicholas Bellinger
23592371e5daSSagi Grimberg isert_rx_login_req(isert_conn);
23602371e5daSSagi Grimberg
2361dac6ab30SSagi Grimberg isert_info("before login_comp conn: %p\n", conn);
2362dac6ab30SSagi Grimberg ret = wait_for_completion_interruptible(&isert_conn->login_comp);
2363b8d26b3bSNicholas Bellinger if (ret)
2364b8d26b3bSNicholas Bellinger return ret;
2365b8d26b3bSNicholas Bellinger
236624f412ddSSagi Grimberg isert_info("processing login->req: %p\n", login->req);
23672371e5daSSagi Grimberg
2368b8d26b3bSNicholas Bellinger return 0;
2369b8d26b3bSNicholas Bellinger }
2370b8d26b3bSNicholas Bellinger
2371b8d26b3bSNicholas Bellinger static void
isert_set_conn_info(struct iscsi_np * np,struct iscsit_conn * conn,struct isert_conn * isert_conn)2372be36d683SMax Gurtovoy isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn,
2373b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn)
2374b8d26b3bSNicholas Bellinger {
2375dac6ab30SSagi Grimberg struct rdma_cm_id *cm_id = isert_conn->cm_id;
2376b8d26b3bSNicholas Bellinger struct rdma_route *cm_route = &cm_id->route;
2377b8d26b3bSNicholas Bellinger
2378b8d26b3bSNicholas Bellinger conn->login_family = np->np_sockaddr.ss_family;
2379b8d26b3bSNicholas Bellinger
2380dc58f760SAndy Grover conn->login_sockaddr = cm_route->addr.dst_addr;
2381dc58f760SAndy Grover conn->local_sockaddr = cm_route->addr.src_addr;
2382b8d26b3bSNicholas Bellinger }
2383b8d26b3bSNicholas Bellinger
2384b8d26b3bSNicholas Bellinger static int
isert_accept_np(struct iscsi_np * np,struct iscsit_conn * conn)2385be36d683SMax Gurtovoy isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn)
2386b8d26b3bSNicholas Bellinger {
23876700425eSSagi Grimberg struct isert_np *isert_np = np->np_context;
2388b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn;
2389c6b8e918SSagi Grimberg int ret;
2390b8d26b3bSNicholas Bellinger
2391b8d26b3bSNicholas Bellinger accept_wait:
2392ed8cb0a4SJenny Derzhavetz ret = down_interruptible(&isert_np->sem);
2393c6b8e918SSagi Grimberg if (ret)
2394b8d26b3bSNicholas Bellinger return -ENODEV;
2395b8d26b3bSNicholas Bellinger
2396b8d26b3bSNicholas Bellinger spin_lock_bh(&np->np_thread_lock);
2397e346ab34SSagi Grimberg if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
2398b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock);
239911378cdbSRasmus Villemoes isert_dbg("np_thread_state %d\n",
2400e346ab34SSagi Grimberg np->np_thread_state);
2401bf194997SLeon Romanovsky /*
2402e346ab34SSagi Grimberg * No point in stalling here when np_thread
2403e346ab34SSagi Grimberg * is in state RESET/SHUTDOWN/EXIT - bail
2404bf194997SLeon Romanovsky */
2405b8d26b3bSNicholas Bellinger return -ENODEV;
2406b8d26b3bSNicholas Bellinger }
2407b8d26b3bSNicholas Bellinger spin_unlock_bh(&np->np_thread_lock);
2408b8d26b3bSNicholas Bellinger
2409ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex);
2410bd379220SJenny Derzhavetz if (list_empty(&isert_np->pending)) {
2411ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex);
2412b8d26b3bSNicholas Bellinger goto accept_wait;
2413b8d26b3bSNicholas Bellinger }
2414bd379220SJenny Derzhavetz isert_conn = list_first_entry(&isert_np->pending,
2415bd379220SJenny Derzhavetz struct isert_conn, node);
2416bd379220SJenny Derzhavetz list_del_init(&isert_conn->node);
2417ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex);
2418b8d26b3bSNicholas Bellinger
2419b8d26b3bSNicholas Bellinger conn->context = isert_conn;
2420b8d26b3bSNicholas Bellinger isert_conn->conn = conn;
2421aea92980SJenny Derzhavetz isert_conn->state = ISER_CONN_BOUND;
2422b8d26b3bSNicholas Bellinger
2423b8d26b3bSNicholas Bellinger isert_set_conn_info(np, conn, isert_conn);
2424b8d26b3bSNicholas Bellinger
242524f412ddSSagi Grimberg isert_dbg("Processing isert_conn: %p\n", isert_conn);
24262371e5daSSagi Grimberg
2427b8d26b3bSNicholas Bellinger return 0;
2428b8d26b3bSNicholas Bellinger }
2429b8d26b3bSNicholas Bellinger
2430b8d26b3bSNicholas Bellinger static void
isert_free_np(struct iscsi_np * np)2431b8d26b3bSNicholas Bellinger isert_free_np(struct iscsi_np *np)
2432b8d26b3bSNicholas Bellinger {
24336700425eSSagi Grimberg struct isert_np *isert_np = np->np_context;
2434268e6811SSagi Grimberg struct isert_conn *isert_conn, *n;
2435691b0480SSaravanan Vajravel LIST_HEAD(drop_conn_list);
2436b8d26b3bSNicholas Bellinger
2437ed8cb0a4SJenny Derzhavetz if (isert_np->cm_id)
2438ed8cb0a4SJenny Derzhavetz rdma_destroy_id(isert_np->cm_id);
2439b8d26b3bSNicholas Bellinger
2440268e6811SSagi Grimberg /*
2441268e6811SSagi Grimberg * FIXME: At this point we don't have a good way to insure
2442268e6811SSagi Grimberg * that at this point we don't have hanging connections that
2443268e6811SSagi Grimberg * completed RDMA establishment but didn't start iscsi login
2444268e6811SSagi Grimberg * process. So work-around this by cleaning up what ever piled
2445bd379220SJenny Derzhavetz * up in accepted and pending lists.
2446268e6811SSagi Grimberg */
2447ed8cb0a4SJenny Derzhavetz mutex_lock(&isert_np->mutex);
2448bd379220SJenny Derzhavetz if (!list_empty(&isert_np->pending)) {
2449bd379220SJenny Derzhavetz isert_info("Still have isert pending connections\n");
2450268e6811SSagi Grimberg list_for_each_entry_safe(isert_conn, n,
2451bd379220SJenny Derzhavetz &isert_np->pending,
2452bd379220SJenny Derzhavetz node) {
2453bd379220SJenny Derzhavetz isert_info("cleaning isert_conn %p state (%d)\n",
2454bd379220SJenny Derzhavetz isert_conn, isert_conn->state);
2455691b0480SSaravanan Vajravel list_move_tail(&isert_conn->node, &drop_conn_list);
2456bd379220SJenny Derzhavetz }
2457bd379220SJenny Derzhavetz }
2458bd379220SJenny Derzhavetz
2459bd379220SJenny Derzhavetz if (!list_empty(&isert_np->accepted)) {
2460bd379220SJenny Derzhavetz isert_info("Still have isert accepted connections\n");
2461bd379220SJenny Derzhavetz list_for_each_entry_safe(isert_conn, n,
2462bd379220SJenny Derzhavetz &isert_np->accepted,
2463bd379220SJenny Derzhavetz node) {
246424f412ddSSagi Grimberg isert_info("cleaning isert_conn %p state (%d)\n",
2465268e6811SSagi Grimberg isert_conn, isert_conn->state);
2466691b0480SSaravanan Vajravel list_move_tail(&isert_conn->node, &drop_conn_list);
2467268e6811SSagi Grimberg }
2468268e6811SSagi Grimberg }
2469ed8cb0a4SJenny Derzhavetz mutex_unlock(&isert_np->mutex);
2470268e6811SSagi Grimberg
2471691b0480SSaravanan Vajravel list_for_each_entry_safe(isert_conn, n, &drop_conn_list, node) {
2472691b0480SSaravanan Vajravel list_del_init(&isert_conn->node);
2473691b0480SSaravanan Vajravel isert_connect_release(isert_conn);
2474691b0480SSaravanan Vajravel }
2475691b0480SSaravanan Vajravel
2476b8d26b3bSNicholas Bellinger np->np_context = NULL;
2477b8d26b3bSNicholas Bellinger kfree(isert_np);
2478b8d26b3bSNicholas Bellinger }
2479b8d26b3bSNicholas Bellinger
isert_release_work(struct work_struct * work)2480b02efbfcSSagi Grimberg static void isert_release_work(struct work_struct *work)
2481b02efbfcSSagi Grimberg {
2482b02efbfcSSagi Grimberg struct isert_conn *isert_conn = container_of(work,
2483b02efbfcSSagi Grimberg struct isert_conn,
2484b02efbfcSSagi Grimberg release_work);
2485b02efbfcSSagi Grimberg
248624f412ddSSagi Grimberg isert_info("Starting release conn %p\n", isert_conn);
2487b02efbfcSSagi Grimberg
2488dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex);
2489b02efbfcSSagi Grimberg isert_conn->state = ISER_CONN_DOWN;
2490dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex);
2491b02efbfcSSagi Grimberg
249224f412ddSSagi Grimberg isert_info("Destroying conn %p\n", isert_conn);
2493b02efbfcSSagi Grimberg isert_put_conn(isert_conn);
2494b02efbfcSSagi Grimberg }
2495b02efbfcSSagi Grimberg
2496bdf20e72SSagi Grimberg static void
isert_wait4logout(struct isert_conn * isert_conn)2497991bb764SSagi Grimberg isert_wait4logout(struct isert_conn *isert_conn)
2498991bb764SSagi Grimberg {
2499be36d683SMax Gurtovoy struct iscsit_conn *conn = isert_conn->conn;
2500991bb764SSagi Grimberg
25014c22e07fSSagi Grimberg isert_info("conn %p\n", isert_conn);
25024c22e07fSSagi Grimberg
2503991bb764SSagi Grimberg if (isert_conn->logout_posted) {
250424f412ddSSagi Grimberg isert_info("conn %p wait for conn_logout_comp\n", isert_conn);
2505991bb764SSagi Grimberg wait_for_completion_timeout(&conn->conn_logout_comp,
2506991bb764SSagi Grimberg SECONDS_FOR_LOGOUT_COMP * HZ);
2507991bb764SSagi Grimberg }
2508991bb764SSagi Grimberg }
2509991bb764SSagi Grimberg
251076261adaSBart Van Assche static void
isert_wait4cmds(struct iscsit_conn * conn)2511be36d683SMax Gurtovoy isert_wait4cmds(struct iscsit_conn *conn)
251276261adaSBart Van Assche {
2513be36d683SMax Gurtovoy isert_info("iscsit_conn %p\n", conn);
251476261adaSBart Van Assche
251576261adaSBart Van Assche if (conn->sess) {
25166d256beeSMike Christie target_stop_cmd_counter(conn->cmd_cnt);
25176d256beeSMike Christie target_wait_for_cmds(conn->cmd_cnt);
251876261adaSBart Van Assche }
251976261adaSBart Van Assche }
252076261adaSBart Van Assche
25213e03c4b0SJenny Derzhavetz /**
25223e03c4b0SJenny Derzhavetz * isert_put_unsol_pending_cmds() - Drop commands waiting for
25233e03c4b0SJenny Derzhavetz * unsolicitate dataout
25243e03c4b0SJenny Derzhavetz * @conn: iscsi connection
25253e03c4b0SJenny Derzhavetz *
25263e03c4b0SJenny Derzhavetz * We might still have commands that are waiting for unsolicited
25273e03c4b0SJenny Derzhavetz * dataouts messages. We must put the extra reference on those
25283e03c4b0SJenny Derzhavetz * before blocking on the target_wait_for_session_cmds
25293e03c4b0SJenny Derzhavetz */
25303e03c4b0SJenny Derzhavetz static void
isert_put_unsol_pending_cmds(struct iscsit_conn * conn)2531be36d683SMax Gurtovoy isert_put_unsol_pending_cmds(struct iscsit_conn *conn)
25323e03c4b0SJenny Derzhavetz {
253366cd9d4eSMax Gurtovoy struct iscsit_cmd *cmd, *tmp;
25343e03c4b0SJenny Derzhavetz static LIST_HEAD(drop_cmd_list);
25353e03c4b0SJenny Derzhavetz
25363e03c4b0SJenny Derzhavetz spin_lock_bh(&conn->cmd_lock);
25373e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
25383e03c4b0SJenny Derzhavetz if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
25393e03c4b0SJenny Derzhavetz (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
25403e03c4b0SJenny Derzhavetz (cmd->write_data_done < cmd->se_cmd.data_length))
25413e03c4b0SJenny Derzhavetz list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
25423e03c4b0SJenny Derzhavetz }
25433e03c4b0SJenny Derzhavetz spin_unlock_bh(&conn->cmd_lock);
25443e03c4b0SJenny Derzhavetz
25453e03c4b0SJenny Derzhavetz list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
25463e03c4b0SJenny Derzhavetz list_del_init(&cmd->i_conn_node);
25473e03c4b0SJenny Derzhavetz if (cmd->i_state != ISTATE_REMOVE) {
25483e03c4b0SJenny Derzhavetz struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
25493e03c4b0SJenny Derzhavetz
25503e03c4b0SJenny Derzhavetz isert_info("conn %p dropping cmd %p\n", conn, cmd);
25513e03c4b0SJenny Derzhavetz isert_put_cmd(isert_cmd, true);
25523e03c4b0SJenny Derzhavetz }
25533e03c4b0SJenny Derzhavetz }
25543e03c4b0SJenny Derzhavetz }
25553e03c4b0SJenny Derzhavetz
isert_wait_conn(struct iscsit_conn * conn)2556be36d683SMax Gurtovoy static void isert_wait_conn(struct iscsit_conn *conn)
2557b8d26b3bSNicholas Bellinger {
2558b8d26b3bSNicholas Bellinger struct isert_conn *isert_conn = conn->context;
2559b8d26b3bSNicholas Bellinger
25604c22e07fSSagi Grimberg isert_info("Starting conn %p\n", isert_conn);
2561b8d26b3bSNicholas Bellinger
2562dac6ab30SSagi Grimberg mutex_lock(&isert_conn->mutex);
2563954f2372SSagi Grimberg isert_conn_terminate(isert_conn);
2564dac6ab30SSagi Grimberg mutex_unlock(&isert_conn->mutex);
2565b2cb9649SNicholas Bellinger
2566572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp);
25673e03c4b0SJenny Derzhavetz isert_put_unsol_pending_cmds(conn);
256876261adaSBart Van Assche isert_wait4cmds(conn);
2569991bb764SSagi Grimberg isert_wait4logout(isert_conn);
2570*dfe26110SLeon Romanovsky
2571*dfe26110SLeon Romanovsky queue_work(isert_release_wq, &isert_conn->release_work);
2572defd8848SNicholas Bellinger }
2573defd8848SNicholas Bellinger
isert_free_conn(struct iscsit_conn * conn)2574be36d683SMax Gurtovoy static void isert_free_conn(struct iscsit_conn *conn)
2575defd8848SNicholas Bellinger {
2576defd8848SNicholas Bellinger struct isert_conn *isert_conn = conn->context;
2577b8d26b3bSNicholas Bellinger
2578572a1434SSagi Grimberg ib_drain_qp(isert_conn->qp);
2579b8d26b3bSNicholas Bellinger isert_put_conn(isert_conn);
2580b8d26b3bSNicholas Bellinger }
2581b8d26b3bSNicholas Bellinger
isert_get_rx_pdu(struct iscsit_conn * conn)2582be36d683SMax Gurtovoy static void isert_get_rx_pdu(struct iscsit_conn *conn)
2583187e84f7SVarun Prakash {
2584187e84f7SVarun Prakash struct completion comp;
2585187e84f7SVarun Prakash
2586187e84f7SVarun Prakash init_completion(&comp);
2587187e84f7SVarun Prakash
2588187e84f7SVarun Prakash wait_for_completion_interruptible(&comp);
2589187e84f7SVarun Prakash }
2590187e84f7SVarun Prakash
2591b8d26b3bSNicholas Bellinger static struct iscsit_transport iser_target_transport = {
2592b8d26b3bSNicholas Bellinger .name = "IB/iSER",
2593b8d26b3bSNicholas Bellinger .transport_type = ISCSI_INFINIBAND,
2594bd027d85SNicholas Bellinger .rdma_shutdown = true,
2595d703ce2fSNicholas Bellinger .priv_size = sizeof(struct isert_cmd),
2596b8d26b3bSNicholas Bellinger .owner = THIS_MODULE,
2597b8d26b3bSNicholas Bellinger .iscsit_setup_np = isert_setup_np,
2598b8d26b3bSNicholas Bellinger .iscsit_accept_np = isert_accept_np,
2599b8d26b3bSNicholas Bellinger .iscsit_free_np = isert_free_np,
2600defd8848SNicholas Bellinger .iscsit_wait_conn = isert_wait_conn,
2601b8d26b3bSNicholas Bellinger .iscsit_free_conn = isert_free_conn,
2602b8d26b3bSNicholas Bellinger .iscsit_get_login_rx = isert_get_login_rx,
2603b8d26b3bSNicholas Bellinger .iscsit_put_login_tx = isert_put_login_tx,
2604b8d26b3bSNicholas Bellinger .iscsit_immediate_queue = isert_immediate_queue,
2605b8d26b3bSNicholas Bellinger .iscsit_response_queue = isert_response_queue,
2606b8d26b3bSNicholas Bellinger .iscsit_get_dataout = isert_get_dataout,
2607b8d26b3bSNicholas Bellinger .iscsit_queue_data_in = isert_put_datain,
2608b8d26b3bSNicholas Bellinger .iscsit_queue_status = isert_put_response,
2609131e6abcSNicholas Bellinger .iscsit_aborted_task = isert_aborted_task,
2610187e84f7SVarun Prakash .iscsit_get_rx_pdu = isert_get_rx_pdu,
2611e70beee7SNicholas Bellinger .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
2612b8d26b3bSNicholas Bellinger };
2613b8d26b3bSNicholas Bellinger
isert_init(void)2614b8d26b3bSNicholas Bellinger static int __init isert_init(void)
2615b8d26b3bSNicholas Bellinger {
2616549f39a5STetsuo Handa isert_login_wq = alloc_workqueue("isert_login_wq", 0, 0);
2617549f39a5STetsuo Handa if (!isert_login_wq) {
2618549f39a5STetsuo Handa isert_err("Unable to allocate isert_login_wq\n");
2619549f39a5STetsuo Handa return -ENOMEM;
2620549f39a5STetsuo Handa }
2621b8d26b3bSNicholas Bellinger
2622631af550SSagi Grimberg isert_comp_wq = alloc_workqueue("isert_comp_wq",
2623631af550SSagi Grimberg WQ_UNBOUND | WQ_HIGHPRI, 0);
2624b8d26b3bSNicholas Bellinger if (!isert_comp_wq) {
262524f412ddSSagi Grimberg isert_err("Unable to allocate isert_comp_wq\n");
2626549f39a5STetsuo Handa goto destroy_login_wq;
2627b8d26b3bSNicholas Bellinger }
2628b8d26b3bSNicholas Bellinger
2629b02efbfcSSagi Grimberg isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
2630b02efbfcSSagi Grimberg WQ_UNBOUND_MAX_ACTIVE);
2631b02efbfcSSagi Grimberg if (!isert_release_wq) {
263224f412ddSSagi Grimberg isert_err("Unable to allocate isert_release_wq\n");
2633b02efbfcSSagi Grimberg goto destroy_comp_wq;
2634b02efbfcSSagi Grimberg }
2635b02efbfcSSagi Grimberg
2636b8d26b3bSNicholas Bellinger iscsit_register_transport(&iser_target_transport);
263724f412ddSSagi Grimberg isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
2638b02efbfcSSagi Grimberg
2639b8d26b3bSNicholas Bellinger return 0;
2640b8d26b3bSNicholas Bellinger
2641b02efbfcSSagi Grimberg destroy_comp_wq:
2642b02efbfcSSagi Grimberg destroy_workqueue(isert_comp_wq);
2643549f39a5STetsuo Handa destroy_login_wq:
2644549f39a5STetsuo Handa destroy_workqueue(isert_login_wq);
26456f0fae3dSSagi Grimberg
2646549f39a5STetsuo Handa return -ENOMEM;
2647b8d26b3bSNicholas Bellinger }
2648b8d26b3bSNicholas Bellinger
isert_exit(void)2649b8d26b3bSNicholas Bellinger static void __exit isert_exit(void)
2650b8d26b3bSNicholas Bellinger {
2651549f39a5STetsuo Handa flush_workqueue(isert_login_wq);
2652b02efbfcSSagi Grimberg destroy_workqueue(isert_release_wq);
2653b8d26b3bSNicholas Bellinger destroy_workqueue(isert_comp_wq);
2654b8d26b3bSNicholas Bellinger iscsit_unregister_transport(&iser_target_transport);
26554c22e07fSSagi Grimberg isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
2656549f39a5STetsuo Handa destroy_workqueue(isert_login_wq);
2657b8d26b3bSNicholas Bellinger }
2658b8d26b3bSNicholas Bellinger
2659b8d26b3bSNicholas Bellinger MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2660b8d26b3bSNicholas Bellinger MODULE_AUTHOR("nab@Linux-iSCSI.org");
2661b8d26b3bSNicholas Bellinger MODULE_LICENSE("GPL");
2662b8d26b3bSNicholas Bellinger
2663b8d26b3bSNicholas Bellinger module_init(isert_init);
2664b8d26b3bSNicholas Bellinger module_exit(isert_exit);
2665