14814a0a4SEdward Tomasz Napierala /*-
24814a0a4SEdward Tomasz Napierala * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
34814a0a4SEdward Tomasz Napierala *
44814a0a4SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without
54814a0a4SEdward Tomasz Napierala * modification, are permitted provided that the following conditions
64814a0a4SEdward Tomasz Napierala * are met:
74814a0a4SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright
84814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer.
94814a0a4SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright
104814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the
114814a0a4SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution.
124814a0a4SEdward Tomasz Napierala *
134814a0a4SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
144814a0a4SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
154814a0a4SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
164814a0a4SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
174814a0a4SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
184814a0a4SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
194814a0a4SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
204814a0a4SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
214814a0a4SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
224814a0a4SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
234814a0a4SEdward Tomasz Napierala * SUCH DAMAGE.
244814a0a4SEdward Tomasz Napierala */
254814a0a4SEdward Tomasz Napierala
264814a0a4SEdward Tomasz Napierala #include "icl_iser.h"
274814a0a4SEdward Tomasz Napierala
284814a0a4SEdward Tomasz Napierala static MALLOC_DEFINE(M_ISER_VERBS, "iser_verbs", "iser verbs backend");
294814a0a4SEdward Tomasz Napierala static int iser_cq_poll_limit = 512;
304814a0a4SEdward Tomasz Napierala
314814a0a4SEdward Tomasz Napierala static void
iser_cq_event_callback(struct ib_event * cause,void * context)324814a0a4SEdward Tomasz Napierala iser_cq_event_callback(struct ib_event *cause, void *context)
334814a0a4SEdward Tomasz Napierala {
344814a0a4SEdward Tomasz Napierala ISER_ERR("got cq event %d", cause->event);
354814a0a4SEdward Tomasz Napierala }
364814a0a4SEdward Tomasz Napierala
374814a0a4SEdward Tomasz Napierala static void
iser_qp_event_callback(struct ib_event * cause,void * context)384814a0a4SEdward Tomasz Napierala iser_qp_event_callback(struct ib_event *cause, void *context)
394814a0a4SEdward Tomasz Napierala {
404814a0a4SEdward Tomasz Napierala ISER_ERR("got qp event %d", cause->event);
414814a0a4SEdward Tomasz Napierala }
424814a0a4SEdward Tomasz Napierala
434814a0a4SEdward Tomasz Napierala static void
iser_event_handler(struct ib_event_handler * handler,struct ib_event * event)444814a0a4SEdward Tomasz Napierala iser_event_handler(struct ib_event_handler *handler,
454814a0a4SEdward Tomasz Napierala struct ib_event *event)
464814a0a4SEdward Tomasz Napierala {
474814a0a4SEdward Tomasz Napierala ISER_ERR("async event %d on device %s port %d",
484814a0a4SEdward Tomasz Napierala event->event, event->device->name,
494814a0a4SEdward Tomasz Napierala event->element.port_num);
504814a0a4SEdward Tomasz Napierala }
514814a0a4SEdward Tomasz Napierala
524814a0a4SEdward Tomasz Napierala /**
534814a0a4SEdward Tomasz Napierala * is_iser_tx_desc - Indicate if the completion wr_id
544814a0a4SEdward Tomasz Napierala * is a TX descriptor or not.
554814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection
564814a0a4SEdward Tomasz Napierala * @wr_id: completion WR identifier
574814a0a4SEdward Tomasz Napierala *
584814a0a4SEdward Tomasz Napierala * Since we cannot rely on wc opcode in FLUSH errors
594814a0a4SEdward Tomasz Napierala * we must work around it by checking if the wr_id address
604814a0a4SEdward Tomasz Napierala * falls in the iser connection rx_descs buffer. If so
614814a0a4SEdward Tomasz Napierala * it is an RX descriptor, otherwize it is a TX.
624814a0a4SEdward Tomasz Napierala */
634814a0a4SEdward Tomasz Napierala static inline bool
is_iser_tx_desc(struct iser_conn * iser_conn,void * wr_id)644814a0a4SEdward Tomasz Napierala is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id)
654814a0a4SEdward Tomasz Napierala {
664814a0a4SEdward Tomasz Napierala void *start = iser_conn->rx_descs;
674814a0a4SEdward Tomasz Napierala u64 len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs);
684814a0a4SEdward Tomasz Napierala void *end = (void *)((uintptr_t)start + (uintptr_t)len);
694814a0a4SEdward Tomasz Napierala
704814a0a4SEdward Tomasz Napierala if (start) {
714814a0a4SEdward Tomasz Napierala if (wr_id >= start && wr_id < end)
724814a0a4SEdward Tomasz Napierala return false;
734814a0a4SEdward Tomasz Napierala } else {
744814a0a4SEdward Tomasz Napierala return ((uintptr_t)wr_id != (uintptr_t)iser_conn->login_resp_buf);
754814a0a4SEdward Tomasz Napierala }
764814a0a4SEdward Tomasz Napierala
774814a0a4SEdward Tomasz Napierala return true;
784814a0a4SEdward Tomasz Napierala }
794814a0a4SEdward Tomasz Napierala
804814a0a4SEdward Tomasz Napierala /**
814814a0a4SEdward Tomasz Napierala * iser_handle_comp_error() - Handle error completion
824814a0a4SEdward Tomasz Napierala * @ib_conn: connection RDMA resources
834814a0a4SEdward Tomasz Napierala * @wc: work completion
844814a0a4SEdward Tomasz Napierala *
854814a0a4SEdward Tomasz Napierala * Notes: Update post_recv_buf_count in case of recv error completion.
864814a0a4SEdward Tomasz Napierala * For non-FLUSH error completion we should also notify iscsi layer that
874814a0a4SEdward Tomasz Napierala * connection is failed (in case we passed bind stage).
884814a0a4SEdward Tomasz Napierala */
894814a0a4SEdward Tomasz Napierala static void
iser_handle_comp_error(struct ib_conn * ib_conn,struct ib_wc * wc)904814a0a4SEdward Tomasz Napierala iser_handle_comp_error(struct ib_conn *ib_conn,
914814a0a4SEdward Tomasz Napierala struct ib_wc *wc)
924814a0a4SEdward Tomasz Napierala {
934814a0a4SEdward Tomasz Napierala void *wr_id = (void *)(uintptr_t)wc->wr_id;
944814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
954814a0a4SEdward Tomasz Napierala ib_conn);
964814a0a4SEdward Tomasz Napierala
974814a0a4SEdward Tomasz Napierala if (is_iser_tx_desc(iser_conn, wr_id)) {
984814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got send comp error", iser_conn);
994814a0a4SEdward Tomasz Napierala } else {
1004814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got recv comp error", iser_conn);
1014814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--;
1024814a0a4SEdward Tomasz Napierala }
1034814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR)
1044814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn);
1054814a0a4SEdward Tomasz Napierala }
1064814a0a4SEdward Tomasz Napierala
1074814a0a4SEdward Tomasz Napierala /**
1084814a0a4SEdward Tomasz Napierala * iser_handle_wc - handle a single work completion
1094814a0a4SEdward Tomasz Napierala * @wc: work completion
1104814a0a4SEdward Tomasz Napierala *
1114814a0a4SEdward Tomasz Napierala * Soft-IRQ context, work completion can be either
1124814a0a4SEdward Tomasz Napierala * SEND or RECV, and can turn out successful or
1134814a0a4SEdward Tomasz Napierala * with error (or flush error).
1144814a0a4SEdward Tomasz Napierala */
iser_handle_wc(struct ib_wc * wc)1154814a0a4SEdward Tomasz Napierala static void iser_handle_wc(struct ib_wc *wc)
1164814a0a4SEdward Tomasz Napierala {
1174814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn;
1184814a0a4SEdward Tomasz Napierala struct iser_tx_desc *tx_desc;
1194814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc;
1204814a0a4SEdward Tomasz Napierala
1214814a0a4SEdward Tomasz Napierala ib_conn = wc->qp->qp_context;
1224814a0a4SEdward Tomasz Napierala if (likely(wc->status == IB_WC_SUCCESS)) {
1234814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_RECV) {
1244814a0a4SEdward Tomasz Napierala rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id;
1254814a0a4SEdward Tomasz Napierala iser_rcv_completion(rx_desc, wc->byte_len,
1264814a0a4SEdward Tomasz Napierala ib_conn);
1274814a0a4SEdward Tomasz Napierala } else
1284814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_SEND) {
1294814a0a4SEdward Tomasz Napierala tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
1304814a0a4SEdward Tomasz Napierala iser_snd_completion(tx_desc, ib_conn);
1314814a0a4SEdward Tomasz Napierala } else {
1324814a0a4SEdward Tomasz Napierala ISER_ERR("Unknown wc opcode %d", wc->opcode);
1334814a0a4SEdward Tomasz Napierala }
1344814a0a4SEdward Tomasz Napierala } else {
1354814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
1364814a0a4SEdward Tomasz Napierala ib_conn);
1374814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) {
138e199d806SEdward Tomasz Napierala ISER_ERR("conn %p wr id %llx status %d vend_err %x",
139e199d806SEdward Tomasz Napierala iser_conn, (unsigned long long)wc->wr_id,
140e199d806SEdward Tomasz Napierala wc->status, wc->vendor_err);
1414814a0a4SEdward Tomasz Napierala } else {
142e199d806SEdward Tomasz Napierala ISER_DBG("flush error: conn %p wr id %llx",
143e199d806SEdward Tomasz Napierala iser_conn, (unsigned long long)wc->wr_id);
1444814a0a4SEdward Tomasz Napierala }
1454814a0a4SEdward Tomasz Napierala
1464814a0a4SEdward Tomasz Napierala if (wc->wr_id == ISER_BEACON_WRID) {
1474814a0a4SEdward Tomasz Napierala /* all flush errors were consumed */
1484814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock);
1494814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got ISER_BEACON_WRID", iser_conn);
1504814a0a4SEdward Tomasz Napierala cv_signal(&ib_conn->beacon.flush_cv);
1514814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock);
1524814a0a4SEdward Tomasz Napierala } else {
1534814a0a4SEdward Tomasz Napierala iser_handle_comp_error(ib_conn, wc);
1544814a0a4SEdward Tomasz Napierala }
1554814a0a4SEdward Tomasz Napierala }
1564814a0a4SEdward Tomasz Napierala }
1574814a0a4SEdward Tomasz Napierala
1584814a0a4SEdward Tomasz Napierala static void
iser_cq_tasklet_fn(void * data,int pending)1594814a0a4SEdward Tomasz Napierala iser_cq_tasklet_fn(void *data, int pending)
1604814a0a4SEdward Tomasz Napierala {
1614814a0a4SEdward Tomasz Napierala struct iser_comp *comp = (struct iser_comp *)data;
1624814a0a4SEdward Tomasz Napierala struct ib_cq *cq = comp->cq;
1634814a0a4SEdward Tomasz Napierala struct ib_wc *const wcs = comp->wcs;
1644814a0a4SEdward Tomasz Napierala int completed = 0;
1654814a0a4SEdward Tomasz Napierala int i;
1664814a0a4SEdward Tomasz Napierala int n;
1674814a0a4SEdward Tomasz Napierala
1684814a0a4SEdward Tomasz Napierala while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) {
1694814a0a4SEdward Tomasz Napierala for (i = 0; i < n; i++)
1704814a0a4SEdward Tomasz Napierala iser_handle_wc(&wcs[i]);
1714814a0a4SEdward Tomasz Napierala
1724814a0a4SEdward Tomasz Napierala completed += n;
1734814a0a4SEdward Tomasz Napierala if (completed >= iser_cq_poll_limit)
1744814a0a4SEdward Tomasz Napierala break;
1754814a0a4SEdward Tomasz Napierala }
1764814a0a4SEdward Tomasz Napierala
1774814a0a4SEdward Tomasz Napierala /*
1784814a0a4SEdward Tomasz Napierala * It is assumed here that arming CQ only once its empty
1794814a0a4SEdward Tomasz Napierala * would not cause interrupts to be missed.
1804814a0a4SEdward Tomasz Napierala */
1814814a0a4SEdward Tomasz Napierala ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1824814a0a4SEdward Tomasz Napierala }
1834814a0a4SEdward Tomasz Napierala
1844814a0a4SEdward Tomasz Napierala static void
iser_cq_callback(struct ib_cq * cq,void * cq_context)1854814a0a4SEdward Tomasz Napierala iser_cq_callback(struct ib_cq *cq, void *cq_context)
1864814a0a4SEdward Tomasz Napierala {
1874814a0a4SEdward Tomasz Napierala struct iser_comp *comp = cq_context;
1884814a0a4SEdward Tomasz Napierala
189f83f8891SEdward Tomasz Napierala taskqueue_enqueue(comp->tq, &comp->task);
1904814a0a4SEdward Tomasz Napierala }
1914814a0a4SEdward Tomasz Napierala
1924814a0a4SEdward Tomasz Napierala /**
1934814a0a4SEdward Tomasz Napierala * iser_create_device_ib_res - creates Protection Domain (PD), Completion
1944814a0a4SEdward Tomasz Napierala * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
1954814a0a4SEdward Tomasz Napierala * the adapator.
1964814a0a4SEdward Tomasz Napierala *
1974814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure
1984814a0a4SEdward Tomasz Napierala */
1994814a0a4SEdward Tomasz Napierala static int
iser_create_device_ib_res(struct iser_device * device)2004814a0a4SEdward Tomasz Napierala iser_create_device_ib_res(struct iser_device *device)
2014814a0a4SEdward Tomasz Napierala {
20241dbd9ddSHans Petter Selasky struct ib_device *ib_dev = device->ib_device;
20341dbd9ddSHans Petter Selasky int i, max_cqe;
2044814a0a4SEdward Tomasz Napierala
20541dbd9ddSHans Petter Selasky if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {
2064814a0a4SEdward Tomasz Napierala ISER_ERR("device %s doesn't support Fastreg, "
2074814a0a4SEdward Tomasz Napierala "can't register memory", device->ib_device->name);
2084814a0a4SEdward Tomasz Napierala return (1);
2094814a0a4SEdward Tomasz Napierala }
2104814a0a4SEdward Tomasz Napierala
2114814a0a4SEdward Tomasz Napierala device->comps_used = min(mp_ncpus, device->ib_device->num_comp_vectors);
2124814a0a4SEdward Tomasz Napierala
2134814a0a4SEdward Tomasz Napierala device->comps = malloc(device->comps_used * sizeof(*device->comps),
2144814a0a4SEdward Tomasz Napierala M_ISER_VERBS, M_WAITOK | M_ZERO);
2154814a0a4SEdward Tomasz Napierala
21641dbd9ddSHans Petter Selasky max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe);
2174814a0a4SEdward Tomasz Napierala
2184814a0a4SEdward Tomasz Napierala ISER_DBG("using %d CQs, device %s supports %d vectors max_cqe %d",
2194814a0a4SEdward Tomasz Napierala device->comps_used, device->ib_device->name,
2204814a0a4SEdward Tomasz Napierala device->ib_device->num_comp_vectors, max_cqe);
2214814a0a4SEdward Tomasz Napierala
22241dbd9ddSHans Petter Selasky device->pd = ib_alloc_pd(device->ib_device, IB_PD_UNSAFE_GLOBAL_RKEY);
2234814a0a4SEdward Tomasz Napierala if (IS_ERR(device->pd))
2244814a0a4SEdward Tomasz Napierala goto pd_err;
2254814a0a4SEdward Tomasz Napierala
2264814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) {
2274814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i];
22841dbd9ddSHans Petter Selasky struct ib_cq_init_attr cq_attr = {
22941dbd9ddSHans Petter Selasky .cqe = max_cqe,
23041dbd9ddSHans Petter Selasky .comp_vector = i,
23141dbd9ddSHans Petter Selasky };
2324814a0a4SEdward Tomasz Napierala
2334814a0a4SEdward Tomasz Napierala comp->device = device;
2344814a0a4SEdward Tomasz Napierala comp->cq = ib_create_cq(device->ib_device,
2354814a0a4SEdward Tomasz Napierala iser_cq_callback,
2364814a0a4SEdward Tomasz Napierala iser_cq_event_callback,
2374814a0a4SEdward Tomasz Napierala (void *)comp,
23841dbd9ddSHans Petter Selasky &cq_attr);
2394814a0a4SEdward Tomasz Napierala if (IS_ERR(comp->cq)) {
2404814a0a4SEdward Tomasz Napierala comp->cq = NULL;
2414814a0a4SEdward Tomasz Napierala goto cq_err;
2424814a0a4SEdward Tomasz Napierala }
2434814a0a4SEdward Tomasz Napierala
2444814a0a4SEdward Tomasz Napierala if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP))
2454814a0a4SEdward Tomasz Napierala goto cq_err;
2464814a0a4SEdward Tomasz Napierala
2474814a0a4SEdward Tomasz Napierala TASK_INIT(&comp->task, 0, iser_cq_tasklet_fn, comp);
2484814a0a4SEdward Tomasz Napierala comp->tq = taskqueue_create_fast("iser_taskq", M_NOWAIT,
2494814a0a4SEdward Tomasz Napierala taskqueue_thread_enqueue, &comp->tq);
2504814a0a4SEdward Tomasz Napierala if (!comp->tq)
2514814a0a4SEdward Tomasz Napierala goto tq_err;
2524814a0a4SEdward Tomasz Napierala taskqueue_start_threads(&comp->tq, 1, PI_NET, "iser taskq");
2534814a0a4SEdward Tomasz Napierala }
2544814a0a4SEdward Tomasz Napierala
25541dbd9ddSHans Petter Selasky device->mr = device->pd->__internal_mr;
2564814a0a4SEdward Tomasz Napierala if (IS_ERR(device->mr))
2574814a0a4SEdward Tomasz Napierala goto tq_err;
2584814a0a4SEdward Tomasz Napierala
2594814a0a4SEdward Tomasz Napierala INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device,
2604814a0a4SEdward Tomasz Napierala iser_event_handler);
2614814a0a4SEdward Tomasz Napierala if (ib_register_event_handler(&device->event_handler))
2628d73c9baSHans Petter Selasky goto tq_err;
2634814a0a4SEdward Tomasz Napierala
2644814a0a4SEdward Tomasz Napierala return (0);
2654814a0a4SEdward Tomasz Napierala
2664814a0a4SEdward Tomasz Napierala tq_err:
2674814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) {
2684814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i];
2694814a0a4SEdward Tomasz Napierala if (comp->tq)
2704814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq);
2714814a0a4SEdward Tomasz Napierala }
2724814a0a4SEdward Tomasz Napierala cq_err:
2734814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) {
2744814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i];
2754814a0a4SEdward Tomasz Napierala if (comp->cq)
2764814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq);
2774814a0a4SEdward Tomasz Napierala }
2784814a0a4SEdward Tomasz Napierala ib_dealloc_pd(device->pd);
2794814a0a4SEdward Tomasz Napierala pd_err:
2804814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS);
2814814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate an IB resource");
2824814a0a4SEdward Tomasz Napierala return (1);
2834814a0a4SEdward Tomasz Napierala }
2844814a0a4SEdward Tomasz Napierala
2854814a0a4SEdward Tomasz Napierala /**
2864814a0a4SEdward Tomasz Napierala * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR,
2874814a0a4SEdward Tomasz Napierala * CQ and PD created with the device associated with the adapator.
2884814a0a4SEdward Tomasz Napierala */
2894814a0a4SEdward Tomasz Napierala static void
iser_free_device_ib_res(struct iser_device * device)2904814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(struct iser_device *device)
2914814a0a4SEdward Tomasz Napierala {
2924814a0a4SEdward Tomasz Napierala int i;
2934814a0a4SEdward Tomasz Napierala
2944814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) {
2954814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i];
2964814a0a4SEdward Tomasz Napierala
2974814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq);
2984814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq);
2994814a0a4SEdward Tomasz Napierala comp->cq = NULL;
3004814a0a4SEdward Tomasz Napierala }
3014814a0a4SEdward Tomasz Napierala
3024814a0a4SEdward Tomasz Napierala (void)ib_unregister_event_handler(&device->event_handler);
3034814a0a4SEdward Tomasz Napierala (void)ib_dealloc_pd(device->pd);
3044814a0a4SEdward Tomasz Napierala
3054814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS);
3064814a0a4SEdward Tomasz Napierala device->comps = NULL;
3074814a0a4SEdward Tomasz Napierala
3084814a0a4SEdward Tomasz Napierala device->mr = NULL;
3094814a0a4SEdward Tomasz Napierala device->pd = NULL;
3104814a0a4SEdward Tomasz Napierala }
3114814a0a4SEdward Tomasz Napierala
3124814a0a4SEdward Tomasz Napierala static int
iser_alloc_reg_res(struct ib_device * ib_device,struct ib_pd * pd,struct iser_reg_resources * res)3134814a0a4SEdward Tomasz Napierala iser_alloc_reg_res(struct ib_device *ib_device,
3144814a0a4SEdward Tomasz Napierala struct ib_pd *pd,
3154814a0a4SEdward Tomasz Napierala struct iser_reg_resources *res)
3164814a0a4SEdward Tomasz Napierala {
3174814a0a4SEdward Tomasz Napierala int ret;
3184814a0a4SEdward Tomasz Napierala
31941dbd9ddSHans Petter Selasky res->mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, ISCSI_ISER_SG_TABLESIZE + 1);
3204814a0a4SEdward Tomasz Napierala if (IS_ERR(res->mr)) {
3214814a0a4SEdward Tomasz Napierala ret = -PTR_ERR(res->mr);
3224814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate fast reg mr err=%d", ret);
32341dbd9ddSHans Petter Selasky return (ret);
3244814a0a4SEdward Tomasz Napierala }
3254814a0a4SEdward Tomasz Napierala res->mr_valid = 1;
3264814a0a4SEdward Tomasz Napierala
3274814a0a4SEdward Tomasz Napierala return (0);
3284814a0a4SEdward Tomasz Napierala }
3294814a0a4SEdward Tomasz Napierala
3304814a0a4SEdward Tomasz Napierala static void
iser_free_reg_res(struct iser_reg_resources * rsc)3314814a0a4SEdward Tomasz Napierala iser_free_reg_res(struct iser_reg_resources *rsc)
3324814a0a4SEdward Tomasz Napierala {
3334814a0a4SEdward Tomasz Napierala ib_dereg_mr(rsc->mr);
3344814a0a4SEdward Tomasz Napierala }
3354814a0a4SEdward Tomasz Napierala
3364814a0a4SEdward Tomasz Napierala static struct fast_reg_descriptor *
iser_create_fastreg_desc(struct ib_device * ib_device,struct ib_pd * pd)3374814a0a4SEdward Tomasz Napierala iser_create_fastreg_desc(struct ib_device *ib_device, struct ib_pd *pd)
3384814a0a4SEdward Tomasz Napierala {
3394814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc;
3404814a0a4SEdward Tomasz Napierala int ret;
3414814a0a4SEdward Tomasz Napierala
3424814a0a4SEdward Tomasz Napierala desc = malloc(sizeof(*desc), M_ISER_VERBS, M_WAITOK | M_ZERO);
3434814a0a4SEdward Tomasz Napierala ret = iser_alloc_reg_res(ib_device, pd, &desc->rsc);
3444814a0a4SEdward Tomasz Napierala if (ret) {
3454814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate reg_resources");
3464814a0a4SEdward Tomasz Napierala goto err;
3474814a0a4SEdward Tomasz Napierala }
3484814a0a4SEdward Tomasz Napierala
3494814a0a4SEdward Tomasz Napierala return (desc);
3504814a0a4SEdward Tomasz Napierala err:
3514814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS);
3524814a0a4SEdward Tomasz Napierala return (NULL);
3534814a0a4SEdward Tomasz Napierala }
3544814a0a4SEdward Tomasz Napierala
3554814a0a4SEdward Tomasz Napierala /**
3564814a0a4SEdward Tomasz Napierala * iser_create_fmr_pool - Creates FMR pool and page_vector
3574814a0a4SEdward Tomasz Napierala *
3584814a0a4SEdward Tomasz Napierala * returns 0 on success, or errno code on failure
3594814a0a4SEdward Tomasz Napierala */
3604814a0a4SEdward Tomasz Napierala int
iser_create_fastreg_pool(struct ib_conn * ib_conn,unsigned cmds_max)3614814a0a4SEdward Tomasz Napierala iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max)
3624814a0a4SEdward Tomasz Napierala {
3634814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device;
3644814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc;
3654814a0a4SEdward Tomasz Napierala int i;
3664814a0a4SEdward Tomasz Napierala
3674814a0a4SEdward Tomasz Napierala INIT_LIST_HEAD(&ib_conn->fastreg.pool);
3684814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size = 0;
3694814a0a4SEdward Tomasz Napierala for (i = 0; i < cmds_max; i++) {
3704814a0a4SEdward Tomasz Napierala desc = iser_create_fastreg_desc(device->ib_device, device->pd);
3714814a0a4SEdward Tomasz Napierala if (!desc) {
3724814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to create fastreg descriptor");
3734814a0a4SEdward Tomasz Napierala goto err;
3744814a0a4SEdward Tomasz Napierala }
3754814a0a4SEdward Tomasz Napierala
3764814a0a4SEdward Tomasz Napierala list_add_tail(&desc->list, &ib_conn->fastreg.pool);
3774814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size++;
3784814a0a4SEdward Tomasz Napierala }
3794814a0a4SEdward Tomasz Napierala
3804814a0a4SEdward Tomasz Napierala return (0);
3814814a0a4SEdward Tomasz Napierala
3824814a0a4SEdward Tomasz Napierala err:
3834814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(ib_conn);
3844814a0a4SEdward Tomasz Napierala return (ENOMEM);
3854814a0a4SEdward Tomasz Napierala }
3864814a0a4SEdward Tomasz Napierala
3874814a0a4SEdward Tomasz Napierala /**
3884814a0a4SEdward Tomasz Napierala * iser_free_fmr_pool - releases the FMR pool and page vec
3894814a0a4SEdward Tomasz Napierala */
3904814a0a4SEdward Tomasz Napierala void
iser_free_fastreg_pool(struct ib_conn * ib_conn)3914814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(struct ib_conn *ib_conn)
3924814a0a4SEdward Tomasz Napierala {
3934814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc, *tmp;
3944814a0a4SEdward Tomasz Napierala int i = 0;
3954814a0a4SEdward Tomasz Napierala
3964814a0a4SEdward Tomasz Napierala if (list_empty(&ib_conn->fastreg.pool))
3974814a0a4SEdward Tomasz Napierala return;
3984814a0a4SEdward Tomasz Napierala
3994814a0a4SEdward Tomasz Napierala ISER_DBG("freeing conn %p fr pool", ib_conn);
4004814a0a4SEdward Tomasz Napierala
4014814a0a4SEdward Tomasz Napierala list_for_each_entry_safe(desc, tmp, &ib_conn->fastreg.pool, list) {
4024814a0a4SEdward Tomasz Napierala list_del(&desc->list);
4034814a0a4SEdward Tomasz Napierala iser_free_reg_res(&desc->rsc);
4044814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS);
4054814a0a4SEdward Tomasz Napierala ++i;
4064814a0a4SEdward Tomasz Napierala }
4074814a0a4SEdward Tomasz Napierala
4084814a0a4SEdward Tomasz Napierala if (i < ib_conn->fastreg.pool_size)
4094814a0a4SEdward Tomasz Napierala ISER_WARN("pool still has %d regions registered",
4104814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size - i);
4114814a0a4SEdward Tomasz Napierala }
4124814a0a4SEdward Tomasz Napierala
4134814a0a4SEdward Tomasz Napierala /**
4144814a0a4SEdward Tomasz Napierala * iser_create_ib_conn_res - Queue-Pair (QP)
4154814a0a4SEdward Tomasz Napierala *
4164814a0a4SEdward Tomasz Napierala * returns 0 on success, 1 on failure
4174814a0a4SEdward Tomasz Napierala */
4184814a0a4SEdward Tomasz Napierala static int
iser_create_ib_conn_res(struct ib_conn * ib_conn)4194814a0a4SEdward Tomasz Napierala iser_create_ib_conn_res(struct ib_conn *ib_conn)
4204814a0a4SEdward Tomasz Napierala {
4214814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn;
4224814a0a4SEdward Tomasz Napierala struct iser_device *device;
4234814a0a4SEdward Tomasz Napierala struct ib_device_attr *dev_attr;
4244814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr;
4254814a0a4SEdward Tomasz Napierala int index, min_index = 0;
4264814a0a4SEdward Tomasz Napierala int ret = -ENOMEM;
4274814a0a4SEdward Tomasz Napierala
4284814a0a4SEdward Tomasz Napierala iser_conn = container_of(ib_conn, struct iser_conn, ib_conn);
4294814a0a4SEdward Tomasz Napierala device = ib_conn->device;
4304814a0a4SEdward Tomasz Napierala dev_attr = &device->dev_attr;
4314814a0a4SEdward Tomasz Napierala
4324814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex);
4334814a0a4SEdward Tomasz Napierala /* select the CQ with the minimal number of usages */
4344814a0a4SEdward Tomasz Napierala for (index = 0; index < device->comps_used; index++) {
4354814a0a4SEdward Tomasz Napierala if (device->comps[index].active_qps <
4364814a0a4SEdward Tomasz Napierala device->comps[min_index].active_qps)
4374814a0a4SEdward Tomasz Napierala min_index = index;
4384814a0a4SEdward Tomasz Napierala }
4394814a0a4SEdward Tomasz Napierala ib_conn->comp = &device->comps[min_index];
4404814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps++;
4414814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex);
4424814a0a4SEdward Tomasz Napierala ISER_INFO("cq index %d used for ib_conn %p", min_index, ib_conn);
4434814a0a4SEdward Tomasz Napierala
4444814a0a4SEdward Tomasz Napierala memset(&init_attr, 0, sizeof init_attr);
4454814a0a4SEdward Tomasz Napierala init_attr.event_handler = iser_qp_event_callback;
4464814a0a4SEdward Tomasz Napierala init_attr.qp_context = (void *)ib_conn;
4474814a0a4SEdward Tomasz Napierala init_attr.send_cq = ib_conn->comp->cq;
4484814a0a4SEdward Tomasz Napierala init_attr.recv_cq = ib_conn->comp->cq;
4494814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;
4504814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_sge = 2;
4514814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_sge = 1;
4524814a0a4SEdward Tomasz Napierala init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
4534814a0a4SEdward Tomasz Napierala init_attr.qp_type = IB_QPT_RC;
4544814a0a4SEdward Tomasz Napierala
4554814a0a4SEdward Tomasz Napierala if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
4564814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS;
4574814a0a4SEdward Tomasz Napierala iser_conn->max_cmds =
4584814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
4594814a0a4SEdward Tomasz Napierala } else {
4604814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = dev_attr->max_qp_wr;
4614814a0a4SEdward Tomasz Napierala iser_conn->max_cmds =
4624814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr);
4634814a0a4SEdward Tomasz Napierala }
4644814a0a4SEdward Tomasz Napierala ISER_DBG("device %s supports max_send_wr %d",
4654814a0a4SEdward Tomasz Napierala device->ib_device->name, dev_attr->max_qp_wr);
4664814a0a4SEdward Tomasz Napierala
4674814a0a4SEdward Tomasz Napierala ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
4684814a0a4SEdward Tomasz Napierala if (ret)
4694814a0a4SEdward Tomasz Napierala goto out_err;
4704814a0a4SEdward Tomasz Napierala
4714814a0a4SEdward Tomasz Napierala ib_conn->qp = ib_conn->cma_id->qp;
4724814a0a4SEdward Tomasz Napierala ISER_DBG("setting conn %p cma_id %p qp %p",
4734814a0a4SEdward Tomasz Napierala ib_conn, ib_conn->cma_id,
4744814a0a4SEdward Tomasz Napierala ib_conn->cma_id->qp);
4754814a0a4SEdward Tomasz Napierala
4764814a0a4SEdward Tomasz Napierala return (ret);
4774814a0a4SEdward Tomasz Napierala
4784814a0a4SEdward Tomasz Napierala out_err:
4794814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex);
4804814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--;
4814814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex);
4824814a0a4SEdward Tomasz Napierala ISER_ERR("unable to alloc mem or create resource, err %d", ret);
4834814a0a4SEdward Tomasz Napierala
4844814a0a4SEdward Tomasz Napierala return (ret);
4854814a0a4SEdward Tomasz Napierala }
4864814a0a4SEdward Tomasz Napierala
4874814a0a4SEdward Tomasz Napierala /**
4884814a0a4SEdward Tomasz Napierala * based on the resolved device node GUID see if there already allocated
4894814a0a4SEdward Tomasz Napierala * device for this device. If there's no such, create one.
4904814a0a4SEdward Tomasz Napierala */
4914814a0a4SEdward Tomasz Napierala static struct iser_device *
iser_device_find_by_ib_device(struct rdma_cm_id * cma_id)4924814a0a4SEdward Tomasz Napierala iser_device_find_by_ib_device(struct rdma_cm_id *cma_id)
4934814a0a4SEdward Tomasz Napierala {
4944814a0a4SEdward Tomasz Napierala struct iser_device *device;
4954814a0a4SEdward Tomasz Napierala
4964814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex);
4974814a0a4SEdward Tomasz Napierala
4984814a0a4SEdward Tomasz Napierala list_for_each_entry(device, &ig.device_list, ig_list)
4994814a0a4SEdward Tomasz Napierala /* find if there's a match using the node GUID */
5004814a0a4SEdward Tomasz Napierala if (device->ib_device->node_guid == cma_id->device->node_guid)
5014814a0a4SEdward Tomasz Napierala goto inc_refcnt;
5024814a0a4SEdward Tomasz Napierala
5034814a0a4SEdward Tomasz Napierala device = malloc(sizeof *device, M_ISER_VERBS, M_WAITOK | M_ZERO);
5044814a0a4SEdward Tomasz Napierala /* assign this device to the device */
5054814a0a4SEdward Tomasz Napierala device->ib_device = cma_id->device;
5064814a0a4SEdward Tomasz Napierala /* init the device and link it into ig device list */
5074814a0a4SEdward Tomasz Napierala if (iser_create_device_ib_res(device)) {
5084814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS);
5094814a0a4SEdward Tomasz Napierala device = NULL;
5104814a0a4SEdward Tomasz Napierala goto out;
5114814a0a4SEdward Tomasz Napierala }
5124814a0a4SEdward Tomasz Napierala list_add(&device->ig_list, &ig.device_list);
5134814a0a4SEdward Tomasz Napierala
5144814a0a4SEdward Tomasz Napierala inc_refcnt:
5154814a0a4SEdward Tomasz Napierala device->refcount++;
5164814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount);
5174814a0a4SEdward Tomasz Napierala out:
5184814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex);
5194814a0a4SEdward Tomasz Napierala return (device);
5204814a0a4SEdward Tomasz Napierala }
5214814a0a4SEdward Tomasz Napierala
5224814a0a4SEdward Tomasz Napierala /* if there's no demand for this device, release it */
5234814a0a4SEdward Tomasz Napierala static void
iser_device_try_release(struct iser_device * device)5244814a0a4SEdward Tomasz Napierala iser_device_try_release(struct iser_device *device)
5254814a0a4SEdward Tomasz Napierala {
5264814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex);
5274814a0a4SEdward Tomasz Napierala device->refcount--;
5284814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount);
5294814a0a4SEdward Tomasz Napierala if (!device->refcount) {
5304814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(device);
5314814a0a4SEdward Tomasz Napierala list_del(&device->ig_list);
5324814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS);
5334814a0a4SEdward Tomasz Napierala device = NULL;
5344814a0a4SEdward Tomasz Napierala }
5354814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex);
5364814a0a4SEdward Tomasz Napierala }
5374814a0a4SEdward Tomasz Napierala
5384814a0a4SEdward Tomasz Napierala /**
5394814a0a4SEdward Tomasz Napierala * Called with state mutex held
5404814a0a4SEdward Tomasz Napierala **/
iser_conn_state_comp_exch(struct iser_conn * iser_conn,enum iser_conn_state comp,enum iser_conn_state exch)5414814a0a4SEdward Tomasz Napierala static int iser_conn_state_comp_exch(struct iser_conn *iser_conn,
5424814a0a4SEdward Tomasz Napierala enum iser_conn_state comp,
5434814a0a4SEdward Tomasz Napierala enum iser_conn_state exch)
5444814a0a4SEdward Tomasz Napierala {
5454814a0a4SEdward Tomasz Napierala int ret;
5464814a0a4SEdward Tomasz Napierala
5474814a0a4SEdward Tomasz Napierala ret = (iser_conn->state == comp);
5484814a0a4SEdward Tomasz Napierala if (ret)
5494814a0a4SEdward Tomasz Napierala iser_conn->state = exch;
5504814a0a4SEdward Tomasz Napierala
5514814a0a4SEdward Tomasz Napierala return ret;
5524814a0a4SEdward Tomasz Napierala }
5534814a0a4SEdward Tomasz Napierala
5544814a0a4SEdward Tomasz Napierala /**
5554814a0a4SEdward Tomasz Napierala * iser_free_ib_conn_res - release IB related resources
5564814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection struct
5574814a0a4SEdward Tomasz Napierala * @destroy: indicator if we need to try to release the
5584814a0a4SEdward Tomasz Napierala * iser device and memory regoins pool (only iscsi
5594814a0a4SEdward Tomasz Napierala * shutdown and DEVICE_REMOVAL will use this).
5604814a0a4SEdward Tomasz Napierala *
5614814a0a4SEdward Tomasz Napierala * This routine is called with the iser state mutex held
5624814a0a4SEdward Tomasz Napierala * so the cm_id removal is out of here. It is Safe to
5634814a0a4SEdward Tomasz Napierala * be invoked multiple times.
5644814a0a4SEdward Tomasz Napierala */
5654814a0a4SEdward Tomasz Napierala void
iser_free_ib_conn_res(struct iser_conn * iser_conn,bool destroy)5664814a0a4SEdward Tomasz Napierala iser_free_ib_conn_res(struct iser_conn *iser_conn,
5674814a0a4SEdward Tomasz Napierala bool destroy)
5684814a0a4SEdward Tomasz Napierala {
5694814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn;
5704814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device;
5714814a0a4SEdward Tomasz Napierala
5724814a0a4SEdward Tomasz Napierala ISER_INFO("freeing conn %p cma_id %p qp %p",
5734814a0a4SEdward Tomasz Napierala iser_conn, ib_conn->cma_id, ib_conn->qp);
5744814a0a4SEdward Tomasz Napierala
5754814a0a4SEdward Tomasz Napierala if (ib_conn->qp != NULL) {
5764814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex);
5774814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--;
5784814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex);
5794814a0a4SEdward Tomasz Napierala rdma_destroy_qp(ib_conn->cma_id);
5804814a0a4SEdward Tomasz Napierala ib_conn->qp = NULL;
5814814a0a4SEdward Tomasz Napierala }
5824814a0a4SEdward Tomasz Napierala
5834814a0a4SEdward Tomasz Napierala if (destroy) {
5844814a0a4SEdward Tomasz Napierala if (iser_conn->login_buf)
5854814a0a4SEdward Tomasz Napierala iser_free_login_buf(iser_conn);
5864814a0a4SEdward Tomasz Napierala
5874814a0a4SEdward Tomasz Napierala if (iser_conn->rx_descs)
5884814a0a4SEdward Tomasz Napierala iser_free_rx_descriptors(iser_conn);
5894814a0a4SEdward Tomasz Napierala
5904814a0a4SEdward Tomasz Napierala if (device != NULL) {
5914814a0a4SEdward Tomasz Napierala iser_device_try_release(device);
5924814a0a4SEdward Tomasz Napierala ib_conn->device = NULL;
5934814a0a4SEdward Tomasz Napierala }
5944814a0a4SEdward Tomasz Napierala }
5954814a0a4SEdward Tomasz Napierala }
5964814a0a4SEdward Tomasz Napierala
5974814a0a4SEdward Tomasz Napierala /**
5984814a0a4SEdward Tomasz Napierala * triggers start of the disconnect procedures and wait for them to be done
5994814a0a4SEdward Tomasz Napierala * Called with state mutex held
6004814a0a4SEdward Tomasz Napierala */
6014814a0a4SEdward Tomasz Napierala int
iser_conn_terminate(struct iser_conn * iser_conn)6024814a0a4SEdward Tomasz Napierala iser_conn_terminate(struct iser_conn *iser_conn)
6034814a0a4SEdward Tomasz Napierala {
6044814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn;
605*c3987b8eSHans Petter Selasky const struct ib_send_wr *bad_send_wr;
606*c3987b8eSHans Petter Selasky const struct ib_recv_wr *bad_recv_wr;
6074814a0a4SEdward Tomasz Napierala int err = 0;
6084814a0a4SEdward Tomasz Napierala
6094814a0a4SEdward Tomasz Napierala /* terminate the iser conn only if the conn state is UP */
6104814a0a4SEdward Tomasz Napierala if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP,
6114814a0a4SEdward Tomasz Napierala ISER_CONN_TERMINATING))
6124814a0a4SEdward Tomasz Napierala return (0);
6134814a0a4SEdward Tomasz Napierala
6144814a0a4SEdward Tomasz Napierala ISER_INFO("iser_conn %p state %d\n", iser_conn, iser_conn->state);
6154814a0a4SEdward Tomasz Napierala
6164814a0a4SEdward Tomasz Napierala if (ib_conn->qp == NULL) {
6174814a0a4SEdward Tomasz Napierala /* HOW can this be??? */
6184814a0a4SEdward Tomasz Napierala ISER_WARN("qp wasn't created");
6194814a0a4SEdward Tomasz Napierala return (1);
6204814a0a4SEdward Tomasz Napierala }
6214814a0a4SEdward Tomasz Napierala
6224814a0a4SEdward Tomasz Napierala /*
6234814a0a4SEdward Tomasz Napierala * Todo: This is a temporary workaround.
6244814a0a4SEdward Tomasz Napierala * We serialize the connection closure using global lock in order to
6254814a0a4SEdward Tomasz Napierala * receive all posted beacons completions.
6264814a0a4SEdward Tomasz Napierala * Without Serialization, in case we open many connections (QPs) on
6274814a0a4SEdward Tomasz Napierala * the same CQ, we might miss beacons because of missing interrupts.
6284814a0a4SEdward Tomasz Napierala */
6294814a0a4SEdward Tomasz Napierala sx_xlock(&ig.close_conns_mutex);
6304814a0a4SEdward Tomasz Napierala
6314814a0a4SEdward Tomasz Napierala /*
6324814a0a4SEdward Tomasz Napierala * In case we didn't already clean up the cma_id (peer initiated
6334814a0a4SEdward Tomasz Napierala * a disconnection), we need to Cause the CMA to change the QP
6344814a0a4SEdward Tomasz Napierala * state to ERROR.
6354814a0a4SEdward Tomasz Napierala */
6364814a0a4SEdward Tomasz Napierala if (ib_conn->cma_id) {
6374814a0a4SEdward Tomasz Napierala err = rdma_disconnect(ib_conn->cma_id);
6384814a0a4SEdward Tomasz Napierala if (err)
6394814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to disconnect, conn: 0x%p err %d",
6404814a0a4SEdward Tomasz Napierala iser_conn, err);
6414814a0a4SEdward Tomasz Napierala
6424814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock);
6434814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.send, 0, sizeof(struct ib_send_wr));
6444814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.wr_id = ISER_BEACON_WRID;
6454814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.opcode = IB_WR_SEND;
6464814a0a4SEdward Tomasz Napierala /* post an indication that all send flush errors were consumed */
6474814a0a4SEdward Tomasz Napierala err = ib_post_send(ib_conn->qp, &ib_conn->beacon.send, &bad_send_wr);
6484814a0a4SEdward Tomasz Napierala if (err) {
6494814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post send_beacon", ib_conn);
6504814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock);
6514814a0a4SEdward Tomasz Napierala goto out;
6524814a0a4SEdward Tomasz Napierala }
6534814a0a4SEdward Tomasz Napierala
6544814a0a4SEdward Tomasz Napierala ISER_DBG("before send cv_wait: %p", iser_conn);
6554814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock);
6564814a0a4SEdward Tomasz Napierala ISER_DBG("after send cv_wait: %p", iser_conn);
6574814a0a4SEdward Tomasz Napierala
6584814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.recv, 0, sizeof(struct ib_recv_wr));
6594814a0a4SEdward Tomasz Napierala ib_conn->beacon.recv.wr_id = ISER_BEACON_WRID;
6604814a0a4SEdward Tomasz Napierala /* post an indication that all recv flush errors were consumed */
6614814a0a4SEdward Tomasz Napierala err = ib_post_recv(ib_conn->qp, &ib_conn->beacon.recv, &bad_recv_wr);
6624814a0a4SEdward Tomasz Napierala if (err) {
6634814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post recv_beacon", ib_conn);
6644814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock);
6654814a0a4SEdward Tomasz Napierala goto out;
6664814a0a4SEdward Tomasz Napierala }
6674814a0a4SEdward Tomasz Napierala
6684814a0a4SEdward Tomasz Napierala ISER_DBG("before recv cv_wait: %p", iser_conn);
6694814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock);
6704814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock);
6714814a0a4SEdward Tomasz Napierala ISER_DBG("after recv cv_wait: %p", iser_conn);
6724814a0a4SEdward Tomasz Napierala }
6734814a0a4SEdward Tomasz Napierala out:
6744814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.close_conns_mutex);
6754814a0a4SEdward Tomasz Napierala return (1);
6764814a0a4SEdward Tomasz Napierala }
6774814a0a4SEdward Tomasz Napierala
6784814a0a4SEdward Tomasz Napierala /**
6794814a0a4SEdward Tomasz Napierala * Called with state mutex held
6804814a0a4SEdward Tomasz Napierala **/
6814814a0a4SEdward Tomasz Napierala static void
iser_connect_error(struct rdma_cm_id * cma_id)6824814a0a4SEdward Tomasz Napierala iser_connect_error(struct rdma_cm_id *cma_id)
6834814a0a4SEdward Tomasz Napierala {
6844814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn;
6854814a0a4SEdward Tomasz Napierala
6864814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context;
6874814a0a4SEdward Tomasz Napierala
6884814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p", iser_conn);
6894814a0a4SEdward Tomasz Napierala
6904814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_TERMINATING;
6914814a0a4SEdward Tomasz Napierala
6924814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv);
6934814a0a4SEdward Tomasz Napierala }
6944814a0a4SEdward Tomasz Napierala
6954814a0a4SEdward Tomasz Napierala /**
6964814a0a4SEdward Tomasz Napierala * Called with state mutex held
6974814a0a4SEdward Tomasz Napierala **/
6984814a0a4SEdward Tomasz Napierala static void
iser_addr_handler(struct rdma_cm_id * cma_id)6994814a0a4SEdward Tomasz Napierala iser_addr_handler(struct rdma_cm_id *cma_id)
7004814a0a4SEdward Tomasz Napierala {
7014814a0a4SEdward Tomasz Napierala struct iser_device *device;
7024814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn;
7034814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn;
7044814a0a4SEdward Tomasz Napierala int ret;
7054814a0a4SEdward Tomasz Napierala
7064814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context;
7074814a0a4SEdward Tomasz Napierala
7084814a0a4SEdward Tomasz Napierala ib_conn = &iser_conn->ib_conn;
7094814a0a4SEdward Tomasz Napierala device = iser_device_find_by_ib_device(cma_id);
7104814a0a4SEdward Tomasz Napierala if (!device) {
7114814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p device lookup/creation failed",
7124814a0a4SEdward Tomasz Napierala iser_conn);
7134814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id);
7144814a0a4SEdward Tomasz Napierala return;
7154814a0a4SEdward Tomasz Napierala }
7164814a0a4SEdward Tomasz Napierala
7174814a0a4SEdward Tomasz Napierala ib_conn->device = device;
7184814a0a4SEdward Tomasz Napierala
7194814a0a4SEdward Tomasz Napierala ret = rdma_resolve_route(cma_id, 1000);
7204814a0a4SEdward Tomasz Napierala if (ret) {
7214814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p resolve route failed: %d", iser_conn, ret);
7224814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id);
7234814a0a4SEdward Tomasz Napierala return;
7244814a0a4SEdward Tomasz Napierala }
7254814a0a4SEdward Tomasz Napierala }
7264814a0a4SEdward Tomasz Napierala
7274814a0a4SEdward Tomasz Napierala /**
7284814a0a4SEdward Tomasz Napierala * Called with state mutex held
7294814a0a4SEdward Tomasz Napierala **/
7304814a0a4SEdward Tomasz Napierala static void
iser_route_handler(struct rdma_cm_id * cma_id)7314814a0a4SEdward Tomasz Napierala iser_route_handler(struct rdma_cm_id *cma_id)
7324814a0a4SEdward Tomasz Napierala {
7334814a0a4SEdward Tomasz Napierala struct rdma_conn_param conn_param;
7344814a0a4SEdward Tomasz Napierala int ret;
7354814a0a4SEdward Tomasz Napierala struct iser_cm_hdr req_hdr;
7364814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context;
7374814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn;
7384814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device;
7394814a0a4SEdward Tomasz Napierala
7404814a0a4SEdward Tomasz Napierala ret = iser_create_ib_conn_res(ib_conn);
7414814a0a4SEdward Tomasz Napierala if (ret)
7424814a0a4SEdward Tomasz Napierala goto failure;
7434814a0a4SEdward Tomasz Napierala
7444814a0a4SEdward Tomasz Napierala memset(&conn_param, 0, sizeof conn_param);
7454814a0a4SEdward Tomasz Napierala conn_param.responder_resources = device->dev_attr.max_qp_rd_atom;
7464814a0a4SEdward Tomasz Napierala conn_param.retry_count = 7;
7474814a0a4SEdward Tomasz Napierala conn_param.rnr_retry_count = 6;
7484814a0a4SEdward Tomasz Napierala /*
7494814a0a4SEdward Tomasz Napierala * Initiaotr depth should not be set, but in order to compat
7504814a0a4SEdward Tomasz Napierala * with old targets, we keep this value set.
7514814a0a4SEdward Tomasz Napierala */
7524814a0a4SEdward Tomasz Napierala conn_param.initiator_depth = 1;
7534814a0a4SEdward Tomasz Napierala
7544814a0a4SEdward Tomasz Napierala memset(&req_hdr, 0, sizeof(req_hdr));
7554814a0a4SEdward Tomasz Napierala req_hdr.flags = (ISER_ZBVA_NOT_SUPPORTED |
7564814a0a4SEdward Tomasz Napierala ISER_SEND_W_INV_NOT_SUPPORTED);
7574814a0a4SEdward Tomasz Napierala conn_param.private_data = (void *)&req_hdr;
7584814a0a4SEdward Tomasz Napierala conn_param.private_data_len = sizeof(struct iser_cm_hdr);
7594814a0a4SEdward Tomasz Napierala
7604814a0a4SEdward Tomasz Napierala ret = rdma_connect(cma_id, &conn_param);
7614814a0a4SEdward Tomasz Napierala if (ret) {
7624814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failure connecting: %d", iser_conn, ret);
7634814a0a4SEdward Tomasz Napierala goto failure;
7644814a0a4SEdward Tomasz Napierala }
7654814a0a4SEdward Tomasz Napierala
7664814a0a4SEdward Tomasz Napierala return;
7674814a0a4SEdward Tomasz Napierala failure:
7684814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id);
7694814a0a4SEdward Tomasz Napierala }
7704814a0a4SEdward Tomasz Napierala
7714814a0a4SEdward Tomasz Napierala /**
7724814a0a4SEdward Tomasz Napierala * Called with state mutex held
7734814a0a4SEdward Tomasz Napierala **/
7744814a0a4SEdward Tomasz Napierala static void
iser_connected_handler(struct rdma_cm_id * cma_id)7754814a0a4SEdward Tomasz Napierala iser_connected_handler(struct rdma_cm_id *cma_id)
7764814a0a4SEdward Tomasz Napierala {
7774814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn;
7784814a0a4SEdward Tomasz Napierala struct ib_qp_attr attr;
7794814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr;
7804814a0a4SEdward Tomasz Napierala
7814814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context;
7824814a0a4SEdward Tomasz Napierala
7834814a0a4SEdward Tomasz Napierala (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr);
7844814a0a4SEdward Tomasz Napierala
7854814a0a4SEdward Tomasz Napierala ISER_INFO("remote qpn:%x my qpn:%x",
7864814a0a4SEdward Tomasz Napierala attr.dest_qp_num, cma_id->qp->qp_num);
7874814a0a4SEdward Tomasz Napierala
7884814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_UP;
7894814a0a4SEdward Tomasz Napierala
7904814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv);
7914814a0a4SEdward Tomasz Napierala }
7924814a0a4SEdward Tomasz Napierala
7934814a0a4SEdward Tomasz Napierala /**
7944814a0a4SEdward Tomasz Napierala * Called with state mutex held
7954814a0a4SEdward Tomasz Napierala **/
7964814a0a4SEdward Tomasz Napierala static void
iser_cleanup_handler(struct rdma_cm_id * cma_id,bool destroy)7974814a0a4SEdward Tomasz Napierala iser_cleanup_handler(struct rdma_cm_id *cma_id, bool destroy)
7984814a0a4SEdward Tomasz Napierala {
7994814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context;
8004814a0a4SEdward Tomasz Napierala
8014814a0a4SEdward Tomasz Napierala if (iser_conn_terminate(iser_conn))
8024814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn);
8034814a0a4SEdward Tomasz Napierala
8044814a0a4SEdward Tomasz Napierala }
8054814a0a4SEdward Tomasz Napierala
8064814a0a4SEdward Tomasz Napierala int
iser_cma_handler(struct rdma_cm_id * cma_id,struct rdma_cm_event * event)8074814a0a4SEdward Tomasz Napierala iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
8084814a0a4SEdward Tomasz Napierala {
8094814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn;
8104814a0a4SEdward Tomasz Napierala int ret = 0;
8114814a0a4SEdward Tomasz Napierala
8124814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context;
8134814a0a4SEdward Tomasz Napierala ISER_INFO("event %d status %d conn %p id %p",
8144814a0a4SEdward Tomasz Napierala event->event, event->status, cma_id->context, cma_id);
8154814a0a4SEdward Tomasz Napierala
8164814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex);
8174814a0a4SEdward Tomasz Napierala switch (event->event) {
8184814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_RESOLVED:
8194814a0a4SEdward Tomasz Napierala iser_addr_handler(cma_id);
8204814a0a4SEdward Tomasz Napierala break;
8214814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_RESOLVED:
8224814a0a4SEdward Tomasz Napierala iser_route_handler(cma_id);
8234814a0a4SEdward Tomasz Napierala break;
8244814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ESTABLISHED:
8254814a0a4SEdward Tomasz Napierala iser_connected_handler(cma_id);
8264814a0a4SEdward Tomasz Napierala break;
8274814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_ERROR:
8284814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_ERROR:
8294814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_CONNECT_ERROR:
8304814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_UNREACHABLE:
8314814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_REJECTED:
8324814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id);
8334814a0a4SEdward Tomasz Napierala break;
8344814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_DISCONNECTED:
8354814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_CHANGE:
8364814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_TIMEWAIT_EXIT:
8374814a0a4SEdward Tomasz Napierala iser_cleanup_handler(cma_id, false);
8384814a0a4SEdward Tomasz Napierala break;
8394814a0a4SEdward Tomasz Napierala default:
8404814a0a4SEdward Tomasz Napierala ISER_ERR("Unexpected RDMA CM event (%d)", event->event);
8414814a0a4SEdward Tomasz Napierala break;
8424814a0a4SEdward Tomasz Napierala }
8434814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex);
8444814a0a4SEdward Tomasz Napierala
8454814a0a4SEdward Tomasz Napierala return (ret);
8464814a0a4SEdward Tomasz Napierala }
8474814a0a4SEdward Tomasz Napierala
8484814a0a4SEdward Tomasz Napierala int
iser_post_recvl(struct iser_conn * iser_conn)8494814a0a4SEdward Tomasz Napierala iser_post_recvl(struct iser_conn *iser_conn)
8504814a0a4SEdward Tomasz Napierala {
851*c3987b8eSHans Petter Selasky const struct ib_recv_wr *rx_wr_failed;
852*c3987b8eSHans Petter Selasky struct ib_recv_wr rx_wr;
8534814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn;
8544814a0a4SEdward Tomasz Napierala struct ib_sge sge;
8554814a0a4SEdward Tomasz Napierala int ib_ret;
8564814a0a4SEdward Tomasz Napierala
8574814a0a4SEdward Tomasz Napierala sge.addr = iser_conn->login_resp_dma;
8584814a0a4SEdward Tomasz Napierala sge.length = ISER_RX_LOGIN_SIZE;
8594814a0a4SEdward Tomasz Napierala sge.lkey = ib_conn->device->mr->lkey;
8604814a0a4SEdward Tomasz Napierala
8614814a0a4SEdward Tomasz Napierala rx_wr.wr_id = (uintptr_t)iser_conn->login_resp_buf;
8624814a0a4SEdward Tomasz Napierala rx_wr.sg_list = &sge;
8634814a0a4SEdward Tomasz Napierala rx_wr.num_sge = 1;
8644814a0a4SEdward Tomasz Napierala rx_wr.next = NULL;
8654814a0a4SEdward Tomasz Napierala
8664814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count++;
8674814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed);
8684814a0a4SEdward Tomasz Napierala if (ib_ret) {
8694814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret);
8704814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--;
8714814a0a4SEdward Tomasz Napierala }
8724814a0a4SEdward Tomasz Napierala
8734814a0a4SEdward Tomasz Napierala return (ib_ret);
8744814a0a4SEdward Tomasz Napierala }
8754814a0a4SEdward Tomasz Napierala
8764814a0a4SEdward Tomasz Napierala int
iser_post_recvm(struct iser_conn * iser_conn,int count)8774814a0a4SEdward Tomasz Napierala iser_post_recvm(struct iser_conn *iser_conn, int count)
8784814a0a4SEdward Tomasz Napierala {
879*c3987b8eSHans Petter Selasky const struct ib_recv_wr *rx_wr_failed;
880*c3987b8eSHans Petter Selasky struct ib_recv_wr *rx_wr;
8814814a0a4SEdward Tomasz Napierala int i, ib_ret;
8824814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn;
8834814a0a4SEdward Tomasz Napierala unsigned int my_rx_head = iser_conn->rx_desc_head;
8844814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc;
8854814a0a4SEdward Tomasz Napierala
8864814a0a4SEdward Tomasz Napierala for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
8874814a0a4SEdward Tomasz Napierala rx_desc = &iser_conn->rx_descs[my_rx_head];
8884814a0a4SEdward Tomasz Napierala rx_wr->wr_id = (uintptr_t)rx_desc;
8894814a0a4SEdward Tomasz Napierala rx_wr->sg_list = &rx_desc->rx_sg;
8904814a0a4SEdward Tomasz Napierala rx_wr->num_sge = 1;
8914814a0a4SEdward Tomasz Napierala rx_wr->next = rx_wr + 1;
8924814a0a4SEdward Tomasz Napierala my_rx_head = (my_rx_head + 1) % iser_conn->qp_max_recv_dtos;
8934814a0a4SEdward Tomasz Napierala }
8944814a0a4SEdward Tomasz Napierala
8954814a0a4SEdward Tomasz Napierala rx_wr--;
8964814a0a4SEdward Tomasz Napierala rx_wr->next = NULL; /* mark end of work requests list */
8974814a0a4SEdward Tomasz Napierala
8984814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count += count;
8994814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed);
9004814a0a4SEdward Tomasz Napierala if (ib_ret) {
9014814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret);
9024814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count -= count;
9034814a0a4SEdward Tomasz Napierala } else
9044814a0a4SEdward Tomasz Napierala iser_conn->rx_desc_head = my_rx_head;
9054814a0a4SEdward Tomasz Napierala
9064814a0a4SEdward Tomasz Napierala return (ib_ret);
9074814a0a4SEdward Tomasz Napierala }
9084814a0a4SEdward Tomasz Napierala
9094814a0a4SEdward Tomasz Napierala /**
9104814a0a4SEdward Tomasz Napierala * iser_start_send - Initiate a Send DTO operation
9114814a0a4SEdward Tomasz Napierala *
9124814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure
9134814a0a4SEdward Tomasz Napierala */
iser_post_send(struct ib_conn * ib_conn,struct iser_tx_desc * tx_desc,bool signal)9144814a0a4SEdward Tomasz Napierala int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
9154814a0a4SEdward Tomasz Napierala bool signal)
9164814a0a4SEdward Tomasz Napierala {
9174814a0a4SEdward Tomasz Napierala int ib_ret;
918*c3987b8eSHans Petter Selasky const struct ib_send_wr *send_wr_failed;
919*c3987b8eSHans Petter Selasky struct ib_send_wr send_wr;
9204814a0a4SEdward Tomasz Napierala
9214814a0a4SEdward Tomasz Napierala ib_dma_sync_single_for_device(ib_conn->device->ib_device,
9224814a0a4SEdward Tomasz Napierala tx_desc->dma_addr, ISER_HEADERS_LEN,
9234814a0a4SEdward Tomasz Napierala DMA_TO_DEVICE);
9244814a0a4SEdward Tomasz Napierala
9254814a0a4SEdward Tomasz Napierala send_wr.next = NULL;
9264814a0a4SEdward Tomasz Napierala send_wr.wr_id = (uintptr_t)tx_desc;
9274814a0a4SEdward Tomasz Napierala send_wr.sg_list = tx_desc->tx_sg;
9284814a0a4SEdward Tomasz Napierala send_wr.num_sge = tx_desc->num_sge;
9294814a0a4SEdward Tomasz Napierala send_wr.opcode = IB_WR_SEND;
9304814a0a4SEdward Tomasz Napierala send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0;
9314814a0a4SEdward Tomasz Napierala
9324814a0a4SEdward Tomasz Napierala ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed);
9334814a0a4SEdward Tomasz Napierala if (ib_ret)
9344814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_send failed, ret:%d", ib_ret);
9354814a0a4SEdward Tomasz Napierala
9364814a0a4SEdward Tomasz Napierala return (ib_ret);
9374814a0a4SEdward Tomasz Napierala }
938