14814a0a4SEdward Tomasz Napierala /* $FreeBSD$ */ 24814a0a4SEdward Tomasz Napierala /*- 34814a0a4SEdward Tomasz Napierala * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved. 44814a0a4SEdward Tomasz Napierala * 54814a0a4SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 64814a0a4SEdward Tomasz Napierala * modification, are permitted provided that the following conditions 74814a0a4SEdward Tomasz Napierala * are met: 84814a0a4SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 94814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 104814a0a4SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 114814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 124814a0a4SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 134814a0a4SEdward Tomasz Napierala * 144814a0a4SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 154814a0a4SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 164814a0a4SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 174814a0a4SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 184814a0a4SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 194814a0a4SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 204814a0a4SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 214814a0a4SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 224814a0a4SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 234814a0a4SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 244814a0a4SEdward Tomasz Napierala * SUCH DAMAGE. 254814a0a4SEdward Tomasz Napierala */ 264814a0a4SEdward Tomasz Napierala 274814a0a4SEdward Tomasz Napierala #include "icl_iser.h" 284814a0a4SEdward Tomasz Napierala 294814a0a4SEdward Tomasz Napierala static MALLOC_DEFINE(M_ISER_VERBS, "iser_verbs", "iser verbs backend"); 304814a0a4SEdward Tomasz Napierala static int iser_cq_poll_limit = 512; 314814a0a4SEdward Tomasz Napierala 324814a0a4SEdward Tomasz Napierala static void 334814a0a4SEdward Tomasz Napierala iser_cq_event_callback(struct ib_event *cause, void *context) 344814a0a4SEdward Tomasz Napierala { 354814a0a4SEdward Tomasz Napierala ISER_ERR("got cq event %d", cause->event); 364814a0a4SEdward Tomasz Napierala } 374814a0a4SEdward Tomasz Napierala 384814a0a4SEdward Tomasz Napierala static void 394814a0a4SEdward Tomasz Napierala iser_qp_event_callback(struct ib_event *cause, void *context) 404814a0a4SEdward Tomasz Napierala { 414814a0a4SEdward Tomasz Napierala ISER_ERR("got qp event %d", cause->event); 424814a0a4SEdward Tomasz Napierala } 434814a0a4SEdward Tomasz Napierala 444814a0a4SEdward Tomasz Napierala static void 454814a0a4SEdward Tomasz Napierala iser_event_handler(struct ib_event_handler *handler, 464814a0a4SEdward Tomasz Napierala struct ib_event *event) 474814a0a4SEdward Tomasz Napierala { 484814a0a4SEdward Tomasz Napierala ISER_ERR("async event %d on device %s port %d", 494814a0a4SEdward Tomasz Napierala event->event, event->device->name, 504814a0a4SEdward Tomasz Napierala event->element.port_num); 514814a0a4SEdward Tomasz Napierala } 524814a0a4SEdward Tomasz Napierala 534814a0a4SEdward Tomasz Napierala /** 544814a0a4SEdward Tomasz Napierala * is_iser_tx_desc - Indicate if the completion wr_id 554814a0a4SEdward Tomasz Napierala * is a TX descriptor or not. 564814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection 574814a0a4SEdward Tomasz Napierala * @wr_id: completion WR identifier 584814a0a4SEdward Tomasz Napierala * 594814a0a4SEdward Tomasz Napierala * Since we cannot rely on wc opcode in FLUSH errors 604814a0a4SEdward Tomasz Napierala * we must work around it by checking if the wr_id address 614814a0a4SEdward Tomasz Napierala * falls in the iser connection rx_descs buffer. If so 624814a0a4SEdward Tomasz Napierala * it is an RX descriptor, otherwize it is a TX. 634814a0a4SEdward Tomasz Napierala */ 644814a0a4SEdward Tomasz Napierala static inline bool 654814a0a4SEdward Tomasz Napierala is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id) 664814a0a4SEdward Tomasz Napierala { 674814a0a4SEdward Tomasz Napierala void *start = iser_conn->rx_descs; 684814a0a4SEdward Tomasz Napierala u64 len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs); 694814a0a4SEdward Tomasz Napierala void *end = (void *)((uintptr_t)start + (uintptr_t)len); 704814a0a4SEdward Tomasz Napierala 714814a0a4SEdward Tomasz Napierala if (start) { 724814a0a4SEdward Tomasz Napierala if (wr_id >= start && wr_id < end) 734814a0a4SEdward Tomasz Napierala return false; 744814a0a4SEdward Tomasz Napierala } else { 754814a0a4SEdward Tomasz Napierala return ((uintptr_t)wr_id != (uintptr_t)iser_conn->login_resp_buf); 764814a0a4SEdward Tomasz Napierala } 774814a0a4SEdward Tomasz Napierala 784814a0a4SEdward Tomasz Napierala return true; 794814a0a4SEdward Tomasz Napierala } 804814a0a4SEdward Tomasz Napierala 814814a0a4SEdward Tomasz Napierala /** 824814a0a4SEdward Tomasz Napierala * iser_handle_comp_error() - Handle error completion 834814a0a4SEdward Tomasz Napierala * @ib_conn: connection RDMA resources 844814a0a4SEdward Tomasz Napierala * @wc: work completion 854814a0a4SEdward Tomasz Napierala * 864814a0a4SEdward Tomasz Napierala * Notes: Update post_recv_buf_count in case of recv error completion. 874814a0a4SEdward Tomasz Napierala * For non-FLUSH error completion we should also notify iscsi layer that 884814a0a4SEdward Tomasz Napierala * connection is failed (in case we passed bind stage). 894814a0a4SEdward Tomasz Napierala */ 904814a0a4SEdward Tomasz Napierala static void 914814a0a4SEdward Tomasz Napierala iser_handle_comp_error(struct ib_conn *ib_conn, 924814a0a4SEdward Tomasz Napierala struct ib_wc *wc) 934814a0a4SEdward Tomasz Napierala { 944814a0a4SEdward Tomasz Napierala void *wr_id = (void *)(uintptr_t)wc->wr_id; 954814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 964814a0a4SEdward Tomasz Napierala ib_conn); 974814a0a4SEdward Tomasz Napierala 984814a0a4SEdward Tomasz Napierala if (is_iser_tx_desc(iser_conn, wr_id)) { 994814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got send comp error", iser_conn); 1004814a0a4SEdward Tomasz Napierala } else { 1014814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got recv comp error", iser_conn); 1024814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--; 1034814a0a4SEdward Tomasz Napierala } 1044814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) 1054814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn); 1064814a0a4SEdward Tomasz Napierala } 1074814a0a4SEdward Tomasz Napierala 1084814a0a4SEdward Tomasz Napierala /** 1094814a0a4SEdward Tomasz Napierala * iser_handle_wc - handle a single work completion 1104814a0a4SEdward Tomasz Napierala * @wc: work completion 1114814a0a4SEdward Tomasz Napierala * 1124814a0a4SEdward Tomasz Napierala * Soft-IRQ context, work completion can be either 1134814a0a4SEdward Tomasz Napierala * SEND or RECV, and can turn out successful or 1144814a0a4SEdward Tomasz Napierala * with error (or flush error). 1154814a0a4SEdward Tomasz Napierala */ 1164814a0a4SEdward Tomasz Napierala static void iser_handle_wc(struct ib_wc *wc) 1174814a0a4SEdward Tomasz Napierala { 1184814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn; 1194814a0a4SEdward Tomasz Napierala struct iser_tx_desc *tx_desc; 1204814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc; 1214814a0a4SEdward Tomasz Napierala 1224814a0a4SEdward Tomasz Napierala ib_conn = wc->qp->qp_context; 1234814a0a4SEdward Tomasz Napierala if (likely(wc->status == IB_WC_SUCCESS)) { 1244814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_RECV) { 1254814a0a4SEdward Tomasz Napierala rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 1264814a0a4SEdward Tomasz Napierala iser_rcv_completion(rx_desc, wc->byte_len, 1274814a0a4SEdward Tomasz Napierala ib_conn); 1284814a0a4SEdward Tomasz Napierala } else 1294814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_SEND) { 1304814a0a4SEdward Tomasz Napierala tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 1314814a0a4SEdward Tomasz Napierala iser_snd_completion(tx_desc, ib_conn); 1324814a0a4SEdward Tomasz Napierala } else { 1334814a0a4SEdward Tomasz Napierala ISER_ERR("Unknown wc opcode %d", wc->opcode); 1344814a0a4SEdward Tomasz Napierala } 1354814a0a4SEdward Tomasz Napierala } else { 1364814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 1374814a0a4SEdward Tomasz Napierala ib_conn); 1384814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) { 139e199d806SEdward Tomasz Napierala ISER_ERR("conn %p wr id %llx status %d vend_err %x", 140e199d806SEdward Tomasz Napierala iser_conn, (unsigned long long)wc->wr_id, 141e199d806SEdward Tomasz Napierala wc->status, wc->vendor_err); 1424814a0a4SEdward Tomasz Napierala } else { 143e199d806SEdward Tomasz Napierala ISER_DBG("flush error: conn %p wr id %llx", 144e199d806SEdward Tomasz Napierala iser_conn, (unsigned long long)wc->wr_id); 1454814a0a4SEdward Tomasz Napierala } 1464814a0a4SEdward Tomasz Napierala 1474814a0a4SEdward Tomasz Napierala if (wc->wr_id == ISER_BEACON_WRID) { 1484814a0a4SEdward Tomasz Napierala /* all flush errors were consumed */ 1494814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock); 1504814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got ISER_BEACON_WRID", iser_conn); 1514814a0a4SEdward Tomasz Napierala cv_signal(&ib_conn->beacon.flush_cv); 1524814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 1534814a0a4SEdward Tomasz Napierala } else { 1544814a0a4SEdward Tomasz Napierala iser_handle_comp_error(ib_conn, wc); 1554814a0a4SEdward Tomasz Napierala } 1564814a0a4SEdward Tomasz Napierala } 1574814a0a4SEdward Tomasz Napierala } 1584814a0a4SEdward Tomasz Napierala 1594814a0a4SEdward Tomasz Napierala static void 1604814a0a4SEdward Tomasz Napierala iser_cq_tasklet_fn(void *data, int pending) 1614814a0a4SEdward Tomasz Napierala { 1624814a0a4SEdward Tomasz Napierala struct iser_comp *comp = (struct iser_comp *)data; 1634814a0a4SEdward Tomasz Napierala struct ib_cq *cq = comp->cq; 1644814a0a4SEdward Tomasz Napierala struct ib_wc *const wcs = comp->wcs; 1654814a0a4SEdward Tomasz Napierala int completed = 0; 1664814a0a4SEdward Tomasz Napierala int i; 1674814a0a4SEdward Tomasz Napierala int n; 1684814a0a4SEdward Tomasz Napierala 1694814a0a4SEdward Tomasz Napierala while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 1704814a0a4SEdward Tomasz Napierala for (i = 0; i < n; i++) 1714814a0a4SEdward Tomasz Napierala iser_handle_wc(&wcs[i]); 1724814a0a4SEdward Tomasz Napierala 1734814a0a4SEdward Tomasz Napierala completed += n; 1744814a0a4SEdward Tomasz Napierala if (completed >= iser_cq_poll_limit) 1754814a0a4SEdward Tomasz Napierala break; 1764814a0a4SEdward Tomasz Napierala } 1774814a0a4SEdward Tomasz Napierala 1784814a0a4SEdward Tomasz Napierala /* 1794814a0a4SEdward Tomasz Napierala * It is assumed here that arming CQ only once its empty 1804814a0a4SEdward Tomasz Napierala * would not cause interrupts to be missed. 1814814a0a4SEdward Tomasz Napierala */ 1824814a0a4SEdward Tomasz Napierala ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 1834814a0a4SEdward Tomasz Napierala } 1844814a0a4SEdward Tomasz Napierala 1854814a0a4SEdward Tomasz Napierala static void 1864814a0a4SEdward Tomasz Napierala iser_cq_callback(struct ib_cq *cq, void *cq_context) 1874814a0a4SEdward Tomasz Napierala { 1884814a0a4SEdward Tomasz Napierala struct iser_comp *comp = cq_context; 1894814a0a4SEdward Tomasz Napierala 190f83f8891SEdward Tomasz Napierala taskqueue_enqueue(comp->tq, &comp->task); 1914814a0a4SEdward Tomasz Napierala } 1924814a0a4SEdward Tomasz Napierala 1934814a0a4SEdward Tomasz Napierala /** 1944814a0a4SEdward Tomasz Napierala * iser_create_device_ib_res - creates Protection Domain (PD), Completion 1954814a0a4SEdward Tomasz Napierala * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with 1964814a0a4SEdward Tomasz Napierala * the adapator. 1974814a0a4SEdward Tomasz Napierala * 1984814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure 1994814a0a4SEdward Tomasz Napierala */ 2004814a0a4SEdward Tomasz Napierala static int 2014814a0a4SEdward Tomasz Napierala iser_create_device_ib_res(struct iser_device *device) 2024814a0a4SEdward Tomasz Napierala { 20341dbd9ddSHans Petter Selasky struct ib_device *ib_dev = device->ib_device; 20441dbd9ddSHans Petter Selasky int i, max_cqe; 2054814a0a4SEdward Tomasz Napierala 20641dbd9ddSHans Petter Selasky if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { 2074814a0a4SEdward Tomasz Napierala ISER_ERR("device %s doesn't support Fastreg, " 2084814a0a4SEdward Tomasz Napierala "can't register memory", device->ib_device->name); 2094814a0a4SEdward Tomasz Napierala return (1); 2104814a0a4SEdward Tomasz Napierala } 2114814a0a4SEdward Tomasz Napierala 2124814a0a4SEdward Tomasz Napierala device->comps_used = min(mp_ncpus, device->ib_device->num_comp_vectors); 2134814a0a4SEdward Tomasz Napierala 2144814a0a4SEdward Tomasz Napierala device->comps = malloc(device->comps_used * sizeof(*device->comps), 2154814a0a4SEdward Tomasz Napierala M_ISER_VERBS, M_WAITOK | M_ZERO); 2164814a0a4SEdward Tomasz Napierala if (!device->comps) 2174814a0a4SEdward Tomasz Napierala goto comps_err; 2184814a0a4SEdward Tomasz Napierala 21941dbd9ddSHans Petter Selasky max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe); 2204814a0a4SEdward Tomasz Napierala 2214814a0a4SEdward Tomasz Napierala ISER_DBG("using %d CQs, device %s supports %d vectors max_cqe %d", 2224814a0a4SEdward Tomasz Napierala device->comps_used, device->ib_device->name, 2234814a0a4SEdward Tomasz Napierala device->ib_device->num_comp_vectors, max_cqe); 2244814a0a4SEdward Tomasz Napierala 22541dbd9ddSHans Petter Selasky device->pd = ib_alloc_pd(device->ib_device, IB_PD_UNSAFE_GLOBAL_RKEY); 2264814a0a4SEdward Tomasz Napierala if (IS_ERR(device->pd)) 2274814a0a4SEdward Tomasz Napierala goto pd_err; 2284814a0a4SEdward Tomasz Napierala 2294814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2304814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 23141dbd9ddSHans Petter Selasky struct ib_cq_init_attr cq_attr = { 23241dbd9ddSHans Petter Selasky .cqe = max_cqe, 23341dbd9ddSHans Petter Selasky .comp_vector = i, 23441dbd9ddSHans Petter Selasky }; 2354814a0a4SEdward Tomasz Napierala 2364814a0a4SEdward Tomasz Napierala comp->device = device; 2374814a0a4SEdward Tomasz Napierala comp->cq = ib_create_cq(device->ib_device, 2384814a0a4SEdward Tomasz Napierala iser_cq_callback, 2394814a0a4SEdward Tomasz Napierala iser_cq_event_callback, 2404814a0a4SEdward Tomasz Napierala (void *)comp, 24141dbd9ddSHans Petter Selasky &cq_attr); 2424814a0a4SEdward Tomasz Napierala if (IS_ERR(comp->cq)) { 2434814a0a4SEdward Tomasz Napierala comp->cq = NULL; 2444814a0a4SEdward Tomasz Napierala goto cq_err; 2454814a0a4SEdward Tomasz Napierala } 2464814a0a4SEdward Tomasz Napierala 2474814a0a4SEdward Tomasz Napierala if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP)) 2484814a0a4SEdward Tomasz Napierala goto cq_err; 2494814a0a4SEdward Tomasz Napierala 2504814a0a4SEdward Tomasz Napierala TASK_INIT(&comp->task, 0, iser_cq_tasklet_fn, comp); 2514814a0a4SEdward Tomasz Napierala comp->tq = taskqueue_create_fast("iser_taskq", M_NOWAIT, 2524814a0a4SEdward Tomasz Napierala taskqueue_thread_enqueue, &comp->tq); 2534814a0a4SEdward Tomasz Napierala if (!comp->tq) 2544814a0a4SEdward Tomasz Napierala goto tq_err; 2554814a0a4SEdward Tomasz Napierala taskqueue_start_threads(&comp->tq, 1, PI_NET, "iser taskq"); 2564814a0a4SEdward Tomasz Napierala } 2574814a0a4SEdward Tomasz Napierala 25841dbd9ddSHans Petter Selasky device->mr = device->pd->__internal_mr; 2594814a0a4SEdward Tomasz Napierala if (IS_ERR(device->mr)) 2604814a0a4SEdward Tomasz Napierala goto tq_err; 2614814a0a4SEdward Tomasz Napierala 2624814a0a4SEdward Tomasz Napierala INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device, 2634814a0a4SEdward Tomasz Napierala iser_event_handler); 2644814a0a4SEdward Tomasz Napierala if (ib_register_event_handler(&device->event_handler)) 265*8d73c9baSHans Petter Selasky goto tq_err; 2664814a0a4SEdward Tomasz Napierala 2674814a0a4SEdward Tomasz Napierala return (0); 2684814a0a4SEdward Tomasz Napierala 2694814a0a4SEdward Tomasz Napierala tq_err: 2704814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2714814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 2724814a0a4SEdward Tomasz Napierala if (comp->tq) 2734814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq); 2744814a0a4SEdward Tomasz Napierala } 2754814a0a4SEdward Tomasz Napierala cq_err: 2764814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2774814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 2784814a0a4SEdward Tomasz Napierala if (comp->cq) 2794814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq); 2804814a0a4SEdward Tomasz Napierala } 2814814a0a4SEdward Tomasz Napierala ib_dealloc_pd(device->pd); 2824814a0a4SEdward Tomasz Napierala pd_err: 2834814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS); 2844814a0a4SEdward Tomasz Napierala comps_err: 2854814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate an IB resource"); 2864814a0a4SEdward Tomasz Napierala return (1); 2874814a0a4SEdward Tomasz Napierala } 2884814a0a4SEdward Tomasz Napierala 2894814a0a4SEdward Tomasz Napierala /** 2904814a0a4SEdward Tomasz Napierala * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR, 2914814a0a4SEdward Tomasz Napierala * CQ and PD created with the device associated with the adapator. 2924814a0a4SEdward Tomasz Napierala */ 2934814a0a4SEdward Tomasz Napierala static void 2944814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(struct iser_device *device) 2954814a0a4SEdward Tomasz Napierala { 2964814a0a4SEdward Tomasz Napierala int i; 2974814a0a4SEdward Tomasz Napierala 2984814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 2994814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 3004814a0a4SEdward Tomasz Napierala 3014814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq); 3024814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq); 3034814a0a4SEdward Tomasz Napierala comp->cq = NULL; 3044814a0a4SEdward Tomasz Napierala } 3054814a0a4SEdward Tomasz Napierala 3064814a0a4SEdward Tomasz Napierala (void)ib_unregister_event_handler(&device->event_handler); 3074814a0a4SEdward Tomasz Napierala (void)ib_dealloc_pd(device->pd); 3084814a0a4SEdward Tomasz Napierala 3094814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS); 3104814a0a4SEdward Tomasz Napierala device->comps = NULL; 3114814a0a4SEdward Tomasz Napierala 3124814a0a4SEdward Tomasz Napierala device->mr = NULL; 3134814a0a4SEdward Tomasz Napierala device->pd = NULL; 3144814a0a4SEdward Tomasz Napierala } 3154814a0a4SEdward Tomasz Napierala 3164814a0a4SEdward Tomasz Napierala static int 3174814a0a4SEdward Tomasz Napierala iser_alloc_reg_res(struct ib_device *ib_device, 3184814a0a4SEdward Tomasz Napierala struct ib_pd *pd, 3194814a0a4SEdward Tomasz Napierala struct iser_reg_resources *res) 3204814a0a4SEdward Tomasz Napierala { 3214814a0a4SEdward Tomasz Napierala int ret; 3224814a0a4SEdward Tomasz Napierala 32341dbd9ddSHans Petter Selasky res->mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, ISCSI_ISER_SG_TABLESIZE + 1); 3244814a0a4SEdward Tomasz Napierala if (IS_ERR(res->mr)) { 3254814a0a4SEdward Tomasz Napierala ret = -PTR_ERR(res->mr); 3264814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate fast reg mr err=%d", ret); 32741dbd9ddSHans Petter Selasky return (ret); 3284814a0a4SEdward Tomasz Napierala } 3294814a0a4SEdward Tomasz Napierala res->mr_valid = 1; 3304814a0a4SEdward Tomasz Napierala 3314814a0a4SEdward Tomasz Napierala return (0); 3324814a0a4SEdward Tomasz Napierala } 3334814a0a4SEdward Tomasz Napierala 3344814a0a4SEdward Tomasz Napierala static void 3354814a0a4SEdward Tomasz Napierala iser_free_reg_res(struct iser_reg_resources *rsc) 3364814a0a4SEdward Tomasz Napierala { 3374814a0a4SEdward Tomasz Napierala ib_dereg_mr(rsc->mr); 3384814a0a4SEdward Tomasz Napierala } 3394814a0a4SEdward Tomasz Napierala 3404814a0a4SEdward Tomasz Napierala static struct fast_reg_descriptor * 3414814a0a4SEdward Tomasz Napierala iser_create_fastreg_desc(struct ib_device *ib_device, struct ib_pd *pd) 3424814a0a4SEdward Tomasz Napierala { 3434814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc; 3444814a0a4SEdward Tomasz Napierala int ret; 3454814a0a4SEdward Tomasz Napierala 3464814a0a4SEdward Tomasz Napierala desc = malloc(sizeof(*desc), M_ISER_VERBS, M_WAITOK | M_ZERO); 3474814a0a4SEdward Tomasz Napierala if (!desc) { 3484814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate a new fastreg descriptor"); 3494814a0a4SEdward Tomasz Napierala return (NULL); 3504814a0a4SEdward Tomasz Napierala } 3514814a0a4SEdward Tomasz Napierala 3524814a0a4SEdward Tomasz Napierala ret = iser_alloc_reg_res(ib_device, pd, &desc->rsc); 3534814a0a4SEdward Tomasz Napierala if (ret) { 3544814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate reg_resources"); 3554814a0a4SEdward Tomasz Napierala goto err; 3564814a0a4SEdward Tomasz Napierala } 3574814a0a4SEdward Tomasz Napierala 3584814a0a4SEdward Tomasz Napierala return (desc); 3594814a0a4SEdward Tomasz Napierala err: 3604814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS); 3614814a0a4SEdward Tomasz Napierala return (NULL); 3624814a0a4SEdward Tomasz Napierala } 3634814a0a4SEdward Tomasz Napierala 3644814a0a4SEdward Tomasz Napierala /** 3654814a0a4SEdward Tomasz Napierala * iser_create_fmr_pool - Creates FMR pool and page_vector 3664814a0a4SEdward Tomasz Napierala * 3674814a0a4SEdward Tomasz Napierala * returns 0 on success, or errno code on failure 3684814a0a4SEdward Tomasz Napierala */ 3694814a0a4SEdward Tomasz Napierala int 3704814a0a4SEdward Tomasz Napierala iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max) 3714814a0a4SEdward Tomasz Napierala { 3724814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 3734814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc; 3744814a0a4SEdward Tomasz Napierala int i; 3754814a0a4SEdward Tomasz Napierala 3764814a0a4SEdward Tomasz Napierala INIT_LIST_HEAD(&ib_conn->fastreg.pool); 3774814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size = 0; 3784814a0a4SEdward Tomasz Napierala for (i = 0; i < cmds_max; i++) { 3794814a0a4SEdward Tomasz Napierala desc = iser_create_fastreg_desc(device->ib_device, device->pd); 3804814a0a4SEdward Tomasz Napierala if (!desc) { 3814814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to create fastreg descriptor"); 3824814a0a4SEdward Tomasz Napierala goto err; 3834814a0a4SEdward Tomasz Napierala } 3844814a0a4SEdward Tomasz Napierala 3854814a0a4SEdward Tomasz Napierala list_add_tail(&desc->list, &ib_conn->fastreg.pool); 3864814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size++; 3874814a0a4SEdward Tomasz Napierala } 3884814a0a4SEdward Tomasz Napierala 3894814a0a4SEdward Tomasz Napierala return (0); 3904814a0a4SEdward Tomasz Napierala 3914814a0a4SEdward Tomasz Napierala err: 3924814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(ib_conn); 3934814a0a4SEdward Tomasz Napierala return (ENOMEM); 3944814a0a4SEdward Tomasz Napierala } 3954814a0a4SEdward Tomasz Napierala 3964814a0a4SEdward Tomasz Napierala /** 3974814a0a4SEdward Tomasz Napierala * iser_free_fmr_pool - releases the FMR pool and page vec 3984814a0a4SEdward Tomasz Napierala */ 3994814a0a4SEdward Tomasz Napierala void 4004814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(struct ib_conn *ib_conn) 4014814a0a4SEdward Tomasz Napierala { 4024814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc, *tmp; 4034814a0a4SEdward Tomasz Napierala int i = 0; 4044814a0a4SEdward Tomasz Napierala 4054814a0a4SEdward Tomasz Napierala if (list_empty(&ib_conn->fastreg.pool)) 4064814a0a4SEdward Tomasz Napierala return; 4074814a0a4SEdward Tomasz Napierala 4084814a0a4SEdward Tomasz Napierala ISER_DBG("freeing conn %p fr pool", ib_conn); 4094814a0a4SEdward Tomasz Napierala 4104814a0a4SEdward Tomasz Napierala list_for_each_entry_safe(desc, tmp, &ib_conn->fastreg.pool, list) { 4114814a0a4SEdward Tomasz Napierala list_del(&desc->list); 4124814a0a4SEdward Tomasz Napierala iser_free_reg_res(&desc->rsc); 4134814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS); 4144814a0a4SEdward Tomasz Napierala ++i; 4154814a0a4SEdward Tomasz Napierala } 4164814a0a4SEdward Tomasz Napierala 4174814a0a4SEdward Tomasz Napierala if (i < ib_conn->fastreg.pool_size) 4184814a0a4SEdward Tomasz Napierala ISER_WARN("pool still has %d regions registered", 4194814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size - i); 4204814a0a4SEdward Tomasz Napierala } 4214814a0a4SEdward Tomasz Napierala 4224814a0a4SEdward Tomasz Napierala /** 4234814a0a4SEdward Tomasz Napierala * iser_create_ib_conn_res - Queue-Pair (QP) 4244814a0a4SEdward Tomasz Napierala * 4254814a0a4SEdward Tomasz Napierala * returns 0 on success, 1 on failure 4264814a0a4SEdward Tomasz Napierala */ 4274814a0a4SEdward Tomasz Napierala static int 4284814a0a4SEdward Tomasz Napierala iser_create_ib_conn_res(struct ib_conn *ib_conn) 4294814a0a4SEdward Tomasz Napierala { 4304814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 4314814a0a4SEdward Tomasz Napierala struct iser_device *device; 4324814a0a4SEdward Tomasz Napierala struct ib_device_attr *dev_attr; 4334814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr; 4344814a0a4SEdward Tomasz Napierala int index, min_index = 0; 4354814a0a4SEdward Tomasz Napierala int ret = -ENOMEM; 4364814a0a4SEdward Tomasz Napierala 4374814a0a4SEdward Tomasz Napierala iser_conn = container_of(ib_conn, struct iser_conn, ib_conn); 4384814a0a4SEdward Tomasz Napierala device = ib_conn->device; 4394814a0a4SEdward Tomasz Napierala dev_attr = &device->dev_attr; 4404814a0a4SEdward Tomasz Napierala 4414814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 4424814a0a4SEdward Tomasz Napierala /* select the CQ with the minimal number of usages */ 4434814a0a4SEdward Tomasz Napierala for (index = 0; index < device->comps_used; index++) { 4444814a0a4SEdward Tomasz Napierala if (device->comps[index].active_qps < 4454814a0a4SEdward Tomasz Napierala device->comps[min_index].active_qps) 4464814a0a4SEdward Tomasz Napierala min_index = index; 4474814a0a4SEdward Tomasz Napierala } 4484814a0a4SEdward Tomasz Napierala ib_conn->comp = &device->comps[min_index]; 4494814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps++; 4504814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 4514814a0a4SEdward Tomasz Napierala ISER_INFO("cq index %d used for ib_conn %p", min_index, ib_conn); 4524814a0a4SEdward Tomasz Napierala 4534814a0a4SEdward Tomasz Napierala memset(&init_attr, 0, sizeof init_attr); 4544814a0a4SEdward Tomasz Napierala init_attr.event_handler = iser_qp_event_callback; 4554814a0a4SEdward Tomasz Napierala init_attr.qp_context = (void *)ib_conn; 4564814a0a4SEdward Tomasz Napierala init_attr.send_cq = ib_conn->comp->cq; 4574814a0a4SEdward Tomasz Napierala init_attr.recv_cq = ib_conn->comp->cq; 4584814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; 4594814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_sge = 2; 4604814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_sge = 1; 4614814a0a4SEdward Tomasz Napierala init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 4624814a0a4SEdward Tomasz Napierala init_attr.qp_type = IB_QPT_RC; 4634814a0a4SEdward Tomasz Napierala 4644814a0a4SEdward Tomasz Napierala if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) { 4654814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS; 4664814a0a4SEdward Tomasz Napierala iser_conn->max_cmds = 4674814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS); 4684814a0a4SEdward Tomasz Napierala } else { 4694814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = dev_attr->max_qp_wr; 4704814a0a4SEdward Tomasz Napierala iser_conn->max_cmds = 4714814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr); 4724814a0a4SEdward Tomasz Napierala } 4734814a0a4SEdward Tomasz Napierala ISER_DBG("device %s supports max_send_wr %d", 4744814a0a4SEdward Tomasz Napierala device->ib_device->name, dev_attr->max_qp_wr); 4754814a0a4SEdward Tomasz Napierala 4764814a0a4SEdward Tomasz Napierala ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); 4774814a0a4SEdward Tomasz Napierala if (ret) 4784814a0a4SEdward Tomasz Napierala goto out_err; 4794814a0a4SEdward Tomasz Napierala 4804814a0a4SEdward Tomasz Napierala ib_conn->qp = ib_conn->cma_id->qp; 4814814a0a4SEdward Tomasz Napierala ISER_DBG("setting conn %p cma_id %p qp %p", 4824814a0a4SEdward Tomasz Napierala ib_conn, ib_conn->cma_id, 4834814a0a4SEdward Tomasz Napierala ib_conn->cma_id->qp); 4844814a0a4SEdward Tomasz Napierala 4854814a0a4SEdward Tomasz Napierala return (ret); 4864814a0a4SEdward Tomasz Napierala 4874814a0a4SEdward Tomasz Napierala out_err: 4884814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 4894814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--; 4904814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 4914814a0a4SEdward Tomasz Napierala ISER_ERR("unable to alloc mem or create resource, err %d", ret); 4924814a0a4SEdward Tomasz Napierala 4934814a0a4SEdward Tomasz Napierala return (ret); 4944814a0a4SEdward Tomasz Napierala } 4954814a0a4SEdward Tomasz Napierala 4964814a0a4SEdward Tomasz Napierala /** 4974814a0a4SEdward Tomasz Napierala * based on the resolved device node GUID see if there already allocated 4984814a0a4SEdward Tomasz Napierala * device for this device. If there's no such, create one. 4994814a0a4SEdward Tomasz Napierala */ 5004814a0a4SEdward Tomasz Napierala static struct iser_device * 5014814a0a4SEdward Tomasz Napierala iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) 5024814a0a4SEdward Tomasz Napierala { 5034814a0a4SEdward Tomasz Napierala struct iser_device *device; 5044814a0a4SEdward Tomasz Napierala 5054814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex); 5064814a0a4SEdward Tomasz Napierala 5074814a0a4SEdward Tomasz Napierala list_for_each_entry(device, &ig.device_list, ig_list) 5084814a0a4SEdward Tomasz Napierala /* find if there's a match using the node GUID */ 5094814a0a4SEdward Tomasz Napierala if (device->ib_device->node_guid == cma_id->device->node_guid) 5104814a0a4SEdward Tomasz Napierala goto inc_refcnt; 5114814a0a4SEdward Tomasz Napierala 5124814a0a4SEdward Tomasz Napierala device = malloc(sizeof *device, M_ISER_VERBS, M_WAITOK | M_ZERO); 5134814a0a4SEdward Tomasz Napierala if (device == NULL) 5144814a0a4SEdward Tomasz Napierala goto out; 5154814a0a4SEdward Tomasz Napierala 5164814a0a4SEdward Tomasz Napierala /* assign this device to the device */ 5174814a0a4SEdward Tomasz Napierala device->ib_device = cma_id->device; 5184814a0a4SEdward Tomasz Napierala /* init the device and link it into ig device list */ 5194814a0a4SEdward Tomasz Napierala if (iser_create_device_ib_res(device)) { 5204814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS); 5214814a0a4SEdward Tomasz Napierala device = NULL; 5224814a0a4SEdward Tomasz Napierala goto out; 5234814a0a4SEdward Tomasz Napierala } 5244814a0a4SEdward Tomasz Napierala list_add(&device->ig_list, &ig.device_list); 5254814a0a4SEdward Tomasz Napierala 5264814a0a4SEdward Tomasz Napierala inc_refcnt: 5274814a0a4SEdward Tomasz Napierala device->refcount++; 5284814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount); 5294814a0a4SEdward Tomasz Napierala out: 5304814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex); 5314814a0a4SEdward Tomasz Napierala return (device); 5324814a0a4SEdward Tomasz Napierala } 5334814a0a4SEdward Tomasz Napierala 5344814a0a4SEdward Tomasz Napierala /* if there's no demand for this device, release it */ 5354814a0a4SEdward Tomasz Napierala static void 5364814a0a4SEdward Tomasz Napierala iser_device_try_release(struct iser_device *device) 5374814a0a4SEdward Tomasz Napierala { 5384814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex); 5394814a0a4SEdward Tomasz Napierala device->refcount--; 5404814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount); 5414814a0a4SEdward Tomasz Napierala if (!device->refcount) { 5424814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(device); 5434814a0a4SEdward Tomasz Napierala list_del(&device->ig_list); 5444814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS); 5454814a0a4SEdward Tomasz Napierala device = NULL; 5464814a0a4SEdward Tomasz Napierala } 5474814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex); 5484814a0a4SEdward Tomasz Napierala } 5494814a0a4SEdward Tomasz Napierala 5504814a0a4SEdward Tomasz Napierala /** 5514814a0a4SEdward Tomasz Napierala * Called with state mutex held 5524814a0a4SEdward Tomasz Napierala **/ 5534814a0a4SEdward Tomasz Napierala static int iser_conn_state_comp_exch(struct iser_conn *iser_conn, 5544814a0a4SEdward Tomasz Napierala enum iser_conn_state comp, 5554814a0a4SEdward Tomasz Napierala enum iser_conn_state exch) 5564814a0a4SEdward Tomasz Napierala { 5574814a0a4SEdward Tomasz Napierala int ret; 5584814a0a4SEdward Tomasz Napierala 5594814a0a4SEdward Tomasz Napierala ret = (iser_conn->state == comp); 5604814a0a4SEdward Tomasz Napierala if (ret) 5614814a0a4SEdward Tomasz Napierala iser_conn->state = exch; 5624814a0a4SEdward Tomasz Napierala 5634814a0a4SEdward Tomasz Napierala return ret; 5644814a0a4SEdward Tomasz Napierala } 5654814a0a4SEdward Tomasz Napierala 5664814a0a4SEdward Tomasz Napierala /** 5674814a0a4SEdward Tomasz Napierala * iser_free_ib_conn_res - release IB related resources 5684814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection struct 5694814a0a4SEdward Tomasz Napierala * @destroy: indicator if we need to try to release the 5704814a0a4SEdward Tomasz Napierala * iser device and memory regoins pool (only iscsi 5714814a0a4SEdward Tomasz Napierala * shutdown and DEVICE_REMOVAL will use this). 5724814a0a4SEdward Tomasz Napierala * 5734814a0a4SEdward Tomasz Napierala * This routine is called with the iser state mutex held 5744814a0a4SEdward Tomasz Napierala * so the cm_id removal is out of here. It is Safe to 5754814a0a4SEdward Tomasz Napierala * be invoked multiple times. 5764814a0a4SEdward Tomasz Napierala */ 5774814a0a4SEdward Tomasz Napierala void 5784814a0a4SEdward Tomasz Napierala iser_free_ib_conn_res(struct iser_conn *iser_conn, 5794814a0a4SEdward Tomasz Napierala bool destroy) 5804814a0a4SEdward Tomasz Napierala { 5814814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 5824814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 5834814a0a4SEdward Tomasz Napierala 5844814a0a4SEdward Tomasz Napierala ISER_INFO("freeing conn %p cma_id %p qp %p", 5854814a0a4SEdward Tomasz Napierala iser_conn, ib_conn->cma_id, ib_conn->qp); 5864814a0a4SEdward Tomasz Napierala 5874814a0a4SEdward Tomasz Napierala if (ib_conn->qp != NULL) { 5884814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 5894814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--; 5904814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 5914814a0a4SEdward Tomasz Napierala rdma_destroy_qp(ib_conn->cma_id); 5924814a0a4SEdward Tomasz Napierala ib_conn->qp = NULL; 5934814a0a4SEdward Tomasz Napierala } 5944814a0a4SEdward Tomasz Napierala 5954814a0a4SEdward Tomasz Napierala if (destroy) { 5964814a0a4SEdward Tomasz Napierala if (iser_conn->login_buf) 5974814a0a4SEdward Tomasz Napierala iser_free_login_buf(iser_conn); 5984814a0a4SEdward Tomasz Napierala 5994814a0a4SEdward Tomasz Napierala if (iser_conn->rx_descs) 6004814a0a4SEdward Tomasz Napierala iser_free_rx_descriptors(iser_conn); 6014814a0a4SEdward Tomasz Napierala 6024814a0a4SEdward Tomasz Napierala if (device != NULL) { 6034814a0a4SEdward Tomasz Napierala iser_device_try_release(device); 6044814a0a4SEdward Tomasz Napierala ib_conn->device = NULL; 6054814a0a4SEdward Tomasz Napierala } 6064814a0a4SEdward Tomasz Napierala } 6074814a0a4SEdward Tomasz Napierala } 6084814a0a4SEdward Tomasz Napierala 6094814a0a4SEdward Tomasz Napierala /** 6104814a0a4SEdward Tomasz Napierala * triggers start of the disconnect procedures and wait for them to be done 6114814a0a4SEdward Tomasz Napierala * Called with state mutex held 6124814a0a4SEdward Tomasz Napierala */ 6134814a0a4SEdward Tomasz Napierala int 6144814a0a4SEdward Tomasz Napierala iser_conn_terminate(struct iser_conn *iser_conn) 6154814a0a4SEdward Tomasz Napierala { 6164814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 6174814a0a4SEdward Tomasz Napierala struct ib_send_wr *bad_send_wr; 6184814a0a4SEdward Tomasz Napierala struct ib_recv_wr *bad_recv_wr; 6194814a0a4SEdward Tomasz Napierala int err = 0; 6204814a0a4SEdward Tomasz Napierala 6214814a0a4SEdward Tomasz Napierala /* terminate the iser conn only if the conn state is UP */ 6224814a0a4SEdward Tomasz Napierala if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP, 6234814a0a4SEdward Tomasz Napierala ISER_CONN_TERMINATING)) 6244814a0a4SEdward Tomasz Napierala return (0); 6254814a0a4SEdward Tomasz Napierala 6264814a0a4SEdward Tomasz Napierala ISER_INFO("iser_conn %p state %d\n", iser_conn, iser_conn->state); 6274814a0a4SEdward Tomasz Napierala 6284814a0a4SEdward Tomasz Napierala if (ib_conn->qp == NULL) { 6294814a0a4SEdward Tomasz Napierala /* HOW can this be??? */ 6304814a0a4SEdward Tomasz Napierala ISER_WARN("qp wasn't created"); 6314814a0a4SEdward Tomasz Napierala return (1); 6324814a0a4SEdward Tomasz Napierala } 6334814a0a4SEdward Tomasz Napierala 6344814a0a4SEdward Tomasz Napierala /* 6354814a0a4SEdward Tomasz Napierala * Todo: This is a temporary workaround. 6364814a0a4SEdward Tomasz Napierala * We serialize the connection closure using global lock in order to 6374814a0a4SEdward Tomasz Napierala * receive all posted beacons completions. 6384814a0a4SEdward Tomasz Napierala * Without Serialization, in case we open many connections (QPs) on 6394814a0a4SEdward Tomasz Napierala * the same CQ, we might miss beacons because of missing interrupts. 6404814a0a4SEdward Tomasz Napierala */ 6414814a0a4SEdward Tomasz Napierala sx_xlock(&ig.close_conns_mutex); 6424814a0a4SEdward Tomasz Napierala 6434814a0a4SEdward Tomasz Napierala /* 6444814a0a4SEdward Tomasz Napierala * In case we didn't already clean up the cma_id (peer initiated 6454814a0a4SEdward Tomasz Napierala * a disconnection), we need to Cause the CMA to change the QP 6464814a0a4SEdward Tomasz Napierala * state to ERROR. 6474814a0a4SEdward Tomasz Napierala */ 6484814a0a4SEdward Tomasz Napierala if (ib_conn->cma_id) { 6494814a0a4SEdward Tomasz Napierala err = rdma_disconnect(ib_conn->cma_id); 6504814a0a4SEdward Tomasz Napierala if (err) 6514814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to disconnect, conn: 0x%p err %d", 6524814a0a4SEdward Tomasz Napierala iser_conn, err); 6534814a0a4SEdward Tomasz Napierala 6544814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock); 6554814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.send, 0, sizeof(struct ib_send_wr)); 6564814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.wr_id = ISER_BEACON_WRID; 6574814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.opcode = IB_WR_SEND; 6584814a0a4SEdward Tomasz Napierala /* post an indication that all send flush errors were consumed */ 6594814a0a4SEdward Tomasz Napierala err = ib_post_send(ib_conn->qp, &ib_conn->beacon.send, &bad_send_wr); 6604814a0a4SEdward Tomasz Napierala if (err) { 6614814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post send_beacon", ib_conn); 6624814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 6634814a0a4SEdward Tomasz Napierala goto out; 6644814a0a4SEdward Tomasz Napierala } 6654814a0a4SEdward Tomasz Napierala 6664814a0a4SEdward Tomasz Napierala ISER_DBG("before send cv_wait: %p", iser_conn); 6674814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock); 6684814a0a4SEdward Tomasz Napierala ISER_DBG("after send cv_wait: %p", iser_conn); 6694814a0a4SEdward Tomasz Napierala 6704814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.recv, 0, sizeof(struct ib_recv_wr)); 6714814a0a4SEdward Tomasz Napierala ib_conn->beacon.recv.wr_id = ISER_BEACON_WRID; 6724814a0a4SEdward Tomasz Napierala /* post an indication that all recv flush errors were consumed */ 6734814a0a4SEdward Tomasz Napierala err = ib_post_recv(ib_conn->qp, &ib_conn->beacon.recv, &bad_recv_wr); 6744814a0a4SEdward Tomasz Napierala if (err) { 6754814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post recv_beacon", ib_conn); 6764814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 6774814a0a4SEdward Tomasz Napierala goto out; 6784814a0a4SEdward Tomasz Napierala } 6794814a0a4SEdward Tomasz Napierala 6804814a0a4SEdward Tomasz Napierala ISER_DBG("before recv cv_wait: %p", iser_conn); 6814814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock); 6824814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 6834814a0a4SEdward Tomasz Napierala ISER_DBG("after recv cv_wait: %p", iser_conn); 6844814a0a4SEdward Tomasz Napierala } 6854814a0a4SEdward Tomasz Napierala out: 6864814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.close_conns_mutex); 6874814a0a4SEdward Tomasz Napierala return (1); 6884814a0a4SEdward Tomasz Napierala } 6894814a0a4SEdward Tomasz Napierala 6904814a0a4SEdward Tomasz Napierala /** 6914814a0a4SEdward Tomasz Napierala * Called with state mutex held 6924814a0a4SEdward Tomasz Napierala **/ 6934814a0a4SEdward Tomasz Napierala static void 6944814a0a4SEdward Tomasz Napierala iser_connect_error(struct rdma_cm_id *cma_id) 6954814a0a4SEdward Tomasz Napierala { 6964814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 6974814a0a4SEdward Tomasz Napierala 6984814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 6994814a0a4SEdward Tomasz Napierala 7004814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p", iser_conn); 7014814a0a4SEdward Tomasz Napierala 7024814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_TERMINATING; 7034814a0a4SEdward Tomasz Napierala 7044814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 7054814a0a4SEdward Tomasz Napierala } 7064814a0a4SEdward Tomasz Napierala 7074814a0a4SEdward Tomasz Napierala /** 7084814a0a4SEdward Tomasz Napierala * Called with state mutex held 7094814a0a4SEdward Tomasz Napierala **/ 7104814a0a4SEdward Tomasz Napierala static void 7114814a0a4SEdward Tomasz Napierala iser_addr_handler(struct rdma_cm_id *cma_id) 7124814a0a4SEdward Tomasz Napierala { 7134814a0a4SEdward Tomasz Napierala struct iser_device *device; 7144814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 7154814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn; 7164814a0a4SEdward Tomasz Napierala int ret; 7174814a0a4SEdward Tomasz Napierala 7184814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 7194814a0a4SEdward Tomasz Napierala 7204814a0a4SEdward Tomasz Napierala ib_conn = &iser_conn->ib_conn; 7214814a0a4SEdward Tomasz Napierala device = iser_device_find_by_ib_device(cma_id); 7224814a0a4SEdward Tomasz Napierala if (!device) { 7234814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p device lookup/creation failed", 7244814a0a4SEdward Tomasz Napierala iser_conn); 7254814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 7264814a0a4SEdward Tomasz Napierala return; 7274814a0a4SEdward Tomasz Napierala } 7284814a0a4SEdward Tomasz Napierala 7294814a0a4SEdward Tomasz Napierala ib_conn->device = device; 7304814a0a4SEdward Tomasz Napierala 7314814a0a4SEdward Tomasz Napierala ret = rdma_resolve_route(cma_id, 1000); 7324814a0a4SEdward Tomasz Napierala if (ret) { 7334814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p resolve route failed: %d", iser_conn, ret); 7344814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 7354814a0a4SEdward Tomasz Napierala return; 7364814a0a4SEdward Tomasz Napierala } 7374814a0a4SEdward Tomasz Napierala } 7384814a0a4SEdward Tomasz Napierala 7394814a0a4SEdward Tomasz Napierala /** 7404814a0a4SEdward Tomasz Napierala * Called with state mutex held 7414814a0a4SEdward Tomasz Napierala **/ 7424814a0a4SEdward Tomasz Napierala static void 7434814a0a4SEdward Tomasz Napierala iser_route_handler(struct rdma_cm_id *cma_id) 7444814a0a4SEdward Tomasz Napierala { 7454814a0a4SEdward Tomasz Napierala struct rdma_conn_param conn_param; 7464814a0a4SEdward Tomasz Napierala int ret; 7474814a0a4SEdward Tomasz Napierala struct iser_cm_hdr req_hdr; 7484814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context; 7494814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 7504814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 7514814a0a4SEdward Tomasz Napierala 7524814a0a4SEdward Tomasz Napierala ret = iser_create_ib_conn_res(ib_conn); 7534814a0a4SEdward Tomasz Napierala if (ret) 7544814a0a4SEdward Tomasz Napierala goto failure; 7554814a0a4SEdward Tomasz Napierala 7564814a0a4SEdward Tomasz Napierala memset(&conn_param, 0, sizeof conn_param); 7574814a0a4SEdward Tomasz Napierala conn_param.responder_resources = device->dev_attr.max_qp_rd_atom; 7584814a0a4SEdward Tomasz Napierala conn_param.retry_count = 7; 7594814a0a4SEdward Tomasz Napierala conn_param.rnr_retry_count = 6; 7604814a0a4SEdward Tomasz Napierala /* 7614814a0a4SEdward Tomasz Napierala * Initiaotr depth should not be set, but in order to compat 7624814a0a4SEdward Tomasz Napierala * with old targets, we keep this value set. 7634814a0a4SEdward Tomasz Napierala */ 7644814a0a4SEdward Tomasz Napierala conn_param.initiator_depth = 1; 7654814a0a4SEdward Tomasz Napierala 7664814a0a4SEdward Tomasz Napierala memset(&req_hdr, 0, sizeof(req_hdr)); 7674814a0a4SEdward Tomasz Napierala req_hdr.flags = (ISER_ZBVA_NOT_SUPPORTED | 7684814a0a4SEdward Tomasz Napierala ISER_SEND_W_INV_NOT_SUPPORTED); 7694814a0a4SEdward Tomasz Napierala conn_param.private_data = (void *)&req_hdr; 7704814a0a4SEdward Tomasz Napierala conn_param.private_data_len = sizeof(struct iser_cm_hdr); 7714814a0a4SEdward Tomasz Napierala 7724814a0a4SEdward Tomasz Napierala ret = rdma_connect(cma_id, &conn_param); 7734814a0a4SEdward Tomasz Napierala if (ret) { 7744814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failure connecting: %d", iser_conn, ret); 7754814a0a4SEdward Tomasz Napierala goto failure; 7764814a0a4SEdward Tomasz Napierala } 7774814a0a4SEdward Tomasz Napierala 7784814a0a4SEdward Tomasz Napierala return; 7794814a0a4SEdward Tomasz Napierala failure: 7804814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 7814814a0a4SEdward Tomasz Napierala } 7824814a0a4SEdward Tomasz Napierala 7834814a0a4SEdward Tomasz Napierala /** 7844814a0a4SEdward Tomasz Napierala * Called with state mutex held 7854814a0a4SEdward Tomasz Napierala **/ 7864814a0a4SEdward Tomasz Napierala static void 7874814a0a4SEdward Tomasz Napierala iser_connected_handler(struct rdma_cm_id *cma_id) 7884814a0a4SEdward Tomasz Napierala { 7894814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 7904814a0a4SEdward Tomasz Napierala struct ib_qp_attr attr; 7914814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr; 7924814a0a4SEdward Tomasz Napierala 7934814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 7944814a0a4SEdward Tomasz Napierala 7954814a0a4SEdward Tomasz Napierala (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); 7964814a0a4SEdward Tomasz Napierala 7974814a0a4SEdward Tomasz Napierala ISER_INFO("remote qpn:%x my qpn:%x", 7984814a0a4SEdward Tomasz Napierala attr.dest_qp_num, cma_id->qp->qp_num); 7994814a0a4SEdward Tomasz Napierala 8004814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_UP; 8014814a0a4SEdward Tomasz Napierala 8024814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 8034814a0a4SEdward Tomasz Napierala } 8044814a0a4SEdward Tomasz Napierala 8054814a0a4SEdward Tomasz Napierala /** 8064814a0a4SEdward Tomasz Napierala * Called with state mutex held 8074814a0a4SEdward Tomasz Napierala **/ 8084814a0a4SEdward Tomasz Napierala static void 8094814a0a4SEdward Tomasz Napierala iser_cleanup_handler(struct rdma_cm_id *cma_id, bool destroy) 8104814a0a4SEdward Tomasz Napierala { 8114814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context; 8124814a0a4SEdward Tomasz Napierala 8134814a0a4SEdward Tomasz Napierala if (iser_conn_terminate(iser_conn)) 8144814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn); 8154814a0a4SEdward Tomasz Napierala 8164814a0a4SEdward Tomasz Napierala } 8174814a0a4SEdward Tomasz Napierala 8184814a0a4SEdward Tomasz Napierala int 8194814a0a4SEdward Tomasz Napierala iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 8204814a0a4SEdward Tomasz Napierala { 8214814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 8224814a0a4SEdward Tomasz Napierala int ret = 0; 8234814a0a4SEdward Tomasz Napierala 8244814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 8254814a0a4SEdward Tomasz Napierala ISER_INFO("event %d status %d conn %p id %p", 8264814a0a4SEdward Tomasz Napierala event->event, event->status, cma_id->context, cma_id); 8274814a0a4SEdward Tomasz Napierala 8284814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 8294814a0a4SEdward Tomasz Napierala switch (event->event) { 8304814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_RESOLVED: 8314814a0a4SEdward Tomasz Napierala iser_addr_handler(cma_id); 8324814a0a4SEdward Tomasz Napierala break; 8334814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_RESOLVED: 8344814a0a4SEdward Tomasz Napierala iser_route_handler(cma_id); 8354814a0a4SEdward Tomasz Napierala break; 8364814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ESTABLISHED: 8374814a0a4SEdward Tomasz Napierala iser_connected_handler(cma_id); 8384814a0a4SEdward Tomasz Napierala break; 8394814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_ERROR: 8404814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_ERROR: 8414814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_CONNECT_ERROR: 8424814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_UNREACHABLE: 8434814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_REJECTED: 8444814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 8454814a0a4SEdward Tomasz Napierala break; 8464814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_DISCONNECTED: 8474814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_CHANGE: 8484814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_TIMEWAIT_EXIT: 8494814a0a4SEdward Tomasz Napierala iser_cleanup_handler(cma_id, false); 8504814a0a4SEdward Tomasz Napierala break; 8514814a0a4SEdward Tomasz Napierala default: 8524814a0a4SEdward Tomasz Napierala ISER_ERR("Unexpected RDMA CM event (%d)", event->event); 8534814a0a4SEdward Tomasz Napierala break; 8544814a0a4SEdward Tomasz Napierala } 8554814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 8564814a0a4SEdward Tomasz Napierala 8574814a0a4SEdward Tomasz Napierala return (ret); 8584814a0a4SEdward Tomasz Napierala } 8594814a0a4SEdward Tomasz Napierala 8604814a0a4SEdward Tomasz Napierala int 8614814a0a4SEdward Tomasz Napierala iser_post_recvl(struct iser_conn *iser_conn) 8624814a0a4SEdward Tomasz Napierala { 8634814a0a4SEdward Tomasz Napierala struct ib_recv_wr rx_wr, *rx_wr_failed; 8644814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 8654814a0a4SEdward Tomasz Napierala struct ib_sge sge; 8664814a0a4SEdward Tomasz Napierala int ib_ret; 8674814a0a4SEdward Tomasz Napierala 8684814a0a4SEdward Tomasz Napierala sge.addr = iser_conn->login_resp_dma; 8694814a0a4SEdward Tomasz Napierala sge.length = ISER_RX_LOGIN_SIZE; 8704814a0a4SEdward Tomasz Napierala sge.lkey = ib_conn->device->mr->lkey; 8714814a0a4SEdward Tomasz Napierala 8724814a0a4SEdward Tomasz Napierala rx_wr.wr_id = (uintptr_t)iser_conn->login_resp_buf; 8734814a0a4SEdward Tomasz Napierala rx_wr.sg_list = &sge; 8744814a0a4SEdward Tomasz Napierala rx_wr.num_sge = 1; 8754814a0a4SEdward Tomasz Napierala rx_wr.next = NULL; 8764814a0a4SEdward Tomasz Napierala 8774814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count++; 8784814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed); 8794814a0a4SEdward Tomasz Napierala if (ib_ret) { 8804814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret); 8814814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--; 8824814a0a4SEdward Tomasz Napierala } 8834814a0a4SEdward Tomasz Napierala 8844814a0a4SEdward Tomasz Napierala return (ib_ret); 8854814a0a4SEdward Tomasz Napierala } 8864814a0a4SEdward Tomasz Napierala 8874814a0a4SEdward Tomasz Napierala int 8884814a0a4SEdward Tomasz Napierala iser_post_recvm(struct iser_conn *iser_conn, int count) 8894814a0a4SEdward Tomasz Napierala { 8904814a0a4SEdward Tomasz Napierala struct ib_recv_wr *rx_wr, *rx_wr_failed; 8914814a0a4SEdward Tomasz Napierala int i, ib_ret; 8924814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 8934814a0a4SEdward Tomasz Napierala unsigned int my_rx_head = iser_conn->rx_desc_head; 8944814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc; 8954814a0a4SEdward Tomasz Napierala 8964814a0a4SEdward Tomasz Napierala for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 8974814a0a4SEdward Tomasz Napierala rx_desc = &iser_conn->rx_descs[my_rx_head]; 8984814a0a4SEdward Tomasz Napierala rx_wr->wr_id = (uintptr_t)rx_desc; 8994814a0a4SEdward Tomasz Napierala rx_wr->sg_list = &rx_desc->rx_sg; 9004814a0a4SEdward Tomasz Napierala rx_wr->num_sge = 1; 9014814a0a4SEdward Tomasz Napierala rx_wr->next = rx_wr + 1; 9024814a0a4SEdward Tomasz Napierala my_rx_head = (my_rx_head + 1) % iser_conn->qp_max_recv_dtos; 9034814a0a4SEdward Tomasz Napierala } 9044814a0a4SEdward Tomasz Napierala 9054814a0a4SEdward Tomasz Napierala rx_wr--; 9064814a0a4SEdward Tomasz Napierala rx_wr->next = NULL; /* mark end of work requests list */ 9074814a0a4SEdward Tomasz Napierala 9084814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count += count; 9094814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed); 9104814a0a4SEdward Tomasz Napierala if (ib_ret) { 9114814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret); 9124814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count -= count; 9134814a0a4SEdward Tomasz Napierala } else 9144814a0a4SEdward Tomasz Napierala iser_conn->rx_desc_head = my_rx_head; 9154814a0a4SEdward Tomasz Napierala 9164814a0a4SEdward Tomasz Napierala return (ib_ret); 9174814a0a4SEdward Tomasz Napierala } 9184814a0a4SEdward Tomasz Napierala 9194814a0a4SEdward Tomasz Napierala /** 9204814a0a4SEdward Tomasz Napierala * iser_start_send - Initiate a Send DTO operation 9214814a0a4SEdward Tomasz Napierala * 9224814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure 9234814a0a4SEdward Tomasz Napierala */ 9244814a0a4SEdward Tomasz Napierala int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, 9254814a0a4SEdward Tomasz Napierala bool signal) 9264814a0a4SEdward Tomasz Napierala { 9274814a0a4SEdward Tomasz Napierala int ib_ret; 9284814a0a4SEdward Tomasz Napierala struct ib_send_wr send_wr, *send_wr_failed; 9294814a0a4SEdward Tomasz Napierala 9304814a0a4SEdward Tomasz Napierala ib_dma_sync_single_for_device(ib_conn->device->ib_device, 9314814a0a4SEdward Tomasz Napierala tx_desc->dma_addr, ISER_HEADERS_LEN, 9324814a0a4SEdward Tomasz Napierala DMA_TO_DEVICE); 9334814a0a4SEdward Tomasz Napierala 9344814a0a4SEdward Tomasz Napierala send_wr.next = NULL; 9354814a0a4SEdward Tomasz Napierala send_wr.wr_id = (uintptr_t)tx_desc; 9364814a0a4SEdward Tomasz Napierala send_wr.sg_list = tx_desc->tx_sg; 9374814a0a4SEdward Tomasz Napierala send_wr.num_sge = tx_desc->num_sge; 9384814a0a4SEdward Tomasz Napierala send_wr.opcode = IB_WR_SEND; 9394814a0a4SEdward Tomasz Napierala send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0; 9404814a0a4SEdward Tomasz Napierala 9414814a0a4SEdward Tomasz Napierala ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed); 9424814a0a4SEdward Tomasz Napierala if (ib_ret) 9434814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_send failed, ret:%d", ib_ret); 9444814a0a4SEdward Tomasz Napierala 9454814a0a4SEdward Tomasz Napierala return (ib_ret); 9464814a0a4SEdward Tomasz Napierala } 947