1*4814a0a4SEdward Tomasz Napierala /* $FreeBSD$ */ 2*4814a0a4SEdward Tomasz Napierala /*- 3*4814a0a4SEdward Tomasz Napierala * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved. 4*4814a0a4SEdward Tomasz Napierala * 5*4814a0a4SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 6*4814a0a4SEdward Tomasz Napierala * modification, are permitted provided that the following conditions 7*4814a0a4SEdward Tomasz Napierala * are met: 8*4814a0a4SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 9*4814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 10*4814a0a4SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 11*4814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 12*4814a0a4SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 13*4814a0a4SEdward Tomasz Napierala * 14*4814a0a4SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15*4814a0a4SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16*4814a0a4SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17*4814a0a4SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18*4814a0a4SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19*4814a0a4SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20*4814a0a4SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21*4814a0a4SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22*4814a0a4SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23*4814a0a4SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24*4814a0a4SEdward Tomasz Napierala * SUCH DAMAGE. 25*4814a0a4SEdward Tomasz Napierala */ 26*4814a0a4SEdward Tomasz Napierala 27*4814a0a4SEdward Tomasz Napierala #include "icl_iser.h" 28*4814a0a4SEdward Tomasz Napierala 29*4814a0a4SEdward Tomasz Napierala static MALLOC_DEFINE(M_ISER_VERBS, "iser_verbs", "iser verbs backend"); 30*4814a0a4SEdward Tomasz Napierala static int iser_cq_poll_limit = 512; 31*4814a0a4SEdward Tomasz Napierala 32*4814a0a4SEdward Tomasz Napierala static void 33*4814a0a4SEdward Tomasz Napierala iser_cq_event_callback(struct ib_event *cause, void *context) 34*4814a0a4SEdward Tomasz Napierala { 35*4814a0a4SEdward Tomasz Napierala ISER_ERR("got cq event %d", cause->event); 36*4814a0a4SEdward Tomasz Napierala } 37*4814a0a4SEdward Tomasz Napierala 38*4814a0a4SEdward Tomasz Napierala static void 39*4814a0a4SEdward Tomasz Napierala iser_qp_event_callback(struct ib_event *cause, void *context) 40*4814a0a4SEdward Tomasz Napierala { 41*4814a0a4SEdward Tomasz Napierala ISER_ERR("got qp event %d", cause->event); 42*4814a0a4SEdward Tomasz Napierala } 43*4814a0a4SEdward Tomasz Napierala 44*4814a0a4SEdward Tomasz Napierala static void 45*4814a0a4SEdward Tomasz Napierala iser_event_handler(struct ib_event_handler *handler, 46*4814a0a4SEdward Tomasz Napierala struct ib_event *event) 47*4814a0a4SEdward Tomasz Napierala { 48*4814a0a4SEdward Tomasz Napierala ISER_ERR("async event %d on device %s port %d", 49*4814a0a4SEdward Tomasz Napierala event->event, event->device->name, 50*4814a0a4SEdward Tomasz Napierala event->element.port_num); 51*4814a0a4SEdward Tomasz Napierala } 52*4814a0a4SEdward Tomasz Napierala 53*4814a0a4SEdward Tomasz Napierala /** 54*4814a0a4SEdward Tomasz Napierala * is_iser_tx_desc - Indicate if the completion wr_id 55*4814a0a4SEdward Tomasz Napierala * is a TX descriptor or not. 56*4814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection 57*4814a0a4SEdward Tomasz Napierala * @wr_id: completion WR identifier 58*4814a0a4SEdward Tomasz Napierala * 59*4814a0a4SEdward Tomasz Napierala * Since we cannot rely on wc opcode in FLUSH errors 60*4814a0a4SEdward Tomasz Napierala * we must work around it by checking if the wr_id address 61*4814a0a4SEdward Tomasz Napierala * falls in the iser connection rx_descs buffer. If so 62*4814a0a4SEdward Tomasz Napierala * it is an RX descriptor, otherwize it is a TX. 63*4814a0a4SEdward Tomasz Napierala */ 64*4814a0a4SEdward Tomasz Napierala static inline bool 65*4814a0a4SEdward Tomasz Napierala is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id) 66*4814a0a4SEdward Tomasz Napierala { 67*4814a0a4SEdward Tomasz Napierala void *start = iser_conn->rx_descs; 68*4814a0a4SEdward Tomasz Napierala u64 len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs); 69*4814a0a4SEdward Tomasz Napierala void *end = (void *)((uintptr_t)start + (uintptr_t)len); 70*4814a0a4SEdward Tomasz Napierala 71*4814a0a4SEdward Tomasz Napierala if (start) { 72*4814a0a4SEdward Tomasz Napierala if (wr_id >= start && wr_id < end) 73*4814a0a4SEdward Tomasz Napierala return false; 74*4814a0a4SEdward Tomasz Napierala } else { 75*4814a0a4SEdward Tomasz Napierala return ((uintptr_t)wr_id != (uintptr_t)iser_conn->login_resp_buf); 76*4814a0a4SEdward Tomasz Napierala } 77*4814a0a4SEdward Tomasz Napierala 78*4814a0a4SEdward Tomasz Napierala return true; 79*4814a0a4SEdward Tomasz Napierala } 80*4814a0a4SEdward Tomasz Napierala 81*4814a0a4SEdward Tomasz Napierala /** 82*4814a0a4SEdward Tomasz Napierala * iser_handle_comp_error() - Handle error completion 83*4814a0a4SEdward Tomasz Napierala * @ib_conn: connection RDMA resources 84*4814a0a4SEdward Tomasz Napierala * @wc: work completion 85*4814a0a4SEdward Tomasz Napierala * 86*4814a0a4SEdward Tomasz Napierala * Notes: Update post_recv_buf_count in case of recv error completion. 87*4814a0a4SEdward Tomasz Napierala * For non-FLUSH error completion we should also notify iscsi layer that 88*4814a0a4SEdward Tomasz Napierala * connection is failed (in case we passed bind stage). 89*4814a0a4SEdward Tomasz Napierala */ 90*4814a0a4SEdward Tomasz Napierala static void 91*4814a0a4SEdward Tomasz Napierala iser_handle_comp_error(struct ib_conn *ib_conn, 92*4814a0a4SEdward Tomasz Napierala struct ib_wc *wc) 93*4814a0a4SEdward Tomasz Napierala { 94*4814a0a4SEdward Tomasz Napierala void *wr_id = (void *)(uintptr_t)wc->wr_id; 95*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 96*4814a0a4SEdward Tomasz Napierala ib_conn); 97*4814a0a4SEdward Tomasz Napierala 98*4814a0a4SEdward Tomasz Napierala if (is_iser_tx_desc(iser_conn, wr_id)) { 99*4814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got send comp error", iser_conn); 100*4814a0a4SEdward Tomasz Napierala } else { 101*4814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got recv comp error", iser_conn); 102*4814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--; 103*4814a0a4SEdward Tomasz Napierala } 104*4814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) 105*4814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn); 106*4814a0a4SEdward Tomasz Napierala } 107*4814a0a4SEdward Tomasz Napierala 108*4814a0a4SEdward Tomasz Napierala /** 109*4814a0a4SEdward Tomasz Napierala * iser_handle_wc - handle a single work completion 110*4814a0a4SEdward Tomasz Napierala * @wc: work completion 111*4814a0a4SEdward Tomasz Napierala * 112*4814a0a4SEdward Tomasz Napierala * Soft-IRQ context, work completion can be either 113*4814a0a4SEdward Tomasz Napierala * SEND or RECV, and can turn out successful or 114*4814a0a4SEdward Tomasz Napierala * with error (or flush error). 115*4814a0a4SEdward Tomasz Napierala */ 116*4814a0a4SEdward Tomasz Napierala static void iser_handle_wc(struct ib_wc *wc) 117*4814a0a4SEdward Tomasz Napierala { 118*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn; 119*4814a0a4SEdward Tomasz Napierala struct iser_tx_desc *tx_desc; 120*4814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc; 121*4814a0a4SEdward Tomasz Napierala 122*4814a0a4SEdward Tomasz Napierala ib_conn = wc->qp->qp_context; 123*4814a0a4SEdward Tomasz Napierala if (likely(wc->status == IB_WC_SUCCESS)) { 124*4814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_RECV) { 125*4814a0a4SEdward Tomasz Napierala rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 126*4814a0a4SEdward Tomasz Napierala iser_rcv_completion(rx_desc, wc->byte_len, 127*4814a0a4SEdward Tomasz Napierala ib_conn); 128*4814a0a4SEdward Tomasz Napierala } else 129*4814a0a4SEdward Tomasz Napierala if (wc->opcode == IB_WC_SEND) { 130*4814a0a4SEdward Tomasz Napierala tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 131*4814a0a4SEdward Tomasz Napierala iser_snd_completion(tx_desc, ib_conn); 132*4814a0a4SEdward Tomasz Napierala } else { 133*4814a0a4SEdward Tomasz Napierala ISER_ERR("Unknown wc opcode %d", wc->opcode); 134*4814a0a4SEdward Tomasz Napierala } 135*4814a0a4SEdward Tomasz Napierala } else { 136*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn, 137*4814a0a4SEdward Tomasz Napierala ib_conn); 138*4814a0a4SEdward Tomasz Napierala if (wc->status != IB_WC_WR_FLUSH_ERR) { 139*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p wr id %lx status %d vend_err %x", 140*4814a0a4SEdward Tomasz Napierala iser_conn, wc->wr_id, wc->status, wc->vendor_err); 141*4814a0a4SEdward Tomasz Napierala } else { 142*4814a0a4SEdward Tomasz Napierala ISER_DBG("flush error: conn %p wr id %lx", iser_conn, wc->wr_id); 143*4814a0a4SEdward Tomasz Napierala } 144*4814a0a4SEdward Tomasz Napierala 145*4814a0a4SEdward Tomasz Napierala if (wc->wr_id == ISER_BEACON_WRID) { 146*4814a0a4SEdward Tomasz Napierala /* all flush errors were consumed */ 147*4814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock); 148*4814a0a4SEdward Tomasz Napierala ISER_DBG("conn %p got ISER_BEACON_WRID", iser_conn); 149*4814a0a4SEdward Tomasz Napierala cv_signal(&ib_conn->beacon.flush_cv); 150*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 151*4814a0a4SEdward Tomasz Napierala } else { 152*4814a0a4SEdward Tomasz Napierala iser_handle_comp_error(ib_conn, wc); 153*4814a0a4SEdward Tomasz Napierala } 154*4814a0a4SEdward Tomasz Napierala } 155*4814a0a4SEdward Tomasz Napierala } 156*4814a0a4SEdward Tomasz Napierala 157*4814a0a4SEdward Tomasz Napierala static void 158*4814a0a4SEdward Tomasz Napierala iser_cq_tasklet_fn(void *data, int pending) 159*4814a0a4SEdward Tomasz Napierala { 160*4814a0a4SEdward Tomasz Napierala struct iser_comp *comp = (struct iser_comp *)data; 161*4814a0a4SEdward Tomasz Napierala struct ib_cq *cq = comp->cq; 162*4814a0a4SEdward Tomasz Napierala struct ib_wc *const wcs = comp->wcs; 163*4814a0a4SEdward Tomasz Napierala int completed = 0; 164*4814a0a4SEdward Tomasz Napierala int i; 165*4814a0a4SEdward Tomasz Napierala int n; 166*4814a0a4SEdward Tomasz Napierala 167*4814a0a4SEdward Tomasz Napierala while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 168*4814a0a4SEdward Tomasz Napierala for (i = 0; i < n; i++) 169*4814a0a4SEdward Tomasz Napierala iser_handle_wc(&wcs[i]); 170*4814a0a4SEdward Tomasz Napierala 171*4814a0a4SEdward Tomasz Napierala completed += n; 172*4814a0a4SEdward Tomasz Napierala if (completed >= iser_cq_poll_limit) 173*4814a0a4SEdward Tomasz Napierala break; 174*4814a0a4SEdward Tomasz Napierala } 175*4814a0a4SEdward Tomasz Napierala 176*4814a0a4SEdward Tomasz Napierala /* 177*4814a0a4SEdward Tomasz Napierala * It is assumed here that arming CQ only once its empty 178*4814a0a4SEdward Tomasz Napierala * would not cause interrupts to be missed. 179*4814a0a4SEdward Tomasz Napierala */ 180*4814a0a4SEdward Tomasz Napierala ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 181*4814a0a4SEdward Tomasz Napierala } 182*4814a0a4SEdward Tomasz Napierala 183*4814a0a4SEdward Tomasz Napierala static void 184*4814a0a4SEdward Tomasz Napierala iser_cq_callback(struct ib_cq *cq, void *cq_context) 185*4814a0a4SEdward Tomasz Napierala { 186*4814a0a4SEdward Tomasz Napierala struct iser_comp *comp = cq_context; 187*4814a0a4SEdward Tomasz Napierala 188*4814a0a4SEdward Tomasz Napierala taskqueue_enqueue_fast(comp->tq, &comp->task); 189*4814a0a4SEdward Tomasz Napierala } 190*4814a0a4SEdward Tomasz Napierala 191*4814a0a4SEdward Tomasz Napierala /** 192*4814a0a4SEdward Tomasz Napierala * iser_create_device_ib_res - creates Protection Domain (PD), Completion 193*4814a0a4SEdward Tomasz Napierala * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with 194*4814a0a4SEdward Tomasz Napierala * the adapator. 195*4814a0a4SEdward Tomasz Napierala * 196*4814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure 197*4814a0a4SEdward Tomasz Napierala */ 198*4814a0a4SEdward Tomasz Napierala static int 199*4814a0a4SEdward Tomasz Napierala iser_create_device_ib_res(struct iser_device *device) 200*4814a0a4SEdward Tomasz Napierala { 201*4814a0a4SEdward Tomasz Napierala struct ib_device_attr *dev_attr = &device->dev_attr; 202*4814a0a4SEdward Tomasz Napierala int ret, i, max_cqe; 203*4814a0a4SEdward Tomasz Napierala 204*4814a0a4SEdward Tomasz Napierala ret = ib_query_device(device->ib_device, dev_attr); 205*4814a0a4SEdward Tomasz Napierala if (ret) { 206*4814a0a4SEdward Tomasz Napierala ISER_ERR("Query device failed for %s", device->ib_device->name); 207*4814a0a4SEdward Tomasz Napierala return (ret); 208*4814a0a4SEdward Tomasz Napierala } 209*4814a0a4SEdward Tomasz Napierala 210*4814a0a4SEdward Tomasz Napierala if (!(dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { 211*4814a0a4SEdward Tomasz Napierala ISER_ERR("device %s doesn't support Fastreg, " 212*4814a0a4SEdward Tomasz Napierala "can't register memory", device->ib_device->name); 213*4814a0a4SEdward Tomasz Napierala return (1); 214*4814a0a4SEdward Tomasz Napierala } 215*4814a0a4SEdward Tomasz Napierala 216*4814a0a4SEdward Tomasz Napierala device->comps_used = min(mp_ncpus, device->ib_device->num_comp_vectors); 217*4814a0a4SEdward Tomasz Napierala 218*4814a0a4SEdward Tomasz Napierala device->comps = malloc(device->comps_used * sizeof(*device->comps), 219*4814a0a4SEdward Tomasz Napierala M_ISER_VERBS, M_WAITOK | M_ZERO); 220*4814a0a4SEdward Tomasz Napierala if (!device->comps) 221*4814a0a4SEdward Tomasz Napierala goto comps_err; 222*4814a0a4SEdward Tomasz Napierala 223*4814a0a4SEdward Tomasz Napierala max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe); 224*4814a0a4SEdward Tomasz Napierala 225*4814a0a4SEdward Tomasz Napierala ISER_DBG("using %d CQs, device %s supports %d vectors max_cqe %d", 226*4814a0a4SEdward Tomasz Napierala device->comps_used, device->ib_device->name, 227*4814a0a4SEdward Tomasz Napierala device->ib_device->num_comp_vectors, max_cqe); 228*4814a0a4SEdward Tomasz Napierala 229*4814a0a4SEdward Tomasz Napierala device->pd = ib_alloc_pd(device->ib_device); 230*4814a0a4SEdward Tomasz Napierala if (IS_ERR(device->pd)) 231*4814a0a4SEdward Tomasz Napierala goto pd_err; 232*4814a0a4SEdward Tomasz Napierala 233*4814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 234*4814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 235*4814a0a4SEdward Tomasz Napierala 236*4814a0a4SEdward Tomasz Napierala comp->device = device; 237*4814a0a4SEdward Tomasz Napierala comp->cq = ib_create_cq(device->ib_device, 238*4814a0a4SEdward Tomasz Napierala iser_cq_callback, 239*4814a0a4SEdward Tomasz Napierala iser_cq_event_callback, 240*4814a0a4SEdward Tomasz Napierala (void *)comp, 241*4814a0a4SEdward Tomasz Napierala max_cqe, i); 242*4814a0a4SEdward Tomasz Napierala if (IS_ERR(comp->cq)) { 243*4814a0a4SEdward Tomasz Napierala comp->cq = NULL; 244*4814a0a4SEdward Tomasz Napierala goto cq_err; 245*4814a0a4SEdward Tomasz Napierala } 246*4814a0a4SEdward Tomasz Napierala 247*4814a0a4SEdward Tomasz Napierala if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP)) 248*4814a0a4SEdward Tomasz Napierala goto cq_err; 249*4814a0a4SEdward Tomasz Napierala 250*4814a0a4SEdward Tomasz Napierala TASK_INIT(&comp->task, 0, iser_cq_tasklet_fn, comp); 251*4814a0a4SEdward Tomasz Napierala comp->tq = taskqueue_create_fast("iser_taskq", M_NOWAIT, 252*4814a0a4SEdward Tomasz Napierala taskqueue_thread_enqueue, &comp->tq); 253*4814a0a4SEdward Tomasz Napierala if (!comp->tq) 254*4814a0a4SEdward Tomasz Napierala goto tq_err; 255*4814a0a4SEdward Tomasz Napierala taskqueue_start_threads(&comp->tq, 1, PI_NET, "iser taskq"); 256*4814a0a4SEdward Tomasz Napierala } 257*4814a0a4SEdward Tomasz Napierala 258*4814a0a4SEdward Tomasz Napierala device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE | 259*4814a0a4SEdward Tomasz Napierala IB_ACCESS_REMOTE_WRITE | 260*4814a0a4SEdward Tomasz Napierala IB_ACCESS_REMOTE_READ); 261*4814a0a4SEdward Tomasz Napierala if (IS_ERR(device->mr)) 262*4814a0a4SEdward Tomasz Napierala goto tq_err; 263*4814a0a4SEdward Tomasz Napierala 264*4814a0a4SEdward Tomasz Napierala INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device, 265*4814a0a4SEdward Tomasz Napierala iser_event_handler); 266*4814a0a4SEdward Tomasz Napierala if (ib_register_event_handler(&device->event_handler)) 267*4814a0a4SEdward Tomasz Napierala goto handler_err; 268*4814a0a4SEdward Tomasz Napierala 269*4814a0a4SEdward Tomasz Napierala return (0); 270*4814a0a4SEdward Tomasz Napierala 271*4814a0a4SEdward Tomasz Napierala handler_err: 272*4814a0a4SEdward Tomasz Napierala ib_dereg_mr(device->mr); 273*4814a0a4SEdward Tomasz Napierala tq_err: 274*4814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 275*4814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 276*4814a0a4SEdward Tomasz Napierala if (comp->tq) 277*4814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq); 278*4814a0a4SEdward Tomasz Napierala } 279*4814a0a4SEdward Tomasz Napierala cq_err: 280*4814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 281*4814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 282*4814a0a4SEdward Tomasz Napierala if (comp->cq) 283*4814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq); 284*4814a0a4SEdward Tomasz Napierala } 285*4814a0a4SEdward Tomasz Napierala ib_dealloc_pd(device->pd); 286*4814a0a4SEdward Tomasz Napierala pd_err: 287*4814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS); 288*4814a0a4SEdward Tomasz Napierala comps_err: 289*4814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate an IB resource"); 290*4814a0a4SEdward Tomasz Napierala return (1); 291*4814a0a4SEdward Tomasz Napierala } 292*4814a0a4SEdward Tomasz Napierala 293*4814a0a4SEdward Tomasz Napierala /** 294*4814a0a4SEdward Tomasz Napierala * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR, 295*4814a0a4SEdward Tomasz Napierala * CQ and PD created with the device associated with the adapator. 296*4814a0a4SEdward Tomasz Napierala */ 297*4814a0a4SEdward Tomasz Napierala static void 298*4814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(struct iser_device *device) 299*4814a0a4SEdward Tomasz Napierala { 300*4814a0a4SEdward Tomasz Napierala int i; 301*4814a0a4SEdward Tomasz Napierala 302*4814a0a4SEdward Tomasz Napierala for (i = 0; i < device->comps_used; i++) { 303*4814a0a4SEdward Tomasz Napierala struct iser_comp *comp = &device->comps[i]; 304*4814a0a4SEdward Tomasz Napierala 305*4814a0a4SEdward Tomasz Napierala taskqueue_free(comp->tq); 306*4814a0a4SEdward Tomasz Napierala ib_destroy_cq(comp->cq); 307*4814a0a4SEdward Tomasz Napierala comp->cq = NULL; 308*4814a0a4SEdward Tomasz Napierala } 309*4814a0a4SEdward Tomasz Napierala 310*4814a0a4SEdward Tomasz Napierala (void)ib_unregister_event_handler(&device->event_handler); 311*4814a0a4SEdward Tomasz Napierala (void)ib_dereg_mr(device->mr); 312*4814a0a4SEdward Tomasz Napierala (void)ib_dealloc_pd(device->pd); 313*4814a0a4SEdward Tomasz Napierala 314*4814a0a4SEdward Tomasz Napierala free(device->comps, M_ISER_VERBS); 315*4814a0a4SEdward Tomasz Napierala device->comps = NULL; 316*4814a0a4SEdward Tomasz Napierala 317*4814a0a4SEdward Tomasz Napierala device->mr = NULL; 318*4814a0a4SEdward Tomasz Napierala device->pd = NULL; 319*4814a0a4SEdward Tomasz Napierala } 320*4814a0a4SEdward Tomasz Napierala 321*4814a0a4SEdward Tomasz Napierala static int 322*4814a0a4SEdward Tomasz Napierala iser_alloc_reg_res(struct ib_device *ib_device, 323*4814a0a4SEdward Tomasz Napierala struct ib_pd *pd, 324*4814a0a4SEdward Tomasz Napierala struct iser_reg_resources *res) 325*4814a0a4SEdward Tomasz Napierala { 326*4814a0a4SEdward Tomasz Napierala int ret; 327*4814a0a4SEdward Tomasz Napierala 328*4814a0a4SEdward Tomasz Napierala res->frpl = ib_alloc_fast_reg_page_list(ib_device, 329*4814a0a4SEdward Tomasz Napierala ISCSI_ISER_SG_TABLESIZE + 1); 330*4814a0a4SEdward Tomasz Napierala if (IS_ERR(res->frpl)) { 331*4814a0a4SEdward Tomasz Napierala ret = -PTR_ERR(res->frpl); 332*4814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate fast reg page list err=%d", ret); 333*4814a0a4SEdward Tomasz Napierala return (ret); 334*4814a0a4SEdward Tomasz Napierala } 335*4814a0a4SEdward Tomasz Napierala 336*4814a0a4SEdward Tomasz Napierala res->mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE + 1); 337*4814a0a4SEdward Tomasz Napierala if (IS_ERR(res->mr)) { 338*4814a0a4SEdward Tomasz Napierala ret = -PTR_ERR(res->mr); 339*4814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate fast reg mr err=%d", ret); 340*4814a0a4SEdward Tomasz Napierala goto fast_reg_mr_failure; 341*4814a0a4SEdward Tomasz Napierala } 342*4814a0a4SEdward Tomasz Napierala res->mr_valid = 1; 343*4814a0a4SEdward Tomasz Napierala 344*4814a0a4SEdward Tomasz Napierala return (0); 345*4814a0a4SEdward Tomasz Napierala 346*4814a0a4SEdward Tomasz Napierala fast_reg_mr_failure: 347*4814a0a4SEdward Tomasz Napierala ib_free_fast_reg_page_list(res->frpl); 348*4814a0a4SEdward Tomasz Napierala 349*4814a0a4SEdward Tomasz Napierala return (ret); 350*4814a0a4SEdward Tomasz Napierala } 351*4814a0a4SEdward Tomasz Napierala 352*4814a0a4SEdward Tomasz Napierala static void 353*4814a0a4SEdward Tomasz Napierala iser_free_reg_res(struct iser_reg_resources *rsc) 354*4814a0a4SEdward Tomasz Napierala { 355*4814a0a4SEdward Tomasz Napierala ib_dereg_mr(rsc->mr); 356*4814a0a4SEdward Tomasz Napierala ib_free_fast_reg_page_list(rsc->frpl); 357*4814a0a4SEdward Tomasz Napierala } 358*4814a0a4SEdward Tomasz Napierala 359*4814a0a4SEdward Tomasz Napierala static struct fast_reg_descriptor * 360*4814a0a4SEdward Tomasz Napierala iser_create_fastreg_desc(struct ib_device *ib_device, struct ib_pd *pd) 361*4814a0a4SEdward Tomasz Napierala { 362*4814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc; 363*4814a0a4SEdward Tomasz Napierala int ret; 364*4814a0a4SEdward Tomasz Napierala 365*4814a0a4SEdward Tomasz Napierala desc = malloc(sizeof(*desc), M_ISER_VERBS, M_WAITOK | M_ZERO); 366*4814a0a4SEdward Tomasz Napierala if (!desc) { 367*4814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to allocate a new fastreg descriptor"); 368*4814a0a4SEdward Tomasz Napierala return (NULL); 369*4814a0a4SEdward Tomasz Napierala } 370*4814a0a4SEdward Tomasz Napierala 371*4814a0a4SEdward Tomasz Napierala ret = iser_alloc_reg_res(ib_device, pd, &desc->rsc); 372*4814a0a4SEdward Tomasz Napierala if (ret) { 373*4814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate reg_resources"); 374*4814a0a4SEdward Tomasz Napierala goto err; 375*4814a0a4SEdward Tomasz Napierala } 376*4814a0a4SEdward Tomasz Napierala 377*4814a0a4SEdward Tomasz Napierala return (desc); 378*4814a0a4SEdward Tomasz Napierala err: 379*4814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS); 380*4814a0a4SEdward Tomasz Napierala return (NULL); 381*4814a0a4SEdward Tomasz Napierala } 382*4814a0a4SEdward Tomasz Napierala 383*4814a0a4SEdward Tomasz Napierala /** 384*4814a0a4SEdward Tomasz Napierala * iser_create_fmr_pool - Creates FMR pool and page_vector 385*4814a0a4SEdward Tomasz Napierala * 386*4814a0a4SEdward Tomasz Napierala * returns 0 on success, or errno code on failure 387*4814a0a4SEdward Tomasz Napierala */ 388*4814a0a4SEdward Tomasz Napierala int 389*4814a0a4SEdward Tomasz Napierala iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max) 390*4814a0a4SEdward Tomasz Napierala { 391*4814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 392*4814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc; 393*4814a0a4SEdward Tomasz Napierala int i; 394*4814a0a4SEdward Tomasz Napierala 395*4814a0a4SEdward Tomasz Napierala INIT_LIST_HEAD(&ib_conn->fastreg.pool); 396*4814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size = 0; 397*4814a0a4SEdward Tomasz Napierala for (i = 0; i < cmds_max; i++) { 398*4814a0a4SEdward Tomasz Napierala desc = iser_create_fastreg_desc(device->ib_device, device->pd); 399*4814a0a4SEdward Tomasz Napierala if (!desc) { 400*4814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to create fastreg descriptor"); 401*4814a0a4SEdward Tomasz Napierala goto err; 402*4814a0a4SEdward Tomasz Napierala } 403*4814a0a4SEdward Tomasz Napierala 404*4814a0a4SEdward Tomasz Napierala list_add_tail(&desc->list, &ib_conn->fastreg.pool); 405*4814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size++; 406*4814a0a4SEdward Tomasz Napierala } 407*4814a0a4SEdward Tomasz Napierala 408*4814a0a4SEdward Tomasz Napierala return (0); 409*4814a0a4SEdward Tomasz Napierala 410*4814a0a4SEdward Tomasz Napierala err: 411*4814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(ib_conn); 412*4814a0a4SEdward Tomasz Napierala return (ENOMEM); 413*4814a0a4SEdward Tomasz Napierala } 414*4814a0a4SEdward Tomasz Napierala 415*4814a0a4SEdward Tomasz Napierala /** 416*4814a0a4SEdward Tomasz Napierala * iser_free_fmr_pool - releases the FMR pool and page vec 417*4814a0a4SEdward Tomasz Napierala */ 418*4814a0a4SEdward Tomasz Napierala void 419*4814a0a4SEdward Tomasz Napierala iser_free_fastreg_pool(struct ib_conn *ib_conn) 420*4814a0a4SEdward Tomasz Napierala { 421*4814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc, *tmp; 422*4814a0a4SEdward Tomasz Napierala int i = 0; 423*4814a0a4SEdward Tomasz Napierala 424*4814a0a4SEdward Tomasz Napierala if (list_empty(&ib_conn->fastreg.pool)) 425*4814a0a4SEdward Tomasz Napierala return; 426*4814a0a4SEdward Tomasz Napierala 427*4814a0a4SEdward Tomasz Napierala ISER_DBG("freeing conn %p fr pool", ib_conn); 428*4814a0a4SEdward Tomasz Napierala 429*4814a0a4SEdward Tomasz Napierala list_for_each_entry_safe(desc, tmp, &ib_conn->fastreg.pool, list) { 430*4814a0a4SEdward Tomasz Napierala list_del(&desc->list); 431*4814a0a4SEdward Tomasz Napierala iser_free_reg_res(&desc->rsc); 432*4814a0a4SEdward Tomasz Napierala free(desc, M_ISER_VERBS); 433*4814a0a4SEdward Tomasz Napierala ++i; 434*4814a0a4SEdward Tomasz Napierala } 435*4814a0a4SEdward Tomasz Napierala 436*4814a0a4SEdward Tomasz Napierala if (i < ib_conn->fastreg.pool_size) 437*4814a0a4SEdward Tomasz Napierala ISER_WARN("pool still has %d regions registered", 438*4814a0a4SEdward Tomasz Napierala ib_conn->fastreg.pool_size - i); 439*4814a0a4SEdward Tomasz Napierala } 440*4814a0a4SEdward Tomasz Napierala 441*4814a0a4SEdward Tomasz Napierala /** 442*4814a0a4SEdward Tomasz Napierala * iser_create_ib_conn_res - Queue-Pair (QP) 443*4814a0a4SEdward Tomasz Napierala * 444*4814a0a4SEdward Tomasz Napierala * returns 0 on success, 1 on failure 445*4814a0a4SEdward Tomasz Napierala */ 446*4814a0a4SEdward Tomasz Napierala static int 447*4814a0a4SEdward Tomasz Napierala iser_create_ib_conn_res(struct ib_conn *ib_conn) 448*4814a0a4SEdward Tomasz Napierala { 449*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 450*4814a0a4SEdward Tomasz Napierala struct iser_device *device; 451*4814a0a4SEdward Tomasz Napierala struct ib_device_attr *dev_attr; 452*4814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr; 453*4814a0a4SEdward Tomasz Napierala int index, min_index = 0; 454*4814a0a4SEdward Tomasz Napierala int ret = -ENOMEM; 455*4814a0a4SEdward Tomasz Napierala 456*4814a0a4SEdward Tomasz Napierala iser_conn = container_of(ib_conn, struct iser_conn, ib_conn); 457*4814a0a4SEdward Tomasz Napierala device = ib_conn->device; 458*4814a0a4SEdward Tomasz Napierala dev_attr = &device->dev_attr; 459*4814a0a4SEdward Tomasz Napierala 460*4814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 461*4814a0a4SEdward Tomasz Napierala /* select the CQ with the minimal number of usages */ 462*4814a0a4SEdward Tomasz Napierala for (index = 0; index < device->comps_used; index++) { 463*4814a0a4SEdward Tomasz Napierala if (device->comps[index].active_qps < 464*4814a0a4SEdward Tomasz Napierala device->comps[min_index].active_qps) 465*4814a0a4SEdward Tomasz Napierala min_index = index; 466*4814a0a4SEdward Tomasz Napierala } 467*4814a0a4SEdward Tomasz Napierala ib_conn->comp = &device->comps[min_index]; 468*4814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps++; 469*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 470*4814a0a4SEdward Tomasz Napierala ISER_INFO("cq index %d used for ib_conn %p", min_index, ib_conn); 471*4814a0a4SEdward Tomasz Napierala 472*4814a0a4SEdward Tomasz Napierala memset(&init_attr, 0, sizeof init_attr); 473*4814a0a4SEdward Tomasz Napierala init_attr.event_handler = iser_qp_event_callback; 474*4814a0a4SEdward Tomasz Napierala init_attr.qp_context = (void *)ib_conn; 475*4814a0a4SEdward Tomasz Napierala init_attr.send_cq = ib_conn->comp->cq; 476*4814a0a4SEdward Tomasz Napierala init_attr.recv_cq = ib_conn->comp->cq; 477*4814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; 478*4814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_sge = 2; 479*4814a0a4SEdward Tomasz Napierala init_attr.cap.max_recv_sge = 1; 480*4814a0a4SEdward Tomasz Napierala init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 481*4814a0a4SEdward Tomasz Napierala init_attr.qp_type = IB_QPT_RC; 482*4814a0a4SEdward Tomasz Napierala 483*4814a0a4SEdward Tomasz Napierala if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) { 484*4814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS; 485*4814a0a4SEdward Tomasz Napierala iser_conn->max_cmds = 486*4814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS); 487*4814a0a4SEdward Tomasz Napierala } else { 488*4814a0a4SEdward Tomasz Napierala init_attr.cap.max_send_wr = dev_attr->max_qp_wr; 489*4814a0a4SEdward Tomasz Napierala iser_conn->max_cmds = 490*4814a0a4SEdward Tomasz Napierala ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr); 491*4814a0a4SEdward Tomasz Napierala } 492*4814a0a4SEdward Tomasz Napierala ISER_DBG("device %s supports max_send_wr %d", 493*4814a0a4SEdward Tomasz Napierala device->ib_device->name, dev_attr->max_qp_wr); 494*4814a0a4SEdward Tomasz Napierala 495*4814a0a4SEdward Tomasz Napierala ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); 496*4814a0a4SEdward Tomasz Napierala if (ret) 497*4814a0a4SEdward Tomasz Napierala goto out_err; 498*4814a0a4SEdward Tomasz Napierala 499*4814a0a4SEdward Tomasz Napierala ib_conn->qp = ib_conn->cma_id->qp; 500*4814a0a4SEdward Tomasz Napierala ISER_DBG("setting conn %p cma_id %p qp %p", 501*4814a0a4SEdward Tomasz Napierala ib_conn, ib_conn->cma_id, 502*4814a0a4SEdward Tomasz Napierala ib_conn->cma_id->qp); 503*4814a0a4SEdward Tomasz Napierala 504*4814a0a4SEdward Tomasz Napierala return (ret); 505*4814a0a4SEdward Tomasz Napierala 506*4814a0a4SEdward Tomasz Napierala out_err: 507*4814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 508*4814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--; 509*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 510*4814a0a4SEdward Tomasz Napierala ISER_ERR("unable to alloc mem or create resource, err %d", ret); 511*4814a0a4SEdward Tomasz Napierala 512*4814a0a4SEdward Tomasz Napierala return (ret); 513*4814a0a4SEdward Tomasz Napierala } 514*4814a0a4SEdward Tomasz Napierala 515*4814a0a4SEdward Tomasz Napierala /** 516*4814a0a4SEdward Tomasz Napierala * based on the resolved device node GUID see if there already allocated 517*4814a0a4SEdward Tomasz Napierala * device for this device. If there's no such, create one. 518*4814a0a4SEdward Tomasz Napierala */ 519*4814a0a4SEdward Tomasz Napierala static struct iser_device * 520*4814a0a4SEdward Tomasz Napierala iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) 521*4814a0a4SEdward Tomasz Napierala { 522*4814a0a4SEdward Tomasz Napierala struct iser_device *device; 523*4814a0a4SEdward Tomasz Napierala 524*4814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex); 525*4814a0a4SEdward Tomasz Napierala 526*4814a0a4SEdward Tomasz Napierala list_for_each_entry(device, &ig.device_list, ig_list) 527*4814a0a4SEdward Tomasz Napierala /* find if there's a match using the node GUID */ 528*4814a0a4SEdward Tomasz Napierala if (device->ib_device->node_guid == cma_id->device->node_guid) 529*4814a0a4SEdward Tomasz Napierala goto inc_refcnt; 530*4814a0a4SEdward Tomasz Napierala 531*4814a0a4SEdward Tomasz Napierala device = malloc(sizeof *device, M_ISER_VERBS, M_WAITOK | M_ZERO); 532*4814a0a4SEdward Tomasz Napierala if (device == NULL) 533*4814a0a4SEdward Tomasz Napierala goto out; 534*4814a0a4SEdward Tomasz Napierala 535*4814a0a4SEdward Tomasz Napierala /* assign this device to the device */ 536*4814a0a4SEdward Tomasz Napierala device->ib_device = cma_id->device; 537*4814a0a4SEdward Tomasz Napierala /* init the device and link it into ig device list */ 538*4814a0a4SEdward Tomasz Napierala if (iser_create_device_ib_res(device)) { 539*4814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS); 540*4814a0a4SEdward Tomasz Napierala device = NULL; 541*4814a0a4SEdward Tomasz Napierala goto out; 542*4814a0a4SEdward Tomasz Napierala } 543*4814a0a4SEdward Tomasz Napierala list_add(&device->ig_list, &ig.device_list); 544*4814a0a4SEdward Tomasz Napierala 545*4814a0a4SEdward Tomasz Napierala inc_refcnt: 546*4814a0a4SEdward Tomasz Napierala device->refcount++; 547*4814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount); 548*4814a0a4SEdward Tomasz Napierala out: 549*4814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex); 550*4814a0a4SEdward Tomasz Napierala return (device); 551*4814a0a4SEdward Tomasz Napierala } 552*4814a0a4SEdward Tomasz Napierala 553*4814a0a4SEdward Tomasz Napierala /* if there's no demand for this device, release it */ 554*4814a0a4SEdward Tomasz Napierala static void 555*4814a0a4SEdward Tomasz Napierala iser_device_try_release(struct iser_device *device) 556*4814a0a4SEdward Tomasz Napierala { 557*4814a0a4SEdward Tomasz Napierala sx_xlock(&ig.device_list_mutex); 558*4814a0a4SEdward Tomasz Napierala device->refcount--; 559*4814a0a4SEdward Tomasz Napierala ISER_INFO("device %p refcount %d", device, device->refcount); 560*4814a0a4SEdward Tomasz Napierala if (!device->refcount) { 561*4814a0a4SEdward Tomasz Napierala iser_free_device_ib_res(device); 562*4814a0a4SEdward Tomasz Napierala list_del(&device->ig_list); 563*4814a0a4SEdward Tomasz Napierala free(device, M_ISER_VERBS); 564*4814a0a4SEdward Tomasz Napierala device = NULL; 565*4814a0a4SEdward Tomasz Napierala } 566*4814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.device_list_mutex); 567*4814a0a4SEdward Tomasz Napierala } 568*4814a0a4SEdward Tomasz Napierala 569*4814a0a4SEdward Tomasz Napierala /** 570*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 571*4814a0a4SEdward Tomasz Napierala **/ 572*4814a0a4SEdward Tomasz Napierala static int iser_conn_state_comp_exch(struct iser_conn *iser_conn, 573*4814a0a4SEdward Tomasz Napierala enum iser_conn_state comp, 574*4814a0a4SEdward Tomasz Napierala enum iser_conn_state exch) 575*4814a0a4SEdward Tomasz Napierala { 576*4814a0a4SEdward Tomasz Napierala int ret; 577*4814a0a4SEdward Tomasz Napierala 578*4814a0a4SEdward Tomasz Napierala ret = (iser_conn->state == comp); 579*4814a0a4SEdward Tomasz Napierala if (ret) 580*4814a0a4SEdward Tomasz Napierala iser_conn->state = exch; 581*4814a0a4SEdward Tomasz Napierala 582*4814a0a4SEdward Tomasz Napierala return ret; 583*4814a0a4SEdward Tomasz Napierala } 584*4814a0a4SEdward Tomasz Napierala 585*4814a0a4SEdward Tomasz Napierala /** 586*4814a0a4SEdward Tomasz Napierala * iser_free_ib_conn_res - release IB related resources 587*4814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection struct 588*4814a0a4SEdward Tomasz Napierala * @destroy: indicator if we need to try to release the 589*4814a0a4SEdward Tomasz Napierala * iser device and memory regoins pool (only iscsi 590*4814a0a4SEdward Tomasz Napierala * shutdown and DEVICE_REMOVAL will use this). 591*4814a0a4SEdward Tomasz Napierala * 592*4814a0a4SEdward Tomasz Napierala * This routine is called with the iser state mutex held 593*4814a0a4SEdward Tomasz Napierala * so the cm_id removal is out of here. It is Safe to 594*4814a0a4SEdward Tomasz Napierala * be invoked multiple times. 595*4814a0a4SEdward Tomasz Napierala */ 596*4814a0a4SEdward Tomasz Napierala void 597*4814a0a4SEdward Tomasz Napierala iser_free_ib_conn_res(struct iser_conn *iser_conn, 598*4814a0a4SEdward Tomasz Napierala bool destroy) 599*4814a0a4SEdward Tomasz Napierala { 600*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 601*4814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 602*4814a0a4SEdward Tomasz Napierala 603*4814a0a4SEdward Tomasz Napierala ISER_INFO("freeing conn %p cma_id %p qp %p", 604*4814a0a4SEdward Tomasz Napierala iser_conn, ib_conn->cma_id, ib_conn->qp); 605*4814a0a4SEdward Tomasz Napierala 606*4814a0a4SEdward Tomasz Napierala if (ib_conn->qp != NULL) { 607*4814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 608*4814a0a4SEdward Tomasz Napierala ib_conn->comp->active_qps--; 609*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 610*4814a0a4SEdward Tomasz Napierala rdma_destroy_qp(ib_conn->cma_id); 611*4814a0a4SEdward Tomasz Napierala ib_conn->qp = NULL; 612*4814a0a4SEdward Tomasz Napierala } 613*4814a0a4SEdward Tomasz Napierala 614*4814a0a4SEdward Tomasz Napierala if (destroy) { 615*4814a0a4SEdward Tomasz Napierala if (iser_conn->login_buf) 616*4814a0a4SEdward Tomasz Napierala iser_free_login_buf(iser_conn); 617*4814a0a4SEdward Tomasz Napierala 618*4814a0a4SEdward Tomasz Napierala if (iser_conn->rx_descs) 619*4814a0a4SEdward Tomasz Napierala iser_free_rx_descriptors(iser_conn); 620*4814a0a4SEdward Tomasz Napierala 621*4814a0a4SEdward Tomasz Napierala if (device != NULL) { 622*4814a0a4SEdward Tomasz Napierala iser_device_try_release(device); 623*4814a0a4SEdward Tomasz Napierala ib_conn->device = NULL; 624*4814a0a4SEdward Tomasz Napierala } 625*4814a0a4SEdward Tomasz Napierala } 626*4814a0a4SEdward Tomasz Napierala } 627*4814a0a4SEdward Tomasz Napierala 628*4814a0a4SEdward Tomasz Napierala /** 629*4814a0a4SEdward Tomasz Napierala * triggers start of the disconnect procedures and wait for them to be done 630*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 631*4814a0a4SEdward Tomasz Napierala */ 632*4814a0a4SEdward Tomasz Napierala int 633*4814a0a4SEdward Tomasz Napierala iser_conn_terminate(struct iser_conn *iser_conn) 634*4814a0a4SEdward Tomasz Napierala { 635*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 636*4814a0a4SEdward Tomasz Napierala struct ib_send_wr *bad_send_wr; 637*4814a0a4SEdward Tomasz Napierala struct ib_recv_wr *bad_recv_wr; 638*4814a0a4SEdward Tomasz Napierala int err = 0; 639*4814a0a4SEdward Tomasz Napierala 640*4814a0a4SEdward Tomasz Napierala /* terminate the iser conn only if the conn state is UP */ 641*4814a0a4SEdward Tomasz Napierala if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP, 642*4814a0a4SEdward Tomasz Napierala ISER_CONN_TERMINATING)) 643*4814a0a4SEdward Tomasz Napierala return (0); 644*4814a0a4SEdward Tomasz Napierala 645*4814a0a4SEdward Tomasz Napierala ISER_INFO("iser_conn %p state %d\n", iser_conn, iser_conn->state); 646*4814a0a4SEdward Tomasz Napierala 647*4814a0a4SEdward Tomasz Napierala if (ib_conn->qp == NULL) { 648*4814a0a4SEdward Tomasz Napierala /* HOW can this be??? */ 649*4814a0a4SEdward Tomasz Napierala ISER_WARN("qp wasn't created"); 650*4814a0a4SEdward Tomasz Napierala return (1); 651*4814a0a4SEdward Tomasz Napierala } 652*4814a0a4SEdward Tomasz Napierala 653*4814a0a4SEdward Tomasz Napierala /* 654*4814a0a4SEdward Tomasz Napierala * Todo: This is a temporary workaround. 655*4814a0a4SEdward Tomasz Napierala * We serialize the connection closure using global lock in order to 656*4814a0a4SEdward Tomasz Napierala * receive all posted beacons completions. 657*4814a0a4SEdward Tomasz Napierala * Without Serialization, in case we open many connections (QPs) on 658*4814a0a4SEdward Tomasz Napierala * the same CQ, we might miss beacons because of missing interrupts. 659*4814a0a4SEdward Tomasz Napierala */ 660*4814a0a4SEdward Tomasz Napierala sx_xlock(&ig.close_conns_mutex); 661*4814a0a4SEdward Tomasz Napierala 662*4814a0a4SEdward Tomasz Napierala /* 663*4814a0a4SEdward Tomasz Napierala * In case we didn't already clean up the cma_id (peer initiated 664*4814a0a4SEdward Tomasz Napierala * a disconnection), we need to Cause the CMA to change the QP 665*4814a0a4SEdward Tomasz Napierala * state to ERROR. 666*4814a0a4SEdward Tomasz Napierala */ 667*4814a0a4SEdward Tomasz Napierala if (ib_conn->cma_id) { 668*4814a0a4SEdward Tomasz Napierala err = rdma_disconnect(ib_conn->cma_id); 669*4814a0a4SEdward Tomasz Napierala if (err) 670*4814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to disconnect, conn: 0x%p err %d", 671*4814a0a4SEdward Tomasz Napierala iser_conn, err); 672*4814a0a4SEdward Tomasz Napierala 673*4814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->beacon.flush_lock); 674*4814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.send, 0, sizeof(struct ib_send_wr)); 675*4814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.wr_id = ISER_BEACON_WRID; 676*4814a0a4SEdward Tomasz Napierala ib_conn->beacon.send.opcode = IB_WR_SEND; 677*4814a0a4SEdward Tomasz Napierala /* post an indication that all send flush errors were consumed */ 678*4814a0a4SEdward Tomasz Napierala err = ib_post_send(ib_conn->qp, &ib_conn->beacon.send, &bad_send_wr); 679*4814a0a4SEdward Tomasz Napierala if (err) { 680*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post send_beacon", ib_conn); 681*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 682*4814a0a4SEdward Tomasz Napierala goto out; 683*4814a0a4SEdward Tomasz Napierala } 684*4814a0a4SEdward Tomasz Napierala 685*4814a0a4SEdward Tomasz Napierala ISER_DBG("before send cv_wait: %p", iser_conn); 686*4814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock); 687*4814a0a4SEdward Tomasz Napierala ISER_DBG("after send cv_wait: %p", iser_conn); 688*4814a0a4SEdward Tomasz Napierala 689*4814a0a4SEdward Tomasz Napierala memset(&ib_conn->beacon.recv, 0, sizeof(struct ib_recv_wr)); 690*4814a0a4SEdward Tomasz Napierala ib_conn->beacon.recv.wr_id = ISER_BEACON_WRID; 691*4814a0a4SEdward Tomasz Napierala /* post an indication that all recv flush errors were consumed */ 692*4814a0a4SEdward Tomasz Napierala err = ib_post_recv(ib_conn->qp, &ib_conn->beacon.recv, &bad_recv_wr); 693*4814a0a4SEdward Tomasz Napierala if (err) { 694*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failed to post recv_beacon", ib_conn); 695*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 696*4814a0a4SEdward Tomasz Napierala goto out; 697*4814a0a4SEdward Tomasz Napierala } 698*4814a0a4SEdward Tomasz Napierala 699*4814a0a4SEdward Tomasz Napierala ISER_DBG("before recv cv_wait: %p", iser_conn); 700*4814a0a4SEdward Tomasz Napierala cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock); 701*4814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->beacon.flush_lock); 702*4814a0a4SEdward Tomasz Napierala ISER_DBG("after recv cv_wait: %p", iser_conn); 703*4814a0a4SEdward Tomasz Napierala } 704*4814a0a4SEdward Tomasz Napierala out: 705*4814a0a4SEdward Tomasz Napierala sx_xunlock(&ig.close_conns_mutex); 706*4814a0a4SEdward Tomasz Napierala return (1); 707*4814a0a4SEdward Tomasz Napierala } 708*4814a0a4SEdward Tomasz Napierala 709*4814a0a4SEdward Tomasz Napierala /** 710*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 711*4814a0a4SEdward Tomasz Napierala **/ 712*4814a0a4SEdward Tomasz Napierala static void 713*4814a0a4SEdward Tomasz Napierala iser_connect_error(struct rdma_cm_id *cma_id) 714*4814a0a4SEdward Tomasz Napierala { 715*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 716*4814a0a4SEdward Tomasz Napierala 717*4814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 718*4814a0a4SEdward Tomasz Napierala 719*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p", iser_conn); 720*4814a0a4SEdward Tomasz Napierala 721*4814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_TERMINATING; 722*4814a0a4SEdward Tomasz Napierala 723*4814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 724*4814a0a4SEdward Tomasz Napierala } 725*4814a0a4SEdward Tomasz Napierala 726*4814a0a4SEdward Tomasz Napierala /** 727*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 728*4814a0a4SEdward Tomasz Napierala **/ 729*4814a0a4SEdward Tomasz Napierala static void 730*4814a0a4SEdward Tomasz Napierala iser_addr_handler(struct rdma_cm_id *cma_id) 731*4814a0a4SEdward Tomasz Napierala { 732*4814a0a4SEdward Tomasz Napierala struct iser_device *device; 733*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 734*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn; 735*4814a0a4SEdward Tomasz Napierala int ret; 736*4814a0a4SEdward Tomasz Napierala 737*4814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 738*4814a0a4SEdward Tomasz Napierala 739*4814a0a4SEdward Tomasz Napierala ib_conn = &iser_conn->ib_conn; 740*4814a0a4SEdward Tomasz Napierala device = iser_device_find_by_ib_device(cma_id); 741*4814a0a4SEdward Tomasz Napierala if (!device) { 742*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p device lookup/creation failed", 743*4814a0a4SEdward Tomasz Napierala iser_conn); 744*4814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 745*4814a0a4SEdward Tomasz Napierala return; 746*4814a0a4SEdward Tomasz Napierala } 747*4814a0a4SEdward Tomasz Napierala 748*4814a0a4SEdward Tomasz Napierala ib_conn->device = device; 749*4814a0a4SEdward Tomasz Napierala 750*4814a0a4SEdward Tomasz Napierala ret = rdma_resolve_route(cma_id, 1000); 751*4814a0a4SEdward Tomasz Napierala if (ret) { 752*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p resolve route failed: %d", iser_conn, ret); 753*4814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 754*4814a0a4SEdward Tomasz Napierala return; 755*4814a0a4SEdward Tomasz Napierala } 756*4814a0a4SEdward Tomasz Napierala } 757*4814a0a4SEdward Tomasz Napierala 758*4814a0a4SEdward Tomasz Napierala /** 759*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 760*4814a0a4SEdward Tomasz Napierala **/ 761*4814a0a4SEdward Tomasz Napierala static void 762*4814a0a4SEdward Tomasz Napierala iser_route_handler(struct rdma_cm_id *cma_id) 763*4814a0a4SEdward Tomasz Napierala { 764*4814a0a4SEdward Tomasz Napierala struct rdma_conn_param conn_param; 765*4814a0a4SEdward Tomasz Napierala int ret; 766*4814a0a4SEdward Tomasz Napierala struct iser_cm_hdr req_hdr; 767*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context; 768*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 769*4814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device; 770*4814a0a4SEdward Tomasz Napierala 771*4814a0a4SEdward Tomasz Napierala ret = iser_create_ib_conn_res(ib_conn); 772*4814a0a4SEdward Tomasz Napierala if (ret) 773*4814a0a4SEdward Tomasz Napierala goto failure; 774*4814a0a4SEdward Tomasz Napierala 775*4814a0a4SEdward Tomasz Napierala memset(&conn_param, 0, sizeof conn_param); 776*4814a0a4SEdward Tomasz Napierala conn_param.responder_resources = device->dev_attr.max_qp_rd_atom; 777*4814a0a4SEdward Tomasz Napierala conn_param.retry_count = 7; 778*4814a0a4SEdward Tomasz Napierala conn_param.rnr_retry_count = 6; 779*4814a0a4SEdward Tomasz Napierala /* 780*4814a0a4SEdward Tomasz Napierala * Initiaotr depth should not be set, but in order to compat 781*4814a0a4SEdward Tomasz Napierala * with old targets, we keep this value set. 782*4814a0a4SEdward Tomasz Napierala */ 783*4814a0a4SEdward Tomasz Napierala conn_param.initiator_depth = 1; 784*4814a0a4SEdward Tomasz Napierala 785*4814a0a4SEdward Tomasz Napierala memset(&req_hdr, 0, sizeof(req_hdr)); 786*4814a0a4SEdward Tomasz Napierala req_hdr.flags = (ISER_ZBVA_NOT_SUPPORTED | 787*4814a0a4SEdward Tomasz Napierala ISER_SEND_W_INV_NOT_SUPPORTED); 788*4814a0a4SEdward Tomasz Napierala conn_param.private_data = (void *)&req_hdr; 789*4814a0a4SEdward Tomasz Napierala conn_param.private_data_len = sizeof(struct iser_cm_hdr); 790*4814a0a4SEdward Tomasz Napierala 791*4814a0a4SEdward Tomasz Napierala ret = rdma_connect(cma_id, &conn_param); 792*4814a0a4SEdward Tomasz Napierala if (ret) { 793*4814a0a4SEdward Tomasz Napierala ISER_ERR("conn %p failure connecting: %d", iser_conn, ret); 794*4814a0a4SEdward Tomasz Napierala goto failure; 795*4814a0a4SEdward Tomasz Napierala } 796*4814a0a4SEdward Tomasz Napierala 797*4814a0a4SEdward Tomasz Napierala return; 798*4814a0a4SEdward Tomasz Napierala failure: 799*4814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 800*4814a0a4SEdward Tomasz Napierala } 801*4814a0a4SEdward Tomasz Napierala 802*4814a0a4SEdward Tomasz Napierala /** 803*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 804*4814a0a4SEdward Tomasz Napierala **/ 805*4814a0a4SEdward Tomasz Napierala static void 806*4814a0a4SEdward Tomasz Napierala iser_connected_handler(struct rdma_cm_id *cma_id) 807*4814a0a4SEdward Tomasz Napierala { 808*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 809*4814a0a4SEdward Tomasz Napierala struct ib_qp_attr attr; 810*4814a0a4SEdward Tomasz Napierala struct ib_qp_init_attr init_attr; 811*4814a0a4SEdward Tomasz Napierala 812*4814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 813*4814a0a4SEdward Tomasz Napierala 814*4814a0a4SEdward Tomasz Napierala (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); 815*4814a0a4SEdward Tomasz Napierala 816*4814a0a4SEdward Tomasz Napierala ISER_INFO("remote qpn:%x my qpn:%x", 817*4814a0a4SEdward Tomasz Napierala attr.dest_qp_num, cma_id->qp->qp_num); 818*4814a0a4SEdward Tomasz Napierala 819*4814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_UP; 820*4814a0a4SEdward Tomasz Napierala 821*4814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 822*4814a0a4SEdward Tomasz Napierala } 823*4814a0a4SEdward Tomasz Napierala 824*4814a0a4SEdward Tomasz Napierala /** 825*4814a0a4SEdward Tomasz Napierala * Called with state mutex held 826*4814a0a4SEdward Tomasz Napierala **/ 827*4814a0a4SEdward Tomasz Napierala static void 828*4814a0a4SEdward Tomasz Napierala iser_cleanup_handler(struct rdma_cm_id *cma_id, bool destroy) 829*4814a0a4SEdward Tomasz Napierala { 830*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = cma_id->context; 831*4814a0a4SEdward Tomasz Napierala 832*4814a0a4SEdward Tomasz Napierala if (iser_conn_terminate(iser_conn)) 833*4814a0a4SEdward Tomasz Napierala iser_conn->icl_conn.ic_error(&iser_conn->icl_conn); 834*4814a0a4SEdward Tomasz Napierala 835*4814a0a4SEdward Tomasz Napierala } 836*4814a0a4SEdward Tomasz Napierala 837*4814a0a4SEdward Tomasz Napierala int 838*4814a0a4SEdward Tomasz Napierala iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 839*4814a0a4SEdward Tomasz Napierala { 840*4814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 841*4814a0a4SEdward Tomasz Napierala int ret = 0; 842*4814a0a4SEdward Tomasz Napierala 843*4814a0a4SEdward Tomasz Napierala iser_conn = cma_id->context; 844*4814a0a4SEdward Tomasz Napierala ISER_INFO("event %d status %d conn %p id %p", 845*4814a0a4SEdward Tomasz Napierala event->event, event->status, cma_id->context, cma_id); 846*4814a0a4SEdward Tomasz Napierala 847*4814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 848*4814a0a4SEdward Tomasz Napierala switch (event->event) { 849*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_RESOLVED: 850*4814a0a4SEdward Tomasz Napierala iser_addr_handler(cma_id); 851*4814a0a4SEdward Tomasz Napierala break; 852*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_RESOLVED: 853*4814a0a4SEdward Tomasz Napierala iser_route_handler(cma_id); 854*4814a0a4SEdward Tomasz Napierala break; 855*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ESTABLISHED: 856*4814a0a4SEdward Tomasz Napierala iser_connected_handler(cma_id); 857*4814a0a4SEdward Tomasz Napierala break; 858*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_ERROR: 859*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ROUTE_ERROR: 860*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_CONNECT_ERROR: 861*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_UNREACHABLE: 862*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_REJECTED: 863*4814a0a4SEdward Tomasz Napierala iser_connect_error(cma_id); 864*4814a0a4SEdward Tomasz Napierala break; 865*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_DISCONNECTED: 866*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_ADDR_CHANGE: 867*4814a0a4SEdward Tomasz Napierala case RDMA_CM_EVENT_TIMEWAIT_EXIT: 868*4814a0a4SEdward Tomasz Napierala iser_cleanup_handler(cma_id, false); 869*4814a0a4SEdward Tomasz Napierala break; 870*4814a0a4SEdward Tomasz Napierala default: 871*4814a0a4SEdward Tomasz Napierala ISER_ERR("Unexpected RDMA CM event (%d)", event->event); 872*4814a0a4SEdward Tomasz Napierala break; 873*4814a0a4SEdward Tomasz Napierala } 874*4814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 875*4814a0a4SEdward Tomasz Napierala 876*4814a0a4SEdward Tomasz Napierala return (ret); 877*4814a0a4SEdward Tomasz Napierala } 878*4814a0a4SEdward Tomasz Napierala 879*4814a0a4SEdward Tomasz Napierala int 880*4814a0a4SEdward Tomasz Napierala iser_post_recvl(struct iser_conn *iser_conn) 881*4814a0a4SEdward Tomasz Napierala { 882*4814a0a4SEdward Tomasz Napierala struct ib_recv_wr rx_wr, *rx_wr_failed; 883*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 884*4814a0a4SEdward Tomasz Napierala struct ib_sge sge; 885*4814a0a4SEdward Tomasz Napierala int ib_ret; 886*4814a0a4SEdward Tomasz Napierala 887*4814a0a4SEdward Tomasz Napierala sge.addr = iser_conn->login_resp_dma; 888*4814a0a4SEdward Tomasz Napierala sge.length = ISER_RX_LOGIN_SIZE; 889*4814a0a4SEdward Tomasz Napierala sge.lkey = ib_conn->device->mr->lkey; 890*4814a0a4SEdward Tomasz Napierala 891*4814a0a4SEdward Tomasz Napierala rx_wr.wr_id = (uintptr_t)iser_conn->login_resp_buf; 892*4814a0a4SEdward Tomasz Napierala rx_wr.sg_list = &sge; 893*4814a0a4SEdward Tomasz Napierala rx_wr.num_sge = 1; 894*4814a0a4SEdward Tomasz Napierala rx_wr.next = NULL; 895*4814a0a4SEdward Tomasz Napierala 896*4814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count++; 897*4814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed); 898*4814a0a4SEdward Tomasz Napierala if (ib_ret) { 899*4814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret); 900*4814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count--; 901*4814a0a4SEdward Tomasz Napierala } 902*4814a0a4SEdward Tomasz Napierala 903*4814a0a4SEdward Tomasz Napierala return (ib_ret); 904*4814a0a4SEdward Tomasz Napierala } 905*4814a0a4SEdward Tomasz Napierala 906*4814a0a4SEdward Tomasz Napierala int 907*4814a0a4SEdward Tomasz Napierala iser_post_recvm(struct iser_conn *iser_conn, int count) 908*4814a0a4SEdward Tomasz Napierala { 909*4814a0a4SEdward Tomasz Napierala struct ib_recv_wr *rx_wr, *rx_wr_failed; 910*4814a0a4SEdward Tomasz Napierala int i, ib_ret; 911*4814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 912*4814a0a4SEdward Tomasz Napierala unsigned int my_rx_head = iser_conn->rx_desc_head; 913*4814a0a4SEdward Tomasz Napierala struct iser_rx_desc *rx_desc; 914*4814a0a4SEdward Tomasz Napierala 915*4814a0a4SEdward Tomasz Napierala for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 916*4814a0a4SEdward Tomasz Napierala rx_desc = &iser_conn->rx_descs[my_rx_head]; 917*4814a0a4SEdward Tomasz Napierala rx_wr->wr_id = (uintptr_t)rx_desc; 918*4814a0a4SEdward Tomasz Napierala rx_wr->sg_list = &rx_desc->rx_sg; 919*4814a0a4SEdward Tomasz Napierala rx_wr->num_sge = 1; 920*4814a0a4SEdward Tomasz Napierala rx_wr->next = rx_wr + 1; 921*4814a0a4SEdward Tomasz Napierala my_rx_head = (my_rx_head + 1) % iser_conn->qp_max_recv_dtos; 922*4814a0a4SEdward Tomasz Napierala } 923*4814a0a4SEdward Tomasz Napierala 924*4814a0a4SEdward Tomasz Napierala rx_wr--; 925*4814a0a4SEdward Tomasz Napierala rx_wr->next = NULL; /* mark end of work requests list */ 926*4814a0a4SEdward Tomasz Napierala 927*4814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count += count; 928*4814a0a4SEdward Tomasz Napierala ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed); 929*4814a0a4SEdward Tomasz Napierala if (ib_ret) { 930*4814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_recv failed ret=%d", ib_ret); 931*4814a0a4SEdward Tomasz Napierala ib_conn->post_recv_buf_count -= count; 932*4814a0a4SEdward Tomasz Napierala } else 933*4814a0a4SEdward Tomasz Napierala iser_conn->rx_desc_head = my_rx_head; 934*4814a0a4SEdward Tomasz Napierala 935*4814a0a4SEdward Tomasz Napierala return (ib_ret); 936*4814a0a4SEdward Tomasz Napierala } 937*4814a0a4SEdward Tomasz Napierala 938*4814a0a4SEdward Tomasz Napierala /** 939*4814a0a4SEdward Tomasz Napierala * iser_start_send - Initiate a Send DTO operation 940*4814a0a4SEdward Tomasz Napierala * 941*4814a0a4SEdward Tomasz Napierala * returns 0 on success, -1 on failure 942*4814a0a4SEdward Tomasz Napierala */ 943*4814a0a4SEdward Tomasz Napierala int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, 944*4814a0a4SEdward Tomasz Napierala bool signal) 945*4814a0a4SEdward Tomasz Napierala { 946*4814a0a4SEdward Tomasz Napierala int ib_ret; 947*4814a0a4SEdward Tomasz Napierala struct ib_send_wr send_wr, *send_wr_failed; 948*4814a0a4SEdward Tomasz Napierala 949*4814a0a4SEdward Tomasz Napierala ib_dma_sync_single_for_device(ib_conn->device->ib_device, 950*4814a0a4SEdward Tomasz Napierala tx_desc->dma_addr, ISER_HEADERS_LEN, 951*4814a0a4SEdward Tomasz Napierala DMA_TO_DEVICE); 952*4814a0a4SEdward Tomasz Napierala 953*4814a0a4SEdward Tomasz Napierala send_wr.next = NULL; 954*4814a0a4SEdward Tomasz Napierala send_wr.wr_id = (uintptr_t)tx_desc; 955*4814a0a4SEdward Tomasz Napierala send_wr.sg_list = tx_desc->tx_sg; 956*4814a0a4SEdward Tomasz Napierala send_wr.num_sge = tx_desc->num_sge; 957*4814a0a4SEdward Tomasz Napierala send_wr.opcode = IB_WR_SEND; 958*4814a0a4SEdward Tomasz Napierala send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0; 959*4814a0a4SEdward Tomasz Napierala 960*4814a0a4SEdward Tomasz Napierala ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed); 961*4814a0a4SEdward Tomasz Napierala if (ib_ret) 962*4814a0a4SEdward Tomasz Napierala ISER_ERR("ib_post_send failed, ret:%d", ib_ret); 963*4814a0a4SEdward Tomasz Napierala 964*4814a0a4SEdward Tomasz Napierala return (ib_ret); 965*4814a0a4SEdward Tomasz Napierala } 966