Lines Matching refs:ch
108 static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
171 static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new) in srpt_set_ch_state() argument
177 spin_lock_irqsave(&ch->spinlock, flags); in srpt_set_ch_state()
178 prev = ch->state; in srpt_set_ch_state()
180 ch->state = new; in srpt_set_ch_state()
183 spin_unlock_irqrestore(&ch->spinlock, flags); in srpt_set_ch_state()
280 struct srpt_rdma_ch *ch = ptr; in srpt_qp_event() local
283 event->event, ch, ch->sess_name, ch->qp->qp_num, in srpt_qp_event()
284 get_ch_state_name(ch->state)); in srpt_qp_event()
288 if (ch->using_rdma_cm) in srpt_qp_event()
289 rdma_notify(ch->rdma_cm.cm_id, event->event); in srpt_qp_event()
291 ib_cm_notify(ch->ib_cm.cm_id, event->event); in srpt_qp_event()
295 ch->sess_name, ch->qp->qp_num, in srpt_qp_event()
296 get_ch_state_name(ch->state)); in srpt_qp_event()
887 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_rdma_ch *ch, in srpt_post_recv() argument
907 return ib_post_recv(ch->qp, &wr, NULL); in srpt_post_recv()
919 static int srpt_zerolength_write(struct srpt_rdma_ch *ch) in srpt_zerolength_write() argument
924 { .wr_cqe = &ch->zw_cqe, }, in srpt_zerolength_write()
930 pr_debug("%s-%d: queued zerolength write\n", ch->sess_name, in srpt_zerolength_write()
931 ch->qp->qp_num); in srpt_zerolength_write()
933 return ib_post_send(ch->qp, &wr.wr, NULL); in srpt_zerolength_write()
938 struct srpt_rdma_ch *ch = wc->qp->qp_context; in srpt_zerolength_write_done() local
940 pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num, in srpt_zerolength_write_done()
944 srpt_process_wait_list(ch); in srpt_zerolength_write_done()
946 if (srpt_set_ch_state(ch, CH_DISCONNECTED)) in srpt_zerolength_write_done()
947 schedule_work(&ch->release_work); in srpt_zerolength_write_done()
950 ch->sess_name, ch->qp->qp_num); in srpt_zerolength_write_done()
959 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_alloc_rw_ctxs() local
984 ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port, in srpt_alloc_rw_ctxs()
1013 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port, in srpt_alloc_rw_ctxs()
1022 static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch, in srpt_free_rw_ctxs() argument
1031 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port, in srpt_free_rw_ctxs()
1176 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_init_ch_qp() argument
1181 WARN_ON_ONCE(ch->using_rdma_cm); in srpt_init_ch_qp()
1189 attr->port_num = ch->sport->port; in srpt_init_ch_qp()
1191 ret = ib_find_cached_pkey(ch->sport->sdev->device, ch->sport->port, in srpt_init_ch_qp()
1192 ch->pkey, &attr->pkey_index); in srpt_init_ch_qp()
1195 ch->pkey, ret); in srpt_init_ch_qp()
1216 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_ch_qp_rtr() argument
1222 WARN_ON_ONCE(ch->using_rdma_cm); in srpt_ch_qp_rtr()
1225 ret = ib_cm_init_qp_attr(ch->ib_cm.cm_id, &qp_attr, &attr_mask); in srpt_ch_qp_rtr()
1248 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp) in srpt_ch_qp_rts() argument
1255 ret = ib_cm_init_qp_attr(ch->ib_cm.cm_id, &qp_attr, &attr_mask); in srpt_ch_qp_rts()
1271 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch) in srpt_ch_qp_err() argument
1276 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE); in srpt_ch_qp_err()
1283 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) in srpt_get_send_ioctx() argument
1288 BUG_ON(!ch); in srpt_get_send_ioctx()
1290 tag = sbitmap_queue_get(&ch->sess->sess_tag_pool, &cpu); in srpt_get_send_ioctx()
1294 ioctx = ch->ioctx_ring[tag]; in srpt_get_send_ioctx()
1295 BUG_ON(ioctx->ch != ch); in srpt_get_send_ioctx()
1391 struct srpt_rdma_ch *ch = wc->qp->qp_context; in srpt_rdma_read_done() local
1396 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_rdma_read_done()
1429 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch, in srpt_build_cmd_rsp() argument
1455 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); in srpt_build_cmd_rsp()
1483 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp); in srpt_build_cmd_rsp()
1511 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch, in srpt_build_tskmgmt_rsp() argument
1528 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); in srpt_build_tskmgmt_rsp()
1552 static void srpt_handle_cmd(struct srpt_rdma_ch *ch, in srpt_handle_cmd() argument
1587 &sg, &sg_cnt, &data_len, ch->imm_data_offset); in srpt_handle_cmd()
1596 rc = target_init_cmd(cmd, ch->sess, &send_ioctx->sense_data[0], in srpt_handle_cmd()
1645 static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch, in srpt_handle_tsk_mgmt() argument
1651 struct se_session *sess = ch->sess; in srpt_handle_tsk_mgmt()
1661 srp_tsk->tsk_mgmt_func, srp_tsk->task_tag, srp_tsk->tag, ch, in srpt_handle_tsk_mgmt()
1662 ch->sess); in srpt_handle_tsk_mgmt()
1684 srpt_handle_new_iu(struct srpt_rdma_ch *ch, struct srpt_recv_ioctx *recv_ioctx) in srpt_handle_new_iu() argument
1691 BUG_ON(!ch); in srpt_handle_new_iu()
1694 if (unlikely(ch->state == CH_CONNECTING)) in srpt_handle_new_iu()
1697 ib_dma_sync_single_for_cpu(ch->sport->sdev->device, in srpt_handle_new_iu()
1705 send_ioctx = srpt_get_send_ioctx(ch); in srpt_handle_new_iu()
1711 WARN_ON_ONCE(!ch->processing_wait_list); in srpt_handle_new_iu()
1717 srpt_handle_cmd(ch, recv_ioctx, send_ioctx); in srpt_handle_new_iu()
1720 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx); in srpt_handle_new_iu()
1740 srpt_post_recv(ch->sport->sdev, ch, recv_ioctx); in srpt_handle_new_iu()
1748 WARN_ON_ONCE(ch->processing_wait_list); in srpt_handle_new_iu()
1749 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list); in srpt_handle_new_iu()
1756 struct srpt_rdma_ch *ch = wc->qp->qp_context; in srpt_recv_done() local
1763 req_lim = atomic_dec_return(&ch->req_lim); in srpt_recv_done()
1767 srpt_handle_new_iu(ch, ioctx); in srpt_recv_done()
1779 static void srpt_process_wait_list(struct srpt_rdma_ch *ch) in srpt_process_wait_list() argument
1783 WARN_ON_ONCE(ch->state == CH_CONNECTING); in srpt_process_wait_list()
1785 if (list_empty(&ch->cmd_wait_list)) in srpt_process_wait_list()
1788 WARN_ON_ONCE(ch->processing_wait_list); in srpt_process_wait_list()
1789 ch->processing_wait_list = true; in srpt_process_wait_list()
1790 list_for_each_entry_safe(recv_ioctx, tmp, &ch->cmd_wait_list, in srpt_process_wait_list()
1792 if (!srpt_handle_new_iu(ch, recv_ioctx)) in srpt_process_wait_list()
1795 ch->processing_wait_list = false; in srpt_process_wait_list()
1817 struct srpt_rdma_ch *ch = wc->qp->qp_context; in srpt_send_done() local
1827 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail); in srpt_send_done()
1840 srpt_process_wait_list(ch); in srpt_send_done()
1847 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) in srpt_create_ch_ib() argument
1850 struct srpt_port *sport = ch->sport; in srpt_create_ch_ib()
1856 WARN_ON(ch->rq_size < 1); in srpt_create_ch_ib()
1864 ch->cq = ib_cq_pool_get(sdev->device, ch->rq_size + sq_size, -1, in srpt_create_ch_ib()
1866 if (IS_ERR(ch->cq)) { in srpt_create_ch_ib()
1867 ret = PTR_ERR(ch->cq); in srpt_create_ch_ib()
1869 ch->rq_size + sq_size, ret); in srpt_create_ch_ib()
1872 ch->cq_size = ch->rq_size + sq_size; in srpt_create_ch_ib()
1874 qp_init->qp_context = (void *)ch; in srpt_create_ch_ib()
1876 qp_init->send_cq = ch->cq; in srpt_create_ch_ib()
1877 qp_init->recv_cq = ch->cq; in srpt_create_ch_ib()
1891 qp_init->port_num = ch->sport->port; in srpt_create_ch_ib()
1895 qp_init->cap.max_recv_wr = ch->rq_size; in srpt_create_ch_ib()
1897 if (ch->using_rdma_cm) { in srpt_create_ch_ib()
1898 ret = rdma_create_qp(ch->rdma_cm.cm_id, sdev->pd, qp_init); in srpt_create_ch_ib()
1899 ch->qp = ch->rdma_cm.cm_id->qp; in srpt_create_ch_ib()
1901 ch->qp = ib_create_qp(sdev->pd, qp_init); in srpt_create_ch_ib()
1902 if (!IS_ERR(ch->qp)) { in srpt_create_ch_ib()
1903 ret = srpt_init_ch_qp(ch, ch->qp); in srpt_create_ch_ib()
1905 ib_destroy_qp(ch->qp); in srpt_create_ch_ib()
1907 ret = PTR_ERR(ch->qp); in srpt_create_ch_ib()
1916 ib_cq_pool_put(ch->cq, ch->cq_size); in srpt_create_ch_ib()
1926 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr); in srpt_create_ch_ib()
1929 __func__, ch->cq->cqe, qp_init->cap.max_send_sge, in srpt_create_ch_ib()
1930 qp_init->cap.max_send_wr, ch); in srpt_create_ch_ib()
1933 for (i = 0; i < ch->rq_size; i++) in srpt_create_ch_ib()
1934 srpt_post_recv(sdev, ch, ch->ioctx_recv_ring[i]); in srpt_create_ch_ib()
1941 ch->qp = NULL; in srpt_create_ch_ib()
1942 ib_cq_pool_put(ch->cq, ch->cq_size); in srpt_create_ch_ib()
1946 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch) in srpt_destroy_ch_ib() argument
1948 ib_destroy_qp(ch->qp); in srpt_destroy_ch_ib()
1949 ib_cq_pool_put(ch->cq, ch->cq_size); in srpt_destroy_ch_ib()
1962 static bool srpt_close_ch(struct srpt_rdma_ch *ch) in srpt_close_ch() argument
1966 if (!srpt_set_ch_state(ch, CH_DRAINING)) { in srpt_close_ch()
1967 pr_debug("%s: already closed\n", ch->sess_name); in srpt_close_ch()
1971 kref_get(&ch->kref); in srpt_close_ch()
1973 ret = srpt_ch_qp_err(ch); in srpt_close_ch()
1976 ch->sess_name, ch->qp->qp_num, ret); in srpt_close_ch()
1978 ret = srpt_zerolength_write(ch); in srpt_close_ch()
1981 ch->sess_name, ch->qp->qp_num, ret); in srpt_close_ch()
1982 if (srpt_set_ch_state(ch, CH_DISCONNECTED)) in srpt_close_ch()
1983 schedule_work(&ch->release_work); in srpt_close_ch()
1988 kref_put(&ch->kref, srpt_free_ch); in srpt_close_ch()
2002 static int srpt_disconnect_ch(struct srpt_rdma_ch *ch) in srpt_disconnect_ch() argument
2006 if (!srpt_set_ch_state(ch, CH_DISCONNECTING)) in srpt_disconnect_ch()
2009 if (ch->using_rdma_cm) { in srpt_disconnect_ch()
2010 ret = rdma_disconnect(ch->rdma_cm.cm_id); in srpt_disconnect_ch()
2012 ret = ib_send_cm_dreq(ch->ib_cm.cm_id, NULL, 0); in srpt_disconnect_ch()
2014 ret = ib_send_cm_drep(ch->ib_cm.cm_id, NULL, 0); in srpt_disconnect_ch()
2017 if (ret < 0 && srpt_close_ch(ch)) in srpt_disconnect_ch()
2024 static void srpt_disconnect_ch_sync(struct srpt_rdma_ch *ch) in srpt_disconnect_ch_sync() argument
2027 struct srpt_port *sport = ch->sport; in srpt_disconnect_ch_sync()
2029 pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num, in srpt_disconnect_ch_sync()
2030 ch->state); in srpt_disconnect_ch_sync()
2032 ch->closed = &closed; in srpt_disconnect_ch_sync()
2035 srpt_disconnect_ch(ch); in srpt_disconnect_ch_sync()
2040 ch->sess_name, ch->qp->qp_num, ch->state); in srpt_disconnect_ch_sync()
2047 struct srpt_rdma_ch *ch; in __srpt_close_all_ch() local
2052 list_for_each_entry(ch, &nexus->ch_list, list) { in __srpt_close_all_ch()
2053 if (srpt_disconnect_ch(ch) >= 0) in __srpt_close_all_ch()
2055 ch->sess_name, ch->qp->qp_num, in __srpt_close_all_ch()
2058 srpt_close_ch(ch); in __srpt_close_all_ch()
2126 struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref); in srpt_free_ch() local
2128 srpt_drop_sport_ref(ch->sport); in srpt_free_ch()
2129 kfree_rcu(ch, rcu); in srpt_free_ch()
2142 struct srpt_rdma_ch *ch; in srpt_release_channel_work() local
2147 ch = container_of(w, struct srpt_rdma_ch, release_work); in srpt_release_channel_work()
2148 pr_debug("%s-%d\n", ch->sess_name, ch->qp->qp_num); in srpt_release_channel_work()
2150 sdev = ch->sport->sdev; in srpt_release_channel_work()
2153 se_sess = ch->sess; in srpt_release_channel_work()
2160 ch->sess = NULL; in srpt_release_channel_work()
2162 if (ch->using_rdma_cm) in srpt_release_channel_work()
2163 rdma_destroy_id(ch->rdma_cm.cm_id); in srpt_release_channel_work()
2165 ib_destroy_cm_id(ch->ib_cm.cm_id); in srpt_release_channel_work()
2167 sport = ch->sport; in srpt_release_channel_work()
2169 list_del_rcu(&ch->list); in srpt_release_channel_work()
2172 if (ch->closed) in srpt_release_channel_work()
2173 complete(ch->closed); in srpt_release_channel_work()
2175 srpt_destroy_ch_ib(ch); in srpt_release_channel_work()
2177 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, in srpt_release_channel_work()
2178 ch->sport->sdev, ch->rq_size, in srpt_release_channel_work()
2179 ch->rsp_buf_cache, DMA_TO_DEVICE); in srpt_release_channel_work()
2181 srpt_cache_put(ch->rsp_buf_cache); in srpt_release_channel_work()
2183 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring, in srpt_release_channel_work()
2184 sdev, ch->rq_size, in srpt_release_channel_work()
2185 ch->req_buf_cache, DMA_FROM_DEVICE); in srpt_release_channel_work()
2187 srpt_cache_put(ch->req_buf_cache); in srpt_release_channel_work()
2189 kref_put(&ch->kref, srpt_free_ch); in srpt_release_channel_work()
2221 struct srpt_rdma_ch *ch = NULL; in srpt_cm_req_recv() local
2275 ch = kzalloc(sizeof(*ch), GFP_KERNEL); in srpt_cm_req_recv()
2276 if (!ch) { in srpt_cm_req_recv()
2282 kref_init(&ch->kref); in srpt_cm_req_recv()
2283 ch->pkey = be16_to_cpu(pkey); in srpt_cm_req_recv()
2284 ch->nexus = nexus; in srpt_cm_req_recv()
2285 ch->zw_cqe.done = srpt_zerolength_write_done; in srpt_cm_req_recv()
2286 INIT_WORK(&ch->release_work, srpt_release_channel_work); in srpt_cm_req_recv()
2287 ch->sport = sport; in srpt_cm_req_recv()
2289 ch->using_rdma_cm = true; in srpt_cm_req_recv()
2290 ch->rdma_cm.cm_id = rdma_cm_id; in srpt_cm_req_recv()
2291 rdma_cm_id->context = ch; in srpt_cm_req_recv()
2293 ch->ib_cm.cm_id = ib_cm_id; in srpt_cm_req_recv()
2294 ib_cm_id->context = ch; in srpt_cm_req_recv()
2301 ch->rq_size = min(MAX_SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr); in srpt_cm_req_recv()
2302 spin_lock_init(&ch->spinlock); in srpt_cm_req_recv()
2303 ch->state = CH_CONNECTING; in srpt_cm_req_recv()
2304 INIT_LIST_HEAD(&ch->cmd_wait_list); in srpt_cm_req_recv()
2305 ch->max_rsp_size = ch->sport->port_attrib.srp_max_rsp_size; in srpt_cm_req_recv()
2307 ch->rsp_buf_cache = srpt_cache_get(ch->max_rsp_size); in srpt_cm_req_recv()
2308 if (!ch->rsp_buf_cache) in srpt_cm_req_recv()
2311 ch->ioctx_ring = (struct srpt_send_ioctx **) in srpt_cm_req_recv()
2312 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2313 sizeof(*ch->ioctx_ring[0]), in srpt_cm_req_recv()
2314 ch->rsp_buf_cache, 0, DMA_TO_DEVICE); in srpt_cm_req_recv()
2315 if (!ch->ioctx_ring) { in srpt_cm_req_recv()
2321 for (i = 0; i < ch->rq_size; i++) in srpt_cm_req_recv()
2322 ch->ioctx_ring[i]->ch = ch; in srpt_cm_req_recv()
2333 ch->imm_data_offset = imm_data_offset; in srpt_cm_req_recv()
2336 ch->imm_data_offset = 0; in srpt_cm_req_recv()
2341 ch->req_buf_cache = srpt_cache_get(req_sz); in srpt_cm_req_recv()
2342 if (!ch->req_buf_cache) in srpt_cm_req_recv()
2345 ch->ioctx_recv_ring = (struct srpt_recv_ioctx **) in srpt_cm_req_recv()
2346 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2347 sizeof(*ch->ioctx_recv_ring[0]), in srpt_cm_req_recv()
2348 ch->req_buf_cache, in srpt_cm_req_recv()
2351 if (!ch->ioctx_recv_ring) { in srpt_cm_req_recv()
2357 for (i = 0; i < ch->rq_size; i++) in srpt_cm_req_recv()
2358 INIT_LIST_HEAD(&ch->ioctx_recv_ring[i]->wait_list); in srpt_cm_req_recv()
2361 ret = srpt_create_ch_ib(ch); in srpt_cm_req_recv()
2368 strscpy(ch->sess_name, src_addr, sizeof(ch->sess_name)); in srpt_cm_req_recv()
2373 pr_debug("registering src addr %s or i_port_id %s\n", ch->sess_name, in srpt_cm_req_recv()
2376 tag_num = ch->rq_size; in srpt_cm_req_recv()
2382 if (!IS_ERR_OR_NULL(ch->sess)) in srpt_cm_req_recv()
2384 ch->sess = target_setup_session(&stpg->tpg, tag_num, in srpt_cm_req_recv()
2386 ch->sess_name, ch, NULL); in srpt_cm_req_recv()
2394 if (!IS_ERR_OR_NULL(ch->sess)) in srpt_cm_req_recv()
2396 ch->sess = target_setup_session(&stpg->tpg, tag_num, in srpt_cm_req_recv()
2398 ch, NULL); in srpt_cm_req_recv()
2399 if (!IS_ERR_OR_NULL(ch->sess)) in srpt_cm_req_recv()
2402 ch->sess = target_setup_session(&stpg->tpg, tag_num, in srpt_cm_req_recv()
2404 i_port_id + 2, ch, NULL); in srpt_cm_req_recv()
2409 if (IS_ERR_OR_NULL(ch->sess)) { in srpt_cm_req_recv()
2410 WARN_ON_ONCE(ch->sess == NULL); in srpt_cm_req_recv()
2411 ret = PTR_ERR(ch->sess); in srpt_cm_req_recv()
2412 ch->sess = NULL; in srpt_cm_req_recv()
2414 ch->sess_name, ret); in srpt_cm_req_recv()
2443 list_add_tail_rcu(&ch->list, &nexus->ch_list); in srpt_cm_req_recv()
2457 ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rtr(ch, ch->qp); in srpt_cm_req_recv()
2465 pr_debug("Establish connection sess=%p name=%s ch=%p\n", ch->sess, in srpt_cm_req_recv()
2466 ch->sess_name, ch); in srpt_cm_req_recv()
2473 ch->max_ti_iu_len = it_iu_len; in srpt_cm_req_recv()
2476 rsp->req_lim_delta = cpu_to_be32(ch->rq_size); in srpt_cm_req_recv()
2477 atomic_set(&ch->req_lim, ch->rq_size); in srpt_cm_req_recv()
2478 atomic_set(&ch->req_lim_delta, 0); in srpt_cm_req_recv()
2481 if (ch->using_rdma_cm) { in srpt_cm_req_recv()
2489 rep_param->ib_cm.qp_num = ch->qp->qp_num; in srpt_cm_req_recv()
2505 if (sport->enabled && ch->state == CH_CONNECTING) { in srpt_cm_req_recv()
2506 if (ch->using_rdma_cm) in srpt_cm_req_recv()
2530 srpt_destroy_ch_ib(ch); in srpt_cm_req_recv()
2533 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring, in srpt_cm_req_recv()
2534 ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2535 ch->req_buf_cache, DMA_FROM_DEVICE); in srpt_cm_req_recv()
2538 srpt_cache_put(ch->req_buf_cache); in srpt_cm_req_recv()
2541 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, in srpt_cm_req_recv()
2542 ch->sport->sdev, ch->rq_size, in srpt_cm_req_recv()
2543 ch->rsp_buf_cache, DMA_TO_DEVICE); in srpt_cm_req_recv()
2546 srpt_cache_put(ch->rsp_buf_cache); in srpt_cm_req_recv()
2553 kfree(ch); in srpt_cm_req_recv()
2554 ch = NULL; in srpt_cm_req_recv()
2572 if (ch && ch->sess) { in srpt_cm_req_recv()
2573 srpt_close_ch(ch); in srpt_cm_req_recv()
2638 static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch, in srpt_cm_rej_recv() argument
2652 ch->sess_name, ch->qp->qp_num, reason, private_data_len ? in srpt_cm_rej_recv()
2664 static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch) in srpt_cm_rtu_recv() argument
2668 ret = ch->using_rdma_cm ? 0 : srpt_ch_qp_rts(ch, ch->qp); in srpt_cm_rtu_recv()
2670 pr_err("%s-%d: QP transition to RTS failed\n", ch->sess_name, in srpt_cm_rtu_recv()
2671 ch->qp->qp_num); in srpt_cm_rtu_recv()
2672 srpt_close_ch(ch); in srpt_cm_rtu_recv()
2681 if (!srpt_set_ch_state(ch, CH_LIVE)) { in srpt_cm_rtu_recv()
2683 ch->sess_name, ch->qp->qp_num); in srpt_cm_rtu_recv()
2688 ret = srpt_zerolength_write(ch); in srpt_cm_rtu_recv()
2707 struct srpt_rdma_ch *ch = cm_id->context; in srpt_cm_handler() local
2717 srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason, in srpt_cm_handler()
2723 srpt_cm_rtu_recv(ch); in srpt_cm_handler()
2726 srpt_disconnect_ch(ch); in srpt_cm_handler()
2730 ch->sess_name, ch->qp->qp_num); in srpt_cm_handler()
2731 srpt_close_ch(ch); in srpt_cm_handler()
2735 ch->sess_name, ch->qp->qp_num); in srpt_cm_handler()
2736 srpt_close_ch(ch); in srpt_cm_handler()
2739 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name, in srpt_cm_handler()
2740 ch->qp->qp_num); in srpt_cm_handler()
2759 struct srpt_rdma_ch *ch = cm_id->context; in srpt_rdma_cm_handler() local
2767 srpt_cm_rej_recv(ch, event->status, in srpt_rdma_cm_handler()
2772 srpt_cm_rtu_recv(ch); in srpt_rdma_cm_handler()
2775 if (ch->state < CH_DISCONNECTING) in srpt_rdma_cm_handler()
2776 srpt_disconnect_ch(ch); in srpt_rdma_cm_handler()
2778 srpt_close_ch(ch); in srpt_rdma_cm_handler()
2781 srpt_close_ch(ch); in srpt_rdma_cm_handler()
2784 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name, in srpt_rdma_cm_handler()
2785 ch->qp->qp_num); in srpt_rdma_cm_handler()
2806 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_write_pending() local
2821 if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) { in srpt_write_pending()
2832 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port, in srpt_write_pending()
2837 ret = ib_post_send(ch->qp, first_wr, NULL); in srpt_write_pending()
2841 atomic_read(&ch->sq_wr_avail)); in srpt_write_pending()
2847 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); in srpt_write_pending()
2873 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_queue_response() local
2874 struct srpt_device *sdev = ch->sport->sdev; in srpt_queue_response()
2892 ch, ioctx->ioctx.index, ioctx->state); in srpt_queue_response()
2906 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, in srpt_queue_response()
2907 ch->sport->port, NULL, first_wr); in srpt_queue_response()
2912 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag, in srpt_queue_response()
2917 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status, in srpt_queue_response()
2921 atomic_inc(&ch->req_lim); in srpt_queue_response()
2924 &ch->sq_wr_avail) < 0)) { in srpt_queue_response()
2945 ret = ib_post_send(ch->qp, first_wr, NULL); in srpt_queue_response()
2955 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail); in srpt_queue_response()
2956 atomic_dec(&ch->req_lim); in srpt_queue_response()
2982 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_aborted_task() local
2984 atomic_inc(&ch->req_lim_delta); in srpt_aborted_task()
3016 struct srpt_rdma_ch *ch; in srpt_release_sport() local
3033 list_for_each_entry(ch, &nexus->ch_list, list) { in srpt_release_sport()
3035 ch->sess_name, ch->qp->qp_num, in srpt_release_sport()
3036 get_ch_state_name(ch->state)); in srpt_release_sport()
3399 struct srpt_rdma_ch *ch = ioctx->ch; in srpt_release_cmd() local
3408 srpt_post_recv(ch->sport->sdev, ch, recv_ioctx); in srpt_release_cmd()
3412 srpt_free_rw_ctxs(ch, ioctx); in srpt_release_cmd()
3429 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr; in srpt_close_session() local
3431 srpt_disconnect_ch_sync(ch); in srpt_close_session()