66 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt);
67 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt);
68 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
70 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
71 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
72 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
73 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
74 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
89 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_drain()  argument
91 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_drain()
130 void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc)  in rpcrdma_flush_disconnect()  argument
133 		rpcrdma_force_disconnect(r_xprt->rx_ep);  in rpcrdma_flush_disconnect()
147 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_send()  local
151 	rpcrdma_sendctx_put_locked(r_xprt, sc);  in rpcrdma_wc_send()
152 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_send()
166 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_receive()  local
170 	--r_xprt->rx_ep->re_receive_count;  in rpcrdma_wc_receive()
187 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_receive()
188 	rpcrdma_rep_put(&r_xprt->rx_buf, rep);  in rpcrdma_wc_receive()
288 static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt,  in rpcrdma_create_id()  argument
292 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_create_id()
378 static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_ep_create()  argument
389 	ep->re_xprt = &r_xprt->rx_xprt;  in rpcrdma_ep_create()
392 	id = rpcrdma_create_id(r_xprt, ep);  in rpcrdma_ep_create()
402 	ep->re_max_requests = r_xprt->rx_xprt.max_reqs;  in rpcrdma_ep_create()
409 	r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests);  in rpcrdma_ep_create()
421 	ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
430 	ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
482 	r_xprt->rx_ep = ep;  in rpcrdma_ep_create()
497 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_connect()  argument
499 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_xprt_connect()
503 	rc = rpcrdma_ep_create(r_xprt);  in rpcrdma_xprt_connect()
506 	ep = r_xprt->rx_ep;  in rpcrdma_xprt_connect()
509 	rpcrdma_reset_cwnd(r_xprt);  in rpcrdma_xprt_connect()
515 	rpcrdma_post_recvs(r_xprt, 1);  in rpcrdma_xprt_connect()
530 	rc = rpcrdma_sendctxs_create(r_xprt);  in rpcrdma_xprt_connect()
536 	rc = rpcrdma_reqs_setup(r_xprt);  in rpcrdma_xprt_connect()
541 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_xprt_connect()
542 	frwr_wp_create(r_xprt);  in rpcrdma_xprt_connect()
545 	trace_xprtrdma_connect(r_xprt, rc);  in rpcrdma_xprt_connect()
559 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_disconnect()  argument
561 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_disconnect()
570 	trace_xprtrdma_disconnect(r_xprt, rc);  in rpcrdma_xprt_disconnect()
572 	rpcrdma_xprt_drain(r_xprt);  in rpcrdma_xprt_disconnect()
573 	rpcrdma_reps_unmap(r_xprt);  in rpcrdma_xprt_disconnect()
574 	rpcrdma_reqs_reset(r_xprt);  in rpcrdma_xprt_disconnect()
575 	rpcrdma_mrs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
576 	rpcrdma_sendctxs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
581 	r_xprt->rx_ep = NULL;  in rpcrdma_xprt_disconnect()
601 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_destroy()  argument
603 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_destroy()
630 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_create()  argument
632 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_create()
641 	i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS;  in rpcrdma_sendctxs_create()
648 		sc = rpcrdma_sendctx_create(r_xprt->rx_ep);  in rpcrdma_sendctxs_create()
683 struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctx_get_locked()  argument
685 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_get_locked()
709 	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_get_locked()
710 	r_xprt->rx_stats.empty_sendctx_q++;  in rpcrdma_sendctx_get_locked()
724 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,  in rpcrdma_sendctx_put_locked()  argument
727 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_put_locked()
745 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_put_locked()
749 rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_create()  argument
751 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_create()
752 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_create()
766 		rc = frwr_mr_init(r_xprt, mr);  in rpcrdma_mrs_create()
778 	r_xprt->rx_stats.mrs_allocated += count;  in rpcrdma_mrs_create()
779 	trace_xprtrdma_createmrs(r_xprt, count);  in rpcrdma_mrs_create()
787 	struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,  in rpcrdma_mr_refresh_worker()  local
790 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_mr_refresh_worker()
791 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_mr_refresh_worker()
799 void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_refresh()  argument
801 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_refresh()
802 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_refresh()
819 struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,  in rpcrdma_req_create()  argument
822 	struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;  in rpcrdma_req_create()
859 int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_req_setup()  argument
866 		     r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz;  in rpcrdma_req_setup()
873 	if (!__rpcrdma_regbuf_dma_map(r_xprt, rb))  in rpcrdma_req_setup()
891 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_setup()  argument
893 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_setup()
898 		rc = rpcrdma_req_setup(r_xprt, req);  in rpcrdma_reqs_setup()
938 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_reset()  argument
940 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_reset()
948 struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_rep_create()  argument
950 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_rep_create()
951 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_rep_create()
966 		atomic_inc_return(&r_xprt->rx_ep->re_completion_ids);  in rpcrdma_rep_create()
971 	rep->rr_rxprt = r_xprt;  in rpcrdma_rep_create()
1020 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reps_unmap()  argument
1022 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reps_unmap()
1053 int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_buffer_create()  argument
1055 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_buffer_create()
1069 	for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) {  in rpcrdma_buffer_create()
1072 		req = rpcrdma_req_create(r_xprt,  in rpcrdma_buffer_create()
1123 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_destroy()  argument
1125 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_destroy()
1176 rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mr_get()  argument
1178 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mr_get()
1301 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,  in __rpcrdma_regbuf_dma_map()  argument
1304 	struct ib_device *device = r_xprt->rx_ep->re_id->device;  in __rpcrdma_regbuf_dma_map()
1317 	rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey;  in __rpcrdma_regbuf_dma_map()
1348 void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed)  in rpcrdma_post_recvs()  argument
1350 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_post_recvs()
1351 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_recvs()
1372 			rep = rpcrdma_rep_create(r_xprt);  in rpcrdma_post_recvs()
1375 		if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) {  in rpcrdma_post_recvs()
1393 		trace_xprtrdma_post_recvs_err(r_xprt, rc);  in rpcrdma_post_recvs()
1407 	trace_xprtrdma_post_recvs(r_xprt, count);  in rpcrdma_post_recvs()