1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /*
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the BSD-type
10 * license below:
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 *
16 * Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 *
19 * Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials provided
22 * with the distribution.
23 *
24 * Neither the name of the Network Appliance, Inc. nor the names of
25 * its contributors may be used to endorse or promote products
26 * derived from this software without specific prior written
27 * permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 */
41
42 #ifndef _LINUX_SUNRPC_XPRT_RDMA_H
43 #define _LINUX_SUNRPC_XPRT_RDMA_H
44
45 #include <linux/wait.h> /* wait_queue_head_t, etc */
46 #include <linux/spinlock.h> /* spinlock_t, etc */
47 #include <linux/atomic.h> /* atomic_t, etc */
48 #include <linux/kref.h> /* struct kref */
49 #include <linux/workqueue.h> /* struct work_struct */
50 #include <linux/llist.h>
51
52 #include <rdma/rdma_cm.h> /* RDMA connection api */
53 #include <rdma/ib_verbs.h> /* RDMA verbs api */
54
55 #include <linux/sunrpc/clnt.h> /* rpc_xprt */
56 #include <linux/sunrpc/rpc_rdma_cid.h> /* completion IDs */
57 #include <linux/sunrpc/rpc_rdma.h> /* RPC/RDMA protocol */
58 #include <linux/sunrpc/xprtrdma.h> /* xprt parameters */
59 #include <linux/sunrpc/rdma_rn.h> /* removal notifications */
60
61 #define RDMA_RESOLVE_TIMEOUT (5000) /* 5 seconds */
62 #define RDMA_CONNECT_RETRY_MAX (2) /* retries if no listener backlog */
63
64 #define RPCRDMA_BIND_TO (60U * HZ)
65 #define RPCRDMA_INIT_REEST_TO (5U * HZ)
66 #define RPCRDMA_MAX_REEST_TO (30U * HZ)
67 #define RPCRDMA_IDLE_DISC_TO (5U * 60 * HZ)
68
69 /*
70 * RDMA Endpoint -- connection endpoint details
71 */
72 struct rpcrdma_mr;
73 struct rpcrdma_ep {
74 struct kref re_kref;
75 struct rdma_cm_id *re_id;
76 struct ib_pd *re_pd;
77 unsigned int re_max_rdma_segs;
78 unsigned int re_max_fr_depth;
79 struct rpcrdma_mr *re_write_pad_mr;
80 enum ib_mr_type re_mrtype;
81 struct completion re_done;
82 unsigned int re_send_count;
83 unsigned int re_send_batch;
84 unsigned int re_max_inline_send;
85 unsigned int re_max_inline_recv;
86 int re_async_rc;
87 int re_connect_status;
88 atomic_t re_receiving;
89 atomic_t re_force_disconnect;
90 struct ib_qp_init_attr re_attr;
91 wait_queue_head_t re_connect_wait;
92 struct rpc_xprt *re_xprt;
93 struct rpcrdma_connect_private
94 re_cm_private;
95 struct rdma_conn_param re_remote_cma;
96 struct rpcrdma_notification re_rn;
97 int re_receive_count;
98 unsigned int re_max_requests; /* depends on device */
99 unsigned int re_recv_batch;
100 unsigned int re_inline_send; /* negotiated */
101 unsigned int re_inline_recv; /* negotiated */
102
103 atomic_t re_completion_ids;
104
105 char re_write_pad[XDR_UNIT];
106 };
107
108 /* Pre-allocate extra Work Requests for handling reverse-direction
109 * Receives and Sends. This is a fixed value because the Work Queues
110 * are allocated when the forward channel is set up, long before the
111 * backchannel is provisioned. This value is two times
112 * NFS4_DEF_CB_SLOT_TABLE_SIZE.
113 */
114 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
115 #define RPCRDMA_BACKWARD_WRS (32)
116 #else
117 #define RPCRDMA_BACKWARD_WRS (0)
118 #endif
119
120 /* Registered buffer -- registered kmalloc'd memory for RDMA SEND/RECV
121 */
122
123 struct rpcrdma_regbuf {
124 struct ib_sge rg_iov;
125 struct ib_device *rg_device;
126 enum dma_data_direction rg_direction;
127 void *rg_data;
128 };
129
rdmab_addr(struct rpcrdma_regbuf * rb)130 static inline u64 rdmab_addr(struct rpcrdma_regbuf *rb)
131 {
132 return rb->rg_iov.addr;
133 }
134
rdmab_length(struct rpcrdma_regbuf * rb)135 static inline u32 rdmab_length(struct rpcrdma_regbuf *rb)
136 {
137 return rb->rg_iov.length;
138 }
139
rdmab_lkey(struct rpcrdma_regbuf * rb)140 static inline u32 rdmab_lkey(struct rpcrdma_regbuf *rb)
141 {
142 return rb->rg_iov.lkey;
143 }
144
rdmab_device(struct rpcrdma_regbuf * rb)145 static inline struct ib_device *rdmab_device(struct rpcrdma_regbuf *rb)
146 {
147 return rb->rg_device;
148 }
149
rdmab_data(const struct rpcrdma_regbuf * rb)150 static inline void *rdmab_data(const struct rpcrdma_regbuf *rb)
151 {
152 return rb->rg_data;
153 }
154
155 /* Do not use emergency memory reserves, and fail quickly if memory
156 * cannot be allocated easily. These flags may be used wherever there
157 * is robust logic to handle a failure to allocate.
158 */
159 #define XPRTRDMA_GFP_FLAGS (__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN)
160
161 /* To ensure a transport can always make forward progress,
162 * the number of RDMA segments allowed in header chunk lists
163 * is capped at 16. This prevents less-capable devices from
164 * overrunning the Send buffer while building chunk lists.
165 *
166 * Elements of the Read list take up more room than the
167 * Write list or Reply chunk. 16 read segments means the
168 * chunk lists cannot consume more than
169 *
170 * ((16 + 2) * read segment size) + 1 XDR words,
171 *
172 * or about 400 bytes. The fixed part of the header is
173 * another 24 bytes. Thus when the inline threshold is
174 * 1024 bytes, at least 600 bytes are available for RPC
175 * message bodies.
176 */
177 enum {
178 RPCRDMA_MAX_HDR_SEGS = 16,
179 };
180
181 /*
182 * struct rpcrdma_rep -- this structure encapsulates state required
183 * to receive and complete an RPC Reply, asychronously. It needs
184 * several pieces of state:
185 *
186 * o receive buffer and ib_sge (donated to provider)
187 * o status of receive (success or not, length, inv rkey)
188 * o bookkeeping state to get run by reply handler (XDR stream)
189 *
190 * These structures are allocated during transport initialization.
191 * N of these are associated with a transport instance, managed by
192 * struct rpcrdma_buffer. N is the max number of outstanding RPCs.
193 */
194
195 struct rpcrdma_rep {
196 struct ib_cqe rr_cqe;
197 struct rpc_rdma_cid rr_cid;
198
199 __be32 rr_xid;
200 __be32 rr_vers;
201 __be32 rr_proc;
202 int rr_wc_flags;
203 u32 rr_inv_rkey;
204 struct rpcrdma_regbuf *rr_rdmabuf;
205 struct rpcrdma_xprt *rr_rxprt;
206 struct rpc_rqst *rr_rqst;
207 struct xdr_buf rr_hdrbuf;
208 struct xdr_stream rr_stream;
209 struct llist_node rr_node;
210 struct ib_recv_wr rr_recv_wr;
211 struct list_head rr_all;
212 };
213
214 /* To reduce the rate at which a transport invokes ib_post_recv
215 * (and thus the hardware doorbell rate), xprtrdma posts Receive
216 * WRs in batches.
217 *
218 * Setting this to zero disables Receive post batching.
219 */
220 enum {
221 RPCRDMA_MAX_RECV_BATCH = 7,
222 };
223
224 /* struct rpcrdma_sendctx - DMA mapped SGEs to unmap after Send completes
225 */
226 struct rpcrdma_req;
227 struct rpcrdma_sendctx {
228 struct ib_cqe sc_cqe;
229 struct rpc_rdma_cid sc_cid;
230 struct rpcrdma_req *sc_req;
231 unsigned int sc_unmap_count;
232 struct ib_sge sc_sges[];
233 };
234
235 /*
236 * struct rpcrdma_mr - external memory region metadata
237 *
238 * An external memory region is any buffer or page that is registered
239 * on the fly (ie, not pre-registered).
240 */
241 struct rpcrdma_req;
242 struct rpcrdma_mr {
243 struct list_head mr_list;
244 struct rpcrdma_req *mr_req;
245
246 struct ib_mr *mr_ibmr;
247 struct ib_device *mr_device;
248 struct scatterlist *mr_sg;
249 int mr_nents;
250 enum dma_data_direction mr_dir;
251 struct ib_cqe mr_cqe;
252 struct completion mr_linv_done;
253 union {
254 struct ib_reg_wr mr_regwr;
255 struct ib_send_wr mr_invwr;
256 };
257 struct rpcrdma_xprt *mr_xprt;
258 u32 mr_handle;
259 u32 mr_length;
260 u64 mr_offset;
261 struct list_head mr_all;
262 struct rpc_rdma_cid mr_cid;
263 };
264
265 /*
266 * struct rpcrdma_req -- structure central to the request/reply sequence.
267 *
268 * N of these are associated with a transport instance, and stored in
269 * struct rpcrdma_buffer. N is the max number of outstanding requests.
270 *
271 * It includes pre-registered buffer memory for send AND recv.
272 * The recv buffer, however, is not owned by this structure, and
273 * is "donated" to the hardware when a recv is posted. When a
274 * reply is handled, the recv buffer used is given back to the
275 * struct rpcrdma_req associated with the request.
276 *
277 * In addition to the basic memory, this structure includes an array
278 * of iovs for send operations. The reason is that the iovs passed to
279 * ib_post_{send,recv} must not be modified until the work request
280 * completes.
281 */
282
283 /* Maximum number of page-sized "segments" per chunk list to be
284 * registered or invalidated. Must handle a Reply chunk:
285 */
286 enum {
287 RPCRDMA_MAX_IOV_SEGS = 3, /* head, page-boundary, tail */
288 RPCRDMA_MAX_DATA_SEGS = ((1 * 1024 * 1024) / PAGE_SIZE) + 1,
289 RPCRDMA_MAX_SEGS = RPCRDMA_MAX_DATA_SEGS +
290 RPCRDMA_MAX_IOV_SEGS,
291 };
292
293 /**
294 * struct rpcrdma_xdr_cursor - tracks position within an xdr_buf
295 * for iterative MR registration
296 * @xc_buf: the xdr_buf being iterated
297 * @xc_page_offset: byte offset into the page region consumed so far
298 * @xc_flags: combination of XC_* bits
299 *
300 * Each XC_*_DONE flag indicates that this region has no
301 * remaining MR registration work. That condition holds both when the region
302 * has already been registered by a prior frwr_map() call and
303 * when the region is excluded from this chunk type (pre-set
304 * at init time by rpcrdma_xdr_cursor_init()). frwr_map()
305 * treats the two cases identically: skip the region.
306 */
307 struct rpcrdma_xdr_cursor {
308 const struct xdr_buf *xc_buf;
309 unsigned int xc_page_offset;
310 unsigned int xc_flags;
311 };
312
313 #define XC_HEAD_DONE BIT(0)
314 #define XC_PAGES_DONE BIT(1)
315 #define XC_TAIL_DONE BIT(2)
316
317 /* The Send SGE array is provisioned to send a maximum size
318 * inline request:
319 * - RPC-over-RDMA header
320 * - xdr_buf head iovec
321 * - RPCRDMA_MAX_INLINE bytes, in pages
322 * - xdr_buf tail iovec
323 *
324 * The actual number of array elements consumed by each RPC
325 * depends on the device's max_sge limit.
326 */
327 enum {
328 RPCRDMA_MIN_SEND_SGES = 3,
329 RPCRDMA_MAX_PAGE_SGES = RPCRDMA_MAX_INLINE >> PAGE_SHIFT,
330 RPCRDMA_MAX_SEND_SGES = 1 + 1 + RPCRDMA_MAX_PAGE_SGES + 1,
331 };
332
333 struct rpcrdma_buffer;
334 struct rpcrdma_req {
335 struct list_head rl_list;
336 struct rpc_rqst rl_slot;
337 struct rpcrdma_rep *rl_reply;
338 struct xdr_stream rl_stream;
339 struct xdr_buf rl_hdrbuf;
340 struct ib_send_wr rl_wr;
341 struct rpcrdma_sendctx *rl_sendctx;
342 struct rpcrdma_regbuf *rl_rdmabuf; /* xprt header */
343 struct rpcrdma_regbuf *rl_sendbuf; /* rq_snd_buf */
344 struct rpcrdma_regbuf *rl_recvbuf; /* rq_rcv_buf */
345
346 struct list_head rl_all;
347 struct kref rl_kref;
348
349 struct list_head rl_free_mrs;
350 struct list_head rl_registered;
351 };
352
353 static inline struct rpcrdma_req *
rpcr_to_rdmar(const struct rpc_rqst * rqst)354 rpcr_to_rdmar(const struct rpc_rqst *rqst)
355 {
356 return container_of(rqst, struct rpcrdma_req, rl_slot);
357 }
358
359 static inline void
rpcrdma_mr_push(struct rpcrdma_mr * mr,struct list_head * list)360 rpcrdma_mr_push(struct rpcrdma_mr *mr, struct list_head *list)
361 {
362 list_add(&mr->mr_list, list);
363 }
364
365 static inline struct rpcrdma_mr *
rpcrdma_mr_pop(struct list_head * list)366 rpcrdma_mr_pop(struct list_head *list)
367 {
368 struct rpcrdma_mr *mr;
369
370 mr = list_first_entry_or_null(list, struct rpcrdma_mr, mr_list);
371 if (mr)
372 list_del_init(&mr->mr_list);
373 return mr;
374 }
375
376 /*
377 * struct rpcrdma_buffer -- holds list/queue of pre-registered memory for
378 * inline requests/replies, and client/server credits.
379 *
380 * One of these is associated with a transport instance
381 */
382 struct rpcrdma_buffer {
383 spinlock_t rb_lock;
384 struct list_head rb_send_bufs;
385 struct list_head rb_mrs;
386
387 unsigned long rb_sc_head;
388 unsigned long rb_sc_tail;
389 unsigned long rb_sc_last;
390 struct rpcrdma_sendctx **rb_sc_ctxs;
391
392 struct list_head rb_allreqs;
393 struct list_head rb_all_mrs;
394 struct list_head rb_all_reps;
395
396 struct llist_head rb_free_reps;
397
398 __be32 rb_max_requests;
399 u32 rb_credits; /* most recent credit grant */
400
401 u32 rb_bc_srv_max_requests;
402 u32 rb_bc_max_requests;
403
404 struct work_struct rb_refresh_worker;
405 };
406
407 /*
408 * Statistics for RPCRDMA
409 */
410 struct rpcrdma_stats {
411 /* accessed when sending a call */
412 unsigned long read_chunk_count;
413 unsigned long write_chunk_count;
414 unsigned long reply_chunk_count;
415 unsigned long long total_rdma_request;
416
417 /* rarely accessed error counters */
418 unsigned long long pullup_copy_count;
419 unsigned long hardway_register_count;
420 unsigned long failed_marshal_count;
421 unsigned long bad_reply_count;
422 unsigned long mrs_recycled;
423 unsigned long mrs_orphaned;
424 unsigned long mrs_allocated;
425 unsigned long empty_sendctx_q;
426
427 /* accessed when receiving a reply */
428 unsigned long long total_rdma_reply;
429 unsigned long long fixup_copy_count;
430 unsigned long reply_waits_for_send;
431 unsigned long local_inv_needed;
432 unsigned long nomsg_call_count;
433 unsigned long bcall_count;
434 };
435
436 /*
437 * RPCRDMA transport -- encapsulates the structures above for
438 * integration with RPC.
439 *
440 * The contained structures are embedded, not pointers,
441 * for convenience. This structure need not be visible externally.
442 *
443 * It is allocated and initialized during mount, and released
444 * during unmount.
445 */
446 struct rpcrdma_xprt {
447 struct rpc_xprt rx_xprt;
448 struct rpcrdma_ep *rx_ep;
449 struct rpcrdma_buffer rx_buf;
450 struct delayed_work rx_connect_worker;
451 struct rpc_timeout rx_timeout;
452 struct rpcrdma_stats rx_stats;
453 };
454
455 #define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt)
456
457 static inline const char *
rpcrdma_addrstr(const struct rpcrdma_xprt * r_xprt)458 rpcrdma_addrstr(const struct rpcrdma_xprt *r_xprt)
459 {
460 return r_xprt->rx_xprt.address_strings[RPC_DISPLAY_ADDR];
461 }
462
463 static inline const char *
rpcrdma_portstr(const struct rpcrdma_xprt * r_xprt)464 rpcrdma_portstr(const struct rpcrdma_xprt *r_xprt)
465 {
466 return r_xprt->rx_xprt.address_strings[RPC_DISPLAY_PORT];
467 }
468
469 /* Setting this to 0 ensures interoperability with early servers.
470 * Setting this to 1 enhances unaligned read/write performance.
471 * Default is 0, see sysctl entry and rpc_rdma.c */
472 extern int xprt_rdma_pad_optimize;
473
474 /* This setting controls the hunt for a supported memory
475 * registration strategy.
476 */
477 extern unsigned int xprt_rdma_memreg_strategy;
478
479 /*
480 * Endpoint calls - xprtrdma/verbs.c
481 */
482 void rpcrdma_force_disconnect(struct rpcrdma_ep *ep);
483 void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc);
484 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt);
485 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt);
486
487 void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed);
488
489 /*
490 * Buffer calls - xprtrdma/verbs.c
491 */
492 struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,
493 size_t size);
494 int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
495 void rpcrdma_req_destroy(struct rpcrdma_req *req);
496 int rpcrdma_buffer_create(struct rpcrdma_xprt *);
497 void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
498 struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt);
499
500 struct rpcrdma_mr *rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt);
501 void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt);
502
503 struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
504 void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers,
505 struct rpcrdma_req *req);
506 void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep);
507 void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req);
508
509 bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size,
510 gfp_t flags);
511 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
512 struct rpcrdma_regbuf *rb);
513
514 /**
515 * rpcrdma_regbuf_is_mapped - check if buffer is DMA mapped
516 *
517 * Returns true if the buffer is now mapped to rb->rg_device.
518 */
rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf * rb)519 static inline bool rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb)
520 {
521 return rb->rg_device != NULL;
522 }
523
524 /**
525 * rpcrdma_regbuf_dma_map - DMA-map a regbuf
526 * @r_xprt: controlling transport instance
527 * @rb: regbuf to be mapped
528 *
529 * Returns true if the buffer is currently DMA mapped.
530 */
rpcrdma_regbuf_dma_map(struct rpcrdma_xprt * r_xprt,struct rpcrdma_regbuf * rb)531 static inline bool rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
532 struct rpcrdma_regbuf *rb)
533 {
534 if (likely(rpcrdma_regbuf_is_mapped(rb)))
535 return true;
536 return __rpcrdma_regbuf_dma_map(r_xprt, rb);
537 }
538
539 /*
540 * Wrappers for chunk registration, shared by read/write chunk code.
541 */
542
543 static inline enum dma_data_direction
rpcrdma_data_dir(bool writing)544 rpcrdma_data_dir(bool writing)
545 {
546 return writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
547 }
548
549 /* Memory registration calls xprtrdma/frwr_ops.c
550 */
551 void frwr_reset(struct rpcrdma_req *req);
552 int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device);
553 int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr);
554 void frwr_mr_release(struct rpcrdma_mr *mr);
555 int frwr_map(struct rpcrdma_xprt *r_xprt,
556 struct rpcrdma_xdr_cursor *cur,
557 bool writing, __be32 xid,
558 struct rpcrdma_mr *mr);
559 int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
560 void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs);
561 void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
562 void frwr_unmap_async(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);
563 int frwr_wp_create(struct rpcrdma_xprt *r_xprt);
564
565 /*
566 * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
567 */
568
569 enum rpcrdma_chunktype {
570 rpcrdma_noch = 0,
571 rpcrdma_noch_pullup,
572 rpcrdma_noch_mapped,
573 rpcrdma_readch,
574 rpcrdma_areadch,
575 rpcrdma_writech,
576 rpcrdma_replych
577 };
578
579 int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
580 struct rpcrdma_req *req, u32 hdrlen,
581 struct xdr_buf *xdr,
582 enum rpcrdma_chunktype rtype);
583 void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc);
584 int rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst);
585 void rpcrdma_set_max_header_sizes(struct rpcrdma_ep *ep);
586 void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt);
587 void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);
588 void rpcrdma_unpin_rqst(struct rpcrdma_rep *rep);
589 void rpcrdma_reply_handler(struct rpcrdma_rep *rep);
590
rpcrdma_set_xdrlen(struct xdr_buf * xdr,size_t len)591 static inline void rpcrdma_set_xdrlen(struct xdr_buf *xdr, size_t len)
592 {
593 xdr->head[0].iov_len = len;
594 xdr->len = len;
595 }
596
597 /* RPC/RDMA module init - xprtrdma/transport.c
598 */
599 extern unsigned int xprt_rdma_max_inline_read;
600 extern unsigned int xprt_rdma_max_inline_write;
601 void xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap);
602 void xprt_rdma_free_addresses(struct rpc_xprt *xprt);
603 void xprt_rdma_close(struct rpc_xprt *xprt);
604 void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq);
605 int xprt_rdma_init(void);
606 void xprt_rdma_cleanup(void);
607
608 /* Backchannel calls - xprtrdma/backchannel.c
609 */
610 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
611 int xprt_rdma_bc_setup(struct rpc_xprt *, unsigned int);
612 size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *);
613 unsigned int xprt_rdma_bc_max_slots(struct rpc_xprt *);
614 void rpcrdma_bc_receive_call(struct rpcrdma_xprt *, struct rpcrdma_rep *);
615 int xprt_rdma_bc_send_reply(struct rpc_rqst *rqst);
616 void xprt_rdma_bc_free_rqst(struct rpc_rqst *);
617 void xprt_rdma_bc_destroy(struct rpc_xprt *, unsigned int);
618 #endif /* CONFIG_SUNRPC_BACKCHANNEL */
619
620 extern struct xprt_class xprt_rdma_bc;
621
622 #endif /* _LINUX_SUNRPC_XPRT_RDMA_H */
623