1 #ifndef _SDP_H_
2 #define _SDP_H_
3
4 #define LINUXKPI_PARAM_PREFIX ib_sdp_
5
6 #include "opt_ddb.h"
7 #include "opt_inet.h"
8 #include "opt_ofed.h"
9
10 #include <sys/param.h>
11 #include <sys/systm.h>
12 #include <sys/malloc.h>
13 #include <sys/kernel.h>
14 #include <sys/sysctl.h>
15 #include <sys/mbuf.h>
16 #include <sys/lock.h>
17 #include <sys/rwlock.h>
18 #include <sys/socket.h>
19 #include <sys/socketvar.h>
20 #include <sys/protosw.h>
21 #include <sys/proc.h>
22 #include <sys/jail.h>
23 #include <sys/domain.h>
24
25 #ifdef DDB
26 #include <ddb/ddb.h>
27 #endif
28
29 #include <net/if.h>
30 #include <net/if_var.h>
31 #include <net/route.h>
32 #include <net/vnet.h>
33
34 #include <netinet/in.h>
35 #include <netinet/in_systm.h>
36 #include <netinet/in_var.h>
37 #include <netinet/in_pcb.h>
38 #include <netinet/tcp.h>
39 #include <netinet/tcp_fsm.h>
40 #include <netinet/tcp_timer.h>
41 #include <netinet/tcp_var.h>
42
43 #include <linux/device.h>
44 #include <linux/err.h>
45 #include <linux/sched.h>
46 #include <linux/workqueue.h>
47 #include <linux/wait.h>
48 #include <linux/module.h>
49 #include <linux/moduleparam.h>
50 #include <linux/pci.h>
51
52 #include <rdma/ib_verbs.h>
53 #include <rdma/rdma_cm.h>
54 #include <rdma/ib_cm.h>
55 #include <rdma/ib_fmr_pool.h>
56 #include <rdma/rdma_sdp.h>
57
58 #ifdef SDP_DEBUG
59 #define CONFIG_INFINIBAND_SDP_DEBUG
60 #endif
61
62 #include "sdp_dbg.h"
63
64 #undef LIST_HEAD
65 /* From sys/queue.h */
66 #define LIST_HEAD(name, type) \
67 struct name { \
68 struct type *lh_first; /* first element */ \
69 }
70
71 /* Interval between successive polls in the Tx routine when polling is used
72 instead of interrupts (in per-core Tx rings) - should be power of 2 */
73 #define SDP_TX_POLL_MODER 16
74 #define SDP_TX_POLL_TIMEOUT (HZ / 20)
75 #define SDP_NAGLE_TIMEOUT (HZ / 10)
76
77 #define SDP_SRCAVAIL_CANCEL_TIMEOUT (HZ * 5)
78 #define SDP_SRCAVAIL_ADV_TIMEOUT (1 * HZ)
79 #define SDP_SRCAVAIL_PAYLOAD_LEN 1
80
81 #define SDP_RESOLVE_TIMEOUT 1000
82 #define SDP_ROUTE_TIMEOUT 1000
83 #define SDP_RETRY_COUNT 5
84 #define SDP_KEEPALIVE_TIME (120 * 60 * HZ)
85 #define SDP_FIN_WAIT_TIMEOUT (60 * HZ) /* like TCP_FIN_TIMEOUT */
86
87 #define SDP_TX_SIZE 0x40
88 #define SDP_RX_SIZE 0x40
89
90 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64))
91 #define SDP_FMR_POOL_SIZE 1024
92 #define SDP_FMR_DIRTY_SIZE ( SDP_FMR_POOL_SIZE / 4 )
93
94 #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2))
95
96 /* mb inlined data len - rest will be rx'ed into frags */
97 #define SDP_HEAD_SIZE (sizeof(struct sdp_bsdh))
98
99 /* limit tx payload len, if the sink supports bigger buffers than the source
100 * can handle.
101 * or rx fragment size (limited by sge->length size) */
102 #define SDP_MAX_PACKET (1 << 16)
103 #define SDP_MAX_PAYLOAD (SDP_MAX_PACKET - SDP_HEAD_SIZE)
104
105 #define SDP_MAX_RECV_SGES (SDP_MAX_PACKET / MCLBYTES)
106 #define SDP_MAX_SEND_SGES (SDP_MAX_PACKET / MCLBYTES) + 2
107
108 #define SDP_NUM_WC 4
109
110 #define SDP_DEF_ZCOPY_THRESH 64*1024
111 #define SDP_MIN_ZCOPY_THRESH PAGE_SIZE
112 #define SDP_MAX_ZCOPY_THRESH 1048576
113
114 #define SDP_OP_RECV 0x800000000LL
115 #define SDP_OP_SEND 0x400000000LL
116 #define SDP_OP_RDMA 0x200000000LL
117 #define SDP_OP_NOP 0x100000000LL
118
119 /* how long (in jiffies) to block sender till tx completion*/
120 #define SDP_BZCOPY_POLL_TIMEOUT (HZ / 10)
121
122 #define SDP_AUTO_CONF 0xffff
123 #define AUTO_MOD_DELAY (HZ / 4)
124
125 struct sdp_mb_cb {
126 __u32 seq; /* Starting sequence number */
127 struct bzcopy_state *bz;
128 struct rx_srcavail_state *rx_sa;
129 struct tx_srcavail_state *tx_sa;
130 };
131
132 #define M_PUSH M_PROTO1 /* Do a 'push'. */
133 #define M_URG M_PROTO2 /* Mark as urgent (oob). */
134
135 #define SDP_SKB_CB(__mb) ((struct sdp_mb_cb *)&((__mb)->cb[0]))
136 #define BZCOPY_STATE(mb) (SDP_SKB_CB(mb)->bz)
137 #define RX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->rx_sa)
138 #define TX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->tx_sa)
139
140 #ifndef MIN
141 #define MIN(a, b) (a < b ? a : b)
142 #endif
143
144 #define ring_head(ring) (atomic_read(&(ring).head))
145 #define ring_tail(ring) (atomic_read(&(ring).tail))
146 #define ring_posted(ring) (ring_head(ring) - ring_tail(ring))
147
148 #define rx_ring_posted(ssk) ring_posted(ssk->rx_ring)
149 #ifdef SDP_ZCOPY
150 #define tx_ring_posted(ssk) (ring_posted(ssk->tx_ring) + \
151 (ssk->tx_ring.rdma_inflight ? ssk->tx_ring.rdma_inflight->busy : 0))
152 #else
153 #define tx_ring_posted(ssk) ring_posted(ssk->tx_ring)
154 #endif
155
156 extern int sdp_zcopy_thresh;
157 extern int rcvbuf_initial_size;
158 extern struct workqueue_struct *rx_comp_wq;
159 extern struct ib_client sdp_client;
160
161 enum sdp_mid {
162 SDP_MID_HELLO = 0x0,
163 SDP_MID_HELLO_ACK = 0x1,
164 SDP_MID_DISCONN = 0x2,
165 SDP_MID_ABORT = 0x3,
166 SDP_MID_SENDSM = 0x4,
167 SDP_MID_RDMARDCOMPL = 0x6,
168 SDP_MID_SRCAVAIL_CANCEL = 0x8,
169 SDP_MID_CHRCVBUF = 0xB,
170 SDP_MID_CHRCVBUF_ACK = 0xC,
171 SDP_MID_SINKAVAIL = 0xFD,
172 SDP_MID_SRCAVAIL = 0xFE,
173 SDP_MID_DATA = 0xFF,
174 };
175
176 enum sdp_flags {
177 SDP_OOB_PRES = 1 << 0,
178 SDP_OOB_PEND = 1 << 1,
179 };
180
181 enum {
182 SDP_MIN_TX_CREDITS = 2
183 };
184
185 enum {
186 SDP_ERR_ERROR = -4,
187 SDP_ERR_FAULT = -3,
188 SDP_NEW_SEG = -2,
189 SDP_DO_WAIT_MEM = -1
190 };
191
192 struct sdp_rrch {
193 __u32 len;
194 } __attribute__((__packed__));
195
196 struct sdp_srcah {
197 __u32 len;
198 __u32 rkey;
199 __u64 vaddr;
200 } __attribute__((__packed__));
201
202 struct sdp_buf {
203 struct mbuf *mb;
204 u64 mapping[SDP_MAX_SEND_SGES];
205 } __attribute__((__packed__));
206
207 struct sdp_chrecvbuf {
208 u32 size;
209 } __attribute__((__packed__));
210
211 /* Context used for synchronous zero copy bcopy (BZCOPY) */
212 struct bzcopy_state {
213 unsigned char __user *u_base;
214 int u_len;
215 int left;
216 int page_cnt;
217 int cur_page;
218 int cur_offset;
219 int busy;
220 struct sdp_sock *ssk;
221 struct page **pages;
222 };
223
224 enum rx_sa_flag {
225 RX_SA_ABORTED = 2,
226 };
227
228 enum tx_sa_flag {
229 TX_SA_SENDSM = 0x01,
230 TX_SA_CROSS_SEND = 0x02,
231 TX_SA_INTRRUPTED = 0x04,
232 TX_SA_TIMEDOUT = 0x08,
233 TX_SA_ERROR = 0x10,
234 };
235
236 struct rx_srcavail_state {
237 /* Advertised buffer stuff */
238 u32 mseq;
239 u32 used;
240 u32 reported;
241 u32 len;
242 u32 rkey;
243 u64 vaddr;
244
245 /* Dest buff info */
246 struct ib_umem *umem;
247 struct ib_pool_fmr *fmr;
248
249 /* Utility */
250 u8 busy;
251 enum rx_sa_flag flags;
252 };
253
254 struct tx_srcavail_state {
255 /* Data below 'busy' will be reset */
256 u8 busy;
257
258 struct ib_umem *umem;
259 struct ib_pool_fmr *fmr;
260
261 u32 bytes_sent;
262 u32 bytes_acked;
263
264 enum tx_sa_flag abort_flags;
265 u8 posted;
266
267 u32 mseq;
268 };
269
270 struct sdp_tx_ring {
271 #ifdef SDP_ZCOPY
272 struct rx_srcavail_state *rdma_inflight;
273 #endif
274 struct sdp_buf *buffer;
275 atomic_t head;
276 atomic_t tail;
277 struct ib_cq *cq;
278
279 atomic_t credits;
280 #define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits))
281
282 struct callout timer;
283 u16 poll_cnt;
284 };
285
286 struct sdp_rx_ring {
287 struct sdp_buf *buffer;
288 atomic_t head;
289 atomic_t tail;
290 struct ib_cq *cq;
291
292 int destroyed;
293 struct rwlock destroyed_lock;
294 };
295
296 struct sdp_device {
297 struct ib_pd *pd;
298 struct ib_fmr_pool *fmr_pool;
299 };
300
301 struct sdp_moderation {
302 unsigned long last_moder_packets;
303 unsigned long last_moder_tx_packets;
304 unsigned long last_moder_bytes;
305 unsigned long last_moder_jiffies;
306 int last_moder_time;
307 u16 rx_usecs;
308 u16 rx_frames;
309 u16 tx_usecs;
310 u32 pkt_rate_low;
311 u16 rx_usecs_low;
312 u32 pkt_rate_high;
313 u16 rx_usecs_high;
314 u16 sample_interval;
315 u16 adaptive_rx_coal;
316 u32 msg_enable;
317
318 int moder_cnt;
319 int moder_time;
320 };
321
322 /* These are flags fields. */
323 #define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */
324 #define SDP_DROPPED 0x0002 /* Socket has been dropped. */
325 #define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */
326 #define SDP_NODELAY 0x0008 /* Disble nagle. */
327 #define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */
328 #define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */
329 #define SDP_DESTROY 0x0040 /* Being destroyed. */
330 #define SDP_DISCON 0x0080 /* rdma_disconnect is owed. */
331
332 /* These are oobflags */
333 #define SDP_HADOOB 0x0001 /* Had OOB data. */
334 #define SDP_HAVEOOB 0x0002 /* Have OOB data. */
335
336 struct sdp_sock {
337 LIST_ENTRY(sdp_sock) list;
338 struct socket *socket;
339 struct rdma_cm_id *id;
340 struct ib_device *ib_device;
341 struct sdp_device *sdp_dev;
342 struct ib_qp *qp;
343 struct ucred *cred;
344 struct callout keep2msl; /* 2msl and keepalive timer. */
345 struct callout nagle_timer; /* timeout waiting for ack */
346 struct ib_ucontext context;
347 in_port_t lport;
348 in_addr_t laddr;
349 in_port_t fport;
350 in_addr_t faddr;
351 int flags;
352 int oobflags; /* protected by rx lock. */
353 int state;
354 int softerror;
355 int recv_bytes; /* Bytes per recv. buf including header */
356 int xmit_size_goal;
357 char iobc;
358
359 struct sdp_rx_ring rx_ring;
360 struct sdp_tx_ring tx_ring;
361 struct rwlock lock;
362 struct mbufq rxctlq; /* received control packets */
363
364 int qp_active; /* XXX Flag. */
365 int max_sge;
366 struct work_struct rx_comp_work;
367 #define rcv_nxt(ssk) atomic_read(&(ssk->rcv_nxt))
368 atomic_t rcv_nxt;
369
370 /* SDP specific */
371 atomic_t mseq_ack;
372 #define mseq_ack(ssk) (atomic_read(&ssk->mseq_ack))
373 unsigned max_bufs; /* Initial buffers offered by other side */
374 unsigned min_bufs; /* Low water mark to wake senders */
375
376 unsigned long nagle_last_unacked; /* mseq of lastest unacked packet */
377
378 atomic_t remote_credits;
379 #define remote_credits(ssk) (atomic_read(&ssk->remote_credits))
380 int poll_cq;
381
382 /* SDP slow start */
383 int recv_request_head; /* mark the rx_head when the resize request
384 was received */
385 int recv_request; /* XXX flag if request to resize was received */
386
387 unsigned long tx_packets;
388 unsigned long rx_packets;
389 unsigned long tx_bytes;
390 unsigned long rx_bytes;
391 struct sdp_moderation auto_mod;
392 struct task shutdown_task;
393 #ifdef SDP_ZCOPY
394 struct tx_srcavail_state *tx_sa;
395 struct rx_srcavail_state *rx_sa;
396 spinlock_t tx_sa_lock;
397 struct delayed_work srcavail_cancel_work;
398 int srcavail_cancel_mseq;
399 /* ZCOPY data: -1:use global; 0:disable zcopy; >0: zcopy threshold */
400 int zcopy_thresh;
401 #endif
402 };
403
404 #define sdp_sk(so) ((struct sdp_sock *)(so->so_pcb))
405
406 #define SDP_RLOCK(ssk) rw_rlock(&(ssk)->lock)
407 #define SDP_WLOCK(ssk) rw_wlock(&(ssk)->lock)
408 #define SDP_RUNLOCK(ssk) rw_runlock(&(ssk)->lock)
409 #define SDP_WUNLOCK(ssk) rw_wunlock(&(ssk)->lock)
410 #define SDP_WLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_WLOCKED)
411 #define SDP_RLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_RLOCKED)
412 #define SDP_LOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_LOCKED)
413
414 MALLOC_DECLARE(M_SDP);
415 SYSCTL_DECL(_net_inet_sdp);
416
tx_sa_reset(struct tx_srcavail_state * tx_sa)417 static inline void tx_sa_reset(struct tx_srcavail_state *tx_sa)
418 {
419 memset((void *)&tx_sa->busy, 0,
420 sizeof(*tx_sa) - offsetof(typeof(*tx_sa), busy));
421 }
422
rx_ring_unlock(struct sdp_rx_ring * rx_ring)423 static inline void rx_ring_unlock(struct sdp_rx_ring *rx_ring)
424 {
425 rw_runlock(&rx_ring->destroyed_lock);
426 }
427
rx_ring_trylock(struct sdp_rx_ring * rx_ring)428 static inline int rx_ring_trylock(struct sdp_rx_ring *rx_ring)
429 {
430 rw_rlock(&rx_ring->destroyed_lock);
431 if (rx_ring->destroyed) {
432 rx_ring_unlock(rx_ring);
433 return 0;
434 }
435 return 1;
436 }
437
rx_ring_destroy_lock(struct sdp_rx_ring * rx_ring)438 static inline void rx_ring_destroy_lock(struct sdp_rx_ring *rx_ring)
439 {
440 rw_wlock(&rx_ring->destroyed_lock);
441 rx_ring->destroyed = 1;
442 rw_wunlock(&rx_ring->destroyed_lock);
443 }
444
sdp_arm_rx_cq(struct sdp_sock * ssk)445 static inline void sdp_arm_rx_cq(struct sdp_sock *ssk)
446 {
447 sdp_prf(ssk->socket, NULL, "Arming RX cq");
448 sdp_dbg_data(ssk->socket, "Arming RX cq\n");
449
450 ib_req_notify_cq(ssk->rx_ring.cq, IB_CQ_NEXT_COMP);
451 }
452
sdp_arm_tx_cq(struct sdp_sock * ssk)453 static inline void sdp_arm_tx_cq(struct sdp_sock *ssk)
454 {
455 sdp_prf(ssk->socket, NULL, "Arming TX cq");
456 sdp_dbg_data(ssk->socket, "Arming TX cq. credits: %d, posted: %d\n",
457 tx_credits(ssk), tx_ring_posted(ssk));
458
459 ib_req_notify_cq(ssk->tx_ring.cq, IB_CQ_NEXT_COMP);
460 }
461
462 /* return the min of:
463 * - tx credits
464 * - free slots in tx_ring (not including SDP_MIN_TX_CREDITS
465 */
tx_slots_free(struct sdp_sock * ssk)466 static inline int tx_slots_free(struct sdp_sock *ssk)
467 {
468 int min_free;
469
470 min_free = MIN(tx_credits(ssk),
471 SDP_TX_SIZE - tx_ring_posted(ssk));
472 if (min_free < SDP_MIN_TX_CREDITS)
473 return 0;
474
475 return min_free - SDP_MIN_TX_CREDITS;
476 };
477
478 /* utilities */
mid2str(int mid)479 static inline char *mid2str(int mid)
480 {
481 #define ENUM2STR(e) [e] = #e
482 static char *mid2str[] = {
483 ENUM2STR(SDP_MID_HELLO),
484 ENUM2STR(SDP_MID_HELLO_ACK),
485 ENUM2STR(SDP_MID_ABORT),
486 ENUM2STR(SDP_MID_DISCONN),
487 ENUM2STR(SDP_MID_SENDSM),
488 ENUM2STR(SDP_MID_RDMARDCOMPL),
489 ENUM2STR(SDP_MID_SRCAVAIL_CANCEL),
490 ENUM2STR(SDP_MID_CHRCVBUF),
491 ENUM2STR(SDP_MID_CHRCVBUF_ACK),
492 ENUM2STR(SDP_MID_DATA),
493 ENUM2STR(SDP_MID_SRCAVAIL),
494 ENUM2STR(SDP_MID_SINKAVAIL),
495 };
496
497 if (mid >= ARRAY_SIZE(mid2str))
498 return NULL;
499
500 return mid2str[mid];
501 }
502
503 static inline struct mbuf *
sdp_alloc_mb(struct socket * sk,u8 mid,int size,int wait)504 sdp_alloc_mb(struct socket *sk, u8 mid, int size, int wait)
505 {
506 struct sdp_bsdh *h;
507 struct mbuf *mb;
508
509 MGETHDR(mb, wait, MT_DATA);
510 if (mb == NULL)
511 return (NULL);
512 mb->m_pkthdr.len = mb->m_len = sizeof(struct sdp_bsdh);
513 h = mtod(mb, struct sdp_bsdh *);
514 h->mid = mid;
515
516 return mb;
517 }
518 static inline struct mbuf *
sdp_alloc_mb_data(struct socket * sk,int wait)519 sdp_alloc_mb_data(struct socket *sk, int wait)
520 {
521 return sdp_alloc_mb(sk, SDP_MID_DATA, 0, wait);
522 }
523
524 static inline struct mbuf *
sdp_alloc_mb_disconnect(struct socket * sk,int wait)525 sdp_alloc_mb_disconnect(struct socket *sk, int wait)
526 {
527 return sdp_alloc_mb(sk, SDP_MID_DISCONN, 0, wait);
528 }
529
530 static inline void *
mb_put(struct mbuf * mb,int len)531 mb_put(struct mbuf *mb, int len)
532 {
533 uint8_t *data;
534
535 data = mb->m_data;
536 data += mb->m_len;
537 mb->m_len += len;
538 return (void *)data;
539 }
540
541 static inline struct mbuf *
sdp_alloc_mb_chrcvbuf_ack(struct socket * sk,int size,int wait)542 sdp_alloc_mb_chrcvbuf_ack(struct socket *sk, int size, int wait)
543 {
544 struct mbuf *mb;
545 struct sdp_chrecvbuf *resp_size;
546
547 mb = sdp_alloc_mb(sk, SDP_MID_CHRCVBUF_ACK, sizeof(*resp_size), wait);
548 if (mb == NULL)
549 return (NULL);
550 resp_size = (struct sdp_chrecvbuf *)mb_put(mb, sizeof *resp_size);
551 resp_size->size = htonl(size);
552
553 return mb;
554 }
555
556 static inline struct mbuf *
sdp_alloc_mb_srcavail(struct socket * sk,u32 len,u32 rkey,u64 vaddr,int wait)557 sdp_alloc_mb_srcavail(struct socket *sk, u32 len, u32 rkey, u64 vaddr, int wait)
558 {
559 struct mbuf *mb;
560 struct sdp_srcah *srcah;
561
562 mb = sdp_alloc_mb(sk, SDP_MID_SRCAVAIL, sizeof(*srcah), wait);
563 if (mb == NULL)
564 return (NULL);
565 srcah = (struct sdp_srcah *)mb_put(mb, sizeof(*srcah));
566 srcah->len = htonl(len);
567 srcah->rkey = htonl(rkey);
568 srcah->vaddr = cpu_to_be64(vaddr);
569
570 return mb;
571 }
572
573 static inline struct mbuf *
sdp_alloc_mb_srcavail_cancel(struct socket * sk,int wait)574 sdp_alloc_mb_srcavail_cancel(struct socket *sk, int wait)
575 {
576 return sdp_alloc_mb(sk, SDP_MID_SRCAVAIL_CANCEL, 0, wait);
577 }
578
579 static inline struct mbuf *
sdp_alloc_mb_rdmardcompl(struct socket * sk,u32 len,int wait)580 sdp_alloc_mb_rdmardcompl(struct socket *sk, u32 len, int wait)
581 {
582 struct mbuf *mb;
583 struct sdp_rrch *rrch;
584
585 mb = sdp_alloc_mb(sk, SDP_MID_RDMARDCOMPL, sizeof(*rrch), wait);
586 if (mb == NULL)
587 return (NULL);
588 rrch = (struct sdp_rrch *)mb_put(mb, sizeof(*rrch));
589 rrch->len = htonl(len);
590
591 return mb;
592 }
593
594 static inline struct mbuf *
sdp_alloc_mb_sendsm(struct socket * sk,int wait)595 sdp_alloc_mb_sendsm(struct socket *sk, int wait)
596 {
597 return sdp_alloc_mb(sk, SDP_MID_SENDSM, 0, wait);
598 }
sdp_tx_ring_slots_left(struct sdp_sock * ssk)599 static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk)
600 {
601 return SDP_TX_SIZE - tx_ring_posted(ssk);
602 }
603
credit_update_needed(struct sdp_sock * ssk)604 static inline int credit_update_needed(struct sdp_sock *ssk)
605 {
606 int c;
607
608 c = remote_credits(ssk);
609 if (likely(c > SDP_MIN_TX_CREDITS))
610 c += c/2;
611 return unlikely(c < rx_ring_posted(ssk)) &&
612 likely(tx_credits(ssk) > 0) &&
613 likely(sdp_tx_ring_slots_left(ssk));
614 }
615
616
617 #define SDPSTATS_COUNTER_INC(stat)
618 #define SDPSTATS_COUNTER_ADD(stat, val)
619 #define SDPSTATS_COUNTER_MID_INC(stat, mid)
620 #define SDPSTATS_HIST_LINEAR(stat, size)
621 #define SDPSTATS_HIST(stat, size)
622
623 static inline void
sdp_cleanup_sdp_buf(struct sdp_sock * ssk,struct sdp_buf * sbuf,enum dma_data_direction dir)624 sdp_cleanup_sdp_buf(struct sdp_sock *ssk, struct sdp_buf *sbuf,
625 enum dma_data_direction dir)
626 {
627 struct ib_device *dev;
628 struct mbuf *mb;
629 int i;
630
631 dev = ssk->ib_device;
632 for (i = 0, mb = sbuf->mb; mb != NULL; mb = mb->m_next, i++)
633 ib_dma_unmap_single(dev, sbuf->mapping[i], mb->m_len, dir);
634 }
635
636 /* sdp_main.c */
637 void sdp_set_default_moderation(struct sdp_sock *ssk);
638 void sdp_start_keepalive_timer(struct socket *sk);
639 void sdp_urg(struct sdp_sock *ssk, struct mbuf *mb);
640 void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk);
641 void sdp_abort(struct socket *sk);
642 struct sdp_sock *sdp_notify(struct sdp_sock *ssk, int error);
643
644
645 /* sdp_cma.c */
646 int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *);
647
648 /* sdp_tx.c */
649 int sdp_tx_ring_create(struct sdp_sock *ssk, struct ib_device *device);
650 void sdp_tx_ring_destroy(struct sdp_sock *ssk);
651 int sdp_xmit_poll(struct sdp_sock *ssk, int force);
652 void sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb);
653 void sdp_post_sends(struct sdp_sock *ssk, int wait);
654 void sdp_post_keepalive(struct sdp_sock *ssk);
655
656 /* sdp_rx.c */
657 void sdp_rx_ring_init(struct sdp_sock *ssk);
658 int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device);
659 void sdp_rx_ring_destroy(struct sdp_sock *ssk);
660 int sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size);
661 int sdp_init_buffers(struct sdp_sock *ssk, u32 new_size);
662 void sdp_do_posts(struct sdp_sock *ssk);
663 void sdp_rx_comp_full(struct sdp_sock *ssk);
664
665 /* sdp_zcopy.c */
666 struct kiocb;
667 int sdp_sendmsg_zcopy(struct kiocb *iocb, struct socket *sk, struct iovec *iov);
668 int sdp_handle_srcavail(struct sdp_sock *ssk, struct sdp_srcah *srcah);
669 void sdp_handle_sendsm(struct sdp_sock *ssk, u32 mseq_ack);
670 void sdp_handle_rdma_read_compl(struct sdp_sock *ssk, u32 mseq_ack,
671 u32 bytes_completed);
672 int sdp_handle_rdma_read_cqe(struct sdp_sock *ssk);
673 int sdp_rdma_to_iovec(struct socket *sk, struct iovec *iov, struct mbuf *mb,
674 unsigned long *used);
675 int sdp_post_rdma_rd_compl(struct sdp_sock *ssk,
676 struct rx_srcavail_state *rx_sa);
677 int sdp_post_sendsm(struct socket *sk);
678 void srcavail_cancel_timeout(struct work_struct *work);
679 void sdp_abort_srcavail(struct socket *sk);
680 void sdp_abort_rdma_read(struct socket *sk);
681 int sdp_process_rx(struct sdp_sock *ssk);
682
683 #endif
684