xref: /freebsd/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h (revision 7661de35d15f582ab33e3bd6b8d909601557e436)
1 #ifndef _SDP_H_
2 #define _SDP_H_
3 
4 #include "opt_ddb.h"
5 #include "opt_inet.h"
6 #include "opt_ofed.h"
7 
8 #include <sys/param.h>
9 #include <sys/systm.h>
10 #include <sys/malloc.h>
11 #include <sys/kernel.h>
12 #include <sys/sysctl.h>
13 #include <sys/mbuf.h>
14 #include <sys/lock.h>
15 #include <sys/rwlock.h>
16 #include <sys/socket.h>
17 #include <sys/socketvar.h>
18 #include <sys/protosw.h>
19 #include <sys/proc.h>
20 #include <sys/jail.h>
21 #include <sys/domain.h>
22 
23 #ifdef DDB
24 #include <ddb/ddb.h>
25 #endif
26 
27 #include <net/if.h>
28 #include <net/if_var.h>
29 #include <net/route.h>
30 #include <net/vnet.h>
31 
32 #include <netinet/in.h>
33 #include <netinet/in_systm.h>
34 #include <netinet/in_var.h>
35 #include <netinet/in_pcb.h>
36 #include <netinet/tcp.h>
37 #include <netinet/tcp_fsm.h>
38 #include <netinet/tcp_timer.h>
39 #include <netinet/tcp_var.h>
40 
41 #include <linux/device.h>
42 #include <linux/err.h>
43 #include <linux/sched.h>
44 #include <linux/workqueue.h>
45 #include <linux/wait.h>
46 #include <linux/module.h>
47 #include <linux/moduleparam.h>
48 #include <linux/pci.h>
49 
50 #include <rdma/ib_verbs.h>
51 #include <rdma/rdma_cm.h>
52 #include <rdma/ib_cm.h>
53 #include <rdma/sdp_socket.h>
54 #include <rdma/ib_fmr_pool.h>
55 
56 #ifdef SDP_DEBUG
57 #define	CONFIG_INFINIBAND_SDP_DEBUG
58 #endif
59 
60 #include "sdp_dbg.h"
61 
62 #undef LIST_HEAD
63 /* From sys/queue.h */
64 #define LIST_HEAD(name, type)                                           \
65 struct name {                                                           \
66         struct type *lh_first;  /* first element */                     \
67 }
68 
69 /* Interval between sucessive polls in the Tx routine when polling is used
70    instead of interrupts (in per-core Tx rings) - should be power of 2 */
71 #define SDP_TX_POLL_MODER	16
72 #define SDP_TX_POLL_TIMEOUT	(HZ / 20)
73 #define SDP_NAGLE_TIMEOUT (HZ / 10)
74 
75 #define SDP_SRCAVAIL_CANCEL_TIMEOUT (HZ * 5)
76 #define SDP_SRCAVAIL_ADV_TIMEOUT (1 * HZ)
77 #define SDP_SRCAVAIL_PAYLOAD_LEN 1
78 
79 #define SDP_RESOLVE_TIMEOUT 1000
80 #define SDP_ROUTE_TIMEOUT 1000
81 #define SDP_RETRY_COUNT 5
82 #define SDP_KEEPALIVE_TIME (120 * 60 * HZ)
83 #define SDP_FIN_WAIT_TIMEOUT (60 * HZ) /* like TCP_FIN_TIMEOUT */
84 
85 #define SDP_TX_SIZE 0x40
86 #define SDP_RX_SIZE 0x40
87 
88 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64))
89 #define SDP_FMR_POOL_SIZE	1024
90 #define SDP_FMR_DIRTY_SIZE	( SDP_FMR_POOL_SIZE / 4 )
91 
92 #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2))
93 
94 /* mb inlined data len - rest will be rx'ed into frags */
95 #define SDP_HEAD_SIZE (sizeof(struct sdp_bsdh))
96 
97 /* limit tx payload len, if the sink supports bigger buffers than the source
98  * can handle.
99  * or rx fragment size (limited by sge->length size) */
100 #define	SDP_MAX_PACKET	(1 << 16)
101 #define SDP_MAX_PAYLOAD (SDP_MAX_PACKET - SDP_HEAD_SIZE)
102 
103 #define SDP_MAX_RECV_SGES (SDP_MAX_PACKET / MCLBYTES)
104 #define SDP_MAX_SEND_SGES (SDP_MAX_PACKET / MCLBYTES) + 2
105 
106 #define SDP_NUM_WC 4
107 
108 #define SDP_DEF_ZCOPY_THRESH 64*1024
109 #define SDP_MIN_ZCOPY_THRESH PAGE_SIZE
110 #define SDP_MAX_ZCOPY_THRESH 1048576
111 
112 #define SDP_OP_RECV 0x800000000LL
113 #define SDP_OP_SEND 0x400000000LL
114 #define SDP_OP_RDMA 0x200000000LL
115 #define SDP_OP_NOP  0x100000000LL
116 
117 /* how long (in jiffies) to block sender till tx completion*/
118 #define SDP_BZCOPY_POLL_TIMEOUT (HZ / 10)
119 
120 #define SDP_AUTO_CONF	0xffff
121 #define AUTO_MOD_DELAY (HZ / 4)
122 
123 struct sdp_mb_cb {
124 	__u32		seq;		/* Starting sequence number	*/
125 	struct bzcopy_state      *bz;
126 	struct rx_srcavail_state *rx_sa;
127 	struct tx_srcavail_state *tx_sa;
128 };
129 
130 #define	M_PUSH	M_PROTO1	/* Do a 'push'. */
131 #define	M_URG	M_PROTO2	/* Mark as urgent (oob). */
132 
133 #define SDP_SKB_CB(__mb)      ((struct sdp_mb_cb *)&((__mb)->cb[0]))
134 #define BZCOPY_STATE(mb)      (SDP_SKB_CB(mb)->bz)
135 #define RX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->rx_sa)
136 #define TX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->tx_sa)
137 
138 #ifndef MIN
139 #define MIN(a, b) (a < b ? a : b)
140 #endif
141 
142 #define ring_head(ring)   (atomic_read(&(ring).head))
143 #define ring_tail(ring)   (atomic_read(&(ring).tail))
144 #define ring_posted(ring) (ring_head(ring) - ring_tail(ring))
145 
146 #define rx_ring_posted(ssk) ring_posted(ssk->rx_ring)
147 #ifdef SDP_ZCOPY
148 #define tx_ring_posted(ssk) (ring_posted(ssk->tx_ring) + \
149 	(ssk->tx_ring.rdma_inflight ? ssk->tx_ring.rdma_inflight->busy : 0))
150 #else
151 #define tx_ring_posted(ssk) ring_posted(ssk->tx_ring)
152 #endif
153 
154 extern int sdp_zcopy_thresh;
155 extern int rcvbuf_initial_size;
156 extern struct workqueue_struct *rx_comp_wq;
157 extern struct ib_client sdp_client;
158 
159 enum sdp_mid {
160 	SDP_MID_HELLO = 0x0,
161 	SDP_MID_HELLO_ACK = 0x1,
162 	SDP_MID_DISCONN = 0x2,
163 	SDP_MID_ABORT = 0x3,
164 	SDP_MID_SENDSM = 0x4,
165 	SDP_MID_RDMARDCOMPL = 0x6,
166 	SDP_MID_SRCAVAIL_CANCEL = 0x8,
167 	SDP_MID_CHRCVBUF = 0xB,
168 	SDP_MID_CHRCVBUF_ACK = 0xC,
169 	SDP_MID_SINKAVAIL = 0xFD,
170 	SDP_MID_SRCAVAIL = 0xFE,
171 	SDP_MID_DATA = 0xFF,
172 };
173 
174 enum sdp_flags {
175         SDP_OOB_PRES = 1 << 0,
176         SDP_OOB_PEND = 1 << 1,
177 };
178 
179 enum {
180 	SDP_MIN_TX_CREDITS = 2
181 };
182 
183 enum {
184 	SDP_ERR_ERROR   = -4,
185 	SDP_ERR_FAULT   = -3,
186 	SDP_NEW_SEG     = -2,
187 	SDP_DO_WAIT_MEM = -1
188 };
189 
190 struct sdp_bsdh {
191 	u8 mid;
192 	u8 flags;
193 	__u16 bufs;
194 	__u32 len;
195 	__u32 mseq;
196 	__u32 mseq_ack;
197 } __attribute__((__packed__));
198 
199 union cma_ip_addr {
200 	struct in6_addr ip6;
201 	struct {
202 		__u32 pad[3];
203 		__u32 addr;
204 	} ip4;
205 } __attribute__((__packed__));
206 
207 /* TODO: too much? Can I avoid having the src/dst and port here? */
208 struct sdp_hh {
209 	struct sdp_bsdh bsdh;
210 	u8 majv_minv;
211 	u8 ipv_cap;
212 	u8 rsvd1;
213 	u8 max_adverts;
214 	__u32 desremrcvsz;
215 	__u32 localrcvsz;
216 	__u16 port;
217 	__u16 rsvd2;
218 	union cma_ip_addr src_addr;
219 	union cma_ip_addr dst_addr;
220 	u8 rsvd3[IB_CM_REQ_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 48];
221 } __attribute__((__packed__));
222 
223 struct sdp_hah {
224 	struct sdp_bsdh bsdh;
225 	u8 majv_minv;
226 	u8 ipv_cap;
227 	u8 rsvd1;
228 	u8 ext_max_adverts;
229 	__u32 actrcvsz;
230 	u8 rsvd2[IB_CM_REP_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 8];
231 } __attribute__((__packed__));
232 
233 struct sdp_rrch {
234 	__u32 len;
235 } __attribute__((__packed__));
236 
237 struct sdp_srcah {
238 	__u32 len;
239 	__u32 rkey;
240 	__u64 vaddr;
241 } __attribute__((__packed__));
242 
243 struct sdp_buf {
244         struct mbuf *mb;
245         u64             mapping[SDP_MAX_SEND_SGES];
246 } __attribute__((__packed__));
247 
248 struct sdp_chrecvbuf {
249 	u32 size;
250 } __attribute__((__packed__));
251 
252 /* Context used for synchronous zero copy bcopy (BZCOPY) */
253 struct bzcopy_state {
254 	unsigned char __user  *u_base;
255 	int                    u_len;
256 	int                    left;
257 	int                    page_cnt;
258 	int                    cur_page;
259 	int                    cur_offset;
260 	int                    busy;
261 	struct sdp_sock      *ssk;
262 	struct page         **pages;
263 };
264 
265 enum rx_sa_flag {
266 	RX_SA_ABORTED    = 2,
267 };
268 
269 enum tx_sa_flag {
270 	TX_SA_SENDSM     = 0x01,
271 	TX_SA_CROSS_SEND = 0x02,
272 	TX_SA_INTRRUPTED = 0x04,
273 	TX_SA_TIMEDOUT   = 0x08,
274 	TX_SA_ERROR      = 0x10,
275 };
276 
277 struct rx_srcavail_state {
278 	/* Advertised buffer stuff */
279 	u32 mseq;
280 	u32 used;
281 	u32 reported;
282 	u32 len;
283 	u32 rkey;
284 	u64 vaddr;
285 
286 	/* Dest buff info */
287 	struct ib_umem *umem;
288 	struct ib_pool_fmr *fmr;
289 
290 	/* Utility */
291 	u8  busy;
292 	enum rx_sa_flag  flags;
293 };
294 
295 struct tx_srcavail_state {
296 	/* Data below 'busy' will be reset */
297 	u8		busy;
298 
299 	struct ib_umem *umem;
300 	struct ib_pool_fmr *fmr;
301 
302 	u32		bytes_sent;
303 	u32		bytes_acked;
304 
305 	enum tx_sa_flag	abort_flags;
306 	u8		posted;
307 
308 	u32		mseq;
309 };
310 
311 struct sdp_tx_ring {
312 #ifdef SDP_ZCOPY
313 	struct rx_srcavail_state *rdma_inflight;
314 #endif
315 	struct sdp_buf   	*buffer;
316 	atomic_t          	head;
317 	atomic_t          	tail;
318 	struct ib_cq 	 	*cq;
319 
320 	atomic_t 	  	credits;
321 #define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits))
322 
323 	struct callout		timer;
324 	u16 		  	poll_cnt;
325 };
326 
327 struct sdp_rx_ring {
328 	struct sdp_buf   *buffer;
329 	atomic_t          head;
330 	atomic_t          tail;
331 	struct ib_cq 	 *cq;
332 
333 	int		 destroyed;
334 	struct rwlock	 destroyed_lock;
335 };
336 
337 struct sdp_device {
338 	struct ib_pd 		*pd;
339 	struct ib_mr 		*mr;
340 	struct ib_fmr_pool 	*fmr_pool;
341 };
342 
343 struct sdp_moderation {
344 	unsigned long last_moder_packets;
345 	unsigned long last_moder_tx_packets;
346 	unsigned long last_moder_bytes;
347 	unsigned long last_moder_jiffies;
348 	int last_moder_time;
349 	u16 rx_usecs;
350 	u16 rx_frames;
351 	u16 tx_usecs;
352 	u32 pkt_rate_low;
353 	u16 rx_usecs_low;
354 	u32 pkt_rate_high;
355 	u16 rx_usecs_high;
356 	u16 sample_interval;
357 	u16 adaptive_rx_coal;
358 	u32 msg_enable;
359 
360 	int moder_cnt;
361 	int moder_time;
362 };
363 
364 /* These are flags fields. */
365 #define	SDP_TIMEWAIT	0x0001		/* In ssk timewait state. */
366 #define	SDP_DROPPED	0x0002		/* Socket has been dropped. */
367 #define	SDP_SOCKREF	0x0004		/* Holding a sockref for close. */
368 #define	SDP_NODELAY	0x0008		/* Disble nagle. */
369 #define	SDP_NEEDFIN	0x0010		/* Send a fin on the next tx. */
370 #define	SDP_DREQWAIT	0x0020		/* Waiting on DREQ. */
371 #define	SDP_DESTROY	0x0040		/* Being destroyed. */
372 #define	SDP_DISCON	0x0080		/* rdma_disconnect is owed. */
373 
374 /* These are oobflags */
375 #define	SDP_HADOOB	0x0001		/* Had OOB data. */
376 #define	SDP_HAVEOOB	0x0002		/* Have OOB data. */
377 
378 struct sdp_sock {
379 	LIST_ENTRY(sdp_sock) list;
380 	struct socket *socket;
381 	struct rdma_cm_id *id;
382 	struct ib_device *ib_device;
383 	struct sdp_device *sdp_dev;
384 	struct ib_qp *qp;
385 	struct ucred *cred;
386 	struct callout keep2msl;	/* 2msl and keepalive timer. */
387 	struct callout nagle_timer;	/* timeout waiting for ack */
388 	struct ib_ucontext context;
389 	in_port_t lport;
390 	in_addr_t laddr;
391 	in_port_t fport;
392 	in_addr_t faddr;
393 	int flags;
394 	int oobflags;		/* protected by rx lock. */
395 	int state;
396 	int softerror;
397 	int recv_bytes;		/* Bytes per recv. buf including header */
398 	int xmit_size_goal;
399 	char iobc;
400 
401 	struct sdp_rx_ring rx_ring;
402 	struct sdp_tx_ring tx_ring;
403 	struct rwlock	lock;
404 	struct mbuf *rx_ctl_q;
405 	struct mbuf *rx_ctl_tail;
406 
407 	int qp_active;	/* XXX Flag. */
408 	int max_sge;
409 	struct work_struct rx_comp_work;
410 #define rcv_nxt(ssk) atomic_read(&(ssk->rcv_nxt))
411 	atomic_t rcv_nxt;
412 
413 	/* SDP specific */
414 	atomic_t mseq_ack;
415 #define mseq_ack(ssk) (atomic_read(&ssk->mseq_ack))
416 	unsigned max_bufs;	/* Initial buffers offered by other side */
417 	unsigned min_bufs;	/* Low water mark to wake senders */
418 
419 	unsigned long nagle_last_unacked; /* mseq of lastest unacked packet */
420 
421 	atomic_t               remote_credits;
422 #define remote_credits(ssk) (atomic_read(&ssk->remote_credits))
423 	int 		  poll_cq;
424 
425 	/* SDP slow start */
426 	int recv_request_head; 	/* mark the rx_head when the resize request
427 				   was recieved */
428 	int recv_request; 	/* XXX flag if request to resize was recieved */
429 
430 	unsigned long tx_packets;
431 	unsigned long rx_packets;
432 	unsigned long tx_bytes;
433 	unsigned long rx_bytes;
434 	struct sdp_moderation auto_mod;
435 	struct task shutdown_task;
436 #ifdef SDP_ZCOPY
437 	struct tx_srcavail_state *tx_sa;
438 	struct rx_srcavail_state *rx_sa;
439 	spinlock_t tx_sa_lock;
440 	struct delayed_work srcavail_cancel_work;
441 	int srcavail_cancel_mseq;
442 	/* ZCOPY data: -1:use global; 0:disable zcopy; >0: zcopy threshold */
443 	int zcopy_thresh;
444 #endif
445 };
446 
447 #define	sdp_sk(so)	((struct sdp_sock *)(so->so_pcb))
448 
449 #define	SDP_RLOCK(ssk)		rw_rlock(&(ssk)->lock)
450 #define	SDP_WLOCK(ssk)		rw_wlock(&(ssk)->lock)
451 #define	SDP_RUNLOCK(ssk)	rw_runlock(&(ssk)->lock)
452 #define	SDP_WUNLOCK(ssk)	rw_wunlock(&(ssk)->lock)
453 #define	SDP_WLOCK_ASSERT(ssk)	rw_assert(&(ssk)->lock, RA_WLOCKED)
454 #define	SDP_RLOCK_ASSERT(ssk)	rw_assert(&(ssk)->lock, RA_RLOCKED)
455 #define	SDP_LOCK_ASSERT(ssk)	rw_assert(&(ssk)->lock, RA_LOCKED)
456 
457 static inline void tx_sa_reset(struct tx_srcavail_state *tx_sa)
458 {
459 	memset((void *)&tx_sa->busy, 0,
460 			sizeof(*tx_sa) - offsetof(typeof(*tx_sa), busy));
461 }
462 
463 static inline void rx_ring_unlock(struct sdp_rx_ring *rx_ring)
464 {
465 	rw_runlock(&rx_ring->destroyed_lock);
466 }
467 
468 static inline int rx_ring_trylock(struct sdp_rx_ring *rx_ring)
469 {
470 	rw_rlock(&rx_ring->destroyed_lock);
471 	if (rx_ring->destroyed) {
472 		rx_ring_unlock(rx_ring);
473 		return 0;
474 	}
475 	return 1;
476 }
477 
478 static inline void rx_ring_destroy_lock(struct sdp_rx_ring *rx_ring)
479 {
480 	rw_wlock(&rx_ring->destroyed_lock);
481 	rx_ring->destroyed = 1;
482 	rw_wunlock(&rx_ring->destroyed_lock);
483 }
484 
485 static inline void sdp_arm_rx_cq(struct sdp_sock *ssk)
486 {
487 	sdp_prf(ssk->socket, NULL, "Arming RX cq");
488 	sdp_dbg_data(ssk->socket, "Arming RX cq\n");
489 
490 	ib_req_notify_cq(ssk->rx_ring.cq, IB_CQ_NEXT_COMP);
491 }
492 
493 static inline void sdp_arm_tx_cq(struct sdp_sock *ssk)
494 {
495 	sdp_prf(ssk->socket, NULL, "Arming TX cq");
496 	sdp_dbg_data(ssk->socket, "Arming TX cq. credits: %d, posted: %d\n",
497 		tx_credits(ssk), tx_ring_posted(ssk));
498 
499 	ib_req_notify_cq(ssk->tx_ring.cq, IB_CQ_NEXT_COMP);
500 }
501 
502 /* return the min of:
503  * - tx credits
504  * - free slots in tx_ring (not including SDP_MIN_TX_CREDITS
505  */
506 static inline int tx_slots_free(struct sdp_sock *ssk)
507 {
508 	int min_free;
509 
510 	min_free = MIN(tx_credits(ssk),
511 			SDP_TX_SIZE - tx_ring_posted(ssk));
512 	if (min_free < SDP_MIN_TX_CREDITS)
513 		return 0;
514 
515 	return min_free - SDP_MIN_TX_CREDITS;
516 };
517 
518 /* utilities */
519 static inline char *mid2str(int mid)
520 {
521 #define ENUM2STR(e) [e] = #e
522 	static char *mid2str[] = {
523 		ENUM2STR(SDP_MID_HELLO),
524 		ENUM2STR(SDP_MID_HELLO_ACK),
525 		ENUM2STR(SDP_MID_ABORT),
526 		ENUM2STR(SDP_MID_DISCONN),
527 		ENUM2STR(SDP_MID_SENDSM),
528 		ENUM2STR(SDP_MID_RDMARDCOMPL),
529 		ENUM2STR(SDP_MID_SRCAVAIL_CANCEL),
530 		ENUM2STR(SDP_MID_CHRCVBUF),
531 		ENUM2STR(SDP_MID_CHRCVBUF_ACK),
532 		ENUM2STR(SDP_MID_DATA),
533 		ENUM2STR(SDP_MID_SRCAVAIL),
534 		ENUM2STR(SDP_MID_SINKAVAIL),
535 	};
536 
537 	if (mid >= ARRAY_SIZE(mid2str))
538 		return NULL;
539 
540 	return mid2str[mid];
541 }
542 
543 static inline struct mbuf *
544 sdp_alloc_mb(struct socket *sk, u8 mid, int size, int wait)
545 {
546 	struct sdp_bsdh *h;
547 	struct mbuf *mb;
548 
549 	MGETHDR(mb, wait, MT_DATA);
550 	if (mb == NULL)
551 		return (NULL);
552 	mb->m_pkthdr.len = mb->m_len = sizeof(struct sdp_bsdh);
553 	h = mtod(mb, struct sdp_bsdh *);
554 	h->mid = mid;
555 
556 	return mb;
557 }
558 static inline struct mbuf *
559 sdp_alloc_mb_data(struct socket *sk, int wait)
560 {
561 	return sdp_alloc_mb(sk, SDP_MID_DATA, 0, wait);
562 }
563 
564 static inline struct mbuf *
565 sdp_alloc_mb_disconnect(struct socket *sk, int wait)
566 {
567 	return sdp_alloc_mb(sk, SDP_MID_DISCONN, 0, wait);
568 }
569 
570 static inline void *
571 mb_put(struct mbuf *mb, int len)
572 {
573 	uint8_t *data;
574 
575 	data = mb->m_data;
576 	data += mb->m_len;
577 	mb->m_len += len;
578 	return (void *)data;
579 }
580 
581 static inline struct mbuf *
582 sdp_alloc_mb_chrcvbuf_ack(struct socket *sk, int size, int wait)
583 {
584 	struct mbuf *mb;
585 	struct sdp_chrecvbuf *resp_size;
586 
587 	mb = sdp_alloc_mb(sk, SDP_MID_CHRCVBUF_ACK, sizeof(*resp_size), wait);
588 	if (mb == NULL)
589 		return (NULL);
590 	resp_size = (struct sdp_chrecvbuf *)mb_put(mb, sizeof *resp_size);
591 	resp_size->size = htonl(size);
592 
593 	return mb;
594 }
595 
596 static inline struct mbuf *
597 sdp_alloc_mb_srcavail(struct socket *sk, u32 len, u32 rkey, u64 vaddr, int wait)
598 {
599 	struct mbuf *mb;
600 	struct sdp_srcah *srcah;
601 
602 	mb = sdp_alloc_mb(sk, SDP_MID_SRCAVAIL, sizeof(*srcah), wait);
603 	if (mb == NULL)
604 		return (NULL);
605 	srcah = (struct sdp_srcah *)mb_put(mb, sizeof(*srcah));
606 	srcah->len = htonl(len);
607 	srcah->rkey = htonl(rkey);
608 	srcah->vaddr = cpu_to_be64(vaddr);
609 
610 	return mb;
611 }
612 
613 static inline struct mbuf *
614 sdp_alloc_mb_srcavail_cancel(struct socket *sk, int wait)
615 {
616 	return sdp_alloc_mb(sk, SDP_MID_SRCAVAIL_CANCEL, 0, wait);
617 }
618 
619 static inline struct mbuf *
620 sdp_alloc_mb_rdmardcompl(struct socket *sk, u32 len, int wait)
621 {
622 	struct mbuf *mb;
623 	struct sdp_rrch *rrch;
624 
625 	mb = sdp_alloc_mb(sk, SDP_MID_RDMARDCOMPL, sizeof(*rrch), wait);
626 	if (mb == NULL)
627 		return (NULL);
628 	rrch = (struct sdp_rrch *)mb_put(mb, sizeof(*rrch));
629 	rrch->len = htonl(len);
630 
631 	return mb;
632 }
633 
634 static inline struct mbuf *
635 sdp_alloc_mb_sendsm(struct socket *sk, int wait)
636 {
637 	return sdp_alloc_mb(sk, SDP_MID_SENDSM, 0, wait);
638 }
639 static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk)
640 {
641 	return SDP_TX_SIZE - tx_ring_posted(ssk);
642 }
643 
644 static inline int credit_update_needed(struct sdp_sock *ssk)
645 {
646 	int c;
647 
648 	c = remote_credits(ssk);
649 	if (likely(c > SDP_MIN_TX_CREDITS))
650 		c += c/2;
651 	return unlikely(c < rx_ring_posted(ssk)) &&
652 	    likely(tx_credits(ssk) > 0) &&
653 	    likely(sdp_tx_ring_slots_left(ssk));
654 }
655 
656 
657 #define SDPSTATS_COUNTER_INC(stat)
658 #define SDPSTATS_COUNTER_ADD(stat, val)
659 #define SDPSTATS_COUNTER_MID_INC(stat, mid)
660 #define SDPSTATS_HIST_LINEAR(stat, size)
661 #define SDPSTATS_HIST(stat, size)
662 
663 static inline void
664 sdp_cleanup_sdp_buf(struct sdp_sock *ssk, struct sdp_buf *sbuf,
665     enum dma_data_direction dir)
666 {
667 	struct ib_device *dev;
668 	struct mbuf *mb;
669 	int i;
670 
671 	dev = ssk->ib_device;
672 	for (i = 0, mb = sbuf->mb; mb != NULL; mb = mb->m_next, i++)
673 		ib_dma_unmap_single(dev, sbuf->mapping[i], mb->m_len, dir);
674 }
675 
676 /* sdp_main.c */
677 void sdp_set_default_moderation(struct sdp_sock *ssk);
678 void sdp_start_keepalive_timer(struct socket *sk);
679 void sdp_urg(struct sdp_sock *ssk, struct mbuf *mb);
680 void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk);
681 void sdp_abort(struct socket *sk);
682 struct sdp_sock *sdp_notify(struct sdp_sock *ssk, int error);
683 
684 
685 /* sdp_cma.c */
686 int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *);
687 
688 /* sdp_tx.c */
689 int sdp_tx_ring_create(struct sdp_sock *ssk, struct ib_device *device);
690 void sdp_tx_ring_destroy(struct sdp_sock *ssk);
691 int sdp_xmit_poll(struct sdp_sock *ssk, int force);
692 void sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb);
693 void sdp_post_sends(struct sdp_sock *ssk, int wait);
694 void sdp_post_keepalive(struct sdp_sock *ssk);
695 
696 /* sdp_rx.c */
697 void sdp_rx_ring_init(struct sdp_sock *ssk);
698 int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device);
699 void sdp_rx_ring_destroy(struct sdp_sock *ssk);
700 int sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size);
701 int sdp_init_buffers(struct sdp_sock *ssk, u32 new_size);
702 void sdp_do_posts(struct sdp_sock *ssk);
703 void sdp_rx_comp_full(struct sdp_sock *ssk);
704 
705 /* sdp_zcopy.c */
706 int sdp_sendmsg_zcopy(struct kiocb *iocb, struct socket *sk, struct iovec *iov);
707 int sdp_handle_srcavail(struct sdp_sock *ssk, struct sdp_srcah *srcah);
708 void sdp_handle_sendsm(struct sdp_sock *ssk, u32 mseq_ack);
709 void sdp_handle_rdma_read_compl(struct sdp_sock *ssk, u32 mseq_ack,
710 		u32 bytes_completed);
711 int sdp_handle_rdma_read_cqe(struct sdp_sock *ssk);
712 int sdp_rdma_to_iovec(struct socket *sk, struct iovec *iov, struct mbuf *mb,
713 		unsigned long *used);
714 int sdp_post_rdma_rd_compl(struct sdp_sock *ssk,
715 		struct rx_srcavail_state *rx_sa);
716 int sdp_post_sendsm(struct socket *sk);
717 void srcavail_cancel_timeout(struct work_struct *work);
718 void sdp_abort_srcavail(struct socket *sk);
719 void sdp_abort_rdma_read(struct socket *sk);
720 int sdp_process_rx(struct sdp_sock *ssk);
721 
722 #endif
723