1aa0a1e58SJeff Roberson #ifndef _SDP_H_ 2aa0a1e58SJeff Roberson #define _SDP_H_ 3aa0a1e58SJeff Roberson 4*fa201e28SHans Petter Selasky #define LINUXKPI_PARAM_PREFIX ib_sdp_ 5*fa201e28SHans Petter Selasky 6aa0a1e58SJeff Roberson #include "opt_ddb.h" 7aa0a1e58SJeff Roberson #include "opt_inet.h" 8aa0a1e58SJeff Roberson #include "opt_ofed.h" 9aa0a1e58SJeff Roberson 10aa0a1e58SJeff Roberson #include <sys/param.h> 11aa0a1e58SJeff Roberson #include <sys/systm.h> 12aa0a1e58SJeff Roberson #include <sys/malloc.h> 13aa0a1e58SJeff Roberson #include <sys/kernel.h> 14aa0a1e58SJeff Roberson #include <sys/sysctl.h> 15aa0a1e58SJeff Roberson #include <sys/mbuf.h> 16aa0a1e58SJeff Roberson #include <sys/lock.h> 17aa0a1e58SJeff Roberson #include <sys/rwlock.h> 18aa0a1e58SJeff Roberson #include <sys/socket.h> 19aa0a1e58SJeff Roberson #include <sys/socketvar.h> 20aa0a1e58SJeff Roberson #include <sys/protosw.h> 21aa0a1e58SJeff Roberson #include <sys/proc.h> 22aa0a1e58SJeff Roberson #include <sys/jail.h> 23aa0a1e58SJeff Roberson #include <sys/domain.h> 24aa0a1e58SJeff Roberson 25aa0a1e58SJeff Roberson #ifdef DDB 26aa0a1e58SJeff Roberson #include <ddb/ddb.h> 27aa0a1e58SJeff Roberson #endif 28aa0a1e58SJeff Roberson 29aa0a1e58SJeff Roberson #include <net/if.h> 3076039bc8SGleb Smirnoff #include <net/if_var.h> 31aa0a1e58SJeff Roberson #include <net/route.h> 32aa0a1e58SJeff Roberson #include <net/vnet.h> 33aa0a1e58SJeff Roberson 34aa0a1e58SJeff Roberson #include <netinet/in.h> 35aa0a1e58SJeff Roberson #include <netinet/in_systm.h> 36aa0a1e58SJeff Roberson #include <netinet/in_var.h> 37aa0a1e58SJeff Roberson #include <netinet/in_pcb.h> 38aa0a1e58SJeff Roberson #include <netinet/tcp.h> 39aa0a1e58SJeff Roberson #include <netinet/tcp_fsm.h> 40aa0a1e58SJeff Roberson #include <netinet/tcp_timer.h> 41aa0a1e58SJeff Roberson #include <netinet/tcp_var.h> 42aa0a1e58SJeff Roberson 43aa0a1e58SJeff Roberson #include <linux/device.h> 44aa0a1e58SJeff Roberson #include <linux/err.h> 45aa0a1e58SJeff Roberson #include <linux/sched.h> 46aa0a1e58SJeff Roberson #include <linux/workqueue.h> 47aa0a1e58SJeff Roberson #include <linux/wait.h> 48aa0a1e58SJeff Roberson #include <linux/module.h> 49aa0a1e58SJeff Roberson #include <linux/moduleparam.h> 50aa0a1e58SJeff Roberson #include <linux/pci.h> 51aa0a1e58SJeff Roberson 52aa0a1e58SJeff Roberson #include <rdma/ib_verbs.h> 53aa0a1e58SJeff Roberson #include <rdma/rdma_cm.h> 54aa0a1e58SJeff Roberson #include <rdma/ib_cm.h> 55aa0a1e58SJeff Roberson #include <rdma/sdp_socket.h> 56aa0a1e58SJeff Roberson #include <rdma/ib_fmr_pool.h> 57aa0a1e58SJeff Roberson 58aa0a1e58SJeff Roberson #ifdef SDP_DEBUG 59aa0a1e58SJeff Roberson #define CONFIG_INFINIBAND_SDP_DEBUG 60aa0a1e58SJeff Roberson #endif 61aa0a1e58SJeff Roberson 62aa0a1e58SJeff Roberson #include "sdp_dbg.h" 63aa0a1e58SJeff Roberson 64aa0a1e58SJeff Roberson #undef LIST_HEAD 65aa0a1e58SJeff Roberson /* From sys/queue.h */ 66aa0a1e58SJeff Roberson #define LIST_HEAD(name, type) \ 67aa0a1e58SJeff Roberson struct name { \ 68aa0a1e58SJeff Roberson struct type *lh_first; /* first element */ \ 69aa0a1e58SJeff Roberson } 70aa0a1e58SJeff Roberson 71bf5cba36SPedro F. Giffuni /* Interval between successive polls in the Tx routine when polling is used 72aa0a1e58SJeff Roberson instead of interrupts (in per-core Tx rings) - should be power of 2 */ 73aa0a1e58SJeff Roberson #define SDP_TX_POLL_MODER 16 74aa0a1e58SJeff Roberson #define SDP_TX_POLL_TIMEOUT (HZ / 20) 75aa0a1e58SJeff Roberson #define SDP_NAGLE_TIMEOUT (HZ / 10) 76aa0a1e58SJeff Roberson 77aa0a1e58SJeff Roberson #define SDP_SRCAVAIL_CANCEL_TIMEOUT (HZ * 5) 78aa0a1e58SJeff Roberson #define SDP_SRCAVAIL_ADV_TIMEOUT (1 * HZ) 79aa0a1e58SJeff Roberson #define SDP_SRCAVAIL_PAYLOAD_LEN 1 80aa0a1e58SJeff Roberson 81aa0a1e58SJeff Roberson #define SDP_RESOLVE_TIMEOUT 1000 82aa0a1e58SJeff Roberson #define SDP_ROUTE_TIMEOUT 1000 83aa0a1e58SJeff Roberson #define SDP_RETRY_COUNT 5 84aa0a1e58SJeff Roberson #define SDP_KEEPALIVE_TIME (120 * 60 * HZ) 85aa0a1e58SJeff Roberson #define SDP_FIN_WAIT_TIMEOUT (60 * HZ) /* like TCP_FIN_TIMEOUT */ 86aa0a1e58SJeff Roberson 87aa0a1e58SJeff Roberson #define SDP_TX_SIZE 0x40 88aa0a1e58SJeff Roberson #define SDP_RX_SIZE 0x40 89aa0a1e58SJeff Roberson 90aa0a1e58SJeff Roberson #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64)) 91aa0a1e58SJeff Roberson #define SDP_FMR_POOL_SIZE 1024 92aa0a1e58SJeff Roberson #define SDP_FMR_DIRTY_SIZE ( SDP_FMR_POOL_SIZE / 4 ) 93aa0a1e58SJeff Roberson 94aa0a1e58SJeff Roberson #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2)) 95aa0a1e58SJeff Roberson 96aa0a1e58SJeff Roberson /* mb inlined data len - rest will be rx'ed into frags */ 97aa0a1e58SJeff Roberson #define SDP_HEAD_SIZE (sizeof(struct sdp_bsdh)) 98aa0a1e58SJeff Roberson 99aa0a1e58SJeff Roberson /* limit tx payload len, if the sink supports bigger buffers than the source 100aa0a1e58SJeff Roberson * can handle. 101aa0a1e58SJeff Roberson * or rx fragment size (limited by sge->length size) */ 102aa0a1e58SJeff Roberson #define SDP_MAX_PACKET (1 << 16) 103aa0a1e58SJeff Roberson #define SDP_MAX_PAYLOAD (SDP_MAX_PACKET - SDP_HEAD_SIZE) 104aa0a1e58SJeff Roberson 105aa0a1e58SJeff Roberson #define SDP_MAX_RECV_SGES (SDP_MAX_PACKET / MCLBYTES) 106aa0a1e58SJeff Roberson #define SDP_MAX_SEND_SGES (SDP_MAX_PACKET / MCLBYTES) + 2 107aa0a1e58SJeff Roberson 108aa0a1e58SJeff Roberson #define SDP_NUM_WC 4 109aa0a1e58SJeff Roberson 110aa0a1e58SJeff Roberson #define SDP_DEF_ZCOPY_THRESH 64*1024 111aa0a1e58SJeff Roberson #define SDP_MIN_ZCOPY_THRESH PAGE_SIZE 112aa0a1e58SJeff Roberson #define SDP_MAX_ZCOPY_THRESH 1048576 113aa0a1e58SJeff Roberson 114aa0a1e58SJeff Roberson #define SDP_OP_RECV 0x800000000LL 115aa0a1e58SJeff Roberson #define SDP_OP_SEND 0x400000000LL 116aa0a1e58SJeff Roberson #define SDP_OP_RDMA 0x200000000LL 117aa0a1e58SJeff Roberson #define SDP_OP_NOP 0x100000000LL 118aa0a1e58SJeff Roberson 119aa0a1e58SJeff Roberson /* how long (in jiffies) to block sender till tx completion*/ 120aa0a1e58SJeff Roberson #define SDP_BZCOPY_POLL_TIMEOUT (HZ / 10) 121aa0a1e58SJeff Roberson 122aa0a1e58SJeff Roberson #define SDP_AUTO_CONF 0xffff 123aa0a1e58SJeff Roberson #define AUTO_MOD_DELAY (HZ / 4) 124aa0a1e58SJeff Roberson 125aa0a1e58SJeff Roberson struct sdp_mb_cb { 126aa0a1e58SJeff Roberson __u32 seq; /* Starting sequence number */ 127aa0a1e58SJeff Roberson struct bzcopy_state *bz; 128aa0a1e58SJeff Roberson struct rx_srcavail_state *rx_sa; 129aa0a1e58SJeff Roberson struct tx_srcavail_state *tx_sa; 130aa0a1e58SJeff Roberson }; 131aa0a1e58SJeff Roberson 132aa0a1e58SJeff Roberson #define M_PUSH M_PROTO1 /* Do a 'push'. */ 133aa0a1e58SJeff Roberson #define M_URG M_PROTO2 /* Mark as urgent (oob). */ 134aa0a1e58SJeff Roberson 135aa0a1e58SJeff Roberson #define SDP_SKB_CB(__mb) ((struct sdp_mb_cb *)&((__mb)->cb[0])) 136aa0a1e58SJeff Roberson #define BZCOPY_STATE(mb) (SDP_SKB_CB(mb)->bz) 137aa0a1e58SJeff Roberson #define RX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->rx_sa) 138aa0a1e58SJeff Roberson #define TX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->tx_sa) 139aa0a1e58SJeff Roberson 140aa0a1e58SJeff Roberson #ifndef MIN 141aa0a1e58SJeff Roberson #define MIN(a, b) (a < b ? a : b) 142aa0a1e58SJeff Roberson #endif 143aa0a1e58SJeff Roberson 144aa0a1e58SJeff Roberson #define ring_head(ring) (atomic_read(&(ring).head)) 145aa0a1e58SJeff Roberson #define ring_tail(ring) (atomic_read(&(ring).tail)) 146aa0a1e58SJeff Roberson #define ring_posted(ring) (ring_head(ring) - ring_tail(ring)) 147aa0a1e58SJeff Roberson 148aa0a1e58SJeff Roberson #define rx_ring_posted(ssk) ring_posted(ssk->rx_ring) 149aa0a1e58SJeff Roberson #ifdef SDP_ZCOPY 150aa0a1e58SJeff Roberson #define tx_ring_posted(ssk) (ring_posted(ssk->tx_ring) + \ 151aa0a1e58SJeff Roberson (ssk->tx_ring.rdma_inflight ? ssk->tx_ring.rdma_inflight->busy : 0)) 152aa0a1e58SJeff Roberson #else 153aa0a1e58SJeff Roberson #define tx_ring_posted(ssk) ring_posted(ssk->tx_ring) 154aa0a1e58SJeff Roberson #endif 155aa0a1e58SJeff Roberson 156aa0a1e58SJeff Roberson extern int sdp_zcopy_thresh; 157aa0a1e58SJeff Roberson extern int rcvbuf_initial_size; 158aa0a1e58SJeff Roberson extern struct workqueue_struct *rx_comp_wq; 159aa0a1e58SJeff Roberson extern struct ib_client sdp_client; 160aa0a1e58SJeff Roberson 161aa0a1e58SJeff Roberson enum sdp_mid { 162aa0a1e58SJeff Roberson SDP_MID_HELLO = 0x0, 163aa0a1e58SJeff Roberson SDP_MID_HELLO_ACK = 0x1, 164aa0a1e58SJeff Roberson SDP_MID_DISCONN = 0x2, 165aa0a1e58SJeff Roberson SDP_MID_ABORT = 0x3, 166aa0a1e58SJeff Roberson SDP_MID_SENDSM = 0x4, 167aa0a1e58SJeff Roberson SDP_MID_RDMARDCOMPL = 0x6, 168aa0a1e58SJeff Roberson SDP_MID_SRCAVAIL_CANCEL = 0x8, 169aa0a1e58SJeff Roberson SDP_MID_CHRCVBUF = 0xB, 170aa0a1e58SJeff Roberson SDP_MID_CHRCVBUF_ACK = 0xC, 171aa0a1e58SJeff Roberson SDP_MID_SINKAVAIL = 0xFD, 172aa0a1e58SJeff Roberson SDP_MID_SRCAVAIL = 0xFE, 173aa0a1e58SJeff Roberson SDP_MID_DATA = 0xFF, 174aa0a1e58SJeff Roberson }; 175aa0a1e58SJeff Roberson 176aa0a1e58SJeff Roberson enum sdp_flags { 177aa0a1e58SJeff Roberson SDP_OOB_PRES = 1 << 0, 178aa0a1e58SJeff Roberson SDP_OOB_PEND = 1 << 1, 179aa0a1e58SJeff Roberson }; 180aa0a1e58SJeff Roberson 181aa0a1e58SJeff Roberson enum { 182aa0a1e58SJeff Roberson SDP_MIN_TX_CREDITS = 2 183aa0a1e58SJeff Roberson }; 184aa0a1e58SJeff Roberson 185aa0a1e58SJeff Roberson enum { 186aa0a1e58SJeff Roberson SDP_ERR_ERROR = -4, 187aa0a1e58SJeff Roberson SDP_ERR_FAULT = -3, 188aa0a1e58SJeff Roberson SDP_NEW_SEG = -2, 189aa0a1e58SJeff Roberson SDP_DO_WAIT_MEM = -1 190aa0a1e58SJeff Roberson }; 191aa0a1e58SJeff Roberson 192aa0a1e58SJeff Roberson struct sdp_bsdh { 193aa0a1e58SJeff Roberson u8 mid; 194aa0a1e58SJeff Roberson u8 flags; 195aa0a1e58SJeff Roberson __u16 bufs; 196aa0a1e58SJeff Roberson __u32 len; 197aa0a1e58SJeff Roberson __u32 mseq; 198aa0a1e58SJeff Roberson __u32 mseq_ack; 199aa0a1e58SJeff Roberson } __attribute__((__packed__)); 200aa0a1e58SJeff Roberson 201aa0a1e58SJeff Roberson union cma_ip_addr { 202aa0a1e58SJeff Roberson struct in6_addr ip6; 203aa0a1e58SJeff Roberson struct { 204aa0a1e58SJeff Roberson __u32 pad[3]; 205aa0a1e58SJeff Roberson __u32 addr; 206aa0a1e58SJeff Roberson } ip4; 207aa0a1e58SJeff Roberson } __attribute__((__packed__)); 208aa0a1e58SJeff Roberson 209aa0a1e58SJeff Roberson /* TODO: too much? Can I avoid having the src/dst and port here? */ 210aa0a1e58SJeff Roberson struct sdp_hh { 211aa0a1e58SJeff Roberson struct sdp_bsdh bsdh; 212aa0a1e58SJeff Roberson u8 majv_minv; 213aa0a1e58SJeff Roberson u8 ipv_cap; 214aa0a1e58SJeff Roberson u8 rsvd1; 215aa0a1e58SJeff Roberson u8 max_adverts; 216aa0a1e58SJeff Roberson __u32 desremrcvsz; 217aa0a1e58SJeff Roberson __u32 localrcvsz; 218aa0a1e58SJeff Roberson __u16 port; 219aa0a1e58SJeff Roberson __u16 rsvd2; 220aa0a1e58SJeff Roberson union cma_ip_addr src_addr; 221aa0a1e58SJeff Roberson union cma_ip_addr dst_addr; 222aa0a1e58SJeff Roberson u8 rsvd3[IB_CM_REQ_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 48]; 223aa0a1e58SJeff Roberson } __attribute__((__packed__)); 224aa0a1e58SJeff Roberson 225aa0a1e58SJeff Roberson struct sdp_hah { 226aa0a1e58SJeff Roberson struct sdp_bsdh bsdh; 227aa0a1e58SJeff Roberson u8 majv_minv; 228aa0a1e58SJeff Roberson u8 ipv_cap; 229aa0a1e58SJeff Roberson u8 rsvd1; 230aa0a1e58SJeff Roberson u8 ext_max_adverts; 231aa0a1e58SJeff Roberson __u32 actrcvsz; 232aa0a1e58SJeff Roberson u8 rsvd2[IB_CM_REP_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 8]; 233aa0a1e58SJeff Roberson } __attribute__((__packed__)); 234aa0a1e58SJeff Roberson 235aa0a1e58SJeff Roberson struct sdp_rrch { 236aa0a1e58SJeff Roberson __u32 len; 237aa0a1e58SJeff Roberson } __attribute__((__packed__)); 238aa0a1e58SJeff Roberson 239aa0a1e58SJeff Roberson struct sdp_srcah { 240aa0a1e58SJeff Roberson __u32 len; 241aa0a1e58SJeff Roberson __u32 rkey; 242aa0a1e58SJeff Roberson __u64 vaddr; 243aa0a1e58SJeff Roberson } __attribute__((__packed__)); 244aa0a1e58SJeff Roberson 245aa0a1e58SJeff Roberson struct sdp_buf { 246aa0a1e58SJeff Roberson struct mbuf *mb; 247aa0a1e58SJeff Roberson u64 mapping[SDP_MAX_SEND_SGES]; 248aa0a1e58SJeff Roberson } __attribute__((__packed__)); 249aa0a1e58SJeff Roberson 250aa0a1e58SJeff Roberson struct sdp_chrecvbuf { 251aa0a1e58SJeff Roberson u32 size; 252aa0a1e58SJeff Roberson } __attribute__((__packed__)); 253aa0a1e58SJeff Roberson 254aa0a1e58SJeff Roberson /* Context used for synchronous zero copy bcopy (BZCOPY) */ 255aa0a1e58SJeff Roberson struct bzcopy_state { 256aa0a1e58SJeff Roberson unsigned char __user *u_base; 257aa0a1e58SJeff Roberson int u_len; 258aa0a1e58SJeff Roberson int left; 259aa0a1e58SJeff Roberson int page_cnt; 260aa0a1e58SJeff Roberson int cur_page; 261aa0a1e58SJeff Roberson int cur_offset; 262aa0a1e58SJeff Roberson int busy; 263aa0a1e58SJeff Roberson struct sdp_sock *ssk; 264aa0a1e58SJeff Roberson struct page **pages; 265aa0a1e58SJeff Roberson }; 266aa0a1e58SJeff Roberson 267aa0a1e58SJeff Roberson enum rx_sa_flag { 268aa0a1e58SJeff Roberson RX_SA_ABORTED = 2, 269aa0a1e58SJeff Roberson }; 270aa0a1e58SJeff Roberson 271aa0a1e58SJeff Roberson enum tx_sa_flag { 272aa0a1e58SJeff Roberson TX_SA_SENDSM = 0x01, 273aa0a1e58SJeff Roberson TX_SA_CROSS_SEND = 0x02, 274aa0a1e58SJeff Roberson TX_SA_INTRRUPTED = 0x04, 275aa0a1e58SJeff Roberson TX_SA_TIMEDOUT = 0x08, 276aa0a1e58SJeff Roberson TX_SA_ERROR = 0x10, 277aa0a1e58SJeff Roberson }; 278aa0a1e58SJeff Roberson 279aa0a1e58SJeff Roberson struct rx_srcavail_state { 280aa0a1e58SJeff Roberson /* Advertised buffer stuff */ 281aa0a1e58SJeff Roberson u32 mseq; 282aa0a1e58SJeff Roberson u32 used; 283aa0a1e58SJeff Roberson u32 reported; 284aa0a1e58SJeff Roberson u32 len; 285aa0a1e58SJeff Roberson u32 rkey; 286aa0a1e58SJeff Roberson u64 vaddr; 287aa0a1e58SJeff Roberson 288aa0a1e58SJeff Roberson /* Dest buff info */ 289aa0a1e58SJeff Roberson struct ib_umem *umem; 290aa0a1e58SJeff Roberson struct ib_pool_fmr *fmr; 291aa0a1e58SJeff Roberson 292aa0a1e58SJeff Roberson /* Utility */ 293aa0a1e58SJeff Roberson u8 busy; 294aa0a1e58SJeff Roberson enum rx_sa_flag flags; 295aa0a1e58SJeff Roberson }; 296aa0a1e58SJeff Roberson 297aa0a1e58SJeff Roberson struct tx_srcavail_state { 298aa0a1e58SJeff Roberson /* Data below 'busy' will be reset */ 299aa0a1e58SJeff Roberson u8 busy; 300aa0a1e58SJeff Roberson 301aa0a1e58SJeff Roberson struct ib_umem *umem; 302aa0a1e58SJeff Roberson struct ib_pool_fmr *fmr; 303aa0a1e58SJeff Roberson 304aa0a1e58SJeff Roberson u32 bytes_sent; 305aa0a1e58SJeff Roberson u32 bytes_acked; 306aa0a1e58SJeff Roberson 307aa0a1e58SJeff Roberson enum tx_sa_flag abort_flags; 308aa0a1e58SJeff Roberson u8 posted; 309aa0a1e58SJeff Roberson 310aa0a1e58SJeff Roberson u32 mseq; 311aa0a1e58SJeff Roberson }; 312aa0a1e58SJeff Roberson 313aa0a1e58SJeff Roberson struct sdp_tx_ring { 314aa0a1e58SJeff Roberson #ifdef SDP_ZCOPY 315aa0a1e58SJeff Roberson struct rx_srcavail_state *rdma_inflight; 316aa0a1e58SJeff Roberson #endif 317aa0a1e58SJeff Roberson struct sdp_buf *buffer; 318aa0a1e58SJeff Roberson atomic_t head; 319aa0a1e58SJeff Roberson atomic_t tail; 320aa0a1e58SJeff Roberson struct ib_cq *cq; 321aa0a1e58SJeff Roberson 322aa0a1e58SJeff Roberson atomic_t credits; 323aa0a1e58SJeff Roberson #define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits)) 324aa0a1e58SJeff Roberson 325aa0a1e58SJeff Roberson struct callout timer; 326aa0a1e58SJeff Roberson u16 poll_cnt; 327aa0a1e58SJeff Roberson }; 328aa0a1e58SJeff Roberson 329aa0a1e58SJeff Roberson struct sdp_rx_ring { 330aa0a1e58SJeff Roberson struct sdp_buf *buffer; 331aa0a1e58SJeff Roberson atomic_t head; 332aa0a1e58SJeff Roberson atomic_t tail; 333aa0a1e58SJeff Roberson struct ib_cq *cq; 334aa0a1e58SJeff Roberson 335aa0a1e58SJeff Roberson int destroyed; 336aa0a1e58SJeff Roberson struct rwlock destroyed_lock; 337aa0a1e58SJeff Roberson }; 338aa0a1e58SJeff Roberson 339aa0a1e58SJeff Roberson struct sdp_device { 340aa0a1e58SJeff Roberson struct ib_pd *pd; 341aa0a1e58SJeff Roberson struct ib_mr *mr; 342aa0a1e58SJeff Roberson struct ib_fmr_pool *fmr_pool; 343aa0a1e58SJeff Roberson }; 344aa0a1e58SJeff Roberson 345aa0a1e58SJeff Roberson struct sdp_moderation { 346aa0a1e58SJeff Roberson unsigned long last_moder_packets; 347aa0a1e58SJeff Roberson unsigned long last_moder_tx_packets; 348aa0a1e58SJeff Roberson unsigned long last_moder_bytes; 349aa0a1e58SJeff Roberson unsigned long last_moder_jiffies; 350aa0a1e58SJeff Roberson int last_moder_time; 351aa0a1e58SJeff Roberson u16 rx_usecs; 352aa0a1e58SJeff Roberson u16 rx_frames; 353aa0a1e58SJeff Roberson u16 tx_usecs; 354aa0a1e58SJeff Roberson u32 pkt_rate_low; 355aa0a1e58SJeff Roberson u16 rx_usecs_low; 356aa0a1e58SJeff Roberson u32 pkt_rate_high; 357aa0a1e58SJeff Roberson u16 rx_usecs_high; 358aa0a1e58SJeff Roberson u16 sample_interval; 359aa0a1e58SJeff Roberson u16 adaptive_rx_coal; 360aa0a1e58SJeff Roberson u32 msg_enable; 361aa0a1e58SJeff Roberson 362aa0a1e58SJeff Roberson int moder_cnt; 363aa0a1e58SJeff Roberson int moder_time; 364aa0a1e58SJeff Roberson }; 365aa0a1e58SJeff Roberson 366aa0a1e58SJeff Roberson /* These are flags fields. */ 367aa0a1e58SJeff Roberson #define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */ 368aa0a1e58SJeff Roberson #define SDP_DROPPED 0x0002 /* Socket has been dropped. */ 369aa0a1e58SJeff Roberson #define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */ 370aa0a1e58SJeff Roberson #define SDP_NODELAY 0x0008 /* Disble nagle. */ 371aa0a1e58SJeff Roberson #define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */ 372aa0a1e58SJeff Roberson #define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */ 373aa0a1e58SJeff Roberson #define SDP_DESTROY 0x0040 /* Being destroyed. */ 374aa0a1e58SJeff Roberson #define SDP_DISCON 0x0080 /* rdma_disconnect is owed. */ 375aa0a1e58SJeff Roberson 376aa0a1e58SJeff Roberson /* These are oobflags */ 377aa0a1e58SJeff Roberson #define SDP_HADOOB 0x0001 /* Had OOB data. */ 378aa0a1e58SJeff Roberson #define SDP_HAVEOOB 0x0002 /* Have OOB data. */ 379aa0a1e58SJeff Roberson 380aa0a1e58SJeff Roberson struct sdp_sock { 381aa0a1e58SJeff Roberson LIST_ENTRY(sdp_sock) list; 382aa0a1e58SJeff Roberson struct socket *socket; 383aa0a1e58SJeff Roberson struct rdma_cm_id *id; 384aa0a1e58SJeff Roberson struct ib_device *ib_device; 385aa0a1e58SJeff Roberson struct sdp_device *sdp_dev; 386aa0a1e58SJeff Roberson struct ib_qp *qp; 387aa0a1e58SJeff Roberson struct ucred *cred; 388aa0a1e58SJeff Roberson struct callout keep2msl; /* 2msl and keepalive timer. */ 389aa0a1e58SJeff Roberson struct callout nagle_timer; /* timeout waiting for ack */ 390aa0a1e58SJeff Roberson struct ib_ucontext context; 391aa0a1e58SJeff Roberson in_port_t lport; 392aa0a1e58SJeff Roberson in_addr_t laddr; 393aa0a1e58SJeff Roberson in_port_t fport; 394aa0a1e58SJeff Roberson in_addr_t faddr; 395aa0a1e58SJeff Roberson int flags; 396aa0a1e58SJeff Roberson int oobflags; /* protected by rx lock. */ 397aa0a1e58SJeff Roberson int state; 398aa0a1e58SJeff Roberson int softerror; 399aa0a1e58SJeff Roberson int recv_bytes; /* Bytes per recv. buf including header */ 400aa0a1e58SJeff Roberson int xmit_size_goal; 401aa0a1e58SJeff Roberson char iobc; 402aa0a1e58SJeff Roberson 403aa0a1e58SJeff Roberson struct sdp_rx_ring rx_ring; 404aa0a1e58SJeff Roberson struct sdp_tx_ring tx_ring; 405aa0a1e58SJeff Roberson struct rwlock lock; 406aa0a1e58SJeff Roberson struct mbuf *rx_ctl_q; 407aa0a1e58SJeff Roberson struct mbuf *rx_ctl_tail; 408aa0a1e58SJeff Roberson 409aa0a1e58SJeff Roberson int qp_active; /* XXX Flag. */ 410aa0a1e58SJeff Roberson int max_sge; 411aa0a1e58SJeff Roberson struct work_struct rx_comp_work; 412aa0a1e58SJeff Roberson #define rcv_nxt(ssk) atomic_read(&(ssk->rcv_nxt)) 413aa0a1e58SJeff Roberson atomic_t rcv_nxt; 414aa0a1e58SJeff Roberson 415aa0a1e58SJeff Roberson /* SDP specific */ 416aa0a1e58SJeff Roberson atomic_t mseq_ack; 417aa0a1e58SJeff Roberson #define mseq_ack(ssk) (atomic_read(&ssk->mseq_ack)) 418aa0a1e58SJeff Roberson unsigned max_bufs; /* Initial buffers offered by other side */ 419aa0a1e58SJeff Roberson unsigned min_bufs; /* Low water mark to wake senders */ 420aa0a1e58SJeff Roberson 421aa0a1e58SJeff Roberson unsigned long nagle_last_unacked; /* mseq of lastest unacked packet */ 422aa0a1e58SJeff Roberson 423aa0a1e58SJeff Roberson atomic_t remote_credits; 424aa0a1e58SJeff Roberson #define remote_credits(ssk) (atomic_read(&ssk->remote_credits)) 425aa0a1e58SJeff Roberson int poll_cq; 426aa0a1e58SJeff Roberson 427aa0a1e58SJeff Roberson /* SDP slow start */ 428aa0a1e58SJeff Roberson int recv_request_head; /* mark the rx_head when the resize request 429bf5cba36SPedro F. Giffuni was received */ 430bf5cba36SPedro F. Giffuni int recv_request; /* XXX flag if request to resize was received */ 431aa0a1e58SJeff Roberson 432aa0a1e58SJeff Roberson unsigned long tx_packets; 433aa0a1e58SJeff Roberson unsigned long rx_packets; 434aa0a1e58SJeff Roberson unsigned long tx_bytes; 435aa0a1e58SJeff Roberson unsigned long rx_bytes; 436aa0a1e58SJeff Roberson struct sdp_moderation auto_mod; 437aa0a1e58SJeff Roberson struct task shutdown_task; 438aa0a1e58SJeff Roberson #ifdef SDP_ZCOPY 439aa0a1e58SJeff Roberson struct tx_srcavail_state *tx_sa; 440aa0a1e58SJeff Roberson struct rx_srcavail_state *rx_sa; 441aa0a1e58SJeff Roberson spinlock_t tx_sa_lock; 442aa0a1e58SJeff Roberson struct delayed_work srcavail_cancel_work; 443aa0a1e58SJeff Roberson int srcavail_cancel_mseq; 444aa0a1e58SJeff Roberson /* ZCOPY data: -1:use global; 0:disable zcopy; >0: zcopy threshold */ 445aa0a1e58SJeff Roberson int zcopy_thresh; 446aa0a1e58SJeff Roberson #endif 447aa0a1e58SJeff Roberson }; 448aa0a1e58SJeff Roberson 449aa0a1e58SJeff Roberson #define sdp_sk(so) ((struct sdp_sock *)(so->so_pcb)) 450aa0a1e58SJeff Roberson 451aa0a1e58SJeff Roberson #define SDP_RLOCK(ssk) rw_rlock(&(ssk)->lock) 452aa0a1e58SJeff Roberson #define SDP_WLOCK(ssk) rw_wlock(&(ssk)->lock) 453aa0a1e58SJeff Roberson #define SDP_RUNLOCK(ssk) rw_runlock(&(ssk)->lock) 454aa0a1e58SJeff Roberson #define SDP_WUNLOCK(ssk) rw_wunlock(&(ssk)->lock) 455aa0a1e58SJeff Roberson #define SDP_WLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_WLOCKED) 456aa0a1e58SJeff Roberson #define SDP_RLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_RLOCKED) 457aa0a1e58SJeff Roberson #define SDP_LOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_LOCKED) 458aa0a1e58SJeff Roberson 459aa0a1e58SJeff Roberson static inline void tx_sa_reset(struct tx_srcavail_state *tx_sa) 460aa0a1e58SJeff Roberson { 461aa0a1e58SJeff Roberson memset((void *)&tx_sa->busy, 0, 462aa0a1e58SJeff Roberson sizeof(*tx_sa) - offsetof(typeof(*tx_sa), busy)); 463aa0a1e58SJeff Roberson } 464aa0a1e58SJeff Roberson 465aa0a1e58SJeff Roberson static inline void rx_ring_unlock(struct sdp_rx_ring *rx_ring) 466aa0a1e58SJeff Roberson { 467aa0a1e58SJeff Roberson rw_runlock(&rx_ring->destroyed_lock); 468aa0a1e58SJeff Roberson } 469aa0a1e58SJeff Roberson 470aa0a1e58SJeff Roberson static inline int rx_ring_trylock(struct sdp_rx_ring *rx_ring) 471aa0a1e58SJeff Roberson { 472aa0a1e58SJeff Roberson rw_rlock(&rx_ring->destroyed_lock); 473aa0a1e58SJeff Roberson if (rx_ring->destroyed) { 474aa0a1e58SJeff Roberson rx_ring_unlock(rx_ring); 475aa0a1e58SJeff Roberson return 0; 476aa0a1e58SJeff Roberson } 477aa0a1e58SJeff Roberson return 1; 478aa0a1e58SJeff Roberson } 479aa0a1e58SJeff Roberson 480aa0a1e58SJeff Roberson static inline void rx_ring_destroy_lock(struct sdp_rx_ring *rx_ring) 481aa0a1e58SJeff Roberson { 482aa0a1e58SJeff Roberson rw_wlock(&rx_ring->destroyed_lock); 483aa0a1e58SJeff Roberson rx_ring->destroyed = 1; 484aa0a1e58SJeff Roberson rw_wunlock(&rx_ring->destroyed_lock); 485aa0a1e58SJeff Roberson } 486aa0a1e58SJeff Roberson 487aa0a1e58SJeff Roberson static inline void sdp_arm_rx_cq(struct sdp_sock *ssk) 488aa0a1e58SJeff Roberson { 489aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "Arming RX cq"); 490aa0a1e58SJeff Roberson sdp_dbg_data(ssk->socket, "Arming RX cq\n"); 491aa0a1e58SJeff Roberson 492aa0a1e58SJeff Roberson ib_req_notify_cq(ssk->rx_ring.cq, IB_CQ_NEXT_COMP); 493aa0a1e58SJeff Roberson } 494aa0a1e58SJeff Roberson 495aa0a1e58SJeff Roberson static inline void sdp_arm_tx_cq(struct sdp_sock *ssk) 496aa0a1e58SJeff Roberson { 497aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "Arming TX cq"); 498aa0a1e58SJeff Roberson sdp_dbg_data(ssk->socket, "Arming TX cq. credits: %d, posted: %d\n", 499aa0a1e58SJeff Roberson tx_credits(ssk), tx_ring_posted(ssk)); 500aa0a1e58SJeff Roberson 501aa0a1e58SJeff Roberson ib_req_notify_cq(ssk->tx_ring.cq, IB_CQ_NEXT_COMP); 502aa0a1e58SJeff Roberson } 503aa0a1e58SJeff Roberson 504aa0a1e58SJeff Roberson /* return the min of: 505aa0a1e58SJeff Roberson * - tx credits 506aa0a1e58SJeff Roberson * - free slots in tx_ring (not including SDP_MIN_TX_CREDITS 507aa0a1e58SJeff Roberson */ 508aa0a1e58SJeff Roberson static inline int tx_slots_free(struct sdp_sock *ssk) 509aa0a1e58SJeff Roberson { 510aa0a1e58SJeff Roberson int min_free; 511aa0a1e58SJeff Roberson 512aa0a1e58SJeff Roberson min_free = MIN(tx_credits(ssk), 513aa0a1e58SJeff Roberson SDP_TX_SIZE - tx_ring_posted(ssk)); 514aa0a1e58SJeff Roberson if (min_free < SDP_MIN_TX_CREDITS) 515aa0a1e58SJeff Roberson return 0; 516aa0a1e58SJeff Roberson 517aa0a1e58SJeff Roberson return min_free - SDP_MIN_TX_CREDITS; 518aa0a1e58SJeff Roberson }; 519aa0a1e58SJeff Roberson 520aa0a1e58SJeff Roberson /* utilities */ 521aa0a1e58SJeff Roberson static inline char *mid2str(int mid) 522aa0a1e58SJeff Roberson { 523aa0a1e58SJeff Roberson #define ENUM2STR(e) [e] = #e 524aa0a1e58SJeff Roberson static char *mid2str[] = { 525aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_HELLO), 526aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_HELLO_ACK), 527aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_ABORT), 528aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_DISCONN), 529aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_SENDSM), 530aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_RDMARDCOMPL), 531aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_SRCAVAIL_CANCEL), 532aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_CHRCVBUF), 533aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_CHRCVBUF_ACK), 534aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_DATA), 535aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_SRCAVAIL), 536aa0a1e58SJeff Roberson ENUM2STR(SDP_MID_SINKAVAIL), 537aa0a1e58SJeff Roberson }; 538aa0a1e58SJeff Roberson 539aa0a1e58SJeff Roberson if (mid >= ARRAY_SIZE(mid2str)) 540aa0a1e58SJeff Roberson return NULL; 541aa0a1e58SJeff Roberson 542aa0a1e58SJeff Roberson return mid2str[mid]; 543aa0a1e58SJeff Roberson } 544aa0a1e58SJeff Roberson 545aa0a1e58SJeff Roberson static inline struct mbuf * 546aa0a1e58SJeff Roberson sdp_alloc_mb(struct socket *sk, u8 mid, int size, int wait) 547aa0a1e58SJeff Roberson { 548aa0a1e58SJeff Roberson struct sdp_bsdh *h; 549aa0a1e58SJeff Roberson struct mbuf *mb; 550aa0a1e58SJeff Roberson 551aa0a1e58SJeff Roberson MGETHDR(mb, wait, MT_DATA); 552aa0a1e58SJeff Roberson if (mb == NULL) 553aa0a1e58SJeff Roberson return (NULL); 554aa0a1e58SJeff Roberson mb->m_pkthdr.len = mb->m_len = sizeof(struct sdp_bsdh); 555aa0a1e58SJeff Roberson h = mtod(mb, struct sdp_bsdh *); 556aa0a1e58SJeff Roberson h->mid = mid; 557aa0a1e58SJeff Roberson 558aa0a1e58SJeff Roberson return mb; 559aa0a1e58SJeff Roberson } 560aa0a1e58SJeff Roberson static inline struct mbuf * 561aa0a1e58SJeff Roberson sdp_alloc_mb_data(struct socket *sk, int wait) 562aa0a1e58SJeff Roberson { 563aa0a1e58SJeff Roberson return sdp_alloc_mb(sk, SDP_MID_DATA, 0, wait); 564aa0a1e58SJeff Roberson } 565aa0a1e58SJeff Roberson 566aa0a1e58SJeff Roberson static inline struct mbuf * 567aa0a1e58SJeff Roberson sdp_alloc_mb_disconnect(struct socket *sk, int wait) 568aa0a1e58SJeff Roberson { 569aa0a1e58SJeff Roberson return sdp_alloc_mb(sk, SDP_MID_DISCONN, 0, wait); 570aa0a1e58SJeff Roberson } 571aa0a1e58SJeff Roberson 572aa0a1e58SJeff Roberson static inline void * 573aa0a1e58SJeff Roberson mb_put(struct mbuf *mb, int len) 574aa0a1e58SJeff Roberson { 575aa0a1e58SJeff Roberson uint8_t *data; 576aa0a1e58SJeff Roberson 577aa0a1e58SJeff Roberson data = mb->m_data; 578aa0a1e58SJeff Roberson data += mb->m_len; 579aa0a1e58SJeff Roberson mb->m_len += len; 580aa0a1e58SJeff Roberson return (void *)data; 581aa0a1e58SJeff Roberson } 582aa0a1e58SJeff Roberson 583aa0a1e58SJeff Roberson static inline struct mbuf * 584aa0a1e58SJeff Roberson sdp_alloc_mb_chrcvbuf_ack(struct socket *sk, int size, int wait) 585aa0a1e58SJeff Roberson { 586aa0a1e58SJeff Roberson struct mbuf *mb; 587aa0a1e58SJeff Roberson struct sdp_chrecvbuf *resp_size; 588aa0a1e58SJeff Roberson 589aa0a1e58SJeff Roberson mb = sdp_alloc_mb(sk, SDP_MID_CHRCVBUF_ACK, sizeof(*resp_size), wait); 590aa0a1e58SJeff Roberson if (mb == NULL) 591aa0a1e58SJeff Roberson return (NULL); 592aa0a1e58SJeff Roberson resp_size = (struct sdp_chrecvbuf *)mb_put(mb, sizeof *resp_size); 593aa0a1e58SJeff Roberson resp_size->size = htonl(size); 594aa0a1e58SJeff Roberson 595aa0a1e58SJeff Roberson return mb; 596aa0a1e58SJeff Roberson } 597aa0a1e58SJeff Roberson 598aa0a1e58SJeff Roberson static inline struct mbuf * 599aa0a1e58SJeff Roberson sdp_alloc_mb_srcavail(struct socket *sk, u32 len, u32 rkey, u64 vaddr, int wait) 600aa0a1e58SJeff Roberson { 601aa0a1e58SJeff Roberson struct mbuf *mb; 602aa0a1e58SJeff Roberson struct sdp_srcah *srcah; 603aa0a1e58SJeff Roberson 604aa0a1e58SJeff Roberson mb = sdp_alloc_mb(sk, SDP_MID_SRCAVAIL, sizeof(*srcah), wait); 605aa0a1e58SJeff Roberson if (mb == NULL) 606aa0a1e58SJeff Roberson return (NULL); 607aa0a1e58SJeff Roberson srcah = (struct sdp_srcah *)mb_put(mb, sizeof(*srcah)); 608aa0a1e58SJeff Roberson srcah->len = htonl(len); 609aa0a1e58SJeff Roberson srcah->rkey = htonl(rkey); 610aa0a1e58SJeff Roberson srcah->vaddr = cpu_to_be64(vaddr); 611aa0a1e58SJeff Roberson 612aa0a1e58SJeff Roberson return mb; 613aa0a1e58SJeff Roberson } 614aa0a1e58SJeff Roberson 615aa0a1e58SJeff Roberson static inline struct mbuf * 616aa0a1e58SJeff Roberson sdp_alloc_mb_srcavail_cancel(struct socket *sk, int wait) 617aa0a1e58SJeff Roberson { 618aa0a1e58SJeff Roberson return sdp_alloc_mb(sk, SDP_MID_SRCAVAIL_CANCEL, 0, wait); 619aa0a1e58SJeff Roberson } 620aa0a1e58SJeff Roberson 621aa0a1e58SJeff Roberson static inline struct mbuf * 622aa0a1e58SJeff Roberson sdp_alloc_mb_rdmardcompl(struct socket *sk, u32 len, int wait) 623aa0a1e58SJeff Roberson { 624aa0a1e58SJeff Roberson struct mbuf *mb; 625aa0a1e58SJeff Roberson struct sdp_rrch *rrch; 626aa0a1e58SJeff Roberson 627aa0a1e58SJeff Roberson mb = sdp_alloc_mb(sk, SDP_MID_RDMARDCOMPL, sizeof(*rrch), wait); 628aa0a1e58SJeff Roberson if (mb == NULL) 629aa0a1e58SJeff Roberson return (NULL); 630aa0a1e58SJeff Roberson rrch = (struct sdp_rrch *)mb_put(mb, sizeof(*rrch)); 631aa0a1e58SJeff Roberson rrch->len = htonl(len); 632aa0a1e58SJeff Roberson 633aa0a1e58SJeff Roberson return mb; 634aa0a1e58SJeff Roberson } 635aa0a1e58SJeff Roberson 636aa0a1e58SJeff Roberson static inline struct mbuf * 637aa0a1e58SJeff Roberson sdp_alloc_mb_sendsm(struct socket *sk, int wait) 638aa0a1e58SJeff Roberson { 639aa0a1e58SJeff Roberson return sdp_alloc_mb(sk, SDP_MID_SENDSM, 0, wait); 640aa0a1e58SJeff Roberson } 641aa0a1e58SJeff Roberson static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk) 642aa0a1e58SJeff Roberson { 643aa0a1e58SJeff Roberson return SDP_TX_SIZE - tx_ring_posted(ssk); 644aa0a1e58SJeff Roberson } 645aa0a1e58SJeff Roberson 646aa0a1e58SJeff Roberson static inline int credit_update_needed(struct sdp_sock *ssk) 647aa0a1e58SJeff Roberson { 648aa0a1e58SJeff Roberson int c; 649aa0a1e58SJeff Roberson 650aa0a1e58SJeff Roberson c = remote_credits(ssk); 651aa0a1e58SJeff Roberson if (likely(c > SDP_MIN_TX_CREDITS)) 652aa0a1e58SJeff Roberson c += c/2; 653aa0a1e58SJeff Roberson return unlikely(c < rx_ring_posted(ssk)) && 654aa0a1e58SJeff Roberson likely(tx_credits(ssk) > 0) && 655aa0a1e58SJeff Roberson likely(sdp_tx_ring_slots_left(ssk)); 656aa0a1e58SJeff Roberson } 657aa0a1e58SJeff Roberson 658aa0a1e58SJeff Roberson 659aa0a1e58SJeff Roberson #define SDPSTATS_COUNTER_INC(stat) 660aa0a1e58SJeff Roberson #define SDPSTATS_COUNTER_ADD(stat, val) 661aa0a1e58SJeff Roberson #define SDPSTATS_COUNTER_MID_INC(stat, mid) 662aa0a1e58SJeff Roberson #define SDPSTATS_HIST_LINEAR(stat, size) 663aa0a1e58SJeff Roberson #define SDPSTATS_HIST(stat, size) 664aa0a1e58SJeff Roberson 665aa0a1e58SJeff Roberson static inline void 666aa0a1e58SJeff Roberson sdp_cleanup_sdp_buf(struct sdp_sock *ssk, struct sdp_buf *sbuf, 667aa0a1e58SJeff Roberson enum dma_data_direction dir) 668aa0a1e58SJeff Roberson { 669aa0a1e58SJeff Roberson struct ib_device *dev; 670aa0a1e58SJeff Roberson struct mbuf *mb; 671aa0a1e58SJeff Roberson int i; 672aa0a1e58SJeff Roberson 673aa0a1e58SJeff Roberson dev = ssk->ib_device; 674aa0a1e58SJeff Roberson for (i = 0, mb = sbuf->mb; mb != NULL; mb = mb->m_next, i++) 675aa0a1e58SJeff Roberson ib_dma_unmap_single(dev, sbuf->mapping[i], mb->m_len, dir); 676aa0a1e58SJeff Roberson } 677aa0a1e58SJeff Roberson 678aa0a1e58SJeff Roberson /* sdp_main.c */ 679aa0a1e58SJeff Roberson void sdp_set_default_moderation(struct sdp_sock *ssk); 680aa0a1e58SJeff Roberson void sdp_start_keepalive_timer(struct socket *sk); 681aa0a1e58SJeff Roberson void sdp_urg(struct sdp_sock *ssk, struct mbuf *mb); 682aa0a1e58SJeff Roberson void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk); 683aa0a1e58SJeff Roberson void sdp_abort(struct socket *sk); 684aa0a1e58SJeff Roberson struct sdp_sock *sdp_notify(struct sdp_sock *ssk, int error); 685aa0a1e58SJeff Roberson 686aa0a1e58SJeff Roberson 687aa0a1e58SJeff Roberson /* sdp_cma.c */ 688aa0a1e58SJeff Roberson int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *); 689aa0a1e58SJeff Roberson 690aa0a1e58SJeff Roberson /* sdp_tx.c */ 691aa0a1e58SJeff Roberson int sdp_tx_ring_create(struct sdp_sock *ssk, struct ib_device *device); 692aa0a1e58SJeff Roberson void sdp_tx_ring_destroy(struct sdp_sock *ssk); 693aa0a1e58SJeff Roberson int sdp_xmit_poll(struct sdp_sock *ssk, int force); 694aa0a1e58SJeff Roberson void sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb); 695aa0a1e58SJeff Roberson void sdp_post_sends(struct sdp_sock *ssk, int wait); 696aa0a1e58SJeff Roberson void sdp_post_keepalive(struct sdp_sock *ssk); 697aa0a1e58SJeff Roberson 698aa0a1e58SJeff Roberson /* sdp_rx.c */ 699aa0a1e58SJeff Roberson void sdp_rx_ring_init(struct sdp_sock *ssk); 700aa0a1e58SJeff Roberson int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device); 701aa0a1e58SJeff Roberson void sdp_rx_ring_destroy(struct sdp_sock *ssk); 702aa0a1e58SJeff Roberson int sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size); 703aa0a1e58SJeff Roberson int sdp_init_buffers(struct sdp_sock *ssk, u32 new_size); 704aa0a1e58SJeff Roberson void sdp_do_posts(struct sdp_sock *ssk); 705aa0a1e58SJeff Roberson void sdp_rx_comp_full(struct sdp_sock *ssk); 706aa0a1e58SJeff Roberson 707aa0a1e58SJeff Roberson /* sdp_zcopy.c */ 708a1081609SBjoern A. Zeeb struct kiocb; 709aa0a1e58SJeff Roberson int sdp_sendmsg_zcopy(struct kiocb *iocb, struct socket *sk, struct iovec *iov); 710aa0a1e58SJeff Roberson int sdp_handle_srcavail(struct sdp_sock *ssk, struct sdp_srcah *srcah); 711aa0a1e58SJeff Roberson void sdp_handle_sendsm(struct sdp_sock *ssk, u32 mseq_ack); 712aa0a1e58SJeff Roberson void sdp_handle_rdma_read_compl(struct sdp_sock *ssk, u32 mseq_ack, 713aa0a1e58SJeff Roberson u32 bytes_completed); 714aa0a1e58SJeff Roberson int sdp_handle_rdma_read_cqe(struct sdp_sock *ssk); 715aa0a1e58SJeff Roberson int sdp_rdma_to_iovec(struct socket *sk, struct iovec *iov, struct mbuf *mb, 716aa0a1e58SJeff Roberson unsigned long *used); 717aa0a1e58SJeff Roberson int sdp_post_rdma_rd_compl(struct sdp_sock *ssk, 718aa0a1e58SJeff Roberson struct rx_srcavail_state *rx_sa); 719aa0a1e58SJeff Roberson int sdp_post_sendsm(struct socket *sk); 720aa0a1e58SJeff Roberson void srcavail_cancel_timeout(struct work_struct *work); 721aa0a1e58SJeff Roberson void sdp_abort_srcavail(struct socket *sk); 722aa0a1e58SJeff Roberson void sdp_abort_rdma_read(struct socket *sk); 723aa0a1e58SJeff Roberson int sdp_process_rx(struct sdp_sock *ssk); 724aa0a1e58SJeff Roberson 725aa0a1e58SJeff Roberson #endif 726