1 #ifndef _SDP_H_ 2 #define _SDP_H_ 3 4 #include "opt_ddb.h" 5 #include "opt_inet.h" 6 #include "opt_ofed.h" 7 8 #include <sys/param.h> 9 #include <sys/systm.h> 10 #include <sys/malloc.h> 11 #include <sys/kernel.h> 12 #include <sys/sysctl.h> 13 #include <sys/mbuf.h> 14 #include <sys/lock.h> 15 #include <sys/rwlock.h> 16 #include <sys/socket.h> 17 #include <sys/socketvar.h> 18 #include <sys/protosw.h> 19 #include <sys/proc.h> 20 #include <sys/jail.h> 21 #include <sys/domain.h> 22 23 #ifdef DDB 24 #include <ddb/ddb.h> 25 #endif 26 27 #include <net/if.h> 28 #include <net/if_var.h> 29 #include <net/route.h> 30 #include <net/vnet.h> 31 32 #include <netinet/in.h> 33 #include <netinet/in_systm.h> 34 #include <netinet/in_var.h> 35 #include <netinet/in_pcb.h> 36 #include <netinet/tcp.h> 37 #include <netinet/tcp_fsm.h> 38 #include <netinet/tcp_timer.h> 39 #include <netinet/tcp_var.h> 40 41 #include <linux/device.h> 42 #include <linux/err.h> 43 #include <linux/sched.h> 44 #include <linux/workqueue.h> 45 #include <linux/wait.h> 46 #include <linux/module.h> 47 #include <linux/moduleparam.h> 48 #include <linux/pci.h> 49 50 #include <rdma/ib_verbs.h> 51 #include <rdma/rdma_cm.h> 52 #include <rdma/ib_cm.h> 53 #include <rdma/sdp_socket.h> 54 #include <rdma/ib_fmr_pool.h> 55 56 #ifdef SDP_DEBUG 57 #define CONFIG_INFINIBAND_SDP_DEBUG 58 #endif 59 60 #include "sdp_dbg.h" 61 62 #undef LIST_HEAD 63 /* From sys/queue.h */ 64 #define LIST_HEAD(name, type) \ 65 struct name { \ 66 struct type *lh_first; /* first element */ \ 67 } 68 69 /* Interval between sucessive polls in the Tx routine when polling is used 70 instead of interrupts (in per-core Tx rings) - should be power of 2 */ 71 #define SDP_TX_POLL_MODER 16 72 #define SDP_TX_POLL_TIMEOUT (HZ / 20) 73 #define SDP_NAGLE_TIMEOUT (HZ / 10) 74 75 #define SDP_SRCAVAIL_CANCEL_TIMEOUT (HZ * 5) 76 #define SDP_SRCAVAIL_ADV_TIMEOUT (1 * HZ) 77 #define SDP_SRCAVAIL_PAYLOAD_LEN 1 78 79 #define SDP_RESOLVE_TIMEOUT 1000 80 #define SDP_ROUTE_TIMEOUT 1000 81 #define SDP_RETRY_COUNT 5 82 #define SDP_KEEPALIVE_TIME (120 * 60 * HZ) 83 #define SDP_FIN_WAIT_TIMEOUT (60 * HZ) /* like TCP_FIN_TIMEOUT */ 84 85 #define SDP_TX_SIZE 0x40 86 #define SDP_RX_SIZE 0x40 87 88 #define SDP_FMR_SIZE (MIN(0x1000, PAGE_SIZE) / sizeof(u64)) 89 #define SDP_FMR_POOL_SIZE 1024 90 #define SDP_FMR_DIRTY_SIZE ( SDP_FMR_POOL_SIZE / 4 ) 91 92 #define SDP_MAX_RDMA_READ_LEN (PAGE_SIZE * (SDP_FMR_SIZE - 2)) 93 94 /* mb inlined data len - rest will be rx'ed into frags */ 95 #define SDP_HEAD_SIZE (sizeof(struct sdp_bsdh)) 96 97 /* limit tx payload len, if the sink supports bigger buffers than the source 98 * can handle. 99 * or rx fragment size (limited by sge->length size) */ 100 #define SDP_MAX_PACKET (1 << 16) 101 #define SDP_MAX_PAYLOAD (SDP_MAX_PACKET - SDP_HEAD_SIZE) 102 103 #define SDP_MAX_RECV_SGES (SDP_MAX_PACKET / MCLBYTES) 104 #define SDP_MAX_SEND_SGES (SDP_MAX_PACKET / MCLBYTES) + 2 105 106 #define SDP_NUM_WC 4 107 108 #define SDP_DEF_ZCOPY_THRESH 64*1024 109 #define SDP_MIN_ZCOPY_THRESH PAGE_SIZE 110 #define SDP_MAX_ZCOPY_THRESH 1048576 111 112 #define SDP_OP_RECV 0x800000000LL 113 #define SDP_OP_SEND 0x400000000LL 114 #define SDP_OP_RDMA 0x200000000LL 115 #define SDP_OP_NOP 0x100000000LL 116 117 /* how long (in jiffies) to block sender till tx completion*/ 118 #define SDP_BZCOPY_POLL_TIMEOUT (HZ / 10) 119 120 #define SDP_AUTO_CONF 0xffff 121 #define AUTO_MOD_DELAY (HZ / 4) 122 123 struct sdp_mb_cb { 124 __u32 seq; /* Starting sequence number */ 125 struct bzcopy_state *bz; 126 struct rx_srcavail_state *rx_sa; 127 struct tx_srcavail_state *tx_sa; 128 }; 129 130 #define M_PUSH M_PROTO1 /* Do a 'push'. */ 131 #define M_URG M_PROTO2 /* Mark as urgent (oob). */ 132 133 #define SDP_SKB_CB(__mb) ((struct sdp_mb_cb *)&((__mb)->cb[0])) 134 #define BZCOPY_STATE(mb) (SDP_SKB_CB(mb)->bz) 135 #define RX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->rx_sa) 136 #define TX_SRCAVAIL_STATE(mb) (SDP_SKB_CB(mb)->tx_sa) 137 138 #ifndef MIN 139 #define MIN(a, b) (a < b ? a : b) 140 #endif 141 142 #define ring_head(ring) (atomic_read(&(ring).head)) 143 #define ring_tail(ring) (atomic_read(&(ring).tail)) 144 #define ring_posted(ring) (ring_head(ring) - ring_tail(ring)) 145 146 #define rx_ring_posted(ssk) ring_posted(ssk->rx_ring) 147 #ifdef SDP_ZCOPY 148 #define tx_ring_posted(ssk) (ring_posted(ssk->tx_ring) + \ 149 (ssk->tx_ring.rdma_inflight ? ssk->tx_ring.rdma_inflight->busy : 0)) 150 #else 151 #define tx_ring_posted(ssk) ring_posted(ssk->tx_ring) 152 #endif 153 154 extern int sdp_zcopy_thresh; 155 extern int rcvbuf_initial_size; 156 extern struct workqueue_struct *rx_comp_wq; 157 extern struct ib_client sdp_client; 158 159 enum sdp_mid { 160 SDP_MID_HELLO = 0x0, 161 SDP_MID_HELLO_ACK = 0x1, 162 SDP_MID_DISCONN = 0x2, 163 SDP_MID_ABORT = 0x3, 164 SDP_MID_SENDSM = 0x4, 165 SDP_MID_RDMARDCOMPL = 0x6, 166 SDP_MID_SRCAVAIL_CANCEL = 0x8, 167 SDP_MID_CHRCVBUF = 0xB, 168 SDP_MID_CHRCVBUF_ACK = 0xC, 169 SDP_MID_SINKAVAIL = 0xFD, 170 SDP_MID_SRCAVAIL = 0xFE, 171 SDP_MID_DATA = 0xFF, 172 }; 173 174 enum sdp_flags { 175 SDP_OOB_PRES = 1 << 0, 176 SDP_OOB_PEND = 1 << 1, 177 }; 178 179 enum { 180 SDP_MIN_TX_CREDITS = 2 181 }; 182 183 enum { 184 SDP_ERR_ERROR = -4, 185 SDP_ERR_FAULT = -3, 186 SDP_NEW_SEG = -2, 187 SDP_DO_WAIT_MEM = -1 188 }; 189 190 struct sdp_bsdh { 191 u8 mid; 192 u8 flags; 193 __u16 bufs; 194 __u32 len; 195 __u32 mseq; 196 __u32 mseq_ack; 197 } __attribute__((__packed__)); 198 199 union cma_ip_addr { 200 struct in6_addr ip6; 201 struct { 202 __u32 pad[3]; 203 __u32 addr; 204 } ip4; 205 } __attribute__((__packed__)); 206 207 /* TODO: too much? Can I avoid having the src/dst and port here? */ 208 struct sdp_hh { 209 struct sdp_bsdh bsdh; 210 u8 majv_minv; 211 u8 ipv_cap; 212 u8 rsvd1; 213 u8 max_adverts; 214 __u32 desremrcvsz; 215 __u32 localrcvsz; 216 __u16 port; 217 __u16 rsvd2; 218 union cma_ip_addr src_addr; 219 union cma_ip_addr dst_addr; 220 u8 rsvd3[IB_CM_REQ_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 48]; 221 } __attribute__((__packed__)); 222 223 struct sdp_hah { 224 struct sdp_bsdh bsdh; 225 u8 majv_minv; 226 u8 ipv_cap; 227 u8 rsvd1; 228 u8 ext_max_adverts; 229 __u32 actrcvsz; 230 u8 rsvd2[IB_CM_REP_PRIVATE_DATA_SIZE - sizeof(struct sdp_bsdh) - 8]; 231 } __attribute__((__packed__)); 232 233 struct sdp_rrch { 234 __u32 len; 235 } __attribute__((__packed__)); 236 237 struct sdp_srcah { 238 __u32 len; 239 __u32 rkey; 240 __u64 vaddr; 241 } __attribute__((__packed__)); 242 243 struct sdp_buf { 244 struct mbuf *mb; 245 u64 mapping[SDP_MAX_SEND_SGES]; 246 } __attribute__((__packed__)); 247 248 struct sdp_chrecvbuf { 249 u32 size; 250 } __attribute__((__packed__)); 251 252 /* Context used for synchronous zero copy bcopy (BZCOPY) */ 253 struct bzcopy_state { 254 unsigned char __user *u_base; 255 int u_len; 256 int left; 257 int page_cnt; 258 int cur_page; 259 int cur_offset; 260 int busy; 261 struct sdp_sock *ssk; 262 struct page **pages; 263 }; 264 265 enum rx_sa_flag { 266 RX_SA_ABORTED = 2, 267 }; 268 269 enum tx_sa_flag { 270 TX_SA_SENDSM = 0x01, 271 TX_SA_CROSS_SEND = 0x02, 272 TX_SA_INTRRUPTED = 0x04, 273 TX_SA_TIMEDOUT = 0x08, 274 TX_SA_ERROR = 0x10, 275 }; 276 277 struct rx_srcavail_state { 278 /* Advertised buffer stuff */ 279 u32 mseq; 280 u32 used; 281 u32 reported; 282 u32 len; 283 u32 rkey; 284 u64 vaddr; 285 286 /* Dest buff info */ 287 struct ib_umem *umem; 288 struct ib_pool_fmr *fmr; 289 290 /* Utility */ 291 u8 busy; 292 enum rx_sa_flag flags; 293 }; 294 295 struct tx_srcavail_state { 296 /* Data below 'busy' will be reset */ 297 u8 busy; 298 299 struct ib_umem *umem; 300 struct ib_pool_fmr *fmr; 301 302 u32 bytes_sent; 303 u32 bytes_acked; 304 305 enum tx_sa_flag abort_flags; 306 u8 posted; 307 308 u32 mseq; 309 }; 310 311 struct sdp_tx_ring { 312 #ifdef SDP_ZCOPY 313 struct rx_srcavail_state *rdma_inflight; 314 #endif 315 struct sdp_buf *buffer; 316 atomic_t head; 317 atomic_t tail; 318 struct ib_cq *cq; 319 320 atomic_t credits; 321 #define tx_credits(ssk) (atomic_read(&ssk->tx_ring.credits)) 322 323 struct callout timer; 324 u16 poll_cnt; 325 }; 326 327 struct sdp_rx_ring { 328 struct sdp_buf *buffer; 329 atomic_t head; 330 atomic_t tail; 331 struct ib_cq *cq; 332 333 int destroyed; 334 struct rwlock destroyed_lock; 335 }; 336 337 struct sdp_device { 338 struct ib_pd *pd; 339 struct ib_mr *mr; 340 struct ib_fmr_pool *fmr_pool; 341 }; 342 343 struct sdp_moderation { 344 unsigned long last_moder_packets; 345 unsigned long last_moder_tx_packets; 346 unsigned long last_moder_bytes; 347 unsigned long last_moder_jiffies; 348 int last_moder_time; 349 u16 rx_usecs; 350 u16 rx_frames; 351 u16 tx_usecs; 352 u32 pkt_rate_low; 353 u16 rx_usecs_low; 354 u32 pkt_rate_high; 355 u16 rx_usecs_high; 356 u16 sample_interval; 357 u16 adaptive_rx_coal; 358 u32 msg_enable; 359 360 int moder_cnt; 361 int moder_time; 362 }; 363 364 /* These are flags fields. */ 365 #define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */ 366 #define SDP_DROPPED 0x0002 /* Socket has been dropped. */ 367 #define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */ 368 #define SDP_NODELAY 0x0008 /* Disble nagle. */ 369 #define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */ 370 #define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */ 371 #define SDP_DESTROY 0x0040 /* Being destroyed. */ 372 #define SDP_DISCON 0x0080 /* rdma_disconnect is owed. */ 373 374 /* These are oobflags */ 375 #define SDP_HADOOB 0x0001 /* Had OOB data. */ 376 #define SDP_HAVEOOB 0x0002 /* Have OOB data. */ 377 378 struct sdp_sock { 379 LIST_ENTRY(sdp_sock) list; 380 struct socket *socket; 381 struct rdma_cm_id *id; 382 struct ib_device *ib_device; 383 struct sdp_device *sdp_dev; 384 struct ib_qp *qp; 385 struct ucred *cred; 386 struct callout keep2msl; /* 2msl and keepalive timer. */ 387 struct callout nagle_timer; /* timeout waiting for ack */ 388 struct ib_ucontext context; 389 in_port_t lport; 390 in_addr_t laddr; 391 in_port_t fport; 392 in_addr_t faddr; 393 int flags; 394 int oobflags; /* protected by rx lock. */ 395 int state; 396 int softerror; 397 int recv_bytes; /* Bytes per recv. buf including header */ 398 int xmit_size_goal; 399 char iobc; 400 401 struct sdp_rx_ring rx_ring; 402 struct sdp_tx_ring tx_ring; 403 struct rwlock lock; 404 struct mbuf *rx_ctl_q; 405 struct mbuf *rx_ctl_tail; 406 407 int qp_active; /* XXX Flag. */ 408 int max_sge; 409 struct work_struct rx_comp_work; 410 #define rcv_nxt(ssk) atomic_read(&(ssk->rcv_nxt)) 411 atomic_t rcv_nxt; 412 413 /* SDP specific */ 414 atomic_t mseq_ack; 415 #define mseq_ack(ssk) (atomic_read(&ssk->mseq_ack)) 416 unsigned max_bufs; /* Initial buffers offered by other side */ 417 unsigned min_bufs; /* Low water mark to wake senders */ 418 419 unsigned long nagle_last_unacked; /* mseq of lastest unacked packet */ 420 421 atomic_t remote_credits; 422 #define remote_credits(ssk) (atomic_read(&ssk->remote_credits)) 423 int poll_cq; 424 425 /* SDP slow start */ 426 int recv_request_head; /* mark the rx_head when the resize request 427 was recieved */ 428 int recv_request; /* XXX flag if request to resize was recieved */ 429 430 unsigned long tx_packets; 431 unsigned long rx_packets; 432 unsigned long tx_bytes; 433 unsigned long rx_bytes; 434 struct sdp_moderation auto_mod; 435 struct task shutdown_task; 436 #ifdef SDP_ZCOPY 437 struct tx_srcavail_state *tx_sa; 438 struct rx_srcavail_state *rx_sa; 439 spinlock_t tx_sa_lock; 440 struct delayed_work srcavail_cancel_work; 441 int srcavail_cancel_mseq; 442 /* ZCOPY data: -1:use global; 0:disable zcopy; >0: zcopy threshold */ 443 int zcopy_thresh; 444 #endif 445 }; 446 447 #define sdp_sk(so) ((struct sdp_sock *)(so->so_pcb)) 448 449 #define SDP_RLOCK(ssk) rw_rlock(&(ssk)->lock) 450 #define SDP_WLOCK(ssk) rw_wlock(&(ssk)->lock) 451 #define SDP_RUNLOCK(ssk) rw_runlock(&(ssk)->lock) 452 #define SDP_WUNLOCK(ssk) rw_wunlock(&(ssk)->lock) 453 #define SDP_WLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_WLOCKED) 454 #define SDP_RLOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_RLOCKED) 455 #define SDP_LOCK_ASSERT(ssk) rw_assert(&(ssk)->lock, RA_LOCKED) 456 457 static inline void tx_sa_reset(struct tx_srcavail_state *tx_sa) 458 { 459 memset((void *)&tx_sa->busy, 0, 460 sizeof(*tx_sa) - offsetof(typeof(*tx_sa), busy)); 461 } 462 463 static inline void rx_ring_unlock(struct sdp_rx_ring *rx_ring) 464 { 465 rw_runlock(&rx_ring->destroyed_lock); 466 } 467 468 static inline int rx_ring_trylock(struct sdp_rx_ring *rx_ring) 469 { 470 rw_rlock(&rx_ring->destroyed_lock); 471 if (rx_ring->destroyed) { 472 rx_ring_unlock(rx_ring); 473 return 0; 474 } 475 return 1; 476 } 477 478 static inline void rx_ring_destroy_lock(struct sdp_rx_ring *rx_ring) 479 { 480 rw_wlock(&rx_ring->destroyed_lock); 481 rx_ring->destroyed = 1; 482 rw_wunlock(&rx_ring->destroyed_lock); 483 } 484 485 static inline void sdp_arm_rx_cq(struct sdp_sock *ssk) 486 { 487 sdp_prf(ssk->socket, NULL, "Arming RX cq"); 488 sdp_dbg_data(ssk->socket, "Arming RX cq\n"); 489 490 ib_req_notify_cq(ssk->rx_ring.cq, IB_CQ_NEXT_COMP); 491 } 492 493 static inline void sdp_arm_tx_cq(struct sdp_sock *ssk) 494 { 495 sdp_prf(ssk->socket, NULL, "Arming TX cq"); 496 sdp_dbg_data(ssk->socket, "Arming TX cq. credits: %d, posted: %d\n", 497 tx_credits(ssk), tx_ring_posted(ssk)); 498 499 ib_req_notify_cq(ssk->tx_ring.cq, IB_CQ_NEXT_COMP); 500 } 501 502 /* return the min of: 503 * - tx credits 504 * - free slots in tx_ring (not including SDP_MIN_TX_CREDITS 505 */ 506 static inline int tx_slots_free(struct sdp_sock *ssk) 507 { 508 int min_free; 509 510 min_free = MIN(tx_credits(ssk), 511 SDP_TX_SIZE - tx_ring_posted(ssk)); 512 if (min_free < SDP_MIN_TX_CREDITS) 513 return 0; 514 515 return min_free - SDP_MIN_TX_CREDITS; 516 }; 517 518 /* utilities */ 519 static inline char *mid2str(int mid) 520 { 521 #define ENUM2STR(e) [e] = #e 522 static char *mid2str[] = { 523 ENUM2STR(SDP_MID_HELLO), 524 ENUM2STR(SDP_MID_HELLO_ACK), 525 ENUM2STR(SDP_MID_ABORT), 526 ENUM2STR(SDP_MID_DISCONN), 527 ENUM2STR(SDP_MID_SENDSM), 528 ENUM2STR(SDP_MID_RDMARDCOMPL), 529 ENUM2STR(SDP_MID_SRCAVAIL_CANCEL), 530 ENUM2STR(SDP_MID_CHRCVBUF), 531 ENUM2STR(SDP_MID_CHRCVBUF_ACK), 532 ENUM2STR(SDP_MID_DATA), 533 ENUM2STR(SDP_MID_SRCAVAIL), 534 ENUM2STR(SDP_MID_SINKAVAIL), 535 }; 536 537 if (mid >= ARRAY_SIZE(mid2str)) 538 return NULL; 539 540 return mid2str[mid]; 541 } 542 543 static inline struct mbuf * 544 sdp_alloc_mb(struct socket *sk, u8 mid, int size, int wait) 545 { 546 struct sdp_bsdh *h; 547 struct mbuf *mb; 548 549 MGETHDR(mb, wait, MT_DATA); 550 if (mb == NULL) 551 return (NULL); 552 mb->m_pkthdr.len = mb->m_len = sizeof(struct sdp_bsdh); 553 h = mtod(mb, struct sdp_bsdh *); 554 h->mid = mid; 555 556 return mb; 557 } 558 static inline struct mbuf * 559 sdp_alloc_mb_data(struct socket *sk, int wait) 560 { 561 return sdp_alloc_mb(sk, SDP_MID_DATA, 0, wait); 562 } 563 564 static inline struct mbuf * 565 sdp_alloc_mb_disconnect(struct socket *sk, int wait) 566 { 567 return sdp_alloc_mb(sk, SDP_MID_DISCONN, 0, wait); 568 } 569 570 static inline void * 571 mb_put(struct mbuf *mb, int len) 572 { 573 uint8_t *data; 574 575 data = mb->m_data; 576 data += mb->m_len; 577 mb->m_len += len; 578 return (void *)data; 579 } 580 581 static inline struct mbuf * 582 sdp_alloc_mb_chrcvbuf_ack(struct socket *sk, int size, int wait) 583 { 584 struct mbuf *mb; 585 struct sdp_chrecvbuf *resp_size; 586 587 mb = sdp_alloc_mb(sk, SDP_MID_CHRCVBUF_ACK, sizeof(*resp_size), wait); 588 if (mb == NULL) 589 return (NULL); 590 resp_size = (struct sdp_chrecvbuf *)mb_put(mb, sizeof *resp_size); 591 resp_size->size = htonl(size); 592 593 return mb; 594 } 595 596 static inline struct mbuf * 597 sdp_alloc_mb_srcavail(struct socket *sk, u32 len, u32 rkey, u64 vaddr, int wait) 598 { 599 struct mbuf *mb; 600 struct sdp_srcah *srcah; 601 602 mb = sdp_alloc_mb(sk, SDP_MID_SRCAVAIL, sizeof(*srcah), wait); 603 if (mb == NULL) 604 return (NULL); 605 srcah = (struct sdp_srcah *)mb_put(mb, sizeof(*srcah)); 606 srcah->len = htonl(len); 607 srcah->rkey = htonl(rkey); 608 srcah->vaddr = cpu_to_be64(vaddr); 609 610 return mb; 611 } 612 613 static inline struct mbuf * 614 sdp_alloc_mb_srcavail_cancel(struct socket *sk, int wait) 615 { 616 return sdp_alloc_mb(sk, SDP_MID_SRCAVAIL_CANCEL, 0, wait); 617 } 618 619 static inline struct mbuf * 620 sdp_alloc_mb_rdmardcompl(struct socket *sk, u32 len, int wait) 621 { 622 struct mbuf *mb; 623 struct sdp_rrch *rrch; 624 625 mb = sdp_alloc_mb(sk, SDP_MID_RDMARDCOMPL, sizeof(*rrch), wait); 626 if (mb == NULL) 627 return (NULL); 628 rrch = (struct sdp_rrch *)mb_put(mb, sizeof(*rrch)); 629 rrch->len = htonl(len); 630 631 return mb; 632 } 633 634 static inline struct mbuf * 635 sdp_alloc_mb_sendsm(struct socket *sk, int wait) 636 { 637 return sdp_alloc_mb(sk, SDP_MID_SENDSM, 0, wait); 638 } 639 static inline int sdp_tx_ring_slots_left(struct sdp_sock *ssk) 640 { 641 return SDP_TX_SIZE - tx_ring_posted(ssk); 642 } 643 644 static inline int credit_update_needed(struct sdp_sock *ssk) 645 { 646 int c; 647 648 c = remote_credits(ssk); 649 if (likely(c > SDP_MIN_TX_CREDITS)) 650 c += c/2; 651 return unlikely(c < rx_ring_posted(ssk)) && 652 likely(tx_credits(ssk) > 0) && 653 likely(sdp_tx_ring_slots_left(ssk)); 654 } 655 656 657 #define SDPSTATS_COUNTER_INC(stat) 658 #define SDPSTATS_COUNTER_ADD(stat, val) 659 #define SDPSTATS_COUNTER_MID_INC(stat, mid) 660 #define SDPSTATS_HIST_LINEAR(stat, size) 661 #define SDPSTATS_HIST(stat, size) 662 663 static inline void 664 sdp_cleanup_sdp_buf(struct sdp_sock *ssk, struct sdp_buf *sbuf, 665 enum dma_data_direction dir) 666 { 667 struct ib_device *dev; 668 struct mbuf *mb; 669 int i; 670 671 dev = ssk->ib_device; 672 for (i = 0, mb = sbuf->mb; mb != NULL; mb = mb->m_next, i++) 673 ib_dma_unmap_single(dev, sbuf->mapping[i], mb->m_len, dir); 674 } 675 676 /* sdp_main.c */ 677 void sdp_set_default_moderation(struct sdp_sock *ssk); 678 void sdp_start_keepalive_timer(struct socket *sk); 679 void sdp_urg(struct sdp_sock *ssk, struct mbuf *mb); 680 void sdp_cancel_dreq_wait_timeout(struct sdp_sock *ssk); 681 void sdp_abort(struct socket *sk); 682 struct sdp_sock *sdp_notify(struct sdp_sock *ssk, int error); 683 684 685 /* sdp_cma.c */ 686 int sdp_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *); 687 688 /* sdp_tx.c */ 689 int sdp_tx_ring_create(struct sdp_sock *ssk, struct ib_device *device); 690 void sdp_tx_ring_destroy(struct sdp_sock *ssk); 691 int sdp_xmit_poll(struct sdp_sock *ssk, int force); 692 void sdp_post_send(struct sdp_sock *ssk, struct mbuf *mb); 693 void sdp_post_sends(struct sdp_sock *ssk, int wait); 694 void sdp_post_keepalive(struct sdp_sock *ssk); 695 696 /* sdp_rx.c */ 697 void sdp_rx_ring_init(struct sdp_sock *ssk); 698 int sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device); 699 void sdp_rx_ring_destroy(struct sdp_sock *ssk); 700 int sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size); 701 int sdp_init_buffers(struct sdp_sock *ssk, u32 new_size); 702 void sdp_do_posts(struct sdp_sock *ssk); 703 void sdp_rx_comp_full(struct sdp_sock *ssk); 704 705 /* sdp_zcopy.c */ 706 struct kiocb; 707 int sdp_sendmsg_zcopy(struct kiocb *iocb, struct socket *sk, struct iovec *iov); 708 int sdp_handle_srcavail(struct sdp_sock *ssk, struct sdp_srcah *srcah); 709 void sdp_handle_sendsm(struct sdp_sock *ssk, u32 mseq_ack); 710 void sdp_handle_rdma_read_compl(struct sdp_sock *ssk, u32 mseq_ack, 711 u32 bytes_completed); 712 int sdp_handle_rdma_read_cqe(struct sdp_sock *ssk); 713 int sdp_rdma_to_iovec(struct socket *sk, struct iovec *iov, struct mbuf *mb, 714 unsigned long *used); 715 int sdp_post_rdma_rd_compl(struct sdp_sock *ssk, 716 struct rx_srcavail_state *rx_sa); 717 int sdp_post_sendsm(struct socket *sk); 718 void srcavail_cancel_timeout(struct work_struct *work); 719 void sdp_abort_srcavail(struct socket *sk); 720 void sdp_abort_rdma_read(struct socket *sk); 721 int sdp_process_rx(struct sdp_sock *ssk); 722 723 #endif 724