1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #ifndef RXE_VERBS_H 8 #define RXE_VERBS_H 9 10 #include <linux/interrupt.h> 11 #include <linux/workqueue.h> 12 #include "rxe_pool.h" 13 #include "rxe_task.h" 14 #include "rxe_hw_counters.h" 15 16 static inline int pkey_match(u16 key1, u16 key2) 17 { 18 return (((key1 & 0x7fff) != 0) && 19 ((key1 & 0x7fff) == (key2 & 0x7fff)) && 20 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0; 21 } 22 23 /* Return >0 if psn_a > psn_b 24 * 0 if psn_a == psn_b 25 * <0 if psn_a < psn_b 26 */ 27 static inline int psn_compare(u32 psn_a, u32 psn_b) 28 { 29 s32 diff; 30 31 diff = (psn_a - psn_b) << 8; 32 return diff; 33 } 34 35 struct rxe_ucontext { 36 struct ib_ucontext ibuc; 37 struct rxe_pool_elem elem; 38 }; 39 40 struct rxe_pd { 41 struct ib_pd ibpd; 42 struct rxe_pool_elem elem; 43 }; 44 45 struct rxe_ah { 46 struct ib_ah ibah; 47 struct rxe_pool_elem elem; 48 struct rxe_av av; 49 bool is_user; 50 int ah_num; 51 }; 52 53 struct rxe_cqe { 54 union { 55 struct ib_wc ibwc; 56 struct ib_uverbs_wc uibwc; 57 }; 58 }; 59 60 struct rxe_cq { 61 struct ib_cq ibcq; 62 struct rxe_pool_elem elem; 63 struct rxe_queue *queue; 64 spinlock_t cq_lock; 65 u8 notify; 66 bool is_user; 67 atomic_t num_wq; 68 }; 69 70 enum wqe_state { 71 wqe_state_posted, 72 wqe_state_processing, 73 wqe_state_pending, 74 wqe_state_done, 75 wqe_state_error, 76 }; 77 78 struct rxe_sq { 79 int max_wr; 80 int max_sge; 81 int max_inline; 82 spinlock_t sq_lock; /* guard queue */ 83 struct rxe_queue *queue; 84 }; 85 86 struct rxe_rq { 87 int max_wr; 88 int max_sge; 89 spinlock_t producer_lock; /* guard queue producer */ 90 spinlock_t consumer_lock; /* guard queue consumer */ 91 struct rxe_queue *queue; 92 }; 93 94 struct rxe_srq { 95 struct ib_srq ibsrq; 96 struct rxe_pool_elem elem; 97 struct rxe_pd *pd; 98 struct rxe_rq rq; 99 u32 srq_num; 100 101 int limit; 102 int error; 103 }; 104 105 struct rxe_req_info { 106 int wqe_index; 107 u32 psn; 108 int opcode; 109 atomic_t rd_atomic; 110 int wait_fence; 111 int need_rd_atomic; 112 int wait_psn; 113 int need_retry; 114 int wait_for_rnr_timer; 115 int noack_pkts; 116 int again; 117 }; 118 119 struct rxe_comp_info { 120 u32 psn; 121 int opcode; 122 int timeout; 123 int timeout_retry; 124 int started_retry; 125 u32 retry_cnt; 126 u32 rnr_retry; 127 }; 128 129 /* responder states */ 130 enum resp_states { 131 RESPST_NONE, 132 RESPST_GET_REQ, 133 RESPST_CHK_PSN, 134 RESPST_CHK_OP_SEQ, 135 RESPST_CHK_OP_VALID, 136 RESPST_CHK_RESOURCE, 137 RESPST_CHK_LENGTH, 138 RESPST_CHK_RKEY, 139 RESPST_EXECUTE, 140 RESPST_READ_REPLY, 141 RESPST_ATOMIC_REPLY, 142 RESPST_ATOMIC_WRITE_REPLY, 143 RESPST_PROCESS_FLUSH, 144 RESPST_COMPLETE, 145 RESPST_ACKNOWLEDGE, 146 RESPST_CLEANUP, 147 RESPST_DUPLICATE_REQUEST, 148 RESPST_ERR_MALFORMED_WQE, 149 RESPST_ERR_UNSUPPORTED_OPCODE, 150 RESPST_ERR_MISALIGNED_ATOMIC, 151 RESPST_ERR_PSN_OUT_OF_SEQ, 152 RESPST_ERR_MISSING_OPCODE_FIRST, 153 RESPST_ERR_MISSING_OPCODE_LAST_C, 154 RESPST_ERR_MISSING_OPCODE_LAST_D1E, 155 RESPST_ERR_TOO_MANY_RDMA_ATM_REQ, 156 RESPST_ERR_RNR, 157 RESPST_ERR_RKEY_VIOLATION, 158 RESPST_ERR_INVALIDATE_RKEY, 159 RESPST_ERR_LENGTH, 160 RESPST_ERR_CQ_OVERFLOW, 161 RESPST_ERROR, 162 RESPST_DONE, 163 RESPST_EXIT, 164 }; 165 166 enum rdatm_res_state { 167 rdatm_res_state_next, 168 rdatm_res_state_new, 169 rdatm_res_state_replay, 170 }; 171 172 struct resp_res { 173 int type; 174 int replay; 175 u32 first_psn; 176 u32 last_psn; 177 u32 cur_psn; 178 enum rdatm_res_state state; 179 180 union { 181 struct { 182 u64 orig_val; 183 } atomic; 184 struct { 185 u64 va_org; 186 u32 rkey; 187 u32 length; 188 u64 va; 189 u32 resid; 190 } read; 191 struct { 192 u32 length; 193 u64 va; 194 u8 type; 195 u8 level; 196 } flush; 197 }; 198 }; 199 200 struct rxe_resp_info { 201 u32 msn; 202 u32 psn; 203 u32 ack_psn; 204 int opcode; 205 int drop_msg; 206 int goto_error; 207 int sent_psn_nak; 208 enum ib_wc_status status; 209 u8 aeth_syndrome; 210 211 /* Receive only */ 212 struct rxe_recv_wqe *wqe; 213 214 /* RDMA read / atomic only */ 215 u64 va; 216 u64 offset; 217 struct rxe_mr *mr; 218 u32 resid; 219 u32 rkey; 220 u32 length; 221 222 /* SRQ only */ 223 struct { 224 struct rxe_recv_wqe wqe; 225 struct ib_sge sge[RXE_MAX_SGE]; 226 } srq_wqe; 227 228 /* Responder resources. It's a circular list where the oldest 229 * resource is dropped first. 230 */ 231 struct resp_res *resources; 232 unsigned int res_head; 233 unsigned int res_tail; 234 struct resp_res *res; 235 }; 236 237 struct rxe_qp { 238 struct ib_qp ibqp; 239 struct rxe_pool_elem elem; 240 struct ib_qp_attr attr; 241 unsigned int valid; 242 unsigned int mtu; 243 bool is_user; 244 245 struct rxe_pd *pd; 246 struct rxe_srq *srq; 247 struct rxe_cq *scq; 248 struct rxe_cq *rcq; 249 250 enum ib_sig_type sq_sig_type; 251 252 struct rxe_sq sq; 253 struct rxe_rq rq; 254 255 struct socket *sk; 256 u32 dst_cookie; 257 u16 src_port; 258 259 struct rxe_av pri_av; 260 struct rxe_av alt_av; 261 262 atomic_t mcg_num; 263 264 struct sk_buff_head req_pkts; 265 struct sk_buff_head resp_pkts; 266 267 struct rxe_task send_task; 268 struct rxe_task recv_task; 269 270 struct rxe_req_info req; 271 struct rxe_comp_info comp; 272 struct rxe_resp_info resp; 273 274 atomic_t ssn; 275 atomic_t skb_out; 276 int need_req_skb; 277 278 /* Timer for retranmitting packet when ACKs have been lost. RC 279 * only. The requester sets it when it is not already 280 * started. The responder resets it whenever an ack is 281 * received. 282 */ 283 struct timer_list retrans_timer; 284 u64 qp_timeout_jiffies; 285 286 /* Timer for handling RNR NAKS. */ 287 struct timer_list rnr_nak_timer; 288 289 spinlock_t state_lock; /* guard requester and completer */ 290 291 struct execute_work cleanup_work; 292 }; 293 294 enum { 295 RXE_ACCESS_REMOTE = IB_ACCESS_REMOTE_READ 296 | IB_ACCESS_REMOTE_WRITE 297 | IB_ACCESS_REMOTE_ATOMIC, 298 RXE_ACCESS_SUPPORTED_MR = RXE_ACCESS_REMOTE 299 | IB_ACCESS_LOCAL_WRITE 300 | IB_ACCESS_MW_BIND 301 | IB_ACCESS_ON_DEMAND 302 | IB_ACCESS_FLUSH_GLOBAL 303 | IB_ACCESS_FLUSH_PERSISTENT 304 | IB_ACCESS_OPTIONAL, 305 RXE_ACCESS_SUPPORTED_QP = RXE_ACCESS_SUPPORTED_MR, 306 RXE_ACCESS_SUPPORTED_MW = RXE_ACCESS_SUPPORTED_MR 307 | IB_ZERO_BASED, 308 }; 309 310 enum rxe_mr_state { 311 RXE_MR_STATE_INVALID, 312 RXE_MR_STATE_FREE, 313 RXE_MR_STATE_VALID, 314 }; 315 316 enum rxe_mr_copy_dir { 317 RXE_TO_MR_OBJ, 318 RXE_FROM_MR_OBJ, 319 }; 320 321 enum rxe_mr_lookup_type { 322 RXE_LOOKUP_LOCAL, 323 RXE_LOOKUP_REMOTE, 324 }; 325 326 enum rxe_rereg { 327 RXE_MR_REREG_SUPPORTED = IB_MR_REREG_PD 328 | IB_MR_REREG_ACCESS, 329 }; 330 331 static inline int rkey_is_mw(u32 rkey) 332 { 333 u32 index = rkey >> 8; 334 335 return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX); 336 } 337 338 struct rxe_mr { 339 struct rxe_pool_elem elem; 340 struct ib_mr ibmr; 341 342 struct ib_umem *umem; 343 344 u32 lkey; 345 u32 rkey; 346 enum rxe_mr_state state; 347 int access; 348 atomic_t num_mw; 349 350 unsigned int page_offset; 351 unsigned int page_shift; 352 u64 page_mask; 353 354 u32 num_buf; 355 u32 nbuf; 356 357 struct xarray page_list; 358 }; 359 360 static inline unsigned int mr_page_size(struct rxe_mr *mr) 361 { 362 return mr ? mr->ibmr.page_size : PAGE_SIZE; 363 } 364 365 enum rxe_mw_state { 366 RXE_MW_STATE_INVALID = RXE_MR_STATE_INVALID, 367 RXE_MW_STATE_FREE = RXE_MR_STATE_FREE, 368 RXE_MW_STATE_VALID = RXE_MR_STATE_VALID, 369 }; 370 371 struct rxe_mw { 372 struct ib_mw ibmw; 373 struct rxe_pool_elem elem; 374 spinlock_t lock; 375 enum rxe_mw_state state; 376 struct rxe_qp *qp; /* Type 2 only */ 377 struct rxe_mr *mr; 378 u32 rkey; 379 int access; 380 u64 addr; 381 u64 length; 382 }; 383 384 struct rxe_mcg { 385 struct rb_node node; 386 struct kref ref_cnt; 387 struct rxe_dev *rxe; 388 struct list_head qp_list; 389 union ib_gid mgid; 390 atomic_t qp_num; 391 u32 qkey; 392 u16 pkey; 393 }; 394 395 struct rxe_mca { 396 struct list_head qp_list; 397 struct rxe_qp *qp; 398 }; 399 400 struct rxe_port { 401 struct ib_port_attr attr; 402 __be64 port_guid; 403 __be64 subnet_prefix; 404 spinlock_t port_lock; /* guard port */ 405 unsigned int mtu_cap; 406 /* special QPs */ 407 u32 qp_gsi_index; 408 }; 409 410 #define RXE_PORT 1 411 struct rxe_dev { 412 struct ib_device ib_dev; 413 struct ib_device_attr attr; 414 int max_ucontext; 415 int max_inline_data; 416 struct mutex usdev_lock; 417 418 char raw_gid[ETH_ALEN]; 419 420 struct rxe_pool uc_pool; 421 struct rxe_pool pd_pool; 422 struct rxe_pool ah_pool; 423 struct rxe_pool srq_pool; 424 struct rxe_pool qp_pool; 425 struct rxe_pool cq_pool; 426 struct rxe_pool mr_pool; 427 struct rxe_pool mw_pool; 428 429 /* multicast support */ 430 spinlock_t mcg_lock; 431 struct rb_root mcg_tree; 432 atomic_t mcg_num; 433 atomic_t mcg_attach; 434 435 spinlock_t pending_lock; /* guard pending_mmaps */ 436 struct list_head pending_mmaps; 437 438 spinlock_t mmap_offset_lock; /* guard mmap_offset */ 439 u64 mmap_offset; 440 441 atomic64_t stats_counters[RXE_NUM_OF_COUNTERS]; 442 443 struct rxe_port port; 444 }; 445 446 static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev) 447 { 448 return ib_device_get_netdev(dev, RXE_PORT); 449 } 450 451 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index) 452 { 453 atomic64_inc(&rxe->stats_counters[index]); 454 } 455 456 static inline struct rxe_dev *to_rdev(struct ib_device *dev) 457 { 458 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL; 459 } 460 461 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc) 462 { 463 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL; 464 } 465 466 static inline struct rxe_pd *to_rpd(struct ib_pd *pd) 467 { 468 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL; 469 } 470 471 static inline struct rxe_ah *to_rah(struct ib_ah *ah) 472 { 473 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL; 474 } 475 476 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq) 477 { 478 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL; 479 } 480 481 static inline struct rxe_qp *to_rqp(struct ib_qp *qp) 482 { 483 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL; 484 } 485 486 static inline struct rxe_cq *to_rcq(struct ib_cq *cq) 487 { 488 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL; 489 } 490 491 static inline struct rxe_mr *to_rmr(struct ib_mr *mr) 492 { 493 return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL; 494 } 495 496 static inline struct rxe_mw *to_rmw(struct ib_mw *mw) 497 { 498 return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL; 499 } 500 501 static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah) 502 { 503 return to_rpd(ah->ibah.pd); 504 } 505 506 static inline struct rxe_pd *mr_pd(struct rxe_mr *mr) 507 { 508 return to_rpd(mr->ibmr.pd); 509 } 510 511 static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw) 512 { 513 return to_rpd(mw->ibmw.pd); 514 } 515 516 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name, 517 struct net_device *ndev); 518 519 #endif /* RXE_VERBS_H */ 520