1 /* 2 * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifndef MLX5_H 34 #define MLX5_H 35 36 #include <stddef.h> 37 #include <stdio.h> 38 #include <stdatomic.h> 39 #include <stdbool.h> 40 41 #include <sys/queue.h> 42 43 #include <infiniband/driver.h> 44 #include <infiniband/udma_barrier.h> 45 #include "mlx5-abi.h" 46 #include "bitmap.h" 47 #include "mlx5dv.h" 48 49 #define PFX "mlx5: " 50 51 52 enum { 53 MLX5_IB_MMAP_CMD_SHIFT = 8, 54 MLX5_IB_MMAP_CMD_MASK = 0xff, 55 }; 56 57 enum { 58 MLX5_MMAP_GET_REGULAR_PAGES_CMD = 0, 59 MLX5_MMAP_GET_CONTIGUOUS_PAGES_CMD = 1, 60 MLX5_MMAP_GET_CORE_CLOCK_CMD = 5 61 }; 62 63 enum { 64 MLX5_CQE_VERSION_V0 = 0, 65 MLX5_CQE_VERSION_V1 = 1, 66 }; 67 68 enum { 69 MLX5_ADAPTER_PAGE_SIZE = 4096, 70 }; 71 72 #define MLX5_CQ_PREFIX "MLX_CQ" 73 #define MLX5_QP_PREFIX "MLX_QP" 74 #define MLX5_MR_PREFIX "MLX_MR" 75 #define MLX5_RWQ_PREFIX "MLX_RWQ" 76 #define MLX5_MAX_LOG2_CONTIG_BLOCK_SIZE 23 77 #define MLX5_MIN_LOG2_CONTIG_BLOCK_SIZE 12 78 79 enum { 80 MLX5_DBG_QP = 1 << 0, 81 MLX5_DBG_CQ = 1 << 1, 82 MLX5_DBG_QP_SEND = 1 << 2, 83 MLX5_DBG_QP_SEND_ERR = 1 << 3, 84 MLX5_DBG_CQ_CQE = 1 << 4, 85 MLX5_DBG_CONTIG = 1 << 5, 86 }; 87 88 extern uint32_t mlx5_debug_mask; 89 extern int mlx5_freeze_on_error_cqe; 90 91 #ifdef MLX5_DEBUG 92 #define mlx5_dbg(fp, mask, format, arg...) \ 93 do { \ 94 if (mask & mlx5_debug_mask) \ 95 fprintf(fp, "%s:%d: " format, __func__, __LINE__, ##arg); \ 96 } while (0) 97 98 #else 99 static inline void mlx5_dbg(FILE *fp, uint32_t mask, const char *fmt, ...) 100 __attribute__((format(printf, 3, 4))); 101 static inline void mlx5_dbg(FILE *fp, uint32_t mask, const char *fmt, ...) 102 { 103 } 104 #endif 105 106 enum { 107 MLX5_STAT_RATE_OFFSET = 5 108 }; 109 110 enum { 111 MLX5_QP_TABLE_SHIFT = 12, 112 MLX5_QP_TABLE_MASK = (1 << MLX5_QP_TABLE_SHIFT) - 1, 113 MLX5_QP_TABLE_SIZE = 1 << (24 - MLX5_QP_TABLE_SHIFT), 114 }; 115 116 enum { 117 MLX5_UIDX_TABLE_SHIFT = 12, 118 MLX5_UIDX_TABLE_MASK = (1 << MLX5_UIDX_TABLE_SHIFT) - 1, 119 MLX5_UIDX_TABLE_SIZE = 1 << (24 - MLX5_UIDX_TABLE_SHIFT), 120 }; 121 122 enum { 123 MLX5_SRQ_TABLE_SHIFT = 12, 124 MLX5_SRQ_TABLE_MASK = (1 << MLX5_SRQ_TABLE_SHIFT) - 1, 125 MLX5_SRQ_TABLE_SIZE = 1 << (24 - MLX5_SRQ_TABLE_SHIFT), 126 }; 127 128 enum { 129 MLX5_BF_OFFSET = 0x800 130 }; 131 132 enum { 133 MLX5_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, 134 MLX5_RECV_OPCODE_SEND = 0x01, 135 MLX5_RECV_OPCODE_SEND_IMM = 0x02, 136 MLX5_RECV_OPCODE_SEND_INVAL = 0x03, 137 138 MLX5_CQE_OPCODE_ERROR = 0x1e, 139 MLX5_CQE_OPCODE_RESIZE = 0x16, 140 }; 141 142 enum { 143 MLX5_SRQ_FLAG_SIGNATURE = 1 << 0, 144 }; 145 146 enum { 147 MLX5_MAX_PORTS_NUM = 2, 148 }; 149 150 enum { 151 MLX5_CSUM_SUPPORT_RAW_OVER_ETH = (1 << 0), 152 /* 153 * Only report rx checksum when the validation 154 * is valid. 155 */ 156 MLX5_RX_CSUM_VALID = (1 << 16), 157 }; 158 159 enum mlx5_alloc_type { 160 MLX5_ALLOC_TYPE_ANON, 161 MLX5_ALLOC_TYPE_HUGE, 162 MLX5_ALLOC_TYPE_CONTIG, 163 MLX5_ALLOC_TYPE_PREFER_HUGE, 164 MLX5_ALLOC_TYPE_PREFER_CONTIG, 165 MLX5_ALLOC_TYPE_ALL 166 }; 167 168 enum mlx5_rsc_type { 169 MLX5_RSC_TYPE_QP, 170 MLX5_RSC_TYPE_XSRQ, 171 MLX5_RSC_TYPE_SRQ, 172 MLX5_RSC_TYPE_RWQ, 173 MLX5_RSC_TYPE_INVAL, 174 }; 175 176 enum { 177 MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE = 1 << 0, 178 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH = 1 << 1, 179 }; 180 181 enum mlx5_vendor_cap_flags { 182 MLX5_VENDOR_CAP_FLAGS_MPW = 1 << 0, 183 }; 184 185 enum { 186 MLX5_FLOW_TAG_MASK = 0x000fffff, 187 }; 188 189 struct mlx5_resource { 190 enum mlx5_rsc_type type; 191 uint32_t rsn; 192 }; 193 194 struct mlx5_device { 195 struct verbs_device verbs_dev; 196 int page_size; 197 int driver_abi_ver; 198 }; 199 200 struct mlx5_db_page; 201 202 struct mlx5_spinlock { 203 pthread_spinlock_t lock; 204 int in_use; 205 }; 206 207 struct mlx5_hugetlb_mem; 208 209 struct mlx5_context { 210 struct ibv_context ibv_ctx; 211 int max_num_qps; 212 int bf_reg_size; 213 int tot_uuars; 214 int low_lat_uuars; 215 int num_uars_per_page; 216 int bf_regs_per_page; 217 int num_bf_regs; 218 int prefer_bf; 219 int shut_up_bf; 220 struct { 221 struct mlx5_qp **table; 222 int refcnt; 223 } qp_table[MLX5_QP_TABLE_SIZE]; 224 pthread_mutex_t qp_table_mutex; 225 226 struct { 227 struct mlx5_srq **table; 228 int refcnt; 229 } srq_table[MLX5_SRQ_TABLE_SIZE]; 230 pthread_mutex_t srq_table_mutex; 231 232 struct { 233 struct mlx5_resource **table; 234 int refcnt; 235 } uidx_table[MLX5_UIDX_TABLE_SIZE]; 236 pthread_mutex_t uidx_table_mutex; 237 238 void *uar[MLX5_MAX_UARS]; 239 struct mlx5_spinlock lock32; 240 struct mlx5_db_page *db_list; 241 pthread_mutex_t db_list_mutex; 242 int cache_line_size; 243 int max_sq_desc_sz; 244 int max_rq_desc_sz; 245 int max_send_wqebb; 246 int max_recv_wr; 247 unsigned max_srq_recv_wr; 248 int num_ports; 249 int stall_enable; 250 int stall_adaptive_enable; 251 int stall_cycles; 252 struct mlx5_bf *bfs; 253 FILE *dbg_fp; 254 char hostname[40]; 255 struct mlx5_spinlock hugetlb_lock; 256 TAILQ_HEAD(,mlx5_hugetlb_mem) hugetlb_list; 257 int cqe_version; 258 uint8_t cached_link_layer[MLX5_MAX_PORTS_NUM]; 259 int cached_device_cap_flags; 260 enum ibv_atomic_cap atomic_cap; 261 struct { 262 uint64_t offset; 263 uint64_t mask; 264 } core_clock; 265 void *hca_core_clock; 266 struct ibv_tso_caps cached_tso_caps; 267 int cmds_supp_uhw; 268 uint32_t uar_size; 269 uint64_t vendor_cap_flags; /* Use enum mlx5_vendor_cap_flags */ 270 struct mlx5dv_cqe_comp_caps cqe_comp_caps; 271 }; 272 273 struct mlx5_bitmap { 274 uint32_t last; 275 uint32_t top; 276 uint32_t max; 277 uint32_t avail; 278 uint32_t mask; 279 unsigned long *table; 280 }; 281 282 struct mlx5_hugetlb_mem { 283 int shmid; 284 void *shmaddr; 285 struct mlx5_bitmap bitmap; 286 TAILQ_ENTRY(mlx5_hugetlb_mem) entry; 287 }; 288 289 struct mlx5_buf { 290 void *buf; 291 size_t length; 292 int base; 293 struct mlx5_hugetlb_mem *hmem; 294 enum mlx5_alloc_type type; 295 }; 296 297 struct mlx5_pd { 298 struct ibv_pd ibv_pd; 299 uint32_t pdn; 300 }; 301 302 enum { 303 MLX5_CQ_SET_CI = 0, 304 MLX5_CQ_ARM_DB = 1, 305 }; 306 307 enum { 308 MLX5_CQ_FLAGS_RX_CSUM_VALID = 1 << 0, 309 MLX5_CQ_FLAGS_EMPTY_DURING_POLL = 1 << 1, 310 MLX5_CQ_FLAGS_FOUND_CQES = 1 << 2, 311 MLX5_CQ_FLAGS_EXTENDED = 1 << 3, 312 MLX5_CQ_FLAGS_SINGLE_THREADED = 1 << 4, 313 MLX5_CQ_FLAGS_DV_OWNED = 1 << 5, 314 }; 315 316 struct mlx5_cq { 317 /* ibv_cq should always be subset of ibv_cq_ex */ 318 struct ibv_cq_ex ibv_cq; 319 struct mlx5_buf buf_a; 320 struct mlx5_buf buf_b; 321 struct mlx5_buf *active_buf; 322 struct mlx5_buf *resize_buf; 323 int resize_cqes; 324 int active_cqes; 325 struct mlx5_spinlock lock; 326 uint32_t cqn; 327 uint32_t cons_index; 328 uint32_t *dbrec; 329 int arm_sn; 330 int cqe_sz; 331 int resize_cqe_sz; 332 int stall_next_poll; 333 int stall_enable; 334 uint64_t stall_last_count; 335 int stall_adaptive_enable; 336 int stall_cycles; 337 struct mlx5_resource *cur_rsc; 338 struct mlx5_srq *cur_srq; 339 struct mlx5_cqe64 *cqe64; 340 uint32_t flags; 341 int umr_opcode; 342 }; 343 344 struct mlx5_srq { 345 struct mlx5_resource rsc; /* This struct must be first */ 346 struct verbs_srq vsrq; 347 struct mlx5_buf buf; 348 struct mlx5_spinlock lock; 349 uint64_t *wrid; 350 uint32_t srqn; 351 int max; 352 int max_gs; 353 int wqe_shift; 354 int head; 355 int tail; 356 uint32_t *db; 357 uint16_t counter; 358 int wq_sig; 359 }; 360 361 struct wr_list { 362 uint16_t opcode; 363 uint16_t next; 364 }; 365 366 struct mlx5_wq { 367 uint64_t *wrid; 368 unsigned *wqe_head; 369 struct mlx5_spinlock lock; 370 unsigned wqe_cnt; 371 unsigned max_post; 372 unsigned head; 373 unsigned tail; 374 unsigned cur_post; 375 int max_gs; 376 int wqe_shift; 377 int offset; 378 void *qend; 379 uint32_t *wr_data; 380 }; 381 382 struct mlx5_bf { 383 void *reg; 384 int need_lock; 385 struct mlx5_spinlock lock; 386 unsigned offset; 387 unsigned buf_size; 388 unsigned uuarn; 389 }; 390 391 struct mlx5_mr { 392 struct ibv_mr ibv_mr; 393 struct mlx5_buf buf; 394 uint32_t alloc_flags; 395 }; 396 397 struct mlx5_qp { 398 struct mlx5_resource rsc; /* This struct must be first */ 399 struct verbs_qp verbs_qp; 400 struct ibv_qp *ibv_qp; 401 struct mlx5_buf buf; 402 void *sq_start; 403 int max_inline_data; 404 int buf_size; 405 /* For Raw Packet QP, use different buffers for the SQ and RQ */ 406 struct mlx5_buf sq_buf; 407 int sq_buf_size; 408 struct mlx5_bf *bf; 409 410 uint8_t fm_cache; 411 uint8_t sq_signal_bits; 412 struct mlx5_wq sq; 413 414 uint32_t *db; 415 struct mlx5_wq rq; 416 int wq_sig; 417 uint32_t qp_cap_cache; 418 int atomics_enabled; 419 uint32_t max_tso; 420 uint16_t max_tso_header; 421 int rss_qp; 422 }; 423 424 struct mlx5_ah { 425 struct ibv_ah ibv_ah; 426 struct mlx5_wqe_av av; 427 bool kern_ah; 428 }; 429 430 struct mlx5_rwq { 431 struct mlx5_resource rsc; 432 struct ibv_wq wq; 433 struct mlx5_buf buf; 434 int buf_size; 435 struct mlx5_wq rq; 436 uint32_t *db; 437 void *pbuff; 438 uint32_t *recv_db; 439 int wq_sig; 440 }; 441 442 static inline int mlx5_ilog2(int n) 443 { 444 int t; 445 446 if (n <= 0) 447 return -1; 448 449 t = 0; 450 while ((1 << t) < n) 451 ++t; 452 453 return t; 454 } 455 456 extern int mlx5_stall_num_loop; 457 extern int mlx5_stall_cq_poll_min; 458 extern int mlx5_stall_cq_poll_max; 459 extern int mlx5_stall_cq_inc_step; 460 extern int mlx5_stall_cq_dec_step; 461 extern int mlx5_single_threaded; 462 463 static inline unsigned DIV_ROUND_UP(unsigned n, unsigned d) 464 { 465 return (n + d - 1u) / d; 466 } 467 468 static inline unsigned long align(unsigned long val, unsigned long align) 469 { 470 return (val + align - 1) & ~(align - 1); 471 } 472 473 #define to_mxxx(xxx, type) \ 474 ((struct mlx5_##type *) \ 475 ((void *) ib##xxx - offsetof(struct mlx5_##type, ibv_##xxx))) 476 477 static inline struct mlx5_device *to_mdev(struct ibv_device *ibdev) 478 { 479 struct mlx5_device *ret; 480 481 ret = (void *)ibdev - offsetof(struct mlx5_device, verbs_dev); 482 return ret; 483 } 484 485 static inline struct mlx5_context *to_mctx(struct ibv_context *ibctx) 486 { 487 return to_mxxx(ctx, context); 488 } 489 490 static inline struct mlx5_pd *to_mpd(struct ibv_pd *ibpd) 491 { 492 return to_mxxx(pd, pd); 493 } 494 495 static inline struct mlx5_cq *to_mcq(struct ibv_cq *ibcq) 496 { 497 return to_mxxx(cq, cq); 498 } 499 500 static inline struct mlx5_srq *to_msrq(struct ibv_srq *ibsrq) 501 { 502 struct verbs_srq *vsrq = (struct verbs_srq *)ibsrq; 503 504 return container_of(vsrq, struct mlx5_srq, vsrq); 505 } 506 507 static inline struct mlx5_qp *to_mqp(struct ibv_qp *ibqp) 508 { 509 struct verbs_qp *vqp = (struct verbs_qp *)ibqp; 510 511 return container_of(vqp, struct mlx5_qp, verbs_qp); 512 } 513 514 static inline struct mlx5_rwq *to_mrwq(struct ibv_wq *ibwq) 515 { 516 return container_of(ibwq, struct mlx5_rwq, wq); 517 } 518 519 static inline struct mlx5_mr *to_mmr(struct ibv_mr *ibmr) 520 { 521 return to_mxxx(mr, mr); 522 } 523 524 static inline struct mlx5_ah *to_mah(struct ibv_ah *ibah) 525 { 526 return to_mxxx(ah, ah); 527 } 528 529 static inline int max_int(int a, int b) 530 { 531 return a > b ? a : b; 532 } 533 534 static inline struct mlx5_qp *rsc_to_mqp(struct mlx5_resource *rsc) 535 { 536 return (struct mlx5_qp *)rsc; 537 } 538 539 static inline struct mlx5_srq *rsc_to_msrq(struct mlx5_resource *rsc) 540 { 541 return (struct mlx5_srq *)rsc; 542 } 543 544 static inline struct mlx5_rwq *rsc_to_mrwq(struct mlx5_resource *rsc) 545 { 546 return (struct mlx5_rwq *)rsc; 547 } 548 549 int mlx5_alloc_buf(struct mlx5_buf *buf, size_t size, int page_size); 550 void mlx5_free_buf(struct mlx5_buf *buf); 551 int mlx5_alloc_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf, 552 size_t size, int page_size, const char *component); 553 void mlx5_free_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf); 554 int mlx5_alloc_prefered_buf(struct mlx5_context *mctx, 555 struct mlx5_buf *buf, 556 size_t size, int page_size, 557 enum mlx5_alloc_type alloc_type, 558 const char *component); 559 int mlx5_free_actual_buf(struct mlx5_context *ctx, struct mlx5_buf *buf); 560 void mlx5_get_alloc_type(const char *component, 561 enum mlx5_alloc_type *alloc_type, 562 enum mlx5_alloc_type default_alloc_type); 563 int mlx5_use_huge(const char *key); 564 565 uint32_t *mlx5_alloc_dbrec(struct mlx5_context *context); 566 void mlx5_free_db(struct mlx5_context *context, uint32_t *db); 567 568 int mlx5_query_device(struct ibv_context *context, 569 struct ibv_device_attr *attr); 570 int mlx5_query_device_ex(struct ibv_context *context, 571 const struct ibv_query_device_ex_input *input, 572 struct ibv_device_attr_ex *attr, 573 size_t attr_size); 574 int mlx5_query_rt_values(struct ibv_context *context, 575 struct ibv_values_ex *values); 576 struct ibv_qp *mlx5_create_qp_ex(struct ibv_context *context, 577 struct ibv_qp_init_attr_ex *attr); 578 int mlx5_query_port(struct ibv_context *context, uint8_t port, 579 struct ibv_port_attr *attr); 580 581 struct ibv_pd *mlx5_alloc_pd(struct ibv_context *context); 582 int mlx5_free_pd(struct ibv_pd *pd); 583 584 struct ibv_mr *mlx5_reg_mr(struct ibv_pd *pd, void *addr, 585 size_t length, int access); 586 int mlx5_rereg_mr(struct ibv_mr *mr, int flags, struct ibv_pd *pd, void *addr, 587 size_t length, int access); 588 int mlx5_dereg_mr(struct ibv_mr *mr); 589 struct ibv_mw *mlx5_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type); 590 int mlx5_dealloc_mw(struct ibv_mw *mw); 591 int mlx5_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, 592 struct ibv_mw_bind *mw_bind); 593 594 struct ibv_cq *mlx5_create_cq(struct ibv_context *context, int cqe, 595 struct ibv_comp_channel *channel, 596 int comp_vector); 597 struct ibv_cq_ex *mlx5_create_cq_ex(struct ibv_context *context, 598 struct ibv_cq_init_attr_ex *cq_attr); 599 void mlx5_cq_fill_pfns(struct mlx5_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr); 600 int mlx5_alloc_cq_buf(struct mlx5_context *mctx, struct mlx5_cq *cq, 601 struct mlx5_buf *buf, int nent, int cqe_sz); 602 int mlx5_free_cq_buf(struct mlx5_context *ctx, struct mlx5_buf *buf); 603 int mlx5_resize_cq(struct ibv_cq *cq, int cqe); 604 int mlx5_destroy_cq(struct ibv_cq *cq); 605 int mlx5_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); 606 int mlx5_poll_cq_v1(struct ibv_cq *cq, int ne, struct ibv_wc *wc); 607 int mlx5_arm_cq(struct ibv_cq *cq, int solicited); 608 void mlx5_cq_event(struct ibv_cq *cq); 609 void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq); 610 void mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq); 611 void mlx5_cq_resize_copy_cqes(struct mlx5_cq *cq); 612 613 struct ibv_srq *mlx5_create_srq(struct ibv_pd *pd, 614 struct ibv_srq_init_attr *attr); 615 int mlx5_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, 616 int mask); 617 int mlx5_query_srq(struct ibv_srq *srq, 618 struct ibv_srq_attr *attr); 619 int mlx5_destroy_srq(struct ibv_srq *srq); 620 int mlx5_alloc_srq_buf(struct ibv_context *context, struct mlx5_srq *srq); 621 void mlx5_free_srq_wqe(struct mlx5_srq *srq, int ind); 622 int mlx5_post_srq_recv(struct ibv_srq *ibsrq, 623 struct ibv_recv_wr *wr, 624 struct ibv_recv_wr **bad_wr); 625 626 struct ibv_qp *mlx5_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); 627 int mlx5_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 628 int attr_mask, 629 struct ibv_qp_init_attr *init_attr); 630 int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 631 int attr_mask); 632 int mlx5_destroy_qp(struct ibv_qp *qp); 633 void mlx5_init_qp_indices(struct mlx5_qp *qp); 634 void mlx5_init_rwq_indices(struct mlx5_rwq *rwq); 635 int mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 636 struct ibv_send_wr **bad_wr); 637 int mlx5_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 638 struct ibv_recv_wr **bad_wr); 639 int mlx5_post_wq_recv(struct ibv_wq *ibwq, struct ibv_recv_wr *wr, 640 struct ibv_recv_wr **bad_wr); 641 void mlx5_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type, 642 struct mlx5_qp *qp); 643 void mlx5_set_sq_sizes(struct mlx5_qp *qp, struct ibv_qp_cap *cap, 644 enum ibv_qp_type type); 645 struct mlx5_qp *mlx5_find_qp(struct mlx5_context *ctx, uint32_t qpn); 646 int mlx5_store_qp(struct mlx5_context *ctx, uint32_t qpn, struct mlx5_qp *qp); 647 void mlx5_clear_qp(struct mlx5_context *ctx, uint32_t qpn); 648 int32_t mlx5_store_uidx(struct mlx5_context *ctx, void *rsc); 649 void mlx5_clear_uidx(struct mlx5_context *ctx, uint32_t uidx); 650 struct mlx5_srq *mlx5_find_srq(struct mlx5_context *ctx, uint32_t srqn); 651 int mlx5_store_srq(struct mlx5_context *ctx, uint32_t srqn, 652 struct mlx5_srq *srq); 653 void mlx5_clear_srq(struct mlx5_context *ctx, uint32_t srqn); 654 struct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); 655 int mlx5_destroy_ah(struct ibv_ah *ah); 656 int mlx5_alloc_av(struct mlx5_pd *pd, struct ibv_ah_attr *attr, 657 struct mlx5_ah *ah); 658 void mlx5_free_av(struct mlx5_ah *ah); 659 int mlx5_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); 660 int mlx5_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); 661 int mlx5_round_up_power_of_two(long long sz); 662 void *mlx5_get_atomic_laddr(struct mlx5_qp *qp, uint16_t idx, int *byte_count); 663 void *mlx5_get_send_wqe(struct mlx5_qp *qp, int n); 664 int mlx5_copy_to_recv_wqe(struct mlx5_qp *qp, int idx, void *buf, int size); 665 int mlx5_copy_to_send_wqe(struct mlx5_qp *qp, int idx, void *buf, int size); 666 int mlx5_copy_to_recv_srq(struct mlx5_srq *srq, int idx, void *buf, int size); 667 struct ibv_xrcd *mlx5_open_xrcd(struct ibv_context *context, 668 struct ibv_xrcd_init_attr *xrcd_init_attr); 669 int mlx5_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num); 670 int mlx5_close_xrcd(struct ibv_xrcd *ib_xrcd); 671 struct ibv_wq *mlx5_create_wq(struct ibv_context *context, 672 struct ibv_wq_init_attr *attr); 673 int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr); 674 int mlx5_destroy_wq(struct ibv_wq *wq); 675 struct ibv_rwq_ind_table *mlx5_create_rwq_ind_table(struct ibv_context *context, 676 struct ibv_rwq_ind_table_init_attr *init_attr); 677 int mlx5_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table); 678 struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context, 679 struct ibv_srq_init_attr_ex *attr); 680 681 static inline void *mlx5_find_uidx(struct mlx5_context *ctx, uint32_t uidx) 682 { 683 int tind = uidx >> MLX5_UIDX_TABLE_SHIFT; 684 685 if (likely(ctx->uidx_table[tind].refcnt)) 686 return ctx->uidx_table[tind].table[uidx & MLX5_UIDX_TABLE_MASK]; 687 688 return NULL; 689 } 690 691 static inline int mlx5_spin_lock(struct mlx5_spinlock *lock) 692 { 693 if (!mlx5_single_threaded) 694 return pthread_spin_lock(&lock->lock); 695 696 if (unlikely(lock->in_use)) { 697 fprintf(stderr, "*** ERROR: multithreading vilation ***\n" 698 "You are running a multithreaded application but\n" 699 "you set MLX5_SINGLE_THREADED=1. Please unset it.\n"); 700 abort(); 701 } else { 702 lock->in_use = 1; 703 /* 704 * This fence is not at all correct, but it increases the 705 * chance that in_use is detected by another thread without 706 * much runtime cost. */ 707 atomic_thread_fence(memory_order_acq_rel); 708 } 709 710 return 0; 711 } 712 713 static inline int mlx5_spin_unlock(struct mlx5_spinlock *lock) 714 { 715 if (!mlx5_single_threaded) 716 return pthread_spin_unlock(&lock->lock); 717 718 lock->in_use = 0; 719 720 return 0; 721 } 722 723 static inline int mlx5_spinlock_init(struct mlx5_spinlock *lock) 724 { 725 lock->in_use = 0; 726 return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE); 727 } 728 729 static inline int mlx5_spinlock_destroy(struct mlx5_spinlock *lock) 730 { 731 return pthread_spin_destroy(&lock->lock); 732 } 733 734 static inline void set_command(int command, off_t *offset) 735 { 736 *offset |= (command << MLX5_IB_MMAP_CMD_SHIFT); 737 } 738 739 static inline void set_arg(int arg, off_t *offset) 740 { 741 *offset |= arg; 742 } 743 744 static inline void set_order(int order, off_t *offset) 745 { 746 set_arg(order, offset); 747 } 748 749 static inline void set_index(int index, off_t *offset) 750 { 751 set_arg(index, offset); 752 } 753 754 static inline uint8_t calc_sig(void *wqe, int size) 755 { 756 int i; 757 uint8_t *p = wqe; 758 uint8_t res = 0; 759 760 for (i = 0; i < size; ++i) 761 res ^= p[i]; 762 763 return ~res; 764 } 765 766 #endif /* MLX5_H */ 767