1 /*
2 * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #ifndef MLX5_H
34 #define MLX5_H
35
36 #include <stddef.h>
37 #include <stdio.h>
38 #include <stdatomic.h>
39 #include <stdbool.h>
40
41 #include <sys/queue.h>
42
43 #include <infiniband/driver.h>
44 #include <infiniband/udma_barrier.h>
45 #include "mlx5-abi.h"
46 #include "bitmap.h"
47 #include "mlx5dv.h"
48
49 #define PFX "mlx5: "
50
51
52 enum {
53 MLX5_IB_MMAP_CMD_SHIFT = 8,
54 MLX5_IB_MMAP_CMD_MASK = 0xff,
55 };
56
57 enum {
58 MLX5_MMAP_GET_REGULAR_PAGES_CMD = 0,
59 MLX5_MMAP_GET_CONTIGUOUS_PAGES_CMD = 1,
60 MLX5_MMAP_GET_CORE_CLOCK_CMD = 5
61 };
62
63 enum {
64 MLX5_CQE_VERSION_V0 = 0,
65 MLX5_CQE_VERSION_V1 = 1,
66 };
67
68 enum {
69 MLX5_ADAPTER_PAGE_SIZE = 4096,
70 };
71
72 #define MLX5_CQ_PREFIX "MLX_CQ"
73 #define MLX5_QP_PREFIX "MLX_QP"
74 #define MLX5_MR_PREFIX "MLX_MR"
75 #define MLX5_RWQ_PREFIX "MLX_RWQ"
76 #define MLX5_MAX_LOG2_CONTIG_BLOCK_SIZE 23
77 #define MLX5_MIN_LOG2_CONTIG_BLOCK_SIZE 12
78
79 enum {
80 MLX5_DBG_QP = 1 << 0,
81 MLX5_DBG_CQ = 1 << 1,
82 MLX5_DBG_QP_SEND = 1 << 2,
83 MLX5_DBG_QP_SEND_ERR = 1 << 3,
84 MLX5_DBG_CQ_CQE = 1 << 4,
85 MLX5_DBG_CONTIG = 1 << 5,
86 };
87
88 extern uint32_t mlx5_debug_mask;
89 extern int mlx5_freeze_on_error_cqe;
90
91 #ifdef MLX5_DEBUG
92 #define mlx5_dbg(fp, mask, format, arg...) \
93 do { \
94 if (mask & mlx5_debug_mask) \
95 fprintf(fp, "%s:%d: " format, __func__, __LINE__, ##arg); \
96 } while (0)
97
98 #else
99 static inline void mlx5_dbg(FILE *fp, uint32_t mask, const char *fmt, ...)
100 __attribute__((format(printf, 3, 4)));
mlx5_dbg(FILE * fp,uint32_t mask,const char * fmt,...)101 static inline void mlx5_dbg(FILE *fp, uint32_t mask, const char *fmt, ...)
102 {
103 }
104 #endif
105
106 enum {
107 MLX5_STAT_RATE_OFFSET = 5
108 };
109
110 enum {
111 MLX5_QP_TABLE_SHIFT = 12,
112 MLX5_QP_TABLE_MASK = (1 << MLX5_QP_TABLE_SHIFT) - 1,
113 MLX5_QP_TABLE_SIZE = 1 << (24 - MLX5_QP_TABLE_SHIFT),
114 };
115
116 enum {
117 MLX5_UIDX_TABLE_SHIFT = 12,
118 MLX5_UIDX_TABLE_MASK = (1 << MLX5_UIDX_TABLE_SHIFT) - 1,
119 MLX5_UIDX_TABLE_SIZE = 1 << (24 - MLX5_UIDX_TABLE_SHIFT),
120 };
121
122 enum {
123 MLX5_SRQ_TABLE_SHIFT = 12,
124 MLX5_SRQ_TABLE_MASK = (1 << MLX5_SRQ_TABLE_SHIFT) - 1,
125 MLX5_SRQ_TABLE_SIZE = 1 << (24 - MLX5_SRQ_TABLE_SHIFT),
126 };
127
128 enum {
129 MLX5_BF_OFFSET = 0x800
130 };
131
132 enum {
133 MLX5_RECV_OPCODE_RDMA_WRITE_IMM = 0x00,
134 MLX5_RECV_OPCODE_SEND = 0x01,
135 MLX5_RECV_OPCODE_SEND_IMM = 0x02,
136 MLX5_RECV_OPCODE_SEND_INVAL = 0x03,
137
138 MLX5_CQE_OPCODE_ERROR = 0x1e,
139 MLX5_CQE_OPCODE_RESIZE = 0x16,
140 };
141
142 enum {
143 MLX5_SRQ_FLAG_SIGNATURE = 1 << 0,
144 };
145
146 enum {
147 MLX5_MAX_PORTS_NUM = 2,
148 };
149
150 enum {
151 MLX5_CSUM_SUPPORT_RAW_OVER_ETH = (1 << 0),
152 /*
153 * Only report rx checksum when the validation
154 * is valid.
155 */
156 MLX5_RX_CSUM_VALID = (1 << 16),
157 };
158
159 enum mlx5_alloc_type {
160 MLX5_ALLOC_TYPE_ANON,
161 MLX5_ALLOC_TYPE_HUGE,
162 MLX5_ALLOC_TYPE_CONTIG,
163 MLX5_ALLOC_TYPE_PREFER_HUGE,
164 MLX5_ALLOC_TYPE_PREFER_CONTIG,
165 MLX5_ALLOC_TYPE_EXTERNAL,
166 MLX5_ALLOC_TYPE_ALL
167 };
168
169 enum mlx5_rsc_type {
170 MLX5_RSC_TYPE_QP,
171 MLX5_RSC_TYPE_XSRQ,
172 MLX5_RSC_TYPE_SRQ,
173 MLX5_RSC_TYPE_RWQ,
174 MLX5_RSC_TYPE_INVAL,
175 };
176
177 enum {
178 MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE = 1 << 0,
179 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH = 1 << 1,
180 };
181
182 enum mlx5_vendor_cap_flags {
183 MLX5_VENDOR_CAP_FLAGS_MPW = 1 << 0, /* Obsoleted */
184 MLX5_VENDOR_CAP_FLAGS_MPW_ALLOWED = 1 << 1,
185 MLX5_VENDOR_CAP_FLAGS_ENHANCED_MPW = 1 << 2,
186 };
187
188 enum {
189 MLX5_FLOW_TAG_MASK = 0x000fffff,
190 };
191
192 struct mlx5_resource {
193 enum mlx5_rsc_type type;
194 uint32_t rsn;
195 };
196
197 struct mlx5_device {
198 struct verbs_device verbs_dev;
199 int page_size;
200 int driver_abi_ver;
201 };
202
203 struct mlx5_db_page;
204
205 struct mlx5_spinlock {
206 pthread_spinlock_t lock;
207 int in_use;
208 };
209
210 struct mlx5_hugetlb_mem;
211
212 struct mlx5_context {
213 struct ibv_context ibv_ctx;
214 int max_num_qps;
215 int bf_reg_size;
216 int tot_uuars;
217 int low_lat_uuars;
218 int num_uars_per_page;
219 int bf_regs_per_page;
220 int num_bf_regs;
221 int prefer_bf;
222 int shut_up_bf;
223 struct {
224 struct mlx5_qp **table;
225 int refcnt;
226 } qp_table[MLX5_QP_TABLE_SIZE];
227 pthread_mutex_t qp_table_mutex;
228
229 struct {
230 struct mlx5_srq **table;
231 int refcnt;
232 } srq_table[MLX5_SRQ_TABLE_SIZE];
233 pthread_mutex_t srq_table_mutex;
234
235 struct {
236 struct mlx5_resource **table;
237 int refcnt;
238 } uidx_table[MLX5_UIDX_TABLE_SIZE];
239 pthread_mutex_t uidx_table_mutex;
240
241 void *uar[MLX5_MAX_UARS];
242 struct mlx5_spinlock lock32;
243 struct mlx5_db_page *db_list;
244 pthread_mutex_t db_list_mutex;
245 int cache_line_size;
246 int max_sq_desc_sz;
247 int max_rq_desc_sz;
248 int max_send_wqebb;
249 int max_recv_wr;
250 unsigned max_srq_recv_wr;
251 int num_ports;
252 int stall_enable;
253 int stall_adaptive_enable;
254 int stall_cycles;
255 struct mlx5_bf *bfs;
256 FILE *dbg_fp;
257 char hostname[40];
258 struct mlx5_spinlock hugetlb_lock;
259 TAILQ_HEAD(,mlx5_hugetlb_mem) hugetlb_list;
260 int cqe_version;
261 uint8_t cached_link_layer[MLX5_MAX_PORTS_NUM];
262 int cached_device_cap_flags;
263 enum ibv_atomic_cap atomic_cap;
264 struct {
265 uint64_t offset;
266 uint64_t mask;
267 } core_clock;
268 void *hca_core_clock;
269 struct ibv_tso_caps cached_tso_caps;
270 int cmds_supp_uhw;
271 uint32_t uar_size;
272 uint64_t vendor_cap_flags; /* Use enum mlx5_vendor_cap_flags */
273 struct mlx5dv_cqe_comp_caps cqe_comp_caps;
274 struct mlx5dv_ctx_allocators extern_alloc;
275 struct mlx5dv_sw_parsing_caps sw_parsing_caps;
276 struct mlx5dv_striding_rq_caps striding_rq_caps;
277 uint32_t tunnel_offloads_caps;
278 };
279
280 struct mlx5_bitmap {
281 uint32_t last;
282 uint32_t top;
283 uint32_t max;
284 uint32_t avail;
285 uint32_t mask;
286 unsigned long *table;
287 };
288
289 struct mlx5_hugetlb_mem {
290 int shmid;
291 void *shmaddr;
292 struct mlx5_bitmap bitmap;
293 TAILQ_ENTRY(mlx5_hugetlb_mem) entry;
294 };
295
296 struct mlx5_buf {
297 void *buf;
298 size_t length;
299 int base;
300 struct mlx5_hugetlb_mem *hmem;
301 enum mlx5_alloc_type type;
302 };
303
304 struct mlx5_pd {
305 struct ibv_pd ibv_pd;
306 uint32_t pdn;
307 };
308
309 enum {
310 MLX5_CQ_SET_CI = 0,
311 MLX5_CQ_ARM_DB = 1,
312 };
313
314 enum {
315 MLX5_CQ_FLAGS_RX_CSUM_VALID = 1 << 0,
316 MLX5_CQ_FLAGS_EMPTY_DURING_POLL = 1 << 1,
317 MLX5_CQ_FLAGS_FOUND_CQES = 1 << 2,
318 MLX5_CQ_FLAGS_EXTENDED = 1 << 3,
319 MLX5_CQ_FLAGS_SINGLE_THREADED = 1 << 4,
320 MLX5_CQ_FLAGS_DV_OWNED = 1 << 5,
321 };
322
323 struct mlx5_cq {
324 /* ibv_cq should always be subset of ibv_cq_ex */
325 struct ibv_cq_ex ibv_cq;
326 struct mlx5_buf buf_a;
327 struct mlx5_buf buf_b;
328 struct mlx5_buf *active_buf;
329 struct mlx5_buf *resize_buf;
330 int resize_cqes;
331 int active_cqes;
332 struct mlx5_spinlock lock;
333 uint32_t cqn;
334 uint32_t cons_index;
335 __be32 *dbrec;
336 int arm_sn;
337 int cqe_sz;
338 int resize_cqe_sz;
339 int stall_next_poll;
340 int stall_enable;
341 uint64_t stall_last_count;
342 int stall_adaptive_enable;
343 int stall_cycles;
344 struct mlx5_resource *cur_rsc;
345 struct mlx5_srq *cur_srq;
346 struct mlx5_cqe64 *cqe64;
347 uint32_t flags;
348 int umr_opcode;
349 };
350
351 struct mlx5_srq {
352 struct mlx5_resource rsc; /* This struct must be first */
353 struct verbs_srq vsrq;
354 struct mlx5_buf buf;
355 struct mlx5_spinlock lock;
356 uint64_t *wrid;
357 uint32_t srqn;
358 int max;
359 int max_gs;
360 int wqe_shift;
361 int head;
362 int tail;
363 __be32 *db;
364 uint16_t counter;
365 int wq_sig;
366 struct ibv_qp *cmd_qp;
367 };
368
369 struct wr_list {
370 uint16_t opcode;
371 uint16_t next;
372 };
373
374 struct mlx5_wq {
375 uint64_t *wrid;
376 unsigned *wqe_head;
377 struct mlx5_spinlock lock;
378 unsigned wqe_cnt;
379 unsigned max_post;
380 unsigned head;
381 unsigned tail;
382 unsigned cur_post;
383 int max_gs;
384 int wqe_shift;
385 int offset;
386 void *qend;
387 uint32_t *wr_data;
388 };
389
390 struct mlx5_bf {
391 void *reg;
392 int need_lock;
393 struct mlx5_spinlock lock;
394 unsigned offset;
395 unsigned buf_size;
396 unsigned uuarn;
397 off_t uar_mmap_offset;
398 };
399
400 struct mlx5_mr {
401 struct ibv_mr ibv_mr;
402 struct mlx5_buf buf;
403 uint32_t alloc_flags;
404 };
405
406 struct mlx5_qp {
407 struct mlx5_resource rsc; /* This struct must be first */
408 struct verbs_qp verbs_qp;
409 struct ibv_qp *ibv_qp;
410 struct mlx5_buf buf;
411 void *sq_start;
412 int max_inline_data;
413 int buf_size;
414 /* For Raw Packet QP, use different buffers for the SQ and RQ */
415 struct mlx5_buf sq_buf;
416 int sq_buf_size;
417 struct mlx5_bf *bf;
418
419 uint8_t fm_cache;
420 uint8_t sq_signal_bits;
421 struct mlx5_wq sq;
422
423 __be32 *db;
424 struct mlx5_wq rq;
425 int wq_sig;
426 uint32_t qp_cap_cache;
427 int atomics_enabled;
428 uint32_t max_tso;
429 uint16_t max_tso_header;
430 int rss_qp;
431 };
432
433 struct mlx5_ah {
434 struct ibv_ah ibv_ah;
435 struct mlx5_wqe_av av;
436 bool kern_ah;
437 };
438
439 struct mlx5_rwq {
440 struct mlx5_resource rsc;
441 struct ibv_wq wq;
442 struct mlx5_buf buf;
443 int buf_size;
444 struct mlx5_wq rq;
445 __be32 *db;
446 void *pbuff;
447 __be32 *recv_db;
448 int wq_sig;
449 };
450
mlx5_ilog2(int n)451 static inline int mlx5_ilog2(int n)
452 {
453 int t;
454
455 if (n <= 0)
456 return -1;
457
458 t = 0;
459 while ((1 << t) < n)
460 ++t;
461
462 return t;
463 }
464
465 extern int mlx5_stall_num_loop;
466 extern int mlx5_stall_cq_poll_min;
467 extern int mlx5_stall_cq_poll_max;
468 extern int mlx5_stall_cq_inc_step;
469 extern int mlx5_stall_cq_dec_step;
470 extern int mlx5_single_threaded;
471
DIV_ROUND_UP(unsigned n,unsigned d)472 static inline unsigned DIV_ROUND_UP(unsigned n, unsigned d)
473 {
474 return (n + d - 1u) / d;
475 }
476
align(unsigned long val,unsigned long align)477 static inline unsigned long align(unsigned long val, unsigned long align)
478 {
479 return (val + align - 1) & ~(align - 1);
480 }
481
482 #define to_mxxx(xxx, type) \
483 ((struct mlx5_##type *) \
484 ((void *) ib##xxx - offsetof(struct mlx5_##type, ibv_##xxx)))
485
to_mdev(struct ibv_device * ibdev)486 static inline struct mlx5_device *to_mdev(struct ibv_device *ibdev)
487 {
488 struct mlx5_device *ret;
489
490 ret = (void *)ibdev - offsetof(struct mlx5_device, verbs_dev);
491 return ret;
492 }
493
to_mctx(struct ibv_context * ibctx)494 static inline struct mlx5_context *to_mctx(struct ibv_context *ibctx)
495 {
496 return to_mxxx(ctx, context);
497 }
498
to_mpd(struct ibv_pd * ibpd)499 static inline struct mlx5_pd *to_mpd(struct ibv_pd *ibpd)
500 {
501 return to_mxxx(pd, pd);
502 }
503
to_mcq(struct ibv_cq * ibcq)504 static inline struct mlx5_cq *to_mcq(struct ibv_cq *ibcq)
505 {
506 return to_mxxx(cq, cq);
507 }
508
to_msrq(struct ibv_srq * ibsrq)509 static inline struct mlx5_srq *to_msrq(struct ibv_srq *ibsrq)
510 {
511 struct verbs_srq *vsrq = (struct verbs_srq *)ibsrq;
512
513 return container_of(vsrq, struct mlx5_srq, vsrq);
514 }
515
to_mqp(struct ibv_qp * ibqp)516 static inline struct mlx5_qp *to_mqp(struct ibv_qp *ibqp)
517 {
518 struct verbs_qp *vqp = (struct verbs_qp *)ibqp;
519
520 return container_of(vqp, struct mlx5_qp, verbs_qp);
521 }
522
to_mrwq(struct ibv_wq * ibwq)523 static inline struct mlx5_rwq *to_mrwq(struct ibv_wq *ibwq)
524 {
525 return container_of(ibwq, struct mlx5_rwq, wq);
526 }
527
to_mmr(struct ibv_mr * ibmr)528 static inline struct mlx5_mr *to_mmr(struct ibv_mr *ibmr)
529 {
530 return to_mxxx(mr, mr);
531 }
532
to_mah(struct ibv_ah * ibah)533 static inline struct mlx5_ah *to_mah(struct ibv_ah *ibah)
534 {
535 return to_mxxx(ah, ah);
536 }
537
max_int(int a,int b)538 static inline int max_int(int a, int b)
539 {
540 return a > b ? a : b;
541 }
542
rsc_to_mqp(struct mlx5_resource * rsc)543 static inline struct mlx5_qp *rsc_to_mqp(struct mlx5_resource *rsc)
544 {
545 return (struct mlx5_qp *)rsc;
546 }
547
rsc_to_msrq(struct mlx5_resource * rsc)548 static inline struct mlx5_srq *rsc_to_msrq(struct mlx5_resource *rsc)
549 {
550 return (struct mlx5_srq *)rsc;
551 }
552
rsc_to_mrwq(struct mlx5_resource * rsc)553 static inline struct mlx5_rwq *rsc_to_mrwq(struct mlx5_resource *rsc)
554 {
555 return (struct mlx5_rwq *)rsc;
556 }
557
558 int mlx5_alloc_buf(struct mlx5_buf *buf, size_t size, int page_size);
559 void mlx5_free_buf(struct mlx5_buf *buf);
560 int mlx5_alloc_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf,
561 size_t size, int page_size, const char *component);
562 void mlx5_free_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf);
563 int mlx5_alloc_prefered_buf(struct mlx5_context *mctx,
564 struct mlx5_buf *buf,
565 size_t size, int page_size,
566 enum mlx5_alloc_type alloc_type,
567 const char *component);
568 int mlx5_free_actual_buf(struct mlx5_context *ctx, struct mlx5_buf *buf);
569 void mlx5_get_alloc_type(struct mlx5_context *context,
570 const char *component,
571 enum mlx5_alloc_type *alloc_type,
572 enum mlx5_alloc_type default_alloc_type);
573 int mlx5_use_huge(const char *key);
574 bool mlx5_is_extern_alloc(struct mlx5_context *context);
575 int mlx5_alloc_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf,
576 size_t size);
577 void mlx5_free_buf_extern(struct mlx5_context *ctx, struct mlx5_buf *buf);
578
579 __be32 *mlx5_alloc_dbrec(struct mlx5_context *context);
580 void mlx5_free_db(struct mlx5_context *context, __be32 *db);
581
582 int mlx5_query_device(struct ibv_context *context,
583 struct ibv_device_attr *attr);
584 int mlx5_query_device_ex(struct ibv_context *context,
585 const struct ibv_query_device_ex_input *input,
586 struct ibv_device_attr_ex *attr,
587 size_t attr_size);
588 int mlx5_query_rt_values(struct ibv_context *context,
589 struct ibv_values_ex *values);
590 struct ibv_qp *mlx5_create_qp_ex(struct ibv_context *context,
591 struct ibv_qp_init_attr_ex *attr);
592 int mlx5_query_port(struct ibv_context *context, uint8_t port,
593 struct ibv_port_attr *attr);
594
595 struct ibv_pd *mlx5_alloc_pd(struct ibv_context *context);
596 int mlx5_free_pd(struct ibv_pd *pd);
597
598 struct ibv_mr *mlx5_reg_mr(struct ibv_pd *pd, void *addr,
599 size_t length, int access);
600 int mlx5_rereg_mr(struct ibv_mr *mr, int flags, struct ibv_pd *pd, void *addr,
601 size_t length, int access);
602 int mlx5_dereg_mr(struct ibv_mr *mr);
603 struct ibv_mw *mlx5_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type);
604 int mlx5_dealloc_mw(struct ibv_mw *mw);
605 int mlx5_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
606 struct ibv_mw_bind *mw_bind);
607
608 struct ibv_cq *mlx5_create_cq(struct ibv_context *context, int cqe,
609 struct ibv_comp_channel *channel,
610 int comp_vector);
611 struct ibv_cq_ex *mlx5_create_cq_ex(struct ibv_context *context,
612 struct ibv_cq_init_attr_ex *cq_attr);
613 void mlx5_cq_fill_pfns(struct mlx5_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr);
614 int mlx5_alloc_cq_buf(struct mlx5_context *mctx, struct mlx5_cq *cq,
615 struct mlx5_buf *buf, int nent, int cqe_sz);
616 int mlx5_free_cq_buf(struct mlx5_context *ctx, struct mlx5_buf *buf);
617 int mlx5_resize_cq(struct ibv_cq *cq, int cqe);
618 int mlx5_destroy_cq(struct ibv_cq *cq);
619 int mlx5_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
620 int mlx5_poll_cq_v1(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
621 int mlx5_arm_cq(struct ibv_cq *cq, int solicited);
622 void mlx5_cq_event(struct ibv_cq *cq);
623 void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq);
624 void mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq);
625 void mlx5_cq_resize_copy_cqes(struct mlx5_cq *cq);
626
627 struct ibv_srq *mlx5_create_srq(struct ibv_pd *pd,
628 struct ibv_srq_init_attr *attr);
629 int mlx5_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr,
630 int mask);
631 int mlx5_query_srq(struct ibv_srq *srq,
632 struct ibv_srq_attr *attr);
633 int mlx5_destroy_srq(struct ibv_srq *srq);
634 int mlx5_alloc_srq_buf(struct ibv_context *context, struct mlx5_srq *srq);
635 void mlx5_free_srq_wqe(struct mlx5_srq *srq, int ind);
636 int mlx5_post_srq_recv(struct ibv_srq *ibsrq,
637 struct ibv_recv_wr *wr,
638 struct ibv_recv_wr **bad_wr);
639
640 struct ibv_qp *mlx5_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
641 int mlx5_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
642 int attr_mask,
643 struct ibv_qp_init_attr *init_attr);
644 int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
645 int attr_mask);
646 int mlx5_destroy_qp(struct ibv_qp *qp);
647 void mlx5_init_qp_indices(struct mlx5_qp *qp);
648 void mlx5_init_rwq_indices(struct mlx5_rwq *rwq);
649 int mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
650 struct ibv_send_wr **bad_wr);
651 int mlx5_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
652 struct ibv_recv_wr **bad_wr);
653 int mlx5_post_wq_recv(struct ibv_wq *ibwq, struct ibv_recv_wr *wr,
654 struct ibv_recv_wr **bad_wr);
655 void mlx5_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
656 struct mlx5_qp *qp);
657 void mlx5_set_sq_sizes(struct mlx5_qp *qp, struct ibv_qp_cap *cap,
658 enum ibv_qp_type type);
659 struct mlx5_qp *mlx5_find_qp(struct mlx5_context *ctx, uint32_t qpn);
660 int mlx5_store_qp(struct mlx5_context *ctx, uint32_t qpn, struct mlx5_qp *qp);
661 void mlx5_clear_qp(struct mlx5_context *ctx, uint32_t qpn);
662 int32_t mlx5_store_uidx(struct mlx5_context *ctx, void *rsc);
663 void mlx5_clear_uidx(struct mlx5_context *ctx, uint32_t uidx);
664 struct mlx5_srq *mlx5_find_srq(struct mlx5_context *ctx, uint32_t srqn);
665 int mlx5_store_srq(struct mlx5_context *ctx, uint32_t srqn,
666 struct mlx5_srq *srq);
667 void mlx5_clear_srq(struct mlx5_context *ctx, uint32_t srqn);
668 struct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
669 int mlx5_destroy_ah(struct ibv_ah *ah);
670 int mlx5_alloc_av(struct mlx5_pd *pd, struct ibv_ah_attr *attr,
671 struct mlx5_ah *ah);
672 void mlx5_free_av(struct mlx5_ah *ah);
673 int mlx5_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid);
674 int mlx5_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid);
675 int mlx5_round_up_power_of_two(long long sz);
676 void *mlx5_get_atomic_laddr(struct mlx5_qp *qp, uint16_t idx, int *byte_count);
677 void *mlx5_get_send_wqe(struct mlx5_qp *qp, int n);
678 int mlx5_copy_to_recv_wqe(struct mlx5_qp *qp, int idx, void *buf, int size);
679 int mlx5_copy_to_send_wqe(struct mlx5_qp *qp, int idx, void *buf, int size);
680 int mlx5_copy_to_recv_srq(struct mlx5_srq *srq, int idx, void *buf, int size);
681 struct ibv_xrcd *mlx5_open_xrcd(struct ibv_context *context,
682 struct ibv_xrcd_init_attr *xrcd_init_attr);
683 int mlx5_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num);
684 int mlx5_close_xrcd(struct ibv_xrcd *ib_xrcd);
685 struct ibv_wq *mlx5_create_wq(struct ibv_context *context,
686 struct ibv_wq_init_attr *attr);
687 int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr);
688 int mlx5_destroy_wq(struct ibv_wq *wq);
689 struct ibv_rwq_ind_table *mlx5_create_rwq_ind_table(struct ibv_context *context,
690 struct ibv_rwq_ind_table_init_attr *init_attr);
691 int mlx5_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table);
692 struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context,
693 struct ibv_srq_init_attr_ex *attr);
694
mlx5_find_uidx(struct mlx5_context * ctx,uint32_t uidx)695 static inline void *mlx5_find_uidx(struct mlx5_context *ctx, uint32_t uidx)
696 {
697 int tind = uidx >> MLX5_UIDX_TABLE_SHIFT;
698
699 if (likely(ctx->uidx_table[tind].refcnt))
700 return ctx->uidx_table[tind].table[uidx & MLX5_UIDX_TABLE_MASK];
701
702 return NULL;
703 }
704
mlx5_spin_lock(struct mlx5_spinlock * lock)705 static inline int mlx5_spin_lock(struct mlx5_spinlock *lock)
706 {
707 if (!mlx5_single_threaded)
708 return pthread_spin_lock(&lock->lock);
709
710 if (unlikely(lock->in_use)) {
711 fprintf(stderr, "*** ERROR: multithreading vilation ***\n"
712 "You are running a multithreaded application but\n"
713 "you set MLX5_SINGLE_THREADED=1. Please unset it.\n");
714 abort();
715 } else {
716 lock->in_use = 1;
717 /*
718 * This fence is not at all correct, but it increases the
719 * chance that in_use is detected by another thread without
720 * much runtime cost. */
721 atomic_thread_fence(memory_order_acq_rel);
722 }
723
724 return 0;
725 }
726
mlx5_spin_unlock(struct mlx5_spinlock * lock)727 static inline int mlx5_spin_unlock(struct mlx5_spinlock *lock)
728 {
729 if (!mlx5_single_threaded)
730 return pthread_spin_unlock(&lock->lock);
731
732 lock->in_use = 0;
733
734 return 0;
735 }
736
mlx5_spinlock_init(struct mlx5_spinlock * lock)737 static inline int mlx5_spinlock_init(struct mlx5_spinlock *lock)
738 {
739 lock->in_use = 0;
740 return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE);
741 }
742
mlx5_spinlock_destroy(struct mlx5_spinlock * lock)743 static inline int mlx5_spinlock_destroy(struct mlx5_spinlock *lock)
744 {
745 return pthread_spin_destroy(&lock->lock);
746 }
747
set_command(int command,off_t * offset)748 static inline void set_command(int command, off_t *offset)
749 {
750 *offset |= (command << MLX5_IB_MMAP_CMD_SHIFT);
751 }
752
set_arg(int arg,off_t * offset)753 static inline void set_arg(int arg, off_t *offset)
754 {
755 *offset |= arg;
756 }
757
set_order(int order,off_t * offset)758 static inline void set_order(int order, off_t *offset)
759 {
760 set_arg(order, offset);
761 }
762
set_index(int index,off_t * offset)763 static inline void set_index(int index, off_t *offset)
764 {
765 set_arg(index, offset);
766 }
767
calc_sig(void * wqe,int size)768 static inline uint8_t calc_sig(void *wqe, int size)
769 {
770 int i;
771 uint8_t *p = wqe;
772 uint8_t res = 0;
773
774 for (i = 0; i < size; ++i)
775 res ^= p[i];
776
777 return ~res;
778 }
779
780 #endif /* MLX5_H */
781