1 /*-
2 * Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #ifndef MLX5_IB_H
27 #define MLX5_IB_H
28
29 #include <linux/kernel.h>
30 #include <linux/sched.h>
31 #include <linux/printk.h>
32 #include <linux/netdevice.h>
33 #include <rdma/ib_verbs.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_smi.h>
36 #include <dev/mlx5/cq.h>
37 #include <dev/mlx5/qp.h>
38 #include <dev/mlx5/srq.h>
39 #include <linux/types.h>
40 #include <dev/mlx5/mlx5_core/transobj.h>
41 #include <rdma/ib_user_verbs.h>
42 #include <rdma/mlx5-abi.h>
43 #include <rdma/uverbs_ioctl.h>
44
45 #define mlx5_ib_dbg(dev, format, arg...) \
46 pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
47 __LINE__, current->pid, ##arg)
48
49 #define mlx5_ib_err(dev, format, arg...) \
50 pr_err("%s: ERR: %s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
51 __LINE__, current->pid, ##arg)
52
53 #define mlx5_ib_warn(dev, format, arg...) \
54 pr_warn("%s: WARN: %s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
55 __LINE__, current->pid, ##arg)
56
57 #define field_avail(type, fld, sz) (offsetof(type, fld) + \
58 sizeof(((type *)0)->fld) <= (sz))
59 #define MLX5_IB_DEFAULT_UIDX 0xffffff
60 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
61
62 #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
63
64 enum {
65 MLX5_IB_MMAP_CMD_SHIFT = 8,
66 MLX5_IB_MMAP_CMD_MASK = 0xff,
67 };
68
69 /*
70 * Reserved mmap command range used to encode rdma_user_mmap entry page
71 * offsets (e.g. dynamically allocated UARs). Keeping these above the legacy
72 * MLX5_IB_MMAP_* commands ensures mlx5_ib_mmap() routes them to the
73 * rdma_user_mmap offset handler instead of the legacy bfreg uar_mmap() path.
74 */
75 enum {
76 MLX5_IB_MMAP_OFFSET_START = 9,
77 MLX5_IB_MMAP_OFFSET_END = 255,
78 };
79
80 enum {
81 MLX5_RES_SCAT_DATA32_CQE = 0x1,
82 MLX5_RES_SCAT_DATA64_CQE = 0x2,
83 MLX5_REQ_SCAT_DATA32_CQE = 0x11,
84 MLX5_REQ_SCAT_DATA64_CQE = 0x22,
85 };
86
87 enum mlx5_ib_latency_class {
88 MLX5_IB_LATENCY_CLASS_LOW,
89 MLX5_IB_LATENCY_CLASS_MEDIUM,
90 MLX5_IB_LATENCY_CLASS_HIGH,
91 MLX5_IB_LATENCY_CLASS_FAST_PATH
92 };
93
94 enum mlx5_ib_mad_ifc_flags {
95 MLX5_MAD_IFC_IGNORE_MKEY = 1,
96 MLX5_MAD_IFC_IGNORE_BKEY = 2,
97 MLX5_MAD_IFC_NET_VIEW = 4,
98 };
99
100 enum {
101 MLX5_CROSS_CHANNEL_BFREG = 0,
102 };
103
104 enum {
105 MLX5_CQE_VERSION_V0,
106 MLX5_CQE_VERSION_V1,
107 };
108
109 enum {
110 MLX5_IB_INVALID_UAR_INDEX = BIT(31),
111 MLX5_IB_INVALID_BFREG = BIT(31),
112 };
113
114 enum mlx5_ib_mmap_type {
115 MLX5_IB_MMAP_TYPE_MEMIC = 1,
116 MLX5_IB_MMAP_TYPE_VAR = 2,
117 MLX5_IB_MMAP_TYPE_UAR_WC = 3,
118 MLX5_IB_MMAP_TYPE_UAR_NC = 4,
119 };
120
121 struct mlx5_bfreg_info {
122 u32 *sys_pages;
123 int num_low_latency_bfregs;
124 unsigned int *count;
125
126 /*
127 * protect bfreg allocation data structs
128 */
129 struct mutex lock;
130 u32 ver;
131 u8 lib_uar_4k : 1;
132 u8 lib_uar_dyn : 1;
133 u32 num_sys_pages;
134 u32 num_static_sys_pages;
135 u32 total_num_bfregs;
136 u32 num_dyn_bfregs;
137 };
138
139 struct mlx5_ib_ucontext {
140 struct ib_ucontext ibucontext;
141 struct list_head db_page_list;
142
143 /* protect doorbell record alloc/free
144 */
145 struct mutex db_page_mutex;
146 struct mlx5_bfreg_info bfregi;
147 u8 cqe_version;
148 /* Transport Domain number */
149 u32 tdn;
150
151 u64 lib_caps;
152 u16 devx_uid;
153 };
154
to_mucontext(struct ib_ucontext * ibucontext)155 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
156 {
157 return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
158 }
159
160 struct mlx5_ib_pd {
161 struct ib_pd ibpd;
162 u32 pdn;
163 u16 uid;
164 };
165
166 #define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
167 #define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
168 #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
169 #error "Invalid number of bypass priorities"
170 #endif
171 #define MLX5_IB_FLOW_LEFTOVERS_PRIO (MLX5_IB_FLOW_MCAST_PRIO + 1)
172
173 #define MLX5_IB_NUM_FLOW_FT (MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
174 #define MLX5_IB_NUM_SNIFFER_FTS 2
175 struct mlx5_ib_flow_prio {
176 struct mlx5_flow_table *flow_table;
177 unsigned int refcount;
178 };
179
180 struct mlx5_ib_flow_handler {
181 struct list_head list;
182 struct ib_flow ibflow;
183 struct mlx5_ib_flow_prio *prio;
184 struct mlx5_flow_handle *rule;
185 };
186
187 struct mlx5_ib_flow_db {
188 struct mlx5_ib_flow_prio prios[MLX5_IB_NUM_FLOW_FT];
189 struct mlx5_ib_flow_prio sniffer[MLX5_IB_NUM_SNIFFER_FTS];
190 struct mlx5_flow_table *lag_demux_ft;
191 /* Protect flow steering bypass flow tables
192 * when add/del flow rules.
193 * only single add/removal of flow steering rule could be done
194 * simultaneously.
195 */
196 struct mutex lock;
197 };
198
199 /* Use macros here so that don't have to duplicate
200 * enum ib_send_flags and enum ib_qp_type for low-level driver
201 */
202
203 #define MLX5_IB_SEND_UMR_UNREG IB_SEND_RESERVED_START
204 #define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 1)
205 #define MLX5_IB_SEND_UMR_UPDATE_MTT (IB_SEND_RESERVED_START << 2)
206
207 #define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 3)
208 #define MLX5_IB_SEND_UMR_UPDATE_PD (IB_SEND_RESERVED_START << 4)
209 #define MLX5_IB_SEND_UMR_UPDATE_ACCESS IB_SEND_RESERVED_END
210
211 #define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
212 /*
213 * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
214 * creates the actual hardware QP.
215 */
216 #define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
217 #define MLX5_IB_QPT_DCI IB_QPT_RESERVED3
218 #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4
219 #define MLX5_IB_WR_UMR IB_WR_RESERVED1
220
221 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
222 *
223 * These flags are intended for internal use by the mlx5_ib driver, and they
224 * rely on the range reserved for that use in the ib_qp_create_flags enum.
225 */
226 #define MLX5_IB_QP_CREATE_SQPN_QP1 IB_QP_CREATE_RESERVED_START
227 #define MLX5_IB_QP_CREATE_WC_TEST (IB_QP_CREATE_RESERVED_START << 1)
228
229 struct wr_list {
230 u16 opcode;
231 u16 next;
232 };
233
234 struct mlx5_ib_wq {
235 u64 *wrid;
236 u32 *wr_data;
237 struct wr_list *w_list;
238 unsigned *wqe_head;
239 u16 unsig_count;
240
241 /* serialize post to the work queue
242 */
243 spinlock_t lock;
244 int wqe_cnt;
245 int max_post;
246 int max_gs;
247 int offset;
248 int wqe_shift;
249 unsigned head;
250 unsigned tail;
251 u16 cur_post;
252 u16 last_poll;
253 void *qend;
254 };
255
256 struct mlx5_ib_rwq {
257 struct ib_wq ibwq;
258 struct mlx5_core_qp core_qp;
259 u32 rq_num_pas;
260 u32 log_rq_stride;
261 u32 log_rq_size;
262 u32 rq_page_offset;
263 u32 log_page_size;
264 struct ib_umem *umem;
265 size_t buf_size;
266 unsigned int page_shift;
267 int create_type;
268 struct mlx5_db db;
269 u32 user_index;
270 u32 wqe_count;
271 u32 wqe_shift;
272 int wq_sig;
273 };
274
275 enum {
276 MLX5_QP_USER,
277 MLX5_QP_KERNEL,
278 MLX5_QP_EMPTY
279 };
280
281 enum {
282 MLX5_WQ_USER,
283 MLX5_WQ_KERNEL
284 };
285
286 struct mlx5_ib_rwq_ind_table {
287 struct ib_rwq_ind_table ib_rwq_ind_tbl;
288 u32 rqtn;
289 u16 uid;
290 };
291
292 /*
293 * Connect-IB can trigger up to four concurrent pagefaults
294 * per-QP.
295 */
296 enum mlx5_ib_pagefault_context {
297 MLX5_IB_PAGEFAULT_RESPONDER_READ,
298 MLX5_IB_PAGEFAULT_REQUESTOR_READ,
299 MLX5_IB_PAGEFAULT_RESPONDER_WRITE,
300 MLX5_IB_PAGEFAULT_REQUESTOR_WRITE,
301 MLX5_IB_PAGEFAULT_CONTEXTS
302 };
303
304 static inline enum mlx5_ib_pagefault_context
mlx5_ib_get_pagefault_context(struct mlx5_pagefault * pagefault)305 mlx5_ib_get_pagefault_context(struct mlx5_pagefault *pagefault)
306 {
307 return pagefault->flags & (MLX5_PFAULT_REQUESTOR | MLX5_PFAULT_WRITE);
308 }
309
310 struct mlx5_ib_pfault {
311 struct work_struct work;
312 struct mlx5_pagefault mpfault;
313 };
314
315 struct mlx5_ib_ubuffer {
316 struct ib_umem *umem;
317 int buf_size;
318 u64 buf_addr;
319 };
320
321 struct mlx5_ib_qp_base {
322 struct mlx5_ib_qp *container_mibqp;
323 struct mlx5_core_qp mqp;
324 struct mlx5_ib_ubuffer ubuffer;
325 };
326
327 struct mlx5_ib_qp_trans {
328 struct mlx5_ib_qp_base base;
329 u16 xrcdn;
330 u8 alt_port;
331 u8 atomic_rd_en;
332 u8 resp_depth;
333 };
334
335 struct mlx5_ib_rss_qp {
336 u32 tirn;
337 };
338
339 struct mlx5_ib_rq {
340 struct mlx5_ib_qp_base base;
341 struct mlx5_ib_wq *rq;
342 struct mlx5_ib_ubuffer ubuffer;
343 struct mlx5_db *doorbell;
344 u32 tirn;
345 u8 state;
346 };
347
348 struct mlx5_ib_sq {
349 struct mlx5_ib_qp_base base;
350 struct mlx5_ib_wq *sq;
351 struct mlx5_ib_ubuffer ubuffer;
352 struct mlx5_db *doorbell;
353 u32 tisn;
354 u8 state;
355 };
356
357 struct mlx5_ib_raw_packet_qp {
358 struct mlx5_ib_sq sq;
359 struct mlx5_ib_rq rq;
360 };
361
362 struct mlx5_bf {
363 int buf_size;
364 unsigned long offset;
365 struct mlx5_sq_bfreg *bfreg;
366 spinlock_t lock32;
367 };
368
369 struct mlx5_ib_dct {
370 struct mlx5_core_dct mdct;
371 u32 *in;
372 };
373
374 struct mlx5_ib_qp {
375 struct ib_qp ibqp;
376 union {
377 struct mlx5_ib_qp_trans trans_qp;
378 struct mlx5_ib_raw_packet_qp raw_packet_qp;
379 struct mlx5_ib_rss_qp rss_qp;
380 struct mlx5_ib_dct dct;
381 };
382 struct mlx5_buf buf;
383
384 struct mlx5_db db;
385 struct mlx5_ib_wq rq;
386
387 u8 sq_signal_bits;
388 u8 fm_cache;
389 struct mlx5_ib_wq sq;
390
391 /* serialize qp state modifications
392 */
393 struct mutex mutex;
394 u32 flags;
395 u8 port;
396 u8 state;
397 int wq_sig;
398 int scat_cqe;
399 int max_inline_data;
400 struct mlx5_bf bf;
401 int has_rq;
402
403 /* only for user space QPs. For kernel
404 * we have it from the bf object
405 */
406 int bfregn;
407
408 int create_type;
409
410 /* Store signature errors */
411 bool signature_en;
412
413 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
414 /*
415 * A flag that is true for QP's that are in a state that doesn't
416 * allow page faults, and shouldn't schedule any more faults.
417 */
418 int disable_page_faults;
419 /*
420 * The disable_page_faults_lock protects a QP's disable_page_faults
421 * field, allowing for a thread to atomically check whether the QP
422 * allows page faults, and if so schedule a page fault.
423 */
424 spinlock_t disable_page_faults_lock;
425 struct mlx5_ib_pfault pagefaults[MLX5_IB_PAGEFAULT_CONTEXTS];
426 #endif
427 struct list_head qps_list;
428 struct list_head cq_recv_list;
429 struct list_head cq_send_list;
430 };
431
432 struct mlx5_ib_cq_buf {
433 struct mlx5_buf buf;
434 struct ib_umem *umem;
435 int cqe_size;
436 int nent;
437 };
438
439 enum mlx5_ib_qp_flags {
440 MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
441 MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
442 MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
443 MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
444 MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
445 MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
446 /* QP uses 1 as its source QP number */
447 MLX5_IB_QP_SQPN_QP1 = 1 << 6,
448 MLX5_IB_QP_CAP_SCATTER_FCS = 1 << 7,
449 MLX5_IB_QP_RSS = 1 << 8,
450 MLX5_IB_QP_UNDERLAY = 1 << 10,
451 };
452
453 struct mlx5_umr_wr {
454 struct ib_send_wr wr;
455 union {
456 u64 virt_addr;
457 u64 offset;
458 } target;
459 struct ib_pd *pd;
460 unsigned int page_shift;
461 unsigned int npages;
462 u32 length;
463 int access_flags;
464 u32 mkey;
465 };
466
umr_wr(const struct ib_send_wr * wr)467 static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)
468 {
469 return container_of(wr, struct mlx5_umr_wr, wr);
470 }
471
472 struct mlx5_shared_mr_info {
473 int mr_id;
474 struct ib_umem *umem;
475 };
476
477 struct mlx5_ib_cq {
478 struct ib_cq ibcq;
479 struct mlx5_core_cq mcq;
480 struct mlx5_ib_cq_buf buf;
481 struct mlx5_db db;
482
483 /* serialize access to the CQ
484 */
485 spinlock_t lock;
486
487 /* protect resize cq
488 */
489 struct mutex resize_mutex;
490 struct mlx5_ib_cq_buf *resize_buf;
491 struct ib_umem *resize_umem;
492 int cqe_size;
493 struct list_head list_send_qp;
494 struct list_head list_recv_qp;
495 u32 create_flags;
496 struct list_head wc_list;
497 enum ib_cq_notify_flags notify_flags;
498 struct work_struct notify_work;
499 };
500
501 struct mlx5_ib_wc {
502 struct ib_wc wc;
503 struct list_head list;
504 };
505
506 struct mlx5_ib_srq {
507 struct ib_srq ibsrq;
508 struct mlx5_core_srq msrq;
509 struct mlx5_buf buf;
510 struct mlx5_db db;
511 u64 *wrid;
512 /* protect SRQ hanlding
513 */
514 spinlock_t lock;
515 int head;
516 int tail;
517 u16 wqe_ctr;
518 struct ib_umem *umem;
519 /* serialize arming a SRQ
520 */
521 struct mutex mutex;
522 int wq_sig;
523 };
524
525 struct mlx5_ib_xrcd {
526 struct ib_xrcd ibxrcd;
527 u32 xrcdn;
528 };
529
530 enum mlx5_ib_mtt_access_flags {
531 MLX5_IB_MTT_READ = (1 << 0),
532 MLX5_IB_MTT_WRITE = (1 << 1),
533 };
534
535 struct mlx5_user_mmap_entry {
536 struct rdma_user_mmap_entry rdma_entry;
537 u8 mmap_flag;
538 u64 address;
539 u32 page_idx;
540 };
541
542 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
543
544 struct mlx5_ib_mr {
545 struct ib_mr ibmr;
546 void *descs;
547 dma_addr_t desc_map;
548 int ndescs;
549 int max_descs;
550 int desc_size;
551 int access_mode;
552 struct mlx5_core_mkey mmkey;
553 struct ib_umem *umem;
554 struct mlx5_shared_mr_info *smr_info;
555 struct list_head list;
556 int order;
557 int umred;
558 int npages;
559 struct mlx5_ib_dev *dev;
560 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
561 struct mlx5_core_sig_ctx *sig;
562 int live;
563 void *descs_alloc;
564 int access_flags; /* Needed for rereg MR */
565 struct mlx5_async_work cb_work;
566 };
567
568 struct mlx5_ib_mw {
569 struct ib_mw ibmw;
570 struct mlx5_core_mkey mmkey;
571 };
572
573 struct mlx5_ib_devx_mr {
574 struct mlx5_core_mkey mmkey;
575 int ndescs;
576 };
577
578 struct mlx5_ib_umr_context {
579 struct ib_cqe cqe;
580 enum ib_wc_status status;
581 struct completion done;
582 };
583
584 struct umr_common {
585 struct ib_pd *pd;
586 struct ib_cq *cq;
587 struct ib_qp *qp;
588 /* control access to UMR QP
589 */
590 struct semaphore sem;
591 };
592
593 enum {
594 MLX5_FMR_INVALID,
595 MLX5_FMR_VALID,
596 MLX5_FMR_BUSY,
597 };
598
599 struct mlx5_cache_ent {
600 struct list_head head;
601 /* sync access to the cahce entry
602 */
603 spinlock_t lock;
604
605
606 char name[4];
607 u32 order;
608 u32 size;
609 u32 cur;
610 u32 miss;
611 u32 limit;
612
613 struct mlx5_ib_dev *dev;
614 struct work_struct work;
615 struct delayed_work dwork;
616 int pending;
617 };
618
619 struct mlx5_mr_cache {
620 struct workqueue_struct *wq;
621 struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES];
622 int stopped;
623 unsigned long last_add;
624 };
625
626 struct mlx5_ib_gsi_qp;
627
628 struct mlx5_ib_port_resources {
629 struct mlx5_ib_resources *devr;
630 struct mlx5_ib_gsi_qp *gsi;
631 struct work_struct pkey_change_work;
632 };
633
634 struct mlx5_ib_resources {
635 struct ib_cq *c0;
636 struct ib_xrcd *x0;
637 struct ib_xrcd *x1;
638 struct ib_pd *p0;
639 struct ib_srq *s0;
640 struct ib_srq *s1;
641 struct mlx5_ib_port_resources ports[2];
642 /* Protects changes to the port resources */
643 struct mutex mutex;
644 };
645
646 struct mlx5_ib_port {
647 u16 q_cnt_id;
648 };
649
650 struct mlx5_roce {
651 /* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
652 * netdev pointer
653 */
654 rwlock_t netdev_lock;
655 if_t netdev;
656 struct notifier_block nb;
657 atomic_t next_port;
658 };
659
660 #define MLX5_IB_STATS_COUNT(a,...) a
661 #define MLX5_IB_STATS_VAR(a,b,c,...) b c;
662 #define MLX5_IB_STATS_DESC(a,b,c,d,e,...) d, e,
663
664 #define MLX5_IB_CONG_PARAMS(m) \
665 /* ECN RP */ \
666 m(+1, u64, rp_clamp_tgt_rate, "rp_clamp_tgt_rate", "If set, whenever a CNP is processed, the target rate is updated to be the current rate") \
667 m(+1, u64, rp_clamp_tgt_rate_ati, "rp_clamp_tgt_rate_ati", "If set, when receiving a CNP, the target rate should be updated if the transission rate was increased due to the timer, and not only due to the byte counter") \
668 m(+1, u64, rp_time_reset, "rp_time_reset", "Time in microseconds between rate increases if no CNPs are received") \
669 m(+1, u64, rp_byte_reset, "rp_byte_reset", "Transmitted data in bytes between rate increases if no CNP's are received. A value of zero means disabled.") \
670 m(+1, u64, rp_threshold, "rp_threshold", "The number of times rpByteStage or rpTimeStage can count before the RP rate control state machine advances states") \
671 m(+1, u64, rp_ai_rate, "rp_ai_rate", "The rate, in Mbits per second, used to increase rpTargetRate in the active increase state") \
672 m(+1, u64, rp_hai_rate, "rp_hai_rate", "The rate, in Mbits per second, used to increase rpTargetRate in the hyper increase state") \
673 m(+1, u64, rp_min_dec_fac, "rp_min_dec_fac", "The minimum factor by which the current transmit rate can be changed when processing a CNP. Value is given as a percentage, [1 .. 100]") \
674 m(+1, u64, rp_min_rate, "rp_min_rate", "The minimum value, in Mbps per second, for rate to limit") \
675 m(+1, u64, rp_rate_to_set_on_first_cnp, "rp_rate_to_set_on_first_cnp", "The rate that is set for the flow when a rate limiter is allocated to it upon first CNP received, in Mbps. A value of zero means use full port speed") \
676 m(+1, u64, rp_dce_tcp_g, "rp_dce_tcp_g", "Used to update the congestion estimator, alpha, once every dce_tcp_rtt once every dce_tcp_rtt microseconds") \
677 m(+1, u64, rp_dce_tcp_rtt, "rp_dce_tcp_rtt", "The time between updates of the aolpha value, in microseconds") \
678 m(+1, u64, rp_rate_reduce_monitor_period, "rp_rate_reduce_monitor_period", "The minimum time between two consecutive rate reductions for a single flow") \
679 m(+1, u64, rp_initial_alpha_value, "rp_initial_alpha_value", "The initial value of alpha to use when receiving the first CNP for a flow") \
680 m(+1, u64, rp_gd, "rp_gd", "If a CNP is received, the flow rate is reduced at the beginning of the next rate_reduce_monitor_period interval") \
681 /* ECN NP */ \
682 m(+1, u64, np_cnp_dscp, "np_cnp_dscp", "The DiffServ Code Point of the generated CNP for this port") \
683 m(+1, u64, np_cnp_prio_mode, "np_cnp_prio_mode", "The 802.1p priority value of the generated CNP for this port") \
684 m(+1, u64, np_cnp_prio, "np_cnp_prio", "The 802.1p priority value of the generated CNP for this port")
685
686 #define MLX5_IB_CONG_PARAMS_NUM (0 MLX5_IB_CONG_PARAMS(MLX5_IB_STATS_COUNT))
687
688 #define MLX5_IB_CONG_STATS(m) \
689 m(+1, u64, syndrome, "syndrome", "Syndrome number") \
690 m(+1, u64, rp_cur_flows, "rp_cur_flows", "Number of flows limited") \
691 m(+1, u64, sum_flows, "sum_flows", "Sum of the number of flows limited over time") \
692 m(+1, u64, rp_cnp_ignored, "rp_cnp_ignored", "Number of CNPs and CNMs ignored") \
693 m(+1, u64, rp_cnp_handled, "rp_cnp_handled", "Number of CNPs and CNMs successfully handled") \
694 m(+1, u64, time_stamp, "time_stamp", "Time stamp in microseconds") \
695 m(+1, u64, accumulators_period, "accumulators_period", "The value of X variable for accumulating counters") \
696 m(+1, u64, np_ecn_marked_roce_packets, "np_ecn_marked_roce_packets", "Number of ECN marked packets seen") \
697 m(+1, u64, np_cnp_sent, "np_cnp_sent", "Number of CNPs sent")
698
699 #define MLX5_IB_CONG_STATS_NUM (0 MLX5_IB_CONG_STATS(MLX5_IB_STATS_COUNT))
700
701 #define MLX5_IB_CONG_STATUS(m) \
702 /* ECN RP */ \
703 m(+1, u64, rp_0_enable, "rp_0_enable", "Enable reaction point, priority 0", MLX5_IB_RROCE_ECN_RP, 0, enable) \
704 m(+1, u64, rp_1_enable, "rp_1_enable", "Enable reaction point, priority 1", MLX5_IB_RROCE_ECN_RP, 1, enable) \
705 m(+1, u64, rp_2_enable, "rp_2_enable", "Enable reaction point, priority 2", MLX5_IB_RROCE_ECN_RP, 2, enable) \
706 m(+1, u64, rp_3_enable, "rp_3_enable", "Enable reaction point, priority 3", MLX5_IB_RROCE_ECN_RP, 3, enable) \
707 m(+1, u64, rp_4_enable, "rp_4_enable", "Enable reaction point, priority 4", MLX5_IB_RROCE_ECN_RP, 4, enable) \
708 m(+1, u64, rp_5_enable, "rp_5_enable", "Enable reaction point, priority 5", MLX5_IB_RROCE_ECN_RP, 5, enable) \
709 m(+1, u64, rp_6_enable, "rp_6_enable", "Enable reaction point, priority 6", MLX5_IB_RROCE_ECN_RP, 6, enable) \
710 m(+1, u64, rp_7_enable, "rp_7_enable", "Enable reaction point, priority 7", MLX5_IB_RROCE_ECN_RP, 7, enable) \
711 m(+1, u64, rp_8_enable, "rp_8_enable", "Enable reaction point, priority 8", MLX5_IB_RROCE_ECN_RP, 8, enable) \
712 m(+1, u64, rp_9_enable, "rp_9_enable", "Enable reaction point, priority 9", MLX5_IB_RROCE_ECN_RP, 9, enable) \
713 m(+1, u64, rp_10_enable, "rp_10_enable", "Enable reaction point, priority 10", MLX5_IB_RROCE_ECN_RP, 10, enable) \
714 m(+1, u64, rp_11_enable, "rp_11_enable", "Enable reaction point, priority 11", MLX5_IB_RROCE_ECN_RP, 11, enable) \
715 m(+1, u64, rp_12_enable, "rp_12_enable", "Enable reaction point, priority 12", MLX5_IB_RROCE_ECN_RP, 12, enable) \
716 m(+1, u64, rp_13_enable, "rp_13_enable", "Enable reaction point, priority 13", MLX5_IB_RROCE_ECN_RP, 13, enable) \
717 m(+1, u64, rp_14_enable, "rp_14_enable", "Enable reaction point, priority 14", MLX5_IB_RROCE_ECN_RP, 14, enable) \
718 m(+1, u64, rp_15_enable, "rp_15_enable", "Enable reaction point, priority 15", MLX5_IB_RROCE_ECN_RP, 15, enable) \
719 /* ECN NP */ \
720 m(+1, u64, np_0_enable, "np_0_enable", "Enable notification point, priority 0", MLX5_IB_RROCE_ECN_NP, 0, enable) \
721 m(+1, u64, np_1_enable, "np_1_enable", "Enable notification point, priority 1", MLX5_IB_RROCE_ECN_NP, 1, enable) \
722 m(+1, u64, np_2_enable, "np_2_enable", "Enable notification point, priority 2", MLX5_IB_RROCE_ECN_NP, 2, enable) \
723 m(+1, u64, np_3_enable, "np_3_enable", "Enable notification point, priority 3", MLX5_IB_RROCE_ECN_NP, 3, enable) \
724 m(+1, u64, np_4_enable, "np_4_enable", "Enable notification point, priority 4", MLX5_IB_RROCE_ECN_NP, 4, enable) \
725 m(+1, u64, np_5_enable, "np_5_enable", "Enable notification point, priority 5", MLX5_IB_RROCE_ECN_NP, 5, enable) \
726 m(+1, u64, np_6_enable, "np_6_enable", "Enable notification point, priority 6", MLX5_IB_RROCE_ECN_NP, 6, enable) \
727 m(+1, u64, np_7_enable, "np_7_enable", "Enable notification point, priority 7", MLX5_IB_RROCE_ECN_NP, 7, enable) \
728 m(+1, u64, np_8_enable, "np_8_enable", "Enable notification point, priority 8", MLX5_IB_RROCE_ECN_NP, 8, enable) \
729 m(+1, u64, np_9_enable, "np_9_enable", "Enable notification point, priority 9", MLX5_IB_RROCE_ECN_NP, 9, enable) \
730 m(+1, u64, np_10_enable, "np_10_enable", "Enable notification point, priority 10", MLX5_IB_RROCE_ECN_NP, 10, enable) \
731 m(+1, u64, np_11_enable, "np_11_enable", "Enable notification point, priority 11", MLX5_IB_RROCE_ECN_NP, 11, enable) \
732 m(+1, u64, np_12_enable, "np_12_enable", "Enable notification point, priority 12", MLX5_IB_RROCE_ECN_NP, 12, enable) \
733 m(+1, u64, np_13_enable, "np_13_enable", "Enable notification point, priority 13", MLX5_IB_RROCE_ECN_NP, 13, enable) \
734 m(+1, u64, np_14_enable, "np_14_enable", "Enable notification point, priority 14", MLX5_IB_RROCE_ECN_NP, 14, enable) \
735 m(+1, u64, np_15_enable, "np_15_enable", "Enable notification point, priority 15", MLX5_IB_RROCE_ECN_NP, 15, enable) \
736
737 #define MLX5_IB_CONG_STATUS_NUM (0 MLX5_IB_CONG_STATUS(MLX5_IB_STATS_COUNT))
738
739 struct mlx5_ib_congestion {
740 struct sysctl_ctx_list ctx;
741 struct sx lock;
742 struct delayed_work dwork;
743 union {
744 u64 arg[1];
745 struct {
746 MLX5_IB_CONG_PARAMS(MLX5_IB_STATS_VAR)
747 MLX5_IB_CONG_STATS(MLX5_IB_STATS_VAR)
748 MLX5_IB_CONG_STATUS(MLX5_IB_STATS_VAR)
749 };
750 };
751 };
752
753 struct mlx5_devx_event_table {
754 /* serialize updating the event_xa */
755 struct mutex event_xa_lock;
756 struct xarray event_xa;
757 };
758
759 struct mlx5_ib_dev {
760 struct ib_device ib_dev;
761 struct mlx5_core_dev *mdev;
762 struct mlx5_roce roce;
763 MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
764 int num_ports;
765 /* serialize update of capability mask
766 */
767 struct mutex cap_mask_mutex;
768 u8 ib_active:1;
769 u8 wc_support:1;
770 struct umr_common umrc;
771 /* sync used page count stats
772 */
773 struct mlx5_ib_resources devr;
774 struct mlx5_mr_cache cache;
775 struct timer_list delay_timer;
776 /* Prevents soft lock on massive reg MRs */
777 struct mutex slow_path_mutex;
778 int fill_delay;
779 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
780 struct ib_odp_caps odp_caps;
781 /*
782 * Sleepable RCU that prevents destruction of MRs while they are still
783 * being used by a page fault handler.
784 */
785 struct srcu_struct mr_srcu;
786 #endif
787 struct mlx5_ib_flow_db flow_db;
788 /* protect resources needed as part of reset flow */
789 spinlock_t reset_flow_resource_lock;
790 struct list_head qp_list;
791 /* Array with num_ports elements */
792 struct mlx5_ib_port *port;
793 struct mlx5_sq_bfreg bfreg;
794 struct mlx5_sq_bfreg wc_bfreg;
795 struct mlx5_sq_bfreg fp_bfreg;
796 struct mlx5_devx_event_table devx_event_table;
797 struct mlx5_ib_congestion congestion;
798
799 struct mlx5_async_ctx async_ctx;
800
801 /* protect the user_td */
802 struct mutex lb_mutex;
803 u32 user_td;
804 };
805
to_mibcq(struct mlx5_core_cq * mcq)806 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
807 {
808 return container_of(mcq, struct mlx5_ib_cq, mcq);
809 }
810
to_mxrcd(struct ib_xrcd * ibxrcd)811 static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
812 {
813 return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
814 }
815
to_mdev(struct ib_device * ibdev)816 static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
817 {
818 return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
819 }
820
mlx5_udata_to_mdev(struct ib_udata * udata)821 static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)
822 {
823 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
824 udata, struct mlx5_ib_ucontext, ibucontext);
825
826 return to_mdev(context->ibucontext.device);
827 }
828
to_mcq(struct ib_cq * ibcq)829 static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
830 {
831 return container_of(ibcq, struct mlx5_ib_cq, ibcq);
832 }
833
to_mibqp(struct mlx5_core_qp * mqp)834 static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
835 {
836 return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
837 }
838
to_mibrwq(struct mlx5_core_qp * core_qp)839 static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
840 {
841 return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
842 }
843
to_mibmr(struct mlx5_core_mkey * mmkey)844 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
845 {
846 return container_of(mmkey, struct mlx5_ib_mr, mmkey);
847 }
848
to_mpd(struct ib_pd * ibpd)849 static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
850 {
851 return container_of(ibpd, struct mlx5_ib_pd, ibpd);
852 }
853
to_msrq(struct ib_srq * ibsrq)854 static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
855 {
856 return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
857 }
858
to_mqp(struct ib_qp * ibqp)859 static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
860 {
861 return container_of(ibqp, struct mlx5_ib_qp, ibqp);
862 }
863
to_mrwq(struct ib_wq * ibwq)864 static inline struct mlx5_ib_rwq *to_mrwq(struct ib_wq *ibwq)
865 {
866 return container_of(ibwq, struct mlx5_ib_rwq, ibwq);
867 }
868
to_mrwq_ind_table(struct ib_rwq_ind_table * ib_rwq_ind_tbl)869 static inline struct mlx5_ib_rwq_ind_table *to_mrwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
870 {
871 return container_of(ib_rwq_ind_tbl, struct mlx5_ib_rwq_ind_table, ib_rwq_ind_tbl);
872 }
873
to_mibsrq(struct mlx5_core_srq * msrq)874 static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
875 {
876 return container_of(msrq, struct mlx5_ib_srq, msrq);
877 }
878
to_mmr(struct ib_mr * ibmr)879 static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
880 {
881 return container_of(ibmr, struct mlx5_ib_mr, ibmr);
882 }
883
to_mmw(struct ib_mw * ibmw)884 static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
885 {
886 return container_of(ibmw, struct mlx5_ib_mw, ibmw);
887 }
888
889 struct mlx5_ib_ah {
890 struct ib_ah ibah;
891 struct mlx5_av av;
892 };
893
to_mah(struct ib_ah * ibah)894 static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
895 {
896 return container_of(ibah, struct mlx5_ib_ah, ibah);
897 }
898
899 static inline struct mlx5_user_mmap_entry *
to_mmmap(struct rdma_user_mmap_entry * rdma_entry)900 to_mmmap(struct rdma_user_mmap_entry *rdma_entry)
901 {
902 return container_of(rdma_entry,
903 struct mlx5_user_mmap_entry, rdma_entry);
904 }
905
906 int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
907 struct mlx5_db *db);
908 void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
909 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
910 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
911 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
912 int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
913 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
914 const void *in_mad, void *response_mad);
915 int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
916 struct ib_udata *udata);
917 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
918 void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
919 int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
920 struct ib_udata *udata);
921 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
922 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
923 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
924 void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
925 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
926 const struct ib_recv_wr **bad_wr);
927 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
928 struct ib_qp_init_attr *init_attr,
929 struct ib_udata *udata);
930 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
931 int attr_mask, struct ib_udata *udata);
932 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
933 struct ib_qp_init_attr *qp_init_attr);
934 int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
935 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
936 const struct ib_send_wr **bad_wr);
937 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
938 const struct ib_recv_wr **bad_wr);
939 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
940 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
941 void *buffer, u32 length,
942 struct mlx5_ib_qp_base *base);
943 int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
944 struct ib_udata *udata);
945 void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
946 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
947 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
948 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
949 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
950 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
951 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
952 u64 virt_addr, int access_flags,
953 struct ib_udata *udata);
954 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
955 struct ib_udata *udata);
956 int mlx5_ib_dealloc_mw(struct ib_mw *mw);
957 int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index,
958 int npages, int zap);
959 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
960 u64 length, u64 virt_addr, int access_flags,
961 struct ib_pd *pd, struct ib_udata *udata);
962 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
963 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
964 u32 max_num_sg, struct ib_udata *udata);
965 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
966 unsigned int *sg_offset);
967 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
968 const struct ib_wc *in_wc, const struct ib_grh *in_grh,
969 const struct ib_mad_hdr *in, size_t in_mad_size,
970 struct ib_mad_hdr *out, size_t *out_mad_size,
971 u16 *out_mad_pkey_index);
972 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
973 struct ib_udata *udata);
974 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
975 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
976 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
977 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
978 struct ib_smp *out_mad);
979 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
980 __be64 *sys_image_guid);
981 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
982 u16 *max_pkeys);
983 int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,
984 u32 *vendor_id);
985 int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);
986 int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);
987 int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,
988 u16 *pkey);
989 int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,
990 union ib_gid *gid);
991 int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
992 struct ib_port_attr *props);
993 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
994 struct ib_port_attr *props);
995 int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
996 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
997 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
998 unsigned long max_page_shift,
999 int *count, int *shift,
1000 int *ncont, int *order);
1001 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1002 int page_shift, size_t offset, size_t num_pages,
1003 __be64 *pas, int access_flags);
1004 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1005 int page_shift, __be64 *pas, int access_flags);
1006 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
1007 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
1008 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
1009 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
1010 int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
1011 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1012 struct ib_mr_status *mr_status);
1013 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
1014 struct ib_wq_init_attr *init_attr,
1015 struct ib_udata *udata);
1016 void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
1017 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
1018 u32 wq_attr_mask, struct ib_udata *udata);
1019 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
1020 struct ib_rwq_ind_table_init_attr *init_attr,
1021 struct ib_udata *udata);
1022 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
1023
1024 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1025 extern struct workqueue_struct *mlx5_ib_page_fault_wq;
1026
1027 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
1028 void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
1029 struct mlx5_ib_pfault *pfault);
1030 void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp);
1031 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
1032 void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
1033 int __init mlx5_ib_odp_init(void);
1034 void mlx5_ib_odp_cleanup(void);
1035 void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp);
1036 void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp);
1037 void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
1038 unsigned long end);
1039 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev * dev)1040 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
1041 {
1042 return;
1043 }
1044
mlx5_ib_odp_create_qp(struct mlx5_ib_qp * qp)1045 static inline void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp) {}
mlx5_ib_odp_init_one(struct mlx5_ib_dev * ibdev)1046 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
mlx5_ib_odp_remove_one(struct mlx5_ib_dev * ibdev)1047 static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev) {}
mlx5_ib_odp_init(void)1048 static inline int mlx5_ib_odp_init(void) { return 0; }
mlx5_ib_odp_cleanup(void)1049 static inline void mlx5_ib_odp_cleanup(void) {}
mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp * qp)1050 static inline void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp) {}
mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp * qp)1051 static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {}
1052
1053 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1054
1055 int mlx5_ib_get_vf_config(struct ib_device *device, int vf,
1056 u8 port, struct ifla_vf_info *info);
1057 int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
1058 u8 port, int state);
1059 int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,
1060 u8 port, struct ifla_vf_stats *stats);
1061 int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
1062 u64 guid, int type);
1063
1064 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
1065 const struct ib_gid_attr *attr);
1066
1067 /* GSI QP helper functions */
1068 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
1069 struct ib_qp_init_attr *init_attr);
1070 int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
1071 int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1072 int attr_mask);
1073 int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
1074 int qp_attr_mask,
1075 struct ib_qp_init_attr *qp_init_attr);
1076 int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,
1077 const struct ib_send_wr **bad_wr);
1078 int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,
1079 const struct ib_recv_wr **bad_wr);
1080 void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
1081
1082 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
1083
1084 void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
1085 int bfregn);
1086
1087 #if 1 /* IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) */
1088 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
1089 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
1090 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
1091 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
1092 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);
1093 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id);
1094 extern const struct uapi_definition mlx5_ib_devx_defs[];
1095 #else
1096 static inline int
mlx5_ib_devx_create(struct mlx5_ib_dev * dev,bool is_user)1097 mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
1098 bool is_user) { return -EOPNOTSUPP; }
mlx5_ib_devx_destroy(struct mlx5_ib_dev * dev,u16 uid)1099 static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
mlx5_ib_devx_init_event_table(struct mlx5_ib_dev * dev)1100 static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev * dev)1101 static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
mlx5_ib_devx_is_flow_dest(void * obj,int * dest_id,int * dest_type)1102 static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
1103 int *dest_type)
1104 {
1105 return false;
1106 }
1107 #endif
1108
init_query_mad(struct ib_smp * mad)1109 static inline void init_query_mad(struct ib_smp *mad)
1110 {
1111 mad->base_version = 1;
1112 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1113 mad->class_version = 1;
1114 mad->method = IB_MGMT_METHOD_GET;
1115 }
1116
convert_access(int acc)1117 static inline u8 convert_access(int acc)
1118 {
1119 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
1120 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
1121 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
1122 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
1123 MLX5_PERM_LOCAL_READ;
1124 }
1125
is_qp1(enum ib_qp_type qp_type)1126 static inline int is_qp1(enum ib_qp_type qp_type)
1127 {
1128 return qp_type == MLX5_IB_QPT_HW_GSI;
1129 }
1130
1131 #define MLX5_MAX_UMR_SHIFT 16
1132 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1133
check_cq_create_flags(u32 flags)1134 static inline u32 check_cq_create_flags(u32 flags)
1135 {
1136 /*
1137 * It returns non-zero value for unsupported CQ
1138 * create flags, otherwise it returns zero.
1139 */
1140 return (flags & ~(IB_CQ_FLAGS_IGNORE_OVERRUN |
1141 IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
1142 }
1143
verify_assign_uidx(u8 cqe_version,u32 cmd_uidx,u32 * user_index)1144 static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
1145 u32 *user_index)
1146 {
1147 if (cqe_version) {
1148 if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
1149 (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
1150 return -EINVAL;
1151 *user_index = cmd_uidx;
1152 } else {
1153 *user_index = MLX5_IB_DEFAULT_UIDX;
1154 }
1155
1156 return 0;
1157 }
1158
get_qp_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_qp * ucmd,int inlen,u32 * user_index)1159 static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
1160 struct mlx5_ib_create_qp *ucmd,
1161 int inlen,
1162 u32 *user_index)
1163 {
1164 u8 cqe_version = ucontext->cqe_version;
1165
1166 if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
1167 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1168 return 0;
1169
1170 if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
1171 !!cqe_version))
1172 return -EINVAL;
1173
1174 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1175 }
1176
get_srq_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_srq * ucmd,int inlen,u32 * user_index)1177 static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
1178 struct mlx5_ib_create_srq *ucmd,
1179 int inlen,
1180 u32 *user_index)
1181 {
1182 u8 cqe_version = ucontext->cqe_version;
1183
1184 if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
1185 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1186 return 0;
1187
1188 if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
1189 !!cqe_version))
1190 return -EINVAL;
1191
1192 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1193 }
1194
1195 void mlx5_ib_cleanup_congestion(struct mlx5_ib_dev *);
1196 int mlx5_ib_init_congestion(struct mlx5_ib_dev *);
1197
get_uars_per_sys_page(struct mlx5_ib_dev * dev,bool lib_support)1198 static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
1199 {
1200 return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1201 MLX5_UARS_IN_PAGE : 1;
1202 }
1203
get_num_static_uars(struct mlx5_ib_dev * dev,struct mlx5_bfreg_info * bfregi)1204 static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
1205 struct mlx5_bfreg_info *bfregi)
1206 {
1207 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
1208 }
1209
1210 int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
1211 struct mlx5_bfreg_info *bfregi, u32 bfregn,
1212 bool dyn_bfreg);
1213
1214 #endif /* MLX5_IB_H */
1215