1 /*-
2 * Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #ifndef MLX5_IB_H
27 #define MLX5_IB_H
28
29 #include <linux/kernel.h>
30 #include <linux/sched.h>
31 #include <linux/printk.h>
32 #include <linux/netdevice.h>
33 #include <rdma/ib_verbs.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_smi.h>
36 #include <dev/mlx5/cq.h>
37 #include <dev/mlx5/qp.h>
38 #include <dev/mlx5/srq.h>
39 #include <linux/types.h>
40 #include <dev/mlx5/mlx5_core/transobj.h>
41 #include <rdma/ib_user_verbs.h>
42 #include <rdma/mlx5-abi.h>
43 #include <rdma/uverbs_ioctl.h>
44
45 #define mlx5_ib_dbg(dev, format, arg...) \
46 pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
47 __LINE__, current->pid, ##arg)
48
49 #define mlx5_ib_err(dev, format, arg...) \
50 pr_err("%s: ERR: %s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
51 __LINE__, current->pid, ##arg)
52
53 #define mlx5_ib_warn(dev, format, arg...) \
54 pr_warn("%s: WARN: %s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
55 __LINE__, current->pid, ##arg)
56
57 #define field_avail(type, fld, sz) (offsetof(type, fld) + \
58 sizeof(((type *)0)->fld) <= (sz))
59 #define MLX5_IB_DEFAULT_UIDX 0xffffff
60 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
61
62 #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
63
64 enum {
65 MLX5_IB_MMAP_CMD_SHIFT = 8,
66 MLX5_IB_MMAP_CMD_MASK = 0xff,
67 };
68
69 /*
70 * Reserved mmap command range used to encode rdma_user_mmap entry page
71 * offsets (e.g. dynamically allocated UARs). Keeping these above the legacy
72 * MLX5_IB_MMAP_* commands ensures mlx5_ib_mmap() routes them to the
73 * rdma_user_mmap offset handler instead of the legacy bfreg uar_mmap() path.
74 */
75 enum {
76 MLX5_IB_MMAP_OFFSET_START = 9,
77 MLX5_IB_MMAP_OFFSET_END = 255,
78 };
79
80 enum {
81 MLX5_RES_SCAT_DATA32_CQE = 0x1,
82 MLX5_RES_SCAT_DATA64_CQE = 0x2,
83 MLX5_REQ_SCAT_DATA32_CQE = 0x11,
84 MLX5_REQ_SCAT_DATA64_CQE = 0x22,
85 };
86
87 enum mlx5_ib_latency_class {
88 MLX5_IB_LATENCY_CLASS_LOW,
89 MLX5_IB_LATENCY_CLASS_MEDIUM,
90 MLX5_IB_LATENCY_CLASS_HIGH,
91 MLX5_IB_LATENCY_CLASS_FAST_PATH
92 };
93
94 enum mlx5_ib_mad_ifc_flags {
95 MLX5_MAD_IFC_IGNORE_MKEY = 1,
96 MLX5_MAD_IFC_IGNORE_BKEY = 2,
97 MLX5_MAD_IFC_NET_VIEW = 4,
98 };
99
100 enum {
101 MLX5_CROSS_CHANNEL_BFREG = 0,
102 };
103
104 enum {
105 MLX5_CQE_VERSION_V0,
106 MLX5_CQE_VERSION_V1,
107 };
108
109 enum {
110 MLX5_IB_INVALID_UAR_INDEX = BIT(31),
111 MLX5_IB_INVALID_BFREG = BIT(31),
112 };
113
114 enum mlx5_ib_mmap_type {
115 MLX5_IB_MMAP_TYPE_MEMIC = 1,
116 MLX5_IB_MMAP_TYPE_VAR = 2,
117 MLX5_IB_MMAP_TYPE_UAR_WC = 3,
118 MLX5_IB_MMAP_TYPE_UAR_NC = 4,
119 };
120
121 struct mlx5_bfreg_info {
122 u32 *sys_pages;
123 int num_low_latency_bfregs;
124 unsigned int *count;
125
126 /*
127 * protect bfreg allocation data structs
128 */
129 struct mutex lock;
130 u32 ver;
131 u8 lib_uar_4k : 1;
132 u8 lib_uar_dyn : 1;
133 u32 num_sys_pages;
134 u32 num_static_sys_pages;
135 u32 total_num_bfregs;
136 u32 num_dyn_bfregs;
137 };
138
139 struct mlx5_ib_ucontext {
140 struct ib_ucontext ibucontext;
141 struct list_head db_page_list;
142
143 /* protect doorbell record alloc/free
144 */
145 struct mutex db_page_mutex;
146 struct mlx5_bfreg_info bfregi;
147 u8 cqe_version;
148 /* Transport Domain number */
149 u32 tdn;
150
151 u64 lib_caps;
152 u16 devx_uid;
153 };
154
to_mucontext(struct ib_ucontext * ibucontext)155 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
156 {
157 return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
158 }
159
160 struct mlx5_ib_pd {
161 struct ib_pd ibpd;
162 u32 pdn;
163 u16 uid;
164 };
165
166 #define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
167 #define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
168 #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
169 #error "Invalid number of bypass priorities"
170 #endif
171 #define MLX5_IB_FLOW_LEFTOVERS_PRIO (MLX5_IB_FLOW_MCAST_PRIO + 1)
172
173 #define MLX5_IB_NUM_FLOW_FT (MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
174 #define MLX5_IB_NUM_SNIFFER_FTS 2
175 struct mlx5_ib_flow_prio {
176 struct mlx5_flow_table *flow_table;
177 unsigned int refcount;
178 };
179
180 struct mlx5_ib_flow_handler {
181 struct list_head list;
182 struct ib_flow ibflow;
183 struct mlx5_ib_flow_prio *prio;
184 struct mlx5_flow_handle *rule;
185 };
186
187 struct mlx5_ib_flow_db {
188 struct mlx5_ib_flow_prio prios[MLX5_IB_NUM_FLOW_FT];
189 struct mlx5_ib_flow_prio sniffer[MLX5_IB_NUM_SNIFFER_FTS];
190 struct mlx5_flow_table *lag_demux_ft;
191 /* Protect flow steering bypass flow tables
192 * when add/del flow rules.
193 * only single add/removal of flow steering rule could be done
194 * simultaneously.
195 */
196 struct mutex lock;
197 };
198
199 /* Use macros here so that don't have to duplicate
200 * enum ib_send_flags and enum ib_qp_type for low-level driver
201 */
202
203 #define MLX5_IB_SEND_UMR_UNREG IB_SEND_RESERVED_START
204 #define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 1)
205 #define MLX5_IB_SEND_UMR_UPDATE_MTT (IB_SEND_RESERVED_START << 2)
206
207 #define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 3)
208 #define MLX5_IB_SEND_UMR_UPDATE_PD (IB_SEND_RESERVED_START << 4)
209 #define MLX5_IB_SEND_UMR_UPDATE_ACCESS IB_SEND_RESERVED_END
210
211 #define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
212 /*
213 * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
214 * creates the actual hardware QP.
215 */
216 #define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
217 #define MLX5_IB_QPT_DCI IB_QPT_RESERVED3
218 #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4
219 #define MLX5_IB_WR_UMR IB_WR_RESERVED1
220
221 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
222 *
223 * These flags are intended for internal use by the mlx5_ib driver, and they
224 * rely on the range reserved for that use in the ib_qp_create_flags enum.
225 */
226 #define MLX5_IB_QP_CREATE_SQPN_QP1 IB_QP_CREATE_RESERVED_START
227 #define MLX5_IB_QP_CREATE_WC_TEST (IB_QP_CREATE_RESERVED_START << 1)
228
229 struct wr_list {
230 u16 opcode;
231 u16 next;
232 };
233
234 enum mlx5_ib_wq_flags {
235 MLX5_IB_WQ_FLAGS_STRIDING_RQ = 0x2,
236 };
237
238 enum mlx5_ib_rq_flags {
239 MLX5_IB_RQ_CVLAN_STRIPPING = 1 << 0,
240 };
241
242 struct mlx5_ib_wq {
243 u64 *wrid;
244 u32 *wr_data;
245 struct wr_list *w_list;
246 unsigned *wqe_head;
247 u16 unsig_count;
248
249 /* serialize post to the work queue
250 */
251 spinlock_t lock;
252 int wqe_cnt;
253 int max_post;
254 int max_gs;
255 int offset;
256 int wqe_shift;
257 unsigned head;
258 unsigned tail;
259 u16 cur_post;
260 u16 last_poll;
261 void *qend;
262 };
263
264 #define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
265 #define MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES 16
266 #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
267 #define MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES 13
268
269 struct mlx5_ib_rwq {
270 struct ib_wq ibwq;
271 struct mlx5_core_qp core_qp;
272 u32 rq_num_pas;
273 u32 log_rq_stride;
274 u32 log_rq_size;
275 u32 rq_page_offset;
276 u32 log_page_size;
277 u32 log_num_strides;
278 u32 two_byte_shift_en;
279 u32 single_stride_log_num_of_bytes;
280 struct ib_umem *umem;
281 size_t buf_size;
282 unsigned int page_shift;
283 int create_type;
284 struct mlx5_db db;
285 u32 user_index;
286 u32 wqe_count;
287 u32 wqe_shift;
288 int wq_sig;
289 u32 create_flags; /* Use enum mlx5_ib_wq_flags */
290 };
291
292 enum {
293 MLX5_QP_USER,
294 MLX5_QP_KERNEL,
295 MLX5_QP_EMPTY
296 };
297
298 enum {
299 MLX5_WQ_USER,
300 MLX5_WQ_KERNEL
301 };
302
303 struct mlx5_ib_rwq_ind_table {
304 struct ib_rwq_ind_table ib_rwq_ind_tbl;
305 u32 rqtn;
306 u16 uid;
307 };
308
309 /*
310 * Connect-IB can trigger up to four concurrent pagefaults
311 * per-QP.
312 */
313 enum mlx5_ib_pagefault_context {
314 MLX5_IB_PAGEFAULT_RESPONDER_READ,
315 MLX5_IB_PAGEFAULT_REQUESTOR_READ,
316 MLX5_IB_PAGEFAULT_RESPONDER_WRITE,
317 MLX5_IB_PAGEFAULT_REQUESTOR_WRITE,
318 MLX5_IB_PAGEFAULT_CONTEXTS
319 };
320
321 static inline enum mlx5_ib_pagefault_context
mlx5_ib_get_pagefault_context(struct mlx5_pagefault * pagefault)322 mlx5_ib_get_pagefault_context(struct mlx5_pagefault *pagefault)
323 {
324 return pagefault->flags & (MLX5_PFAULT_REQUESTOR | MLX5_PFAULT_WRITE);
325 }
326
327 struct mlx5_ib_pfault {
328 struct work_struct work;
329 struct mlx5_pagefault mpfault;
330 };
331
332 struct mlx5_ib_ubuffer {
333 struct ib_umem *umem;
334 int buf_size;
335 u64 buf_addr;
336 };
337
338 struct mlx5_ib_qp_base {
339 struct mlx5_ib_qp *container_mibqp;
340 struct mlx5_core_qp mqp;
341 struct mlx5_ib_ubuffer ubuffer;
342 };
343
344 struct mlx5_ib_qp_trans {
345 struct mlx5_ib_qp_base base;
346 u16 xrcdn;
347 u8 alt_port;
348 u8 atomic_rd_en;
349 u8 resp_depth;
350 };
351
352 struct mlx5_ib_rss_qp {
353 u32 tirn;
354 };
355
356 struct mlx5_ib_rq {
357 struct mlx5_ib_qp_base base;
358 struct mlx5_ib_wq *rq;
359 struct mlx5_ib_ubuffer ubuffer;
360 struct mlx5_db *doorbell;
361 u32 tirn;
362 u8 state;
363 u32 flags;
364 };
365
366 struct mlx5_ib_sq {
367 struct mlx5_ib_qp_base base;
368 struct mlx5_ib_wq *sq;
369 struct mlx5_ib_ubuffer ubuffer;
370 struct mlx5_db *doorbell;
371 u32 tisn;
372 u8 state;
373 };
374
375 struct mlx5_ib_raw_packet_qp {
376 struct mlx5_ib_sq sq;
377 struct mlx5_ib_rq rq;
378 };
379
380 struct mlx5_bf {
381 int buf_size;
382 unsigned long offset;
383 struct mlx5_sq_bfreg *bfreg;
384 spinlock_t lock32;
385 };
386
387 struct mlx5_ib_dct {
388 struct mlx5_core_dct mdct;
389 u32 *in;
390 };
391
392 struct mlx5_ib_qp {
393 struct ib_qp ibqp;
394 union {
395 struct mlx5_ib_qp_trans trans_qp;
396 struct mlx5_ib_raw_packet_qp raw_packet_qp;
397 struct mlx5_ib_rss_qp rss_qp;
398 struct mlx5_ib_dct dct;
399 };
400 struct mlx5_buf buf;
401
402 struct mlx5_db db;
403 struct mlx5_ib_wq rq;
404
405 u8 sq_signal_bits;
406 u8 fm_cache;
407 struct mlx5_ib_wq sq;
408
409 /* serialize qp state modifications
410 */
411 struct mutex mutex;
412 u32 flags;
413 u8 port;
414 u8 state;
415 int wq_sig;
416 int scat_cqe;
417 int max_inline_data;
418 struct mlx5_bf bf;
419 int has_rq;
420
421 /* only for user space QPs. For kernel
422 * we have it from the bf object
423 */
424 int bfregn;
425
426 int create_type;
427
428 /* Store signature errors */
429 bool signature_en;
430
431 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
432 /*
433 * A flag that is true for QP's that are in a state that doesn't
434 * allow page faults, and shouldn't schedule any more faults.
435 */
436 int disable_page_faults;
437 /*
438 * The disable_page_faults_lock protects a QP's disable_page_faults
439 * field, allowing for a thread to atomically check whether the QP
440 * allows page faults, and if so schedule a page fault.
441 */
442 spinlock_t disable_page_faults_lock;
443 struct mlx5_ib_pfault pagefaults[MLX5_IB_PAGEFAULT_CONTEXTS];
444 #endif
445 struct list_head qps_list;
446 struct list_head cq_recv_list;
447 struct list_head cq_send_list;
448 bool tunnel_offload_en;
449 };
450
451 struct mlx5_ib_cq_buf {
452 struct mlx5_buf buf;
453 struct ib_umem *umem;
454 int cqe_size;
455 int nent;
456 };
457
458 enum mlx5_ib_qp_flags {
459 MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
460 MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
461 MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
462 MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
463 MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
464 MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
465 /* QP uses 1 as its source QP number */
466 MLX5_IB_QP_SQPN_QP1 = 1 << 6,
467 MLX5_IB_QP_CAP_SCATTER_FCS = 1 << 7,
468 MLX5_IB_QP_RSS = 1 << 8,
469 MLX5_IB_QP_CVLAN_STRIPPING = 1 << 9,
470 MLX5_IB_QP_UNDERLAY = 1 << 10,
471 };
472
473 struct mlx5_umr_wr {
474 struct ib_send_wr wr;
475 union {
476 u64 virt_addr;
477 u64 offset;
478 } target;
479 struct ib_pd *pd;
480 unsigned int page_shift;
481 unsigned int npages;
482 u64 length;
483 int access_flags;
484 u32 mkey;
485 };
486
umr_wr(const struct ib_send_wr * wr)487 static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)
488 {
489 return container_of(wr, struct mlx5_umr_wr, wr);
490 }
491
492 struct mlx5_shared_mr_info {
493 int mr_id;
494 struct ib_umem *umem;
495 };
496
497 struct mlx5_ib_cq {
498 struct ib_cq ibcq;
499 struct mlx5_core_cq mcq;
500 struct mlx5_ib_cq_buf buf;
501 struct mlx5_db db;
502
503 /* serialize access to the CQ
504 */
505 spinlock_t lock;
506
507 /* protect resize cq
508 */
509 struct mutex resize_mutex;
510 struct mlx5_ib_cq_buf *resize_buf;
511 struct ib_umem *resize_umem;
512 int cqe_size;
513 struct list_head list_send_qp;
514 struct list_head list_recv_qp;
515 u32 create_flags;
516 struct list_head wc_list;
517 enum ib_cq_notify_flags notify_flags;
518 struct work_struct notify_work;
519 };
520
521 struct mlx5_ib_wc {
522 struct ib_wc wc;
523 struct list_head list;
524 };
525
526 struct mlx5_ib_srq {
527 struct ib_srq ibsrq;
528 struct mlx5_core_srq msrq;
529 struct mlx5_buf buf;
530 struct mlx5_db db;
531 u64 *wrid;
532 /* protect SRQ hanlding
533 */
534 spinlock_t lock;
535 int head;
536 int tail;
537 u16 wqe_ctr;
538 struct ib_umem *umem;
539 /* serialize arming a SRQ
540 */
541 struct mutex mutex;
542 int wq_sig;
543 };
544
545 struct mlx5_ib_xrcd {
546 struct ib_xrcd ibxrcd;
547 u32 xrcdn;
548 };
549
550 enum mlx5_ib_mtt_access_flags {
551 MLX5_IB_MTT_READ = (1 << 0),
552 MLX5_IB_MTT_WRITE = (1 << 1),
553 };
554
555 struct mlx5_user_mmap_entry {
556 struct rdma_user_mmap_entry rdma_entry;
557 u8 mmap_flag;
558 u64 address;
559 u32 page_idx;
560 };
561
562 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
563
564 struct mlx5_ib_mr {
565 struct ib_mr ibmr;
566 void *descs;
567 dma_addr_t desc_map;
568 int ndescs;
569 int max_descs;
570 int desc_size;
571 int access_mode;
572 struct mlx5_core_mkey mmkey;
573 struct ib_umem *umem;
574 struct mlx5_shared_mr_info *smr_info;
575 struct list_head list;
576 int order;
577 int umred;
578 int npages;
579 struct mlx5_ib_dev *dev;
580 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
581 struct mlx5_core_sig_ctx *sig;
582 int live;
583 void *descs_alloc;
584 int access_flags; /* Needed for rereg MR */
585 struct mlx5_async_work cb_work;
586 };
587
588 struct mlx5_ib_mw {
589 struct ib_mw ibmw;
590 struct mlx5_core_mkey mmkey;
591 };
592
593 struct mlx5_ib_devx_mr {
594 struct mlx5_core_mkey mmkey;
595 int ndescs;
596 };
597
598 struct mlx5_ib_umr_context {
599 struct ib_cqe cqe;
600 enum ib_wc_status status;
601 struct completion done;
602 };
603
604 struct umr_common {
605 struct ib_pd *pd;
606 struct ib_cq *cq;
607 struct ib_qp *qp;
608 /* control access to UMR QP
609 */
610 struct semaphore sem;
611 };
612
613 enum {
614 MLX5_FMR_INVALID,
615 MLX5_FMR_VALID,
616 MLX5_FMR_BUSY,
617 };
618
619 struct mlx5_cache_ent {
620 struct list_head head;
621 /* sync access to the cahce entry
622 */
623 spinlock_t lock;
624
625
626 char name[4];
627 u32 order;
628 u32 size;
629 u32 cur;
630 u32 miss;
631 u32 limit;
632
633 struct mlx5_ib_dev *dev;
634 struct work_struct work;
635 struct delayed_work dwork;
636 int pending;
637 };
638
639 struct mlx5_mr_cache {
640 struct workqueue_struct *wq;
641 struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES];
642 int stopped;
643 unsigned long last_add;
644 };
645
646 struct mlx5_ib_gsi_qp;
647
648 struct mlx5_ib_port_resources {
649 struct mlx5_ib_resources *devr;
650 struct mlx5_ib_gsi_qp *gsi;
651 struct work_struct pkey_change_work;
652 };
653
654 struct mlx5_ib_resources {
655 struct ib_cq *c0;
656 struct ib_xrcd *x0;
657 struct ib_xrcd *x1;
658 struct ib_pd *p0;
659 struct ib_srq *s0;
660 struct ib_srq *s1;
661 struct mlx5_ib_port_resources ports[2];
662 /* Protects changes to the port resources */
663 struct mutex mutex;
664 };
665
666 struct mlx5_ib_port {
667 u16 q_cnt_id;
668 };
669
670 struct mlx5_roce {
671 /* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
672 * netdev pointer
673 */
674 rwlock_t netdev_lock;
675 if_t netdev;
676 struct notifier_block nb;
677 atomic_t next_port;
678 };
679
680 #define MLX5_IB_STATS_COUNT(a,...) a
681 #define MLX5_IB_STATS_VAR(a,b,c,...) b c;
682 #define MLX5_IB_STATS_DESC(a,b,c,d,e,...) d, e,
683
684 #define MLX5_IB_CONG_PARAMS(m) \
685 /* ECN RP */ \
686 m(+1, u64, rp_clamp_tgt_rate, "rp_clamp_tgt_rate", "If set, whenever a CNP is processed, the target rate is updated to be the current rate") \
687 m(+1, u64, rp_clamp_tgt_rate_ati, "rp_clamp_tgt_rate_ati", "If set, when receiving a CNP, the target rate should be updated if the transission rate was increased due to the timer, and not only due to the byte counter") \
688 m(+1, u64, rp_time_reset, "rp_time_reset", "Time in microseconds between rate increases if no CNPs are received") \
689 m(+1, u64, rp_byte_reset, "rp_byte_reset", "Transmitted data in bytes between rate increases if no CNP's are received. A value of zero means disabled.") \
690 m(+1, u64, rp_threshold, "rp_threshold", "The number of times rpByteStage or rpTimeStage can count before the RP rate control state machine advances states") \
691 m(+1, u64, rp_ai_rate, "rp_ai_rate", "The rate, in Mbits per second, used to increase rpTargetRate in the active increase state") \
692 m(+1, u64, rp_hai_rate, "rp_hai_rate", "The rate, in Mbits per second, used to increase rpTargetRate in the hyper increase state") \
693 m(+1, u64, rp_min_dec_fac, "rp_min_dec_fac", "The minimum factor by which the current transmit rate can be changed when processing a CNP. Value is given as a percentage, [1 .. 100]") \
694 m(+1, u64, rp_min_rate, "rp_min_rate", "The minimum value, in Mbps per second, for rate to limit") \
695 m(+1, u64, rp_rate_to_set_on_first_cnp, "rp_rate_to_set_on_first_cnp", "The rate that is set for the flow when a rate limiter is allocated to it upon first CNP received, in Mbps. A value of zero means use full port speed") \
696 m(+1, u64, rp_dce_tcp_g, "rp_dce_tcp_g", "Used to update the congestion estimator, alpha, once every dce_tcp_rtt once every dce_tcp_rtt microseconds") \
697 m(+1, u64, rp_dce_tcp_rtt, "rp_dce_tcp_rtt", "The time between updates of the aolpha value, in microseconds") \
698 m(+1, u64, rp_rate_reduce_monitor_period, "rp_rate_reduce_monitor_period", "The minimum time between two consecutive rate reductions for a single flow") \
699 m(+1, u64, rp_initial_alpha_value, "rp_initial_alpha_value", "The initial value of alpha to use when receiving the first CNP for a flow") \
700 m(+1, u64, rp_gd, "rp_gd", "If a CNP is received, the flow rate is reduced at the beginning of the next rate_reduce_monitor_period interval") \
701 /* ECN NP */ \
702 m(+1, u64, np_cnp_dscp, "np_cnp_dscp", "The DiffServ Code Point of the generated CNP for this port") \
703 m(+1, u64, np_cnp_prio_mode, "np_cnp_prio_mode", "The 802.1p priority value of the generated CNP for this port") \
704 m(+1, u64, np_cnp_prio, "np_cnp_prio", "The 802.1p priority value of the generated CNP for this port")
705
706 #define MLX5_IB_CONG_PARAMS_NUM (0 MLX5_IB_CONG_PARAMS(MLX5_IB_STATS_COUNT))
707
708 #define MLX5_IB_CONG_STATS(m) \
709 m(+1, u64, syndrome, "syndrome", "Syndrome number") \
710 m(+1, u64, rp_cur_flows, "rp_cur_flows", "Number of flows limited") \
711 m(+1, u64, sum_flows, "sum_flows", "Sum of the number of flows limited over time") \
712 m(+1, u64, rp_cnp_ignored, "rp_cnp_ignored", "Number of CNPs and CNMs ignored") \
713 m(+1, u64, rp_cnp_handled, "rp_cnp_handled", "Number of CNPs and CNMs successfully handled") \
714 m(+1, u64, time_stamp, "time_stamp", "Time stamp in microseconds") \
715 m(+1, u64, accumulators_period, "accumulators_period", "The value of X variable for accumulating counters") \
716 m(+1, u64, np_ecn_marked_roce_packets, "np_ecn_marked_roce_packets", "Number of ECN marked packets seen") \
717 m(+1, u64, np_cnp_sent, "np_cnp_sent", "Number of CNPs sent")
718
719 #define MLX5_IB_CONG_STATS_NUM (0 MLX5_IB_CONG_STATS(MLX5_IB_STATS_COUNT))
720
721 #define MLX5_IB_CONG_STATUS(m) \
722 /* ECN RP */ \
723 m(+1, u64, rp_0_enable, "rp_0_enable", "Enable reaction point, priority 0", MLX5_IB_RROCE_ECN_RP, 0, enable) \
724 m(+1, u64, rp_1_enable, "rp_1_enable", "Enable reaction point, priority 1", MLX5_IB_RROCE_ECN_RP, 1, enable) \
725 m(+1, u64, rp_2_enable, "rp_2_enable", "Enable reaction point, priority 2", MLX5_IB_RROCE_ECN_RP, 2, enable) \
726 m(+1, u64, rp_3_enable, "rp_3_enable", "Enable reaction point, priority 3", MLX5_IB_RROCE_ECN_RP, 3, enable) \
727 m(+1, u64, rp_4_enable, "rp_4_enable", "Enable reaction point, priority 4", MLX5_IB_RROCE_ECN_RP, 4, enable) \
728 m(+1, u64, rp_5_enable, "rp_5_enable", "Enable reaction point, priority 5", MLX5_IB_RROCE_ECN_RP, 5, enable) \
729 m(+1, u64, rp_6_enable, "rp_6_enable", "Enable reaction point, priority 6", MLX5_IB_RROCE_ECN_RP, 6, enable) \
730 m(+1, u64, rp_7_enable, "rp_7_enable", "Enable reaction point, priority 7", MLX5_IB_RROCE_ECN_RP, 7, enable) \
731 m(+1, u64, rp_8_enable, "rp_8_enable", "Enable reaction point, priority 8", MLX5_IB_RROCE_ECN_RP, 8, enable) \
732 m(+1, u64, rp_9_enable, "rp_9_enable", "Enable reaction point, priority 9", MLX5_IB_RROCE_ECN_RP, 9, enable) \
733 m(+1, u64, rp_10_enable, "rp_10_enable", "Enable reaction point, priority 10", MLX5_IB_RROCE_ECN_RP, 10, enable) \
734 m(+1, u64, rp_11_enable, "rp_11_enable", "Enable reaction point, priority 11", MLX5_IB_RROCE_ECN_RP, 11, enable) \
735 m(+1, u64, rp_12_enable, "rp_12_enable", "Enable reaction point, priority 12", MLX5_IB_RROCE_ECN_RP, 12, enable) \
736 m(+1, u64, rp_13_enable, "rp_13_enable", "Enable reaction point, priority 13", MLX5_IB_RROCE_ECN_RP, 13, enable) \
737 m(+1, u64, rp_14_enable, "rp_14_enable", "Enable reaction point, priority 14", MLX5_IB_RROCE_ECN_RP, 14, enable) \
738 m(+1, u64, rp_15_enable, "rp_15_enable", "Enable reaction point, priority 15", MLX5_IB_RROCE_ECN_RP, 15, enable) \
739 /* ECN NP */ \
740 m(+1, u64, np_0_enable, "np_0_enable", "Enable notification point, priority 0", MLX5_IB_RROCE_ECN_NP, 0, enable) \
741 m(+1, u64, np_1_enable, "np_1_enable", "Enable notification point, priority 1", MLX5_IB_RROCE_ECN_NP, 1, enable) \
742 m(+1, u64, np_2_enable, "np_2_enable", "Enable notification point, priority 2", MLX5_IB_RROCE_ECN_NP, 2, enable) \
743 m(+1, u64, np_3_enable, "np_3_enable", "Enable notification point, priority 3", MLX5_IB_RROCE_ECN_NP, 3, enable) \
744 m(+1, u64, np_4_enable, "np_4_enable", "Enable notification point, priority 4", MLX5_IB_RROCE_ECN_NP, 4, enable) \
745 m(+1, u64, np_5_enable, "np_5_enable", "Enable notification point, priority 5", MLX5_IB_RROCE_ECN_NP, 5, enable) \
746 m(+1, u64, np_6_enable, "np_6_enable", "Enable notification point, priority 6", MLX5_IB_RROCE_ECN_NP, 6, enable) \
747 m(+1, u64, np_7_enable, "np_7_enable", "Enable notification point, priority 7", MLX5_IB_RROCE_ECN_NP, 7, enable) \
748 m(+1, u64, np_8_enable, "np_8_enable", "Enable notification point, priority 8", MLX5_IB_RROCE_ECN_NP, 8, enable) \
749 m(+1, u64, np_9_enable, "np_9_enable", "Enable notification point, priority 9", MLX5_IB_RROCE_ECN_NP, 9, enable) \
750 m(+1, u64, np_10_enable, "np_10_enable", "Enable notification point, priority 10", MLX5_IB_RROCE_ECN_NP, 10, enable) \
751 m(+1, u64, np_11_enable, "np_11_enable", "Enable notification point, priority 11", MLX5_IB_RROCE_ECN_NP, 11, enable) \
752 m(+1, u64, np_12_enable, "np_12_enable", "Enable notification point, priority 12", MLX5_IB_RROCE_ECN_NP, 12, enable) \
753 m(+1, u64, np_13_enable, "np_13_enable", "Enable notification point, priority 13", MLX5_IB_RROCE_ECN_NP, 13, enable) \
754 m(+1, u64, np_14_enable, "np_14_enable", "Enable notification point, priority 14", MLX5_IB_RROCE_ECN_NP, 14, enable) \
755 m(+1, u64, np_15_enable, "np_15_enable", "Enable notification point, priority 15", MLX5_IB_RROCE_ECN_NP, 15, enable) \
756
757 #define MLX5_IB_CONG_STATUS_NUM (0 MLX5_IB_CONG_STATUS(MLX5_IB_STATS_COUNT))
758
759 struct mlx5_ib_congestion {
760 struct sysctl_ctx_list ctx;
761 struct sx lock;
762 struct delayed_work dwork;
763 union {
764 u64 arg[1];
765 struct {
766 MLX5_IB_CONG_PARAMS(MLX5_IB_STATS_VAR)
767 MLX5_IB_CONG_STATS(MLX5_IB_STATS_VAR)
768 MLX5_IB_CONG_STATUS(MLX5_IB_STATS_VAR)
769 };
770 };
771 };
772
773 struct mlx5_devx_event_table {
774 /* serialize updating the event_xa */
775 struct mutex event_xa_lock;
776 struct xarray event_xa;
777 };
778
779 struct mlx5_ib_dev {
780 struct ib_device ib_dev;
781 struct mlx5_core_dev *mdev;
782 struct mlx5_roce roce;
783 MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
784 int num_ports;
785 /* serialize update of capability mask
786 */
787 struct mutex cap_mask_mutex;
788 u8 ib_active:1;
789 u8 wc_support:1;
790 struct umr_common umrc;
791 /* sync used page count stats
792 */
793 struct mlx5_ib_resources devr;
794 struct mlx5_mr_cache cache;
795 struct timer_list delay_timer;
796 /* Prevents soft lock on massive reg MRs */
797 struct mutex slow_path_mutex;
798 int fill_delay;
799 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
800 struct ib_odp_caps odp_caps;
801 /*
802 * Sleepable RCU that prevents destruction of MRs while they are still
803 * being used by a page fault handler.
804 */
805 struct srcu_struct mr_srcu;
806 #endif
807 struct mlx5_ib_flow_db flow_db;
808 /* protect resources needed as part of reset flow */
809 spinlock_t reset_flow_resource_lock;
810 struct list_head qp_list;
811 /* Array with num_ports elements */
812 struct mlx5_ib_port *port;
813 struct mlx5_sq_bfreg bfreg;
814 struct mlx5_sq_bfreg wc_bfreg;
815 struct mlx5_sq_bfreg fp_bfreg;
816 struct mlx5_devx_event_table devx_event_table;
817 struct mlx5_ib_congestion congestion;
818
819 struct mlx5_async_ctx async_ctx;
820
821 /* protect the user_td */
822 struct mutex lb_mutex;
823 u32 user_td;
824 };
825
to_mibcq(struct mlx5_core_cq * mcq)826 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
827 {
828 return container_of(mcq, struct mlx5_ib_cq, mcq);
829 }
830
to_mxrcd(struct ib_xrcd * ibxrcd)831 static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
832 {
833 return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
834 }
835
to_mdev(struct ib_device * ibdev)836 static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
837 {
838 return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
839 }
840
mlx5_udata_to_mdev(struct ib_udata * udata)841 static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)
842 {
843 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
844 udata, struct mlx5_ib_ucontext, ibucontext);
845
846 return to_mdev(context->ibucontext.device);
847 }
848
to_mcq(struct ib_cq * ibcq)849 static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
850 {
851 return container_of(ibcq, struct mlx5_ib_cq, ibcq);
852 }
853
to_mibqp(struct mlx5_core_qp * mqp)854 static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
855 {
856 return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
857 }
858
to_mibrwq(struct mlx5_core_qp * core_qp)859 static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
860 {
861 return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
862 }
863
to_mibmr(struct mlx5_core_mkey * mmkey)864 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
865 {
866 return container_of(mmkey, struct mlx5_ib_mr, mmkey);
867 }
868
to_mpd(struct ib_pd * ibpd)869 static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
870 {
871 return container_of(ibpd, struct mlx5_ib_pd, ibpd);
872 }
873
to_msrq(struct ib_srq * ibsrq)874 static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
875 {
876 return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
877 }
878
to_mqp(struct ib_qp * ibqp)879 static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
880 {
881 return container_of(ibqp, struct mlx5_ib_qp, ibqp);
882 }
883
to_mrwq(struct ib_wq * ibwq)884 static inline struct mlx5_ib_rwq *to_mrwq(struct ib_wq *ibwq)
885 {
886 return container_of(ibwq, struct mlx5_ib_rwq, ibwq);
887 }
888
to_mrwq_ind_table(struct ib_rwq_ind_table * ib_rwq_ind_tbl)889 static inline struct mlx5_ib_rwq_ind_table *to_mrwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
890 {
891 return container_of(ib_rwq_ind_tbl, struct mlx5_ib_rwq_ind_table, ib_rwq_ind_tbl);
892 }
893
to_mibsrq(struct mlx5_core_srq * msrq)894 static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
895 {
896 return container_of(msrq, struct mlx5_ib_srq, msrq);
897 }
898
to_mmr(struct ib_mr * ibmr)899 static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
900 {
901 return container_of(ibmr, struct mlx5_ib_mr, ibmr);
902 }
903
to_mmw(struct ib_mw * ibmw)904 static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
905 {
906 return container_of(ibmw, struct mlx5_ib_mw, ibmw);
907 }
908
909 struct mlx5_ib_ah {
910 struct ib_ah ibah;
911 struct mlx5_av av;
912 };
913
to_mah(struct ib_ah * ibah)914 static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
915 {
916 return container_of(ibah, struct mlx5_ib_ah, ibah);
917 }
918
919 static inline struct mlx5_user_mmap_entry *
to_mmmap(struct rdma_user_mmap_entry * rdma_entry)920 to_mmmap(struct rdma_user_mmap_entry *rdma_entry)
921 {
922 return container_of(rdma_entry,
923 struct mlx5_user_mmap_entry, rdma_entry);
924 }
925
926 int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
927 struct mlx5_db *db);
928 void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
929 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
930 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
931 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
932 int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
933 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
934 const void *in_mad, void *response_mad);
935 int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
936 struct ib_udata *udata);
937 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
938 void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
939 int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
940 struct ib_udata *udata);
941 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
942 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
943 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
944 void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
945 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
946 const struct ib_recv_wr **bad_wr);
947 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
948 struct ib_qp_init_attr *init_attr,
949 struct ib_udata *udata);
950 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
951 int attr_mask, struct ib_udata *udata);
952 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
953 struct ib_qp_init_attr *qp_init_attr);
954 int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
955 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
956 const struct ib_send_wr **bad_wr);
957 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
958 const struct ib_recv_wr **bad_wr);
959 void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
960 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
961 void *buffer, u32 length,
962 struct mlx5_ib_qp_base *base);
963 int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
964 struct ib_udata *udata);
965 void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
966 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
967 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
968 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
969 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
970 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
971 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
972 u64 virt_addr, int access_flags,
973 struct ib_udata *udata);
974 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
975 struct ib_udata *udata);
976 int mlx5_ib_dealloc_mw(struct ib_mw *mw);
977 int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index,
978 int npages, int zap);
979 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
980 u64 length, u64 virt_addr, int access_flags,
981 struct ib_pd *pd, struct ib_udata *udata);
982 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
983 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
984 u32 max_num_sg, struct ib_udata *udata);
985 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
986 unsigned int *sg_offset);
987 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
988 const struct ib_wc *in_wc, const struct ib_grh *in_grh,
989 const struct ib_mad_hdr *in, size_t in_mad_size,
990 struct ib_mad_hdr *out, size_t *out_mad_size,
991 u16 *out_mad_pkey_index);
992 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
993 struct ib_udata *udata);
994 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
995 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
996 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
997 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
998 struct ib_smp *out_mad);
999 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
1000 __be64 *sys_image_guid);
1001 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
1002 u16 *max_pkeys);
1003 int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,
1004 u32 *vendor_id);
1005 int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);
1006 int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);
1007 int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,
1008 u16 *pkey);
1009 int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,
1010 union ib_gid *gid);
1011 int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
1012 struct ib_port_attr *props);
1013 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
1014 struct ib_port_attr *props);
1015 int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
1016 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
1017 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
1018 unsigned long max_page_shift,
1019 int *count, int *shift,
1020 int *ncont, int *order);
1021 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1022 int page_shift, size_t offset, size_t num_pages,
1023 __be64 *pas, int access_flags);
1024 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1025 int page_shift, __be64 *pas, int access_flags);
1026 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
1027 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
1028 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
1029 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
1030 int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
1031 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1032 struct ib_mr_status *mr_status);
1033 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
1034 struct ib_wq_init_attr *init_attr,
1035 struct ib_udata *udata);
1036 void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
1037 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
1038 u32 wq_attr_mask, struct ib_udata *udata);
1039 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
1040 struct ib_rwq_ind_table_init_attr *init_attr,
1041 struct ib_udata *udata);
1042 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
1043
1044 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1045 extern struct workqueue_struct *mlx5_ib_page_fault_wq;
1046
1047 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
1048 void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,
1049 struct mlx5_ib_pfault *pfault);
1050 void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp);
1051 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
1052 void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
1053 int __init mlx5_ib_odp_init(void);
1054 void mlx5_ib_odp_cleanup(void);
1055 void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp);
1056 void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp);
1057 void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
1058 unsigned long end);
1059 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev * dev)1060 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
1061 {
1062 return;
1063 }
1064
mlx5_ib_odp_create_qp(struct mlx5_ib_qp * qp)1065 static inline void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp) {}
mlx5_ib_odp_init_one(struct mlx5_ib_dev * ibdev)1066 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
mlx5_ib_odp_remove_one(struct mlx5_ib_dev * ibdev)1067 static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev) {}
mlx5_ib_odp_init(void)1068 static inline int mlx5_ib_odp_init(void) { return 0; }
mlx5_ib_odp_cleanup(void)1069 static inline void mlx5_ib_odp_cleanup(void) {}
mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp * qp)1070 static inline void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp) {}
mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp * qp)1071 static inline void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp) {}
1072
1073 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1074
1075 int mlx5_ib_get_vf_config(struct ib_device *device, int vf,
1076 u8 port, struct ifla_vf_info *info);
1077 int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
1078 u8 port, int state);
1079 int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,
1080 u8 port, struct ifla_vf_stats *stats);
1081 int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
1082 u64 guid, int type);
1083
1084 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
1085 const struct ib_gid_attr *attr);
1086
1087 /* GSI QP helper functions */
1088 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
1089 struct ib_qp_init_attr *init_attr);
1090 int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
1091 int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1092 int attr_mask);
1093 int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
1094 int qp_attr_mask,
1095 struct ib_qp_init_attr *qp_init_attr);
1096 int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,
1097 const struct ib_send_wr **bad_wr);
1098 int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,
1099 const struct ib_recv_wr **bad_wr);
1100 void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
1101
1102 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
1103
1104 void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
1105 int bfregn);
1106
1107 #if 1 /* IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) */
1108 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
1109 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
1110 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
1111 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
1112 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);
1113 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id);
1114 extern const struct uapi_definition mlx5_ib_devx_defs[];
1115 #else
1116 static inline int
mlx5_ib_devx_create(struct mlx5_ib_dev * dev,bool is_user)1117 mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
1118 bool is_user) { return -EOPNOTSUPP; }
mlx5_ib_devx_destroy(struct mlx5_ib_dev * dev,u16 uid)1119 static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
mlx5_ib_devx_init_event_table(struct mlx5_ib_dev * dev)1120 static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev * dev)1121 static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
mlx5_ib_devx_is_flow_dest(void * obj,int * dest_id,int * dest_type)1122 static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
1123 int *dest_type)
1124 {
1125 return false;
1126 }
1127 #endif
1128
init_query_mad(struct ib_smp * mad)1129 static inline void init_query_mad(struct ib_smp *mad)
1130 {
1131 mad->base_version = 1;
1132 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1133 mad->class_version = 1;
1134 mad->method = IB_MGMT_METHOD_GET;
1135 }
1136
convert_access(int acc)1137 static inline u8 convert_access(int acc)
1138 {
1139 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
1140 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
1141 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
1142 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
1143 MLX5_PERM_LOCAL_READ;
1144 }
1145
is_qp1(enum ib_qp_type qp_type)1146 static inline int is_qp1(enum ib_qp_type qp_type)
1147 {
1148 return qp_type == MLX5_IB_QPT_HW_GSI;
1149 }
1150
1151 #define MLX5_MAX_UMR_SHIFT 16
1152 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1153
check_cq_create_flags(u32 flags)1154 static inline u32 check_cq_create_flags(u32 flags)
1155 {
1156 /*
1157 * It returns non-zero value for unsupported CQ
1158 * create flags, otherwise it returns zero.
1159 */
1160 return (flags & ~(IB_CQ_FLAGS_IGNORE_OVERRUN |
1161 IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
1162 }
1163
verify_assign_uidx(u8 cqe_version,u32 cmd_uidx,u32 * user_index)1164 static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
1165 u32 *user_index)
1166 {
1167 if (cqe_version) {
1168 if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
1169 (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
1170 return -EINVAL;
1171 *user_index = cmd_uidx;
1172 } else {
1173 *user_index = MLX5_IB_DEFAULT_UIDX;
1174 }
1175
1176 return 0;
1177 }
1178
get_qp_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_qp * ucmd,int inlen,u32 * user_index)1179 static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
1180 struct mlx5_ib_create_qp *ucmd,
1181 int inlen,
1182 u32 *user_index)
1183 {
1184 u8 cqe_version = ucontext->cqe_version;
1185
1186 if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
1187 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1188 return 0;
1189
1190 if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
1191 !!cqe_version))
1192 return -EINVAL;
1193
1194 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1195 }
1196
get_srq_user_index(struct mlx5_ib_ucontext * ucontext,struct mlx5_ib_create_srq * ucmd,int inlen,u32 * user_index)1197 static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
1198 struct mlx5_ib_create_srq *ucmd,
1199 int inlen,
1200 u32 *user_index)
1201 {
1202 u8 cqe_version = ucontext->cqe_version;
1203
1204 if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
1205 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1206 return 0;
1207
1208 if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
1209 !!cqe_version))
1210 return -EINVAL;
1211
1212 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1213 }
1214
1215 void mlx5_ib_cleanup_congestion(struct mlx5_ib_dev *);
1216 int mlx5_ib_init_congestion(struct mlx5_ib_dev *);
1217
get_uars_per_sys_page(struct mlx5_ib_dev * dev,bool lib_support)1218 static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
1219 {
1220 return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1221 MLX5_UARS_IN_PAGE : 1;
1222 }
1223
get_num_static_uars(struct mlx5_ib_dev * dev,struct mlx5_bfreg_info * bfregi)1224 static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
1225 struct mlx5_bfreg_info *bfregi)
1226 {
1227 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
1228 }
1229
1230 int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
1231 struct mlx5_bfreg_info *bfregi, u32 bfregn,
1232 bool dyn_bfreg);
1233
1234 #endif /* MLX5_IB_H */
1235