xref: /linux/drivers/infiniband/hw/mlx5/mlx5_ib.h (revision 2c97b5ae83dca56718774e7b4bf9640f05d11867)
1 /*
2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #ifndef MLX5_IB_H
34 #define MLX5_IB_H
35 
36 #include <linux/kernel.h>
37 #include <linux/sched.h>
38 #include <rdma/ib_verbs.h>
39 #include <rdma/ib_umem.h>
40 #include <rdma/ib_smi.h>
41 #include <linux/mlx5/driver.h>
42 #include <linux/mlx5/cq.h>
43 #include <linux/mlx5/fs.h>
44 #include <linux/mlx5/qp.h>
45 #include <linux/types.h>
46 #include <linux/mlx5/transobj.h>
47 #include <rdma/ib_user_verbs.h>
48 #include <rdma/mlx5-abi.h>
49 #include <rdma/uverbs_ioctl.h>
50 #include <rdma/mlx5_user_ioctl_cmds.h>
51 #include <rdma/mlx5_user_ioctl_verbs.h>
52 
53 #include "srq.h"
54 
55 #define mlx5_ib_dbg(_dev, format, arg...)                                      \
56 	dev_dbg(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__,      \
57 		__LINE__, current->pid, ##arg)
58 
59 #define mlx5_ib_err(_dev, format, arg...)                                      \
60 	dev_err(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__,      \
61 		__LINE__, current->pid, ##arg)
62 
63 #define mlx5_ib_warn(_dev, format, arg...)                                     \
64 	dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__,     \
65 		 __LINE__, current->pid, ##arg)
66 
67 #define field_avail(type, fld, sz) (offsetof(type, fld) +		\
68 				    sizeof(((type *)0)->fld) <= (sz))
69 #define MLX5_IB_DEFAULT_UIDX 0xffffff
70 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
71 
72 #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
73 
74 enum {
75 	MLX5_IB_MMAP_CMD_SHIFT	= 8,
76 	MLX5_IB_MMAP_CMD_MASK	= 0xff,
77 };
78 
79 enum {
80 	MLX5_RES_SCAT_DATA32_CQE	= 0x1,
81 	MLX5_RES_SCAT_DATA64_CQE	= 0x2,
82 	MLX5_REQ_SCAT_DATA32_CQE	= 0x11,
83 	MLX5_REQ_SCAT_DATA64_CQE	= 0x22,
84 };
85 
86 enum mlx5_ib_mad_ifc_flags {
87 	MLX5_MAD_IFC_IGNORE_MKEY	= 1,
88 	MLX5_MAD_IFC_IGNORE_BKEY	= 2,
89 	MLX5_MAD_IFC_NET_VIEW		= 4,
90 };
91 
92 enum {
93 	MLX5_CROSS_CHANNEL_BFREG         = 0,
94 };
95 
96 enum {
97 	MLX5_CQE_VERSION_V0,
98 	MLX5_CQE_VERSION_V1,
99 };
100 
101 enum {
102 	MLX5_TM_MAX_RNDV_MSG_SIZE	= 64,
103 	MLX5_TM_MAX_SGE			= 1,
104 };
105 
106 enum {
107 	MLX5_IB_INVALID_UAR_INDEX	= BIT(31),
108 	MLX5_IB_INVALID_BFREG		= BIT(31),
109 };
110 
111 enum {
112 	MLX5_MAX_MEMIC_PAGES = 0x100,
113 	MLX5_MEMIC_ALLOC_SIZE_MASK = 0x3f,
114 };
115 
116 enum {
117 	MLX5_MEMIC_BASE_ALIGN	= 6,
118 	MLX5_MEMIC_BASE_SIZE	= 1 << MLX5_MEMIC_BASE_ALIGN,
119 };
120 
121 #define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)                                        \
122 	(MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity))
123 #define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))
124 
125 struct mlx5_ib_ucontext {
126 	struct ib_ucontext	ibucontext;
127 	struct list_head	db_page_list;
128 
129 	/* protect doorbell record alloc/free
130 	 */
131 	struct mutex		db_page_mutex;
132 	struct mlx5_bfreg_info	bfregi;
133 	u8			cqe_version;
134 	/* Transport Domain number */
135 	u32			tdn;
136 
137 	u64			lib_caps;
138 	DECLARE_BITMAP(dm_pages, MLX5_MAX_MEMIC_PAGES);
139 	u16			devx_uid;
140 	/* For RoCE LAG TX affinity */
141 	atomic_t		tx_port_affinity;
142 };
143 
144 static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
145 {
146 	return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
147 }
148 
149 struct mlx5_ib_pd {
150 	struct ib_pd		ibpd;
151 	u32			pdn;
152 	u16			uid;
153 };
154 
155 enum {
156 	MLX5_IB_FLOW_ACTION_MODIFY_HEADER,
157 	MLX5_IB_FLOW_ACTION_PACKET_REFORMAT,
158 	MLX5_IB_FLOW_ACTION_DECAP,
159 };
160 
161 #define MLX5_IB_FLOW_MCAST_PRIO		(MLX5_BY_PASS_NUM_PRIOS - 1)
162 #define MLX5_IB_FLOW_LAST_PRIO		(MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
163 #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
164 #error "Invalid number of bypass priorities"
165 #endif
166 #define MLX5_IB_FLOW_LEFTOVERS_PRIO	(MLX5_IB_FLOW_MCAST_PRIO + 1)
167 
168 #define MLX5_IB_NUM_FLOW_FT		(MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
169 #define MLX5_IB_NUM_SNIFFER_FTS		2
170 #define MLX5_IB_NUM_EGRESS_FTS		1
171 struct mlx5_ib_flow_prio {
172 	struct mlx5_flow_table		*flow_table;
173 	unsigned int			refcount;
174 };
175 
176 struct mlx5_ib_flow_handler {
177 	struct list_head		list;
178 	struct ib_flow			ibflow;
179 	struct mlx5_ib_flow_prio	*prio;
180 	struct mlx5_flow_handle		*rule;
181 	struct ib_counters		*ibcounters;
182 	struct mlx5_ib_dev		*dev;
183 	struct mlx5_ib_flow_matcher	*flow_matcher;
184 };
185 
186 struct mlx5_ib_flow_matcher {
187 	struct mlx5_ib_match_params matcher_mask;
188 	int			mask_len;
189 	enum mlx5_ib_flow_type	flow_type;
190 	enum mlx5_flow_namespace_type ns_type;
191 	u16			priority;
192 	struct mlx5_core_dev	*mdev;
193 	atomic_t		usecnt;
194 	u8			match_criteria_enable;
195 };
196 
197 struct mlx5_ib_flow_db {
198 	struct mlx5_ib_flow_prio	prios[MLX5_IB_NUM_FLOW_FT];
199 	struct mlx5_ib_flow_prio	egress_prios[MLX5_IB_NUM_FLOW_FT];
200 	struct mlx5_ib_flow_prio	sniffer[MLX5_IB_NUM_SNIFFER_FTS];
201 	struct mlx5_ib_flow_prio	egress[MLX5_IB_NUM_EGRESS_FTS];
202 	struct mlx5_ib_flow_prio	fdb;
203 	struct mlx5_ib_flow_prio	rdma_rx[MLX5_IB_NUM_FLOW_FT];
204 	struct mlx5_flow_table		*lag_demux_ft;
205 	/* Protect flow steering bypass flow tables
206 	 * when add/del flow rules.
207 	 * only single add/removal of flow steering rule could be done
208 	 * simultaneously.
209 	 */
210 	struct mutex			lock;
211 };
212 
213 /* Use macros here so that don't have to duplicate
214  * enum ib_send_flags and enum ib_qp_type for low-level driver
215  */
216 
217 #define MLX5_IB_SEND_UMR_ENABLE_MR	       (IB_SEND_RESERVED_START << 0)
218 #define MLX5_IB_SEND_UMR_DISABLE_MR	       (IB_SEND_RESERVED_START << 1)
219 #define MLX5_IB_SEND_UMR_FAIL_IF_FREE	       (IB_SEND_RESERVED_START << 2)
220 #define MLX5_IB_SEND_UMR_UPDATE_XLT	       (IB_SEND_RESERVED_START << 3)
221 #define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION    (IB_SEND_RESERVED_START << 4)
222 #define MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS       IB_SEND_RESERVED_END
223 
224 #define MLX5_IB_QPT_REG_UMR	IB_QPT_RESERVED1
225 /*
226  * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
227  * creates the actual hardware QP.
228  */
229 #define MLX5_IB_QPT_HW_GSI	IB_QPT_RESERVED2
230 #define MLX5_IB_QPT_DCI		IB_QPT_RESERVED3
231 #define MLX5_IB_QPT_DCT		IB_QPT_RESERVED4
232 #define MLX5_IB_WR_UMR		IB_WR_RESERVED1
233 
234 #define MLX5_IB_UMR_OCTOWORD	       16
235 #define MLX5_IB_UMR_XLT_ALIGNMENT      64
236 
237 #define MLX5_IB_UPD_XLT_ZAP	      BIT(0)
238 #define MLX5_IB_UPD_XLT_ENABLE	      BIT(1)
239 #define MLX5_IB_UPD_XLT_ATOMIC	      BIT(2)
240 #define MLX5_IB_UPD_XLT_ADDR	      BIT(3)
241 #define MLX5_IB_UPD_XLT_PD	      BIT(4)
242 #define MLX5_IB_UPD_XLT_ACCESS	      BIT(5)
243 #define MLX5_IB_UPD_XLT_INDIRECT      BIT(6)
244 
245 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
246  *
247  * These flags are intended for internal use by the mlx5_ib driver, and they
248  * rely on the range reserved for that use in the ib_qp_create_flags enum.
249  */
250 #define MLX5_IB_QP_CREATE_SQPN_QP1	IB_QP_CREATE_RESERVED_START
251 #define MLX5_IB_QP_CREATE_WC_TEST	(IB_QP_CREATE_RESERVED_START << 1)
252 
253 struct wr_list {
254 	u16	opcode;
255 	u16	next;
256 };
257 
258 enum mlx5_ib_rq_flags {
259 	MLX5_IB_RQ_CVLAN_STRIPPING	= 1 << 0,
260 	MLX5_IB_RQ_PCI_WRITE_END_PADDING	= 1 << 1,
261 };
262 
263 struct mlx5_ib_wq {
264 	struct mlx5_frag_buf_ctrl fbc;
265 	u64		       *wrid;
266 	u32		       *wr_data;
267 	struct wr_list	       *w_list;
268 	unsigned	       *wqe_head;
269 	u16		        unsig_count;
270 
271 	/* serialize post to the work queue
272 	 */
273 	spinlock_t		lock;
274 	int			wqe_cnt;
275 	int			max_post;
276 	int			max_gs;
277 	int			offset;
278 	int			wqe_shift;
279 	unsigned		head;
280 	unsigned		tail;
281 	u16			cur_post;
282 	void			*cur_edge;
283 };
284 
285 enum mlx5_ib_wq_flags {
286 	MLX5_IB_WQ_FLAGS_DELAY_DROP = 0x1,
287 	MLX5_IB_WQ_FLAGS_STRIDING_RQ = 0x2,
288 };
289 
290 #define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
291 #define MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES 16
292 #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
293 #define MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES 13
294 #define MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES 3
295 
296 struct mlx5_ib_rwq {
297 	struct ib_wq		ibwq;
298 	struct mlx5_core_qp	core_qp;
299 	u32			rq_num_pas;
300 	u32			log_rq_stride;
301 	u32			log_rq_size;
302 	u32			rq_page_offset;
303 	u32			log_page_size;
304 	u32			log_num_strides;
305 	u32			two_byte_shift_en;
306 	u32			single_stride_log_num_of_bytes;
307 	struct ib_umem		*umem;
308 	size_t			buf_size;
309 	unsigned int		page_shift;
310 	int			create_type;
311 	struct mlx5_db		db;
312 	u32			user_index;
313 	u32			wqe_count;
314 	u32			wqe_shift;
315 	int			wq_sig;
316 	u32			create_flags; /* Use enum mlx5_ib_wq_flags */
317 };
318 
319 enum {
320 	MLX5_QP_USER,
321 	MLX5_QP_KERNEL,
322 	MLX5_QP_EMPTY
323 };
324 
325 enum {
326 	MLX5_WQ_USER,
327 	MLX5_WQ_KERNEL
328 };
329 
330 struct mlx5_ib_rwq_ind_table {
331 	struct ib_rwq_ind_table ib_rwq_ind_tbl;
332 	u32			rqtn;
333 	u16			uid;
334 };
335 
336 struct mlx5_ib_ubuffer {
337 	struct ib_umem	       *umem;
338 	int			buf_size;
339 	u64			buf_addr;
340 };
341 
342 struct mlx5_ib_qp_base {
343 	struct mlx5_ib_qp	*container_mibqp;
344 	struct mlx5_core_qp	mqp;
345 	struct mlx5_ib_ubuffer	ubuffer;
346 };
347 
348 struct mlx5_ib_qp_trans {
349 	struct mlx5_ib_qp_base	base;
350 	u16			xrcdn;
351 	u8			alt_port;
352 	u8			atomic_rd_en;
353 	u8			resp_depth;
354 };
355 
356 struct mlx5_ib_rss_qp {
357 	u32	tirn;
358 };
359 
360 struct mlx5_ib_rq {
361 	struct mlx5_ib_qp_base base;
362 	struct mlx5_ib_wq	*rq;
363 	struct mlx5_ib_ubuffer	ubuffer;
364 	struct mlx5_db		*doorbell;
365 	u32			tirn;
366 	u8			state;
367 	u32			flags;
368 };
369 
370 struct mlx5_ib_sq {
371 	struct mlx5_ib_qp_base base;
372 	struct mlx5_ib_wq	*sq;
373 	struct mlx5_ib_ubuffer  ubuffer;
374 	struct mlx5_db		*doorbell;
375 	struct mlx5_flow_handle	*flow_rule;
376 	u32			tisn;
377 	u8			state;
378 };
379 
380 struct mlx5_ib_raw_packet_qp {
381 	struct mlx5_ib_sq sq;
382 	struct mlx5_ib_rq rq;
383 };
384 
385 struct mlx5_bf {
386 	int			buf_size;
387 	unsigned long		offset;
388 	struct mlx5_sq_bfreg   *bfreg;
389 };
390 
391 struct mlx5_ib_dct {
392 	struct mlx5_core_dct    mdct;
393 	u32                     *in;
394 };
395 
396 struct mlx5_ib_qp {
397 	struct ib_qp		ibqp;
398 	union {
399 		struct mlx5_ib_qp_trans trans_qp;
400 		struct mlx5_ib_raw_packet_qp raw_packet_qp;
401 		struct mlx5_ib_rss_qp rss_qp;
402 		struct mlx5_ib_dct dct;
403 	};
404 	struct mlx5_frag_buf	buf;
405 
406 	struct mlx5_db		db;
407 	struct mlx5_ib_wq	rq;
408 
409 	u8			sq_signal_bits;
410 	u8			next_fence;
411 	struct mlx5_ib_wq	sq;
412 
413 	/* serialize qp state modifications
414 	 */
415 	struct mutex		mutex;
416 	u32			flags;
417 	u8			port;
418 	u8			state;
419 	int			wq_sig;
420 	int			scat_cqe;
421 	int			max_inline_data;
422 	struct mlx5_bf	        bf;
423 	int			has_rq;
424 
425 	/* only for user space QPs. For kernel
426 	 * we have it from the bf object
427 	 */
428 	int			bfregn;
429 
430 	int			create_type;
431 
432 	struct list_head	qps_list;
433 	struct list_head	cq_recv_list;
434 	struct list_head	cq_send_list;
435 	struct mlx5_rate_limit	rl;
436 	u32                     underlay_qpn;
437 	u32			flags_en;
438 	/* storage for qp sub type when core qp type is IB_QPT_DRIVER */
439 	enum ib_qp_type		qp_sub_type;
440 	/* A flag to indicate if there's a new counter is configured
441 	 * but not take effective
442 	 */
443 	u32                     counter_pending;
444 };
445 
446 struct mlx5_ib_cq_buf {
447 	struct mlx5_frag_buf_ctrl fbc;
448 	struct mlx5_frag_buf    frag_buf;
449 	struct ib_umem		*umem;
450 	int			cqe_size;
451 	int			nent;
452 };
453 
454 enum mlx5_ib_qp_flags {
455 	MLX5_IB_QP_LSO                          = IB_QP_CREATE_IPOIB_UD_LSO,
456 	MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK     = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
457 	MLX5_IB_QP_CROSS_CHANNEL            = IB_QP_CREATE_CROSS_CHANNEL,
458 	MLX5_IB_QP_MANAGED_SEND             = IB_QP_CREATE_MANAGED_SEND,
459 	MLX5_IB_QP_MANAGED_RECV             = IB_QP_CREATE_MANAGED_RECV,
460 	MLX5_IB_QP_SIGNATURE_HANDLING           = 1 << 5,
461 	/* QP uses 1 as its source QP number */
462 	MLX5_IB_QP_SQPN_QP1			= 1 << 6,
463 	MLX5_IB_QP_CAP_SCATTER_FCS		= 1 << 7,
464 	MLX5_IB_QP_RSS				= 1 << 8,
465 	MLX5_IB_QP_CVLAN_STRIPPING		= 1 << 9,
466 	MLX5_IB_QP_UNDERLAY			= 1 << 10,
467 	MLX5_IB_QP_PCI_WRITE_END_PADDING	= 1 << 11,
468 	MLX5_IB_QP_TUNNEL_OFFLOAD		= 1 << 12,
469 	MLX5_IB_QP_PACKET_BASED_CREDIT		= 1 << 13,
470 };
471 
472 struct mlx5_umr_wr {
473 	struct ib_send_wr		wr;
474 	u64				virt_addr;
475 	u64				offset;
476 	struct ib_pd		       *pd;
477 	unsigned int			page_shift;
478 	unsigned int			xlt_size;
479 	u64				length;
480 	int				access_flags;
481 	u32				mkey;
482 	u8				ignore_free_state:1;
483 };
484 
485 static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr)
486 {
487 	return container_of(wr, struct mlx5_umr_wr, wr);
488 }
489 
490 struct mlx5_shared_mr_info {
491 	int mr_id;
492 	struct ib_umem		*umem;
493 };
494 
495 enum mlx5_ib_cq_pr_flags {
496 	MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD	= 1 << 0,
497 };
498 
499 struct mlx5_ib_cq {
500 	struct ib_cq		ibcq;
501 	struct mlx5_core_cq	mcq;
502 	struct mlx5_ib_cq_buf	buf;
503 	struct mlx5_db		db;
504 
505 	/* serialize access to the CQ
506 	 */
507 	spinlock_t		lock;
508 
509 	/* protect resize cq
510 	 */
511 	struct mutex		resize_mutex;
512 	struct mlx5_ib_cq_buf  *resize_buf;
513 	struct ib_umem	       *resize_umem;
514 	int			cqe_size;
515 	struct list_head	list_send_qp;
516 	struct list_head	list_recv_qp;
517 	u32			create_flags;
518 	struct list_head	wc_list;
519 	enum ib_cq_notify_flags notify_flags;
520 	struct work_struct	notify_work;
521 	u16			private_flags; /* Use mlx5_ib_cq_pr_flags */
522 };
523 
524 struct mlx5_ib_wc {
525 	struct ib_wc wc;
526 	struct list_head list;
527 };
528 
529 struct mlx5_ib_srq {
530 	struct ib_srq		ibsrq;
531 	struct mlx5_core_srq	msrq;
532 	struct mlx5_frag_buf	buf;
533 	struct mlx5_db		db;
534 	struct mlx5_frag_buf_ctrl fbc;
535 	u64		       *wrid;
536 	/* protect SRQ hanlding
537 	 */
538 	spinlock_t		lock;
539 	int			head;
540 	int			tail;
541 	u16			wqe_ctr;
542 	struct ib_umem	       *umem;
543 	/* serialize arming a SRQ
544 	 */
545 	struct mutex		mutex;
546 	int			wq_sig;
547 };
548 
549 struct mlx5_ib_xrcd {
550 	struct ib_xrcd		ibxrcd;
551 	u32			xrcdn;
552 };
553 
554 enum mlx5_ib_mtt_access_flags {
555 	MLX5_IB_MTT_READ  = (1 << 0),
556 	MLX5_IB_MTT_WRITE = (1 << 1),
557 };
558 
559 struct mlx5_ib_dm {
560 	struct ib_dm		ibdm;
561 	phys_addr_t		dev_addr;
562 	u32			type;
563 	size_t			size;
564 	union {
565 		struct {
566 			u32	obj_id;
567 		} icm_dm;
568 		/* other dm types specific params should be added here */
569 	};
570 };
571 
572 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
573 
574 #define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE   |\
575 					 IB_ACCESS_REMOTE_WRITE  |\
576 					 IB_ACCESS_REMOTE_READ   |\
577 					 IB_ACCESS_REMOTE_ATOMIC |\
578 					 IB_ZERO_BASED)
579 
580 #define MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE   |\
581 					  IB_ACCESS_REMOTE_WRITE  |\
582 					  IB_ACCESS_REMOTE_READ   |\
583 					  IB_ZERO_BASED)
584 
585 #define mlx5_update_odp_stats(mr, counter_name, value)		\
586 	atomic64_add(value, &((mr)->odp_stats.counter_name))
587 
588 struct mlx5_ib_mr {
589 	struct ib_mr		ibmr;
590 	void			*descs;
591 	dma_addr_t		desc_map;
592 	int			ndescs;
593 	int			data_length;
594 	int			meta_ndescs;
595 	int			meta_length;
596 	int			max_descs;
597 	int			desc_size;
598 	int			access_mode;
599 	struct mlx5_core_mkey	mmkey;
600 	struct ib_umem	       *umem;
601 	struct mlx5_shared_mr_info	*smr_info;
602 	struct list_head	list;
603 	int			order;
604 	bool			allocated_from_cache;
605 	int			npages;
606 	struct mlx5_ib_dev     *dev;
607 	u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
608 	struct mlx5_core_sig_ctx    *sig;
609 	void			*descs_alloc;
610 	int			access_flags; /* Needed for rereg MR */
611 
612 	struct mlx5_ib_mr      *parent;
613 	/* Needed for IB_MR_TYPE_INTEGRITY */
614 	struct mlx5_ib_mr      *pi_mr;
615 	struct mlx5_ib_mr      *klm_mr;
616 	struct mlx5_ib_mr      *mtt_mr;
617 	u64			data_iova;
618 	u64			pi_iova;
619 
620 	/* For ODP and implicit */
621 	atomic_t		num_deferred_work;
622 	struct xarray		implicit_children;
623 	union {
624 		struct rcu_head rcu;
625 		struct list_head elm;
626 		struct work_struct work;
627 	} odp_destroy;
628 	struct ib_odp_counters	odp_stats;
629 	bool			is_odp_implicit;
630 
631 	struct mlx5_async_work  cb_work;
632 };
633 
634 static inline bool is_odp_mr(struct mlx5_ib_mr *mr)
635 {
636 	return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem &&
637 	       mr->umem->is_odp;
638 }
639 
640 struct mlx5_ib_mw {
641 	struct ib_mw		ibmw;
642 	struct mlx5_core_mkey	mmkey;
643 	int			ndescs;
644 };
645 
646 struct mlx5_ib_devx_mr {
647 	struct mlx5_core_mkey	mmkey;
648 	int			ndescs;
649 };
650 
651 struct mlx5_ib_umr_context {
652 	struct ib_cqe		cqe;
653 	enum ib_wc_status	status;
654 	struct completion	done;
655 };
656 
657 struct umr_common {
658 	struct ib_pd	*pd;
659 	struct ib_cq	*cq;
660 	struct ib_qp	*qp;
661 	/* control access to UMR QP
662 	 */
663 	struct semaphore	sem;
664 };
665 
666 enum {
667 	MLX5_FMR_INVALID,
668 	MLX5_FMR_VALID,
669 	MLX5_FMR_BUSY,
670 };
671 
672 struct mlx5_cache_ent {
673 	struct list_head	head;
674 	/* sync access to the cahce entry
675 	 */
676 	spinlock_t		lock;
677 
678 
679 	char                    name[4];
680 	u32                     order;
681 	u32			xlt;
682 	u32			access_mode;
683 	u32			page;
684 
685 	u32			size;
686 	u32                     cur;
687 	u32                     miss;
688 	u32			limit;
689 
690 	struct mlx5_ib_dev     *dev;
691 	struct work_struct	work;
692 	struct delayed_work	dwork;
693 	int			pending;
694 	struct completion	compl;
695 };
696 
697 struct mlx5_mr_cache {
698 	struct workqueue_struct *wq;
699 	struct mlx5_cache_ent	ent[MAX_MR_CACHE_ENTRIES];
700 	int			stopped;
701 	struct dentry		*root;
702 	unsigned long		last_add;
703 };
704 
705 struct mlx5_ib_gsi_qp;
706 
707 struct mlx5_ib_port_resources {
708 	struct mlx5_ib_resources *devr;
709 	struct mlx5_ib_gsi_qp *gsi;
710 	struct work_struct pkey_change_work;
711 };
712 
713 struct mlx5_ib_resources {
714 	struct ib_cq	*c0;
715 	struct ib_xrcd	*x0;
716 	struct ib_xrcd	*x1;
717 	struct ib_pd	*p0;
718 	struct ib_srq	*s0;
719 	struct ib_srq	*s1;
720 	struct mlx5_ib_port_resources ports[2];
721 	/* Protects changes to the port resources */
722 	struct mutex	mutex;
723 };
724 
725 struct mlx5_ib_counters {
726 	const char **names;
727 	size_t *offsets;
728 	u32 num_q_counters;
729 	u32 num_cong_counters;
730 	u32 num_ext_ppcnt_counters;
731 	u16 set_id;
732 	bool set_id_valid;
733 };
734 
735 struct mlx5_ib_multiport_info;
736 
737 struct mlx5_ib_multiport {
738 	struct mlx5_ib_multiport_info *mpi;
739 	/* To be held when accessing the multiport info */
740 	spinlock_t mpi_lock;
741 };
742 
743 struct mlx5_roce {
744 	/* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
745 	 * netdev pointer
746 	 */
747 	rwlock_t		netdev_lock;
748 	struct net_device	*netdev;
749 	struct notifier_block	nb;
750 	atomic_t		tx_port_affinity;
751 	enum ib_port_state last_port_state;
752 	struct mlx5_ib_dev	*dev;
753 	u8			native_port_num;
754 };
755 
756 struct mlx5_ib_port {
757 	struct mlx5_ib_counters cnts;
758 	struct mlx5_ib_multiport mp;
759 	struct mlx5_ib_dbg_cc_params *dbg_cc_params;
760 	struct mlx5_roce roce;
761 	struct mlx5_eswitch_rep		*rep;
762 };
763 
764 struct mlx5_ib_dbg_param {
765 	int			offset;
766 	struct mlx5_ib_dev	*dev;
767 	struct dentry		*dentry;
768 	u8			port_num;
769 };
770 
771 enum mlx5_ib_dbg_cc_types {
772 	MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE,
773 	MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI,
774 	MLX5_IB_DBG_CC_RP_TIME_RESET,
775 	MLX5_IB_DBG_CC_RP_BYTE_RESET,
776 	MLX5_IB_DBG_CC_RP_THRESHOLD,
777 	MLX5_IB_DBG_CC_RP_AI_RATE,
778 	MLX5_IB_DBG_CC_RP_HAI_RATE,
779 	MLX5_IB_DBG_CC_RP_MIN_DEC_FAC,
780 	MLX5_IB_DBG_CC_RP_MIN_RATE,
781 	MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP,
782 	MLX5_IB_DBG_CC_RP_DCE_TCP_G,
783 	MLX5_IB_DBG_CC_RP_DCE_TCP_RTT,
784 	MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD,
785 	MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE,
786 	MLX5_IB_DBG_CC_RP_GD,
787 	MLX5_IB_DBG_CC_NP_CNP_DSCP,
788 	MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
789 	MLX5_IB_DBG_CC_NP_CNP_PRIO,
790 	MLX5_IB_DBG_CC_MAX,
791 };
792 
793 struct mlx5_ib_dbg_cc_params {
794 	struct dentry			*root;
795 	struct mlx5_ib_dbg_param	params[MLX5_IB_DBG_CC_MAX];
796 };
797 
798 enum {
799 	MLX5_MAX_DELAY_DROP_TIMEOUT_MS = 100,
800 };
801 
802 struct mlx5_ib_delay_drop {
803 	struct mlx5_ib_dev     *dev;
804 	struct work_struct	delay_drop_work;
805 	/* serialize setting of delay drop */
806 	struct mutex		lock;
807 	u32			timeout;
808 	bool			activate;
809 	atomic_t		events_cnt;
810 	atomic_t		rqs_cnt;
811 	struct dentry		*dir_debugfs;
812 };
813 
814 enum mlx5_ib_stages {
815 	MLX5_IB_STAGE_INIT,
816 	MLX5_IB_STAGE_FLOW_DB,
817 	MLX5_IB_STAGE_CAPS,
818 	MLX5_IB_STAGE_NON_DEFAULT_CB,
819 	MLX5_IB_STAGE_ROCE,
820 	MLX5_IB_STAGE_SRQ,
821 	MLX5_IB_STAGE_DEVICE_RESOURCES,
822 	MLX5_IB_STAGE_DEVICE_NOTIFIER,
823 	MLX5_IB_STAGE_ODP,
824 	MLX5_IB_STAGE_COUNTERS,
825 	MLX5_IB_STAGE_CONG_DEBUGFS,
826 	MLX5_IB_STAGE_UAR,
827 	MLX5_IB_STAGE_BFREG,
828 	MLX5_IB_STAGE_PRE_IB_REG_UMR,
829 	MLX5_IB_STAGE_WHITELIST_UID,
830 	MLX5_IB_STAGE_IB_REG,
831 	MLX5_IB_STAGE_POST_IB_REG_UMR,
832 	MLX5_IB_STAGE_DELAY_DROP,
833 	MLX5_IB_STAGE_CLASS_ATTR,
834 	MLX5_IB_STAGE_MAX,
835 };
836 
837 struct mlx5_ib_stage {
838 	int (*init)(struct mlx5_ib_dev *dev);
839 	void (*cleanup)(struct mlx5_ib_dev *dev);
840 };
841 
842 #define STAGE_CREATE(_stage, _init, _cleanup) \
843 	.stage[_stage] = {.init = _init, .cleanup = _cleanup}
844 
845 struct mlx5_ib_profile {
846 	struct mlx5_ib_stage stage[MLX5_IB_STAGE_MAX];
847 };
848 
849 struct mlx5_ib_multiport_info {
850 	struct list_head list;
851 	struct mlx5_ib_dev *ibdev;
852 	struct mlx5_core_dev *mdev;
853 	struct notifier_block mdev_events;
854 	struct completion unref_comp;
855 	u64 sys_image_guid;
856 	u32 mdev_refcnt;
857 	bool is_master;
858 	bool unaffiliate;
859 };
860 
861 struct mlx5_ib_flow_action {
862 	struct ib_flow_action		ib_action;
863 	union {
864 		struct {
865 			u64			    ib_flags;
866 			struct mlx5_accel_esp_xfrm *ctx;
867 		} esp_aes_gcm;
868 		struct {
869 			struct mlx5_ib_dev *dev;
870 			u32 sub_type;
871 			union {
872 				struct mlx5_modify_hdr *modify_hdr;
873 				struct mlx5_pkt_reformat *pkt_reformat;
874 			};
875 		} flow_action_raw;
876 	};
877 };
878 
879 struct mlx5_dm {
880 	struct mlx5_core_dev *dev;
881 	/* This lock is used to protect the access to the shared
882 	 * allocation map when concurrent requests by different
883 	 * processes are handled.
884 	 */
885 	spinlock_t lock;
886 	DECLARE_BITMAP(memic_alloc_pages, MLX5_MAX_MEMIC_PAGES);
887 };
888 
889 struct mlx5_read_counters_attr {
890 	struct mlx5_fc *hw_cntrs_hndl;
891 	u64 *out;
892 	u32 flags;
893 };
894 
895 enum mlx5_ib_counters_type {
896 	MLX5_IB_COUNTERS_FLOW,
897 };
898 
899 struct mlx5_ib_mcounters {
900 	struct ib_counters ibcntrs;
901 	enum mlx5_ib_counters_type type;
902 	/* number of counters supported for this counters type */
903 	u32 counters_num;
904 	struct mlx5_fc *hw_cntrs_hndl;
905 	/* read function for this counters type */
906 	int (*read_counters)(struct ib_device *ibdev,
907 			     struct mlx5_read_counters_attr *read_attr);
908 	/* max index set as part of create_flow */
909 	u32 cntrs_max_index;
910 	/* number of counters data entries (<description,index> pair) */
911 	u32 ncounters;
912 	/* counters data array for descriptions and indexes */
913 	struct mlx5_ib_flow_counters_desc *counters_data;
914 	/* protects access to mcounters internal data */
915 	struct mutex mcntrs_mutex;
916 };
917 
918 static inline struct mlx5_ib_mcounters *
919 to_mcounters(struct ib_counters *ibcntrs)
920 {
921 	return container_of(ibcntrs, struct mlx5_ib_mcounters, ibcntrs);
922 }
923 
924 int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
925 			   bool is_egress,
926 			   struct mlx5_flow_act *action);
927 struct mlx5_ib_lb_state {
928 	/* protect the user_td */
929 	struct mutex		mutex;
930 	u32			user_td;
931 	int			qps;
932 	bool			enabled;
933 };
934 
935 struct mlx5_ib_pf_eq {
936 	struct notifier_block irq_nb;
937 	struct mlx5_ib_dev *dev;
938 	struct mlx5_eq *core;
939 	struct work_struct work;
940 	spinlock_t lock; /* Pagefaults spinlock */
941 	struct workqueue_struct *wq;
942 	mempool_t *pool;
943 };
944 
945 struct mlx5_devx_event_table {
946 	struct mlx5_nb devx_nb;
947 	/* serialize updating the event_xa */
948 	struct mutex event_xa_lock;
949 	struct xarray event_xa;
950 };
951 
952 struct mlx5_ib_dev {
953 	struct ib_device		ib_dev;
954 	struct mlx5_core_dev		*mdev;
955 	struct notifier_block		mdev_events;
956 	int				num_ports;
957 	/* serialize update of capability mask
958 	 */
959 	struct mutex			cap_mask_mutex;
960 	u8				ib_active:1;
961 	u8				fill_delay:1;
962 	u8				is_rep:1;
963 	u8				lag_active:1;
964 	u8				wc_support:1;
965 	struct umr_common		umrc;
966 	/* sync used page count stats
967 	 */
968 	struct mlx5_ib_resources	devr;
969 	struct mlx5_mr_cache		cache;
970 	struct timer_list		delay_timer;
971 	/* Prevents soft lock on massive reg MRs */
972 	struct mutex			slow_path_mutex;
973 	struct ib_odp_caps	odp_caps;
974 	u64			odp_max_size;
975 	struct mlx5_ib_pf_eq	odp_pf_eq;
976 
977 	/*
978 	 * Sleepable RCU that prevents destruction of MRs while they are still
979 	 * being used by a page fault handler.
980 	 */
981 	struct srcu_struct      odp_srcu;
982 	struct xarray		odp_mkeys;
983 
984 	u32			null_mkey;
985 	struct mlx5_ib_flow_db	*flow_db;
986 	/* protect resources needed as part of reset flow */
987 	spinlock_t		reset_flow_resource_lock;
988 	struct list_head	qp_list;
989 	/* Array with num_ports elements */
990 	struct mlx5_ib_port	*port;
991 	struct mlx5_sq_bfreg	bfreg;
992 	struct mlx5_sq_bfreg	wc_bfreg;
993 	struct mlx5_sq_bfreg	fp_bfreg;
994 	struct mlx5_ib_delay_drop	delay_drop;
995 	const struct mlx5_ib_profile	*profile;
996 
997 	struct mlx5_ib_lb_state		lb;
998 	u8			umr_fence;
999 	struct list_head	ib_dev_list;
1000 	u64			sys_image_guid;
1001 	struct mlx5_dm		dm;
1002 	u16			devx_whitelist_uid;
1003 	struct mlx5_srq_table   srq_table;
1004 	struct mlx5_async_ctx   async_ctx;
1005 	struct mlx5_devx_event_table devx_event_table;
1006 
1007 	struct xarray sig_mrs;
1008 };
1009 
1010 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
1011 {
1012 	return container_of(mcq, struct mlx5_ib_cq, mcq);
1013 }
1014 
1015 static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
1016 {
1017 	return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
1018 }
1019 
1020 static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
1021 {
1022 	return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
1023 }
1024 
1025 static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)
1026 {
1027 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1028 		udata, struct mlx5_ib_ucontext, ibucontext);
1029 
1030 	return to_mdev(context->ibucontext.device);
1031 }
1032 
1033 static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
1034 {
1035 	return container_of(ibcq, struct mlx5_ib_cq, ibcq);
1036 }
1037 
1038 static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
1039 {
1040 	return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
1041 }
1042 
1043 static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
1044 {
1045 	return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
1046 }
1047 
1048 static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
1049 {
1050 	return container_of(mmkey, struct mlx5_ib_mr, mmkey);
1051 }
1052 
1053 static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
1054 {
1055 	return container_of(ibpd, struct mlx5_ib_pd, ibpd);
1056 }
1057 
1058 static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
1059 {
1060 	return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
1061 }
1062 
1063 static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
1064 {
1065 	return container_of(ibqp, struct mlx5_ib_qp, ibqp);
1066 }
1067 
1068 static inline struct mlx5_ib_rwq *to_mrwq(struct ib_wq *ibwq)
1069 {
1070 	return container_of(ibwq, struct mlx5_ib_rwq, ibwq);
1071 }
1072 
1073 static inline struct mlx5_ib_rwq_ind_table *to_mrwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
1074 {
1075 	return container_of(ib_rwq_ind_tbl, struct mlx5_ib_rwq_ind_table, ib_rwq_ind_tbl);
1076 }
1077 
1078 static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
1079 {
1080 	return container_of(msrq, struct mlx5_ib_srq, msrq);
1081 }
1082 
1083 static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm)
1084 {
1085 	return container_of(ibdm, struct mlx5_ib_dm, ibdm);
1086 }
1087 
1088 static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
1089 {
1090 	return container_of(ibmr, struct mlx5_ib_mr, ibmr);
1091 }
1092 
1093 static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
1094 {
1095 	return container_of(ibmw, struct mlx5_ib_mw, ibmw);
1096 }
1097 
1098 static inline struct mlx5_ib_flow_action *
1099 to_mflow_act(struct ib_flow_action *ibact)
1100 {
1101 	return container_of(ibact, struct mlx5_ib_flow_action, ib_action);
1102 }
1103 
1104 int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context,
1105 			struct ib_udata *udata, unsigned long virt,
1106 			struct mlx5_db *db);
1107 void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
1108 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
1109 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
1110 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
1111 int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
1112 		      struct ib_udata *udata);
1113 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
1114 void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
1115 int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
1116 		       struct ib_udata *udata);
1117 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1118 		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
1119 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
1120 void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
1121 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
1122 			  const struct ib_recv_wr **bad_wr);
1123 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp);
1124 void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp);
1125 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
1126 				struct ib_qp_init_attr *init_attr,
1127 				struct ib_udata *udata);
1128 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1129 		      int attr_mask, struct ib_udata *udata);
1130 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
1131 		     struct ib_qp_init_attr *qp_init_attr);
1132 int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
1133 void mlx5_ib_drain_sq(struct ib_qp *qp);
1134 void mlx5_ib_drain_rq(struct ib_qp *qp);
1135 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1136 		      const struct ib_send_wr **bad_wr);
1137 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1138 		      const struct ib_recv_wr **bad_wr);
1139 int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
1140 			     int buflen, size_t *bc);
1141 int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
1142 			     int buflen, size_t *bc);
1143 int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
1144 			      void *buffer, int buflen, size_t *bc);
1145 int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1146 		      struct ib_udata *udata);
1147 void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
1148 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
1149 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
1150 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
1151 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
1152 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
1153 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1154 				  u64 virt_addr, int access_flags,
1155 				  struct ib_udata *udata);
1156 int mlx5_ib_advise_mr(struct ib_pd *pd,
1157 		      enum ib_uverbs_advise_mr_advice advice,
1158 		      u32 flags,
1159 		      struct ib_sge *sg_list,
1160 		      u32 num_sge,
1161 		      struct uverbs_attr_bundle *attrs);
1162 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
1163 			       struct ib_udata *udata);
1164 int mlx5_ib_dealloc_mw(struct ib_mw *mw);
1165 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
1166 		       int page_shift, int flags);
1167 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
1168 					     struct ib_udata *udata,
1169 					     int access_flags);
1170 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);
1171 void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr);
1172 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1173 			  u64 length, u64 virt_addr, int access_flags,
1174 			  struct ib_pd *pd, struct ib_udata *udata);
1175 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
1176 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1177 			       u32 max_num_sg, struct ib_udata *udata);
1178 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
1179 					 u32 max_num_sg,
1180 					 u32 max_num_meta_sg);
1181 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1182 		      unsigned int *sg_offset);
1183 int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
1184 			 int data_sg_nents, unsigned int *data_sg_offset,
1185 			 struct scatterlist *meta_sg, int meta_sg_nents,
1186 			 unsigned int *meta_sg_offset);
1187 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
1188 			const struct ib_wc *in_wc, const struct ib_grh *in_grh,
1189 			const struct ib_mad *in, struct ib_mad *out,
1190 			size_t *out_mad_size, u16 *out_mad_pkey_index);
1191 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
1192 				   struct ib_udata *udata);
1193 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
1194 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
1195 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
1196 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
1197 					  struct ib_smp *out_mad);
1198 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
1199 					 __be64 *sys_image_guid);
1200 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
1201 				 u16 *max_pkeys);
1202 int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,
1203 				 u32 *vendor_id);
1204 int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);
1205 int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);
1206 int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,
1207 			    u16 *pkey);
1208 int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,
1209 			    union ib_gid *gid);
1210 int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
1211 			    struct ib_port_attr *props);
1212 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
1213 		       struct ib_port_attr *props);
1214 int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
1215 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
1216 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
1217 			unsigned long max_page_shift,
1218 			int *count, int *shift,
1219 			int *ncont, int *order);
1220 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1221 			    int page_shift, size_t offset, size_t num_pages,
1222 			    __be64 *pas, int access_flags);
1223 void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1224 			  int page_shift, __be64 *pas, int access_flags);
1225 void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
1226 int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);
1227 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
1228 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
1229 
1230 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
1231 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
1232 int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr);
1233 
1234 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1235 			    struct ib_mr_status *mr_status);
1236 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
1237 				struct ib_wq_init_attr *init_attr,
1238 				struct ib_udata *udata);
1239 void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
1240 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
1241 		      u32 wq_attr_mask, struct ib_udata *udata);
1242 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
1243 						      struct ib_rwq_ind_table_init_attr *init_attr,
1244 						      struct ib_udata *udata);
1245 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
1246 struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
1247 			       struct ib_ucontext *context,
1248 			       struct ib_dm_alloc_attr *attr,
1249 			       struct uverbs_attr_bundle *attrs);
1250 int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs);
1251 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1252 				struct ib_dm_mr_attr *attr,
1253 				struct uverbs_attr_bundle *attrs);
1254 
1255 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1256 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
1257 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
1258 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev);
1259 int __init mlx5_ib_odp_init(void);
1260 void mlx5_ib_odp_cleanup(void);
1261 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent);
1262 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
1263 			   size_t nentries, struct mlx5_ib_mr *mr, int flags);
1264 
1265 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
1266 			       enum ib_uverbs_advise_mr_advice advice,
1267 			       u32 flags, struct ib_sge *sg_list, u32 num_sge);
1268 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1269 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
1270 {
1271 	return;
1272 }
1273 
1274 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
1275 static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *ibdev) {}
1276 static inline int mlx5_ib_odp_init(void) { return 0; }
1277 static inline void mlx5_ib_odp_cleanup(void)				    {}
1278 static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) {}
1279 static inline void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
1280 					 size_t nentries, struct mlx5_ib_mr *mr,
1281 					 int flags) {}
1282 
1283 static inline int
1284 mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
1285 			   enum ib_uverbs_advise_mr_advice advice, u32 flags,
1286 			   struct ib_sge *sg_list, u32 num_sge)
1287 {
1288 	return -EOPNOTSUPP;
1289 }
1290 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1291 
1292 extern const struct mmu_interval_notifier_ops mlx5_mn_ops;
1293 
1294 /* Needed for rep profile */
1295 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
1296 		      const struct mlx5_ib_profile *profile,
1297 		      int stage);
1298 void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
1299 		    const struct mlx5_ib_profile *profile);
1300 
1301 int mlx5_ib_get_vf_config(struct ib_device *device, int vf,
1302 			  u8 port, struct ifla_vf_info *info);
1303 int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
1304 			      u8 port, int state);
1305 int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,
1306 			 u8 port, struct ifla_vf_stats *stats);
1307 int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u8 port,
1308 			struct ifla_vf_guid *node_guid,
1309 			struct ifla_vf_guid *port_guid);
1310 int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
1311 			u64 guid, int type);
1312 
1313 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
1314 			       const struct ib_gid_attr *attr);
1315 
1316 void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
1317 void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
1318 
1319 /* GSI QP helper functions */
1320 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
1321 				    struct ib_qp_init_attr *init_attr);
1322 int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
1323 int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1324 			  int attr_mask);
1325 int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
1326 			 int qp_attr_mask,
1327 			 struct ib_qp_init_attr *qp_init_attr);
1328 int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,
1329 			  const struct ib_send_wr **bad_wr);
1330 int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,
1331 			  const struct ib_recv_wr **bad_wr);
1332 void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
1333 
1334 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
1335 
1336 void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
1337 			int bfregn);
1338 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi);
1339 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev,
1340 						   u8 ib_port_num,
1341 						   u8 *native_port_num);
1342 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
1343 				  u8 port_num);
1344 int mlx5_ib_fill_res_entry(struct sk_buff *msg,
1345 			   struct rdma_restrack_entry *res);
1346 int mlx5_ib_fill_stat_entry(struct sk_buff *msg,
1347 			    struct rdma_restrack_entry *res);
1348 
1349 #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
1350 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
1351 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
1352 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
1353 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
1354 const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void);
1355 extern const struct uapi_definition mlx5_ib_devx_defs[];
1356 extern const struct uapi_definition mlx5_ib_flow_defs[];
1357 struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add(
1358 	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
1359 	struct mlx5_flow_context *flow_context,
1360 	struct mlx5_flow_act *flow_act, u32 counter_id,
1361 	void *cmd_in, int inlen, int dest_id, int dest_type);
1362 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);
1363 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id);
1364 int mlx5_ib_get_flow_trees(const struct uverbs_object_tree_def **root);
1365 void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction);
1366 #else
1367 static inline int
1368 mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
1369 			   bool is_user) { return -EOPNOTSUPP; }
1370 static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
1371 static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
1372 static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
1373 static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
1374 					     int *dest_type)
1375 {
1376 	return false;
1377 }
1378 static inline void
1379 mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
1380 {
1381 	return;
1382 };
1383 #endif
1384 static inline void init_query_mad(struct ib_smp *mad)
1385 {
1386 	mad->base_version  = 1;
1387 	mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1388 	mad->class_version = 1;
1389 	mad->method	   = IB_MGMT_METHOD_GET;
1390 }
1391 
1392 static inline u8 convert_access(int acc)
1393 {
1394 	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
1395 	       (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
1396 	       (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
1397 	       (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
1398 	       MLX5_PERM_LOCAL_READ;
1399 }
1400 
1401 static inline int is_qp1(enum ib_qp_type qp_type)
1402 {
1403 	return qp_type == MLX5_IB_QPT_HW_GSI;
1404 }
1405 
1406 #define MLX5_MAX_UMR_SHIFT 16
1407 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1408 
1409 static inline u32 check_cq_create_flags(u32 flags)
1410 {
1411 	/*
1412 	 * It returns non-zero value for unsupported CQ
1413 	 * create flags, otherwise it returns zero.
1414 	 */
1415 	return (flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN |
1416 			  IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION));
1417 }
1418 
1419 static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
1420 				     u32 *user_index)
1421 {
1422 	if (cqe_version) {
1423 		if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
1424 		    (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
1425 			return -EINVAL;
1426 		*user_index = cmd_uidx;
1427 	} else {
1428 		*user_index = MLX5_IB_DEFAULT_UIDX;
1429 	}
1430 
1431 	return 0;
1432 }
1433 
1434 static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
1435 				    struct mlx5_ib_create_qp *ucmd,
1436 				    int inlen,
1437 				    u32 *user_index)
1438 {
1439 	u8 cqe_version = ucontext->cqe_version;
1440 
1441 	if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
1442 	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1443 		return 0;
1444 
1445 	if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
1446 	       !!cqe_version))
1447 		return -EINVAL;
1448 
1449 	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1450 }
1451 
1452 static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
1453 				     struct mlx5_ib_create_srq *ucmd,
1454 				     int inlen,
1455 				     u32 *user_index)
1456 {
1457 	u8 cqe_version = ucontext->cqe_version;
1458 
1459 	if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
1460 	    !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1461 		return 0;
1462 
1463 	if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
1464 	       !!cqe_version))
1465 		return -EINVAL;
1466 
1467 	return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1468 }
1469 
1470 static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
1471 {
1472 	return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1473 				MLX5_UARS_IN_PAGE : 1;
1474 }
1475 
1476 static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
1477 				      struct mlx5_bfreg_info *bfregi)
1478 {
1479 	return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
1480 }
1481 
1482 unsigned long mlx5_ib_get_xlt_emergency_page(void);
1483 void mlx5_ib_put_xlt_emergency_page(void);
1484 
1485 int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
1486 			struct mlx5_bfreg_info *bfregi, u32 bfregn,
1487 			bool dyn_bfreg);
1488 
1489 int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter);
1490 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num);
1491 
1492 static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
1493 				       bool do_modify_atomic)
1494 {
1495 	if (MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
1496 		return false;
1497 
1498 	if (do_modify_atomic &&
1499 	    MLX5_CAP_GEN(dev->mdev, atomic) &&
1500 	    MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
1501 		return false;
1502 
1503 	return true;
1504 }
1505 
1506 int mlx5_ib_enable_driver(struct ib_device *dev);
1507 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
1508 #endif /* MLX5_IB_H */
1509