xref: /linux/drivers/infiniband/hw/ionic/ionic_ibdev.h (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3 
4 #ifndef _IONIC_IBDEV_H_
5 #define _IONIC_IBDEV_H_
6 
7 #include <rdma/ib_verbs.h>
8 #include <rdma/ib_pack.h>
9 #include <rdma/iter.h>
10 #include <rdma/uverbs_ioctl.h>
11 
12 #include <rdma/ionic-abi.h>
13 #include <ionic_api.h>
14 #include <ionic_regs.h>
15 
16 #include "ionic_fw.h"
17 #include "ionic_queue.h"
18 #include "ionic_res.h"
19 
20 #include "ionic_lif_cfg.h"
21 
22 /* Config knobs */
23 #define IONIC_EQ_DEPTH 511
24 #define IONIC_EQ_COUNT 32
25 #define IONIC_AQ_DEPTH 63
26 #define IONIC_AQ_COUNT 4
27 #define IONIC_EQ_ISR_BUDGET 10
28 #define IONIC_EQ_WORK_BUDGET 1000
29 #define IONIC_MAX_RD_ATOM 16
30 #define IONIC_PKEY_TBL_LEN 1
31 #define IONIC_GID_TBL_LEN 256
32 
33 #define IONIC_MAX_QPID 0xffffff
34 #define IONIC_SPEC_HIGH 8
35 #define IONIC_MAX_PD 1024
36 #define IONIC_SQCMB_ORDER 5
37 #define IONIC_RQCMB_ORDER 0
38 
39 #define IONIC_MAX_SRQ_SGES	2
40 #define IONIC_MAX_SRQ_LIMIT	0xffff
41 #define IONIC_MAX_SRQ_DEPTH	0xffff
42 
43 #define IONIC_META_LAST		((void *)1ul)
44 #define IONIC_META_POSTED	((void *)2ul)
45 
46 #define IONIC_CQ_GRACE 100
47 
48 #define IONIC_ROCE_UDP_SPORT	28272
49 #define IONIC_DMA_LKEY		0
50 #define IONIC_DMA_RKEY		IONIC_DMA_LKEY
51 
52 #define IONIC_CMB_SUPPORTED \
53 	(IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
54 	 IONIC_CMB_WC | IONIC_CMB_UC)
55 
56 /* resource is not reserved on the device, indicated in tbl_order */
57 #define IONIC_RES_INVALID	-1
58 
59 struct ionic_aq;
60 struct ionic_cq;
61 struct ionic_eq;
62 struct ionic_vcq;
63 
64 enum ionic_admin_state {
65 	IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
66 	IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
67 	IONIC_ADMIN_KILLED, /* not submitting, locally completed */
68 };
69 
70 enum ionic_admin_flags {
71 	IONIC_ADMIN_F_BUSYWAIT  = BIT(0),	/* Don't sleep */
72 	IONIC_ADMIN_F_TEARDOWN  = BIT(1),	/* In destroy path */
73 	IONIC_ADMIN_F_INTERRUPT = BIT(2),	/* Interruptible w/timeout */
74 };
75 
76 enum ionic_mmap_flag {
77 	IONIC_MMAP_WC = BIT(0),
78 	IONIC_MMAP_PHC = BIT(1),
79 };
80 
81 struct ionic_mmap_entry {
82 	struct rdma_user_mmap_entry rdma_entry;
83 	unsigned long size;
84 	unsigned long pfn;
85 	u8 mmap_flags;
86 };
87 
88 struct ionic_ibdev {
89 	struct ib_device	ibdev;
90 
91 	struct ionic_lif_cfg	lif_cfg;
92 
93 	struct xarray		qp_tbl;
94 	struct xarray		cq_tbl;
95 	struct xarray		srq_tbl;
96 
97 	struct ionic_resid_bits	inuse_dbid;
98 	struct ionic_resid_bits	inuse_pdid;
99 	struct ionic_resid_bits	inuse_ahid;
100 	struct ionic_resid_bits	inuse_mrid;
101 	struct ionic_resid_bits	inuse_qpid;
102 	struct ionic_resid_bits	inuse_cqid;
103 
104 	u8			half_cqid_udma_shift;
105 	u8			half_qpid_udma_shift;
106 	u8			next_qpid_udma_idx;
107 	u8			next_mrkey;
108 
109 	struct work_struct	reset_work;
110 	bool			reset_posted;
111 	u32			reset_cnt;
112 
113 	struct delayed_work	admin_dwork;
114 	struct ionic_aq		**aq_vec;
115 	atomic_t		admin_state;
116 
117 	struct ionic_eq		**eq_vec;
118 
119 	struct ionic_v1_stat	*hw_stats;
120 	void			*hw_stats_buf;
121 	struct rdma_stat_desc	*hw_stats_hdrs;
122 	struct ionic_counter_stats *counter_stats;
123 	int			hw_stats_count;
124 };
125 
126 struct ionic_eq {
127 	struct ionic_ibdev	*dev;
128 
129 	u32			eqid;
130 	u32			intr;
131 
132 	struct ionic_queue	q;
133 
134 	int			armed;
135 	bool			enable;
136 
137 	struct work_struct	work;
138 
139 	int			irq;
140 	char			name[32];
141 };
142 
143 struct ionic_admin_wr {
144 	struct completion		work;
145 	struct list_head		aq_ent;
146 	struct ionic_v1_admin_wqe	wqe;
147 	struct ionic_v1_cqe		cqe;
148 	struct ionic_aq			*aq;
149 	int				status;
150 };
151 
152 struct ionic_admin_wr_q {
153 	struct ionic_admin_wr	*wr;
154 	int			wqe_strides;
155 };
156 
157 struct ionic_aq {
158 	struct ionic_ibdev	*dev;
159 	struct ionic_vcq	*vcq;
160 
161 	struct work_struct	work;
162 
163 	atomic_t		admin_state;
164 	unsigned long		stamp;
165 	bool			armed;
166 
167 	u32			aqid;
168 	u32			cqid;
169 
170 	spinlock_t		lock; /* for posting */
171 	struct ionic_queue	q;
172 	struct ionic_admin_wr_q	*q_wr;
173 	struct list_head	wr_prod;
174 	struct list_head	wr_post;
175 };
176 
177 struct ionic_ctx {
178 	struct ib_ucontext	ibctx;
179 	u32			dbid;
180 	struct rdma_user_mmap_entry	*mmap_dbell;
181 	struct rdma_user_mmap_entry	*mmap_phc;
182 };
183 
184 struct ionic_tbl_buf {
185 	u32		tbl_limit;
186 	u32		tbl_pages;
187 	size_t		tbl_size;
188 	__le64		*tbl_buf;
189 	dma_addr_t	tbl_dma;
190 	u8		page_size_log2;
191 };
192 
193 struct ionic_pd {
194 	struct ib_pd		ibpd;
195 
196 	u32			pdid;
197 	u32			flags;
198 };
199 
200 struct ionic_cq {
201 	struct ionic_vcq	*vcq;
202 
203 	u32			cqid;
204 	u32			eqid;
205 
206 	spinlock_t		lock; /* for polling */
207 	struct list_head	poll_sq;
208 	bool			flush;
209 	struct list_head	flush_sq;
210 	struct list_head	flush_rq;
211 	struct list_head	ibkill_flush_ent;
212 
213 	struct ionic_queue	q;
214 	bool			color;
215 	int			credit;
216 	u16			arm_any_prod;
217 	u16			arm_sol_prod;
218 
219 	struct kref		cq_kref;
220 	struct completion	cq_rel_comp;
221 
222 	/* infrequently accessed, keep at end */
223 	struct ib_umem		*umem;
224 };
225 
226 struct ionic_vcq {
227 	struct ib_cq		ibcq;
228 	struct ionic_cq		cq[2];
229 	u8			udma_mask;
230 	u8			poll_idx;
231 };
232 
233 struct ionic_sq_meta {
234 	u64			wrid;
235 	u32			len;
236 	u16			seq;
237 	u8			ibop;
238 	u8			ibsts;
239 	u8			remote:1;
240 	u8			signal:1;
241 	u8			local_comp:1;
242 };
243 
244 struct ionic_rq_meta {
245 	struct ionic_rq_meta	*next;
246 	u64			wrid;
247 };
248 
249 struct ionic_rq {
250 	struct ionic_queue		q;
251 	u32				qid;
252 	u32				cqid;
253 	struct list_head		cq_flush;
254 	spinlock_t			lock; /* for posting and polling */
255 
256 	phys_addr_t			cmb_addr;
257 	struct rdma_user_mmap_entry	*mmap_cmb;
258 
259 	struct ionic_rq_meta		*meta;
260 	struct ionic_rq_meta		*meta_head;
261 	struct ib_umem			*umem;
262 
263 	int				spec;
264 	int				cmb_order;
265 	u32				cmb_pgid;
266 	u16				old_prod;
267 	u8				cmb;
268 	bool				flush;
269 };
270 
271 struct ionic_srq {
272 	struct ib_srq		ibsrq;
273 	struct ionic_rq		rq;
274 	struct kref		kref;
275 	struct completion	rel_comp;
276 	u16			srq_limit;
277 	u8			udma_idx;
278 };
279 
280 struct ionic_qp {
281 	struct ib_qp		ibqp;
282 	enum ib_qp_state	state;
283 
284 	u32			qpid;
285 	u32			ahid;
286 	u32			sq_cqid;
287 	u8			udma_idx;
288 	u8			has_ah:1;
289 	u8			has_sq:1;
290 	u8			sig_all:1;
291 
292 	struct list_head	qp_list_counter;
293 
294 	struct list_head	cq_poll_sq;
295 	struct list_head	cq_flush_sq;
296 	struct list_head	ibkill_flush_ent;
297 
298 	spinlock_t		sq_lock; /* for posting and polling */
299 	struct ionic_queue	sq;
300 	struct ionic_sq_meta	*sq_meta;
301 	u16			*sq_msn_idx;
302 	int			sq_spec;
303 	u16			sq_old_prod;
304 	u16			sq_msn_prod;
305 	u16			sq_msn_cons;
306 	u8			sq_cmb;
307 	bool			sq_flush;
308 	bool			sq_flush_rcvd;
309 
310 	struct ionic_rq		rq;
311 
312 	struct kref		qp_kref;
313 	struct completion	qp_rel_comp;
314 
315 	/* infrequently accessed, keep at end */
316 	int			sgid_index;
317 	int			sq_cmb_order;
318 	u32			sq_cmb_pgid;
319 	phys_addr_t		sq_cmb_addr;
320 	struct rdma_user_mmap_entry *mmap_sq_cmb;
321 
322 	struct ib_umem		*sq_umem;
323 
324 	int			dcqcn_profile;
325 
326 	struct ib_ud_header	*hdr;
327 	struct ionic_srq	*srq;
328 };
329 
330 struct ionic_ah {
331 	struct ib_ah		ibah;
332 	u32			ahid;
333 	int			sgid_index;
334 	struct ib_ud_header	hdr;
335 };
336 
337 struct ionic_mr {
338 	union {
339 		struct ib_mr	ibmr;
340 		struct ib_mw	ibmw;
341 	};
342 
343 	u32			mrid;
344 	int			flags;
345 
346 	struct ib_umem		*umem;
347 	struct ionic_tbl_buf	buf;
348 	bool			created;
349 };
350 
351 struct ionic_counter_stats {
352 	int queue_stats_count;
353 	struct ionic_v1_stat *hdr;
354 	struct rdma_stat_desc *stats_hdrs;
355 	struct ida counter_ida;
356 };
357 
358 struct ionic_rdma_counter {
359 	struct rdma_counter rdma_counter;
360 	void *vals;
361 	struct list_head qp_list;
362 };
363 
364 static inline struct ionic_rdma_counter *
365 to_ionic_rdma_counter(struct rdma_counter *counter)
366 {
367 	return container_of(counter, struct ionic_rdma_counter, rdma_counter);
368 }
369 
370 static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
371 {
372 	return container_of(ibdev, struct ionic_ibdev, ibdev);
373 }
374 
375 static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)
376 {
377 	return container_of(ibctx, struct ionic_ctx, ibctx);
378 }
379 
380 static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)
381 {
382 	if (!uobj)
383 		return NULL;
384 
385 	if (!uobj->context)
386 		return NULL;
387 
388 	return to_ionic_ctx(uobj->context);
389 }
390 
391 static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)
392 {
393 	return container_of(ibpd, struct ionic_pd, ibpd);
394 }
395 
396 static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)
397 {
398 	return container_of(ibmr, struct ionic_mr, ibmr);
399 }
400 
401 static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)
402 {
403 	return container_of(ibmw, struct ionic_mr, ibmw);
404 }
405 
406 static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)
407 {
408 	return container_of(ibcq, struct ionic_vcq, ibcq);
409 }
410 
411 static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,
412 					       uint8_t udma_idx)
413 {
414 	return &to_ionic_vcq(ibcq)->cq[udma_idx];
415 }
416 
417 static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
418 {
419 	return container_of(ibqp, struct ionic_qp, ibqp);
420 }
421 
422 static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq)
423 {
424 	return container_of(ibsrq, struct ionic_srq, ibsrq);
425 }
426 
427 static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
428 {
429 	return container_of(ibah, struct ionic_ah, ibah);
430 }
431 
432 static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,
433 				 struct ionic_ctx *ctx)
434 {
435 	if (!ctx)
436 		return dev->lif_cfg.dbid;
437 
438 	return ctx->dbid;
439 }
440 
441 static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,
442 				 struct ib_uobject *uobj)
443 {
444 	return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));
445 }
446 
447 static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)
448 {
449 	return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;
450 }
451 
452 static inline void ionic_qp_complete(struct kref *kref)
453 {
454 	struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);
455 
456 	complete(&qp->qp_rel_comp);
457 }
458 
459 static inline void ionic_cq_complete(struct kref *kref)
460 {
461 	struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);
462 
463 	complete(&cq->cq_rel_comp);
464 }
465 
466 static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev,
467 					  enum ionic_lif_rdma_alloc_qid qtype)
468 {
469 	return dev->lif_cfg.alloc_qid_cap & qtype;
470 }
471 
472 static inline void ionic_srq_complete(struct kref *kref)
473 {
474 	struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref);
475 
476 	complete(&srq->rel_comp);
477 }
478 
479 /* ionic_admin.c */
480 extern struct workqueue_struct *ionic_evt_workq;
481 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
482 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
483 		     enum ionic_admin_flags);
484 
485 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);
486 
487 int ionic_create_rdma_admin(struct ionic_ibdev *dev);
488 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);
489 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);
490 
491 /* ionic_controlpath.c */
492 int ionic_create_cq_common(struct ionic_vcq *vcq,
493 			   struct ionic_tbl_buf *buf,
494 			   const struct ib_cq_init_attr *attr,
495 			   struct ionic_ctx *ctx,
496 			   struct ib_udata *udata,
497 			   struct ionic_qdesc *req_cq,
498 			   int udma_idx);
499 int ionic_post_create_cq_cmd(struct ionic_cq *cq,
500 			     struct ib_udata *udata,
501 			     __u32 *resp_cqid);
502 void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq);
503 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
504 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
505 void ionic_notify_flush_cq(struct ionic_cq *cq);
506 
507 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);
508 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);
509 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);
510 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
511 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
512 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
513 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
514 		    struct ib_udata *udata);
515 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
516 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);
517 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);
518 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
519 				u64 addr, int access, struct ib_dmah *dmah,
520 				struct ib_udata *udata);
521 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
522 				       u64 length, u64 addr, int fd, int access,
523 					   struct ib_dmah *dmah,
524 				       struct uverbs_attr_bundle *attrs);
525 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
526 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
527 			     u32 max_sg);
528 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
529 		    unsigned int *sg_offset);
530 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
531 int ionic_dealloc_mw(struct ib_mw *ibmw);
532 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
533 		    struct uverbs_attr_bundle *attrs);
534 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
535 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
536 		    struct ib_udata *udata);
537 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
538 		    struct ib_udata *udata);
539 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
540 		   struct ib_qp_init_attr *init_attr);
541 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
542 int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr,
543 		     struct ib_udata *udata);
544 int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
545 int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
546 int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
547 		     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
548 
549 /* ionic_datapath.c */
550 int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
551 		    const struct ib_send_wr **bad);
552 int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
553 		    const struct ib_recv_wr **bad);
554 int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);
555 int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
556 
557 /* ionic_hw_stats.c */
558 void ionic_stats_init(struct ionic_ibdev *dev);
559 void ionic_stats_cleanup(struct ionic_ibdev *dev);
560 
561 /* ionic_pgtbl.c */
562 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);
563 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);
564 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);
565 int ionic_pgtbl_init(struct ionic_ibdev *dev,
566 		     struct ionic_tbl_buf *buf,
567 		     struct ib_umem *umem,
568 		     dma_addr_t dma,
569 		     int limit,
570 		     u64 page_size);
571 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);
572 #endif /* _IONIC_IBDEV_H_ */
573