1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #ifndef _IONIC_IBDEV_H_ 5 #define _IONIC_IBDEV_H_ 6 7 #include <rdma/ib_umem.h> 8 #include <rdma/ib_verbs.h> 9 #include <rdma/ib_pack.h> 10 #include <rdma/uverbs_ioctl.h> 11 12 #include <rdma/ionic-abi.h> 13 #include <ionic_api.h> 14 #include <ionic_regs.h> 15 16 #include "ionic_fw.h" 17 #include "ionic_queue.h" 18 #include "ionic_res.h" 19 20 #include "ionic_lif_cfg.h" 21 22 /* Config knobs */ 23 #define IONIC_EQ_DEPTH 511 24 #define IONIC_EQ_COUNT 32 25 #define IONIC_AQ_DEPTH 63 26 #define IONIC_AQ_COUNT 4 27 #define IONIC_EQ_ISR_BUDGET 10 28 #define IONIC_EQ_WORK_BUDGET 1000 29 #define IONIC_MAX_PD 1024 30 #define IONIC_SPEC_HIGH 8 31 #define IONIC_SQCMB_ORDER 5 32 #define IONIC_RQCMB_ORDER 0 33 34 #define IONIC_META_LAST ((void *)1ul) 35 #define IONIC_META_POSTED ((void *)2ul) 36 37 #define IONIC_CQ_GRACE 100 38 39 #define IONIC_ROCE_UDP_SPORT 28272 40 #define IONIC_DMA_LKEY 0 41 #define IONIC_DMA_RKEY IONIC_DMA_LKEY 42 43 #define IONIC_CMB_SUPPORTED \ 44 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ 45 IONIC_CMB_WC | IONIC_CMB_UC) 46 47 /* resource is not reserved on the device, indicated in tbl_order */ 48 #define IONIC_RES_INVALID -1 49 50 struct ionic_aq; 51 struct ionic_cq; 52 struct ionic_eq; 53 struct ionic_vcq; 54 55 enum ionic_admin_state { 56 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ 57 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ 58 IONIC_ADMIN_KILLED, /* not submitting, locally completed */ 59 }; 60 61 enum ionic_admin_flags { 62 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ 63 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ 64 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */ 65 }; 66 67 enum ionic_mmap_flag { 68 IONIC_MMAP_WC = BIT(0), 69 }; 70 71 struct ionic_mmap_entry { 72 struct rdma_user_mmap_entry rdma_entry; 73 unsigned long size; 74 unsigned long pfn; 75 u8 mmap_flags; 76 }; 77 78 struct ionic_ibdev { 79 struct ib_device ibdev; 80 81 struct ionic_lif_cfg lif_cfg; 82 83 struct xarray qp_tbl; 84 struct xarray cq_tbl; 85 86 struct ionic_resid_bits inuse_dbid; 87 struct ionic_resid_bits inuse_pdid; 88 struct ionic_resid_bits inuse_ahid; 89 struct ionic_resid_bits inuse_mrid; 90 struct ionic_resid_bits inuse_qpid; 91 struct ionic_resid_bits inuse_cqid; 92 93 u8 half_cqid_udma_shift; 94 u8 half_qpid_udma_shift; 95 u8 next_qpid_udma_idx; 96 u8 next_mrkey; 97 98 struct work_struct reset_work; 99 bool reset_posted; 100 u32 reset_cnt; 101 102 struct delayed_work admin_dwork; 103 struct ionic_aq **aq_vec; 104 atomic_t admin_state; 105 106 struct ionic_eq **eq_vec; 107 }; 108 109 struct ionic_eq { 110 struct ionic_ibdev *dev; 111 112 u32 eqid; 113 u32 intr; 114 115 struct ionic_queue q; 116 117 bool armed; 118 bool enable; 119 120 struct work_struct work; 121 122 int irq; 123 char name[32]; 124 }; 125 126 struct ionic_admin_wr { 127 struct completion work; 128 struct list_head aq_ent; 129 struct ionic_v1_admin_wqe wqe; 130 struct ionic_v1_cqe cqe; 131 struct ionic_aq *aq; 132 int status; 133 }; 134 135 struct ionic_admin_wr_q { 136 struct ionic_admin_wr *wr; 137 int wqe_strides; 138 }; 139 140 struct ionic_aq { 141 struct ionic_ibdev *dev; 142 struct ionic_vcq *vcq; 143 144 struct work_struct work; 145 146 atomic_t admin_state; 147 unsigned long stamp; 148 bool armed; 149 150 u32 aqid; 151 u32 cqid; 152 153 spinlock_t lock; /* for posting */ 154 struct ionic_queue q; 155 struct ionic_admin_wr_q *q_wr; 156 struct list_head wr_prod; 157 struct list_head wr_post; 158 }; 159 160 struct ionic_ctx { 161 struct ib_ucontext ibctx; 162 u32 dbid; 163 struct rdma_user_mmap_entry *mmap_dbell; 164 }; 165 166 struct ionic_tbl_buf { 167 u32 tbl_limit; 168 u32 tbl_pages; 169 size_t tbl_size; 170 __le64 *tbl_buf; 171 dma_addr_t tbl_dma; 172 u8 page_size_log2; 173 }; 174 175 struct ionic_pd { 176 struct ib_pd ibpd; 177 178 u32 pdid; 179 u32 flags; 180 }; 181 182 struct ionic_cq { 183 struct ionic_vcq *vcq; 184 185 u32 cqid; 186 u32 eqid; 187 188 spinlock_t lock; /* for polling */ 189 struct list_head poll_sq; 190 bool flush; 191 struct list_head flush_sq; 192 struct list_head flush_rq; 193 struct list_head ibkill_flush_ent; 194 195 struct ionic_queue q; 196 bool color; 197 int credit; 198 u16 arm_any_prod; 199 u16 arm_sol_prod; 200 201 struct kref cq_kref; 202 struct completion cq_rel_comp; 203 204 /* infrequently accessed, keep at end */ 205 struct ib_umem *umem; 206 }; 207 208 struct ionic_vcq { 209 struct ib_cq ibcq; 210 struct ionic_cq cq[2]; 211 u8 udma_mask; 212 u8 poll_idx; 213 }; 214 215 struct ionic_sq_meta { 216 u64 wrid; 217 u32 len; 218 u16 seq; 219 u8 ibop; 220 u8 ibsts; 221 u8 remote:1; 222 u8 signal:1; 223 u8 local_comp:1; 224 }; 225 226 struct ionic_rq_meta { 227 struct ionic_rq_meta *next; 228 u64 wrid; 229 }; 230 231 struct ionic_qp { 232 struct ib_qp ibqp; 233 enum ib_qp_state state; 234 235 u32 qpid; 236 u32 ahid; 237 u32 sq_cqid; 238 u32 rq_cqid; 239 u8 udma_idx; 240 u8 has_ah:1; 241 u8 has_sq:1; 242 u8 has_rq:1; 243 u8 sig_all:1; 244 245 struct list_head qp_list_counter; 246 247 struct list_head cq_poll_sq; 248 struct list_head cq_flush_sq; 249 struct list_head cq_flush_rq; 250 struct list_head ibkill_flush_ent; 251 252 spinlock_t sq_lock; /* for posting and polling */ 253 struct ionic_queue sq; 254 struct ionic_sq_meta *sq_meta; 255 u16 *sq_msn_idx; 256 int sq_spec; 257 u16 sq_old_prod; 258 u16 sq_msn_prod; 259 u16 sq_msn_cons; 260 u8 sq_cmb; 261 bool sq_flush; 262 bool sq_flush_rcvd; 263 264 spinlock_t rq_lock; /* for posting and polling */ 265 struct ionic_queue rq; 266 struct ionic_rq_meta *rq_meta; 267 struct ionic_rq_meta *rq_meta_head; 268 int rq_spec; 269 u16 rq_old_prod; 270 u8 rq_cmb; 271 bool rq_flush; 272 273 struct kref qp_kref; 274 struct completion qp_rel_comp; 275 276 /* infrequently accessed, keep at end */ 277 int sgid_index; 278 int sq_cmb_order; 279 u32 sq_cmb_pgid; 280 phys_addr_t sq_cmb_addr; 281 struct rdma_user_mmap_entry *mmap_sq_cmb; 282 283 struct ib_umem *sq_umem; 284 285 int rq_cmb_order; 286 u32 rq_cmb_pgid; 287 phys_addr_t rq_cmb_addr; 288 struct rdma_user_mmap_entry *mmap_rq_cmb; 289 290 struct ib_umem *rq_umem; 291 292 int dcqcn_profile; 293 294 struct ib_ud_header *hdr; 295 }; 296 297 struct ionic_ah { 298 struct ib_ah ibah; 299 u32 ahid; 300 int sgid_index; 301 struct ib_ud_header hdr; 302 }; 303 304 struct ionic_mr { 305 union { 306 struct ib_mr ibmr; 307 struct ib_mw ibmw; 308 }; 309 310 u32 mrid; 311 int flags; 312 313 struct ib_umem *umem; 314 struct ionic_tbl_buf buf; 315 bool created; 316 }; 317 318 static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) 319 { 320 return container_of(ibdev, struct ionic_ibdev, ibdev); 321 } 322 323 static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx) 324 { 325 return container_of(ibctx, struct ionic_ctx, ibctx); 326 } 327 328 static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj) 329 { 330 if (!uobj) 331 return NULL; 332 333 if (!uobj->context) 334 return NULL; 335 336 return to_ionic_ctx(uobj->context); 337 } 338 339 static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd) 340 { 341 return container_of(ibpd, struct ionic_pd, ibpd); 342 } 343 344 static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr) 345 { 346 return container_of(ibmr, struct ionic_mr, ibmr); 347 } 348 349 static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw) 350 { 351 return container_of(ibmw, struct ionic_mr, ibmw); 352 } 353 354 static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq) 355 { 356 return container_of(ibcq, struct ionic_vcq, ibcq); 357 } 358 359 static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, 360 uint8_t udma_idx) 361 { 362 return &to_ionic_vcq(ibcq)->cq[udma_idx]; 363 } 364 365 static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) 366 { 367 return container_of(ibqp, struct ionic_qp, ibqp); 368 } 369 370 static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) 371 { 372 return container_of(ibah, struct ionic_ah, ibah); 373 } 374 375 static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, 376 struct ionic_ctx *ctx) 377 { 378 if (!ctx) 379 return dev->lif_cfg.dbid; 380 381 return ctx->dbid; 382 } 383 384 static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, 385 struct ib_uobject *uobj) 386 { 387 return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj)); 388 } 389 390 static inline bool ionic_ibop_is_local(enum ib_wr_opcode op) 391 { 392 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR; 393 } 394 395 static inline void ionic_qp_complete(struct kref *kref) 396 { 397 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); 398 399 complete(&qp->qp_rel_comp); 400 } 401 402 static inline void ionic_cq_complete(struct kref *kref) 403 { 404 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); 405 406 complete(&cq->cq_rel_comp); 407 } 408 409 /* ionic_admin.c */ 410 extern struct workqueue_struct *ionic_evt_workq; 411 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); 412 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, 413 enum ionic_admin_flags); 414 415 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); 416 417 int ionic_create_rdma_admin(struct ionic_ibdev *dev); 418 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev); 419 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); 420 421 /* ionic_controlpath.c */ 422 int ionic_create_cq_common(struct ionic_vcq *vcq, 423 struct ionic_tbl_buf *buf, 424 const struct ib_cq_init_attr *attr, 425 struct ionic_ctx *ctx, 426 struct ib_udata *udata, 427 struct ionic_qdesc *req_cq, 428 __u32 *resp_cqid, 429 int udma_idx); 430 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); 431 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); 432 void ionic_notify_flush_cq(struct ionic_cq *cq); 433 434 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata); 435 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx); 436 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma); 437 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry); 438 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 439 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 440 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 441 struct ib_udata *udata); 442 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); 443 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags); 444 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access); 445 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 446 u64 addr, int access, struct ib_dmah *dmah, 447 struct ib_udata *udata); 448 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 449 u64 length, u64 addr, int fd, int access, 450 struct ib_dmah *dmah, 451 struct uverbs_attr_bundle *attrs); 452 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); 453 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 454 u32 max_sg); 455 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 456 unsigned int *sg_offset); 457 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); 458 int ionic_dealloc_mw(struct ib_mw *ibmw); 459 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 460 struct uverbs_attr_bundle *attrs); 461 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 462 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 463 struct ib_udata *udata); 464 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 465 struct ib_udata *udata); 466 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 467 struct ib_qp_init_attr *init_attr); 468 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 469 470 /* ionic_datapath.c */ 471 int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 472 const struct ib_send_wr **bad); 473 int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 474 const struct ib_recv_wr **bad); 475 int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc); 476 int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 477 478 /* ionic_pgtbl.c */ 479 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va); 480 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va); 481 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma); 482 int ionic_pgtbl_init(struct ionic_ibdev *dev, 483 struct ionic_tbl_buf *buf, 484 struct ib_umem *umem, 485 dma_addr_t dma, 486 int limit, 487 u64 page_size); 488 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf); 489 #endif /* _IONIC_IBDEV_H_ */ 490