1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #ifndef _IONIC_IBDEV_H_ 5 #define _IONIC_IBDEV_H_ 6 7 #include <rdma/ib_verbs.h> 8 #include <rdma/ib_pack.h> 9 #include <rdma/iter.h> 10 #include <rdma/uverbs_ioctl.h> 11 12 #include <rdma/ionic-abi.h> 13 #include <ionic_api.h> 14 #include <ionic_regs.h> 15 16 #include "ionic_fw.h" 17 #include "ionic_queue.h" 18 #include "ionic_res.h" 19 20 #include "ionic_lif_cfg.h" 21 22 /* Config knobs */ 23 #define IONIC_EQ_DEPTH 511 24 #define IONIC_EQ_COUNT 32 25 #define IONIC_AQ_DEPTH 63 26 #define IONIC_AQ_COUNT 4 27 #define IONIC_EQ_ISR_BUDGET 10 28 #define IONIC_EQ_WORK_BUDGET 1000 29 #define IONIC_MAX_RD_ATOM 16 30 #define IONIC_PKEY_TBL_LEN 1 31 #define IONIC_GID_TBL_LEN 256 32 33 #define IONIC_MAX_QPID 0xffffff 34 #define IONIC_SPEC_HIGH 8 35 #define IONIC_MAX_PD 1024 36 #define IONIC_SQCMB_ORDER 5 37 #define IONIC_RQCMB_ORDER 0 38 39 #define IONIC_MAX_SRQ_SGES 2 40 #define IONIC_MAX_SRQ_LIMIT 0xffff 41 #define IONIC_MAX_SRQ_DEPTH 0xffff 42 43 #define IONIC_META_LAST ((void *)1ul) 44 #define IONIC_META_POSTED ((void *)2ul) 45 46 #define IONIC_CQ_GRACE 100 47 48 #define IONIC_ROCE_UDP_SPORT 28272 49 #define IONIC_DMA_LKEY 0 50 #define IONIC_DMA_RKEY IONIC_DMA_LKEY 51 52 #define IONIC_CMB_SUPPORTED \ 53 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ 54 IONIC_CMB_WC | IONIC_CMB_UC) 55 56 /* resource is not reserved on the device, indicated in tbl_order */ 57 #define IONIC_RES_INVALID -1 58 59 struct ionic_aq; 60 struct ionic_cq; 61 struct ionic_eq; 62 struct ionic_vcq; 63 64 enum ionic_admin_state { 65 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ 66 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ 67 IONIC_ADMIN_KILLED, /* not submitting, locally completed */ 68 }; 69 70 enum ionic_admin_flags { 71 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ 72 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ 73 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */ 74 }; 75 76 enum ionic_mmap_flag { 77 IONIC_MMAP_WC = BIT(0), 78 IONIC_MMAP_PHC = BIT(1), 79 }; 80 81 struct ionic_mmap_entry { 82 struct rdma_user_mmap_entry rdma_entry; 83 unsigned long size; 84 unsigned long pfn; 85 u8 mmap_flags; 86 }; 87 88 struct ionic_ibdev { 89 struct ib_device ibdev; 90 91 struct ionic_lif_cfg lif_cfg; 92 93 struct xarray qp_tbl; 94 struct xarray cq_tbl; 95 struct xarray srq_tbl; 96 97 struct ionic_resid_bits inuse_dbid; 98 struct ionic_resid_bits inuse_pdid; 99 struct ionic_resid_bits inuse_ahid; 100 struct ionic_resid_bits inuse_mrid; 101 struct ionic_resid_bits inuse_qpid; 102 struct ionic_resid_bits inuse_cqid; 103 104 u8 half_cqid_udma_shift; 105 u8 half_qpid_udma_shift; 106 u8 next_qpid_udma_idx; 107 u8 next_mrkey; 108 109 struct work_struct reset_work; 110 bool reset_posted; 111 u32 reset_cnt; 112 113 struct delayed_work admin_dwork; 114 struct ionic_aq **aq_vec; 115 atomic_t admin_state; 116 117 struct ionic_eq **eq_vec; 118 119 struct ionic_v1_stat *hw_stats; 120 void *hw_stats_buf; 121 struct rdma_stat_desc *hw_stats_hdrs; 122 struct ionic_counter_stats *counter_stats; 123 int hw_stats_count; 124 }; 125 126 struct ionic_eq { 127 struct ionic_ibdev *dev; 128 129 u32 eqid; 130 u32 intr; 131 132 struct ionic_queue q; 133 134 int armed; 135 bool enable; 136 137 struct work_struct work; 138 139 int irq; 140 char name[32]; 141 }; 142 143 struct ionic_admin_wr { 144 struct completion work; 145 struct list_head aq_ent; 146 struct ionic_v1_admin_wqe wqe; 147 struct ionic_v1_cqe cqe; 148 struct ionic_aq *aq; 149 int status; 150 }; 151 152 struct ionic_admin_wr_q { 153 struct ionic_admin_wr *wr; 154 int wqe_strides; 155 }; 156 157 struct ionic_aq { 158 struct ionic_ibdev *dev; 159 struct ionic_vcq *vcq; 160 161 struct work_struct work; 162 163 atomic_t admin_state; 164 unsigned long stamp; 165 bool armed; 166 167 u32 aqid; 168 u32 cqid; 169 170 spinlock_t lock; /* for posting */ 171 struct ionic_queue q; 172 struct ionic_admin_wr_q *q_wr; 173 struct list_head wr_prod; 174 struct list_head wr_post; 175 }; 176 177 struct ionic_ctx { 178 struct ib_ucontext ibctx; 179 u32 dbid; 180 struct rdma_user_mmap_entry *mmap_dbell; 181 struct rdma_user_mmap_entry *mmap_phc; 182 }; 183 184 struct ionic_tbl_buf { 185 u32 tbl_limit; 186 u32 tbl_pages; 187 size_t tbl_size; 188 __le64 *tbl_buf; 189 dma_addr_t tbl_dma; 190 u8 page_size_log2; 191 }; 192 193 struct ionic_pd { 194 struct ib_pd ibpd; 195 196 u32 pdid; 197 u32 flags; 198 }; 199 200 struct ionic_cq { 201 struct ionic_vcq *vcq; 202 203 u32 cqid; 204 u32 eqid; 205 206 spinlock_t lock; /* for polling */ 207 struct list_head poll_sq; 208 bool flush; 209 struct list_head flush_sq; 210 struct list_head flush_rq; 211 struct list_head ibkill_flush_ent; 212 213 struct ionic_queue q; 214 bool color; 215 int credit; 216 u16 arm_any_prod; 217 u16 arm_sol_prod; 218 219 struct kref cq_kref; 220 struct completion cq_rel_comp; 221 222 /* infrequently accessed, keep at end */ 223 struct ib_umem *umem; 224 }; 225 226 struct ionic_vcq { 227 struct ib_cq ibcq; 228 struct ionic_cq cq[2]; 229 u8 udma_mask; 230 u8 poll_idx; 231 }; 232 233 struct ionic_sq_meta { 234 u64 wrid; 235 u32 len; 236 u16 seq; 237 u8 ibop; 238 u8 ibsts; 239 u8 remote:1; 240 u8 signal:1; 241 u8 local_comp:1; 242 }; 243 244 struct ionic_rq_meta { 245 struct ionic_rq_meta *next; 246 u64 wrid; 247 }; 248 249 struct ionic_rq { 250 struct ionic_queue q; 251 u32 qid; 252 u32 cqid; 253 struct list_head cq_flush; 254 spinlock_t lock; /* for posting and polling */ 255 256 phys_addr_t cmb_addr; 257 struct rdma_user_mmap_entry *mmap_cmb; 258 259 struct ionic_rq_meta *meta; 260 struct ionic_rq_meta *meta_head; 261 struct ib_umem *umem; 262 263 int spec; 264 int cmb_order; 265 u32 cmb_pgid; 266 u16 old_prod; 267 u8 cmb; 268 bool flush; 269 }; 270 271 struct ionic_srq { 272 struct ib_srq ibsrq; 273 struct ionic_rq rq; 274 struct kref kref; 275 struct completion rel_comp; 276 u16 srq_limit; 277 u8 udma_idx; 278 }; 279 280 struct ionic_qp { 281 struct ib_qp ibqp; 282 enum ib_qp_state state; 283 284 u32 qpid; 285 u32 ahid; 286 u32 sq_cqid; 287 u8 udma_idx; 288 u8 has_ah:1; 289 u8 has_sq:1; 290 u8 sig_all:1; 291 292 struct list_head qp_list_counter; 293 294 struct list_head cq_poll_sq; 295 struct list_head cq_flush_sq; 296 struct list_head ibkill_flush_ent; 297 298 spinlock_t sq_lock; /* for posting and polling */ 299 struct ionic_queue sq; 300 struct ionic_sq_meta *sq_meta; 301 u16 *sq_msn_idx; 302 int sq_spec; 303 u16 sq_old_prod; 304 u16 sq_msn_prod; 305 u16 sq_msn_cons; 306 u8 sq_cmb; 307 bool sq_flush; 308 bool sq_flush_rcvd; 309 310 struct ionic_rq rq; 311 312 struct kref qp_kref; 313 struct completion qp_rel_comp; 314 315 /* infrequently accessed, keep at end */ 316 int sgid_index; 317 int sq_cmb_order; 318 u32 sq_cmb_pgid; 319 phys_addr_t sq_cmb_addr; 320 struct rdma_user_mmap_entry *mmap_sq_cmb; 321 322 struct ib_umem *sq_umem; 323 324 int dcqcn_profile; 325 326 struct ib_ud_header *hdr; 327 struct ionic_srq *srq; 328 }; 329 330 struct ionic_ah { 331 struct ib_ah ibah; 332 u32 ahid; 333 int sgid_index; 334 struct ib_ud_header hdr; 335 }; 336 337 struct ionic_mr { 338 union { 339 struct ib_mr ibmr; 340 struct ib_mw ibmw; 341 }; 342 343 u32 mrid; 344 int flags; 345 346 struct ib_umem *umem; 347 struct ionic_tbl_buf buf; 348 bool created; 349 }; 350 351 struct ionic_counter_stats { 352 int queue_stats_count; 353 struct ionic_v1_stat *hdr; 354 struct rdma_stat_desc *stats_hdrs; 355 struct ida counter_ida; 356 }; 357 358 struct ionic_rdma_counter { 359 struct rdma_counter rdma_counter; 360 void *vals; 361 struct list_head qp_list; 362 }; 363 364 static inline struct ionic_rdma_counter * 365 to_ionic_rdma_counter(struct rdma_counter *counter) 366 { 367 return container_of(counter, struct ionic_rdma_counter, rdma_counter); 368 } 369 370 static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) 371 { 372 return container_of(ibdev, struct ionic_ibdev, ibdev); 373 } 374 375 static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx) 376 { 377 return container_of(ibctx, struct ionic_ctx, ibctx); 378 } 379 380 static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj) 381 { 382 if (!uobj) 383 return NULL; 384 385 if (!uobj->context) 386 return NULL; 387 388 return to_ionic_ctx(uobj->context); 389 } 390 391 static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd) 392 { 393 return container_of(ibpd, struct ionic_pd, ibpd); 394 } 395 396 static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr) 397 { 398 return container_of(ibmr, struct ionic_mr, ibmr); 399 } 400 401 static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw) 402 { 403 return container_of(ibmw, struct ionic_mr, ibmw); 404 } 405 406 static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq) 407 { 408 return container_of(ibcq, struct ionic_vcq, ibcq); 409 } 410 411 static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, 412 uint8_t udma_idx) 413 { 414 return &to_ionic_vcq(ibcq)->cq[udma_idx]; 415 } 416 417 static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) 418 { 419 return container_of(ibqp, struct ionic_qp, ibqp); 420 } 421 422 static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq) 423 { 424 return container_of(ibsrq, struct ionic_srq, ibsrq); 425 } 426 427 static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) 428 { 429 return container_of(ibah, struct ionic_ah, ibah); 430 } 431 432 static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, 433 struct ionic_ctx *ctx) 434 { 435 if (!ctx) 436 return dev->lif_cfg.dbid; 437 438 return ctx->dbid; 439 } 440 441 static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, 442 struct ib_uobject *uobj) 443 { 444 return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj)); 445 } 446 447 static inline bool ionic_ibop_is_local(enum ib_wr_opcode op) 448 { 449 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR; 450 } 451 452 static inline void ionic_qp_complete(struct kref *kref) 453 { 454 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); 455 456 complete(&qp->qp_rel_comp); 457 } 458 459 static inline void ionic_cq_complete(struct kref *kref) 460 { 461 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); 462 463 complete(&cq->cq_rel_comp); 464 } 465 466 static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev, 467 enum ionic_lif_rdma_alloc_qid qtype) 468 { 469 return dev->lif_cfg.alloc_qid_cap & qtype; 470 } 471 472 static inline void ionic_srq_complete(struct kref *kref) 473 { 474 struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref); 475 476 complete(&srq->rel_comp); 477 } 478 479 /* ionic_admin.c */ 480 extern struct workqueue_struct *ionic_evt_workq; 481 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); 482 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, 483 enum ionic_admin_flags); 484 485 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); 486 487 int ionic_create_rdma_admin(struct ionic_ibdev *dev); 488 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev); 489 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); 490 491 /* ionic_controlpath.c */ 492 int ionic_create_cq_common(struct ionic_vcq *vcq, 493 struct ionic_tbl_buf *buf, 494 const struct ib_cq_init_attr *attr, 495 struct ionic_ctx *ctx, 496 struct ib_udata *udata, 497 struct ionic_qdesc *req_cq, 498 int udma_idx); 499 int ionic_post_create_cq_cmd(struct ionic_cq *cq, 500 struct ib_udata *udata, 501 __u32 *resp_cqid); 502 void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq); 503 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); 504 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); 505 void ionic_notify_flush_cq(struct ionic_cq *cq); 506 507 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata); 508 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx); 509 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma); 510 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry); 511 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 512 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 513 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 514 struct ib_udata *udata); 515 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); 516 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags); 517 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access); 518 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 519 u64 addr, int access, struct ib_dmah *dmah, 520 struct ib_udata *udata); 521 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 522 u64 length, u64 addr, int fd, int access, 523 struct ib_dmah *dmah, 524 struct uverbs_attr_bundle *attrs); 525 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); 526 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 527 u32 max_sg); 528 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 529 unsigned int *sg_offset); 530 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); 531 int ionic_dealloc_mw(struct ib_mw *ibmw); 532 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 533 struct uverbs_attr_bundle *attrs); 534 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 535 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 536 struct ib_udata *udata); 537 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 538 struct ib_udata *udata); 539 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 540 struct ib_qp_init_attr *init_attr); 541 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 542 int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, 543 struct ib_udata *udata); 544 int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); 545 int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); 546 int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 547 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); 548 549 /* ionic_datapath.c */ 550 int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 551 const struct ib_send_wr **bad); 552 int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 553 const struct ib_recv_wr **bad); 554 int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc); 555 int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 556 557 /* ionic_hw_stats.c */ 558 void ionic_stats_init(struct ionic_ibdev *dev); 559 void ionic_stats_cleanup(struct ionic_ibdev *dev); 560 561 /* ionic_pgtbl.c */ 562 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va); 563 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va); 564 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma); 565 int ionic_pgtbl_init(struct ionic_ibdev *dev, 566 struct ionic_tbl_buf *buf, 567 struct ib_umem *umem, 568 dma_addr_t dma, 569 int limit, 570 u64 page_size); 571 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf); 572 #endif /* _IONIC_IBDEV_H_ */ 573