1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #ifndef _IONIC_IBDEV_H_ 5 #define _IONIC_IBDEV_H_ 6 7 #include <rdma/ib_verbs.h> 8 #include <rdma/ib_pack.h> 9 #include <rdma/iter.h> 10 #include <rdma/uverbs_ioctl.h> 11 12 #include <rdma/ionic-abi.h> 13 #include <ionic_api.h> 14 #include <ionic_regs.h> 15 16 #include "ionic_fw.h" 17 #include "ionic_queue.h" 18 #include "ionic_res.h" 19 20 #include "ionic_lif_cfg.h" 21 22 /* Config knobs */ 23 #define IONIC_EQ_DEPTH 511 24 #define IONIC_EQ_COUNT 32 25 #define IONIC_AQ_DEPTH 63 26 #define IONIC_AQ_COUNT 4 27 #define IONIC_EQ_ISR_BUDGET 10 28 #define IONIC_EQ_WORK_BUDGET 1000 29 #define IONIC_MAX_RD_ATOM 16 30 #define IONIC_PKEY_TBL_LEN 1 31 #define IONIC_GID_TBL_LEN 256 32 33 #define IONIC_MAX_QPID 0xffffff 34 #define IONIC_SPEC_HIGH 8 35 #define IONIC_MAX_PD 1024 36 #define IONIC_SQCMB_ORDER 5 37 #define IONIC_RQCMB_ORDER 0 38 39 #define IONIC_META_LAST ((void *)1ul) 40 #define IONIC_META_POSTED ((void *)2ul) 41 42 #define IONIC_CQ_GRACE 100 43 44 #define IONIC_ROCE_UDP_SPORT 28272 45 #define IONIC_DMA_LKEY 0 46 #define IONIC_DMA_RKEY IONIC_DMA_LKEY 47 48 #define IONIC_CMB_SUPPORTED \ 49 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ 50 IONIC_CMB_WC | IONIC_CMB_UC) 51 52 /* resource is not reserved on the device, indicated in tbl_order */ 53 #define IONIC_RES_INVALID -1 54 55 struct ionic_aq; 56 struct ionic_cq; 57 struct ionic_eq; 58 struct ionic_vcq; 59 60 enum ionic_admin_state { 61 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ 62 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ 63 IONIC_ADMIN_KILLED, /* not submitting, locally completed */ 64 }; 65 66 enum ionic_admin_flags { 67 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ 68 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ 69 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */ 70 }; 71 72 enum ionic_mmap_flag { 73 IONIC_MMAP_WC = BIT(0), 74 IONIC_MMAP_PHC = BIT(1), 75 }; 76 77 struct ionic_mmap_entry { 78 struct rdma_user_mmap_entry rdma_entry; 79 unsigned long size; 80 unsigned long pfn; 81 u8 mmap_flags; 82 }; 83 84 struct ionic_ibdev { 85 struct ib_device ibdev; 86 87 struct ionic_lif_cfg lif_cfg; 88 89 struct xarray qp_tbl; 90 struct xarray cq_tbl; 91 92 struct ionic_resid_bits inuse_dbid; 93 struct ionic_resid_bits inuse_pdid; 94 struct ionic_resid_bits inuse_ahid; 95 struct ionic_resid_bits inuse_mrid; 96 struct ionic_resid_bits inuse_qpid; 97 struct ionic_resid_bits inuse_cqid; 98 99 u8 half_cqid_udma_shift; 100 u8 half_qpid_udma_shift; 101 u8 next_qpid_udma_idx; 102 u8 next_mrkey; 103 104 struct work_struct reset_work; 105 bool reset_posted; 106 u32 reset_cnt; 107 108 struct delayed_work admin_dwork; 109 struct ionic_aq **aq_vec; 110 atomic_t admin_state; 111 112 struct ionic_eq **eq_vec; 113 114 struct ionic_v1_stat *hw_stats; 115 void *hw_stats_buf; 116 struct rdma_stat_desc *hw_stats_hdrs; 117 struct ionic_counter_stats *counter_stats; 118 int hw_stats_count; 119 }; 120 121 struct ionic_eq { 122 struct ionic_ibdev *dev; 123 124 u32 eqid; 125 u32 intr; 126 127 struct ionic_queue q; 128 129 int armed; 130 bool enable; 131 132 struct work_struct work; 133 134 int irq; 135 char name[32]; 136 }; 137 138 struct ionic_admin_wr { 139 struct completion work; 140 struct list_head aq_ent; 141 struct ionic_v1_admin_wqe wqe; 142 struct ionic_v1_cqe cqe; 143 struct ionic_aq *aq; 144 int status; 145 }; 146 147 struct ionic_admin_wr_q { 148 struct ionic_admin_wr *wr; 149 int wqe_strides; 150 }; 151 152 struct ionic_aq { 153 struct ionic_ibdev *dev; 154 struct ionic_vcq *vcq; 155 156 struct work_struct work; 157 158 atomic_t admin_state; 159 unsigned long stamp; 160 bool armed; 161 162 u32 aqid; 163 u32 cqid; 164 165 spinlock_t lock; /* for posting */ 166 struct ionic_queue q; 167 struct ionic_admin_wr_q *q_wr; 168 struct list_head wr_prod; 169 struct list_head wr_post; 170 }; 171 172 struct ionic_ctx { 173 struct ib_ucontext ibctx; 174 u32 dbid; 175 struct rdma_user_mmap_entry *mmap_dbell; 176 struct rdma_user_mmap_entry *mmap_phc; 177 }; 178 179 struct ionic_tbl_buf { 180 u32 tbl_limit; 181 u32 tbl_pages; 182 size_t tbl_size; 183 __le64 *tbl_buf; 184 dma_addr_t tbl_dma; 185 u8 page_size_log2; 186 }; 187 188 struct ionic_pd { 189 struct ib_pd ibpd; 190 191 u32 pdid; 192 u32 flags; 193 }; 194 195 struct ionic_cq { 196 struct ionic_vcq *vcq; 197 198 u32 cqid; 199 u32 eqid; 200 201 spinlock_t lock; /* for polling */ 202 struct list_head poll_sq; 203 bool flush; 204 struct list_head flush_sq; 205 struct list_head flush_rq; 206 struct list_head ibkill_flush_ent; 207 208 struct ionic_queue q; 209 bool color; 210 int credit; 211 u16 arm_any_prod; 212 u16 arm_sol_prod; 213 214 struct kref cq_kref; 215 struct completion cq_rel_comp; 216 217 /* infrequently accessed, keep at end */ 218 struct ib_umem *umem; 219 }; 220 221 struct ionic_vcq { 222 struct ib_cq ibcq; 223 struct ionic_cq cq[2]; 224 u8 udma_mask; 225 u8 poll_idx; 226 }; 227 228 struct ionic_sq_meta { 229 u64 wrid; 230 u32 len; 231 u16 seq; 232 u8 ibop; 233 u8 ibsts; 234 u8 remote:1; 235 u8 signal:1; 236 u8 local_comp:1; 237 }; 238 239 struct ionic_rq_meta { 240 struct ionic_rq_meta *next; 241 u64 wrid; 242 }; 243 244 struct ionic_qp { 245 struct ib_qp ibqp; 246 enum ib_qp_state state; 247 248 u32 qpid; 249 u32 ahid; 250 u32 sq_cqid; 251 u32 rq_cqid; 252 u8 udma_idx; 253 u8 has_ah:1; 254 u8 has_sq:1; 255 u8 has_rq:1; 256 u8 sig_all:1; 257 258 struct list_head qp_list_counter; 259 260 struct list_head cq_poll_sq; 261 struct list_head cq_flush_sq; 262 struct list_head cq_flush_rq; 263 struct list_head ibkill_flush_ent; 264 265 spinlock_t sq_lock; /* for posting and polling */ 266 struct ionic_queue sq; 267 struct ionic_sq_meta *sq_meta; 268 u16 *sq_msn_idx; 269 int sq_spec; 270 u16 sq_old_prod; 271 u16 sq_msn_prod; 272 u16 sq_msn_cons; 273 u8 sq_cmb; 274 bool sq_flush; 275 bool sq_flush_rcvd; 276 277 spinlock_t rq_lock; /* for posting and polling */ 278 struct ionic_queue rq; 279 struct ionic_rq_meta *rq_meta; 280 struct ionic_rq_meta *rq_meta_head; 281 int rq_spec; 282 u16 rq_old_prod; 283 u8 rq_cmb; 284 bool rq_flush; 285 286 struct kref qp_kref; 287 struct completion qp_rel_comp; 288 289 /* infrequently accessed, keep at end */ 290 int sgid_index; 291 int sq_cmb_order; 292 u32 sq_cmb_pgid; 293 phys_addr_t sq_cmb_addr; 294 struct rdma_user_mmap_entry *mmap_sq_cmb; 295 296 struct ib_umem *sq_umem; 297 298 int rq_cmb_order; 299 u32 rq_cmb_pgid; 300 phys_addr_t rq_cmb_addr; 301 struct rdma_user_mmap_entry *mmap_rq_cmb; 302 303 struct ib_umem *rq_umem; 304 305 int dcqcn_profile; 306 307 struct ib_ud_header *hdr; 308 }; 309 310 struct ionic_ah { 311 struct ib_ah ibah; 312 u32 ahid; 313 int sgid_index; 314 struct ib_ud_header hdr; 315 }; 316 317 struct ionic_mr { 318 union { 319 struct ib_mr ibmr; 320 struct ib_mw ibmw; 321 }; 322 323 u32 mrid; 324 int flags; 325 326 struct ib_umem *umem; 327 struct ionic_tbl_buf buf; 328 bool created; 329 }; 330 331 struct ionic_counter_stats { 332 int queue_stats_count; 333 struct ionic_v1_stat *hdr; 334 struct rdma_stat_desc *stats_hdrs; 335 struct ida counter_ida; 336 }; 337 338 struct ionic_rdma_counter { 339 struct rdma_counter rdma_counter; 340 void *vals; 341 struct list_head qp_list; 342 }; 343 344 static inline struct ionic_rdma_counter * 345 to_ionic_rdma_counter(struct rdma_counter *counter) 346 { 347 return container_of(counter, struct ionic_rdma_counter, rdma_counter); 348 } 349 350 static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) 351 { 352 return container_of(ibdev, struct ionic_ibdev, ibdev); 353 } 354 355 static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx) 356 { 357 return container_of(ibctx, struct ionic_ctx, ibctx); 358 } 359 360 static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj) 361 { 362 if (!uobj) 363 return NULL; 364 365 if (!uobj->context) 366 return NULL; 367 368 return to_ionic_ctx(uobj->context); 369 } 370 371 static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd) 372 { 373 return container_of(ibpd, struct ionic_pd, ibpd); 374 } 375 376 static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr) 377 { 378 return container_of(ibmr, struct ionic_mr, ibmr); 379 } 380 381 static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw) 382 { 383 return container_of(ibmw, struct ionic_mr, ibmw); 384 } 385 386 static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq) 387 { 388 return container_of(ibcq, struct ionic_vcq, ibcq); 389 } 390 391 static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, 392 uint8_t udma_idx) 393 { 394 return &to_ionic_vcq(ibcq)->cq[udma_idx]; 395 } 396 397 static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) 398 { 399 return container_of(ibqp, struct ionic_qp, ibqp); 400 } 401 402 static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) 403 { 404 return container_of(ibah, struct ionic_ah, ibah); 405 } 406 407 static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, 408 struct ionic_ctx *ctx) 409 { 410 if (!ctx) 411 return dev->lif_cfg.dbid; 412 413 return ctx->dbid; 414 } 415 416 static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, 417 struct ib_uobject *uobj) 418 { 419 return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj)); 420 } 421 422 static inline bool ionic_ibop_is_local(enum ib_wr_opcode op) 423 { 424 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR; 425 } 426 427 static inline void ionic_qp_complete(struct kref *kref) 428 { 429 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); 430 431 complete(&qp->qp_rel_comp); 432 } 433 434 static inline void ionic_cq_complete(struct kref *kref) 435 { 436 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); 437 438 complete(&cq->cq_rel_comp); 439 } 440 441 /* ionic_admin.c */ 442 extern struct workqueue_struct *ionic_evt_workq; 443 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); 444 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, 445 enum ionic_admin_flags); 446 447 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); 448 449 int ionic_create_rdma_admin(struct ionic_ibdev *dev); 450 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev); 451 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); 452 453 /* ionic_controlpath.c */ 454 int ionic_create_cq_common(struct ionic_vcq *vcq, 455 struct ionic_tbl_buf *buf, 456 const struct ib_cq_init_attr *attr, 457 struct ionic_ctx *ctx, 458 struct ib_udata *udata, 459 struct ionic_qdesc *req_cq, 460 __u32 *resp_cqid, 461 int udma_idx); 462 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); 463 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); 464 void ionic_notify_flush_cq(struct ionic_cq *cq); 465 466 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata); 467 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx); 468 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma); 469 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry); 470 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 471 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 472 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 473 struct ib_udata *udata); 474 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); 475 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags); 476 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access); 477 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 478 u64 addr, int access, struct ib_dmah *dmah, 479 struct ib_udata *udata); 480 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 481 u64 length, u64 addr, int fd, int access, 482 struct ib_dmah *dmah, 483 struct uverbs_attr_bundle *attrs); 484 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); 485 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 486 u32 max_sg); 487 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 488 unsigned int *sg_offset); 489 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); 490 int ionic_dealloc_mw(struct ib_mw *ibmw); 491 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 492 struct uverbs_attr_bundle *attrs); 493 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 494 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 495 struct ib_udata *udata); 496 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 497 struct ib_udata *udata); 498 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 499 struct ib_qp_init_attr *init_attr); 500 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 501 502 /* ionic_datapath.c */ 503 int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 504 const struct ib_send_wr **bad); 505 int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 506 const struct ib_recv_wr **bad); 507 int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc); 508 int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 509 510 /* ionic_hw_stats.c */ 511 void ionic_stats_init(struct ionic_ibdev *dev); 512 void ionic_stats_cleanup(struct ionic_ibdev *dev); 513 514 /* ionic_pgtbl.c */ 515 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va); 516 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va); 517 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma); 518 int ionic_pgtbl_init(struct ionic_ibdev *dev, 519 struct ionic_tbl_buf *buf, 520 struct ib_umem *umem, 521 dma_addr_t dma, 522 int limit, 523 u64 page_size); 524 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf); 525 #endif /* _IONIC_IBDEV_H_ */ 526