1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #ifndef _IONIC_IBDEV_H_ 5 #define _IONIC_IBDEV_H_ 6 7 #include <rdma/ib_umem.h> 8 #include <rdma/ib_verbs.h> 9 #include <rdma/ib_pack.h> 10 #include <rdma/uverbs_ioctl.h> 11 12 #include <rdma/ionic-abi.h> 13 #include <ionic_api.h> 14 #include <ionic_regs.h> 15 16 #include "ionic_fw.h" 17 #include "ionic_queue.h" 18 #include "ionic_res.h" 19 20 #include "ionic_lif_cfg.h" 21 22 /* Config knobs */ 23 #define IONIC_EQ_DEPTH 511 24 #define IONIC_EQ_COUNT 32 25 #define IONIC_AQ_DEPTH 63 26 #define IONIC_AQ_COUNT 4 27 #define IONIC_EQ_ISR_BUDGET 10 28 #define IONIC_EQ_WORK_BUDGET 1000 29 #define IONIC_MAX_RD_ATOM 16 30 #define IONIC_PKEY_TBL_LEN 1 31 #define IONIC_GID_TBL_LEN 256 32 33 #define IONIC_MAX_QPID 0xffffff 34 #define IONIC_SPEC_HIGH 8 35 #define IONIC_MAX_PD 1024 36 #define IONIC_SPEC_HIGH 8 37 #define IONIC_SQCMB_ORDER 5 38 #define IONIC_RQCMB_ORDER 0 39 40 #define IONIC_META_LAST ((void *)1ul) 41 #define IONIC_META_POSTED ((void *)2ul) 42 43 #define IONIC_CQ_GRACE 100 44 45 #define IONIC_ROCE_UDP_SPORT 28272 46 #define IONIC_DMA_LKEY 0 47 #define IONIC_DMA_RKEY IONIC_DMA_LKEY 48 49 #define IONIC_CMB_SUPPORTED \ 50 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ 51 IONIC_CMB_WC | IONIC_CMB_UC) 52 53 /* resource is not reserved on the device, indicated in tbl_order */ 54 #define IONIC_RES_INVALID -1 55 56 struct ionic_aq; 57 struct ionic_cq; 58 struct ionic_eq; 59 struct ionic_vcq; 60 61 enum ionic_admin_state { 62 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ 63 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ 64 IONIC_ADMIN_KILLED, /* not submitting, locally completed */ 65 }; 66 67 enum ionic_admin_flags { 68 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ 69 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ 70 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */ 71 }; 72 73 enum ionic_mmap_flag { 74 IONIC_MMAP_WC = BIT(0), 75 }; 76 77 struct ionic_mmap_entry { 78 struct rdma_user_mmap_entry rdma_entry; 79 unsigned long size; 80 unsigned long pfn; 81 u8 mmap_flags; 82 }; 83 84 struct ionic_ibdev { 85 struct ib_device ibdev; 86 87 struct ionic_lif_cfg lif_cfg; 88 89 struct xarray qp_tbl; 90 struct xarray cq_tbl; 91 92 struct ionic_resid_bits inuse_dbid; 93 struct ionic_resid_bits inuse_pdid; 94 struct ionic_resid_bits inuse_ahid; 95 struct ionic_resid_bits inuse_mrid; 96 struct ionic_resid_bits inuse_qpid; 97 struct ionic_resid_bits inuse_cqid; 98 99 u8 half_cqid_udma_shift; 100 u8 half_qpid_udma_shift; 101 u8 next_qpid_udma_idx; 102 u8 next_mrkey; 103 104 struct work_struct reset_work; 105 bool reset_posted; 106 u32 reset_cnt; 107 108 struct delayed_work admin_dwork; 109 struct ionic_aq **aq_vec; 110 atomic_t admin_state; 111 112 struct ionic_eq **eq_vec; 113 114 struct ionic_v1_stat *hw_stats; 115 void *hw_stats_buf; 116 struct rdma_stat_desc *hw_stats_hdrs; 117 struct ionic_counter_stats *counter_stats; 118 int hw_stats_count; 119 }; 120 121 struct ionic_eq { 122 struct ionic_ibdev *dev; 123 124 u32 eqid; 125 u32 intr; 126 127 struct ionic_queue q; 128 129 int armed; 130 bool enable; 131 132 struct work_struct work; 133 134 int irq; 135 char name[32]; 136 }; 137 138 struct ionic_admin_wr { 139 struct completion work; 140 struct list_head aq_ent; 141 struct ionic_v1_admin_wqe wqe; 142 struct ionic_v1_cqe cqe; 143 struct ionic_aq *aq; 144 int status; 145 }; 146 147 struct ionic_admin_wr_q { 148 struct ionic_admin_wr *wr; 149 int wqe_strides; 150 }; 151 152 struct ionic_aq { 153 struct ionic_ibdev *dev; 154 struct ionic_vcq *vcq; 155 156 struct work_struct work; 157 158 atomic_t admin_state; 159 unsigned long stamp; 160 bool armed; 161 162 u32 aqid; 163 u32 cqid; 164 165 spinlock_t lock; /* for posting */ 166 struct ionic_queue q; 167 struct ionic_admin_wr_q *q_wr; 168 struct list_head wr_prod; 169 struct list_head wr_post; 170 }; 171 172 struct ionic_ctx { 173 struct ib_ucontext ibctx; 174 u32 dbid; 175 struct rdma_user_mmap_entry *mmap_dbell; 176 }; 177 178 struct ionic_tbl_buf { 179 u32 tbl_limit; 180 u32 tbl_pages; 181 size_t tbl_size; 182 __le64 *tbl_buf; 183 dma_addr_t tbl_dma; 184 u8 page_size_log2; 185 }; 186 187 struct ionic_pd { 188 struct ib_pd ibpd; 189 190 u32 pdid; 191 u32 flags; 192 }; 193 194 struct ionic_cq { 195 struct ionic_vcq *vcq; 196 197 u32 cqid; 198 u32 eqid; 199 200 spinlock_t lock; /* for polling */ 201 struct list_head poll_sq; 202 bool flush; 203 struct list_head flush_sq; 204 struct list_head flush_rq; 205 struct list_head ibkill_flush_ent; 206 207 struct ionic_queue q; 208 bool color; 209 int credit; 210 u16 arm_any_prod; 211 u16 arm_sol_prod; 212 213 struct kref cq_kref; 214 struct completion cq_rel_comp; 215 216 /* infrequently accessed, keep at end */ 217 struct ib_umem *umem; 218 }; 219 220 struct ionic_vcq { 221 struct ib_cq ibcq; 222 struct ionic_cq cq[2]; 223 u8 udma_mask; 224 u8 poll_idx; 225 }; 226 227 struct ionic_sq_meta { 228 u64 wrid; 229 u32 len; 230 u16 seq; 231 u8 ibop; 232 u8 ibsts; 233 u8 remote:1; 234 u8 signal:1; 235 u8 local_comp:1; 236 }; 237 238 struct ionic_rq_meta { 239 struct ionic_rq_meta *next; 240 u64 wrid; 241 }; 242 243 struct ionic_qp { 244 struct ib_qp ibqp; 245 enum ib_qp_state state; 246 247 u32 qpid; 248 u32 ahid; 249 u32 sq_cqid; 250 u32 rq_cqid; 251 u8 udma_idx; 252 u8 has_ah:1; 253 u8 has_sq:1; 254 u8 has_rq:1; 255 u8 sig_all:1; 256 257 struct list_head qp_list_counter; 258 259 struct list_head cq_poll_sq; 260 struct list_head cq_flush_sq; 261 struct list_head cq_flush_rq; 262 struct list_head ibkill_flush_ent; 263 264 spinlock_t sq_lock; /* for posting and polling */ 265 struct ionic_queue sq; 266 struct ionic_sq_meta *sq_meta; 267 u16 *sq_msn_idx; 268 int sq_spec; 269 u16 sq_old_prod; 270 u16 sq_msn_prod; 271 u16 sq_msn_cons; 272 u8 sq_cmb; 273 bool sq_flush; 274 bool sq_flush_rcvd; 275 276 spinlock_t rq_lock; /* for posting and polling */ 277 struct ionic_queue rq; 278 struct ionic_rq_meta *rq_meta; 279 struct ionic_rq_meta *rq_meta_head; 280 int rq_spec; 281 u16 rq_old_prod; 282 u8 rq_cmb; 283 bool rq_flush; 284 285 struct kref qp_kref; 286 struct completion qp_rel_comp; 287 288 /* infrequently accessed, keep at end */ 289 int sgid_index; 290 int sq_cmb_order; 291 u32 sq_cmb_pgid; 292 phys_addr_t sq_cmb_addr; 293 struct rdma_user_mmap_entry *mmap_sq_cmb; 294 295 struct ib_umem *sq_umem; 296 297 int rq_cmb_order; 298 u32 rq_cmb_pgid; 299 phys_addr_t rq_cmb_addr; 300 struct rdma_user_mmap_entry *mmap_rq_cmb; 301 302 struct ib_umem *rq_umem; 303 304 int dcqcn_profile; 305 306 struct ib_ud_header *hdr; 307 }; 308 309 struct ionic_ah { 310 struct ib_ah ibah; 311 u32 ahid; 312 int sgid_index; 313 struct ib_ud_header hdr; 314 }; 315 316 struct ionic_mr { 317 union { 318 struct ib_mr ibmr; 319 struct ib_mw ibmw; 320 }; 321 322 u32 mrid; 323 int flags; 324 325 struct ib_umem *umem; 326 struct ionic_tbl_buf buf; 327 bool created; 328 }; 329 330 struct ionic_counter_stats { 331 int queue_stats_count; 332 struct ionic_v1_stat *hdr; 333 struct rdma_stat_desc *stats_hdrs; 334 struct xarray xa_counters; 335 }; 336 337 struct ionic_counter { 338 void *vals; 339 struct list_head qp_list; 340 }; 341 342 static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) 343 { 344 return container_of(ibdev, struct ionic_ibdev, ibdev); 345 } 346 347 static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx) 348 { 349 return container_of(ibctx, struct ionic_ctx, ibctx); 350 } 351 352 static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj) 353 { 354 if (!uobj) 355 return NULL; 356 357 if (!uobj->context) 358 return NULL; 359 360 return to_ionic_ctx(uobj->context); 361 } 362 363 static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd) 364 { 365 return container_of(ibpd, struct ionic_pd, ibpd); 366 } 367 368 static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr) 369 { 370 return container_of(ibmr, struct ionic_mr, ibmr); 371 } 372 373 static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw) 374 { 375 return container_of(ibmw, struct ionic_mr, ibmw); 376 } 377 378 static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq) 379 { 380 return container_of(ibcq, struct ionic_vcq, ibcq); 381 } 382 383 static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, 384 uint8_t udma_idx) 385 { 386 return &to_ionic_vcq(ibcq)->cq[udma_idx]; 387 } 388 389 static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) 390 { 391 return container_of(ibqp, struct ionic_qp, ibqp); 392 } 393 394 static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) 395 { 396 return container_of(ibah, struct ionic_ah, ibah); 397 } 398 399 static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, 400 struct ionic_ctx *ctx) 401 { 402 if (!ctx) 403 return dev->lif_cfg.dbid; 404 405 return ctx->dbid; 406 } 407 408 static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, 409 struct ib_uobject *uobj) 410 { 411 return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj)); 412 } 413 414 static inline bool ionic_ibop_is_local(enum ib_wr_opcode op) 415 { 416 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR; 417 } 418 419 static inline void ionic_qp_complete(struct kref *kref) 420 { 421 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); 422 423 complete(&qp->qp_rel_comp); 424 } 425 426 static inline void ionic_cq_complete(struct kref *kref) 427 { 428 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); 429 430 complete(&cq->cq_rel_comp); 431 } 432 433 /* ionic_admin.c */ 434 extern struct workqueue_struct *ionic_evt_workq; 435 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); 436 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, 437 enum ionic_admin_flags); 438 439 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); 440 441 int ionic_create_rdma_admin(struct ionic_ibdev *dev); 442 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev); 443 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); 444 445 /* ionic_controlpath.c */ 446 int ionic_create_cq_common(struct ionic_vcq *vcq, 447 struct ionic_tbl_buf *buf, 448 const struct ib_cq_init_attr *attr, 449 struct ionic_ctx *ctx, 450 struct ib_udata *udata, 451 struct ionic_qdesc *req_cq, 452 __u32 *resp_cqid, 453 int udma_idx); 454 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); 455 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); 456 void ionic_notify_flush_cq(struct ionic_cq *cq); 457 458 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata); 459 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx); 460 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma); 461 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry); 462 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 463 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 464 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 465 struct ib_udata *udata); 466 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); 467 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags); 468 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access); 469 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, 470 u64 addr, int access, struct ib_dmah *dmah, 471 struct ib_udata *udata); 472 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, 473 u64 length, u64 addr, int fd, int access, 474 struct ib_dmah *dmah, 475 struct uverbs_attr_bundle *attrs); 476 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); 477 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, 478 u32 max_sg); 479 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 480 unsigned int *sg_offset); 481 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); 482 int ionic_dealloc_mw(struct ib_mw *ibmw); 483 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 484 struct uverbs_attr_bundle *attrs); 485 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 486 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 487 struct ib_udata *udata); 488 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 489 struct ib_udata *udata); 490 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 491 struct ib_qp_init_attr *init_attr); 492 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 493 494 /* ionic_datapath.c */ 495 int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 496 const struct ib_send_wr **bad); 497 int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 498 const struct ib_recv_wr **bad); 499 int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc); 500 int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 501 502 /* ionic_hw_stats.c */ 503 void ionic_stats_init(struct ionic_ibdev *dev); 504 void ionic_stats_cleanup(struct ionic_ibdev *dev); 505 506 /* ionic_pgtbl.c */ 507 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va); 508 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va); 509 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma); 510 int ionic_pgtbl_init(struct ionic_ibdev *dev, 511 struct ionic_tbl_buf *buf, 512 struct ib_umem *umem, 513 dma_addr_t dma, 514 int limit, 515 u64 page_size); 516 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf); 517 #endif /* _IONIC_IBDEV_H_ */ 518