1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2022 Microsoft Corporation. All rights reserved. 4 */ 5 6 #ifndef _MANA_IB_H_ 7 #define _MANA_IB_H_ 8 9 #include <rdma/ib_verbs.h> 10 #include <rdma/ib_mad.h> 11 #include <rdma/ib_umem.h> 12 #include <rdma/mana-abi.h> 13 #include <rdma/uverbs_ioctl.h> 14 #include <linux/dmapool.h> 15 16 #include <net/mana/mana.h> 17 #include "shadow_queue.h" 18 #include "counters.h" 19 20 #define PAGE_SZ_BM \ 21 (SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K | \ 22 SZ_512K | SZ_1M | SZ_2M) 23 24 /* MANA doesn't have any limit for MR size */ 25 #define MANA_IB_MAX_MR_SIZE U64_MAX 26 27 /* Send queue ID mask */ 28 #define MANA_SENDQ_MASK BIT(31) 29 30 /* 31 * The hardware limit of number of MRs is greater than maximum number of MRs 32 * that can possibly represent in 24 bits 33 */ 34 #define MANA_IB_MAX_MR 0xFFFFFFu 35 36 /* 37 * The CA timeout is approx. 260ms (4us * 2^(DELAY)) 38 */ 39 #define MANA_CA_ACK_DELAY 16 40 41 /* 42 * The buffer used for writing AV 43 */ 44 #define MANA_AV_BUFFER_SIZE 64 45 46 struct mana_ib_adapter_caps { 47 u32 max_sq_id; 48 u32 max_rq_id; 49 u32 max_cq_id; 50 u32 max_qp_count; 51 u32 max_cq_count; 52 u32 max_mr_count; 53 u32 max_pd_count; 54 u32 max_inbound_read_limit; 55 u32 max_outbound_read_limit; 56 u32 mw_count; 57 u32 max_srq_count; 58 u32 max_qp_wr; 59 u32 max_send_sge_count; 60 u32 max_recv_sge_count; 61 u32 max_inline_data_size; 62 u64 feature_flags; 63 }; 64 65 struct mana_ib_queue { 66 struct ib_umem *umem; 67 struct gdma_queue *kmem; 68 u64 gdma_region; 69 u64 id; 70 }; 71 72 struct mana_ib_dev { 73 struct ib_device ib_dev; 74 struct gdma_dev *gdma_dev; 75 mana_handle_t adapter_handle; 76 struct gdma_queue *fatal_err_eq; 77 struct gdma_queue **eqs; 78 struct xarray qp_table_wq; 79 struct mana_ib_adapter_caps adapter_caps; 80 struct dma_pool *av_pool; 81 netdevice_tracker dev_tracker; 82 struct notifier_block nb; 83 }; 84 85 struct mana_ib_wq { 86 struct ib_wq ibwq; 87 struct mana_ib_queue queue; 88 int wqe; 89 u32 wq_buf_size; 90 mana_handle_t rx_object; 91 }; 92 93 struct mana_ib_pd { 94 struct ib_pd ibpd; 95 u32 pdn; 96 mana_handle_t pd_handle; 97 98 /* Mutex for sharing access to vport_use_count */ 99 struct mutex vport_mutex; 100 int vport_use_count; 101 102 bool tx_shortform_allowed; 103 u32 tx_vp_offset; 104 }; 105 106 struct mana_ib_av { 107 u8 dest_ip[16]; 108 u8 dest_mac[ETH_ALEN]; 109 u16 udp_src_port; 110 u8 src_ip[16]; 111 u32 hop_limit : 8; 112 u32 reserved1 : 12; 113 u32 dscp : 6; 114 u32 reserved2 : 5; 115 u32 is_ipv6 : 1; 116 u32 reserved3 : 32; 117 }; 118 119 struct mana_ib_ah { 120 struct ib_ah ibah; 121 struct mana_ib_av *av; 122 dma_addr_t dma_handle; 123 }; 124 125 struct mana_ib_mr { 126 struct ib_mr ibmr; 127 struct ib_umem *umem; 128 mana_handle_t mr_handle; 129 }; 130 131 struct mana_ib_cq { 132 struct ib_cq ibcq; 133 struct mana_ib_queue queue; 134 /* protects CQ polling */ 135 spinlock_t cq_lock; 136 struct list_head list_send_qp; 137 struct list_head list_recv_qp; 138 int cqe; 139 u32 comp_vector; 140 mana_handle_t cq_handle; 141 }; 142 143 enum mana_rc_queue_type { 144 MANA_RC_SEND_QUEUE_REQUESTER = 0, 145 MANA_RC_SEND_QUEUE_RESPONDER, 146 MANA_RC_SEND_QUEUE_FMR, 147 MANA_RC_RECV_QUEUE_REQUESTER, 148 MANA_RC_RECV_QUEUE_RESPONDER, 149 MANA_RC_QUEUE_TYPE_MAX, 150 }; 151 152 struct mana_ib_rc_qp { 153 struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX]; 154 }; 155 156 enum mana_ud_queue_type { 157 MANA_UD_SEND_QUEUE = 0, 158 MANA_UD_RECV_QUEUE, 159 MANA_UD_QUEUE_TYPE_MAX, 160 }; 161 162 struct mana_ib_ud_qp { 163 struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX]; 164 u32 sq_psn; 165 }; 166 167 struct mana_ib_qp { 168 struct ib_qp ibqp; 169 170 mana_handle_t qp_handle; 171 union { 172 struct mana_ib_queue raw_sq; 173 struct mana_ib_rc_qp rc_qp; 174 struct mana_ib_ud_qp ud_qp; 175 }; 176 177 /* The port on the IB device, starting with 1 */ 178 u32 port; 179 180 struct list_head cq_send_list; 181 struct list_head cq_recv_list; 182 struct shadow_queue shadow_rq; 183 struct shadow_queue shadow_sq; 184 185 refcount_t refcount; 186 struct completion free; 187 }; 188 189 struct mana_ib_ucontext { 190 struct ib_ucontext ibucontext; 191 u32 doorbell; 192 }; 193 194 struct mana_ib_rwq_ind_table { 195 struct ib_rwq_ind_table ib_ind_table; 196 }; 197 198 enum mana_ib_command_code { 199 MANA_IB_GET_ADAPTER_CAP = 0x30001, 200 MANA_IB_CREATE_ADAPTER = 0x30002, 201 MANA_IB_DESTROY_ADAPTER = 0x30003, 202 MANA_IB_CONFIG_IP_ADDR = 0x30004, 203 MANA_IB_CONFIG_MAC_ADDR = 0x30005, 204 MANA_IB_CREATE_UD_QP = 0x30006, 205 MANA_IB_DESTROY_UD_QP = 0x30007, 206 MANA_IB_CREATE_CQ = 0x30008, 207 MANA_IB_DESTROY_CQ = 0x30009, 208 MANA_IB_CREATE_RC_QP = 0x3000a, 209 MANA_IB_DESTROY_RC_QP = 0x3000b, 210 MANA_IB_SET_QP_STATE = 0x3000d, 211 MANA_IB_QUERY_VF_COUNTERS = 0x30022, 212 }; 213 214 struct mana_ib_query_adapter_caps_req { 215 struct gdma_req_hdr hdr; 216 }; /*HW Data */ 217 218 enum mana_ib_adapter_features { 219 MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4), 220 }; 221 222 struct mana_ib_query_adapter_caps_resp { 223 struct gdma_resp_hdr hdr; 224 u32 max_sq_id; 225 u32 max_rq_id; 226 u32 max_cq_id; 227 u32 max_qp_count; 228 u32 max_cq_count; 229 u32 max_mr_count; 230 u32 max_pd_count; 231 u32 max_inbound_read_limit; 232 u32 max_outbound_read_limit; 233 u32 mw_count; 234 u32 max_srq_count; 235 u32 max_requester_sq_size; 236 u32 max_responder_sq_size; 237 u32 max_requester_rq_size; 238 u32 max_responder_rq_size; 239 u32 max_send_sge_count; 240 u32 max_recv_sge_count; 241 u32 max_inline_data_size; 242 u64 feature_flags; 243 }; /* HW Data */ 244 245 enum mana_ib_adapter_features_request { 246 MANA_IB_FEATURE_CLIENT_ERROR_CQE_REQUEST = BIT(1), 247 }; /*HW Data */ 248 249 struct mana_rnic_create_adapter_req { 250 struct gdma_req_hdr hdr; 251 u32 notify_eq_id; 252 u32 reserved; 253 u64 feature_flags; 254 }; /*HW Data */ 255 256 struct mana_rnic_create_adapter_resp { 257 struct gdma_resp_hdr hdr; 258 mana_handle_t adapter; 259 }; /* HW Data */ 260 261 struct mana_rnic_destroy_adapter_req { 262 struct gdma_req_hdr hdr; 263 mana_handle_t adapter; 264 }; /*HW Data */ 265 266 struct mana_rnic_destroy_adapter_resp { 267 struct gdma_resp_hdr hdr; 268 }; /* HW Data */ 269 270 enum mana_ib_addr_op { 271 ADDR_OP_ADD = 1, 272 ADDR_OP_REMOVE = 2, 273 }; 274 275 enum sgid_entry_type { 276 SGID_TYPE_IPV4 = 1, 277 SGID_TYPE_IPV6 = 2, 278 }; 279 280 struct mana_rnic_config_addr_req { 281 struct gdma_req_hdr hdr; 282 mana_handle_t adapter; 283 enum mana_ib_addr_op op; 284 enum sgid_entry_type sgid_type; 285 u8 ip_addr[16]; 286 }; /* HW Data */ 287 288 struct mana_rnic_config_addr_resp { 289 struct gdma_resp_hdr hdr; 290 }; /* HW Data */ 291 292 struct mana_rnic_config_mac_addr_req { 293 struct gdma_req_hdr hdr; 294 mana_handle_t adapter; 295 enum mana_ib_addr_op op; 296 u8 mac_addr[ETH_ALEN]; 297 u8 reserved[6]; 298 }; /* HW Data */ 299 300 struct mana_rnic_config_mac_addr_resp { 301 struct gdma_resp_hdr hdr; 302 }; /* HW Data */ 303 304 struct mana_rnic_create_cq_req { 305 struct gdma_req_hdr hdr; 306 mana_handle_t adapter; 307 u64 gdma_region; 308 u32 eq_id; 309 u32 doorbell_page; 310 }; /* HW Data */ 311 312 struct mana_rnic_create_cq_resp { 313 struct gdma_resp_hdr hdr; 314 mana_handle_t cq_handle; 315 u32 cq_id; 316 u32 reserved; 317 }; /* HW Data */ 318 319 struct mana_rnic_destroy_cq_req { 320 struct gdma_req_hdr hdr; 321 mana_handle_t adapter; 322 mana_handle_t cq_handle; 323 }; /* HW Data */ 324 325 struct mana_rnic_destroy_cq_resp { 326 struct gdma_resp_hdr hdr; 327 }; /* HW Data */ 328 329 enum mana_rnic_create_rc_flags { 330 MANA_RC_FLAG_NO_FMR = 2, 331 }; 332 333 struct mana_rnic_create_qp_req { 334 struct gdma_req_hdr hdr; 335 mana_handle_t adapter; 336 mana_handle_t pd_handle; 337 mana_handle_t send_cq_handle; 338 mana_handle_t recv_cq_handle; 339 u64 dma_region[MANA_RC_QUEUE_TYPE_MAX]; 340 u64 deprecated[2]; 341 u64 flags; 342 u32 doorbell_page; 343 u32 max_send_wr; 344 u32 max_recv_wr; 345 u32 max_send_sge; 346 u32 max_recv_sge; 347 u32 reserved; 348 }; /* HW Data */ 349 350 struct mana_rnic_create_qp_resp { 351 struct gdma_resp_hdr hdr; 352 mana_handle_t rc_qp_handle; 353 u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX]; 354 u32 reserved; 355 }; /* HW Data*/ 356 357 struct mana_rnic_destroy_rc_qp_req { 358 struct gdma_req_hdr hdr; 359 mana_handle_t adapter; 360 mana_handle_t rc_qp_handle; 361 }; /* HW Data */ 362 363 struct mana_rnic_destroy_rc_qp_resp { 364 struct gdma_resp_hdr hdr; 365 }; /* HW Data */ 366 367 struct mana_rnic_create_udqp_req { 368 struct gdma_req_hdr hdr; 369 mana_handle_t adapter; 370 mana_handle_t pd_handle; 371 mana_handle_t send_cq_handle; 372 mana_handle_t recv_cq_handle; 373 u64 dma_region[MANA_UD_QUEUE_TYPE_MAX]; 374 u32 qp_type; 375 u32 doorbell_page; 376 u32 max_send_wr; 377 u32 max_recv_wr; 378 u32 max_send_sge; 379 u32 max_recv_sge; 380 }; /* HW Data */ 381 382 struct mana_rnic_create_udqp_resp { 383 struct gdma_resp_hdr hdr; 384 mana_handle_t qp_handle; 385 u32 queue_ids[MANA_UD_QUEUE_TYPE_MAX]; 386 }; /* HW Data*/ 387 388 struct mana_rnic_destroy_udqp_req { 389 struct gdma_req_hdr hdr; 390 mana_handle_t adapter; 391 mana_handle_t qp_handle; 392 }; /* HW Data */ 393 394 struct mana_rnic_destroy_udqp_resp { 395 struct gdma_resp_hdr hdr; 396 }; /* HW Data */ 397 398 struct mana_ib_ah_attr { 399 u8 src_addr[16]; 400 u8 dest_addr[16]; 401 u8 src_mac[ETH_ALEN]; 402 u8 dest_mac[ETH_ALEN]; 403 u8 src_addr_type; 404 u8 dest_addr_type; 405 u8 hop_limit; 406 u8 traffic_class; 407 u16 src_port; 408 u16 dest_port; 409 u32 reserved; 410 }; 411 412 struct mana_rnic_set_qp_state_req { 413 struct gdma_req_hdr hdr; 414 mana_handle_t adapter; 415 mana_handle_t qp_handle; 416 u64 attr_mask; 417 u32 qp_state; 418 u32 path_mtu; 419 u32 rq_psn; 420 u32 sq_psn; 421 u32 dest_qpn; 422 u32 max_dest_rd_atomic; 423 u32 retry_cnt; 424 u32 rnr_retry; 425 u32 min_rnr_timer; 426 u32 reserved; 427 struct mana_ib_ah_attr ah_attr; 428 }; /* HW Data */ 429 430 struct mana_rnic_set_qp_state_resp { 431 struct gdma_resp_hdr hdr; 432 }; /* HW Data */ 433 434 enum WQE_OPCODE_TYPES { 435 WQE_TYPE_UD_SEND = 0, 436 WQE_TYPE_UD_RECV = 8, 437 }; /* HW DATA */ 438 439 struct rdma_send_oob { 440 u32 wqe_type : 5; 441 u32 fence : 1; 442 u32 signaled : 1; 443 u32 solicited : 1; 444 u32 psn : 24; 445 446 u32 ssn_or_rqpn : 24; 447 u32 reserved1 : 8; 448 union { 449 struct { 450 u32 remote_qkey; 451 u32 immediate; 452 u32 reserved1; 453 u32 reserved2; 454 } ud_send; 455 }; 456 }; /* HW DATA */ 457 458 struct mana_rdma_cqe { 459 union { 460 struct { 461 u8 cqe_type; 462 u8 data[GDMA_COMP_DATA_SIZE - 1]; 463 }; 464 struct { 465 u32 cqe_type : 8; 466 u32 vendor_error : 9; 467 u32 reserved1 : 15; 468 u32 sge_offset : 5; 469 u32 tx_wqe_offset : 27; 470 } ud_send; 471 struct { 472 u32 cqe_type : 8; 473 u32 reserved1 : 24; 474 u32 msg_len; 475 u32 src_qpn : 24; 476 u32 reserved2 : 8; 477 u32 imm_data; 478 u32 rx_wqe_offset; 479 } ud_recv; 480 }; 481 }; /* HW DATA */ 482 483 struct mana_rnic_query_vf_cntrs_req { 484 struct gdma_req_hdr hdr; 485 mana_handle_t adapter; 486 }; /* HW Data */ 487 488 struct mana_rnic_query_vf_cntrs_resp { 489 struct gdma_resp_hdr hdr; 490 u64 requester_timeout; 491 u64 requester_oos_nak; 492 u64 requester_rnr_nak; 493 u64 responder_rnr_nak; 494 u64 responder_oos; 495 u64 responder_dup_request; 496 u64 requester_implicit_nak; 497 u64 requester_readresp_psn_mismatch; 498 u64 nak_inv_req; 499 u64 nak_access_err; 500 u64 nak_opp_err; 501 u64 nak_inv_read; 502 u64 responder_local_len_err; 503 u64 requestor_local_prot_err; 504 u64 responder_rem_access_err; 505 u64 responder_local_qp_err; 506 u64 responder_malformed_wqe; 507 u64 general_hw_err; 508 u64 requester_rnr_nak_retries_exceeded; 509 u64 requester_retries_exceeded; 510 u64 total_fatal_err; 511 u64 received_cnps; 512 u64 num_qps_congested; 513 u64 rate_inc_events; 514 u64 num_qps_recovered; 515 u64 current_rate; 516 }; /* HW Data */ 517 518 static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) 519 { 520 return mdev->gdma_dev->gdma_context; 521 } 522 523 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev, 524 u32 qid, bool is_sq) 525 { 526 struct mana_ib_qp *qp; 527 unsigned long flag; 528 529 if (is_sq) 530 qid |= MANA_SENDQ_MASK; 531 532 xa_lock_irqsave(&mdev->qp_table_wq, flag); 533 qp = xa_load(&mdev->qp_table_wq, qid); 534 if (qp) 535 refcount_inc(&qp->refcount); 536 xa_unlock_irqrestore(&mdev->qp_table_wq, flag); 537 return qp; 538 } 539 540 static inline void mana_put_qp_ref(struct mana_ib_qp *qp) 541 { 542 if (refcount_dec_and_test(&qp->refcount)) 543 complete(&qp->free); 544 } 545 546 static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port) 547 { 548 struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 549 struct gdma_context *gc = mdev_to_gc(mdev); 550 struct mana_context *mc = gc->mana.driver_data; 551 552 if (port < 1 || port > mc->num_ports) 553 return NULL; 554 return mc->ports[port - 1]; 555 } 556 557 static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size) 558 { 559 u32 i; 560 561 for (i = 0; i < size; i++) 562 dst[size - 1 - i] = src[i]; 563 } 564 565 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); 566 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); 567 568 int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 569 mana_handle_t *gdma_region); 570 571 int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 572 mana_handle_t *gdma_region, u64 virt); 573 574 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, 575 mana_handle_t gdma_region); 576 577 int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type, 578 struct mana_ib_queue *queue); 579 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, 580 struct mana_ib_queue *queue); 581 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); 582 583 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, 584 struct ib_wq_init_attr *init_attr, 585 struct ib_udata *udata); 586 587 int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, 588 u32 wq_attr_mask, struct ib_udata *udata); 589 590 int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata); 591 592 int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, 593 struct ib_rwq_ind_table_init_attr *init_attr, 594 struct ib_udata *udata); 595 596 int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl); 597 598 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags); 599 600 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 601 u64 iova, int access_flags, 602 struct ib_udata *udata); 603 604 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); 605 606 int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr, 607 struct ib_udata *udata); 608 609 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 610 int attr_mask, struct ib_udata *udata); 611 612 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 613 614 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id, 615 struct mana_ib_pd *pd, u32 doorbell_id); 616 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, 617 u32 port); 618 619 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 620 struct uverbs_attr_bundle *attrs); 621 622 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 623 624 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 625 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 626 627 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, 628 struct ib_udata *udata); 629 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext); 630 631 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma); 632 633 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, 634 struct ib_port_immutable *immutable); 635 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 636 struct ib_udata *uhw); 637 int mana_ib_query_port(struct ib_device *ibdev, u32 port, 638 struct ib_port_attr *props); 639 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index, 640 union ib_gid *gid); 641 642 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext); 643 644 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); 645 646 int mana_ib_create_eqs(struct mana_ib_dev *mdev); 647 648 void mana_ib_destroy_eqs(struct mana_ib_dev *mdev); 649 650 int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev); 651 652 int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev); 653 654 int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); 655 656 enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num); 657 658 int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context); 659 660 int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context); 661 662 int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac); 663 664 int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell); 665 666 int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); 667 668 int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, 669 struct ib_qp_init_attr *attr, u32 doorbell, u64 flags); 670 int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp); 671 672 int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, 673 struct ib_qp_init_attr *attr, u32 doorbell, u32 type); 674 int mana_ib_gd_destroy_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp); 675 676 int mana_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 677 struct ib_udata *udata); 678 int mana_ib_destroy_ah(struct ib_ah *ah, u32 flags); 679 680 int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 681 const struct ib_recv_wr **bad_wr); 682 int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 683 const struct ib_send_wr **bad_wr); 684 685 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); 686 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 687 688 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length, 689 u64 iova, int fd, int mr_access_flags, 690 struct uverbs_attr_bundle *attrs); 691 #endif 692