1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2022 Microsoft Corporation. All rights reserved. 4 */ 5 6 #ifndef _MANA_IB_H_ 7 #define _MANA_IB_H_ 8 9 #include <rdma/ib_verbs.h> 10 #include <rdma/ib_mad.h> 11 #include <rdma/ib_umem.h> 12 #include <rdma/mana-abi.h> 13 #include <rdma/uverbs_ioctl.h> 14 15 #include <net/mana/mana.h> 16 17 #define PAGE_SZ_BM \ 18 (SZ_4K | SZ_8K | SZ_16K | SZ_32K | SZ_64K | SZ_128K | SZ_256K | \ 19 SZ_512K | SZ_1M | SZ_2M) 20 21 /* MANA doesn't have any limit for MR size */ 22 #define MANA_IB_MAX_MR_SIZE U64_MAX 23 24 /* 25 * The hardware limit of number of MRs is greater than maximum number of MRs 26 * that can possibly represent in 24 bits 27 */ 28 #define MANA_IB_MAX_MR 0xFFFFFFu 29 30 /* 31 * The CA timeout is approx. 260ms (4us * 2^(DELAY)) 32 */ 33 #define MANA_CA_ACK_DELAY 16 34 35 struct mana_ib_adapter_caps { 36 u32 max_sq_id; 37 u32 max_rq_id; 38 u32 max_cq_id; 39 u32 max_qp_count; 40 u32 max_cq_count; 41 u32 max_mr_count; 42 u32 max_pd_count; 43 u32 max_inbound_read_limit; 44 u32 max_outbound_read_limit; 45 u32 mw_count; 46 u32 max_srq_count; 47 u32 max_qp_wr; 48 u32 max_send_sge_count; 49 u32 max_recv_sge_count; 50 u32 max_inline_data_size; 51 }; 52 53 struct mana_ib_queue { 54 struct ib_umem *umem; 55 u64 gdma_region; 56 u64 id; 57 }; 58 59 struct mana_ib_dev { 60 struct ib_device ib_dev; 61 struct gdma_dev *gdma_dev; 62 mana_handle_t adapter_handle; 63 struct gdma_queue *fatal_err_eq; 64 struct gdma_queue **eqs; 65 struct xarray qp_table_wq; 66 struct mana_ib_adapter_caps adapter_caps; 67 }; 68 69 struct mana_ib_wq { 70 struct ib_wq ibwq; 71 struct mana_ib_queue queue; 72 int wqe; 73 u32 wq_buf_size; 74 mana_handle_t rx_object; 75 }; 76 77 struct mana_ib_pd { 78 struct ib_pd ibpd; 79 u32 pdn; 80 mana_handle_t pd_handle; 81 82 /* Mutex for sharing access to vport_use_count */ 83 struct mutex vport_mutex; 84 int vport_use_count; 85 86 bool tx_shortform_allowed; 87 u32 tx_vp_offset; 88 }; 89 90 struct mana_ib_mr { 91 struct ib_mr ibmr; 92 struct ib_umem *umem; 93 mana_handle_t mr_handle; 94 }; 95 96 struct mana_ib_cq { 97 struct ib_cq ibcq; 98 struct mana_ib_queue queue; 99 int cqe; 100 u32 comp_vector; 101 mana_handle_t cq_handle; 102 }; 103 104 enum mana_rc_queue_type { 105 MANA_RC_SEND_QUEUE_REQUESTER = 0, 106 MANA_RC_SEND_QUEUE_RESPONDER, 107 MANA_RC_SEND_QUEUE_FMR, 108 MANA_RC_RECV_QUEUE_REQUESTER, 109 MANA_RC_RECV_QUEUE_RESPONDER, 110 MANA_RC_QUEUE_TYPE_MAX, 111 }; 112 113 struct mana_ib_rc_qp { 114 struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX]; 115 }; 116 117 struct mana_ib_qp { 118 struct ib_qp ibqp; 119 120 mana_handle_t qp_handle; 121 union { 122 struct mana_ib_queue raw_sq; 123 struct mana_ib_rc_qp rc_qp; 124 }; 125 126 /* The port on the IB device, starting with 1 */ 127 u32 port; 128 129 refcount_t refcount; 130 struct completion free; 131 }; 132 133 struct mana_ib_ucontext { 134 struct ib_ucontext ibucontext; 135 u32 doorbell; 136 }; 137 138 struct mana_ib_rwq_ind_table { 139 struct ib_rwq_ind_table ib_ind_table; 140 }; 141 142 enum mana_ib_command_code { 143 MANA_IB_GET_ADAPTER_CAP = 0x30001, 144 MANA_IB_CREATE_ADAPTER = 0x30002, 145 MANA_IB_DESTROY_ADAPTER = 0x30003, 146 MANA_IB_CONFIG_IP_ADDR = 0x30004, 147 MANA_IB_CONFIG_MAC_ADDR = 0x30005, 148 MANA_IB_CREATE_CQ = 0x30008, 149 MANA_IB_DESTROY_CQ = 0x30009, 150 MANA_IB_CREATE_RC_QP = 0x3000a, 151 MANA_IB_DESTROY_RC_QP = 0x3000b, 152 MANA_IB_SET_QP_STATE = 0x3000d, 153 }; 154 155 struct mana_ib_query_adapter_caps_req { 156 struct gdma_req_hdr hdr; 157 }; /*HW Data */ 158 159 struct mana_ib_query_adapter_caps_resp { 160 struct gdma_resp_hdr hdr; 161 u32 max_sq_id; 162 u32 max_rq_id; 163 u32 max_cq_id; 164 u32 max_qp_count; 165 u32 max_cq_count; 166 u32 max_mr_count; 167 u32 max_pd_count; 168 u32 max_inbound_read_limit; 169 u32 max_outbound_read_limit; 170 u32 mw_count; 171 u32 max_srq_count; 172 u32 max_requester_sq_size; 173 u32 max_responder_sq_size; 174 u32 max_requester_rq_size; 175 u32 max_responder_rq_size; 176 u32 max_send_sge_count; 177 u32 max_recv_sge_count; 178 u32 max_inline_data_size; 179 }; /* HW Data */ 180 181 struct mana_rnic_create_adapter_req { 182 struct gdma_req_hdr hdr; 183 u32 notify_eq_id; 184 u32 reserved; 185 u64 feature_flags; 186 }; /*HW Data */ 187 188 struct mana_rnic_create_adapter_resp { 189 struct gdma_resp_hdr hdr; 190 mana_handle_t adapter; 191 }; /* HW Data */ 192 193 struct mana_rnic_destroy_adapter_req { 194 struct gdma_req_hdr hdr; 195 mana_handle_t adapter; 196 }; /*HW Data */ 197 198 struct mana_rnic_destroy_adapter_resp { 199 struct gdma_resp_hdr hdr; 200 }; /* HW Data */ 201 202 enum mana_ib_addr_op { 203 ADDR_OP_ADD = 1, 204 ADDR_OP_REMOVE = 2, 205 }; 206 207 enum sgid_entry_type { 208 SGID_TYPE_IPV4 = 1, 209 SGID_TYPE_IPV6 = 2, 210 }; 211 212 struct mana_rnic_config_addr_req { 213 struct gdma_req_hdr hdr; 214 mana_handle_t adapter; 215 enum mana_ib_addr_op op; 216 enum sgid_entry_type sgid_type; 217 u8 ip_addr[16]; 218 }; /* HW Data */ 219 220 struct mana_rnic_config_addr_resp { 221 struct gdma_resp_hdr hdr; 222 }; /* HW Data */ 223 224 struct mana_rnic_config_mac_addr_req { 225 struct gdma_req_hdr hdr; 226 mana_handle_t adapter; 227 enum mana_ib_addr_op op; 228 u8 mac_addr[ETH_ALEN]; 229 u8 reserved[6]; 230 }; /* HW Data */ 231 232 struct mana_rnic_config_mac_addr_resp { 233 struct gdma_resp_hdr hdr; 234 }; /* HW Data */ 235 236 struct mana_rnic_create_cq_req { 237 struct gdma_req_hdr hdr; 238 mana_handle_t adapter; 239 u64 gdma_region; 240 u32 eq_id; 241 u32 doorbell_page; 242 }; /* HW Data */ 243 244 struct mana_rnic_create_cq_resp { 245 struct gdma_resp_hdr hdr; 246 mana_handle_t cq_handle; 247 u32 cq_id; 248 u32 reserved; 249 }; /* HW Data */ 250 251 struct mana_rnic_destroy_cq_req { 252 struct gdma_req_hdr hdr; 253 mana_handle_t adapter; 254 mana_handle_t cq_handle; 255 }; /* HW Data */ 256 257 struct mana_rnic_destroy_cq_resp { 258 struct gdma_resp_hdr hdr; 259 }; /* HW Data */ 260 261 enum mana_rnic_create_rc_flags { 262 MANA_RC_FLAG_NO_FMR = 2, 263 }; 264 265 struct mana_rnic_create_qp_req { 266 struct gdma_req_hdr hdr; 267 mana_handle_t adapter; 268 mana_handle_t pd_handle; 269 mana_handle_t send_cq_handle; 270 mana_handle_t recv_cq_handle; 271 u64 dma_region[MANA_RC_QUEUE_TYPE_MAX]; 272 u64 deprecated[2]; 273 u64 flags; 274 u32 doorbell_page; 275 u32 max_send_wr; 276 u32 max_recv_wr; 277 u32 max_send_sge; 278 u32 max_recv_sge; 279 u32 reserved; 280 }; /* HW Data */ 281 282 struct mana_rnic_create_qp_resp { 283 struct gdma_resp_hdr hdr; 284 mana_handle_t rc_qp_handle; 285 u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX]; 286 u32 reserved; 287 }; /* HW Data*/ 288 289 struct mana_rnic_destroy_rc_qp_req { 290 struct gdma_req_hdr hdr; 291 mana_handle_t adapter; 292 mana_handle_t rc_qp_handle; 293 }; /* HW Data */ 294 295 struct mana_rnic_destroy_rc_qp_resp { 296 struct gdma_resp_hdr hdr; 297 }; /* HW Data */ 298 299 struct mana_ib_ah_attr { 300 u8 src_addr[16]; 301 u8 dest_addr[16]; 302 u8 src_mac[ETH_ALEN]; 303 u8 dest_mac[ETH_ALEN]; 304 u8 src_addr_type; 305 u8 dest_addr_type; 306 u8 hop_limit; 307 u8 traffic_class; 308 u16 src_port; 309 u16 dest_port; 310 u32 reserved; 311 }; 312 313 struct mana_rnic_set_qp_state_req { 314 struct gdma_req_hdr hdr; 315 mana_handle_t adapter; 316 mana_handle_t qp_handle; 317 u64 attr_mask; 318 u32 qp_state; 319 u32 path_mtu; 320 u32 rq_psn; 321 u32 sq_psn; 322 u32 dest_qpn; 323 u32 max_dest_rd_atomic; 324 u32 retry_cnt; 325 u32 rnr_retry; 326 u32 min_rnr_timer; 327 u32 reserved; 328 struct mana_ib_ah_attr ah_attr; 329 }; /* HW Data */ 330 331 struct mana_rnic_set_qp_state_resp { 332 struct gdma_resp_hdr hdr; 333 }; /* HW Data */ 334 335 static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev) 336 { 337 return mdev->gdma_dev->gdma_context; 338 } 339 340 static inline struct mana_ib_qp *mana_get_qp_ref(struct mana_ib_dev *mdev, 341 uint32_t qid) 342 { 343 struct mana_ib_qp *qp; 344 unsigned long flag; 345 346 xa_lock_irqsave(&mdev->qp_table_wq, flag); 347 qp = xa_load(&mdev->qp_table_wq, qid); 348 if (qp) 349 refcount_inc(&qp->refcount); 350 xa_unlock_irqrestore(&mdev->qp_table_wq, flag); 351 return qp; 352 } 353 354 static inline void mana_put_qp_ref(struct mana_ib_qp *qp) 355 { 356 if (refcount_dec_and_test(&qp->refcount)) 357 complete(&qp->free); 358 } 359 360 static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibdev, u32 port) 361 { 362 struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 363 struct gdma_context *gc = mdev_to_gc(mdev); 364 struct mana_context *mc = gc->mana.driver_data; 365 366 if (port < 1 || port > mc->num_ports) 367 return NULL; 368 return mc->ports[port - 1]; 369 } 370 371 static inline void copy_in_reverse(u8 *dst, const u8 *src, u32 size) 372 { 373 u32 i; 374 375 for (i = 0; i < size; i++) 376 dst[size - 1 - i] = src[i]; 377 } 378 379 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); 380 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); 381 382 int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 383 mana_handle_t *gdma_region); 384 385 int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 386 mana_handle_t *gdma_region, u64 virt); 387 388 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, 389 mana_handle_t gdma_region); 390 391 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, 392 struct mana_ib_queue *queue); 393 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); 394 395 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, 396 struct ib_wq_init_attr *init_attr, 397 struct ib_udata *udata); 398 399 int mana_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, 400 u32 wq_attr_mask, struct ib_udata *udata); 401 402 int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata); 403 404 int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, 405 struct ib_rwq_ind_table_init_attr *init_attr, 406 struct ib_udata *udata); 407 408 int mana_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl); 409 410 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags); 411 412 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 413 u64 iova, int access_flags, 414 struct ib_udata *udata); 415 416 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); 417 418 int mana_ib_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr, 419 struct ib_udata *udata); 420 421 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 422 int attr_mask, struct ib_udata *udata); 423 424 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 425 426 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port_id, 427 struct mana_ib_pd *pd, u32 doorbell_id); 428 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, 429 u32 port); 430 431 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 432 struct uverbs_attr_bundle *attrs); 433 434 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 435 436 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 437 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 438 439 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, 440 struct ib_udata *udata); 441 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext); 442 443 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma); 444 445 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, 446 struct ib_port_immutable *immutable); 447 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 448 struct ib_udata *uhw); 449 int mana_ib_query_port(struct ib_device *ibdev, u32 port, 450 struct ib_port_attr *props); 451 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index, 452 union ib_gid *gid); 453 454 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext); 455 456 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); 457 458 int mana_ib_create_eqs(struct mana_ib_dev *mdev); 459 460 void mana_ib_destroy_eqs(struct mana_ib_dev *mdev); 461 462 int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev); 463 464 int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev); 465 466 int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); 467 468 enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num); 469 470 int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context); 471 472 int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context); 473 474 int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac); 475 476 int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell); 477 478 int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq); 479 480 int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, 481 struct ib_qp_init_attr *attr, u32 doorbell, u64 flags); 482 int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp); 483 #endif 484