1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #ifndef __QEDR_H__ 33 #define __QEDR_H__ 34 35 #include <linux/pci.h> 36 #include <linux/xarray.h> 37 #include <rdma/ib_addr.h> 38 #include <linux/qed/qed_if.h> 39 #include <linux/qed/qed_chain.h> 40 #include <linux/qed/qed_rdma_if.h> 41 #include <linux/qed/qede_rdma.h> 42 #include <linux/qed/roce_common.h> 43 #include <linux/completion.h> 44 #include "qedr_hsi_rdma.h" 45 46 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA" 47 #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev) 48 #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP) 49 #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE) 50 51 #define DP_DEBUG(dev, module, fmt, ...) \ 52 pr_debug("(%s) " module ": " fmt, \ 53 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__) 54 55 #define QEDR_MSG_INIT "INIT" 56 #define QEDR_MSG_CQ " CQ" 57 #define QEDR_MSG_MR " MR" 58 #define QEDR_MSG_QP " QP" 59 #define QEDR_MSG_SRQ " SRQ" 60 #define QEDR_MSG_GSI " GSI" 61 #define QEDR_MSG_IWARP " IW" 62 63 #define QEDR_CQ_MAGIC_NUMBER (0x11223344) 64 65 #define FW_PAGE_SHIFT (12) 66 67 struct qedr_dev; 68 69 struct qedr_cnq { 70 struct qedr_dev *dev; 71 struct qed_chain pbl; 72 struct qed_sb_info *sb; 73 char name[32]; 74 u64 n_comp; 75 __le16 *hw_cons_ptr; 76 u8 index; 77 }; 78 79 #define QEDR_MAX_SGID 128 80 81 struct qedr_device_attr { 82 u32 vendor_id; 83 u32 vendor_part_id; 84 u32 hw_ver; 85 u64 fw_ver; 86 u64 node_guid; 87 u64 sys_image_guid; 88 u8 max_cnq; 89 u8 max_sge; 90 u16 max_inline; 91 u32 max_sqe; 92 u32 max_rqe; 93 u8 max_qp_resp_rd_atomic_resc; 94 u8 max_qp_req_rd_atomic_resc; 95 u64 max_dev_resp_rd_atomic_resc; 96 u32 max_cq; 97 u32 max_qp; 98 u32 max_mr; 99 u64 max_mr_size; 100 u32 max_cqe; 101 u32 max_mw; 102 u32 max_mr_mw_fmr_pbl; 103 u64 max_mr_mw_fmr_size; 104 u32 max_pd; 105 u32 max_ah; 106 u8 max_pkey; 107 u32 max_srq; 108 u32 max_srq_wr; 109 u8 max_srq_sge; 110 u8 max_stats_queues; 111 u32 dev_caps; 112 113 u64 page_size_caps; 114 u8 dev_ack_delay; 115 u32 reserved_lkey; 116 u32 bad_pkey_counter; 117 struct qed_rdma_events events; 118 }; 119 120 #define QEDR_ENET_STATE_BIT (0) 121 122 struct qedr_dev { 123 struct ib_device ibdev; 124 struct qed_dev *cdev; 125 struct pci_dev *pdev; 126 struct net_device *ndev; 127 128 enum ib_atomic_cap atomic_cap; 129 130 void *rdma_ctx; 131 struct qedr_device_attr attr; 132 133 const struct qed_rdma_ops *ops; 134 struct qed_int_info int_info; 135 136 struct qed_sb_info *sb_array; 137 struct qedr_cnq *cnq_array; 138 int num_cnq; 139 int sb_start; 140 141 void __iomem *db_addr; 142 u64 db_phys_addr; 143 u32 db_size; 144 u16 dpi; 145 146 union ib_gid *sgid_tbl; 147 148 /* Lock for sgid table */ 149 spinlock_t sgid_lock; 150 151 u64 guid; 152 153 u32 dp_module; 154 u8 dp_level; 155 u8 num_hwfns; 156 #define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1) 157 u8 affin_hwfn_idx; 158 u8 gsi_ll2_handle; 159 160 uint wq_multiplier; 161 u8 gsi_ll2_mac_address[ETH_ALEN]; 162 int gsi_qp_created; 163 struct qedr_cq *gsi_sqcq; 164 struct qedr_cq *gsi_rqcq; 165 struct qedr_qp *gsi_qp; 166 enum qed_rdma_type rdma_type; 167 struct xarray qps; 168 struct xarray srqs; 169 struct workqueue_struct *iwarp_wq; 170 u16 iwarp_max_mtu; 171 172 unsigned long enet_state; 173 174 u8 user_dpm_enabled; 175 }; 176 177 #define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *)) 178 #define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge)) 179 #define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \ 180 QEDR_SQE_ELEMENT_SIZE) 181 #define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\ 182 (RDMA_RING_PAGE_SIZE) / \ 183 (QEDR_SQE_ELEMENT_SIZE) /\ 184 (QEDR_MAX_SQE_ELEMENTS_PER_SQE)) 185 /* RQ */ 186 #define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *)) 187 #define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge)) 188 #define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE) 189 #define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\ 190 (RDMA_RING_PAGE_SIZE) / \ 191 (QEDR_RQE_ELEMENT_SIZE) /\ 192 (QEDR_MAX_RQE_ELEMENTS_PER_RQE)) 193 194 #define QEDR_CQE_SIZE (sizeof(union rdma_cqe)) 195 #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024) 196 #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \ 197 sizeof(u64)) - 1) 198 #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \ 199 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE)) 200 201 #define QEDR_ROCE_MAX_CNQ_SIZE (0x4000) 202 203 #define QEDR_PORT (1) 204 205 #define QEDR_ROCE_PKEY_TABLE_LEN 1 206 #define QEDR_ROCE_PKEY_DEFAULT 0xffff 207 208 struct qedr_pbl { 209 struct list_head list_entry; 210 void *va; 211 dma_addr_t pa; 212 }; 213 214 struct qedr_ucontext { 215 struct ib_ucontext ibucontext; 216 struct qedr_dev *dev; 217 struct qedr_pd *pd; 218 void __iomem *dpi_addr; 219 struct rdma_user_mmap_entry *db_mmap_entry; 220 u64 dpi_phys_addr; 221 u32 dpi_size; 222 u16 dpi; 223 bool db_rec; 224 u8 edpm_mode; 225 }; 226 227 union db_prod32 { 228 struct rdma_pwm_val16_data data; 229 u32 raw; 230 }; 231 232 union db_prod64 { 233 struct rdma_pwm_val32_data data; 234 u64 raw; 235 }; 236 237 enum qedr_cq_type { 238 QEDR_CQ_TYPE_GSI, 239 QEDR_CQ_TYPE_KERNEL, 240 QEDR_CQ_TYPE_USER, 241 }; 242 243 struct qedr_pbl_info { 244 u32 num_pbls; 245 u32 num_pbes; 246 u32 pbl_size; 247 u32 pbe_size; 248 bool two_layered; 249 }; 250 251 struct qedr_userq { 252 struct ib_umem *umem; 253 struct qedr_pbl_info pbl_info; 254 struct qedr_pbl *pbl_tbl; 255 u64 buf_addr; 256 size_t buf_len; 257 258 /* doorbell recovery */ 259 void __iomem *db_addr; 260 struct qedr_user_db_rec *db_rec_data; 261 struct rdma_user_mmap_entry *db_mmap_entry; 262 void __iomem *db_rec_db2_addr; 263 union db_prod32 db_rec_db2_data; 264 }; 265 266 struct qedr_cq { 267 struct ib_cq ibcq; 268 269 enum qedr_cq_type cq_type; 270 u32 sig; 271 272 u16 icid; 273 274 /* Lock to protect multiplem CQ's */ 275 spinlock_t cq_lock; 276 u8 arm_flags; 277 struct qed_chain pbl; 278 279 void __iomem *db_addr; 280 union db_prod64 db; 281 282 u8 pbl_toggle; 283 union rdma_cqe *latest_cqe; 284 union rdma_cqe *toggle_cqe; 285 286 u32 cq_cons; 287 288 struct qedr_userq q; 289 u8 destroyed; 290 u16 cnq_notif; 291 }; 292 293 struct qedr_pd { 294 struct ib_pd ibpd; 295 u32 pd_id; 296 struct qedr_ucontext *uctx; 297 }; 298 299 struct qedr_xrcd { 300 struct ib_xrcd ibxrcd; 301 u16 xrcd_id; 302 }; 303 304 struct qedr_qp_hwq_info { 305 /* WQE Elements */ 306 struct qed_chain pbl; 307 u64 p_phys_addr_tbl; 308 u32 max_sges; 309 310 /* WQE */ 311 u16 prod; 312 u16 cons; 313 u16 wqe_cons; 314 u16 gsi_cons; 315 u16 max_wr; 316 317 /* DB */ 318 void __iomem *db; 319 union db_prod32 db_data; 320 321 void __iomem *iwarp_db2; 322 union db_prod32 iwarp_db2_data; 323 }; 324 325 struct qedr_srq_hwq_info { 326 u32 max_sges; 327 u32 max_wr; 328 struct qed_chain pbl; 329 u64 p_phys_addr_tbl; 330 u32 wqe_prod; 331 u32 sge_prod; 332 u32 wr_prod_cnt; 333 atomic_t wr_cons_cnt; 334 u32 num_elems; 335 336 struct rdma_srq_producers *virt_prod_pair_addr; 337 dma_addr_t phy_prod_pair_addr; 338 }; 339 340 struct qedr_srq { 341 struct ib_srq ibsrq; 342 struct qedr_dev *dev; 343 344 struct qedr_userq usrq; 345 struct qedr_srq_hwq_info hw_srq; 346 struct ib_umem *prod_umem; 347 u16 srq_id; 348 u32 srq_limit; 349 bool is_xrc; 350 /* lock to protect srq recv post */ 351 spinlock_t lock; 352 }; 353 354 enum qedr_qp_err_bitmap { 355 QEDR_QP_ERR_SQ_FULL = 1, 356 QEDR_QP_ERR_RQ_FULL = 2, 357 QEDR_QP_ERR_BAD_SR = 4, 358 QEDR_QP_ERR_BAD_RR = 8, 359 QEDR_QP_ERR_SQ_PBL_FULL = 16, 360 QEDR_QP_ERR_RQ_PBL_FULL = 32, 361 }; 362 363 enum qedr_qp_create_type { 364 QEDR_QP_CREATE_NONE, 365 QEDR_QP_CREATE_USER, 366 QEDR_QP_CREATE_KERNEL, 367 }; 368 369 enum qedr_iwarp_cm_flags { 370 QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0), 371 QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1), 372 }; 373 374 struct qedr_qp { 375 struct ib_qp ibqp; /* must be first */ 376 struct qedr_dev *dev; 377 struct qedr_qp_hwq_info sq; 378 struct qedr_qp_hwq_info rq; 379 380 u32 max_inline_data; 381 382 /* Lock for QP's */ 383 spinlock_t q_lock; 384 struct qedr_cq *sq_cq; 385 struct qedr_cq *rq_cq; 386 struct qedr_srq *srq; 387 enum qed_roce_qp_state state; 388 u32 id; 389 struct qedr_pd *pd; 390 enum ib_qp_type qp_type; 391 enum qedr_qp_create_type create_type; 392 struct qed_rdma_qp *qed_qp; 393 u32 qp_id; 394 u16 icid; 395 u16 mtu; 396 int sgid_idx; 397 u32 rq_psn; 398 u32 sq_psn; 399 u32 qkey; 400 u32 dest_qp_num; 401 u8 timeout; 402 403 /* Relevant to qps created from kernel space only (ULPs) */ 404 u8 prev_wqe_size; 405 u16 wqe_cons; 406 u32 err_bitmap; 407 bool signaled; 408 409 /* SQ shadow */ 410 struct { 411 u64 wr_id; 412 enum ib_wc_opcode opcode; 413 u32 bytes_len; 414 u8 wqe_size; 415 bool signaled; 416 dma_addr_t icrc_mapping; 417 u32 *icrc; 418 struct qedr_mr *mr; 419 } *wqe_wr_id; 420 421 /* RQ shadow */ 422 struct { 423 u64 wr_id; 424 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE]; 425 u8 wqe_size; 426 427 u8 smac[ETH_ALEN]; 428 u16 vlan; 429 int rc; 430 } *rqe_wr_id; 431 432 /* Relevant to qps created from user space only (applications) */ 433 struct qedr_userq usq; 434 struct qedr_userq urq; 435 436 /* synchronization objects used with iwarp ep */ 437 struct kref refcnt; 438 struct completion iwarp_cm_comp; 439 struct completion qp_rel_comp; 440 unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */ 441 }; 442 443 struct qedr_ah { 444 struct ib_ah ibah; 445 struct rdma_ah_attr attr; 446 }; 447 448 enum qedr_mr_type { 449 QEDR_MR_USER, 450 QEDR_MR_KERNEL, 451 QEDR_MR_DMA, 452 QEDR_MR_FRMR, 453 }; 454 455 struct mr_info { 456 struct qedr_pbl *pbl_table; 457 struct qedr_pbl_info pbl_info; 458 struct list_head free_pbl_list; 459 struct list_head inuse_pbl_list; 460 u32 completed; 461 u32 completed_handled; 462 }; 463 464 struct qedr_mr { 465 struct ib_mr ibmr; 466 struct ib_umem *umem; 467 468 struct qed_rdma_register_tid_in_params hw_mr; 469 enum qedr_mr_type type; 470 471 struct qedr_dev *dev; 472 struct mr_info info; 473 474 u64 *pages; 475 u32 npages; 476 }; 477 478 struct qedr_user_mmap_entry { 479 struct rdma_user_mmap_entry rdma_entry; 480 struct qedr_dev *dev; 481 union { 482 u64 io_address; 483 void *address; 484 }; 485 size_t length; 486 u16 dpi; 487 u8 mmap_flag; 488 }; 489 490 #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT))) 491 492 #define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \ 493 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT) 494 #define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \ 495 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT) 496 #define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \ 497 RDMA_CQE_RESPONDER_INV_FLG_SHIFT) 498 499 static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info) 500 { 501 info->cons = (info->cons + 1) % info->max_wr; 502 info->wqe_cons++; 503 } 504 505 static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info) 506 { 507 info->prod = (info->prod + 1) % info->max_wr; 508 } 509 510 static inline int qedr_get_dmac(struct qedr_dev *dev, 511 struct rdma_ah_attr *ah_attr, u8 *mac_addr) 512 { 513 union ib_gid zero_sgid = { { 0 } }; 514 struct in6_addr in6; 515 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 516 u8 *dmac; 517 518 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) { 519 DP_ERR(dev, "Local port GID not supported\n"); 520 eth_zero_addr(mac_addr); 521 return -EINVAL; 522 } 523 524 memcpy(&in6, grh->dgid.raw, sizeof(in6)); 525 dmac = rdma_ah_retrieve_dmac(ah_attr); 526 if (!dmac) 527 return -EINVAL; 528 ether_addr_copy(mac_addr, dmac); 529 530 return 0; 531 } 532 533 struct qedr_iw_listener { 534 struct qedr_dev *dev; 535 struct iw_cm_id *cm_id; 536 int backlog; 537 void *qed_handle; 538 }; 539 540 struct qedr_iw_ep { 541 struct qedr_dev *dev; 542 struct iw_cm_id *cm_id; 543 struct qedr_qp *qp; 544 void *qed_context; 545 struct kref refcnt; 546 }; 547 548 static inline 549 struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext) 550 { 551 return container_of(ibucontext, struct qedr_ucontext, ibucontext); 552 } 553 554 static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev) 555 { 556 return container_of(ibdev, struct qedr_dev, ibdev); 557 } 558 559 static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd) 560 { 561 return container_of(ibpd, struct qedr_pd, ibpd); 562 } 563 564 static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd) 565 { 566 return container_of(ibxrcd, struct qedr_xrcd, ibxrcd); 567 } 568 569 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq) 570 { 571 return container_of(ibcq, struct qedr_cq, ibcq); 572 } 573 574 static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp) 575 { 576 return container_of(ibqp, struct qedr_qp, ibqp); 577 } 578 579 static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah) 580 { 581 return container_of(ibah, struct qedr_ah, ibah); 582 } 583 584 static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr) 585 { 586 return container_of(ibmr, struct qedr_mr, ibmr); 587 } 588 589 static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq) 590 { 591 return container_of(ibsrq, struct qedr_srq, ibsrq); 592 } 593 594 static inline bool qedr_qp_has_srq(struct qedr_qp *qp) 595 { 596 return qp->srq; 597 } 598 599 static inline bool qedr_qp_has_sq(struct qedr_qp *qp) 600 { 601 if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT) 602 return false; 603 604 return true; 605 } 606 607 static inline bool qedr_qp_has_rq(struct qedr_qp *qp) 608 { 609 if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI || 610 qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp)) 611 return false; 612 613 return true; 614 } 615 616 static inline struct qedr_user_mmap_entry * 617 get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry) 618 { 619 return container_of(rdma_entry, struct qedr_user_mmap_entry, 620 rdma_entry); 621 } 622 #endif 623