1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 #include "qedr_iw_cm.h" 55 56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 57 #define RDMA_MAX_SGE_PER_SRQ (4) 58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 59 60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 61 62 enum { 63 QEDR_USER_MMAP_IO_WC = 0, 64 QEDR_USER_MMAP_PHYS_PAGE, 65 }; 66 67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 68 size_t len) 69 { 70 size_t min_len = min_t(size_t, len, udata->outlen); 71 72 return ib_copy_to_udata(udata, src, min_len); 73 } 74 75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 76 { 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 78 return -EINVAL; 79 80 *pkey = QEDR_ROCE_PKEY_DEFAULT; 81 return 0; 82 } 83 84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, 85 int index, union ib_gid *sgid) 86 { 87 struct qedr_dev *dev = get_qedr_dev(ibdev); 88 89 memset(sgid->raw, 0, sizeof(sgid->raw)); 90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 91 92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 93 sgid->global.interface_id, sgid->global.subnet_prefix); 94 95 return 0; 96 } 97 98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 99 { 100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 101 struct qedr_device_attr *qattr = &dev->attr; 102 struct qedr_srq *srq = get_qedr_srq(ibsrq); 103 104 srq_attr->srq_limit = srq->srq_limit; 105 srq_attr->max_wr = qattr->max_srq_wr; 106 srq_attr->max_sge = qattr->max_sge; 107 108 return 0; 109 } 110 111 int qedr_query_device(struct ib_device *ibdev, 112 struct ib_device_attr *attr, struct ib_udata *udata) 113 { 114 struct qedr_dev *dev = get_qedr_dev(ibdev); 115 struct qedr_device_attr *qattr = &dev->attr; 116 117 if (!dev->rdma_ctx) { 118 DP_ERR(dev, 119 "qedr_query_device called with invalid params rdma_ctx=%p\n", 120 dev->rdma_ctx); 121 return -EINVAL; 122 } 123 124 memset(attr, 0, sizeof(*attr)); 125 126 attr->fw_ver = qattr->fw_ver; 127 attr->sys_image_guid = qattr->sys_image_guid; 128 attr->max_mr_size = qattr->max_mr_size; 129 attr->page_size_cap = qattr->page_size_caps; 130 attr->vendor_id = qattr->vendor_id; 131 attr->vendor_part_id = qattr->vendor_part_id; 132 attr->hw_ver = qattr->hw_ver; 133 attr->max_qp = qattr->max_qp; 134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 136 IB_DEVICE_RC_RNR_NAK_GEN | 137 IB_DEVICE_MEM_MGT_EXTENSIONS; 138 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 139 140 if (!rdma_protocol_iwarp(&dev->ibdev, 1)) 141 attr->device_cap_flags |= IB_DEVICE_XRC; 142 attr->max_send_sge = qattr->max_sge; 143 attr->max_recv_sge = qattr->max_sge; 144 attr->max_sge_rd = qattr->max_sge; 145 attr->max_cq = qattr->max_cq; 146 attr->max_cqe = qattr->max_cqe; 147 attr->max_mr = qattr->max_mr; 148 attr->max_mw = qattr->max_mw; 149 attr->max_pd = qattr->max_pd; 150 attr->atomic_cap = dev->atomic_cap; 151 attr->max_qp_init_rd_atom = 152 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 153 attr->max_qp_rd_atom = 154 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 155 attr->max_qp_init_rd_atom); 156 157 attr->max_srq = qattr->max_srq; 158 attr->max_srq_sge = qattr->max_srq_sge; 159 attr->max_srq_wr = qattr->max_srq_wr; 160 161 attr->local_ca_ack_delay = qattr->dev_ack_delay; 162 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 163 attr->max_pkeys = qattr->max_pkey; 164 attr->max_ah = qattr->max_ah; 165 166 return 0; 167 } 168 169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed, 170 u8 *ib_width) 171 { 172 switch (speed) { 173 case 1000: 174 *ib_speed = IB_SPEED_SDR; 175 *ib_width = IB_WIDTH_1X; 176 break; 177 case 10000: 178 *ib_speed = IB_SPEED_QDR; 179 *ib_width = IB_WIDTH_1X; 180 break; 181 182 case 20000: 183 *ib_speed = IB_SPEED_DDR; 184 *ib_width = IB_WIDTH_4X; 185 break; 186 187 case 25000: 188 *ib_speed = IB_SPEED_EDR; 189 *ib_width = IB_WIDTH_1X; 190 break; 191 192 case 40000: 193 *ib_speed = IB_SPEED_QDR; 194 *ib_width = IB_WIDTH_4X; 195 break; 196 197 case 50000: 198 *ib_speed = IB_SPEED_HDR; 199 *ib_width = IB_WIDTH_1X; 200 break; 201 202 case 100000: 203 *ib_speed = IB_SPEED_EDR; 204 *ib_width = IB_WIDTH_4X; 205 break; 206 207 default: 208 /* Unsupported */ 209 *ib_speed = IB_SPEED_SDR; 210 *ib_width = IB_WIDTH_1X; 211 } 212 } 213 214 int qedr_query_port(struct ib_device *ibdev, u32 port, 215 struct ib_port_attr *attr) 216 { 217 struct qedr_dev *dev; 218 struct qed_rdma_port *rdma_port; 219 220 dev = get_qedr_dev(ibdev); 221 222 if (!dev->rdma_ctx) { 223 DP_ERR(dev, "rdma_ctx is NULL\n"); 224 return -EINVAL; 225 } 226 227 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 228 229 /* *attr being zeroed by the caller, avoid zeroing it here */ 230 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 231 attr->state = IB_PORT_ACTIVE; 232 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 233 } else { 234 attr->state = IB_PORT_DOWN; 235 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 236 } 237 attr->max_mtu = IB_MTU_4096; 238 attr->lid = 0; 239 attr->lmc = 0; 240 attr->sm_lid = 0; 241 attr->sm_sl = 0; 242 attr->ip_gids = true; 243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 244 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu); 245 attr->gid_tbl_len = 1; 246 } else { 247 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 248 attr->gid_tbl_len = QEDR_MAX_SGID; 249 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 250 } 251 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 252 attr->qkey_viol_cntr = 0; 253 get_link_speed_and_width(rdma_port->link_speed, 254 &attr->active_speed, &attr->active_width); 255 attr->max_msg_sz = rdma_port->max_msg_size; 256 attr->max_vl_num = 4; 257 258 return 0; 259 } 260 261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 262 { 263 struct ib_device *ibdev = uctx->device; 264 int rc; 265 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 266 struct qedr_alloc_ucontext_resp uresp = {}; 267 struct qedr_alloc_ucontext_req ureq = {}; 268 struct qedr_dev *dev = get_qedr_dev(ibdev); 269 struct qed_rdma_add_user_out_params oparams; 270 struct qedr_user_mmap_entry *entry; 271 272 if (!udata) 273 return -EFAULT; 274 275 if (udata->inlen) { 276 rc = ib_copy_from_udata(&ureq, udata, 277 min(sizeof(ureq), udata->inlen)); 278 if (rc) { 279 DP_ERR(dev, "Problem copying data from user space\n"); 280 return -EFAULT; 281 } 282 ctx->edpm_mode = !!(ureq.context_flags & 283 QEDR_ALLOC_UCTX_EDPM_MODE); 284 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); 285 } 286 287 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 288 if (rc) { 289 DP_ERR(dev, 290 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 291 rc); 292 return rc; 293 } 294 295 ctx->dpi = oparams.dpi; 296 ctx->dpi_addr = oparams.dpi_addr; 297 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 298 ctx->dpi_size = oparams.dpi_size; 299 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 300 if (!entry) { 301 rc = -ENOMEM; 302 goto err; 303 } 304 305 entry->io_address = ctx->dpi_phys_addr; 306 entry->length = ctx->dpi_size; 307 entry->mmap_flag = QEDR_USER_MMAP_IO_WC; 308 entry->dpi = ctx->dpi; 309 entry->dev = dev; 310 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry, 311 ctx->dpi_size); 312 if (rc) { 313 kfree(entry); 314 goto err; 315 } 316 ctx->db_mmap_entry = &entry->rdma_entry; 317 318 if (!dev->user_dpm_enabled) 319 uresp.dpm_flags = 0; 320 else if (rdma_protocol_iwarp(&dev->ibdev, 1)) 321 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; 322 else 323 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | 324 QEDR_DPM_TYPE_ROCE_LEGACY | 325 QEDR_DPM_TYPE_ROCE_EDPM_MODE; 326 327 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) { 328 uresp.dpm_flags |= QEDR_DPM_SIZES_SET; 329 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; 330 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; 331 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE; 332 } 333 334 uresp.wids_enabled = 1; 335 uresp.wid_count = oparams.wid_count; 336 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); 337 uresp.db_size = ctx->dpi_size; 338 uresp.max_send_wr = dev->attr.max_sqe; 339 uresp.max_recv_wr = dev->attr.max_rqe; 340 uresp.max_srq_wr = dev->attr.max_srq_wr; 341 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 342 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 343 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 344 uresp.max_cqes = QEDR_MAX_CQES; 345 346 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 347 if (rc) 348 goto err; 349 350 ctx->dev = dev; 351 352 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 353 &ctx->ibucontext); 354 return 0; 355 356 err: 357 if (!ctx->db_mmap_entry) 358 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi); 359 else 360 rdma_user_mmap_entry_remove(ctx->db_mmap_entry); 361 362 return rc; 363 } 364 365 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 366 { 367 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 368 369 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 370 uctx); 371 372 rdma_user_mmap_entry_remove(uctx->db_mmap_entry); 373 } 374 375 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 376 { 377 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry); 378 struct qedr_dev *dev = entry->dev; 379 380 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE) 381 free_page((unsigned long)entry->address); 382 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC) 383 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi); 384 385 kfree(entry); 386 } 387 388 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) 389 { 390 struct ib_device *dev = ucontext->device; 391 size_t length = vma->vm_end - vma->vm_start; 392 struct rdma_user_mmap_entry *rdma_entry; 393 struct qedr_user_mmap_entry *entry; 394 int rc = 0; 395 u64 pfn; 396 397 ibdev_dbg(dev, 398 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 399 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 400 401 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma); 402 if (!rdma_entry) { 403 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n", 404 vma->vm_pgoff); 405 return -EINVAL; 406 } 407 entry = get_qedr_mmap_entry(rdma_entry); 408 ibdev_dbg(dev, 409 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 410 entry->io_address, length, entry->mmap_flag); 411 412 switch (entry->mmap_flag) { 413 case QEDR_USER_MMAP_IO_WC: 414 pfn = entry->io_address >> PAGE_SHIFT; 415 rc = rdma_user_mmap_io(ucontext, vma, pfn, length, 416 pgprot_writecombine(vma->vm_page_prot), 417 rdma_entry); 418 break; 419 case QEDR_USER_MMAP_PHYS_PAGE: 420 rc = vm_insert_page(vma, vma->vm_start, 421 virt_to_page(entry->address)); 422 break; 423 default: 424 rc = -EINVAL; 425 } 426 427 if (rc) 428 ibdev_dbg(dev, 429 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 430 entry->io_address, length, entry->mmap_flag, rc); 431 432 rdma_user_mmap_entry_put(rdma_entry); 433 return rc; 434 } 435 436 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 437 { 438 struct ib_device *ibdev = ibpd->device; 439 struct qedr_dev *dev = get_qedr_dev(ibdev); 440 struct qedr_pd *pd = get_qedr_pd(ibpd); 441 u16 pd_id; 442 int rc; 443 444 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 445 udata ? "User Lib" : "Kernel"); 446 447 if (!dev->rdma_ctx) { 448 DP_ERR(dev, "invalid RDMA context\n"); 449 return -EINVAL; 450 } 451 452 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 453 if (rc) 454 return rc; 455 456 pd->pd_id = pd_id; 457 458 if (udata) { 459 struct qedr_alloc_pd_uresp uresp = { 460 .pd_id = pd_id, 461 }; 462 struct qedr_ucontext *context = rdma_udata_to_drv_context( 463 udata, struct qedr_ucontext, ibucontext); 464 465 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 466 if (rc) { 467 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 468 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 469 return rc; 470 } 471 472 pd->uctx = context; 473 pd->uctx->pd = pd; 474 } 475 476 return 0; 477 } 478 479 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 480 { 481 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 482 struct qedr_pd *pd = get_qedr_pd(ibpd); 483 484 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 485 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 486 return 0; 487 } 488 489 490 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 491 { 492 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 493 struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd); 494 495 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id); 496 } 497 498 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 499 { 500 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 501 u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id; 502 503 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id); 504 return 0; 505 } 506 static void qedr_free_pbl(struct qedr_dev *dev, 507 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 508 { 509 struct pci_dev *pdev = dev->pdev; 510 int i; 511 512 for (i = 0; i < pbl_info->num_pbls; i++) { 513 if (!pbl[i].va) 514 continue; 515 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 516 pbl[i].va, pbl[i].pa); 517 } 518 519 kfree(pbl); 520 } 521 522 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 523 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 524 525 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 526 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 527 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 528 529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 530 struct qedr_pbl_info *pbl_info, 531 gfp_t flags) 532 { 533 struct pci_dev *pdev = dev->pdev; 534 struct qedr_pbl *pbl_table; 535 dma_addr_t *pbl_main_tbl; 536 dma_addr_t pa; 537 void *va; 538 int i; 539 540 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 541 if (!pbl_table) 542 return ERR_PTR(-ENOMEM); 543 544 for (i = 0; i < pbl_info->num_pbls; i++) { 545 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 546 flags); 547 if (!va) 548 goto err; 549 550 pbl_table[i].va = va; 551 pbl_table[i].pa = pa; 552 } 553 554 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 555 * the first one with physical pointers to all of the rest 556 */ 557 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 558 for (i = 0; i < pbl_info->num_pbls - 1; i++) 559 pbl_main_tbl[i] = pbl_table[i + 1].pa; 560 561 return pbl_table; 562 563 err: 564 for (i--; i >= 0; i--) 565 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 566 pbl_table[i].va, pbl_table[i].pa); 567 568 qedr_free_pbl(dev, pbl_info, pbl_table); 569 570 return ERR_PTR(-ENOMEM); 571 } 572 573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 574 struct qedr_pbl_info *pbl_info, 575 u32 num_pbes, int two_layer_capable) 576 { 577 u32 pbl_capacity; 578 u32 pbl_size; 579 u32 num_pbls; 580 581 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 582 if (num_pbes > MAX_PBES_TWO_LAYER) { 583 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 584 num_pbes); 585 return -EINVAL; 586 } 587 588 /* calculate required pbl page size */ 589 pbl_size = MIN_FW_PBL_PAGE_SIZE; 590 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 591 NUM_PBES_ON_PAGE(pbl_size); 592 593 while (pbl_capacity < num_pbes) { 594 pbl_size *= 2; 595 pbl_capacity = pbl_size / sizeof(u64); 596 pbl_capacity = pbl_capacity * pbl_capacity; 597 } 598 599 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 600 num_pbls++; /* One for the layer0 ( points to the pbls) */ 601 pbl_info->two_layered = true; 602 } else { 603 /* One layered PBL */ 604 num_pbls = 1; 605 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 606 roundup_pow_of_two((num_pbes * sizeof(u64)))); 607 pbl_info->two_layered = false; 608 } 609 610 pbl_info->num_pbls = num_pbls; 611 pbl_info->pbl_size = pbl_size; 612 pbl_info->num_pbes = num_pbes; 613 614 DP_DEBUG(dev, QEDR_MSG_MR, 615 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 616 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 617 618 return 0; 619 } 620 621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 622 struct qedr_pbl *pbl, 623 struct qedr_pbl_info *pbl_info, u32 pg_shift) 624 { 625 int pbe_cnt, total_num_pbes = 0; 626 struct qedr_pbl *pbl_tbl; 627 struct ib_block_iter biter; 628 struct regpair *pbe; 629 630 if (!pbl_info->num_pbes) 631 return; 632 633 /* If we have a two layered pbl, the first pbl points to the rest 634 * of the pbls and the first entry lays on the second pbl in the table 635 */ 636 if (pbl_info->two_layered) 637 pbl_tbl = &pbl[1]; 638 else 639 pbl_tbl = pbl; 640 641 pbe = (struct regpair *)pbl_tbl->va; 642 if (!pbe) { 643 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 644 return; 645 } 646 647 pbe_cnt = 0; 648 649 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { 650 u64 pg_addr = rdma_block_iter_dma_address(&biter); 651 652 pbe->lo = cpu_to_le32(pg_addr); 653 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 654 655 pbe_cnt++; 656 total_num_pbes++; 657 pbe++; 658 659 if (total_num_pbes == pbl_info->num_pbes) 660 return; 661 662 /* If the given pbl is full storing the pbes, move to next pbl. 663 */ 664 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 665 pbl_tbl++; 666 pbe = (struct regpair *)pbl_tbl->va; 667 pbe_cnt = 0; 668 } 669 } 670 } 671 672 static int qedr_db_recovery_add(struct qedr_dev *dev, 673 void __iomem *db_addr, 674 void *db_data, 675 enum qed_db_rec_width db_width, 676 enum qed_db_rec_space db_space) 677 { 678 if (!db_data) { 679 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 680 return 0; 681 } 682 683 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data, 684 db_width, db_space); 685 } 686 687 static void qedr_db_recovery_del(struct qedr_dev *dev, 688 void __iomem *db_addr, 689 void *db_data) 690 { 691 if (!db_data) { 692 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 693 return; 694 } 695 696 /* Ignore return code as there is not much we can do about it. Error 697 * log will be printed inside. 698 */ 699 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); 700 } 701 702 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 703 struct qedr_cq *cq, struct ib_udata *udata, 704 u32 db_offset) 705 { 706 struct qedr_create_cq_uresp uresp; 707 int rc; 708 709 memset(&uresp, 0, sizeof(uresp)); 710 711 uresp.db_offset = db_offset; 712 uresp.icid = cq->icid; 713 if (cq->q.db_mmap_entry) 714 uresp.db_rec_addr = 715 rdma_user_mmap_get_offset(cq->q.db_mmap_entry); 716 717 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 718 if (rc) 719 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 720 721 return rc; 722 } 723 724 static void consume_cqe(struct qedr_cq *cq) 725 { 726 if (cq->latest_cqe == cq->toggle_cqe) 727 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 728 729 cq->latest_cqe = qed_chain_consume(&cq->pbl); 730 } 731 732 static inline int qedr_align_cq_entries(int entries) 733 { 734 u64 size, aligned_size; 735 736 /* We allocate an extra entry that we don't report to the FW. */ 737 size = (entries + 1) * QEDR_CQE_SIZE; 738 aligned_size = ALIGN(size, PAGE_SIZE); 739 740 return aligned_size / QEDR_CQE_SIZE; 741 } 742 743 static int qedr_init_user_db_rec(struct ib_udata *udata, 744 struct qedr_dev *dev, struct qedr_userq *q, 745 bool requires_db_rec) 746 { 747 struct qedr_ucontext *uctx = 748 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 749 ibucontext); 750 struct qedr_user_mmap_entry *entry; 751 int rc; 752 753 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */ 754 if (requires_db_rec == 0 || !uctx->db_rec) 755 return 0; 756 757 /* Allocate a page for doorbell recovery, add to mmap */ 758 q->db_rec_data = (void *)get_zeroed_page(GFP_USER); 759 if (!q->db_rec_data) { 760 DP_ERR(dev, "get_zeroed_page failed\n"); 761 return -ENOMEM; 762 } 763 764 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 765 if (!entry) 766 goto err_free_db_data; 767 768 entry->address = q->db_rec_data; 769 entry->length = PAGE_SIZE; 770 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE; 771 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext, 772 &entry->rdma_entry, 773 PAGE_SIZE); 774 if (rc) 775 goto err_free_entry; 776 777 q->db_mmap_entry = &entry->rdma_entry; 778 779 return 0; 780 781 err_free_entry: 782 kfree(entry); 783 784 err_free_db_data: 785 free_page((unsigned long)q->db_rec_data); 786 q->db_rec_data = NULL; 787 return -ENOMEM; 788 } 789 790 static inline int qedr_init_user_queue(struct ib_udata *udata, 791 struct qedr_dev *dev, 792 struct qedr_userq *q, u64 buf_addr, 793 size_t buf_len, bool requires_db_rec, 794 int access, 795 int alloc_and_init) 796 { 797 u32 fw_pages; 798 int rc; 799 800 q->buf_addr = buf_addr; 801 q->buf_len = buf_len; 802 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); 803 if (IS_ERR(q->umem)) { 804 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 805 PTR_ERR(q->umem)); 806 return PTR_ERR(q->umem); 807 } 808 809 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT); 810 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 811 if (rc) 812 goto err0; 813 814 if (alloc_and_init) { 815 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 816 if (IS_ERR(q->pbl_tbl)) { 817 rc = PTR_ERR(q->pbl_tbl); 818 goto err0; 819 } 820 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 821 FW_PAGE_SHIFT); 822 } else { 823 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 824 if (!q->pbl_tbl) { 825 rc = -ENOMEM; 826 goto err0; 827 } 828 } 829 830 /* mmap the user address used to store doorbell data for recovery */ 831 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); 832 833 err0: 834 ib_umem_release(q->umem); 835 q->umem = NULL; 836 837 return rc; 838 } 839 840 static inline void qedr_init_cq_params(struct qedr_cq *cq, 841 struct qedr_ucontext *ctx, 842 struct qedr_dev *dev, int vector, 843 int chain_entries, int page_cnt, 844 u64 pbl_ptr, 845 struct qed_rdma_create_cq_in_params 846 *params) 847 { 848 memset(params, 0, sizeof(*params)); 849 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 850 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 851 params->cnq_id = vector; 852 params->cq_size = chain_entries - 1; 853 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 854 params->pbl_num_pages = page_cnt; 855 params->pbl_ptr = pbl_ptr; 856 params->pbl_two_level = 0; 857 } 858 859 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 860 { 861 cq->db.data.agg_flags = flags; 862 cq->db.data.value = cpu_to_le32(cons); 863 writeq(cq->db.raw, cq->db_addr); 864 } 865 866 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 867 { 868 struct qedr_cq *cq = get_qedr_cq(ibcq); 869 unsigned long sflags; 870 struct qedr_dev *dev; 871 872 dev = get_qedr_dev(ibcq->device); 873 874 if (cq->destroyed) { 875 DP_ERR(dev, 876 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 877 cq, cq->icid); 878 return -EINVAL; 879 } 880 881 882 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 883 return 0; 884 885 spin_lock_irqsave(&cq->cq_lock, sflags); 886 887 cq->arm_flags = 0; 888 889 if (flags & IB_CQ_SOLICITED) 890 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 891 892 if (flags & IB_CQ_NEXT_COMP) 893 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 894 895 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 896 897 spin_unlock_irqrestore(&cq->cq_lock, sflags); 898 899 return 0; 900 } 901 902 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 903 struct uverbs_attr_bundle *attrs) 904 { 905 struct ib_udata *udata = &attrs->driver_udata; 906 struct ib_device *ibdev = ibcq->device; 907 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 908 udata, struct qedr_ucontext, ibucontext); 909 struct qed_rdma_destroy_cq_out_params destroy_oparams; 910 struct qed_rdma_destroy_cq_in_params destroy_iparams; 911 struct qed_chain_init_params chain_params = { 912 .mode = QED_CHAIN_MODE_PBL, 913 .intended_use = QED_CHAIN_USE_TO_CONSUME, 914 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 915 .elem_size = sizeof(union rdma_cqe), 916 }; 917 struct qedr_dev *dev = get_qedr_dev(ibdev); 918 struct qed_rdma_create_cq_in_params params; 919 struct qedr_create_cq_ureq ureq = {}; 920 int vector = attr->comp_vector; 921 int entries = attr->cqe; 922 struct qedr_cq *cq = get_qedr_cq(ibcq); 923 int chain_entries; 924 u32 db_offset; 925 int page_cnt; 926 u64 pbl_ptr; 927 u16 icid; 928 int rc; 929 930 DP_DEBUG(dev, QEDR_MSG_INIT, 931 "create_cq: called from %s. entries=%d, vector=%d\n", 932 udata ? "User Lib" : "Kernel", entries, vector); 933 934 if (attr->flags) 935 return -EOPNOTSUPP; 936 937 if (entries > QEDR_MAX_CQES) { 938 DP_ERR(dev, 939 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 940 entries, QEDR_MAX_CQES); 941 return -EINVAL; 942 } 943 944 chain_entries = qedr_align_cq_entries(entries); 945 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 946 chain_params.num_elems = chain_entries; 947 948 /* calc db offset. user will add DPI base, kernel will add db addr */ 949 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 950 951 if (udata) { 952 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 953 udata->inlen))) { 954 DP_ERR(dev, 955 "create cq: problem copying data from user space\n"); 956 goto err0; 957 } 958 959 if (!ureq.len) { 960 DP_ERR(dev, 961 "create cq: cannot create a cq with 0 entries\n"); 962 goto err0; 963 } 964 965 cq->cq_type = QEDR_CQ_TYPE_USER; 966 967 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 968 ureq.len, true, IB_ACCESS_LOCAL_WRITE, 969 1); 970 if (rc) 971 goto err0; 972 973 pbl_ptr = cq->q.pbl_tbl->pa; 974 page_cnt = cq->q.pbl_info.num_pbes; 975 976 cq->ibcq.cqe = chain_entries; 977 cq->q.db_addr = ctx->dpi_addr + db_offset; 978 } else { 979 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 980 981 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl, 982 &chain_params); 983 if (rc) 984 goto err0; 985 986 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 987 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 988 cq->ibcq.cqe = cq->pbl.capacity; 989 } 990 991 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 992 pbl_ptr, ¶ms); 993 994 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 995 if (rc) 996 goto err1; 997 998 cq->icid = icid; 999 cq->sig = QEDR_CQ_MAGIC_NUMBER; 1000 spin_lock_init(&cq->cq_lock); 1001 1002 if (udata) { 1003 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); 1004 if (rc) 1005 goto err2; 1006 1007 rc = qedr_db_recovery_add(dev, cq->q.db_addr, 1008 &cq->q.db_rec_data->db_data, 1009 DB_REC_WIDTH_64B, 1010 DB_REC_USER); 1011 if (rc) 1012 goto err2; 1013 1014 } else { 1015 /* Generate doorbell address. */ 1016 cq->db.data.icid = cq->icid; 1017 cq->db_addr = dev->db_addr + db_offset; 1018 cq->db.data.params = DB_AGG_CMD_MAX << 1019 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1020 1021 /* point to the very last element, passing it we will toggle */ 1022 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 1023 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 1024 cq->latest_cqe = NULL; 1025 consume_cqe(cq); 1026 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 1027 1028 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, 1029 DB_REC_WIDTH_64B, DB_REC_KERNEL); 1030 if (rc) 1031 goto err2; 1032 } 1033 1034 DP_DEBUG(dev, QEDR_MSG_CQ, 1035 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1036 cq->icid, cq, params.cq_size); 1037 1038 return 0; 1039 1040 err2: 1041 destroy_iparams.icid = cq->icid; 1042 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1043 &destroy_oparams); 1044 err1: 1045 if (udata) { 1046 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1047 ib_umem_release(cq->q.umem); 1048 if (cq->q.db_mmap_entry) 1049 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1050 } else { 1051 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1052 } 1053 err0: 1054 return -EINVAL; 1055 } 1056 1057 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1058 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1059 1060 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1061 { 1062 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1063 struct qed_rdma_destroy_cq_out_params oparams; 1064 struct qed_rdma_destroy_cq_in_params iparams; 1065 struct qedr_cq *cq = get_qedr_cq(ibcq); 1066 int iter; 1067 1068 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1069 1070 cq->destroyed = 1; 1071 1072 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1073 if (cq->cq_type == QEDR_CQ_TYPE_GSI) { 1074 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1075 return 0; 1076 } 1077 1078 iparams.icid = cq->icid; 1079 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1080 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1081 1082 if (udata) { 1083 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1084 ib_umem_release(cq->q.umem); 1085 1086 if (cq->q.db_rec_data) { 1087 qedr_db_recovery_del(dev, cq->q.db_addr, 1088 &cq->q.db_rec_data->db_data); 1089 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1090 } 1091 } else { 1092 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1093 } 1094 1095 /* We don't want the IRQ handler to handle a non-existing CQ so we 1096 * wait until all CNQ interrupts, if any, are received. This will always 1097 * happen and will always happen very fast. If not, then a serious error 1098 * has occured. That is why we can use a long delay. 1099 * We spin for a short time so we don’t lose time on context switching 1100 * in case all the completions are handled in that span. Otherwise 1101 * we sleep for a while and check again. Since the CNQ may be 1102 * associated with (only) the current CPU we use msleep to allow the 1103 * current CPU to be freed. 1104 * The CNQ notification is increased in qedr_irq_handler(). 1105 */ 1106 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1107 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1108 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1109 iter--; 1110 } 1111 1112 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1113 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1114 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1115 iter--; 1116 } 1117 1118 /* Note that we don't need to have explicit code to wait for the 1119 * completion of the event handler because it is invoked from the EQ. 1120 * Since the destroy CQ ramrod has also been received on the EQ we can 1121 * be certain that there's no event handler in process. 1122 */ 1123 return 0; 1124 } 1125 1126 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1127 struct ib_qp_attr *attr, 1128 int attr_mask, 1129 struct qed_rdma_modify_qp_in_params 1130 *qp_params) 1131 { 1132 const struct ib_gid_attr *gid_attr; 1133 enum rdma_network_type nw_type; 1134 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1135 u32 ipv4_addr; 1136 int ret; 1137 int i; 1138 1139 gid_attr = grh->sgid_attr; 1140 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1141 if (ret) 1142 return ret; 1143 1144 nw_type = rdma_gid_attr_network_type(gid_attr); 1145 switch (nw_type) { 1146 case RDMA_NETWORK_IPV6: 1147 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1148 sizeof(qp_params->sgid)); 1149 memcpy(&qp_params->dgid.bytes[0], 1150 &grh->dgid, 1151 sizeof(qp_params->dgid)); 1152 qp_params->roce_mode = ROCE_V2_IPV6; 1153 SET_FIELD(qp_params->modify_flags, 1154 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1155 break; 1156 case RDMA_NETWORK_ROCE_V1: 1157 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1158 sizeof(qp_params->sgid)); 1159 memcpy(&qp_params->dgid.bytes[0], 1160 &grh->dgid, 1161 sizeof(qp_params->dgid)); 1162 qp_params->roce_mode = ROCE_V1; 1163 break; 1164 case RDMA_NETWORK_IPV4: 1165 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1166 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1167 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1168 qp_params->sgid.ipv4_addr = ipv4_addr; 1169 ipv4_addr = 1170 qedr_get_ipv4_from_gid(grh->dgid.raw); 1171 qp_params->dgid.ipv4_addr = ipv4_addr; 1172 SET_FIELD(qp_params->modify_flags, 1173 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1174 qp_params->roce_mode = ROCE_V2_IPV4; 1175 break; 1176 default: 1177 return -EINVAL; 1178 } 1179 1180 for (i = 0; i < 4; i++) { 1181 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1182 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1183 } 1184 1185 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1186 qp_params->vlan_id = 0; 1187 1188 return 0; 1189 } 1190 1191 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1192 struct ib_qp_init_attr *attrs, 1193 struct ib_udata *udata) 1194 { 1195 struct qedr_device_attr *qattr = &dev->attr; 1196 1197 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1198 if (attrs->qp_type != IB_QPT_RC && 1199 attrs->qp_type != IB_QPT_GSI && 1200 attrs->qp_type != IB_QPT_XRC_INI && 1201 attrs->qp_type != IB_QPT_XRC_TGT) { 1202 DP_DEBUG(dev, QEDR_MSG_QP, 1203 "create qp: unsupported qp type=0x%x requested\n", 1204 attrs->qp_type); 1205 return -EOPNOTSUPP; 1206 } 1207 1208 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1209 DP_ERR(dev, 1210 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1211 attrs->cap.max_send_wr, qattr->max_sqe); 1212 return -EINVAL; 1213 } 1214 1215 if (attrs->cap.max_inline_data > qattr->max_inline) { 1216 DP_ERR(dev, 1217 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1218 attrs->cap.max_inline_data, qattr->max_inline); 1219 return -EINVAL; 1220 } 1221 1222 if (attrs->cap.max_send_sge > qattr->max_sge) { 1223 DP_ERR(dev, 1224 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1225 attrs->cap.max_send_sge, qattr->max_sge); 1226 return -EINVAL; 1227 } 1228 1229 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1230 DP_ERR(dev, 1231 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1232 attrs->cap.max_recv_sge, qattr->max_sge); 1233 return -EINVAL; 1234 } 1235 1236 /* verify consumer QPs are not trying to use GSI QP's CQ. 1237 * TGT QP isn't associated with RQ/SQ 1238 */ 1239 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) && 1240 (attrs->qp_type != IB_QPT_XRC_TGT) && 1241 (attrs->qp_type != IB_QPT_XRC_INI)) { 1242 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq); 1243 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq); 1244 1245 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) || 1246 (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) { 1247 DP_ERR(dev, 1248 "create qp: consumer QP cannot use GSI CQs.\n"); 1249 return -EINVAL; 1250 } 1251 } 1252 1253 return 0; 1254 } 1255 1256 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1257 struct qedr_srq *srq, struct ib_udata *udata) 1258 { 1259 struct qedr_create_srq_uresp uresp = {}; 1260 int rc; 1261 1262 uresp.srq_id = srq->srq_id; 1263 1264 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1265 if (rc) 1266 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1267 1268 return rc; 1269 } 1270 1271 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1272 struct qedr_create_qp_uresp *uresp, 1273 struct qedr_qp *qp) 1274 { 1275 /* iWARP requires two doorbells per RQ. */ 1276 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1277 uresp->rq_db_offset = 1278 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1279 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1280 } else { 1281 uresp->rq_db_offset = 1282 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1283 } 1284 1285 uresp->rq_icid = qp->icid; 1286 if (qp->urq.db_mmap_entry) 1287 uresp->rq_db_rec_addr = 1288 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry); 1289 } 1290 1291 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1292 struct qedr_create_qp_uresp *uresp, 1293 struct qedr_qp *qp) 1294 { 1295 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1296 1297 /* iWARP uses the same cid for rq and sq */ 1298 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1299 uresp->sq_icid = qp->icid; 1300 else 1301 uresp->sq_icid = qp->icid + 1; 1302 1303 if (qp->usq.db_mmap_entry) 1304 uresp->sq_db_rec_addr = 1305 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry); 1306 } 1307 1308 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1309 struct qedr_qp *qp, struct ib_udata *udata, 1310 struct qedr_create_qp_uresp *uresp) 1311 { 1312 int rc; 1313 1314 memset(uresp, 0, sizeof(*uresp)); 1315 1316 if (qedr_qp_has_sq(qp)) 1317 qedr_copy_sq_uresp(dev, uresp, qp); 1318 1319 if (qedr_qp_has_rq(qp)) 1320 qedr_copy_rq_uresp(dev, uresp, qp); 1321 1322 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1323 uresp->qp_id = qp->qp_id; 1324 1325 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); 1326 if (rc) 1327 DP_ERR(dev, 1328 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1329 qp->icid); 1330 1331 return rc; 1332 } 1333 1334 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1335 { 1336 qed_chain_reset(&qph->pbl); 1337 qph->prod = 0; 1338 qph->cons = 0; 1339 qph->wqe_cons = 0; 1340 qph->db_data.data.value = cpu_to_le16(0); 1341 } 1342 1343 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1344 struct qedr_qp *qp, 1345 struct qedr_pd *pd, 1346 struct ib_qp_init_attr *attrs) 1347 { 1348 spin_lock_init(&qp->q_lock); 1349 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1350 kref_init(&qp->refcnt); 1351 init_completion(&qp->iwarp_cm_comp); 1352 init_completion(&qp->qp_rel_comp); 1353 } 1354 1355 qp->pd = pd; 1356 qp->qp_type = attrs->qp_type; 1357 qp->max_inline_data = attrs->cap.max_inline_data; 1358 qp->state = QED_ROCE_QP_STATE_RESET; 1359 1360 qp->prev_wqe_size = 0; 1361 1362 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; 1363 qp->dev = dev; 1364 if (qedr_qp_has_sq(qp)) { 1365 qedr_reset_qp_hwq_info(&qp->sq); 1366 qp->sq.max_sges = attrs->cap.max_send_sge; 1367 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1368 DP_DEBUG(dev, QEDR_MSG_QP, 1369 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1370 qp->sq.max_sges, qp->sq_cq->icid); 1371 } 1372 1373 if (attrs->srq) 1374 qp->srq = get_qedr_srq(attrs->srq); 1375 1376 if (qedr_qp_has_rq(qp)) { 1377 qedr_reset_qp_hwq_info(&qp->rq); 1378 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1379 qp->rq.max_sges = attrs->cap.max_recv_sge; 1380 DP_DEBUG(dev, QEDR_MSG_QP, 1381 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1382 qp->rq.max_sges, qp->rq_cq->icid); 1383 } 1384 1385 DP_DEBUG(dev, QEDR_MSG_QP, 1386 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1387 pd->pd_id, qp->qp_type, qp->max_inline_data, 1388 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1389 DP_DEBUG(dev, QEDR_MSG_QP, 1390 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1391 qp->sq.max_sges, qp->sq_cq->icid); 1392 } 1393 1394 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1395 { 1396 int rc = 0; 1397 1398 if (qedr_qp_has_sq(qp)) { 1399 qp->sq.db = dev->db_addr + 1400 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1401 qp->sq.db_data.data.icid = qp->icid + 1; 1402 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data, 1403 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1404 if (rc) 1405 return rc; 1406 } 1407 1408 if (qedr_qp_has_rq(qp)) { 1409 qp->rq.db = dev->db_addr + 1410 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1411 qp->rq.db_data.data.icid = qp->icid; 1412 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data, 1413 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1414 if (rc && qedr_qp_has_sq(qp)) 1415 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 1416 } 1417 1418 return rc; 1419 } 1420 1421 static int qedr_check_srq_params(struct qedr_dev *dev, 1422 struct ib_srq_init_attr *attrs, 1423 struct ib_udata *udata) 1424 { 1425 struct qedr_device_attr *qattr = &dev->attr; 1426 1427 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1428 DP_ERR(dev, 1429 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1430 attrs->attr.max_wr, qattr->max_srq_wr); 1431 return -EINVAL; 1432 } 1433 1434 if (attrs->attr.max_sge > qattr->max_sge) { 1435 DP_ERR(dev, 1436 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1437 attrs->attr.max_sge, qattr->max_sge); 1438 } 1439 1440 if (!udata && attrs->srq_type == IB_SRQT_XRC) { 1441 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n"); 1442 return -EINVAL; 1443 } 1444 1445 return 0; 1446 } 1447 1448 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1449 { 1450 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1451 ib_umem_release(srq->usrq.umem); 1452 ib_umem_release(srq->prod_umem); 1453 } 1454 1455 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1456 { 1457 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1458 struct qedr_dev *dev = srq->dev; 1459 1460 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1461 1462 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1463 hw_srq->virt_prod_pair_addr, 1464 hw_srq->phy_prod_pair_addr); 1465 } 1466 1467 static int qedr_init_srq_user_params(struct ib_udata *udata, 1468 struct qedr_srq *srq, 1469 struct qedr_create_srq_ureq *ureq, 1470 int access) 1471 { 1472 struct scatterlist *sg; 1473 int rc; 1474 1475 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1476 ureq->srq_len, false, access, 1); 1477 if (rc) 1478 return rc; 1479 1480 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, 1481 sizeof(struct rdma_srq_producers), access); 1482 if (IS_ERR(srq->prod_umem)) { 1483 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1484 ib_umem_release(srq->usrq.umem); 1485 DP_ERR(srq->dev, 1486 "create srq: failed ib_umem_get for producer, got %ld\n", 1487 PTR_ERR(srq->prod_umem)); 1488 return PTR_ERR(srq->prod_umem); 1489 } 1490 1491 sg = srq->prod_umem->sgt_append.sgt.sgl; 1492 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1493 1494 return 0; 1495 } 1496 1497 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1498 struct qedr_dev *dev, 1499 struct ib_srq_init_attr *init_attr) 1500 { 1501 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1502 struct qed_chain_init_params params = { 1503 .mode = QED_CHAIN_MODE_PBL, 1504 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1505 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 1506 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE, 1507 }; 1508 dma_addr_t phy_prod_pair_addr; 1509 u32 num_elems; 1510 void *va; 1511 int rc; 1512 1513 va = dma_alloc_coherent(&dev->pdev->dev, 1514 sizeof(struct rdma_srq_producers), 1515 &phy_prod_pair_addr, GFP_KERNEL); 1516 if (!va) { 1517 DP_ERR(dev, 1518 "create srq: failed to allocate dma memory for producer\n"); 1519 return -ENOMEM; 1520 } 1521 1522 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1523 hw_srq->virt_prod_pair_addr = va; 1524 1525 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1526 params.num_elems = num_elems; 1527 1528 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, ¶ms); 1529 if (rc) 1530 goto err0; 1531 1532 hw_srq->num_elems = num_elems; 1533 1534 return 0; 1535 1536 err0: 1537 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1538 va, phy_prod_pair_addr); 1539 return rc; 1540 } 1541 1542 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1543 struct ib_udata *udata) 1544 { 1545 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1546 struct qed_rdma_create_srq_in_params in_params = {}; 1547 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1548 struct qed_rdma_create_srq_out_params out_params; 1549 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1550 struct qedr_create_srq_ureq ureq = {}; 1551 u64 pbl_base_addr, phy_prod_pair_addr; 1552 struct qedr_srq_hwq_info *hw_srq; 1553 u32 page_cnt, page_size; 1554 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1555 int rc = 0; 1556 1557 DP_DEBUG(dev, QEDR_MSG_QP, 1558 "create SRQ called from %s (pd %p)\n", 1559 (udata) ? "User lib" : "kernel", pd); 1560 1561 if (init_attr->srq_type != IB_SRQT_BASIC && 1562 init_attr->srq_type != IB_SRQT_XRC) 1563 return -EOPNOTSUPP; 1564 1565 rc = qedr_check_srq_params(dev, init_attr, udata); 1566 if (rc) 1567 return -EINVAL; 1568 1569 srq->dev = dev; 1570 srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC); 1571 hw_srq = &srq->hw_srq; 1572 spin_lock_init(&srq->lock); 1573 1574 hw_srq->max_wr = init_attr->attr.max_wr; 1575 hw_srq->max_sges = init_attr->attr.max_sge; 1576 1577 if (udata) { 1578 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1579 udata->inlen))) { 1580 DP_ERR(dev, 1581 "create srq: problem copying data from user space\n"); 1582 goto err0; 1583 } 1584 1585 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0); 1586 if (rc) 1587 goto err0; 1588 1589 page_cnt = srq->usrq.pbl_info.num_pbes; 1590 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1591 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1592 page_size = PAGE_SIZE; 1593 } else { 1594 struct qed_chain *pbl; 1595 1596 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1597 if (rc) 1598 goto err0; 1599 1600 pbl = &hw_srq->pbl; 1601 page_cnt = qed_chain_get_page_cnt(pbl); 1602 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1603 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1604 page_size = QED_CHAIN_PAGE_SIZE; 1605 } 1606 1607 in_params.pd_id = pd->pd_id; 1608 in_params.pbl_base_addr = pbl_base_addr; 1609 in_params.prod_pair_addr = phy_prod_pair_addr; 1610 in_params.num_pages = page_cnt; 1611 in_params.page_size = page_size; 1612 if (srq->is_xrc) { 1613 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd); 1614 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq); 1615 1616 in_params.is_xrc = 1; 1617 in_params.xrcd_id = xrcd->xrcd_id; 1618 in_params.cq_cid = cq->icid; 1619 } 1620 1621 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1622 if (rc) 1623 goto err1; 1624 1625 srq->srq_id = out_params.srq_id; 1626 1627 if (udata) { 1628 rc = qedr_copy_srq_uresp(dev, srq, udata); 1629 if (rc) 1630 goto err2; 1631 } 1632 1633 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1634 if (rc) 1635 goto err2; 1636 1637 DP_DEBUG(dev, QEDR_MSG_SRQ, 1638 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1639 return 0; 1640 1641 err2: 1642 destroy_in_params.srq_id = srq->srq_id; 1643 1644 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1645 err1: 1646 if (udata) 1647 qedr_free_srq_user_params(srq); 1648 else 1649 qedr_free_srq_kernel_params(srq); 1650 err0: 1651 return -EFAULT; 1652 } 1653 1654 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1655 { 1656 struct qed_rdma_destroy_srq_in_params in_params = {}; 1657 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1658 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1659 1660 xa_erase_irq(&dev->srqs, srq->srq_id); 1661 in_params.srq_id = srq->srq_id; 1662 in_params.is_xrc = srq->is_xrc; 1663 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1664 1665 if (ibsrq->uobject) 1666 qedr_free_srq_user_params(srq); 1667 else 1668 qedr_free_srq_kernel_params(srq); 1669 1670 DP_DEBUG(dev, QEDR_MSG_SRQ, 1671 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1672 srq->srq_id); 1673 return 0; 1674 } 1675 1676 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1677 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1678 { 1679 struct qed_rdma_modify_srq_in_params in_params = {}; 1680 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1681 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1682 int rc; 1683 1684 if (attr_mask & IB_SRQ_MAX_WR) { 1685 DP_ERR(dev, 1686 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1687 attr_mask, srq); 1688 return -EINVAL; 1689 } 1690 1691 if (attr_mask & IB_SRQ_LIMIT) { 1692 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1693 DP_ERR(dev, 1694 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1695 attr->srq_limit, srq->hw_srq.max_wr); 1696 return -EINVAL; 1697 } 1698 1699 in_params.srq_id = srq->srq_id; 1700 in_params.wqe_limit = attr->srq_limit; 1701 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1702 if (rc) 1703 return rc; 1704 } 1705 1706 srq->srq_limit = attr->srq_limit; 1707 1708 DP_DEBUG(dev, QEDR_MSG_SRQ, 1709 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1710 1711 return 0; 1712 } 1713 1714 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type) 1715 { 1716 switch (ib_qp_type) { 1717 case IB_QPT_RC: 1718 return QED_RDMA_QP_TYPE_RC; 1719 case IB_QPT_XRC_INI: 1720 return QED_RDMA_QP_TYPE_XRC_INI; 1721 case IB_QPT_XRC_TGT: 1722 return QED_RDMA_QP_TYPE_XRC_TGT; 1723 default: 1724 return QED_RDMA_QP_TYPE_INVAL; 1725 } 1726 } 1727 1728 static inline void 1729 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1730 struct qedr_pd *pd, 1731 struct qedr_qp *qp, 1732 struct ib_qp_init_attr *attrs, 1733 bool fmr_and_reserved_lkey, 1734 struct qed_rdma_create_qp_in_params *params) 1735 { 1736 /* QP handle to be written in an async event */ 1737 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1738 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1739 1740 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1741 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1742 params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type); 1743 params->stats_queue = 0; 1744 1745 if (pd) { 1746 params->pd = pd->pd_id; 1747 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1748 } 1749 1750 if (qedr_qp_has_sq(qp)) 1751 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1752 1753 if (qedr_qp_has_rq(qp)) 1754 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1755 1756 if (qedr_qp_has_srq(qp)) { 1757 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1758 params->srq_id = qp->srq->srq_id; 1759 params->use_srq = true; 1760 } else { 1761 params->srq_id = 0; 1762 params->use_srq = false; 1763 } 1764 } 1765 1766 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1767 { 1768 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1769 "qp=%p. " 1770 "sq_addr=0x%llx, " 1771 "sq_len=%zd, " 1772 "rq_addr=0x%llx, " 1773 "rq_len=%zd" 1774 "\n", 1775 qp, 1776 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0, 1777 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0, 1778 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0, 1779 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0); 1780 } 1781 1782 static inline void 1783 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1784 struct qedr_qp *qp, 1785 struct qed_rdma_create_qp_out_params *out_params) 1786 { 1787 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1788 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1789 1790 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1791 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1792 if (!qp->srq) { 1793 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1794 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1795 } 1796 1797 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1798 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1799 } 1800 1801 static void qedr_cleanup_user(struct qedr_dev *dev, 1802 struct qedr_ucontext *ctx, 1803 struct qedr_qp *qp) 1804 { 1805 if (qedr_qp_has_sq(qp)) { 1806 ib_umem_release(qp->usq.umem); 1807 qp->usq.umem = NULL; 1808 } 1809 1810 if (qedr_qp_has_rq(qp)) { 1811 ib_umem_release(qp->urq.umem); 1812 qp->urq.umem = NULL; 1813 } 1814 1815 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1816 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); 1817 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); 1818 } else { 1819 kfree(qp->usq.pbl_tbl); 1820 kfree(qp->urq.pbl_tbl); 1821 } 1822 1823 if (qp->usq.db_rec_data) { 1824 qedr_db_recovery_del(dev, qp->usq.db_addr, 1825 &qp->usq.db_rec_data->db_data); 1826 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry); 1827 } 1828 1829 if (qp->urq.db_rec_data) { 1830 qedr_db_recovery_del(dev, qp->urq.db_addr, 1831 &qp->urq.db_rec_data->db_data); 1832 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry); 1833 } 1834 1835 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1836 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr, 1837 &qp->urq.db_rec_db2_data); 1838 } 1839 1840 static int qedr_create_user_qp(struct qedr_dev *dev, 1841 struct qedr_qp *qp, 1842 struct ib_pd *ibpd, 1843 struct ib_udata *udata, 1844 struct ib_qp_init_attr *attrs) 1845 { 1846 struct qed_rdma_create_qp_in_params in_params; 1847 struct qed_rdma_create_qp_out_params out_params; 1848 struct qedr_create_qp_uresp uresp = {}; 1849 struct qedr_create_qp_ureq ureq = {}; 1850 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1851 struct qedr_ucontext *ctx = NULL; 1852 struct qedr_pd *pd = NULL; 1853 int rc = 0; 1854 1855 qp->create_type = QEDR_QP_CREATE_USER; 1856 1857 if (ibpd) { 1858 pd = get_qedr_pd(ibpd); 1859 ctx = pd->uctx; 1860 } 1861 1862 if (udata) { 1863 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1864 udata->inlen)); 1865 if (rc) { 1866 DP_ERR(dev, "Problem copying data from user space\n"); 1867 return rc; 1868 } 1869 } 1870 1871 if (qedr_qp_has_sq(qp)) { 1872 /* SQ - read access only (0) */ 1873 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1874 ureq.sq_len, true, 0, alloc_and_init); 1875 if (rc) 1876 return rc; 1877 } 1878 1879 if (qedr_qp_has_rq(qp)) { 1880 /* RQ - read access only (0) */ 1881 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1882 ureq.rq_len, true, 0, alloc_and_init); 1883 if (rc) { 1884 ib_umem_release(qp->usq.umem); 1885 qp->usq.umem = NULL; 1886 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1887 qedr_free_pbl(dev, &qp->usq.pbl_info, 1888 qp->usq.pbl_tbl); 1889 } else { 1890 kfree(qp->usq.pbl_tbl); 1891 } 1892 return rc; 1893 } 1894 } 1895 1896 memset(&in_params, 0, sizeof(in_params)); 1897 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1898 in_params.qp_handle_lo = ureq.qp_handle_lo; 1899 in_params.qp_handle_hi = ureq.qp_handle_hi; 1900 1901 if (qp->qp_type == IB_QPT_XRC_TGT) { 1902 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd); 1903 1904 in_params.xrcd_id = xrcd->xrcd_id; 1905 in_params.qp_handle_lo = qp->qp_id; 1906 in_params.use_srq = 1; 1907 } 1908 1909 if (qedr_qp_has_sq(qp)) { 1910 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1911 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1912 } 1913 1914 if (qedr_qp_has_rq(qp)) { 1915 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1916 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1917 } 1918 1919 if (ctx) 1920 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode); 1921 1922 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1923 &in_params, &out_params); 1924 1925 if (!qp->qed_qp) { 1926 rc = -ENOMEM; 1927 goto err1; 1928 } 1929 1930 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1931 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1932 1933 qp->qp_id = out_params.qp_id; 1934 qp->icid = out_params.icid; 1935 1936 if (udata) { 1937 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); 1938 if (rc) 1939 goto err; 1940 } 1941 1942 /* db offset was calculated in copy_qp_uresp, now set in the user q */ 1943 if (qedr_qp_has_sq(qp)) { 1944 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; 1945 qp->sq.max_wr = attrs->cap.max_send_wr; 1946 rc = qedr_db_recovery_add(dev, qp->usq.db_addr, 1947 &qp->usq.db_rec_data->db_data, 1948 DB_REC_WIDTH_32B, 1949 DB_REC_USER); 1950 if (rc) 1951 goto err; 1952 } 1953 1954 if (qedr_qp_has_rq(qp)) { 1955 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; 1956 qp->rq.max_wr = attrs->cap.max_recv_wr; 1957 rc = qedr_db_recovery_add(dev, qp->urq.db_addr, 1958 &qp->urq.db_rec_data->db_data, 1959 DB_REC_WIDTH_32B, 1960 DB_REC_USER); 1961 if (rc) 1962 goto err; 1963 } 1964 1965 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1966 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; 1967 1968 /* calculate the db_rec_db2 data since it is constant so no 1969 * need to reflect from user 1970 */ 1971 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); 1972 qp->urq.db_rec_db2_data.data.value = 1973 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); 1974 1975 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, 1976 &qp->urq.db_rec_db2_data, 1977 DB_REC_WIDTH_32B, 1978 DB_REC_USER); 1979 if (rc) 1980 goto err; 1981 } 1982 qedr_qp_user_print(dev, qp); 1983 return rc; 1984 err: 1985 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1986 if (rc) 1987 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1988 1989 err1: 1990 qedr_cleanup_user(dev, ctx, qp); 1991 return rc; 1992 } 1993 1994 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1995 { 1996 int rc; 1997 1998 qp->sq.db = dev->db_addr + 1999 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 2000 qp->sq.db_data.data.icid = qp->icid; 2001 2002 rc = qedr_db_recovery_add(dev, qp->sq.db, 2003 &qp->sq.db_data, 2004 DB_REC_WIDTH_32B, 2005 DB_REC_KERNEL); 2006 if (rc) 2007 return rc; 2008 2009 qp->rq.db = dev->db_addr + 2010 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 2011 qp->rq.db_data.data.icid = qp->icid; 2012 qp->rq.iwarp_db2 = dev->db_addr + 2013 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2014 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2015 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2016 2017 rc = qedr_db_recovery_add(dev, qp->rq.db, 2018 &qp->rq.db_data, 2019 DB_REC_WIDTH_32B, 2020 DB_REC_KERNEL); 2021 if (rc) 2022 return rc; 2023 2024 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2, 2025 &qp->rq.iwarp_db2_data, 2026 DB_REC_WIDTH_32B, 2027 DB_REC_KERNEL); 2028 return rc; 2029 } 2030 2031 static int 2032 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 2033 struct qedr_qp *qp, 2034 struct qed_rdma_create_qp_in_params *in_params, 2035 u32 n_sq_elems, u32 n_rq_elems) 2036 { 2037 struct qed_rdma_create_qp_out_params out_params; 2038 struct qed_chain_init_params params = { 2039 .mode = QED_CHAIN_MODE_PBL, 2040 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2041 }; 2042 int rc; 2043 2044 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2045 params.num_elems = n_sq_elems; 2046 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2047 2048 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2049 if (rc) 2050 return rc; 2051 2052 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 2053 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 2054 2055 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2056 params.num_elems = n_rq_elems; 2057 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2058 2059 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2060 if (rc) 2061 return rc; 2062 2063 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 2064 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 2065 2066 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2067 in_params, &out_params); 2068 2069 if (!qp->qed_qp) 2070 return -EINVAL; 2071 2072 qp->qp_id = out_params.qp_id; 2073 qp->icid = out_params.icid; 2074 2075 return qedr_set_roce_db_info(dev, qp); 2076 } 2077 2078 static int 2079 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 2080 struct qedr_qp *qp, 2081 struct qed_rdma_create_qp_in_params *in_params, 2082 u32 n_sq_elems, u32 n_rq_elems) 2083 { 2084 struct qed_rdma_create_qp_out_params out_params; 2085 struct qed_chain_init_params params = { 2086 .mode = QED_CHAIN_MODE_PBL, 2087 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2088 }; 2089 int rc; 2090 2091 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 2092 QEDR_SQE_ELEMENT_SIZE, 2093 QED_CHAIN_PAGE_SIZE, 2094 QED_CHAIN_MODE_PBL); 2095 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 2096 QEDR_RQE_ELEMENT_SIZE, 2097 QED_CHAIN_PAGE_SIZE, 2098 QED_CHAIN_MODE_PBL); 2099 2100 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2101 in_params, &out_params); 2102 2103 if (!qp->qed_qp) 2104 return -EINVAL; 2105 2106 /* Now we allocate the chain */ 2107 2108 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2109 params.num_elems = n_sq_elems; 2110 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2111 params.ext_pbl_virt = out_params.sq_pbl_virt; 2112 params.ext_pbl_phys = out_params.sq_pbl_phys; 2113 2114 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2115 if (rc) 2116 goto err; 2117 2118 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2119 params.num_elems = n_rq_elems; 2120 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2121 params.ext_pbl_virt = out_params.rq_pbl_virt; 2122 params.ext_pbl_phys = out_params.rq_pbl_phys; 2123 2124 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2125 if (rc) 2126 goto err; 2127 2128 qp->qp_id = out_params.qp_id; 2129 qp->icid = out_params.icid; 2130 2131 return qedr_set_iwarp_db_info(dev, qp); 2132 2133 err: 2134 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2135 2136 return rc; 2137 } 2138 2139 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 2140 { 2141 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 2142 kfree(qp->wqe_wr_id); 2143 2144 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 2145 kfree(qp->rqe_wr_id); 2146 2147 /* GSI qp is not registered to db mechanism so no need to delete */ 2148 if (qp->qp_type == IB_QPT_GSI) 2149 return; 2150 2151 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 2152 2153 if (!qp->srq) { 2154 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data); 2155 2156 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2157 qedr_db_recovery_del(dev, qp->rq.iwarp_db2, 2158 &qp->rq.iwarp_db2_data); 2159 } 2160 } 2161 2162 static int qedr_create_kernel_qp(struct qedr_dev *dev, 2163 struct qedr_qp *qp, 2164 struct ib_pd *ibpd, 2165 struct ib_qp_init_attr *attrs) 2166 { 2167 struct qed_rdma_create_qp_in_params in_params; 2168 struct qedr_pd *pd = get_qedr_pd(ibpd); 2169 int rc = -EINVAL; 2170 u32 n_rq_elems; 2171 u32 n_sq_elems; 2172 u32 n_sq_entries; 2173 2174 memset(&in_params, 0, sizeof(in_params)); 2175 qp->create_type = QEDR_QP_CREATE_KERNEL; 2176 2177 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 2178 * the ring. The ring should allow at least a single WR, even if the 2179 * user requested none, due to allocation issues. 2180 * We should add an extra WR since the prod and cons indices of 2181 * wqe_wr_id are managed in such a way that the WQ is considered full 2182 * when (prod+1)%max_wr==cons. We currently don't do that because we 2183 * double the number of entries due an iSER issue that pushes far more 2184 * WRs than indicated. If we decline its ib_post_send() then we get 2185 * error prints in the dmesg we'd like to avoid. 2186 */ 2187 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2188 dev->attr.max_sqe); 2189 2190 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 2191 GFP_KERNEL); 2192 if (!qp->wqe_wr_id) { 2193 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 2194 return -ENOMEM; 2195 } 2196 2197 /* QP handle to be written in CQE */ 2198 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 2199 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 2200 2201 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 2202 * the ring. There ring should allow at least a single WR, even if the 2203 * user requested none, due to allocation issues. 2204 */ 2205 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 2206 2207 /* Allocate driver internal RQ array */ 2208 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 2209 GFP_KERNEL); 2210 if (!qp->rqe_wr_id) { 2211 DP_ERR(dev, 2212 "create qp: failed RQ shadow memory allocation\n"); 2213 kfree(qp->wqe_wr_id); 2214 return -ENOMEM; 2215 } 2216 2217 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 2218 2219 n_sq_entries = attrs->cap.max_send_wr; 2220 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 2221 n_sq_entries = max_t(u32, n_sq_entries, 1); 2222 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2223 2224 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 2225 2226 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2227 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 2228 n_sq_elems, n_rq_elems); 2229 else 2230 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 2231 n_sq_elems, n_rq_elems); 2232 if (rc) 2233 qedr_cleanup_kernel(dev, qp); 2234 2235 return rc; 2236 } 2237 2238 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2239 struct ib_udata *udata) 2240 { 2241 struct qedr_ucontext *ctx = 2242 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 2243 ibucontext); 2244 int rc; 2245 2246 if (qp->qp_type != IB_QPT_GSI) { 2247 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2248 if (rc) 2249 return rc; 2250 } 2251 2252 if (qp->create_type == QEDR_QP_CREATE_USER) 2253 qedr_cleanup_user(dev, ctx, qp); 2254 else 2255 qedr_cleanup_kernel(dev, qp); 2256 2257 return 0; 2258 } 2259 2260 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 2261 struct ib_udata *udata) 2262 { 2263 struct qedr_xrcd *xrcd = NULL; 2264 struct ib_pd *ibpd = ibqp->pd; 2265 struct qedr_pd *pd = get_qedr_pd(ibpd); 2266 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2267 struct qedr_qp *qp = get_qedr_qp(ibqp); 2268 int rc = 0; 2269 2270 if (attrs->create_flags) 2271 return -EOPNOTSUPP; 2272 2273 if (attrs->qp_type == IB_QPT_XRC_TGT) 2274 xrcd = get_qedr_xrcd(attrs->xrcd); 2275 else 2276 pd = get_qedr_pd(ibpd); 2277 2278 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2279 udata ? "user library" : "kernel", pd); 2280 2281 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 2282 if (rc) 2283 return rc; 2284 2285 DP_DEBUG(dev, QEDR_MSG_QP, 2286 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2287 udata ? "user library" : "kernel", attrs->event_handler, pd, 2288 get_qedr_cq(attrs->send_cq), 2289 get_qedr_cq(attrs->send_cq)->icid, 2290 get_qedr_cq(attrs->recv_cq), 2291 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2292 2293 qedr_set_common_qp_params(dev, qp, pd, attrs); 2294 2295 if (attrs->qp_type == IB_QPT_GSI) 2296 return qedr_create_gsi_qp(dev, attrs, qp); 2297 2298 if (udata || xrcd) 2299 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2300 else 2301 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2302 2303 if (rc) 2304 return rc; 2305 2306 qp->ibqp.qp_num = qp->qp_id; 2307 2308 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2309 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 2310 if (rc) 2311 goto out_free_qp_resources; 2312 } 2313 2314 return 0; 2315 2316 out_free_qp_resources: 2317 qedr_free_qp_resources(dev, qp, udata); 2318 return -EFAULT; 2319 } 2320 2321 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2322 { 2323 switch (qp_state) { 2324 case QED_ROCE_QP_STATE_RESET: 2325 return IB_QPS_RESET; 2326 case QED_ROCE_QP_STATE_INIT: 2327 return IB_QPS_INIT; 2328 case QED_ROCE_QP_STATE_RTR: 2329 return IB_QPS_RTR; 2330 case QED_ROCE_QP_STATE_RTS: 2331 return IB_QPS_RTS; 2332 case QED_ROCE_QP_STATE_SQD: 2333 return IB_QPS_SQD; 2334 case QED_ROCE_QP_STATE_ERR: 2335 return IB_QPS_ERR; 2336 case QED_ROCE_QP_STATE_SQE: 2337 return IB_QPS_SQE; 2338 } 2339 return IB_QPS_ERR; 2340 } 2341 2342 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2343 enum ib_qp_state qp_state) 2344 { 2345 switch (qp_state) { 2346 case IB_QPS_RESET: 2347 return QED_ROCE_QP_STATE_RESET; 2348 case IB_QPS_INIT: 2349 return QED_ROCE_QP_STATE_INIT; 2350 case IB_QPS_RTR: 2351 return QED_ROCE_QP_STATE_RTR; 2352 case IB_QPS_RTS: 2353 return QED_ROCE_QP_STATE_RTS; 2354 case IB_QPS_SQD: 2355 return QED_ROCE_QP_STATE_SQD; 2356 case IB_QPS_ERR: 2357 return QED_ROCE_QP_STATE_ERR; 2358 default: 2359 return QED_ROCE_QP_STATE_ERR; 2360 } 2361 } 2362 2363 static int qedr_update_qp_state(struct qedr_dev *dev, 2364 struct qedr_qp *qp, 2365 enum qed_roce_qp_state cur_state, 2366 enum qed_roce_qp_state new_state) 2367 { 2368 int status = 0; 2369 2370 if (new_state == cur_state) 2371 return 0; 2372 2373 switch (cur_state) { 2374 case QED_ROCE_QP_STATE_RESET: 2375 switch (new_state) { 2376 case QED_ROCE_QP_STATE_INIT: 2377 break; 2378 default: 2379 status = -EINVAL; 2380 break; 2381 } 2382 break; 2383 case QED_ROCE_QP_STATE_INIT: 2384 switch (new_state) { 2385 case QED_ROCE_QP_STATE_RTR: 2386 /* Update doorbell (in case post_recv was 2387 * done before move to RTR) 2388 */ 2389 2390 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2391 writel(qp->rq.db_data.raw, qp->rq.db); 2392 } 2393 break; 2394 case QED_ROCE_QP_STATE_ERR: 2395 break; 2396 default: 2397 /* Invalid state change. */ 2398 status = -EINVAL; 2399 break; 2400 } 2401 break; 2402 case QED_ROCE_QP_STATE_RTR: 2403 /* RTR->XXX */ 2404 switch (new_state) { 2405 case QED_ROCE_QP_STATE_RTS: 2406 break; 2407 case QED_ROCE_QP_STATE_ERR: 2408 break; 2409 default: 2410 /* Invalid state change. */ 2411 status = -EINVAL; 2412 break; 2413 } 2414 break; 2415 case QED_ROCE_QP_STATE_RTS: 2416 /* RTS->XXX */ 2417 switch (new_state) { 2418 case QED_ROCE_QP_STATE_SQD: 2419 break; 2420 case QED_ROCE_QP_STATE_ERR: 2421 break; 2422 default: 2423 /* Invalid state change. */ 2424 status = -EINVAL; 2425 break; 2426 } 2427 break; 2428 case QED_ROCE_QP_STATE_SQD: 2429 /* SQD->XXX */ 2430 switch (new_state) { 2431 case QED_ROCE_QP_STATE_RTS: 2432 case QED_ROCE_QP_STATE_ERR: 2433 break; 2434 default: 2435 /* Invalid state change. */ 2436 status = -EINVAL; 2437 break; 2438 } 2439 break; 2440 case QED_ROCE_QP_STATE_ERR: 2441 /* ERR->XXX */ 2442 switch (new_state) { 2443 case QED_ROCE_QP_STATE_RESET: 2444 if ((qp->rq.prod != qp->rq.cons) || 2445 (qp->sq.prod != qp->sq.cons)) { 2446 DP_NOTICE(dev, 2447 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2448 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2449 qp->sq.cons); 2450 status = -EINVAL; 2451 } 2452 break; 2453 default: 2454 status = -EINVAL; 2455 break; 2456 } 2457 break; 2458 default: 2459 status = -EINVAL; 2460 break; 2461 } 2462 2463 return status; 2464 } 2465 2466 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2467 int attr_mask, struct ib_udata *udata) 2468 { 2469 struct qedr_qp *qp = get_qedr_qp(ibqp); 2470 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2471 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2472 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2473 enum ib_qp_state old_qp_state, new_qp_state; 2474 enum qed_roce_qp_state cur_state; 2475 int rc = 0; 2476 2477 DP_DEBUG(dev, QEDR_MSG_QP, 2478 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2479 attr->qp_state); 2480 2481 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 2482 return -EOPNOTSUPP; 2483 2484 old_qp_state = qedr_get_ibqp_state(qp->state); 2485 if (attr_mask & IB_QP_STATE) 2486 new_qp_state = attr->qp_state; 2487 else 2488 new_qp_state = old_qp_state; 2489 2490 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2491 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2492 ibqp->qp_type, attr_mask)) { 2493 DP_ERR(dev, 2494 "modify qp: invalid attribute mask=0x%x specified for\n" 2495 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2496 attr_mask, qp->qp_id, ibqp->qp_type, 2497 old_qp_state, new_qp_state); 2498 rc = -EINVAL; 2499 goto err; 2500 } 2501 } 2502 2503 /* Translate the masks... */ 2504 if (attr_mask & IB_QP_STATE) { 2505 SET_FIELD(qp_params.modify_flags, 2506 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2507 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2508 } 2509 2510 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2511 qp_params.sqd_async = true; 2512 2513 if (attr_mask & IB_QP_PKEY_INDEX) { 2514 SET_FIELD(qp_params.modify_flags, 2515 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2516 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2517 rc = -EINVAL; 2518 goto err; 2519 } 2520 2521 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2522 } 2523 2524 if (attr_mask & IB_QP_QKEY) 2525 qp->qkey = attr->qkey; 2526 2527 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2528 SET_FIELD(qp_params.modify_flags, 2529 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2530 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2531 IB_ACCESS_REMOTE_READ; 2532 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2533 IB_ACCESS_REMOTE_WRITE; 2534 qp_params.incoming_atomic_en = attr->qp_access_flags & 2535 IB_ACCESS_REMOTE_ATOMIC; 2536 } 2537 2538 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2539 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2540 return -EINVAL; 2541 2542 if (attr_mask & IB_QP_PATH_MTU) { 2543 if (attr->path_mtu < IB_MTU_256 || 2544 attr->path_mtu > IB_MTU_4096) { 2545 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2546 rc = -EINVAL; 2547 goto err; 2548 } 2549 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2550 ib_mtu_enum_to_int(iboe_get_mtu 2551 (dev->ndev->mtu))); 2552 } 2553 2554 if (!qp->mtu) { 2555 qp->mtu = 2556 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2557 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2558 } 2559 2560 SET_FIELD(qp_params.modify_flags, 2561 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2562 2563 qp_params.traffic_class_tos = grh->traffic_class; 2564 qp_params.flow_label = grh->flow_label; 2565 qp_params.hop_limit_ttl = grh->hop_limit; 2566 2567 qp->sgid_idx = grh->sgid_index; 2568 2569 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2570 if (rc) { 2571 DP_ERR(dev, 2572 "modify qp: problems with GID index %d (rc=%d)\n", 2573 grh->sgid_index, rc); 2574 return rc; 2575 } 2576 2577 rc = qedr_get_dmac(dev, &attr->ah_attr, 2578 qp_params.remote_mac_addr); 2579 if (rc) 2580 return rc; 2581 2582 qp_params.use_local_mac = true; 2583 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2584 2585 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2586 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2587 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2588 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2589 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2590 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2591 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2592 qp_params.remote_mac_addr); 2593 2594 qp_params.mtu = qp->mtu; 2595 qp_params.lb_indication = false; 2596 } 2597 2598 if (!qp_params.mtu) { 2599 /* Stay with current MTU */ 2600 if (qp->mtu) 2601 qp_params.mtu = qp->mtu; 2602 else 2603 qp_params.mtu = 2604 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2605 } 2606 2607 if (attr_mask & IB_QP_TIMEOUT) { 2608 SET_FIELD(qp_params.modify_flags, 2609 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2610 2611 /* The received timeout value is an exponent used like this: 2612 * "12.7.34 LOCAL ACK TIMEOUT 2613 * Value representing the transport (ACK) timeout for use by 2614 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2615 * The FW expects timeout in msec so we need to divide the usec 2616 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2617 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2618 * The value of zero means infinite so we use a 'max_t' to make 2619 * sure that sub 1 msec values will be configured as 1 msec. 2620 */ 2621 if (attr->timeout) 2622 qp_params.ack_timeout = 2623 1 << max_t(int, attr->timeout - 8, 0); 2624 else 2625 qp_params.ack_timeout = 0; 2626 2627 qp->timeout = attr->timeout; 2628 } 2629 2630 if (attr_mask & IB_QP_RETRY_CNT) { 2631 SET_FIELD(qp_params.modify_flags, 2632 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2633 qp_params.retry_cnt = attr->retry_cnt; 2634 } 2635 2636 if (attr_mask & IB_QP_RNR_RETRY) { 2637 SET_FIELD(qp_params.modify_flags, 2638 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2639 qp_params.rnr_retry_cnt = attr->rnr_retry; 2640 } 2641 2642 if (attr_mask & IB_QP_RQ_PSN) { 2643 SET_FIELD(qp_params.modify_flags, 2644 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2645 qp_params.rq_psn = attr->rq_psn; 2646 qp->rq_psn = attr->rq_psn; 2647 } 2648 2649 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2650 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2651 rc = -EINVAL; 2652 DP_ERR(dev, 2653 "unsupported max_rd_atomic=%d, supported=%d\n", 2654 attr->max_rd_atomic, 2655 dev->attr.max_qp_req_rd_atomic_resc); 2656 goto err; 2657 } 2658 2659 SET_FIELD(qp_params.modify_flags, 2660 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2661 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2662 } 2663 2664 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2665 SET_FIELD(qp_params.modify_flags, 2666 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2667 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2668 } 2669 2670 if (attr_mask & IB_QP_SQ_PSN) { 2671 SET_FIELD(qp_params.modify_flags, 2672 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2673 qp_params.sq_psn = attr->sq_psn; 2674 qp->sq_psn = attr->sq_psn; 2675 } 2676 2677 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2678 if (attr->max_dest_rd_atomic > 2679 dev->attr.max_qp_resp_rd_atomic_resc) { 2680 DP_ERR(dev, 2681 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2682 attr->max_dest_rd_atomic, 2683 dev->attr.max_qp_resp_rd_atomic_resc); 2684 2685 rc = -EINVAL; 2686 goto err; 2687 } 2688 2689 SET_FIELD(qp_params.modify_flags, 2690 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2691 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2692 } 2693 2694 if (attr_mask & IB_QP_DEST_QPN) { 2695 SET_FIELD(qp_params.modify_flags, 2696 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2697 2698 qp_params.dest_qp = attr->dest_qp_num; 2699 qp->dest_qp_num = attr->dest_qp_num; 2700 } 2701 2702 cur_state = qp->state; 2703 2704 /* Update the QP state before the actual ramrod to prevent a race with 2705 * fast path. Modifying the QP state to error will cause the device to 2706 * flush the CQEs and while polling the flushed CQEs will considered as 2707 * a potential issue if the QP isn't in error state. 2708 */ 2709 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2710 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2711 qp->state = QED_ROCE_QP_STATE_ERR; 2712 2713 if (qp->qp_type != IB_QPT_GSI) 2714 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2715 qp->qed_qp, &qp_params); 2716 2717 if (attr_mask & IB_QP_STATE) { 2718 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2719 rc = qedr_update_qp_state(dev, qp, cur_state, 2720 qp_params.new_state); 2721 qp->state = qp_params.new_state; 2722 } 2723 2724 err: 2725 return rc; 2726 } 2727 2728 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2729 { 2730 int ib_qp_acc_flags = 0; 2731 2732 if (params->incoming_rdma_write_en) 2733 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2734 if (params->incoming_rdma_read_en) 2735 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2736 if (params->incoming_atomic_en) 2737 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2738 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2739 return ib_qp_acc_flags; 2740 } 2741 2742 int qedr_query_qp(struct ib_qp *ibqp, 2743 struct ib_qp_attr *qp_attr, 2744 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2745 { 2746 struct qed_rdma_query_qp_out_params params; 2747 struct qedr_qp *qp = get_qedr_qp(ibqp); 2748 struct qedr_dev *dev = qp->dev; 2749 int rc = 0; 2750 2751 memset(¶ms, 0, sizeof(params)); 2752 memset(qp_attr, 0, sizeof(*qp_attr)); 2753 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2754 2755 if (qp->qp_type != IB_QPT_GSI) { 2756 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2757 if (rc) 2758 goto err; 2759 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2760 } else { 2761 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS); 2762 } 2763 2764 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2765 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2766 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2767 qp_attr->rq_psn = params.rq_psn; 2768 qp_attr->sq_psn = params.sq_psn; 2769 qp_attr->dest_qp_num = params.dest_qp; 2770 2771 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2772 2773 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2774 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2775 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2776 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2777 qp_attr->cap.max_inline_data = dev->attr.max_inline; 2778 qp_init_attr->cap = qp_attr->cap; 2779 2780 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2781 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2782 params.flow_label, qp->sgid_idx, 2783 params.hop_limit_ttl, params.traffic_class_tos); 2784 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2785 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2786 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2787 qp_attr->timeout = qp->timeout; 2788 qp_attr->rnr_retry = params.rnr_retry; 2789 qp_attr->retry_cnt = params.retry_cnt; 2790 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2791 qp_attr->pkey_index = params.pkey_index; 2792 qp_attr->port_num = 1; 2793 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2794 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2795 qp_attr->alt_pkey_index = 0; 2796 qp_attr->alt_port_num = 0; 2797 qp_attr->alt_timeout = 0; 2798 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2799 2800 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2801 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2802 qp_attr->max_rd_atomic = params.max_rd_atomic; 2803 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2804 2805 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2806 qp_attr->cap.max_inline_data); 2807 2808 err: 2809 return rc; 2810 } 2811 2812 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2813 { 2814 struct qedr_qp *qp = get_qedr_qp(ibqp); 2815 struct qedr_dev *dev = qp->dev; 2816 struct ib_qp_attr attr; 2817 int attr_mask = 0; 2818 2819 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2820 qp, qp->qp_type); 2821 2822 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2823 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2824 (qp->state != QED_ROCE_QP_STATE_ERR) && 2825 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2826 2827 attr.qp_state = IB_QPS_ERR; 2828 attr_mask |= IB_QP_STATE; 2829 2830 /* Change the QP state to ERROR */ 2831 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2832 } 2833 } else { 2834 /* If connection establishment started the WAIT_FOR_CONNECT 2835 * bit will be on and we need to Wait for the establishment 2836 * to complete before destroying the qp. 2837 */ 2838 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, 2839 &qp->iwarp_cm_flags)) 2840 wait_for_completion(&qp->iwarp_cm_comp); 2841 2842 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT 2843 * bit will be on, and we need to wait for the disconnect to 2844 * complete before continuing. We can use the same completion, 2845 * iwarp_cm_comp, since this is the only place that waits for 2846 * this completion and it is sequential. In addition, 2847 * disconnect can't occur before the connection is fully 2848 * established, therefore if WAIT_FOR_DISCONNECT is on it 2849 * means WAIT_FOR_CONNECT is also on and the completion for 2850 * CONNECT already occurred. 2851 */ 2852 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT, 2853 &qp->iwarp_cm_flags)) 2854 wait_for_completion(&qp->iwarp_cm_comp); 2855 } 2856 2857 if (qp->qp_type == IB_QPT_GSI) 2858 qedr_destroy_gsi_qp(dev); 2859 2860 /* We need to remove the entry from the xarray before we release the 2861 * qp_id to avoid a race of the qp_id being reallocated and failing 2862 * on xa_insert 2863 */ 2864 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2865 xa_erase(&dev->qps, qp->qp_id); 2866 2867 qedr_free_qp_resources(dev, qp, udata); 2868 2869 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2870 qedr_iw_qp_rem_ref(&qp->ibqp); 2871 wait_for_completion(&qp->qp_rel_comp); 2872 } 2873 2874 return 0; 2875 } 2876 2877 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 2878 struct ib_udata *udata) 2879 { 2880 struct qedr_ah *ah = get_qedr_ah(ibah); 2881 2882 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr); 2883 2884 return 0; 2885 } 2886 2887 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2888 { 2889 struct qedr_ah *ah = get_qedr_ah(ibah); 2890 2891 rdma_destroy_ah_attr(&ah->attr); 2892 return 0; 2893 } 2894 2895 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2896 { 2897 struct qedr_pbl *pbl, *tmp; 2898 2899 if (info->pbl_table) 2900 list_add_tail(&info->pbl_table->list_entry, 2901 &info->free_pbl_list); 2902 2903 if (!list_empty(&info->inuse_pbl_list)) 2904 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2905 2906 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2907 list_del(&pbl->list_entry); 2908 qedr_free_pbl(dev, &info->pbl_info, pbl); 2909 } 2910 } 2911 2912 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2913 size_t page_list_len, bool two_layered) 2914 { 2915 struct qedr_pbl *tmp; 2916 int rc; 2917 2918 INIT_LIST_HEAD(&info->free_pbl_list); 2919 INIT_LIST_HEAD(&info->inuse_pbl_list); 2920 2921 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2922 page_list_len, two_layered); 2923 if (rc) 2924 goto done; 2925 2926 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2927 if (IS_ERR(info->pbl_table)) { 2928 rc = PTR_ERR(info->pbl_table); 2929 goto done; 2930 } 2931 2932 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2933 &info->pbl_table->pa); 2934 2935 /* in usual case we use 2 PBLs, so we add one to free 2936 * list and allocating another one 2937 */ 2938 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2939 if (IS_ERR(tmp)) { 2940 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2941 goto done; 2942 } 2943 2944 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2945 2946 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2947 2948 done: 2949 if (rc) 2950 free_mr_info(dev, info); 2951 2952 return rc; 2953 } 2954 2955 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2956 u64 usr_addr, int acc, struct ib_dmah *dmah, 2957 struct ib_udata *udata) 2958 { 2959 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2960 struct qedr_mr *mr; 2961 struct qedr_pd *pd; 2962 int rc = -ENOMEM; 2963 2964 if (dmah) 2965 return ERR_PTR(-EOPNOTSUPP); 2966 2967 pd = get_qedr_pd(ibpd); 2968 DP_DEBUG(dev, QEDR_MSG_MR, 2969 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2970 pd->pd_id, start, len, usr_addr, acc); 2971 2972 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2973 return ERR_PTR(-EINVAL); 2974 2975 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2976 if (!mr) 2977 return ERR_PTR(rc); 2978 2979 mr->type = QEDR_MR_USER; 2980 2981 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 2982 if (IS_ERR(mr->umem)) { 2983 rc = -EFAULT; 2984 goto err0; 2985 } 2986 2987 rc = init_mr_info(dev, &mr->info, 2988 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1); 2989 if (rc) 2990 goto err1; 2991 2992 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2993 &mr->info.pbl_info, PAGE_SHIFT); 2994 2995 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2996 if (rc) { 2997 if (rc == -EINVAL) 2998 DP_ERR(dev, "Out of MR resources\n"); 2999 else 3000 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3001 3002 goto err1; 3003 } 3004 3005 /* Index only, 18 bit long, lkey = itid << 8 | key */ 3006 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3007 mr->hw_mr.key = 0; 3008 mr->hw_mr.pd = pd->pd_id; 3009 mr->hw_mr.local_read = 1; 3010 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3011 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3012 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3013 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3014 mr->hw_mr.mw_bind = false; 3015 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 3016 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3017 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3018 mr->hw_mr.page_size_log = PAGE_SHIFT; 3019 mr->hw_mr.length = len; 3020 mr->hw_mr.vaddr = usr_addr; 3021 mr->hw_mr.phy_mr = false; 3022 mr->hw_mr.dma_mr = false; 3023 3024 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3025 if (rc) { 3026 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3027 goto err2; 3028 } 3029 3030 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3031 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3032 mr->hw_mr.remote_atomic) 3033 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3034 3035 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 3036 mr->ibmr.lkey); 3037 return &mr->ibmr; 3038 3039 err2: 3040 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3041 err1: 3042 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3043 err0: 3044 kfree(mr); 3045 return ERR_PTR(rc); 3046 } 3047 3048 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3049 { 3050 struct qedr_mr *mr = get_qedr_mr(ib_mr); 3051 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 3052 int rc = 0; 3053 3054 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 3055 if (rc) 3056 return rc; 3057 3058 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3059 3060 if (mr->type != QEDR_MR_DMA) 3061 free_mr_info(dev, &mr->info); 3062 3063 /* it could be user registered memory. */ 3064 ib_umem_release(mr->umem); 3065 3066 kfree(mr); 3067 3068 return rc; 3069 } 3070 3071 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 3072 int max_page_list_len) 3073 { 3074 struct qedr_pd *pd = get_qedr_pd(ibpd); 3075 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3076 struct qedr_mr *mr; 3077 int rc = -ENOMEM; 3078 3079 DP_DEBUG(dev, QEDR_MSG_MR, 3080 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 3081 max_page_list_len); 3082 3083 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3084 if (!mr) 3085 return ERR_PTR(rc); 3086 3087 mr->dev = dev; 3088 mr->type = QEDR_MR_FRMR; 3089 3090 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 3091 if (rc) 3092 goto err0; 3093 3094 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3095 if (rc) { 3096 if (rc == -EINVAL) 3097 DP_ERR(dev, "Out of MR resources\n"); 3098 else 3099 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3100 3101 goto err1; 3102 } 3103 3104 /* Index only, 18 bit long, lkey = itid << 8 | key */ 3105 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 3106 mr->hw_mr.key = 0; 3107 mr->hw_mr.pd = pd->pd_id; 3108 mr->hw_mr.local_read = 1; 3109 mr->hw_mr.local_write = 0; 3110 mr->hw_mr.remote_read = 0; 3111 mr->hw_mr.remote_write = 0; 3112 mr->hw_mr.remote_atomic = 0; 3113 mr->hw_mr.mw_bind = false; 3114 mr->hw_mr.pbl_ptr = 0; 3115 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3116 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3117 mr->hw_mr.length = 0; 3118 mr->hw_mr.vaddr = 0; 3119 mr->hw_mr.phy_mr = true; 3120 mr->hw_mr.dma_mr = false; 3121 3122 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3123 if (rc) { 3124 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3125 goto err2; 3126 } 3127 3128 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3129 mr->ibmr.rkey = mr->ibmr.lkey; 3130 3131 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 3132 return mr; 3133 3134 err2: 3135 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3136 err1: 3137 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3138 err0: 3139 kfree(mr); 3140 return ERR_PTR(rc); 3141 } 3142 3143 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 3144 u32 max_num_sg) 3145 { 3146 struct qedr_mr *mr; 3147 3148 if (mr_type != IB_MR_TYPE_MEM_REG) 3149 return ERR_PTR(-EINVAL); 3150 3151 mr = __qedr_alloc_mr(ibpd, max_num_sg); 3152 3153 if (IS_ERR(mr)) 3154 return ERR_PTR(-EINVAL); 3155 3156 return &mr->ibmr; 3157 } 3158 3159 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 3160 { 3161 struct qedr_mr *mr = get_qedr_mr(ibmr); 3162 struct qedr_pbl *pbl_table; 3163 struct regpair *pbe; 3164 u32 pbes_in_page; 3165 3166 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 3167 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 3168 return -ENOMEM; 3169 } 3170 3171 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 3172 mr->npages, addr); 3173 3174 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 3175 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 3176 pbe = (struct regpair *)pbl_table->va; 3177 pbe += mr->npages % pbes_in_page; 3178 pbe->lo = cpu_to_le32((u32)addr); 3179 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 3180 3181 mr->npages++; 3182 3183 return 0; 3184 } 3185 3186 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 3187 { 3188 int work = info->completed - info->completed_handled - 1; 3189 3190 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 3191 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 3192 struct qedr_pbl *pbl; 3193 3194 /* Free all the page list that are possible to be freed 3195 * (all the ones that were invalidated), under the assumption 3196 * that if an FMR was completed successfully that means that 3197 * if there was an invalidate operation before it also ended 3198 */ 3199 pbl = list_first_entry(&info->inuse_pbl_list, 3200 struct qedr_pbl, list_entry); 3201 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 3202 info->completed_handled++; 3203 } 3204 } 3205 3206 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3207 int sg_nents, unsigned int *sg_offset) 3208 { 3209 struct qedr_mr *mr = get_qedr_mr(ibmr); 3210 3211 mr->npages = 0; 3212 3213 handle_completed_mrs(mr->dev, &mr->info); 3214 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 3215 } 3216 3217 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 3218 { 3219 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3220 struct qedr_pd *pd = get_qedr_pd(ibpd); 3221 struct qedr_mr *mr; 3222 int rc; 3223 3224 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3225 if (!mr) 3226 return ERR_PTR(-ENOMEM); 3227 3228 mr->type = QEDR_MR_DMA; 3229 3230 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3231 if (rc) { 3232 if (rc == -EINVAL) 3233 DP_ERR(dev, "Out of MR resources\n"); 3234 else 3235 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3236 3237 goto err1; 3238 } 3239 3240 /* index only, 18 bit long, lkey = itid << 8 | key */ 3241 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3242 mr->hw_mr.pd = pd->pd_id; 3243 mr->hw_mr.local_read = 1; 3244 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3245 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3246 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3247 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3248 mr->hw_mr.dma_mr = true; 3249 3250 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3251 if (rc) { 3252 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3253 goto err2; 3254 } 3255 3256 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3257 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3258 mr->hw_mr.remote_atomic) 3259 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3260 3261 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3262 return &mr->ibmr; 3263 3264 err2: 3265 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3266 err1: 3267 kfree(mr); 3268 return ERR_PTR(rc); 3269 } 3270 3271 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3272 { 3273 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3274 } 3275 3276 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3277 { 3278 int i, len = 0; 3279 3280 for (i = 0; i < num_sge; i++) 3281 len += sg_list[i].length; 3282 3283 return len; 3284 } 3285 3286 static void swap_wqe_data64(u64 *p) 3287 { 3288 int i; 3289 3290 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3291 *p = cpu_to_be64(cpu_to_le64(*p)); 3292 } 3293 3294 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3295 struct qedr_qp *qp, u8 *wqe_size, 3296 const struct ib_send_wr *wr, 3297 const struct ib_send_wr **bad_wr, 3298 u8 *bits, u8 bit) 3299 { 3300 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3301 char *seg_prt, *wqe; 3302 int i, seg_siz; 3303 3304 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3305 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3306 *bad_wr = wr; 3307 return 0; 3308 } 3309 3310 if (!data_size) 3311 return data_size; 3312 3313 *bits |= bit; 3314 3315 seg_prt = NULL; 3316 wqe = NULL; 3317 seg_siz = 0; 3318 3319 /* Copy data inline */ 3320 for (i = 0; i < wr->num_sge; i++) { 3321 u32 len = wr->sg_list[i].length; 3322 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3323 3324 while (len > 0) { 3325 u32 cur; 3326 3327 /* New segment required */ 3328 if (!seg_siz) { 3329 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3330 seg_prt = wqe; 3331 seg_siz = sizeof(struct rdma_sq_common_wqe); 3332 (*wqe_size)++; 3333 } 3334 3335 /* Calculate currently allowed length */ 3336 cur = min_t(u32, len, seg_siz); 3337 memcpy(seg_prt, src, cur); 3338 3339 /* Update segment variables */ 3340 seg_prt += cur; 3341 seg_siz -= cur; 3342 3343 /* Update sge variables */ 3344 src += cur; 3345 len -= cur; 3346 3347 /* Swap fully-completed segments */ 3348 if (!seg_siz) 3349 swap_wqe_data64((u64 *)wqe); 3350 } 3351 } 3352 3353 /* swap last not completed segment */ 3354 if (seg_siz) 3355 swap_wqe_data64((u64 *)wqe); 3356 3357 return data_size; 3358 } 3359 3360 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3361 do { \ 3362 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3363 (sge)->length = cpu_to_le32(vlength); \ 3364 (sge)->flags = cpu_to_le32(vflags); \ 3365 } while (0) 3366 3367 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3368 do { \ 3369 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3370 (hdr)->num_sges = num_sge; \ 3371 } while (0) 3372 3373 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3374 do { \ 3375 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3376 (sge)->length = cpu_to_le32(vlength); \ 3377 (sge)->l_key = cpu_to_le32(vlkey); \ 3378 } while (0) 3379 3380 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3381 const struct ib_send_wr *wr) 3382 { 3383 u32 data_size = 0; 3384 int i; 3385 3386 for (i = 0; i < wr->num_sge; i++) { 3387 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3388 3389 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3390 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3391 sge->length = cpu_to_le32(wr->sg_list[i].length); 3392 data_size += wr->sg_list[i].length; 3393 } 3394 3395 if (wqe_size) 3396 *wqe_size += wr->num_sge; 3397 3398 return data_size; 3399 } 3400 3401 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3402 struct qedr_qp *qp, 3403 struct rdma_sq_rdma_wqe_1st *rwqe, 3404 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3405 const struct ib_send_wr *wr, 3406 const struct ib_send_wr **bad_wr) 3407 { 3408 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3409 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3410 3411 if (wr->send_flags & IB_SEND_INLINE && 3412 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3413 wr->opcode == IB_WR_RDMA_WRITE)) { 3414 u8 flags = 0; 3415 3416 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3417 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3418 bad_wr, &rwqe->flags, flags); 3419 } 3420 3421 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3422 } 3423 3424 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3425 struct qedr_qp *qp, 3426 struct rdma_sq_send_wqe_1st *swqe, 3427 struct rdma_sq_send_wqe_2st *swqe2, 3428 const struct ib_send_wr *wr, 3429 const struct ib_send_wr **bad_wr) 3430 { 3431 memset(swqe2, 0, sizeof(*swqe2)); 3432 if (wr->send_flags & IB_SEND_INLINE) { 3433 u8 flags = 0; 3434 3435 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3436 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3437 bad_wr, &swqe->flags, flags); 3438 } 3439 3440 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3441 } 3442 3443 static int qedr_prepare_reg(struct qedr_qp *qp, 3444 struct rdma_sq_fmr_wqe_1st *fwqe1, 3445 const struct ib_reg_wr *wr) 3446 { 3447 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3448 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3449 3450 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3451 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3452 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3453 fwqe1->l_key = wr->key; 3454 3455 fwqe2->access_ctrl = 0; 3456 3457 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3458 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3459 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3460 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3461 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3462 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3463 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3464 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3465 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3466 fwqe2->fmr_ctrl = 0; 3467 3468 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3469 ilog2(mr->ibmr.page_size) - 12); 3470 3471 fwqe2->length_hi = 0; 3472 fwqe2->length_lo = mr->ibmr.length; 3473 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3474 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3475 3476 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3477 3478 return 0; 3479 } 3480 3481 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3482 { 3483 switch (opcode) { 3484 case IB_WR_RDMA_WRITE: 3485 case IB_WR_RDMA_WRITE_WITH_IMM: 3486 return IB_WC_RDMA_WRITE; 3487 case IB_WR_SEND_WITH_IMM: 3488 case IB_WR_SEND: 3489 case IB_WR_SEND_WITH_INV: 3490 return IB_WC_SEND; 3491 case IB_WR_RDMA_READ: 3492 case IB_WR_RDMA_READ_WITH_INV: 3493 return IB_WC_RDMA_READ; 3494 case IB_WR_ATOMIC_CMP_AND_SWP: 3495 return IB_WC_COMP_SWAP; 3496 case IB_WR_ATOMIC_FETCH_AND_ADD: 3497 return IB_WC_FETCH_ADD; 3498 case IB_WR_REG_MR: 3499 return IB_WC_REG_MR; 3500 case IB_WR_LOCAL_INV: 3501 return IB_WC_LOCAL_INV; 3502 default: 3503 return IB_WC_SEND; 3504 } 3505 } 3506 3507 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3508 const struct ib_send_wr *wr) 3509 { 3510 int wq_is_full, err_wr, pbl_is_full; 3511 struct qedr_dev *dev = qp->dev; 3512 3513 /* prevent SQ overflow and/or processing of a bad WR */ 3514 err_wr = wr->num_sge > qp->sq.max_sges; 3515 wq_is_full = qedr_wq_is_full(&qp->sq); 3516 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3517 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3518 if (wq_is_full || err_wr || pbl_is_full) { 3519 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3520 DP_ERR(dev, 3521 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3522 qp); 3523 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3524 } 3525 3526 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3527 DP_ERR(dev, 3528 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3529 qp); 3530 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3531 } 3532 3533 if (pbl_is_full && 3534 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3535 DP_ERR(dev, 3536 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3537 qp); 3538 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3539 } 3540 return false; 3541 } 3542 return true; 3543 } 3544 3545 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3546 const struct ib_send_wr **bad_wr) 3547 { 3548 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3549 struct qedr_qp *qp = get_qedr_qp(ibqp); 3550 struct rdma_sq_atomic_wqe_1st *awqe1; 3551 struct rdma_sq_atomic_wqe_2nd *awqe2; 3552 struct rdma_sq_atomic_wqe_3rd *awqe3; 3553 struct rdma_sq_send_wqe_2st *swqe2; 3554 struct rdma_sq_local_inv_wqe *iwqe; 3555 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3556 struct rdma_sq_send_wqe_1st *swqe; 3557 struct rdma_sq_rdma_wqe_1st *rwqe; 3558 struct rdma_sq_fmr_wqe_1st *fwqe1; 3559 struct rdma_sq_common_wqe *wqe; 3560 u32 length; 3561 int rc = 0; 3562 bool comp; 3563 3564 if (!qedr_can_post_send(qp, wr)) { 3565 *bad_wr = wr; 3566 return -ENOMEM; 3567 } 3568 3569 wqe = qed_chain_produce(&qp->sq.pbl); 3570 qp->wqe_wr_id[qp->sq.prod].signaled = 3571 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3572 3573 wqe->flags = 0; 3574 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3575 !!(wr->send_flags & IB_SEND_SOLICITED)); 3576 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3577 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3578 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3579 !!(wr->send_flags & IB_SEND_FENCE)); 3580 wqe->prev_wqe_size = qp->prev_wqe_size; 3581 3582 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3583 3584 switch (wr->opcode) { 3585 case IB_WR_SEND_WITH_IMM: 3586 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3587 rc = -EINVAL; 3588 *bad_wr = wr; 3589 break; 3590 } 3591 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3592 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3593 swqe->wqe_size = 2; 3594 swqe2 = qed_chain_produce(&qp->sq.pbl); 3595 3596 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3597 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3598 wr, bad_wr); 3599 swqe->length = cpu_to_le32(length); 3600 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3601 qp->prev_wqe_size = swqe->wqe_size; 3602 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3603 break; 3604 case IB_WR_SEND: 3605 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3606 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3607 3608 swqe->wqe_size = 2; 3609 swqe2 = qed_chain_produce(&qp->sq.pbl); 3610 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3611 wr, bad_wr); 3612 swqe->length = cpu_to_le32(length); 3613 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3614 qp->prev_wqe_size = swqe->wqe_size; 3615 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3616 break; 3617 case IB_WR_SEND_WITH_INV: 3618 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3619 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3620 swqe2 = qed_chain_produce(&qp->sq.pbl); 3621 swqe->wqe_size = 2; 3622 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3623 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3624 wr, bad_wr); 3625 swqe->length = cpu_to_le32(length); 3626 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3627 qp->prev_wqe_size = swqe->wqe_size; 3628 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3629 break; 3630 3631 case IB_WR_RDMA_WRITE_WITH_IMM: 3632 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3633 rc = -EINVAL; 3634 *bad_wr = wr; 3635 break; 3636 } 3637 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3638 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3639 3640 rwqe->wqe_size = 2; 3641 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3642 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3643 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3644 wr, bad_wr); 3645 rwqe->length = cpu_to_le32(length); 3646 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3647 qp->prev_wqe_size = rwqe->wqe_size; 3648 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3649 break; 3650 case IB_WR_RDMA_WRITE: 3651 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3652 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3653 3654 rwqe->wqe_size = 2; 3655 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3656 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3657 wr, bad_wr); 3658 rwqe->length = cpu_to_le32(length); 3659 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3660 qp->prev_wqe_size = rwqe->wqe_size; 3661 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3662 break; 3663 case IB_WR_RDMA_READ_WITH_INV: 3664 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3665 fallthrough; /* same is identical to RDMA READ */ 3666 3667 case IB_WR_RDMA_READ: 3668 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3669 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3670 3671 rwqe->wqe_size = 2; 3672 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3673 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3674 wr, bad_wr); 3675 rwqe->length = cpu_to_le32(length); 3676 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3677 qp->prev_wqe_size = rwqe->wqe_size; 3678 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3679 break; 3680 3681 case IB_WR_ATOMIC_CMP_AND_SWP: 3682 case IB_WR_ATOMIC_FETCH_AND_ADD: 3683 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3684 awqe1->wqe_size = 4; 3685 3686 awqe2 = qed_chain_produce(&qp->sq.pbl); 3687 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3688 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3689 3690 awqe3 = qed_chain_produce(&qp->sq.pbl); 3691 3692 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3693 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3694 DMA_REGPAIR_LE(awqe3->swap_data, 3695 atomic_wr(wr)->compare_add); 3696 } else { 3697 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3698 DMA_REGPAIR_LE(awqe3->swap_data, 3699 atomic_wr(wr)->swap); 3700 DMA_REGPAIR_LE(awqe3->cmp_data, 3701 atomic_wr(wr)->compare_add); 3702 } 3703 3704 qedr_prepare_sq_sges(qp, NULL, wr); 3705 3706 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3707 qp->prev_wqe_size = awqe1->wqe_size; 3708 break; 3709 3710 case IB_WR_LOCAL_INV: 3711 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3712 iwqe->wqe_size = 1; 3713 3714 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3715 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3716 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3717 qp->prev_wqe_size = iwqe->wqe_size; 3718 break; 3719 case IB_WR_REG_MR: 3720 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3721 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3722 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3723 fwqe1->wqe_size = 2; 3724 3725 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3726 if (rc) { 3727 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3728 *bad_wr = wr; 3729 break; 3730 } 3731 3732 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3733 qp->prev_wqe_size = fwqe1->wqe_size; 3734 break; 3735 default: 3736 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3737 rc = -EINVAL; 3738 *bad_wr = wr; 3739 break; 3740 } 3741 3742 if (*bad_wr) { 3743 u16 value; 3744 3745 /* Restore prod to its position before 3746 * this WR was processed 3747 */ 3748 value = le16_to_cpu(qp->sq.db_data.data.value); 3749 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3750 3751 /* Restore prev_wqe_size */ 3752 qp->prev_wqe_size = wqe->prev_wqe_size; 3753 rc = -EINVAL; 3754 DP_ERR(dev, "POST SEND FAILED\n"); 3755 } 3756 3757 return rc; 3758 } 3759 3760 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3761 const struct ib_send_wr **bad_wr) 3762 { 3763 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3764 struct qedr_qp *qp = get_qedr_qp(ibqp); 3765 unsigned long flags; 3766 int rc = 0; 3767 3768 *bad_wr = NULL; 3769 3770 if (qp->qp_type == IB_QPT_GSI) 3771 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3772 3773 spin_lock_irqsave(&qp->q_lock, flags); 3774 3775 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3776 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3777 (qp->state != QED_ROCE_QP_STATE_ERR) && 3778 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3779 spin_unlock_irqrestore(&qp->q_lock, flags); 3780 *bad_wr = wr; 3781 DP_DEBUG(dev, QEDR_MSG_CQ, 3782 "QP in wrong state! QP icid=0x%x state %d\n", 3783 qp->icid, qp->state); 3784 return -EINVAL; 3785 } 3786 } 3787 3788 while (wr) { 3789 rc = __qedr_post_send(ibqp, wr, bad_wr); 3790 if (rc) 3791 break; 3792 3793 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3794 3795 qedr_inc_sw_prod(&qp->sq); 3796 3797 qp->sq.db_data.data.value++; 3798 3799 wr = wr->next; 3800 } 3801 3802 /* Trigger doorbell 3803 * If there was a failure in the first WR then it will be triggered in 3804 * vane. However this is not harmful (as long as the producer value is 3805 * unchanged). For performance reasons we avoid checking for this 3806 * redundant doorbell. 3807 * 3808 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3809 * soon as we give the doorbell, we could get a completion 3810 * for this wr, therefore we need to make sure that the 3811 * memory is updated before giving the doorbell. 3812 * During qedr_poll_cq, rmb is called before accessing the 3813 * cqe. This covers for the smp_rmb as well. 3814 */ 3815 smp_wmb(); 3816 writel(qp->sq.db_data.raw, qp->sq.db); 3817 3818 spin_unlock_irqrestore(&qp->q_lock, flags); 3819 3820 return rc; 3821 } 3822 3823 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3824 { 3825 u32 used; 3826 3827 /* Calculate number of elements used based on producer 3828 * count and consumer count and subtract it from max 3829 * work request supported so that we get elements left. 3830 */ 3831 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt); 3832 3833 return hw_srq->max_wr - used; 3834 } 3835 3836 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3837 const struct ib_recv_wr **bad_wr) 3838 { 3839 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3840 struct qedr_srq_hwq_info *hw_srq; 3841 struct qedr_dev *dev = srq->dev; 3842 struct qed_chain *pbl; 3843 unsigned long flags; 3844 int status = 0; 3845 u32 num_sge; 3846 3847 spin_lock_irqsave(&srq->lock, flags); 3848 3849 hw_srq = &srq->hw_srq; 3850 pbl = &srq->hw_srq.pbl; 3851 while (wr) { 3852 struct rdma_srq_wqe_header *hdr; 3853 int i; 3854 3855 if (!qedr_srq_elem_left(hw_srq) || 3856 wr->num_sge > srq->hw_srq.max_sges) { 3857 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3858 hw_srq->wr_prod_cnt, 3859 atomic_read(&hw_srq->wr_cons_cnt), 3860 wr->num_sge, srq->hw_srq.max_sges); 3861 status = -ENOMEM; 3862 *bad_wr = wr; 3863 break; 3864 } 3865 3866 hdr = qed_chain_produce(pbl); 3867 num_sge = wr->num_sge; 3868 /* Set number of sge and work request id in header */ 3869 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3870 3871 srq->hw_srq.wr_prod_cnt++; 3872 hw_srq->wqe_prod++; 3873 hw_srq->sge_prod++; 3874 3875 DP_DEBUG(dev, QEDR_MSG_SRQ, 3876 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3877 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3878 3879 for (i = 0; i < wr->num_sge; i++) { 3880 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3881 3882 /* Set SGE length, lkey and address */ 3883 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3884 wr->sg_list[i].length, wr->sg_list[i].lkey); 3885 3886 DP_DEBUG(dev, QEDR_MSG_SRQ, 3887 "[%d]: len %d key %x addr %x:%x\n", 3888 i, srq_sge->length, srq_sge->l_key, 3889 srq_sge->addr.hi, srq_sge->addr.lo); 3890 hw_srq->sge_prod++; 3891 } 3892 3893 /* Update WQE and SGE information before 3894 * updating producer. 3895 */ 3896 dma_wmb(); 3897 3898 /* SRQ producer is 8 bytes. Need to update SGE producer index 3899 * in first 4 bytes and need to update WQE producer in 3900 * next 4 bytes. 3901 */ 3902 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod); 3903 /* Make sure sge producer is updated first */ 3904 dma_wmb(); 3905 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod); 3906 3907 wr = wr->next; 3908 } 3909 3910 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3911 qed_chain_get_elem_left(pbl)); 3912 spin_unlock_irqrestore(&srq->lock, flags); 3913 3914 return status; 3915 } 3916 3917 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3918 const struct ib_recv_wr **bad_wr) 3919 { 3920 struct qedr_qp *qp = get_qedr_qp(ibqp); 3921 struct qedr_dev *dev = qp->dev; 3922 unsigned long flags; 3923 int status = 0; 3924 3925 if (qp->qp_type == IB_QPT_GSI) 3926 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3927 3928 spin_lock_irqsave(&qp->q_lock, flags); 3929 3930 while (wr) { 3931 int i; 3932 3933 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3934 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3935 wr->num_sge > qp->rq.max_sges) { 3936 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3937 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3938 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3939 qp->rq.max_sges); 3940 status = -ENOMEM; 3941 *bad_wr = wr; 3942 break; 3943 } 3944 for (i = 0; i < wr->num_sge; i++) { 3945 u32 flags = 0; 3946 struct rdma_rq_sge *rqe = 3947 qed_chain_produce(&qp->rq.pbl); 3948 3949 /* First one must include the number 3950 * of SGE in the list 3951 */ 3952 if (!i) 3953 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3954 wr->num_sge); 3955 3956 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3957 wr->sg_list[i].lkey); 3958 3959 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3960 wr->sg_list[i].length, flags); 3961 } 3962 3963 /* Special case of no sges. FW requires between 1-4 sges... 3964 * in this case we need to post 1 sge with length zero. this is 3965 * because rdma write with immediate consumes an RQ. 3966 */ 3967 if (!wr->num_sge) { 3968 u32 flags = 0; 3969 struct rdma_rq_sge *rqe = 3970 qed_chain_produce(&qp->rq.pbl); 3971 3972 /* First one must include the number 3973 * of SGE in the list 3974 */ 3975 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3976 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3977 3978 RQ_SGE_SET(rqe, 0, 0, flags); 3979 i = 1; 3980 } 3981 3982 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3983 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3984 3985 qedr_inc_sw_prod(&qp->rq); 3986 3987 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3988 * soon as we give the doorbell, we could get a completion 3989 * for this wr, therefore we need to make sure that the 3990 * memory is update before giving the doorbell. 3991 * During qedr_poll_cq, rmb is called before accessing the 3992 * cqe. This covers for the smp_rmb as well. 3993 */ 3994 smp_wmb(); 3995 3996 qp->rq.db_data.data.value++; 3997 3998 writel(qp->rq.db_data.raw, qp->rq.db); 3999 4000 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 4001 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 4002 } 4003 4004 wr = wr->next; 4005 } 4006 4007 spin_unlock_irqrestore(&qp->q_lock, flags); 4008 4009 return status; 4010 } 4011 4012 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 4013 { 4014 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4015 4016 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 4017 cq->pbl_toggle; 4018 } 4019 4020 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 4021 { 4022 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4023 struct qedr_qp *qp; 4024 4025 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 4026 resp_cqe->qp_handle.lo, 4027 u64); 4028 return qp; 4029 } 4030 4031 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 4032 { 4033 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4034 4035 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 4036 } 4037 4038 /* Return latest CQE (needs processing) */ 4039 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 4040 { 4041 return cq->latest_cqe; 4042 } 4043 4044 /* In fmr we need to increase the number of fmr completed counter for the fmr 4045 * algorithm determining whether we can free a pbl or not. 4046 * we need to perform this whether the work request was signaled or not. for 4047 * this purpose we call this function from the condition that checks if a wr 4048 * should be skipped, to make sure we don't miss it ( possibly this fmr 4049 * operation was not signalted) 4050 */ 4051 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 4052 { 4053 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 4054 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4055 } 4056 4057 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 4058 struct qedr_cq *cq, int num_entries, 4059 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 4060 int force) 4061 { 4062 u16 cnt = 0; 4063 4064 while (num_entries && qp->sq.wqe_cons != hw_cons) { 4065 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 4066 qedr_chk_if_fmr(qp); 4067 /* skip WC */ 4068 goto next_cqe; 4069 } 4070 4071 /* fill WC */ 4072 wc->status = status; 4073 wc->vendor_err = 0; 4074 wc->wc_flags = 0; 4075 wc->src_qp = qp->id; 4076 wc->qp = &qp->ibqp; 4077 4078 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 4079 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 4080 4081 switch (wc->opcode) { 4082 case IB_WC_RDMA_WRITE: 4083 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4084 break; 4085 case IB_WC_COMP_SWAP: 4086 case IB_WC_FETCH_ADD: 4087 wc->byte_len = 8; 4088 break; 4089 case IB_WC_REG_MR: 4090 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4091 break; 4092 case IB_WC_RDMA_READ: 4093 case IB_WC_SEND: 4094 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4095 break; 4096 default: 4097 break; 4098 } 4099 4100 num_entries--; 4101 wc++; 4102 cnt++; 4103 next_cqe: 4104 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 4105 qed_chain_consume(&qp->sq.pbl); 4106 qedr_inc_sw_cons(&qp->sq); 4107 } 4108 4109 return cnt; 4110 } 4111 4112 static int qedr_poll_cq_req(struct qedr_dev *dev, 4113 struct qedr_qp *qp, struct qedr_cq *cq, 4114 int num_entries, struct ib_wc *wc, 4115 struct rdma_cqe_requester *req) 4116 { 4117 int cnt = 0; 4118 4119 switch (req->status) { 4120 case RDMA_CQE_REQ_STS_OK: 4121 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4122 IB_WC_SUCCESS, 0); 4123 break; 4124 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 4125 if (qp->state != QED_ROCE_QP_STATE_ERR) 4126 DP_DEBUG(dev, QEDR_MSG_CQ, 4127 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4128 cq->icid, qp->icid); 4129 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4130 IB_WC_WR_FLUSH_ERR, 1); 4131 break; 4132 default: 4133 /* process all WQE before the cosumer */ 4134 qp->state = QED_ROCE_QP_STATE_ERR; 4135 cnt = process_req(dev, qp, cq, num_entries, wc, 4136 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4137 wc += cnt; 4138 /* if we have extra WC fill it with actual error info */ 4139 if (cnt < num_entries) { 4140 enum ib_wc_status wc_status; 4141 4142 switch (req->status) { 4143 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4144 DP_ERR(dev, 4145 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4146 cq->icid, qp->icid); 4147 wc_status = IB_WC_BAD_RESP_ERR; 4148 break; 4149 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4150 DP_ERR(dev, 4151 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4152 cq->icid, qp->icid); 4153 wc_status = IB_WC_LOC_LEN_ERR; 4154 break; 4155 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4156 DP_ERR(dev, 4157 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4158 cq->icid, qp->icid); 4159 wc_status = IB_WC_LOC_QP_OP_ERR; 4160 break; 4161 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4162 DP_ERR(dev, 4163 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4164 cq->icid, qp->icid); 4165 wc_status = IB_WC_LOC_PROT_ERR; 4166 break; 4167 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4168 DP_ERR(dev, 4169 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4170 cq->icid, qp->icid); 4171 wc_status = IB_WC_MW_BIND_ERR; 4172 break; 4173 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4174 DP_ERR(dev, 4175 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4176 cq->icid, qp->icid); 4177 wc_status = IB_WC_REM_INV_REQ_ERR; 4178 break; 4179 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4180 DP_ERR(dev, 4181 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4182 cq->icid, qp->icid); 4183 wc_status = IB_WC_REM_ACCESS_ERR; 4184 break; 4185 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4186 DP_ERR(dev, 4187 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4188 cq->icid, qp->icid); 4189 wc_status = IB_WC_REM_OP_ERR; 4190 break; 4191 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4192 DP_ERR(dev, 4193 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4194 cq->icid, qp->icid); 4195 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4196 break; 4197 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4198 DP_ERR(dev, 4199 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4200 cq->icid, qp->icid); 4201 wc_status = IB_WC_RETRY_EXC_ERR; 4202 break; 4203 default: 4204 DP_ERR(dev, 4205 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4206 cq->icid, qp->icid); 4207 wc_status = IB_WC_GENERAL_ERR; 4208 } 4209 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4210 wc_status, 1); 4211 } 4212 } 4213 4214 return cnt; 4215 } 4216 4217 static inline int qedr_cqe_resp_status_to_ib(u8 status) 4218 { 4219 switch (status) { 4220 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4221 return IB_WC_LOC_ACCESS_ERR; 4222 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4223 return IB_WC_LOC_LEN_ERR; 4224 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4225 return IB_WC_LOC_QP_OP_ERR; 4226 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4227 return IB_WC_LOC_PROT_ERR; 4228 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4229 return IB_WC_MW_BIND_ERR; 4230 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4231 return IB_WC_REM_INV_RD_REQ_ERR; 4232 case RDMA_CQE_RESP_STS_OK: 4233 return IB_WC_SUCCESS; 4234 default: 4235 return IB_WC_GENERAL_ERR; 4236 } 4237 } 4238 4239 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 4240 struct ib_wc *wc) 4241 { 4242 wc->status = IB_WC_SUCCESS; 4243 wc->byte_len = le32_to_cpu(resp->length); 4244 4245 if (resp->flags & QEDR_RESP_IMM) { 4246 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4247 wc->wc_flags |= IB_WC_WITH_IMM; 4248 4249 if (resp->flags & QEDR_RESP_RDMA) 4250 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4251 4252 if (resp->flags & QEDR_RESP_INV) 4253 return -EINVAL; 4254 4255 } else if (resp->flags & QEDR_RESP_INV) { 4256 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4257 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4258 4259 if (resp->flags & QEDR_RESP_RDMA) 4260 return -EINVAL; 4261 4262 } else if (resp->flags & QEDR_RESP_RDMA) { 4263 return -EINVAL; 4264 } 4265 4266 return 0; 4267 } 4268 4269 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4270 struct qedr_cq *cq, struct ib_wc *wc, 4271 struct rdma_cqe_responder *resp, u64 wr_id) 4272 { 4273 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4274 wc->opcode = IB_WC_RECV; 4275 wc->wc_flags = 0; 4276 4277 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4278 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4279 DP_ERR(dev, 4280 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4281 cq, cq->icid, resp->flags); 4282 4283 } else { 4284 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4285 if (wc->status == IB_WC_GENERAL_ERR) 4286 DP_ERR(dev, 4287 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4288 cq, cq->icid, resp->status); 4289 } 4290 4291 /* Fill the rest of the WC */ 4292 wc->vendor_err = 0; 4293 wc->src_qp = qp->id; 4294 wc->qp = &qp->ibqp; 4295 wc->wr_id = wr_id; 4296 } 4297 4298 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4299 struct qedr_cq *cq, struct ib_wc *wc, 4300 struct rdma_cqe_responder *resp) 4301 { 4302 struct qedr_srq *srq = qp->srq; 4303 u64 wr_id; 4304 4305 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4306 le32_to_cpu(resp->srq_wr_id.lo), u64); 4307 4308 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4309 wc->status = IB_WC_WR_FLUSH_ERR; 4310 wc->vendor_err = 0; 4311 wc->wr_id = wr_id; 4312 wc->byte_len = 0; 4313 wc->src_qp = qp->id; 4314 wc->qp = &qp->ibqp; 4315 wc->wr_id = wr_id; 4316 } else { 4317 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4318 } 4319 atomic_inc(&srq->hw_srq.wr_cons_cnt); 4320 4321 return 1; 4322 } 4323 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4324 struct qedr_cq *cq, struct ib_wc *wc, 4325 struct rdma_cqe_responder *resp) 4326 { 4327 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4328 4329 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4330 4331 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4332 qed_chain_consume(&qp->rq.pbl); 4333 qedr_inc_sw_cons(&qp->rq); 4334 4335 return 1; 4336 } 4337 4338 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4339 int num_entries, struct ib_wc *wc, u16 hw_cons) 4340 { 4341 u16 cnt = 0; 4342 4343 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4344 /* fill WC */ 4345 wc->status = IB_WC_WR_FLUSH_ERR; 4346 wc->vendor_err = 0; 4347 wc->wc_flags = 0; 4348 wc->src_qp = qp->id; 4349 wc->byte_len = 0; 4350 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4351 wc->qp = &qp->ibqp; 4352 num_entries--; 4353 wc++; 4354 cnt++; 4355 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4356 qed_chain_consume(&qp->rq.pbl); 4357 qedr_inc_sw_cons(&qp->rq); 4358 } 4359 4360 return cnt; 4361 } 4362 4363 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4364 struct rdma_cqe_responder *resp, int *update) 4365 { 4366 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4367 consume_cqe(cq); 4368 *update |= 1; 4369 } 4370 } 4371 4372 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4373 struct qedr_cq *cq, int num_entries, 4374 struct ib_wc *wc, 4375 struct rdma_cqe_responder *resp) 4376 { 4377 int cnt; 4378 4379 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4380 consume_cqe(cq); 4381 4382 return cnt; 4383 } 4384 4385 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4386 struct qedr_cq *cq, int num_entries, 4387 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4388 int *update) 4389 { 4390 int cnt; 4391 4392 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4393 cnt = process_resp_flush(qp, cq, num_entries, wc, 4394 resp->rq_cons_or_srq_id); 4395 try_consume_resp_cqe(cq, qp, resp, update); 4396 } else { 4397 cnt = process_resp_one(dev, qp, cq, wc, resp); 4398 consume_cqe(cq); 4399 *update |= 1; 4400 } 4401 4402 return cnt; 4403 } 4404 4405 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4406 struct rdma_cqe_requester *req, int *update) 4407 { 4408 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4409 consume_cqe(cq); 4410 *update |= 1; 4411 } 4412 } 4413 4414 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4415 { 4416 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4417 struct qedr_cq *cq = get_qedr_cq(ibcq); 4418 union rdma_cqe *cqe; 4419 u32 old_cons, new_cons; 4420 unsigned long flags; 4421 int update = 0; 4422 int done = 0; 4423 4424 if (cq->destroyed) { 4425 DP_ERR(dev, 4426 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4427 cq, cq->icid); 4428 return 0; 4429 } 4430 4431 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4432 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4433 4434 spin_lock_irqsave(&cq->cq_lock, flags); 4435 cqe = cq->latest_cqe; 4436 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4437 while (num_entries && is_valid_cqe(cq, cqe)) { 4438 struct qedr_qp *qp; 4439 int cnt = 0; 4440 4441 /* prevent speculative reads of any field of CQE */ 4442 rmb(); 4443 4444 qp = cqe_get_qp(cqe); 4445 if (!qp) { 4446 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4447 break; 4448 } 4449 4450 wc->qp = &qp->ibqp; 4451 4452 switch (cqe_get_type(cqe)) { 4453 case RDMA_CQE_TYPE_REQUESTER: 4454 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4455 &cqe->req); 4456 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4457 break; 4458 case RDMA_CQE_TYPE_RESPONDER_RQ: 4459 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4460 &cqe->resp, &update); 4461 break; 4462 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4463 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4464 wc, &cqe->resp); 4465 update = 1; 4466 break; 4467 case RDMA_CQE_TYPE_INVALID: 4468 default: 4469 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4470 cqe_get_type(cqe)); 4471 } 4472 num_entries -= cnt; 4473 wc += cnt; 4474 done += cnt; 4475 4476 cqe = get_cqe(cq); 4477 } 4478 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4479 4480 cq->cq_cons += new_cons - old_cons; 4481 4482 if (update) 4483 /* doorbell notifies abount latest VALID entry, 4484 * but chain already point to the next INVALID one 4485 */ 4486 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4487 4488 spin_unlock_irqrestore(&cq->cq_lock, flags); 4489 return done; 4490 } 4491 4492 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4493 u32 port_num, const struct ib_wc *in_wc, 4494 const struct ib_grh *in_grh, const struct ib_mad *in, 4495 struct ib_mad *out_mad, size_t *out_mad_size, 4496 u16 *out_mad_pkey_index) 4497 { 4498 return IB_MAD_RESULT_SUCCESS; 4499 } 4500