1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_addr.h> 43 #include <rdma/ib_cache.h> 44 #include <rdma/iter.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 #include "qedr_iw_cm.h" 55 56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 57 #define RDMA_MAX_SGE_PER_SRQ (4) 58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 59 60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 61 62 enum { 63 QEDR_USER_MMAP_IO_WC = 0, 64 QEDR_USER_MMAP_PHYS_PAGE, 65 }; 66 67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 68 size_t len) 69 { 70 size_t min_len = min_t(size_t, len, udata->outlen); 71 72 return ib_copy_to_udata(udata, src, min_len); 73 } 74 75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 76 { 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 78 return -EINVAL; 79 80 *pkey = QEDR_ROCE_PKEY_DEFAULT; 81 return 0; 82 } 83 84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, 85 int index, union ib_gid *sgid) 86 { 87 struct qedr_dev *dev = get_qedr_dev(ibdev); 88 89 memset(sgid->raw, 0, sizeof(sgid->raw)); 90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 91 92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 93 sgid->global.interface_id, sgid->global.subnet_prefix); 94 95 return 0; 96 } 97 98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 99 { 100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 101 struct qedr_device_attr *qattr = &dev->attr; 102 struct qedr_srq *srq = get_qedr_srq(ibsrq); 103 104 srq_attr->srq_limit = srq->srq_limit; 105 srq_attr->max_wr = qattr->max_srq_wr; 106 srq_attr->max_sge = qattr->max_sge; 107 108 return 0; 109 } 110 111 int qedr_query_device(struct ib_device *ibdev, 112 struct ib_device_attr *attr, struct ib_udata *udata) 113 { 114 struct qedr_dev *dev = get_qedr_dev(ibdev); 115 struct qedr_device_attr *qattr = &dev->attr; 116 117 if (!dev->rdma_ctx) { 118 DP_ERR(dev, 119 "qedr_query_device called with invalid params rdma_ctx=%p\n", 120 dev->rdma_ctx); 121 return -EINVAL; 122 } 123 124 memset(attr, 0, sizeof(*attr)); 125 126 attr->fw_ver = qattr->fw_ver; 127 attr->sys_image_guid = qattr->sys_image_guid; 128 attr->max_mr_size = qattr->max_mr_size; 129 attr->page_size_cap = qattr->page_size_caps; 130 attr->vendor_id = qattr->vendor_id; 131 attr->vendor_part_id = qattr->vendor_part_id; 132 attr->hw_ver = qattr->hw_ver; 133 attr->max_qp = qattr->max_qp; 134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 136 IB_DEVICE_RC_RNR_NAK_GEN | 137 IB_DEVICE_MEM_MGT_EXTENSIONS; 138 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 139 140 if (!rdma_protocol_iwarp(&dev->ibdev, 1)) 141 attr->device_cap_flags |= IB_DEVICE_XRC; 142 attr->max_send_sge = qattr->max_sge; 143 attr->max_recv_sge = qattr->max_sge; 144 attr->max_sge_rd = qattr->max_sge; 145 attr->max_cq = qattr->max_cq; 146 attr->max_cqe = qattr->max_cqe; 147 attr->max_mr = qattr->max_mr; 148 attr->max_mw = qattr->max_mw; 149 attr->max_pd = qattr->max_pd; 150 attr->atomic_cap = dev->atomic_cap; 151 attr->max_qp_init_rd_atom = 152 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 153 attr->max_qp_rd_atom = 154 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 155 attr->max_qp_init_rd_atom); 156 157 attr->max_srq = qattr->max_srq; 158 attr->max_srq_sge = qattr->max_srq_sge; 159 attr->max_srq_wr = qattr->max_srq_wr; 160 161 attr->local_ca_ack_delay = qattr->dev_ack_delay; 162 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 163 attr->max_pkeys = qattr->max_pkey; 164 attr->max_ah = qattr->max_ah; 165 166 return 0; 167 } 168 169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed, 170 u8 *ib_width) 171 { 172 switch (speed) { 173 case 1000: 174 *ib_speed = IB_SPEED_SDR; 175 *ib_width = IB_WIDTH_1X; 176 break; 177 case 10000: 178 *ib_speed = IB_SPEED_QDR; 179 *ib_width = IB_WIDTH_1X; 180 break; 181 182 case 20000: 183 *ib_speed = IB_SPEED_DDR; 184 *ib_width = IB_WIDTH_4X; 185 break; 186 187 case 25000: 188 *ib_speed = IB_SPEED_EDR; 189 *ib_width = IB_WIDTH_1X; 190 break; 191 192 case 40000: 193 *ib_speed = IB_SPEED_QDR; 194 *ib_width = IB_WIDTH_4X; 195 break; 196 197 case 50000: 198 *ib_speed = IB_SPEED_HDR; 199 *ib_width = IB_WIDTH_1X; 200 break; 201 202 case 100000: 203 *ib_speed = IB_SPEED_EDR; 204 *ib_width = IB_WIDTH_4X; 205 break; 206 207 default: 208 /* Unsupported */ 209 *ib_speed = IB_SPEED_SDR; 210 *ib_width = IB_WIDTH_1X; 211 } 212 } 213 214 int qedr_query_port(struct ib_device *ibdev, u32 port, 215 struct ib_port_attr *attr) 216 { 217 struct qedr_dev *dev; 218 struct qed_rdma_port *rdma_port; 219 220 dev = get_qedr_dev(ibdev); 221 222 if (!dev->rdma_ctx) { 223 DP_ERR(dev, "rdma_ctx is NULL\n"); 224 return -EINVAL; 225 } 226 227 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 228 229 /* *attr being zeroed by the caller, avoid zeroing it here */ 230 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 231 attr->state = IB_PORT_ACTIVE; 232 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 233 } else { 234 attr->state = IB_PORT_DOWN; 235 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 236 } 237 attr->max_mtu = IB_MTU_4096; 238 attr->lid = 0; 239 attr->lmc = 0; 240 attr->sm_lid = 0; 241 attr->sm_sl = 0; 242 attr->ip_gids = true; 243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 244 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu); 245 attr->gid_tbl_len = 1; 246 } else { 247 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 248 attr->gid_tbl_len = QEDR_MAX_SGID; 249 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 250 } 251 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 252 attr->qkey_viol_cntr = 0; 253 get_link_speed_and_width(rdma_port->link_speed, 254 &attr->active_speed, &attr->active_width); 255 attr->max_msg_sz = rdma_port->max_msg_size; 256 attr->max_vl_num = 4; 257 258 return 0; 259 } 260 261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 262 { 263 struct ib_device *ibdev = uctx->device; 264 int rc; 265 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 266 struct qedr_alloc_ucontext_resp uresp = {}; 267 struct qedr_alloc_ucontext_req ureq; 268 struct qedr_dev *dev = get_qedr_dev(ibdev); 269 struct qed_rdma_add_user_out_params oparams; 270 struct qedr_user_mmap_entry *entry; 271 272 if (!udata) 273 return -EFAULT; 274 275 if (udata->inlen) { 276 rc = ib_copy_validate_udata_in(udata, ureq, reserved); 277 if (rc) 278 return rc; 279 ctx->edpm_mode = !!(ureq.context_flags & 280 QEDR_ALLOC_UCTX_EDPM_MODE); 281 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); 282 } 283 284 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 285 if (rc) { 286 DP_ERR(dev, 287 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 288 rc); 289 return rc; 290 } 291 292 ctx->dpi = oparams.dpi; 293 ctx->dpi_addr = oparams.dpi_addr; 294 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 295 ctx->dpi_size = oparams.dpi_size; 296 entry = kzalloc_obj(*entry); 297 if (!entry) { 298 rc = -ENOMEM; 299 goto err; 300 } 301 302 entry->io_address = ctx->dpi_phys_addr; 303 entry->length = ctx->dpi_size; 304 entry->mmap_flag = QEDR_USER_MMAP_IO_WC; 305 entry->dpi = ctx->dpi; 306 entry->dev = dev; 307 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry, 308 ctx->dpi_size); 309 if (rc) { 310 kfree(entry); 311 goto err; 312 } 313 ctx->db_mmap_entry = &entry->rdma_entry; 314 315 if (!dev->user_dpm_enabled) 316 uresp.dpm_flags = 0; 317 else if (rdma_protocol_iwarp(&dev->ibdev, 1)) 318 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; 319 else 320 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | 321 QEDR_DPM_TYPE_ROCE_LEGACY | 322 QEDR_DPM_TYPE_ROCE_EDPM_MODE; 323 324 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) { 325 uresp.dpm_flags |= QEDR_DPM_SIZES_SET; 326 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; 327 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; 328 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE; 329 } 330 331 uresp.wids_enabled = 1; 332 uresp.wid_count = oparams.wid_count; 333 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); 334 uresp.db_size = ctx->dpi_size; 335 uresp.max_send_wr = dev->attr.max_sqe; 336 uresp.max_recv_wr = dev->attr.max_rqe; 337 uresp.max_srq_wr = dev->attr.max_srq_wr; 338 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 339 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 340 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 341 uresp.max_cqes = QEDR_MAX_CQES; 342 343 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 344 if (rc) 345 goto err; 346 347 ctx->dev = dev; 348 349 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 350 &ctx->ibucontext); 351 return 0; 352 353 err: 354 if (!ctx->db_mmap_entry) 355 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi); 356 else 357 rdma_user_mmap_entry_remove(ctx->db_mmap_entry); 358 359 return rc; 360 } 361 362 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 363 { 364 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 365 366 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 367 uctx); 368 369 rdma_user_mmap_entry_remove(uctx->db_mmap_entry); 370 } 371 372 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 373 { 374 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry); 375 struct qedr_dev *dev = entry->dev; 376 377 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE) 378 free_page((unsigned long)entry->address); 379 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC) 380 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi); 381 382 kfree(entry); 383 } 384 385 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) 386 { 387 struct ib_device *dev = ucontext->device; 388 size_t length = vma->vm_end - vma->vm_start; 389 struct rdma_user_mmap_entry *rdma_entry; 390 struct qedr_user_mmap_entry *entry; 391 int rc = 0; 392 u64 pfn; 393 394 ibdev_dbg(dev, 395 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 396 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 397 398 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma); 399 if (!rdma_entry) { 400 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n", 401 vma->vm_pgoff); 402 return -EINVAL; 403 } 404 entry = get_qedr_mmap_entry(rdma_entry); 405 ibdev_dbg(dev, 406 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 407 entry->io_address, length, entry->mmap_flag); 408 409 switch (entry->mmap_flag) { 410 case QEDR_USER_MMAP_IO_WC: 411 pfn = entry->io_address >> PAGE_SHIFT; 412 rc = rdma_user_mmap_io(ucontext, vma, pfn, length, 413 pgprot_writecombine(vma->vm_page_prot), 414 rdma_entry); 415 break; 416 case QEDR_USER_MMAP_PHYS_PAGE: 417 rc = vm_insert_page(vma, vma->vm_start, 418 virt_to_page(entry->address)); 419 break; 420 default: 421 rc = -EINVAL; 422 } 423 424 if (rc) 425 ibdev_dbg(dev, 426 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 427 entry->io_address, length, entry->mmap_flag, rc); 428 429 rdma_user_mmap_entry_put(rdma_entry); 430 return rc; 431 } 432 433 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 434 { 435 struct ib_device *ibdev = ibpd->device; 436 struct qedr_dev *dev = get_qedr_dev(ibdev); 437 struct qedr_pd *pd = get_qedr_pd(ibpd); 438 u16 pd_id; 439 int rc; 440 441 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 442 udata ? "User Lib" : "Kernel"); 443 444 if (!dev->rdma_ctx) { 445 DP_ERR(dev, "invalid RDMA context\n"); 446 return -EINVAL; 447 } 448 449 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 450 if (rc) 451 return rc; 452 453 pd->pd_id = pd_id; 454 455 if (udata) { 456 struct qedr_alloc_pd_uresp uresp = { 457 .pd_id = pd_id, 458 }; 459 struct qedr_ucontext *context = rdma_udata_to_drv_context( 460 udata, struct qedr_ucontext, ibucontext); 461 462 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 463 if (rc) { 464 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 465 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 466 return rc; 467 } 468 469 pd->uctx = context; 470 pd->uctx->pd = pd; 471 } 472 473 return 0; 474 } 475 476 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 477 { 478 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 479 struct qedr_pd *pd = get_qedr_pd(ibpd); 480 481 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 482 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 483 return 0; 484 } 485 486 487 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 488 { 489 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 490 struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd); 491 492 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id); 493 } 494 495 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) 496 { 497 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device); 498 u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id; 499 500 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id); 501 return 0; 502 } 503 static void qedr_free_pbl(struct qedr_dev *dev, 504 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 505 { 506 struct pci_dev *pdev = dev->pdev; 507 int i; 508 509 for (i = 0; i < pbl_info->num_pbls; i++) { 510 if (!pbl[i].va) 511 continue; 512 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 513 pbl[i].va, pbl[i].pa); 514 } 515 516 kfree(pbl); 517 } 518 519 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 520 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 521 522 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 523 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 524 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 525 526 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 527 struct qedr_pbl_info *pbl_info, 528 gfp_t flags) 529 { 530 struct pci_dev *pdev = dev->pdev; 531 struct qedr_pbl *pbl_table; 532 dma_addr_t *pbl_main_tbl; 533 dma_addr_t pa; 534 void *va; 535 int i; 536 537 pbl_table = kzalloc_objs(*pbl_table, pbl_info->num_pbls, flags); 538 if (!pbl_table) 539 return ERR_PTR(-ENOMEM); 540 541 for (i = 0; i < pbl_info->num_pbls; i++) { 542 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 543 flags); 544 if (!va) 545 goto err; 546 547 pbl_table[i].va = va; 548 pbl_table[i].pa = pa; 549 } 550 551 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 552 * the first one with physical pointers to all of the rest 553 */ 554 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 555 for (i = 0; i < pbl_info->num_pbls - 1; i++) 556 pbl_main_tbl[i] = pbl_table[i + 1].pa; 557 558 return pbl_table; 559 560 err: 561 for (i--; i >= 0; i--) 562 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 563 pbl_table[i].va, pbl_table[i].pa); 564 565 qedr_free_pbl(dev, pbl_info, pbl_table); 566 567 return ERR_PTR(-ENOMEM); 568 } 569 570 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 571 struct qedr_pbl_info *pbl_info, 572 u32 num_pbes, int two_layer_capable) 573 { 574 u32 pbl_capacity; 575 u32 pbl_size; 576 u32 num_pbls; 577 578 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 579 if (num_pbes > MAX_PBES_TWO_LAYER) { 580 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 581 num_pbes); 582 return -EINVAL; 583 } 584 585 /* calculate required pbl page size */ 586 pbl_size = MIN_FW_PBL_PAGE_SIZE; 587 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 588 NUM_PBES_ON_PAGE(pbl_size); 589 590 while (pbl_capacity < num_pbes) { 591 pbl_size *= 2; 592 pbl_capacity = pbl_size / sizeof(u64); 593 pbl_capacity = pbl_capacity * pbl_capacity; 594 } 595 596 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 597 num_pbls++; /* One for the layer0 ( points to the pbls) */ 598 pbl_info->two_layered = true; 599 } else { 600 /* One layered PBL */ 601 num_pbls = 1; 602 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 603 roundup_pow_of_two((num_pbes * sizeof(u64)))); 604 pbl_info->two_layered = false; 605 } 606 607 pbl_info->num_pbls = num_pbls; 608 pbl_info->pbl_size = pbl_size; 609 pbl_info->num_pbes = num_pbes; 610 611 DP_DEBUG(dev, QEDR_MSG_MR, 612 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 613 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 614 615 return 0; 616 } 617 618 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 619 struct qedr_pbl *pbl, 620 struct qedr_pbl_info *pbl_info, u32 pg_shift) 621 { 622 int pbe_cnt, total_num_pbes = 0; 623 struct qedr_pbl *pbl_tbl; 624 struct ib_block_iter biter; 625 struct regpair *pbe; 626 627 if (!pbl_info->num_pbes) 628 return; 629 630 /* If we have a two layered pbl, the first pbl points to the rest 631 * of the pbls and the first entry lays on the second pbl in the table 632 */ 633 if (pbl_info->two_layered) 634 pbl_tbl = &pbl[1]; 635 else 636 pbl_tbl = pbl; 637 638 pbe = (struct regpair *)pbl_tbl->va; 639 if (!pbe) { 640 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 641 return; 642 } 643 644 pbe_cnt = 0; 645 646 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) { 647 u64 pg_addr = rdma_block_iter_dma_address(&biter); 648 649 pbe->lo = cpu_to_le32(pg_addr); 650 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 651 652 pbe_cnt++; 653 total_num_pbes++; 654 pbe++; 655 656 if (total_num_pbes == pbl_info->num_pbes) 657 return; 658 659 /* If the given pbl is full storing the pbes, move to next pbl. 660 */ 661 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 662 pbl_tbl++; 663 pbe = (struct regpair *)pbl_tbl->va; 664 pbe_cnt = 0; 665 } 666 } 667 } 668 669 static int qedr_db_recovery_add(struct qedr_dev *dev, 670 void __iomem *db_addr, 671 void *db_data, 672 enum qed_db_rec_width db_width, 673 enum qed_db_rec_space db_space) 674 { 675 if (!db_data) { 676 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 677 return 0; 678 } 679 680 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data, 681 db_width, db_space); 682 } 683 684 static void qedr_db_recovery_del(struct qedr_dev *dev, 685 void __iomem *db_addr, 686 void *db_data) 687 { 688 if (!db_data) { 689 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 690 return; 691 } 692 693 /* Ignore return code as there is not much we can do about it. Error 694 * log will be printed inside. 695 */ 696 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); 697 } 698 699 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 700 struct qedr_cq *cq, struct ib_udata *udata, 701 u32 db_offset) 702 { 703 struct qedr_create_cq_uresp uresp; 704 int rc; 705 706 memset(&uresp, 0, sizeof(uresp)); 707 708 uresp.db_offset = db_offset; 709 uresp.icid = cq->icid; 710 if (cq->q.db_mmap_entry) 711 uresp.db_rec_addr = 712 rdma_user_mmap_get_offset(cq->q.db_mmap_entry); 713 714 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 715 if (rc) 716 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 717 718 return rc; 719 } 720 721 static void consume_cqe(struct qedr_cq *cq) 722 { 723 if (cq->latest_cqe == cq->toggle_cqe) 724 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 725 726 cq->latest_cqe = qed_chain_consume(&cq->pbl); 727 } 728 729 static inline int qedr_align_cq_entries(int entries) 730 { 731 u64 size, aligned_size; 732 733 /* We allocate an extra entry that we don't report to the FW. */ 734 size = (entries + 1) * QEDR_CQE_SIZE; 735 aligned_size = ALIGN(size, PAGE_SIZE); 736 737 return aligned_size / QEDR_CQE_SIZE; 738 } 739 740 static int qedr_init_user_db_rec(struct ib_udata *udata, 741 struct qedr_dev *dev, struct qedr_userq *q, 742 bool requires_db_rec) 743 { 744 struct qedr_ucontext *uctx = 745 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 746 ibucontext); 747 struct qedr_user_mmap_entry *entry; 748 int rc; 749 750 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */ 751 if (requires_db_rec == 0 || !uctx->db_rec) 752 return 0; 753 754 /* Allocate a page for doorbell recovery, add to mmap */ 755 q->db_rec_data = (void *)get_zeroed_page(GFP_USER); 756 if (!q->db_rec_data) { 757 DP_ERR(dev, "get_zeroed_page failed\n"); 758 return -ENOMEM; 759 } 760 761 entry = kzalloc_obj(*entry); 762 if (!entry) 763 goto err_free_db_data; 764 765 entry->address = q->db_rec_data; 766 entry->length = PAGE_SIZE; 767 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE; 768 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext, 769 &entry->rdma_entry, 770 PAGE_SIZE); 771 if (rc) 772 goto err_free_entry; 773 774 q->db_mmap_entry = &entry->rdma_entry; 775 776 return 0; 777 778 err_free_entry: 779 kfree(entry); 780 781 err_free_db_data: 782 free_page((unsigned long)q->db_rec_data); 783 q->db_rec_data = NULL; 784 return -ENOMEM; 785 } 786 787 static inline int qedr_init_user_queue(struct ib_udata *udata, 788 struct qedr_dev *dev, 789 struct qedr_userq *q, u64 buf_addr, 790 size_t buf_len, bool requires_db_rec, 791 int access, 792 int alloc_and_init) 793 { 794 u32 fw_pages; 795 int rc; 796 797 q->buf_addr = buf_addr; 798 q->buf_len = buf_len; 799 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); 800 if (IS_ERR(q->umem)) { 801 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 802 PTR_ERR(q->umem)); 803 return PTR_ERR(q->umem); 804 } 805 806 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT); 807 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 808 if (rc) 809 goto err0; 810 811 if (alloc_and_init) { 812 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 813 if (IS_ERR(q->pbl_tbl)) { 814 rc = PTR_ERR(q->pbl_tbl); 815 goto err0; 816 } 817 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 818 FW_PAGE_SHIFT); 819 } else { 820 q->pbl_tbl = kzalloc_obj(*q->pbl_tbl); 821 if (!q->pbl_tbl) { 822 rc = -ENOMEM; 823 goto err0; 824 } 825 } 826 827 /* mmap the user address used to store doorbell data for recovery */ 828 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); 829 830 err0: 831 ib_umem_release(q->umem); 832 q->umem = NULL; 833 834 return rc; 835 } 836 837 static inline void qedr_init_cq_params(struct qedr_cq *cq, 838 struct qedr_ucontext *ctx, 839 struct qedr_dev *dev, int vector, 840 int chain_entries, int page_cnt, 841 u64 pbl_ptr, 842 struct qed_rdma_create_cq_in_params 843 *params) 844 { 845 memset(params, 0, sizeof(*params)); 846 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 847 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 848 params->cnq_id = vector; 849 params->cq_size = chain_entries - 1; 850 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 851 params->pbl_num_pages = page_cnt; 852 params->pbl_ptr = pbl_ptr; 853 params->pbl_two_level = 0; 854 } 855 856 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 857 { 858 cq->db.data.agg_flags = flags; 859 cq->db.data.value = cpu_to_le32(cons); 860 writeq(cq->db.raw, cq->db_addr); 861 } 862 863 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 864 { 865 struct qedr_cq *cq = get_qedr_cq(ibcq); 866 unsigned long sflags; 867 struct qedr_dev *dev; 868 869 dev = get_qedr_dev(ibcq->device); 870 871 if (cq->destroyed) { 872 DP_ERR(dev, 873 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 874 cq, cq->icid); 875 return -EINVAL; 876 } 877 878 879 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 880 return 0; 881 882 spin_lock_irqsave(&cq->cq_lock, sflags); 883 884 cq->arm_flags = 0; 885 886 if (flags & IB_CQ_SOLICITED) 887 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 888 889 if (flags & IB_CQ_NEXT_COMP) 890 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 891 892 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 893 894 spin_unlock_irqrestore(&cq->cq_lock, sflags); 895 896 return 0; 897 } 898 899 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 900 struct uverbs_attr_bundle *attrs) 901 { 902 struct ib_udata *udata = &attrs->driver_udata; 903 struct ib_device *ibdev = ibcq->device; 904 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 905 udata, struct qedr_ucontext, ibucontext); 906 struct qed_rdma_destroy_cq_out_params destroy_oparams; 907 struct qed_rdma_destroy_cq_in_params destroy_iparams; 908 struct qed_chain_init_params chain_params = { 909 .mode = QED_CHAIN_MODE_PBL, 910 .intended_use = QED_CHAIN_USE_TO_CONSUME, 911 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 912 .elem_size = sizeof(union rdma_cqe), 913 }; 914 struct qedr_dev *dev = get_qedr_dev(ibdev); 915 struct qed_rdma_create_cq_in_params params; 916 struct qedr_create_cq_ureq ureq; 917 int vector = attr->comp_vector; 918 int entries = attr->cqe; 919 struct qedr_cq *cq = get_qedr_cq(ibcq); 920 int chain_entries; 921 u32 db_offset; 922 int page_cnt; 923 u64 pbl_ptr; 924 u16 icid; 925 int rc; 926 927 DP_DEBUG(dev, QEDR_MSG_INIT, 928 "create_cq: called from %s. entries=%d, vector=%d\n", 929 udata ? "User Lib" : "Kernel", entries, vector); 930 931 if (attr->flags) 932 return -EOPNOTSUPP; 933 934 if (entries > QEDR_MAX_CQES) { 935 DP_ERR(dev, 936 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 937 entries, QEDR_MAX_CQES); 938 return -EINVAL; 939 } 940 941 chain_entries = qedr_align_cq_entries(entries); 942 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 943 chain_params.num_elems = chain_entries; 944 945 /* calc db offset. user will add DPI base, kernel will add db addr */ 946 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 947 948 if (udata) { 949 rc = ib_copy_validate_udata_in(udata, ureq, len); 950 if (rc) 951 return rc; 952 953 if (!ureq.len) { 954 DP_ERR(dev, 955 "create cq: cannot create a cq with 0 entries\n"); 956 goto err0; 957 } 958 959 cq->cq_type = QEDR_CQ_TYPE_USER; 960 961 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 962 ureq.len, true, IB_ACCESS_LOCAL_WRITE, 963 1); 964 if (rc) 965 goto err0; 966 967 pbl_ptr = cq->q.pbl_tbl->pa; 968 page_cnt = cq->q.pbl_info.num_pbes; 969 970 cq->ibcq.cqe = chain_entries; 971 cq->q.db_addr = ctx->dpi_addr + db_offset; 972 } else { 973 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 974 975 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl, 976 &chain_params); 977 if (rc) 978 goto err0; 979 980 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 981 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 982 cq->ibcq.cqe = cq->pbl.capacity; 983 } 984 985 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 986 pbl_ptr, ¶ms); 987 988 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 989 if (rc) 990 goto err1; 991 992 cq->icid = icid; 993 cq->sig = QEDR_CQ_MAGIC_NUMBER; 994 spin_lock_init(&cq->cq_lock); 995 996 if (udata) { 997 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); 998 if (rc) 999 goto err2; 1000 1001 rc = qedr_db_recovery_add(dev, cq->q.db_addr, 1002 &cq->q.db_rec_data->db_data, 1003 DB_REC_WIDTH_64B, 1004 DB_REC_USER); 1005 if (rc) 1006 goto err2; 1007 1008 } else { 1009 /* Generate doorbell address. */ 1010 cq->db.data.icid = cq->icid; 1011 cq->db_addr = dev->db_addr + db_offset; 1012 cq->db.data.params = DB_AGG_CMD_MAX << 1013 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1014 1015 /* point to the very last element, passing it we will toggle */ 1016 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 1017 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 1018 cq->latest_cqe = NULL; 1019 consume_cqe(cq); 1020 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 1021 1022 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, 1023 DB_REC_WIDTH_64B, DB_REC_KERNEL); 1024 if (rc) 1025 goto err2; 1026 } 1027 1028 DP_DEBUG(dev, QEDR_MSG_CQ, 1029 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1030 cq->icid, cq, params.cq_size); 1031 1032 return 0; 1033 1034 err2: 1035 destroy_iparams.icid = cq->icid; 1036 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1037 &destroy_oparams); 1038 err1: 1039 if (udata) { 1040 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1041 ib_umem_release(cq->q.umem); 1042 if (cq->q.db_mmap_entry) 1043 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1044 } else { 1045 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1046 } 1047 err0: 1048 return -EINVAL; 1049 } 1050 1051 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1052 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1053 1054 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1055 { 1056 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1057 struct qed_rdma_destroy_cq_out_params oparams; 1058 struct qed_rdma_destroy_cq_in_params iparams; 1059 struct qedr_cq *cq = get_qedr_cq(ibcq); 1060 int iter; 1061 1062 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1063 1064 cq->destroyed = 1; 1065 1066 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1067 if (cq->cq_type == QEDR_CQ_TYPE_GSI) { 1068 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1069 return 0; 1070 } 1071 1072 iparams.icid = cq->icid; 1073 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1074 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1075 1076 if (udata) { 1077 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1078 ib_umem_release(cq->q.umem); 1079 1080 if (cq->q.db_rec_data) { 1081 qedr_db_recovery_del(dev, cq->q.db_addr, 1082 &cq->q.db_rec_data->db_data); 1083 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1084 } 1085 } else { 1086 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1087 } 1088 1089 /* We don't want the IRQ handler to handle a non-existing CQ so we 1090 * wait until all CNQ interrupts, if any, are received. This will always 1091 * happen and will always happen very fast. If not, then a serious error 1092 * has occured. That is why we can use a long delay. 1093 * We spin for a short time so we don’t lose time on context switching 1094 * in case all the completions are handled in that span. Otherwise 1095 * we sleep for a while and check again. Since the CNQ may be 1096 * associated with (only) the current CPU we use msleep to allow the 1097 * current CPU to be freed. 1098 * The CNQ notification is increased in qedr_irq_handler(). 1099 */ 1100 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1101 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1102 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1103 iter--; 1104 } 1105 1106 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1107 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1108 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1109 iter--; 1110 } 1111 1112 /* Note that we don't need to have explicit code to wait for the 1113 * completion of the event handler because it is invoked from the EQ. 1114 * Since the destroy CQ ramrod has also been received on the EQ we can 1115 * be certain that there's no event handler in process. 1116 */ 1117 return 0; 1118 } 1119 1120 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1121 struct ib_qp_attr *attr, 1122 int attr_mask, 1123 struct qed_rdma_modify_qp_in_params 1124 *qp_params) 1125 { 1126 const struct ib_gid_attr *gid_attr; 1127 enum rdma_network_type nw_type; 1128 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1129 u32 ipv4_addr; 1130 int ret; 1131 int i; 1132 1133 gid_attr = grh->sgid_attr; 1134 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1135 if (ret) 1136 return ret; 1137 1138 nw_type = rdma_gid_attr_network_type(gid_attr); 1139 switch (nw_type) { 1140 case RDMA_NETWORK_IPV6: 1141 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1142 sizeof(qp_params->sgid)); 1143 memcpy(&qp_params->dgid.bytes[0], 1144 &grh->dgid, 1145 sizeof(qp_params->dgid)); 1146 qp_params->roce_mode = ROCE_V2_IPV6; 1147 SET_FIELD(qp_params->modify_flags, 1148 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1149 break; 1150 case RDMA_NETWORK_ROCE_V1: 1151 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1152 sizeof(qp_params->sgid)); 1153 memcpy(&qp_params->dgid.bytes[0], 1154 &grh->dgid, 1155 sizeof(qp_params->dgid)); 1156 qp_params->roce_mode = ROCE_V1; 1157 break; 1158 case RDMA_NETWORK_IPV4: 1159 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1160 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1161 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1162 qp_params->sgid.ipv4_addr = ipv4_addr; 1163 ipv4_addr = 1164 qedr_get_ipv4_from_gid(grh->dgid.raw); 1165 qp_params->dgid.ipv4_addr = ipv4_addr; 1166 SET_FIELD(qp_params->modify_flags, 1167 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1168 qp_params->roce_mode = ROCE_V2_IPV4; 1169 break; 1170 default: 1171 return -EINVAL; 1172 } 1173 1174 for (i = 0; i < 4; i++) { 1175 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1176 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1177 } 1178 1179 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1180 qp_params->vlan_id = 0; 1181 1182 return 0; 1183 } 1184 1185 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1186 struct ib_qp_init_attr *attrs, 1187 struct ib_udata *udata) 1188 { 1189 struct qedr_device_attr *qattr = &dev->attr; 1190 1191 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1192 if (attrs->qp_type != IB_QPT_RC && 1193 attrs->qp_type != IB_QPT_GSI && 1194 attrs->qp_type != IB_QPT_XRC_INI && 1195 attrs->qp_type != IB_QPT_XRC_TGT) { 1196 DP_DEBUG(dev, QEDR_MSG_QP, 1197 "create qp: unsupported qp type=0x%x requested\n", 1198 attrs->qp_type); 1199 return -EOPNOTSUPP; 1200 } 1201 1202 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1203 DP_ERR(dev, 1204 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1205 attrs->cap.max_send_wr, qattr->max_sqe); 1206 return -EINVAL; 1207 } 1208 1209 if (attrs->cap.max_inline_data > qattr->max_inline) { 1210 DP_ERR(dev, 1211 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1212 attrs->cap.max_inline_data, qattr->max_inline); 1213 return -EINVAL; 1214 } 1215 1216 if (attrs->cap.max_send_sge > qattr->max_sge) { 1217 DP_ERR(dev, 1218 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1219 attrs->cap.max_send_sge, qattr->max_sge); 1220 return -EINVAL; 1221 } 1222 1223 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1224 DP_ERR(dev, 1225 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1226 attrs->cap.max_recv_sge, qattr->max_sge); 1227 return -EINVAL; 1228 } 1229 1230 /* verify consumer QPs are not trying to use GSI QP's CQ. 1231 * TGT QP isn't associated with RQ/SQ 1232 */ 1233 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) && 1234 (attrs->qp_type != IB_QPT_XRC_TGT) && 1235 (attrs->qp_type != IB_QPT_XRC_INI)) { 1236 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq); 1237 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq); 1238 1239 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) || 1240 (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) { 1241 DP_ERR(dev, 1242 "create qp: consumer QP cannot use GSI CQs.\n"); 1243 return -EINVAL; 1244 } 1245 } 1246 1247 return 0; 1248 } 1249 1250 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1251 struct qedr_srq *srq, struct ib_udata *udata) 1252 { 1253 struct qedr_create_srq_uresp uresp = {}; 1254 int rc; 1255 1256 uresp.srq_id = srq->srq_id; 1257 1258 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1259 if (rc) 1260 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1261 1262 return rc; 1263 } 1264 1265 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1266 struct qedr_create_qp_uresp *uresp, 1267 struct qedr_qp *qp) 1268 { 1269 /* iWARP requires two doorbells per RQ. */ 1270 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1271 uresp->rq_db_offset = 1272 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1273 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1274 } else { 1275 uresp->rq_db_offset = 1276 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1277 } 1278 1279 uresp->rq_icid = qp->icid; 1280 if (qp->urq.db_mmap_entry) 1281 uresp->rq_db_rec_addr = 1282 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry); 1283 } 1284 1285 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1286 struct qedr_create_qp_uresp *uresp, 1287 struct qedr_qp *qp) 1288 { 1289 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1290 1291 /* iWARP uses the same cid for rq and sq */ 1292 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1293 uresp->sq_icid = qp->icid; 1294 else 1295 uresp->sq_icid = qp->icid + 1; 1296 1297 if (qp->usq.db_mmap_entry) 1298 uresp->sq_db_rec_addr = 1299 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry); 1300 } 1301 1302 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1303 struct qedr_qp *qp, struct ib_udata *udata, 1304 struct qedr_create_qp_uresp *uresp) 1305 { 1306 int rc; 1307 1308 memset(uresp, 0, sizeof(*uresp)); 1309 1310 if (qedr_qp_has_sq(qp)) 1311 qedr_copy_sq_uresp(dev, uresp, qp); 1312 1313 if (qedr_qp_has_rq(qp)) 1314 qedr_copy_rq_uresp(dev, uresp, qp); 1315 1316 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1317 uresp->qp_id = qp->qp_id; 1318 1319 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); 1320 if (rc) 1321 DP_ERR(dev, 1322 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1323 qp->icid); 1324 1325 return rc; 1326 } 1327 1328 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 1329 { 1330 qed_chain_reset(&qph->pbl); 1331 qph->prod = 0; 1332 qph->cons = 0; 1333 qph->wqe_cons = 0; 1334 qph->db_data.data.value = cpu_to_le16(0); 1335 } 1336 1337 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1338 struct qedr_qp *qp, 1339 struct qedr_pd *pd, 1340 struct ib_qp_init_attr *attrs) 1341 { 1342 spin_lock_init(&qp->q_lock); 1343 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1344 kref_init(&qp->refcnt); 1345 init_completion(&qp->iwarp_cm_comp); 1346 init_completion(&qp->qp_rel_comp); 1347 } 1348 1349 qp->pd = pd; 1350 qp->qp_type = attrs->qp_type; 1351 qp->max_inline_data = attrs->cap.max_inline_data; 1352 qp->state = QED_ROCE_QP_STATE_RESET; 1353 1354 qp->prev_wqe_size = 0; 1355 1356 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; 1357 qp->dev = dev; 1358 if (qedr_qp_has_sq(qp)) { 1359 qedr_reset_qp_hwq_info(&qp->sq); 1360 qp->sq.max_sges = attrs->cap.max_send_sge; 1361 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1362 DP_DEBUG(dev, QEDR_MSG_QP, 1363 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1364 qp->sq.max_sges, qp->sq_cq->icid); 1365 } 1366 1367 if (attrs->srq) 1368 qp->srq = get_qedr_srq(attrs->srq); 1369 1370 if (qedr_qp_has_rq(qp)) { 1371 qedr_reset_qp_hwq_info(&qp->rq); 1372 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1373 qp->rq.max_sges = attrs->cap.max_recv_sge; 1374 DP_DEBUG(dev, QEDR_MSG_QP, 1375 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1376 qp->rq.max_sges, qp->rq_cq->icid); 1377 } 1378 1379 DP_DEBUG(dev, QEDR_MSG_QP, 1380 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1381 pd->pd_id, qp->qp_type, qp->max_inline_data, 1382 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1383 DP_DEBUG(dev, QEDR_MSG_QP, 1384 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1385 qp->sq.max_sges, qp->sq_cq->icid); 1386 } 1387 1388 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1389 { 1390 int rc = 0; 1391 1392 if (qedr_qp_has_sq(qp)) { 1393 qp->sq.db = dev->db_addr + 1394 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1395 qp->sq.db_data.data.icid = qp->icid + 1; 1396 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data, 1397 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1398 if (rc) 1399 return rc; 1400 } 1401 1402 if (qedr_qp_has_rq(qp)) { 1403 qp->rq.db = dev->db_addr + 1404 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1405 qp->rq.db_data.data.icid = qp->icid; 1406 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data, 1407 DB_REC_WIDTH_32B, DB_REC_KERNEL); 1408 if (rc && qedr_qp_has_sq(qp)) 1409 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 1410 } 1411 1412 return rc; 1413 } 1414 1415 static int qedr_check_srq_params(struct qedr_dev *dev, 1416 struct ib_srq_init_attr *attrs, 1417 struct ib_udata *udata) 1418 { 1419 struct qedr_device_attr *qattr = &dev->attr; 1420 1421 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1422 DP_ERR(dev, 1423 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1424 attrs->attr.max_wr, qattr->max_srq_wr); 1425 return -EINVAL; 1426 } 1427 1428 if (attrs->attr.max_sge > qattr->max_sge) { 1429 DP_ERR(dev, 1430 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1431 attrs->attr.max_sge, qattr->max_sge); 1432 } 1433 1434 if (!udata && attrs->srq_type == IB_SRQT_XRC) { 1435 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n"); 1436 return -EINVAL; 1437 } 1438 1439 return 0; 1440 } 1441 1442 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1443 { 1444 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1445 ib_umem_release(srq->usrq.umem); 1446 ib_umem_release(srq->prod_umem); 1447 } 1448 1449 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1450 { 1451 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1452 struct qedr_dev *dev = srq->dev; 1453 1454 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1455 1456 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1457 hw_srq->virt_prod_pair_addr, 1458 hw_srq->phy_prod_pair_addr); 1459 } 1460 1461 static int qedr_init_srq_user_params(struct ib_udata *udata, 1462 struct qedr_srq *srq, 1463 struct qedr_create_srq_ureq *ureq, 1464 int access) 1465 { 1466 struct scatterlist *sg; 1467 int rc; 1468 1469 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1470 ureq->srq_len, false, access, 1); 1471 if (rc) 1472 return rc; 1473 1474 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, 1475 sizeof(struct rdma_srq_producers), access); 1476 if (IS_ERR(srq->prod_umem)) { 1477 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1478 ib_umem_release(srq->usrq.umem); 1479 DP_ERR(srq->dev, 1480 "create srq: failed ib_umem_get for producer, got %ld\n", 1481 PTR_ERR(srq->prod_umem)); 1482 return PTR_ERR(srq->prod_umem); 1483 } 1484 1485 sg = srq->prod_umem->sgt_append.sgt.sgl; 1486 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1487 1488 return 0; 1489 } 1490 1491 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1492 struct qedr_dev *dev, 1493 struct ib_srq_init_attr *init_attr) 1494 { 1495 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1496 struct qed_chain_init_params params = { 1497 .mode = QED_CHAIN_MODE_PBL, 1498 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1499 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 1500 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE, 1501 }; 1502 dma_addr_t phy_prod_pair_addr; 1503 u32 num_elems; 1504 void *va; 1505 int rc; 1506 1507 va = dma_alloc_coherent(&dev->pdev->dev, 1508 sizeof(struct rdma_srq_producers), 1509 &phy_prod_pair_addr, GFP_KERNEL); 1510 if (!va) { 1511 DP_ERR(dev, 1512 "create srq: failed to allocate dma memory for producer\n"); 1513 return -ENOMEM; 1514 } 1515 1516 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1517 hw_srq->virt_prod_pair_addr = va; 1518 1519 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1520 params.num_elems = num_elems; 1521 1522 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, ¶ms); 1523 if (rc) 1524 goto err0; 1525 1526 hw_srq->num_elems = num_elems; 1527 1528 return 0; 1529 1530 err0: 1531 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1532 va, phy_prod_pair_addr); 1533 return rc; 1534 } 1535 1536 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1537 struct ib_udata *udata) 1538 { 1539 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1540 struct qed_rdma_create_srq_in_params in_params = {}; 1541 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1542 struct qed_rdma_create_srq_out_params out_params; 1543 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1544 struct qedr_create_srq_ureq ureq; 1545 u64 pbl_base_addr, phy_prod_pair_addr; 1546 struct qedr_srq_hwq_info *hw_srq; 1547 u32 page_cnt, page_size; 1548 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1549 int rc = 0; 1550 1551 DP_DEBUG(dev, QEDR_MSG_QP, 1552 "create SRQ called from %s (pd %p)\n", 1553 (udata) ? "User lib" : "kernel", pd); 1554 1555 if (init_attr->srq_type != IB_SRQT_BASIC && 1556 init_attr->srq_type != IB_SRQT_XRC) 1557 return -EOPNOTSUPP; 1558 1559 rc = qedr_check_srq_params(dev, init_attr, udata); 1560 if (rc) 1561 return -EINVAL; 1562 1563 srq->dev = dev; 1564 srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC); 1565 hw_srq = &srq->hw_srq; 1566 spin_lock_init(&srq->lock); 1567 1568 hw_srq->max_wr = init_attr->attr.max_wr; 1569 hw_srq->max_sges = init_attr->attr.max_sge; 1570 1571 if (udata) { 1572 rc = ib_copy_validate_udata_in(udata, ureq, srq_len); 1573 if (rc) 1574 return rc; 1575 1576 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0); 1577 if (rc) 1578 goto err0; 1579 1580 page_cnt = srq->usrq.pbl_info.num_pbes; 1581 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1582 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1583 page_size = PAGE_SIZE; 1584 } else { 1585 struct qed_chain *pbl; 1586 1587 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1588 if (rc) 1589 goto err0; 1590 1591 pbl = &hw_srq->pbl; 1592 page_cnt = qed_chain_get_page_cnt(pbl); 1593 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1594 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1595 page_size = QED_CHAIN_PAGE_SIZE; 1596 } 1597 1598 in_params.pd_id = pd->pd_id; 1599 in_params.pbl_base_addr = pbl_base_addr; 1600 in_params.prod_pair_addr = phy_prod_pair_addr; 1601 in_params.num_pages = page_cnt; 1602 in_params.page_size = page_size; 1603 if (srq->is_xrc) { 1604 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd); 1605 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq); 1606 1607 in_params.is_xrc = 1; 1608 in_params.xrcd_id = xrcd->xrcd_id; 1609 in_params.cq_cid = cq->icid; 1610 } 1611 1612 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1613 if (rc) 1614 goto err1; 1615 1616 srq->srq_id = out_params.srq_id; 1617 1618 if (udata) { 1619 rc = qedr_copy_srq_uresp(dev, srq, udata); 1620 if (rc) 1621 goto err2; 1622 } 1623 1624 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1625 if (rc) 1626 goto err2; 1627 1628 DP_DEBUG(dev, QEDR_MSG_SRQ, 1629 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1630 return 0; 1631 1632 err2: 1633 destroy_in_params.srq_id = srq->srq_id; 1634 1635 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1636 err1: 1637 if (udata) 1638 qedr_free_srq_user_params(srq); 1639 else 1640 qedr_free_srq_kernel_params(srq); 1641 err0: 1642 return -EFAULT; 1643 } 1644 1645 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1646 { 1647 struct qed_rdma_destroy_srq_in_params in_params = {}; 1648 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1649 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1650 1651 xa_erase_irq(&dev->srqs, srq->srq_id); 1652 in_params.srq_id = srq->srq_id; 1653 in_params.is_xrc = srq->is_xrc; 1654 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1655 1656 if (ibsrq->uobject) 1657 qedr_free_srq_user_params(srq); 1658 else 1659 qedr_free_srq_kernel_params(srq); 1660 1661 DP_DEBUG(dev, QEDR_MSG_SRQ, 1662 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1663 srq->srq_id); 1664 return 0; 1665 } 1666 1667 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1668 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1669 { 1670 struct qed_rdma_modify_srq_in_params in_params = {}; 1671 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1672 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1673 int rc; 1674 1675 if (attr_mask & IB_SRQ_MAX_WR) { 1676 DP_ERR(dev, 1677 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1678 attr_mask, srq); 1679 return -EINVAL; 1680 } 1681 1682 if (attr_mask & IB_SRQ_LIMIT) { 1683 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1684 DP_ERR(dev, 1685 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1686 attr->srq_limit, srq->hw_srq.max_wr); 1687 return -EINVAL; 1688 } 1689 1690 in_params.srq_id = srq->srq_id; 1691 in_params.wqe_limit = attr->srq_limit; 1692 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1693 if (rc) 1694 return rc; 1695 } 1696 1697 srq->srq_limit = attr->srq_limit; 1698 1699 DP_DEBUG(dev, QEDR_MSG_SRQ, 1700 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1701 1702 return 0; 1703 } 1704 1705 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type) 1706 { 1707 switch (ib_qp_type) { 1708 case IB_QPT_RC: 1709 return QED_RDMA_QP_TYPE_RC; 1710 case IB_QPT_XRC_INI: 1711 return QED_RDMA_QP_TYPE_XRC_INI; 1712 case IB_QPT_XRC_TGT: 1713 return QED_RDMA_QP_TYPE_XRC_TGT; 1714 default: 1715 return QED_RDMA_QP_TYPE_INVAL; 1716 } 1717 } 1718 1719 static inline void 1720 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1721 struct qedr_pd *pd, 1722 struct qedr_qp *qp, 1723 struct ib_qp_init_attr *attrs, 1724 bool fmr_and_reserved_lkey, 1725 struct qed_rdma_create_qp_in_params *params) 1726 { 1727 /* QP handle to be written in an async event */ 1728 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1729 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1730 1731 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1732 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1733 params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type); 1734 params->stats_queue = 0; 1735 1736 if (pd) { 1737 params->pd = pd->pd_id; 1738 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1739 } 1740 1741 if (qedr_qp_has_sq(qp)) 1742 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1743 1744 if (qedr_qp_has_rq(qp)) 1745 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1746 1747 if (qedr_qp_has_srq(qp)) { 1748 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1749 params->srq_id = qp->srq->srq_id; 1750 params->use_srq = true; 1751 } else { 1752 params->srq_id = 0; 1753 params->use_srq = false; 1754 } 1755 } 1756 1757 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1758 { 1759 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1760 "qp=%p. " 1761 "sq_addr=0x%llx, " 1762 "sq_len=%zd, " 1763 "rq_addr=0x%llx, " 1764 "rq_len=%zd" 1765 "\n", 1766 qp, 1767 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0, 1768 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0, 1769 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0, 1770 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0); 1771 } 1772 1773 static inline void 1774 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1775 struct qedr_qp *qp, 1776 struct qed_rdma_create_qp_out_params *out_params) 1777 { 1778 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1779 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1780 1781 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1782 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1783 if (!qp->srq) { 1784 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1785 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1786 } 1787 1788 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1789 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1790 } 1791 1792 static void qedr_cleanup_user(struct qedr_dev *dev, 1793 struct qedr_ucontext *ctx, 1794 struct qedr_qp *qp) 1795 { 1796 if (qedr_qp_has_sq(qp)) { 1797 ib_umem_release(qp->usq.umem); 1798 qp->usq.umem = NULL; 1799 } 1800 1801 if (qedr_qp_has_rq(qp)) { 1802 ib_umem_release(qp->urq.umem); 1803 qp->urq.umem = NULL; 1804 } 1805 1806 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1807 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); 1808 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); 1809 } else { 1810 kfree(qp->usq.pbl_tbl); 1811 kfree(qp->urq.pbl_tbl); 1812 } 1813 1814 if (qp->usq.db_rec_data) { 1815 qedr_db_recovery_del(dev, qp->usq.db_addr, 1816 &qp->usq.db_rec_data->db_data); 1817 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry); 1818 } 1819 1820 if (qp->urq.db_rec_data) { 1821 qedr_db_recovery_del(dev, qp->urq.db_addr, 1822 &qp->urq.db_rec_data->db_data); 1823 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry); 1824 } 1825 1826 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1827 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr, 1828 &qp->urq.db_rec_db2_data); 1829 } 1830 1831 static int qedr_create_user_qp(struct qedr_dev *dev, 1832 struct qedr_qp *qp, 1833 struct ib_pd *ibpd, 1834 struct ib_udata *udata, 1835 struct ib_qp_init_attr *attrs) 1836 { 1837 struct qed_rdma_create_qp_in_params in_params; 1838 struct qed_rdma_create_qp_out_params out_params; 1839 struct qedr_create_qp_uresp uresp = {}; 1840 struct qedr_create_qp_ureq ureq; 1841 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1842 struct qedr_ucontext *ctx = NULL; 1843 struct qedr_pd *pd = NULL; 1844 int rc = 0; 1845 1846 qp->create_type = QEDR_QP_CREATE_USER; 1847 1848 if (ibpd) { 1849 pd = get_qedr_pd(ibpd); 1850 ctx = pd->uctx; 1851 } 1852 1853 if (udata) { 1854 rc = ib_copy_validate_udata_in(udata, ureq, rq_len); 1855 if (rc) 1856 return rc; 1857 } 1858 1859 if (qedr_qp_has_sq(qp)) { 1860 /* SQ - read access only (0) */ 1861 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1862 ureq.sq_len, true, 0, alloc_and_init); 1863 if (rc) 1864 return rc; 1865 } 1866 1867 if (qedr_qp_has_rq(qp)) { 1868 /* RQ - read access only (0) */ 1869 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1870 ureq.rq_len, true, 0, alloc_and_init); 1871 if (rc) { 1872 ib_umem_release(qp->usq.umem); 1873 qp->usq.umem = NULL; 1874 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1875 qedr_free_pbl(dev, &qp->usq.pbl_info, 1876 qp->usq.pbl_tbl); 1877 } else { 1878 kfree(qp->usq.pbl_tbl); 1879 } 1880 return rc; 1881 } 1882 } 1883 1884 memset(&in_params, 0, sizeof(in_params)); 1885 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1886 in_params.qp_handle_lo = ureq.qp_handle_lo; 1887 in_params.qp_handle_hi = ureq.qp_handle_hi; 1888 1889 if (qp->qp_type == IB_QPT_XRC_TGT) { 1890 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd); 1891 1892 in_params.xrcd_id = xrcd->xrcd_id; 1893 in_params.qp_handle_lo = qp->qp_id; 1894 in_params.use_srq = 1; 1895 } 1896 1897 if (qedr_qp_has_sq(qp)) { 1898 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1899 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1900 } 1901 1902 if (qedr_qp_has_rq(qp)) { 1903 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1904 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1905 } 1906 1907 if (ctx) 1908 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode); 1909 1910 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1911 &in_params, &out_params); 1912 1913 if (!qp->qed_qp) { 1914 rc = -ENOMEM; 1915 goto err1; 1916 } 1917 1918 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1919 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1920 1921 qp->qp_id = out_params.qp_id; 1922 qp->icid = out_params.icid; 1923 1924 if (udata) { 1925 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); 1926 if (rc) 1927 goto err; 1928 } 1929 1930 /* db offset was calculated in copy_qp_uresp, now set in the user q */ 1931 if (qedr_qp_has_sq(qp)) { 1932 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; 1933 qp->sq.max_wr = attrs->cap.max_send_wr; 1934 rc = qedr_db_recovery_add(dev, qp->usq.db_addr, 1935 &qp->usq.db_rec_data->db_data, 1936 DB_REC_WIDTH_32B, 1937 DB_REC_USER); 1938 if (rc) 1939 goto err; 1940 } 1941 1942 if (qedr_qp_has_rq(qp)) { 1943 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; 1944 qp->rq.max_wr = attrs->cap.max_recv_wr; 1945 rc = qedr_db_recovery_add(dev, qp->urq.db_addr, 1946 &qp->urq.db_rec_data->db_data, 1947 DB_REC_WIDTH_32B, 1948 DB_REC_USER); 1949 if (rc) 1950 goto err; 1951 } 1952 1953 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1954 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; 1955 1956 /* calculate the db_rec_db2 data since it is constant so no 1957 * need to reflect from user 1958 */ 1959 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); 1960 qp->urq.db_rec_db2_data.data.value = 1961 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); 1962 1963 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, 1964 &qp->urq.db_rec_db2_data, 1965 DB_REC_WIDTH_32B, 1966 DB_REC_USER); 1967 if (rc) 1968 goto err; 1969 } 1970 qedr_qp_user_print(dev, qp); 1971 return rc; 1972 err: 1973 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1974 if (rc) 1975 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1976 1977 err1: 1978 qedr_cleanup_user(dev, ctx, qp); 1979 return rc; 1980 } 1981 1982 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1983 { 1984 int rc; 1985 1986 qp->sq.db = dev->db_addr + 1987 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1988 qp->sq.db_data.data.icid = qp->icid; 1989 1990 rc = qedr_db_recovery_add(dev, qp->sq.db, 1991 &qp->sq.db_data, 1992 DB_REC_WIDTH_32B, 1993 DB_REC_KERNEL); 1994 if (rc) 1995 return rc; 1996 1997 qp->rq.db = dev->db_addr + 1998 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1999 qp->rq.db_data.data.icid = qp->icid; 2000 qp->rq.iwarp_db2 = dev->db_addr + 2001 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 2002 qp->rq.iwarp_db2_data.data.icid = qp->icid; 2003 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 2004 2005 rc = qedr_db_recovery_add(dev, qp->rq.db, 2006 &qp->rq.db_data, 2007 DB_REC_WIDTH_32B, 2008 DB_REC_KERNEL); 2009 if (rc) 2010 return rc; 2011 2012 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2, 2013 &qp->rq.iwarp_db2_data, 2014 DB_REC_WIDTH_32B, 2015 DB_REC_KERNEL); 2016 return rc; 2017 } 2018 2019 static int 2020 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 2021 struct qedr_qp *qp, 2022 struct qed_rdma_create_qp_in_params *in_params, 2023 u32 n_sq_elems, u32 n_rq_elems) 2024 { 2025 struct qed_rdma_create_qp_out_params out_params; 2026 struct qed_chain_init_params params = { 2027 .mode = QED_CHAIN_MODE_PBL, 2028 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2029 }; 2030 int rc; 2031 2032 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2033 params.num_elems = n_sq_elems; 2034 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2035 2036 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2037 if (rc) 2038 return rc; 2039 2040 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 2041 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 2042 2043 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2044 params.num_elems = n_rq_elems; 2045 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2046 2047 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2048 if (rc) 2049 return rc; 2050 2051 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 2052 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 2053 2054 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2055 in_params, &out_params); 2056 2057 if (!qp->qed_qp) 2058 return -EINVAL; 2059 2060 qp->qp_id = out_params.qp_id; 2061 qp->icid = out_params.icid; 2062 2063 return qedr_set_roce_db_info(dev, qp); 2064 } 2065 2066 static int 2067 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 2068 struct qedr_qp *qp, 2069 struct qed_rdma_create_qp_in_params *in_params, 2070 u32 n_sq_elems, u32 n_rq_elems) 2071 { 2072 struct qed_rdma_create_qp_out_params out_params; 2073 struct qed_chain_init_params params = { 2074 .mode = QED_CHAIN_MODE_PBL, 2075 .cnt_type = QED_CHAIN_CNT_TYPE_U32, 2076 }; 2077 int rc; 2078 2079 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 2080 QEDR_SQE_ELEMENT_SIZE, 2081 QED_CHAIN_PAGE_SIZE, 2082 QED_CHAIN_MODE_PBL); 2083 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 2084 QEDR_RQE_ELEMENT_SIZE, 2085 QED_CHAIN_PAGE_SIZE, 2086 QED_CHAIN_MODE_PBL); 2087 2088 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 2089 in_params, &out_params); 2090 2091 if (!qp->qed_qp) 2092 return -EINVAL; 2093 2094 /* Now we allocate the chain */ 2095 2096 params.intended_use = QED_CHAIN_USE_TO_PRODUCE; 2097 params.num_elems = n_sq_elems; 2098 params.elem_size = QEDR_SQE_ELEMENT_SIZE; 2099 params.ext_pbl_virt = out_params.sq_pbl_virt; 2100 params.ext_pbl_phys = out_params.sq_pbl_phys; 2101 2102 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms); 2103 if (rc) 2104 goto err; 2105 2106 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE; 2107 params.num_elems = n_rq_elems; 2108 params.elem_size = QEDR_RQE_ELEMENT_SIZE; 2109 params.ext_pbl_virt = out_params.rq_pbl_virt; 2110 params.ext_pbl_phys = out_params.rq_pbl_phys; 2111 2112 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms); 2113 if (rc) 2114 goto err; 2115 2116 qp->qp_id = out_params.qp_id; 2117 qp->icid = out_params.icid; 2118 2119 return qedr_set_iwarp_db_info(dev, qp); 2120 2121 err: 2122 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2123 2124 return rc; 2125 } 2126 2127 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 2128 { 2129 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 2130 kfree(qp->wqe_wr_id); 2131 2132 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 2133 kfree(qp->rqe_wr_id); 2134 2135 /* GSI qp is not registered to db mechanism so no need to delete */ 2136 if (qp->qp_type == IB_QPT_GSI) 2137 return; 2138 2139 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 2140 2141 if (!qp->srq) { 2142 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data); 2143 2144 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2145 qedr_db_recovery_del(dev, qp->rq.iwarp_db2, 2146 &qp->rq.iwarp_db2_data); 2147 } 2148 } 2149 2150 static int qedr_create_kernel_qp(struct qedr_dev *dev, 2151 struct qedr_qp *qp, 2152 struct ib_pd *ibpd, 2153 struct ib_qp_init_attr *attrs) 2154 { 2155 struct qed_rdma_create_qp_in_params in_params; 2156 struct qedr_pd *pd = get_qedr_pd(ibpd); 2157 int rc = -EINVAL; 2158 u32 n_rq_elems; 2159 u32 n_sq_elems; 2160 u32 n_sq_entries; 2161 2162 memset(&in_params, 0, sizeof(in_params)); 2163 qp->create_type = QEDR_QP_CREATE_KERNEL; 2164 2165 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 2166 * the ring. The ring should allow at least a single WR, even if the 2167 * user requested none, due to allocation issues. 2168 * We should add an extra WR since the prod and cons indices of 2169 * wqe_wr_id are managed in such a way that the WQ is considered full 2170 * when (prod+1)%max_wr==cons. We currently don't do that because we 2171 * double the number of entries due an iSER issue that pushes far more 2172 * WRs than indicated. If we decline its ib_post_send() then we get 2173 * error prints in the dmesg we'd like to avoid. 2174 */ 2175 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2176 dev->attr.max_sqe); 2177 2178 qp->wqe_wr_id = kzalloc_objs(*qp->wqe_wr_id, qp->sq.max_wr); 2179 if (!qp->wqe_wr_id) { 2180 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 2181 return -ENOMEM; 2182 } 2183 2184 /* QP handle to be written in CQE */ 2185 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 2186 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 2187 2188 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 2189 * the ring. There ring should allow at least a single WR, even if the 2190 * user requested none, due to allocation issues. 2191 */ 2192 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 2193 2194 /* Allocate driver internal RQ array */ 2195 qp->rqe_wr_id = kzalloc_objs(*qp->rqe_wr_id, qp->rq.max_wr); 2196 if (!qp->rqe_wr_id) { 2197 DP_ERR(dev, 2198 "create qp: failed RQ shadow memory allocation\n"); 2199 kfree(qp->wqe_wr_id); 2200 return -ENOMEM; 2201 } 2202 2203 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 2204 2205 n_sq_entries = attrs->cap.max_send_wr; 2206 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 2207 n_sq_entries = max_t(u32, n_sq_entries, 1); 2208 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2209 2210 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 2211 2212 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2213 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 2214 n_sq_elems, n_rq_elems); 2215 else 2216 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 2217 n_sq_elems, n_rq_elems); 2218 if (rc) 2219 qedr_cleanup_kernel(dev, qp); 2220 2221 return rc; 2222 } 2223 2224 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2225 struct ib_udata *udata) 2226 { 2227 struct qedr_ucontext *ctx = 2228 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 2229 ibucontext); 2230 int rc; 2231 2232 if (qp->qp_type != IB_QPT_GSI) { 2233 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2234 if (rc) 2235 return rc; 2236 } 2237 2238 if (qp->create_type == QEDR_QP_CREATE_USER) 2239 qedr_cleanup_user(dev, ctx, qp); 2240 else 2241 qedr_cleanup_kernel(dev, qp); 2242 2243 return 0; 2244 } 2245 2246 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 2247 struct ib_udata *udata) 2248 { 2249 struct qedr_xrcd *xrcd = NULL; 2250 struct ib_pd *ibpd = ibqp->pd; 2251 struct qedr_pd *pd = get_qedr_pd(ibpd); 2252 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 2253 struct qedr_qp *qp = get_qedr_qp(ibqp); 2254 int rc = 0; 2255 2256 if (attrs->create_flags) 2257 return -EOPNOTSUPP; 2258 2259 if (attrs->qp_type == IB_QPT_XRC_TGT) 2260 xrcd = get_qedr_xrcd(attrs->xrcd); 2261 else 2262 pd = get_qedr_pd(ibpd); 2263 2264 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2265 udata ? "user library" : "kernel", pd); 2266 2267 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 2268 if (rc) 2269 return rc; 2270 2271 DP_DEBUG(dev, QEDR_MSG_QP, 2272 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2273 udata ? "user library" : "kernel", attrs->event_handler, pd, 2274 get_qedr_cq(attrs->send_cq), 2275 get_qedr_cq(attrs->send_cq)->icid, 2276 get_qedr_cq(attrs->recv_cq), 2277 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2278 2279 qedr_set_common_qp_params(dev, qp, pd, attrs); 2280 2281 if (attrs->qp_type == IB_QPT_GSI) 2282 return qedr_create_gsi_qp(dev, attrs, qp); 2283 2284 if (udata || xrcd) 2285 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2286 else 2287 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2288 2289 if (rc) 2290 return rc; 2291 2292 qp->ibqp.qp_num = qp->qp_id; 2293 2294 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2295 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 2296 if (rc) 2297 goto out_free_qp_resources; 2298 } 2299 2300 return 0; 2301 2302 out_free_qp_resources: 2303 qedr_free_qp_resources(dev, qp, udata); 2304 return -EFAULT; 2305 } 2306 2307 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2308 { 2309 switch (qp_state) { 2310 case QED_ROCE_QP_STATE_RESET: 2311 return IB_QPS_RESET; 2312 case QED_ROCE_QP_STATE_INIT: 2313 return IB_QPS_INIT; 2314 case QED_ROCE_QP_STATE_RTR: 2315 return IB_QPS_RTR; 2316 case QED_ROCE_QP_STATE_RTS: 2317 return IB_QPS_RTS; 2318 case QED_ROCE_QP_STATE_SQD: 2319 return IB_QPS_SQD; 2320 case QED_ROCE_QP_STATE_ERR: 2321 return IB_QPS_ERR; 2322 case QED_ROCE_QP_STATE_SQE: 2323 return IB_QPS_SQE; 2324 } 2325 return IB_QPS_ERR; 2326 } 2327 2328 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2329 enum ib_qp_state qp_state) 2330 { 2331 switch (qp_state) { 2332 case IB_QPS_RESET: 2333 return QED_ROCE_QP_STATE_RESET; 2334 case IB_QPS_INIT: 2335 return QED_ROCE_QP_STATE_INIT; 2336 case IB_QPS_RTR: 2337 return QED_ROCE_QP_STATE_RTR; 2338 case IB_QPS_RTS: 2339 return QED_ROCE_QP_STATE_RTS; 2340 case IB_QPS_SQD: 2341 return QED_ROCE_QP_STATE_SQD; 2342 case IB_QPS_ERR: 2343 return QED_ROCE_QP_STATE_ERR; 2344 default: 2345 return QED_ROCE_QP_STATE_ERR; 2346 } 2347 } 2348 2349 static int qedr_update_qp_state(struct qedr_dev *dev, 2350 struct qedr_qp *qp, 2351 enum qed_roce_qp_state cur_state, 2352 enum qed_roce_qp_state new_state) 2353 { 2354 int status = 0; 2355 2356 if (new_state == cur_state) 2357 return 0; 2358 2359 switch (cur_state) { 2360 case QED_ROCE_QP_STATE_RESET: 2361 switch (new_state) { 2362 case QED_ROCE_QP_STATE_INIT: 2363 break; 2364 default: 2365 status = -EINVAL; 2366 break; 2367 } 2368 break; 2369 case QED_ROCE_QP_STATE_INIT: 2370 switch (new_state) { 2371 case QED_ROCE_QP_STATE_RTR: 2372 /* Update doorbell (in case post_recv was 2373 * done before move to RTR) 2374 */ 2375 2376 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2377 writel(qp->rq.db_data.raw, qp->rq.db); 2378 } 2379 break; 2380 case QED_ROCE_QP_STATE_ERR: 2381 break; 2382 default: 2383 /* Invalid state change. */ 2384 status = -EINVAL; 2385 break; 2386 } 2387 break; 2388 case QED_ROCE_QP_STATE_RTR: 2389 /* RTR->XXX */ 2390 switch (new_state) { 2391 case QED_ROCE_QP_STATE_RTS: 2392 break; 2393 case QED_ROCE_QP_STATE_ERR: 2394 break; 2395 default: 2396 /* Invalid state change. */ 2397 status = -EINVAL; 2398 break; 2399 } 2400 break; 2401 case QED_ROCE_QP_STATE_RTS: 2402 /* RTS->XXX */ 2403 switch (new_state) { 2404 case QED_ROCE_QP_STATE_SQD: 2405 break; 2406 case QED_ROCE_QP_STATE_ERR: 2407 break; 2408 default: 2409 /* Invalid state change. */ 2410 status = -EINVAL; 2411 break; 2412 } 2413 break; 2414 case QED_ROCE_QP_STATE_SQD: 2415 /* SQD->XXX */ 2416 switch (new_state) { 2417 case QED_ROCE_QP_STATE_RTS: 2418 case QED_ROCE_QP_STATE_ERR: 2419 break; 2420 default: 2421 /* Invalid state change. */ 2422 status = -EINVAL; 2423 break; 2424 } 2425 break; 2426 case QED_ROCE_QP_STATE_ERR: 2427 /* ERR->XXX */ 2428 switch (new_state) { 2429 case QED_ROCE_QP_STATE_RESET: 2430 if ((qp->rq.prod != qp->rq.cons) || 2431 (qp->sq.prod != qp->sq.cons)) { 2432 DP_NOTICE(dev, 2433 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2434 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2435 qp->sq.cons); 2436 status = -EINVAL; 2437 } 2438 break; 2439 default: 2440 status = -EINVAL; 2441 break; 2442 } 2443 break; 2444 default: 2445 status = -EINVAL; 2446 break; 2447 } 2448 2449 return status; 2450 } 2451 2452 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2453 int attr_mask, struct ib_udata *udata) 2454 { 2455 struct qedr_qp *qp = get_qedr_qp(ibqp); 2456 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2457 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2458 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2459 enum ib_qp_state old_qp_state, new_qp_state; 2460 enum qed_roce_qp_state cur_state; 2461 int rc = 0; 2462 2463 DP_DEBUG(dev, QEDR_MSG_QP, 2464 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2465 attr->qp_state); 2466 2467 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 2468 return -EOPNOTSUPP; 2469 2470 old_qp_state = qedr_get_ibqp_state(qp->state); 2471 if (attr_mask & IB_QP_STATE) 2472 new_qp_state = attr->qp_state; 2473 else 2474 new_qp_state = old_qp_state; 2475 2476 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2477 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2478 ibqp->qp_type, attr_mask)) { 2479 DP_ERR(dev, 2480 "modify qp: invalid attribute mask=0x%x specified for\n" 2481 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2482 attr_mask, qp->qp_id, ibqp->qp_type, 2483 old_qp_state, new_qp_state); 2484 rc = -EINVAL; 2485 goto err; 2486 } 2487 } 2488 2489 /* Translate the masks... */ 2490 if (attr_mask & IB_QP_STATE) { 2491 SET_FIELD(qp_params.modify_flags, 2492 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2493 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2494 } 2495 2496 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2497 qp_params.sqd_async = true; 2498 2499 if (attr_mask & IB_QP_PKEY_INDEX) { 2500 SET_FIELD(qp_params.modify_flags, 2501 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2502 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2503 rc = -EINVAL; 2504 goto err; 2505 } 2506 2507 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2508 } 2509 2510 if (attr_mask & IB_QP_QKEY) 2511 qp->qkey = attr->qkey; 2512 2513 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2514 SET_FIELD(qp_params.modify_flags, 2515 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2516 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2517 IB_ACCESS_REMOTE_READ; 2518 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2519 IB_ACCESS_REMOTE_WRITE; 2520 qp_params.incoming_atomic_en = attr->qp_access_flags & 2521 IB_ACCESS_REMOTE_ATOMIC; 2522 } 2523 2524 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2525 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2526 return -EINVAL; 2527 2528 if (attr_mask & IB_QP_PATH_MTU) { 2529 if (attr->path_mtu < IB_MTU_256 || 2530 attr->path_mtu > IB_MTU_4096) { 2531 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2532 rc = -EINVAL; 2533 goto err; 2534 } 2535 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2536 ib_mtu_enum_to_int(iboe_get_mtu 2537 (dev->ndev->mtu))); 2538 } 2539 2540 if (!qp->mtu) { 2541 qp->mtu = 2542 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2543 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2544 } 2545 2546 SET_FIELD(qp_params.modify_flags, 2547 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2548 2549 qp_params.traffic_class_tos = grh->traffic_class; 2550 qp_params.flow_label = grh->flow_label; 2551 qp_params.hop_limit_ttl = grh->hop_limit; 2552 2553 qp->sgid_idx = grh->sgid_index; 2554 2555 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2556 if (rc) { 2557 DP_ERR(dev, 2558 "modify qp: problems with GID index %d (rc=%d)\n", 2559 grh->sgid_index, rc); 2560 return rc; 2561 } 2562 2563 rc = qedr_get_dmac(dev, &attr->ah_attr, 2564 qp_params.remote_mac_addr); 2565 if (rc) 2566 return rc; 2567 2568 qp_params.use_local_mac = true; 2569 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2570 2571 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2572 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2573 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2574 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2575 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2576 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2577 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2578 qp_params.remote_mac_addr); 2579 2580 qp_params.mtu = qp->mtu; 2581 qp_params.lb_indication = false; 2582 } 2583 2584 if (!qp_params.mtu) { 2585 /* Stay with current MTU */ 2586 if (qp->mtu) 2587 qp_params.mtu = qp->mtu; 2588 else 2589 qp_params.mtu = 2590 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2591 } 2592 2593 if (attr_mask & IB_QP_TIMEOUT) { 2594 SET_FIELD(qp_params.modify_flags, 2595 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2596 2597 /* The received timeout value is an exponent used like this: 2598 * "12.7.34 LOCAL ACK TIMEOUT 2599 * Value representing the transport (ACK) timeout for use by 2600 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2601 * The FW expects timeout in msec so we need to divide the usec 2602 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2603 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2604 * The value of zero means infinite so we use a 'max_t' to make 2605 * sure that sub 1 msec values will be configured as 1 msec. 2606 */ 2607 if (attr->timeout) 2608 qp_params.ack_timeout = 2609 1 << max_t(int, attr->timeout - 8, 0); 2610 else 2611 qp_params.ack_timeout = 0; 2612 2613 qp->timeout = attr->timeout; 2614 } 2615 2616 if (attr_mask & IB_QP_RETRY_CNT) { 2617 SET_FIELD(qp_params.modify_flags, 2618 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2619 qp_params.retry_cnt = attr->retry_cnt; 2620 } 2621 2622 if (attr_mask & IB_QP_RNR_RETRY) { 2623 SET_FIELD(qp_params.modify_flags, 2624 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2625 qp_params.rnr_retry_cnt = attr->rnr_retry; 2626 } 2627 2628 if (attr_mask & IB_QP_RQ_PSN) { 2629 SET_FIELD(qp_params.modify_flags, 2630 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2631 qp_params.rq_psn = attr->rq_psn; 2632 qp->rq_psn = attr->rq_psn; 2633 } 2634 2635 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2636 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2637 rc = -EINVAL; 2638 DP_ERR(dev, 2639 "unsupported max_rd_atomic=%d, supported=%d\n", 2640 attr->max_rd_atomic, 2641 dev->attr.max_qp_req_rd_atomic_resc); 2642 goto err; 2643 } 2644 2645 SET_FIELD(qp_params.modify_flags, 2646 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2647 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2648 } 2649 2650 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2651 SET_FIELD(qp_params.modify_flags, 2652 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2653 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2654 } 2655 2656 if (attr_mask & IB_QP_SQ_PSN) { 2657 SET_FIELD(qp_params.modify_flags, 2658 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2659 qp_params.sq_psn = attr->sq_psn; 2660 qp->sq_psn = attr->sq_psn; 2661 } 2662 2663 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2664 if (attr->max_dest_rd_atomic > 2665 dev->attr.max_qp_resp_rd_atomic_resc) { 2666 DP_ERR(dev, 2667 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2668 attr->max_dest_rd_atomic, 2669 dev->attr.max_qp_resp_rd_atomic_resc); 2670 2671 rc = -EINVAL; 2672 goto err; 2673 } 2674 2675 SET_FIELD(qp_params.modify_flags, 2676 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2677 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2678 } 2679 2680 if (attr_mask & IB_QP_DEST_QPN) { 2681 SET_FIELD(qp_params.modify_flags, 2682 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2683 2684 qp_params.dest_qp = attr->dest_qp_num; 2685 qp->dest_qp_num = attr->dest_qp_num; 2686 } 2687 2688 cur_state = qp->state; 2689 2690 /* Update the QP state before the actual ramrod to prevent a race with 2691 * fast path. Modifying the QP state to error will cause the device to 2692 * flush the CQEs and while polling the flushed CQEs will considered as 2693 * a potential issue if the QP isn't in error state. 2694 */ 2695 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2696 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2697 qp->state = QED_ROCE_QP_STATE_ERR; 2698 2699 if (qp->qp_type != IB_QPT_GSI) 2700 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2701 qp->qed_qp, &qp_params); 2702 2703 if (attr_mask & IB_QP_STATE) { 2704 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2705 rc = qedr_update_qp_state(dev, qp, cur_state, 2706 qp_params.new_state); 2707 qp->state = qp_params.new_state; 2708 } 2709 2710 err: 2711 return rc; 2712 } 2713 2714 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2715 { 2716 int ib_qp_acc_flags = 0; 2717 2718 if (params->incoming_rdma_write_en) 2719 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2720 if (params->incoming_rdma_read_en) 2721 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2722 if (params->incoming_atomic_en) 2723 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2724 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2725 return ib_qp_acc_flags; 2726 } 2727 2728 int qedr_query_qp(struct ib_qp *ibqp, 2729 struct ib_qp_attr *qp_attr, 2730 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2731 { 2732 struct qed_rdma_query_qp_out_params params; 2733 struct qedr_qp *qp = get_qedr_qp(ibqp); 2734 struct qedr_dev *dev = qp->dev; 2735 int rc = 0; 2736 2737 memset(¶ms, 0, sizeof(params)); 2738 memset(qp_attr, 0, sizeof(*qp_attr)); 2739 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2740 2741 if (qp->qp_type != IB_QPT_GSI) { 2742 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2743 if (rc) 2744 goto err; 2745 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2746 } else { 2747 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS); 2748 } 2749 2750 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2751 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2752 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2753 qp_attr->rq_psn = params.rq_psn; 2754 qp_attr->sq_psn = params.sq_psn; 2755 qp_attr->dest_qp_num = params.dest_qp; 2756 2757 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2758 2759 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2760 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2761 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2762 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2763 qp_attr->cap.max_inline_data = dev->attr.max_inline; 2764 qp_init_attr->cap = qp_attr->cap; 2765 2766 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2767 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2768 params.flow_label, qp->sgid_idx, 2769 params.hop_limit_ttl, params.traffic_class_tos); 2770 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2771 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2772 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2773 qp_attr->timeout = qp->timeout; 2774 qp_attr->rnr_retry = params.rnr_retry; 2775 qp_attr->retry_cnt = params.retry_cnt; 2776 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2777 qp_attr->pkey_index = params.pkey_index; 2778 qp_attr->port_num = 1; 2779 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2780 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2781 qp_attr->alt_pkey_index = 0; 2782 qp_attr->alt_port_num = 0; 2783 qp_attr->alt_timeout = 0; 2784 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2785 2786 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2787 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2788 qp_attr->max_rd_atomic = params.max_rd_atomic; 2789 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2790 2791 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2792 qp_attr->cap.max_inline_data); 2793 2794 err: 2795 return rc; 2796 } 2797 2798 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2799 { 2800 struct qedr_qp *qp = get_qedr_qp(ibqp); 2801 struct qedr_dev *dev = qp->dev; 2802 struct ib_qp_attr attr; 2803 int attr_mask = 0; 2804 2805 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2806 qp, qp->qp_type); 2807 2808 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2809 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2810 (qp->state != QED_ROCE_QP_STATE_ERR) && 2811 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2812 2813 attr.qp_state = IB_QPS_ERR; 2814 attr_mask |= IB_QP_STATE; 2815 2816 /* Change the QP state to ERROR */ 2817 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2818 } 2819 } else { 2820 /* If connection establishment started the WAIT_FOR_CONNECT 2821 * bit will be on and we need to Wait for the establishment 2822 * to complete before destroying the qp. 2823 */ 2824 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, 2825 &qp->iwarp_cm_flags)) 2826 wait_for_completion(&qp->iwarp_cm_comp); 2827 2828 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT 2829 * bit will be on, and we need to wait for the disconnect to 2830 * complete before continuing. We can use the same completion, 2831 * iwarp_cm_comp, since this is the only place that waits for 2832 * this completion and it is sequential. In addition, 2833 * disconnect can't occur before the connection is fully 2834 * established, therefore if WAIT_FOR_DISCONNECT is on it 2835 * means WAIT_FOR_CONNECT is also on and the completion for 2836 * CONNECT already occurred. 2837 */ 2838 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT, 2839 &qp->iwarp_cm_flags)) 2840 wait_for_completion(&qp->iwarp_cm_comp); 2841 } 2842 2843 if (qp->qp_type == IB_QPT_GSI) 2844 qedr_destroy_gsi_qp(dev); 2845 2846 /* We need to remove the entry from the xarray before we release the 2847 * qp_id to avoid a race of the qp_id being reallocated and failing 2848 * on xa_insert 2849 */ 2850 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2851 xa_erase(&dev->qps, qp->qp_id); 2852 2853 qedr_free_qp_resources(dev, qp, udata); 2854 2855 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2856 qedr_iw_qp_rem_ref(&qp->ibqp); 2857 wait_for_completion(&qp->qp_rel_comp); 2858 } 2859 2860 return 0; 2861 } 2862 2863 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 2864 struct ib_udata *udata) 2865 { 2866 struct qedr_ah *ah = get_qedr_ah(ibah); 2867 2868 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr); 2869 2870 return 0; 2871 } 2872 2873 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2874 { 2875 struct qedr_ah *ah = get_qedr_ah(ibah); 2876 2877 rdma_destroy_ah_attr(&ah->attr); 2878 return 0; 2879 } 2880 2881 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2882 { 2883 struct qedr_pbl *pbl, *tmp; 2884 2885 if (info->pbl_table) 2886 list_add_tail(&info->pbl_table->list_entry, 2887 &info->free_pbl_list); 2888 2889 if (!list_empty(&info->inuse_pbl_list)) 2890 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2891 2892 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2893 list_del(&pbl->list_entry); 2894 qedr_free_pbl(dev, &info->pbl_info, pbl); 2895 } 2896 } 2897 2898 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2899 size_t page_list_len, bool two_layered) 2900 { 2901 struct qedr_pbl *tmp; 2902 int rc; 2903 2904 INIT_LIST_HEAD(&info->free_pbl_list); 2905 INIT_LIST_HEAD(&info->inuse_pbl_list); 2906 2907 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2908 page_list_len, two_layered); 2909 if (rc) 2910 goto done; 2911 2912 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2913 if (IS_ERR(info->pbl_table)) { 2914 rc = PTR_ERR(info->pbl_table); 2915 goto done; 2916 } 2917 2918 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2919 &info->pbl_table->pa); 2920 2921 /* in usual case we use 2 PBLs, so we add one to free 2922 * list and allocating another one 2923 */ 2924 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2925 if (IS_ERR(tmp)) { 2926 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2927 goto done; 2928 } 2929 2930 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2931 2932 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2933 2934 done: 2935 if (rc) 2936 free_mr_info(dev, info); 2937 2938 return rc; 2939 } 2940 2941 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2942 u64 usr_addr, int acc, struct ib_dmah *dmah, 2943 struct ib_udata *udata) 2944 { 2945 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2946 struct qedr_mr *mr; 2947 struct qedr_pd *pd; 2948 int rc = -ENOMEM; 2949 2950 if (dmah) 2951 return ERR_PTR(-EOPNOTSUPP); 2952 2953 pd = get_qedr_pd(ibpd); 2954 DP_DEBUG(dev, QEDR_MSG_MR, 2955 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2956 pd->pd_id, start, len, usr_addr, acc); 2957 2958 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2959 return ERR_PTR(-EINVAL); 2960 2961 mr = kzalloc_obj(*mr); 2962 if (!mr) 2963 return ERR_PTR(rc); 2964 2965 mr->type = QEDR_MR_USER; 2966 2967 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 2968 if (IS_ERR(mr->umem)) { 2969 rc = -EFAULT; 2970 goto err0; 2971 } 2972 2973 rc = init_mr_info(dev, &mr->info, 2974 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1); 2975 if (rc) 2976 goto err1; 2977 2978 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2979 &mr->info.pbl_info, PAGE_SHIFT); 2980 2981 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2982 if (rc) { 2983 if (rc == -EINVAL) 2984 DP_ERR(dev, "Out of MR resources\n"); 2985 else 2986 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 2987 2988 goto err1; 2989 } 2990 2991 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2992 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2993 mr->hw_mr.key = 0; 2994 mr->hw_mr.pd = pd->pd_id; 2995 mr->hw_mr.local_read = 1; 2996 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2997 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2998 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2999 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3000 mr->hw_mr.mw_bind = false; 3001 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 3002 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3003 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3004 mr->hw_mr.page_size_log = PAGE_SHIFT; 3005 mr->hw_mr.length = len; 3006 mr->hw_mr.vaddr = usr_addr; 3007 mr->hw_mr.phy_mr = false; 3008 mr->hw_mr.dma_mr = false; 3009 3010 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3011 if (rc) { 3012 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3013 goto err2; 3014 } 3015 3016 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3017 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3018 mr->hw_mr.remote_atomic) 3019 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3020 3021 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 3022 mr->ibmr.lkey); 3023 return &mr->ibmr; 3024 3025 err2: 3026 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3027 err1: 3028 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3029 err0: 3030 kfree(mr); 3031 return ERR_PTR(rc); 3032 } 3033 3034 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3035 { 3036 struct qedr_mr *mr = get_qedr_mr(ib_mr); 3037 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 3038 int rc = 0; 3039 3040 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 3041 if (rc) 3042 return rc; 3043 3044 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3045 3046 if (mr->type != QEDR_MR_DMA) 3047 free_mr_info(dev, &mr->info); 3048 3049 /* it could be user registered memory. */ 3050 ib_umem_release(mr->umem); 3051 3052 kfree(mr); 3053 3054 return rc; 3055 } 3056 3057 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 3058 int max_page_list_len) 3059 { 3060 struct qedr_pd *pd = get_qedr_pd(ibpd); 3061 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3062 struct qedr_mr *mr; 3063 int rc = -ENOMEM; 3064 3065 DP_DEBUG(dev, QEDR_MSG_MR, 3066 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 3067 max_page_list_len); 3068 3069 mr = kzalloc_obj(*mr); 3070 if (!mr) 3071 return ERR_PTR(rc); 3072 3073 mr->dev = dev; 3074 mr->type = QEDR_MR_FRMR; 3075 3076 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 3077 if (rc) 3078 goto err0; 3079 3080 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3081 if (rc) { 3082 if (rc == -EINVAL) 3083 DP_ERR(dev, "Out of MR resources\n"); 3084 else 3085 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3086 3087 goto err1; 3088 } 3089 3090 /* Index only, 18 bit long, lkey = itid << 8 | key */ 3091 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 3092 mr->hw_mr.key = 0; 3093 mr->hw_mr.pd = pd->pd_id; 3094 mr->hw_mr.local_read = 1; 3095 mr->hw_mr.local_write = 0; 3096 mr->hw_mr.remote_read = 0; 3097 mr->hw_mr.remote_write = 0; 3098 mr->hw_mr.remote_atomic = 0; 3099 mr->hw_mr.mw_bind = false; 3100 mr->hw_mr.pbl_ptr = 0; 3101 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 3102 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 3103 mr->hw_mr.length = 0; 3104 mr->hw_mr.vaddr = 0; 3105 mr->hw_mr.phy_mr = true; 3106 mr->hw_mr.dma_mr = false; 3107 3108 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3109 if (rc) { 3110 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3111 goto err2; 3112 } 3113 3114 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3115 mr->ibmr.rkey = mr->ibmr.lkey; 3116 3117 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 3118 return mr; 3119 3120 err2: 3121 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3122 err1: 3123 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 3124 err0: 3125 kfree(mr); 3126 return ERR_PTR(rc); 3127 } 3128 3129 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 3130 u32 max_num_sg) 3131 { 3132 struct qedr_mr *mr; 3133 3134 if (mr_type != IB_MR_TYPE_MEM_REG) 3135 return ERR_PTR(-EINVAL); 3136 3137 mr = __qedr_alloc_mr(ibpd, max_num_sg); 3138 3139 if (IS_ERR(mr)) 3140 return ERR_PTR(-EINVAL); 3141 3142 return &mr->ibmr; 3143 } 3144 3145 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 3146 { 3147 struct qedr_mr *mr = get_qedr_mr(ibmr); 3148 struct qedr_pbl *pbl_table; 3149 struct regpair *pbe; 3150 u32 pbes_in_page; 3151 3152 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 3153 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 3154 return -ENOMEM; 3155 } 3156 3157 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 3158 mr->npages, addr); 3159 3160 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 3161 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 3162 pbe = (struct regpair *)pbl_table->va; 3163 pbe += mr->npages % pbes_in_page; 3164 pbe->lo = cpu_to_le32((u32)addr); 3165 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 3166 3167 mr->npages++; 3168 3169 return 0; 3170 } 3171 3172 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 3173 { 3174 int work = info->completed - info->completed_handled - 1; 3175 3176 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 3177 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 3178 struct qedr_pbl *pbl; 3179 3180 /* Free all the page list that are possible to be freed 3181 * (all the ones that were invalidated), under the assumption 3182 * that if an FMR was completed successfully that means that 3183 * if there was an invalidate operation before it also ended 3184 */ 3185 pbl = list_first_entry(&info->inuse_pbl_list, 3186 struct qedr_pbl, list_entry); 3187 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 3188 info->completed_handled++; 3189 } 3190 } 3191 3192 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3193 int sg_nents, unsigned int *sg_offset) 3194 { 3195 struct qedr_mr *mr = get_qedr_mr(ibmr); 3196 3197 mr->npages = 0; 3198 3199 handle_completed_mrs(mr->dev, &mr->info); 3200 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 3201 } 3202 3203 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 3204 { 3205 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3206 struct qedr_pd *pd = get_qedr_pd(ibpd); 3207 struct qedr_mr *mr; 3208 int rc; 3209 3210 mr = kzalloc_obj(*mr); 3211 if (!mr) 3212 return ERR_PTR(-ENOMEM); 3213 3214 mr->type = QEDR_MR_DMA; 3215 3216 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3217 if (rc) { 3218 if (rc == -EINVAL) 3219 DP_ERR(dev, "Out of MR resources\n"); 3220 else 3221 DP_ERR(dev, "roce alloc tid returned error %d\n", rc); 3222 3223 goto err1; 3224 } 3225 3226 /* index only, 18 bit long, lkey = itid << 8 | key */ 3227 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3228 mr->hw_mr.pd = pd->pd_id; 3229 mr->hw_mr.local_read = 1; 3230 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3231 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3232 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3233 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3234 mr->hw_mr.dma_mr = true; 3235 3236 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3237 if (rc) { 3238 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3239 goto err2; 3240 } 3241 3242 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3243 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3244 mr->hw_mr.remote_atomic) 3245 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3246 3247 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3248 return &mr->ibmr; 3249 3250 err2: 3251 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3252 err1: 3253 kfree(mr); 3254 return ERR_PTR(rc); 3255 } 3256 3257 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3258 { 3259 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3260 } 3261 3262 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3263 { 3264 int i, len = 0; 3265 3266 for (i = 0; i < num_sge; i++) 3267 len += sg_list[i].length; 3268 3269 return len; 3270 } 3271 3272 static void swap_wqe_data64(u64 *p) 3273 { 3274 int i; 3275 3276 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3277 *p = cpu_to_be64(cpu_to_le64(*p)); 3278 } 3279 3280 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3281 struct qedr_qp *qp, u8 *wqe_size, 3282 const struct ib_send_wr *wr, 3283 const struct ib_send_wr **bad_wr, 3284 u8 *bits, u8 bit) 3285 { 3286 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3287 char *seg_prt, *wqe; 3288 int i, seg_siz; 3289 3290 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3291 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3292 *bad_wr = wr; 3293 return 0; 3294 } 3295 3296 if (!data_size) 3297 return data_size; 3298 3299 *bits |= bit; 3300 3301 seg_prt = NULL; 3302 wqe = NULL; 3303 seg_siz = 0; 3304 3305 /* Copy data inline */ 3306 for (i = 0; i < wr->num_sge; i++) { 3307 u32 len = wr->sg_list[i].length; 3308 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3309 3310 while (len > 0) { 3311 u32 cur; 3312 3313 /* New segment required */ 3314 if (!seg_siz) { 3315 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3316 seg_prt = wqe; 3317 seg_siz = sizeof(struct rdma_sq_common_wqe); 3318 (*wqe_size)++; 3319 } 3320 3321 /* Calculate currently allowed length */ 3322 cur = min_t(u32, len, seg_siz); 3323 memcpy(seg_prt, src, cur); 3324 3325 /* Update segment variables */ 3326 seg_prt += cur; 3327 seg_siz -= cur; 3328 3329 /* Update sge variables */ 3330 src += cur; 3331 len -= cur; 3332 3333 /* Swap fully-completed segments */ 3334 if (!seg_siz) 3335 swap_wqe_data64((u64 *)wqe); 3336 } 3337 } 3338 3339 /* swap last not completed segment */ 3340 if (seg_siz) 3341 swap_wqe_data64((u64 *)wqe); 3342 3343 return data_size; 3344 } 3345 3346 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3347 do { \ 3348 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3349 (sge)->length = cpu_to_le32(vlength); \ 3350 (sge)->flags = cpu_to_le32(vflags); \ 3351 } while (0) 3352 3353 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3354 do { \ 3355 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3356 (hdr)->num_sges = num_sge; \ 3357 } while (0) 3358 3359 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3360 do { \ 3361 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3362 (sge)->length = cpu_to_le32(vlength); \ 3363 (sge)->l_key = cpu_to_le32(vlkey); \ 3364 } while (0) 3365 3366 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3367 const struct ib_send_wr *wr) 3368 { 3369 u32 data_size = 0; 3370 int i; 3371 3372 for (i = 0; i < wr->num_sge; i++) { 3373 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3374 3375 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3376 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3377 sge->length = cpu_to_le32(wr->sg_list[i].length); 3378 data_size += wr->sg_list[i].length; 3379 } 3380 3381 if (wqe_size) 3382 *wqe_size += wr->num_sge; 3383 3384 return data_size; 3385 } 3386 3387 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3388 struct qedr_qp *qp, 3389 struct rdma_sq_rdma_wqe_1st *rwqe, 3390 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3391 const struct ib_send_wr *wr, 3392 const struct ib_send_wr **bad_wr) 3393 { 3394 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3395 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3396 3397 if (wr->send_flags & IB_SEND_INLINE && 3398 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3399 wr->opcode == IB_WR_RDMA_WRITE)) { 3400 u8 flags = 0; 3401 3402 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3403 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3404 bad_wr, &rwqe->flags, flags); 3405 } 3406 3407 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3408 } 3409 3410 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3411 struct qedr_qp *qp, 3412 struct rdma_sq_send_wqe_1st *swqe, 3413 struct rdma_sq_send_wqe_2st *swqe2, 3414 const struct ib_send_wr *wr, 3415 const struct ib_send_wr **bad_wr) 3416 { 3417 memset(swqe2, 0, sizeof(*swqe2)); 3418 if (wr->send_flags & IB_SEND_INLINE) { 3419 u8 flags = 0; 3420 3421 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3422 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3423 bad_wr, &swqe->flags, flags); 3424 } 3425 3426 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3427 } 3428 3429 static int qedr_prepare_reg(struct qedr_qp *qp, 3430 struct rdma_sq_fmr_wqe_1st *fwqe1, 3431 const struct ib_reg_wr *wr) 3432 { 3433 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3434 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3435 3436 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3437 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3438 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3439 fwqe1->l_key = wr->key; 3440 3441 fwqe2->access_ctrl = 0; 3442 3443 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3444 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3445 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3446 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3447 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3448 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3449 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3450 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3451 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3452 fwqe2->fmr_ctrl = 0; 3453 3454 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3455 ilog2(mr->ibmr.page_size) - 12); 3456 3457 fwqe2->length_hi = 0; 3458 fwqe2->length_lo = mr->ibmr.length; 3459 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3460 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3461 3462 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3463 3464 return 0; 3465 } 3466 3467 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3468 { 3469 switch (opcode) { 3470 case IB_WR_RDMA_WRITE: 3471 case IB_WR_RDMA_WRITE_WITH_IMM: 3472 return IB_WC_RDMA_WRITE; 3473 case IB_WR_SEND_WITH_IMM: 3474 case IB_WR_SEND: 3475 case IB_WR_SEND_WITH_INV: 3476 return IB_WC_SEND; 3477 case IB_WR_RDMA_READ: 3478 case IB_WR_RDMA_READ_WITH_INV: 3479 return IB_WC_RDMA_READ; 3480 case IB_WR_ATOMIC_CMP_AND_SWP: 3481 return IB_WC_COMP_SWAP; 3482 case IB_WR_ATOMIC_FETCH_AND_ADD: 3483 return IB_WC_FETCH_ADD; 3484 case IB_WR_REG_MR: 3485 return IB_WC_REG_MR; 3486 case IB_WR_LOCAL_INV: 3487 return IB_WC_LOCAL_INV; 3488 default: 3489 return IB_WC_SEND; 3490 } 3491 } 3492 3493 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3494 const struct ib_send_wr *wr) 3495 { 3496 int wq_is_full, err_wr, pbl_is_full; 3497 struct qedr_dev *dev = qp->dev; 3498 3499 /* prevent SQ overflow and/or processing of a bad WR */ 3500 err_wr = wr->num_sge > qp->sq.max_sges; 3501 wq_is_full = qedr_wq_is_full(&qp->sq); 3502 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3503 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3504 if (wq_is_full || err_wr || pbl_is_full) { 3505 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3506 DP_ERR(dev, 3507 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3508 qp); 3509 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3510 } 3511 3512 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3513 DP_ERR(dev, 3514 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3515 qp); 3516 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3517 } 3518 3519 if (pbl_is_full && 3520 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3521 DP_ERR(dev, 3522 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3523 qp); 3524 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3525 } 3526 return false; 3527 } 3528 return true; 3529 } 3530 3531 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3532 const struct ib_send_wr **bad_wr) 3533 { 3534 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3535 struct qedr_qp *qp = get_qedr_qp(ibqp); 3536 struct rdma_sq_atomic_wqe_1st *awqe1; 3537 struct rdma_sq_atomic_wqe_2nd *awqe2; 3538 struct rdma_sq_atomic_wqe_3rd *awqe3; 3539 struct rdma_sq_send_wqe_2st *swqe2; 3540 struct rdma_sq_local_inv_wqe *iwqe; 3541 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3542 struct rdma_sq_send_wqe_1st *swqe; 3543 struct rdma_sq_rdma_wqe_1st *rwqe; 3544 struct rdma_sq_fmr_wqe_1st *fwqe1; 3545 struct rdma_sq_common_wqe *wqe; 3546 u32 length; 3547 int rc = 0; 3548 bool comp; 3549 3550 if (!qedr_can_post_send(qp, wr)) { 3551 *bad_wr = wr; 3552 return -ENOMEM; 3553 } 3554 3555 wqe = qed_chain_produce(&qp->sq.pbl); 3556 qp->wqe_wr_id[qp->sq.prod].signaled = 3557 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3558 3559 wqe->flags = 0; 3560 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3561 !!(wr->send_flags & IB_SEND_SOLICITED)); 3562 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3563 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3564 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3565 !!(wr->send_flags & IB_SEND_FENCE)); 3566 wqe->prev_wqe_size = qp->prev_wqe_size; 3567 3568 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3569 3570 switch (wr->opcode) { 3571 case IB_WR_SEND_WITH_IMM: 3572 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3573 rc = -EINVAL; 3574 *bad_wr = wr; 3575 break; 3576 } 3577 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3578 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3579 swqe->wqe_size = 2; 3580 swqe2 = qed_chain_produce(&qp->sq.pbl); 3581 3582 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3583 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3584 wr, bad_wr); 3585 swqe->length = cpu_to_le32(length); 3586 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3587 qp->prev_wqe_size = swqe->wqe_size; 3588 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3589 break; 3590 case IB_WR_SEND: 3591 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3592 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3593 3594 swqe->wqe_size = 2; 3595 swqe2 = qed_chain_produce(&qp->sq.pbl); 3596 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3597 wr, bad_wr); 3598 swqe->length = cpu_to_le32(length); 3599 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3600 qp->prev_wqe_size = swqe->wqe_size; 3601 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3602 break; 3603 case IB_WR_SEND_WITH_INV: 3604 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3605 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3606 swqe2 = qed_chain_produce(&qp->sq.pbl); 3607 swqe->wqe_size = 2; 3608 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3609 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3610 wr, bad_wr); 3611 swqe->length = cpu_to_le32(length); 3612 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3613 qp->prev_wqe_size = swqe->wqe_size; 3614 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3615 break; 3616 3617 case IB_WR_RDMA_WRITE_WITH_IMM: 3618 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3619 rc = -EINVAL; 3620 *bad_wr = wr; 3621 break; 3622 } 3623 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3624 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3625 3626 rwqe->wqe_size = 2; 3627 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3628 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3629 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3630 wr, bad_wr); 3631 rwqe->length = cpu_to_le32(length); 3632 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3633 qp->prev_wqe_size = rwqe->wqe_size; 3634 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3635 break; 3636 case IB_WR_RDMA_WRITE: 3637 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3638 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3639 3640 rwqe->wqe_size = 2; 3641 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3642 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3643 wr, bad_wr); 3644 rwqe->length = cpu_to_le32(length); 3645 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3646 qp->prev_wqe_size = rwqe->wqe_size; 3647 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3648 break; 3649 case IB_WR_RDMA_READ_WITH_INV: 3650 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3651 fallthrough; /* same is identical to RDMA READ */ 3652 3653 case IB_WR_RDMA_READ: 3654 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3655 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3656 3657 rwqe->wqe_size = 2; 3658 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3659 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3660 wr, bad_wr); 3661 rwqe->length = cpu_to_le32(length); 3662 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3663 qp->prev_wqe_size = rwqe->wqe_size; 3664 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3665 break; 3666 3667 case IB_WR_ATOMIC_CMP_AND_SWP: 3668 case IB_WR_ATOMIC_FETCH_AND_ADD: 3669 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3670 awqe1->wqe_size = 4; 3671 3672 awqe2 = qed_chain_produce(&qp->sq.pbl); 3673 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3674 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3675 3676 awqe3 = qed_chain_produce(&qp->sq.pbl); 3677 3678 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3679 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3680 DMA_REGPAIR_LE(awqe3->swap_data, 3681 atomic_wr(wr)->compare_add); 3682 } else { 3683 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3684 DMA_REGPAIR_LE(awqe3->swap_data, 3685 atomic_wr(wr)->swap); 3686 DMA_REGPAIR_LE(awqe3->cmp_data, 3687 atomic_wr(wr)->compare_add); 3688 } 3689 3690 qedr_prepare_sq_sges(qp, NULL, wr); 3691 3692 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3693 qp->prev_wqe_size = awqe1->wqe_size; 3694 break; 3695 3696 case IB_WR_LOCAL_INV: 3697 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3698 iwqe->wqe_size = 1; 3699 3700 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3701 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3702 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3703 qp->prev_wqe_size = iwqe->wqe_size; 3704 break; 3705 case IB_WR_REG_MR: 3706 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3707 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3708 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3709 fwqe1->wqe_size = 2; 3710 3711 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3712 if (rc) { 3713 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3714 *bad_wr = wr; 3715 break; 3716 } 3717 3718 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3719 qp->prev_wqe_size = fwqe1->wqe_size; 3720 break; 3721 default: 3722 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3723 rc = -EINVAL; 3724 *bad_wr = wr; 3725 break; 3726 } 3727 3728 if (*bad_wr) { 3729 u16 value; 3730 3731 /* Restore prod to its position before 3732 * this WR was processed 3733 */ 3734 value = le16_to_cpu(qp->sq.db_data.data.value); 3735 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3736 3737 /* Restore prev_wqe_size */ 3738 qp->prev_wqe_size = wqe->prev_wqe_size; 3739 rc = -EINVAL; 3740 DP_ERR(dev, "POST SEND FAILED\n"); 3741 } 3742 3743 return rc; 3744 } 3745 3746 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3747 const struct ib_send_wr **bad_wr) 3748 { 3749 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3750 struct qedr_qp *qp = get_qedr_qp(ibqp); 3751 unsigned long flags; 3752 int rc = 0; 3753 3754 *bad_wr = NULL; 3755 3756 if (qp->qp_type == IB_QPT_GSI) 3757 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3758 3759 spin_lock_irqsave(&qp->q_lock, flags); 3760 3761 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3762 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3763 (qp->state != QED_ROCE_QP_STATE_ERR) && 3764 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3765 spin_unlock_irqrestore(&qp->q_lock, flags); 3766 *bad_wr = wr; 3767 DP_DEBUG(dev, QEDR_MSG_CQ, 3768 "QP in wrong state! QP icid=0x%x state %d\n", 3769 qp->icid, qp->state); 3770 return -EINVAL; 3771 } 3772 } 3773 3774 while (wr) { 3775 rc = __qedr_post_send(ibqp, wr, bad_wr); 3776 if (rc) 3777 break; 3778 3779 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3780 3781 qedr_inc_sw_prod(&qp->sq); 3782 3783 qp->sq.db_data.data.value++; 3784 3785 wr = wr->next; 3786 } 3787 3788 /* Trigger doorbell 3789 * If there was a failure in the first WR then it will be triggered in 3790 * vane. However this is not harmful (as long as the producer value is 3791 * unchanged). For performance reasons we avoid checking for this 3792 * redundant doorbell. 3793 * 3794 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3795 * soon as we give the doorbell, we could get a completion 3796 * for this wr, therefore we need to make sure that the 3797 * memory is updated before giving the doorbell. 3798 * During qedr_poll_cq, rmb is called before accessing the 3799 * cqe. This covers for the smp_rmb as well. 3800 */ 3801 smp_wmb(); 3802 writel(qp->sq.db_data.raw, qp->sq.db); 3803 3804 spin_unlock_irqrestore(&qp->q_lock, flags); 3805 3806 return rc; 3807 } 3808 3809 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3810 { 3811 u32 used; 3812 3813 /* Calculate number of elements used based on producer 3814 * count and consumer count and subtract it from max 3815 * work request supported so that we get elements left. 3816 */ 3817 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt); 3818 3819 return hw_srq->max_wr - used; 3820 } 3821 3822 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3823 const struct ib_recv_wr **bad_wr) 3824 { 3825 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3826 struct qedr_srq_hwq_info *hw_srq; 3827 struct qedr_dev *dev = srq->dev; 3828 struct qed_chain *pbl; 3829 unsigned long flags; 3830 int status = 0; 3831 u32 num_sge; 3832 3833 spin_lock_irqsave(&srq->lock, flags); 3834 3835 hw_srq = &srq->hw_srq; 3836 pbl = &srq->hw_srq.pbl; 3837 while (wr) { 3838 struct rdma_srq_wqe_header *hdr; 3839 int i; 3840 3841 if (!qedr_srq_elem_left(hw_srq) || 3842 wr->num_sge > srq->hw_srq.max_sges) { 3843 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3844 hw_srq->wr_prod_cnt, 3845 atomic_read(&hw_srq->wr_cons_cnt), 3846 wr->num_sge, srq->hw_srq.max_sges); 3847 status = -ENOMEM; 3848 *bad_wr = wr; 3849 break; 3850 } 3851 3852 hdr = qed_chain_produce(pbl); 3853 num_sge = wr->num_sge; 3854 /* Set number of sge and work request id in header */ 3855 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3856 3857 srq->hw_srq.wr_prod_cnt++; 3858 hw_srq->wqe_prod++; 3859 hw_srq->sge_prod++; 3860 3861 DP_DEBUG(dev, QEDR_MSG_SRQ, 3862 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3863 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3864 3865 for (i = 0; i < wr->num_sge; i++) { 3866 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3867 3868 /* Set SGE length, lkey and address */ 3869 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3870 wr->sg_list[i].length, wr->sg_list[i].lkey); 3871 3872 DP_DEBUG(dev, QEDR_MSG_SRQ, 3873 "[%d]: len %d key %x addr %x:%x\n", 3874 i, srq_sge->length, srq_sge->l_key, 3875 srq_sge->addr.hi, srq_sge->addr.lo); 3876 hw_srq->sge_prod++; 3877 } 3878 3879 /* Update WQE and SGE information before 3880 * updating producer. 3881 */ 3882 dma_wmb(); 3883 3884 /* SRQ producer is 8 bytes. Need to update SGE producer index 3885 * in first 4 bytes and need to update WQE producer in 3886 * next 4 bytes. 3887 */ 3888 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod); 3889 /* Make sure sge producer is updated first */ 3890 dma_wmb(); 3891 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod); 3892 3893 wr = wr->next; 3894 } 3895 3896 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3897 qed_chain_get_elem_left(pbl)); 3898 spin_unlock_irqrestore(&srq->lock, flags); 3899 3900 return status; 3901 } 3902 3903 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3904 const struct ib_recv_wr **bad_wr) 3905 { 3906 struct qedr_qp *qp = get_qedr_qp(ibqp); 3907 struct qedr_dev *dev = qp->dev; 3908 unsigned long flags; 3909 int status = 0; 3910 3911 if (qp->qp_type == IB_QPT_GSI) 3912 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3913 3914 spin_lock_irqsave(&qp->q_lock, flags); 3915 3916 while (wr) { 3917 int i; 3918 3919 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3920 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3921 wr->num_sge > qp->rq.max_sges) { 3922 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3923 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3924 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3925 qp->rq.max_sges); 3926 status = -ENOMEM; 3927 *bad_wr = wr; 3928 break; 3929 } 3930 for (i = 0; i < wr->num_sge; i++) { 3931 u32 flags = 0; 3932 struct rdma_rq_sge *rqe = 3933 qed_chain_produce(&qp->rq.pbl); 3934 3935 /* First one must include the number 3936 * of SGE in the list 3937 */ 3938 if (!i) 3939 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3940 wr->num_sge); 3941 3942 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3943 wr->sg_list[i].lkey); 3944 3945 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3946 wr->sg_list[i].length, flags); 3947 } 3948 3949 /* Special case of no sges. FW requires between 1-4 sges... 3950 * in this case we need to post 1 sge with length zero. this is 3951 * because rdma write with immediate consumes an RQ. 3952 */ 3953 if (!wr->num_sge) { 3954 u32 flags = 0; 3955 struct rdma_rq_sge *rqe = 3956 qed_chain_produce(&qp->rq.pbl); 3957 3958 /* First one must include the number 3959 * of SGE in the list 3960 */ 3961 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3962 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3963 3964 RQ_SGE_SET(rqe, 0, 0, flags); 3965 i = 1; 3966 } 3967 3968 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3969 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3970 3971 qedr_inc_sw_prod(&qp->rq); 3972 3973 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3974 * soon as we give the doorbell, we could get a completion 3975 * for this wr, therefore we need to make sure that the 3976 * memory is update before giving the doorbell. 3977 * During qedr_poll_cq, rmb is called before accessing the 3978 * cqe. This covers for the smp_rmb as well. 3979 */ 3980 smp_wmb(); 3981 3982 qp->rq.db_data.data.value++; 3983 3984 writel(qp->rq.db_data.raw, qp->rq.db); 3985 3986 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3987 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3988 } 3989 3990 wr = wr->next; 3991 } 3992 3993 spin_unlock_irqrestore(&qp->q_lock, flags); 3994 3995 return status; 3996 } 3997 3998 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3999 { 4000 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4001 4002 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 4003 cq->pbl_toggle; 4004 } 4005 4006 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 4007 { 4008 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4009 struct qedr_qp *qp; 4010 4011 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 4012 resp_cqe->qp_handle.lo, 4013 u64); 4014 return qp; 4015 } 4016 4017 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 4018 { 4019 struct rdma_cqe_requester *resp_cqe = &cqe->req; 4020 4021 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 4022 } 4023 4024 /* Return latest CQE (needs processing) */ 4025 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 4026 { 4027 return cq->latest_cqe; 4028 } 4029 4030 /* In fmr we need to increase the number of fmr completed counter for the fmr 4031 * algorithm determining whether we can free a pbl or not. 4032 * we need to perform this whether the work request was signaled or not. for 4033 * this purpose we call this function from the condition that checks if a wr 4034 * should be skipped, to make sure we don't miss it ( possibly this fmr 4035 * operation was not signalted) 4036 */ 4037 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 4038 { 4039 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 4040 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4041 } 4042 4043 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 4044 struct qedr_cq *cq, int num_entries, 4045 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 4046 int force) 4047 { 4048 u16 cnt = 0; 4049 4050 while (num_entries && qp->sq.wqe_cons != hw_cons) { 4051 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 4052 qedr_chk_if_fmr(qp); 4053 /* skip WC */ 4054 goto next_cqe; 4055 } 4056 4057 /* fill WC */ 4058 wc->status = status; 4059 wc->vendor_err = 0; 4060 wc->wc_flags = 0; 4061 wc->src_qp = qp->id; 4062 wc->qp = &qp->ibqp; 4063 4064 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 4065 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 4066 4067 switch (wc->opcode) { 4068 case IB_WC_RDMA_WRITE: 4069 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4070 break; 4071 case IB_WC_COMP_SWAP: 4072 case IB_WC_FETCH_ADD: 4073 wc->byte_len = 8; 4074 break; 4075 case IB_WC_REG_MR: 4076 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 4077 break; 4078 case IB_WC_RDMA_READ: 4079 case IB_WC_SEND: 4080 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 4081 break; 4082 default: 4083 break; 4084 } 4085 4086 num_entries--; 4087 wc++; 4088 cnt++; 4089 next_cqe: 4090 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 4091 qed_chain_consume(&qp->sq.pbl); 4092 qedr_inc_sw_cons(&qp->sq); 4093 } 4094 4095 return cnt; 4096 } 4097 4098 static int qedr_poll_cq_req(struct qedr_dev *dev, 4099 struct qedr_qp *qp, struct qedr_cq *cq, 4100 int num_entries, struct ib_wc *wc, 4101 struct rdma_cqe_requester *req) 4102 { 4103 int cnt = 0; 4104 4105 switch (req->status) { 4106 case RDMA_CQE_REQ_STS_OK: 4107 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4108 IB_WC_SUCCESS, 0); 4109 break; 4110 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 4111 if (qp->state != QED_ROCE_QP_STATE_ERR) 4112 DP_DEBUG(dev, QEDR_MSG_CQ, 4113 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4114 cq->icid, qp->icid); 4115 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 4116 IB_WC_WR_FLUSH_ERR, 1); 4117 break; 4118 default: 4119 /* process all WQE before the cosumer */ 4120 qp->state = QED_ROCE_QP_STATE_ERR; 4121 cnt = process_req(dev, qp, cq, num_entries, wc, 4122 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4123 wc += cnt; 4124 /* if we have extra WC fill it with actual error info */ 4125 if (cnt < num_entries) { 4126 enum ib_wc_status wc_status; 4127 4128 switch (req->status) { 4129 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4130 DP_ERR(dev, 4131 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4132 cq->icid, qp->icid); 4133 wc_status = IB_WC_BAD_RESP_ERR; 4134 break; 4135 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4136 DP_ERR(dev, 4137 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4138 cq->icid, qp->icid); 4139 wc_status = IB_WC_LOC_LEN_ERR; 4140 break; 4141 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4142 DP_ERR(dev, 4143 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4144 cq->icid, qp->icid); 4145 wc_status = IB_WC_LOC_QP_OP_ERR; 4146 break; 4147 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4148 DP_ERR(dev, 4149 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4150 cq->icid, qp->icid); 4151 wc_status = IB_WC_LOC_PROT_ERR; 4152 break; 4153 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4154 DP_ERR(dev, 4155 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4156 cq->icid, qp->icid); 4157 wc_status = IB_WC_MW_BIND_ERR; 4158 break; 4159 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4160 DP_ERR(dev, 4161 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4162 cq->icid, qp->icid); 4163 wc_status = IB_WC_REM_INV_REQ_ERR; 4164 break; 4165 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4166 DP_ERR(dev, 4167 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4168 cq->icid, qp->icid); 4169 wc_status = IB_WC_REM_ACCESS_ERR; 4170 break; 4171 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4172 DP_ERR(dev, 4173 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4174 cq->icid, qp->icid); 4175 wc_status = IB_WC_REM_OP_ERR; 4176 break; 4177 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4178 DP_ERR(dev, 4179 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4180 cq->icid, qp->icid); 4181 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4182 break; 4183 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4184 DP_ERR(dev, 4185 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4186 cq->icid, qp->icid); 4187 wc_status = IB_WC_RETRY_EXC_ERR; 4188 break; 4189 default: 4190 DP_ERR(dev, 4191 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4192 cq->icid, qp->icid); 4193 wc_status = IB_WC_GENERAL_ERR; 4194 } 4195 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4196 wc_status, 1); 4197 } 4198 } 4199 4200 return cnt; 4201 } 4202 4203 static inline int qedr_cqe_resp_status_to_ib(u8 status) 4204 { 4205 switch (status) { 4206 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4207 return IB_WC_LOC_ACCESS_ERR; 4208 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4209 return IB_WC_LOC_LEN_ERR; 4210 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4211 return IB_WC_LOC_QP_OP_ERR; 4212 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4213 return IB_WC_LOC_PROT_ERR; 4214 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4215 return IB_WC_MW_BIND_ERR; 4216 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4217 return IB_WC_REM_INV_RD_REQ_ERR; 4218 case RDMA_CQE_RESP_STS_OK: 4219 return IB_WC_SUCCESS; 4220 default: 4221 return IB_WC_GENERAL_ERR; 4222 } 4223 } 4224 4225 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 4226 struct ib_wc *wc) 4227 { 4228 wc->status = IB_WC_SUCCESS; 4229 wc->byte_len = le32_to_cpu(resp->length); 4230 4231 if (resp->flags & QEDR_RESP_IMM) { 4232 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4233 wc->wc_flags |= IB_WC_WITH_IMM; 4234 4235 if (resp->flags & QEDR_RESP_RDMA) 4236 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4237 4238 if (resp->flags & QEDR_RESP_INV) 4239 return -EINVAL; 4240 4241 } else if (resp->flags & QEDR_RESP_INV) { 4242 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4243 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4244 4245 if (resp->flags & QEDR_RESP_RDMA) 4246 return -EINVAL; 4247 4248 } else if (resp->flags & QEDR_RESP_RDMA) { 4249 return -EINVAL; 4250 } 4251 4252 return 0; 4253 } 4254 4255 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4256 struct qedr_cq *cq, struct ib_wc *wc, 4257 struct rdma_cqe_responder *resp, u64 wr_id) 4258 { 4259 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4260 wc->opcode = IB_WC_RECV; 4261 wc->wc_flags = 0; 4262 4263 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4264 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4265 DP_ERR(dev, 4266 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4267 cq, cq->icid, resp->flags); 4268 4269 } else { 4270 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4271 if (wc->status == IB_WC_GENERAL_ERR) 4272 DP_ERR(dev, 4273 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4274 cq, cq->icid, resp->status); 4275 } 4276 4277 /* Fill the rest of the WC */ 4278 wc->vendor_err = 0; 4279 wc->src_qp = qp->id; 4280 wc->qp = &qp->ibqp; 4281 wc->wr_id = wr_id; 4282 } 4283 4284 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4285 struct qedr_cq *cq, struct ib_wc *wc, 4286 struct rdma_cqe_responder *resp) 4287 { 4288 struct qedr_srq *srq = qp->srq; 4289 u64 wr_id; 4290 4291 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4292 le32_to_cpu(resp->srq_wr_id.lo), u64); 4293 4294 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4295 wc->status = IB_WC_WR_FLUSH_ERR; 4296 wc->vendor_err = 0; 4297 wc->wr_id = wr_id; 4298 wc->byte_len = 0; 4299 wc->src_qp = qp->id; 4300 wc->qp = &qp->ibqp; 4301 wc->wr_id = wr_id; 4302 } else { 4303 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4304 } 4305 atomic_inc(&srq->hw_srq.wr_cons_cnt); 4306 4307 return 1; 4308 } 4309 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4310 struct qedr_cq *cq, struct ib_wc *wc, 4311 struct rdma_cqe_responder *resp) 4312 { 4313 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4314 4315 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4316 4317 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4318 qed_chain_consume(&qp->rq.pbl); 4319 qedr_inc_sw_cons(&qp->rq); 4320 4321 return 1; 4322 } 4323 4324 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4325 int num_entries, struct ib_wc *wc, u16 hw_cons) 4326 { 4327 u16 cnt = 0; 4328 4329 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4330 /* fill WC */ 4331 wc->status = IB_WC_WR_FLUSH_ERR; 4332 wc->vendor_err = 0; 4333 wc->wc_flags = 0; 4334 wc->src_qp = qp->id; 4335 wc->byte_len = 0; 4336 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4337 wc->qp = &qp->ibqp; 4338 num_entries--; 4339 wc++; 4340 cnt++; 4341 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4342 qed_chain_consume(&qp->rq.pbl); 4343 qedr_inc_sw_cons(&qp->rq); 4344 } 4345 4346 return cnt; 4347 } 4348 4349 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4350 struct rdma_cqe_responder *resp, int *update) 4351 { 4352 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4353 consume_cqe(cq); 4354 *update |= 1; 4355 } 4356 } 4357 4358 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4359 struct qedr_cq *cq, int num_entries, 4360 struct ib_wc *wc, 4361 struct rdma_cqe_responder *resp) 4362 { 4363 int cnt; 4364 4365 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4366 consume_cqe(cq); 4367 4368 return cnt; 4369 } 4370 4371 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4372 struct qedr_cq *cq, int num_entries, 4373 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4374 int *update) 4375 { 4376 int cnt; 4377 4378 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4379 cnt = process_resp_flush(qp, cq, num_entries, wc, 4380 resp->rq_cons_or_srq_id); 4381 try_consume_resp_cqe(cq, qp, resp, update); 4382 } else { 4383 cnt = process_resp_one(dev, qp, cq, wc, resp); 4384 consume_cqe(cq); 4385 *update |= 1; 4386 } 4387 4388 return cnt; 4389 } 4390 4391 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4392 struct rdma_cqe_requester *req, int *update) 4393 { 4394 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4395 consume_cqe(cq); 4396 *update |= 1; 4397 } 4398 } 4399 4400 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4401 { 4402 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4403 struct qedr_cq *cq = get_qedr_cq(ibcq); 4404 union rdma_cqe *cqe; 4405 u32 old_cons, new_cons; 4406 unsigned long flags; 4407 int update = 0; 4408 int done = 0; 4409 4410 if (cq->destroyed) { 4411 DP_ERR(dev, 4412 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4413 cq, cq->icid); 4414 return 0; 4415 } 4416 4417 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4418 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4419 4420 spin_lock_irqsave(&cq->cq_lock, flags); 4421 cqe = cq->latest_cqe; 4422 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4423 while (num_entries && is_valid_cqe(cq, cqe)) { 4424 struct qedr_qp *qp; 4425 int cnt = 0; 4426 4427 /* prevent speculative reads of any field of CQE */ 4428 rmb(); 4429 4430 qp = cqe_get_qp(cqe); 4431 if (!qp) { 4432 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4433 break; 4434 } 4435 4436 wc->qp = &qp->ibqp; 4437 4438 switch (cqe_get_type(cqe)) { 4439 case RDMA_CQE_TYPE_REQUESTER: 4440 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4441 &cqe->req); 4442 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4443 break; 4444 case RDMA_CQE_TYPE_RESPONDER_RQ: 4445 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4446 &cqe->resp, &update); 4447 break; 4448 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4449 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4450 wc, &cqe->resp); 4451 update = 1; 4452 break; 4453 case RDMA_CQE_TYPE_INVALID: 4454 default: 4455 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4456 cqe_get_type(cqe)); 4457 } 4458 num_entries -= cnt; 4459 wc += cnt; 4460 done += cnt; 4461 4462 cqe = get_cqe(cq); 4463 } 4464 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4465 4466 cq->cq_cons += new_cons - old_cons; 4467 4468 if (update) 4469 /* doorbell notifies abount latest VALID entry, 4470 * but chain already point to the next INVALID one 4471 */ 4472 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4473 4474 spin_unlock_irqrestore(&cq->cq_lock, flags); 4475 return done; 4476 } 4477 4478 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4479 u32 port_num, const struct ib_wc *in_wc, 4480 const struct ib_grh *in_grh, const struct ib_mad *in, 4481 struct ib_mad *out_mad, size_t *out_mad_size, 4482 u16 *out_mad_pkey_index) 4483 { 4484 return IB_MAD_RESULT_SUCCESS; 4485 } 4486