1 /* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43 #include <linux/dma-mapping.h> 44 #include <net/addrconf.h> 45 #include <rdma/ib_verbs.h> 46 #include <rdma/ib_user_verbs.h> 47 #include <rdma/iw_cm.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_cache.h> 50 #include <rdma/iter.h> 51 #include <rdma/uverbs_ioctl.h> 52 53 #include "ocrdma.h" 54 #include "ocrdma_hw.h" 55 #include "ocrdma_verbs.h" 56 #include <rdma/ocrdma-abi.h> 57 58 int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 59 { 60 if (index > 0) 61 return -EINVAL; 62 63 *pkey = 0xffff; 64 return 0; 65 } 66 67 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 68 struct ib_udata *uhw) 69 { 70 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 71 int err; 72 73 err = ib_is_udata_in_empty(uhw); 74 if (err) 75 return err; 76 77 memset(attr, 0, sizeof *attr); 78 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], 79 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); 80 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 81 dev->nic_info.mac_addr); 82 attr->max_mr_size = dev->attr.max_mr_size; 83 attr->page_size_cap = 0xffff000; 84 attr->vendor_id = dev->nic_info.pdev->vendor; 85 attr->vendor_part_id = dev->nic_info.pdev->device; 86 attr->hw_ver = dev->asic_id; 87 attr->max_qp = dev->attr.max_qp; 88 attr->max_ah = OCRDMA_MAX_AH; 89 attr->max_qp_wr = dev->attr.max_wqe; 90 91 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 92 IB_DEVICE_RC_RNR_NAK_GEN | 93 IB_DEVICE_SHUTDOWN_PORT | 94 IB_DEVICE_SYS_IMAGE_GUID | 95 IB_DEVICE_MEM_MGT_EXTENSIONS; 96 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 97 attr->max_send_sge = dev->attr.max_send_sge; 98 attr->max_recv_sge = dev->attr.max_recv_sge; 99 attr->max_sge_rd = dev->attr.max_rdma_sge; 100 attr->max_cq = dev->attr.max_cq; 101 attr->max_cqe = dev->attr.max_cqe; 102 attr->max_mr = dev->attr.max_mr; 103 attr->max_mw = dev->attr.max_mw; 104 attr->max_pd = dev->attr.max_pd; 105 attr->atomic_cap = 0; 106 attr->max_qp_rd_atom = 107 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 108 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 109 attr->max_srq = dev->attr.max_srq; 110 attr->max_srq_sge = dev->attr.max_srq_sge; 111 attr->max_srq_wr = dev->attr.max_rqe; 112 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 113 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; 114 attr->max_pkeys = 1; 115 return ib_respond_empty_udata(uhw); 116 } 117 118 static inline void get_link_speed_and_width(struct ocrdma_dev *dev, 119 u16 *ib_speed, u8 *ib_width) 120 { 121 int status; 122 u8 speed; 123 124 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL); 125 if (status) 126 speed = OCRDMA_PHYS_LINK_SPEED_ZERO; 127 128 switch (speed) { 129 case OCRDMA_PHYS_LINK_SPEED_1GBPS: 130 *ib_speed = IB_SPEED_SDR; 131 *ib_width = IB_WIDTH_1X; 132 break; 133 134 case OCRDMA_PHYS_LINK_SPEED_10GBPS: 135 *ib_speed = IB_SPEED_QDR; 136 *ib_width = IB_WIDTH_1X; 137 break; 138 139 case OCRDMA_PHYS_LINK_SPEED_20GBPS: 140 *ib_speed = IB_SPEED_DDR; 141 *ib_width = IB_WIDTH_4X; 142 break; 143 144 case OCRDMA_PHYS_LINK_SPEED_40GBPS: 145 *ib_speed = IB_SPEED_QDR; 146 *ib_width = IB_WIDTH_4X; 147 break; 148 149 default: 150 /* Unsupported */ 151 *ib_speed = IB_SPEED_SDR; 152 *ib_width = IB_WIDTH_1X; 153 } 154 } 155 156 int ocrdma_query_port(struct ib_device *ibdev, 157 u32 port, struct ib_port_attr *props) 158 { 159 enum ib_port_state port_state; 160 struct ocrdma_dev *dev; 161 struct net_device *netdev; 162 163 /* props being zeroed by the caller, avoid zeroing it here */ 164 dev = get_ocrdma_dev(ibdev); 165 netdev = dev->nic_info.netdev; 166 if (netif_running(netdev) && netif_oper_up(netdev)) { 167 port_state = IB_PORT_ACTIVE; 168 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 169 } else { 170 port_state = IB_PORT_DOWN; 171 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 172 } 173 props->max_mtu = IB_MTU_4096; 174 props->active_mtu = iboe_get_mtu(netdev->mtu); 175 props->lid = 0; 176 props->lmc = 0; 177 props->sm_lid = 0; 178 props->sm_sl = 0; 179 props->state = port_state; 180 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 181 IB_PORT_DEVICE_MGMT_SUP | 182 IB_PORT_VENDOR_CLASS_SUP; 183 props->ip_gids = true; 184 props->gid_tbl_len = OCRDMA_MAX_SGID; 185 props->pkey_tbl_len = 1; 186 props->bad_pkey_cntr = 0; 187 props->qkey_viol_cntr = 0; 188 get_link_speed_and_width(dev, &props->active_speed, 189 &props->active_width); 190 props->max_msg_sz = 0x80000000; 191 props->max_vl_num = 4; 192 return 0; 193 } 194 195 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 196 unsigned long len) 197 { 198 struct ocrdma_mm *mm; 199 200 mm = kzalloc_obj(*mm); 201 if (mm == NULL) 202 return -ENOMEM; 203 mm->key.phy_addr = phy_addr; 204 mm->key.len = len; 205 INIT_LIST_HEAD(&mm->entry); 206 207 mutex_lock(&uctx->mm_list_lock); 208 list_add_tail(&mm->entry, &uctx->mm_head); 209 mutex_unlock(&uctx->mm_list_lock); 210 return 0; 211 } 212 213 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 214 unsigned long len) 215 { 216 struct ocrdma_mm *mm, *tmp; 217 218 mutex_lock(&uctx->mm_list_lock); 219 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 220 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 221 continue; 222 223 list_del(&mm->entry); 224 kfree(mm); 225 break; 226 } 227 mutex_unlock(&uctx->mm_list_lock); 228 } 229 230 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 231 unsigned long len) 232 { 233 bool found = false; 234 struct ocrdma_mm *mm; 235 236 mutex_lock(&uctx->mm_list_lock); 237 list_for_each_entry(mm, &uctx->mm_head, entry) { 238 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 239 continue; 240 241 found = true; 242 break; 243 } 244 mutex_unlock(&uctx->mm_list_lock); 245 return found; 246 } 247 248 249 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) 250 { 251 u16 pd_bitmap_idx = 0; 252 unsigned long *pd_bitmap; 253 254 if (dpp_pool) { 255 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; 256 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 257 dev->pd_mgr->max_dpp_pd); 258 __set_bit(pd_bitmap_idx, pd_bitmap); 259 dev->pd_mgr->pd_dpp_count++; 260 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) 261 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; 262 } else { 263 pd_bitmap = dev->pd_mgr->pd_norm_bitmap; 264 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 265 dev->pd_mgr->max_normal_pd); 266 __set_bit(pd_bitmap_idx, pd_bitmap); 267 dev->pd_mgr->pd_norm_count++; 268 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) 269 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; 270 } 271 return pd_bitmap_idx; 272 } 273 274 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, 275 bool dpp_pool) 276 { 277 u16 pd_count; 278 u16 pd_bit_index; 279 280 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count : 281 dev->pd_mgr->pd_norm_count; 282 if (pd_count == 0) 283 return -EINVAL; 284 285 if (dpp_pool) { 286 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start; 287 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) { 288 return -EINVAL; 289 } else { 290 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap); 291 dev->pd_mgr->pd_dpp_count--; 292 } 293 } else { 294 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start; 295 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) { 296 return -EINVAL; 297 } else { 298 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap); 299 dev->pd_mgr->pd_norm_count--; 300 } 301 } 302 303 return 0; 304 } 305 306 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, 307 bool dpp_pool) 308 { 309 int status; 310 311 mutex_lock(&dev->dev_lock); 312 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool); 313 mutex_unlock(&dev->dev_lock); 314 return status; 315 } 316 317 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 318 { 319 u16 pd_idx = 0; 320 int status = 0; 321 322 mutex_lock(&dev->dev_lock); 323 if (pd->dpp_enabled) { 324 /* try allocating DPP PD, if not available then normal PD */ 325 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) { 326 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true); 327 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx; 328 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx; 329 } else if (dev->pd_mgr->pd_norm_count < 330 dev->pd_mgr->max_normal_pd) { 331 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 332 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 333 pd->dpp_enabled = false; 334 } else { 335 status = -EINVAL; 336 } 337 } else { 338 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) { 339 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 340 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 341 } else { 342 status = -EINVAL; 343 } 344 } 345 mutex_unlock(&dev->dev_lock); 346 return status; 347 } 348 349 /* 350 * NOTE: 351 * 352 * ocrdma_ucontext must be used here because this function is also 353 * called from ocrdma_alloc_ucontext where ib_udata does not have 354 * valid ib_ucontext pointer. ib_uverbs_get_context does not call 355 * uobj_{alloc|get_xxx} helpers which are used to store the 356 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so 357 * ib_udata does NOT imply valid ib_ucontext here! 358 */ 359 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 360 struct ocrdma_ucontext *uctx, 361 struct ib_udata *udata) 362 { 363 int status; 364 365 if (udata && uctx && dev->attr.max_dpp_pds) { 366 pd->dpp_enabled = 367 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R; 368 pd->num_dpp_qp = 369 pd->dpp_enabled ? (dev->nic_info.db_page_size / 370 dev->attr.wqe_size) : 0; 371 } 372 373 if (dev->pd_mgr->pd_prealloc_valid) 374 return ocrdma_get_pd_num(dev, pd); 375 376 retry: 377 status = ocrdma_mbx_alloc_pd(dev, pd); 378 if (status) { 379 if (pd->dpp_enabled) { 380 pd->dpp_enabled = false; 381 pd->num_dpp_qp = 0; 382 goto retry; 383 } 384 return status; 385 } 386 387 return 0; 388 } 389 390 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, 391 struct ocrdma_pd *pd) 392 { 393 return (uctx->cntxt_pd == pd); 394 } 395 396 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev, 397 struct ocrdma_pd *pd) 398 { 399 if (dev->pd_mgr->pd_prealloc_valid) 400 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled); 401 else 402 ocrdma_mbx_dealloc_pd(dev, pd); 403 } 404 405 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, 406 struct ocrdma_ucontext *uctx, 407 struct ib_udata *udata) 408 { 409 struct ib_device *ibdev = &dev->ibdev; 410 struct ib_pd *pd; 411 int status; 412 413 pd = rdma_zalloc_drv_obj(ibdev, ib_pd); 414 if (!pd) 415 return -ENOMEM; 416 417 pd->device = ibdev; 418 uctx->cntxt_pd = get_ocrdma_pd(pd); 419 420 status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata); 421 if (status) { 422 kfree(uctx->cntxt_pd); 423 goto err; 424 } 425 426 uctx->cntxt_pd->uctx = uctx; 427 uctx->cntxt_pd->ibpd.device = &dev->ibdev; 428 err: 429 return status; 430 } 431 432 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) 433 { 434 struct ocrdma_pd *pd = uctx->cntxt_pd; 435 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 436 437 if (uctx->pd_in_use) { 438 pr_err("%s(%d) Freeing in use pdid=0x%x.\n", 439 __func__, dev->id, pd->id); 440 } 441 uctx->cntxt_pd = NULL; 442 _ocrdma_dealloc_pd(dev, pd); 443 kfree(pd); 444 } 445 446 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) 447 { 448 struct ocrdma_pd *pd = NULL; 449 450 mutex_lock(&uctx->mm_list_lock); 451 if (!uctx->pd_in_use) { 452 uctx->pd_in_use = true; 453 pd = uctx->cntxt_pd; 454 } 455 mutex_unlock(&uctx->mm_list_lock); 456 457 return pd; 458 } 459 460 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) 461 { 462 mutex_lock(&uctx->mm_list_lock); 463 uctx->pd_in_use = false; 464 mutex_unlock(&uctx->mm_list_lock); 465 } 466 467 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 468 { 469 struct ib_device *ibdev = uctx->device; 470 int status; 471 struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx); 472 struct ocrdma_alloc_ucontext_resp resp = {}; 473 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 474 struct pci_dev *pdev = dev->nic_info.pdev; 475 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE); 476 477 if (!udata) 478 return -EFAULT; 479 INIT_LIST_HEAD(&ctx->mm_head); 480 mutex_init(&ctx->mm_list_lock); 481 482 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len, 483 &ctx->ah_tbl.pa, GFP_KERNEL); 484 if (!ctx->ah_tbl.va) 485 return -ENOMEM; 486 487 ctx->ah_tbl.len = map_len; 488 489 resp.ah_tbl_len = ctx->ah_tbl.len; 490 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 491 492 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 493 if (status) 494 goto map_err; 495 496 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata); 497 if (status) 498 goto pd_err; 499 500 resp.dev_id = dev->id; 501 resp.max_inline_data = dev->attr.max_inline_data; 502 resp.wqe_size = dev->attr.wqe_size; 503 resp.rqe_size = dev->attr.rqe_size; 504 resp.dpp_wqe_size = dev->attr.wqe_size; 505 506 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); 507 status = ib_respond_udata(udata, resp); 508 if (status) 509 goto cpy_err; 510 return 0; 511 512 cpy_err: 513 ocrdma_dealloc_ucontext_pd(ctx); 514 pd_err: 515 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 516 map_err: 517 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 518 ctx->ah_tbl.pa); 519 return status; 520 } 521 522 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 523 { 524 struct ocrdma_mm *mm, *tmp; 525 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 526 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device); 527 struct pci_dev *pdev = dev->nic_info.pdev; 528 529 ocrdma_dealloc_ucontext_pd(uctx); 530 531 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 532 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 533 uctx->ah_tbl.pa); 534 535 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 536 list_del(&mm->entry); 537 kfree(mm); 538 } 539 } 540 541 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 542 { 543 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 544 struct ocrdma_dev *dev = get_ocrdma_dev(context->device); 545 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 546 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 547 unsigned long len = (vma->vm_end - vma->vm_start); 548 int status; 549 bool found; 550 551 if (vma->vm_start & (PAGE_SIZE - 1)) 552 return -EINVAL; 553 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len); 554 if (!found) 555 return -EINVAL; 556 557 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 558 dev->nic_info.db_total_size)) && 559 (len <= dev->nic_info.db_page_size)) { 560 if (vma->vm_flags & VM_READ) 561 return -EPERM; 562 563 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 564 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 565 len, vma->vm_page_prot); 566 } else if (dev->nic_info.dpp_unmapped_len && 567 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) && 568 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 569 dev->nic_info.dpp_unmapped_len)) && 570 (len <= dev->nic_info.dpp_unmapped_len)) { 571 if (vma->vm_flags & VM_READ) 572 return -EPERM; 573 574 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 575 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 576 len, vma->vm_page_prot); 577 } else { 578 status = remap_pfn_range(vma, vma->vm_start, 579 vma->vm_pgoff, len, vma->vm_page_prot); 580 } 581 return status; 582 } 583 584 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 585 struct ib_udata *udata) 586 { 587 int status; 588 u64 db_page_addr; 589 u64 dpp_page_addr = 0; 590 u32 db_page_size; 591 struct ocrdma_alloc_pd_uresp rsp = {}; 592 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 593 udata, struct ocrdma_ucontext, ibucontext); 594 595 rsp.id = pd->id; 596 rsp.dpp_enabled = pd->dpp_enabled; 597 db_page_addr = ocrdma_get_db_addr(dev, pd->id); 598 db_page_size = dev->nic_info.db_page_size; 599 600 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 601 if (status) 602 return status; 603 604 if (pd->dpp_enabled) { 605 dpp_page_addr = dev->nic_info.dpp_unmapped_addr + 606 (pd->id * PAGE_SIZE); 607 status = ocrdma_add_mmap(uctx, dpp_page_addr, 608 PAGE_SIZE); 609 if (status) 610 goto dpp_map_err; 611 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 612 rsp.dpp_page_addr_lo = dpp_page_addr; 613 } 614 615 status = ib_respond_udata(udata, rsp); 616 if (status) 617 goto ucopy_err; 618 619 pd->uctx = uctx; 620 return 0; 621 622 ucopy_err: 623 if (pd->dpp_enabled) 624 ocrdma_del_mmap(uctx, dpp_page_addr, PAGE_SIZE); 625 dpp_map_err: 626 ocrdma_del_mmap(uctx, db_page_addr, db_page_size); 627 return status; 628 } 629 630 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 631 { 632 struct ib_device *ibdev = ibpd->device; 633 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 634 struct ocrdma_pd *pd; 635 int status; 636 u8 is_uctx_pd = false; 637 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 638 udata, struct ocrdma_ucontext, ibucontext); 639 640 if (udata) { 641 pd = ocrdma_get_ucontext_pd(uctx); 642 if (pd) { 643 is_uctx_pd = true; 644 goto pd_mapping; 645 } 646 } 647 648 pd = get_ocrdma_pd(ibpd); 649 status = _ocrdma_alloc_pd(dev, pd, uctx, udata); 650 if (status) 651 goto exit; 652 653 pd_mapping: 654 if (udata) { 655 status = ocrdma_copy_pd_uresp(dev, pd, udata); 656 if (status) 657 goto err; 658 } 659 return 0; 660 661 err: 662 if (is_uctx_pd) 663 ocrdma_release_ucontext_pd(uctx); 664 else 665 _ocrdma_dealloc_pd(dev, pd); 666 exit: 667 return status; 668 } 669 670 int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 671 { 672 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 673 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 674 struct ocrdma_ucontext *uctx = NULL; 675 u64 usr_db; 676 677 uctx = pd->uctx; 678 if (uctx) { 679 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 680 (pd->id * PAGE_SIZE); 681 if (pd->dpp_enabled) 682 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE); 683 usr_db = ocrdma_get_db_addr(dev, pd->id); 684 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 685 686 if (is_ucontext_pd(uctx, pd)) { 687 ocrdma_release_ucontext_pd(uctx); 688 return 0; 689 } 690 } 691 _ocrdma_dealloc_pd(dev, pd); 692 return 0; 693 } 694 695 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 696 u32 pdid, int acc, u32 num_pbls, u32 addr_check) 697 { 698 int status; 699 700 mr->hwmr.fr_mr = 0; 701 mr->hwmr.local_rd = 1; 702 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 703 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 704 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 705 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0; 706 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 707 mr->hwmr.num_pbls = num_pbls; 708 709 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check); 710 if (status) 711 return status; 712 713 mr->ibmr.lkey = mr->hwmr.lkey; 714 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 715 mr->ibmr.rkey = mr->hwmr.lkey; 716 return 0; 717 } 718 719 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 720 { 721 int status; 722 struct ocrdma_mr *mr; 723 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 724 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 725 726 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 727 pr_err("%s err, invalid access rights\n", __func__); 728 return ERR_PTR(-EINVAL); 729 } 730 731 mr = kzalloc_obj(*mr); 732 if (!mr) 733 return ERR_PTR(-ENOMEM); 734 735 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0, 736 OCRDMA_ADDR_CHECK_DISABLE); 737 if (status) { 738 kfree(mr); 739 return ERR_PTR(status); 740 } 741 742 return &mr->ibmr; 743 } 744 745 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, 746 struct ocrdma_hw_mr *mr) 747 { 748 struct pci_dev *pdev = dev->nic_info.pdev; 749 int i = 0; 750 751 if (mr->pbl_table) { 752 for (i = 0; i < mr->num_pbls; i++) { 753 if (!mr->pbl_table[i].va) 754 continue; 755 dma_free_coherent(&pdev->dev, mr->pbl_size, 756 mr->pbl_table[i].va, 757 mr->pbl_table[i].pa); 758 } 759 kfree(mr->pbl_table); 760 mr->pbl_table = NULL; 761 } 762 } 763 764 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 765 u32 num_pbes) 766 { 767 u32 num_pbls = 0; 768 u32 idx = 0; 769 int status = 0; 770 u32 pbl_size; 771 772 do { 773 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx); 774 if (pbl_size > MAX_OCRDMA_PBL_SIZE) { 775 status = -EFAULT; 776 break; 777 } 778 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 779 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 780 idx++; 781 } while (num_pbls >= dev->attr.max_num_mr_pbl); 782 783 mr->hwmr.num_pbes = num_pbes; 784 mr->hwmr.num_pbls = num_pbls; 785 mr->hwmr.pbl_size = pbl_size; 786 return status; 787 } 788 789 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) 790 { 791 int status = 0; 792 int i; 793 u32 dma_len = mr->pbl_size; 794 struct pci_dev *pdev = dev->nic_info.pdev; 795 void *va; 796 dma_addr_t pa; 797 798 mr->pbl_table = kzalloc_objs(*mr->pbl_table, mr->num_pbls); 799 800 if (!mr->pbl_table) 801 return -ENOMEM; 802 803 for (i = 0; i < mr->num_pbls; i++) { 804 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL); 805 if (!va) { 806 ocrdma_free_mr_pbl_tbl(dev, mr); 807 status = -ENOMEM; 808 break; 809 } 810 mr->pbl_table[i].va = va; 811 mr->pbl_table[i].pa = pa; 812 } 813 return status; 814 } 815 816 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr) 817 { 818 struct ocrdma_pbe *pbe; 819 struct ib_block_iter biter; 820 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 821 int pbe_cnt; 822 u64 pg_addr; 823 824 if (!mr->hwmr.num_pbes) 825 return; 826 827 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 828 pbe_cnt = 0; 829 830 rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) { 831 /* store the page address in pbe */ 832 pg_addr = rdma_block_iter_dma_address(&biter); 833 pbe->pa_lo = cpu_to_le32(pg_addr); 834 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr)); 835 pbe_cnt += 1; 836 pbe++; 837 838 /* if the given pbl is full storing the pbes, 839 * move to next pbl. 840 */ 841 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) { 842 pbl_tbl++; 843 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 844 pbe_cnt = 0; 845 } 846 } 847 } 848 849 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 850 u64 usr_addr, int acc, struct ib_dmah *dmah, 851 struct ib_udata *udata) 852 { 853 int status = -ENOMEM; 854 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 855 struct ocrdma_mr *mr; 856 struct ocrdma_pd *pd; 857 858 if (dmah) 859 return ERR_PTR(-EOPNOTSUPP); 860 861 pd = get_ocrdma_pd(ibpd); 862 863 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 864 return ERR_PTR(-EINVAL); 865 866 mr = kzalloc_obj(*mr); 867 if (!mr) 868 return ERR_PTR(status); 869 mr->umem = ib_umem_get_va(ibpd->device, start, len, acc); 870 if (IS_ERR(mr->umem)) { 871 status = -EFAULT; 872 goto umem_err; 873 } 874 status = ocrdma_get_pbl_info( 875 dev, mr, ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE)); 876 if (status) 877 goto umem_err; 878 879 mr->hwmr.pbe_size = PAGE_SIZE; 880 mr->hwmr.va = usr_addr; 881 mr->hwmr.len = len; 882 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 883 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 884 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 885 mr->hwmr.local_rd = 1; 886 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 887 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 888 if (status) 889 goto umem_err; 890 build_user_pbes(dev, mr); 891 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 892 if (status) 893 goto mbx_err; 894 mr->ibmr.lkey = mr->hwmr.lkey; 895 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 896 mr->ibmr.rkey = mr->hwmr.lkey; 897 898 return &mr->ibmr; 899 900 mbx_err: 901 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 902 umem_err: 903 kfree(mr); 904 return ERR_PTR(status); 905 } 906 907 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 908 { 909 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 910 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device); 911 912 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 913 914 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 915 916 /* it could be user registered memory. */ 917 ib_umem_release(mr->umem); 918 kfree(mr); 919 920 /* Don't stop cleanup, in case FW is unresponsive */ 921 if (dev->mqe_ctx.fw_error_state) { 922 pr_err("%s(%d) fw not responding.\n", 923 __func__, dev->id); 924 } 925 return 0; 926 } 927 928 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 929 struct ib_udata *udata) 930 { 931 int status; 932 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 933 udata, struct ocrdma_ucontext, ibucontext); 934 struct ocrdma_create_cq_uresp uresp = {}; 935 936 /* this must be user flow! */ 937 if (!udata) 938 return -EINVAL; 939 940 uresp.cq_id = cq->id; 941 uresp.page_size = PAGE_ALIGN(cq->len); 942 uresp.num_pages = 1; 943 uresp.max_hw_cqe = cq->max_hw_cqe; 944 uresp.page_addr[0] = virt_to_phys(cq->va); 945 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 946 uresp.db_page_size = dev->nic_info.db_page_size; 947 uresp.phase_change = cq->phase_change ? 1 : 0; 948 status = ib_respond_udata(udata, uresp); 949 if (status) 950 goto err; 951 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 952 if (status) 953 goto err; 954 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size); 955 if (status) { 956 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 957 goto err; 958 } 959 cq->ucontext = uctx; 960 err: 961 return status; 962 } 963 964 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 965 struct uverbs_attr_bundle *attrs) 966 { 967 struct ib_udata *udata = &attrs->driver_udata; 968 struct ib_device *ibdev = ibcq->device; 969 int entries = attr->cqe; 970 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 971 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 972 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 973 udata, struct ocrdma_ucontext, ibucontext); 974 u16 pd_id = 0; 975 int status; 976 struct ocrdma_create_cq_ureq ureq; 977 978 if (attr->flags) 979 return -EOPNOTSUPP; 980 981 if (udata) { 982 status = ib_copy_validate_udata_in(udata, ureq, rsvd); 983 if (status) 984 return status; 985 } else 986 ureq.dpp_cq = 0; 987 988 spin_lock_init(&cq->cq_lock); 989 spin_lock_init(&cq->comp_handler_lock); 990 INIT_LIST_HEAD(&cq->sq_head); 991 INIT_LIST_HEAD(&cq->rq_head); 992 993 if (udata) 994 pd_id = uctx->cntxt_pd->id; 995 996 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); 997 if (status) 998 return status; 999 1000 if (udata) { 1001 status = ocrdma_copy_cq_uresp(dev, cq, udata); 1002 if (status) 1003 goto ctx_err; 1004 } 1005 cq->phase = OCRDMA_CQE_VALID; 1006 dev->cq_tbl[cq->id] = cq; 1007 return 0; 1008 1009 ctx_err: 1010 ocrdma_mbx_destroy_cq(dev, cq); 1011 return status; 1012 } 1013 1014 int ocrdma_resize_cq(struct ib_cq *ibcq, unsigned int new_cnt, 1015 struct ib_udata *udata) 1016 { 1017 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1018 1019 if (new_cnt > cq->max_hw_cqe) 1020 return -EINVAL; 1021 1022 ibcq->cqe = new_cnt; 1023 return 0; 1024 } 1025 1026 static void ocrdma_flush_cq(struct ocrdma_cq *cq) 1027 { 1028 int cqe_cnt; 1029 int valid_count = 0; 1030 unsigned long flags; 1031 1032 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 1033 struct ocrdma_cqe *cqe = NULL; 1034 1035 cqe = cq->va; 1036 cqe_cnt = cq->cqe_cnt; 1037 1038 /* Last irq might have scheduled a polling thread 1039 * sync-up with it before hard flushing. 1040 */ 1041 spin_lock_irqsave(&cq->cq_lock, flags); 1042 while (cqe_cnt) { 1043 if (is_cqe_valid(cq, cqe)) 1044 valid_count++; 1045 cqe++; 1046 cqe_cnt--; 1047 } 1048 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count); 1049 spin_unlock_irqrestore(&cq->cq_lock, flags); 1050 } 1051 1052 int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1053 { 1054 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1055 struct ocrdma_eq *eq = NULL; 1056 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 1057 int pdid = 0; 1058 u32 irq, indx; 1059 1060 dev->cq_tbl[cq->id] = NULL; 1061 indx = ocrdma_get_eq_table_index(dev, cq->eqn); 1062 1063 eq = &dev->eq_tbl[indx]; 1064 irq = ocrdma_get_irq(dev, eq); 1065 synchronize_irq(irq); 1066 ocrdma_flush_cq(cq); 1067 1068 ocrdma_mbx_destroy_cq(dev, cq); 1069 if (cq->ucontext) { 1070 pdid = cq->ucontext->cntxt_pd->id; 1071 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, 1072 PAGE_ALIGN(cq->len)); 1073 ocrdma_del_mmap(cq->ucontext, 1074 ocrdma_get_db_addr(dev, pdid), 1075 dev->nic_info.db_page_size); 1076 } 1077 return 0; 1078 } 1079 1080 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1081 { 1082 int status = -EINVAL; 1083 1084 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { 1085 dev->qp_tbl[qp->id] = qp; 1086 status = 0; 1087 } 1088 return status; 1089 } 1090 1091 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1092 { 1093 dev->qp_tbl[qp->id] = NULL; 1094 } 1095 1096 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 1097 struct ib_qp_init_attr *attrs, 1098 struct ib_udata *udata) 1099 { 1100 if ((attrs->qp_type != IB_QPT_GSI) && 1101 (attrs->qp_type != IB_QPT_RC) && 1102 (attrs->qp_type != IB_QPT_UC) && 1103 (attrs->qp_type != IB_QPT_UD)) { 1104 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1105 __func__, dev->id, attrs->qp_type); 1106 return -EOPNOTSUPP; 1107 } 1108 /* Skip the check for QP1 to support CM size of 128 */ 1109 if ((attrs->qp_type != IB_QPT_GSI) && 1110 (attrs->cap.max_send_wr > dev->attr.max_wqe)) { 1111 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1112 __func__, dev->id, attrs->cap.max_send_wr); 1113 pr_err("%s(%d) supported send_wr=0x%x\n", 1114 __func__, dev->id, dev->attr.max_wqe); 1115 return -EINVAL; 1116 } 1117 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) { 1118 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n", 1119 __func__, dev->id, attrs->cap.max_recv_wr); 1120 pr_err("%s(%d) supported recv_wr=0x%x\n", 1121 __func__, dev->id, dev->attr.max_rqe); 1122 return -EINVAL; 1123 } 1124 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) { 1125 pr_err("%s(%d) unsupported inline data size=0x%x requested\n", 1126 __func__, dev->id, attrs->cap.max_inline_data); 1127 pr_err("%s(%d) supported inline data size=0x%x\n", 1128 __func__, dev->id, dev->attr.max_inline_data); 1129 return -EINVAL; 1130 } 1131 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) { 1132 pr_err("%s(%d) unsupported send_sge=0x%x requested\n", 1133 __func__, dev->id, attrs->cap.max_send_sge); 1134 pr_err("%s(%d) supported send_sge=0x%x\n", 1135 __func__, dev->id, dev->attr.max_send_sge); 1136 return -EINVAL; 1137 } 1138 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) { 1139 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n", 1140 __func__, dev->id, attrs->cap.max_recv_sge); 1141 pr_err("%s(%d) supported recv_sge=0x%x\n", 1142 __func__, dev->id, dev->attr.max_recv_sge); 1143 return -EINVAL; 1144 } 1145 /* unprivileged user space cannot create special QP */ 1146 if (udata && attrs->qp_type == IB_QPT_GSI) { 1147 pr_err 1148 ("%s(%d) Userspace can't create special QPs of type=0x%x\n", 1149 __func__, dev->id, attrs->qp_type); 1150 return -EINVAL; 1151 } 1152 /* allow creating only one GSI type of QP */ 1153 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 1154 pr_err("%s(%d) GSI special QPs already created.\n", 1155 __func__, dev->id); 1156 return -EINVAL; 1157 } 1158 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1159 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1160 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1161 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) { 1162 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1163 __func__, dev->id); 1164 return -EINVAL; 1165 } 1166 } 1167 return 0; 1168 } 1169 1170 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, 1171 struct ib_udata *udata, int dpp_offset, 1172 int dpp_credit_lmt, int srq) 1173 { 1174 int status; 1175 u64 usr_db; 1176 struct ocrdma_create_qp_uresp uresp = {}; 1177 struct ocrdma_pd *pd = qp->pd; 1178 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 1179 1180 usr_db = dev->nic_info.unmapped_db + 1181 (pd->id * dev->nic_info.db_page_size); 1182 uresp.qp_id = qp->id; 1183 uresp.sq_dbid = qp->sq.dbid; 1184 uresp.num_sq_pages = 1; 1185 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1186 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1187 uresp.num_wqe_allocated = qp->sq.max_cnt; 1188 if (!srq) { 1189 uresp.rq_dbid = qp->rq.dbid; 1190 uresp.num_rq_pages = 1; 1191 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1192 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1193 uresp.num_rqe_allocated = qp->rq.max_cnt; 1194 } 1195 uresp.db_page_addr = usr_db; 1196 uresp.db_page_size = dev->nic_info.db_page_size; 1197 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1198 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1199 uresp.db_shift = OCRDMA_DB_RQ_SHIFT; 1200 1201 if (qp->dpp_enabled) { 1202 uresp.dpp_credit = dpp_credit_lmt; 1203 uresp.dpp_offset = dpp_offset; 1204 } 1205 status = ib_respond_udata(udata, uresp); 1206 if (status) 1207 goto err; 1208 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], 1209 uresp.sq_page_size); 1210 if (status) 1211 goto err; 1212 1213 if (!srq) { 1214 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0], 1215 uresp.rq_page_size); 1216 if (status) 1217 goto rq_map_err; 1218 } 1219 return status; 1220 rq_map_err: 1221 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); 1222 err: 1223 return status; 1224 } 1225 1226 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, 1227 struct ocrdma_pd *pd) 1228 { 1229 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1230 qp->sq_db = dev->nic_info.db + 1231 (pd->id * dev->nic_info.db_page_size) + 1232 OCRDMA_DB_GEN2_SQ_OFFSET; 1233 qp->rq_db = dev->nic_info.db + 1234 (pd->id * dev->nic_info.db_page_size) + 1235 OCRDMA_DB_GEN2_RQ_OFFSET; 1236 } else { 1237 qp->sq_db = dev->nic_info.db + 1238 (pd->id * dev->nic_info.db_page_size) + 1239 OCRDMA_DB_SQ_OFFSET; 1240 qp->rq_db = dev->nic_info.db + 1241 (pd->id * dev->nic_info.db_page_size) + 1242 OCRDMA_DB_RQ_OFFSET; 1243 } 1244 } 1245 1246 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) 1247 { 1248 qp->wqe_wr_id_tbl = kzalloc_objs(*qp->wqe_wr_id_tbl, qp->sq.max_cnt); 1249 if (qp->wqe_wr_id_tbl == NULL) 1250 return -ENOMEM; 1251 1252 qp->rqe_wr_id_tbl = kzalloc_objs(*qp->rqe_wr_id_tbl, qp->rq.max_cnt); 1253 if (qp->rqe_wr_id_tbl == NULL) 1254 return -ENOMEM; 1255 1256 return 0; 1257 } 1258 1259 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, 1260 struct ocrdma_pd *pd, 1261 struct ib_qp_init_attr *attrs) 1262 { 1263 qp->pd = pd; 1264 spin_lock_init(&qp->q_lock); 1265 INIT_LIST_HEAD(&qp->sq_entry); 1266 INIT_LIST_HEAD(&qp->rq_entry); 1267 1268 qp->qp_type = attrs->qp_type; 1269 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; 1270 qp->max_inline_data = attrs->cap.max_inline_data; 1271 qp->sq.max_sges = attrs->cap.max_send_sge; 1272 qp->rq.max_sges = attrs->cap.max_recv_sge; 1273 qp->state = OCRDMA_QPS_RST; 1274 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; 1275 } 1276 1277 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, 1278 struct ib_qp_init_attr *attrs) 1279 { 1280 if (attrs->qp_type == IB_QPT_GSI) { 1281 dev->gsi_qp_created = 1; 1282 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq); 1283 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq); 1284 } 1285 } 1286 1287 int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 1288 struct ib_udata *udata) 1289 { 1290 int status; 1291 struct ib_pd *ibpd = ibqp->pd; 1292 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1293 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1294 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1295 struct ocrdma_create_qp_ureq ureq; 1296 u16 dpp_credit_lmt, dpp_offset; 1297 1298 if (attrs->create_flags) 1299 return -EOPNOTSUPP; 1300 1301 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata); 1302 if (status) 1303 goto gen_err; 1304 1305 if (udata) { 1306 status = ib_copy_validate_udata_in(udata, ureq, rsvd1); 1307 if (status) 1308 return status; 1309 } else { 1310 memset(&ureq, 0, sizeof(ureq)); 1311 } 1312 1313 ocrdma_set_qp_init_params(qp, pd, attrs); 1314 if (udata == NULL) 1315 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | 1316 OCRDMA_QP_FAST_REG); 1317 1318 mutex_lock(&dev->dev_lock); 1319 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1320 ureq.dpp_cq_id, 1321 &dpp_offset, &dpp_credit_lmt); 1322 if (status) 1323 goto mbx_err; 1324 1325 /* user space QP's wr_id table are managed in library */ 1326 if (udata == NULL) { 1327 status = ocrdma_alloc_wr_id_tbl(qp); 1328 if (status) 1329 goto map_err; 1330 } 1331 1332 status = ocrdma_add_qpn_map(dev, qp); 1333 if (status) 1334 goto map_err; 1335 ocrdma_set_qp_db(dev, qp, pd); 1336 if (udata) { 1337 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, 1338 dpp_credit_lmt, 1339 (attrs->srq != NULL)); 1340 if (status) 1341 goto cpy_err; 1342 } 1343 ocrdma_store_gsi_qp_cq(dev, attrs); 1344 qp->ibqp.qp_num = qp->id; 1345 mutex_unlock(&dev->dev_lock); 1346 return 0; 1347 1348 cpy_err: 1349 ocrdma_del_qpn_map(dev, qp); 1350 map_err: 1351 ocrdma_mbx_destroy_qp(dev, qp); 1352 mbx_err: 1353 mutex_unlock(&dev->dev_lock); 1354 kfree(qp->wqe_wr_id_tbl); 1355 kfree(qp->rqe_wr_id_tbl); 1356 pr_err("%s(%d) error=%d\n", __func__, dev->id, status); 1357 gen_err: 1358 return status; 1359 } 1360 1361 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1362 int attr_mask) 1363 { 1364 int status = 0; 1365 struct ocrdma_qp *qp; 1366 struct ocrdma_dev *dev; 1367 enum ib_qp_state old_qps; 1368 1369 qp = get_ocrdma_qp(ibqp); 1370 dev = get_ocrdma_dev(ibqp->device); 1371 if (attr_mask & IB_QP_STATE) 1372 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); 1373 /* if new and previous states are same hw doesn't need to 1374 * know about it. 1375 */ 1376 if (status < 0) 1377 return status; 1378 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); 1379 } 1380 1381 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1382 int attr_mask, struct ib_udata *udata) 1383 { 1384 unsigned long flags; 1385 int status = -EINVAL; 1386 struct ocrdma_qp *qp; 1387 struct ocrdma_dev *dev; 1388 enum ib_qp_state old_qps, new_qps; 1389 1390 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1391 return -EOPNOTSUPP; 1392 1393 qp = get_ocrdma_qp(ibqp); 1394 dev = get_ocrdma_dev(ibqp->device); 1395 1396 /* syncronize with multiple context trying to change, retrive qps */ 1397 mutex_lock(&dev->dev_lock); 1398 /* syncronize with wqe, rqe posting and cqe processing contexts */ 1399 spin_lock_irqsave(&qp->q_lock, flags); 1400 old_qps = get_ibqp_state(qp->state); 1401 if (attr_mask & IB_QP_STATE) 1402 new_qps = attr->qp_state; 1403 else 1404 new_qps = old_qps; 1405 spin_unlock_irqrestore(&qp->q_lock, flags); 1406 1407 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) { 1408 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n" 1409 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n", 1410 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, 1411 old_qps, new_qps); 1412 goto param_err; 1413 } 1414 1415 status = _ocrdma_modify_qp(ibqp, attr, attr_mask); 1416 if (status > 0) 1417 status = 0; 1418 param_err: 1419 mutex_unlock(&dev->dev_lock); 1420 return status; 1421 } 1422 1423 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) 1424 { 1425 switch (mtu) { 1426 case 256: 1427 return IB_MTU_256; 1428 case 512: 1429 return IB_MTU_512; 1430 case 1024: 1431 return IB_MTU_1024; 1432 case 2048: 1433 return IB_MTU_2048; 1434 case 4096: 1435 return IB_MTU_4096; 1436 default: 1437 return IB_MTU_1024; 1438 } 1439 } 1440 1441 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) 1442 { 1443 int ib_qp_acc_flags = 0; 1444 1445 if (qp_cap_flags & OCRDMA_QP_INB_WR) 1446 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1447 if (qp_cap_flags & OCRDMA_QP_INB_RD) 1448 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 1449 return ib_qp_acc_flags; 1450 } 1451 1452 int ocrdma_query_qp(struct ib_qp *ibqp, 1453 struct ib_qp_attr *qp_attr, 1454 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 1455 { 1456 int status; 1457 u32 qp_state; 1458 struct ocrdma_qp_params params; 1459 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1460 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1461 1462 memset(¶ms, 0, sizeof(params)); 1463 mutex_lock(&dev->dev_lock); 1464 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); 1465 mutex_unlock(&dev->dev_lock); 1466 if (status) 1467 goto mbx_err; 1468 if (qp->qp_type == IB_QPT_UD) 1469 qp_attr->qkey = params.qkey; 1470 qp_attr->path_mtu = 1471 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx & 1472 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >> 1473 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT; 1474 qp_attr->path_mig_state = IB_MIG_MIGRATED; 1475 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK; 1476 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK; 1477 qp_attr->dest_qp_num = 1478 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK; 1479 1480 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); 1481 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; 1482 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1483 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1484 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1485 qp_attr->cap.max_inline_data = qp->max_inline_data; 1486 qp_init_attr->cap = qp_attr->cap; 1487 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1488 1489 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 1490 params.rnt_rc_sl_fl & 1491 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK, 1492 qp->sgid_idx, 1493 (params.hop_lmt_rq_psn & 1494 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >> 1495 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT, 1496 (params.tclass_sq_psn & 1497 OCRDMA_QP_PARAMS_TCLASS_MASK) >> 1498 OCRDMA_QP_PARAMS_TCLASS_SHIFT); 1499 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]); 1500 1501 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 1502 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl & 1503 OCRDMA_QP_PARAMS_SL_MASK) >> 1504 OCRDMA_QP_PARAMS_SL_SHIFT); 1505 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn & 1506 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >> 1507 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT; 1508 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn & 1509 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >> 1510 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT; 1511 qp_attr->retry_cnt = 1512 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >> 1513 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT; 1514 qp_attr->min_rnr_timer = 0; 1515 qp_attr->pkey_index = 0; 1516 qp_attr->port_num = 1; 1517 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 1518 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 1519 qp_attr->alt_pkey_index = 0; 1520 qp_attr->alt_port_num = 0; 1521 qp_attr->alt_timeout = 0; 1522 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 1523 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >> 1524 OCRDMA_QP_PARAMS_STATE_SHIFT; 1525 qp_attr->qp_state = get_ibqp_state(qp_state); 1526 qp_attr->cur_qp_state = qp_attr->qp_state; 1527 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0; 1528 qp_attr->max_dest_rd_atomic = 1529 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT; 1530 qp_attr->max_rd_atomic = 1531 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK; 1532 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags & 1533 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0; 1534 /* Sync driver QP state with FW */ 1535 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); 1536 mbx_err: 1537 return status; 1538 } 1539 1540 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) 1541 { 1542 unsigned int i = idx / 32; 1543 u32 mask = (1U << (idx % 32)); 1544 1545 srq->idx_bit_fields[i] ^= mask; 1546 } 1547 1548 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1549 { 1550 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt; 1551 } 1552 1553 static int is_hw_sq_empty(struct ocrdma_qp *qp) 1554 { 1555 return (qp->sq.tail == qp->sq.head); 1556 } 1557 1558 static int is_hw_rq_empty(struct ocrdma_qp *qp) 1559 { 1560 return (qp->rq.tail == qp->rq.head); 1561 } 1562 1563 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1564 { 1565 return q->va + (q->head * q->entry_size); 1566 } 1567 1568 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, 1569 u32 idx) 1570 { 1571 return q->va + (idx * q->entry_size); 1572 } 1573 1574 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) 1575 { 1576 q->head = (q->head + 1) & q->max_wqe_idx; 1577 } 1578 1579 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) 1580 { 1581 q->tail = (q->tail + 1) & q->max_wqe_idx; 1582 } 1583 1584 /* discard the cqe for a given QP */ 1585 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) 1586 { 1587 unsigned long cq_flags; 1588 unsigned long flags; 1589 u32 cur_getp, stop_getp; 1590 struct ocrdma_cqe *cqe; 1591 u32 qpn = 0, wqe_idx = 0; 1592 1593 spin_lock_irqsave(&cq->cq_lock, cq_flags); 1594 1595 /* traverse through the CQEs in the hw CQ, 1596 * find the matching CQE for a given qp, 1597 * mark the matching one discarded by clearing qpn. 1598 * ring the doorbell in the poll_cq() as 1599 * we don't complete out of order cqe. 1600 */ 1601 1602 cur_getp = cq->getp; 1603 /* find upto when do we reap the cq. */ 1604 stop_getp = cur_getp; 1605 do { 1606 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) 1607 break; 1608 1609 cqe = cq->va + cur_getp; 1610 /* if (a) done reaping whole hw cq, or 1611 * (b) qp_xq becomes empty. 1612 * then exit 1613 */ 1614 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK; 1615 /* if previously discarded cqe found, skip that too. */ 1616 /* check for matching qp */ 1617 if (qpn == 0 || qpn != qp->id) 1618 goto skip_cqe; 1619 1620 if (is_cqe_for_sq(cqe)) { 1621 ocrdma_hwq_inc_tail(&qp->sq); 1622 } else { 1623 if (qp->srq) { 1624 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 1625 OCRDMA_CQE_BUFTAG_SHIFT) & 1626 qp->srq->rq.max_wqe_idx; 1627 BUG_ON(wqe_idx < 1); 1628 spin_lock_irqsave(&qp->srq->q_lock, flags); 1629 ocrdma_hwq_inc_tail(&qp->srq->rq); 1630 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); 1631 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1632 1633 } else { 1634 ocrdma_hwq_inc_tail(&qp->rq); 1635 } 1636 } 1637 /* mark cqe discarded so that it is not picked up later 1638 * in the poll_cq(). 1639 */ 1640 cqe->cmn.qpn = 0; 1641 skip_cqe: 1642 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1643 } while (cur_getp != stop_getp); 1644 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1645 } 1646 1647 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1648 { 1649 int found = false; 1650 unsigned long flags; 1651 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 1652 /* sync with any active CQ poll */ 1653 1654 spin_lock_irqsave(&dev->flush_q_lock, flags); 1655 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); 1656 if (found) 1657 list_del(&qp->sq_entry); 1658 if (!qp->srq) { 1659 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); 1660 if (found) 1661 list_del(&qp->rq_entry); 1662 } 1663 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 1664 } 1665 1666 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1667 { 1668 struct ocrdma_pd *pd; 1669 struct ocrdma_qp *qp; 1670 struct ocrdma_dev *dev; 1671 struct ib_qp_attr attrs; 1672 int attr_mask; 1673 unsigned long flags; 1674 1675 qp = get_ocrdma_qp(ibqp); 1676 dev = get_ocrdma_dev(ibqp->device); 1677 1678 pd = qp->pd; 1679 1680 /* change the QP state to ERROR */ 1681 if (qp->state != OCRDMA_QPS_RST) { 1682 attrs.qp_state = IB_QPS_ERR; 1683 attr_mask = IB_QP_STATE; 1684 _ocrdma_modify_qp(ibqp, &attrs, attr_mask); 1685 } 1686 /* ensure that CQEs for newly created QP (whose id may be same with 1687 * one which just getting destroyed are same), dont get 1688 * discarded until the old CQEs are discarded. 1689 */ 1690 mutex_lock(&dev->dev_lock); 1691 (void) ocrdma_mbx_destroy_qp(dev, qp); 1692 1693 /* 1694 * acquire CQ lock while destroy is in progress, in order to 1695 * protect against proessing in-flight CQEs for this QP. 1696 */ 1697 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); 1698 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) { 1699 spin_lock(&qp->rq_cq->cq_lock); 1700 ocrdma_del_qpn_map(dev, qp); 1701 spin_unlock(&qp->rq_cq->cq_lock); 1702 } else { 1703 ocrdma_del_qpn_map(dev, qp); 1704 } 1705 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); 1706 1707 if (!pd->uctx) { 1708 ocrdma_discard_cqes(qp, qp->sq_cq); 1709 ocrdma_discard_cqes(qp, qp->rq_cq); 1710 } 1711 mutex_unlock(&dev->dev_lock); 1712 1713 if (pd->uctx) { 1714 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, 1715 PAGE_ALIGN(qp->sq.len)); 1716 if (!qp->srq) 1717 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, 1718 PAGE_ALIGN(qp->rq.len)); 1719 } 1720 1721 ocrdma_del_flush_qp(qp); 1722 1723 kfree(qp->wqe_wr_id_tbl); 1724 kfree(qp->rqe_wr_id_tbl); 1725 return 0; 1726 } 1727 1728 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, 1729 struct ib_udata *udata) 1730 { 1731 int status; 1732 struct ocrdma_create_srq_uresp uresp = {}; 1733 1734 uresp.rq_dbid = srq->rq.dbid; 1735 uresp.num_rq_pages = 1; 1736 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1737 uresp.rq_page_size = srq->rq.len; 1738 uresp.db_page_addr = dev->nic_info.unmapped_db + 1739 (srq->pd->id * dev->nic_info.db_page_size); 1740 uresp.db_page_size = dev->nic_info.db_page_size; 1741 uresp.num_rqe_allocated = srq->rq.max_cnt; 1742 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1743 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1744 uresp.db_shift = 24; 1745 } else { 1746 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1747 uresp.db_shift = 16; 1748 } 1749 1750 status = ib_respond_udata(udata, uresp); 1751 if (status) 1752 return status; 1753 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], 1754 uresp.rq_page_size); 1755 if (status) 1756 return status; 1757 return status; 1758 } 1759 1760 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1761 struct ib_udata *udata) 1762 { 1763 int status; 1764 struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd); 1765 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1766 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq); 1767 1768 if (init_attr->srq_type != IB_SRQT_BASIC) 1769 return -EOPNOTSUPP; 1770 1771 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1772 return -EINVAL; 1773 if (init_attr->attr.max_wr > dev->attr.max_rqe) 1774 return -EINVAL; 1775 1776 spin_lock_init(&srq->q_lock); 1777 srq->pd = pd; 1778 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1779 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd); 1780 if (status) 1781 return status; 1782 1783 if (!udata) { 1784 srq->rqe_wr_id_tbl = 1785 kzalloc_objs(*srq->rqe_wr_id_tbl, srq->rq.max_cnt); 1786 if (!srq->rqe_wr_id_tbl) { 1787 status = -ENOMEM; 1788 goto arm_err; 1789 } 1790 1791 srq->bit_fields_len = (srq->rq.max_cnt / 32) + 1792 (srq->rq.max_cnt % 32 ? 1 : 0); 1793 srq->idx_bit_fields = 1794 kmalloc_array(srq->bit_fields_len, sizeof(u32), 1795 GFP_KERNEL); 1796 if (!srq->idx_bit_fields) { 1797 status = -ENOMEM; 1798 goto arm_err; 1799 } 1800 memset(srq->idx_bit_fields, 0xff, 1801 srq->bit_fields_len * sizeof(u32)); 1802 } 1803 1804 if (init_attr->attr.srq_limit) { 1805 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr); 1806 if (status) 1807 goto arm_err; 1808 } 1809 1810 if (udata) { 1811 status = ocrdma_copy_srq_uresp(dev, srq, udata); 1812 if (status) 1813 goto arm_err; 1814 } 1815 1816 return 0; 1817 1818 arm_err: 1819 ocrdma_mbx_destroy_srq(dev, srq); 1820 kfree(srq->rqe_wr_id_tbl); 1821 kfree(srq->idx_bit_fields); 1822 return status; 1823 } 1824 1825 int ocrdma_modify_srq(struct ib_srq *ibsrq, 1826 struct ib_srq_attr *srq_attr, 1827 enum ib_srq_attr_mask srq_attr_mask, 1828 struct ib_udata *udata) 1829 { 1830 int status; 1831 struct ocrdma_srq *srq; 1832 1833 srq = get_ocrdma_srq(ibsrq); 1834 if (srq_attr_mask & IB_SRQ_MAX_WR) 1835 status = -EINVAL; 1836 else 1837 status = ocrdma_mbx_modify_srq(srq, srq_attr); 1838 return status; 1839 } 1840 1841 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 1842 { 1843 struct ocrdma_srq *srq; 1844 1845 srq = get_ocrdma_srq(ibsrq); 1846 return ocrdma_mbx_query_srq(srq, srq_attr); 1847 } 1848 1849 int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1850 { 1851 struct ocrdma_srq *srq; 1852 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1853 1854 srq = get_ocrdma_srq(ibsrq); 1855 1856 ocrdma_mbx_destroy_srq(dev, srq); 1857 1858 if (srq->pd->uctx) 1859 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, 1860 PAGE_ALIGN(srq->rq.len)); 1861 1862 kfree(srq->idx_bit_fields); 1863 kfree(srq->rqe_wr_id_tbl); 1864 return 0; 1865 } 1866 1867 /* unprivileged verbs and their support functions. */ 1868 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, 1869 struct ocrdma_hdr_wqe *hdr, 1870 const struct ib_send_wr *wr) 1871 { 1872 struct ocrdma_ewqe_ud_hdr *ud_hdr = 1873 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1); 1874 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah); 1875 1876 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn; 1877 if (qp->qp_type == IB_QPT_GSI) 1878 ud_hdr->qkey = qp->qkey; 1879 else 1880 ud_hdr->qkey = ud_wr(wr)->remote_qkey; 1881 ud_hdr->rsvd_ahid = ah->id; 1882 ud_hdr->hdr_type = ah->hdr_type; 1883 if (ah->av->valid & OCRDMA_AV_VLAN_VALID) 1884 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT); 1885 } 1886 1887 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, 1888 struct ocrdma_sge *sge, int num_sge, 1889 struct ib_sge *sg_list) 1890 { 1891 int i; 1892 1893 for (i = 0; i < num_sge; i++) { 1894 sge[i].lrkey = sg_list[i].lkey; 1895 sge[i].addr_lo = sg_list[i].addr; 1896 sge[i].addr_hi = upper_32_bits(sg_list[i].addr); 1897 sge[i].len = sg_list[i].length; 1898 hdr->total_len += sg_list[i].length; 1899 } 1900 if (num_sge == 0) 1901 memset(sge, 0, sizeof(*sge)); 1902 } 1903 1904 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) 1905 { 1906 uint32_t total_len = 0, i; 1907 1908 for (i = 0; i < num_sge; i++) 1909 total_len += sg_list[i].length; 1910 return total_len; 1911 } 1912 1913 1914 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 1915 struct ocrdma_hdr_wqe *hdr, 1916 struct ocrdma_sge *sge, 1917 const struct ib_send_wr *wr, u32 wqe_size) 1918 { 1919 int i; 1920 char *dpp_addr; 1921 1922 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { 1923 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge); 1924 if (unlikely(hdr->total_len > qp->max_inline_data)) { 1925 pr_err("%s() supported_len=0x%x,\n" 1926 " unsupported len req=0x%x\n", __func__, 1927 qp->max_inline_data, hdr->total_len); 1928 return -EINVAL; 1929 } 1930 dpp_addr = (char *)sge; 1931 for (i = 0; i < wr->num_sge; i++) { 1932 memcpy(dpp_addr, 1933 (void *)(unsigned long)wr->sg_list[i].addr, 1934 wr->sg_list[i].length); 1935 dpp_addr += wr->sg_list[i].length; 1936 } 1937 1938 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 1939 if (0 == hdr->total_len) 1940 wqe_size += sizeof(struct ocrdma_sge); 1941 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 1942 } else { 1943 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1944 if (wr->num_sge) 1945 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge)); 1946 else 1947 wqe_size += sizeof(struct ocrdma_sge); 1948 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 1949 } 1950 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1951 return 0; 1952 } 1953 1954 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1955 const struct ib_send_wr *wr) 1956 { 1957 struct ocrdma_sge *sge; 1958 u32 wqe_size = sizeof(*hdr); 1959 1960 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 1961 ocrdma_build_ud_hdr(qp, hdr, wr); 1962 sge = (struct ocrdma_sge *)(hdr + 2); 1963 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 1964 } else { 1965 sge = (struct ocrdma_sge *)(hdr + 1); 1966 } 1967 1968 return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1969 } 1970 1971 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1972 const struct ib_send_wr *wr) 1973 { 1974 int status; 1975 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1976 struct ocrdma_sge *sge = ext_rw + 1; 1977 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw); 1978 1979 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1980 if (status) 1981 return status; 1982 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 1983 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 1984 ext_rw->lrkey = rdma_wr(wr)->rkey; 1985 ext_rw->len = hdr->total_len; 1986 return 0; 1987 } 1988 1989 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1990 const struct ib_send_wr *wr) 1991 { 1992 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1993 struct ocrdma_sge *sge = ext_rw + 1; 1994 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) + 1995 sizeof(struct ocrdma_hdr_wqe); 1996 1997 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1998 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1999 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT); 2000 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2001 2002 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2003 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2004 ext_rw->lrkey = rdma_wr(wr)->rkey; 2005 ext_rw->len = hdr->total_len; 2006 } 2007 2008 static int get_encoded_page_size(int pg_sz) 2009 { 2010 /* Max size is 256M 4096 << 16 */ 2011 int i = 0; 2012 for (; i < 17; i++) 2013 if (pg_sz == (4096 << i)) 2014 break; 2015 return i; 2016 } 2017 2018 static int ocrdma_build_reg(struct ocrdma_qp *qp, 2019 struct ocrdma_hdr_wqe *hdr, 2020 const struct ib_reg_wr *wr) 2021 { 2022 u64 fbo; 2023 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1); 2024 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr); 2025 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 2026 struct ocrdma_pbe *pbe; 2027 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr); 2028 int num_pbes = 0, i; 2029 2030 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES); 2031 2032 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT); 2033 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2034 2035 if (wr->access & IB_ACCESS_LOCAL_WRITE) 2036 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR; 2037 if (wr->access & IB_ACCESS_REMOTE_WRITE) 2038 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR; 2039 if (wr->access & IB_ACCESS_REMOTE_READ) 2040 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD; 2041 hdr->lkey = wr->key; 2042 hdr->total_len = mr->ibmr.length; 2043 2044 fbo = mr->ibmr.iova - mr->pages[0]; 2045 2046 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova); 2047 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff); 2048 fast_reg->fbo_hi = upper_32_bits(fbo); 2049 fast_reg->fbo_lo = (u32) fbo & 0xffffffff; 2050 fast_reg->num_sges = mr->npages; 2051 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size); 2052 2053 pbe = pbl_tbl->va; 2054 for (i = 0; i < mr->npages; i++) { 2055 u64 buf_addr = mr->pages[i]; 2056 2057 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK)); 2058 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr)); 2059 num_pbes += 1; 2060 pbe++; 2061 2062 /* if the pbl is full storing the pbes, 2063 * move to next pbl. 2064 */ 2065 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) { 2066 pbl_tbl++; 2067 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 2068 } 2069 } 2070 2071 return 0; 2072 } 2073 2074 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2075 { 2076 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); 2077 2078 iowrite32(val, qp->sq_db); 2079 } 2080 2081 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 2082 const struct ib_send_wr **bad_wr) 2083 { 2084 int status = 0; 2085 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2086 struct ocrdma_hdr_wqe *hdr; 2087 unsigned long flags; 2088 2089 spin_lock_irqsave(&qp->q_lock, flags); 2090 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { 2091 spin_unlock_irqrestore(&qp->q_lock, flags); 2092 *bad_wr = wr; 2093 return -EINVAL; 2094 } 2095 2096 while (wr) { 2097 if (qp->qp_type == IB_QPT_UD && 2098 (wr->opcode != IB_WR_SEND && 2099 wr->opcode != IB_WR_SEND_WITH_IMM)) { 2100 *bad_wr = wr; 2101 status = -EINVAL; 2102 break; 2103 } 2104 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || 2105 wr->num_sge > qp->sq.max_sges) { 2106 *bad_wr = wr; 2107 status = -ENOMEM; 2108 break; 2109 } 2110 hdr = ocrdma_hwq_head(&qp->sq); 2111 hdr->cw = 0; 2112 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2113 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2114 if (wr->send_flags & IB_SEND_FENCE) 2115 hdr->cw |= 2116 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT); 2117 if (wr->send_flags & IB_SEND_SOLICITED) 2118 hdr->cw |= 2119 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT); 2120 hdr->total_len = 0; 2121 switch (wr->opcode) { 2122 case IB_WR_SEND_WITH_IMM: 2123 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2124 hdr->immdt = ntohl(wr->ex.imm_data); 2125 fallthrough; 2126 case IB_WR_SEND: 2127 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2128 ocrdma_build_send(qp, hdr, wr); 2129 break; 2130 case IB_WR_SEND_WITH_INV: 2131 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT); 2132 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2133 hdr->lkey = wr->ex.invalidate_rkey; 2134 status = ocrdma_build_send(qp, hdr, wr); 2135 break; 2136 case IB_WR_RDMA_WRITE_WITH_IMM: 2137 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2138 hdr->immdt = ntohl(wr->ex.imm_data); 2139 fallthrough; 2140 case IB_WR_RDMA_WRITE: 2141 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT); 2142 status = ocrdma_build_write(qp, hdr, wr); 2143 break; 2144 case IB_WR_RDMA_READ: 2145 ocrdma_build_read(qp, hdr, wr); 2146 break; 2147 case IB_WR_LOCAL_INV: 2148 hdr->cw |= 2149 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2150 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) + 2151 sizeof(struct ocrdma_sge)) / 2152 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2153 hdr->lkey = wr->ex.invalidate_rkey; 2154 break; 2155 case IB_WR_REG_MR: 2156 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); 2157 break; 2158 default: 2159 status = -EINVAL; 2160 break; 2161 } 2162 if (status) { 2163 *bad_wr = wr; 2164 break; 2165 } 2166 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2167 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2168 else 2169 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2170 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; 2171 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) & 2172 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE); 2173 /* make sure wqe is written before adapter can access it */ 2174 wmb(); 2175 /* inform hw to start processing it */ 2176 ocrdma_ring_sq_db(qp); 2177 2178 /* update pointer, counter for next wr */ 2179 ocrdma_hwq_inc_head(&qp->sq); 2180 wr = wr->next; 2181 } 2182 spin_unlock_irqrestore(&qp->q_lock, flags); 2183 return status; 2184 } 2185 2186 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) 2187 { 2188 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); 2189 2190 iowrite32(val, qp->rq_db); 2191 } 2192 2193 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, 2194 const struct ib_recv_wr *wr, u16 tag) 2195 { 2196 u32 wqe_size = 0; 2197 struct ocrdma_sge *sge; 2198 if (wr->num_sge) 2199 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe); 2200 else 2201 wqe_size = sizeof(*sge) + sizeof(*rqe); 2202 2203 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << 2204 OCRDMA_WQE_SIZE_SHIFT); 2205 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2206 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2207 rqe->total_len = 0; 2208 rqe->rsvd_tag = tag; 2209 sge = (struct ocrdma_sge *)(rqe + 1); 2210 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list); 2211 ocrdma_cpu_to_le32(rqe, wqe_size); 2212 } 2213 2214 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 2215 const struct ib_recv_wr **bad_wr) 2216 { 2217 int status = 0; 2218 unsigned long flags; 2219 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2220 struct ocrdma_hdr_wqe *rqe; 2221 2222 spin_lock_irqsave(&qp->q_lock, flags); 2223 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { 2224 spin_unlock_irqrestore(&qp->q_lock, flags); 2225 *bad_wr = wr; 2226 return -EINVAL; 2227 } 2228 while (wr) { 2229 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || 2230 wr->num_sge > qp->rq.max_sges) { 2231 *bad_wr = wr; 2232 status = -ENOMEM; 2233 break; 2234 } 2235 rqe = ocrdma_hwq_head(&qp->rq); 2236 ocrdma_build_rqe(rqe, wr, 0); 2237 2238 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; 2239 /* make sure rqe is written before adapter can access it */ 2240 wmb(); 2241 2242 /* inform hw to start processing it */ 2243 ocrdma_ring_rq_db(qp); 2244 2245 /* update pointer, counter for next wr */ 2246 ocrdma_hwq_inc_head(&qp->rq); 2247 wr = wr->next; 2248 } 2249 spin_unlock_irqrestore(&qp->q_lock, flags); 2250 return status; 2251 } 2252 2253 /* cqe for srq's rqe can potentially arrive out of order. 2254 * index gives the entry in the shadow table where to store 2255 * the wr_id. tag/index is returned in cqe to reference back 2256 * for a given rqe. 2257 */ 2258 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) 2259 { 2260 int row = 0; 2261 int indx = 0; 2262 2263 for (row = 0; row < srq->bit_fields_len; row++) { 2264 if (srq->idx_bit_fields[row]) { 2265 indx = ffs(srq->idx_bit_fields[row]); 2266 indx = (row * 32) + (indx - 1); 2267 BUG_ON(indx >= srq->rq.max_cnt); 2268 ocrdma_srq_toggle_bit(srq, indx); 2269 break; 2270 } 2271 } 2272 2273 BUG_ON(row == srq->bit_fields_len); 2274 return indx + 1; /* Use from index 1 */ 2275 } 2276 2277 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) 2278 { 2279 u32 val = srq->rq.dbid | (1 << 16); 2280 2281 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET); 2282 } 2283 2284 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 2285 const struct ib_recv_wr **bad_wr) 2286 { 2287 int status = 0; 2288 unsigned long flags; 2289 struct ocrdma_srq *srq; 2290 struct ocrdma_hdr_wqe *rqe; 2291 u16 tag; 2292 2293 srq = get_ocrdma_srq(ibsrq); 2294 2295 spin_lock_irqsave(&srq->q_lock, flags); 2296 while (wr) { 2297 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 || 2298 wr->num_sge > srq->rq.max_sges) { 2299 status = -ENOMEM; 2300 *bad_wr = wr; 2301 break; 2302 } 2303 tag = ocrdma_srq_get_idx(srq); 2304 rqe = ocrdma_hwq_head(&srq->rq); 2305 ocrdma_build_rqe(rqe, wr, tag); 2306 2307 srq->rqe_wr_id_tbl[tag] = wr->wr_id; 2308 /* make sure rqe is written before adapter can perform DMA */ 2309 wmb(); 2310 /* inform hw to start processing it */ 2311 ocrdma_ring_srq_db(srq); 2312 /* update pointer, counter for next wr */ 2313 ocrdma_hwq_inc_head(&srq->rq); 2314 wr = wr->next; 2315 } 2316 spin_unlock_irqrestore(&srq->q_lock, flags); 2317 return status; 2318 } 2319 2320 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2321 { 2322 enum ib_wc_status ibwc_status; 2323 2324 switch (status) { 2325 case OCRDMA_CQE_GENERAL_ERR: 2326 ibwc_status = IB_WC_GENERAL_ERR; 2327 break; 2328 case OCRDMA_CQE_LOC_LEN_ERR: 2329 ibwc_status = IB_WC_LOC_LEN_ERR; 2330 break; 2331 case OCRDMA_CQE_LOC_QP_OP_ERR: 2332 ibwc_status = IB_WC_LOC_QP_OP_ERR; 2333 break; 2334 case OCRDMA_CQE_LOC_EEC_OP_ERR: 2335 ibwc_status = IB_WC_LOC_EEC_OP_ERR; 2336 break; 2337 case OCRDMA_CQE_LOC_PROT_ERR: 2338 ibwc_status = IB_WC_LOC_PROT_ERR; 2339 break; 2340 case OCRDMA_CQE_WR_FLUSH_ERR: 2341 ibwc_status = IB_WC_WR_FLUSH_ERR; 2342 break; 2343 case OCRDMA_CQE_MW_BIND_ERR: 2344 ibwc_status = IB_WC_MW_BIND_ERR; 2345 break; 2346 case OCRDMA_CQE_BAD_RESP_ERR: 2347 ibwc_status = IB_WC_BAD_RESP_ERR; 2348 break; 2349 case OCRDMA_CQE_LOC_ACCESS_ERR: 2350 ibwc_status = IB_WC_LOC_ACCESS_ERR; 2351 break; 2352 case OCRDMA_CQE_REM_INV_REQ_ERR: 2353 ibwc_status = IB_WC_REM_INV_REQ_ERR; 2354 break; 2355 case OCRDMA_CQE_REM_ACCESS_ERR: 2356 ibwc_status = IB_WC_REM_ACCESS_ERR; 2357 break; 2358 case OCRDMA_CQE_REM_OP_ERR: 2359 ibwc_status = IB_WC_REM_OP_ERR; 2360 break; 2361 case OCRDMA_CQE_RETRY_EXC_ERR: 2362 ibwc_status = IB_WC_RETRY_EXC_ERR; 2363 break; 2364 case OCRDMA_CQE_RNR_RETRY_EXC_ERR: 2365 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR; 2366 break; 2367 case OCRDMA_CQE_LOC_RDD_VIOL_ERR: 2368 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR; 2369 break; 2370 case OCRDMA_CQE_REM_INV_RD_REQ_ERR: 2371 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR; 2372 break; 2373 case OCRDMA_CQE_REM_ABORT_ERR: 2374 ibwc_status = IB_WC_REM_ABORT_ERR; 2375 break; 2376 case OCRDMA_CQE_INV_EECN_ERR: 2377 ibwc_status = IB_WC_INV_EECN_ERR; 2378 break; 2379 case OCRDMA_CQE_INV_EEC_STATE_ERR: 2380 ibwc_status = IB_WC_INV_EEC_STATE_ERR; 2381 break; 2382 case OCRDMA_CQE_FATAL_ERR: 2383 ibwc_status = IB_WC_FATAL_ERR; 2384 break; 2385 case OCRDMA_CQE_RESP_TIMEOUT_ERR: 2386 ibwc_status = IB_WC_RESP_TIMEOUT_ERR; 2387 break; 2388 default: 2389 ibwc_status = IB_WC_GENERAL_ERR; 2390 break; 2391 } 2392 return ibwc_status; 2393 } 2394 2395 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, 2396 u32 wqe_idx) 2397 { 2398 struct ocrdma_hdr_wqe *hdr; 2399 struct ocrdma_sge *rw; 2400 int opcode; 2401 2402 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); 2403 2404 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; 2405 /* Undo the hdr->cw swap */ 2406 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK; 2407 switch (opcode) { 2408 case OCRDMA_WRITE: 2409 ibwc->opcode = IB_WC_RDMA_WRITE; 2410 break; 2411 case OCRDMA_READ: 2412 rw = (struct ocrdma_sge *)(hdr + 1); 2413 ibwc->opcode = IB_WC_RDMA_READ; 2414 ibwc->byte_len = rw->len; 2415 break; 2416 case OCRDMA_SEND: 2417 ibwc->opcode = IB_WC_SEND; 2418 break; 2419 case OCRDMA_FR_MR: 2420 ibwc->opcode = IB_WC_REG_MR; 2421 break; 2422 case OCRDMA_LKEY_INV: 2423 ibwc->opcode = IB_WC_LOCAL_INV; 2424 break; 2425 default: 2426 ibwc->status = IB_WC_GENERAL_ERR; 2427 pr_err("%s() invalid opcode received = 0x%x\n", 2428 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK); 2429 break; 2430 } 2431 } 2432 2433 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, 2434 struct ocrdma_cqe *cqe) 2435 { 2436 if (is_cqe_for_sq(cqe)) { 2437 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2438 cqe->flags_status_srcqpn) & 2439 ~OCRDMA_CQE_STATUS_MASK); 2440 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2441 cqe->flags_status_srcqpn) | 2442 (OCRDMA_CQE_WR_FLUSH_ERR << 2443 OCRDMA_CQE_STATUS_SHIFT)); 2444 } else { 2445 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2446 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2447 cqe->flags_status_srcqpn) & 2448 ~OCRDMA_CQE_UD_STATUS_MASK); 2449 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2450 cqe->flags_status_srcqpn) | 2451 (OCRDMA_CQE_WR_FLUSH_ERR << 2452 OCRDMA_CQE_UD_STATUS_SHIFT)); 2453 } else { 2454 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2455 cqe->flags_status_srcqpn) & 2456 ~OCRDMA_CQE_STATUS_MASK); 2457 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2458 cqe->flags_status_srcqpn) | 2459 (OCRDMA_CQE_WR_FLUSH_ERR << 2460 OCRDMA_CQE_STATUS_SHIFT)); 2461 } 2462 } 2463 } 2464 2465 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2466 struct ocrdma_qp *qp, int status) 2467 { 2468 bool expand = false; 2469 2470 ibwc->byte_len = 0; 2471 ibwc->qp = &qp->ibqp; 2472 ibwc->status = ocrdma_to_ibwc_err(status); 2473 2474 ocrdma_flush_qp(qp); 2475 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); 2476 2477 /* if wqe/rqe pending for which cqe needs to be returned, 2478 * trigger inflating it. 2479 */ 2480 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { 2481 expand = true; 2482 ocrdma_set_cqe_status_flushed(qp, cqe); 2483 } 2484 return expand; 2485 } 2486 2487 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2488 struct ocrdma_qp *qp, int status) 2489 { 2490 ibwc->opcode = IB_WC_RECV; 2491 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2492 ocrdma_hwq_inc_tail(&qp->rq); 2493 2494 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2495 } 2496 2497 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2498 struct ocrdma_qp *qp, int status) 2499 { 2500 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2501 ocrdma_hwq_inc_tail(&qp->sq); 2502 2503 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2504 } 2505 2506 2507 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, 2508 struct ocrdma_cqe *cqe, struct ib_wc *ibwc, 2509 bool *polled, bool *stop) 2510 { 2511 bool expand; 2512 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2513 int status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2514 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2515 if (status < OCRDMA_MAX_CQE_ERR) 2516 atomic_inc(&dev->cqe_err_stats[status]); 2517 2518 /* when hw sq is empty, but rq is not empty, so we continue 2519 * to keep the cqe in order to get the cq event again. 2520 */ 2521 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { 2522 /* when cq for rq and sq is same, it is safe to return 2523 * flush cqe for RQEs. 2524 */ 2525 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2526 *polled = true; 2527 status = OCRDMA_CQE_WR_FLUSH_ERR; 2528 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2529 } else { 2530 /* stop processing further cqe as this cqe is used for 2531 * triggering cq event on buddy cq of RQ. 2532 * When QP is destroyed, this cqe will be removed 2533 * from the cq's hardware q. 2534 */ 2535 *polled = false; 2536 *stop = true; 2537 expand = false; 2538 } 2539 } else if (is_hw_sq_empty(qp)) { 2540 /* Do nothing */ 2541 expand = false; 2542 *polled = false; 2543 *stop = false; 2544 } else { 2545 *polled = true; 2546 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2547 } 2548 return expand; 2549 } 2550 2551 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, 2552 struct ocrdma_cqe *cqe, 2553 struct ib_wc *ibwc, bool *polled) 2554 { 2555 bool expand = false; 2556 int tail = qp->sq.tail; 2557 u32 wqe_idx; 2558 2559 if (!qp->wqe_wr_id_tbl[tail].signaled) { 2560 *polled = false; /* WC cannot be consumed yet */ 2561 } else { 2562 ibwc->status = IB_WC_SUCCESS; 2563 ibwc->wc_flags = 0; 2564 ibwc->qp = &qp->ibqp; 2565 ocrdma_update_wc(qp, ibwc, tail); 2566 *polled = true; 2567 } 2568 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) & 2569 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; 2570 if (tail != wqe_idx) 2571 expand = true; /* Coalesced CQE can't be consumed yet */ 2572 2573 ocrdma_hwq_inc_tail(&qp->sq); 2574 return expand; 2575 } 2576 2577 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2578 struct ib_wc *ibwc, bool *polled, bool *stop) 2579 { 2580 int status; 2581 bool expand; 2582 2583 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2584 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2585 2586 if (status == OCRDMA_CQE_SUCCESS) 2587 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); 2588 else 2589 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); 2590 return expand; 2591 } 2592 2593 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, 2594 struct ocrdma_cqe *cqe) 2595 { 2596 int status; 2597 u16 hdr_type = 0; 2598 2599 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2600 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT; 2601 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) & 2602 OCRDMA_CQE_SRCQP_MASK; 2603 ibwc->pkey_index = 0; 2604 ibwc->wc_flags = IB_WC_GRH; 2605 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2606 OCRDMA_CQE_UD_XFER_LEN_SHIFT) & 2607 OCRDMA_CQE_UD_XFER_LEN_MASK; 2608 2609 if (ocrdma_is_udp_encap_supported(dev)) { 2610 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2611 OCRDMA_CQE_UD_L3TYPE_SHIFT) & 2612 OCRDMA_CQE_UD_L3TYPE_MASK; 2613 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2614 ibwc->network_hdr_type = hdr_type; 2615 } 2616 2617 return status; 2618 } 2619 2620 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, 2621 struct ocrdma_cqe *cqe, 2622 struct ocrdma_qp *qp) 2623 { 2624 unsigned long flags; 2625 struct ocrdma_srq *srq; 2626 u32 wqe_idx; 2627 2628 srq = get_ocrdma_srq(qp->ibqp.srq); 2629 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 2630 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx; 2631 BUG_ON(wqe_idx < 1); 2632 2633 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2634 spin_lock_irqsave(&srq->q_lock, flags); 2635 ocrdma_srq_toggle_bit(srq, wqe_idx - 1); 2636 spin_unlock_irqrestore(&srq->q_lock, flags); 2637 ocrdma_hwq_inc_tail(&srq->rq); 2638 } 2639 2640 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2641 struct ib_wc *ibwc, bool *polled, bool *stop, 2642 int status) 2643 { 2644 bool expand; 2645 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2646 2647 if (status < OCRDMA_MAX_CQE_ERR) 2648 atomic_inc(&dev->cqe_err_stats[status]); 2649 2650 /* when hw_rq is empty, but wq is not empty, so continue 2651 * to keep the cqe to get the cq event again. 2652 */ 2653 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { 2654 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2655 *polled = true; 2656 status = OCRDMA_CQE_WR_FLUSH_ERR; 2657 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2658 } else { 2659 *polled = false; 2660 *stop = true; 2661 expand = false; 2662 } 2663 } else if (is_hw_rq_empty(qp)) { 2664 /* Do nothing */ 2665 expand = false; 2666 *polled = false; 2667 *stop = false; 2668 } else { 2669 *polled = true; 2670 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2671 } 2672 return expand; 2673 } 2674 2675 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, 2676 struct ocrdma_cqe *cqe, struct ib_wc *ibwc) 2677 { 2678 struct ocrdma_dev *dev; 2679 2680 dev = get_ocrdma_dev(qp->ibqp.device); 2681 ibwc->opcode = IB_WC_RECV; 2682 ibwc->qp = &qp->ibqp; 2683 ibwc->status = IB_WC_SUCCESS; 2684 2685 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2686 ocrdma_update_ud_rcqe(dev, ibwc, cqe); 2687 else 2688 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen); 2689 2690 if (is_cqe_imm(cqe)) { 2691 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2692 ibwc->wc_flags |= IB_WC_WITH_IMM; 2693 } else if (is_cqe_wr_imm(cqe)) { 2694 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2695 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2696 ibwc->wc_flags |= IB_WC_WITH_IMM; 2697 } else if (is_cqe_invalidated(cqe)) { 2698 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2699 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2700 } 2701 if (qp->ibqp.srq) { 2702 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2703 } else { 2704 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2705 ocrdma_hwq_inc_tail(&qp->rq); 2706 } 2707 } 2708 2709 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2710 struct ib_wc *ibwc, bool *polled, bool *stop) 2711 { 2712 int status; 2713 bool expand = false; 2714 2715 ibwc->wc_flags = 0; 2716 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2717 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2718 OCRDMA_CQE_UD_STATUS_MASK) >> 2719 OCRDMA_CQE_UD_STATUS_SHIFT; 2720 } else { 2721 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2722 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2723 } 2724 2725 if (status == OCRDMA_CQE_SUCCESS) { 2726 *polled = true; 2727 ocrdma_poll_success_rcqe(qp, cqe, ibwc); 2728 } else { 2729 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, 2730 status); 2731 } 2732 return expand; 2733 } 2734 2735 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, 2736 u16 cur_getp) 2737 { 2738 if (cq->phase_change) { 2739 if (cur_getp == 0) 2740 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2741 } else { 2742 /* clear valid bit */ 2743 cqe->flags_status_srcqpn = 0; 2744 } 2745 } 2746 2747 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2748 struct ib_wc *ibwc) 2749 { 2750 u16 qpn = 0; 2751 int i = 0; 2752 bool expand = false; 2753 int polled_hw_cqes = 0; 2754 struct ocrdma_qp *qp = NULL; 2755 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 2756 struct ocrdma_cqe *cqe; 2757 u16 cur_getp; bool polled = false; bool stop = false; 2758 2759 cur_getp = cq->getp; 2760 while (num_entries) { 2761 cqe = cq->va + cur_getp; 2762 /* check whether valid cqe or not */ 2763 if (!is_cqe_valid(cq, cqe)) 2764 break; 2765 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK); 2766 /* ignore discarded cqe */ 2767 if (qpn == 0) 2768 goto skip_cqe; 2769 qp = dev->qp_tbl[qpn]; 2770 BUG_ON(qp == NULL); 2771 2772 if (is_cqe_for_sq(cqe)) { 2773 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, 2774 &stop); 2775 } else { 2776 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, 2777 &stop); 2778 } 2779 if (expand) 2780 goto expand_cqe; 2781 if (stop) 2782 goto stop_cqe; 2783 /* clear qpn to avoid duplicate processing by discard_cqe() */ 2784 cqe->cmn.qpn = 0; 2785 skip_cqe: 2786 polled_hw_cqes += 1; 2787 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 2788 ocrdma_change_cq_phase(cq, cqe, cur_getp); 2789 expand_cqe: 2790 if (polled) { 2791 num_entries -= 1; 2792 i += 1; 2793 ibwc = ibwc + 1; 2794 polled = false; 2795 } 2796 } 2797 stop_cqe: 2798 cq->getp = cur_getp; 2799 2800 if (polled_hw_cqes) 2801 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes); 2802 2803 return i; 2804 } 2805 2806 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ 2807 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, 2808 struct ocrdma_qp *qp, struct ib_wc *ibwc) 2809 { 2810 int err_cqes = 0; 2811 2812 while (num_entries) { 2813 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) 2814 break; 2815 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { 2816 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2817 ocrdma_hwq_inc_tail(&qp->sq); 2818 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2819 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2820 ocrdma_hwq_inc_tail(&qp->rq); 2821 } else { 2822 return err_cqes; 2823 } 2824 ibwc->byte_len = 0; 2825 ibwc->status = IB_WC_WR_FLUSH_ERR; 2826 ibwc = ibwc + 1; 2827 err_cqes += 1; 2828 num_entries -= 1; 2829 } 2830 return err_cqes; 2831 } 2832 2833 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2834 { 2835 int cqes_to_poll = num_entries; 2836 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2837 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2838 int num_os_cqe = 0, err_cqes = 0; 2839 struct ocrdma_qp *qp; 2840 unsigned long flags; 2841 2842 /* poll cqes from adapter CQ */ 2843 spin_lock_irqsave(&cq->cq_lock, flags); 2844 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc); 2845 spin_unlock_irqrestore(&cq->cq_lock, flags); 2846 cqes_to_poll -= num_os_cqe; 2847 2848 if (cqes_to_poll) { 2849 wc = wc + num_os_cqe; 2850 /* adapter returns single error cqe when qp moves to 2851 * error state. So insert error cqes with wc_status as 2852 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ 2853 * respectively which uses this CQ. 2854 */ 2855 spin_lock_irqsave(&dev->flush_q_lock, flags); 2856 list_for_each_entry(qp, &cq->sq_head, sq_entry) { 2857 if (cqes_to_poll == 0) 2858 break; 2859 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); 2860 cqes_to_poll -= err_cqes; 2861 num_os_cqe += err_cqes; 2862 wc = wc + err_cqes; 2863 } 2864 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 2865 } 2866 return num_os_cqe; 2867 } 2868 2869 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2870 { 2871 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2872 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2873 u16 cq_id; 2874 unsigned long flags; 2875 bool arm_needed = false, sol_needed = false; 2876 2877 cq_id = cq->id; 2878 2879 spin_lock_irqsave(&cq->cq_lock, flags); 2880 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 2881 arm_needed = true; 2882 if (cq_flags & IB_CQ_SOLICITED) 2883 sol_needed = true; 2884 2885 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0); 2886 spin_unlock_irqrestore(&cq->cq_lock, flags); 2887 2888 return 0; 2889 } 2890 2891 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2892 u32 max_num_sg) 2893 { 2894 int status; 2895 struct ocrdma_mr *mr; 2896 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 2897 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 2898 2899 if (mr_type != IB_MR_TYPE_MEM_REG) 2900 return ERR_PTR(-EINVAL); 2901 2902 if (max_num_sg > dev->attr.max_pages_per_frmr) 2903 return ERR_PTR(-EINVAL); 2904 2905 mr = kzalloc_flex(*mr, pages, max_num_sg); 2906 if (!mr) 2907 return ERR_PTR(-ENOMEM); 2908 2909 status = ocrdma_get_pbl_info(dev, mr, max_num_sg); 2910 if (status) 2911 goto pl_err; 2912 mr->hwmr.fr_mr = 1; 2913 mr->hwmr.remote_rd = 0; 2914 mr->hwmr.remote_wr = 0; 2915 mr->hwmr.local_rd = 0; 2916 mr->hwmr.local_wr = 0; 2917 mr->hwmr.mw_bind = 0; 2918 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 2919 if (status) 2920 goto pl_err; 2921 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0); 2922 if (status) 2923 goto mbx_err; 2924 mr->ibmr.rkey = mr->hwmr.lkey; 2925 mr->ibmr.lkey = mr->hwmr.lkey; 2926 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = 2927 (unsigned long) mr; 2928 return &mr->ibmr; 2929 mbx_err: 2930 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 2931 pl_err: 2932 kfree(mr); 2933 return ERR_PTR(-ENOMEM); 2934 } 2935 2936 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) 2937 { 2938 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2939 2940 if (unlikely(mr->npages == mr->hwmr.num_pbes)) 2941 return -ENOMEM; 2942 2943 mr->pages[mr->npages++] = addr; 2944 2945 return 0; 2946 } 2947 2948 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2949 unsigned int *sg_offset) 2950 { 2951 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2952 2953 mr->npages = 0; 2954 2955 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page); 2956 } 2957