1 /* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43 #include <linux/dma-mapping.h> 44 #include <net/addrconf.h> 45 #include <rdma/ib_verbs.h> 46 #include <rdma/ib_user_verbs.h> 47 #include <rdma/iw_cm.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_cache.h> 50 #include <rdma/iter.h> 51 #include <rdma/uverbs_ioctl.h> 52 53 #include "ocrdma.h" 54 #include "ocrdma_hw.h" 55 #include "ocrdma_verbs.h" 56 #include <rdma/ocrdma-abi.h> 57 58 int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 59 { 60 if (index > 0) 61 return -EINVAL; 62 63 *pkey = 0xffff; 64 return 0; 65 } 66 67 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 68 struct ib_udata *uhw) 69 { 70 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 71 int err; 72 73 err = ib_no_udata_io(uhw); 74 if (err) 75 return err; 76 77 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], 78 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); 79 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 80 dev->nic_info.mac_addr); 81 attr->max_mr_size = dev->attr.max_mr_size; 82 attr->page_size_cap = 0xffff000; 83 attr->vendor_id = dev->nic_info.pdev->vendor; 84 attr->vendor_part_id = dev->nic_info.pdev->device; 85 attr->hw_ver = dev->asic_id; 86 attr->max_qp = dev->attr.max_qp; 87 attr->max_ah = OCRDMA_MAX_AH; 88 attr->max_qp_wr = dev->attr.max_wqe; 89 90 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 91 IB_DEVICE_RC_RNR_NAK_GEN | 92 IB_DEVICE_SHUTDOWN_PORT | 93 IB_DEVICE_SYS_IMAGE_GUID | 94 IB_DEVICE_MEM_MGT_EXTENSIONS; 95 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 96 attr->max_send_sge = dev->attr.max_send_sge; 97 attr->max_recv_sge = dev->attr.max_recv_sge; 98 attr->max_sge_rd = dev->attr.max_rdma_sge; 99 attr->max_cq = dev->attr.max_cq; 100 attr->max_cqe = dev->attr.max_cqe; 101 attr->max_mr = dev->attr.max_mr; 102 attr->max_mw = dev->attr.max_mw; 103 attr->max_pd = dev->attr.max_pd; 104 attr->atomic_cap = 0; 105 attr->max_qp_rd_atom = 106 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 107 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 108 attr->max_srq = dev->attr.max_srq; 109 attr->max_srq_sge = dev->attr.max_srq_sge; 110 attr->max_srq_wr = dev->attr.max_rqe; 111 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 112 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; 113 attr->max_pkeys = 1; 114 return 0; 115 } 116 117 static inline void get_link_speed_and_width(struct ocrdma_dev *dev, 118 u16 *ib_speed, u8 *ib_width) 119 { 120 int status; 121 u8 speed; 122 123 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL); 124 if (status) 125 speed = OCRDMA_PHYS_LINK_SPEED_ZERO; 126 127 switch (speed) { 128 case OCRDMA_PHYS_LINK_SPEED_1GBPS: 129 *ib_speed = IB_SPEED_SDR; 130 *ib_width = IB_WIDTH_1X; 131 break; 132 133 case OCRDMA_PHYS_LINK_SPEED_10GBPS: 134 *ib_speed = IB_SPEED_QDR; 135 *ib_width = IB_WIDTH_1X; 136 break; 137 138 case OCRDMA_PHYS_LINK_SPEED_20GBPS: 139 *ib_speed = IB_SPEED_DDR; 140 *ib_width = IB_WIDTH_4X; 141 break; 142 143 case OCRDMA_PHYS_LINK_SPEED_40GBPS: 144 *ib_speed = IB_SPEED_QDR; 145 *ib_width = IB_WIDTH_4X; 146 break; 147 148 default: 149 /* Unsupported */ 150 *ib_speed = IB_SPEED_SDR; 151 *ib_width = IB_WIDTH_1X; 152 } 153 } 154 155 int ocrdma_query_port(struct ib_device *ibdev, 156 u32 port, struct ib_port_attr *props) 157 { 158 enum ib_port_state port_state; 159 struct ocrdma_dev *dev; 160 struct net_device *netdev; 161 162 /* props being zeroed by the caller, avoid zeroing it here */ 163 dev = get_ocrdma_dev(ibdev); 164 netdev = dev->nic_info.netdev; 165 if (netif_running(netdev) && netif_oper_up(netdev)) { 166 port_state = IB_PORT_ACTIVE; 167 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 168 } else { 169 port_state = IB_PORT_DOWN; 170 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 171 } 172 props->max_mtu = IB_MTU_4096; 173 props->active_mtu = iboe_get_mtu(netdev->mtu); 174 props->lid = 0; 175 props->lmc = 0; 176 props->sm_lid = 0; 177 props->sm_sl = 0; 178 props->state = port_state; 179 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 180 IB_PORT_DEVICE_MGMT_SUP | 181 IB_PORT_VENDOR_CLASS_SUP; 182 props->ip_gids = true; 183 props->gid_tbl_len = OCRDMA_MAX_SGID; 184 props->pkey_tbl_len = 1; 185 props->bad_pkey_cntr = 0; 186 props->qkey_viol_cntr = 0; 187 get_link_speed_and_width(dev, &props->active_speed, 188 &props->active_width); 189 props->max_msg_sz = 0x80000000; 190 props->max_vl_num = 4; 191 return 0; 192 } 193 194 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 195 unsigned long len) 196 { 197 struct ocrdma_mm *mm; 198 199 mm = kzalloc_obj(*mm); 200 if (mm == NULL) 201 return -ENOMEM; 202 mm->key.phy_addr = phy_addr; 203 mm->key.len = len; 204 INIT_LIST_HEAD(&mm->entry); 205 206 mutex_lock(&uctx->mm_list_lock); 207 list_add_tail(&mm->entry, &uctx->mm_head); 208 mutex_unlock(&uctx->mm_list_lock); 209 return 0; 210 } 211 212 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 213 unsigned long len) 214 { 215 struct ocrdma_mm *mm, *tmp; 216 217 mutex_lock(&uctx->mm_list_lock); 218 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 219 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 220 continue; 221 222 list_del(&mm->entry); 223 kfree(mm); 224 break; 225 } 226 mutex_unlock(&uctx->mm_list_lock); 227 } 228 229 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 230 unsigned long len) 231 { 232 bool found = false; 233 struct ocrdma_mm *mm; 234 235 mutex_lock(&uctx->mm_list_lock); 236 list_for_each_entry(mm, &uctx->mm_head, entry) { 237 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 238 continue; 239 240 found = true; 241 break; 242 } 243 mutex_unlock(&uctx->mm_list_lock); 244 return found; 245 } 246 247 248 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) 249 { 250 u16 pd_bitmap_idx = 0; 251 unsigned long *pd_bitmap; 252 253 if (dpp_pool) { 254 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; 255 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 256 dev->pd_mgr->max_dpp_pd); 257 __set_bit(pd_bitmap_idx, pd_bitmap); 258 dev->pd_mgr->pd_dpp_count++; 259 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) 260 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; 261 } else { 262 pd_bitmap = dev->pd_mgr->pd_norm_bitmap; 263 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 264 dev->pd_mgr->max_normal_pd); 265 __set_bit(pd_bitmap_idx, pd_bitmap); 266 dev->pd_mgr->pd_norm_count++; 267 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) 268 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; 269 } 270 return pd_bitmap_idx; 271 } 272 273 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, 274 bool dpp_pool) 275 { 276 u16 pd_count; 277 u16 pd_bit_index; 278 279 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count : 280 dev->pd_mgr->pd_norm_count; 281 if (pd_count == 0) 282 return -EINVAL; 283 284 if (dpp_pool) { 285 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start; 286 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) { 287 return -EINVAL; 288 } else { 289 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap); 290 dev->pd_mgr->pd_dpp_count--; 291 } 292 } else { 293 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start; 294 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) { 295 return -EINVAL; 296 } else { 297 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap); 298 dev->pd_mgr->pd_norm_count--; 299 } 300 } 301 302 return 0; 303 } 304 305 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, 306 bool dpp_pool) 307 { 308 int status; 309 310 mutex_lock(&dev->dev_lock); 311 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool); 312 mutex_unlock(&dev->dev_lock); 313 return status; 314 } 315 316 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 317 { 318 u16 pd_idx = 0; 319 int status = 0; 320 321 mutex_lock(&dev->dev_lock); 322 if (pd->dpp_enabled) { 323 /* try allocating DPP PD, if not available then normal PD */ 324 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) { 325 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true); 326 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx; 327 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx; 328 } else if (dev->pd_mgr->pd_norm_count < 329 dev->pd_mgr->max_normal_pd) { 330 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 331 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 332 pd->dpp_enabled = false; 333 } else { 334 status = -EINVAL; 335 } 336 } else { 337 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) { 338 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 339 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 340 } else { 341 status = -EINVAL; 342 } 343 } 344 mutex_unlock(&dev->dev_lock); 345 return status; 346 } 347 348 /* 349 * NOTE: 350 * 351 * ocrdma_ucontext must be used here because this function is also 352 * called from ocrdma_alloc_ucontext where ib_udata does not have 353 * valid ib_ucontext pointer. ib_uverbs_get_context does not call 354 * uobj_{alloc|get_xxx} helpers which are used to store the 355 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so 356 * ib_udata does NOT imply valid ib_ucontext here! 357 */ 358 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 359 struct ocrdma_ucontext *uctx, 360 struct ib_udata *udata) 361 { 362 int status; 363 364 if (udata && uctx && dev->attr.max_dpp_pds) { 365 pd->dpp_enabled = 366 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R; 367 pd->num_dpp_qp = 368 pd->dpp_enabled ? (dev->nic_info.db_page_size / 369 dev->attr.wqe_size) : 0; 370 } 371 372 if (dev->pd_mgr->pd_prealloc_valid) 373 return ocrdma_get_pd_num(dev, pd); 374 375 retry: 376 status = ocrdma_mbx_alloc_pd(dev, pd); 377 if (status) { 378 if (pd->dpp_enabled) { 379 pd->dpp_enabled = false; 380 pd->num_dpp_qp = 0; 381 goto retry; 382 } 383 return status; 384 } 385 386 return 0; 387 } 388 389 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, 390 struct ocrdma_pd *pd) 391 { 392 return (uctx->cntxt_pd == pd); 393 } 394 395 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev, 396 struct ocrdma_pd *pd) 397 { 398 if (dev->pd_mgr->pd_prealloc_valid) 399 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled); 400 else 401 ocrdma_mbx_dealloc_pd(dev, pd); 402 } 403 404 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, 405 struct ocrdma_ucontext *uctx, 406 struct ib_udata *udata) 407 { 408 struct ib_device *ibdev = &dev->ibdev; 409 struct ib_pd *pd; 410 int status; 411 412 pd = rdma_zalloc_drv_obj(ibdev, ib_pd); 413 if (!pd) 414 return -ENOMEM; 415 416 pd->device = ibdev; 417 uctx->cntxt_pd = get_ocrdma_pd(pd); 418 419 status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata); 420 if (status) { 421 kfree(uctx->cntxt_pd); 422 goto err; 423 } 424 425 uctx->cntxt_pd->uctx = uctx; 426 uctx->cntxt_pd->ibpd.device = &dev->ibdev; 427 err: 428 return status; 429 } 430 431 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) 432 { 433 struct ocrdma_pd *pd = uctx->cntxt_pd; 434 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 435 436 if (uctx->pd_in_use) { 437 pr_err("%s(%d) Freeing in use pdid=0x%x.\n", 438 __func__, dev->id, pd->id); 439 } 440 uctx->cntxt_pd = NULL; 441 _ocrdma_dealloc_pd(dev, pd); 442 kfree(pd); 443 } 444 445 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) 446 { 447 struct ocrdma_pd *pd = NULL; 448 449 mutex_lock(&uctx->mm_list_lock); 450 if (!uctx->pd_in_use) { 451 uctx->pd_in_use = true; 452 pd = uctx->cntxt_pd; 453 } 454 mutex_unlock(&uctx->mm_list_lock); 455 456 return pd; 457 } 458 459 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) 460 { 461 mutex_lock(&uctx->mm_list_lock); 462 uctx->pd_in_use = false; 463 mutex_unlock(&uctx->mm_list_lock); 464 } 465 466 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 467 { 468 struct ib_device *ibdev = uctx->device; 469 int status; 470 struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx); 471 struct ocrdma_alloc_ucontext_resp resp = {}; 472 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 473 struct pci_dev *pdev = dev->nic_info.pdev; 474 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE); 475 476 if (!udata) 477 return -EFAULT; 478 INIT_LIST_HEAD(&ctx->mm_head); 479 mutex_init(&ctx->mm_list_lock); 480 481 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len, 482 &ctx->ah_tbl.pa, GFP_KERNEL); 483 if (!ctx->ah_tbl.va) 484 return -ENOMEM; 485 486 ctx->ah_tbl.len = map_len; 487 488 resp.ah_tbl_len = ctx->ah_tbl.len; 489 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 490 491 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 492 if (status) 493 goto map_err; 494 495 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata); 496 if (status) 497 goto pd_err; 498 499 resp.dev_id = dev->id; 500 resp.max_inline_data = dev->attr.max_inline_data; 501 resp.wqe_size = dev->attr.wqe_size; 502 resp.rqe_size = dev->attr.rqe_size; 503 resp.dpp_wqe_size = dev->attr.wqe_size; 504 505 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); 506 status = ib_respond_udata(udata, resp); 507 if (status) 508 goto cpy_err; 509 return 0; 510 511 cpy_err: 512 ocrdma_dealloc_ucontext_pd(ctx); 513 pd_err: 514 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 515 map_err: 516 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 517 ctx->ah_tbl.pa); 518 return status; 519 } 520 521 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 522 { 523 struct ocrdma_mm *mm, *tmp; 524 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 525 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device); 526 struct pci_dev *pdev = dev->nic_info.pdev; 527 528 ocrdma_dealloc_ucontext_pd(uctx); 529 530 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 531 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 532 uctx->ah_tbl.pa); 533 534 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 535 list_del(&mm->entry); 536 kfree(mm); 537 } 538 } 539 540 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 541 { 542 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 543 struct ocrdma_dev *dev = get_ocrdma_dev(context->device); 544 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 545 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 546 unsigned long len = (vma->vm_end - vma->vm_start); 547 int status; 548 bool found; 549 550 if (vma->vm_start & (PAGE_SIZE - 1)) 551 return -EINVAL; 552 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len); 553 if (!found) 554 return -EINVAL; 555 556 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 557 dev->nic_info.db_total_size)) && 558 (len <= dev->nic_info.db_page_size)) { 559 if (vma->vm_flags & VM_READ) 560 return -EPERM; 561 562 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 563 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 564 len, vma->vm_page_prot); 565 } else if (dev->nic_info.dpp_unmapped_len && 566 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) && 567 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 568 dev->nic_info.dpp_unmapped_len)) && 569 (len <= dev->nic_info.dpp_unmapped_len)) { 570 if (vma->vm_flags & VM_READ) 571 return -EPERM; 572 573 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 574 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 575 len, vma->vm_page_prot); 576 } else { 577 status = remap_pfn_range(vma, vma->vm_start, 578 vma->vm_pgoff, len, vma->vm_page_prot); 579 } 580 return status; 581 } 582 583 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 584 struct ib_udata *udata) 585 { 586 int status; 587 u64 db_page_addr; 588 u64 dpp_page_addr = 0; 589 u32 db_page_size; 590 struct ocrdma_alloc_pd_uresp rsp = {}; 591 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 592 udata, struct ocrdma_ucontext, ibucontext); 593 594 rsp.id = pd->id; 595 rsp.dpp_enabled = pd->dpp_enabled; 596 db_page_addr = ocrdma_get_db_addr(dev, pd->id); 597 db_page_size = dev->nic_info.db_page_size; 598 599 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 600 if (status) 601 return status; 602 603 if (pd->dpp_enabled) { 604 dpp_page_addr = dev->nic_info.dpp_unmapped_addr + 605 (pd->id * PAGE_SIZE); 606 status = ocrdma_add_mmap(uctx, dpp_page_addr, 607 PAGE_SIZE); 608 if (status) 609 goto dpp_map_err; 610 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 611 rsp.dpp_page_addr_lo = dpp_page_addr; 612 } 613 614 status = ib_respond_udata(udata, rsp); 615 if (status) 616 goto ucopy_err; 617 618 pd->uctx = uctx; 619 return 0; 620 621 ucopy_err: 622 if (pd->dpp_enabled) 623 ocrdma_del_mmap(uctx, dpp_page_addr, PAGE_SIZE); 624 dpp_map_err: 625 ocrdma_del_mmap(uctx, db_page_addr, db_page_size); 626 return status; 627 } 628 629 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 630 { 631 struct ib_device *ibdev = ibpd->device; 632 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 633 struct ocrdma_pd *pd; 634 int status; 635 u8 is_uctx_pd = false; 636 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 637 udata, struct ocrdma_ucontext, ibucontext); 638 639 if (udata) { 640 pd = ocrdma_get_ucontext_pd(uctx); 641 if (pd) { 642 is_uctx_pd = true; 643 goto pd_mapping; 644 } 645 } 646 647 pd = get_ocrdma_pd(ibpd); 648 status = _ocrdma_alloc_pd(dev, pd, uctx, udata); 649 if (status) 650 goto exit; 651 652 pd_mapping: 653 if (udata) { 654 status = ocrdma_copy_pd_uresp(dev, pd, udata); 655 if (status) 656 goto err; 657 } 658 return 0; 659 660 err: 661 if (is_uctx_pd) 662 ocrdma_release_ucontext_pd(uctx); 663 else 664 _ocrdma_dealloc_pd(dev, pd); 665 exit: 666 return status; 667 } 668 669 int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 670 { 671 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 672 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 673 struct ocrdma_ucontext *uctx = NULL; 674 u64 usr_db; 675 676 uctx = pd->uctx; 677 if (uctx) { 678 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 679 (pd->id * PAGE_SIZE); 680 if (pd->dpp_enabled) 681 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE); 682 usr_db = ocrdma_get_db_addr(dev, pd->id); 683 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 684 685 if (is_ucontext_pd(uctx, pd)) { 686 ocrdma_release_ucontext_pd(uctx); 687 return 0; 688 } 689 } 690 _ocrdma_dealloc_pd(dev, pd); 691 return 0; 692 } 693 694 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 695 u32 pdid, int acc, u32 num_pbls, u32 addr_check) 696 { 697 int status; 698 699 mr->hwmr.fr_mr = 0; 700 mr->hwmr.local_rd = 1; 701 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 702 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 703 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 704 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0; 705 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 706 mr->hwmr.num_pbls = num_pbls; 707 708 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check); 709 if (status) 710 return status; 711 712 mr->ibmr.lkey = mr->hwmr.lkey; 713 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 714 mr->ibmr.rkey = mr->hwmr.lkey; 715 return 0; 716 } 717 718 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 719 { 720 int status; 721 struct ocrdma_mr *mr; 722 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 723 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 724 725 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 726 pr_err("%s err, invalid access rights\n", __func__); 727 return ERR_PTR(-EINVAL); 728 } 729 730 mr = kzalloc_obj(*mr); 731 if (!mr) 732 return ERR_PTR(-ENOMEM); 733 734 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0, 735 OCRDMA_ADDR_CHECK_DISABLE); 736 if (status) { 737 kfree(mr); 738 return ERR_PTR(status); 739 } 740 741 return &mr->ibmr; 742 } 743 744 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, 745 struct ocrdma_hw_mr *mr) 746 { 747 struct pci_dev *pdev = dev->nic_info.pdev; 748 int i = 0; 749 750 if (mr->pbl_table) { 751 for (i = 0; i < mr->num_pbls; i++) { 752 if (!mr->pbl_table[i].va) 753 continue; 754 dma_free_coherent(&pdev->dev, mr->pbl_size, 755 mr->pbl_table[i].va, 756 mr->pbl_table[i].pa); 757 } 758 kfree(mr->pbl_table); 759 mr->pbl_table = NULL; 760 } 761 } 762 763 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 764 u32 num_pbes) 765 { 766 u32 num_pbls = 0; 767 u32 idx = 0; 768 int status = 0; 769 u32 pbl_size; 770 771 do { 772 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx); 773 if (pbl_size > MAX_OCRDMA_PBL_SIZE) { 774 status = -EFAULT; 775 break; 776 } 777 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 778 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 779 idx++; 780 } while (num_pbls >= dev->attr.max_num_mr_pbl); 781 782 mr->hwmr.num_pbes = num_pbes; 783 mr->hwmr.num_pbls = num_pbls; 784 mr->hwmr.pbl_size = pbl_size; 785 return status; 786 } 787 788 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) 789 { 790 int status = 0; 791 int i; 792 u32 dma_len = mr->pbl_size; 793 struct pci_dev *pdev = dev->nic_info.pdev; 794 void *va; 795 dma_addr_t pa; 796 797 mr->pbl_table = kzalloc_objs(*mr->pbl_table, mr->num_pbls); 798 799 if (!mr->pbl_table) 800 return -ENOMEM; 801 802 for (i = 0; i < mr->num_pbls; i++) { 803 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL); 804 if (!va) { 805 ocrdma_free_mr_pbl_tbl(dev, mr); 806 status = -ENOMEM; 807 break; 808 } 809 mr->pbl_table[i].va = va; 810 mr->pbl_table[i].pa = pa; 811 } 812 return status; 813 } 814 815 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr) 816 { 817 struct ocrdma_pbe *pbe; 818 struct ib_block_iter biter; 819 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 820 int pbe_cnt; 821 u64 pg_addr; 822 823 if (!mr->hwmr.num_pbes) 824 return; 825 826 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 827 pbe_cnt = 0; 828 829 rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) { 830 /* store the page address in pbe */ 831 pg_addr = rdma_block_iter_dma_address(&biter); 832 pbe->pa_lo = cpu_to_le32(pg_addr); 833 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr)); 834 pbe_cnt += 1; 835 pbe++; 836 837 /* if the given pbl is full storing the pbes, 838 * move to next pbl. 839 */ 840 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) { 841 pbl_tbl++; 842 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 843 pbe_cnt = 0; 844 } 845 } 846 } 847 848 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 849 u64 usr_addr, int acc, struct ib_dmah *dmah, 850 struct ib_udata *udata) 851 { 852 int status = -ENOMEM; 853 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 854 struct ocrdma_mr *mr; 855 struct ocrdma_pd *pd; 856 857 if (dmah) 858 return ERR_PTR(-EOPNOTSUPP); 859 860 pd = get_ocrdma_pd(ibpd); 861 862 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 863 return ERR_PTR(-EINVAL); 864 865 mr = kzalloc_obj(*mr); 866 if (!mr) 867 return ERR_PTR(status); 868 mr->umem = ib_umem_get_va(ibpd->device, start, len, acc); 869 if (IS_ERR(mr->umem)) { 870 status = -EFAULT; 871 goto umem_err; 872 } 873 status = ocrdma_get_pbl_info( 874 dev, mr, ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE)); 875 if (status) 876 goto umem_err; 877 878 mr->hwmr.pbe_size = PAGE_SIZE; 879 mr->hwmr.va = usr_addr; 880 mr->hwmr.len = len; 881 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 882 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 883 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 884 mr->hwmr.local_rd = 1; 885 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 886 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 887 if (status) 888 goto umem_err; 889 build_user_pbes(dev, mr); 890 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 891 if (status) 892 goto mbx_err; 893 mr->ibmr.lkey = mr->hwmr.lkey; 894 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 895 mr->ibmr.rkey = mr->hwmr.lkey; 896 897 return &mr->ibmr; 898 899 mbx_err: 900 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 901 umem_err: 902 kfree(mr); 903 return ERR_PTR(status); 904 } 905 906 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 907 { 908 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 909 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device); 910 911 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 912 913 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 914 915 /* it could be user registered memory. */ 916 ib_umem_release(mr->umem); 917 kfree(mr); 918 919 /* Don't stop cleanup, in case FW is unresponsive */ 920 if (dev->mqe_ctx.fw_error_state) { 921 pr_err("%s(%d) fw not responding.\n", 922 __func__, dev->id); 923 } 924 return 0; 925 } 926 927 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 928 struct ib_udata *udata) 929 { 930 int status; 931 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 932 udata, struct ocrdma_ucontext, ibucontext); 933 struct ocrdma_create_cq_uresp uresp = {}; 934 935 /* this must be user flow! */ 936 if (!udata) 937 return -EINVAL; 938 939 uresp.cq_id = cq->id; 940 uresp.page_size = PAGE_ALIGN(cq->len); 941 uresp.num_pages = 1; 942 uresp.max_hw_cqe = cq->max_hw_cqe; 943 uresp.page_addr[0] = virt_to_phys(cq->va); 944 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 945 uresp.db_page_size = dev->nic_info.db_page_size; 946 uresp.phase_change = cq->phase_change ? 1 : 0; 947 status = ib_respond_udata(udata, uresp); 948 if (status) 949 goto err; 950 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 951 if (status) 952 goto err; 953 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size); 954 if (status) { 955 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 956 goto err; 957 } 958 cq->ucontext = uctx; 959 err: 960 return status; 961 } 962 963 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 964 struct uverbs_attr_bundle *attrs) 965 { 966 struct ib_udata *udata = &attrs->driver_udata; 967 struct ib_device *ibdev = ibcq->device; 968 int entries = attr->cqe; 969 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 970 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 971 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 972 udata, struct ocrdma_ucontext, ibucontext); 973 u16 pd_id = 0; 974 int status; 975 struct ocrdma_create_cq_ureq ureq; 976 977 if (attr->flags) 978 return -EOPNOTSUPP; 979 980 if (udata) { 981 status = ib_copy_validate_udata_in(udata, ureq, rsvd); 982 if (status) 983 return status; 984 } else 985 ureq.dpp_cq = 0; 986 987 spin_lock_init(&cq->cq_lock); 988 spin_lock_init(&cq->comp_handler_lock); 989 INIT_LIST_HEAD(&cq->sq_head); 990 INIT_LIST_HEAD(&cq->rq_head); 991 992 if (udata) 993 pd_id = uctx->cntxt_pd->id; 994 995 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); 996 if (status) 997 return status; 998 999 if (udata) { 1000 status = ocrdma_copy_cq_uresp(dev, cq, udata); 1001 if (status) 1002 goto ctx_err; 1003 } 1004 cq->phase = OCRDMA_CQE_VALID; 1005 dev->cq_tbl[cq->id] = cq; 1006 return 0; 1007 1008 ctx_err: 1009 ocrdma_mbx_destroy_cq(dev, cq); 1010 return status; 1011 } 1012 1013 int ocrdma_resize_cq(struct ib_cq *ibcq, unsigned int new_cnt, 1014 struct ib_udata *udata) 1015 { 1016 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1017 1018 if (new_cnt > cq->max_hw_cqe) 1019 return -EINVAL; 1020 1021 ibcq->cqe = new_cnt; 1022 return 0; 1023 } 1024 1025 static void ocrdma_flush_cq(struct ocrdma_cq *cq) 1026 { 1027 int cqe_cnt; 1028 int valid_count = 0; 1029 unsigned long flags; 1030 1031 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 1032 struct ocrdma_cqe *cqe = NULL; 1033 1034 cqe = cq->va; 1035 cqe_cnt = cq->cqe_cnt; 1036 1037 /* Last irq might have scheduled a polling thread 1038 * sync-up with it before hard flushing. 1039 */ 1040 spin_lock_irqsave(&cq->cq_lock, flags); 1041 while (cqe_cnt) { 1042 if (is_cqe_valid(cq, cqe)) 1043 valid_count++; 1044 cqe++; 1045 cqe_cnt--; 1046 } 1047 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count); 1048 spin_unlock_irqrestore(&cq->cq_lock, flags); 1049 } 1050 1051 int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1052 { 1053 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1054 struct ocrdma_eq *eq = NULL; 1055 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 1056 int pdid = 0; 1057 u32 irq, indx; 1058 1059 dev->cq_tbl[cq->id] = NULL; 1060 indx = ocrdma_get_eq_table_index(dev, cq->eqn); 1061 1062 eq = &dev->eq_tbl[indx]; 1063 irq = ocrdma_get_irq(dev, eq); 1064 synchronize_irq(irq); 1065 ocrdma_flush_cq(cq); 1066 1067 ocrdma_mbx_destroy_cq(dev, cq); 1068 if (cq->ucontext) { 1069 pdid = cq->ucontext->cntxt_pd->id; 1070 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, 1071 PAGE_ALIGN(cq->len)); 1072 ocrdma_del_mmap(cq->ucontext, 1073 ocrdma_get_db_addr(dev, pdid), 1074 dev->nic_info.db_page_size); 1075 } 1076 return 0; 1077 } 1078 1079 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1080 { 1081 int status = -EINVAL; 1082 1083 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { 1084 dev->qp_tbl[qp->id] = qp; 1085 status = 0; 1086 } 1087 return status; 1088 } 1089 1090 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1091 { 1092 dev->qp_tbl[qp->id] = NULL; 1093 } 1094 1095 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 1096 struct ib_qp_init_attr *attrs, 1097 struct ib_udata *udata) 1098 { 1099 if ((attrs->qp_type != IB_QPT_GSI) && 1100 (attrs->qp_type != IB_QPT_RC) && 1101 (attrs->qp_type != IB_QPT_UC) && 1102 (attrs->qp_type != IB_QPT_UD)) { 1103 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1104 __func__, dev->id, attrs->qp_type); 1105 return -EOPNOTSUPP; 1106 } 1107 /* Skip the check for QP1 to support CM size of 128 */ 1108 if ((attrs->qp_type != IB_QPT_GSI) && 1109 (attrs->cap.max_send_wr > dev->attr.max_wqe)) { 1110 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1111 __func__, dev->id, attrs->cap.max_send_wr); 1112 pr_err("%s(%d) supported send_wr=0x%x\n", 1113 __func__, dev->id, dev->attr.max_wqe); 1114 return -EINVAL; 1115 } 1116 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) { 1117 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n", 1118 __func__, dev->id, attrs->cap.max_recv_wr); 1119 pr_err("%s(%d) supported recv_wr=0x%x\n", 1120 __func__, dev->id, dev->attr.max_rqe); 1121 return -EINVAL; 1122 } 1123 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) { 1124 pr_err("%s(%d) unsupported inline data size=0x%x requested\n", 1125 __func__, dev->id, attrs->cap.max_inline_data); 1126 pr_err("%s(%d) supported inline data size=0x%x\n", 1127 __func__, dev->id, dev->attr.max_inline_data); 1128 return -EINVAL; 1129 } 1130 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) { 1131 pr_err("%s(%d) unsupported send_sge=0x%x requested\n", 1132 __func__, dev->id, attrs->cap.max_send_sge); 1133 pr_err("%s(%d) supported send_sge=0x%x\n", 1134 __func__, dev->id, dev->attr.max_send_sge); 1135 return -EINVAL; 1136 } 1137 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) { 1138 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n", 1139 __func__, dev->id, attrs->cap.max_recv_sge); 1140 pr_err("%s(%d) supported recv_sge=0x%x\n", 1141 __func__, dev->id, dev->attr.max_recv_sge); 1142 return -EINVAL; 1143 } 1144 /* unprivileged user space cannot create special QP */ 1145 if (udata && attrs->qp_type == IB_QPT_GSI) { 1146 pr_err 1147 ("%s(%d) Userspace can't create special QPs of type=0x%x\n", 1148 __func__, dev->id, attrs->qp_type); 1149 return -EINVAL; 1150 } 1151 /* allow creating only one GSI type of QP */ 1152 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 1153 pr_err("%s(%d) GSI special QPs already created.\n", 1154 __func__, dev->id); 1155 return -EINVAL; 1156 } 1157 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1158 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1159 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1160 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) { 1161 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1162 __func__, dev->id); 1163 return -EINVAL; 1164 } 1165 } 1166 return 0; 1167 } 1168 1169 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, 1170 struct ib_udata *udata, int dpp_offset, 1171 int dpp_credit_lmt, int srq) 1172 { 1173 int status; 1174 u64 usr_db; 1175 struct ocrdma_create_qp_uresp uresp = {}; 1176 struct ocrdma_pd *pd = qp->pd; 1177 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 1178 1179 usr_db = dev->nic_info.unmapped_db + 1180 (pd->id * dev->nic_info.db_page_size); 1181 uresp.qp_id = qp->id; 1182 uresp.sq_dbid = qp->sq.dbid; 1183 uresp.num_sq_pages = 1; 1184 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1185 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1186 uresp.num_wqe_allocated = qp->sq.max_cnt; 1187 if (!srq) { 1188 uresp.rq_dbid = qp->rq.dbid; 1189 uresp.num_rq_pages = 1; 1190 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1191 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1192 uresp.num_rqe_allocated = qp->rq.max_cnt; 1193 } 1194 uresp.db_page_addr = usr_db; 1195 uresp.db_page_size = dev->nic_info.db_page_size; 1196 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1197 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1198 uresp.db_shift = OCRDMA_DB_RQ_SHIFT; 1199 1200 if (qp->dpp_enabled) { 1201 uresp.dpp_credit = dpp_credit_lmt; 1202 uresp.dpp_offset = dpp_offset; 1203 } 1204 status = ib_respond_udata(udata, uresp); 1205 if (status) 1206 goto err; 1207 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], 1208 uresp.sq_page_size); 1209 if (status) 1210 goto err; 1211 1212 if (!srq) { 1213 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0], 1214 uresp.rq_page_size); 1215 if (status) 1216 goto rq_map_err; 1217 } 1218 return status; 1219 rq_map_err: 1220 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); 1221 err: 1222 return status; 1223 } 1224 1225 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, 1226 struct ocrdma_pd *pd) 1227 { 1228 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1229 qp->sq_db = dev->nic_info.db + 1230 (pd->id * dev->nic_info.db_page_size) + 1231 OCRDMA_DB_GEN2_SQ_OFFSET; 1232 qp->rq_db = dev->nic_info.db + 1233 (pd->id * dev->nic_info.db_page_size) + 1234 OCRDMA_DB_GEN2_RQ_OFFSET; 1235 } else { 1236 qp->sq_db = dev->nic_info.db + 1237 (pd->id * dev->nic_info.db_page_size) + 1238 OCRDMA_DB_SQ_OFFSET; 1239 qp->rq_db = dev->nic_info.db + 1240 (pd->id * dev->nic_info.db_page_size) + 1241 OCRDMA_DB_RQ_OFFSET; 1242 } 1243 } 1244 1245 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) 1246 { 1247 qp->wqe_wr_id_tbl = kzalloc_objs(*qp->wqe_wr_id_tbl, qp->sq.max_cnt); 1248 if (qp->wqe_wr_id_tbl == NULL) 1249 return -ENOMEM; 1250 1251 qp->rqe_wr_id_tbl = kzalloc_objs(*qp->rqe_wr_id_tbl, qp->rq.max_cnt); 1252 if (qp->rqe_wr_id_tbl == NULL) 1253 return -ENOMEM; 1254 1255 return 0; 1256 } 1257 1258 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, 1259 struct ocrdma_pd *pd, 1260 struct ib_qp_init_attr *attrs) 1261 { 1262 qp->pd = pd; 1263 spin_lock_init(&qp->q_lock); 1264 INIT_LIST_HEAD(&qp->sq_entry); 1265 INIT_LIST_HEAD(&qp->rq_entry); 1266 1267 qp->qp_type = attrs->qp_type; 1268 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; 1269 qp->max_inline_data = attrs->cap.max_inline_data; 1270 qp->sq.max_sges = attrs->cap.max_send_sge; 1271 qp->rq.max_sges = attrs->cap.max_recv_sge; 1272 qp->state = OCRDMA_QPS_RST; 1273 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; 1274 } 1275 1276 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, 1277 struct ib_qp_init_attr *attrs) 1278 { 1279 if (attrs->qp_type == IB_QPT_GSI) { 1280 dev->gsi_qp_created = 1; 1281 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq); 1282 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq); 1283 } 1284 } 1285 1286 int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 1287 struct ib_udata *udata) 1288 { 1289 int status; 1290 struct ib_pd *ibpd = ibqp->pd; 1291 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1292 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1293 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1294 struct ocrdma_create_qp_ureq ureq; 1295 u16 dpp_credit_lmt, dpp_offset; 1296 1297 if (attrs->create_flags) 1298 return -EOPNOTSUPP; 1299 1300 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata); 1301 if (status) 1302 goto gen_err; 1303 1304 if (udata) { 1305 status = ib_copy_validate_udata_in(udata, ureq, rsvd1); 1306 if (status) 1307 return status; 1308 } else { 1309 memset(&ureq, 0, sizeof(ureq)); 1310 } 1311 1312 ocrdma_set_qp_init_params(qp, pd, attrs); 1313 if (udata == NULL) 1314 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | 1315 OCRDMA_QP_FAST_REG); 1316 1317 mutex_lock(&dev->dev_lock); 1318 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1319 ureq.dpp_cq_id, 1320 &dpp_offset, &dpp_credit_lmt); 1321 if (status) 1322 goto mbx_err; 1323 1324 /* user space QP's wr_id table are managed in library */ 1325 if (udata == NULL) { 1326 status = ocrdma_alloc_wr_id_tbl(qp); 1327 if (status) 1328 goto map_err; 1329 } 1330 1331 status = ocrdma_add_qpn_map(dev, qp); 1332 if (status) 1333 goto map_err; 1334 ocrdma_set_qp_db(dev, qp, pd); 1335 if (udata) { 1336 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, 1337 dpp_credit_lmt, 1338 (attrs->srq != NULL)); 1339 if (status) 1340 goto cpy_err; 1341 } 1342 ocrdma_store_gsi_qp_cq(dev, attrs); 1343 qp->ibqp.qp_num = qp->id; 1344 mutex_unlock(&dev->dev_lock); 1345 return 0; 1346 1347 cpy_err: 1348 ocrdma_del_qpn_map(dev, qp); 1349 map_err: 1350 ocrdma_mbx_destroy_qp(dev, qp); 1351 mbx_err: 1352 mutex_unlock(&dev->dev_lock); 1353 kfree(qp->wqe_wr_id_tbl); 1354 kfree(qp->rqe_wr_id_tbl); 1355 pr_err("%s(%d) error=%d\n", __func__, dev->id, status); 1356 gen_err: 1357 return status; 1358 } 1359 1360 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1361 int attr_mask) 1362 { 1363 int status = 0; 1364 struct ocrdma_qp *qp; 1365 struct ocrdma_dev *dev; 1366 enum ib_qp_state old_qps; 1367 1368 qp = get_ocrdma_qp(ibqp); 1369 dev = get_ocrdma_dev(ibqp->device); 1370 if (attr_mask & IB_QP_STATE) 1371 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); 1372 /* if new and previous states are same hw doesn't need to 1373 * know about it. 1374 */ 1375 if (status < 0) 1376 return status; 1377 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); 1378 } 1379 1380 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1381 int attr_mask, struct ib_udata *udata) 1382 { 1383 unsigned long flags; 1384 int status = -EINVAL; 1385 struct ocrdma_qp *qp; 1386 struct ocrdma_dev *dev; 1387 enum ib_qp_state old_qps, new_qps; 1388 1389 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1390 return -EOPNOTSUPP; 1391 1392 qp = get_ocrdma_qp(ibqp); 1393 dev = get_ocrdma_dev(ibqp->device); 1394 1395 /* syncronize with multiple context trying to change, retrive qps */ 1396 mutex_lock(&dev->dev_lock); 1397 /* syncronize with wqe, rqe posting and cqe processing contexts */ 1398 spin_lock_irqsave(&qp->q_lock, flags); 1399 old_qps = get_ibqp_state(qp->state); 1400 if (attr_mask & IB_QP_STATE) 1401 new_qps = attr->qp_state; 1402 else 1403 new_qps = old_qps; 1404 spin_unlock_irqrestore(&qp->q_lock, flags); 1405 1406 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) { 1407 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n" 1408 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n", 1409 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, 1410 old_qps, new_qps); 1411 goto param_err; 1412 } 1413 1414 status = _ocrdma_modify_qp(ibqp, attr, attr_mask); 1415 if (status > 0) 1416 status = 0; 1417 param_err: 1418 mutex_unlock(&dev->dev_lock); 1419 return status; 1420 } 1421 1422 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) 1423 { 1424 switch (mtu) { 1425 case 256: 1426 return IB_MTU_256; 1427 case 512: 1428 return IB_MTU_512; 1429 case 1024: 1430 return IB_MTU_1024; 1431 case 2048: 1432 return IB_MTU_2048; 1433 case 4096: 1434 return IB_MTU_4096; 1435 default: 1436 return IB_MTU_1024; 1437 } 1438 } 1439 1440 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) 1441 { 1442 int ib_qp_acc_flags = 0; 1443 1444 if (qp_cap_flags & OCRDMA_QP_INB_WR) 1445 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1446 if (qp_cap_flags & OCRDMA_QP_INB_RD) 1447 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 1448 return ib_qp_acc_flags; 1449 } 1450 1451 int ocrdma_query_qp(struct ib_qp *ibqp, 1452 struct ib_qp_attr *qp_attr, 1453 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 1454 { 1455 int status; 1456 u32 qp_state; 1457 struct ocrdma_qp_params params; 1458 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1459 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1460 1461 memset(¶ms, 0, sizeof(params)); 1462 mutex_lock(&dev->dev_lock); 1463 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); 1464 mutex_unlock(&dev->dev_lock); 1465 if (status) 1466 goto mbx_err; 1467 if (qp->qp_type == IB_QPT_UD) 1468 qp_attr->qkey = params.qkey; 1469 qp_attr->path_mtu = 1470 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx & 1471 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >> 1472 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT; 1473 qp_attr->path_mig_state = IB_MIG_MIGRATED; 1474 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK; 1475 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK; 1476 qp_attr->dest_qp_num = 1477 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK; 1478 1479 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); 1480 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; 1481 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1482 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1483 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1484 qp_attr->cap.max_inline_data = qp->max_inline_data; 1485 qp_init_attr->cap = qp_attr->cap; 1486 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1487 1488 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 1489 params.rnt_rc_sl_fl & 1490 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK, 1491 qp->sgid_idx, 1492 (params.hop_lmt_rq_psn & 1493 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >> 1494 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT, 1495 (params.tclass_sq_psn & 1496 OCRDMA_QP_PARAMS_TCLASS_MASK) >> 1497 OCRDMA_QP_PARAMS_TCLASS_SHIFT); 1498 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]); 1499 1500 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 1501 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl & 1502 OCRDMA_QP_PARAMS_SL_MASK) >> 1503 OCRDMA_QP_PARAMS_SL_SHIFT); 1504 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn & 1505 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >> 1506 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT; 1507 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn & 1508 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >> 1509 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT; 1510 qp_attr->retry_cnt = 1511 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >> 1512 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT; 1513 qp_attr->min_rnr_timer = 0; 1514 qp_attr->pkey_index = 0; 1515 qp_attr->port_num = 1; 1516 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 1517 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 1518 qp_attr->alt_pkey_index = 0; 1519 qp_attr->alt_port_num = 0; 1520 qp_attr->alt_timeout = 0; 1521 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 1522 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >> 1523 OCRDMA_QP_PARAMS_STATE_SHIFT; 1524 qp_attr->qp_state = get_ibqp_state(qp_state); 1525 qp_attr->cur_qp_state = qp_attr->qp_state; 1526 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0; 1527 qp_attr->max_dest_rd_atomic = 1528 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT; 1529 qp_attr->max_rd_atomic = 1530 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK; 1531 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags & 1532 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0; 1533 /* Sync driver QP state with FW */ 1534 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); 1535 mbx_err: 1536 return status; 1537 } 1538 1539 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) 1540 { 1541 unsigned int i = idx / 32; 1542 u32 mask = (1U << (idx % 32)); 1543 1544 srq->idx_bit_fields[i] ^= mask; 1545 } 1546 1547 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1548 { 1549 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt; 1550 } 1551 1552 static int is_hw_sq_empty(struct ocrdma_qp *qp) 1553 { 1554 return (qp->sq.tail == qp->sq.head); 1555 } 1556 1557 static int is_hw_rq_empty(struct ocrdma_qp *qp) 1558 { 1559 return (qp->rq.tail == qp->rq.head); 1560 } 1561 1562 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1563 { 1564 return q->va + (q->head * q->entry_size); 1565 } 1566 1567 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, 1568 u32 idx) 1569 { 1570 return q->va + (idx * q->entry_size); 1571 } 1572 1573 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) 1574 { 1575 q->head = (q->head + 1) & q->max_wqe_idx; 1576 } 1577 1578 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) 1579 { 1580 q->tail = (q->tail + 1) & q->max_wqe_idx; 1581 } 1582 1583 /* discard the cqe for a given QP */ 1584 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) 1585 { 1586 unsigned long cq_flags; 1587 unsigned long flags; 1588 u32 cur_getp, stop_getp; 1589 struct ocrdma_cqe *cqe; 1590 u32 qpn = 0, wqe_idx = 0; 1591 1592 spin_lock_irqsave(&cq->cq_lock, cq_flags); 1593 1594 /* traverse through the CQEs in the hw CQ, 1595 * find the matching CQE for a given qp, 1596 * mark the matching one discarded by clearing qpn. 1597 * ring the doorbell in the poll_cq() as 1598 * we don't complete out of order cqe. 1599 */ 1600 1601 cur_getp = cq->getp; 1602 /* find upto when do we reap the cq. */ 1603 stop_getp = cur_getp; 1604 do { 1605 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) 1606 break; 1607 1608 cqe = cq->va + cur_getp; 1609 /* if (a) done reaping whole hw cq, or 1610 * (b) qp_xq becomes empty. 1611 * then exit 1612 */ 1613 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK; 1614 /* if previously discarded cqe found, skip that too. */ 1615 /* check for matching qp */ 1616 if (qpn == 0 || qpn != qp->id) 1617 goto skip_cqe; 1618 1619 if (is_cqe_for_sq(cqe)) { 1620 ocrdma_hwq_inc_tail(&qp->sq); 1621 } else { 1622 if (qp->srq) { 1623 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 1624 OCRDMA_CQE_BUFTAG_SHIFT) & 1625 qp->srq->rq.max_wqe_idx; 1626 BUG_ON(wqe_idx < 1); 1627 spin_lock_irqsave(&qp->srq->q_lock, flags); 1628 ocrdma_hwq_inc_tail(&qp->srq->rq); 1629 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); 1630 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1631 1632 } else { 1633 ocrdma_hwq_inc_tail(&qp->rq); 1634 } 1635 } 1636 /* mark cqe discarded so that it is not picked up later 1637 * in the poll_cq(). 1638 */ 1639 cqe->cmn.qpn = 0; 1640 skip_cqe: 1641 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1642 } while (cur_getp != stop_getp); 1643 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1644 } 1645 1646 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1647 { 1648 int found = false; 1649 unsigned long flags; 1650 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 1651 /* sync with any active CQ poll */ 1652 1653 spin_lock_irqsave(&dev->flush_q_lock, flags); 1654 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); 1655 if (found) 1656 list_del(&qp->sq_entry); 1657 if (!qp->srq) { 1658 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); 1659 if (found) 1660 list_del(&qp->rq_entry); 1661 } 1662 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 1663 } 1664 1665 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1666 { 1667 struct ocrdma_pd *pd; 1668 struct ocrdma_qp *qp; 1669 struct ocrdma_dev *dev; 1670 struct ib_qp_attr attrs; 1671 int attr_mask; 1672 unsigned long flags; 1673 1674 qp = get_ocrdma_qp(ibqp); 1675 dev = get_ocrdma_dev(ibqp->device); 1676 1677 pd = qp->pd; 1678 1679 /* change the QP state to ERROR */ 1680 if (qp->state != OCRDMA_QPS_RST) { 1681 attrs.qp_state = IB_QPS_ERR; 1682 attr_mask = IB_QP_STATE; 1683 _ocrdma_modify_qp(ibqp, &attrs, attr_mask); 1684 } 1685 /* ensure that CQEs for newly created QP (whose id may be same with 1686 * one which just getting destroyed are same), dont get 1687 * discarded until the old CQEs are discarded. 1688 */ 1689 mutex_lock(&dev->dev_lock); 1690 (void) ocrdma_mbx_destroy_qp(dev, qp); 1691 1692 /* 1693 * acquire CQ lock while destroy is in progress, in order to 1694 * protect against proessing in-flight CQEs for this QP. 1695 */ 1696 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); 1697 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) { 1698 spin_lock(&qp->rq_cq->cq_lock); 1699 ocrdma_del_qpn_map(dev, qp); 1700 spin_unlock(&qp->rq_cq->cq_lock); 1701 } else { 1702 ocrdma_del_qpn_map(dev, qp); 1703 } 1704 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); 1705 1706 if (!pd->uctx) { 1707 ocrdma_discard_cqes(qp, qp->sq_cq); 1708 ocrdma_discard_cqes(qp, qp->rq_cq); 1709 } 1710 mutex_unlock(&dev->dev_lock); 1711 1712 if (pd->uctx) { 1713 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, 1714 PAGE_ALIGN(qp->sq.len)); 1715 if (!qp->srq) 1716 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, 1717 PAGE_ALIGN(qp->rq.len)); 1718 } 1719 1720 ocrdma_del_flush_qp(qp); 1721 1722 kfree(qp->wqe_wr_id_tbl); 1723 kfree(qp->rqe_wr_id_tbl); 1724 return 0; 1725 } 1726 1727 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, 1728 struct ib_udata *udata) 1729 { 1730 int status; 1731 struct ocrdma_create_srq_uresp uresp = {}; 1732 1733 uresp.rq_dbid = srq->rq.dbid; 1734 uresp.num_rq_pages = 1; 1735 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1736 uresp.rq_page_size = srq->rq.len; 1737 uresp.db_page_addr = dev->nic_info.unmapped_db + 1738 (srq->pd->id * dev->nic_info.db_page_size); 1739 uresp.db_page_size = dev->nic_info.db_page_size; 1740 uresp.num_rqe_allocated = srq->rq.max_cnt; 1741 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1742 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1743 uresp.db_shift = 24; 1744 } else { 1745 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1746 uresp.db_shift = 16; 1747 } 1748 1749 status = ib_respond_udata(udata, uresp); 1750 if (status) 1751 return status; 1752 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], 1753 uresp.rq_page_size); 1754 if (status) 1755 return status; 1756 return status; 1757 } 1758 1759 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1760 struct ib_udata *udata) 1761 { 1762 int status; 1763 struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd); 1764 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1765 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq); 1766 1767 if (init_attr->srq_type != IB_SRQT_BASIC) 1768 return -EOPNOTSUPP; 1769 1770 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1771 return -EINVAL; 1772 if (init_attr->attr.max_wr > dev->attr.max_rqe) 1773 return -EINVAL; 1774 1775 spin_lock_init(&srq->q_lock); 1776 srq->pd = pd; 1777 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1778 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd); 1779 if (status) 1780 return status; 1781 1782 if (!udata) { 1783 srq->rqe_wr_id_tbl = 1784 kzalloc_objs(*srq->rqe_wr_id_tbl, srq->rq.max_cnt); 1785 if (!srq->rqe_wr_id_tbl) { 1786 status = -ENOMEM; 1787 goto arm_err; 1788 } 1789 1790 srq->bit_fields_len = (srq->rq.max_cnt / 32) + 1791 (srq->rq.max_cnt % 32 ? 1 : 0); 1792 srq->idx_bit_fields = 1793 kmalloc_array(srq->bit_fields_len, sizeof(u32), 1794 GFP_KERNEL); 1795 if (!srq->idx_bit_fields) { 1796 status = -ENOMEM; 1797 goto arm_err; 1798 } 1799 memset(srq->idx_bit_fields, 0xff, 1800 srq->bit_fields_len * sizeof(u32)); 1801 } 1802 1803 if (init_attr->attr.srq_limit) { 1804 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr); 1805 if (status) 1806 goto arm_err; 1807 } 1808 1809 if (udata) { 1810 status = ocrdma_copy_srq_uresp(dev, srq, udata); 1811 if (status) 1812 goto arm_err; 1813 } 1814 1815 return 0; 1816 1817 arm_err: 1818 ocrdma_mbx_destroy_srq(dev, srq); 1819 kfree(srq->rqe_wr_id_tbl); 1820 kfree(srq->idx_bit_fields); 1821 return status; 1822 } 1823 1824 int ocrdma_modify_srq(struct ib_srq *ibsrq, 1825 struct ib_srq_attr *srq_attr, 1826 enum ib_srq_attr_mask srq_attr_mask, 1827 struct ib_udata *udata) 1828 { 1829 int status; 1830 struct ocrdma_srq *srq; 1831 1832 srq = get_ocrdma_srq(ibsrq); 1833 if (srq_attr_mask & IB_SRQ_MAX_WR) 1834 status = -EINVAL; 1835 else 1836 status = ocrdma_mbx_modify_srq(srq, srq_attr); 1837 return status; 1838 } 1839 1840 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 1841 { 1842 struct ocrdma_srq *srq; 1843 1844 srq = get_ocrdma_srq(ibsrq); 1845 return ocrdma_mbx_query_srq(srq, srq_attr); 1846 } 1847 1848 int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1849 { 1850 struct ocrdma_srq *srq; 1851 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1852 1853 srq = get_ocrdma_srq(ibsrq); 1854 1855 ocrdma_mbx_destroy_srq(dev, srq); 1856 1857 if (srq->pd->uctx) 1858 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, 1859 PAGE_ALIGN(srq->rq.len)); 1860 1861 kfree(srq->idx_bit_fields); 1862 kfree(srq->rqe_wr_id_tbl); 1863 return 0; 1864 } 1865 1866 /* unprivileged verbs and their support functions. */ 1867 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, 1868 struct ocrdma_hdr_wqe *hdr, 1869 const struct ib_send_wr *wr) 1870 { 1871 struct ocrdma_ewqe_ud_hdr *ud_hdr = 1872 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1); 1873 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah); 1874 1875 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn; 1876 if (qp->qp_type == IB_QPT_GSI) 1877 ud_hdr->qkey = qp->qkey; 1878 else 1879 ud_hdr->qkey = ud_wr(wr)->remote_qkey; 1880 ud_hdr->rsvd_ahid = ah->id; 1881 ud_hdr->hdr_type = ah->hdr_type; 1882 if (ah->av->valid & OCRDMA_AV_VLAN_VALID) 1883 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT); 1884 } 1885 1886 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, 1887 struct ocrdma_sge *sge, int num_sge, 1888 struct ib_sge *sg_list) 1889 { 1890 int i; 1891 1892 for (i = 0; i < num_sge; i++) { 1893 sge[i].lrkey = sg_list[i].lkey; 1894 sge[i].addr_lo = sg_list[i].addr; 1895 sge[i].addr_hi = upper_32_bits(sg_list[i].addr); 1896 sge[i].len = sg_list[i].length; 1897 hdr->total_len += sg_list[i].length; 1898 } 1899 if (num_sge == 0) 1900 memset(sge, 0, sizeof(*sge)); 1901 } 1902 1903 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) 1904 { 1905 uint32_t total_len = 0, i; 1906 1907 for (i = 0; i < num_sge; i++) 1908 total_len += sg_list[i].length; 1909 return total_len; 1910 } 1911 1912 1913 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 1914 struct ocrdma_hdr_wqe *hdr, 1915 struct ocrdma_sge *sge, 1916 const struct ib_send_wr *wr, u32 wqe_size) 1917 { 1918 int i; 1919 char *dpp_addr; 1920 1921 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { 1922 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge); 1923 if (unlikely(hdr->total_len > qp->max_inline_data)) { 1924 pr_err("%s() supported_len=0x%x,\n" 1925 " unsupported len req=0x%x\n", __func__, 1926 qp->max_inline_data, hdr->total_len); 1927 return -EINVAL; 1928 } 1929 dpp_addr = (char *)sge; 1930 for (i = 0; i < wr->num_sge; i++) { 1931 memcpy(dpp_addr, 1932 (void *)(unsigned long)wr->sg_list[i].addr, 1933 wr->sg_list[i].length); 1934 dpp_addr += wr->sg_list[i].length; 1935 } 1936 1937 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 1938 if (0 == hdr->total_len) 1939 wqe_size += sizeof(struct ocrdma_sge); 1940 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 1941 } else { 1942 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1943 if (wr->num_sge) 1944 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge)); 1945 else 1946 wqe_size += sizeof(struct ocrdma_sge); 1947 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 1948 } 1949 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1950 return 0; 1951 } 1952 1953 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1954 const struct ib_send_wr *wr) 1955 { 1956 struct ocrdma_sge *sge; 1957 u32 wqe_size = sizeof(*hdr); 1958 1959 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 1960 ocrdma_build_ud_hdr(qp, hdr, wr); 1961 sge = (struct ocrdma_sge *)(hdr + 2); 1962 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 1963 } else { 1964 sge = (struct ocrdma_sge *)(hdr + 1); 1965 } 1966 1967 return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1968 } 1969 1970 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1971 const struct ib_send_wr *wr) 1972 { 1973 int status; 1974 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1975 struct ocrdma_sge *sge = ext_rw + 1; 1976 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw); 1977 1978 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1979 if (status) 1980 return status; 1981 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 1982 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 1983 ext_rw->lrkey = rdma_wr(wr)->rkey; 1984 ext_rw->len = hdr->total_len; 1985 return 0; 1986 } 1987 1988 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1989 const struct ib_send_wr *wr) 1990 { 1991 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1992 struct ocrdma_sge *sge = ext_rw + 1; 1993 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) + 1994 sizeof(struct ocrdma_hdr_wqe); 1995 1996 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1997 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1998 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT); 1999 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2000 2001 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2002 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2003 ext_rw->lrkey = rdma_wr(wr)->rkey; 2004 ext_rw->len = hdr->total_len; 2005 } 2006 2007 static int get_encoded_page_size(int pg_sz) 2008 { 2009 /* Max size is 256M 4096 << 16 */ 2010 int i = 0; 2011 for (; i < 17; i++) 2012 if (pg_sz == (4096 << i)) 2013 break; 2014 return i; 2015 } 2016 2017 static int ocrdma_build_reg(struct ocrdma_qp *qp, 2018 struct ocrdma_hdr_wqe *hdr, 2019 const struct ib_reg_wr *wr) 2020 { 2021 u64 fbo; 2022 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1); 2023 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr); 2024 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 2025 struct ocrdma_pbe *pbe; 2026 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr); 2027 int num_pbes = 0, i; 2028 2029 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES); 2030 2031 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT); 2032 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2033 2034 if (wr->access & IB_ACCESS_LOCAL_WRITE) 2035 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR; 2036 if (wr->access & IB_ACCESS_REMOTE_WRITE) 2037 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR; 2038 if (wr->access & IB_ACCESS_REMOTE_READ) 2039 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD; 2040 hdr->lkey = wr->key; 2041 hdr->total_len = mr->ibmr.length; 2042 2043 fbo = mr->ibmr.iova - mr->pages[0]; 2044 2045 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova); 2046 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff); 2047 fast_reg->fbo_hi = upper_32_bits(fbo); 2048 fast_reg->fbo_lo = (u32) fbo & 0xffffffff; 2049 fast_reg->num_sges = mr->npages; 2050 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size); 2051 2052 pbe = pbl_tbl->va; 2053 for (i = 0; i < mr->npages; i++) { 2054 u64 buf_addr = mr->pages[i]; 2055 2056 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK)); 2057 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr)); 2058 num_pbes += 1; 2059 pbe++; 2060 2061 /* if the pbl is full storing the pbes, 2062 * move to next pbl. 2063 */ 2064 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) { 2065 pbl_tbl++; 2066 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 2067 } 2068 } 2069 2070 return 0; 2071 } 2072 2073 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2074 { 2075 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); 2076 2077 iowrite32(val, qp->sq_db); 2078 } 2079 2080 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 2081 const struct ib_send_wr **bad_wr) 2082 { 2083 int status = 0; 2084 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2085 struct ocrdma_hdr_wqe *hdr; 2086 unsigned long flags; 2087 2088 spin_lock_irqsave(&qp->q_lock, flags); 2089 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { 2090 spin_unlock_irqrestore(&qp->q_lock, flags); 2091 *bad_wr = wr; 2092 return -EINVAL; 2093 } 2094 2095 while (wr) { 2096 if (qp->qp_type == IB_QPT_UD && 2097 (wr->opcode != IB_WR_SEND && 2098 wr->opcode != IB_WR_SEND_WITH_IMM)) { 2099 *bad_wr = wr; 2100 status = -EINVAL; 2101 break; 2102 } 2103 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || 2104 wr->num_sge > qp->sq.max_sges) { 2105 *bad_wr = wr; 2106 status = -ENOMEM; 2107 break; 2108 } 2109 hdr = ocrdma_hwq_head(&qp->sq); 2110 hdr->cw = 0; 2111 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2112 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2113 if (wr->send_flags & IB_SEND_FENCE) 2114 hdr->cw |= 2115 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT); 2116 if (wr->send_flags & IB_SEND_SOLICITED) 2117 hdr->cw |= 2118 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT); 2119 hdr->total_len = 0; 2120 switch (wr->opcode) { 2121 case IB_WR_SEND_WITH_IMM: 2122 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2123 hdr->immdt = ntohl(wr->ex.imm_data); 2124 fallthrough; 2125 case IB_WR_SEND: 2126 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2127 ocrdma_build_send(qp, hdr, wr); 2128 break; 2129 case IB_WR_SEND_WITH_INV: 2130 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT); 2131 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2132 hdr->lkey = wr->ex.invalidate_rkey; 2133 status = ocrdma_build_send(qp, hdr, wr); 2134 break; 2135 case IB_WR_RDMA_WRITE_WITH_IMM: 2136 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2137 hdr->immdt = ntohl(wr->ex.imm_data); 2138 fallthrough; 2139 case IB_WR_RDMA_WRITE: 2140 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT); 2141 status = ocrdma_build_write(qp, hdr, wr); 2142 break; 2143 case IB_WR_RDMA_READ: 2144 ocrdma_build_read(qp, hdr, wr); 2145 break; 2146 case IB_WR_LOCAL_INV: 2147 hdr->cw |= 2148 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2149 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) + 2150 sizeof(struct ocrdma_sge)) / 2151 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2152 hdr->lkey = wr->ex.invalidate_rkey; 2153 break; 2154 case IB_WR_REG_MR: 2155 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); 2156 break; 2157 default: 2158 status = -EINVAL; 2159 break; 2160 } 2161 if (status) { 2162 *bad_wr = wr; 2163 break; 2164 } 2165 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2166 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2167 else 2168 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2169 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; 2170 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) & 2171 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE); 2172 /* make sure wqe is written before adapter can access it */ 2173 wmb(); 2174 /* inform hw to start processing it */ 2175 ocrdma_ring_sq_db(qp); 2176 2177 /* update pointer, counter for next wr */ 2178 ocrdma_hwq_inc_head(&qp->sq); 2179 wr = wr->next; 2180 } 2181 spin_unlock_irqrestore(&qp->q_lock, flags); 2182 return status; 2183 } 2184 2185 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) 2186 { 2187 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); 2188 2189 iowrite32(val, qp->rq_db); 2190 } 2191 2192 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, 2193 const struct ib_recv_wr *wr, u16 tag) 2194 { 2195 u32 wqe_size = 0; 2196 struct ocrdma_sge *sge; 2197 if (wr->num_sge) 2198 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe); 2199 else 2200 wqe_size = sizeof(*sge) + sizeof(*rqe); 2201 2202 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << 2203 OCRDMA_WQE_SIZE_SHIFT); 2204 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2205 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2206 rqe->total_len = 0; 2207 rqe->rsvd_tag = tag; 2208 sge = (struct ocrdma_sge *)(rqe + 1); 2209 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list); 2210 ocrdma_cpu_to_le32(rqe, wqe_size); 2211 } 2212 2213 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 2214 const struct ib_recv_wr **bad_wr) 2215 { 2216 int status = 0; 2217 unsigned long flags; 2218 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2219 struct ocrdma_hdr_wqe *rqe; 2220 2221 spin_lock_irqsave(&qp->q_lock, flags); 2222 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { 2223 spin_unlock_irqrestore(&qp->q_lock, flags); 2224 *bad_wr = wr; 2225 return -EINVAL; 2226 } 2227 while (wr) { 2228 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || 2229 wr->num_sge > qp->rq.max_sges) { 2230 *bad_wr = wr; 2231 status = -ENOMEM; 2232 break; 2233 } 2234 rqe = ocrdma_hwq_head(&qp->rq); 2235 ocrdma_build_rqe(rqe, wr, 0); 2236 2237 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; 2238 /* make sure rqe is written before adapter can access it */ 2239 wmb(); 2240 2241 /* inform hw to start processing it */ 2242 ocrdma_ring_rq_db(qp); 2243 2244 /* update pointer, counter for next wr */ 2245 ocrdma_hwq_inc_head(&qp->rq); 2246 wr = wr->next; 2247 } 2248 spin_unlock_irqrestore(&qp->q_lock, flags); 2249 return status; 2250 } 2251 2252 /* cqe for srq's rqe can potentially arrive out of order. 2253 * index gives the entry in the shadow table where to store 2254 * the wr_id. tag/index is returned in cqe to reference back 2255 * for a given rqe. 2256 */ 2257 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) 2258 { 2259 int row = 0; 2260 int indx = 0; 2261 2262 for (row = 0; row < srq->bit_fields_len; row++) { 2263 if (srq->idx_bit_fields[row]) { 2264 indx = ffs(srq->idx_bit_fields[row]); 2265 indx = (row * 32) + (indx - 1); 2266 BUG_ON(indx >= srq->rq.max_cnt); 2267 ocrdma_srq_toggle_bit(srq, indx); 2268 break; 2269 } 2270 } 2271 2272 BUG_ON(row == srq->bit_fields_len); 2273 return indx + 1; /* Use from index 1 */ 2274 } 2275 2276 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) 2277 { 2278 u32 val = srq->rq.dbid | (1 << 16); 2279 2280 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET); 2281 } 2282 2283 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 2284 const struct ib_recv_wr **bad_wr) 2285 { 2286 int status = 0; 2287 unsigned long flags; 2288 struct ocrdma_srq *srq; 2289 struct ocrdma_hdr_wqe *rqe; 2290 u16 tag; 2291 2292 srq = get_ocrdma_srq(ibsrq); 2293 2294 spin_lock_irqsave(&srq->q_lock, flags); 2295 while (wr) { 2296 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 || 2297 wr->num_sge > srq->rq.max_sges) { 2298 status = -ENOMEM; 2299 *bad_wr = wr; 2300 break; 2301 } 2302 tag = ocrdma_srq_get_idx(srq); 2303 rqe = ocrdma_hwq_head(&srq->rq); 2304 ocrdma_build_rqe(rqe, wr, tag); 2305 2306 srq->rqe_wr_id_tbl[tag] = wr->wr_id; 2307 /* make sure rqe is written before adapter can perform DMA */ 2308 wmb(); 2309 /* inform hw to start processing it */ 2310 ocrdma_ring_srq_db(srq); 2311 /* update pointer, counter for next wr */ 2312 ocrdma_hwq_inc_head(&srq->rq); 2313 wr = wr->next; 2314 } 2315 spin_unlock_irqrestore(&srq->q_lock, flags); 2316 return status; 2317 } 2318 2319 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2320 { 2321 enum ib_wc_status ibwc_status; 2322 2323 switch (status) { 2324 case OCRDMA_CQE_GENERAL_ERR: 2325 ibwc_status = IB_WC_GENERAL_ERR; 2326 break; 2327 case OCRDMA_CQE_LOC_LEN_ERR: 2328 ibwc_status = IB_WC_LOC_LEN_ERR; 2329 break; 2330 case OCRDMA_CQE_LOC_QP_OP_ERR: 2331 ibwc_status = IB_WC_LOC_QP_OP_ERR; 2332 break; 2333 case OCRDMA_CQE_LOC_EEC_OP_ERR: 2334 ibwc_status = IB_WC_LOC_EEC_OP_ERR; 2335 break; 2336 case OCRDMA_CQE_LOC_PROT_ERR: 2337 ibwc_status = IB_WC_LOC_PROT_ERR; 2338 break; 2339 case OCRDMA_CQE_WR_FLUSH_ERR: 2340 ibwc_status = IB_WC_WR_FLUSH_ERR; 2341 break; 2342 case OCRDMA_CQE_MW_BIND_ERR: 2343 ibwc_status = IB_WC_MW_BIND_ERR; 2344 break; 2345 case OCRDMA_CQE_BAD_RESP_ERR: 2346 ibwc_status = IB_WC_BAD_RESP_ERR; 2347 break; 2348 case OCRDMA_CQE_LOC_ACCESS_ERR: 2349 ibwc_status = IB_WC_LOC_ACCESS_ERR; 2350 break; 2351 case OCRDMA_CQE_REM_INV_REQ_ERR: 2352 ibwc_status = IB_WC_REM_INV_REQ_ERR; 2353 break; 2354 case OCRDMA_CQE_REM_ACCESS_ERR: 2355 ibwc_status = IB_WC_REM_ACCESS_ERR; 2356 break; 2357 case OCRDMA_CQE_REM_OP_ERR: 2358 ibwc_status = IB_WC_REM_OP_ERR; 2359 break; 2360 case OCRDMA_CQE_RETRY_EXC_ERR: 2361 ibwc_status = IB_WC_RETRY_EXC_ERR; 2362 break; 2363 case OCRDMA_CQE_RNR_RETRY_EXC_ERR: 2364 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR; 2365 break; 2366 case OCRDMA_CQE_LOC_RDD_VIOL_ERR: 2367 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR; 2368 break; 2369 case OCRDMA_CQE_REM_INV_RD_REQ_ERR: 2370 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR; 2371 break; 2372 case OCRDMA_CQE_REM_ABORT_ERR: 2373 ibwc_status = IB_WC_REM_ABORT_ERR; 2374 break; 2375 case OCRDMA_CQE_INV_EECN_ERR: 2376 ibwc_status = IB_WC_INV_EECN_ERR; 2377 break; 2378 case OCRDMA_CQE_INV_EEC_STATE_ERR: 2379 ibwc_status = IB_WC_INV_EEC_STATE_ERR; 2380 break; 2381 case OCRDMA_CQE_FATAL_ERR: 2382 ibwc_status = IB_WC_FATAL_ERR; 2383 break; 2384 case OCRDMA_CQE_RESP_TIMEOUT_ERR: 2385 ibwc_status = IB_WC_RESP_TIMEOUT_ERR; 2386 break; 2387 default: 2388 ibwc_status = IB_WC_GENERAL_ERR; 2389 break; 2390 } 2391 return ibwc_status; 2392 } 2393 2394 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, 2395 u32 wqe_idx) 2396 { 2397 struct ocrdma_hdr_wqe *hdr; 2398 struct ocrdma_sge *rw; 2399 int opcode; 2400 2401 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); 2402 2403 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; 2404 /* Undo the hdr->cw swap */ 2405 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK; 2406 switch (opcode) { 2407 case OCRDMA_WRITE: 2408 ibwc->opcode = IB_WC_RDMA_WRITE; 2409 break; 2410 case OCRDMA_READ: 2411 rw = (struct ocrdma_sge *)(hdr + 1); 2412 ibwc->opcode = IB_WC_RDMA_READ; 2413 ibwc->byte_len = rw->len; 2414 break; 2415 case OCRDMA_SEND: 2416 ibwc->opcode = IB_WC_SEND; 2417 break; 2418 case OCRDMA_FR_MR: 2419 ibwc->opcode = IB_WC_REG_MR; 2420 break; 2421 case OCRDMA_LKEY_INV: 2422 ibwc->opcode = IB_WC_LOCAL_INV; 2423 break; 2424 default: 2425 ibwc->status = IB_WC_GENERAL_ERR; 2426 pr_err("%s() invalid opcode received = 0x%x\n", 2427 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK); 2428 break; 2429 } 2430 } 2431 2432 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, 2433 struct ocrdma_cqe *cqe) 2434 { 2435 if (is_cqe_for_sq(cqe)) { 2436 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2437 cqe->flags_status_srcqpn) & 2438 ~OCRDMA_CQE_STATUS_MASK); 2439 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2440 cqe->flags_status_srcqpn) | 2441 (OCRDMA_CQE_WR_FLUSH_ERR << 2442 OCRDMA_CQE_STATUS_SHIFT)); 2443 } else { 2444 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2445 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2446 cqe->flags_status_srcqpn) & 2447 ~OCRDMA_CQE_UD_STATUS_MASK); 2448 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2449 cqe->flags_status_srcqpn) | 2450 (OCRDMA_CQE_WR_FLUSH_ERR << 2451 OCRDMA_CQE_UD_STATUS_SHIFT)); 2452 } else { 2453 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2454 cqe->flags_status_srcqpn) & 2455 ~OCRDMA_CQE_STATUS_MASK); 2456 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2457 cqe->flags_status_srcqpn) | 2458 (OCRDMA_CQE_WR_FLUSH_ERR << 2459 OCRDMA_CQE_STATUS_SHIFT)); 2460 } 2461 } 2462 } 2463 2464 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2465 struct ocrdma_qp *qp, int status) 2466 { 2467 bool expand = false; 2468 2469 ibwc->byte_len = 0; 2470 ibwc->qp = &qp->ibqp; 2471 ibwc->status = ocrdma_to_ibwc_err(status); 2472 2473 ocrdma_flush_qp(qp); 2474 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); 2475 2476 /* if wqe/rqe pending for which cqe needs to be returned, 2477 * trigger inflating it. 2478 */ 2479 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { 2480 expand = true; 2481 ocrdma_set_cqe_status_flushed(qp, cqe); 2482 } 2483 return expand; 2484 } 2485 2486 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2487 struct ocrdma_qp *qp, int status) 2488 { 2489 ibwc->opcode = IB_WC_RECV; 2490 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2491 ocrdma_hwq_inc_tail(&qp->rq); 2492 2493 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2494 } 2495 2496 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2497 struct ocrdma_qp *qp, int status) 2498 { 2499 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2500 ocrdma_hwq_inc_tail(&qp->sq); 2501 2502 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2503 } 2504 2505 2506 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, 2507 struct ocrdma_cqe *cqe, struct ib_wc *ibwc, 2508 bool *polled, bool *stop) 2509 { 2510 bool expand; 2511 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2512 int status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2513 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2514 if (status < OCRDMA_MAX_CQE_ERR) 2515 atomic_inc(&dev->cqe_err_stats[status]); 2516 2517 /* when hw sq is empty, but rq is not empty, so we continue 2518 * to keep the cqe in order to get the cq event again. 2519 */ 2520 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { 2521 /* when cq for rq and sq is same, it is safe to return 2522 * flush cqe for RQEs. 2523 */ 2524 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2525 *polled = true; 2526 status = OCRDMA_CQE_WR_FLUSH_ERR; 2527 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2528 } else { 2529 /* stop processing further cqe as this cqe is used for 2530 * triggering cq event on buddy cq of RQ. 2531 * When QP is destroyed, this cqe will be removed 2532 * from the cq's hardware q. 2533 */ 2534 *polled = false; 2535 *stop = true; 2536 expand = false; 2537 } 2538 } else if (is_hw_sq_empty(qp)) { 2539 /* Do nothing */ 2540 expand = false; 2541 *polled = false; 2542 *stop = false; 2543 } else { 2544 *polled = true; 2545 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2546 } 2547 return expand; 2548 } 2549 2550 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, 2551 struct ocrdma_cqe *cqe, 2552 struct ib_wc *ibwc, bool *polled) 2553 { 2554 bool expand = false; 2555 int tail = qp->sq.tail; 2556 u32 wqe_idx; 2557 2558 if (!qp->wqe_wr_id_tbl[tail].signaled) { 2559 *polled = false; /* WC cannot be consumed yet */ 2560 } else { 2561 ibwc->status = IB_WC_SUCCESS; 2562 ibwc->wc_flags = 0; 2563 ibwc->qp = &qp->ibqp; 2564 ocrdma_update_wc(qp, ibwc, tail); 2565 *polled = true; 2566 } 2567 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) & 2568 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; 2569 if (tail != wqe_idx) 2570 expand = true; /* Coalesced CQE can't be consumed yet */ 2571 2572 ocrdma_hwq_inc_tail(&qp->sq); 2573 return expand; 2574 } 2575 2576 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2577 struct ib_wc *ibwc, bool *polled, bool *stop) 2578 { 2579 int status; 2580 bool expand; 2581 2582 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2583 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2584 2585 if (status == OCRDMA_CQE_SUCCESS) 2586 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); 2587 else 2588 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); 2589 return expand; 2590 } 2591 2592 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, 2593 struct ocrdma_cqe *cqe) 2594 { 2595 int status; 2596 u16 hdr_type = 0; 2597 2598 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2599 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT; 2600 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) & 2601 OCRDMA_CQE_SRCQP_MASK; 2602 ibwc->pkey_index = 0; 2603 ibwc->wc_flags = IB_WC_GRH; 2604 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2605 OCRDMA_CQE_UD_XFER_LEN_SHIFT) & 2606 OCRDMA_CQE_UD_XFER_LEN_MASK; 2607 2608 if (ocrdma_is_udp_encap_supported(dev)) { 2609 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2610 OCRDMA_CQE_UD_L3TYPE_SHIFT) & 2611 OCRDMA_CQE_UD_L3TYPE_MASK; 2612 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2613 ibwc->network_hdr_type = hdr_type; 2614 } 2615 2616 return status; 2617 } 2618 2619 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, 2620 struct ocrdma_cqe *cqe, 2621 struct ocrdma_qp *qp) 2622 { 2623 unsigned long flags; 2624 struct ocrdma_srq *srq; 2625 u32 wqe_idx; 2626 2627 srq = get_ocrdma_srq(qp->ibqp.srq); 2628 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 2629 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx; 2630 BUG_ON(wqe_idx < 1); 2631 2632 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2633 spin_lock_irqsave(&srq->q_lock, flags); 2634 ocrdma_srq_toggle_bit(srq, wqe_idx - 1); 2635 spin_unlock_irqrestore(&srq->q_lock, flags); 2636 ocrdma_hwq_inc_tail(&srq->rq); 2637 } 2638 2639 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2640 struct ib_wc *ibwc, bool *polled, bool *stop, 2641 int status) 2642 { 2643 bool expand; 2644 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2645 2646 if (status < OCRDMA_MAX_CQE_ERR) 2647 atomic_inc(&dev->cqe_err_stats[status]); 2648 2649 /* when hw_rq is empty, but wq is not empty, so continue 2650 * to keep the cqe to get the cq event again. 2651 */ 2652 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { 2653 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2654 *polled = true; 2655 status = OCRDMA_CQE_WR_FLUSH_ERR; 2656 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2657 } else { 2658 *polled = false; 2659 *stop = true; 2660 expand = false; 2661 } 2662 } else if (is_hw_rq_empty(qp)) { 2663 /* Do nothing */ 2664 expand = false; 2665 *polled = false; 2666 *stop = false; 2667 } else { 2668 *polled = true; 2669 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2670 } 2671 return expand; 2672 } 2673 2674 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, 2675 struct ocrdma_cqe *cqe, struct ib_wc *ibwc) 2676 { 2677 struct ocrdma_dev *dev; 2678 2679 dev = get_ocrdma_dev(qp->ibqp.device); 2680 ibwc->opcode = IB_WC_RECV; 2681 ibwc->qp = &qp->ibqp; 2682 ibwc->status = IB_WC_SUCCESS; 2683 2684 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2685 ocrdma_update_ud_rcqe(dev, ibwc, cqe); 2686 else 2687 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen); 2688 2689 if (is_cqe_imm(cqe)) { 2690 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2691 ibwc->wc_flags |= IB_WC_WITH_IMM; 2692 } else if (is_cqe_wr_imm(cqe)) { 2693 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2694 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2695 ibwc->wc_flags |= IB_WC_WITH_IMM; 2696 } else if (is_cqe_invalidated(cqe)) { 2697 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2698 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2699 } 2700 if (qp->ibqp.srq) { 2701 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2702 } else { 2703 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2704 ocrdma_hwq_inc_tail(&qp->rq); 2705 } 2706 } 2707 2708 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2709 struct ib_wc *ibwc, bool *polled, bool *stop) 2710 { 2711 int status; 2712 bool expand = false; 2713 2714 ibwc->wc_flags = 0; 2715 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2716 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2717 OCRDMA_CQE_UD_STATUS_MASK) >> 2718 OCRDMA_CQE_UD_STATUS_SHIFT; 2719 } else { 2720 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2721 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2722 } 2723 2724 if (status == OCRDMA_CQE_SUCCESS) { 2725 *polled = true; 2726 ocrdma_poll_success_rcqe(qp, cqe, ibwc); 2727 } else { 2728 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, 2729 status); 2730 } 2731 return expand; 2732 } 2733 2734 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, 2735 u16 cur_getp) 2736 { 2737 if (cq->phase_change) { 2738 if (cur_getp == 0) 2739 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2740 } else { 2741 /* clear valid bit */ 2742 cqe->flags_status_srcqpn = 0; 2743 } 2744 } 2745 2746 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2747 struct ib_wc *ibwc) 2748 { 2749 u16 qpn = 0; 2750 int i = 0; 2751 bool expand = false; 2752 int polled_hw_cqes = 0; 2753 struct ocrdma_qp *qp = NULL; 2754 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 2755 struct ocrdma_cqe *cqe; 2756 u16 cur_getp; bool polled = false; bool stop = false; 2757 2758 cur_getp = cq->getp; 2759 while (num_entries) { 2760 cqe = cq->va + cur_getp; 2761 /* check whether valid cqe or not */ 2762 if (!is_cqe_valid(cq, cqe)) 2763 break; 2764 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK); 2765 /* ignore discarded cqe */ 2766 if (qpn == 0) 2767 goto skip_cqe; 2768 qp = dev->qp_tbl[qpn]; 2769 BUG_ON(qp == NULL); 2770 2771 if (is_cqe_for_sq(cqe)) { 2772 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, 2773 &stop); 2774 } else { 2775 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, 2776 &stop); 2777 } 2778 if (expand) 2779 goto expand_cqe; 2780 if (stop) 2781 goto stop_cqe; 2782 /* clear qpn to avoid duplicate processing by discard_cqe() */ 2783 cqe->cmn.qpn = 0; 2784 skip_cqe: 2785 polled_hw_cqes += 1; 2786 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 2787 ocrdma_change_cq_phase(cq, cqe, cur_getp); 2788 expand_cqe: 2789 if (polled) { 2790 num_entries -= 1; 2791 i += 1; 2792 ibwc = ibwc + 1; 2793 polled = false; 2794 } 2795 } 2796 stop_cqe: 2797 cq->getp = cur_getp; 2798 2799 if (polled_hw_cqes) 2800 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes); 2801 2802 return i; 2803 } 2804 2805 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ 2806 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, 2807 struct ocrdma_qp *qp, struct ib_wc *ibwc) 2808 { 2809 int err_cqes = 0; 2810 2811 while (num_entries) { 2812 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) 2813 break; 2814 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { 2815 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2816 ocrdma_hwq_inc_tail(&qp->sq); 2817 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2818 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2819 ocrdma_hwq_inc_tail(&qp->rq); 2820 } else { 2821 return err_cqes; 2822 } 2823 ibwc->byte_len = 0; 2824 ibwc->status = IB_WC_WR_FLUSH_ERR; 2825 ibwc = ibwc + 1; 2826 err_cqes += 1; 2827 num_entries -= 1; 2828 } 2829 return err_cqes; 2830 } 2831 2832 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2833 { 2834 int cqes_to_poll = num_entries; 2835 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2836 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2837 int num_os_cqe = 0, err_cqes = 0; 2838 struct ocrdma_qp *qp; 2839 unsigned long flags; 2840 2841 /* poll cqes from adapter CQ */ 2842 spin_lock_irqsave(&cq->cq_lock, flags); 2843 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc); 2844 spin_unlock_irqrestore(&cq->cq_lock, flags); 2845 cqes_to_poll -= num_os_cqe; 2846 2847 if (cqes_to_poll) { 2848 wc = wc + num_os_cqe; 2849 /* adapter returns single error cqe when qp moves to 2850 * error state. So insert error cqes with wc_status as 2851 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ 2852 * respectively which uses this CQ. 2853 */ 2854 spin_lock_irqsave(&dev->flush_q_lock, flags); 2855 list_for_each_entry(qp, &cq->sq_head, sq_entry) { 2856 if (cqes_to_poll == 0) 2857 break; 2858 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); 2859 cqes_to_poll -= err_cqes; 2860 num_os_cqe += err_cqes; 2861 wc = wc + err_cqes; 2862 } 2863 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 2864 } 2865 return num_os_cqe; 2866 } 2867 2868 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2869 { 2870 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2871 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2872 u16 cq_id; 2873 unsigned long flags; 2874 bool arm_needed = false, sol_needed = false; 2875 2876 cq_id = cq->id; 2877 2878 spin_lock_irqsave(&cq->cq_lock, flags); 2879 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 2880 arm_needed = true; 2881 if (cq_flags & IB_CQ_SOLICITED) 2882 sol_needed = true; 2883 2884 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0); 2885 spin_unlock_irqrestore(&cq->cq_lock, flags); 2886 2887 return 0; 2888 } 2889 2890 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2891 u32 max_num_sg) 2892 { 2893 int status; 2894 struct ocrdma_mr *mr; 2895 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 2896 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 2897 2898 if (mr_type != IB_MR_TYPE_MEM_REG) 2899 return ERR_PTR(-EINVAL); 2900 2901 if (max_num_sg > dev->attr.max_pages_per_frmr) 2902 return ERR_PTR(-EINVAL); 2903 2904 mr = kzalloc_flex(*mr, pages, max_num_sg); 2905 if (!mr) 2906 return ERR_PTR(-ENOMEM); 2907 2908 status = ocrdma_get_pbl_info(dev, mr, max_num_sg); 2909 if (status) 2910 goto pl_err; 2911 mr->hwmr.fr_mr = 1; 2912 mr->hwmr.remote_rd = 0; 2913 mr->hwmr.remote_wr = 0; 2914 mr->hwmr.local_rd = 0; 2915 mr->hwmr.local_wr = 0; 2916 mr->hwmr.mw_bind = 0; 2917 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 2918 if (status) 2919 goto pl_err; 2920 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0); 2921 if (status) 2922 goto mbx_err; 2923 mr->ibmr.rkey = mr->hwmr.lkey; 2924 mr->ibmr.lkey = mr->hwmr.lkey; 2925 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = 2926 (unsigned long) mr; 2927 return &mr->ibmr; 2928 mbx_err: 2929 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 2930 pl_err: 2931 kfree(mr); 2932 return ERR_PTR(-ENOMEM); 2933 } 2934 2935 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) 2936 { 2937 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2938 2939 if (unlikely(mr->npages == mr->hwmr.num_pbes)) 2940 return -ENOMEM; 2941 2942 mr->pages[mr->npages++] = addr; 2943 2944 return 0; 2945 } 2946 2947 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2948 unsigned int *sg_offset) 2949 { 2950 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2951 2952 mr->npages = 0; 2953 2954 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page); 2955 } 2956