1 /* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43 #include <linux/dma-mapping.h> 44 #include <rdma/ib_verbs.h> 45 #include <rdma/ib_user_verbs.h> 46 #include <rdma/iw_cm.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_cache.h> 50 #include <rdma/uverbs_ioctl.h> 51 52 #include "ocrdma.h" 53 #include "ocrdma_hw.h" 54 #include "ocrdma_verbs.h" 55 #include <rdma/ocrdma-abi.h> 56 57 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 58 { 59 if (index > 0) 60 return -EINVAL; 61 62 *pkey = 0xffff; 63 return 0; 64 } 65 66 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 67 struct ib_udata *uhw) 68 { 69 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 70 71 if (uhw->inlen || uhw->outlen) 72 return -EINVAL; 73 74 memset(attr, 0, sizeof *attr); 75 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], 76 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); 77 ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid); 78 attr->max_mr_size = dev->attr.max_mr_size; 79 attr->page_size_cap = 0xffff000; 80 attr->vendor_id = dev->nic_info.pdev->vendor; 81 attr->vendor_part_id = dev->nic_info.pdev->device; 82 attr->hw_ver = dev->asic_id; 83 attr->max_qp = dev->attr.max_qp; 84 attr->max_ah = OCRDMA_MAX_AH; 85 attr->max_qp_wr = dev->attr.max_wqe; 86 87 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 88 IB_DEVICE_RC_RNR_NAK_GEN | 89 IB_DEVICE_SHUTDOWN_PORT | 90 IB_DEVICE_SYS_IMAGE_GUID | 91 IB_DEVICE_LOCAL_DMA_LKEY | 92 IB_DEVICE_MEM_MGT_EXTENSIONS; 93 attr->max_send_sge = dev->attr.max_send_sge; 94 attr->max_recv_sge = dev->attr.max_recv_sge; 95 attr->max_sge_rd = dev->attr.max_rdma_sge; 96 attr->max_cq = dev->attr.max_cq; 97 attr->max_cqe = dev->attr.max_cqe; 98 attr->max_mr = dev->attr.max_mr; 99 attr->max_mw = dev->attr.max_mw; 100 attr->max_pd = dev->attr.max_pd; 101 attr->atomic_cap = 0; 102 attr->max_qp_rd_atom = 103 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 104 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 105 attr->max_srq = dev->attr.max_srq; 106 attr->max_srq_sge = dev->attr.max_srq_sge; 107 attr->max_srq_wr = dev->attr.max_rqe; 108 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 109 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; 110 attr->max_pkeys = 1; 111 return 0; 112 } 113 114 static inline void get_link_speed_and_width(struct ocrdma_dev *dev, 115 u16 *ib_speed, u8 *ib_width) 116 { 117 int status; 118 u8 speed; 119 120 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL); 121 if (status) 122 speed = OCRDMA_PHYS_LINK_SPEED_ZERO; 123 124 switch (speed) { 125 case OCRDMA_PHYS_LINK_SPEED_1GBPS: 126 *ib_speed = IB_SPEED_SDR; 127 *ib_width = IB_WIDTH_1X; 128 break; 129 130 case OCRDMA_PHYS_LINK_SPEED_10GBPS: 131 *ib_speed = IB_SPEED_QDR; 132 *ib_width = IB_WIDTH_1X; 133 break; 134 135 case OCRDMA_PHYS_LINK_SPEED_20GBPS: 136 *ib_speed = IB_SPEED_DDR; 137 *ib_width = IB_WIDTH_4X; 138 break; 139 140 case OCRDMA_PHYS_LINK_SPEED_40GBPS: 141 *ib_speed = IB_SPEED_QDR; 142 *ib_width = IB_WIDTH_4X; 143 break; 144 145 default: 146 /* Unsupported */ 147 *ib_speed = IB_SPEED_SDR; 148 *ib_width = IB_WIDTH_1X; 149 } 150 } 151 152 int ocrdma_query_port(struct ib_device *ibdev, 153 u8 port, struct ib_port_attr *props) 154 { 155 enum ib_port_state port_state; 156 struct ocrdma_dev *dev; 157 struct net_device *netdev; 158 159 /* props being zeroed by the caller, avoid zeroing it here */ 160 dev = get_ocrdma_dev(ibdev); 161 netdev = dev->nic_info.netdev; 162 if (netif_running(netdev) && netif_oper_up(netdev)) { 163 port_state = IB_PORT_ACTIVE; 164 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 165 } else { 166 port_state = IB_PORT_DOWN; 167 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 168 } 169 props->max_mtu = IB_MTU_4096; 170 props->active_mtu = iboe_get_mtu(netdev->mtu); 171 props->lid = 0; 172 props->lmc = 0; 173 props->sm_lid = 0; 174 props->sm_sl = 0; 175 props->state = port_state; 176 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 177 IB_PORT_DEVICE_MGMT_SUP | 178 IB_PORT_VENDOR_CLASS_SUP; 179 props->ip_gids = true; 180 props->gid_tbl_len = OCRDMA_MAX_SGID; 181 props->pkey_tbl_len = 1; 182 props->bad_pkey_cntr = 0; 183 props->qkey_viol_cntr = 0; 184 get_link_speed_and_width(dev, &props->active_speed, 185 &props->active_width); 186 props->max_msg_sz = 0x80000000; 187 props->max_vl_num = 4; 188 return 0; 189 } 190 191 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 192 unsigned long len) 193 { 194 struct ocrdma_mm *mm; 195 196 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 197 if (mm == NULL) 198 return -ENOMEM; 199 mm->key.phy_addr = phy_addr; 200 mm->key.len = len; 201 INIT_LIST_HEAD(&mm->entry); 202 203 mutex_lock(&uctx->mm_list_lock); 204 list_add_tail(&mm->entry, &uctx->mm_head); 205 mutex_unlock(&uctx->mm_list_lock); 206 return 0; 207 } 208 209 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 210 unsigned long len) 211 { 212 struct ocrdma_mm *mm, *tmp; 213 214 mutex_lock(&uctx->mm_list_lock); 215 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 216 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 217 continue; 218 219 list_del(&mm->entry); 220 kfree(mm); 221 break; 222 } 223 mutex_unlock(&uctx->mm_list_lock); 224 } 225 226 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 227 unsigned long len) 228 { 229 bool found = false; 230 struct ocrdma_mm *mm; 231 232 mutex_lock(&uctx->mm_list_lock); 233 list_for_each_entry(mm, &uctx->mm_head, entry) { 234 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 235 continue; 236 237 found = true; 238 break; 239 } 240 mutex_unlock(&uctx->mm_list_lock); 241 return found; 242 } 243 244 245 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) 246 { 247 u16 pd_bitmap_idx = 0; 248 const unsigned long *pd_bitmap; 249 250 if (dpp_pool) { 251 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; 252 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 253 dev->pd_mgr->max_dpp_pd); 254 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap); 255 dev->pd_mgr->pd_dpp_count++; 256 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) 257 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; 258 } else { 259 pd_bitmap = dev->pd_mgr->pd_norm_bitmap; 260 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 261 dev->pd_mgr->max_normal_pd); 262 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap); 263 dev->pd_mgr->pd_norm_count++; 264 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) 265 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; 266 } 267 return pd_bitmap_idx; 268 } 269 270 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, 271 bool dpp_pool) 272 { 273 u16 pd_count; 274 u16 pd_bit_index; 275 276 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count : 277 dev->pd_mgr->pd_norm_count; 278 if (pd_count == 0) 279 return -EINVAL; 280 281 if (dpp_pool) { 282 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start; 283 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) { 284 return -EINVAL; 285 } else { 286 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap); 287 dev->pd_mgr->pd_dpp_count--; 288 } 289 } else { 290 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start; 291 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) { 292 return -EINVAL; 293 } else { 294 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap); 295 dev->pd_mgr->pd_norm_count--; 296 } 297 } 298 299 return 0; 300 } 301 302 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, 303 bool dpp_pool) 304 { 305 int status; 306 307 mutex_lock(&dev->dev_lock); 308 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool); 309 mutex_unlock(&dev->dev_lock); 310 return status; 311 } 312 313 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 314 { 315 u16 pd_idx = 0; 316 int status = 0; 317 318 mutex_lock(&dev->dev_lock); 319 if (pd->dpp_enabled) { 320 /* try allocating DPP PD, if not available then normal PD */ 321 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) { 322 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true); 323 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx; 324 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx; 325 } else if (dev->pd_mgr->pd_norm_count < 326 dev->pd_mgr->max_normal_pd) { 327 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 328 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 329 pd->dpp_enabled = false; 330 } else { 331 status = -EINVAL; 332 } 333 } else { 334 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) { 335 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 336 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 337 } else { 338 status = -EINVAL; 339 } 340 } 341 mutex_unlock(&dev->dev_lock); 342 return status; 343 } 344 345 /* 346 * NOTE: 347 * 348 * ocrdma_ucontext must be used here because this function is also 349 * called from ocrdma_alloc_ucontext where ib_udata does not have 350 * valid ib_ucontext pointer. ib_uverbs_get_context does not call 351 * uobj_{alloc|get_xxx} helpers which are used to store the 352 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so 353 * ib_udata does NOT imply valid ib_ucontext here! 354 */ 355 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 356 struct ocrdma_ucontext *uctx, 357 struct ib_udata *udata) 358 { 359 int status; 360 361 if (udata && uctx && dev->attr.max_dpp_pds) { 362 pd->dpp_enabled = 363 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R; 364 pd->num_dpp_qp = 365 pd->dpp_enabled ? (dev->nic_info.db_page_size / 366 dev->attr.wqe_size) : 0; 367 } 368 369 if (dev->pd_mgr->pd_prealloc_valid) 370 return ocrdma_get_pd_num(dev, pd); 371 372 retry: 373 status = ocrdma_mbx_alloc_pd(dev, pd); 374 if (status) { 375 if (pd->dpp_enabled) { 376 pd->dpp_enabled = false; 377 pd->num_dpp_qp = 0; 378 goto retry; 379 } 380 return status; 381 } 382 383 return 0; 384 } 385 386 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, 387 struct ocrdma_pd *pd) 388 { 389 return (uctx->cntxt_pd == pd); 390 } 391 392 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev, 393 struct ocrdma_pd *pd) 394 { 395 if (dev->pd_mgr->pd_prealloc_valid) 396 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled); 397 else 398 ocrdma_mbx_dealloc_pd(dev, pd); 399 } 400 401 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, 402 struct ocrdma_ucontext *uctx, 403 struct ib_udata *udata) 404 { 405 struct ib_device *ibdev = &dev->ibdev; 406 struct ib_pd *pd; 407 int status; 408 409 pd = rdma_zalloc_drv_obj(ibdev, ib_pd); 410 if (!pd) 411 return -ENOMEM; 412 413 pd->device = ibdev; 414 uctx->cntxt_pd = get_ocrdma_pd(pd); 415 416 status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata); 417 if (status) { 418 kfree(uctx->cntxt_pd); 419 goto err; 420 } 421 422 uctx->cntxt_pd->uctx = uctx; 423 uctx->cntxt_pd->ibpd.device = &dev->ibdev; 424 err: 425 return status; 426 } 427 428 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) 429 { 430 struct ocrdma_pd *pd = uctx->cntxt_pd; 431 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 432 433 if (uctx->pd_in_use) { 434 pr_err("%s(%d) Freeing in use pdid=0x%x.\n", 435 __func__, dev->id, pd->id); 436 } 437 uctx->cntxt_pd = NULL; 438 _ocrdma_dealloc_pd(dev, pd); 439 kfree(pd); 440 } 441 442 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) 443 { 444 struct ocrdma_pd *pd = NULL; 445 446 mutex_lock(&uctx->mm_list_lock); 447 if (!uctx->pd_in_use) { 448 uctx->pd_in_use = true; 449 pd = uctx->cntxt_pd; 450 } 451 mutex_unlock(&uctx->mm_list_lock); 452 453 return pd; 454 } 455 456 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) 457 { 458 mutex_lock(&uctx->mm_list_lock); 459 uctx->pd_in_use = false; 460 mutex_unlock(&uctx->mm_list_lock); 461 } 462 463 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 464 { 465 struct ib_device *ibdev = uctx->device; 466 int status; 467 struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx); 468 struct ocrdma_alloc_ucontext_resp resp = {}; 469 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 470 struct pci_dev *pdev = dev->nic_info.pdev; 471 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE); 472 473 if (!udata) 474 return -EFAULT; 475 INIT_LIST_HEAD(&ctx->mm_head); 476 mutex_init(&ctx->mm_list_lock); 477 478 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len, 479 &ctx->ah_tbl.pa, GFP_KERNEL); 480 if (!ctx->ah_tbl.va) 481 return -ENOMEM; 482 483 ctx->ah_tbl.len = map_len; 484 485 resp.ah_tbl_len = ctx->ah_tbl.len; 486 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 487 488 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 489 if (status) 490 goto map_err; 491 492 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata); 493 if (status) 494 goto pd_err; 495 496 resp.dev_id = dev->id; 497 resp.max_inline_data = dev->attr.max_inline_data; 498 resp.wqe_size = dev->attr.wqe_size; 499 resp.rqe_size = dev->attr.rqe_size; 500 resp.dpp_wqe_size = dev->attr.wqe_size; 501 502 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); 503 status = ib_copy_to_udata(udata, &resp, sizeof(resp)); 504 if (status) 505 goto cpy_err; 506 return 0; 507 508 cpy_err: 509 ocrdma_dealloc_ucontext_pd(ctx); 510 pd_err: 511 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 512 map_err: 513 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 514 ctx->ah_tbl.pa); 515 return status; 516 } 517 518 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 519 { 520 struct ocrdma_mm *mm, *tmp; 521 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 522 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device); 523 struct pci_dev *pdev = dev->nic_info.pdev; 524 525 ocrdma_dealloc_ucontext_pd(uctx); 526 527 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 528 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 529 uctx->ah_tbl.pa); 530 531 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 532 list_del(&mm->entry); 533 kfree(mm); 534 } 535 } 536 537 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 538 { 539 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 540 struct ocrdma_dev *dev = get_ocrdma_dev(context->device); 541 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 542 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 543 unsigned long len = (vma->vm_end - vma->vm_start); 544 int status; 545 bool found; 546 547 if (vma->vm_start & (PAGE_SIZE - 1)) 548 return -EINVAL; 549 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len); 550 if (!found) 551 return -EINVAL; 552 553 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 554 dev->nic_info.db_total_size)) && 555 (len <= dev->nic_info.db_page_size)) { 556 if (vma->vm_flags & VM_READ) 557 return -EPERM; 558 559 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 560 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 561 len, vma->vm_page_prot); 562 } else if (dev->nic_info.dpp_unmapped_len && 563 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) && 564 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 565 dev->nic_info.dpp_unmapped_len)) && 566 (len <= dev->nic_info.dpp_unmapped_len)) { 567 if (vma->vm_flags & VM_READ) 568 return -EPERM; 569 570 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 571 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 572 len, vma->vm_page_prot); 573 } else { 574 status = remap_pfn_range(vma, vma->vm_start, 575 vma->vm_pgoff, len, vma->vm_page_prot); 576 } 577 return status; 578 } 579 580 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 581 struct ib_udata *udata) 582 { 583 int status; 584 u64 db_page_addr; 585 u64 dpp_page_addr = 0; 586 u32 db_page_size; 587 struct ocrdma_alloc_pd_uresp rsp; 588 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 589 udata, struct ocrdma_ucontext, ibucontext); 590 591 memset(&rsp, 0, sizeof(rsp)); 592 rsp.id = pd->id; 593 rsp.dpp_enabled = pd->dpp_enabled; 594 db_page_addr = ocrdma_get_db_addr(dev, pd->id); 595 db_page_size = dev->nic_info.db_page_size; 596 597 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 598 if (status) 599 return status; 600 601 if (pd->dpp_enabled) { 602 dpp_page_addr = dev->nic_info.dpp_unmapped_addr + 603 (pd->id * PAGE_SIZE); 604 status = ocrdma_add_mmap(uctx, dpp_page_addr, 605 PAGE_SIZE); 606 if (status) 607 goto dpp_map_err; 608 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 609 rsp.dpp_page_addr_lo = dpp_page_addr; 610 } 611 612 status = ib_copy_to_udata(udata, &rsp, sizeof(rsp)); 613 if (status) 614 goto ucopy_err; 615 616 pd->uctx = uctx; 617 return 0; 618 619 ucopy_err: 620 if (pd->dpp_enabled) 621 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE); 622 dpp_map_err: 623 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size); 624 return status; 625 } 626 627 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 628 { 629 struct ib_device *ibdev = ibpd->device; 630 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 631 struct ocrdma_pd *pd; 632 int status; 633 u8 is_uctx_pd = false; 634 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 635 udata, struct ocrdma_ucontext, ibucontext); 636 637 if (udata) { 638 pd = ocrdma_get_ucontext_pd(uctx); 639 if (pd) { 640 is_uctx_pd = true; 641 goto pd_mapping; 642 } 643 } 644 645 pd = get_ocrdma_pd(ibpd); 646 status = _ocrdma_alloc_pd(dev, pd, uctx, udata); 647 if (status) 648 goto exit; 649 650 pd_mapping: 651 if (udata) { 652 status = ocrdma_copy_pd_uresp(dev, pd, udata); 653 if (status) 654 goto err; 655 } 656 return 0; 657 658 err: 659 if (is_uctx_pd) 660 ocrdma_release_ucontext_pd(uctx); 661 else 662 _ocrdma_dealloc_pd(dev, pd); 663 exit: 664 return status; 665 } 666 667 int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 668 { 669 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 670 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 671 struct ocrdma_ucontext *uctx = NULL; 672 u64 usr_db; 673 674 uctx = pd->uctx; 675 if (uctx) { 676 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 677 (pd->id * PAGE_SIZE); 678 if (pd->dpp_enabled) 679 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE); 680 usr_db = ocrdma_get_db_addr(dev, pd->id); 681 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 682 683 if (is_ucontext_pd(uctx, pd)) { 684 ocrdma_release_ucontext_pd(uctx); 685 return 0; 686 } 687 } 688 _ocrdma_dealloc_pd(dev, pd); 689 return 0; 690 } 691 692 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 693 u32 pdid, int acc, u32 num_pbls, u32 addr_check) 694 { 695 int status; 696 697 mr->hwmr.fr_mr = 0; 698 mr->hwmr.local_rd = 1; 699 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 700 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 701 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 702 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0; 703 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 704 mr->hwmr.num_pbls = num_pbls; 705 706 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check); 707 if (status) 708 return status; 709 710 mr->ibmr.lkey = mr->hwmr.lkey; 711 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 712 mr->ibmr.rkey = mr->hwmr.lkey; 713 return 0; 714 } 715 716 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 717 { 718 int status; 719 struct ocrdma_mr *mr; 720 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 721 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 722 723 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 724 pr_err("%s err, invalid access rights\n", __func__); 725 return ERR_PTR(-EINVAL); 726 } 727 728 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 729 if (!mr) 730 return ERR_PTR(-ENOMEM); 731 732 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0, 733 OCRDMA_ADDR_CHECK_DISABLE); 734 if (status) { 735 kfree(mr); 736 return ERR_PTR(status); 737 } 738 739 return &mr->ibmr; 740 } 741 742 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, 743 struct ocrdma_hw_mr *mr) 744 { 745 struct pci_dev *pdev = dev->nic_info.pdev; 746 int i = 0; 747 748 if (mr->pbl_table) { 749 for (i = 0; i < mr->num_pbls; i++) { 750 if (!mr->pbl_table[i].va) 751 continue; 752 dma_free_coherent(&pdev->dev, mr->pbl_size, 753 mr->pbl_table[i].va, 754 mr->pbl_table[i].pa); 755 } 756 kfree(mr->pbl_table); 757 mr->pbl_table = NULL; 758 } 759 } 760 761 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 762 u32 num_pbes) 763 { 764 u32 num_pbls = 0; 765 u32 idx = 0; 766 int status = 0; 767 u32 pbl_size; 768 769 do { 770 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx); 771 if (pbl_size > MAX_OCRDMA_PBL_SIZE) { 772 status = -EFAULT; 773 break; 774 } 775 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 776 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 777 idx++; 778 } while (num_pbls >= dev->attr.max_num_mr_pbl); 779 780 mr->hwmr.num_pbes = num_pbes; 781 mr->hwmr.num_pbls = num_pbls; 782 mr->hwmr.pbl_size = pbl_size; 783 return status; 784 } 785 786 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) 787 { 788 int status = 0; 789 int i; 790 u32 dma_len = mr->pbl_size; 791 struct pci_dev *pdev = dev->nic_info.pdev; 792 void *va; 793 dma_addr_t pa; 794 795 mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl), 796 GFP_KERNEL); 797 798 if (!mr->pbl_table) 799 return -ENOMEM; 800 801 for (i = 0; i < mr->num_pbls; i++) { 802 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL); 803 if (!va) { 804 ocrdma_free_mr_pbl_tbl(dev, mr); 805 status = -ENOMEM; 806 break; 807 } 808 mr->pbl_table[i].va = va; 809 mr->pbl_table[i].pa = pa; 810 } 811 return status; 812 } 813 814 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr) 815 { 816 struct ocrdma_pbe *pbe; 817 struct ib_block_iter biter; 818 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 819 int pbe_cnt; 820 u64 pg_addr; 821 822 if (!mr->hwmr.num_pbes) 823 return; 824 825 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 826 pbe_cnt = 0; 827 828 rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) { 829 /* store the page address in pbe */ 830 pg_addr = rdma_block_iter_dma_address(&biter); 831 pbe->pa_lo = cpu_to_le32(pg_addr); 832 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr)); 833 pbe_cnt += 1; 834 pbe++; 835 836 /* if the given pbl is full storing the pbes, 837 * move to next pbl. 838 */ 839 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) { 840 pbl_tbl++; 841 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 842 pbe_cnt = 0; 843 } 844 } 845 } 846 847 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 848 u64 usr_addr, int acc, struct ib_udata *udata) 849 { 850 int status = -ENOMEM; 851 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 852 struct ocrdma_mr *mr; 853 struct ocrdma_pd *pd; 854 855 pd = get_ocrdma_pd(ibpd); 856 857 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 858 return ERR_PTR(-EINVAL); 859 860 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 861 if (!mr) 862 return ERR_PTR(status); 863 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 864 if (IS_ERR(mr->umem)) { 865 status = -EFAULT; 866 goto umem_err; 867 } 868 status = ocrdma_get_pbl_info( 869 dev, mr, ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE)); 870 if (status) 871 goto umem_err; 872 873 mr->hwmr.pbe_size = PAGE_SIZE; 874 mr->hwmr.va = usr_addr; 875 mr->hwmr.len = len; 876 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 877 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 878 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 879 mr->hwmr.local_rd = 1; 880 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 881 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 882 if (status) 883 goto umem_err; 884 build_user_pbes(dev, mr); 885 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 886 if (status) 887 goto mbx_err; 888 mr->ibmr.lkey = mr->hwmr.lkey; 889 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 890 mr->ibmr.rkey = mr->hwmr.lkey; 891 892 return &mr->ibmr; 893 894 mbx_err: 895 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 896 umem_err: 897 kfree(mr); 898 return ERR_PTR(status); 899 } 900 901 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 902 { 903 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 904 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device); 905 906 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 907 908 kfree(mr->pages); 909 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 910 911 /* it could be user registered memory. */ 912 ib_umem_release(mr->umem); 913 kfree(mr); 914 915 /* Don't stop cleanup, in case FW is unresponsive */ 916 if (dev->mqe_ctx.fw_error_state) { 917 pr_err("%s(%d) fw not responding.\n", 918 __func__, dev->id); 919 } 920 return 0; 921 } 922 923 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 924 struct ib_udata *udata) 925 { 926 int status; 927 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 928 udata, struct ocrdma_ucontext, ibucontext); 929 struct ocrdma_create_cq_uresp uresp; 930 931 /* this must be user flow! */ 932 if (!udata) 933 return -EINVAL; 934 935 memset(&uresp, 0, sizeof(uresp)); 936 uresp.cq_id = cq->id; 937 uresp.page_size = PAGE_ALIGN(cq->len); 938 uresp.num_pages = 1; 939 uresp.max_hw_cqe = cq->max_hw_cqe; 940 uresp.page_addr[0] = virt_to_phys(cq->va); 941 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 942 uresp.db_page_size = dev->nic_info.db_page_size; 943 uresp.phase_change = cq->phase_change ? 1 : 0; 944 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 945 if (status) { 946 pr_err("%s(%d) copy error cqid=0x%x.\n", 947 __func__, dev->id, cq->id); 948 goto err; 949 } 950 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 951 if (status) 952 goto err; 953 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size); 954 if (status) { 955 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 956 goto err; 957 } 958 cq->ucontext = uctx; 959 err: 960 return status; 961 } 962 963 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 964 struct ib_udata *udata) 965 { 966 struct ib_device *ibdev = ibcq->device; 967 int entries = attr->cqe; 968 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 969 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 970 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 971 udata, struct ocrdma_ucontext, ibucontext); 972 u16 pd_id = 0; 973 int status; 974 struct ocrdma_create_cq_ureq ureq; 975 976 if (attr->flags) 977 return -EOPNOTSUPP; 978 979 if (udata) { 980 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 981 return -EFAULT; 982 } else 983 ureq.dpp_cq = 0; 984 985 spin_lock_init(&cq->cq_lock); 986 spin_lock_init(&cq->comp_handler_lock); 987 INIT_LIST_HEAD(&cq->sq_head); 988 INIT_LIST_HEAD(&cq->rq_head); 989 990 if (udata) 991 pd_id = uctx->cntxt_pd->id; 992 993 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); 994 if (status) 995 return status; 996 997 if (udata) { 998 status = ocrdma_copy_cq_uresp(dev, cq, udata); 999 if (status) 1000 goto ctx_err; 1001 } 1002 cq->phase = OCRDMA_CQE_VALID; 1003 dev->cq_tbl[cq->id] = cq; 1004 return 0; 1005 1006 ctx_err: 1007 ocrdma_mbx_destroy_cq(dev, cq); 1008 return status; 1009 } 1010 1011 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, 1012 struct ib_udata *udata) 1013 { 1014 int status = 0; 1015 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1016 1017 if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) { 1018 status = -EINVAL; 1019 return status; 1020 } 1021 ibcq->cqe = new_cnt; 1022 return status; 1023 } 1024 1025 static void ocrdma_flush_cq(struct ocrdma_cq *cq) 1026 { 1027 int cqe_cnt; 1028 int valid_count = 0; 1029 unsigned long flags; 1030 1031 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 1032 struct ocrdma_cqe *cqe = NULL; 1033 1034 cqe = cq->va; 1035 cqe_cnt = cq->cqe_cnt; 1036 1037 /* Last irq might have scheduled a polling thread 1038 * sync-up with it before hard flushing. 1039 */ 1040 spin_lock_irqsave(&cq->cq_lock, flags); 1041 while (cqe_cnt) { 1042 if (is_cqe_valid(cq, cqe)) 1043 valid_count++; 1044 cqe++; 1045 cqe_cnt--; 1046 } 1047 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count); 1048 spin_unlock_irqrestore(&cq->cq_lock, flags); 1049 } 1050 1051 int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1052 { 1053 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1054 struct ocrdma_eq *eq = NULL; 1055 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 1056 int pdid = 0; 1057 u32 irq, indx; 1058 1059 dev->cq_tbl[cq->id] = NULL; 1060 indx = ocrdma_get_eq_table_index(dev, cq->eqn); 1061 1062 eq = &dev->eq_tbl[indx]; 1063 irq = ocrdma_get_irq(dev, eq); 1064 synchronize_irq(irq); 1065 ocrdma_flush_cq(cq); 1066 1067 ocrdma_mbx_destroy_cq(dev, cq); 1068 if (cq->ucontext) { 1069 pdid = cq->ucontext->cntxt_pd->id; 1070 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, 1071 PAGE_ALIGN(cq->len)); 1072 ocrdma_del_mmap(cq->ucontext, 1073 ocrdma_get_db_addr(dev, pdid), 1074 dev->nic_info.db_page_size); 1075 } 1076 return 0; 1077 } 1078 1079 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1080 { 1081 int status = -EINVAL; 1082 1083 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { 1084 dev->qp_tbl[qp->id] = qp; 1085 status = 0; 1086 } 1087 return status; 1088 } 1089 1090 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1091 { 1092 dev->qp_tbl[qp->id] = NULL; 1093 } 1094 1095 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 1096 struct ib_qp_init_attr *attrs, 1097 struct ib_udata *udata) 1098 { 1099 if ((attrs->qp_type != IB_QPT_GSI) && 1100 (attrs->qp_type != IB_QPT_RC) && 1101 (attrs->qp_type != IB_QPT_UC) && 1102 (attrs->qp_type != IB_QPT_UD)) { 1103 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1104 __func__, dev->id, attrs->qp_type); 1105 return -EOPNOTSUPP; 1106 } 1107 /* Skip the check for QP1 to support CM size of 128 */ 1108 if ((attrs->qp_type != IB_QPT_GSI) && 1109 (attrs->cap.max_send_wr > dev->attr.max_wqe)) { 1110 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1111 __func__, dev->id, attrs->cap.max_send_wr); 1112 pr_err("%s(%d) supported send_wr=0x%x\n", 1113 __func__, dev->id, dev->attr.max_wqe); 1114 return -EINVAL; 1115 } 1116 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) { 1117 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n", 1118 __func__, dev->id, attrs->cap.max_recv_wr); 1119 pr_err("%s(%d) supported recv_wr=0x%x\n", 1120 __func__, dev->id, dev->attr.max_rqe); 1121 return -EINVAL; 1122 } 1123 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) { 1124 pr_err("%s(%d) unsupported inline data size=0x%x requested\n", 1125 __func__, dev->id, attrs->cap.max_inline_data); 1126 pr_err("%s(%d) supported inline data size=0x%x\n", 1127 __func__, dev->id, dev->attr.max_inline_data); 1128 return -EINVAL; 1129 } 1130 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) { 1131 pr_err("%s(%d) unsupported send_sge=0x%x requested\n", 1132 __func__, dev->id, attrs->cap.max_send_sge); 1133 pr_err("%s(%d) supported send_sge=0x%x\n", 1134 __func__, dev->id, dev->attr.max_send_sge); 1135 return -EINVAL; 1136 } 1137 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) { 1138 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n", 1139 __func__, dev->id, attrs->cap.max_recv_sge); 1140 pr_err("%s(%d) supported recv_sge=0x%x\n", 1141 __func__, dev->id, dev->attr.max_recv_sge); 1142 return -EINVAL; 1143 } 1144 /* unprivileged user space cannot create special QP */ 1145 if (udata && attrs->qp_type == IB_QPT_GSI) { 1146 pr_err 1147 ("%s(%d) Userspace can't create special QPs of type=0x%x\n", 1148 __func__, dev->id, attrs->qp_type); 1149 return -EINVAL; 1150 } 1151 /* allow creating only one GSI type of QP */ 1152 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 1153 pr_err("%s(%d) GSI special QPs already created.\n", 1154 __func__, dev->id); 1155 return -EINVAL; 1156 } 1157 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1158 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1159 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1160 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) { 1161 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1162 __func__, dev->id); 1163 return -EINVAL; 1164 } 1165 } 1166 return 0; 1167 } 1168 1169 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, 1170 struct ib_udata *udata, int dpp_offset, 1171 int dpp_credit_lmt, int srq) 1172 { 1173 int status; 1174 u64 usr_db; 1175 struct ocrdma_create_qp_uresp uresp; 1176 struct ocrdma_pd *pd = qp->pd; 1177 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 1178 1179 memset(&uresp, 0, sizeof(uresp)); 1180 usr_db = dev->nic_info.unmapped_db + 1181 (pd->id * dev->nic_info.db_page_size); 1182 uresp.qp_id = qp->id; 1183 uresp.sq_dbid = qp->sq.dbid; 1184 uresp.num_sq_pages = 1; 1185 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1186 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1187 uresp.num_wqe_allocated = qp->sq.max_cnt; 1188 if (!srq) { 1189 uresp.rq_dbid = qp->rq.dbid; 1190 uresp.num_rq_pages = 1; 1191 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1192 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1193 uresp.num_rqe_allocated = qp->rq.max_cnt; 1194 } 1195 uresp.db_page_addr = usr_db; 1196 uresp.db_page_size = dev->nic_info.db_page_size; 1197 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1198 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1199 uresp.db_shift = OCRDMA_DB_RQ_SHIFT; 1200 1201 if (qp->dpp_enabled) { 1202 uresp.dpp_credit = dpp_credit_lmt; 1203 uresp.dpp_offset = dpp_offset; 1204 } 1205 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1206 if (status) { 1207 pr_err("%s(%d) user copy error.\n", __func__, dev->id); 1208 goto err; 1209 } 1210 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], 1211 uresp.sq_page_size); 1212 if (status) 1213 goto err; 1214 1215 if (!srq) { 1216 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0], 1217 uresp.rq_page_size); 1218 if (status) 1219 goto rq_map_err; 1220 } 1221 return status; 1222 rq_map_err: 1223 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); 1224 err: 1225 return status; 1226 } 1227 1228 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, 1229 struct ocrdma_pd *pd) 1230 { 1231 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1232 qp->sq_db = dev->nic_info.db + 1233 (pd->id * dev->nic_info.db_page_size) + 1234 OCRDMA_DB_GEN2_SQ_OFFSET; 1235 qp->rq_db = dev->nic_info.db + 1236 (pd->id * dev->nic_info.db_page_size) + 1237 OCRDMA_DB_GEN2_RQ_OFFSET; 1238 } else { 1239 qp->sq_db = dev->nic_info.db + 1240 (pd->id * dev->nic_info.db_page_size) + 1241 OCRDMA_DB_SQ_OFFSET; 1242 qp->rq_db = dev->nic_info.db + 1243 (pd->id * dev->nic_info.db_page_size) + 1244 OCRDMA_DB_RQ_OFFSET; 1245 } 1246 } 1247 1248 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) 1249 { 1250 qp->wqe_wr_id_tbl = 1251 kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)), 1252 GFP_KERNEL); 1253 if (qp->wqe_wr_id_tbl == NULL) 1254 return -ENOMEM; 1255 qp->rqe_wr_id_tbl = 1256 kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL); 1257 if (qp->rqe_wr_id_tbl == NULL) 1258 return -ENOMEM; 1259 1260 return 0; 1261 } 1262 1263 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, 1264 struct ocrdma_pd *pd, 1265 struct ib_qp_init_attr *attrs) 1266 { 1267 qp->pd = pd; 1268 spin_lock_init(&qp->q_lock); 1269 INIT_LIST_HEAD(&qp->sq_entry); 1270 INIT_LIST_HEAD(&qp->rq_entry); 1271 1272 qp->qp_type = attrs->qp_type; 1273 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; 1274 qp->max_inline_data = attrs->cap.max_inline_data; 1275 qp->sq.max_sges = attrs->cap.max_send_sge; 1276 qp->rq.max_sges = attrs->cap.max_recv_sge; 1277 qp->state = OCRDMA_QPS_RST; 1278 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1279 } 1280 1281 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, 1282 struct ib_qp_init_attr *attrs) 1283 { 1284 if (attrs->qp_type == IB_QPT_GSI) { 1285 dev->gsi_qp_created = 1; 1286 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq); 1287 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq); 1288 } 1289 } 1290 1291 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, 1292 struct ib_qp_init_attr *attrs, 1293 struct ib_udata *udata) 1294 { 1295 int status; 1296 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1297 struct ocrdma_qp *qp; 1298 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 1299 struct ocrdma_create_qp_ureq ureq; 1300 u16 dpp_credit_lmt, dpp_offset; 1301 1302 if (attrs->create_flags) 1303 return ERR_PTR(-EOPNOTSUPP); 1304 1305 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata); 1306 if (status) 1307 goto gen_err; 1308 1309 memset(&ureq, 0, sizeof(ureq)); 1310 if (udata) { 1311 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1312 return ERR_PTR(-EFAULT); 1313 } 1314 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1315 if (!qp) { 1316 status = -ENOMEM; 1317 goto gen_err; 1318 } 1319 ocrdma_set_qp_init_params(qp, pd, attrs); 1320 if (udata == NULL) 1321 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | 1322 OCRDMA_QP_FAST_REG); 1323 1324 mutex_lock(&dev->dev_lock); 1325 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1326 ureq.dpp_cq_id, 1327 &dpp_offset, &dpp_credit_lmt); 1328 if (status) 1329 goto mbx_err; 1330 1331 /* user space QP's wr_id table are managed in library */ 1332 if (udata == NULL) { 1333 status = ocrdma_alloc_wr_id_tbl(qp); 1334 if (status) 1335 goto map_err; 1336 } 1337 1338 status = ocrdma_add_qpn_map(dev, qp); 1339 if (status) 1340 goto map_err; 1341 ocrdma_set_qp_db(dev, qp, pd); 1342 if (udata) { 1343 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, 1344 dpp_credit_lmt, 1345 (attrs->srq != NULL)); 1346 if (status) 1347 goto cpy_err; 1348 } 1349 ocrdma_store_gsi_qp_cq(dev, attrs); 1350 qp->ibqp.qp_num = qp->id; 1351 mutex_unlock(&dev->dev_lock); 1352 return &qp->ibqp; 1353 1354 cpy_err: 1355 ocrdma_del_qpn_map(dev, qp); 1356 map_err: 1357 ocrdma_mbx_destroy_qp(dev, qp); 1358 mbx_err: 1359 mutex_unlock(&dev->dev_lock); 1360 kfree(qp->wqe_wr_id_tbl); 1361 kfree(qp->rqe_wr_id_tbl); 1362 kfree(qp); 1363 pr_err("%s(%d) error=%d\n", __func__, dev->id, status); 1364 gen_err: 1365 return ERR_PTR(status); 1366 } 1367 1368 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1369 int attr_mask) 1370 { 1371 int status = 0; 1372 struct ocrdma_qp *qp; 1373 struct ocrdma_dev *dev; 1374 enum ib_qp_state old_qps; 1375 1376 qp = get_ocrdma_qp(ibqp); 1377 dev = get_ocrdma_dev(ibqp->device); 1378 if (attr_mask & IB_QP_STATE) 1379 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); 1380 /* if new and previous states are same hw doesn't need to 1381 * know about it. 1382 */ 1383 if (status < 0) 1384 return status; 1385 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); 1386 } 1387 1388 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1389 int attr_mask, struct ib_udata *udata) 1390 { 1391 unsigned long flags; 1392 int status = -EINVAL; 1393 struct ocrdma_qp *qp; 1394 struct ocrdma_dev *dev; 1395 enum ib_qp_state old_qps, new_qps; 1396 1397 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1398 return -EOPNOTSUPP; 1399 1400 qp = get_ocrdma_qp(ibqp); 1401 dev = get_ocrdma_dev(ibqp->device); 1402 1403 /* syncronize with multiple context trying to change, retrive qps */ 1404 mutex_lock(&dev->dev_lock); 1405 /* syncronize with wqe, rqe posting and cqe processing contexts */ 1406 spin_lock_irqsave(&qp->q_lock, flags); 1407 old_qps = get_ibqp_state(qp->state); 1408 if (attr_mask & IB_QP_STATE) 1409 new_qps = attr->qp_state; 1410 else 1411 new_qps = old_qps; 1412 spin_unlock_irqrestore(&qp->q_lock, flags); 1413 1414 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) { 1415 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n" 1416 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n", 1417 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, 1418 old_qps, new_qps); 1419 goto param_err; 1420 } 1421 1422 status = _ocrdma_modify_qp(ibqp, attr, attr_mask); 1423 if (status > 0) 1424 status = 0; 1425 param_err: 1426 mutex_unlock(&dev->dev_lock); 1427 return status; 1428 } 1429 1430 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) 1431 { 1432 switch (mtu) { 1433 case 256: 1434 return IB_MTU_256; 1435 case 512: 1436 return IB_MTU_512; 1437 case 1024: 1438 return IB_MTU_1024; 1439 case 2048: 1440 return IB_MTU_2048; 1441 case 4096: 1442 return IB_MTU_4096; 1443 default: 1444 return IB_MTU_1024; 1445 } 1446 } 1447 1448 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) 1449 { 1450 int ib_qp_acc_flags = 0; 1451 1452 if (qp_cap_flags & OCRDMA_QP_INB_WR) 1453 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1454 if (qp_cap_flags & OCRDMA_QP_INB_RD) 1455 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 1456 return ib_qp_acc_flags; 1457 } 1458 1459 int ocrdma_query_qp(struct ib_qp *ibqp, 1460 struct ib_qp_attr *qp_attr, 1461 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 1462 { 1463 int status; 1464 u32 qp_state; 1465 struct ocrdma_qp_params params; 1466 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1467 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1468 1469 memset(¶ms, 0, sizeof(params)); 1470 mutex_lock(&dev->dev_lock); 1471 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); 1472 mutex_unlock(&dev->dev_lock); 1473 if (status) 1474 goto mbx_err; 1475 if (qp->qp_type == IB_QPT_UD) 1476 qp_attr->qkey = params.qkey; 1477 qp_attr->path_mtu = 1478 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx & 1479 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >> 1480 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT; 1481 qp_attr->path_mig_state = IB_MIG_MIGRATED; 1482 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK; 1483 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK; 1484 qp_attr->dest_qp_num = 1485 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK; 1486 1487 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); 1488 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; 1489 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1490 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1491 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1492 qp_attr->cap.max_inline_data = qp->max_inline_data; 1493 qp_init_attr->cap = qp_attr->cap; 1494 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1495 1496 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 1497 params.rnt_rc_sl_fl & 1498 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK, 1499 qp->sgid_idx, 1500 (params.hop_lmt_rq_psn & 1501 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >> 1502 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT, 1503 (params.tclass_sq_psn & 1504 OCRDMA_QP_PARAMS_TCLASS_MASK) >> 1505 OCRDMA_QP_PARAMS_TCLASS_SHIFT); 1506 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]); 1507 1508 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 1509 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl & 1510 OCRDMA_QP_PARAMS_SL_MASK) >> 1511 OCRDMA_QP_PARAMS_SL_SHIFT); 1512 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn & 1513 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >> 1514 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT; 1515 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn & 1516 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >> 1517 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT; 1518 qp_attr->retry_cnt = 1519 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >> 1520 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT; 1521 qp_attr->min_rnr_timer = 0; 1522 qp_attr->pkey_index = 0; 1523 qp_attr->port_num = 1; 1524 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 1525 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 1526 qp_attr->alt_pkey_index = 0; 1527 qp_attr->alt_port_num = 0; 1528 qp_attr->alt_timeout = 0; 1529 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 1530 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >> 1531 OCRDMA_QP_PARAMS_STATE_SHIFT; 1532 qp_attr->qp_state = get_ibqp_state(qp_state); 1533 qp_attr->cur_qp_state = qp_attr->qp_state; 1534 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0; 1535 qp_attr->max_dest_rd_atomic = 1536 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT; 1537 qp_attr->max_rd_atomic = 1538 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK; 1539 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags & 1540 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0; 1541 /* Sync driver QP state with FW */ 1542 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); 1543 mbx_err: 1544 return status; 1545 } 1546 1547 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) 1548 { 1549 unsigned int i = idx / 32; 1550 u32 mask = (1U << (idx % 32)); 1551 1552 srq->idx_bit_fields[i] ^= mask; 1553 } 1554 1555 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1556 { 1557 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt; 1558 } 1559 1560 static int is_hw_sq_empty(struct ocrdma_qp *qp) 1561 { 1562 return (qp->sq.tail == qp->sq.head); 1563 } 1564 1565 static int is_hw_rq_empty(struct ocrdma_qp *qp) 1566 { 1567 return (qp->rq.tail == qp->rq.head); 1568 } 1569 1570 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1571 { 1572 return q->va + (q->head * q->entry_size); 1573 } 1574 1575 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, 1576 u32 idx) 1577 { 1578 return q->va + (idx * q->entry_size); 1579 } 1580 1581 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) 1582 { 1583 q->head = (q->head + 1) & q->max_wqe_idx; 1584 } 1585 1586 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) 1587 { 1588 q->tail = (q->tail + 1) & q->max_wqe_idx; 1589 } 1590 1591 /* discard the cqe for a given QP */ 1592 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) 1593 { 1594 unsigned long cq_flags; 1595 unsigned long flags; 1596 int discard_cnt = 0; 1597 u32 cur_getp, stop_getp; 1598 struct ocrdma_cqe *cqe; 1599 u32 qpn = 0, wqe_idx = 0; 1600 1601 spin_lock_irqsave(&cq->cq_lock, cq_flags); 1602 1603 /* traverse through the CQEs in the hw CQ, 1604 * find the matching CQE for a given qp, 1605 * mark the matching one discarded by clearing qpn. 1606 * ring the doorbell in the poll_cq() as 1607 * we don't complete out of order cqe. 1608 */ 1609 1610 cur_getp = cq->getp; 1611 /* find upto when do we reap the cq. */ 1612 stop_getp = cur_getp; 1613 do { 1614 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) 1615 break; 1616 1617 cqe = cq->va + cur_getp; 1618 /* if (a) done reaping whole hw cq, or 1619 * (b) qp_xq becomes empty. 1620 * then exit 1621 */ 1622 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK; 1623 /* if previously discarded cqe found, skip that too. */ 1624 /* check for matching qp */ 1625 if (qpn == 0 || qpn != qp->id) 1626 goto skip_cqe; 1627 1628 if (is_cqe_for_sq(cqe)) { 1629 ocrdma_hwq_inc_tail(&qp->sq); 1630 } else { 1631 if (qp->srq) { 1632 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 1633 OCRDMA_CQE_BUFTAG_SHIFT) & 1634 qp->srq->rq.max_wqe_idx; 1635 BUG_ON(wqe_idx < 1); 1636 spin_lock_irqsave(&qp->srq->q_lock, flags); 1637 ocrdma_hwq_inc_tail(&qp->srq->rq); 1638 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); 1639 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1640 1641 } else { 1642 ocrdma_hwq_inc_tail(&qp->rq); 1643 } 1644 } 1645 /* mark cqe discarded so that it is not picked up later 1646 * in the poll_cq(). 1647 */ 1648 discard_cnt += 1; 1649 cqe->cmn.qpn = 0; 1650 skip_cqe: 1651 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1652 } while (cur_getp != stop_getp); 1653 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1654 } 1655 1656 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1657 { 1658 int found = false; 1659 unsigned long flags; 1660 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 1661 /* sync with any active CQ poll */ 1662 1663 spin_lock_irqsave(&dev->flush_q_lock, flags); 1664 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); 1665 if (found) 1666 list_del(&qp->sq_entry); 1667 if (!qp->srq) { 1668 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); 1669 if (found) 1670 list_del(&qp->rq_entry); 1671 } 1672 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 1673 } 1674 1675 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1676 { 1677 struct ocrdma_pd *pd; 1678 struct ocrdma_qp *qp; 1679 struct ocrdma_dev *dev; 1680 struct ib_qp_attr attrs; 1681 int attr_mask; 1682 unsigned long flags; 1683 1684 qp = get_ocrdma_qp(ibqp); 1685 dev = get_ocrdma_dev(ibqp->device); 1686 1687 pd = qp->pd; 1688 1689 /* change the QP state to ERROR */ 1690 if (qp->state != OCRDMA_QPS_RST) { 1691 attrs.qp_state = IB_QPS_ERR; 1692 attr_mask = IB_QP_STATE; 1693 _ocrdma_modify_qp(ibqp, &attrs, attr_mask); 1694 } 1695 /* ensure that CQEs for newly created QP (whose id may be same with 1696 * one which just getting destroyed are same), dont get 1697 * discarded until the old CQEs are discarded. 1698 */ 1699 mutex_lock(&dev->dev_lock); 1700 (void) ocrdma_mbx_destroy_qp(dev, qp); 1701 1702 /* 1703 * acquire CQ lock while destroy is in progress, in order to 1704 * protect against proessing in-flight CQEs for this QP. 1705 */ 1706 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); 1707 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) { 1708 spin_lock(&qp->rq_cq->cq_lock); 1709 ocrdma_del_qpn_map(dev, qp); 1710 spin_unlock(&qp->rq_cq->cq_lock); 1711 } else { 1712 ocrdma_del_qpn_map(dev, qp); 1713 } 1714 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); 1715 1716 if (!pd->uctx) { 1717 ocrdma_discard_cqes(qp, qp->sq_cq); 1718 ocrdma_discard_cqes(qp, qp->rq_cq); 1719 } 1720 mutex_unlock(&dev->dev_lock); 1721 1722 if (pd->uctx) { 1723 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, 1724 PAGE_ALIGN(qp->sq.len)); 1725 if (!qp->srq) 1726 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, 1727 PAGE_ALIGN(qp->rq.len)); 1728 } 1729 1730 ocrdma_del_flush_qp(qp); 1731 1732 kfree(qp->wqe_wr_id_tbl); 1733 kfree(qp->rqe_wr_id_tbl); 1734 kfree(qp); 1735 return 0; 1736 } 1737 1738 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, 1739 struct ib_udata *udata) 1740 { 1741 int status; 1742 struct ocrdma_create_srq_uresp uresp; 1743 1744 memset(&uresp, 0, sizeof(uresp)); 1745 uresp.rq_dbid = srq->rq.dbid; 1746 uresp.num_rq_pages = 1; 1747 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1748 uresp.rq_page_size = srq->rq.len; 1749 uresp.db_page_addr = dev->nic_info.unmapped_db + 1750 (srq->pd->id * dev->nic_info.db_page_size); 1751 uresp.db_page_size = dev->nic_info.db_page_size; 1752 uresp.num_rqe_allocated = srq->rq.max_cnt; 1753 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1754 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1755 uresp.db_shift = 24; 1756 } else { 1757 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1758 uresp.db_shift = 16; 1759 } 1760 1761 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1762 if (status) 1763 return status; 1764 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], 1765 uresp.rq_page_size); 1766 if (status) 1767 return status; 1768 return status; 1769 } 1770 1771 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1772 struct ib_udata *udata) 1773 { 1774 int status; 1775 struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd); 1776 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1777 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq); 1778 1779 if (init_attr->srq_type != IB_SRQT_BASIC) 1780 return -EOPNOTSUPP; 1781 1782 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1783 return -EINVAL; 1784 if (init_attr->attr.max_wr > dev->attr.max_rqe) 1785 return -EINVAL; 1786 1787 spin_lock_init(&srq->q_lock); 1788 srq->pd = pd; 1789 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1790 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd); 1791 if (status) 1792 return status; 1793 1794 if (!udata) { 1795 srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64), 1796 GFP_KERNEL); 1797 if (!srq->rqe_wr_id_tbl) { 1798 status = -ENOMEM; 1799 goto arm_err; 1800 } 1801 1802 srq->bit_fields_len = (srq->rq.max_cnt / 32) + 1803 (srq->rq.max_cnt % 32 ? 1 : 0); 1804 srq->idx_bit_fields = 1805 kmalloc_array(srq->bit_fields_len, sizeof(u32), 1806 GFP_KERNEL); 1807 if (!srq->idx_bit_fields) { 1808 status = -ENOMEM; 1809 goto arm_err; 1810 } 1811 memset(srq->idx_bit_fields, 0xff, 1812 srq->bit_fields_len * sizeof(u32)); 1813 } 1814 1815 if (init_attr->attr.srq_limit) { 1816 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr); 1817 if (status) 1818 goto arm_err; 1819 } 1820 1821 if (udata) { 1822 status = ocrdma_copy_srq_uresp(dev, srq, udata); 1823 if (status) 1824 goto arm_err; 1825 } 1826 1827 return 0; 1828 1829 arm_err: 1830 ocrdma_mbx_destroy_srq(dev, srq); 1831 kfree(srq->rqe_wr_id_tbl); 1832 kfree(srq->idx_bit_fields); 1833 return status; 1834 } 1835 1836 int ocrdma_modify_srq(struct ib_srq *ibsrq, 1837 struct ib_srq_attr *srq_attr, 1838 enum ib_srq_attr_mask srq_attr_mask, 1839 struct ib_udata *udata) 1840 { 1841 int status; 1842 struct ocrdma_srq *srq; 1843 1844 srq = get_ocrdma_srq(ibsrq); 1845 if (srq_attr_mask & IB_SRQ_MAX_WR) 1846 status = -EINVAL; 1847 else 1848 status = ocrdma_mbx_modify_srq(srq, srq_attr); 1849 return status; 1850 } 1851 1852 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 1853 { 1854 int status; 1855 struct ocrdma_srq *srq; 1856 1857 srq = get_ocrdma_srq(ibsrq); 1858 status = ocrdma_mbx_query_srq(srq, srq_attr); 1859 return status; 1860 } 1861 1862 int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1863 { 1864 struct ocrdma_srq *srq; 1865 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1866 1867 srq = get_ocrdma_srq(ibsrq); 1868 1869 ocrdma_mbx_destroy_srq(dev, srq); 1870 1871 if (srq->pd->uctx) 1872 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, 1873 PAGE_ALIGN(srq->rq.len)); 1874 1875 kfree(srq->idx_bit_fields); 1876 kfree(srq->rqe_wr_id_tbl); 1877 return 0; 1878 } 1879 1880 /* unprivileged verbs and their support functions. */ 1881 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, 1882 struct ocrdma_hdr_wqe *hdr, 1883 const struct ib_send_wr *wr) 1884 { 1885 struct ocrdma_ewqe_ud_hdr *ud_hdr = 1886 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1); 1887 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah); 1888 1889 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn; 1890 if (qp->qp_type == IB_QPT_GSI) 1891 ud_hdr->qkey = qp->qkey; 1892 else 1893 ud_hdr->qkey = ud_wr(wr)->remote_qkey; 1894 ud_hdr->rsvd_ahid = ah->id; 1895 ud_hdr->hdr_type = ah->hdr_type; 1896 if (ah->av->valid & OCRDMA_AV_VLAN_VALID) 1897 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT); 1898 } 1899 1900 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, 1901 struct ocrdma_sge *sge, int num_sge, 1902 struct ib_sge *sg_list) 1903 { 1904 int i; 1905 1906 for (i = 0; i < num_sge; i++) { 1907 sge[i].lrkey = sg_list[i].lkey; 1908 sge[i].addr_lo = sg_list[i].addr; 1909 sge[i].addr_hi = upper_32_bits(sg_list[i].addr); 1910 sge[i].len = sg_list[i].length; 1911 hdr->total_len += sg_list[i].length; 1912 } 1913 if (num_sge == 0) 1914 memset(sge, 0, sizeof(*sge)); 1915 } 1916 1917 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) 1918 { 1919 uint32_t total_len = 0, i; 1920 1921 for (i = 0; i < num_sge; i++) 1922 total_len += sg_list[i].length; 1923 return total_len; 1924 } 1925 1926 1927 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 1928 struct ocrdma_hdr_wqe *hdr, 1929 struct ocrdma_sge *sge, 1930 const struct ib_send_wr *wr, u32 wqe_size) 1931 { 1932 int i; 1933 char *dpp_addr; 1934 1935 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { 1936 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge); 1937 if (unlikely(hdr->total_len > qp->max_inline_data)) { 1938 pr_err("%s() supported_len=0x%x,\n" 1939 " unsupported len req=0x%x\n", __func__, 1940 qp->max_inline_data, hdr->total_len); 1941 return -EINVAL; 1942 } 1943 dpp_addr = (char *)sge; 1944 for (i = 0; i < wr->num_sge; i++) { 1945 memcpy(dpp_addr, 1946 (void *)(unsigned long)wr->sg_list[i].addr, 1947 wr->sg_list[i].length); 1948 dpp_addr += wr->sg_list[i].length; 1949 } 1950 1951 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 1952 if (0 == hdr->total_len) 1953 wqe_size += sizeof(struct ocrdma_sge); 1954 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 1955 } else { 1956 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1957 if (wr->num_sge) 1958 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge)); 1959 else 1960 wqe_size += sizeof(struct ocrdma_sge); 1961 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 1962 } 1963 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1964 return 0; 1965 } 1966 1967 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1968 const struct ib_send_wr *wr) 1969 { 1970 int status; 1971 struct ocrdma_sge *sge; 1972 u32 wqe_size = sizeof(*hdr); 1973 1974 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 1975 ocrdma_build_ud_hdr(qp, hdr, wr); 1976 sge = (struct ocrdma_sge *)(hdr + 2); 1977 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 1978 } else { 1979 sge = (struct ocrdma_sge *)(hdr + 1); 1980 } 1981 1982 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1983 return status; 1984 } 1985 1986 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1987 const struct ib_send_wr *wr) 1988 { 1989 int status; 1990 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1991 struct ocrdma_sge *sge = ext_rw + 1; 1992 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw); 1993 1994 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1995 if (status) 1996 return status; 1997 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 1998 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 1999 ext_rw->lrkey = rdma_wr(wr)->rkey; 2000 ext_rw->len = hdr->total_len; 2001 return 0; 2002 } 2003 2004 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 2005 const struct ib_send_wr *wr) 2006 { 2007 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 2008 struct ocrdma_sge *sge = ext_rw + 1; 2009 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) + 2010 sizeof(struct ocrdma_hdr_wqe); 2011 2012 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 2013 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2014 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT); 2015 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2016 2017 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2018 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2019 ext_rw->lrkey = rdma_wr(wr)->rkey; 2020 ext_rw->len = hdr->total_len; 2021 } 2022 2023 static int get_encoded_page_size(int pg_sz) 2024 { 2025 /* Max size is 256M 4096 << 16 */ 2026 int i = 0; 2027 for (; i < 17; i++) 2028 if (pg_sz == (4096 << i)) 2029 break; 2030 return i; 2031 } 2032 2033 static int ocrdma_build_reg(struct ocrdma_qp *qp, 2034 struct ocrdma_hdr_wqe *hdr, 2035 const struct ib_reg_wr *wr) 2036 { 2037 u64 fbo; 2038 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1); 2039 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr); 2040 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 2041 struct ocrdma_pbe *pbe; 2042 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr); 2043 int num_pbes = 0, i; 2044 2045 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES); 2046 2047 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT); 2048 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2049 2050 if (wr->access & IB_ACCESS_LOCAL_WRITE) 2051 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR; 2052 if (wr->access & IB_ACCESS_REMOTE_WRITE) 2053 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR; 2054 if (wr->access & IB_ACCESS_REMOTE_READ) 2055 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD; 2056 hdr->lkey = wr->key; 2057 hdr->total_len = mr->ibmr.length; 2058 2059 fbo = mr->ibmr.iova - mr->pages[0]; 2060 2061 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova); 2062 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff); 2063 fast_reg->fbo_hi = upper_32_bits(fbo); 2064 fast_reg->fbo_lo = (u32) fbo & 0xffffffff; 2065 fast_reg->num_sges = mr->npages; 2066 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size); 2067 2068 pbe = pbl_tbl->va; 2069 for (i = 0; i < mr->npages; i++) { 2070 u64 buf_addr = mr->pages[i]; 2071 2072 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK)); 2073 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr)); 2074 num_pbes += 1; 2075 pbe++; 2076 2077 /* if the pbl is full storing the pbes, 2078 * move to next pbl. 2079 */ 2080 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) { 2081 pbl_tbl++; 2082 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 2083 } 2084 } 2085 2086 return 0; 2087 } 2088 2089 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2090 { 2091 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); 2092 2093 iowrite32(val, qp->sq_db); 2094 } 2095 2096 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 2097 const struct ib_send_wr **bad_wr) 2098 { 2099 int status = 0; 2100 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2101 struct ocrdma_hdr_wqe *hdr; 2102 unsigned long flags; 2103 2104 spin_lock_irqsave(&qp->q_lock, flags); 2105 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { 2106 spin_unlock_irqrestore(&qp->q_lock, flags); 2107 *bad_wr = wr; 2108 return -EINVAL; 2109 } 2110 2111 while (wr) { 2112 if (qp->qp_type == IB_QPT_UD && 2113 (wr->opcode != IB_WR_SEND && 2114 wr->opcode != IB_WR_SEND_WITH_IMM)) { 2115 *bad_wr = wr; 2116 status = -EINVAL; 2117 break; 2118 } 2119 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || 2120 wr->num_sge > qp->sq.max_sges) { 2121 *bad_wr = wr; 2122 status = -ENOMEM; 2123 break; 2124 } 2125 hdr = ocrdma_hwq_head(&qp->sq); 2126 hdr->cw = 0; 2127 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2128 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2129 if (wr->send_flags & IB_SEND_FENCE) 2130 hdr->cw |= 2131 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT); 2132 if (wr->send_flags & IB_SEND_SOLICITED) 2133 hdr->cw |= 2134 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT); 2135 hdr->total_len = 0; 2136 switch (wr->opcode) { 2137 case IB_WR_SEND_WITH_IMM: 2138 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2139 hdr->immdt = ntohl(wr->ex.imm_data); 2140 fallthrough; 2141 case IB_WR_SEND: 2142 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2143 ocrdma_build_send(qp, hdr, wr); 2144 break; 2145 case IB_WR_SEND_WITH_INV: 2146 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT); 2147 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2148 hdr->lkey = wr->ex.invalidate_rkey; 2149 status = ocrdma_build_send(qp, hdr, wr); 2150 break; 2151 case IB_WR_RDMA_WRITE_WITH_IMM: 2152 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2153 hdr->immdt = ntohl(wr->ex.imm_data); 2154 fallthrough; 2155 case IB_WR_RDMA_WRITE: 2156 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT); 2157 status = ocrdma_build_write(qp, hdr, wr); 2158 break; 2159 case IB_WR_RDMA_READ: 2160 ocrdma_build_read(qp, hdr, wr); 2161 break; 2162 case IB_WR_LOCAL_INV: 2163 hdr->cw |= 2164 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2165 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) + 2166 sizeof(struct ocrdma_sge)) / 2167 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2168 hdr->lkey = wr->ex.invalidate_rkey; 2169 break; 2170 case IB_WR_REG_MR: 2171 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); 2172 break; 2173 default: 2174 status = -EINVAL; 2175 break; 2176 } 2177 if (status) { 2178 *bad_wr = wr; 2179 break; 2180 } 2181 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2182 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2183 else 2184 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2185 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; 2186 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) & 2187 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE); 2188 /* make sure wqe is written before adapter can access it */ 2189 wmb(); 2190 /* inform hw to start processing it */ 2191 ocrdma_ring_sq_db(qp); 2192 2193 /* update pointer, counter for next wr */ 2194 ocrdma_hwq_inc_head(&qp->sq); 2195 wr = wr->next; 2196 } 2197 spin_unlock_irqrestore(&qp->q_lock, flags); 2198 return status; 2199 } 2200 2201 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) 2202 { 2203 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); 2204 2205 iowrite32(val, qp->rq_db); 2206 } 2207 2208 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, 2209 const struct ib_recv_wr *wr, u16 tag) 2210 { 2211 u32 wqe_size = 0; 2212 struct ocrdma_sge *sge; 2213 if (wr->num_sge) 2214 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe); 2215 else 2216 wqe_size = sizeof(*sge) + sizeof(*rqe); 2217 2218 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << 2219 OCRDMA_WQE_SIZE_SHIFT); 2220 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2221 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2222 rqe->total_len = 0; 2223 rqe->rsvd_tag = tag; 2224 sge = (struct ocrdma_sge *)(rqe + 1); 2225 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list); 2226 ocrdma_cpu_to_le32(rqe, wqe_size); 2227 } 2228 2229 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 2230 const struct ib_recv_wr **bad_wr) 2231 { 2232 int status = 0; 2233 unsigned long flags; 2234 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2235 struct ocrdma_hdr_wqe *rqe; 2236 2237 spin_lock_irqsave(&qp->q_lock, flags); 2238 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { 2239 spin_unlock_irqrestore(&qp->q_lock, flags); 2240 *bad_wr = wr; 2241 return -EINVAL; 2242 } 2243 while (wr) { 2244 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || 2245 wr->num_sge > qp->rq.max_sges) { 2246 *bad_wr = wr; 2247 status = -ENOMEM; 2248 break; 2249 } 2250 rqe = ocrdma_hwq_head(&qp->rq); 2251 ocrdma_build_rqe(rqe, wr, 0); 2252 2253 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; 2254 /* make sure rqe is written before adapter can access it */ 2255 wmb(); 2256 2257 /* inform hw to start processing it */ 2258 ocrdma_ring_rq_db(qp); 2259 2260 /* update pointer, counter for next wr */ 2261 ocrdma_hwq_inc_head(&qp->rq); 2262 wr = wr->next; 2263 } 2264 spin_unlock_irqrestore(&qp->q_lock, flags); 2265 return status; 2266 } 2267 2268 /* cqe for srq's rqe can potentially arrive out of order. 2269 * index gives the entry in the shadow table where to store 2270 * the wr_id. tag/index is returned in cqe to reference back 2271 * for a given rqe. 2272 */ 2273 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) 2274 { 2275 int row = 0; 2276 int indx = 0; 2277 2278 for (row = 0; row < srq->bit_fields_len; row++) { 2279 if (srq->idx_bit_fields[row]) { 2280 indx = ffs(srq->idx_bit_fields[row]); 2281 indx = (row * 32) + (indx - 1); 2282 BUG_ON(indx >= srq->rq.max_cnt); 2283 ocrdma_srq_toggle_bit(srq, indx); 2284 break; 2285 } 2286 } 2287 2288 BUG_ON(row == srq->bit_fields_len); 2289 return indx + 1; /* Use from index 1 */ 2290 } 2291 2292 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) 2293 { 2294 u32 val = srq->rq.dbid | (1 << 16); 2295 2296 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET); 2297 } 2298 2299 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 2300 const struct ib_recv_wr **bad_wr) 2301 { 2302 int status = 0; 2303 unsigned long flags; 2304 struct ocrdma_srq *srq; 2305 struct ocrdma_hdr_wqe *rqe; 2306 u16 tag; 2307 2308 srq = get_ocrdma_srq(ibsrq); 2309 2310 spin_lock_irqsave(&srq->q_lock, flags); 2311 while (wr) { 2312 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 || 2313 wr->num_sge > srq->rq.max_sges) { 2314 status = -ENOMEM; 2315 *bad_wr = wr; 2316 break; 2317 } 2318 tag = ocrdma_srq_get_idx(srq); 2319 rqe = ocrdma_hwq_head(&srq->rq); 2320 ocrdma_build_rqe(rqe, wr, tag); 2321 2322 srq->rqe_wr_id_tbl[tag] = wr->wr_id; 2323 /* make sure rqe is written before adapter can perform DMA */ 2324 wmb(); 2325 /* inform hw to start processing it */ 2326 ocrdma_ring_srq_db(srq); 2327 /* update pointer, counter for next wr */ 2328 ocrdma_hwq_inc_head(&srq->rq); 2329 wr = wr->next; 2330 } 2331 spin_unlock_irqrestore(&srq->q_lock, flags); 2332 return status; 2333 } 2334 2335 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2336 { 2337 enum ib_wc_status ibwc_status; 2338 2339 switch (status) { 2340 case OCRDMA_CQE_GENERAL_ERR: 2341 ibwc_status = IB_WC_GENERAL_ERR; 2342 break; 2343 case OCRDMA_CQE_LOC_LEN_ERR: 2344 ibwc_status = IB_WC_LOC_LEN_ERR; 2345 break; 2346 case OCRDMA_CQE_LOC_QP_OP_ERR: 2347 ibwc_status = IB_WC_LOC_QP_OP_ERR; 2348 break; 2349 case OCRDMA_CQE_LOC_EEC_OP_ERR: 2350 ibwc_status = IB_WC_LOC_EEC_OP_ERR; 2351 break; 2352 case OCRDMA_CQE_LOC_PROT_ERR: 2353 ibwc_status = IB_WC_LOC_PROT_ERR; 2354 break; 2355 case OCRDMA_CQE_WR_FLUSH_ERR: 2356 ibwc_status = IB_WC_WR_FLUSH_ERR; 2357 break; 2358 case OCRDMA_CQE_MW_BIND_ERR: 2359 ibwc_status = IB_WC_MW_BIND_ERR; 2360 break; 2361 case OCRDMA_CQE_BAD_RESP_ERR: 2362 ibwc_status = IB_WC_BAD_RESP_ERR; 2363 break; 2364 case OCRDMA_CQE_LOC_ACCESS_ERR: 2365 ibwc_status = IB_WC_LOC_ACCESS_ERR; 2366 break; 2367 case OCRDMA_CQE_REM_INV_REQ_ERR: 2368 ibwc_status = IB_WC_REM_INV_REQ_ERR; 2369 break; 2370 case OCRDMA_CQE_REM_ACCESS_ERR: 2371 ibwc_status = IB_WC_REM_ACCESS_ERR; 2372 break; 2373 case OCRDMA_CQE_REM_OP_ERR: 2374 ibwc_status = IB_WC_REM_OP_ERR; 2375 break; 2376 case OCRDMA_CQE_RETRY_EXC_ERR: 2377 ibwc_status = IB_WC_RETRY_EXC_ERR; 2378 break; 2379 case OCRDMA_CQE_RNR_RETRY_EXC_ERR: 2380 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR; 2381 break; 2382 case OCRDMA_CQE_LOC_RDD_VIOL_ERR: 2383 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR; 2384 break; 2385 case OCRDMA_CQE_REM_INV_RD_REQ_ERR: 2386 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR; 2387 break; 2388 case OCRDMA_CQE_REM_ABORT_ERR: 2389 ibwc_status = IB_WC_REM_ABORT_ERR; 2390 break; 2391 case OCRDMA_CQE_INV_EECN_ERR: 2392 ibwc_status = IB_WC_INV_EECN_ERR; 2393 break; 2394 case OCRDMA_CQE_INV_EEC_STATE_ERR: 2395 ibwc_status = IB_WC_INV_EEC_STATE_ERR; 2396 break; 2397 case OCRDMA_CQE_FATAL_ERR: 2398 ibwc_status = IB_WC_FATAL_ERR; 2399 break; 2400 case OCRDMA_CQE_RESP_TIMEOUT_ERR: 2401 ibwc_status = IB_WC_RESP_TIMEOUT_ERR; 2402 break; 2403 default: 2404 ibwc_status = IB_WC_GENERAL_ERR; 2405 break; 2406 } 2407 return ibwc_status; 2408 } 2409 2410 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, 2411 u32 wqe_idx) 2412 { 2413 struct ocrdma_hdr_wqe *hdr; 2414 struct ocrdma_sge *rw; 2415 int opcode; 2416 2417 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); 2418 2419 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; 2420 /* Undo the hdr->cw swap */ 2421 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK; 2422 switch (opcode) { 2423 case OCRDMA_WRITE: 2424 ibwc->opcode = IB_WC_RDMA_WRITE; 2425 break; 2426 case OCRDMA_READ: 2427 rw = (struct ocrdma_sge *)(hdr + 1); 2428 ibwc->opcode = IB_WC_RDMA_READ; 2429 ibwc->byte_len = rw->len; 2430 break; 2431 case OCRDMA_SEND: 2432 ibwc->opcode = IB_WC_SEND; 2433 break; 2434 case OCRDMA_FR_MR: 2435 ibwc->opcode = IB_WC_REG_MR; 2436 break; 2437 case OCRDMA_LKEY_INV: 2438 ibwc->opcode = IB_WC_LOCAL_INV; 2439 break; 2440 default: 2441 ibwc->status = IB_WC_GENERAL_ERR; 2442 pr_err("%s() invalid opcode received = 0x%x\n", 2443 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK); 2444 break; 2445 } 2446 } 2447 2448 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, 2449 struct ocrdma_cqe *cqe) 2450 { 2451 if (is_cqe_for_sq(cqe)) { 2452 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2453 cqe->flags_status_srcqpn) & 2454 ~OCRDMA_CQE_STATUS_MASK); 2455 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2456 cqe->flags_status_srcqpn) | 2457 (OCRDMA_CQE_WR_FLUSH_ERR << 2458 OCRDMA_CQE_STATUS_SHIFT)); 2459 } else { 2460 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2461 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2462 cqe->flags_status_srcqpn) & 2463 ~OCRDMA_CQE_UD_STATUS_MASK); 2464 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2465 cqe->flags_status_srcqpn) | 2466 (OCRDMA_CQE_WR_FLUSH_ERR << 2467 OCRDMA_CQE_UD_STATUS_SHIFT)); 2468 } else { 2469 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2470 cqe->flags_status_srcqpn) & 2471 ~OCRDMA_CQE_STATUS_MASK); 2472 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2473 cqe->flags_status_srcqpn) | 2474 (OCRDMA_CQE_WR_FLUSH_ERR << 2475 OCRDMA_CQE_STATUS_SHIFT)); 2476 } 2477 } 2478 } 2479 2480 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2481 struct ocrdma_qp *qp, int status) 2482 { 2483 bool expand = false; 2484 2485 ibwc->byte_len = 0; 2486 ibwc->qp = &qp->ibqp; 2487 ibwc->status = ocrdma_to_ibwc_err(status); 2488 2489 ocrdma_flush_qp(qp); 2490 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); 2491 2492 /* if wqe/rqe pending for which cqe needs to be returned, 2493 * trigger inflating it. 2494 */ 2495 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { 2496 expand = true; 2497 ocrdma_set_cqe_status_flushed(qp, cqe); 2498 } 2499 return expand; 2500 } 2501 2502 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2503 struct ocrdma_qp *qp, int status) 2504 { 2505 ibwc->opcode = IB_WC_RECV; 2506 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2507 ocrdma_hwq_inc_tail(&qp->rq); 2508 2509 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2510 } 2511 2512 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2513 struct ocrdma_qp *qp, int status) 2514 { 2515 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2516 ocrdma_hwq_inc_tail(&qp->sq); 2517 2518 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2519 } 2520 2521 2522 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, 2523 struct ocrdma_cqe *cqe, struct ib_wc *ibwc, 2524 bool *polled, bool *stop) 2525 { 2526 bool expand; 2527 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2528 int status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2529 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2530 if (status < OCRDMA_MAX_CQE_ERR) 2531 atomic_inc(&dev->cqe_err_stats[status]); 2532 2533 /* when hw sq is empty, but rq is not empty, so we continue 2534 * to keep the cqe in order to get the cq event again. 2535 */ 2536 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { 2537 /* when cq for rq and sq is same, it is safe to return 2538 * flush cqe for RQEs. 2539 */ 2540 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2541 *polled = true; 2542 status = OCRDMA_CQE_WR_FLUSH_ERR; 2543 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2544 } else { 2545 /* stop processing further cqe as this cqe is used for 2546 * triggering cq event on buddy cq of RQ. 2547 * When QP is destroyed, this cqe will be removed 2548 * from the cq's hardware q. 2549 */ 2550 *polled = false; 2551 *stop = true; 2552 expand = false; 2553 } 2554 } else if (is_hw_sq_empty(qp)) { 2555 /* Do nothing */ 2556 expand = false; 2557 *polled = false; 2558 *stop = false; 2559 } else { 2560 *polled = true; 2561 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2562 } 2563 return expand; 2564 } 2565 2566 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, 2567 struct ocrdma_cqe *cqe, 2568 struct ib_wc *ibwc, bool *polled) 2569 { 2570 bool expand = false; 2571 int tail = qp->sq.tail; 2572 u32 wqe_idx; 2573 2574 if (!qp->wqe_wr_id_tbl[tail].signaled) { 2575 *polled = false; /* WC cannot be consumed yet */ 2576 } else { 2577 ibwc->status = IB_WC_SUCCESS; 2578 ibwc->wc_flags = 0; 2579 ibwc->qp = &qp->ibqp; 2580 ocrdma_update_wc(qp, ibwc, tail); 2581 *polled = true; 2582 } 2583 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) & 2584 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; 2585 if (tail != wqe_idx) 2586 expand = true; /* Coalesced CQE can't be consumed yet */ 2587 2588 ocrdma_hwq_inc_tail(&qp->sq); 2589 return expand; 2590 } 2591 2592 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2593 struct ib_wc *ibwc, bool *polled, bool *stop) 2594 { 2595 int status; 2596 bool expand; 2597 2598 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2599 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2600 2601 if (status == OCRDMA_CQE_SUCCESS) 2602 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); 2603 else 2604 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); 2605 return expand; 2606 } 2607 2608 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, 2609 struct ocrdma_cqe *cqe) 2610 { 2611 int status; 2612 u16 hdr_type = 0; 2613 2614 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2615 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT; 2616 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) & 2617 OCRDMA_CQE_SRCQP_MASK; 2618 ibwc->pkey_index = 0; 2619 ibwc->wc_flags = IB_WC_GRH; 2620 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2621 OCRDMA_CQE_UD_XFER_LEN_SHIFT) & 2622 OCRDMA_CQE_UD_XFER_LEN_MASK; 2623 2624 if (ocrdma_is_udp_encap_supported(dev)) { 2625 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2626 OCRDMA_CQE_UD_L3TYPE_SHIFT) & 2627 OCRDMA_CQE_UD_L3TYPE_MASK; 2628 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2629 ibwc->network_hdr_type = hdr_type; 2630 } 2631 2632 return status; 2633 } 2634 2635 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, 2636 struct ocrdma_cqe *cqe, 2637 struct ocrdma_qp *qp) 2638 { 2639 unsigned long flags; 2640 struct ocrdma_srq *srq; 2641 u32 wqe_idx; 2642 2643 srq = get_ocrdma_srq(qp->ibqp.srq); 2644 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 2645 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx; 2646 BUG_ON(wqe_idx < 1); 2647 2648 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2649 spin_lock_irqsave(&srq->q_lock, flags); 2650 ocrdma_srq_toggle_bit(srq, wqe_idx - 1); 2651 spin_unlock_irqrestore(&srq->q_lock, flags); 2652 ocrdma_hwq_inc_tail(&srq->rq); 2653 } 2654 2655 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2656 struct ib_wc *ibwc, bool *polled, bool *stop, 2657 int status) 2658 { 2659 bool expand; 2660 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2661 2662 if (status < OCRDMA_MAX_CQE_ERR) 2663 atomic_inc(&dev->cqe_err_stats[status]); 2664 2665 /* when hw_rq is empty, but wq is not empty, so continue 2666 * to keep the cqe to get the cq event again. 2667 */ 2668 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { 2669 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2670 *polled = true; 2671 status = OCRDMA_CQE_WR_FLUSH_ERR; 2672 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2673 } else { 2674 *polled = false; 2675 *stop = true; 2676 expand = false; 2677 } 2678 } else if (is_hw_rq_empty(qp)) { 2679 /* Do nothing */ 2680 expand = false; 2681 *polled = false; 2682 *stop = false; 2683 } else { 2684 *polled = true; 2685 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2686 } 2687 return expand; 2688 } 2689 2690 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, 2691 struct ocrdma_cqe *cqe, struct ib_wc *ibwc) 2692 { 2693 struct ocrdma_dev *dev; 2694 2695 dev = get_ocrdma_dev(qp->ibqp.device); 2696 ibwc->opcode = IB_WC_RECV; 2697 ibwc->qp = &qp->ibqp; 2698 ibwc->status = IB_WC_SUCCESS; 2699 2700 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2701 ocrdma_update_ud_rcqe(dev, ibwc, cqe); 2702 else 2703 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen); 2704 2705 if (is_cqe_imm(cqe)) { 2706 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2707 ibwc->wc_flags |= IB_WC_WITH_IMM; 2708 } else if (is_cqe_wr_imm(cqe)) { 2709 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2710 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2711 ibwc->wc_flags |= IB_WC_WITH_IMM; 2712 } else if (is_cqe_invalidated(cqe)) { 2713 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2714 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2715 } 2716 if (qp->ibqp.srq) { 2717 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2718 } else { 2719 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2720 ocrdma_hwq_inc_tail(&qp->rq); 2721 } 2722 } 2723 2724 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2725 struct ib_wc *ibwc, bool *polled, bool *stop) 2726 { 2727 int status; 2728 bool expand = false; 2729 2730 ibwc->wc_flags = 0; 2731 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2732 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2733 OCRDMA_CQE_UD_STATUS_MASK) >> 2734 OCRDMA_CQE_UD_STATUS_SHIFT; 2735 } else { 2736 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2737 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2738 } 2739 2740 if (status == OCRDMA_CQE_SUCCESS) { 2741 *polled = true; 2742 ocrdma_poll_success_rcqe(qp, cqe, ibwc); 2743 } else { 2744 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, 2745 status); 2746 } 2747 return expand; 2748 } 2749 2750 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, 2751 u16 cur_getp) 2752 { 2753 if (cq->phase_change) { 2754 if (cur_getp == 0) 2755 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2756 } else { 2757 /* clear valid bit */ 2758 cqe->flags_status_srcqpn = 0; 2759 } 2760 } 2761 2762 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2763 struct ib_wc *ibwc) 2764 { 2765 u16 qpn = 0; 2766 int i = 0; 2767 bool expand = false; 2768 int polled_hw_cqes = 0; 2769 struct ocrdma_qp *qp = NULL; 2770 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 2771 struct ocrdma_cqe *cqe; 2772 u16 cur_getp; bool polled = false; bool stop = false; 2773 2774 cur_getp = cq->getp; 2775 while (num_entries) { 2776 cqe = cq->va + cur_getp; 2777 /* check whether valid cqe or not */ 2778 if (!is_cqe_valid(cq, cqe)) 2779 break; 2780 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK); 2781 /* ignore discarded cqe */ 2782 if (qpn == 0) 2783 goto skip_cqe; 2784 qp = dev->qp_tbl[qpn]; 2785 BUG_ON(qp == NULL); 2786 2787 if (is_cqe_for_sq(cqe)) { 2788 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, 2789 &stop); 2790 } else { 2791 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, 2792 &stop); 2793 } 2794 if (expand) 2795 goto expand_cqe; 2796 if (stop) 2797 goto stop_cqe; 2798 /* clear qpn to avoid duplicate processing by discard_cqe() */ 2799 cqe->cmn.qpn = 0; 2800 skip_cqe: 2801 polled_hw_cqes += 1; 2802 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 2803 ocrdma_change_cq_phase(cq, cqe, cur_getp); 2804 expand_cqe: 2805 if (polled) { 2806 num_entries -= 1; 2807 i += 1; 2808 ibwc = ibwc + 1; 2809 polled = false; 2810 } 2811 } 2812 stop_cqe: 2813 cq->getp = cur_getp; 2814 2815 if (polled_hw_cqes) 2816 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes); 2817 2818 return i; 2819 } 2820 2821 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ 2822 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, 2823 struct ocrdma_qp *qp, struct ib_wc *ibwc) 2824 { 2825 int err_cqes = 0; 2826 2827 while (num_entries) { 2828 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) 2829 break; 2830 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { 2831 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2832 ocrdma_hwq_inc_tail(&qp->sq); 2833 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2834 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2835 ocrdma_hwq_inc_tail(&qp->rq); 2836 } else { 2837 return err_cqes; 2838 } 2839 ibwc->byte_len = 0; 2840 ibwc->status = IB_WC_WR_FLUSH_ERR; 2841 ibwc = ibwc + 1; 2842 err_cqes += 1; 2843 num_entries -= 1; 2844 } 2845 return err_cqes; 2846 } 2847 2848 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2849 { 2850 int cqes_to_poll = num_entries; 2851 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2852 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2853 int num_os_cqe = 0, err_cqes = 0; 2854 struct ocrdma_qp *qp; 2855 unsigned long flags; 2856 2857 /* poll cqes from adapter CQ */ 2858 spin_lock_irqsave(&cq->cq_lock, flags); 2859 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc); 2860 spin_unlock_irqrestore(&cq->cq_lock, flags); 2861 cqes_to_poll -= num_os_cqe; 2862 2863 if (cqes_to_poll) { 2864 wc = wc + num_os_cqe; 2865 /* adapter returns single error cqe when qp moves to 2866 * error state. So insert error cqes with wc_status as 2867 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ 2868 * respectively which uses this CQ. 2869 */ 2870 spin_lock_irqsave(&dev->flush_q_lock, flags); 2871 list_for_each_entry(qp, &cq->sq_head, sq_entry) { 2872 if (cqes_to_poll == 0) 2873 break; 2874 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); 2875 cqes_to_poll -= err_cqes; 2876 num_os_cqe += err_cqes; 2877 wc = wc + err_cqes; 2878 } 2879 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 2880 } 2881 return num_os_cqe; 2882 } 2883 2884 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2885 { 2886 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2887 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2888 u16 cq_id; 2889 unsigned long flags; 2890 bool arm_needed = false, sol_needed = false; 2891 2892 cq_id = cq->id; 2893 2894 spin_lock_irqsave(&cq->cq_lock, flags); 2895 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 2896 arm_needed = true; 2897 if (cq_flags & IB_CQ_SOLICITED) 2898 sol_needed = true; 2899 2900 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0); 2901 spin_unlock_irqrestore(&cq->cq_lock, flags); 2902 2903 return 0; 2904 } 2905 2906 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2907 u32 max_num_sg) 2908 { 2909 int status; 2910 struct ocrdma_mr *mr; 2911 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 2912 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 2913 2914 if (mr_type != IB_MR_TYPE_MEM_REG) 2915 return ERR_PTR(-EINVAL); 2916 2917 if (max_num_sg > dev->attr.max_pages_per_frmr) 2918 return ERR_PTR(-EINVAL); 2919 2920 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2921 if (!mr) 2922 return ERR_PTR(-ENOMEM); 2923 2924 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 2925 if (!mr->pages) { 2926 status = -ENOMEM; 2927 goto pl_err; 2928 } 2929 2930 status = ocrdma_get_pbl_info(dev, mr, max_num_sg); 2931 if (status) 2932 goto pbl_err; 2933 mr->hwmr.fr_mr = 1; 2934 mr->hwmr.remote_rd = 0; 2935 mr->hwmr.remote_wr = 0; 2936 mr->hwmr.local_rd = 0; 2937 mr->hwmr.local_wr = 0; 2938 mr->hwmr.mw_bind = 0; 2939 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 2940 if (status) 2941 goto pbl_err; 2942 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0); 2943 if (status) 2944 goto mbx_err; 2945 mr->ibmr.rkey = mr->hwmr.lkey; 2946 mr->ibmr.lkey = mr->hwmr.lkey; 2947 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = 2948 (unsigned long) mr; 2949 return &mr->ibmr; 2950 mbx_err: 2951 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 2952 pbl_err: 2953 kfree(mr->pages); 2954 pl_err: 2955 kfree(mr); 2956 return ERR_PTR(-ENOMEM); 2957 } 2958 2959 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) 2960 { 2961 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2962 2963 if (unlikely(mr->npages == mr->hwmr.num_pbes)) 2964 return -ENOMEM; 2965 2966 mr->pages[mr->npages++] = addr; 2967 2968 return 0; 2969 } 2970 2971 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2972 unsigned int *sg_offset) 2973 { 2974 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2975 2976 mr->npages = 0; 2977 2978 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page); 2979 } 2980