1 /* This file is part of the Emulex RoCE Device Driver for 2 * RoCE (RDMA over Converged Ethernet) adapters. 3 * Copyright (C) 2012-2015 Emulex. All rights reserved. 4 * EMULEX and SLI are trademarks of Emulex. 5 * www.emulex.com 6 * 7 * This software is available to you under a choice of one of two licenses. 8 * You may choose to be licensed under the terms of the GNU General Public 9 * License (GPL) Version 2, available from the file COPYING in the main 10 * directory of this source tree, or the BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * - Redistributions of source code must retain the above copyright notice, 17 * this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * Contact Information: 36 * linux-drivers@emulex.com 37 * 38 * Emulex 39 * 3333 Susan Street 40 * Costa Mesa, CA 92626 41 */ 42 43 #include <linux/dma-mapping.h> 44 #include <rdma/ib_verbs.h> 45 #include <rdma/ib_user_verbs.h> 46 #include <rdma/iw_cm.h> 47 #include <rdma/ib_umem.h> 48 #include <rdma/ib_addr.h> 49 #include <rdma/ib_cache.h> 50 #include <rdma/uverbs_ioctl.h> 51 52 #include "ocrdma.h" 53 #include "ocrdma_hw.h" 54 #include "ocrdma_verbs.h" 55 #include <rdma/ocrdma-abi.h> 56 57 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 58 { 59 if (index > 0) 60 return -EINVAL; 61 62 *pkey = 0xffff; 63 return 0; 64 } 65 66 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 67 struct ib_udata *uhw) 68 { 69 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 70 71 if (uhw->inlen || uhw->outlen) 72 return -EINVAL; 73 74 memset(attr, 0, sizeof *attr); 75 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], 76 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); 77 ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid); 78 attr->max_mr_size = dev->attr.max_mr_size; 79 attr->page_size_cap = 0xffff000; 80 attr->vendor_id = dev->nic_info.pdev->vendor; 81 attr->vendor_part_id = dev->nic_info.pdev->device; 82 attr->hw_ver = dev->asic_id; 83 attr->max_qp = dev->attr.max_qp; 84 attr->max_ah = OCRDMA_MAX_AH; 85 attr->max_qp_wr = dev->attr.max_wqe; 86 87 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 88 IB_DEVICE_RC_RNR_NAK_GEN | 89 IB_DEVICE_SHUTDOWN_PORT | 90 IB_DEVICE_SYS_IMAGE_GUID | 91 IB_DEVICE_LOCAL_DMA_LKEY | 92 IB_DEVICE_MEM_MGT_EXTENSIONS; 93 attr->max_send_sge = dev->attr.max_send_sge; 94 attr->max_recv_sge = dev->attr.max_recv_sge; 95 attr->max_sge_rd = dev->attr.max_rdma_sge; 96 attr->max_cq = dev->attr.max_cq; 97 attr->max_cqe = dev->attr.max_cqe; 98 attr->max_mr = dev->attr.max_mr; 99 attr->max_mw = dev->attr.max_mw; 100 attr->max_pd = dev->attr.max_pd; 101 attr->atomic_cap = 0; 102 attr->max_fmr = 0; 103 attr->max_map_per_fmr = 0; 104 attr->max_qp_rd_atom = 105 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 106 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 107 attr->max_srq = dev->attr.max_srq; 108 attr->max_srq_sge = dev->attr.max_srq_sge; 109 attr->max_srq_wr = dev->attr.max_rqe; 110 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 111 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; 112 attr->max_pkeys = 1; 113 return 0; 114 } 115 116 static inline void get_link_speed_and_width(struct ocrdma_dev *dev, 117 u8 *ib_speed, u8 *ib_width) 118 { 119 int status; 120 u8 speed; 121 122 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL); 123 if (status) 124 speed = OCRDMA_PHYS_LINK_SPEED_ZERO; 125 126 switch (speed) { 127 case OCRDMA_PHYS_LINK_SPEED_1GBPS: 128 *ib_speed = IB_SPEED_SDR; 129 *ib_width = IB_WIDTH_1X; 130 break; 131 132 case OCRDMA_PHYS_LINK_SPEED_10GBPS: 133 *ib_speed = IB_SPEED_QDR; 134 *ib_width = IB_WIDTH_1X; 135 break; 136 137 case OCRDMA_PHYS_LINK_SPEED_20GBPS: 138 *ib_speed = IB_SPEED_DDR; 139 *ib_width = IB_WIDTH_4X; 140 break; 141 142 case OCRDMA_PHYS_LINK_SPEED_40GBPS: 143 *ib_speed = IB_SPEED_QDR; 144 *ib_width = IB_WIDTH_4X; 145 break; 146 147 default: 148 /* Unsupported */ 149 *ib_speed = IB_SPEED_SDR; 150 *ib_width = IB_WIDTH_1X; 151 } 152 } 153 154 int ocrdma_query_port(struct ib_device *ibdev, 155 u8 port, struct ib_port_attr *props) 156 { 157 enum ib_port_state port_state; 158 struct ocrdma_dev *dev; 159 struct net_device *netdev; 160 161 /* props being zeroed by the caller, avoid zeroing it here */ 162 dev = get_ocrdma_dev(ibdev); 163 netdev = dev->nic_info.netdev; 164 if (netif_running(netdev) && netif_oper_up(netdev)) { 165 port_state = IB_PORT_ACTIVE; 166 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 167 } else { 168 port_state = IB_PORT_DOWN; 169 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 170 } 171 props->max_mtu = IB_MTU_4096; 172 props->active_mtu = iboe_get_mtu(netdev->mtu); 173 props->lid = 0; 174 props->lmc = 0; 175 props->sm_lid = 0; 176 props->sm_sl = 0; 177 props->state = port_state; 178 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 179 IB_PORT_DEVICE_MGMT_SUP | 180 IB_PORT_VENDOR_CLASS_SUP; 181 props->ip_gids = true; 182 props->gid_tbl_len = OCRDMA_MAX_SGID; 183 props->pkey_tbl_len = 1; 184 props->bad_pkey_cntr = 0; 185 props->qkey_viol_cntr = 0; 186 get_link_speed_and_width(dev, &props->active_speed, 187 &props->active_width); 188 props->max_msg_sz = 0x80000000; 189 props->max_vl_num = 4; 190 return 0; 191 } 192 193 int ocrdma_modify_port(struct ib_device *ibdev, u8 port, int mask, 194 struct ib_port_modify *props) 195 { 196 return 0; 197 } 198 199 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 200 unsigned long len) 201 { 202 struct ocrdma_mm *mm; 203 204 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 205 if (mm == NULL) 206 return -ENOMEM; 207 mm->key.phy_addr = phy_addr; 208 mm->key.len = len; 209 INIT_LIST_HEAD(&mm->entry); 210 211 mutex_lock(&uctx->mm_list_lock); 212 list_add_tail(&mm->entry, &uctx->mm_head); 213 mutex_unlock(&uctx->mm_list_lock); 214 return 0; 215 } 216 217 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 218 unsigned long len) 219 { 220 struct ocrdma_mm *mm, *tmp; 221 222 mutex_lock(&uctx->mm_list_lock); 223 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 224 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 225 continue; 226 227 list_del(&mm->entry); 228 kfree(mm); 229 break; 230 } 231 mutex_unlock(&uctx->mm_list_lock); 232 } 233 234 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, 235 unsigned long len) 236 { 237 bool found = false; 238 struct ocrdma_mm *mm; 239 240 mutex_lock(&uctx->mm_list_lock); 241 list_for_each_entry(mm, &uctx->mm_head, entry) { 242 if (len != mm->key.len && phy_addr != mm->key.phy_addr) 243 continue; 244 245 found = true; 246 break; 247 } 248 mutex_unlock(&uctx->mm_list_lock); 249 return found; 250 } 251 252 253 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) 254 { 255 u16 pd_bitmap_idx = 0; 256 const unsigned long *pd_bitmap; 257 258 if (dpp_pool) { 259 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap; 260 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 261 dev->pd_mgr->max_dpp_pd); 262 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap); 263 dev->pd_mgr->pd_dpp_count++; 264 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh) 265 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count; 266 } else { 267 pd_bitmap = dev->pd_mgr->pd_norm_bitmap; 268 pd_bitmap_idx = find_first_zero_bit(pd_bitmap, 269 dev->pd_mgr->max_normal_pd); 270 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap); 271 dev->pd_mgr->pd_norm_count++; 272 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh) 273 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count; 274 } 275 return pd_bitmap_idx; 276 } 277 278 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, 279 bool dpp_pool) 280 { 281 u16 pd_count; 282 u16 pd_bit_index; 283 284 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count : 285 dev->pd_mgr->pd_norm_count; 286 if (pd_count == 0) 287 return -EINVAL; 288 289 if (dpp_pool) { 290 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start; 291 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) { 292 return -EINVAL; 293 } else { 294 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap); 295 dev->pd_mgr->pd_dpp_count--; 296 } 297 } else { 298 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start; 299 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) { 300 return -EINVAL; 301 } else { 302 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap); 303 dev->pd_mgr->pd_norm_count--; 304 } 305 } 306 307 return 0; 308 } 309 310 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, 311 bool dpp_pool) 312 { 313 int status; 314 315 mutex_lock(&dev->dev_lock); 316 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool); 317 mutex_unlock(&dev->dev_lock); 318 return status; 319 } 320 321 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 322 { 323 u16 pd_idx = 0; 324 int status = 0; 325 326 mutex_lock(&dev->dev_lock); 327 if (pd->dpp_enabled) { 328 /* try allocating DPP PD, if not available then normal PD */ 329 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) { 330 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true); 331 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx; 332 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx; 333 } else if (dev->pd_mgr->pd_norm_count < 334 dev->pd_mgr->max_normal_pd) { 335 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 336 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 337 pd->dpp_enabled = false; 338 } else { 339 status = -EINVAL; 340 } 341 } else { 342 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) { 343 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false); 344 pd->id = dev->pd_mgr->pd_norm_start + pd_idx; 345 } else { 346 status = -EINVAL; 347 } 348 } 349 mutex_unlock(&dev->dev_lock); 350 return status; 351 } 352 353 /* 354 * NOTE: 355 * 356 * ocrdma_ucontext must be used here because this function is also 357 * called from ocrdma_alloc_ucontext where ib_udata does not have 358 * valid ib_ucontext pointer. ib_uverbs_get_context does not call 359 * uobj_{alloc|get_xxx} helpers which are used to store the 360 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so 361 * ib_udata does NOT imply valid ib_ucontext here! 362 */ 363 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 364 struct ocrdma_ucontext *uctx, 365 struct ib_udata *udata) 366 { 367 int status; 368 369 if (udata && uctx && dev->attr.max_dpp_pds) { 370 pd->dpp_enabled = 371 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R; 372 pd->num_dpp_qp = 373 pd->dpp_enabled ? (dev->nic_info.db_page_size / 374 dev->attr.wqe_size) : 0; 375 } 376 377 if (dev->pd_mgr->pd_prealloc_valid) 378 return ocrdma_get_pd_num(dev, pd); 379 380 retry: 381 status = ocrdma_mbx_alloc_pd(dev, pd); 382 if (status) { 383 if (pd->dpp_enabled) { 384 pd->dpp_enabled = false; 385 pd->num_dpp_qp = 0; 386 goto retry; 387 } 388 return status; 389 } 390 391 return 0; 392 } 393 394 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, 395 struct ocrdma_pd *pd) 396 { 397 return (uctx->cntxt_pd == pd); 398 } 399 400 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev, 401 struct ocrdma_pd *pd) 402 { 403 if (dev->pd_mgr->pd_prealloc_valid) 404 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled); 405 else 406 ocrdma_mbx_dealloc_pd(dev, pd); 407 } 408 409 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, 410 struct ocrdma_ucontext *uctx, 411 struct ib_udata *udata) 412 { 413 struct ib_device *ibdev = &dev->ibdev; 414 struct ib_pd *pd; 415 int status; 416 417 pd = rdma_zalloc_drv_obj(ibdev, ib_pd); 418 if (!pd) 419 return -ENOMEM; 420 421 pd->device = ibdev; 422 uctx->cntxt_pd = get_ocrdma_pd(pd); 423 424 status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata); 425 if (status) { 426 kfree(uctx->cntxt_pd); 427 goto err; 428 } 429 430 uctx->cntxt_pd->uctx = uctx; 431 uctx->cntxt_pd->ibpd.device = &dev->ibdev; 432 err: 433 return status; 434 } 435 436 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) 437 { 438 struct ocrdma_pd *pd = uctx->cntxt_pd; 439 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 440 441 if (uctx->pd_in_use) { 442 pr_err("%s(%d) Freeing in use pdid=0x%x.\n", 443 __func__, dev->id, pd->id); 444 } 445 kfree(uctx->cntxt_pd); 446 uctx->cntxt_pd = NULL; 447 _ocrdma_dealloc_pd(dev, pd); 448 } 449 450 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) 451 { 452 struct ocrdma_pd *pd = NULL; 453 454 mutex_lock(&uctx->mm_list_lock); 455 if (!uctx->pd_in_use) { 456 uctx->pd_in_use = true; 457 pd = uctx->cntxt_pd; 458 } 459 mutex_unlock(&uctx->mm_list_lock); 460 461 return pd; 462 } 463 464 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) 465 { 466 mutex_lock(&uctx->mm_list_lock); 467 uctx->pd_in_use = false; 468 mutex_unlock(&uctx->mm_list_lock); 469 } 470 471 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 472 { 473 struct ib_device *ibdev = uctx->device; 474 int status; 475 struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx); 476 struct ocrdma_alloc_ucontext_resp resp = {}; 477 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 478 struct pci_dev *pdev = dev->nic_info.pdev; 479 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE); 480 481 if (!udata) 482 return -EFAULT; 483 INIT_LIST_HEAD(&ctx->mm_head); 484 mutex_init(&ctx->mm_list_lock); 485 486 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len, 487 &ctx->ah_tbl.pa, GFP_KERNEL); 488 if (!ctx->ah_tbl.va) 489 return -ENOMEM; 490 491 ctx->ah_tbl.len = map_len; 492 493 resp.ah_tbl_len = ctx->ah_tbl.len; 494 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va); 495 496 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 497 if (status) 498 goto map_err; 499 500 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata); 501 if (status) 502 goto pd_err; 503 504 resp.dev_id = dev->id; 505 resp.max_inline_data = dev->attr.max_inline_data; 506 resp.wqe_size = dev->attr.wqe_size; 507 resp.rqe_size = dev->attr.rqe_size; 508 resp.dpp_wqe_size = dev->attr.wqe_size; 509 510 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver)); 511 status = ib_copy_to_udata(udata, &resp, sizeof(resp)); 512 if (status) 513 goto cpy_err; 514 return 0; 515 516 cpy_err: 517 ocrdma_dealloc_ucontext_pd(ctx); 518 pd_err: 519 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 520 map_err: 521 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 522 ctx->ah_tbl.pa); 523 return status; 524 } 525 526 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 527 { 528 struct ocrdma_mm *mm, *tmp; 529 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 530 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device); 531 struct pci_dev *pdev = dev->nic_info.pdev; 532 533 ocrdma_dealloc_ucontext_pd(uctx); 534 535 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 536 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 537 uctx->ah_tbl.pa); 538 539 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 540 list_del(&mm->entry); 541 kfree(mm); 542 } 543 } 544 545 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 546 { 547 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 548 struct ocrdma_dev *dev = get_ocrdma_dev(context->device); 549 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 550 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 551 unsigned long len = (vma->vm_end - vma->vm_start); 552 int status; 553 bool found; 554 555 if (vma->vm_start & (PAGE_SIZE - 1)) 556 return -EINVAL; 557 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len); 558 if (!found) 559 return -EINVAL; 560 561 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 562 dev->nic_info.db_total_size)) && 563 (len <= dev->nic_info.db_page_size)) { 564 if (vma->vm_flags & VM_READ) 565 return -EPERM; 566 567 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 568 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 569 len, vma->vm_page_prot); 570 } else if (dev->nic_info.dpp_unmapped_len && 571 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) && 572 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 573 dev->nic_info.dpp_unmapped_len)) && 574 (len <= dev->nic_info.dpp_unmapped_len)) { 575 if (vma->vm_flags & VM_READ) 576 return -EPERM; 577 578 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 579 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 580 len, vma->vm_page_prot); 581 } else { 582 status = remap_pfn_range(vma, vma->vm_start, 583 vma->vm_pgoff, len, vma->vm_page_prot); 584 } 585 return status; 586 } 587 588 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, 589 struct ib_udata *udata) 590 { 591 int status; 592 u64 db_page_addr; 593 u64 dpp_page_addr = 0; 594 u32 db_page_size; 595 struct ocrdma_alloc_pd_uresp rsp; 596 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 597 udata, struct ocrdma_ucontext, ibucontext); 598 599 memset(&rsp, 0, sizeof(rsp)); 600 rsp.id = pd->id; 601 rsp.dpp_enabled = pd->dpp_enabled; 602 db_page_addr = ocrdma_get_db_addr(dev, pd->id); 603 db_page_size = dev->nic_info.db_page_size; 604 605 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 606 if (status) 607 return status; 608 609 if (pd->dpp_enabled) { 610 dpp_page_addr = dev->nic_info.dpp_unmapped_addr + 611 (pd->id * PAGE_SIZE); 612 status = ocrdma_add_mmap(uctx, dpp_page_addr, 613 PAGE_SIZE); 614 if (status) 615 goto dpp_map_err; 616 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 617 rsp.dpp_page_addr_lo = dpp_page_addr; 618 } 619 620 status = ib_copy_to_udata(udata, &rsp, sizeof(rsp)); 621 if (status) 622 goto ucopy_err; 623 624 pd->uctx = uctx; 625 return 0; 626 627 ucopy_err: 628 if (pd->dpp_enabled) 629 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE); 630 dpp_map_err: 631 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size); 632 return status; 633 } 634 635 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 636 { 637 struct ib_device *ibdev = ibpd->device; 638 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 639 struct ocrdma_pd *pd; 640 int status; 641 u8 is_uctx_pd = false; 642 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 643 udata, struct ocrdma_ucontext, ibucontext); 644 645 if (udata) { 646 pd = ocrdma_get_ucontext_pd(uctx); 647 if (pd) { 648 is_uctx_pd = true; 649 goto pd_mapping; 650 } 651 } 652 653 pd = get_ocrdma_pd(ibpd); 654 status = _ocrdma_alloc_pd(dev, pd, uctx, udata); 655 if (status) 656 goto exit; 657 658 pd_mapping: 659 if (udata) { 660 status = ocrdma_copy_pd_uresp(dev, pd, udata); 661 if (status) 662 goto err; 663 } 664 return 0; 665 666 err: 667 if (is_uctx_pd) 668 ocrdma_release_ucontext_pd(uctx); 669 else 670 _ocrdma_dealloc_pd(dev, pd); 671 exit: 672 return status; 673 } 674 675 void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 676 { 677 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 678 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 679 struct ocrdma_ucontext *uctx = NULL; 680 u64 usr_db; 681 682 uctx = pd->uctx; 683 if (uctx) { 684 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 685 (pd->id * PAGE_SIZE); 686 if (pd->dpp_enabled) 687 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE); 688 usr_db = ocrdma_get_db_addr(dev, pd->id); 689 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 690 691 if (is_ucontext_pd(uctx, pd)) { 692 ocrdma_release_ucontext_pd(uctx); 693 return; 694 } 695 } 696 _ocrdma_dealloc_pd(dev, pd); 697 } 698 699 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 700 u32 pdid, int acc, u32 num_pbls, u32 addr_check) 701 { 702 int status; 703 704 mr->hwmr.fr_mr = 0; 705 mr->hwmr.local_rd = 1; 706 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 707 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 708 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 709 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0; 710 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 711 mr->hwmr.num_pbls = num_pbls; 712 713 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check); 714 if (status) 715 return status; 716 717 mr->ibmr.lkey = mr->hwmr.lkey; 718 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 719 mr->ibmr.rkey = mr->hwmr.lkey; 720 return 0; 721 } 722 723 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 724 { 725 int status; 726 struct ocrdma_mr *mr; 727 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 728 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 729 730 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { 731 pr_err("%s err, invalid access rights\n", __func__); 732 return ERR_PTR(-EINVAL); 733 } 734 735 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 736 if (!mr) 737 return ERR_PTR(-ENOMEM); 738 739 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0, 740 OCRDMA_ADDR_CHECK_DISABLE); 741 if (status) { 742 kfree(mr); 743 return ERR_PTR(status); 744 } 745 746 return &mr->ibmr; 747 } 748 749 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, 750 struct ocrdma_hw_mr *mr) 751 { 752 struct pci_dev *pdev = dev->nic_info.pdev; 753 int i = 0; 754 755 if (mr->pbl_table) { 756 for (i = 0; i < mr->num_pbls; i++) { 757 if (!mr->pbl_table[i].va) 758 continue; 759 dma_free_coherent(&pdev->dev, mr->pbl_size, 760 mr->pbl_table[i].va, 761 mr->pbl_table[i].pa); 762 } 763 kfree(mr->pbl_table); 764 mr->pbl_table = NULL; 765 } 766 } 767 768 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 769 u32 num_pbes) 770 { 771 u32 num_pbls = 0; 772 u32 idx = 0; 773 int status = 0; 774 u32 pbl_size; 775 776 do { 777 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx); 778 if (pbl_size > MAX_OCRDMA_PBL_SIZE) { 779 status = -EFAULT; 780 break; 781 } 782 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 783 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 784 idx++; 785 } while (num_pbls >= dev->attr.max_num_mr_pbl); 786 787 mr->hwmr.num_pbes = num_pbes; 788 mr->hwmr.num_pbls = num_pbls; 789 mr->hwmr.pbl_size = pbl_size; 790 return status; 791 } 792 793 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) 794 { 795 int status = 0; 796 int i; 797 u32 dma_len = mr->pbl_size; 798 struct pci_dev *pdev = dev->nic_info.pdev; 799 void *va; 800 dma_addr_t pa; 801 802 mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl), 803 GFP_KERNEL); 804 805 if (!mr->pbl_table) 806 return -ENOMEM; 807 808 for (i = 0; i < mr->num_pbls; i++) { 809 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL); 810 if (!va) { 811 ocrdma_free_mr_pbl_tbl(dev, mr); 812 status = -ENOMEM; 813 break; 814 } 815 mr->pbl_table[i].va = va; 816 mr->pbl_table[i].pa = pa; 817 } 818 return status; 819 } 820 821 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr, 822 u32 num_pbes) 823 { 824 struct ocrdma_pbe *pbe; 825 struct sg_dma_page_iter sg_iter; 826 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 827 struct ib_umem *umem = mr->umem; 828 int pbe_cnt, total_num_pbes = 0; 829 u64 pg_addr; 830 831 if (!mr->hwmr.num_pbes) 832 return; 833 834 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 835 pbe_cnt = 0; 836 837 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 838 /* store the page address in pbe */ 839 pg_addr = sg_page_iter_dma_address(&sg_iter); 840 pbe->pa_lo = cpu_to_le32(pg_addr); 841 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr)); 842 pbe_cnt += 1; 843 total_num_pbes += 1; 844 pbe++; 845 846 /* if done building pbes, issue the mbx cmd. */ 847 if (total_num_pbes == num_pbes) 848 return; 849 850 /* if the given pbl is full storing the pbes, 851 * move to next pbl. 852 */ 853 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) { 854 pbl_tbl++; 855 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 856 pbe_cnt = 0; 857 } 858 } 859 } 860 861 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 862 u64 usr_addr, int acc, struct ib_udata *udata) 863 { 864 int status = -ENOMEM; 865 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 866 struct ocrdma_mr *mr; 867 struct ocrdma_pd *pd; 868 u32 num_pbes; 869 870 pd = get_ocrdma_pd(ibpd); 871 872 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 873 return ERR_PTR(-EINVAL); 874 875 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 876 if (!mr) 877 return ERR_PTR(status); 878 mr->umem = ib_umem_get(udata, start, len, acc, 0); 879 if (IS_ERR(mr->umem)) { 880 status = -EFAULT; 881 goto umem_err; 882 } 883 num_pbes = ib_umem_page_count(mr->umem); 884 status = ocrdma_get_pbl_info(dev, mr, num_pbes); 885 if (status) 886 goto umem_err; 887 888 mr->hwmr.pbe_size = PAGE_SIZE; 889 mr->hwmr.fbo = ib_umem_offset(mr->umem); 890 mr->hwmr.va = usr_addr; 891 mr->hwmr.len = len; 892 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 893 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 894 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 895 mr->hwmr.local_rd = 1; 896 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 897 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 898 if (status) 899 goto umem_err; 900 build_user_pbes(dev, mr, num_pbes); 901 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 902 if (status) 903 goto mbx_err; 904 mr->ibmr.lkey = mr->hwmr.lkey; 905 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 906 mr->ibmr.rkey = mr->hwmr.lkey; 907 908 return &mr->ibmr; 909 910 mbx_err: 911 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 912 umem_err: 913 kfree(mr); 914 return ERR_PTR(status); 915 } 916 917 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 918 { 919 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 920 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device); 921 922 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 923 924 kfree(mr->pages); 925 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 926 927 /* it could be user registered memory. */ 928 ib_umem_release(mr->umem); 929 kfree(mr); 930 931 /* Don't stop cleanup, in case FW is unresponsive */ 932 if (dev->mqe_ctx.fw_error_state) { 933 pr_err("%s(%d) fw not responding.\n", 934 __func__, dev->id); 935 } 936 return 0; 937 } 938 939 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 940 struct ib_udata *udata) 941 { 942 int status; 943 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 944 udata, struct ocrdma_ucontext, ibucontext); 945 struct ocrdma_create_cq_uresp uresp; 946 947 /* this must be user flow! */ 948 if (!udata) 949 return -EINVAL; 950 951 memset(&uresp, 0, sizeof(uresp)); 952 uresp.cq_id = cq->id; 953 uresp.page_size = PAGE_ALIGN(cq->len); 954 uresp.num_pages = 1; 955 uresp.max_hw_cqe = cq->max_hw_cqe; 956 uresp.page_addr[0] = virt_to_phys(cq->va); 957 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 958 uresp.db_page_size = dev->nic_info.db_page_size; 959 uresp.phase_change = cq->phase_change ? 1 : 0; 960 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 961 if (status) { 962 pr_err("%s(%d) copy error cqid=0x%x.\n", 963 __func__, dev->id, cq->id); 964 goto err; 965 } 966 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 967 if (status) 968 goto err; 969 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size); 970 if (status) { 971 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 972 goto err; 973 } 974 cq->ucontext = uctx; 975 err: 976 return status; 977 } 978 979 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 980 struct ib_udata *udata) 981 { 982 struct ib_device *ibdev = ibcq->device; 983 int entries = attr->cqe; 984 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 985 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 986 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( 987 udata, struct ocrdma_ucontext, ibucontext); 988 u16 pd_id = 0; 989 int status; 990 struct ocrdma_create_cq_ureq ureq; 991 992 if (attr->flags) 993 return -EINVAL; 994 995 if (udata) { 996 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 997 return -EFAULT; 998 } else 999 ureq.dpp_cq = 0; 1000 1001 spin_lock_init(&cq->cq_lock); 1002 spin_lock_init(&cq->comp_handler_lock); 1003 INIT_LIST_HEAD(&cq->sq_head); 1004 INIT_LIST_HEAD(&cq->rq_head); 1005 1006 if (udata) 1007 pd_id = uctx->cntxt_pd->id; 1008 1009 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); 1010 if (status) 1011 return status; 1012 1013 if (udata) { 1014 status = ocrdma_copy_cq_uresp(dev, cq, udata); 1015 if (status) 1016 goto ctx_err; 1017 } 1018 cq->phase = OCRDMA_CQE_VALID; 1019 dev->cq_tbl[cq->id] = cq; 1020 return 0; 1021 1022 ctx_err: 1023 ocrdma_mbx_destroy_cq(dev, cq); 1024 return status; 1025 } 1026 1027 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, 1028 struct ib_udata *udata) 1029 { 1030 int status = 0; 1031 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1032 1033 if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) { 1034 status = -EINVAL; 1035 return status; 1036 } 1037 ibcq->cqe = new_cnt; 1038 return status; 1039 } 1040 1041 static void ocrdma_flush_cq(struct ocrdma_cq *cq) 1042 { 1043 int cqe_cnt; 1044 int valid_count = 0; 1045 unsigned long flags; 1046 1047 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 1048 struct ocrdma_cqe *cqe = NULL; 1049 1050 cqe = cq->va; 1051 cqe_cnt = cq->cqe_cnt; 1052 1053 /* Last irq might have scheduled a polling thread 1054 * sync-up with it before hard flushing. 1055 */ 1056 spin_lock_irqsave(&cq->cq_lock, flags); 1057 while (cqe_cnt) { 1058 if (is_cqe_valid(cq, cqe)) 1059 valid_count++; 1060 cqe++; 1061 cqe_cnt--; 1062 } 1063 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count); 1064 spin_unlock_irqrestore(&cq->cq_lock, flags); 1065 } 1066 1067 void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1068 { 1069 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 1070 struct ocrdma_eq *eq = NULL; 1071 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 1072 int pdid = 0; 1073 u32 irq, indx; 1074 1075 dev->cq_tbl[cq->id] = NULL; 1076 indx = ocrdma_get_eq_table_index(dev, cq->eqn); 1077 1078 eq = &dev->eq_tbl[indx]; 1079 irq = ocrdma_get_irq(dev, eq); 1080 synchronize_irq(irq); 1081 ocrdma_flush_cq(cq); 1082 1083 ocrdma_mbx_destroy_cq(dev, cq); 1084 if (cq->ucontext) { 1085 pdid = cq->ucontext->cntxt_pd->id; 1086 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, 1087 PAGE_ALIGN(cq->len)); 1088 ocrdma_del_mmap(cq->ucontext, 1089 ocrdma_get_db_addr(dev, pdid), 1090 dev->nic_info.db_page_size); 1091 } 1092 } 1093 1094 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1095 { 1096 int status = -EINVAL; 1097 1098 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) { 1099 dev->qp_tbl[qp->id] = qp; 1100 status = 0; 1101 } 1102 return status; 1103 } 1104 1105 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) 1106 { 1107 dev->qp_tbl[qp->id] = NULL; 1108 } 1109 1110 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 1111 struct ib_qp_init_attr *attrs, 1112 struct ib_udata *udata) 1113 { 1114 if ((attrs->qp_type != IB_QPT_GSI) && 1115 (attrs->qp_type != IB_QPT_RC) && 1116 (attrs->qp_type != IB_QPT_UC) && 1117 (attrs->qp_type != IB_QPT_UD)) { 1118 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1119 __func__, dev->id, attrs->qp_type); 1120 return -EINVAL; 1121 } 1122 /* Skip the check for QP1 to support CM size of 128 */ 1123 if ((attrs->qp_type != IB_QPT_GSI) && 1124 (attrs->cap.max_send_wr > dev->attr.max_wqe)) { 1125 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1126 __func__, dev->id, attrs->cap.max_send_wr); 1127 pr_err("%s(%d) supported send_wr=0x%x\n", 1128 __func__, dev->id, dev->attr.max_wqe); 1129 return -EINVAL; 1130 } 1131 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) { 1132 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n", 1133 __func__, dev->id, attrs->cap.max_recv_wr); 1134 pr_err("%s(%d) supported recv_wr=0x%x\n", 1135 __func__, dev->id, dev->attr.max_rqe); 1136 return -EINVAL; 1137 } 1138 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) { 1139 pr_err("%s(%d) unsupported inline data size=0x%x requested\n", 1140 __func__, dev->id, attrs->cap.max_inline_data); 1141 pr_err("%s(%d) supported inline data size=0x%x\n", 1142 __func__, dev->id, dev->attr.max_inline_data); 1143 return -EINVAL; 1144 } 1145 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) { 1146 pr_err("%s(%d) unsupported send_sge=0x%x requested\n", 1147 __func__, dev->id, attrs->cap.max_send_sge); 1148 pr_err("%s(%d) supported send_sge=0x%x\n", 1149 __func__, dev->id, dev->attr.max_send_sge); 1150 return -EINVAL; 1151 } 1152 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) { 1153 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n", 1154 __func__, dev->id, attrs->cap.max_recv_sge); 1155 pr_err("%s(%d) supported recv_sge=0x%x\n", 1156 __func__, dev->id, dev->attr.max_recv_sge); 1157 return -EINVAL; 1158 } 1159 /* unprivileged user space cannot create special QP */ 1160 if (udata && attrs->qp_type == IB_QPT_GSI) { 1161 pr_err 1162 ("%s(%d) Userspace can't create special QPs of type=0x%x\n", 1163 __func__, dev->id, attrs->qp_type); 1164 return -EINVAL; 1165 } 1166 /* allow creating only one GSI type of QP */ 1167 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { 1168 pr_err("%s(%d) GSI special QPs already created.\n", 1169 __func__, dev->id); 1170 return -EINVAL; 1171 } 1172 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1173 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1174 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1175 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) { 1176 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1177 __func__, dev->id); 1178 return -EINVAL; 1179 } 1180 } 1181 return 0; 1182 } 1183 1184 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, 1185 struct ib_udata *udata, int dpp_offset, 1186 int dpp_credit_lmt, int srq) 1187 { 1188 int status; 1189 u64 usr_db; 1190 struct ocrdma_create_qp_uresp uresp; 1191 struct ocrdma_pd *pd = qp->pd; 1192 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device); 1193 1194 memset(&uresp, 0, sizeof(uresp)); 1195 usr_db = dev->nic_info.unmapped_db + 1196 (pd->id * dev->nic_info.db_page_size); 1197 uresp.qp_id = qp->id; 1198 uresp.sq_dbid = qp->sq.dbid; 1199 uresp.num_sq_pages = 1; 1200 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1201 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va); 1202 uresp.num_wqe_allocated = qp->sq.max_cnt; 1203 if (!srq) { 1204 uresp.rq_dbid = qp->rq.dbid; 1205 uresp.num_rq_pages = 1; 1206 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1207 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va); 1208 uresp.num_rqe_allocated = qp->rq.max_cnt; 1209 } 1210 uresp.db_page_addr = usr_db; 1211 uresp.db_page_size = dev->nic_info.db_page_size; 1212 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1213 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1214 uresp.db_shift = OCRDMA_DB_RQ_SHIFT; 1215 1216 if (qp->dpp_enabled) { 1217 uresp.dpp_credit = dpp_credit_lmt; 1218 uresp.dpp_offset = dpp_offset; 1219 } 1220 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1221 if (status) { 1222 pr_err("%s(%d) user copy error.\n", __func__, dev->id); 1223 goto err; 1224 } 1225 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0], 1226 uresp.sq_page_size); 1227 if (status) 1228 goto err; 1229 1230 if (!srq) { 1231 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0], 1232 uresp.rq_page_size); 1233 if (status) 1234 goto rq_map_err; 1235 } 1236 return status; 1237 rq_map_err: 1238 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size); 1239 err: 1240 return status; 1241 } 1242 1243 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, 1244 struct ocrdma_pd *pd) 1245 { 1246 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1247 qp->sq_db = dev->nic_info.db + 1248 (pd->id * dev->nic_info.db_page_size) + 1249 OCRDMA_DB_GEN2_SQ_OFFSET; 1250 qp->rq_db = dev->nic_info.db + 1251 (pd->id * dev->nic_info.db_page_size) + 1252 OCRDMA_DB_GEN2_RQ_OFFSET; 1253 } else { 1254 qp->sq_db = dev->nic_info.db + 1255 (pd->id * dev->nic_info.db_page_size) + 1256 OCRDMA_DB_SQ_OFFSET; 1257 qp->rq_db = dev->nic_info.db + 1258 (pd->id * dev->nic_info.db_page_size) + 1259 OCRDMA_DB_RQ_OFFSET; 1260 } 1261 } 1262 1263 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) 1264 { 1265 qp->wqe_wr_id_tbl = 1266 kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)), 1267 GFP_KERNEL); 1268 if (qp->wqe_wr_id_tbl == NULL) 1269 return -ENOMEM; 1270 qp->rqe_wr_id_tbl = 1271 kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL); 1272 if (qp->rqe_wr_id_tbl == NULL) 1273 return -ENOMEM; 1274 1275 return 0; 1276 } 1277 1278 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, 1279 struct ocrdma_pd *pd, 1280 struct ib_qp_init_attr *attrs) 1281 { 1282 qp->pd = pd; 1283 spin_lock_init(&qp->q_lock); 1284 INIT_LIST_HEAD(&qp->sq_entry); 1285 INIT_LIST_HEAD(&qp->rq_entry); 1286 1287 qp->qp_type = attrs->qp_type; 1288 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR; 1289 qp->max_inline_data = attrs->cap.max_inline_data; 1290 qp->sq.max_sges = attrs->cap.max_send_sge; 1291 qp->rq.max_sges = attrs->cap.max_recv_sge; 1292 qp->state = OCRDMA_QPS_RST; 1293 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1294 } 1295 1296 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, 1297 struct ib_qp_init_attr *attrs) 1298 { 1299 if (attrs->qp_type == IB_QPT_GSI) { 1300 dev->gsi_qp_created = 1; 1301 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq); 1302 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq); 1303 } 1304 } 1305 1306 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd, 1307 struct ib_qp_init_attr *attrs, 1308 struct ib_udata *udata) 1309 { 1310 int status; 1311 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1312 struct ocrdma_qp *qp; 1313 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 1314 struct ocrdma_create_qp_ureq ureq; 1315 u16 dpp_credit_lmt, dpp_offset; 1316 1317 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata); 1318 if (status) 1319 goto gen_err; 1320 1321 memset(&ureq, 0, sizeof(ureq)); 1322 if (udata) { 1323 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1324 return ERR_PTR(-EFAULT); 1325 } 1326 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1327 if (!qp) { 1328 status = -ENOMEM; 1329 goto gen_err; 1330 } 1331 ocrdma_set_qp_init_params(qp, pd, attrs); 1332 if (udata == NULL) 1333 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 | 1334 OCRDMA_QP_FAST_REG); 1335 1336 mutex_lock(&dev->dev_lock); 1337 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1338 ureq.dpp_cq_id, 1339 &dpp_offset, &dpp_credit_lmt); 1340 if (status) 1341 goto mbx_err; 1342 1343 /* user space QP's wr_id table are managed in library */ 1344 if (udata == NULL) { 1345 status = ocrdma_alloc_wr_id_tbl(qp); 1346 if (status) 1347 goto map_err; 1348 } 1349 1350 status = ocrdma_add_qpn_map(dev, qp); 1351 if (status) 1352 goto map_err; 1353 ocrdma_set_qp_db(dev, qp, pd); 1354 if (udata) { 1355 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset, 1356 dpp_credit_lmt, 1357 (attrs->srq != NULL)); 1358 if (status) 1359 goto cpy_err; 1360 } 1361 ocrdma_store_gsi_qp_cq(dev, attrs); 1362 qp->ibqp.qp_num = qp->id; 1363 mutex_unlock(&dev->dev_lock); 1364 return &qp->ibqp; 1365 1366 cpy_err: 1367 ocrdma_del_qpn_map(dev, qp); 1368 map_err: 1369 ocrdma_mbx_destroy_qp(dev, qp); 1370 mbx_err: 1371 mutex_unlock(&dev->dev_lock); 1372 kfree(qp->wqe_wr_id_tbl); 1373 kfree(qp->rqe_wr_id_tbl); 1374 kfree(qp); 1375 pr_err("%s(%d) error=%d\n", __func__, dev->id, status); 1376 gen_err: 1377 return ERR_PTR(status); 1378 } 1379 1380 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1381 int attr_mask) 1382 { 1383 int status = 0; 1384 struct ocrdma_qp *qp; 1385 struct ocrdma_dev *dev; 1386 enum ib_qp_state old_qps; 1387 1388 qp = get_ocrdma_qp(ibqp); 1389 dev = get_ocrdma_dev(ibqp->device); 1390 if (attr_mask & IB_QP_STATE) 1391 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps); 1392 /* if new and previous states are same hw doesn't need to 1393 * know about it. 1394 */ 1395 if (status < 0) 1396 return status; 1397 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask); 1398 } 1399 1400 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1401 int attr_mask, struct ib_udata *udata) 1402 { 1403 unsigned long flags; 1404 int status = -EINVAL; 1405 struct ocrdma_qp *qp; 1406 struct ocrdma_dev *dev; 1407 enum ib_qp_state old_qps, new_qps; 1408 1409 qp = get_ocrdma_qp(ibqp); 1410 dev = get_ocrdma_dev(ibqp->device); 1411 1412 /* syncronize with multiple context trying to change, retrive qps */ 1413 mutex_lock(&dev->dev_lock); 1414 /* syncronize with wqe, rqe posting and cqe processing contexts */ 1415 spin_lock_irqsave(&qp->q_lock, flags); 1416 old_qps = get_ibqp_state(qp->state); 1417 if (attr_mask & IB_QP_STATE) 1418 new_qps = attr->qp_state; 1419 else 1420 new_qps = old_qps; 1421 spin_unlock_irqrestore(&qp->q_lock, flags); 1422 1423 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) { 1424 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n" 1425 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n", 1426 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type, 1427 old_qps, new_qps); 1428 goto param_err; 1429 } 1430 1431 status = _ocrdma_modify_qp(ibqp, attr, attr_mask); 1432 if (status > 0) 1433 status = 0; 1434 param_err: 1435 mutex_unlock(&dev->dev_lock); 1436 return status; 1437 } 1438 1439 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) 1440 { 1441 switch (mtu) { 1442 case 256: 1443 return IB_MTU_256; 1444 case 512: 1445 return IB_MTU_512; 1446 case 1024: 1447 return IB_MTU_1024; 1448 case 2048: 1449 return IB_MTU_2048; 1450 case 4096: 1451 return IB_MTU_4096; 1452 default: 1453 return IB_MTU_1024; 1454 } 1455 } 1456 1457 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) 1458 { 1459 int ib_qp_acc_flags = 0; 1460 1461 if (qp_cap_flags & OCRDMA_QP_INB_WR) 1462 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 1463 if (qp_cap_flags & OCRDMA_QP_INB_RD) 1464 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 1465 return ib_qp_acc_flags; 1466 } 1467 1468 int ocrdma_query_qp(struct ib_qp *ibqp, 1469 struct ib_qp_attr *qp_attr, 1470 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 1471 { 1472 int status; 1473 u32 qp_state; 1474 struct ocrdma_qp_params params; 1475 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 1476 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device); 1477 1478 memset(¶ms, 0, sizeof(params)); 1479 mutex_lock(&dev->dev_lock); 1480 status = ocrdma_mbx_query_qp(dev, qp, ¶ms); 1481 mutex_unlock(&dev->dev_lock); 1482 if (status) 1483 goto mbx_err; 1484 if (qp->qp_type == IB_QPT_UD) 1485 qp_attr->qkey = params.qkey; 1486 qp_attr->path_mtu = 1487 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx & 1488 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >> 1489 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT; 1490 qp_attr->path_mig_state = IB_MIG_MIGRATED; 1491 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK; 1492 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK; 1493 qp_attr->dest_qp_num = 1494 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK; 1495 1496 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags); 1497 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1; 1498 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1499 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1500 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1501 qp_attr->cap.max_inline_data = qp->max_inline_data; 1502 qp_init_attr->cap = qp_attr->cap; 1503 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 1504 1505 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 1506 params.rnt_rc_sl_fl & 1507 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK, 1508 qp->sgid_idx, 1509 (params.hop_lmt_rq_psn & 1510 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >> 1511 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT, 1512 (params.tclass_sq_psn & 1513 OCRDMA_QP_PARAMS_TCLASS_MASK) >> 1514 OCRDMA_QP_PARAMS_TCLASS_SHIFT); 1515 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]); 1516 1517 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 1518 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl & 1519 OCRDMA_QP_PARAMS_SL_MASK) >> 1520 OCRDMA_QP_PARAMS_SL_SHIFT); 1521 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn & 1522 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >> 1523 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT; 1524 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn & 1525 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >> 1526 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT; 1527 qp_attr->retry_cnt = 1528 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >> 1529 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT; 1530 qp_attr->min_rnr_timer = 0; 1531 qp_attr->pkey_index = 0; 1532 qp_attr->port_num = 1; 1533 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 1534 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 1535 qp_attr->alt_pkey_index = 0; 1536 qp_attr->alt_port_num = 0; 1537 qp_attr->alt_timeout = 0; 1538 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 1539 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >> 1540 OCRDMA_QP_PARAMS_STATE_SHIFT; 1541 qp_attr->qp_state = get_ibqp_state(qp_state); 1542 qp_attr->cur_qp_state = qp_attr->qp_state; 1543 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0; 1544 qp_attr->max_dest_rd_atomic = 1545 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT; 1546 qp_attr->max_rd_atomic = 1547 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK; 1548 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags & 1549 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0; 1550 /* Sync driver QP state with FW */ 1551 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL); 1552 mbx_err: 1553 return status; 1554 } 1555 1556 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) 1557 { 1558 unsigned int i = idx / 32; 1559 u32 mask = (1U << (idx % 32)); 1560 1561 srq->idx_bit_fields[i] ^= mask; 1562 } 1563 1564 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1565 { 1566 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt; 1567 } 1568 1569 static int is_hw_sq_empty(struct ocrdma_qp *qp) 1570 { 1571 return (qp->sq.tail == qp->sq.head); 1572 } 1573 1574 static int is_hw_rq_empty(struct ocrdma_qp *qp) 1575 { 1576 return (qp->rq.tail == qp->rq.head); 1577 } 1578 1579 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1580 { 1581 return q->va + (q->head * q->entry_size); 1582 } 1583 1584 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, 1585 u32 idx) 1586 { 1587 return q->va + (idx * q->entry_size); 1588 } 1589 1590 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) 1591 { 1592 q->head = (q->head + 1) & q->max_wqe_idx; 1593 } 1594 1595 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) 1596 { 1597 q->tail = (q->tail + 1) & q->max_wqe_idx; 1598 } 1599 1600 /* discard the cqe for a given QP */ 1601 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) 1602 { 1603 unsigned long cq_flags; 1604 unsigned long flags; 1605 int discard_cnt = 0; 1606 u32 cur_getp, stop_getp; 1607 struct ocrdma_cqe *cqe; 1608 u32 qpn = 0, wqe_idx = 0; 1609 1610 spin_lock_irqsave(&cq->cq_lock, cq_flags); 1611 1612 /* traverse through the CQEs in the hw CQ, 1613 * find the matching CQE for a given qp, 1614 * mark the matching one discarded by clearing qpn. 1615 * ring the doorbell in the poll_cq() as 1616 * we don't complete out of order cqe. 1617 */ 1618 1619 cur_getp = cq->getp; 1620 /* find upto when do we reap the cq. */ 1621 stop_getp = cur_getp; 1622 do { 1623 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) 1624 break; 1625 1626 cqe = cq->va + cur_getp; 1627 /* if (a) done reaping whole hw cq, or 1628 * (b) qp_xq becomes empty. 1629 * then exit 1630 */ 1631 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK; 1632 /* if previously discarded cqe found, skip that too. */ 1633 /* check for matching qp */ 1634 if (qpn == 0 || qpn != qp->id) 1635 goto skip_cqe; 1636 1637 if (is_cqe_for_sq(cqe)) { 1638 ocrdma_hwq_inc_tail(&qp->sq); 1639 } else { 1640 if (qp->srq) { 1641 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 1642 OCRDMA_CQE_BUFTAG_SHIFT) & 1643 qp->srq->rq.max_wqe_idx; 1644 BUG_ON(wqe_idx < 1); 1645 spin_lock_irqsave(&qp->srq->q_lock, flags); 1646 ocrdma_hwq_inc_tail(&qp->srq->rq); 1647 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1); 1648 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1649 1650 } else { 1651 ocrdma_hwq_inc_tail(&qp->rq); 1652 } 1653 } 1654 /* mark cqe discarded so that it is not picked up later 1655 * in the poll_cq(). 1656 */ 1657 discard_cnt += 1; 1658 cqe->cmn.qpn = 0; 1659 skip_cqe: 1660 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1661 } while (cur_getp != stop_getp); 1662 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1663 } 1664 1665 void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1666 { 1667 int found = false; 1668 unsigned long flags; 1669 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 1670 /* sync with any active CQ poll */ 1671 1672 spin_lock_irqsave(&dev->flush_q_lock, flags); 1673 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp); 1674 if (found) 1675 list_del(&qp->sq_entry); 1676 if (!qp->srq) { 1677 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp); 1678 if (found) 1679 list_del(&qp->rq_entry); 1680 } 1681 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 1682 } 1683 1684 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1685 { 1686 struct ocrdma_pd *pd; 1687 struct ocrdma_qp *qp; 1688 struct ocrdma_dev *dev; 1689 struct ib_qp_attr attrs; 1690 int attr_mask; 1691 unsigned long flags; 1692 1693 qp = get_ocrdma_qp(ibqp); 1694 dev = get_ocrdma_dev(ibqp->device); 1695 1696 pd = qp->pd; 1697 1698 /* change the QP state to ERROR */ 1699 if (qp->state != OCRDMA_QPS_RST) { 1700 attrs.qp_state = IB_QPS_ERR; 1701 attr_mask = IB_QP_STATE; 1702 _ocrdma_modify_qp(ibqp, &attrs, attr_mask); 1703 } 1704 /* ensure that CQEs for newly created QP (whose id may be same with 1705 * one which just getting destroyed are same), dont get 1706 * discarded until the old CQEs are discarded. 1707 */ 1708 mutex_lock(&dev->dev_lock); 1709 (void) ocrdma_mbx_destroy_qp(dev, qp); 1710 1711 /* 1712 * acquire CQ lock while destroy is in progress, in order to 1713 * protect against proessing in-flight CQEs for this QP. 1714 */ 1715 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags); 1716 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) { 1717 spin_lock(&qp->rq_cq->cq_lock); 1718 ocrdma_del_qpn_map(dev, qp); 1719 spin_unlock(&qp->rq_cq->cq_lock); 1720 } else { 1721 ocrdma_del_qpn_map(dev, qp); 1722 } 1723 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags); 1724 1725 if (!pd->uctx) { 1726 ocrdma_discard_cqes(qp, qp->sq_cq); 1727 ocrdma_discard_cqes(qp, qp->rq_cq); 1728 } 1729 mutex_unlock(&dev->dev_lock); 1730 1731 if (pd->uctx) { 1732 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, 1733 PAGE_ALIGN(qp->sq.len)); 1734 if (!qp->srq) 1735 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, 1736 PAGE_ALIGN(qp->rq.len)); 1737 } 1738 1739 ocrdma_del_flush_qp(qp); 1740 1741 kfree(qp->wqe_wr_id_tbl); 1742 kfree(qp->rqe_wr_id_tbl); 1743 kfree(qp); 1744 return 0; 1745 } 1746 1747 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, 1748 struct ib_udata *udata) 1749 { 1750 int status; 1751 struct ocrdma_create_srq_uresp uresp; 1752 1753 memset(&uresp, 0, sizeof(uresp)); 1754 uresp.rq_dbid = srq->rq.dbid; 1755 uresp.num_rq_pages = 1; 1756 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va); 1757 uresp.rq_page_size = srq->rq.len; 1758 uresp.db_page_addr = dev->nic_info.unmapped_db + 1759 (srq->pd->id * dev->nic_info.db_page_size); 1760 uresp.db_page_size = dev->nic_info.db_page_size; 1761 uresp.num_rqe_allocated = srq->rq.max_cnt; 1762 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) { 1763 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET; 1764 uresp.db_shift = 24; 1765 } else { 1766 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1767 uresp.db_shift = 16; 1768 } 1769 1770 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1771 if (status) 1772 return status; 1773 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0], 1774 uresp.rq_page_size); 1775 if (status) 1776 return status; 1777 return status; 1778 } 1779 1780 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1781 struct ib_udata *udata) 1782 { 1783 int status; 1784 struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd); 1785 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1786 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq); 1787 1788 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1789 return -EINVAL; 1790 if (init_attr->attr.max_wr > dev->attr.max_rqe) 1791 return -EINVAL; 1792 1793 spin_lock_init(&srq->q_lock); 1794 srq->pd = pd; 1795 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1796 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd); 1797 if (status) 1798 return status; 1799 1800 if (!udata) { 1801 srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64), 1802 GFP_KERNEL); 1803 if (!srq->rqe_wr_id_tbl) { 1804 status = -ENOMEM; 1805 goto arm_err; 1806 } 1807 1808 srq->bit_fields_len = (srq->rq.max_cnt / 32) + 1809 (srq->rq.max_cnt % 32 ? 1 : 0); 1810 srq->idx_bit_fields = 1811 kmalloc_array(srq->bit_fields_len, sizeof(u32), 1812 GFP_KERNEL); 1813 if (!srq->idx_bit_fields) { 1814 status = -ENOMEM; 1815 goto arm_err; 1816 } 1817 memset(srq->idx_bit_fields, 0xff, 1818 srq->bit_fields_len * sizeof(u32)); 1819 } 1820 1821 if (init_attr->attr.srq_limit) { 1822 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr); 1823 if (status) 1824 goto arm_err; 1825 } 1826 1827 if (udata) { 1828 status = ocrdma_copy_srq_uresp(dev, srq, udata); 1829 if (status) 1830 goto arm_err; 1831 } 1832 1833 return 0; 1834 1835 arm_err: 1836 ocrdma_mbx_destroy_srq(dev, srq); 1837 kfree(srq->rqe_wr_id_tbl); 1838 kfree(srq->idx_bit_fields); 1839 return status; 1840 } 1841 1842 int ocrdma_modify_srq(struct ib_srq *ibsrq, 1843 struct ib_srq_attr *srq_attr, 1844 enum ib_srq_attr_mask srq_attr_mask, 1845 struct ib_udata *udata) 1846 { 1847 int status; 1848 struct ocrdma_srq *srq; 1849 1850 srq = get_ocrdma_srq(ibsrq); 1851 if (srq_attr_mask & IB_SRQ_MAX_WR) 1852 status = -EINVAL; 1853 else 1854 status = ocrdma_mbx_modify_srq(srq, srq_attr); 1855 return status; 1856 } 1857 1858 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 1859 { 1860 int status; 1861 struct ocrdma_srq *srq; 1862 1863 srq = get_ocrdma_srq(ibsrq); 1864 status = ocrdma_mbx_query_srq(srq, srq_attr); 1865 return status; 1866 } 1867 1868 void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1869 { 1870 struct ocrdma_srq *srq; 1871 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device); 1872 1873 srq = get_ocrdma_srq(ibsrq); 1874 1875 ocrdma_mbx_destroy_srq(dev, srq); 1876 1877 if (srq->pd->uctx) 1878 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, 1879 PAGE_ALIGN(srq->rq.len)); 1880 1881 kfree(srq->idx_bit_fields); 1882 kfree(srq->rqe_wr_id_tbl); 1883 } 1884 1885 /* unprivileged verbs and their support functions. */ 1886 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, 1887 struct ocrdma_hdr_wqe *hdr, 1888 const struct ib_send_wr *wr) 1889 { 1890 struct ocrdma_ewqe_ud_hdr *ud_hdr = 1891 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1); 1892 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah); 1893 1894 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn; 1895 if (qp->qp_type == IB_QPT_GSI) 1896 ud_hdr->qkey = qp->qkey; 1897 else 1898 ud_hdr->qkey = ud_wr(wr)->remote_qkey; 1899 ud_hdr->rsvd_ahid = ah->id; 1900 ud_hdr->hdr_type = ah->hdr_type; 1901 if (ah->av->valid & OCRDMA_AV_VLAN_VALID) 1902 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT); 1903 } 1904 1905 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, 1906 struct ocrdma_sge *sge, int num_sge, 1907 struct ib_sge *sg_list) 1908 { 1909 int i; 1910 1911 for (i = 0; i < num_sge; i++) { 1912 sge[i].lrkey = sg_list[i].lkey; 1913 sge[i].addr_lo = sg_list[i].addr; 1914 sge[i].addr_hi = upper_32_bits(sg_list[i].addr); 1915 sge[i].len = sg_list[i].length; 1916 hdr->total_len += sg_list[i].length; 1917 } 1918 if (num_sge == 0) 1919 memset(sge, 0, sizeof(*sge)); 1920 } 1921 1922 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) 1923 { 1924 uint32_t total_len = 0, i; 1925 1926 for (i = 0; i < num_sge; i++) 1927 total_len += sg_list[i].length; 1928 return total_len; 1929 } 1930 1931 1932 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 1933 struct ocrdma_hdr_wqe *hdr, 1934 struct ocrdma_sge *sge, 1935 const struct ib_send_wr *wr, u32 wqe_size) 1936 { 1937 int i; 1938 char *dpp_addr; 1939 1940 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) { 1941 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge); 1942 if (unlikely(hdr->total_len > qp->max_inline_data)) { 1943 pr_err("%s() supported_len=0x%x,\n" 1944 " unsupported len req=0x%x\n", __func__, 1945 qp->max_inline_data, hdr->total_len); 1946 return -EINVAL; 1947 } 1948 dpp_addr = (char *)sge; 1949 for (i = 0; i < wr->num_sge; i++) { 1950 memcpy(dpp_addr, 1951 (void *)(unsigned long)wr->sg_list[i].addr, 1952 wr->sg_list[i].length); 1953 dpp_addr += wr->sg_list[i].length; 1954 } 1955 1956 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 1957 if (0 == hdr->total_len) 1958 wqe_size += sizeof(struct ocrdma_sge); 1959 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 1960 } else { 1961 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1962 if (wr->num_sge) 1963 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge)); 1964 else 1965 wqe_size += sizeof(struct ocrdma_sge); 1966 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 1967 } 1968 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 1969 return 0; 1970 } 1971 1972 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1973 const struct ib_send_wr *wr) 1974 { 1975 int status; 1976 struct ocrdma_sge *sge; 1977 u32 wqe_size = sizeof(*hdr); 1978 1979 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 1980 ocrdma_build_ud_hdr(qp, hdr, wr); 1981 sge = (struct ocrdma_sge *)(hdr + 2); 1982 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 1983 } else { 1984 sge = (struct ocrdma_sge *)(hdr + 1); 1985 } 1986 1987 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1988 return status; 1989 } 1990 1991 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 1992 const struct ib_send_wr *wr) 1993 { 1994 int status; 1995 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 1996 struct ocrdma_sge *sge = ext_rw + 1; 1997 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw); 1998 1999 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 2000 if (status) 2001 return status; 2002 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2003 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2004 ext_rw->lrkey = rdma_wr(wr)->rkey; 2005 ext_rw->len = hdr->total_len; 2006 return 0; 2007 } 2008 2009 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, 2010 const struct ib_send_wr *wr) 2011 { 2012 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1); 2013 struct ocrdma_sge *sge = ext_rw + 1; 2014 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) + 2015 sizeof(struct ocrdma_hdr_wqe); 2016 2017 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 2018 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2019 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT); 2020 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2021 2022 ext_rw->addr_lo = rdma_wr(wr)->remote_addr; 2023 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr); 2024 ext_rw->lrkey = rdma_wr(wr)->rkey; 2025 ext_rw->len = hdr->total_len; 2026 } 2027 2028 static int get_encoded_page_size(int pg_sz) 2029 { 2030 /* Max size is 256M 4096 << 16 */ 2031 int i = 0; 2032 for (; i < 17; i++) 2033 if (pg_sz == (4096 << i)) 2034 break; 2035 return i; 2036 } 2037 2038 static int ocrdma_build_reg(struct ocrdma_qp *qp, 2039 struct ocrdma_hdr_wqe *hdr, 2040 const struct ib_reg_wr *wr) 2041 { 2042 u64 fbo; 2043 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1); 2044 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr); 2045 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table; 2046 struct ocrdma_pbe *pbe; 2047 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr); 2048 int num_pbes = 0, i; 2049 2050 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES); 2051 2052 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT); 2053 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT); 2054 2055 if (wr->access & IB_ACCESS_LOCAL_WRITE) 2056 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR; 2057 if (wr->access & IB_ACCESS_REMOTE_WRITE) 2058 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR; 2059 if (wr->access & IB_ACCESS_REMOTE_READ) 2060 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD; 2061 hdr->lkey = wr->key; 2062 hdr->total_len = mr->ibmr.length; 2063 2064 fbo = mr->ibmr.iova - mr->pages[0]; 2065 2066 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova); 2067 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff); 2068 fast_reg->fbo_hi = upper_32_bits(fbo); 2069 fast_reg->fbo_lo = (u32) fbo & 0xffffffff; 2070 fast_reg->num_sges = mr->npages; 2071 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size); 2072 2073 pbe = pbl_tbl->va; 2074 for (i = 0; i < mr->npages; i++) { 2075 u64 buf_addr = mr->pages[i]; 2076 2077 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK)); 2078 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr)); 2079 num_pbes += 1; 2080 pbe++; 2081 2082 /* if the pbl is full storing the pbes, 2083 * move to next pbl. 2084 */ 2085 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) { 2086 pbl_tbl++; 2087 pbe = (struct ocrdma_pbe *)pbl_tbl->va; 2088 } 2089 } 2090 2091 return 0; 2092 } 2093 2094 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2095 { 2096 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT); 2097 2098 iowrite32(val, qp->sq_db); 2099 } 2100 2101 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 2102 const struct ib_send_wr **bad_wr) 2103 { 2104 int status = 0; 2105 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2106 struct ocrdma_hdr_wqe *hdr; 2107 unsigned long flags; 2108 2109 spin_lock_irqsave(&qp->q_lock, flags); 2110 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) { 2111 spin_unlock_irqrestore(&qp->q_lock, flags); 2112 *bad_wr = wr; 2113 return -EINVAL; 2114 } 2115 2116 while (wr) { 2117 if (qp->qp_type == IB_QPT_UD && 2118 (wr->opcode != IB_WR_SEND && 2119 wr->opcode != IB_WR_SEND_WITH_IMM)) { 2120 *bad_wr = wr; 2121 status = -EINVAL; 2122 break; 2123 } 2124 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 || 2125 wr->num_sge > qp->sq.max_sges) { 2126 *bad_wr = wr; 2127 status = -ENOMEM; 2128 break; 2129 } 2130 hdr = ocrdma_hwq_head(&qp->sq); 2131 hdr->cw = 0; 2132 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2133 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2134 if (wr->send_flags & IB_SEND_FENCE) 2135 hdr->cw |= 2136 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT); 2137 if (wr->send_flags & IB_SEND_SOLICITED) 2138 hdr->cw |= 2139 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT); 2140 hdr->total_len = 0; 2141 switch (wr->opcode) { 2142 case IB_WR_SEND_WITH_IMM: 2143 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2144 hdr->immdt = ntohl(wr->ex.imm_data); 2145 /* fall through */ 2146 case IB_WR_SEND: 2147 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2148 ocrdma_build_send(qp, hdr, wr); 2149 break; 2150 case IB_WR_SEND_WITH_INV: 2151 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT); 2152 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT); 2153 hdr->lkey = wr->ex.invalidate_rkey; 2154 status = ocrdma_build_send(qp, hdr, wr); 2155 break; 2156 case IB_WR_RDMA_WRITE_WITH_IMM: 2157 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT); 2158 hdr->immdt = ntohl(wr->ex.imm_data); 2159 /* fall through */ 2160 case IB_WR_RDMA_WRITE: 2161 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT); 2162 status = ocrdma_build_write(qp, hdr, wr); 2163 break; 2164 case IB_WR_RDMA_READ: 2165 ocrdma_build_read(qp, hdr, wr); 2166 break; 2167 case IB_WR_LOCAL_INV: 2168 hdr->cw |= 2169 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2170 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) + 2171 sizeof(struct ocrdma_sge)) / 2172 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2173 hdr->lkey = wr->ex.invalidate_rkey; 2174 break; 2175 case IB_WR_REG_MR: 2176 status = ocrdma_build_reg(qp, hdr, reg_wr(wr)); 2177 break; 2178 default: 2179 status = -EINVAL; 2180 break; 2181 } 2182 if (status) { 2183 *bad_wr = wr; 2184 break; 2185 } 2186 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled) 2187 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2188 else 2189 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2190 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id; 2191 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) & 2192 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE); 2193 /* make sure wqe is written before adapter can access it */ 2194 wmb(); 2195 /* inform hw to start processing it */ 2196 ocrdma_ring_sq_db(qp); 2197 2198 /* update pointer, counter for next wr */ 2199 ocrdma_hwq_inc_head(&qp->sq); 2200 wr = wr->next; 2201 } 2202 spin_unlock_irqrestore(&qp->q_lock, flags); 2203 return status; 2204 } 2205 2206 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) 2207 { 2208 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT); 2209 2210 iowrite32(val, qp->rq_db); 2211 } 2212 2213 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, 2214 const struct ib_recv_wr *wr, u16 tag) 2215 { 2216 u32 wqe_size = 0; 2217 struct ocrdma_sge *sge; 2218 if (wr->num_sge) 2219 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe); 2220 else 2221 wqe_size = sizeof(*sge) + sizeof(*rqe); 2222 2223 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) << 2224 OCRDMA_WQE_SIZE_SHIFT); 2225 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2226 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT); 2227 rqe->total_len = 0; 2228 rqe->rsvd_tag = tag; 2229 sge = (struct ocrdma_sge *)(rqe + 1); 2230 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list); 2231 ocrdma_cpu_to_le32(rqe, wqe_size); 2232 } 2233 2234 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 2235 const struct ib_recv_wr **bad_wr) 2236 { 2237 int status = 0; 2238 unsigned long flags; 2239 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp); 2240 struct ocrdma_hdr_wqe *rqe; 2241 2242 spin_lock_irqsave(&qp->q_lock, flags); 2243 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) { 2244 spin_unlock_irqrestore(&qp->q_lock, flags); 2245 *bad_wr = wr; 2246 return -EINVAL; 2247 } 2248 while (wr) { 2249 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 || 2250 wr->num_sge > qp->rq.max_sges) { 2251 *bad_wr = wr; 2252 status = -ENOMEM; 2253 break; 2254 } 2255 rqe = ocrdma_hwq_head(&qp->rq); 2256 ocrdma_build_rqe(rqe, wr, 0); 2257 2258 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id; 2259 /* make sure rqe is written before adapter can access it */ 2260 wmb(); 2261 2262 /* inform hw to start processing it */ 2263 ocrdma_ring_rq_db(qp); 2264 2265 /* update pointer, counter for next wr */ 2266 ocrdma_hwq_inc_head(&qp->rq); 2267 wr = wr->next; 2268 } 2269 spin_unlock_irqrestore(&qp->q_lock, flags); 2270 return status; 2271 } 2272 2273 /* cqe for srq's rqe can potentially arrive out of order. 2274 * index gives the entry in the shadow table where to store 2275 * the wr_id. tag/index is returned in cqe to reference back 2276 * for a given rqe. 2277 */ 2278 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) 2279 { 2280 int row = 0; 2281 int indx = 0; 2282 2283 for (row = 0; row < srq->bit_fields_len; row++) { 2284 if (srq->idx_bit_fields[row]) { 2285 indx = ffs(srq->idx_bit_fields[row]); 2286 indx = (row * 32) + (indx - 1); 2287 BUG_ON(indx >= srq->rq.max_cnt); 2288 ocrdma_srq_toggle_bit(srq, indx); 2289 break; 2290 } 2291 } 2292 2293 BUG_ON(row == srq->bit_fields_len); 2294 return indx + 1; /* Use from index 1 */ 2295 } 2296 2297 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) 2298 { 2299 u32 val = srq->rq.dbid | (1 << 16); 2300 2301 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET); 2302 } 2303 2304 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 2305 const struct ib_recv_wr **bad_wr) 2306 { 2307 int status = 0; 2308 unsigned long flags; 2309 struct ocrdma_srq *srq; 2310 struct ocrdma_hdr_wqe *rqe; 2311 u16 tag; 2312 2313 srq = get_ocrdma_srq(ibsrq); 2314 2315 spin_lock_irqsave(&srq->q_lock, flags); 2316 while (wr) { 2317 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 || 2318 wr->num_sge > srq->rq.max_sges) { 2319 status = -ENOMEM; 2320 *bad_wr = wr; 2321 break; 2322 } 2323 tag = ocrdma_srq_get_idx(srq); 2324 rqe = ocrdma_hwq_head(&srq->rq); 2325 ocrdma_build_rqe(rqe, wr, tag); 2326 2327 srq->rqe_wr_id_tbl[tag] = wr->wr_id; 2328 /* make sure rqe is written before adapter can perform DMA */ 2329 wmb(); 2330 /* inform hw to start processing it */ 2331 ocrdma_ring_srq_db(srq); 2332 /* update pointer, counter for next wr */ 2333 ocrdma_hwq_inc_head(&srq->rq); 2334 wr = wr->next; 2335 } 2336 spin_unlock_irqrestore(&srq->q_lock, flags); 2337 return status; 2338 } 2339 2340 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2341 { 2342 enum ib_wc_status ibwc_status; 2343 2344 switch (status) { 2345 case OCRDMA_CQE_GENERAL_ERR: 2346 ibwc_status = IB_WC_GENERAL_ERR; 2347 break; 2348 case OCRDMA_CQE_LOC_LEN_ERR: 2349 ibwc_status = IB_WC_LOC_LEN_ERR; 2350 break; 2351 case OCRDMA_CQE_LOC_QP_OP_ERR: 2352 ibwc_status = IB_WC_LOC_QP_OP_ERR; 2353 break; 2354 case OCRDMA_CQE_LOC_EEC_OP_ERR: 2355 ibwc_status = IB_WC_LOC_EEC_OP_ERR; 2356 break; 2357 case OCRDMA_CQE_LOC_PROT_ERR: 2358 ibwc_status = IB_WC_LOC_PROT_ERR; 2359 break; 2360 case OCRDMA_CQE_WR_FLUSH_ERR: 2361 ibwc_status = IB_WC_WR_FLUSH_ERR; 2362 break; 2363 case OCRDMA_CQE_MW_BIND_ERR: 2364 ibwc_status = IB_WC_MW_BIND_ERR; 2365 break; 2366 case OCRDMA_CQE_BAD_RESP_ERR: 2367 ibwc_status = IB_WC_BAD_RESP_ERR; 2368 break; 2369 case OCRDMA_CQE_LOC_ACCESS_ERR: 2370 ibwc_status = IB_WC_LOC_ACCESS_ERR; 2371 break; 2372 case OCRDMA_CQE_REM_INV_REQ_ERR: 2373 ibwc_status = IB_WC_REM_INV_REQ_ERR; 2374 break; 2375 case OCRDMA_CQE_REM_ACCESS_ERR: 2376 ibwc_status = IB_WC_REM_ACCESS_ERR; 2377 break; 2378 case OCRDMA_CQE_REM_OP_ERR: 2379 ibwc_status = IB_WC_REM_OP_ERR; 2380 break; 2381 case OCRDMA_CQE_RETRY_EXC_ERR: 2382 ibwc_status = IB_WC_RETRY_EXC_ERR; 2383 break; 2384 case OCRDMA_CQE_RNR_RETRY_EXC_ERR: 2385 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR; 2386 break; 2387 case OCRDMA_CQE_LOC_RDD_VIOL_ERR: 2388 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR; 2389 break; 2390 case OCRDMA_CQE_REM_INV_RD_REQ_ERR: 2391 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR; 2392 break; 2393 case OCRDMA_CQE_REM_ABORT_ERR: 2394 ibwc_status = IB_WC_REM_ABORT_ERR; 2395 break; 2396 case OCRDMA_CQE_INV_EECN_ERR: 2397 ibwc_status = IB_WC_INV_EECN_ERR; 2398 break; 2399 case OCRDMA_CQE_INV_EEC_STATE_ERR: 2400 ibwc_status = IB_WC_INV_EEC_STATE_ERR; 2401 break; 2402 case OCRDMA_CQE_FATAL_ERR: 2403 ibwc_status = IB_WC_FATAL_ERR; 2404 break; 2405 case OCRDMA_CQE_RESP_TIMEOUT_ERR: 2406 ibwc_status = IB_WC_RESP_TIMEOUT_ERR; 2407 break; 2408 default: 2409 ibwc_status = IB_WC_GENERAL_ERR; 2410 break; 2411 } 2412 return ibwc_status; 2413 } 2414 2415 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, 2416 u32 wqe_idx) 2417 { 2418 struct ocrdma_hdr_wqe *hdr; 2419 struct ocrdma_sge *rw; 2420 int opcode; 2421 2422 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx); 2423 2424 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid; 2425 /* Undo the hdr->cw swap */ 2426 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK; 2427 switch (opcode) { 2428 case OCRDMA_WRITE: 2429 ibwc->opcode = IB_WC_RDMA_WRITE; 2430 break; 2431 case OCRDMA_READ: 2432 rw = (struct ocrdma_sge *)(hdr + 1); 2433 ibwc->opcode = IB_WC_RDMA_READ; 2434 ibwc->byte_len = rw->len; 2435 break; 2436 case OCRDMA_SEND: 2437 ibwc->opcode = IB_WC_SEND; 2438 break; 2439 case OCRDMA_FR_MR: 2440 ibwc->opcode = IB_WC_REG_MR; 2441 break; 2442 case OCRDMA_LKEY_INV: 2443 ibwc->opcode = IB_WC_LOCAL_INV; 2444 break; 2445 default: 2446 ibwc->status = IB_WC_GENERAL_ERR; 2447 pr_err("%s() invalid opcode received = 0x%x\n", 2448 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK); 2449 break; 2450 } 2451 } 2452 2453 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, 2454 struct ocrdma_cqe *cqe) 2455 { 2456 if (is_cqe_for_sq(cqe)) { 2457 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2458 cqe->flags_status_srcqpn) & 2459 ~OCRDMA_CQE_STATUS_MASK); 2460 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2461 cqe->flags_status_srcqpn) | 2462 (OCRDMA_CQE_WR_FLUSH_ERR << 2463 OCRDMA_CQE_STATUS_SHIFT)); 2464 } else { 2465 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2466 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2467 cqe->flags_status_srcqpn) & 2468 ~OCRDMA_CQE_UD_STATUS_MASK); 2469 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2470 cqe->flags_status_srcqpn) | 2471 (OCRDMA_CQE_WR_FLUSH_ERR << 2472 OCRDMA_CQE_UD_STATUS_SHIFT)); 2473 } else { 2474 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2475 cqe->flags_status_srcqpn) & 2476 ~OCRDMA_CQE_STATUS_MASK); 2477 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu( 2478 cqe->flags_status_srcqpn) | 2479 (OCRDMA_CQE_WR_FLUSH_ERR << 2480 OCRDMA_CQE_STATUS_SHIFT)); 2481 } 2482 } 2483 } 2484 2485 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2486 struct ocrdma_qp *qp, int status) 2487 { 2488 bool expand = false; 2489 2490 ibwc->byte_len = 0; 2491 ibwc->qp = &qp->ibqp; 2492 ibwc->status = ocrdma_to_ibwc_err(status); 2493 2494 ocrdma_flush_qp(qp); 2495 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL); 2496 2497 /* if wqe/rqe pending for which cqe needs to be returned, 2498 * trigger inflating it. 2499 */ 2500 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) { 2501 expand = true; 2502 ocrdma_set_cqe_status_flushed(qp, cqe); 2503 } 2504 return expand; 2505 } 2506 2507 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2508 struct ocrdma_qp *qp, int status) 2509 { 2510 ibwc->opcode = IB_WC_RECV; 2511 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2512 ocrdma_hwq_inc_tail(&qp->rq); 2513 2514 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2515 } 2516 2517 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, 2518 struct ocrdma_qp *qp, int status) 2519 { 2520 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2521 ocrdma_hwq_inc_tail(&qp->sq); 2522 2523 return ocrdma_update_err_cqe(ibwc, cqe, qp, status); 2524 } 2525 2526 2527 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, 2528 struct ocrdma_cqe *cqe, struct ib_wc *ibwc, 2529 bool *polled, bool *stop) 2530 { 2531 bool expand; 2532 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2533 int status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2534 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2535 if (status < OCRDMA_MAX_CQE_ERR) 2536 atomic_inc(&dev->cqe_err_stats[status]); 2537 2538 /* when hw sq is empty, but rq is not empty, so we continue 2539 * to keep the cqe in order to get the cq event again. 2540 */ 2541 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) { 2542 /* when cq for rq and sq is same, it is safe to return 2543 * flush cqe for RQEs. 2544 */ 2545 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2546 *polled = true; 2547 status = OCRDMA_CQE_WR_FLUSH_ERR; 2548 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2549 } else { 2550 /* stop processing further cqe as this cqe is used for 2551 * triggering cq event on buddy cq of RQ. 2552 * When QP is destroyed, this cqe will be removed 2553 * from the cq's hardware q. 2554 */ 2555 *polled = false; 2556 *stop = true; 2557 expand = false; 2558 } 2559 } else if (is_hw_sq_empty(qp)) { 2560 /* Do nothing */ 2561 expand = false; 2562 *polled = false; 2563 *stop = false; 2564 } else { 2565 *polled = true; 2566 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2567 } 2568 return expand; 2569 } 2570 2571 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, 2572 struct ocrdma_cqe *cqe, 2573 struct ib_wc *ibwc, bool *polled) 2574 { 2575 bool expand = false; 2576 int tail = qp->sq.tail; 2577 u32 wqe_idx; 2578 2579 if (!qp->wqe_wr_id_tbl[tail].signaled) { 2580 *polled = false; /* WC cannot be consumed yet */ 2581 } else { 2582 ibwc->status = IB_WC_SUCCESS; 2583 ibwc->wc_flags = 0; 2584 ibwc->qp = &qp->ibqp; 2585 ocrdma_update_wc(qp, ibwc, tail); 2586 *polled = true; 2587 } 2588 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) & 2589 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx; 2590 if (tail != wqe_idx) 2591 expand = true; /* Coalesced CQE can't be consumed yet */ 2592 2593 ocrdma_hwq_inc_tail(&qp->sq); 2594 return expand; 2595 } 2596 2597 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2598 struct ib_wc *ibwc, bool *polled, bool *stop) 2599 { 2600 int status; 2601 bool expand; 2602 2603 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2604 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2605 2606 if (status == OCRDMA_CQE_SUCCESS) 2607 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled); 2608 else 2609 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop); 2610 return expand; 2611 } 2612 2613 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, 2614 struct ocrdma_cqe *cqe) 2615 { 2616 int status; 2617 u16 hdr_type = 0; 2618 2619 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2620 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT; 2621 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) & 2622 OCRDMA_CQE_SRCQP_MASK; 2623 ibwc->pkey_index = 0; 2624 ibwc->wc_flags = IB_WC_GRH; 2625 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2626 OCRDMA_CQE_UD_XFER_LEN_SHIFT) & 2627 OCRDMA_CQE_UD_XFER_LEN_MASK; 2628 2629 if (ocrdma_is_udp_encap_supported(dev)) { 2630 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >> 2631 OCRDMA_CQE_UD_L3TYPE_SHIFT) & 2632 OCRDMA_CQE_UD_L3TYPE_MASK; 2633 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 2634 ibwc->network_hdr_type = hdr_type; 2635 } 2636 2637 return status; 2638 } 2639 2640 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, 2641 struct ocrdma_cqe *cqe, 2642 struct ocrdma_qp *qp) 2643 { 2644 unsigned long flags; 2645 struct ocrdma_srq *srq; 2646 u32 wqe_idx; 2647 2648 srq = get_ocrdma_srq(qp->ibqp.srq); 2649 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >> 2650 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx; 2651 BUG_ON(wqe_idx < 1); 2652 2653 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2654 spin_lock_irqsave(&srq->q_lock, flags); 2655 ocrdma_srq_toggle_bit(srq, wqe_idx - 1); 2656 spin_unlock_irqrestore(&srq->q_lock, flags); 2657 ocrdma_hwq_inc_tail(&srq->rq); 2658 } 2659 2660 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2661 struct ib_wc *ibwc, bool *polled, bool *stop, 2662 int status) 2663 { 2664 bool expand; 2665 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device); 2666 2667 if (status < OCRDMA_MAX_CQE_ERR) 2668 atomic_inc(&dev->cqe_err_stats[status]); 2669 2670 /* when hw_rq is empty, but wq is not empty, so continue 2671 * to keep the cqe to get the cq event again. 2672 */ 2673 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) { 2674 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) { 2675 *polled = true; 2676 status = OCRDMA_CQE_WR_FLUSH_ERR; 2677 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status); 2678 } else { 2679 *polled = false; 2680 *stop = true; 2681 expand = false; 2682 } 2683 } else if (is_hw_rq_empty(qp)) { 2684 /* Do nothing */ 2685 expand = false; 2686 *polled = false; 2687 *stop = false; 2688 } else { 2689 *polled = true; 2690 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status); 2691 } 2692 return expand; 2693 } 2694 2695 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, 2696 struct ocrdma_cqe *cqe, struct ib_wc *ibwc) 2697 { 2698 struct ocrdma_dev *dev; 2699 2700 dev = get_ocrdma_dev(qp->ibqp.device); 2701 ibwc->opcode = IB_WC_RECV; 2702 ibwc->qp = &qp->ibqp; 2703 ibwc->status = IB_WC_SUCCESS; 2704 2705 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2706 ocrdma_update_ud_rcqe(dev, ibwc, cqe); 2707 else 2708 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen); 2709 2710 if (is_cqe_imm(cqe)) { 2711 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2712 ibwc->wc_flags |= IB_WC_WITH_IMM; 2713 } else if (is_cqe_wr_imm(cqe)) { 2714 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 2715 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt)); 2716 ibwc->wc_flags |= IB_WC_WITH_IMM; 2717 } else if (is_cqe_invalidated(cqe)) { 2718 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2719 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2720 } 2721 if (qp->ibqp.srq) { 2722 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2723 } else { 2724 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2725 ocrdma_hwq_inc_tail(&qp->rq); 2726 } 2727 } 2728 2729 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, 2730 struct ib_wc *ibwc, bool *polled, bool *stop) 2731 { 2732 int status; 2733 bool expand = false; 2734 2735 ibwc->wc_flags = 0; 2736 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) { 2737 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2738 OCRDMA_CQE_UD_STATUS_MASK) >> 2739 OCRDMA_CQE_UD_STATUS_SHIFT; 2740 } else { 2741 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2742 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2743 } 2744 2745 if (status == OCRDMA_CQE_SUCCESS) { 2746 *polled = true; 2747 ocrdma_poll_success_rcqe(qp, cqe, ibwc); 2748 } else { 2749 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop, 2750 status); 2751 } 2752 return expand; 2753 } 2754 2755 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, 2756 u16 cur_getp) 2757 { 2758 if (cq->phase_change) { 2759 if (cur_getp == 0) 2760 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2761 } else { 2762 /* clear valid bit */ 2763 cqe->flags_status_srcqpn = 0; 2764 } 2765 } 2766 2767 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2768 struct ib_wc *ibwc) 2769 { 2770 u16 qpn = 0; 2771 int i = 0; 2772 bool expand = false; 2773 int polled_hw_cqes = 0; 2774 struct ocrdma_qp *qp = NULL; 2775 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device); 2776 struct ocrdma_cqe *cqe; 2777 u16 cur_getp; bool polled = false; bool stop = false; 2778 2779 cur_getp = cq->getp; 2780 while (num_entries) { 2781 cqe = cq->va + cur_getp; 2782 /* check whether valid cqe or not */ 2783 if (!is_cqe_valid(cq, cqe)) 2784 break; 2785 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK); 2786 /* ignore discarded cqe */ 2787 if (qpn == 0) 2788 goto skip_cqe; 2789 qp = dev->qp_tbl[qpn]; 2790 BUG_ON(qp == NULL); 2791 2792 if (is_cqe_for_sq(cqe)) { 2793 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled, 2794 &stop); 2795 } else { 2796 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled, 2797 &stop); 2798 } 2799 if (expand) 2800 goto expand_cqe; 2801 if (stop) 2802 goto stop_cqe; 2803 /* clear qpn to avoid duplicate processing by discard_cqe() */ 2804 cqe->cmn.qpn = 0; 2805 skip_cqe: 2806 polled_hw_cqes += 1; 2807 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 2808 ocrdma_change_cq_phase(cq, cqe, cur_getp); 2809 expand_cqe: 2810 if (polled) { 2811 num_entries -= 1; 2812 i += 1; 2813 ibwc = ibwc + 1; 2814 polled = false; 2815 } 2816 } 2817 stop_cqe: 2818 cq->getp = cur_getp; 2819 2820 if (polled_hw_cqes) 2821 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes); 2822 2823 return i; 2824 } 2825 2826 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ 2827 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, 2828 struct ocrdma_qp *qp, struct ib_wc *ibwc) 2829 { 2830 int err_cqes = 0; 2831 2832 while (num_entries) { 2833 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp)) 2834 break; 2835 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) { 2836 ocrdma_update_wc(qp, ibwc, qp->sq.tail); 2837 ocrdma_hwq_inc_tail(&qp->sq); 2838 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2839 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2840 ocrdma_hwq_inc_tail(&qp->rq); 2841 } else { 2842 return err_cqes; 2843 } 2844 ibwc->byte_len = 0; 2845 ibwc->status = IB_WC_WR_FLUSH_ERR; 2846 ibwc = ibwc + 1; 2847 err_cqes += 1; 2848 num_entries -= 1; 2849 } 2850 return err_cqes; 2851 } 2852 2853 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2854 { 2855 int cqes_to_poll = num_entries; 2856 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2857 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2858 int num_os_cqe = 0, err_cqes = 0; 2859 struct ocrdma_qp *qp; 2860 unsigned long flags; 2861 2862 /* poll cqes from adapter CQ */ 2863 spin_lock_irqsave(&cq->cq_lock, flags); 2864 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc); 2865 spin_unlock_irqrestore(&cq->cq_lock, flags); 2866 cqes_to_poll -= num_os_cqe; 2867 2868 if (cqes_to_poll) { 2869 wc = wc + num_os_cqe; 2870 /* adapter returns single error cqe when qp moves to 2871 * error state. So insert error cqes with wc_status as 2872 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ 2873 * respectively which uses this CQ. 2874 */ 2875 spin_lock_irqsave(&dev->flush_q_lock, flags); 2876 list_for_each_entry(qp, &cq->sq_head, sq_entry) { 2877 if (cqes_to_poll == 0) 2878 break; 2879 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc); 2880 cqes_to_poll -= err_cqes; 2881 num_os_cqe += err_cqes; 2882 wc = wc + err_cqes; 2883 } 2884 spin_unlock_irqrestore(&dev->flush_q_lock, flags); 2885 } 2886 return num_os_cqe; 2887 } 2888 2889 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2890 { 2891 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 2892 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device); 2893 u16 cq_id; 2894 unsigned long flags; 2895 bool arm_needed = false, sol_needed = false; 2896 2897 cq_id = cq->id; 2898 2899 spin_lock_irqsave(&cq->cq_lock, flags); 2900 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 2901 arm_needed = true; 2902 if (cq_flags & IB_CQ_SOLICITED) 2903 sol_needed = true; 2904 2905 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0); 2906 spin_unlock_irqrestore(&cq->cq_lock, flags); 2907 2908 return 0; 2909 } 2910 2911 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2912 u32 max_num_sg, struct ib_udata *udata) 2913 { 2914 int status; 2915 struct ocrdma_mr *mr; 2916 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 2917 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device); 2918 2919 if (mr_type != IB_MR_TYPE_MEM_REG) 2920 return ERR_PTR(-EINVAL); 2921 2922 if (max_num_sg > dev->attr.max_pages_per_frmr) 2923 return ERR_PTR(-EINVAL); 2924 2925 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2926 if (!mr) 2927 return ERR_PTR(-ENOMEM); 2928 2929 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 2930 if (!mr->pages) { 2931 status = -ENOMEM; 2932 goto pl_err; 2933 } 2934 2935 status = ocrdma_get_pbl_info(dev, mr, max_num_sg); 2936 if (status) 2937 goto pbl_err; 2938 mr->hwmr.fr_mr = 1; 2939 mr->hwmr.remote_rd = 0; 2940 mr->hwmr.remote_wr = 0; 2941 mr->hwmr.local_rd = 0; 2942 mr->hwmr.local_wr = 0; 2943 mr->hwmr.mw_bind = 0; 2944 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr); 2945 if (status) 2946 goto pbl_err; 2947 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0); 2948 if (status) 2949 goto mbx_err; 2950 mr->ibmr.rkey = mr->hwmr.lkey; 2951 mr->ibmr.lkey = mr->hwmr.lkey; 2952 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = 2953 (unsigned long) mr; 2954 return &mr->ibmr; 2955 mbx_err: 2956 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); 2957 pbl_err: 2958 kfree(mr->pages); 2959 pl_err: 2960 kfree(mr); 2961 return ERR_PTR(-ENOMEM); 2962 } 2963 2964 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) 2965 { 2966 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2967 2968 if (unlikely(mr->npages == mr->hwmr.num_pbes)) 2969 return -ENOMEM; 2970 2971 mr->pages[mr->npages++] = addr; 2972 2973 return 0; 2974 } 2975 2976 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2977 unsigned int *sg_offset) 2978 { 2979 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr); 2980 2981 mr->npages = 0; 2982 2983 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page); 2984 } 2985