1 /* 2 * Broadcom NetXtreme-E RoCE driver. 3 * 4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term 5 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in 21 * the documentation and/or other materials provided with the 22 * distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Description: IB Verbs interpreter 37 */ 38 39 #include <linux/interrupt.h> 40 #include <linux/types.h> 41 #include <linux/pci.h> 42 #include <linux/netdevice.h> 43 #include <linux/if_ether.h> 44 #include <net/addrconf.h> 45 46 #include <rdma/ib_verbs.h> 47 #include <rdma/ib_user_verbs.h> 48 #include <rdma/ib_umem.h> 49 #include <rdma/ib_addr.h> 50 #include <rdma/ib_mad.h> 51 #include <rdma/ib_cache.h> 52 #include <rdma/uverbs_ioctl.h> 53 54 #include "bnxt_ulp.h" 55 56 #include "roce_hsi.h" 57 #include "qplib_res.h" 58 #include "qplib_sp.h" 59 #include "qplib_fp.h" 60 #include "qplib_rcfw.h" 61 62 #include "bnxt_re.h" 63 #include "ib_verbs.h" 64 #include <rdma/bnxt_re-abi.h> 65 66 static int __from_ib_access_flags(int iflags) 67 { 68 int qflags = 0; 69 70 if (iflags & IB_ACCESS_LOCAL_WRITE) 71 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 72 if (iflags & IB_ACCESS_REMOTE_READ) 73 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 74 if (iflags & IB_ACCESS_REMOTE_WRITE) 75 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 76 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 77 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 78 if (iflags & IB_ACCESS_MW_BIND) 79 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 80 if (iflags & IB_ZERO_BASED) 81 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 82 if (iflags & IB_ACCESS_ON_DEMAND) 83 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 84 return qflags; 85 }; 86 87 static enum ib_access_flags __to_ib_access_flags(int qflags) 88 { 89 enum ib_access_flags iflags = 0; 90 91 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 92 iflags |= IB_ACCESS_LOCAL_WRITE; 93 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 94 iflags |= IB_ACCESS_REMOTE_WRITE; 95 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 96 iflags |= IB_ACCESS_REMOTE_READ; 97 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 98 iflags |= IB_ACCESS_REMOTE_ATOMIC; 99 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 100 iflags |= IB_ACCESS_MW_BIND; 101 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 102 iflags |= IB_ZERO_BASED; 103 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 104 iflags |= IB_ACCESS_ON_DEMAND; 105 return iflags; 106 }; 107 108 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, 109 struct bnxt_qplib_sge *sg_list, int num) 110 { 111 int i, total = 0; 112 113 for (i = 0; i < num; i++) { 114 sg_list[i].addr = ib_sg_list[i].addr; 115 sg_list[i].lkey = ib_sg_list[i].lkey; 116 sg_list[i].size = ib_sg_list[i].length; 117 total += sg_list[i].size; 118 } 119 return total; 120 } 121 122 /* Device */ 123 int bnxt_re_query_device(struct ib_device *ibdev, 124 struct ib_device_attr *ib_attr, 125 struct ib_udata *udata) 126 { 127 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 128 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 129 130 memset(ib_attr, 0, sizeof(*ib_attr)); 131 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 132 min(sizeof(dev_attr->fw_ver), 133 sizeof(ib_attr->fw_ver))); 134 addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid, 135 rdev->netdev->dev_addr); 136 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 137 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED; 138 139 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 140 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 141 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 142 ib_attr->max_qp = dev_attr->max_qp; 143 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 144 ib_attr->device_cap_flags = 145 IB_DEVICE_CURR_QP_STATE_MOD 146 | IB_DEVICE_RC_RNR_NAK_GEN 147 | IB_DEVICE_SHUTDOWN_PORT 148 | IB_DEVICE_SYS_IMAGE_GUID 149 | IB_DEVICE_RESIZE_MAX_WR 150 | IB_DEVICE_PORT_ACTIVE_EVENT 151 | IB_DEVICE_N_NOTIFY_CQ 152 | IB_DEVICE_MEM_WINDOW 153 | IB_DEVICE_MEM_WINDOW_TYPE_2B 154 | IB_DEVICE_MEM_MGT_EXTENSIONS; 155 ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 156 ib_attr->max_send_sge = dev_attr->max_qp_sges; 157 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 158 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 159 ib_attr->max_cq = dev_attr->max_cq; 160 ib_attr->max_cqe = dev_attr->max_cq_wqes; 161 ib_attr->max_mr = dev_attr->max_mr; 162 ib_attr->max_pd = dev_attr->max_pd; 163 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 164 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 165 ib_attr->atomic_cap = IB_ATOMIC_NONE; 166 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE; 167 if (dev_attr->is_atomic) { 168 ib_attr->atomic_cap = IB_ATOMIC_GLOB; 169 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; 170 } 171 172 ib_attr->max_ee_rd_atom = 0; 173 ib_attr->max_res_rd_atom = 0; 174 ib_attr->max_ee_init_rd_atom = 0; 175 ib_attr->max_ee = 0; 176 ib_attr->max_rdd = 0; 177 ib_attr->max_mw = dev_attr->max_mw; 178 ib_attr->max_raw_ipv6_qp = 0; 179 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 180 ib_attr->max_mcast_grp = 0; 181 ib_attr->max_mcast_qp_attach = 0; 182 ib_attr->max_total_mcast_qp_attach = 0; 183 ib_attr->max_ah = dev_attr->max_ah; 184 185 ib_attr->max_srq = dev_attr->max_srq; 186 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 187 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 188 189 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 190 191 ib_attr->max_pkeys = 1; 192 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 193 return 0; 194 } 195 196 /* Port */ 197 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num, 198 struct ib_port_attr *port_attr) 199 { 200 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 201 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 202 203 memset(port_attr, 0, sizeof(*port_attr)); 204 205 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) { 206 port_attr->state = IB_PORT_ACTIVE; 207 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 208 } else { 209 port_attr->state = IB_PORT_DOWN; 210 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 211 } 212 port_attr->max_mtu = IB_MTU_4096; 213 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu); 214 port_attr->gid_tbl_len = dev_attr->max_sgid; 215 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 216 IB_PORT_DEVICE_MGMT_SUP | 217 IB_PORT_VENDOR_CLASS_SUP; 218 port_attr->ip_gids = true; 219 220 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 221 port_attr->bad_pkey_cntr = 0; 222 port_attr->qkey_viol_cntr = 0; 223 port_attr->pkey_tbl_len = dev_attr->max_pkey; 224 port_attr->lid = 0; 225 port_attr->sm_lid = 0; 226 port_attr->lmc = 0; 227 port_attr->max_vl_num = 4; 228 port_attr->sm_sl = 0; 229 port_attr->subnet_timeout = 0; 230 port_attr->init_type_reply = 0; 231 port_attr->active_speed = rdev->active_speed; 232 port_attr->active_width = rdev->active_width; 233 234 return 0; 235 } 236 237 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num, 238 struct ib_port_immutable *immutable) 239 { 240 struct ib_port_attr port_attr; 241 242 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 243 return -EINVAL; 244 245 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 246 immutable->gid_tbl_len = port_attr.gid_tbl_len; 247 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 248 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; 249 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 250 return 0; 251 } 252 253 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str) 254 { 255 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 256 257 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d", 258 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1], 259 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]); 260 } 261 262 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num, 263 u16 index, u16 *pkey) 264 { 265 if (index > 0) 266 return -EINVAL; 267 268 *pkey = IB_DEFAULT_PKEY_FULL; 269 270 return 0; 271 } 272 273 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num, 274 int index, union ib_gid *gid) 275 { 276 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 277 int rc = 0; 278 279 /* Ignore port_num */ 280 memset(gid, 0, sizeof(*gid)); 281 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 282 &rdev->qplib_res.sgid_tbl, index, 283 (struct bnxt_qplib_gid *)gid); 284 return rc; 285 } 286 287 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) 288 { 289 int rc = 0; 290 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 291 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 292 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 293 struct bnxt_qplib_gid *gid_to_del; 294 u16 vlan_id = 0xFFFF; 295 296 /* Delete the entry from the hardware */ 297 ctx = *context; 298 if (!ctx) 299 return -EINVAL; 300 301 if (sgid_tbl && sgid_tbl->active) { 302 if (ctx->idx >= sgid_tbl->max) 303 return -EINVAL; 304 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; 305 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; 306 /* DEL_GID is called in WQ context(netdevice_event_work_handler) 307 * or via the ib_unregister_device path. In the former case QP1 308 * may not be destroyed yet, in which case just return as FW 309 * needs that entry to be present and will fail it's deletion. 310 * We could get invoked again after QP1 is destroyed OR get an 311 * ADD_GID call with a different GID value for the same index 312 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 313 */ 314 if (ctx->idx == 0 && 315 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 316 ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) { 317 ibdev_dbg(&rdev->ibdev, 318 "Trying to delete GID0 while QP1 is alive\n"); 319 return -EFAULT; 320 } 321 ctx->refcnt--; 322 if (!ctx->refcnt) { 323 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, 324 vlan_id, true); 325 if (rc) { 326 ibdev_err(&rdev->ibdev, 327 "Failed to remove GID: %#x", rc); 328 } else { 329 ctx_tbl = sgid_tbl->ctx; 330 ctx_tbl[ctx->idx] = NULL; 331 kfree(ctx); 332 } 333 } 334 } else { 335 return -EINVAL; 336 } 337 return rc; 338 } 339 340 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) 341 { 342 int rc; 343 u32 tbl_idx = 0; 344 u16 vlan_id = 0xFFFF; 345 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 346 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 347 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 348 349 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL); 350 if (rc) 351 return rc; 352 353 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid, 354 rdev->qplib_res.netdev->dev_addr, 355 vlan_id, true, &tbl_idx); 356 if (rc == -EALREADY) { 357 ctx_tbl = sgid_tbl->ctx; 358 ctx_tbl[tbl_idx]->refcnt++; 359 *context = ctx_tbl[tbl_idx]; 360 return 0; 361 } 362 363 if (rc < 0) { 364 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc); 365 return rc; 366 } 367 368 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 369 if (!ctx) 370 return -ENOMEM; 371 ctx_tbl = sgid_tbl->ctx; 372 ctx->idx = tbl_idx; 373 ctx->refcnt = 1; 374 ctx_tbl[tbl_idx] = ctx; 375 *context = ctx; 376 377 return rc; 378 } 379 380 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 381 u32 port_num) 382 { 383 return IB_LINK_LAYER_ETHERNET; 384 } 385 386 #define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE) 387 388 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd) 389 { 390 struct bnxt_re_fence_data *fence = &pd->fence; 391 struct ib_mr *ib_mr = &fence->mr->ib_mr; 392 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 393 394 memset(wqe, 0, sizeof(*wqe)); 395 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 396 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 397 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 398 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 399 wqe->bind.zero_based = false; 400 wqe->bind.parent_l_key = ib_mr->lkey; 401 wqe->bind.va = (u64)(unsigned long)fence->va; 402 wqe->bind.length = fence->size; 403 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 404 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 405 406 /* Save the initial rkey in fence structure for now; 407 * wqe->bind.r_key will be set at (re)bind time. 408 */ 409 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 410 } 411 412 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 413 { 414 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 415 qplib_qp); 416 struct ib_pd *ib_pd = qp->ib_qp.pd; 417 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 418 struct bnxt_re_fence_data *fence = &pd->fence; 419 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 420 struct bnxt_qplib_swqe wqe; 421 int rc; 422 423 memcpy(&wqe, fence_wqe, sizeof(wqe)); 424 wqe.bind.r_key = fence->bind_rkey; 425 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 426 427 ibdev_dbg(&qp->rdev->ibdev, 428 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 429 wqe.bind.r_key, qp->qplib_qp.id, pd); 430 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 431 if (rc) { 432 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n"); 433 return rc; 434 } 435 bnxt_qplib_post_send_db(&qp->qplib_qp); 436 437 return rc; 438 } 439 440 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd) 441 { 442 struct bnxt_re_fence_data *fence = &pd->fence; 443 struct bnxt_re_dev *rdev = pd->rdev; 444 struct device *dev = &rdev->en_dev->pdev->dev; 445 struct bnxt_re_mr *mr = fence->mr; 446 447 if (fence->mw) { 448 bnxt_re_dealloc_mw(fence->mw); 449 fence->mw = NULL; 450 } 451 if (mr) { 452 if (mr->ib_mr.rkey) 453 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 454 true); 455 if (mr->ib_mr.lkey) 456 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 457 kfree(mr); 458 fence->mr = NULL; 459 } 460 if (fence->dma_addr) { 461 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES, 462 DMA_BIDIRECTIONAL); 463 fence->dma_addr = 0; 464 } 465 } 466 467 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd) 468 { 469 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 470 struct bnxt_re_fence_data *fence = &pd->fence; 471 struct bnxt_re_dev *rdev = pd->rdev; 472 struct device *dev = &rdev->en_dev->pdev->dev; 473 struct bnxt_re_mr *mr = NULL; 474 dma_addr_t dma_addr = 0; 475 struct ib_mw *mw; 476 int rc; 477 478 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES, 479 DMA_BIDIRECTIONAL); 480 rc = dma_mapping_error(dev, dma_addr); 481 if (rc) { 482 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n"); 483 rc = -EIO; 484 fence->dma_addr = 0; 485 goto fail; 486 } 487 fence->dma_addr = dma_addr; 488 489 /* Allocate a MR */ 490 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 491 if (!mr) { 492 rc = -ENOMEM; 493 goto fail; 494 } 495 fence->mr = mr; 496 mr->rdev = rdev; 497 mr->qplib_mr.pd = &pd->qplib_pd; 498 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 499 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 500 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 501 if (rc) { 502 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n"); 503 goto fail; 504 } 505 506 /* Register MR */ 507 mr->ib_mr.lkey = mr->qplib_mr.lkey; 508 mr->qplib_mr.va = (u64)(unsigned long)fence->va; 509 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES; 510 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 511 BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE); 512 if (rc) { 513 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n"); 514 goto fail; 515 } 516 mr->ib_mr.rkey = mr->qplib_mr.rkey; 517 518 /* Create a fence MW only for kernel consumers */ 519 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL); 520 if (IS_ERR(mw)) { 521 ibdev_err(&rdev->ibdev, 522 "Failed to create fence-MW for PD: %p\n", pd); 523 rc = PTR_ERR(mw); 524 goto fail; 525 } 526 fence->mw = mw; 527 528 bnxt_re_create_fence_wqe(pd); 529 return 0; 530 531 fail: 532 bnxt_re_destroy_fence_mr(pd); 533 return rc; 534 } 535 536 /* Protection Domains */ 537 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 538 { 539 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 540 struct bnxt_re_dev *rdev = pd->rdev; 541 542 bnxt_re_destroy_fence_mr(pd); 543 544 if (pd->qplib_pd.id) { 545 if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res, 546 &rdev->qplib_res.pd_tbl, 547 &pd->qplib_pd)) 548 atomic_dec(&rdev->pd_count); 549 } 550 return 0; 551 } 552 553 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 554 { 555 struct ib_device *ibdev = ibpd->device; 556 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 557 struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context( 558 udata, struct bnxt_re_ucontext, ib_uctx); 559 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd); 560 int rc; 561 562 pd->rdev = rdev; 563 if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) { 564 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD"); 565 rc = -ENOMEM; 566 goto fail; 567 } 568 569 if (udata) { 570 struct bnxt_re_pd_resp resp; 571 572 if (!ucntx->dpi.dbr) { 573 /* Allocate DPI in alloc_pd to avoid failing of 574 * ibv_devinfo and family of application when DPIs 575 * are depleted. 576 */ 577 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, 578 &ucntx->dpi, ucntx)) { 579 rc = -ENOMEM; 580 goto dbfail; 581 } 582 } 583 584 resp.pdid = pd->qplib_pd.id; 585 /* Still allow mapping this DBR to the new user PD. */ 586 resp.dpi = ucntx->dpi.dpi; 587 resp.dbr = (u64)ucntx->dpi.umdbr; 588 589 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 590 if (rc) { 591 ibdev_err(&rdev->ibdev, 592 "Failed to copy user response\n"); 593 goto dbfail; 594 } 595 } 596 597 if (!udata) 598 if (bnxt_re_create_fence_mr(pd)) 599 ibdev_warn(&rdev->ibdev, 600 "Failed to create Fence-MR\n"); 601 atomic_inc(&rdev->pd_count); 602 603 return 0; 604 dbfail: 605 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 606 &pd->qplib_pd); 607 fail: 608 return rc; 609 } 610 611 /* Address Handles */ 612 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 613 { 614 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 615 struct bnxt_re_dev *rdev = ah->rdev; 616 617 bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, 618 !(flags & RDMA_DESTROY_AH_SLEEPABLE)); 619 atomic_dec(&rdev->ah_count); 620 621 return 0; 622 } 623 624 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype) 625 { 626 u8 nw_type; 627 628 switch (ntype) { 629 case RDMA_NETWORK_IPV4: 630 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 631 break; 632 case RDMA_NETWORK_IPV6: 633 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 634 break; 635 default: 636 nw_type = CMDQ_CREATE_AH_TYPE_V1; 637 break; 638 } 639 return nw_type; 640 } 641 642 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, 643 struct ib_udata *udata) 644 { 645 struct ib_pd *ib_pd = ib_ah->pd; 646 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 647 struct rdma_ah_attr *ah_attr = init_attr->ah_attr; 648 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 649 struct bnxt_re_dev *rdev = pd->rdev; 650 const struct ib_gid_attr *sgid_attr; 651 struct bnxt_re_gid_ctx *ctx; 652 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 653 u8 nw_type; 654 int rc; 655 656 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) { 657 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set"); 658 return -EINVAL; 659 } 660 661 ah->rdev = rdev; 662 ah->qplib_ah.pd = &pd->qplib_pd; 663 664 /* Supply the configuration for the HW */ 665 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 666 sizeof(union ib_gid)); 667 sgid_attr = grh->sgid_attr; 668 /* Get the HW context of the GID. The reference 669 * of GID table entry is already taken by the caller. 670 */ 671 ctx = rdma_read_gid_hw_context(sgid_attr); 672 ah->qplib_ah.sgid_index = ctx->idx; 673 ah->qplib_ah.host_sgid_index = grh->sgid_index; 674 ah->qplib_ah.traffic_class = grh->traffic_class; 675 ah->qplib_ah.flow_label = grh->flow_label; 676 ah->qplib_ah.hop_limit = grh->hop_limit; 677 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 678 679 /* Get network header type for this GID */ 680 nw_type = rdma_gid_attr_network_type(sgid_attr); 681 ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type); 682 683 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN); 684 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, 685 !(init_attr->flags & 686 RDMA_CREATE_AH_SLEEPABLE)); 687 if (rc) { 688 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH"); 689 return rc; 690 } 691 692 /* Write AVID to shared page. */ 693 if (udata) { 694 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 695 udata, struct bnxt_re_ucontext, ib_uctx); 696 unsigned long flag; 697 u32 *wrptr; 698 699 spin_lock_irqsave(&uctx->sh_lock, flag); 700 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT); 701 *wrptr = ah->qplib_ah.id; 702 wmb(); /* make sure cache is updated. */ 703 spin_unlock_irqrestore(&uctx->sh_lock, flag); 704 } 705 atomic_inc(&rdev->ah_count); 706 707 return 0; 708 } 709 710 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 711 { 712 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah); 713 714 ah_attr->type = ib_ah->type; 715 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 716 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN); 717 rdma_ah_set_grh(ah_attr, NULL, 0, 718 ah->qplib_ah.host_sgid_index, 719 0, ah->qplib_ah.traffic_class); 720 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 721 rdma_ah_set_port_num(ah_attr, 1); 722 rdma_ah_set_static_rate(ah_attr, 0); 723 return 0; 724 } 725 726 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 727 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock) 728 { 729 unsigned long flags; 730 731 spin_lock_irqsave(&qp->scq->cq_lock, flags); 732 if (qp->rcq != qp->scq) 733 spin_lock(&qp->rcq->cq_lock); 734 else 735 __acquire(&qp->rcq->cq_lock); 736 737 return flags; 738 } 739 740 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 741 unsigned long flags) 742 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock) 743 { 744 if (qp->rcq != qp->scq) 745 spin_unlock(&qp->rcq->cq_lock); 746 else 747 __release(&qp->rcq->cq_lock); 748 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 749 } 750 751 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) 752 { 753 struct bnxt_re_qp *gsi_sqp; 754 struct bnxt_re_ah *gsi_sah; 755 struct bnxt_re_dev *rdev; 756 int rc = 0; 757 758 rdev = qp->rdev; 759 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 760 gsi_sah = rdev->gsi_ctx.gsi_sah; 761 762 ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n"); 763 bnxt_qplib_destroy_ah(&rdev->qplib_res, 764 &gsi_sah->qplib_ah, 765 true); 766 atomic_dec(&rdev->ah_count); 767 bnxt_qplib_clean_qp(&qp->qplib_qp); 768 769 ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n"); 770 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); 771 if (rc) { 772 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed"); 773 goto fail; 774 } 775 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); 776 777 /* remove from active qp list */ 778 mutex_lock(&rdev->qp_lock); 779 list_del(&gsi_sqp->list); 780 mutex_unlock(&rdev->qp_lock); 781 atomic_dec(&rdev->qp_count); 782 783 kfree(rdev->gsi_ctx.sqp_tbl); 784 kfree(gsi_sah); 785 kfree(gsi_sqp); 786 rdev->gsi_ctx.gsi_sqp = NULL; 787 rdev->gsi_ctx.gsi_sah = NULL; 788 rdev->gsi_ctx.sqp_tbl = NULL; 789 790 return 0; 791 fail: 792 return rc; 793 } 794 795 /* Queue Pairs */ 796 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) 797 { 798 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 799 struct bnxt_re_dev *rdev = qp->rdev; 800 unsigned int flags; 801 int rc; 802 803 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp); 804 805 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 806 if (rc) { 807 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); 808 return rc; 809 } 810 811 if (rdma_is_kernel_res(&qp->ib_qp.res)) { 812 flags = bnxt_re_lock_cqs(qp); 813 bnxt_qplib_clean_qp(&qp->qplib_qp); 814 bnxt_re_unlock_cqs(qp, flags); 815 } 816 817 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 818 819 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) { 820 rc = bnxt_re_destroy_gsi_sqp(qp); 821 if (rc) 822 return rc; 823 } 824 825 mutex_lock(&rdev->qp_lock); 826 list_del(&qp->list); 827 mutex_unlock(&rdev->qp_lock); 828 atomic_dec(&rdev->qp_count); 829 830 ib_umem_release(qp->rumem); 831 ib_umem_release(qp->sumem); 832 833 return 0; 834 } 835 836 static u8 __from_ib_qp_type(enum ib_qp_type type) 837 { 838 switch (type) { 839 case IB_QPT_GSI: 840 return CMDQ_CREATE_QP1_TYPE_GSI; 841 case IB_QPT_RC: 842 return CMDQ_CREATE_QP_TYPE_RC; 843 case IB_QPT_UD: 844 return CMDQ_CREATE_QP_TYPE_UD; 845 default: 846 return IB_QPT_MAX; 847 } 848 } 849 850 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp, 851 int rsge, int max) 852 { 853 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 854 rsge = max; 855 return bnxt_re_get_rwqe_size(rsge); 856 } 857 858 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge) 859 { 860 u16 wqe_size, calc_ils; 861 862 wqe_size = bnxt_re_get_swqe_size(nsge); 863 if (ilsize) { 864 calc_ils = sizeof(struct sq_send_hdr) + ilsize; 865 wqe_size = max_t(u16, calc_ils, wqe_size); 866 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr)); 867 } 868 return wqe_size; 869 } 870 871 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, 872 struct ib_qp_init_attr *init_attr) 873 { 874 struct bnxt_qplib_dev_attr *dev_attr; 875 struct bnxt_qplib_qp *qplqp; 876 struct bnxt_re_dev *rdev; 877 struct bnxt_qplib_q *sq; 878 int align, ilsize; 879 880 rdev = qp->rdev; 881 qplqp = &qp->qplib_qp; 882 sq = &qplqp->sq; 883 dev_attr = &rdev->dev_attr; 884 885 align = sizeof(struct sq_send_hdr); 886 ilsize = ALIGN(init_attr->cap.max_inline_data, align); 887 888 sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge); 889 if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges)) 890 return -EINVAL; 891 /* For gen p4 and gen p5 backward compatibility mode 892 * wqe size is fixed to 128 bytes 893 */ 894 if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) && 895 qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 896 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges); 897 898 if (init_attr->cap.max_inline_data) { 899 qplqp->max_inline_data = sq->wqe_size - 900 sizeof(struct sq_send_hdr); 901 init_attr->cap.max_inline_data = qplqp->max_inline_data; 902 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 903 sq->max_sge = qplqp->max_inline_data / 904 sizeof(struct sq_sge); 905 } 906 907 return 0; 908 } 909 910 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, 911 struct bnxt_re_qp *qp, struct ib_udata *udata) 912 { 913 struct bnxt_qplib_qp *qplib_qp; 914 struct bnxt_re_ucontext *cntx; 915 struct bnxt_re_qp_req ureq; 916 int bytes = 0, psn_sz; 917 struct ib_umem *umem; 918 int psn_nume; 919 920 qplib_qp = &qp->qplib_qp; 921 cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, 922 ib_uctx); 923 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 924 return -EFAULT; 925 926 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); 927 /* Consider mapping PSN search memory only for RC QPs. */ 928 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { 929 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ? 930 sizeof(struct sq_psn_search_ext) : 931 sizeof(struct sq_psn_search); 932 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 933 qplib_qp->sq.max_wqe : 934 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / 935 sizeof(struct bnxt_qplib_sge)); 936 bytes += (psn_nume * psn_sz); 937 } 938 939 bytes = PAGE_ALIGN(bytes); 940 umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes, 941 IB_ACCESS_LOCAL_WRITE); 942 if (IS_ERR(umem)) 943 return PTR_ERR(umem); 944 945 qp->sumem = umem; 946 qplib_qp->sq.sg_info.umem = umem; 947 qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; 948 qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; 949 qplib_qp->qp_handle = ureq.qp_handle; 950 951 if (!qp->qplib_qp.srq) { 952 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); 953 bytes = PAGE_ALIGN(bytes); 954 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes, 955 IB_ACCESS_LOCAL_WRITE); 956 if (IS_ERR(umem)) 957 goto rqfail; 958 qp->rumem = umem; 959 qplib_qp->rq.sg_info.umem = umem; 960 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; 961 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; 962 } 963 964 qplib_qp->dpi = &cntx->dpi; 965 return 0; 966 rqfail: 967 ib_umem_release(qp->sumem); 968 qp->sumem = NULL; 969 memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); 970 971 return PTR_ERR(umem); 972 } 973 974 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah 975 (struct bnxt_re_pd *pd, 976 struct bnxt_qplib_res *qp1_res, 977 struct bnxt_qplib_qp *qp1_qp) 978 { 979 struct bnxt_re_dev *rdev = pd->rdev; 980 struct bnxt_re_ah *ah; 981 union ib_gid sgid; 982 int rc; 983 984 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 985 if (!ah) 986 return NULL; 987 988 ah->rdev = rdev; 989 ah->qplib_ah.pd = &pd->qplib_pd; 990 991 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 992 if (rc) 993 goto fail; 994 995 /* supply the dgid data same as sgid */ 996 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 997 sizeof(union ib_gid)); 998 ah->qplib_ah.sgid_index = 0; 999 1000 ah->qplib_ah.traffic_class = 0; 1001 ah->qplib_ah.flow_label = 0; 1002 ah->qplib_ah.hop_limit = 1; 1003 ah->qplib_ah.sl = 0; 1004 /* Have DMAC same as SMAC */ 1005 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr); 1006 1007 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false); 1008 if (rc) { 1009 ibdev_err(&rdev->ibdev, 1010 "Failed to allocate HW AH for Shadow QP"); 1011 goto fail; 1012 } 1013 atomic_inc(&rdev->ah_count); 1014 1015 return ah; 1016 1017 fail: 1018 kfree(ah); 1019 return NULL; 1020 } 1021 1022 static struct bnxt_re_qp *bnxt_re_create_shadow_qp 1023 (struct bnxt_re_pd *pd, 1024 struct bnxt_qplib_res *qp1_res, 1025 struct bnxt_qplib_qp *qp1_qp) 1026 { 1027 struct bnxt_re_dev *rdev = pd->rdev; 1028 struct bnxt_re_qp *qp; 1029 int rc; 1030 1031 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1032 if (!qp) 1033 return NULL; 1034 1035 qp->rdev = rdev; 1036 1037 /* Initialize the shadow QP structure from the QP1 values */ 1038 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr); 1039 1040 qp->qplib_qp.pd = &pd->qplib_pd; 1041 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp); 1042 qp->qplib_qp.type = IB_QPT_UD; 1043 1044 qp->qplib_qp.max_inline_data = 0; 1045 qp->qplib_qp.sig_type = true; 1046 1047 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 1048 qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6); 1049 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 1050 qp->qplib_qp.sq.max_sge = 2; 1051 /* Q full delta can be 1 since it is internal QP */ 1052 qp->qplib_qp.sq.q_full_delta = 1; 1053 qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE; 1054 qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT; 1055 1056 qp->qplib_qp.scq = qp1_qp->scq; 1057 qp->qplib_qp.rcq = qp1_qp->rcq; 1058 1059 qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6); 1060 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 1061 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 1062 /* Q full delta can be 1 since it is internal QP */ 1063 qp->qplib_qp.rq.q_full_delta = 1; 1064 qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE; 1065 qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT; 1066 1067 qp->qplib_qp.mtu = qp1_qp->mtu; 1068 1069 qp->qplib_qp.sq_hdr_buf_size = 0; 1070 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 1071 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1072 1073 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 1074 if (rc) 1075 goto fail; 1076 1077 spin_lock_init(&qp->sq_lock); 1078 INIT_LIST_HEAD(&qp->list); 1079 mutex_lock(&rdev->qp_lock); 1080 list_add_tail(&qp->list, &rdev->qp_list); 1081 atomic_inc(&rdev->qp_count); 1082 mutex_unlock(&rdev->qp_lock); 1083 return qp; 1084 fail: 1085 kfree(qp); 1086 return NULL; 1087 } 1088 1089 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, 1090 struct ib_qp_init_attr *init_attr) 1091 { 1092 struct bnxt_qplib_dev_attr *dev_attr; 1093 struct bnxt_qplib_qp *qplqp; 1094 struct bnxt_re_dev *rdev; 1095 struct bnxt_qplib_q *rq; 1096 int entries; 1097 1098 rdev = qp->rdev; 1099 qplqp = &qp->qplib_qp; 1100 rq = &qplqp->rq; 1101 dev_attr = &rdev->dev_attr; 1102 1103 if (init_attr->srq) { 1104 struct bnxt_re_srq *srq; 1105 1106 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq); 1107 qplqp->srq = &srq->qplib_srq; 1108 rq->max_wqe = 0; 1109 } else { 1110 rq->max_sge = init_attr->cap.max_recv_sge; 1111 if (rq->max_sge > dev_attr->max_qp_sges) 1112 rq->max_sge = dev_attr->max_qp_sges; 1113 init_attr->cap.max_recv_sge = rq->max_sge; 1114 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge, 1115 dev_attr->max_qp_sges); 1116 /* Allocate 1 more than what's provided so posting max doesn't 1117 * mean empty. 1118 */ 1119 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1); 1120 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1121 rq->q_full_delta = 0; 1122 rq->sg_info.pgsize = PAGE_SIZE; 1123 rq->sg_info.pgshft = PAGE_SHIFT; 1124 } 1125 1126 return 0; 1127 } 1128 1129 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) 1130 { 1131 struct bnxt_qplib_dev_attr *dev_attr; 1132 struct bnxt_qplib_qp *qplqp; 1133 struct bnxt_re_dev *rdev; 1134 1135 rdev = qp->rdev; 1136 qplqp = &qp->qplib_qp; 1137 dev_attr = &rdev->dev_attr; 1138 1139 if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { 1140 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1141 if (qplqp->rq.max_sge > dev_attr->max_qp_sges) 1142 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1143 qplqp->rq.max_sge = 6; 1144 } 1145 } 1146 1147 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, 1148 struct ib_qp_init_attr *init_attr, 1149 struct ib_udata *udata) 1150 { 1151 struct bnxt_qplib_dev_attr *dev_attr; 1152 struct bnxt_qplib_qp *qplqp; 1153 struct bnxt_re_dev *rdev; 1154 struct bnxt_qplib_q *sq; 1155 int entries; 1156 int diff; 1157 int rc; 1158 1159 rdev = qp->rdev; 1160 qplqp = &qp->qplib_qp; 1161 sq = &qplqp->sq; 1162 dev_attr = &rdev->dev_attr; 1163 1164 sq->max_sge = init_attr->cap.max_send_sge; 1165 if (sq->max_sge > dev_attr->max_qp_sges) { 1166 sq->max_sge = dev_attr->max_qp_sges; 1167 init_attr->cap.max_send_sge = sq->max_sge; 1168 } 1169 1170 rc = bnxt_re_setup_swqe_size(qp, init_attr); 1171 if (rc) 1172 return rc; 1173 1174 entries = init_attr->cap.max_send_wr; 1175 /* Allocate 128 + 1 more than what's provided */ 1176 diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ? 1177 0 : BNXT_QPLIB_RESERVED_QP_WRS; 1178 entries = roundup_pow_of_two(entries + diff + 1); 1179 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); 1180 sq->q_full_delta = diff + 1; 1181 /* 1182 * Reserving one slot for Phantom WQE. Application can 1183 * post one extra entry in this case. But allowing this to avoid 1184 * unexpected Queue full condition 1185 */ 1186 qplqp->sq.q_full_delta -= 1; 1187 qplqp->sq.sg_info.pgsize = PAGE_SIZE; 1188 qplqp->sq.sg_info.pgshft = PAGE_SHIFT; 1189 1190 return 0; 1191 } 1192 1193 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, 1194 struct ib_qp_init_attr *init_attr) 1195 { 1196 struct bnxt_qplib_dev_attr *dev_attr; 1197 struct bnxt_qplib_qp *qplqp; 1198 struct bnxt_re_dev *rdev; 1199 int entries; 1200 1201 rdev = qp->rdev; 1202 qplqp = &qp->qplib_qp; 1203 dev_attr = &rdev->dev_attr; 1204 1205 if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) { 1206 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1); 1207 qplqp->sq.max_wqe = min_t(u32, entries, 1208 dev_attr->max_qp_wqes + 1); 1209 qplqp->sq.q_full_delta = qplqp->sq.max_wqe - 1210 init_attr->cap.max_send_wr; 1211 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ 1212 if (qplqp->sq.max_sge > dev_attr->max_qp_sges) 1213 qplqp->sq.max_sge = dev_attr->max_qp_sges; 1214 } 1215 } 1216 1217 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, 1218 struct ib_qp_init_attr *init_attr) 1219 { 1220 struct bnxt_qplib_chip_ctx *chip_ctx; 1221 int qptype; 1222 1223 chip_ctx = rdev->chip_ctx; 1224 1225 qptype = __from_ib_qp_type(init_attr->qp_type); 1226 if (qptype == IB_QPT_MAX) { 1227 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype); 1228 qptype = -EOPNOTSUPP; 1229 goto out; 1230 } 1231 1232 if (bnxt_qplib_is_chip_gen_p5(chip_ctx) && 1233 init_attr->qp_type == IB_QPT_GSI) 1234 qptype = CMDQ_CREATE_QP_TYPE_GSI; 1235 out: 1236 return qptype; 1237 } 1238 1239 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 1240 struct ib_qp_init_attr *init_attr, 1241 struct ib_udata *udata) 1242 { 1243 struct bnxt_qplib_dev_attr *dev_attr; 1244 struct bnxt_qplib_qp *qplqp; 1245 struct bnxt_re_dev *rdev; 1246 struct bnxt_re_cq *cq; 1247 int rc = 0, qptype; 1248 1249 rdev = qp->rdev; 1250 qplqp = &qp->qplib_qp; 1251 dev_attr = &rdev->dev_attr; 1252 1253 /* Setup misc params */ 1254 ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr); 1255 qplqp->pd = &pd->qplib_pd; 1256 qplqp->qp_handle = (u64)qplqp; 1257 qplqp->max_inline_data = init_attr->cap.max_inline_data; 1258 qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1259 true : false); 1260 qptype = bnxt_re_init_qp_type(rdev, init_attr); 1261 if (qptype < 0) { 1262 rc = qptype; 1263 goto out; 1264 } 1265 qplqp->type = (u8)qptype; 1266 qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode; 1267 1268 if (init_attr->qp_type == IB_QPT_RC) { 1269 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; 1270 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 1271 } 1272 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1273 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ 1274 if (init_attr->create_flags) { 1275 ibdev_dbg(&rdev->ibdev, 1276 "QP create flags 0x%x not supported", 1277 init_attr->create_flags); 1278 return -EOPNOTSUPP; 1279 } 1280 1281 /* Setup CQs */ 1282 if (init_attr->send_cq) { 1283 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq); 1284 qplqp->scq = &cq->qplib_cq; 1285 qp->scq = cq; 1286 } 1287 1288 if (init_attr->recv_cq) { 1289 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq); 1290 qplqp->rcq = &cq->qplib_cq; 1291 qp->rcq = cq; 1292 } 1293 1294 /* Setup RQ/SRQ */ 1295 rc = bnxt_re_init_rq_attr(qp, init_attr); 1296 if (rc) 1297 goto out; 1298 if (init_attr->qp_type == IB_QPT_GSI) 1299 bnxt_re_adjust_gsi_rq_attr(qp); 1300 1301 /* Setup SQ */ 1302 rc = bnxt_re_init_sq_attr(qp, init_attr, udata); 1303 if (rc) 1304 goto out; 1305 if (init_attr->qp_type == IB_QPT_GSI) 1306 bnxt_re_adjust_gsi_sq_attr(qp, init_attr); 1307 1308 if (udata) /* This will update DPI and qp_handle */ 1309 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 1310 out: 1311 return rc; 1312 } 1313 1314 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, 1315 struct bnxt_re_pd *pd) 1316 { 1317 struct bnxt_re_sqp_entries *sqp_tbl; 1318 struct bnxt_re_dev *rdev; 1319 struct bnxt_re_qp *sqp; 1320 struct bnxt_re_ah *sah; 1321 int rc = 0; 1322 1323 rdev = qp->rdev; 1324 /* Create a shadow QP to handle the QP1 traffic */ 1325 sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl), 1326 GFP_KERNEL); 1327 if (!sqp_tbl) 1328 return -ENOMEM; 1329 rdev->gsi_ctx.sqp_tbl = sqp_tbl; 1330 1331 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); 1332 if (!sqp) { 1333 rc = -ENODEV; 1334 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1"); 1335 goto out; 1336 } 1337 rdev->gsi_ctx.gsi_sqp = sqp; 1338 1339 sqp->rcq = qp->rcq; 1340 sqp->scq = qp->scq; 1341 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1342 &qp->qplib_qp); 1343 if (!sah) { 1344 bnxt_qplib_destroy_qp(&rdev->qplib_res, 1345 &sqp->qplib_qp); 1346 rc = -ENODEV; 1347 ibdev_err(&rdev->ibdev, 1348 "Failed to create AH entry for ShadowQP"); 1349 goto out; 1350 } 1351 rdev->gsi_ctx.gsi_sah = sah; 1352 1353 return 0; 1354 out: 1355 kfree(sqp_tbl); 1356 return rc; 1357 } 1358 1359 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 1360 struct ib_qp_init_attr *init_attr) 1361 { 1362 struct bnxt_re_dev *rdev; 1363 struct bnxt_qplib_qp *qplqp; 1364 int rc = 0; 1365 1366 rdev = qp->rdev; 1367 qplqp = &qp->qplib_qp; 1368 1369 qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 1370 qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2; 1371 1372 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp); 1373 if (rc) { 1374 ibdev_err(&rdev->ibdev, "create HW QP1 failed!"); 1375 goto out; 1376 } 1377 1378 rc = bnxt_re_create_shadow_gsi(qp, pd); 1379 out: 1380 return rc; 1381 } 1382 1383 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, 1384 struct ib_qp_init_attr *init_attr, 1385 struct bnxt_qplib_dev_attr *dev_attr) 1386 { 1387 bool rc = true; 1388 1389 if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes || 1390 init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes || 1391 init_attr->cap.max_send_sge > dev_attr->max_qp_sges || 1392 init_attr->cap.max_recv_sge > dev_attr->max_qp_sges || 1393 init_attr->cap.max_inline_data > dev_attr->max_inline_data) { 1394 ibdev_err(&rdev->ibdev, 1395 "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x", 1396 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, 1397 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, 1398 init_attr->cap.max_send_sge, dev_attr->max_qp_sges, 1399 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, 1400 init_attr->cap.max_inline_data, 1401 dev_attr->max_inline_data); 1402 rc = false; 1403 } 1404 return rc; 1405 } 1406 1407 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, 1408 struct ib_udata *udata) 1409 { 1410 struct ib_pd *ib_pd = ib_qp->pd; 1411 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1412 struct bnxt_re_dev *rdev = pd->rdev; 1413 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1414 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1415 int rc; 1416 1417 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); 1418 if (!rc) { 1419 rc = -EINVAL; 1420 goto fail; 1421 } 1422 1423 qp->rdev = rdev; 1424 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); 1425 if (rc) 1426 goto fail; 1427 1428 if (qp_init_attr->qp_type == IB_QPT_GSI && 1429 !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) { 1430 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr); 1431 if (rc == -ENODEV) 1432 goto qp_destroy; 1433 if (rc) 1434 goto fail; 1435 } else { 1436 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 1437 if (rc) { 1438 ibdev_err(&rdev->ibdev, "Failed to create HW QP"); 1439 goto free_umem; 1440 } 1441 if (udata) { 1442 struct bnxt_re_qp_resp resp; 1443 1444 resp.qpid = qp->qplib_qp.id; 1445 resp.rsvd = 0; 1446 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1447 if (rc) { 1448 ibdev_err(&rdev->ibdev, "Failed to copy QP udata"); 1449 goto qp_destroy; 1450 } 1451 } 1452 } 1453 1454 qp->ib_qp.qp_num = qp->qplib_qp.id; 1455 if (qp_init_attr->qp_type == IB_QPT_GSI) 1456 rdev->gsi_ctx.gsi_qp = qp; 1457 spin_lock_init(&qp->sq_lock); 1458 spin_lock_init(&qp->rq_lock); 1459 INIT_LIST_HEAD(&qp->list); 1460 mutex_lock(&rdev->qp_lock); 1461 list_add_tail(&qp->list, &rdev->qp_list); 1462 mutex_unlock(&rdev->qp_lock); 1463 atomic_inc(&rdev->qp_count); 1464 1465 return 0; 1466 qp_destroy: 1467 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1468 free_umem: 1469 ib_umem_release(qp->rumem); 1470 ib_umem_release(qp->sumem); 1471 fail: 1472 return rc; 1473 } 1474 1475 static u8 __from_ib_qp_state(enum ib_qp_state state) 1476 { 1477 switch (state) { 1478 case IB_QPS_RESET: 1479 return CMDQ_MODIFY_QP_NEW_STATE_RESET; 1480 case IB_QPS_INIT: 1481 return CMDQ_MODIFY_QP_NEW_STATE_INIT; 1482 case IB_QPS_RTR: 1483 return CMDQ_MODIFY_QP_NEW_STATE_RTR; 1484 case IB_QPS_RTS: 1485 return CMDQ_MODIFY_QP_NEW_STATE_RTS; 1486 case IB_QPS_SQD: 1487 return CMDQ_MODIFY_QP_NEW_STATE_SQD; 1488 case IB_QPS_SQE: 1489 return CMDQ_MODIFY_QP_NEW_STATE_SQE; 1490 case IB_QPS_ERR: 1491 default: 1492 return CMDQ_MODIFY_QP_NEW_STATE_ERR; 1493 } 1494 } 1495 1496 static enum ib_qp_state __to_ib_qp_state(u8 state) 1497 { 1498 switch (state) { 1499 case CMDQ_MODIFY_QP_NEW_STATE_RESET: 1500 return IB_QPS_RESET; 1501 case CMDQ_MODIFY_QP_NEW_STATE_INIT: 1502 return IB_QPS_INIT; 1503 case CMDQ_MODIFY_QP_NEW_STATE_RTR: 1504 return IB_QPS_RTR; 1505 case CMDQ_MODIFY_QP_NEW_STATE_RTS: 1506 return IB_QPS_RTS; 1507 case CMDQ_MODIFY_QP_NEW_STATE_SQD: 1508 return IB_QPS_SQD; 1509 case CMDQ_MODIFY_QP_NEW_STATE_SQE: 1510 return IB_QPS_SQE; 1511 case CMDQ_MODIFY_QP_NEW_STATE_ERR: 1512 default: 1513 return IB_QPS_ERR; 1514 } 1515 } 1516 1517 static u32 __from_ib_mtu(enum ib_mtu mtu) 1518 { 1519 switch (mtu) { 1520 case IB_MTU_256: 1521 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; 1522 case IB_MTU_512: 1523 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; 1524 case IB_MTU_1024: 1525 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; 1526 case IB_MTU_2048: 1527 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1528 case IB_MTU_4096: 1529 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; 1530 default: 1531 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; 1532 } 1533 } 1534 1535 static enum ib_mtu __to_ib_mtu(u32 mtu) 1536 { 1537 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { 1538 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: 1539 return IB_MTU_256; 1540 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: 1541 return IB_MTU_512; 1542 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: 1543 return IB_MTU_1024; 1544 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: 1545 return IB_MTU_2048; 1546 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: 1547 return IB_MTU_4096; 1548 default: 1549 return IB_MTU_2048; 1550 } 1551 } 1552 1553 /* Shared Receive Queues */ 1554 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) 1555 { 1556 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1557 ib_srq); 1558 struct bnxt_re_dev *rdev = srq->rdev; 1559 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1560 struct bnxt_qplib_nq *nq = NULL; 1561 1562 if (qplib_srq->cq) 1563 nq = qplib_srq->cq->nq; 1564 bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1565 ib_umem_release(srq->umem); 1566 atomic_dec(&rdev->srq_count); 1567 if (nq) 1568 nq->budget--; 1569 return 0; 1570 } 1571 1572 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1573 struct bnxt_re_pd *pd, 1574 struct bnxt_re_srq *srq, 1575 struct ib_udata *udata) 1576 { 1577 struct bnxt_re_srq_req ureq; 1578 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1579 struct ib_umem *umem; 1580 int bytes = 0; 1581 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context( 1582 udata, struct bnxt_re_ucontext, ib_uctx); 1583 1584 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) 1585 return -EFAULT; 1586 1587 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); 1588 bytes = PAGE_ALIGN(bytes); 1589 umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes, 1590 IB_ACCESS_LOCAL_WRITE); 1591 if (IS_ERR(umem)) 1592 return PTR_ERR(umem); 1593 1594 srq->umem = umem; 1595 qplib_srq->sg_info.umem = umem; 1596 qplib_srq->sg_info.pgsize = PAGE_SIZE; 1597 qplib_srq->sg_info.pgshft = PAGE_SHIFT; 1598 qplib_srq->srq_handle = ureq.srq_handle; 1599 qplib_srq->dpi = &cntx->dpi; 1600 1601 return 0; 1602 } 1603 1604 int bnxt_re_create_srq(struct ib_srq *ib_srq, 1605 struct ib_srq_init_attr *srq_init_attr, 1606 struct ib_udata *udata) 1607 { 1608 struct bnxt_qplib_dev_attr *dev_attr; 1609 struct bnxt_qplib_nq *nq = NULL; 1610 struct bnxt_re_dev *rdev; 1611 struct bnxt_re_srq *srq; 1612 struct bnxt_re_pd *pd; 1613 struct ib_pd *ib_pd; 1614 int rc, entries; 1615 1616 ib_pd = ib_srq->pd; 1617 pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 1618 rdev = pd->rdev; 1619 dev_attr = &rdev->dev_attr; 1620 srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); 1621 1622 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1623 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded"); 1624 rc = -EINVAL; 1625 goto exit; 1626 } 1627 1628 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1629 rc = -EOPNOTSUPP; 1630 goto exit; 1631 } 1632 1633 srq->rdev = rdev; 1634 srq->qplib_srq.pd = &pd->qplib_pd; 1635 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1636 /* Allocate 1 more than what's provided so posting max doesn't 1637 * mean empty 1638 */ 1639 entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1); 1640 if (entries > dev_attr->max_srq_wqes + 1) 1641 entries = dev_attr->max_srq_wqes + 1; 1642 srq->qplib_srq.max_wqe = entries; 1643 1644 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1645 /* 128 byte wqe size for SRQ . So use max sges */ 1646 srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges); 1647 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1648 srq->srq_limit = srq_init_attr->attr.srq_limit; 1649 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id; 1650 nq = &rdev->nq[0]; 1651 1652 if (udata) { 1653 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1654 if (rc) 1655 goto fail; 1656 } 1657 1658 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1659 if (rc) { 1660 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!"); 1661 goto fail; 1662 } 1663 1664 if (udata) { 1665 struct bnxt_re_srq_resp resp; 1666 1667 resp.srqid = srq->qplib_srq.id; 1668 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 1669 if (rc) { 1670 ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!"); 1671 bnxt_qplib_destroy_srq(&rdev->qplib_res, 1672 &srq->qplib_srq); 1673 goto fail; 1674 } 1675 } 1676 if (nq) 1677 nq->budget++; 1678 atomic_inc(&rdev->srq_count); 1679 spin_lock_init(&srq->lock); 1680 1681 return 0; 1682 1683 fail: 1684 ib_umem_release(srq->umem); 1685 exit: 1686 return rc; 1687 } 1688 1689 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1690 enum ib_srq_attr_mask srq_attr_mask, 1691 struct ib_udata *udata) 1692 { 1693 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1694 ib_srq); 1695 struct bnxt_re_dev *rdev = srq->rdev; 1696 int rc; 1697 1698 switch (srq_attr_mask) { 1699 case IB_SRQ_MAX_WR: 1700 /* SRQ resize is not supported */ 1701 break; 1702 case IB_SRQ_LIMIT: 1703 /* Change the SRQ threshold */ 1704 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1705 return -EINVAL; 1706 1707 srq->qplib_srq.threshold = srq_attr->srq_limit; 1708 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1709 if (rc) { 1710 ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!"); 1711 return rc; 1712 } 1713 /* On success, update the shadow */ 1714 srq->srq_limit = srq_attr->srq_limit; 1715 /* No need to Build and send response back to udata */ 1716 break; 1717 default: 1718 ibdev_err(&rdev->ibdev, 1719 "Unsupported srq_attr_mask 0x%x", srq_attr_mask); 1720 return -EINVAL; 1721 } 1722 return 0; 1723 } 1724 1725 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1726 { 1727 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1728 ib_srq); 1729 struct bnxt_re_srq tsrq; 1730 struct bnxt_re_dev *rdev = srq->rdev; 1731 int rc; 1732 1733 /* Get live SRQ attr */ 1734 tsrq.qplib_srq.id = srq->qplib_srq.id; 1735 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq); 1736 if (rc) { 1737 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!"); 1738 return rc; 1739 } 1740 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1741 srq_attr->max_sge = srq->qplib_srq.max_sge; 1742 srq_attr->srq_limit = tsrq.qplib_srq.threshold; 1743 1744 return 0; 1745 } 1746 1747 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1748 const struct ib_recv_wr **bad_wr) 1749 { 1750 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, 1751 ib_srq); 1752 struct bnxt_qplib_swqe wqe; 1753 unsigned long flags; 1754 int rc = 0; 1755 1756 spin_lock_irqsave(&srq->lock, flags); 1757 while (wr) { 1758 /* Transcribe each ib_recv_wr to qplib_swqe */ 1759 wqe.num_sge = wr->num_sge; 1760 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 1761 wqe.wr_id = wr->wr_id; 1762 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1763 1764 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1765 if (rc) { 1766 *bad_wr = wr; 1767 break; 1768 } 1769 wr = wr->next; 1770 } 1771 spin_unlock_irqrestore(&srq->lock, flags); 1772 1773 return rc; 1774 } 1775 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 1776 struct bnxt_re_qp *qp1_qp, 1777 int qp_attr_mask) 1778 { 1779 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; 1780 int rc = 0; 1781 1782 if (qp_attr_mask & IB_QP_STATE) { 1783 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1784 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 1785 } 1786 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1787 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1788 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 1789 } 1790 1791 if (qp_attr_mask & IB_QP_QKEY) { 1792 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1793 /* Using a Random QKEY */ 1794 qp->qplib_qp.qkey = 0x81818181; 1795 } 1796 if (qp_attr_mask & IB_QP_SQ_PSN) { 1797 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1798 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 1799 } 1800 1801 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 1802 if (rc) 1803 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1"); 1804 return rc; 1805 } 1806 1807 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 1808 int qp_attr_mask, struct ib_udata *udata) 1809 { 1810 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 1811 struct bnxt_re_dev *rdev = qp->rdev; 1812 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 1813 enum ib_qp_state curr_qp_state, new_qp_state; 1814 int rc, entries; 1815 unsigned int flags; 1816 u8 nw_type; 1817 1818 if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1819 return -EOPNOTSUPP; 1820 1821 qp->qplib_qp.modify_flags = 0; 1822 if (qp_attr_mask & IB_QP_STATE) { 1823 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 1824 new_qp_state = qp_attr->qp_state; 1825 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state, 1826 ib_qp->qp_type, qp_attr_mask)) { 1827 ibdev_err(&rdev->ibdev, 1828 "Invalid attribute mask: %#x specified ", 1829 qp_attr_mask); 1830 ibdev_err(&rdev->ibdev, 1831 "for qpn: %#x type: %#x", 1832 ib_qp->qp_num, ib_qp->qp_type); 1833 ibdev_err(&rdev->ibdev, 1834 "curr_qp_state=0x%x, new_qp_state=0x%x\n", 1835 curr_qp_state, new_qp_state); 1836 return -EINVAL; 1837 } 1838 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 1839 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 1840 1841 if (!qp->sumem && 1842 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 1843 ibdev_dbg(&rdev->ibdev, 1844 "Move QP = %p to flush list\n", qp); 1845 flags = bnxt_re_lock_cqs(qp); 1846 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 1847 bnxt_re_unlock_cqs(qp, flags); 1848 } 1849 if (!qp->sumem && 1850 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 1851 ibdev_dbg(&rdev->ibdev, 1852 "Move QP = %p out of flush list\n", qp); 1853 flags = bnxt_re_lock_cqs(qp); 1854 bnxt_qplib_clean_qp(&qp->qplib_qp); 1855 bnxt_re_unlock_cqs(qp, flags); 1856 } 1857 } 1858 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1859 qp->qplib_qp.modify_flags |= 1860 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 1861 qp->qplib_qp.en_sqd_async_notify = true; 1862 } 1863 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 1864 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 1865 qp->qplib_qp.access = 1866 __from_ib_access_flags(qp_attr->qp_access_flags); 1867 /* LOCAL_WRITE access must be set to allow RC receive */ 1868 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 1869 /* Temp: Set all params on QP as of now */ 1870 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; 1871 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; 1872 } 1873 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 1874 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 1875 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 1876 } 1877 if (qp_attr_mask & IB_QP_QKEY) { 1878 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 1879 qp->qplib_qp.qkey = qp_attr->qkey; 1880 } 1881 if (qp_attr_mask & IB_QP_AV) { 1882 const struct ib_global_route *grh = 1883 rdma_ah_read_grh(&qp_attr->ah_attr); 1884 const struct ib_gid_attr *sgid_attr; 1885 struct bnxt_re_gid_ctx *ctx; 1886 1887 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 1888 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 1889 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 1890 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 1891 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 1892 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 1893 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 1894 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 1895 sizeof(qp->qplib_qp.ah.dgid.data)); 1896 qp->qplib_qp.ah.flow_label = grh->flow_label; 1897 sgid_attr = grh->sgid_attr; 1898 /* Get the HW context of the GID. The reference 1899 * of GID table entry is already taken by the caller. 1900 */ 1901 ctx = rdma_read_gid_hw_context(sgid_attr); 1902 qp->qplib_qp.ah.sgid_index = ctx->idx; 1903 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 1904 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 1905 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 1906 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 1907 ether_addr_copy(qp->qplib_qp.ah.dmac, 1908 qp_attr->ah_attr.roce.dmac); 1909 1910 rc = rdma_read_gid_l2_fields(sgid_attr, NULL, 1911 &qp->qplib_qp.smac[0]); 1912 if (rc) 1913 return rc; 1914 1915 nw_type = rdma_gid_attr_network_type(sgid_attr); 1916 switch (nw_type) { 1917 case RDMA_NETWORK_IPV4: 1918 qp->qplib_qp.nw_type = 1919 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 1920 break; 1921 case RDMA_NETWORK_IPV6: 1922 qp->qplib_qp.nw_type = 1923 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 1924 break; 1925 default: 1926 qp->qplib_qp.nw_type = 1927 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 1928 break; 1929 } 1930 } 1931 1932 if (qp_attr_mask & IB_QP_PATH_MTU) { 1933 qp->qplib_qp.modify_flags |= 1934 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1935 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu); 1936 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu); 1937 } else if (qp_attr->qp_state == IB_QPS_RTR) { 1938 qp->qplib_qp.modify_flags |= 1939 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; 1940 qp->qplib_qp.path_mtu = 1941 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu)); 1942 qp->qplib_qp.mtu = 1943 ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu)); 1944 } 1945 1946 if (qp_attr_mask & IB_QP_TIMEOUT) { 1947 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 1948 qp->qplib_qp.timeout = qp_attr->timeout; 1949 } 1950 if (qp_attr_mask & IB_QP_RETRY_CNT) { 1951 qp->qplib_qp.modify_flags |= 1952 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 1953 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 1954 } 1955 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1956 qp->qplib_qp.modify_flags |= 1957 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 1958 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 1959 } 1960 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 1961 qp->qplib_qp.modify_flags |= 1962 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 1963 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 1964 } 1965 if (qp_attr_mask & IB_QP_RQ_PSN) { 1966 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 1967 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 1968 } 1969 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1970 qp->qplib_qp.modify_flags |= 1971 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 1972 /* Cap the max_rd_atomic to device max */ 1973 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 1974 dev_attr->max_qp_rd_atom); 1975 } 1976 if (qp_attr_mask & IB_QP_SQ_PSN) { 1977 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 1978 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 1979 } 1980 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1981 if (qp_attr->max_dest_rd_atomic > 1982 dev_attr->max_qp_init_rd_atom) { 1983 ibdev_err(&rdev->ibdev, 1984 "max_dest_rd_atomic requested%d is > dev_max%d", 1985 qp_attr->max_dest_rd_atomic, 1986 dev_attr->max_qp_init_rd_atom); 1987 return -EINVAL; 1988 } 1989 1990 qp->qplib_qp.modify_flags |= 1991 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 1992 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 1993 } 1994 if (qp_attr_mask & IB_QP_CAP) { 1995 qp->qplib_qp.modify_flags |= 1996 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 1997 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 1998 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 1999 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 2000 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 2001 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 2002 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 2003 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 2004 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 2005 (qp_attr->cap.max_inline_data >= 2006 dev_attr->max_inline_data)) { 2007 ibdev_err(&rdev->ibdev, 2008 "Create QP failed - max exceeded"); 2009 return -EINVAL; 2010 } 2011 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 2012 qp->qplib_qp.sq.max_wqe = min_t(u32, entries, 2013 dev_attr->max_qp_wqes + 1); 2014 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 2015 qp_attr->cap.max_send_wr; 2016 /* 2017 * Reserving one slot for Phantom WQE. Some application can 2018 * post one extra entry in this case. Allowing this to avoid 2019 * unexpected Queue full condition 2020 */ 2021 qp->qplib_qp.sq.q_full_delta -= 1; 2022 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 2023 if (qp->qplib_qp.rq.max_wqe) { 2024 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 2025 qp->qplib_qp.rq.max_wqe = 2026 min_t(u32, entries, dev_attr->max_qp_wqes + 1); 2027 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 2028 qp_attr->cap.max_recv_wr; 2029 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 2030 } else { 2031 /* SRQ was used prior, just ignore the RQ caps */ 2032 } 2033 } 2034 if (qp_attr_mask & IB_QP_DEST_QPN) { 2035 qp->qplib_qp.modify_flags |= 2036 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 2037 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 2038 } 2039 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2040 if (rc) { 2041 ibdev_err(&rdev->ibdev, "Failed to modify HW QP"); 2042 return rc; 2043 } 2044 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) 2045 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 2046 return rc; 2047 } 2048 2049 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2050 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 2051 { 2052 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2053 struct bnxt_re_dev *rdev = qp->rdev; 2054 struct bnxt_qplib_qp *qplib_qp; 2055 int rc; 2056 2057 qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL); 2058 if (!qplib_qp) 2059 return -ENOMEM; 2060 2061 qplib_qp->id = qp->qplib_qp.id; 2062 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 2063 2064 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 2065 if (rc) { 2066 ibdev_err(&rdev->ibdev, "Failed to query HW QP"); 2067 goto out; 2068 } 2069 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 2070 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); 2071 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 2072 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 2073 qp_attr->pkey_index = qplib_qp->pkey_index; 2074 qp_attr->qkey = qplib_qp->qkey; 2075 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2076 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, 2077 qplib_qp->ah.host_sgid_index, 2078 qplib_qp->ah.hop_limit, 2079 qplib_qp->ah.traffic_class); 2080 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); 2081 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); 2082 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac); 2083 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 2084 qp_attr->timeout = qplib_qp->timeout; 2085 qp_attr->retry_cnt = qplib_qp->retry_cnt; 2086 qp_attr->rnr_retry = qplib_qp->rnr_retry; 2087 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 2088 qp_attr->rq_psn = qplib_qp->rq.psn; 2089 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 2090 qp_attr->sq_psn = qplib_qp->sq.psn; 2091 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 2092 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 2093 IB_SIGNAL_REQ_WR; 2094 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 2095 2096 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 2097 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 2098 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 2099 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 2100 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 2101 qp_init_attr->cap = qp_attr->cap; 2102 2103 out: 2104 kfree(qplib_qp); 2105 return rc; 2106 } 2107 2108 /* Routine for sending QP1 packets for RoCE V1 an V2 2109 */ 2110 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, 2111 const struct ib_send_wr *wr, 2112 struct bnxt_qplib_swqe *wqe, 2113 int payload_size) 2114 { 2115 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, 2116 ib_ah); 2117 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 2118 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr; 2119 struct bnxt_qplib_sge sge; 2120 u8 nw_type; 2121 u16 ether_type; 2122 union ib_gid dgid; 2123 bool is_eth = false; 2124 bool is_vlan = false; 2125 bool is_grh = false; 2126 bool is_udp = false; 2127 u8 ip_version = 0; 2128 u16 vlan_id = 0xFFFF; 2129 void *buf; 2130 int i, rc = 0; 2131 2132 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 2133 2134 rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL); 2135 if (rc) 2136 return rc; 2137 2138 /* Get network header type for this GID */ 2139 nw_type = rdma_gid_attr_network_type(sgid_attr); 2140 switch (nw_type) { 2141 case RDMA_NETWORK_IPV4: 2142 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET; 2143 break; 2144 case RDMA_NETWORK_IPV6: 2145 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET; 2146 break; 2147 default: 2148 nw_type = BNXT_RE_ROCE_V1_PACKET; 2149 break; 2150 } 2151 memcpy(&dgid.raw, &qplib_ah->dgid, 16); 2152 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP; 2153 if (is_udp) { 2154 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { 2155 ip_version = 4; 2156 ether_type = ETH_P_IP; 2157 } else { 2158 ip_version = 6; 2159 ether_type = ETH_P_IPV6; 2160 } 2161 is_grh = false; 2162 } else { 2163 ether_type = ETH_P_IBOE; 2164 is_grh = true; 2165 } 2166 2167 is_eth = true; 2168 is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false; 2169 2170 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh, 2171 ip_version, is_udp, 0, &qp->qp1_hdr); 2172 2173 /* ETH */ 2174 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac); 2175 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac); 2176 2177 /* For vlan, check the sgid for vlan existence */ 2178 2179 if (!is_vlan) { 2180 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type); 2181 } else { 2182 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type); 2183 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id); 2184 } 2185 2186 if (is_grh || (ip_version == 6)) { 2187 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw, 2188 sizeof(sgid_attr->gid)); 2189 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 2190 sizeof(sgid_attr->gid)); 2191 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit; 2192 } 2193 2194 if (ip_version == 4) { 2195 qp->qp1_hdr.ip4.tos = 0; 2196 qp->qp1_hdr.ip4.id = 0; 2197 qp->qp1_hdr.ip4.frag_off = htons(IP_DF); 2198 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit; 2199 2200 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4); 2201 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4); 2202 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr); 2203 } 2204 2205 if (is_udp) { 2206 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT); 2207 qp->qp1_hdr.udp.sport = htons(0x8CD1); 2208 qp->qp1_hdr.udp.csum = 0; 2209 } 2210 2211 /* BTH */ 2212 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 2213 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 2214 qp->qp1_hdr.immediate_present = 1; 2215 } else { 2216 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 2217 } 2218 if (wr->send_flags & IB_SEND_SOLICITED) 2219 qp->qp1_hdr.bth.solicited_event = 1; 2220 /* pad_count */ 2221 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 2222 2223 /* P_key for QP1 is for all members */ 2224 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 2225 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 2226 qp->qp1_hdr.bth.ack_req = 0; 2227 qp->send_psn++; 2228 qp->send_psn &= BTH_PSN_MASK; 2229 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 2230 /* DETH */ 2231 /* Use the priviledged Q_Key for QP1 */ 2232 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 2233 qp->qp1_hdr.deth.source_qpn = IB_QP1; 2234 2235 /* Pack the QP1 to the transmit buffer */ 2236 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 2237 if (buf) { 2238 ib_ud_header_pack(&qp->qp1_hdr, buf); 2239 for (i = wqe->num_sge; i; i--) { 2240 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 2241 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 2242 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 2243 } 2244 2245 /* 2246 * Max Header buf size for IPV6 RoCE V2 is 86, 2247 * which is same as the QP1 SQ header buffer. 2248 * Header buf size for IPV4 RoCE V2 can be 66. 2249 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20). 2250 * Subtract 20 bytes from QP1 SQ header buf size 2251 */ 2252 if (is_udp && ip_version == 4) 2253 sge.size -= 20; 2254 /* 2255 * Max Header buf size for RoCE V1 is 78. 2256 * ETH(14) + VLAN(4) + GRH(40) + BTH(20). 2257 * Subtract 8 bytes from QP1 SQ header buf size 2258 */ 2259 if (!is_udp) 2260 sge.size -= 8; 2261 2262 /* Subtract 4 bytes for non vlan packets */ 2263 if (!is_vlan) 2264 sge.size -= 4; 2265 2266 wqe->sg_list[0].addr = sge.addr; 2267 wqe->sg_list[0].lkey = sge.lkey; 2268 wqe->sg_list[0].size = sge.size; 2269 wqe->num_sge++; 2270 2271 } else { 2272 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!"); 2273 rc = -ENOMEM; 2274 } 2275 return rc; 2276 } 2277 2278 /* For the MAD layer, it only provides the recv SGE the size of 2279 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 2280 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 2281 * receive packet (334 bytes) with no VLAN and then copy the GRH 2282 * and the MAD datagram out to the provided SGE. 2283 */ 2284 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 2285 const struct ib_recv_wr *wr, 2286 struct bnxt_qplib_swqe *wqe, 2287 int payload_size) 2288 { 2289 struct bnxt_re_sqp_entries *sqp_entry; 2290 struct bnxt_qplib_sge ref, sge; 2291 struct bnxt_re_dev *rdev; 2292 u32 rq_prod_index; 2293 2294 rdev = qp->rdev; 2295 2296 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 2297 2298 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) 2299 return -ENOMEM; 2300 2301 /* Create 1 SGE to receive the entire 2302 * ethernet packet 2303 */ 2304 /* Save the reference from ULP */ 2305 ref.addr = wqe->sg_list[0].addr; 2306 ref.lkey = wqe->sg_list[0].lkey; 2307 ref.size = wqe->sg_list[0].size; 2308 2309 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; 2310 2311 /* SGE 1 */ 2312 wqe->sg_list[0].addr = sge.addr; 2313 wqe->sg_list[0].lkey = sge.lkey; 2314 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 2315 sge.size -= wqe->sg_list[0].size; 2316 2317 sqp_entry->sge.addr = ref.addr; 2318 sqp_entry->sge.lkey = ref.lkey; 2319 sqp_entry->sge.size = ref.size; 2320 /* Store the wrid for reporting completion */ 2321 sqp_entry->wrid = wqe->wr_id; 2322 /* change the wqe->wrid to table index */ 2323 wqe->wr_id = rq_prod_index; 2324 return 0; 2325 } 2326 2327 static int is_ud_qp(struct bnxt_re_qp *qp) 2328 { 2329 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || 2330 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); 2331 } 2332 2333 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 2334 const struct ib_send_wr *wr, 2335 struct bnxt_qplib_swqe *wqe) 2336 { 2337 struct bnxt_re_ah *ah = NULL; 2338 2339 if (is_ud_qp(qp)) { 2340 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah); 2341 wqe->send.q_key = ud_wr(wr)->remote_qkey; 2342 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 2343 wqe->send.avid = ah->qplib_ah.id; 2344 } 2345 switch (wr->opcode) { 2346 case IB_WR_SEND: 2347 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 2348 break; 2349 case IB_WR_SEND_WITH_IMM: 2350 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 2351 wqe->send.imm_data = wr->ex.imm_data; 2352 break; 2353 case IB_WR_SEND_WITH_INV: 2354 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 2355 wqe->send.inv_key = wr->ex.invalidate_rkey; 2356 break; 2357 default: 2358 return -EINVAL; 2359 } 2360 if (wr->send_flags & IB_SEND_SIGNALED) 2361 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2362 if (wr->send_flags & IB_SEND_FENCE) 2363 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2364 if (wr->send_flags & IB_SEND_SOLICITED) 2365 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2366 if (wr->send_flags & IB_SEND_INLINE) 2367 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2368 2369 return 0; 2370 } 2371 2372 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 2373 struct bnxt_qplib_swqe *wqe) 2374 { 2375 switch (wr->opcode) { 2376 case IB_WR_RDMA_WRITE: 2377 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 2378 break; 2379 case IB_WR_RDMA_WRITE_WITH_IMM: 2380 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 2381 wqe->rdma.imm_data = wr->ex.imm_data; 2382 break; 2383 case IB_WR_RDMA_READ: 2384 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 2385 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 2386 break; 2387 default: 2388 return -EINVAL; 2389 } 2390 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 2391 wqe->rdma.r_key = rdma_wr(wr)->rkey; 2392 if (wr->send_flags & IB_SEND_SIGNALED) 2393 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2394 if (wr->send_flags & IB_SEND_FENCE) 2395 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2396 if (wr->send_flags & IB_SEND_SOLICITED) 2397 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2398 if (wr->send_flags & IB_SEND_INLINE) 2399 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 2400 2401 return 0; 2402 } 2403 2404 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 2405 struct bnxt_qplib_swqe *wqe) 2406 { 2407 switch (wr->opcode) { 2408 case IB_WR_ATOMIC_CMP_AND_SWP: 2409 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 2410 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2411 wqe->atomic.swap_data = atomic_wr(wr)->swap; 2412 break; 2413 case IB_WR_ATOMIC_FETCH_AND_ADD: 2414 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 2415 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 2416 break; 2417 default: 2418 return -EINVAL; 2419 } 2420 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 2421 wqe->atomic.r_key = atomic_wr(wr)->rkey; 2422 if (wr->send_flags & IB_SEND_SIGNALED) 2423 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2424 if (wr->send_flags & IB_SEND_FENCE) 2425 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2426 if (wr->send_flags & IB_SEND_SOLICITED) 2427 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2428 return 0; 2429 } 2430 2431 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 2432 struct bnxt_qplib_swqe *wqe) 2433 { 2434 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 2435 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 2436 2437 /* Need unconditional fence for local invalidate 2438 * opcode to work as expected. 2439 */ 2440 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2441 2442 if (wr->send_flags & IB_SEND_SIGNALED) 2443 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2444 if (wr->send_flags & IB_SEND_SOLICITED) 2445 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 2446 2447 return 0; 2448 } 2449 2450 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 2451 struct bnxt_qplib_swqe *wqe) 2452 { 2453 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr); 2454 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 2455 int access = wr->access; 2456 2457 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 2458 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 2459 wqe->frmr.page_list = mr->pages; 2460 wqe->frmr.page_list_len = mr->npages; 2461 wqe->frmr.levels = qplib_frpl->hwq.level; 2462 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 2463 2464 /* Need unconditional fence for reg_mr 2465 * opcode to function as expected. 2466 */ 2467 2468 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 2469 2470 if (wr->wr.send_flags & IB_SEND_SIGNALED) 2471 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 2472 2473 if (access & IB_ACCESS_LOCAL_WRITE) 2474 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 2475 if (access & IB_ACCESS_REMOTE_READ) 2476 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 2477 if (access & IB_ACCESS_REMOTE_WRITE) 2478 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 2479 if (access & IB_ACCESS_REMOTE_ATOMIC) 2480 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 2481 if (access & IB_ACCESS_MW_BIND) 2482 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 2483 2484 wqe->frmr.l_key = wr->key; 2485 wqe->frmr.length = wr->mr->length; 2486 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); 2487 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); 2488 wqe->frmr.va = wr->mr->iova; 2489 return 0; 2490 } 2491 2492 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, 2493 const struct ib_send_wr *wr, 2494 struct bnxt_qplib_swqe *wqe) 2495 { 2496 /* Copy the inline data to the data field */ 2497 u8 *in_data; 2498 u32 i, sge_len; 2499 void *sge_addr; 2500 2501 in_data = wqe->inline_data; 2502 for (i = 0; i < wr->num_sge; i++) { 2503 sge_addr = (void *)(unsigned long) 2504 wr->sg_list[i].addr; 2505 sge_len = wr->sg_list[i].length; 2506 2507 if ((sge_len + wqe->inline_len) > 2508 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { 2509 ibdev_err(&rdev->ibdev, 2510 "Inline data size requested > supported value"); 2511 return -EINVAL; 2512 } 2513 sge_len = wr->sg_list[i].length; 2514 2515 memcpy(in_data, sge_addr, sge_len); 2516 in_data += wr->sg_list[i].length; 2517 wqe->inline_len += wr->sg_list[i].length; 2518 } 2519 return wqe->inline_len; 2520 } 2521 2522 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, 2523 const struct ib_send_wr *wr, 2524 struct bnxt_qplib_swqe *wqe) 2525 { 2526 int payload_sz = 0; 2527 2528 if (wr->send_flags & IB_SEND_INLINE) 2529 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); 2530 else 2531 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, 2532 wqe->num_sge); 2533 2534 return payload_sz; 2535 } 2536 2537 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 2538 { 2539 if ((qp->ib_qp.qp_type == IB_QPT_UD || 2540 qp->ib_qp.qp_type == IB_QPT_GSI || 2541 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 2542 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 2543 int qp_attr_mask; 2544 struct ib_qp_attr qp_attr; 2545 2546 qp_attr_mask = IB_QP_STATE; 2547 qp_attr.qp_state = IB_QPS_RTS; 2548 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 2549 qp->qplib_qp.wqe_cnt = 0; 2550 } 2551 } 2552 2553 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 2554 struct bnxt_re_qp *qp, 2555 const struct ib_send_wr *wr) 2556 { 2557 int rc = 0, payload_sz = 0; 2558 unsigned long flags; 2559 2560 spin_lock_irqsave(&qp->sq_lock, flags); 2561 while (wr) { 2562 struct bnxt_qplib_swqe wqe = {}; 2563 2564 /* Common */ 2565 wqe.num_sge = wr->num_sge; 2566 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2567 ibdev_err(&rdev->ibdev, 2568 "Limit exceeded for Send SGEs"); 2569 rc = -EINVAL; 2570 goto bad; 2571 } 2572 2573 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2574 if (payload_sz < 0) { 2575 rc = -EINVAL; 2576 goto bad; 2577 } 2578 wqe.wr_id = wr->wr_id; 2579 2580 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 2581 2582 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2583 if (!rc) 2584 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2585 bad: 2586 if (rc) { 2587 ibdev_err(&rdev->ibdev, 2588 "Post send failed opcode = %#x rc = %d", 2589 wr->opcode, rc); 2590 break; 2591 } 2592 wr = wr->next; 2593 } 2594 bnxt_qplib_post_send_db(&qp->qplib_qp); 2595 bnxt_ud_qp_hw_stall_workaround(qp); 2596 spin_unlock_irqrestore(&qp->sq_lock, flags); 2597 return rc; 2598 } 2599 2600 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 2601 const struct ib_send_wr **bad_wr) 2602 { 2603 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2604 struct bnxt_qplib_swqe wqe; 2605 int rc = 0, payload_sz = 0; 2606 unsigned long flags; 2607 2608 spin_lock_irqsave(&qp->sq_lock, flags); 2609 while (wr) { 2610 /* House keeping */ 2611 memset(&wqe, 0, sizeof(wqe)); 2612 2613 /* Common */ 2614 wqe.num_sge = wr->num_sge; 2615 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 2616 ibdev_err(&qp->rdev->ibdev, 2617 "Limit exceeded for Send SGEs"); 2618 rc = -EINVAL; 2619 goto bad; 2620 } 2621 2622 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); 2623 if (payload_sz < 0) { 2624 rc = -EINVAL; 2625 goto bad; 2626 } 2627 wqe.wr_id = wr->wr_id; 2628 2629 switch (wr->opcode) { 2630 case IB_WR_SEND: 2631 case IB_WR_SEND_WITH_IMM: 2632 if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) { 2633 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe, 2634 payload_sz); 2635 if (rc) 2636 goto bad; 2637 wqe.rawqp1.lflags |= 2638 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2639 } 2640 switch (wr->send_flags) { 2641 case IB_SEND_IP_CSUM: 2642 wqe.rawqp1.lflags |= 2643 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2644 break; 2645 default: 2646 break; 2647 } 2648 fallthrough; 2649 case IB_WR_SEND_WITH_INV: 2650 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 2651 break; 2652 case IB_WR_RDMA_WRITE: 2653 case IB_WR_RDMA_WRITE_WITH_IMM: 2654 case IB_WR_RDMA_READ: 2655 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 2656 break; 2657 case IB_WR_ATOMIC_CMP_AND_SWP: 2658 case IB_WR_ATOMIC_FETCH_AND_ADD: 2659 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 2660 break; 2661 case IB_WR_RDMA_READ_WITH_INV: 2662 ibdev_err(&qp->rdev->ibdev, 2663 "RDMA Read with Invalidate is not supported"); 2664 rc = -EINVAL; 2665 goto bad; 2666 case IB_WR_LOCAL_INV: 2667 rc = bnxt_re_build_inv_wqe(wr, &wqe); 2668 break; 2669 case IB_WR_REG_MR: 2670 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 2671 break; 2672 default: 2673 /* Unsupported WRs */ 2674 ibdev_err(&qp->rdev->ibdev, 2675 "WR (%#x) is not supported", wr->opcode); 2676 rc = -EINVAL; 2677 goto bad; 2678 } 2679 if (!rc) 2680 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 2681 bad: 2682 if (rc) { 2683 ibdev_err(&qp->rdev->ibdev, 2684 "post_send failed op:%#x qps = %#x rc = %d\n", 2685 wr->opcode, qp->qplib_qp.state, rc); 2686 *bad_wr = wr; 2687 break; 2688 } 2689 wr = wr->next; 2690 } 2691 bnxt_qplib_post_send_db(&qp->qplib_qp); 2692 bnxt_ud_qp_hw_stall_workaround(qp); 2693 spin_unlock_irqrestore(&qp->sq_lock, flags); 2694 2695 return rc; 2696 } 2697 2698 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 2699 struct bnxt_re_qp *qp, 2700 const struct ib_recv_wr *wr) 2701 { 2702 struct bnxt_qplib_swqe wqe; 2703 int rc = 0; 2704 2705 memset(&wqe, 0, sizeof(wqe)); 2706 while (wr) { 2707 /* House keeping */ 2708 memset(&wqe, 0, sizeof(wqe)); 2709 2710 /* Common */ 2711 wqe.num_sge = wr->num_sge; 2712 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2713 ibdev_err(&rdev->ibdev, 2714 "Limit exceeded for Receive SGEs"); 2715 rc = -EINVAL; 2716 break; 2717 } 2718 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); 2719 wqe.wr_id = wr->wr_id; 2720 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2721 2722 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2723 if (rc) 2724 break; 2725 2726 wr = wr->next; 2727 } 2728 if (!rc) 2729 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2730 return rc; 2731 } 2732 2733 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 2734 const struct ib_recv_wr **bad_wr) 2735 { 2736 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); 2737 struct bnxt_qplib_swqe wqe; 2738 int rc = 0, payload_sz = 0; 2739 unsigned long flags; 2740 u32 count = 0; 2741 2742 spin_lock_irqsave(&qp->rq_lock, flags); 2743 while (wr) { 2744 /* House keeping */ 2745 memset(&wqe, 0, sizeof(wqe)); 2746 2747 /* Common */ 2748 wqe.num_sge = wr->num_sge; 2749 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 2750 ibdev_err(&qp->rdev->ibdev, 2751 "Limit exceeded for Receive SGEs"); 2752 rc = -EINVAL; 2753 *bad_wr = wr; 2754 break; 2755 } 2756 2757 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, 2758 wr->num_sge); 2759 wqe.wr_id = wr->wr_id; 2760 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 2761 2762 if (ib_qp->qp_type == IB_QPT_GSI && 2763 qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI) 2764 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe, 2765 payload_sz); 2766 if (!rc) 2767 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 2768 if (rc) { 2769 *bad_wr = wr; 2770 break; 2771 } 2772 2773 /* Ring DB if the RQEs posted reaches a threshold value */ 2774 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 2775 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2776 count = 0; 2777 } 2778 2779 wr = wr->next; 2780 } 2781 2782 if (count) 2783 bnxt_qplib_post_recv_db(&qp->qplib_qp); 2784 2785 spin_unlock_irqrestore(&qp->rq_lock, flags); 2786 2787 return rc; 2788 } 2789 2790 /* Completion Queues */ 2791 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 2792 { 2793 struct bnxt_re_cq *cq; 2794 struct bnxt_qplib_nq *nq; 2795 struct bnxt_re_dev *rdev; 2796 2797 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 2798 rdev = cq->rdev; 2799 nq = cq->qplib_cq.nq; 2800 2801 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2802 ib_umem_release(cq->umem); 2803 2804 atomic_dec(&rdev->cq_count); 2805 nq->budget--; 2806 kfree(cq->cql); 2807 return 0; 2808 } 2809 2810 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 2811 struct ib_udata *udata) 2812 { 2813 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev); 2814 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 2815 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); 2816 int rc, entries; 2817 int cqe = attr->cqe; 2818 struct bnxt_qplib_nq *nq = NULL; 2819 unsigned int nq_alloc_cnt; 2820 2821 if (attr->flags) 2822 return -EOPNOTSUPP; 2823 2824 /* Validate CQ fields */ 2825 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 2826 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded"); 2827 return -EINVAL; 2828 } 2829 2830 cq->rdev = rdev; 2831 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); 2832 2833 entries = roundup_pow_of_two(cqe + 1); 2834 if (entries > dev_attr->max_cq_wqes + 1) 2835 entries = dev_attr->max_cq_wqes + 1; 2836 2837 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE; 2838 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT; 2839 if (udata) { 2840 struct bnxt_re_cq_req req; 2841 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context( 2842 udata, struct bnxt_re_ucontext, ib_uctx); 2843 if (ib_copy_from_udata(&req, udata, sizeof(req))) { 2844 rc = -EFAULT; 2845 goto fail; 2846 } 2847 2848 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va, 2849 entries * sizeof(struct cq_base), 2850 IB_ACCESS_LOCAL_WRITE); 2851 if (IS_ERR(cq->umem)) { 2852 rc = PTR_ERR(cq->umem); 2853 goto fail; 2854 } 2855 cq->qplib_cq.sg_info.umem = cq->umem; 2856 cq->qplib_cq.dpi = &uctx->dpi; 2857 } else { 2858 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL); 2859 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 2860 GFP_KERNEL); 2861 if (!cq->cql) { 2862 rc = -ENOMEM; 2863 goto fail; 2864 } 2865 2866 cq->qplib_cq.dpi = &rdev->dpi_privileged; 2867 } 2868 /* 2869 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 2870 * used for getting the NQ index. 2871 */ 2872 nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt); 2873 nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)]; 2874 cq->qplib_cq.max_wqe = entries; 2875 cq->qplib_cq.cnq_hw_ring_id = nq->ring_id; 2876 cq->qplib_cq.nq = nq; 2877 2878 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq); 2879 if (rc) { 2880 ibdev_err(&rdev->ibdev, "Failed to create HW CQ"); 2881 goto fail; 2882 } 2883 2884 cq->ib_cq.cqe = entries; 2885 cq->cq_period = cq->qplib_cq.period; 2886 nq->budget++; 2887 2888 atomic_inc(&rdev->cq_count); 2889 spin_lock_init(&cq->cq_lock); 2890 2891 if (udata) { 2892 struct bnxt_re_cq_resp resp; 2893 2894 resp.cqid = cq->qplib_cq.id; 2895 resp.tail = cq->qplib_cq.hwq.cons; 2896 resp.phase = cq->qplib_cq.period; 2897 resp.rsvd = 0; 2898 rc = ib_copy_to_udata(udata, &resp, sizeof(resp)); 2899 if (rc) { 2900 ibdev_err(&rdev->ibdev, "Failed to copy CQ udata"); 2901 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 2902 goto c2fail; 2903 } 2904 } 2905 2906 return 0; 2907 2908 c2fail: 2909 ib_umem_release(cq->umem); 2910 fail: 2911 kfree(cq->cql); 2912 return rc; 2913 } 2914 2915 static u8 __req_to_ib_wc_status(u8 qstatus) 2916 { 2917 switch (qstatus) { 2918 case CQ_REQ_STATUS_OK: 2919 return IB_WC_SUCCESS; 2920 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 2921 return IB_WC_BAD_RESP_ERR; 2922 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 2923 return IB_WC_LOC_LEN_ERR; 2924 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 2925 return IB_WC_LOC_QP_OP_ERR; 2926 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 2927 return IB_WC_LOC_PROT_ERR; 2928 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 2929 return IB_WC_GENERAL_ERR; 2930 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 2931 return IB_WC_REM_INV_REQ_ERR; 2932 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 2933 return IB_WC_REM_ACCESS_ERR; 2934 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 2935 return IB_WC_REM_OP_ERR; 2936 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 2937 return IB_WC_RNR_RETRY_EXC_ERR; 2938 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 2939 return IB_WC_RETRY_EXC_ERR; 2940 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 2941 return IB_WC_WR_FLUSH_ERR; 2942 default: 2943 return IB_WC_GENERAL_ERR; 2944 } 2945 return 0; 2946 } 2947 2948 static u8 __rawqp1_to_ib_wc_status(u8 qstatus) 2949 { 2950 switch (qstatus) { 2951 case CQ_RES_RAWETH_QP1_STATUS_OK: 2952 return IB_WC_SUCCESS; 2953 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 2954 return IB_WC_LOC_ACCESS_ERR; 2955 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 2956 return IB_WC_LOC_LEN_ERR; 2957 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 2958 return IB_WC_LOC_PROT_ERR; 2959 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 2960 return IB_WC_LOC_QP_OP_ERR; 2961 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 2962 return IB_WC_GENERAL_ERR; 2963 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 2964 return IB_WC_WR_FLUSH_ERR; 2965 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 2966 return IB_WC_WR_FLUSH_ERR; 2967 default: 2968 return IB_WC_GENERAL_ERR; 2969 } 2970 } 2971 2972 static u8 __rc_to_ib_wc_status(u8 qstatus) 2973 { 2974 switch (qstatus) { 2975 case CQ_RES_RC_STATUS_OK: 2976 return IB_WC_SUCCESS; 2977 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 2978 return IB_WC_LOC_ACCESS_ERR; 2979 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 2980 return IB_WC_LOC_LEN_ERR; 2981 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 2982 return IB_WC_LOC_PROT_ERR; 2983 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 2984 return IB_WC_LOC_QP_OP_ERR; 2985 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 2986 return IB_WC_GENERAL_ERR; 2987 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 2988 return IB_WC_REM_INV_REQ_ERR; 2989 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 2990 return IB_WC_WR_FLUSH_ERR; 2991 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 2992 return IB_WC_WR_FLUSH_ERR; 2993 default: 2994 return IB_WC_GENERAL_ERR; 2995 } 2996 } 2997 2998 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 2999 { 3000 switch (cqe->type) { 3001 case BNXT_QPLIB_SWQE_TYPE_SEND: 3002 wc->opcode = IB_WC_SEND; 3003 break; 3004 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 3005 wc->opcode = IB_WC_SEND; 3006 wc->wc_flags |= IB_WC_WITH_IMM; 3007 break; 3008 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 3009 wc->opcode = IB_WC_SEND; 3010 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3011 break; 3012 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 3013 wc->opcode = IB_WC_RDMA_WRITE; 3014 break; 3015 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 3016 wc->opcode = IB_WC_RDMA_WRITE; 3017 wc->wc_flags |= IB_WC_WITH_IMM; 3018 break; 3019 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 3020 wc->opcode = IB_WC_RDMA_READ; 3021 break; 3022 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 3023 wc->opcode = IB_WC_COMP_SWAP; 3024 break; 3025 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 3026 wc->opcode = IB_WC_FETCH_ADD; 3027 break; 3028 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 3029 wc->opcode = IB_WC_LOCAL_INV; 3030 break; 3031 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 3032 wc->opcode = IB_WC_REG_MR; 3033 break; 3034 default: 3035 wc->opcode = IB_WC_SEND; 3036 break; 3037 } 3038 3039 wc->status = __req_to_ib_wc_status(cqe->status); 3040 } 3041 3042 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, 3043 u16 raweth_qp1_flags2) 3044 { 3045 bool is_ipv6 = false, is_ipv4 = false; 3046 3047 /* raweth_qp1_flags Bit 9-6 indicates itype */ 3048 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 3049 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 3050 return -1; 3051 3052 if (raweth_qp1_flags2 & 3053 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 3054 raweth_qp1_flags2 & 3055 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 3056 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 3057 (raweth_qp1_flags2 & 3058 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 3059 (is_ipv6 = true) : (is_ipv4 = true); 3060 return ((is_ipv6) ? 3061 BNXT_RE_ROCEV2_IPV6_PACKET : 3062 BNXT_RE_ROCEV2_IPV4_PACKET); 3063 } else { 3064 return BNXT_RE_ROCE_V1_PACKET; 3065 } 3066 } 3067 3068 static int bnxt_re_to_ib_nw_type(int nw_type) 3069 { 3070 u8 nw_hdr_type = 0xFF; 3071 3072 switch (nw_type) { 3073 case BNXT_RE_ROCE_V1_PACKET: 3074 nw_hdr_type = RDMA_NETWORK_ROCE_V1; 3075 break; 3076 case BNXT_RE_ROCEV2_IPV4_PACKET: 3077 nw_hdr_type = RDMA_NETWORK_IPV4; 3078 break; 3079 case BNXT_RE_ROCEV2_IPV6_PACKET: 3080 nw_hdr_type = RDMA_NETWORK_IPV6; 3081 break; 3082 } 3083 return nw_hdr_type; 3084 } 3085 3086 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 3087 void *rq_hdr_buf) 3088 { 3089 u8 *tmp_buf = NULL; 3090 struct ethhdr *eth_hdr; 3091 u16 eth_type; 3092 bool rc = false; 3093 3094 tmp_buf = (u8 *)rq_hdr_buf; 3095 /* 3096 * If dest mac is not same as I/F mac, this could be a 3097 * loopback address or multicast address, check whether 3098 * it is a loopback packet 3099 */ 3100 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) { 3101 tmp_buf += 4; 3102 /* Check the ether type */ 3103 eth_hdr = (struct ethhdr *)tmp_buf; 3104 eth_type = ntohs(eth_hdr->h_proto); 3105 switch (eth_type) { 3106 case ETH_P_IBOE: 3107 rc = true; 3108 break; 3109 case ETH_P_IP: 3110 case ETH_P_IPV6: { 3111 u32 len; 3112 struct udphdr *udp_hdr; 3113 3114 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) : 3115 sizeof(struct ipv6hdr)); 3116 tmp_buf += sizeof(struct ethhdr) + len; 3117 udp_hdr = (struct udphdr *)tmp_buf; 3118 if (ntohs(udp_hdr->dest) == 3119 ROCE_V2_UDP_DPORT) 3120 rc = true; 3121 break; 3122 } 3123 default: 3124 break; 3125 } 3126 } 3127 3128 return rc; 3129 } 3130 3131 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp, 3132 struct bnxt_qplib_cqe *cqe) 3133 { 3134 struct bnxt_re_dev *rdev = gsi_qp->rdev; 3135 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3136 struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp; 3137 struct bnxt_re_ah *gsi_sah; 3138 struct ib_send_wr *swr; 3139 struct ib_ud_wr udwr; 3140 struct ib_recv_wr rwr; 3141 int pkt_type = 0; 3142 u32 tbl_idx; 3143 void *rq_hdr_buf; 3144 dma_addr_t rq_hdr_buf_map; 3145 dma_addr_t shrq_hdr_buf_map; 3146 u32 offset = 0; 3147 u32 skip_bytes = 0; 3148 struct ib_sge s_sge[2]; 3149 struct ib_sge r_sge[2]; 3150 int rc; 3151 3152 memset(&udwr, 0, sizeof(udwr)); 3153 memset(&rwr, 0, sizeof(rwr)); 3154 memset(&s_sge, 0, sizeof(s_sge)); 3155 memset(&r_sge, 0, sizeof(r_sge)); 3156 3157 swr = &udwr.wr; 3158 tbl_idx = cqe->wr_id; 3159 3160 rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf + 3161 (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size); 3162 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 3163 tbl_idx); 3164 3165 /* Shadow QP header buffer */ 3166 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 3167 tbl_idx); 3168 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3169 3170 /* Store this cqe */ 3171 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 3172 sqp_entry->qp1_qp = gsi_qp; 3173 3174 /* Find packet type from the cqe */ 3175 3176 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 3177 cqe->raweth_qp1_flags2); 3178 if (pkt_type < 0) { 3179 ibdev_err(&rdev->ibdev, "Invalid packet\n"); 3180 return -EINVAL; 3181 } 3182 3183 /* Adjust the offset for the user buffer and post in the rq */ 3184 3185 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 3186 offset = 20; 3187 3188 /* 3189 * QP1 loopback packet has 4 bytes of internal header before 3190 * ether header. Skip these four bytes. 3191 */ 3192 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 3193 skip_bytes = 4; 3194 3195 /* First send SGE . Skip the ether header*/ 3196 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 3197 + skip_bytes; 3198 s_sge[0].lkey = 0xFFFFFFFF; 3199 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 3200 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 3201 3202 /* Second Send SGE */ 3203 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 3204 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 3205 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 3206 s_sge[1].addr += 8; 3207 s_sge[1].lkey = 0xFFFFFFFF; 3208 s_sge[1].length = 256; 3209 3210 /* First recv SGE */ 3211 3212 r_sge[0].addr = shrq_hdr_buf_map; 3213 r_sge[0].lkey = 0xFFFFFFFF; 3214 r_sge[0].length = 40; 3215 3216 r_sge[1].addr = sqp_entry->sge.addr + offset; 3217 r_sge[1].lkey = sqp_entry->sge.lkey; 3218 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 3219 3220 /* Create receive work request */ 3221 rwr.num_sge = 2; 3222 rwr.sg_list = r_sge; 3223 rwr.wr_id = tbl_idx; 3224 rwr.next = NULL; 3225 3226 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); 3227 if (rc) { 3228 ibdev_err(&rdev->ibdev, 3229 "Failed to post Rx buffers to shadow QP"); 3230 return -ENOMEM; 3231 } 3232 3233 swr->num_sge = 2; 3234 swr->sg_list = s_sge; 3235 swr->wr_id = tbl_idx; 3236 swr->opcode = IB_WR_SEND; 3237 swr->next = NULL; 3238 gsi_sah = rdev->gsi_ctx.gsi_sah; 3239 udwr.ah = &gsi_sah->ib_ah; 3240 udwr.remote_qpn = gsi_sqp->qplib_qp.id; 3241 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; 3242 3243 /* post data received in the send queue */ 3244 rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); 3245 3246 return 0; 3247 } 3248 3249 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 3250 struct bnxt_qplib_cqe *cqe) 3251 { 3252 wc->opcode = IB_WC_RECV; 3253 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 3254 wc->wc_flags |= IB_WC_GRH; 3255 } 3256 3257 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, 3258 u16 vlan_id) 3259 { 3260 /* 3261 * Check if the vlan is configured in the host. If not configured, it 3262 * can be a transparent VLAN. So dont report the vlan id. 3263 */ 3264 if (!__vlan_find_dev_deep_rcu(rdev->netdev, 3265 htons(ETH_P_8021Q), vlan_id)) 3266 return false; 3267 return true; 3268 } 3269 3270 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe, 3271 u16 *vid, u8 *sl) 3272 { 3273 bool ret = false; 3274 u32 metadata; 3275 u16 tpid; 3276 3277 metadata = orig_cqe->raweth_qp1_metadata; 3278 if (orig_cqe->raweth_qp1_flags2 & 3279 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 3280 tpid = ((metadata & 3281 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 3282 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 3283 if (tpid == ETH_P_8021Q) { 3284 *vid = metadata & 3285 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 3286 *sl = (metadata & 3287 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 3288 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 3289 ret = true; 3290 } 3291 } 3292 3293 return ret; 3294 } 3295 3296 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 3297 struct bnxt_qplib_cqe *cqe) 3298 { 3299 wc->opcode = IB_WC_RECV; 3300 wc->status = __rc_to_ib_wc_status(cqe->status); 3301 3302 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 3303 wc->wc_flags |= IB_WC_WITH_IMM; 3304 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 3305 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 3306 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 3307 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 3308 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 3309 } 3310 3311 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, 3312 struct ib_wc *wc, 3313 struct bnxt_qplib_cqe *cqe) 3314 { 3315 struct bnxt_re_dev *rdev = gsi_sqp->rdev; 3316 struct bnxt_re_qp *gsi_qp = NULL; 3317 struct bnxt_qplib_cqe *orig_cqe = NULL; 3318 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3319 int nw_type; 3320 u32 tbl_idx; 3321 u16 vlan_id; 3322 u8 sl; 3323 3324 tbl_idx = cqe->wr_id; 3325 3326 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3327 gsi_qp = sqp_entry->qp1_qp; 3328 orig_cqe = &sqp_entry->cqe; 3329 3330 wc->wr_id = sqp_entry->wrid; 3331 wc->byte_len = orig_cqe->length; 3332 wc->qp = &gsi_qp->ib_qp; 3333 3334 wc->ex.imm_data = orig_cqe->immdata; 3335 wc->src_qp = orig_cqe->src_qp; 3336 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 3337 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) { 3338 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 3339 wc->vlan_id = vlan_id; 3340 wc->sl = sl; 3341 wc->wc_flags |= IB_WC_WITH_VLAN; 3342 } 3343 } 3344 wc->port_num = 1; 3345 wc->vendor_err = orig_cqe->status; 3346 3347 wc->opcode = IB_WC_RECV; 3348 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 3349 wc->wc_flags |= IB_WC_GRH; 3350 3351 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 3352 orig_cqe->raweth_qp1_flags2); 3353 if (nw_type >= 0) { 3354 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3355 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3356 } 3357 } 3358 3359 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp, 3360 struct ib_wc *wc, 3361 struct bnxt_qplib_cqe *cqe) 3362 { 3363 struct bnxt_re_dev *rdev; 3364 u16 vlan_id = 0; 3365 u8 nw_type; 3366 3367 rdev = qp->rdev; 3368 wc->opcode = IB_WC_RECV; 3369 wc->status = __rc_to_ib_wc_status(cqe->status); 3370 3371 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) 3372 wc->wc_flags |= IB_WC_WITH_IMM; 3373 /* report only on GSI QP for Thor */ 3374 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) { 3375 wc->wc_flags |= IB_WC_GRH; 3376 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3377 wc->wc_flags |= IB_WC_WITH_SMAC; 3378 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { 3379 vlan_id = (cqe->cfa_meta & 0xFFF); 3380 } 3381 /* Mark only if vlan_id is non zero */ 3382 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 3383 wc->vlan_id = vlan_id; 3384 wc->wc_flags |= IB_WC_WITH_VLAN; 3385 } 3386 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >> 3387 CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT; 3388 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type); 3389 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; 3390 } 3391 3392 } 3393 3394 static int send_phantom_wqe(struct bnxt_re_qp *qp) 3395 { 3396 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 3397 unsigned long flags; 3398 int rc = 0; 3399 3400 spin_lock_irqsave(&qp->sq_lock, flags); 3401 3402 rc = bnxt_re_bind_fence_mw(lib_qp); 3403 if (!rc) { 3404 lib_qp->sq.phantom_wqe_cnt++; 3405 ibdev_dbg(&qp->rdev->ibdev, 3406 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 3407 lib_qp->id, lib_qp->sq.hwq.prod, 3408 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 3409 lib_qp->sq.phantom_wqe_cnt); 3410 } 3411 3412 spin_unlock_irqrestore(&qp->sq_lock, flags); 3413 return rc; 3414 } 3415 3416 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 3417 { 3418 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3419 struct bnxt_re_qp *qp, *sh_qp; 3420 struct bnxt_qplib_cqe *cqe; 3421 int i, ncqe, budget; 3422 struct bnxt_qplib_q *sq; 3423 struct bnxt_qplib_qp *lib_qp; 3424 u32 tbl_idx; 3425 struct bnxt_re_sqp_entries *sqp_entry = NULL; 3426 unsigned long flags; 3427 3428 spin_lock_irqsave(&cq->cq_lock, flags); 3429 budget = min_t(u32, num_entries, cq->max_cql); 3430 num_entries = budget; 3431 if (!cq->cql) { 3432 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use"); 3433 goto exit; 3434 } 3435 cqe = &cq->cql[0]; 3436 while (budget) { 3437 lib_qp = NULL; 3438 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 3439 if (lib_qp) { 3440 sq = &lib_qp->sq; 3441 if (sq->send_phantom) { 3442 qp = container_of(lib_qp, 3443 struct bnxt_re_qp, qplib_qp); 3444 if (send_phantom_wqe(qp) == -ENOMEM) 3445 ibdev_err(&cq->rdev->ibdev, 3446 "Phantom failed! Scheduled to send again\n"); 3447 else 3448 sq->send_phantom = false; 3449 } 3450 } 3451 if (ncqe < budget) 3452 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 3453 cqe + ncqe, 3454 budget - ncqe); 3455 3456 if (!ncqe) 3457 break; 3458 3459 for (i = 0; i < ncqe; i++, cqe++) { 3460 /* Transcribe each qplib_wqe back to ib_wc */ 3461 memset(wc, 0, sizeof(*wc)); 3462 3463 wc->wr_id = cqe->wr_id; 3464 wc->byte_len = cqe->length; 3465 qp = container_of 3466 ((struct bnxt_qplib_qp *) 3467 (unsigned long)(cqe->qp_handle), 3468 struct bnxt_re_qp, qplib_qp); 3469 wc->qp = &qp->ib_qp; 3470 wc->ex.imm_data = cqe->immdata; 3471 wc->src_qp = cqe->src_qp; 3472 memcpy(wc->smac, cqe->smac, ETH_ALEN); 3473 wc->port_num = 1; 3474 wc->vendor_err = cqe->status; 3475 3476 switch (cqe->opcode) { 3477 case CQ_BASE_CQE_TYPE_REQ: 3478 sh_qp = qp->rdev->gsi_ctx.gsi_sqp; 3479 if (sh_qp && 3480 qp->qplib_qp.id == sh_qp->qplib_qp.id) { 3481 /* Handle this completion with 3482 * the stored completion 3483 */ 3484 memset(wc, 0, sizeof(*wc)); 3485 continue; 3486 } 3487 bnxt_re_process_req_wc(wc, cqe); 3488 break; 3489 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 3490 if (!cqe->status) { 3491 int rc = 0; 3492 3493 rc = bnxt_re_process_raw_qp_pkt_rx 3494 (qp, cqe); 3495 if (!rc) { 3496 memset(wc, 0, sizeof(*wc)); 3497 continue; 3498 } 3499 cqe->status = -1; 3500 } 3501 /* Errors need not be looped back. 3502 * But change the wr_id to the one 3503 * stored in the table 3504 */ 3505 tbl_idx = cqe->wr_id; 3506 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx]; 3507 wc->wr_id = sqp_entry->wrid; 3508 bnxt_re_process_res_rawqp1_wc(wc, cqe); 3509 break; 3510 case CQ_BASE_CQE_TYPE_RES_RC: 3511 bnxt_re_process_res_rc_wc(wc, cqe); 3512 break; 3513 case CQ_BASE_CQE_TYPE_RES_UD: 3514 sh_qp = qp->rdev->gsi_ctx.gsi_sqp; 3515 if (sh_qp && 3516 qp->qplib_qp.id == sh_qp->qplib_qp.id) { 3517 /* Handle this completion with 3518 * the stored completion 3519 */ 3520 if (cqe->status) { 3521 continue; 3522 } else { 3523 bnxt_re_process_res_shadow_qp_wc 3524 (qp, wc, cqe); 3525 break; 3526 } 3527 } 3528 bnxt_re_process_res_ud_wc(qp, wc, cqe); 3529 break; 3530 default: 3531 ibdev_err(&cq->rdev->ibdev, 3532 "POLL CQ : type 0x%x not handled", 3533 cqe->opcode); 3534 continue; 3535 } 3536 wc++; 3537 budget--; 3538 } 3539 } 3540 exit: 3541 spin_unlock_irqrestore(&cq->cq_lock, flags); 3542 return num_entries - budget; 3543 } 3544 3545 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 3546 enum ib_cq_notify_flags ib_cqn_flags) 3547 { 3548 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq); 3549 int type = 0, rc = 0; 3550 unsigned long flags; 3551 3552 spin_lock_irqsave(&cq->cq_lock, flags); 3553 /* Trigger on the very next completion */ 3554 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 3555 type = DBC_DBC_TYPE_CQ_ARMALL; 3556 /* Trigger on the next solicited completion */ 3557 else if (ib_cqn_flags & IB_CQ_SOLICITED) 3558 type = DBC_DBC_TYPE_CQ_ARMSE; 3559 3560 /* Poll to see if there are missed events */ 3561 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3562 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) { 3563 rc = 1; 3564 goto exit; 3565 } 3566 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 3567 3568 exit: 3569 spin_unlock_irqrestore(&cq->cq_lock, flags); 3570 return rc; 3571 } 3572 3573 /* Memory Regions */ 3574 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 3575 { 3576 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3577 struct bnxt_re_dev *rdev = pd->rdev; 3578 struct bnxt_re_mr *mr; 3579 int rc; 3580 3581 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3582 if (!mr) 3583 return ERR_PTR(-ENOMEM); 3584 3585 mr->rdev = rdev; 3586 mr->qplib_mr.pd = &pd->qplib_pd; 3587 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3588 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3589 3590 /* Allocate and register 0 as the address */ 3591 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3592 if (rc) 3593 goto fail; 3594 3595 mr->qplib_mr.hwq.level = PBL_LVL_MAX; 3596 mr->qplib_mr.total_size = -1; /* Infinte length */ 3597 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0, 3598 PAGE_SIZE); 3599 if (rc) 3600 goto fail_mr; 3601 3602 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3603 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 3604 IB_ACCESS_REMOTE_ATOMIC)) 3605 mr->ib_mr.rkey = mr->ib_mr.lkey; 3606 atomic_inc(&rdev->mr_count); 3607 3608 return &mr->ib_mr; 3609 3610 fail_mr: 3611 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3612 fail: 3613 kfree(mr); 3614 return ERR_PTR(rc); 3615 } 3616 3617 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3618 { 3619 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3620 struct bnxt_re_dev *rdev = mr->rdev; 3621 int rc; 3622 3623 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3624 if (rc) { 3625 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc); 3626 return rc; 3627 } 3628 3629 if (mr->pages) { 3630 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 3631 &mr->qplib_frpl); 3632 kfree(mr->pages); 3633 mr->npages = 0; 3634 mr->pages = NULL; 3635 } 3636 ib_umem_release(mr->ib_umem); 3637 3638 kfree(mr); 3639 atomic_dec(&rdev->mr_count); 3640 return rc; 3641 } 3642 3643 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 3644 { 3645 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3646 3647 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 3648 return -ENOMEM; 3649 3650 mr->pages[mr->npages++] = addr; 3651 return 0; 3652 } 3653 3654 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, 3655 unsigned int *sg_offset) 3656 { 3657 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr); 3658 3659 mr->npages = 0; 3660 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); 3661 } 3662 3663 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 3664 u32 max_num_sg) 3665 { 3666 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3667 struct bnxt_re_dev *rdev = pd->rdev; 3668 struct bnxt_re_mr *mr = NULL; 3669 int rc; 3670 3671 if (type != IB_MR_TYPE_MEM_REG) { 3672 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type); 3673 return ERR_PTR(-EINVAL); 3674 } 3675 if (max_num_sg > MAX_PBL_LVL_1_PGS) 3676 return ERR_PTR(-EINVAL); 3677 3678 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3679 if (!mr) 3680 return ERR_PTR(-ENOMEM); 3681 3682 mr->rdev = rdev; 3683 mr->qplib_mr.pd = &pd->qplib_pd; 3684 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 3685 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 3686 3687 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3688 if (rc) 3689 goto bail; 3690 3691 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3692 mr->ib_mr.rkey = mr->ib_mr.lkey; 3693 3694 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL); 3695 if (!mr->pages) { 3696 rc = -ENOMEM; 3697 goto fail; 3698 } 3699 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 3700 &mr->qplib_frpl, max_num_sg); 3701 if (rc) { 3702 ibdev_err(&rdev->ibdev, 3703 "Failed to allocate HW FR page list"); 3704 goto fail_mr; 3705 } 3706 3707 atomic_inc(&rdev->mr_count); 3708 return &mr->ib_mr; 3709 3710 fail_mr: 3711 kfree(mr->pages); 3712 fail: 3713 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3714 bail: 3715 kfree(mr); 3716 return ERR_PTR(rc); 3717 } 3718 3719 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 3720 struct ib_udata *udata) 3721 { 3722 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3723 struct bnxt_re_dev *rdev = pd->rdev; 3724 struct bnxt_re_mw *mw; 3725 int rc; 3726 3727 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 3728 if (!mw) 3729 return ERR_PTR(-ENOMEM); 3730 mw->rdev = rdev; 3731 mw->qplib_mw.pd = &pd->qplib_pd; 3732 3733 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 3734 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 3735 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 3736 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 3737 if (rc) { 3738 ibdev_err(&rdev->ibdev, "Allocate MW failed!"); 3739 goto fail; 3740 } 3741 mw->ib_mw.rkey = mw->qplib_mw.rkey; 3742 3743 atomic_inc(&rdev->mw_count); 3744 return &mw->ib_mw; 3745 3746 fail: 3747 kfree(mw); 3748 return ERR_PTR(rc); 3749 } 3750 3751 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 3752 { 3753 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw); 3754 struct bnxt_re_dev *rdev = mw->rdev; 3755 int rc; 3756 3757 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 3758 if (rc) { 3759 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc); 3760 return rc; 3761 } 3762 3763 kfree(mw); 3764 atomic_dec(&rdev->mw_count); 3765 return rc; 3766 } 3767 3768 /* uverbs */ 3769 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 3770 u64 virt_addr, int mr_access_flags, 3771 struct ib_udata *udata) 3772 { 3773 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd); 3774 struct bnxt_re_dev *rdev = pd->rdev; 3775 struct bnxt_re_mr *mr; 3776 struct ib_umem *umem; 3777 unsigned long page_size; 3778 int umem_pgs, rc; 3779 3780 if (length > BNXT_RE_MAX_MR_SIZE) { 3781 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n", 3782 length, BNXT_RE_MAX_MR_SIZE); 3783 return ERR_PTR(-ENOMEM); 3784 } 3785 3786 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3787 if (!mr) 3788 return ERR_PTR(-ENOMEM); 3789 3790 mr->rdev = rdev; 3791 mr->qplib_mr.pd = &pd->qplib_pd; 3792 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 3793 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 3794 3795 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 3796 if (rc) { 3797 ibdev_err(&rdev->ibdev, "Failed to allocate MR"); 3798 goto free_mr; 3799 } 3800 /* The fixed portion of the rkey is the same as the lkey */ 3801 mr->ib_mr.rkey = mr->qplib_mr.rkey; 3802 3803 umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags); 3804 if (IS_ERR(umem)) { 3805 ibdev_err(&rdev->ibdev, "Failed to get umem"); 3806 rc = -EFAULT; 3807 goto free_mrw; 3808 } 3809 mr->ib_umem = umem; 3810 3811 mr->qplib_mr.va = virt_addr; 3812 page_size = ib_umem_find_best_pgsz( 3813 umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr); 3814 if (!page_size) { 3815 ibdev_err(&rdev->ibdev, "umem page size unsupported!"); 3816 rc = -EFAULT; 3817 goto free_umem; 3818 } 3819 mr->qplib_mr.total_size = length; 3820 3821 umem_pgs = ib_umem_num_dma_blocks(umem, page_size); 3822 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem, 3823 umem_pgs, page_size); 3824 if (rc) { 3825 ibdev_err(&rdev->ibdev, "Failed to register user MR"); 3826 goto free_umem; 3827 } 3828 3829 mr->ib_mr.lkey = mr->qplib_mr.lkey; 3830 mr->ib_mr.rkey = mr->qplib_mr.lkey; 3831 atomic_inc(&rdev->mr_count); 3832 3833 return &mr->ib_mr; 3834 free_umem: 3835 ib_umem_release(umem); 3836 free_mrw: 3837 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 3838 free_mr: 3839 kfree(mr); 3840 return ERR_PTR(rc); 3841 } 3842 3843 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) 3844 { 3845 struct ib_device *ibdev = ctx->device; 3846 struct bnxt_re_ucontext *uctx = 3847 container_of(ctx, struct bnxt_re_ucontext, ib_uctx); 3848 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 3849 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; 3850 struct bnxt_re_uctx_resp resp = {}; 3851 u32 chip_met_rev_num = 0; 3852 int rc; 3853 3854 ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver); 3855 3856 if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 3857 ibdev_dbg(ibdev, " is different from the device %d ", 3858 BNXT_RE_ABI_VERSION); 3859 return -EPERM; 3860 } 3861 3862 uctx->rdev = rdev; 3863 3864 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 3865 if (!uctx->shpg) { 3866 rc = -ENOMEM; 3867 goto fail; 3868 } 3869 spin_lock_init(&uctx->sh_lock); 3870 3871 resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; 3872 chip_met_rev_num = rdev->chip_ctx->chip_num; 3873 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) << 3874 BNXT_RE_CHIP_ID0_CHIP_REV_SFT; 3875 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) << 3876 BNXT_RE_CHIP_ID0_CHIP_MET_SFT; 3877 resp.chip_id0 = chip_met_rev_num; 3878 /*Temp, Use xa_alloc instead */ 3879 resp.dev_id = rdev->en_dev->pdev->devfn; 3880 resp.max_qp = rdev->qplib_ctx.qpc_count; 3881 resp.pg_size = PAGE_SIZE; 3882 resp.cqe_sz = sizeof(struct cq_base); 3883 resp.max_cqd = dev_attr->max_cq_wqes; 3884 3885 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE; 3886 resp.mode = rdev->chip_ctx->modes.wqe_mode; 3887 3888 rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); 3889 if (rc) { 3890 ibdev_err(ibdev, "Failed to copy user context"); 3891 rc = -EFAULT; 3892 goto cfail; 3893 } 3894 3895 return 0; 3896 cfail: 3897 free_page((unsigned long)uctx->shpg); 3898 uctx->shpg = NULL; 3899 fail: 3900 return rc; 3901 } 3902 3903 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 3904 { 3905 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3906 struct bnxt_re_ucontext, 3907 ib_uctx); 3908 3909 struct bnxt_re_dev *rdev = uctx->rdev; 3910 3911 if (uctx->shpg) 3912 free_page((unsigned long)uctx->shpg); 3913 3914 if (uctx->dpi.dbr) { 3915 /* Free DPI only if this is the first PD allocated by the 3916 * application and mark the context dpi as NULL 3917 */ 3918 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 3919 &rdev->qplib_res.dpi_tbl, &uctx->dpi); 3920 uctx->dpi.dbr = NULL; 3921 } 3922 } 3923 3924 /* Helper function to mmap the virtual memory from user app */ 3925 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 3926 { 3927 struct bnxt_re_ucontext *uctx = container_of(ib_uctx, 3928 struct bnxt_re_ucontext, 3929 ib_uctx); 3930 struct bnxt_re_dev *rdev = uctx->rdev; 3931 u64 pfn; 3932 3933 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 3934 return -EINVAL; 3935 3936 if (vma->vm_pgoff) { 3937 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 3938 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 3939 PAGE_SIZE, vma->vm_page_prot)) { 3940 ibdev_err(&rdev->ibdev, "Failed to map DPI"); 3941 return -EAGAIN; 3942 } 3943 } else { 3944 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT; 3945 if (remap_pfn_range(vma, vma->vm_start, 3946 pfn, PAGE_SIZE, vma->vm_page_prot)) { 3947 ibdev_err(&rdev->ibdev, "Failed to map shared page"); 3948 return -EAGAIN; 3949 } 3950 } 3951 3952 return 0; 3953 } 3954