1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: IB Verbs interpreter 29 */ 30 31 #include <linux/if_ether.h> 32 #include <linux/etherdevice.h> 33 #include <rdma/uverbs_ioctl.h> 34 35 #include "bnxt_re.h" 36 #include "ib_verbs.h" 37 38 static inline 39 struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap) 40 { 41 42 *nmap = umem->nmap; 43 return umem->sg_head.sgl; 44 } 45 46 static inline void bnxt_re_peer_mem_release(struct ib_umem *umem) 47 { 48 dev_dbg(NULL, "ib_umem_release getting invoked \n"); 49 ib_umem_release(umem); 50 } 51 52 void bnxt_re_resolve_dmac_task(struct work_struct *work) 53 { 54 int rc = -1; 55 struct bnxt_re_dev *rdev; 56 struct ib_ah_attr *ah_attr; 57 struct bnxt_re_resolve_dmac_work *dmac_work = 58 container_of(work, struct bnxt_re_resolve_dmac_work, work); 59 60 rdev = dmac_work->rdev; 61 ah_attr = dmac_work->ah_attr; 62 rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr); 63 if (rc) 64 dev_err(rdev_to_dev(dmac_work->rdev), 65 "Failed to resolve dest mac rc = %d\n", rc); 66 atomic_set(&dmac_work->status_wait, rc << 8); 67 } 68 69 static int __from_ib_access_flags(int iflags) 70 { 71 int qflags = 0; 72 73 if (iflags & IB_ACCESS_LOCAL_WRITE) 74 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 75 if (iflags & IB_ACCESS_REMOTE_READ) 76 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 77 if (iflags & IB_ACCESS_REMOTE_WRITE) 78 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 79 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 80 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 81 if (iflags & IB_ACCESS_MW_BIND) 82 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 83 if (iflags & IB_ZERO_BASED) 84 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 85 if (iflags & IB_ACCESS_ON_DEMAND) 86 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 87 return qflags; 88 }; 89 90 static enum ib_access_flags __to_ib_access_flags(int qflags) 91 { 92 enum ib_access_flags iflags = 0; 93 94 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 95 iflags |= IB_ACCESS_LOCAL_WRITE; 96 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 97 iflags |= IB_ACCESS_REMOTE_WRITE; 98 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 99 iflags |= IB_ACCESS_REMOTE_READ; 100 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 101 iflags |= IB_ACCESS_REMOTE_ATOMIC; 102 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 103 iflags |= IB_ACCESS_MW_BIND; 104 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 105 iflags |= IB_ZERO_BASED; 106 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 107 iflags |= IB_ACCESS_ON_DEMAND; 108 return iflags; 109 }; 110 111 static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data, 112 int len, struct ib_udata *udata) 113 { 114 int rc; 115 116 rc = ib_copy_to_udata(udata, data, len); 117 if (rc) 118 dev_err(rdev_to_dev(rdev), 119 "ucontext copy failed from %ps rc %d\n", 120 __builtin_return_address(0), rc); 121 122 return rc; 123 } 124 125 struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, 126 u8 port_num) 127 { 128 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 129 struct ifnet *netdev = NULL; 130 131 rcu_read_lock(); 132 133 if (!rdev || !rdev->netdev) 134 goto end; 135 136 netdev = rdev->netdev; 137 138 /* In case of active-backup bond mode, return active slave */ 139 if (netdev) 140 dev_hold(netdev); 141 142 end: 143 rcu_read_unlock(); 144 return netdev; 145 } 146 147 int bnxt_re_query_device(struct ib_device *ibdev, 148 struct ib_device_attr *ib_attr, 149 struct ib_udata *udata) 150 { 151 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 152 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; 153 154 memset(ib_attr, 0, sizeof(*ib_attr)); 155 156 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4); 157 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid); 158 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 159 ib_attr->page_size_cap = dev_attr->page_size_cap; 160 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 161 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 162 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 163 ib_attr->max_qp = dev_attr->max_qp; 164 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 165 /* 166 * Read and set from the module param 'min_tx_depth' 167 * only once after the driver load 168 */ 169 if (rdev->min_tx_depth == 1 && 170 min_tx_depth < dev_attr->max_qp_wqes) 171 rdev->min_tx_depth = min_tx_depth; 172 ib_attr->device_cap_flags = 173 IB_DEVICE_CURR_QP_STATE_MOD 174 | IB_DEVICE_RC_RNR_NAK_GEN 175 | IB_DEVICE_SHUTDOWN_PORT 176 | IB_DEVICE_SYS_IMAGE_GUID 177 | IB_DEVICE_LOCAL_DMA_LKEY 178 | IB_DEVICE_RESIZE_MAX_WR 179 | IB_DEVICE_PORT_ACTIVE_EVENT 180 | IB_DEVICE_N_NOTIFY_CQ 181 | IB_DEVICE_MEM_WINDOW 182 | IB_DEVICE_MEM_WINDOW_TYPE_2B 183 | IB_DEVICE_MEM_MGT_EXTENSIONS; 184 ib_attr->max_send_sge = dev_attr->max_qp_sges; 185 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 186 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 187 ib_attr->max_cq = dev_attr->max_cq; 188 ib_attr->max_cqe = dev_attr->max_cq_wqes; 189 ib_attr->max_mr = dev_attr->max_mr; 190 ib_attr->max_pd = dev_attr->max_pd; 191 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 192 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 193 if (dev_attr->is_atomic) { 194 ib_attr->atomic_cap = IB_ATOMIC_GLOB; 195 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; 196 } 197 ib_attr->max_ee_rd_atom = 0; 198 ib_attr->max_res_rd_atom = 0; 199 ib_attr->max_ee_init_rd_atom = 0; 200 ib_attr->max_ee = 0; 201 ib_attr->max_rdd = 0; 202 ib_attr->max_mw = dev_attr->max_mw; 203 ib_attr->max_raw_ipv6_qp = 0; 204 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 205 ib_attr->max_mcast_grp = 0; 206 ib_attr->max_mcast_qp_attach = 0; 207 ib_attr->max_total_mcast_qp_attach = 0; 208 ib_attr->max_ah = dev_attr->max_ah; 209 ib_attr->max_srq = dev_attr->max_srq; 210 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 211 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 212 213 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 214 ib_attr->max_pkeys = 1; 215 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 216 ib_attr->sig_prot_cap = 0; 217 ib_attr->sig_guard_cap = 0; 218 ib_attr->odp_caps.general_caps = 0; 219 220 return 0; 221 } 222 223 int bnxt_re_modify_device(struct ib_device *ibdev, 224 int device_modify_mask, 225 struct ib_device_modify *device_modify) 226 { 227 dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n", 228 device_modify_mask); 229 230 switch (device_modify_mask) { 231 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: 232 /* Modify the GUID requires the modification of the GID table */ 233 /* GUID should be made as READ-ONLY */ 234 break; 235 case IB_DEVICE_MODIFY_NODE_DESC: 236 /* Node Desc should be made as READ-ONLY */ 237 break; 238 default: 239 break; 240 } 241 return 0; 242 } 243 244 static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width) 245 { 246 switch (espeed) { 247 case SPEED_1000: 248 *speed = IB_SPEED_SDR; 249 *width = IB_WIDTH_1X; 250 break; 251 case SPEED_10000: 252 *speed = IB_SPEED_QDR; 253 *width = IB_WIDTH_1X; 254 break; 255 case SPEED_20000: 256 *speed = IB_SPEED_DDR; 257 *width = IB_WIDTH_4X; 258 break; 259 case SPEED_25000: 260 *speed = IB_SPEED_EDR; 261 *width = IB_WIDTH_1X; 262 break; 263 case SPEED_40000: 264 *speed = IB_SPEED_QDR; 265 *width = IB_WIDTH_4X; 266 break; 267 case SPEED_50000: 268 *speed = IB_SPEED_EDR; 269 *width = IB_WIDTH_2X; 270 break; 271 case SPEED_100000: 272 *speed = IB_SPEED_EDR; 273 *width = IB_WIDTH_4X; 274 break; 275 case SPEED_200000: 276 *speed = IB_SPEED_HDR; 277 *width = IB_WIDTH_4X; 278 break; 279 default: 280 *speed = IB_SPEED_SDR; 281 *width = IB_WIDTH_1X; 282 break; 283 } 284 } 285 286 /* Port */ 287 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, 288 struct ib_port_attr *port_attr) 289 { 290 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 291 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; 292 u8 active_speed = 0, active_width = 0; 293 294 dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num); 295 memset(port_attr, 0, sizeof(*port_attr)); 296 297 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 298 port_attr->state = bnxt_re_get_link_state(rdev); 299 if (port_attr->state == IB_PORT_ACTIVE) 300 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 301 port_attr->max_mtu = IB_MTU_4096; 302 port_attr->active_mtu = iboe_get_mtu(if_getmtu(rdev->netdev)); 303 port_attr->gid_tbl_len = dev_attr->max_sgid; 304 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 305 IB_PORT_DEVICE_MGMT_SUP | 306 IB_PORT_VENDOR_CLASS_SUP | 307 IB_PORT_IP_BASED_GIDS; 308 309 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 310 port_attr->bad_pkey_cntr = 0; 311 port_attr->qkey_viol_cntr = 0; 312 port_attr->pkey_tbl_len = dev_attr->max_pkey; 313 port_attr->lid = 0; 314 port_attr->sm_lid = 0; 315 port_attr->lmc = 0; 316 port_attr->max_vl_num = 4; 317 port_attr->sm_sl = 0; 318 port_attr->subnet_timeout = 0; 319 port_attr->init_type_reply = 0; 320 rdev->espeed = rdev->en_dev->espeed; 321 322 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 323 __to_ib_speed_width(rdev->espeed, &active_speed, 324 &active_width); 325 326 port_attr->active_speed = active_speed; 327 port_attr->active_width = active_width; 328 329 return 0; 330 } 331 332 int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, 333 int port_modify_mask, 334 struct ib_port_modify *port_modify) 335 { 336 dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n", 337 port_modify_mask); 338 339 switch (port_modify_mask) { 340 case IB_PORT_SHUTDOWN: 341 break; 342 case IB_PORT_INIT_TYPE: 343 break; 344 case IB_PORT_RESET_QKEY_CNTR: 345 break; 346 default: 347 break; 348 } 349 return 0; 350 } 351 352 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, 353 struct ib_port_immutable *immutable) 354 { 355 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 356 struct ib_port_attr port_attr; 357 358 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 359 return -EINVAL; 360 361 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 362 immutable->gid_tbl_len = port_attr.gid_tbl_len; 363 if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP) 364 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 365 else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP) 366 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 367 else 368 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | 369 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 370 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 371 return 0; 372 } 373 374 void bnxt_re_compat_qfwstr(void) 375 { 376 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 377 378 sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0], 379 rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2], 380 rdev->dev_attr->fw_ver[3]); 381 } 382 383 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, 384 u16 index, u16 *pkey) 385 { 386 if (index > 0) 387 return -EINVAL; 388 389 *pkey = IB_DEFAULT_PKEY_FULL; 390 391 return 0; 392 } 393 394 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, 395 int index, union ib_gid *gid) 396 { 397 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 398 int rc = 0; 399 400 /* Ignore port_num */ 401 memset(gid, 0, sizeof(*gid)); 402 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 403 &rdev->qplib_res.sgid_tbl, index, 404 (struct bnxt_qplib_gid *)gid); 405 return rc; 406 } 407 408 int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, 409 unsigned int index, void **context) 410 { 411 int rc = 0; 412 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 413 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 414 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 415 struct bnxt_qplib_gid *gid_to_del; 416 u16 vlan_id = 0xFFFF; 417 418 /* Delete the entry from the hardware */ 419 ctx = *context; 420 if (!ctx) { 421 dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n"); 422 return -EINVAL; 423 } 424 if (sgid_tbl && sgid_tbl->active) { 425 if (ctx->idx >= sgid_tbl->max) { 426 dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n"); 427 return -EINVAL; 428 } 429 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; 430 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; 431 ctx->refcnt--; 432 /* DEL_GID is called via WQ context(netdevice_event_work_handler) 433 * or via the ib_unregister_device path. In the former case QP1 434 * may not be destroyed yet, in which case just return as FW 435 * needs that entry to be present and will fail it's deletion. 436 * We could get invoked again after QP1 is destroyed OR get an 437 * ADD_GID call with a different GID value for the same index 438 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 439 */ 440 if (ctx->idx == 0 && 441 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 442 (rdev->gsi_ctx.gsi_sqp || 443 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) { 444 dev_dbg(rdev_to_dev(rdev), 445 "Trying to delete GID0 while QP1 is alive\n"); 446 if (!ctx->refcnt) { 447 rdev->gid_map[index] = -1; 448 ctx_tbl = sgid_tbl->ctx; 449 ctx_tbl[ctx->idx] = NULL; 450 kfree(ctx); 451 } 452 return 0; 453 } 454 rdev->gid_map[index] = -1; 455 if (!ctx->refcnt) { 456 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, 457 vlan_id, true); 458 if (!rc) { 459 dev_dbg(rdev_to_dev(rdev), "GID remove success\n"); 460 ctx_tbl = sgid_tbl->ctx; 461 ctx_tbl[ctx->idx] = NULL; 462 kfree(ctx); 463 } else { 464 dev_err(rdev_to_dev(rdev), 465 "Remove GID failed rc = 0x%x\n", rc); 466 } 467 } 468 } else { 469 dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n"); 470 return -EINVAL; 471 } 472 return rc; 473 } 474 475 int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, 476 unsigned int index, const union ib_gid *gid, 477 const struct ib_gid_attr *attr, void **context) 478 { 479 int rc; 480 u32 tbl_idx = 0; 481 u16 vlan_id = 0xFFFF; 482 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 483 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 484 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 485 if ((attr->ndev) && is_vlan_dev(attr->ndev)) 486 vlan_id = vlan_dev_vlan_id(attr->ndev); 487 488 rc = bnxt_qplib_add_sgid(sgid_tbl, gid, 489 rdev->dev_addr, 490 vlan_id, true, &tbl_idx); 491 if (rc == -EALREADY) { 492 dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid); 493 ctx_tbl = sgid_tbl->ctx; 494 if (!ctx_tbl[tbl_idx]) { 495 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 496 if (!ctx) 497 return -ENOMEM; 498 ctx->idx = tbl_idx; 499 ctx->refcnt = 1; 500 ctx_tbl[tbl_idx] = ctx; 501 } else { 502 ctx_tbl[tbl_idx]->refcnt++; 503 } 504 *context = ctx_tbl[tbl_idx]; 505 /* tbl_idx is the HW table index and index is the stack index */ 506 rdev->gid_map[index] = tbl_idx; 507 return 0; 508 } else if (rc < 0) { 509 dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc); 510 return rc; 511 } else { 512 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 513 if (!ctx) { 514 dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n"); 515 return -ENOMEM; 516 } 517 ctx_tbl = sgid_tbl->ctx; 518 ctx->idx = tbl_idx; 519 ctx->refcnt = 1; 520 ctx_tbl[tbl_idx] = ctx; 521 /* tbl_idx is the HW table index and index is the stack index */ 522 rdev->gid_map[index] = tbl_idx; 523 *context = ctx; 524 } 525 return rc; 526 } 527 528 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 529 u8 port_num) 530 { 531 return IB_LINK_LAYER_ETHERNET; 532 } 533 534 static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd) 535 { 536 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 537 struct ib_mr *ib_mr = &fence->mr->ib_mr; 538 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 539 struct bnxt_re_dev *rdev = pd->rdev; 540 541 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 542 return; 543 544 memset(wqe, 0, sizeof(*wqe)); 545 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 546 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 547 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 548 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 549 wqe->bind.zero_based = false; 550 wqe->bind.parent_l_key = ib_mr->lkey; 551 wqe->bind.va = (u64)fence->va; 552 wqe->bind.length = fence->size; 553 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 554 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 555 556 /* Save the initial rkey in fence structure for now; 557 * wqe->bind.r_key will be set at (re)bind time. 558 */ 559 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 560 } 561 562 static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 563 { 564 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 565 qplib_qp); 566 struct ib_pd *ib_pd = qp->ib_qp.pd; 567 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 568 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 569 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 570 struct bnxt_qplib_swqe wqe; 571 int rc; 572 573 /* TODO: Need SQ locking here when Fence WQE 574 * posting moves up into bnxt_re from bnxt_qplib. 575 */ 576 memcpy(&wqe, fence_wqe, sizeof(wqe)); 577 wqe.bind.r_key = fence->bind_rkey; 578 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 579 580 dev_dbg(rdev_to_dev(qp->rdev), 581 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 582 wqe.bind.r_key, qp->qplib_qp.id, pd); 583 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 584 if (rc) { 585 dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); 586 return rc; 587 } 588 bnxt_qplib_post_send_db(&qp->qplib_qp); 589 590 return rc; 591 } 592 593 static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd) 594 { 595 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 596 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 597 struct bnxt_re_dev *rdev = pd->rdev; 598 struct bnxt_qplib_mrinfo mrinfo; 599 struct bnxt_re_mr *mr = NULL; 600 struct ib_mw *ib_mw = NULL; 601 dma_addr_t dma_addr = 0; 602 u32 max_mr_count; 603 u64 pbl_tbl; 604 int rc; 605 606 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 607 return 0; 608 609 memset(&mrinfo, 0, sizeof(mrinfo)); 610 /* Allocate a small chunk of memory and dma-map it */ 611 fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL); 612 if (!fence->va) 613 return -ENOMEM; 614 dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va, 615 BNXT_RE_LEGACY_FENCE_BYTES, 616 DMA_BIDIRECTIONAL); 617 rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr); 618 if (rc) { 619 dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); 620 rc = -EIO; 621 fence->dma_addr = 0; 622 goto free_va; 623 } 624 fence->dma_addr = dma_addr; 625 626 /* Allocate a MR */ 627 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 628 if (!mr) 629 goto free_dma_addr; 630 fence->mr = mr; 631 mr->rdev = rdev; 632 mr->qplib_mr.pd = &pd->qplib_pd; 633 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 634 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 635 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { 636 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 637 if (rc) { 638 dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); 639 goto free_mr; 640 } 641 /* Register MR */ 642 mr->ib_mr.lkey = mr->qplib_mr.lkey; 643 } 644 mr->qplib_mr.va = (u64)fence->va; 645 mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES; 646 pbl_tbl = dma_addr; 647 648 mrinfo.mrw = &mr->qplib_mr; 649 mrinfo.ptes = &pbl_tbl; 650 mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE; 651 652 mrinfo.sg.nmap = 0; 653 mrinfo.sg.sghead = 0; 654 mrinfo.sg.pgshft = PAGE_SHIFT; 655 mrinfo.sg.pgsize = PAGE_SIZE; 656 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 657 if (rc) { 658 dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); 659 goto free_mr; 660 } 661 mr->ib_mr.lkey = mr->qplib_mr.lkey; 662 mr->ib_mr.rkey = mr->qplib_mr.rkey; 663 atomic_inc(&rdev->stats.rsors.mr_count); 664 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 665 if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count))) 666 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 667 668 ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL); 669 /* Create a fence MW only for kernel consumers */ 670 if (!ib_mw) { 671 dev_err(rdev_to_dev(rdev), 672 "Failed to create fence-MW for PD: %p\n", pd); 673 rc = -EINVAL; 674 goto free_mr; 675 } 676 fence->mw = ib_mw; 677 678 bnxt_re_legacy_create_fence_wqe(pd); 679 return 0; 680 681 free_mr: 682 if (mr->ib_mr.lkey) { 683 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 684 atomic_dec(&rdev->stats.rsors.mr_count); 685 } 686 kfree(mr); 687 fence->mr = NULL; 688 689 free_dma_addr: 690 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, 691 BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); 692 fence->dma_addr = 0; 693 694 free_va: 695 kfree(fence->va); 696 fence->va = NULL; 697 return rc; 698 } 699 700 static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd) 701 { 702 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 703 struct bnxt_re_dev *rdev = pd->rdev; 704 struct bnxt_re_mr *mr = fence->mr; 705 706 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 707 return; 708 709 if (fence->mw) { 710 bnxt_re_dealloc_mw(fence->mw); 711 fence->mw = NULL; 712 } 713 if (mr) { 714 if (mr->ib_mr.rkey) 715 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 716 false); 717 if (mr->ib_mr.lkey) 718 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 719 kfree(mr); 720 fence->mr = NULL; 721 atomic_dec(&rdev->stats.rsors.mr_count); 722 } 723 if (fence->dma_addr) { 724 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, 725 BNXT_RE_LEGACY_FENCE_BYTES, 726 DMA_BIDIRECTIONAL); 727 fence->dma_addr = 0; 728 } 729 kfree(fence->va); 730 fence->va = NULL; 731 } 732 733 734 static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev, 735 struct bnxt_re_ucontext *cntx) 736 { 737 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; 738 int ret = 0; 739 u8 type; 740 /* Allocate DPI in alloc_pd or in create_cq to avoid failing of 741 * ibv_devinfo and family of application when DPIs are depleted. 742 */ 743 type = BNXT_QPLIB_DPI_TYPE_UC; 744 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type); 745 if (ret) { 746 dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n"); 747 goto out; 748 } 749 750 if (cctx->modes.db_push) { 751 type = BNXT_QPLIB_DPI_TYPE_WC; 752 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi, 753 cntx, type); 754 if (ret) 755 dev_err(rdev_to_dev(rdev), "push dp alloc failed\n"); 756 } 757 out: 758 return ret; 759 } 760 761 /* Protection Domains */ 762 void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 763 { 764 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 765 struct bnxt_re_dev *rdev = pd->rdev; 766 int rc; 767 768 bnxt_re_legacy_destroy_fence_mr(pd); 769 770 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, 771 &rdev->qplib_res.pd_tbl, 772 &pd->qplib_pd); 773 if (rc) 774 dev_err_ratelimited(rdev_to_dev(rdev), 775 "%s failed rc = %d\n", __func__, rc); 776 atomic_dec(&rdev->stats.rsors.pd_count); 777 778 return; 779 } 780 781 int bnxt_re_alloc_pd(struct ib_pd *pd_in, 782 struct ib_udata *udata) 783 { 784 struct ib_pd *ibpd = pd_in; 785 struct ib_device *ibdev = ibpd->device; 786 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 787 struct bnxt_re_ucontext *ucntx = 788 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, 789 ibucontext); 790 u32 max_pd_count; 791 int rc; 792 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd); 793 794 pd->rdev = rdev; 795 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { 796 dev_err(rdev_to_dev(rdev), 797 "Allocate HW Protection Domain failed!\n"); 798 rc = -ENOMEM; 799 goto fail; 800 } 801 802 if (udata) { 803 struct bnxt_re_pd_resp resp = {}; 804 805 if (!ucntx->dpi.dbr) { 806 rc = bnxt_re_get_user_dpi(rdev, ucntx); 807 if (rc) 808 goto dbfail; 809 } 810 811 resp.pdid = pd->qplib_pd.id; 812 /* Still allow mapping this DBR to the new user PD. */ 813 resp.dpi = ucntx->dpi.dpi; 814 resp.dbr = (u64)ucntx->dpi.umdbr; 815 /* Copy only on a valid wcpdi */ 816 if (ucntx->wcdpi.dpi) { 817 resp.wcdpi = ucntx->wcdpi.dpi; 818 resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI; 819 } 820 if (rdev->dbr_pacing) { 821 WARN_ON(!rdev->dbr_bar_addr); 822 resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr; 823 resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR; 824 } 825 826 rc = bnxt_re_copy_to_udata(rdev, &resp, 827 min(udata->outlen, sizeof(resp)), 828 udata); 829 if (rc) 830 goto dbfail; 831 } 832 833 if (!udata) 834 if (bnxt_re_legacy_create_fence_mr(pd)) 835 dev_warn(rdev_to_dev(rdev), 836 "Failed to create Fence-MR\n"); 837 838 atomic_inc(&rdev->stats.rsors.pd_count); 839 max_pd_count = atomic_read(&rdev->stats.rsors.pd_count); 840 if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count)) 841 atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count); 842 843 return 0; 844 dbfail: 845 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 846 &pd->qplib_pd); 847 fail: 848 return rc; 849 } 850 851 /* Address Handles */ 852 void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 853 { 854 struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); 855 struct bnxt_re_dev *rdev = ah->rdev; 856 int rc = 0; 857 bool block = true; 858 859 block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); 860 861 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); 862 if (rc) 863 dev_err_ratelimited(rdev_to_dev(rdev), 864 "%s id = %d blocking %d failed rc = %d\n", 865 __func__, ah->qplib_ah.id, block, rc); 866 atomic_dec(&rdev->stats.rsors.ah_count); 867 868 return; 869 } 870 871 static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype) 872 { 873 u8 nw_type; 874 switch (ntype) { 875 case RDMA_NETWORK_IPV4: 876 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 877 break; 878 case RDMA_NETWORK_IPV6: 879 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 880 break; 881 default: 882 nw_type = CMDQ_CREATE_AH_TYPE_V1; 883 break; 884 } 885 return nw_type; 886 } 887 888 static inline int 889 bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, 890 union ib_gid *sgid, struct ib_gid_attr **sgid_attr, 891 struct ib_global_route *grh, struct ib_ah *ah) 892 { 893 int ret = 0; 894 895 ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr); 896 return ret; 897 } 898 899 static inline enum rdma_network_type 900 bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, 901 union ib_gid *sgid) 902 { 903 return ib_gid_to_network_type(sgid_attr->gid_type, sgid); 904 } 905 906 static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev, 907 struct ib_ah_attr *ah_attr, 908 struct bnxt_re_ah_info *ah_info) 909 { 910 struct ib_gid_attr *gattr; 911 enum rdma_network_type ib_ntype; 912 u8 ntype; 913 union ib_gid *gid; 914 int rc = 0; 915 916 gid = &ah_info->sgid; 917 gattr = &ah_info->sgid_attr; 918 919 rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index, 920 gid, &gattr, &ah_attr->grh, NULL); 921 if (rc) 922 return rc; 923 924 /* Get vlan tag */ 925 if (gattr->ndev) { 926 if (is_vlan_dev(gattr->ndev)) 927 ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev); 928 if_rele(gattr->ndev); 929 } 930 931 /* Get network header type for this GID */ 932 933 ib_ntype = bnxt_re_gid_to_network_type(gattr, gid); 934 ntype = _to_bnxt_re_nw_type(ib_ntype); 935 ah_info->nw_type = ntype; 936 937 return rc; 938 } 939 940 static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx) 941 { 942 gindx = rdev->gid_map[gindx]; 943 return gindx; 944 } 945 946 static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr, 947 struct bnxt_re_ah_info *ah_info, bool is_user, 948 struct bnxt_re_ah *ah) 949 { 950 int rc = 0; 951 u8 *dmac; 952 953 if (is_user && !rdma_is_multicast_addr((struct in6_addr *) 954 ah_attr->grh.dgid.raw) && 955 !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) { 956 957 u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US; 958 struct bnxt_re_resolve_dmac_work *resolve_dmac_work; 959 960 961 resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC); 962 963 resolve_dmac_work->rdev = rdev; 964 resolve_dmac_work->ah_attr = ah_attr; 965 resolve_dmac_work->ah_info = ah_info; 966 967 atomic_set(&resolve_dmac_work->status_wait, 1); 968 INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task); 969 queue_work(rdev->resolve_wq, &resolve_dmac_work->work); 970 971 do { 972 rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF; 973 if (!rc) 974 break; 975 udelay(1); 976 } while (--retry_count); 977 if (atomic_read(&resolve_dmac_work->status_wait)) { 978 INIT_LIST_HEAD(&resolve_dmac_work->list); 979 list_add_tail(&resolve_dmac_work->list, 980 &rdev->mac_wq_list); 981 return -EFAULT; 982 } 983 kfree(resolve_dmac_work); 984 } 985 dmac = ROCE_DMAC(ah_attr); 986 if (dmac) 987 memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN); 988 return rc; 989 } 990 991 int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, 992 u32 flags, struct ib_udata *udata) 993 { 994 995 struct ib_ah *ib_ah = ah_in; 996 struct ib_pd *ib_pd = ib_ah->pd; 997 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah); 998 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd); 999 struct bnxt_re_dev *rdev = pd->rdev; 1000 struct bnxt_re_ah_info ah_info; 1001 u32 max_ah_count; 1002 bool is_user; 1003 int rc; 1004 bool block = true; 1005 struct ib_ah_attr *ah_attr = attr; 1006 block = !(flags & RDMA_CREATE_AH_SLEEPABLE); 1007 1008 if (!(ah_attr->ah_flags & IB_AH_GRH)) 1009 dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n"); 1010 1011 ah->rdev = rdev; 1012 ah->qplib_ah.pd = &pd->qplib_pd; 1013 is_user = ib_pd->uobject ? true : false; 1014 1015 /* Supply the configuration for the HW */ 1016 memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, 1017 sizeof(union ib_gid)); 1018 ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index); 1019 if (ah->qplib_ah.sgid_index == 0xFF) { 1020 dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n"); 1021 rc = -EINVAL; 1022 goto fail; 1023 } 1024 ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; 1025 ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; 1026 ah->qplib_ah.flow_label = ah_attr->grh.flow_label; 1027 ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; 1028 ah->qplib_ah.sl = ah_attr->sl; 1029 rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info); 1030 if (rc) 1031 goto fail; 1032 ah->qplib_ah.nw_type = ah_info.nw_type; 1033 1034 rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah); 1035 if (rc) 1036 goto fail; 1037 1038 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block); 1039 if (rc) { 1040 dev_err(rdev_to_dev(rdev), 1041 "Allocate HW Address Handle failed!\n"); 1042 goto fail; 1043 } 1044 1045 /* Write AVID to shared page. */ 1046 if (ib_pd->uobject) { 1047 struct ib_ucontext *ib_uctx = ib_pd->uobject->context; 1048 struct bnxt_re_ucontext *uctx; 1049 unsigned long flag; 1050 u32 *wrptr; 1051 1052 uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); 1053 spin_lock_irqsave(&uctx->sh_lock, flag); 1054 wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT); 1055 *wrptr = ah->qplib_ah.id; 1056 wmb(); /* make sure cache is updated. */ 1057 spin_unlock_irqrestore(&uctx->sh_lock, flag); 1058 } 1059 atomic_inc(&rdev->stats.rsors.ah_count); 1060 max_ah_count = atomic_read(&rdev->stats.rsors.ah_count); 1061 if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count)) 1062 atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count); 1063 1064 return 0; 1065 fail: 1066 return rc; 1067 } 1068 1069 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) 1070 { 1071 return 0; 1072 } 1073 1074 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) 1075 { 1076 struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); 1077 1078 memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, 1079 sizeof(union ib_gid)); 1080 ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; 1081 ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; 1082 ah_attr->sl = ah->qplib_ah.sl; 1083 memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN); 1084 ah_attr->ah_flags = IB_AH_GRH; 1085 ah_attr->port_num = 1; 1086 ah_attr->static_rate = 0; 1087 1088 return 0; 1089 } 1090 1091 /* Shared Receive Queues */ 1092 void bnxt_re_destroy_srq(struct ib_srq *ib_srq, 1093 struct ib_udata *udata) 1094 { 1095 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); 1096 struct bnxt_re_dev *rdev = srq->rdev; 1097 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1098 int rc = 0; 1099 1100 1101 rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1102 if (rc) 1103 dev_err_ratelimited(rdev_to_dev(rdev), 1104 "%s id = %d failed rc = %d\n", 1105 __func__, qplib_srq->id, rc); 1106 1107 if (srq->umem && !IS_ERR(srq->umem)) 1108 ib_umem_release(srq->umem); 1109 1110 atomic_dec(&rdev->stats.rsors.srq_count); 1111 1112 return; 1113 } 1114 1115 static u16 _max_rwqe_sz(int nsge) 1116 { 1117 return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge)); 1118 } 1119 1120 static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp, 1121 int rsge, int max) 1122 { 1123 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1124 rsge = max; 1125 1126 return _max_rwqe_sz(rsge); 1127 } 1128 1129 static inline 1130 struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, 1131 struct ib_ucontext *ucontext, 1132 struct ib_udata *udata, 1133 unsigned long addr, 1134 size_t size, int access, int dmasync) 1135 { 1136 return ib_umem_get(ucontext, addr, size, access, dmasync); 1137 } 1138 1139 static inline 1140 struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, 1141 struct ib_ucontext *ucontext, 1142 struct ib_udata *udata, 1143 unsigned long addr, 1144 size_t size, int access, int dmasync) 1145 { 1146 return ib_umem_get_compat(rdev, ucontext, udata, addr, size, 1147 access, 0); 1148 } 1149 1150 static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem) 1151 { 1152 return ib_umem_num_pages(umem); 1153 } 1154 1155 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1156 struct bnxt_re_pd *pd, 1157 struct bnxt_re_srq *srq, 1158 struct ib_udata *udata) 1159 { 1160 struct bnxt_qplib_sg_info *sginfo; 1161 struct bnxt_qplib_srq *qplib_srq; 1162 struct bnxt_re_ucontext *cntx; 1163 struct ib_ucontext *context; 1164 struct bnxt_re_srq_req ureq; 1165 struct ib_umem *umem; 1166 int rc, bytes = 0; 1167 1168 context = pd->ibpd.uobject->context; 1169 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 1170 qplib_srq = &srq->qplib_srq; 1171 sginfo = &qplib_srq->sginfo; 1172 1173 if (udata->inlen < sizeof(ureq)) 1174 dev_warn(rdev_to_dev(rdev), 1175 "Update the library ulen %d klen %d\n", 1176 (unsigned int)udata->inlen, 1177 (unsigned int)sizeof(ureq)); 1178 1179 rc = ib_copy_from_udata(&ureq, udata, 1180 min(udata->inlen, sizeof(ureq))); 1181 if (rc) 1182 return rc; 1183 1184 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); 1185 bytes = PAGE_ALIGN(bytes); 1186 umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes, 1187 IB_ACCESS_LOCAL_WRITE, 1); 1188 if (IS_ERR(umem)) { 1189 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", 1190 __func__, PTR_ERR(umem)); 1191 return PTR_ERR(umem); 1192 } 1193 1194 srq->umem = umem; 1195 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); 1196 sginfo->npages = ib_umem_num_pages_compat(umem); 1197 qplib_srq->srq_handle = ureq.srq_handle; 1198 qplib_srq->dpi = &cntx->dpi; 1199 qplib_srq->is_user = true; 1200 1201 return 0; 1202 } 1203 1204 int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr, 1205 struct ib_udata *udata) 1206 { 1207 struct bnxt_qplib_dev_attr *dev_attr; 1208 struct bnxt_re_ucontext *cntx = NULL; 1209 struct ib_ucontext *context; 1210 struct bnxt_re_dev *rdev; 1211 struct bnxt_re_pd *pd; 1212 int rc, entries; 1213 struct ib_srq *ib_srq = srq_in; 1214 struct ib_pd *ib_pd = ib_srq->pd; 1215 struct bnxt_re_srq *srq = 1216 container_of(ib_srq, struct bnxt_re_srq, ibsrq); 1217 u32 max_srq_count; 1218 1219 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 1220 rdev = pd->rdev; 1221 dev_attr = rdev->dev_attr; 1222 1223 if (rdev->mod_exit) { 1224 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 1225 rc = -EIO; 1226 goto exit; 1227 } 1228 1229 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1230 dev_err(rdev_to_dev(rdev), "SRQ type not supported\n"); 1231 rc = -ENOTSUPP; 1232 goto exit; 1233 } 1234 1235 if (udata) { 1236 context = pd->ibpd.uobject->context; 1237 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 1238 } 1239 1240 if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) { 1241 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n"); 1242 rc = -EINVAL; 1243 goto exit; 1244 } 1245 1246 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1247 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n"); 1248 rc = -EINVAL; 1249 goto exit; 1250 } 1251 1252 srq->rdev = rdev; 1253 srq->qplib_srq.pd = &pd->qplib_pd; 1254 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1255 1256 /* Allocate 1 more than what's provided so posting max doesn't 1257 mean empty */ 1258 entries = srq_init_attr->attr.max_wr + 1; 1259 entries = bnxt_re_init_depth(entries, cntx); 1260 if (entries > dev_attr->max_srq_wqes + 1) 1261 entries = dev_attr->max_srq_wqes + 1; 1262 1263 srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */ 1264 srq->qplib_srq.max_wqe = entries; 1265 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1266 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1267 srq->srq_limit = srq_init_attr->attr.srq_limit; 1268 srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id; 1269 srq->qplib_srq.sginfo.pgsize = PAGE_SIZE; 1270 srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT; 1271 1272 if (udata) { 1273 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1274 if (rc) 1275 goto fail; 1276 } 1277 1278 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1279 if (rc) { 1280 dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n"); 1281 goto fail; 1282 } 1283 1284 if (udata) { 1285 struct bnxt_re_srq_resp resp; 1286 1287 resp.srqid = srq->qplib_srq.id; 1288 rc = bnxt_re_copy_to_udata(rdev, &resp, 1289 min(udata->outlen, sizeof(resp)), 1290 udata); 1291 if (rc) { 1292 bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); 1293 goto fail; 1294 } 1295 } 1296 atomic_inc(&rdev->stats.rsors.srq_count); 1297 max_srq_count = atomic_read(&rdev->stats.rsors.srq_count); 1298 if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count)) 1299 atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count); 1300 spin_lock_init(&srq->lock); 1301 1302 return 0; 1303 fail: 1304 if (udata && srq->umem && !IS_ERR(srq->umem)) { 1305 ib_umem_release(srq->umem); 1306 srq->umem = NULL; 1307 } 1308 exit: 1309 return rc; 1310 } 1311 1312 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1313 enum ib_srq_attr_mask srq_attr_mask, 1314 struct ib_udata *udata) 1315 { 1316 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, 1317 ibsrq); 1318 struct bnxt_re_dev *rdev = srq->rdev; 1319 int rc; 1320 1321 switch (srq_attr_mask) { 1322 case IB_SRQ_MAX_WR: 1323 /* SRQ resize is not supported */ 1324 break; 1325 case IB_SRQ_LIMIT: 1326 /* Change the SRQ threshold */ 1327 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1328 return -EINVAL; 1329 1330 srq->qplib_srq.threshold = srq_attr->srq_limit; 1331 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1332 if (rc) { 1333 dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n"); 1334 return rc; 1335 } 1336 /* On success, update the shadow */ 1337 srq->srq_limit = srq_attr->srq_limit; 1338 1339 if (udata) { 1340 /* Build and send response back to udata */ 1341 rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata); 1342 if (rc) 1343 return rc; 1344 } 1345 break; 1346 default: 1347 dev_err(rdev_to_dev(rdev), 1348 "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask); 1349 return -EINVAL; 1350 } 1351 return 0; 1352 } 1353 1354 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1355 { 1356 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, 1357 ibsrq); 1358 struct bnxt_re_dev *rdev = srq->rdev; 1359 int rc; 1360 1361 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq); 1362 if (rc) { 1363 dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n", 1364 srq->qplib_srq.id, rc); 1365 return rc; 1366 } 1367 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1368 srq_attr->max_sge = srq->qplib_srq.max_sge; 1369 srq_attr->srq_limit = srq->qplib_srq.threshold; 1370 1371 return 0; 1372 } 1373 1374 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1375 const struct ib_recv_wr **bad_wr) 1376 { 1377 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, 1378 ibsrq); 1379 struct bnxt_qplib_swqe wqe = {}; 1380 unsigned long flags; 1381 int rc = 0; 1382 1383 spin_lock_irqsave(&srq->lock, flags); 1384 while (wr) { 1385 /* Transcribe each ib_recv_wr to qplib_swqe */ 1386 wqe.num_sge = wr->num_sge; 1387 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 1388 wqe.wr_id = wr->wr_id; 1389 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1390 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1391 if (rc) { 1392 *bad_wr = wr; 1393 break; 1394 } 1395 wr = wr->next; 1396 } 1397 spin_unlock_irqrestore(&srq->lock, flags); 1398 1399 return rc; 1400 } 1401 1402 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 1403 { 1404 unsigned long flags; 1405 1406 spin_lock_irqsave(&qp->scq->cq_lock, flags); 1407 if (qp->rcq && qp->rcq != qp->scq) 1408 spin_lock(&qp->rcq->cq_lock); 1409 1410 return flags; 1411 } 1412 1413 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 1414 unsigned long flags) 1415 { 1416 if (qp->rcq && qp->rcq != qp->scq) 1417 spin_unlock(&qp->rcq->cq_lock); 1418 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 1419 } 1420 1421 /* Queue Pairs */ 1422 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) 1423 { 1424 struct bnxt_re_qp *gsi_sqp; 1425 struct bnxt_re_ah *gsi_sah; 1426 struct bnxt_re_dev *rdev; 1427 unsigned long flags; 1428 int rc = 0; 1429 1430 rdev = qp->rdev; 1431 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 1432 gsi_sah = rdev->gsi_ctx.gsi_sah; 1433 1434 /* remove from active qp list */ 1435 mutex_lock(&rdev->qp_lock); 1436 list_del(&gsi_sqp->list); 1437 mutex_unlock(&rdev->qp_lock); 1438 1439 if (gsi_sah) { 1440 dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n"); 1441 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, 1442 true); 1443 if (rc) 1444 dev_err(rdev_to_dev(rdev), 1445 "Destroy HW AH for shadow QP failed!\n"); 1446 atomic_dec(&rdev->stats.rsors.ah_count); 1447 } 1448 1449 dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n"); 1450 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); 1451 if (rc) 1452 dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n"); 1453 1454 /* Clean the CQ for shadow QP completions */ 1455 flags = bnxt_re_lock_cqs(gsi_sqp); 1456 bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp); 1457 bnxt_re_unlock_cqs(gsi_sqp, flags); 1458 1459 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); 1460 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp); 1461 kfree(rdev->gsi_ctx.sqp_tbl); 1462 kfree(gsi_sah); 1463 kfree(gsi_sqp); 1464 rdev->gsi_ctx.gsi_sqp = NULL; 1465 rdev->gsi_ctx.gsi_sah = NULL; 1466 rdev->gsi_ctx.sqp_tbl = NULL; 1467 atomic_dec(&rdev->stats.rsors.qp_count); 1468 1469 return 0; 1470 } 1471 1472 static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps) 1473 { 1474 u32 total_qp = 0; 1475 u64 avg_time = 0; 1476 int i; 1477 1478 if (!rdev->rcfw.sp_perf_stats_enabled) 1479 return; 1480 1481 switch (active_qps) { 1482 case 1: 1483 /* Potential hint for Test Stop */ 1484 for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) { 1485 if (rdev->rcfw.qp_destroy_stats[i]) { 1486 total_qp++; 1487 avg_time += rdev->rcfw.qp_destroy_stats[i]; 1488 } 1489 } 1490 if (total_qp >= 0 || avg_time >= 0) 1491 dev_dbg(rdev_to_dev(rdev), 1492 "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n", 1493 __builtin_return_address(0), total_qp, 1494 (long)jiffies_to_msecs(avg_time)); 1495 break; 1496 case 2: 1497 /* Potential hint for Test Start */ 1498 dev_dbg(rdev_to_dev(rdev), 1499 "Perf Debug: %ps active_qps = %d\n", 1500 __builtin_return_address(0), active_qps); 1501 break; 1502 default: 1503 /* Potential hint to know latency of QP destroy. 1504 * Average time taken for 1K QP Destroy. 1505 */ 1506 if (active_qps > 1024 && !(active_qps % 1024)) 1507 dev_dbg(rdev_to_dev(rdev), 1508 "Perf Debug: %ps Active QP (%d) Watermark (%d)\n", 1509 __builtin_return_address(0), active_qps, 1510 atomic_read(&rdev->stats.rsors.max_qp_count)); 1511 break; 1512 } 1513 } 1514 1515 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) 1516 { 1517 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 1518 struct bnxt_re_dev *rdev = qp->rdev; 1519 unsigned long flags; 1520 u32 active_qps; 1521 int rc; 1522 1523 mutex_lock(&rdev->qp_lock); 1524 list_del(&qp->list); 1525 active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count); 1526 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) 1527 atomic_dec(&rdev->stats.rsors.rc_qp_count); 1528 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) 1529 atomic_dec(&rdev->stats.rsors.ud_qp_count); 1530 mutex_unlock(&rdev->qp_lock); 1531 1532 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1533 if (rc) 1534 dev_err_ratelimited(rdev_to_dev(rdev), 1535 "%s id = %d failed rc = %d\n", 1536 __func__, qp->qplib_qp.id, rc); 1537 1538 if (!ib_qp->uobject) { 1539 flags = bnxt_re_lock_cqs(qp); 1540 bnxt_qplib_clean_qp(&qp->qplib_qp); 1541 bnxt_re_unlock_cqs(qp, flags); 1542 } 1543 1544 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 1545 if (ib_qp->qp_type == IB_QPT_GSI && 1546 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 1547 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && 1548 rdev->gsi_ctx.gsi_sqp) { 1549 bnxt_re_destroy_gsi_sqp(qp); 1550 } 1551 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp); 1552 } 1553 1554 if (qp->rumem && !IS_ERR(qp->rumem)) 1555 ib_umem_release(qp->rumem); 1556 if (qp->sumem && !IS_ERR(qp->sumem)) 1557 ib_umem_release(qp->sumem); 1558 kfree(qp); 1559 1560 bnxt_re_dump_debug_stats(rdev, active_qps); 1561 1562 return 0; 1563 } 1564 1565 static u8 __from_ib_qp_type(enum ib_qp_type type) 1566 { 1567 switch (type) { 1568 case IB_QPT_GSI: 1569 return CMDQ_CREATE_QP1_TYPE_GSI; 1570 case IB_QPT_RC: 1571 return CMDQ_CREATE_QP_TYPE_RC; 1572 case IB_QPT_UD: 1573 return CMDQ_CREATE_QP_TYPE_UD; 1574 case IB_QPT_RAW_ETHERTYPE: 1575 return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE; 1576 default: 1577 return IB_QPT_MAX; 1578 } 1579 } 1580 1581 static u16 _get_swqe_sz(int nsge) 1582 { 1583 return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); 1584 } 1585 1586 static int bnxt_re_get_swqe_size(int ilsize, int nsge) 1587 { 1588 u16 wqe_size, calc_ils; 1589 1590 wqe_size = _get_swqe_sz(nsge); 1591 if (ilsize) { 1592 calc_ils = (sizeof(struct sq_send_hdr) + ilsize); 1593 wqe_size = max_t(int, calc_ils, wqe_size); 1594 wqe_size = ALIGN(wqe_size, 32); 1595 } 1596 return wqe_size; 1597 } 1598 1599 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, 1600 struct ib_qp_init_attr *init_attr) 1601 { 1602 struct bnxt_qplib_dev_attr *dev_attr; 1603 struct bnxt_qplib_qp *qplqp; 1604 struct bnxt_re_dev *rdev; 1605 struct bnxt_qplib_q *sq; 1606 int align, ilsize; 1607 1608 rdev = qp->rdev; 1609 qplqp = &qp->qplib_qp; 1610 sq = &qplqp->sq; 1611 dev_attr = rdev->dev_attr; 1612 1613 align = sizeof(struct sq_send_hdr); 1614 ilsize = ALIGN(init_attr->cap.max_inline_data, align); 1615 1616 sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); 1617 if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) 1618 return -EINVAL; 1619 /* For Cu/Wh and gen p5 backward compatibility mode 1620 * wqe size is fixed to 128 bytes 1621 */ 1622 if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) && 1623 qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1624 sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges); 1625 1626 if (init_attr->cap.max_inline_data) { 1627 qplqp->max_inline_data = sq->wqe_size - 1628 sizeof(struct sq_send_hdr); 1629 init_attr->cap.max_inline_data = qplqp->max_inline_data; 1630 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1631 sq->max_sge = qplqp->max_inline_data / 1632 sizeof(struct sq_sge); 1633 } 1634 1635 return 0; 1636 } 1637 1638 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, 1639 struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, 1640 struct ib_udata *udata) 1641 { 1642 struct bnxt_qplib_sg_info *sginfo; 1643 struct bnxt_qplib_qp *qplib_qp; 1644 struct bnxt_re_ucontext *cntx; 1645 struct ib_ucontext *context; 1646 struct bnxt_re_qp_req ureq; 1647 struct ib_umem *umem; 1648 int rc, bytes = 0; 1649 int psn_nume; 1650 int psn_sz; 1651 1652 qplib_qp = &qp->qplib_qp; 1653 context = pd->ibpd.uobject->context; 1654 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 1655 sginfo = &qplib_qp->sq.sginfo; 1656 1657 if (udata->inlen < sizeof(ureq)) 1658 dev_warn(rdev_to_dev(rdev), 1659 "Update the library ulen %d klen %d\n", 1660 (unsigned int)udata->inlen, 1661 (unsigned int)sizeof(ureq)); 1662 1663 rc = ib_copy_from_udata(&ureq, udata, 1664 min(udata->inlen, sizeof(ureq))); 1665 if (rc) 1666 return rc; 1667 1668 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); 1669 /* Consider mapping PSN search memory only for RC QPs. */ 1670 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { 1671 psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? 1672 sizeof(struct sq_psn_search_ext) : 1673 sizeof(struct sq_psn_search); 1674 if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) 1675 psn_sz = sizeof(struct sq_msn_search); 1676 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 1677 qplib_qp->sq.max_wqe : 1678 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / 1679 sizeof(struct bnxt_qplib_sge)); 1680 if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) 1681 psn_nume = roundup_pow_of_two(psn_nume); 1682 1683 bytes += (psn_nume * psn_sz); 1684 } 1685 bytes = PAGE_ALIGN(bytes); 1686 umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, 1687 IB_ACCESS_LOCAL_WRITE, 1); 1688 if (IS_ERR(umem)) { 1689 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", 1690 __func__, PTR_ERR(umem)); 1691 return PTR_ERR(umem); 1692 } 1693 1694 qp->sumem = umem; 1695 /* pgsize and pgshft were initialize already. */ 1696 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); 1697 sginfo->npages = ib_umem_num_pages_compat(umem); 1698 qplib_qp->qp_handle = ureq.qp_handle; 1699 1700 if (!qp->qplib_qp.srq) { 1701 sginfo = &qplib_qp->rq.sginfo; 1702 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); 1703 bytes = PAGE_ALIGN(bytes); 1704 umem = ib_umem_get_compat(rdev, 1705 context, udata, ureq.qprva, bytes, 1706 IB_ACCESS_LOCAL_WRITE, 1); 1707 if (IS_ERR(umem)) { 1708 dev_err(rdev_to_dev(rdev), 1709 "%s: ib_umem_get failed ret =%ld\n", 1710 __func__, PTR_ERR(umem)); 1711 goto rqfail; 1712 } 1713 qp->rumem = umem; 1714 /* pgsize and pgshft were initialize already. */ 1715 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); 1716 sginfo->npages = ib_umem_num_pages_compat(umem); 1717 } 1718 1719 qplib_qp->dpi = &cntx->dpi; 1720 qplib_qp->is_user = true; 1721 1722 return 0; 1723 rqfail: 1724 ib_umem_release(qp->sumem); 1725 qp->sumem = NULL; 1726 qplib_qp->sq.sginfo.sghead = NULL; 1727 qplib_qp->sq.sginfo.nmap = 0; 1728 1729 return PTR_ERR(umem); 1730 } 1731 1732 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd, 1733 struct bnxt_qplib_res *qp1_res, 1734 struct bnxt_qplib_qp *qp1_qp) 1735 { 1736 struct bnxt_re_dev *rdev = pd->rdev; 1737 struct bnxt_re_ah *ah; 1738 union ib_gid sgid; 1739 int rc; 1740 1741 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 1742 if (!ah) { 1743 dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n"); 1744 return NULL; 1745 } 1746 memset(ah, 0, sizeof(*ah)); 1747 ah->rdev = rdev; 1748 ah->qplib_ah.pd = &pd->qplib_pd; 1749 1750 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 1751 if (rc) 1752 goto fail; 1753 1754 /* supply the dgid data same as sgid */ 1755 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 1756 sizeof(union ib_gid)); 1757 ah->qplib_ah.sgid_index = 0; 1758 1759 ah->qplib_ah.traffic_class = 0; 1760 ah->qplib_ah.flow_label = 0; 1761 ah->qplib_ah.hop_limit = 1; 1762 ah->qplib_ah.sl = 0; 1763 /* Have DMAC same as SMAC */ 1764 ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr); 1765 dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n", 1766 ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2], 1767 ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]); 1768 1769 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true); 1770 if (rc) { 1771 dev_err(rdev_to_dev(rdev), 1772 "Allocate HW AH for Shadow QP failed!\n"); 1773 goto fail; 1774 } 1775 dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id); 1776 atomic_inc(&rdev->stats.rsors.ah_count); 1777 1778 return ah; 1779 fail: 1780 kfree(ah); 1781 return NULL; 1782 } 1783 1784 void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev) 1785 { 1786 struct bnxt_re_qp *gsi_qp; 1787 struct bnxt_re_ah *sah; 1788 struct bnxt_re_pd *pd; 1789 struct ib_pd *ib_pd; 1790 int rc; 1791 1792 if (!rdev) 1793 return; 1794 1795 sah = rdev->gsi_ctx.gsi_sah; 1796 1797 dev_dbg(rdev_to_dev(rdev), "Updating the AH\n"); 1798 if (sah) { 1799 /* Check if the AH created with current mac address */ 1800 if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) { 1801 dev_dbg(rdev_to_dev(rdev), 1802 "Not modifying shadow AH during AH update\n"); 1803 return; 1804 } 1805 1806 gsi_qp = rdev->gsi_ctx.gsi_qp; 1807 ib_pd = gsi_qp->ib_qp.pd; 1808 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 1809 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, 1810 &sah->qplib_ah, false); 1811 if (rc) { 1812 dev_err(rdev_to_dev(rdev), 1813 "Failed to destroy shadow AH during AH update\n"); 1814 return; 1815 } 1816 atomic_dec(&rdev->stats.rsors.ah_count); 1817 kfree(sah); 1818 rdev->gsi_ctx.gsi_sah = NULL; 1819 1820 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1821 &gsi_qp->qplib_qp); 1822 if (!sah) { 1823 dev_err(rdev_to_dev(rdev), 1824 "Failed to update AH for ShadowQP\n"); 1825 return; 1826 } 1827 rdev->gsi_ctx.gsi_sah = sah; 1828 atomic_inc(&rdev->stats.rsors.ah_count); 1829 } 1830 } 1831 1832 static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, 1833 struct bnxt_qplib_res *qp1_res, 1834 struct bnxt_qplib_qp *qp1_qp) 1835 { 1836 struct bnxt_re_dev *rdev = pd->rdev; 1837 struct bnxt_re_qp *qp; 1838 int rc; 1839 1840 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1841 if (!qp) { 1842 dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n"); 1843 return NULL; 1844 } 1845 memset(qp, 0, sizeof(*qp)); 1846 qp->rdev = rdev; 1847 1848 /* Initialize the shadow QP structure from the QP1 values */ 1849 ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr); 1850 qp->qplib_qp.pd = &pd->qplib_pd; 1851 qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp; 1852 qp->qplib_qp.type = IB_QPT_UD; 1853 1854 qp->qplib_qp.max_inline_data = 0; 1855 qp->qplib_qp.sig_type = true; 1856 1857 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 1858 qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); 1859 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 1860 qp->qplib_qp.sq.max_sge = 2; 1861 /* Q full delta can be 1 since it is internal QP */ 1862 qp->qplib_qp.sq.q_full_delta = 1; 1863 qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE; 1864 qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT; 1865 1866 qp->qplib_qp.scq = qp1_qp->scq; 1867 qp->qplib_qp.rcq = qp1_qp->rcq; 1868 1869 qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ 1870 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 1871 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 1872 qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; 1873 qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; 1874 /* Q full delta can be 1 since it is internal QP */ 1875 qp->qplib_qp.rq.q_full_delta = 1; 1876 qp->qplib_qp.mtu = qp1_qp->mtu; 1877 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1878 1879 rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0, 1880 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6); 1881 if (rc) 1882 goto fail; 1883 1884 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 1885 if (rc) { 1886 dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); 1887 goto qp_fail; 1888 } 1889 1890 dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n", 1891 qp->qplib_qp.id); 1892 spin_lock_init(&qp->sq_lock); 1893 INIT_LIST_HEAD(&qp->list); 1894 mutex_lock(&rdev->qp_lock); 1895 list_add_tail(&qp->list, &rdev->qp_list); 1896 atomic_inc(&rdev->stats.rsors.qp_count); 1897 mutex_unlock(&rdev->qp_lock); 1898 return qp; 1899 qp_fail: 1900 bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp); 1901 fail: 1902 kfree(qp); 1903 return NULL; 1904 } 1905 1906 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, 1907 struct ib_qp_init_attr *init_attr, void *cntx) 1908 { 1909 struct bnxt_qplib_dev_attr *dev_attr; 1910 struct bnxt_qplib_qp *qplqp; 1911 struct bnxt_re_dev *rdev; 1912 struct bnxt_qplib_q *rq; 1913 int entries; 1914 1915 rdev = qp->rdev; 1916 qplqp = &qp->qplib_qp; 1917 rq = &qplqp->rq; 1918 dev_attr = rdev->dev_attr; 1919 1920 if (init_attr->srq) { 1921 struct bnxt_re_srq *srq; 1922 1923 srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq); 1924 if (!srq) { 1925 dev_err(rdev_to_dev(rdev), "SRQ not found\n"); 1926 return -EINVAL; 1927 } 1928 qplqp->srq = &srq->qplib_srq; 1929 rq->max_wqe = 0; 1930 } else { 1931 rq->max_sge = init_attr->cap.max_recv_sge; 1932 if (rq->max_sge > dev_attr->max_qp_sges) 1933 rq->max_sge = dev_attr->max_qp_sges; 1934 init_attr->cap.max_recv_sge = rq->max_sge; 1935 rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge, 1936 dev_attr->max_qp_sges); 1937 1938 /* Allocate 1 more than what's provided so posting max doesn't 1939 mean empty */ 1940 entries = init_attr->cap.max_recv_wr + 1; 1941 entries = bnxt_re_init_depth(entries, cntx); 1942 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1943 rq->q_full_delta = 0; 1944 rq->sginfo.pgsize = PAGE_SIZE; 1945 rq->sginfo.pgshft = PAGE_SHIFT; 1946 } 1947 1948 return 0; 1949 } 1950 1951 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) 1952 { 1953 struct bnxt_qplib_dev_attr *dev_attr; 1954 struct bnxt_qplib_qp *qplqp; 1955 struct bnxt_re_dev *rdev; 1956 1957 rdev = qp->rdev; 1958 qplqp = &qp->qplib_qp; 1959 dev_attr = rdev->dev_attr; 1960 1961 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) 1962 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1963 } 1964 1965 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, 1966 struct ib_qp_init_attr *init_attr, 1967 void *cntx) 1968 { 1969 struct bnxt_qplib_dev_attr *dev_attr; 1970 struct bnxt_qplib_qp *qplqp; 1971 struct bnxt_re_dev *rdev; 1972 struct bnxt_qplib_q *sq; 1973 int diff = 0; 1974 int entries; 1975 int rc; 1976 1977 rdev = qp->rdev; 1978 qplqp = &qp->qplib_qp; 1979 sq = &qplqp->sq; 1980 dev_attr = rdev->dev_attr; 1981 1982 sq->max_sge = init_attr->cap.max_send_sge; 1983 if (sq->max_sge > dev_attr->max_qp_sges) { 1984 sq->max_sge = dev_attr->max_qp_sges; 1985 init_attr->cap.max_send_sge = sq->max_sge; 1986 } 1987 rc = bnxt_re_setup_swqe_size(qp, init_attr); 1988 if (rc) 1989 return rc; 1990 /* 1991 * Change the SQ depth if user has requested minimum using 1992 * configfs. Only supported for kernel consumers. Setting 1993 * min_tx_depth to 4096 to handle iser SQ full condition 1994 * in most of the newer OS distros 1995 */ 1996 entries = init_attr->cap.max_send_wr; 1997 if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { 1998 /* 1999 * If users specify any value greater than 1 use min_tx_depth 2000 * provided by user for comparison. Else, compare it with the 2001 * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. 2002 */ 2003 if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) 2004 entries = rdev->min_tx_depth; 2005 else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) 2006 entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; 2007 } 2008 diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); 2009 entries = bnxt_re_init_depth(entries + diff + 1, cntx); 2010 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); 2011 sq->q_full_delta = diff + 1; 2012 /* 2013 * Reserving one slot for Phantom WQE. Application can 2014 * post one extra entry in this case. But allowing this to avoid 2015 * unexpected Queue full condition 2016 */ 2017 sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */ 2018 sq->sginfo.pgsize = PAGE_SIZE; 2019 sq->sginfo.pgshft = PAGE_SHIFT; 2020 return 0; 2021 } 2022 2023 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, 2024 struct ib_qp_init_attr *init_attr, 2025 void *cntx) 2026 { 2027 struct bnxt_qplib_dev_attr *dev_attr; 2028 struct bnxt_qplib_qp *qplqp; 2029 struct bnxt_re_dev *rdev; 2030 int entries; 2031 2032 rdev = qp->rdev; 2033 qplqp = &qp->qplib_qp; 2034 dev_attr = rdev->dev_attr; 2035 2036 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 2037 entries = init_attr->cap.max_send_wr + 1; 2038 entries = bnxt_re_init_depth(entries, cntx); 2039 qplqp->sq.max_wqe = min_t(u32, entries, 2040 dev_attr->max_qp_wqes + 1); 2041 qplqp->sq.q_full_delta = qplqp->sq.max_wqe - 2042 init_attr->cap.max_send_wr; 2043 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ 2044 if (qplqp->sq.max_sge > dev_attr->max_qp_sges) 2045 qplqp->sq.max_sge = dev_attr->max_qp_sges; 2046 } 2047 } 2048 2049 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, 2050 struct ib_qp_init_attr *init_attr) 2051 { 2052 struct bnxt_qplib_chip_ctx *chip_ctx; 2053 struct bnxt_re_gsi_context *gsi_ctx; 2054 int qptype; 2055 2056 chip_ctx = rdev->chip_ctx; 2057 gsi_ctx = &rdev->gsi_ctx; 2058 2059 qptype = __from_ib_qp_type(init_attr->qp_type); 2060 if (qptype == IB_QPT_MAX) { 2061 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n", 2062 qptype); 2063 qptype = -EINVAL; 2064 goto out; 2065 } 2066 2067 if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) { 2068 /* For Thor always force UD mode. */ 2069 qptype = CMDQ_CREATE_QP_TYPE_GSI; 2070 gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD; 2071 } 2072 out: 2073 return qptype; 2074 } 2075 2076 static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev) 2077 { 2078 return rdev->chip_ctx->modes.wqe_mode; 2079 } 2080 2081 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 2082 struct ib_qp_init_attr *init_attr, 2083 struct ib_udata *udata) 2084 { 2085 struct bnxt_qplib_dev_attr *dev_attr; 2086 struct bnxt_re_ucontext *cntx = NULL; 2087 struct ib_ucontext *context; 2088 struct bnxt_qplib_qp *qplqp; 2089 struct bnxt_re_dev *rdev; 2090 struct bnxt_re_cq *cq; 2091 int rc = 0, qptype; 2092 2093 rdev = qp->rdev; 2094 qplqp = &qp->qplib_qp; 2095 dev_attr = rdev->dev_attr; 2096 2097 if (udata) { 2098 context = pd->ibpd.uobject->context; 2099 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 2100 } 2101 2102 /* Setup misc params */ 2103 qplqp->is_user = false; 2104 qplqp->pd = &pd->qplib_pd; 2105 qplqp->qp_handle = (u64)qplqp; 2106 qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 2107 true : false); 2108 qptype = bnxt_re_init_qp_type(rdev, init_attr); 2109 if (qptype < 0) { 2110 rc = qptype; 2111 goto out; 2112 } 2113 qplqp->type = (u8)qptype; 2114 qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev); 2115 ether_addr_copy(qplqp->smac, rdev->dev_addr); 2116 2117 if (init_attr->qp_type == IB_QPT_RC) { 2118 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; 2119 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 2120 } 2121 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(if_getmtu(rdev->netdev))); 2122 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ 2123 if (init_attr->create_flags) { 2124 dev_dbg(rdev_to_dev(rdev), 2125 "QP create flags 0x%x not supported\n", 2126 init_attr->create_flags); 2127 return -EOPNOTSUPP; 2128 } 2129 2130 /* Setup CQs */ 2131 if (init_attr->send_cq) { 2132 cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq); 2133 if (!cq) { 2134 dev_err(rdev_to_dev(rdev), "Send CQ not found\n"); 2135 rc = -EINVAL; 2136 goto out; 2137 } 2138 qplqp->scq = &cq->qplib_cq; 2139 qp->scq = cq; 2140 } 2141 2142 if (init_attr->recv_cq) { 2143 cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq); 2144 if (!cq) { 2145 dev_err(rdev_to_dev(rdev), "Receive CQ not found\n"); 2146 rc = -EINVAL; 2147 goto out; 2148 } 2149 qplqp->rcq = &cq->qplib_cq; 2150 qp->rcq = cq; 2151 } 2152 2153 /* Setup RQ/SRQ */ 2154 rc = bnxt_re_init_rq_attr(qp, init_attr, cntx); 2155 if (rc) 2156 goto out; 2157 if (init_attr->qp_type == IB_QPT_GSI) 2158 bnxt_re_adjust_gsi_rq_attr(qp); 2159 2160 /* Setup SQ */ 2161 rc = bnxt_re_init_sq_attr(qp, init_attr, cntx); 2162 if (rc) 2163 goto out; 2164 if (init_attr->qp_type == IB_QPT_GSI) 2165 bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx); 2166 2167 if (udata) /* This will update DPI and qp_handle */ 2168 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 2169 out: 2170 return rc; 2171 } 2172 2173 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, 2174 struct bnxt_re_pd *pd) 2175 { 2176 struct bnxt_re_sqp_entries *sqp_tbl = NULL; 2177 struct bnxt_re_dev *rdev; 2178 struct bnxt_re_qp *sqp; 2179 struct bnxt_re_ah *sah; 2180 int rc = 0; 2181 2182 rdev = qp->rdev; 2183 /* Create a shadow QP to handle the QP1 traffic */ 2184 sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES, 2185 GFP_KERNEL); 2186 if (!sqp_tbl) 2187 return -ENOMEM; 2188 rdev->gsi_ctx.sqp_tbl = sqp_tbl; 2189 2190 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); 2191 if (!sqp) { 2192 rc = -ENODEV; 2193 dev_err(rdev_to_dev(rdev), 2194 "Failed to create Shadow QP for QP1\n"); 2195 goto out; 2196 } 2197 rdev->gsi_ctx.gsi_sqp = sqp; 2198 2199 sqp->rcq = qp->rcq; 2200 sqp->scq = qp->scq; 2201 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 2202 &qp->qplib_qp); 2203 if (!sah) { 2204 bnxt_qplib_destroy_qp(&rdev->qplib_res, 2205 &sqp->qplib_qp); 2206 rc = -ENODEV; 2207 dev_err(rdev_to_dev(rdev), 2208 "Failed to create AH entry for ShadowQP\n"); 2209 goto out; 2210 } 2211 rdev->gsi_ctx.gsi_sah = sah; 2212 2213 return 0; 2214 out: 2215 kfree(sqp_tbl); 2216 return rc; 2217 } 2218 2219 static int __get_rq_hdr_buf_size(u8 gsi_mode) 2220 { 2221 return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ? 2222 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 : 2223 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE; 2224 } 2225 2226 static int __get_sq_hdr_buf_size(u8 gsi_mode) 2227 { 2228 return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ? 2229 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 : 2230 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE; 2231 } 2232 2233 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd) 2234 { 2235 struct bnxt_qplib_qp *qplqp; 2236 struct bnxt_qplib_res *res; 2237 struct bnxt_re_dev *rdev; 2238 u32 sstep, rstep; 2239 u8 gsi_mode; 2240 int rc = 0; 2241 2242 rdev = qp->rdev; 2243 qplqp = &qp->qplib_qp; 2244 res = &rdev->qplib_res; 2245 gsi_mode = rdev->gsi_ctx.gsi_qp_mode; 2246 2247 rstep = __get_rq_hdr_buf_size(gsi_mode); 2248 sstep = __get_sq_hdr_buf_size(gsi_mode); 2249 rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep); 2250 if (rc) 2251 goto out; 2252 2253 rc = bnxt_qplib_create_qp1(res, qplqp); 2254 if (rc) { 2255 dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n"); 2256 goto out; 2257 } 2258 2259 if (gsi_mode == BNXT_RE_GSI_MODE_ALL) 2260 rc = bnxt_re_create_shadow_gsi(qp, pd); 2261 out: 2262 return rc; 2263 } 2264 2265 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, 2266 struct ib_qp_init_attr *init_attr, 2267 struct bnxt_qplib_dev_attr *dev_attr) 2268 { 2269 bool rc = true; 2270 int ilsize; 2271 2272 ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge)); 2273 if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || 2274 (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || 2275 (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || 2276 (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || 2277 (ilsize > dev_attr->max_inline_data)) { 2278 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! " 2279 "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x " 2280 "0x%x/0x%x 0x%x/0x%x\n", 2281 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, 2282 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, 2283 init_attr->cap.max_send_sge, dev_attr->max_qp_sges, 2284 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, 2285 init_attr->cap.max_inline_data, 2286 dev_attr->max_inline_data); 2287 rc = false; 2288 } 2289 return rc; 2290 } 2291 2292 static inline struct 2293 bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, 2294 struct bnxt_re_dev *rdev) 2295 { 2296 struct bnxt_re_qp *qp; 2297 2298 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2299 if (!qp) 2300 dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n"); 2301 return qp; 2302 } 2303 2304 struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, 2305 struct ib_qp_init_attr *qp_init_attr, 2306 struct ib_udata *udata) 2307 { 2308 struct bnxt_re_pd *pd; 2309 struct ib_pd *ib_pd = qp_in; 2310 struct bnxt_qplib_dev_attr *dev_attr; 2311 struct bnxt_re_dev *rdev; 2312 u32 active_qps, tmp_qps; 2313 struct bnxt_re_qp *qp; 2314 int rc; 2315 2316 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 2317 rdev = pd->rdev; 2318 dev_attr = rdev->dev_attr; 2319 if (rdev->mod_exit) { 2320 rc = -EIO; 2321 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 2322 goto exit; 2323 } 2324 2325 if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) { 2326 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n", 2327 atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp); 2328 rc = -EINVAL; 2329 goto exit; 2330 } 2331 2332 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); 2333 if (!rc) { 2334 rc = -EINVAL; 2335 goto exit; 2336 } 2337 qp = __get_qp_from_qp_in(qp_in, rdev); 2338 if (!qp) { 2339 rc = -ENOMEM; 2340 goto exit; 2341 } 2342 qp->rdev = rdev; 2343 2344 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); 2345 if (rc) 2346 goto fail; 2347 2348 if (qp_init_attr->qp_type == IB_QPT_GSI && 2349 !_is_chip_gen_p5_p7(rdev->chip_ctx)) { 2350 rc = bnxt_re_create_gsi_qp(qp, pd); 2351 if (rc == -ENODEV) 2352 goto qp_destroy; 2353 if (rc) 2354 goto fail; 2355 } else { 2356 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 2357 if (rc) { 2358 dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); 2359 goto free_umem; 2360 } 2361 2362 if (udata) { 2363 struct bnxt_re_qp_resp resp; 2364 2365 resp.qpid = qp->qplib_qp.id; 2366 rc = bnxt_re_copy_to_udata(rdev, &resp, 2367 min(udata->outlen, sizeof(resp)), 2368 udata); 2369 if (rc) 2370 goto qp_destroy; 2371 } 2372 } 2373 2374 qp->ib_qp.qp_num = qp->qplib_qp.id; 2375 if (qp_init_attr->qp_type == IB_QPT_GSI) 2376 rdev->gsi_ctx.gsi_qp = qp; 2377 spin_lock_init(&qp->sq_lock); 2378 spin_lock_init(&qp->rq_lock); 2379 INIT_LIST_HEAD(&qp->list); 2380 mutex_lock(&rdev->qp_lock); 2381 list_add_tail(&qp->list, &rdev->qp_list); 2382 mutex_unlock(&rdev->qp_lock); 2383 atomic_inc(&rdev->stats.rsors.qp_count); 2384 active_qps = atomic_read(&rdev->stats.rsors.qp_count); 2385 if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count)) 2386 atomic_set(&rdev->stats.rsors.max_qp_count, active_qps); 2387 2388 bnxt_re_dump_debug_stats(rdev, active_qps); 2389 2390 /* Get the counters for RC QPs and UD QPs */ 2391 if (qp_init_attr->qp_type == IB_QPT_RC) { 2392 tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count); 2393 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count)) 2394 atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps); 2395 } else if (qp_init_attr->qp_type == IB_QPT_UD) { 2396 tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count); 2397 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count)) 2398 atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps); 2399 } 2400 2401 return &qp->ib_qp; 2402 2403 qp_destroy: 2404 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 2405 free_umem: 2406 if (udata) { 2407 if (qp->rumem && !IS_ERR(qp->rumem)) 2408 ib_umem_release(qp->rumem); 2409 if (qp->sumem && !IS_ERR(qp->sumem)) 2410 ib_umem_release(qp->sumem); 2411 } 2412 fail: 2413 kfree(qp); 2414 exit: 2415 return ERR_PTR(rc); 2416 } 2417 2418 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 2419 struct bnxt_re_qp *qp1_qp, 2420 int qp_attr_mask) 2421 { 2422 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; 2423 int rc = 0; 2424 2425 if (qp_attr_mask & IB_QP_STATE) { 2426 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 2427 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 2428 } 2429 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 2430 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 2431 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 2432 } 2433 2434 if (qp_attr_mask & IB_QP_QKEY) { 2435 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 2436 /* Using a Random QKEY */ 2437 qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY; 2438 } 2439 if (qp_attr_mask & IB_QP_SQ_PSN) { 2440 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 2441 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 2442 } 2443 2444 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2445 if (rc) 2446 dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n"); 2447 return rc; 2448 } 2449 2450 static u32 ipv4_from_gid(u8 *gid) 2451 { 2452 return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]); 2453 } 2454 2455 static u16 get_source_port(struct bnxt_re_dev *rdev, 2456 struct bnxt_re_qp *qp) 2457 { 2458 u8 ip_off, data[48], smac[ETH_ALEN]; 2459 u16 crc = 0, buf_len = 0, i; 2460 u8 addr_len; 2461 u32 qpn; 2462 2463 if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) { 2464 addr_len = 6; 2465 ip_off = 10; 2466 } else { 2467 addr_len = 4; 2468 ip_off = 12; 2469 } 2470 2471 memcpy(smac, qp->qplib_qp.smac, ETH_ALEN); 2472 2473 memset(data, 0, 48); 2474 memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN); 2475 buf_len += ETH_ALEN; 2476 2477 memcpy(data + buf_len, smac, ETH_ALEN); 2478 buf_len += ETH_ALEN; 2479 2480 memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len); 2481 buf_len += addr_len; 2482 2483 memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len); 2484 buf_len += addr_len; 2485 2486 qpn = htonl(qp->qplib_qp.dest_qpn); 2487 memcpy(data + buf_len, (u8 *)&qpn + 1, 3); 2488 buf_len += 3; 2489 2490 for (i = 0; i < buf_len; i++) 2491 crc = crc16(crc, (data + i), 1); 2492 2493 return crc; 2494 } 2495 2496 static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) 2497 { 2498 u16 type; 2499 2500 type = __from_hw_to_ib_qp_type(qp->qplib_qp.type); 2501 2502 /* User-space can extract ip address with sgid_index. */ 2503 if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) { 2504 qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw); 2505 qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data); 2506 } else { 2507 memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw, 2508 sizeof(qp->qp_info_entry.s_ip.ipv6_addr)); 2509 memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data, 2510 sizeof(qp->qp_info_entry.d_ip.ipv6_addr)); 2511 } 2512 2513 if (type == IB_QPT_RC && 2514 (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 || 2515 qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) { 2516 qp->qp_info_entry.s_port = get_source_port(rdev, qp); 2517 } 2518 qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT; 2519 } 2520 2521 static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp) 2522 { 2523 struct bnxt_qplib_q *rq, *sq; 2524 unsigned long flags; 2525 2526 if (qp->sumem) 2527 return; 2528 2529 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 2530 rq = &qp->qplib_qp.rq; 2531 sq = &qp->qplib_qp.sq; 2532 2533 dev_dbg(rdev_to_dev(qp->rdev), 2534 "Move QP = %p to flush list\n", qp); 2535 flags = bnxt_re_lock_cqs(qp); 2536 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 2537 bnxt_re_unlock_cqs(qp, flags); 2538 2539 if (sq->hwq.prod != sq->hwq.cons) 2540 bnxt_re_handle_cqn(&qp->scq->qplib_cq); 2541 2542 if (qp->rcq && (qp->rcq != qp->scq) && 2543 (rq->hwq.prod != rq->hwq.cons)) 2544 bnxt_re_handle_cqn(&qp->rcq->qplib_cq); 2545 } 2546 2547 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 2548 dev_dbg(rdev_to_dev(qp->rdev), 2549 "Move QP = %p out of flush list\n", qp); 2550 flags = bnxt_re_lock_cqs(qp); 2551 bnxt_qplib_clean_qp(&qp->qplib_qp); 2552 bnxt_re_unlock_cqs(qp, flags); 2553 } 2554 } 2555 2556 bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, 2557 enum ib_qp_state next_state, 2558 enum ib_qp_type type, 2559 enum ib_qp_attr_mask mask) 2560 { 2561 return (ib_modify_qp_is_ok(cur_state, next_state, 2562 type, mask)); 2563 } 2564 2565 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2566 int qp_attr_mask, struct ib_udata *udata) 2567 { 2568 enum ib_qp_state curr_qp_state, new_qp_state; 2569 struct bnxt_re_modify_qp_ex_resp resp = {}; 2570 struct bnxt_re_modify_qp_ex_req ureq = {}; 2571 struct bnxt_qplib_dev_attr *dev_attr; 2572 struct bnxt_qplib_ppp *ppp = NULL; 2573 struct bnxt_re_dev *rdev; 2574 struct bnxt_re_qp *qp; 2575 struct ib_gid_attr *sgid_attr; 2576 struct ib_gid_attr gid_attr; 2577 union ib_gid sgid, *gid_ptr = NULL; 2578 u8 nw_type; 2579 int rc, entries, status; 2580 bool is_copy_to_udata = false; 2581 bool is_qpmtu_high = false; 2582 2583 qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 2584 rdev = qp->rdev; 2585 dev_attr = rdev->dev_attr; 2586 2587 qp->qplib_qp.modify_flags = 0; 2588 ppp = &qp->qplib_qp.ppp; 2589 if (qp_attr_mask & IB_QP_STATE) { 2590 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 2591 new_qp_state = qp_attr->qp_state; 2592 if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state, 2593 ib_qp->qp_type, qp_attr_mask)) { 2594 dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x" 2595 " specified for qpn=0x%x of type=0x%x" 2596 " current_qp_state=0x%x, new_qp_state=0x%x\n", 2597 qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, 2598 curr_qp_state, new_qp_state); 2599 return -EINVAL; 2600 } 2601 dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x " 2602 "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n", 2603 __func__, __LINE__, qp_attr_mask, ib_qp->qp_num, 2604 ib_qp->qp_type, curr_qp_state, new_qp_state); 2605 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 2606 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 2607 2608 if (udata && curr_qp_state == IB_QPS_RESET && 2609 new_qp_state == IB_QPS_INIT) { 2610 if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 2611 if (ureq.comp_mask & 2612 BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) { 2613 ppp->req = BNXT_QPLIB_PPP_REQ; 2614 ppp->dpi = ureq.dpi; 2615 } 2616 } 2617 } 2618 } 2619 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 2620 qp->qplib_qp.modify_flags |= 2621 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 2622 qp->qplib_qp.en_sqd_async_notify = true; 2623 } 2624 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 2625 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 2626 qp->qplib_qp.access = 2627 __from_ib_access_flags(qp_attr->qp_access_flags); 2628 /* LOCAL_WRITE access must be set to allow RC receive */ 2629 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 2630 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; 2631 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; 2632 } 2633 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 2634 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 2635 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 2636 } 2637 if (qp_attr_mask & IB_QP_QKEY) { 2638 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 2639 qp->qplib_qp.qkey = qp_attr->qkey; 2640 } 2641 if (qp_attr_mask & IB_QP_AV) { 2642 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 2643 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 2644 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 2645 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 2646 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 2647 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 2648 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 2649 memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, 2650 sizeof(qp->qplib_qp.ah.dgid.data)); 2651 qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; 2652 qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev, 2653 qp_attr->ah_attr.grh.sgid_index); 2654 qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index; 2655 qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; 2656 qp->qplib_qp.ah.traffic_class = 2657 qp_attr->ah_attr.grh.traffic_class; 2658 qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; 2659 ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr)); 2660 sgid_attr = &gid_attr; 2661 status = bnxt_re_get_cached_gid(&rdev->ibdev, 1, 2662 qp_attr->ah_attr.grh.sgid_index, 2663 &sgid, &sgid_attr, 2664 &qp_attr->ah_attr.grh, NULL); 2665 if (!status) 2666 if_rele(sgid_attr->ndev); 2667 gid_ptr = &sgid; 2668 if (sgid_attr->ndev) { 2669 memcpy(qp->qplib_qp.smac, rdev->dev_addr, 2670 ETH_ALEN); 2671 nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid); 2672 dev_dbg(rdev_to_dev(rdev), 2673 "Connection using the nw_type %d\n", nw_type); 2674 switch (nw_type) { 2675 case RDMA_NETWORK_IPV4: 2676 qp->qplib_qp.nw_type = 2677 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 2678 break; 2679 case RDMA_NETWORK_IPV6: 2680 qp->qplib_qp.nw_type = 2681 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 2682 break; 2683 default: 2684 qp->qplib_qp.nw_type = 2685 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 2686 break; 2687 } 2688 } 2689 memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid)); 2690 } 2691 2692 /* MTU settings allowed only during INIT -> RTR */ 2693 if (qp_attr->qp_state == IB_QPS_RTR) { 2694 bnxt_re_init_qpmtu(qp, if_getmtu(rdev->netdev), qp_attr_mask, qp_attr, 2695 &is_qpmtu_high); 2696 if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 2697 if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) { 2698 resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK; 2699 resp.path_mtu = qp->qplib_qp.mtu; 2700 is_copy_to_udata = true; 2701 } else if (is_qpmtu_high) { 2702 dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n", 2703 qp->qplib_qp.id); 2704 return -EINVAL; 2705 } 2706 } 2707 } 2708 2709 if (qp_attr_mask & IB_QP_TIMEOUT) { 2710 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 2711 qp->qplib_qp.timeout = qp_attr->timeout; 2712 } 2713 if (qp_attr_mask & IB_QP_RETRY_CNT) { 2714 qp->qplib_qp.modify_flags |= 2715 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 2716 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 2717 } 2718 if (qp_attr_mask & IB_QP_RNR_RETRY) { 2719 qp->qplib_qp.modify_flags |= 2720 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 2721 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 2722 } 2723 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 2724 qp->qplib_qp.modify_flags |= 2725 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 2726 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 2727 } 2728 if (qp_attr_mask & IB_QP_RQ_PSN) { 2729 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 2730 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 2731 } 2732 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2733 qp->qplib_qp.modify_flags |= 2734 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 2735 /* Cap the max_rd_atomic to device max */ 2736 if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom) 2737 dev_dbg(rdev_to_dev(rdev), 2738 "max_rd_atomic requested %d is > device max %d\n", 2739 qp_attr->max_rd_atomic, 2740 dev_attr->max_qp_rd_atom); 2741 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 2742 dev_attr->max_qp_rd_atom); 2743 } 2744 if (qp_attr_mask & IB_QP_SQ_PSN) { 2745 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 2746 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 2747 } 2748 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2749 if (qp_attr->max_dest_rd_atomic > 2750 dev_attr->max_qp_init_rd_atom) { 2751 dev_err(rdev_to_dev(rdev), 2752 "max_dest_rd_atomic requested %d is > device max %d\n", 2753 qp_attr->max_dest_rd_atomic, 2754 dev_attr->max_qp_init_rd_atom); 2755 return -EINVAL; 2756 } 2757 qp->qplib_qp.modify_flags |= 2758 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 2759 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 2760 } 2761 if (qp_attr_mask & IB_QP_CAP) { 2762 qp->qplib_qp.modify_flags |= 2763 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 2764 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 2765 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 2766 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 2767 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 2768 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 2769 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 2770 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 2771 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 2772 (qp_attr->cap.max_inline_data >= 2773 dev_attr->max_inline_data)) { 2774 dev_err(rdev_to_dev(rdev), 2775 "Create QP failed - max exceeded\n"); 2776 return -EINVAL; 2777 } 2778 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 2779 if (entries > dev_attr->max_qp_wqes) 2780 entries = dev_attr->max_qp_wqes; 2781 entries = min_t(u32, entries, dev_attr->max_qp_wqes); 2782 qp->qplib_qp.sq.max_wqe = entries; 2783 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 2784 qp_attr->cap.max_send_wr; 2785 /* 2786 * Reserving one slot for Phantom WQE. Some application can 2787 * post one extra entry in this case. Allowing this to avoid 2788 * unexpected Queue full condition 2789 */ 2790 qp->qplib_qp.sq.q_full_delta -= 1; 2791 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 2792 if (qp->qplib_qp.rq.max_wqe) { 2793 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 2794 if (entries > dev_attr->max_qp_wqes) 2795 entries = dev_attr->max_qp_wqes; 2796 qp->qplib_qp.rq.max_wqe = entries; 2797 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 2798 qp_attr->cap.max_recv_wr; 2799 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 2800 } else { 2801 /* SRQ was used prior, just ignore the RQ caps */ 2802 } 2803 } 2804 if (qp_attr_mask & IB_QP_DEST_QPN) { 2805 qp->qplib_qp.modify_flags |= 2806 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 2807 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 2808 } 2809 2810 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2811 if (rc) { 2812 dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n"); 2813 return rc; 2814 } 2815 if (qp_attr_mask & IB_QP_STATE) 2816 bnxt_qplib_manage_flush_qp(qp); 2817 if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK && 2818 ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) { 2819 resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN; 2820 resp.ppp_st_idx = ppp->st_idx_en >> 2821 BNXT_QPLIB_PPP_ST_IDX_SHIFT; 2822 is_copy_to_udata = true; 2823 } 2824 2825 if (is_copy_to_udata) { 2826 rc = bnxt_re_copy_to_udata(rdev, &resp, 2827 min(udata->outlen, sizeof(resp)), 2828 udata); 2829 if (rc) 2830 return rc; 2831 } 2832 2833 if (ib_qp->qp_type == IB_QPT_GSI && 2834 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && 2835 rdev->gsi_ctx.gsi_sqp) 2836 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 2837 /* 2838 * Update info when qp_info_info 2839 */ 2840 bnxt_re_update_qp_info(rdev, qp); 2841 return rc; 2842 } 2843 2844 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2845 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 2846 { 2847 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 2848 struct bnxt_re_dev *rdev = qp->rdev; 2849 struct bnxt_qplib_qp *qplib_qp; 2850 int rc; 2851 2852 qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL); 2853 if (!qplib_qp) 2854 return -ENOMEM; 2855 2856 qplib_qp->id = qp->qplib_qp.id; 2857 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 2858 2859 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 2860 if (rc) { 2861 dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n", 2862 qplib_qp->id, rc); 2863 goto free_mem; 2864 } 2865 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 2866 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); 2867 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 2868 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 2869 qp_attr->pkey_index = qplib_qp->pkey_index; 2870 qp_attr->qkey = qplib_qp->qkey; 2871 memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data, 2872 sizeof(qplib_qp->ah.dgid.data)); 2873 qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label; 2874 qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index; 2875 qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit; 2876 qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class; 2877 qp_attr->ah_attr.sl = qplib_qp->ah.sl; 2878 ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac); 2879 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 2880 qp_attr->timeout = qplib_qp->timeout; 2881 qp_attr->retry_cnt = qplib_qp->retry_cnt; 2882 qp_attr->rnr_retry = qplib_qp->rnr_retry; 2883 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 2884 qp_attr->rq_psn = qplib_qp->rq.psn; 2885 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 2886 qp_attr->sq_psn = qplib_qp->sq.psn; 2887 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 2888 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 2889 IB_SIGNAL_REQ_WR; 2890 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 2891 2892 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 2893 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 2894 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 2895 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 2896 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 2897 qp_init_attr->cap = qp_attr->cap; 2898 2899 free_mem: 2900 kfree(qplib_qp); 2901 return rc; 2902 } 2903 2904 /* Builders */ 2905 2906 /* For Raw, the application is responsible to build the entire packet */ 2907 static void bnxt_re_build_raw_send(const struct ib_send_wr *wr, 2908 struct bnxt_qplib_swqe *wqe) 2909 { 2910 switch (wr->send_flags) { 2911 case IB_SEND_IP_CSUM: 2912 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2913 break; 2914 default: 2915 /* Pad HW RoCE iCRC */ 2916 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2917 break; 2918 } 2919 } 2920 2921 /* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr 2922 * as according to the sgid and AV 2923 */ 2924 static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, 2925 struct bnxt_qplib_swqe *wqe, int payload_size) 2926 { 2927 struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, 2928 ibah); 2929 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 2930 struct bnxt_qplib_sge sge; 2931 int i, rc = 0; 2932 union ib_gid sgid; 2933 u16 vlan_id; 2934 u8 *ptmac; 2935 void *buf; 2936 2937 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 2938 2939 /* Get sgid */ 2940 rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid); 2941 if (rc) 2942 return rc; 2943 2944 /* ETH */ 2945 qp->qp1_hdr.eth_present = 1; 2946 ptmac = ah->qplib_ah.dmac; 2947 memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4); 2948 ptmac += 4; 2949 memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2); 2950 2951 ptmac = qp->qplib_qp.smac; 2952 memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2); 2953 ptmac += 2; 2954 memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4); 2955 2956 qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1); 2957 2958 /* For vlan, check the sgid for vlan existence */ 2959 vlan_id = rdma_get_vlan_id(&sgid); 2960 if (vlan_id && vlan_id < 0x1000) { 2961 qp->qp1_hdr.vlan_present = 1; 2962 qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q); 2963 } 2964 /* GRH */ 2965 qp->qp1_hdr.grh_present = 1; 2966 qp->qp1_hdr.grh.ip_version = 6; 2967 qp->qp1_hdr.grh.payload_length = 2968 cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7) 2969 & ~3); 2970 qp->qp1_hdr.grh.next_header = 0x1b; 2971 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); 2972 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 2973 sizeof(sgid)); 2974 2975 /* BTH */ 2976 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 2977 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 2978 qp->qp1_hdr.immediate_present = 1; 2979 } else { 2980 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 2981 } 2982 if (wr->send_flags & IB_SEND_SOLICITED) 2983 qp->qp1_hdr.bth.solicited_event = 1; 2984 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 2985 /* P_key for QP1 is for all members */ 2986 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 2987 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 2988 qp->qp1_hdr.bth.ack_req = 0; 2989 qp->send_psn++; 2990 qp->send_psn &= BTH_PSN_MASK; 2991 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 2992 /* DETH */ 2993 /* Use the priviledged Q_Key for QP1 */ 2994 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 2995 qp->qp1_hdr.deth.source_qpn = IB_QP1; 2996 2997 /* Pack the QP1 to the transmit buffer */ 2998 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 2999 if (!buf) { 3000 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); 3001 rc = -ENOMEM; 3002 } 3003 for (i = wqe->num_sge; i; i--) { 3004 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 3005 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 3006 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 3007 } 3008 wqe->sg_list[0].addr = sge.addr; 3009 wqe->sg_list[0].lkey = sge.lkey; 3010 wqe->sg_list[0].size = sge.size; 3011 wqe->num_sge++; 3012 3013 return rc; 3014 } 3015 3016 static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp, 3017 const struct ib_send_wr *wr, 3018 struct bnxt_qplib_swqe *wqe) 3019 { 3020 struct bnxt_re_dev *rdev; 3021 int rc, indx, len = 0; 3022 3023 rdev = qp->rdev; 3024 3025 /* Mode UD is applicable to Gen P5 only */ 3026 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) 3027 return 0; 3028 3029 for (indx = 0; indx < wr->num_sge; indx++) { 3030 wqe->sg_list[indx].addr = wr->sg_list[indx].addr; 3031 wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey; 3032 wqe->sg_list[indx].size = wr->sg_list[indx].length; 3033 len += wr->sg_list[indx].length; 3034 } 3035 rc = bnxt_re_build_qp1_send(qp, wr, wqe, len); 3036 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 3037 3038 return rc; 3039 } 3040 3041 /* For the MAD layer, it only provides the recv SGE the size of 3042 ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 3043 nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 3044 receive packet (334 bytes) with no VLAN and then copy the GRH 3045 and the MAD datagram out to the provided SGE. 3046 */ 3047 3048 static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp, 3049 const struct ib_recv_wr *wr, 3050 struct bnxt_qplib_swqe *wqe) 3051 { 3052 struct bnxt_re_dev *rdev = qp->rdev; 3053 struct bnxt_qplib_sge ref, sge; 3054 u8 udp_hdr_size = 0; 3055 u8 ip_hdr_size = 0; 3056 int rc = 0; 3057 int size; 3058 3059 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { 3060 /* Create 5 SGEs as according to the following: 3061 * Ethernet header (14) 3062 * ib_grh (40) - as provided from the wr 3063 * ib_bth + ib_deth + UDP(RoCE v2 only) (28) 3064 * MAD (256) - as provided from the wr 3065 * iCRC (4) 3066 */ 3067 3068 /* Set RoCE v2 header size and offsets */ 3069 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4) 3070 ip_hdr_size = 20; 3071 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1) 3072 udp_hdr_size = 8; 3073 3074 /* Save the reference from ULP */ 3075 ref.addr = wr->sg_list[0].addr; 3076 ref.lkey = wr->sg_list[0].lkey; 3077 ref.size = wr->sg_list[0].length; 3078 3079 /* SGE 1 */ 3080 size = sge.size; 3081 wqe->sg_list[0].addr = sge.addr; 3082 wqe->sg_list[0].lkey = sge.lkey; 3083 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE; 3084 size -= wqe->sg_list[0].size; 3085 if (size <= 0) { 3086 dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n"); 3087 rc = -ENOMEM; 3088 goto done; 3089 } 3090 sge.size = (u32)size; 3091 sge.addr += wqe->sg_list[0].size; 3092 3093 /* SGE 2 */ 3094 /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */ 3095 wqe->sg_list[1].addr = ref.addr + ip_hdr_size; 3096 wqe->sg_list[1].lkey = ref.lkey; 3097 wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size; 3098 ref.size -= wqe->sg_list[1].size; 3099 if (ref.size <= 0) { 3100 dev_err(rdev_to_dev(qp->rdev), 3101 "QP1 ref buffer is empty!\n"); 3102 rc = -ENOMEM; 3103 goto done; 3104 } 3105 ref.addr += wqe->sg_list[1].size + ip_hdr_size; 3106 3107 /* SGE 3 */ 3108 wqe->sg_list[2].addr = sge.addr; 3109 wqe->sg_list[2].lkey = sge.lkey; 3110 wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE + 3111 udp_hdr_size; 3112 size -= wqe->sg_list[2].size; 3113 if (size <= 0) { 3114 dev_err(rdev_to_dev(qp->rdev), 3115 "QP1 rq buffer is empty!\n"); 3116 rc = -ENOMEM; 3117 goto done; 3118 } 3119 sge.size = (u32)size; 3120 sge.addr += wqe->sg_list[2].size; 3121 3122 /* SGE 4 */ 3123 wqe->sg_list[3].addr = ref.addr; 3124 wqe->sg_list[3].lkey = ref.lkey; 3125 wqe->sg_list[3].size = ref.size; 3126 ref.size -= wqe->sg_list[3].size; 3127 if (ref.size) { 3128 dev_err(rdev_to_dev(qp->rdev), 3129 "QP1 ref buffer is incorrect!\n"); 3130 rc = -ENOMEM; 3131 goto done; 3132 } 3133 /* SGE 5 */ 3134 wqe->sg_list[4].addr = sge.addr; 3135 wqe->sg_list[4].lkey = sge.lkey; 3136 wqe->sg_list[4].size = sge.size; 3137 size -= wqe->sg_list[4].size; 3138 if (size) { 3139 dev_err(rdev_to_dev(qp->rdev), 3140 "QP1 rq buffer is incorrect!\n"); 3141 rc = -ENOMEM; 3142 goto done; 3143 } 3144 sge.size = (u32)size; 3145 wqe->num_sge = 5; 3146 } else { 3147 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); 3148 rc = -ENOMEM; 3149 } 3150 done: 3151 return rc; 3152 } 3153 3154 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 3155 const struct ib_recv_wr *wr, 3156 struct bnxt_qplib_swqe *wqe) 3157 { 3158 struct bnxt_re_sqp_entries *sqp_entry; 3159 struct bnxt_qplib_sge sge; 3160 struct bnxt_re_dev *rdev; 3161 u32 rq_prod_index; 3162 int rc = 0; 3163 3164 rdev = qp->rdev; 3165 3166 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 3167 3168 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { 3169 /* Create 1 SGE to receive the entire 3170 * ethernet packet 3171 */ 3172 /* SGE 1 */ 3173 wqe->sg_list[0].addr = sge.addr; 3174 /* TODO check the lkey to be used */ 3175 wqe->sg_list[0].lkey = sge.lkey; 3176 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 3177 if (sge.size < wqe->sg_list[0].size) { 3178 dev_err(rdev_to_dev(qp->rdev), 3179 "QP1 rq buffer is empty!\n"); 3180 rc = -ENOMEM; 3181 goto done; 3182 } 3183 3184 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; 3185 sqp_entry->sge.addr = wr->sg_list[0].addr; 3186 sqp_entry->sge.lkey = wr->sg_list[0].lkey; 3187 sqp_entry->sge.size = wr->sg_list[0].length; 3188 /* Store the wrid for reporting completion */ 3189 sqp_entry->wrid = wqe->wr_id; 3190 /* change the wqe->wrid to table index */ 3191 wqe->wr_id = rq_prod_index; 3192 } 3193 done: 3194 return rc; 3195 } 3196 3197 static bool is_ud_qp(struct bnxt_re_qp *qp) 3198 { 3199 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || 3200 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); 3201 } 3202 3203 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 3204 const struct ib_send_wr *wr, 3205 struct bnxt_qplib_swqe *wqe) 3206 { 3207 struct bnxt_re_ah *ah = NULL; 3208 3209 if(is_ud_qp(qp)) { 3210 ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); 3211 wqe->send.q_key = ud_wr(wr)->remote_qkey; 3212 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 3213 wqe->send.avid = ah->qplib_ah.id; 3214 } 3215 switch (wr->opcode) { 3216 case IB_WR_SEND: 3217 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 3218 break; 3219 case IB_WR_SEND_WITH_IMM: 3220 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 3221 wqe->send.imm_data = wr->ex.imm_data; 3222 break; 3223 case IB_WR_SEND_WITH_INV: 3224 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 3225 wqe->send.inv_key = wr->ex.invalidate_rkey; 3226 break; 3227 default: 3228 dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n", 3229 __func__, wr->opcode); 3230 return -EINVAL; 3231 } 3232 if (wr->send_flags & IB_SEND_SIGNALED) 3233 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3234 if (wr->send_flags & IB_SEND_FENCE) 3235 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3236 if (wr->send_flags & IB_SEND_SOLICITED) 3237 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3238 if (wr->send_flags & IB_SEND_INLINE) 3239 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 3240 3241 return 0; 3242 } 3243 3244 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 3245 struct bnxt_qplib_swqe *wqe) 3246 { 3247 switch (wr->opcode) { 3248 case IB_WR_RDMA_WRITE: 3249 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 3250 break; 3251 case IB_WR_RDMA_WRITE_WITH_IMM: 3252 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 3253 wqe->rdma.imm_data = wr->ex.imm_data; 3254 break; 3255 case IB_WR_RDMA_READ: 3256 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 3257 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 3258 break; 3259 default: 3260 return -EINVAL; 3261 } 3262 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 3263 wqe->rdma.r_key = rdma_wr(wr)->rkey; 3264 if (wr->send_flags & IB_SEND_SIGNALED) 3265 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3266 if (wr->send_flags & IB_SEND_FENCE) 3267 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3268 if (wr->send_flags & IB_SEND_SOLICITED) 3269 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3270 if (wr->send_flags & IB_SEND_INLINE) 3271 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 3272 3273 return 0; 3274 } 3275 3276 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 3277 struct bnxt_qplib_swqe *wqe) 3278 { 3279 switch (wr->opcode) { 3280 case IB_WR_ATOMIC_CMP_AND_SWP: 3281 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 3282 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 3283 wqe->atomic.swap_data = atomic_wr(wr)->swap; 3284 break; 3285 case IB_WR_ATOMIC_FETCH_AND_ADD: 3286 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 3287 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 3288 break; 3289 default: 3290 return -EINVAL; 3291 } 3292 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 3293 wqe->atomic.r_key = atomic_wr(wr)->rkey; 3294 if (wr->send_flags & IB_SEND_SIGNALED) 3295 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3296 if (wr->send_flags & IB_SEND_FENCE) 3297 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3298 if (wr->send_flags & IB_SEND_SOLICITED) 3299 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3300 return 0; 3301 } 3302 3303 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 3304 struct bnxt_qplib_swqe *wqe) 3305 { 3306 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 3307 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 3308 if (wr->send_flags & IB_SEND_SIGNALED) 3309 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3310 if (wr->send_flags & IB_SEND_FENCE) 3311 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3312 if (wr->send_flags & IB_SEND_SOLICITED) 3313 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3314 3315 return 0; 3316 } 3317 3318 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 3319 struct bnxt_qplib_swqe *wqe) 3320 { 3321 struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr); 3322 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 3323 int reg_len, i, access = wr->access; 3324 3325 if (mr->npages > qplib_frpl->max_pg_ptrs) { 3326 dev_err_ratelimited(rdev_to_dev(mr->rdev), 3327 " %s: failed npages %d > %d\n", __func__, 3328 mr->npages, qplib_frpl->max_pg_ptrs); 3329 return -EINVAL; 3330 } 3331 3332 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 3333 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 3334 wqe->frmr.levels = qplib_frpl->hwq.level; 3335 wqe->frmr.page_list = mr->pages; 3336 wqe->frmr.page_list_len = mr->npages; 3337 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 3338 3339 if (wr->wr.send_flags & IB_SEND_SIGNALED) 3340 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3341 if (access & IB_ACCESS_LOCAL_WRITE) 3342 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 3343 if (access & IB_ACCESS_REMOTE_READ) 3344 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 3345 if (access & IB_ACCESS_REMOTE_WRITE) 3346 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 3347 if (access & IB_ACCESS_REMOTE_ATOMIC) 3348 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 3349 if (access & IB_ACCESS_MW_BIND) 3350 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 3351 3352 /* TODO: OFED provides the rkey of the MR instead of the lkey */ 3353 wqe->frmr.l_key = wr->key; 3354 wqe->frmr.length = wr->mr->length; 3355 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); 3356 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); 3357 wqe->frmr.va = wr->mr->iova; 3358 reg_len = wqe->frmr.page_list_len * wr->mr->page_size; 3359 3360 if (wqe->frmr.length > reg_len) { 3361 dev_err_ratelimited(rdev_to_dev(mr->rdev), 3362 "%s: bnxt_re_mr 0x%px len (%d > %d)\n", 3363 __func__, (void *)mr, wqe->frmr.length, 3364 reg_len); 3365 3366 for (i = 0; i < mr->npages; i++) 3367 dev_dbg(rdev_to_dev(mr->rdev), 3368 "%s: build_reg_wqe page[%d] = 0x%llx\n", 3369 __func__, i, mr->pages[i]); 3370 3371 return -EINVAL; 3372 } 3373 3374 return 0; 3375 } 3376 3377 static void bnxt_re_set_sg_list(const struct ib_send_wr *wr, 3378 struct bnxt_qplib_swqe *wqe) 3379 { 3380 wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 3381 wqe->num_sge = wr->num_sge; 3382 } 3383 3384 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 3385 { 3386 if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI || 3387 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 3388 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 3389 int qp_attr_mask; 3390 struct ib_qp_attr qp_attr; 3391 3392 qp_attr_mask = IB_QP_STATE; 3393 qp_attr.qp_state = IB_QPS_RTS; 3394 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 3395 qp->qplib_qp.wqe_cnt = 0; 3396 } 3397 } 3398 3399 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 3400 struct bnxt_re_qp *qp, 3401 const struct ib_send_wr *wr) 3402 { 3403 struct bnxt_qplib_swqe wqe; 3404 unsigned long flags; 3405 int rc = 0; 3406 3407 spin_lock_irqsave(&qp->sq_lock, flags); 3408 while (wr) { 3409 /* House keeping */ 3410 memset(&wqe, 0, sizeof(wqe)); 3411 /* Common */ 3412 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 3413 dev_err(rdev_to_dev(rdev), 3414 "Limit exceeded for Send SGEs\n"); 3415 rc = -EINVAL; 3416 break; 3417 } 3418 3419 bnxt_re_set_sg_list(wr, &wqe); 3420 wqe.wr_id = wr->wr_id; 3421 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 3422 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 3423 if (rc) 3424 break; 3425 3426 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 3427 if (rc) { 3428 dev_err(rdev_to_dev(rdev), 3429 "bad_wr seen with opcode = 0x%x rc = %d\n", 3430 wr->opcode, rc); 3431 break; 3432 } 3433 wr = wr->next; 3434 } 3435 bnxt_qplib_post_send_db(&qp->qplib_qp); 3436 bnxt_ud_qp_hw_stall_workaround(qp); 3437 spin_unlock_irqrestore(&qp->sq_lock, flags); 3438 return rc; 3439 } 3440 3441 static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe) 3442 { 3443 /* Need unconditional fence for non-wire memory opcode 3444 * to work as expected. 3445 */ 3446 if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || 3447 wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || 3448 wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || 3449 wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) 3450 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3451 } 3452 3453 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 3454 const struct ib_send_wr **bad_wr) 3455 { 3456 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 3457 struct bnxt_qplib_sge sge[6]; 3458 struct bnxt_qplib_swqe wqe; 3459 struct bnxt_re_dev *rdev; 3460 unsigned long flags; 3461 int rc = 0; 3462 3463 rdev = qp->rdev; 3464 spin_lock_irqsave(&qp->sq_lock, flags); 3465 while (wr) { 3466 /* House keeping */ 3467 memset(&wqe, 0, sizeof(wqe)); 3468 /* Common */ 3469 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 3470 dev_err(rdev_to_dev(rdev), 3471 "Limit exceeded for Send SGEs\n"); 3472 rc = -EINVAL; 3473 goto bad; 3474 } 3475 3476 bnxt_re_set_sg_list(wr, &wqe); 3477 wqe.wr_id = wr->wr_id; 3478 3479 switch (wr->opcode) { 3480 case IB_WR_SEND: 3481 case IB_WR_SEND_WITH_IMM: 3482 if (ib_qp->qp_type == IB_QPT_GSI && 3483 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 3484 memset(sge, 0, sizeof(sge)); 3485 wqe.sg_list = sge; 3486 rc = bnxt_re_build_gsi_send(qp, wr, &wqe); 3487 if (rc) 3488 goto bad; 3489 } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) { 3490 bnxt_re_build_raw_send(wr, &wqe); 3491 } 3492 switch (wr->send_flags) { 3493 case IB_SEND_IP_CSUM: 3494 wqe.rawqp1.lflags |= 3495 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 3496 break; 3497 default: 3498 break; 3499 } 3500 fallthrough; 3501 case IB_WR_SEND_WITH_INV: 3502 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 3503 break; 3504 case IB_WR_RDMA_WRITE: 3505 case IB_WR_RDMA_WRITE_WITH_IMM: 3506 case IB_WR_RDMA_READ: 3507 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 3508 break; 3509 case IB_WR_ATOMIC_CMP_AND_SWP: 3510 case IB_WR_ATOMIC_FETCH_AND_ADD: 3511 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 3512 break; 3513 case IB_WR_RDMA_READ_WITH_INV: 3514 dev_err(rdev_to_dev(rdev), 3515 "RDMA Read with Invalidate is not supported\n"); 3516 rc = -EINVAL; 3517 goto bad; 3518 case IB_WR_LOCAL_INV: 3519 rc = bnxt_re_build_inv_wqe(wr, &wqe); 3520 break; 3521 case IB_WR_REG_MR: 3522 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 3523 break; 3524 default: 3525 /* Unsupported WRs */ 3526 dev_err(rdev_to_dev(rdev), 3527 "WR (0x%x) is not supported\n", wr->opcode); 3528 rc = -EINVAL; 3529 goto bad; 3530 } 3531 3532 if (likely(!rc)) { 3533 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 3534 bnxt_re_legacy_set_uc_fence(&wqe); 3535 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 3536 } 3537 bad: 3538 if (unlikely(rc)) { 3539 dev_err(rdev_to_dev(rdev), 3540 "bad_wr seen with opcode = 0x%x\n", wr->opcode); 3541 *bad_wr = wr; 3542 break; 3543 } 3544 wr = wr->next; 3545 } 3546 bnxt_qplib_post_send_db(&qp->qplib_qp); 3547 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 3548 bnxt_ud_qp_hw_stall_workaround(qp); 3549 spin_unlock_irqrestore(&qp->sq_lock, flags); 3550 3551 return rc; 3552 } 3553 3554 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 3555 struct bnxt_re_qp *qp, 3556 struct ib_recv_wr *wr) 3557 { 3558 struct bnxt_qplib_swqe wqe; 3559 int rc = 0; 3560 3561 /* rq lock can be pardoned here. */ 3562 while (wr) { 3563 /* House keeping */ 3564 memset(&wqe, 0, sizeof(wqe)); 3565 /* Common */ 3566 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 3567 dev_err(rdev_to_dev(rdev), 3568 "Limit exceeded for Receive SGEs\n"); 3569 rc = -EINVAL; 3570 goto bad; 3571 } 3572 3573 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 3574 wqe.num_sge = wr->num_sge; 3575 wqe.wr_id = wr->wr_id; 3576 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 3577 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 3578 bad: 3579 if (rc) { 3580 dev_err(rdev_to_dev(rdev), 3581 "bad_wr seen with RQ post\n"); 3582 break; 3583 } 3584 wr = wr->next; 3585 } 3586 bnxt_qplib_post_recv_db(&qp->qplib_qp); 3587 return rc; 3588 } 3589 3590 static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp, 3591 const struct ib_recv_wr *wr, 3592 struct bnxt_qplib_swqe *wqe) 3593 { 3594 struct bnxt_re_dev *rdev = qp->rdev; 3595 int rc = 0; 3596 3597 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) 3598 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe); 3599 else 3600 rc = bnxt_re_build_qp1_recv(qp, wr, wqe); 3601 3602 return rc; 3603 } 3604 3605 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 3606 const struct ib_recv_wr **bad_wr) 3607 { 3608 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 3609 struct bnxt_qplib_sge sge[6]; 3610 struct bnxt_qplib_swqe wqe; 3611 unsigned long flags; 3612 u32 count = 0; 3613 int rc = 0; 3614 3615 spin_lock_irqsave(&qp->rq_lock, flags); 3616 while (wr) { 3617 memset(&wqe, 0, sizeof(wqe)); 3618 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 3619 dev_err(rdev_to_dev(qp->rdev), 3620 "Limit exceeded for Receive SGEs\n"); 3621 rc = -EINVAL; 3622 goto bad; 3623 } 3624 wqe.num_sge = wr->num_sge; 3625 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 3626 wqe.wr_id = wr->wr_id; 3627 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 3628 3629 if (ib_qp->qp_type == IB_QPT_GSI && 3630 qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 3631 memset(sge, 0, sizeof(sge)); 3632 wqe.sg_list = sge; 3633 rc = bnxt_re_build_gsi_recv(qp, wr, &wqe); 3634 if (rc) 3635 goto bad; 3636 } 3637 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 3638 bad: 3639 if (rc) { 3640 dev_err(rdev_to_dev(qp->rdev), 3641 "bad_wr seen with RQ post\n"); 3642 *bad_wr = wr; 3643 break; 3644 } 3645 /* Ring DB if the RQEs posted reaches a threshold value */ 3646 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 3647 bnxt_qplib_post_recv_db(&qp->qplib_qp); 3648 count = 0; 3649 } 3650 wr = wr->next; 3651 } 3652 3653 if (count) 3654 bnxt_qplib_post_recv_db(&qp->qplib_qp); 3655 spin_unlock_irqrestore(&qp->rq_lock, flags); 3656 3657 return rc; 3658 } 3659 3660 /* Completion Queues */ 3661 void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 3662 { 3663 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 3664 struct bnxt_re_dev *rdev = cq->rdev; 3665 int rc = 0; 3666 3667 if (cq->uctx_cq_page) { 3668 BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq); 3669 free_page((u64)cq->uctx_cq_page); 3670 cq->uctx_cq_page = NULL; 3671 } 3672 if (cq->is_dbr_soft_cq && cq->uctx) { 3673 void *dbr_page; 3674 3675 if (cq->uctx->dbr_recov_cq) { 3676 dbr_page = cq->uctx->dbr_recov_cq_page; 3677 cq->uctx->dbr_recov_cq_page = NULL; 3678 cq->uctx->dbr_recov_cq = NULL; 3679 free_page((unsigned long)dbr_page); 3680 } 3681 goto end; 3682 } 3683 /* CQ getting destroyed. Set this state for cqn handler */ 3684 spin_lock_bh(&cq->qplib_cq.compl_lock); 3685 cq->qplib_cq.destroyed = true; 3686 spin_unlock_bh(&cq->qplib_cq.compl_lock); 3687 if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE || 3688 ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE) 3689 cancel_work_sync(&ib_cq->work); 3690 3691 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 3692 if (rc) 3693 dev_err_ratelimited(rdev_to_dev(rdev), 3694 "%s id = %d failed rc = %d\n", 3695 __func__, cq->qplib_cq.id, rc); 3696 3697 bnxt_re_put_nq(rdev, cq->qplib_cq.nq); 3698 if (cq->umem && !IS_ERR(cq->umem)) 3699 ib_umem_release(cq->umem); 3700 3701 kfree(cq->cql); 3702 atomic_dec(&rdev->stats.rsors.cq_count); 3703 end: 3704 return; 3705 } 3706 3707 static inline struct 3708 bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, 3709 struct bnxt_re_dev *rdev) 3710 { 3711 struct bnxt_re_cq *cq; 3712 cq = container_of(cq_in, struct bnxt_re_cq, ibcq); 3713 return cq; 3714 } 3715 3716 int bnxt_re_create_cq(struct ib_cq *cq_in, 3717 const struct ib_cq_init_attr *attr, 3718 struct ib_udata *udata) 3719 { 3720 struct bnxt_qplib_dev_attr *dev_attr; 3721 struct bnxt_re_ucontext *uctx = NULL; 3722 struct ib_ucontext *context = NULL; 3723 struct bnxt_qplib_cq *qplcq; 3724 struct bnxt_re_cq_req ureq; 3725 struct bnxt_re_dev *rdev; 3726 int rc, entries; 3727 struct bnxt_re_cq *cq; 3728 u32 max_active_cqs; 3729 int cqe = attr->cqe; 3730 3731 if (attr->flags) 3732 return -EOPNOTSUPP; 3733 3734 rdev = rdev_from_cq_in(cq_in); 3735 if (rdev->mod_exit) { 3736 rc = -EIO; 3737 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 3738 goto exit; 3739 } 3740 if (udata) { 3741 uctx = rdma_udata_to_drv_context(udata, 3742 struct bnxt_re_ucontext, 3743 ibucontext); 3744 context = &uctx->ibucontext; 3745 } 3746 dev_attr = rdev->dev_attr; 3747 3748 if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) { 3749 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n"); 3750 rc = -EINVAL; 3751 goto exit; 3752 } 3753 /* Validate CQ fields */ 3754 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 3755 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n"); 3756 rc = -EINVAL; 3757 goto exit; 3758 } 3759 3760 cq = __get_cq_from_cq_in(cq_in, rdev); 3761 if (!cq) { 3762 rc = -ENOMEM; 3763 goto exit; 3764 } 3765 cq->rdev = rdev; 3766 cq->uctx = uctx; 3767 qplcq = &cq->qplib_cq; 3768 qplcq->cq_handle = (u64)qplcq; 3769 /* 3770 * Since CQ is for QP1 is shared with Shadow CQ, the size 3771 * should be double the size. There is no way to identify 3772 * whether this CQ is for GSI QP. So assuming that the first 3773 * CQ created is for QP1 3774 */ 3775 if (!udata && !rdev->gsi_ctx.first_cq_created && 3776 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) { 3777 rdev->gsi_ctx.first_cq_created = true; 3778 /* 3779 * Total CQE required for the CQ = CQE for QP1 RQ + 3780 * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs. 3781 * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe 3782 */ 3783 cqe *= 3; 3784 } 3785 3786 entries = bnxt_re_init_depth(cqe + 1, uctx); 3787 if (entries > dev_attr->max_cq_wqes + 1) 3788 entries = dev_attr->max_cq_wqes + 1; 3789 3790 qplcq->sginfo.pgshft = PAGE_SHIFT; 3791 qplcq->sginfo.pgsize = PAGE_SIZE; 3792 if (udata) { 3793 if (udata->inlen < sizeof(ureq)) 3794 dev_warn(rdev_to_dev(rdev), 3795 "Update the library ulen %d klen %d\n", 3796 (unsigned int)udata->inlen, 3797 (unsigned int)sizeof(ureq)); 3798 3799 rc = ib_copy_from_udata(&ureq, udata, 3800 min(udata->inlen, sizeof(ureq))); 3801 if (rc) 3802 goto fail; 3803 3804 if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) { 3805 cq->is_dbr_soft_cq = true; 3806 goto success; 3807 } 3808 3809 if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) { 3810 void *dbr_page; 3811 u32 *epoch; 3812 3813 dbr_page = (void *)__get_free_page(GFP_KERNEL); 3814 if (!dbr_page) { 3815 dev_err(rdev_to_dev(rdev), 3816 "DBR recov CQ page allocation failed!"); 3817 rc = -ENOMEM; 3818 goto fail; 3819 } 3820 3821 /* memset the epoch and epoch_ack to 0 */ 3822 epoch = dbr_page; 3823 epoch[0] = 0x0; 3824 epoch[1] = 0x0; 3825 3826 uctx->dbr_recov_cq = cq; 3827 uctx->dbr_recov_cq_page = dbr_page; 3828 3829 cq->is_dbr_soft_cq = true; 3830 goto success; 3831 } 3832 3833 cq->umem = ib_umem_get_compat 3834 (rdev, context, udata, ureq.cq_va, 3835 entries * sizeof(struct cq_base), 3836 IB_ACCESS_LOCAL_WRITE, 1); 3837 if (IS_ERR(cq->umem)) { 3838 rc = PTR_ERR(cq->umem); 3839 dev_err(rdev_to_dev(rdev), 3840 "%s: ib_umem_get failed! rc = %d\n", 3841 __func__, rc); 3842 goto fail; 3843 } 3844 qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem, 3845 &qplcq->sginfo.nmap); 3846 qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem); 3847 if (!uctx->dpi.dbr) { 3848 rc = bnxt_re_get_user_dpi(rdev, uctx); 3849 if (rc) 3850 goto c2fail; 3851 } 3852 qplcq->dpi = &uctx->dpi; 3853 } else { 3854 cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries; 3855 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 3856 GFP_KERNEL); 3857 if (!cq->cql) { 3858 dev_err(rdev_to_dev(rdev), 3859 "Allocate CQL for %d failed!\n", cq->max_cql); 3860 rc = -ENOMEM; 3861 goto fail; 3862 } 3863 qplcq->dpi = &rdev->dpi_privileged; 3864 } 3865 /* 3866 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 3867 * used for getting the NQ index. 3868 */ 3869 qplcq->max_wqe = entries; 3870 qplcq->nq = bnxt_re_get_nq(rdev); 3871 qplcq->cnq_hw_ring_id = qplcq->nq->ring_id; 3872 3873 rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq); 3874 if (rc) { 3875 dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n"); 3876 goto fail; 3877 } 3878 3879 INIT_LIST_HEAD(&cq->cq_list); 3880 cq->ibcq.cqe = entries; 3881 cq->cq_period = qplcq->period; 3882 3883 atomic_inc(&rdev->stats.rsors.cq_count); 3884 max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count); 3885 if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count)) 3886 atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs); 3887 spin_lock_init(&cq->cq_lock); 3888 3889 if (udata) { 3890 struct bnxt_re_cq_resp resp; 3891 3892 resp.cqid = qplcq->id; 3893 resp.tail = qplcq->hwq.cons; 3894 resp.phase = qplcq->period; 3895 resp.comp_mask = 0; 3896 resp.dbr = (u64)uctx->dpi.umdbr; 3897 resp.dpi = uctx->dpi.dpi; 3898 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO; 3899 /* Copy only on a valid wcpdi */ 3900 if (uctx->wcdpi.dpi) { 3901 resp.wcdpi = uctx->wcdpi.dpi; 3902 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI; 3903 } 3904 3905 if (_is_chip_p7(rdev->chip_ctx)) { 3906 cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL); 3907 3908 if (!cq->uctx_cq_page) { 3909 dev_err(rdev_to_dev(rdev), 3910 "CQ page allocation failed!\n"); 3911 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); 3912 rc = -ENOMEM; 3913 goto c2fail; 3914 } 3915 3916 resp.uctx_cq_page = (u64)cq->uctx_cq_page; 3917 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE; 3918 } 3919 3920 rc = bnxt_re_copy_to_udata(rdev, &resp, 3921 min(udata->outlen, sizeof(resp)), 3922 udata); 3923 if (rc) { 3924 free_page((u64)cq->uctx_cq_page); 3925 cq->uctx_cq_page = NULL; 3926 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); 3927 goto c2fail; 3928 } 3929 3930 if (cq->uctx_cq_page) 3931 BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq); 3932 } 3933 3934 success: 3935 return 0; 3936 c2fail: 3937 if (udata && cq->umem && !IS_ERR(cq->umem)) 3938 ib_umem_release(cq->umem); 3939 fail: 3940 if (cq) { 3941 if (cq->cql) 3942 kfree(cq->cql); 3943 } 3944 exit: 3945 return rc; 3946 } 3947 3948 int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period) 3949 { 3950 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 3951 struct bnxt_re_dev *rdev = cq->rdev; 3952 int rc; 3953 3954 if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) { 3955 cq->qplib_cq.count = cq_count; 3956 cq->qplib_cq.period = cq_period; 3957 rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq); 3958 if (rc) { 3959 dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n", 3960 cq->qplib_cq.id); 3961 return rc; 3962 } 3963 /* On success, update the shadow */ 3964 cq->cq_count = cq_count; 3965 cq->cq_period = cq_period; 3966 } 3967 return 0; 3968 } 3969 3970 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) 3971 { 3972 struct bnxt_re_dev *rdev = cq->rdev; 3973 3974 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); 3975 3976 cq->qplib_cq.max_wqe = cq->resize_cqe; 3977 if (cq->resize_umem) { 3978 ib_umem_release(cq->umem); 3979 cq->umem = cq->resize_umem; 3980 cq->resize_umem = NULL; 3981 cq->resize_cqe = 0; 3982 } 3983 } 3984 3985 int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata) 3986 { 3987 struct bnxt_qplib_sg_info sginfo = {}; 3988 struct bnxt_qplib_dpi *orig_dpi = NULL; 3989 struct bnxt_qplib_dev_attr *dev_attr; 3990 struct bnxt_re_ucontext *uctx = NULL; 3991 struct bnxt_re_resize_cq_req ureq; 3992 struct ib_ucontext *context = NULL; 3993 struct bnxt_re_dev *rdev; 3994 struct bnxt_re_cq *cq; 3995 int rc, entries; 3996 3997 /* Don't allow more than one resize request at the same time. 3998 * TODO: need a mutex here when we support kernel consumers of resize. 3999 */ 4000 cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 4001 rdev = cq->rdev; 4002 dev_attr = rdev->dev_attr; 4003 if (ib_cq->uobject) { 4004 uctx = rdma_udata_to_drv_context(udata, 4005 struct bnxt_re_ucontext, 4006 ibucontext); 4007 context = &uctx->ibucontext; 4008 } 4009 4010 if (cq->resize_umem) { 4011 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n", 4012 cq->qplib_cq.id); 4013 return -EBUSY; 4014 } 4015 4016 /* Check the requested cq depth out of supported depth */ 4017 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 4018 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n", 4019 cq->qplib_cq.id); 4020 return -EINVAL; 4021 } 4022 4023 entries = bnxt_re_init_depth(cqe + 1, uctx); 4024 entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1); 4025 4026 /* Check to see if the new requested size can be handled by already 4027 * existing CQ 4028 */ 4029 if (entries == cq->ibcq.cqe) { 4030 dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe); 4031 return 0; 4032 } 4033 4034 if (ib_cq->uobject && udata) { 4035 if (udata->inlen < sizeof(ureq)) 4036 dev_warn(rdev_to_dev(rdev), 4037 "Update the library ulen %d klen %d\n", 4038 (unsigned int)udata->inlen, 4039 (unsigned int)sizeof(ureq)); 4040 4041 rc = ib_copy_from_udata(&ureq, udata, 4042 min(udata->inlen, sizeof(ureq))); 4043 if (rc) 4044 goto fail; 4045 4046 dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__, 4047 (void *)ureq.cq_va); 4048 cq->resize_umem = ib_umem_get_compat 4049 (rdev, 4050 context, udata, ureq.cq_va, 4051 entries * sizeof(struct cq_base), 4052 IB_ACCESS_LOCAL_WRITE, 1); 4053 if (IS_ERR(cq->resize_umem)) { 4054 rc = PTR_ERR(cq->resize_umem); 4055 cq->resize_umem = NULL; 4056 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", 4057 __func__, rc); 4058 goto fail; 4059 } 4060 cq->resize_cqe = entries; 4061 dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n", 4062 __func__); 4063 memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo)); 4064 orig_dpi = cq->qplib_cq.dpi; 4065 4066 cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem, 4067 &cq->qplib_cq.sginfo.nmap); 4068 cq->qplib_cq.sginfo.npages = 4069 ib_umem_num_pages_compat(cq->resize_umem); 4070 cq->qplib_cq.sginfo.pgsize = PAGE_SIZE; 4071 cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT; 4072 cq->qplib_cq.dpi = &uctx->dpi; 4073 } else { 4074 /* TODO: kernel consumer */ 4075 } 4076 4077 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); 4078 if (rc) { 4079 dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n", 4080 cq->qplib_cq.id); 4081 goto fail; 4082 } 4083 4084 cq->ibcq.cqe = cq->resize_cqe; 4085 /* For kernel consumers complete resize here. For uverbs consumers, 4086 * we complete it in the context of ibv_poll_cq(). 4087 */ 4088 if (!cq->resize_umem) 4089 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); 4090 4091 atomic_inc(&rdev->stats.rsors.resize_count); 4092 return 0; 4093 4094 fail: 4095 if (cq->resize_umem) { 4096 ib_umem_release(cq->resize_umem); 4097 cq->resize_umem = NULL; 4098 cq->resize_cqe = 0; 4099 memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo)); 4100 cq->qplib_cq.dpi = orig_dpi; 4101 } 4102 return rc; 4103 } 4104 4105 static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus) 4106 { 4107 switch(qstatus) { 4108 case CQ_REQ_STATUS_OK: 4109 return IB_WC_SUCCESS; 4110 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 4111 return IB_WC_BAD_RESP_ERR; 4112 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 4113 return IB_WC_LOC_LEN_ERR; 4114 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 4115 return IB_WC_LOC_QP_OP_ERR; 4116 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 4117 return IB_WC_LOC_PROT_ERR; 4118 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 4119 return IB_WC_GENERAL_ERR; 4120 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 4121 return IB_WC_REM_INV_REQ_ERR; 4122 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 4123 return IB_WC_REM_ACCESS_ERR; 4124 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 4125 return IB_WC_REM_OP_ERR; 4126 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 4127 return IB_WC_RNR_RETRY_EXC_ERR; 4128 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 4129 return IB_WC_RETRY_EXC_ERR; 4130 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 4131 return IB_WC_WR_FLUSH_ERR; 4132 default: 4133 return IB_WC_GENERAL_ERR; 4134 } 4135 return 0; 4136 } 4137 4138 static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus) 4139 { 4140 switch(qstatus) { 4141 case CQ_RES_RAWETH_QP1_STATUS_OK: 4142 return IB_WC_SUCCESS; 4143 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 4144 return IB_WC_LOC_ACCESS_ERR; 4145 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 4146 return IB_WC_LOC_LEN_ERR; 4147 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 4148 return IB_WC_LOC_PROT_ERR; 4149 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 4150 return IB_WC_LOC_QP_OP_ERR; 4151 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 4152 return IB_WC_GENERAL_ERR; 4153 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 4154 return IB_WC_WR_FLUSH_ERR; 4155 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 4156 return IB_WC_WR_FLUSH_ERR; 4157 default: 4158 return IB_WC_GENERAL_ERR; 4159 } 4160 } 4161 4162 static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus) 4163 { 4164 switch(qstatus) { 4165 case CQ_RES_RC_STATUS_OK: 4166 return IB_WC_SUCCESS; 4167 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 4168 return IB_WC_LOC_ACCESS_ERR; 4169 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 4170 return IB_WC_LOC_LEN_ERR; 4171 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 4172 return IB_WC_LOC_PROT_ERR; 4173 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 4174 return IB_WC_LOC_QP_OP_ERR; 4175 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 4176 return IB_WC_GENERAL_ERR; 4177 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 4178 return IB_WC_REM_INV_REQ_ERR; 4179 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 4180 return IB_WC_WR_FLUSH_ERR; 4181 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 4182 return IB_WC_WR_FLUSH_ERR; 4183 default: 4184 return IB_WC_GENERAL_ERR; 4185 } 4186 } 4187 4188 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 4189 { 4190 switch (cqe->type) { 4191 case BNXT_QPLIB_SWQE_TYPE_SEND: 4192 wc->opcode = IB_WC_SEND; 4193 break; 4194 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 4195 wc->opcode = IB_WC_SEND; 4196 wc->wc_flags |= IB_WC_WITH_IMM; 4197 break; 4198 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 4199 wc->opcode = IB_WC_SEND; 4200 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4201 break; 4202 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 4203 wc->opcode = IB_WC_RDMA_WRITE; 4204 break; 4205 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 4206 wc->opcode = IB_WC_RDMA_WRITE; 4207 wc->wc_flags |= IB_WC_WITH_IMM; 4208 break; 4209 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 4210 wc->opcode = IB_WC_RDMA_READ; 4211 break; 4212 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 4213 wc->opcode = IB_WC_COMP_SWAP; 4214 break; 4215 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 4216 wc->opcode = IB_WC_FETCH_ADD; 4217 break; 4218 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 4219 wc->opcode = IB_WC_LOCAL_INV; 4220 break; 4221 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 4222 wc->opcode = IB_WC_REG_MR; 4223 break; 4224 default: 4225 wc->opcode = IB_WC_SEND; 4226 break; 4227 } 4228 4229 wc->status = __req_to_ib_wc_status(cqe->status); 4230 } 4231 4232 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2) 4233 { 4234 bool is_ipv6 = false, is_ipv4 = false; 4235 4236 /* raweth_qp1_flags Bit 9-6 indicates itype */ 4237 4238 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 4239 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 4240 return -1; 4241 4242 if (raweth_qp1_flags2 & 4243 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 4244 raweth_qp1_flags2 & 4245 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 4246 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 4247 (raweth_qp1_flags2 & 4248 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 4249 (is_ipv6 = true) : (is_ipv4 = true); 4250 return ((is_ipv6) ? 4251 BNXT_RE_ROCEV2_IPV6_PACKET : 4252 BNXT_RE_ROCEV2_IPV4_PACKET); 4253 } else { 4254 return BNXT_RE_ROCE_V1_PACKET; 4255 } 4256 } 4257 4258 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 4259 void *rq_hdr_buf) 4260 { 4261 u8 *tmp_buf = NULL; 4262 struct ethhdr *eth_hdr; 4263 u16 eth_type; 4264 bool rc = false; 4265 4266 tmp_buf = (u8 *)rq_hdr_buf; 4267 /* 4268 * If dest mac is not same as I/F mac, this could be a 4269 * loopback address or multicast address, check whether 4270 * it is a loopback packet 4271 */ 4272 if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) { 4273 tmp_buf += 4; 4274 /* Check the ether type */ 4275 eth_hdr = (struct ethhdr *)tmp_buf; 4276 eth_type = ntohs(eth_hdr->h_proto); 4277 switch (eth_type) { 4278 case BNXT_QPLIB_ETHTYPE_ROCEV1: 4279 rc = true; 4280 break; 4281 default: 4282 break; 4283 } 4284 } 4285 4286 return rc; 4287 } 4288 4289 static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev, 4290 void *rq_hdr_buf, 4291 struct bnxt_qplib_cqe *cqe) 4292 { 4293 struct vlan_hdr *vlan_hdr; 4294 struct ethhdr *eth_hdr; 4295 u8 *tmp_buf = NULL; 4296 u16 eth_type; 4297 4298 tmp_buf = (u8 *)rq_hdr_buf; 4299 /* Check the ether type */ 4300 eth_hdr = (struct ethhdr *)tmp_buf; 4301 eth_type = ntohs(eth_hdr->h_proto); 4302 if (eth_type == ETH_P_8021Q) { 4303 tmp_buf += sizeof(struct ethhdr); 4304 vlan_hdr = (struct vlan_hdr *)tmp_buf; 4305 cqe->raweth_qp1_metadata = 4306 ntohs(vlan_hdr->h_vlan_TCI) | 4307 (eth_type << 4308 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 4309 cqe->raweth_qp1_flags2 |= 4310 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN; 4311 return true; 4312 } 4313 4314 return false; 4315 } 4316 4317 static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp, 4318 struct bnxt_qplib_cqe *cqe) 4319 { 4320 struct bnxt_re_sqp_entries *sqp_entry = NULL; 4321 struct bnxt_qplib_hdrbuf *hdr_buf; 4322 dma_addr_t shrq_hdr_buf_map; 4323 struct ib_sge s_sge[2] = {}; 4324 struct ib_sge r_sge[2] = {}; 4325 struct ib_recv_wr rwr = {}; 4326 struct bnxt_re_ah *gsi_sah; 4327 struct bnxt_re_qp *gsi_sqp; 4328 dma_addr_t rq_hdr_buf_map; 4329 struct bnxt_re_dev *rdev; 4330 struct ib_send_wr *swr; 4331 u32 skip_bytes = 0; 4332 void *rq_hdr_buf; 4333 int pkt_type = 0; 4334 u32 offset = 0; 4335 u32 tbl_idx; 4336 int rc; 4337 struct ib_ud_wr udwr = {}; 4338 4339 swr = &udwr.wr; 4340 rdev = gsi_qp->rdev; 4341 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 4342 tbl_idx = cqe->wr_id; 4343 4344 hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf; 4345 rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step; 4346 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 4347 tbl_idx); 4348 /* Shadow QP header buffer */ 4349 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp, 4350 tbl_idx); 4351 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 4352 4353 /* Find packet type from the cqe */ 4354 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 4355 cqe->raweth_qp1_flags2); 4356 if (pkt_type < 0) { 4357 dev_err(rdev_to_dev(rdev), "Not handling this packet\n"); 4358 return -EINVAL; 4359 } 4360 4361 /* Adjust the offset for the user buffer and post in the rq */ 4362 4363 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 4364 offset = 20; 4365 4366 /* 4367 * QP1 loopback packet has 4 bytes of internal header before 4368 * ether header. Skip these four bytes. 4369 */ 4370 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 4371 skip_bytes = 4; 4372 4373 if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe)) 4374 skip_bytes += VLAN_HLEN; 4375 4376 /* Store this cqe */ 4377 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 4378 sqp_entry->qp1_qp = gsi_qp; 4379 4380 /* First send SGE . Skip the ether header*/ 4381 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 4382 + skip_bytes; 4383 s_sge[0].lkey = 0xFFFFFFFF; 4384 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 4385 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 4386 4387 /* Second Send SGE */ 4388 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 4389 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 4390 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 4391 s_sge[1].addr += 8; 4392 s_sge[1].lkey = 0xFFFFFFFF; 4393 s_sge[1].length = 256; 4394 4395 /* First recv SGE */ 4396 r_sge[0].addr = shrq_hdr_buf_map; 4397 r_sge[0].lkey = 0xFFFFFFFF; 4398 r_sge[0].length = 40; 4399 4400 r_sge[1].addr = sqp_entry->sge.addr + offset; 4401 r_sge[1].lkey = sqp_entry->sge.lkey; 4402 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 4403 4404 /* Create receive work request */ 4405 rwr.num_sge = 2; 4406 rwr.sg_list = r_sge; 4407 rwr.wr_id = tbl_idx; 4408 rwr.next = NULL; 4409 4410 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); 4411 if (rc) { 4412 dev_err(rdev_to_dev(rdev), 4413 "Failed to post Rx buffers to shadow QP\n"); 4414 return -ENOMEM; 4415 } 4416 4417 swr->num_sge = 2; 4418 swr->sg_list = s_sge; 4419 swr->wr_id = tbl_idx; 4420 swr->opcode = IB_WR_SEND; 4421 swr->next = NULL; 4422 4423 gsi_sah = rdev->gsi_ctx.gsi_sah; 4424 udwr.ah = &gsi_sah->ibah; 4425 udwr.remote_qpn = gsi_sqp->qplib_qp.id; 4426 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; 4427 /* post data received in the send queue */ 4428 rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); 4429 4430 return rc; 4431 } 4432 4433 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 4434 struct bnxt_qplib_cqe *cqe) 4435 { 4436 wc->opcode = IB_WC_RECV; 4437 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 4438 wc->wc_flags |= IB_WC_GRH; 4439 } 4440 4441 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 4442 struct bnxt_qplib_cqe *cqe) 4443 { 4444 wc->opcode = IB_WC_RECV; 4445 wc->status = __rc_to_ib_wc_status(cqe->status); 4446 4447 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 4448 wc->wc_flags |= IB_WC_WITH_IMM; 4449 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 4450 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4451 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 4452 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 4453 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4454 } 4455 4456 /* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */ 4457 static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe, 4458 u16 *vid, u8 *sl) 4459 { 4460 u32 metadata; 4461 u16 tpid; 4462 bool ret = false; 4463 metadata = orig_cqe->raweth_qp1_metadata; 4464 if (orig_cqe->raweth_qp1_flags2 & 4465 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 4466 tpid = ((metadata & 4467 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 4468 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 4469 if (tpid == ETH_P_8021Q) { 4470 *vid = metadata & 4471 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 4472 *sl = (metadata & 4473 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 4474 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 4475 ret = !!(*vid); 4476 } 4477 } 4478 4479 return ret; 4480 } 4481 4482 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, 4483 struct ib_wc *wc, 4484 struct bnxt_qplib_cqe *cqe) 4485 { 4486 u32 tbl_idx; 4487 struct bnxt_re_dev *rdev = gsi_sqp->rdev; 4488 struct bnxt_re_qp *gsi_qp = NULL; 4489 struct bnxt_qplib_cqe *orig_cqe = NULL; 4490 struct bnxt_re_sqp_entries *sqp_entry = NULL; 4491 int nw_type; 4492 u16 vlan_id; 4493 u8 sl; 4494 4495 tbl_idx = cqe->wr_id; 4496 4497 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 4498 gsi_qp = sqp_entry->qp1_qp; 4499 orig_cqe = &sqp_entry->cqe; 4500 4501 wc->wr_id = sqp_entry->wrid; 4502 wc->byte_len = orig_cqe->length; 4503 wc->qp = &gsi_qp->ib_qp; 4504 4505 wc->ex.imm_data = orig_cqe->immdata; 4506 wc->src_qp = orig_cqe->src_qp; 4507 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 4508 if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) { 4509 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 4510 wc->sl = sl; 4511 wc->vlan_id = vlan_id; 4512 wc->wc_flags |= IB_WC_WITH_VLAN; 4513 } 4514 } 4515 wc->port_num = 1; 4516 wc->vendor_err = orig_cqe->status; 4517 4518 wc->opcode = IB_WC_RECV; 4519 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 4520 wc->wc_flags |= IB_WC_GRH; 4521 4522 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 4523 orig_cqe->raweth_qp1_flags2); 4524 if(nw_type >= 0) 4525 dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type); 4526 } 4527 4528 static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev, 4529 struct bnxt_re_qp *qp, struct ib_wc *wc, 4530 struct bnxt_qplib_cqe *cqe) 4531 { 4532 u16 vlan_id = 0; 4533 4534 wc->opcode = IB_WC_RECV; 4535 wc->status = __rc_to_ib_wc_status(cqe->status); 4536 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) 4537 wc->wc_flags |= IB_WC_WITH_IMM; 4538 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 4539 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4540 /* report only on GSI QP for Thor */ 4541 if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id && 4542 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) { 4543 wc->wc_flags |= IB_WC_GRH; 4544 memcpy(wc->smac, cqe->smac, ETH_ALEN); 4545 wc->wc_flags |= IB_WC_WITH_SMAC; 4546 if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) { 4547 if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) { 4548 if (cqe->cfa_meta & 4549 BNXT_QPLIB_CQE_CFA_META1_VALID) 4550 vlan_id = (cqe->cfa_meta & 0xFFF); 4551 } 4552 } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { 4553 vlan_id = (cqe->cfa_meta & 0xFFF); 4554 } 4555 /* Mark only if vlan_id is non zero */ 4556 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 4557 wc->vlan_id = vlan_id; 4558 wc->wc_flags |= IB_WC_WITH_VLAN; 4559 } 4560 } 4561 } 4562 4563 static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp) 4564 { 4565 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 4566 unsigned long flags; 4567 int rc = 0; 4568 4569 spin_lock_irqsave(&qp->sq_lock, flags); 4570 4571 rc = bnxt_re_legacy_bind_fence_mw(lib_qp); 4572 if (!rc) { 4573 lib_qp->sq.phantom_wqe_cnt++; 4574 dev_dbg(&lib_qp->sq.hwq.pdev->dev, 4575 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 4576 lib_qp->id, lib_qp->sq.hwq.prod, 4577 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 4578 lib_qp->sq.phantom_wqe_cnt); 4579 } 4580 4581 spin_unlock_irqrestore(&qp->sq_lock, flags); 4582 return rc; 4583 } 4584 4585 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 4586 { 4587 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 4588 struct bnxt_re_dev *rdev = cq->rdev; 4589 struct bnxt_re_qp *qp; 4590 struct bnxt_qplib_cqe *cqe; 4591 int i, ncqe, budget, init_budget; 4592 struct bnxt_qplib_q *sq; 4593 struct bnxt_qplib_qp *lib_qp; 4594 u32 tbl_idx; 4595 struct bnxt_re_sqp_entries *sqp_entry = NULL; 4596 unsigned long flags; 4597 u8 gsi_mode; 4598 4599 /* 4600 * DB recovery CQ; only process the door bell pacing alert from 4601 * the user lib 4602 */ 4603 if (cq->is_dbr_soft_cq) { 4604 bnxt_re_pacing_alert(rdev); 4605 return 0; 4606 } 4607 4608 /* User CQ; the only processing we do is to 4609 * complete any pending CQ resize operation. 4610 */ 4611 if (cq->umem) { 4612 if (cq->resize_umem) 4613 bnxt_re_resize_cq_complete(cq); 4614 return 0; 4615 } 4616 4617 spin_lock_irqsave(&cq->cq_lock, flags); 4618 4619 budget = min_t(u32, num_entries, cq->max_cql); 4620 init_budget = budget; 4621 if (!cq->cql) { 4622 dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n"); 4623 goto exit; 4624 } 4625 cqe = &cq->cql[0]; 4626 gsi_mode = rdev->gsi_ctx.gsi_qp_mode; 4627 while (budget) { 4628 lib_qp = NULL; 4629 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 4630 if (lib_qp) { 4631 sq = &lib_qp->sq; 4632 if (sq->legacy_send_phantom == true) { 4633 qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp); 4634 if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM) 4635 dev_err(rdev_to_dev(rdev), 4636 "Phantom failed! Scheduled to send again\n"); 4637 else 4638 sq->legacy_send_phantom = false; 4639 } 4640 } 4641 if (ncqe < budget) 4642 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 4643 cqe + ncqe, 4644 budget - ncqe); 4645 4646 if (!ncqe) 4647 break; 4648 4649 for (i = 0; i < ncqe; i++, cqe++) { 4650 /* Transcribe each qplib_wqe back to ib_wc */ 4651 memset(wc, 0, sizeof(*wc)); 4652 4653 wc->wr_id = cqe->wr_id; 4654 wc->byte_len = cqe->length; 4655 qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle, 4656 struct bnxt_re_qp, qplib_qp); 4657 if (!qp) { 4658 dev_err(rdev_to_dev(rdev), 4659 "POLL CQ bad QP handle\n"); 4660 continue; 4661 } 4662 wc->qp = &qp->ib_qp; 4663 wc->ex.imm_data = cqe->immdata; 4664 wc->src_qp = cqe->src_qp; 4665 memcpy(wc->smac, cqe->smac, ETH_ALEN); 4666 wc->port_num = 1; 4667 wc->vendor_err = cqe->status; 4668 4669 switch(cqe->opcode) { 4670 case CQ_BASE_CQE_TYPE_REQ: 4671 if (gsi_mode == BNXT_RE_GSI_MODE_ALL && 4672 qp->qplib_qp.id == 4673 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { 4674 /* Handle this completion with 4675 * the stored completion */ 4676 dev_dbg(rdev_to_dev(rdev), 4677 "Skipping this UD Send CQ\n"); 4678 memset(wc, 0, sizeof(*wc)); 4679 continue; 4680 } 4681 bnxt_re_process_req_wc(wc, cqe); 4682 break; 4683 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 4684 if (gsi_mode == BNXT_RE_GSI_MODE_ALL) { 4685 if (!cqe->status) { 4686 int rc = 0; 4687 rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe); 4688 if (!rc) { 4689 memset(wc, 0, 4690 sizeof(*wc)); 4691 continue; 4692 } 4693 cqe->status = -1; 4694 } 4695 /* Errors need not be looped back. 4696 * But change the wr_id to the one 4697 * stored in the table 4698 */ 4699 tbl_idx = cqe->wr_id; 4700 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 4701 wc->wr_id = sqp_entry->wrid; 4702 } 4703 4704 bnxt_re_process_res_rawqp1_wc(wc, cqe); 4705 break; 4706 case CQ_BASE_CQE_TYPE_RES_RC: 4707 bnxt_re_process_res_rc_wc(wc, cqe); 4708 break; 4709 case CQ_BASE_CQE_TYPE_RES_UD: 4710 if (gsi_mode == BNXT_RE_GSI_MODE_ALL && 4711 qp->qplib_qp.id == 4712 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { 4713 /* Handle this completion with 4714 * the stored completion 4715 */ 4716 dev_dbg(rdev_to_dev(rdev), 4717 "Handling the UD receive CQ\n"); 4718 if (cqe->status) { 4719 /* TODO handle this completion as a failure in 4720 * loopback porocedure 4721 */ 4722 continue; 4723 } else { 4724 bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe); 4725 break; 4726 } 4727 } 4728 bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe); 4729 break; 4730 default: 4731 dev_err(rdev_to_dev(cq->rdev), 4732 "POLL CQ type 0x%x not handled, skip!\n", 4733 cqe->opcode); 4734 continue; 4735 } 4736 wc++; 4737 budget--; 4738 } 4739 } 4740 exit: 4741 spin_unlock_irqrestore(&cq->cq_lock, flags); 4742 return init_budget - budget; 4743 } 4744 4745 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 4746 enum ib_cq_notify_flags ib_cqn_flags) 4747 { 4748 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 4749 int type = 0, rc = 0; 4750 unsigned long flags; 4751 4752 spin_lock_irqsave(&cq->cq_lock, flags); 4753 /* Trigger on the very next completion */ 4754 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 4755 type = DBC_DBC_TYPE_CQ_ARMALL; 4756 /* Trigger on the next solicited completion */ 4757 else if (ib_cqn_flags & IB_CQ_SOLICITED) 4758 type = DBC_DBC_TYPE_CQ_ARMSE; 4759 4760 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 4761 4762 /* Poll to see if there are missed events */ 4763 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 4764 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) 4765 rc = 1; 4766 4767 spin_unlock_irqrestore(&cq->cq_lock, flags); 4768 4769 return rc; 4770 } 4771 4772 /* Memory Regions */ 4773 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 4774 { 4775 struct bnxt_qplib_mrinfo mrinfo; 4776 struct bnxt_re_dev *rdev; 4777 struct bnxt_re_mr *mr; 4778 struct bnxt_re_pd *pd; 4779 u32 max_mr_count; 4780 u64 pbl = 0; 4781 int rc; 4782 4783 memset(&mrinfo, 0, sizeof(mrinfo)); 4784 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 4785 rdev = pd->rdev; 4786 4787 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 4788 if (!mr) { 4789 dev_err(rdev_to_dev(rdev), 4790 "Allocate memory for DMA MR failed!\n"); 4791 return ERR_PTR(-ENOMEM); 4792 } 4793 mr->rdev = rdev; 4794 mr->qplib_mr.pd = &pd->qplib_pd; 4795 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 4796 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 4797 4798 /* Allocate and register 0 as the address */ 4799 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 4800 if (rc) { 4801 dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n"); 4802 goto fail; 4803 } 4804 mr->qplib_mr.total_size = -1; /* Infinite length */ 4805 mrinfo.ptes = &pbl; 4806 mrinfo.sg.npages = 0; 4807 mrinfo.sg.pgsize = PAGE_SIZE; 4808 mrinfo.sg.pgshft = PAGE_SHIFT; 4809 mrinfo.sg.pgsize = PAGE_SIZE; 4810 mrinfo.mrw = &mr->qplib_mr; 4811 mrinfo.is_dma = true; 4812 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 4813 if (rc) { 4814 dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n"); 4815 goto fail_mr; 4816 } 4817 mr->ib_mr.lkey = mr->qplib_mr.lkey; 4818 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 4819 IB_ACCESS_REMOTE_ATOMIC)) 4820 mr->ib_mr.rkey = mr->ib_mr.lkey; 4821 atomic_inc(&rdev->stats.rsors.mr_count); 4822 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 4823 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) 4824 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 4825 4826 return &mr->ib_mr; 4827 4828 fail_mr: 4829 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4830 fail: 4831 kfree(mr); 4832 return ERR_PTR(rc); 4833 } 4834 4835 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 4836 { 4837 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 4838 struct bnxt_re_dev *rdev = mr->rdev; 4839 int rc = 0; 4840 4841 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4842 if (rc) 4843 dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n", 4844 mr->qplib_mr.lkey, rc); 4845 4846 if (mr->pages) { 4847 bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 4848 &mr->qplib_frpl); 4849 kfree(mr->pages); 4850 mr->npages = 0; 4851 mr->pages = NULL; 4852 } 4853 if (!IS_ERR(mr->ib_umem) && mr->ib_umem) { 4854 mr->is_invalcb_active = false; 4855 bnxt_re_peer_mem_release(mr->ib_umem); 4856 } 4857 kfree(mr); 4858 atomic_dec(&rdev->stats.rsors.mr_count); 4859 return 0; 4860 } 4861 4862 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 4863 { 4864 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 4865 4866 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 4867 return -ENOMEM; 4868 4869 mr->pages[mr->npages++] = addr; 4870 dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n", 4871 ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1, 4872 mr->pages[mr->npages - 1]); 4873 return 0; 4874 } 4875 4876 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, 4877 int sg_nents, unsigned int *sg_offset) 4878 { 4879 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 4880 4881 mr->npages = 0; 4882 return ib_sg_to_pages(ib_mr, sg, sg_nents, 4883 sg_offset, bnxt_re_set_page); 4884 } 4885 4886 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 4887 u32 max_num_sg, struct ib_udata *udata) 4888 { 4889 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 4890 struct bnxt_re_dev *rdev = pd->rdev; 4891 struct bnxt_re_mr *mr; 4892 u32 max_mr_count; 4893 int rc; 4894 4895 dev_dbg(rdev_to_dev(rdev), "Alloc MR\n"); 4896 if (type != IB_MR_TYPE_MEM_REG) { 4897 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type); 4898 return ERR_PTR(-EINVAL); 4899 } 4900 if (max_num_sg > MAX_PBL_LVL_1_PGS) { 4901 dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n"); 4902 return ERR_PTR(-EINVAL); 4903 } 4904 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 4905 if (!mr) { 4906 dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n"); 4907 return ERR_PTR(-ENOMEM); 4908 } 4909 mr->rdev = rdev; 4910 mr->qplib_mr.pd = &pd->qplib_pd; 4911 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 4912 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 4913 4914 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 4915 if (rc) { 4916 dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); 4917 goto fail; 4918 } 4919 mr->ib_mr.lkey = mr->qplib_mr.lkey; 4920 mr->ib_mr.rkey = mr->ib_mr.lkey; 4921 mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL); 4922 if (!mr->pages) { 4923 dev_err(rdev_to_dev(rdev), 4924 "Allocate MR page list mem failed!\n"); 4925 rc = -ENOMEM; 4926 goto fail_mr; 4927 } 4928 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 4929 &mr->qplib_frpl, max_num_sg); 4930 if (rc) { 4931 dev_err(rdev_to_dev(rdev), 4932 "Allocate HW Fast reg page list failed!\n"); 4933 goto free_page; 4934 } 4935 dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages); 4936 4937 atomic_inc(&rdev->stats.rsors.mr_count); 4938 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 4939 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) 4940 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 4941 return &mr->ib_mr; 4942 4943 free_page: 4944 kfree(mr->pages); 4945 fail_mr: 4946 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4947 fail: 4948 kfree(mr); 4949 return ERR_PTR(rc); 4950 } 4951 4952 /* Memory Windows */ 4953 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 4954 struct ib_udata *udata) 4955 { 4956 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 4957 struct bnxt_re_dev *rdev = pd->rdev; 4958 struct bnxt_re_mw *mw; 4959 u32 max_mw_count; 4960 int rc; 4961 4962 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 4963 if (!mw) { 4964 dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); 4965 rc = -ENOMEM; 4966 goto exit; 4967 } 4968 mw->rdev = rdev; 4969 mw->qplib_mw.pd = &pd->qplib_pd; 4970 4971 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 4972 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 4973 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 4974 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 4975 if (rc) { 4976 dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); 4977 goto fail; 4978 } 4979 mw->ib_mw.rkey = mw->qplib_mw.rkey; 4980 atomic_inc(&rdev->stats.rsors.mw_count); 4981 max_mw_count = atomic_read(&rdev->stats.rsors.mw_count); 4982 if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count)) 4983 atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count); 4984 4985 return &mw->ib_mw; 4986 fail: 4987 kfree(mw); 4988 exit: 4989 return ERR_PTR(rc); 4990 } 4991 4992 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 4993 { 4994 struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw); 4995 struct bnxt_re_dev *rdev = mw->rdev; 4996 int rc; 4997 4998 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 4999 if (rc) { 5000 dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); 5001 return rc; 5002 } 5003 5004 kfree(mw); 5005 atomic_dec(&rdev->stats.rsors.mw_count); 5006 return rc; 5007 } 5008 5009 static int bnxt_re_page_size_ok(int page_shift) 5010 { 5011 switch (page_shift) { 5012 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: 5013 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: 5014 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: 5015 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: 5016 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: 5017 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: 5018 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: 5019 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB: 5020 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: 5021 return 1; 5022 default: 5023 return 0; 5024 } 5025 } 5026 5027 static int bnxt_re_get_page_shift(struct ib_umem *umem, 5028 u64 va, u64 st, u64 cmask) 5029 { 5030 int pgshft; 5031 5032 pgshft = ilog2(umem->page_size); 5033 5034 return pgshft; 5035 } 5036 5037 static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift) 5038 { 5039 int npages = 0; 5040 5041 if (page_shift == PAGE_SHIFT) { 5042 npages = ib_umem_num_pages_compat(umem); 5043 } else { 5044 npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift); 5045 if (start % BIT(page_shift)) 5046 npages++; 5047 } 5048 return npages; 5049 } 5050 5051 /* uverbs */ 5052 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 5053 u64 virt_addr, int mr_access_flags, 5054 struct ib_udata *udata) 5055 { 5056 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 5057 struct bnxt_re_dev *rdev = pd->rdev; 5058 struct bnxt_qplib_mrinfo mrinfo; 5059 int umem_pgs, page_shift, rc; 5060 struct bnxt_re_mr *mr; 5061 struct ib_umem *umem; 5062 u32 max_mr_count; 5063 int npages; 5064 5065 dev_dbg(rdev_to_dev(rdev), "Reg user MR\n"); 5066 5067 if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr) 5068 return ERR_PTR(-ENOMEM); 5069 5070 if (rdev->mod_exit) { 5071 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 5072 return ERR_PTR(-EIO); 5073 } 5074 memset(&mrinfo, 0, sizeof(mrinfo)); 5075 if (length > BNXT_RE_MAX_MR_SIZE) { 5076 dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu " 5077 "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE); 5078 return ERR_PTR(-ENOMEM); 5079 } 5080 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5081 if (!mr) { 5082 dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); 5083 return ERR_PTR (-ENOMEM); 5084 } 5085 mr->rdev = rdev; 5086 mr->qplib_mr.pd = &pd->qplib_pd; 5087 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 5088 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 5089 5090 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { 5091 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 5092 if (rc) { 5093 dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n"); 5094 goto fail; 5095 } 5096 /* The fixed portion of the rkey is the same as the lkey */ 5097 mr->ib_mr.rkey = mr->qplib_mr.rkey; 5098 } 5099 5100 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, 5101 udata, start, length, 5102 mr_access_flags, 0); 5103 if (IS_ERR(umem)) { 5104 rc = PTR_ERR(umem); 5105 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", 5106 __func__, rc); 5107 goto free_mr; 5108 } 5109 mr->ib_umem = umem; 5110 5111 mr->qplib_mr.va = virt_addr; 5112 umem_pgs = ib_umem_num_pages_compat(umem); 5113 if (!umem_pgs) { 5114 dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); 5115 rc = -EINVAL; 5116 goto free_umem; 5117 } 5118 mr->qplib_mr.total_size = length; 5119 page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, 5120 rdev->dev_attr->page_size_cap); 5121 if (!bnxt_re_page_size_ok(page_shift)) { 5122 dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n"); 5123 rc = -EFAULT; 5124 goto free_umem; 5125 } 5126 npages = bnxt_re_get_num_pages(umem, start, length, page_shift); 5127 5128 /* Map umem buf ptrs to the PBL */ 5129 mrinfo.sg.npages = npages; 5130 mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); 5131 mrinfo.sg.pgshft = page_shift; 5132 mrinfo.sg.pgsize = BIT(page_shift); 5133 5134 mrinfo.mrw = &mr->qplib_mr; 5135 5136 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 5137 if (rc) { 5138 dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n"); 5139 goto free_umem; 5140 } 5141 5142 mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey; 5143 atomic_inc(&rdev->stats.rsors.mr_count); 5144 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 5145 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) 5146 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 5147 5148 return &mr->ib_mr; 5149 5150 free_umem: 5151 bnxt_re_peer_mem_release(mr->ib_umem); 5152 free_mr: 5153 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) 5154 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 5155 fail: 5156 kfree(mr); 5157 return ERR_PTR(rc); 5158 } 5159 5160 int 5161 bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, 5162 u64 virt_addr, int mr_access_flags, 5163 struct ib_pd *ib_pd, struct ib_udata *udata) 5164 { 5165 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 5166 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 5167 int umem_pgs = 0, page_shift = PAGE_SHIFT, rc; 5168 struct bnxt_re_dev *rdev = mr->rdev; 5169 struct bnxt_qplib_mrinfo mrinfo; 5170 struct ib_umem *umem; 5171 u32 npages; 5172 5173 /* TODO: Must decipher what to modify based on the flags */ 5174 memset(&mrinfo, 0, sizeof(mrinfo)); 5175 if (flags & IB_MR_REREG_TRANS) { 5176 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, 5177 udata, start, length, 5178 mr_access_flags, 0); 5179 if (IS_ERR(umem)) { 5180 rc = PTR_ERR(umem); 5181 dev_err(rdev_to_dev(rdev), 5182 "%s: ib_umem_get failed! ret = %d\n", 5183 __func__, rc); 5184 goto fail; 5185 } 5186 mr->ib_umem = umem; 5187 5188 mr->qplib_mr.va = virt_addr; 5189 umem_pgs = ib_umem_num_pages_compat(umem); 5190 if (!umem_pgs) { 5191 dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); 5192 rc = -EINVAL; 5193 goto fail_free_umem; 5194 } 5195 mr->qplib_mr.total_size = length; 5196 page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, 5197 rdev->dev_attr->page_size_cap); 5198 if (!bnxt_re_page_size_ok(page_shift)) { 5199 dev_err(rdev_to_dev(rdev), 5200 "umem page size unsupported!\n"); 5201 rc = -EFAULT; 5202 goto fail_free_umem; 5203 } 5204 npages = bnxt_re_get_num_pages(umem, start, length, page_shift); 5205 /* Map umem buf ptrs to the PBL */ 5206 mrinfo.sg.npages = npages; 5207 mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); 5208 mrinfo.sg.pgshft = page_shift; 5209 mrinfo.sg.pgsize = BIT(page_shift); 5210 } 5211 5212 mrinfo.mrw = &mr->qplib_mr; 5213 if (flags & IB_MR_REREG_PD) 5214 mr->qplib_mr.pd = &pd->qplib_pd; 5215 5216 if (flags & IB_MR_REREG_ACCESS) 5217 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 5218 5219 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 5220 if (rc) { 5221 dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n"); 5222 goto fail_free_umem; 5223 } 5224 mr->ib_mr.rkey = mr->qplib_mr.rkey; 5225 5226 return 0; 5227 5228 fail_free_umem: 5229 bnxt_re_peer_mem_release(mr->ib_umem); 5230 fail: 5231 return rc; 5232 } 5233 5234 static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev) 5235 { 5236 struct ib_device *ibdev = &rdev->ibdev; 5237 u32 uverbs_abi_ver; 5238 5239 uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev); 5240 dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n", 5241 uverbs_abi_ver); 5242 if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 5243 dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n", 5244 BNXT_RE_ABI_VERSION); 5245 return -EPERM; 5246 } 5247 return 0; 5248 } 5249 5250 int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, 5251 struct ib_udata *udata) 5252 { 5253 struct ib_ucontext *ctx = uctx_in; 5254 struct ib_device *ibdev = ctx->device; 5255 struct bnxt_re_ucontext *uctx = 5256 container_of(ctx, struct bnxt_re_ucontext, ibucontext); 5257 5258 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 5259 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; 5260 struct bnxt_re_uctx_resp resp = {}; 5261 struct bnxt_re_uctx_req ureq = {}; 5262 struct bnxt_qplib_chip_ctx *cctx; 5263 u32 chip_met_rev_num; 5264 bool genp5 = false; 5265 int rc; 5266 5267 cctx = rdev->chip_ctx; 5268 rc = bnxt_re_check_abi_version(rdev); 5269 if (rc) 5270 goto fail; 5271 5272 uctx->rdev = rdev; 5273 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 5274 if (!uctx->shpg) { 5275 dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n"); 5276 rc = -ENOMEM; 5277 goto fail; 5278 } 5279 spin_lock_init(&uctx->sh_lock); 5280 if (BNXT_RE_ABI_VERSION >= 4) { 5281 chip_met_rev_num = cctx->chip_num; 5282 chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) << 5283 BNXT_RE_CHIP_ID0_CHIP_REV_SFT; 5284 chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) << 5285 BNXT_RE_CHIP_ID0_CHIP_MET_SFT; 5286 resp.chip_id0 = chip_met_rev_num; 5287 resp.chip_id1 = 0; /* future extension of chip info */ 5288 } 5289 5290 if (BNXT_RE_ABI_VERSION != 4) { 5291 /*Temp, Use idr_alloc instead*/ 5292 resp.dev_id = rdev->en_dev->pdev->devfn; 5293 resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max; 5294 } 5295 5296 genp5 = _is_chip_gen_p5_p7(cctx); 5297 if (BNXT_RE_ABI_VERSION > 5) { 5298 resp.modes = genp5 ? cctx->modes.wqe_mode : 0; 5299 if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) 5300 resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; 5301 } 5302 5303 resp.pg_size = PAGE_SIZE; 5304 resp.cqe_sz = sizeof(struct cq_base); 5305 resp.max_cqd = dev_attr->max_cq_wqes; 5306 if (genp5 && cctx->modes.db_push) { 5307 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; 5308 if (_is_chip_p7(cctx) && 5309 !(dev_attr->dev_cap_flags & 5310 CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE)) 5311 resp.comp_mask &= 5312 ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; 5313 } 5314 5315 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED; 5316 5317 if (rdev->dbr_pacing) 5318 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED; 5319 5320 if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov) 5321 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED; 5322 5323 if (udata->inlen >= sizeof(ureq)) { 5324 rc = ib_copy_from_udata(&ureq, udata, 5325 min(udata->inlen, sizeof(ureq))); 5326 if (rc) 5327 goto cfail; 5328 if (bnxt_re_init_pow2_flag(&ureq, &resp)) 5329 dev_warn(rdev_to_dev(rdev), 5330 "Enabled roundup logic. Library bug?\n"); 5331 if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5)) 5332 dev_warn(rdev_to_dev(rdev), 5333 "Rsvd wqe in use! Try the updated library.\n"); 5334 } else { 5335 dev_warn(rdev_to_dev(rdev), 5336 "Enabled roundup logic. Update the library!\n"); 5337 resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; 5338 5339 dev_warn(rdev_to_dev(rdev), 5340 "Rsvd wqe in use. Update the library!\n"); 5341 resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; 5342 } 5343 5344 uctx->cmask = (uint64_t)resp.comp_mask; 5345 rc = bnxt_re_copy_to_udata(rdev, &resp, 5346 min(udata->outlen, sizeof(resp)), 5347 udata); 5348 if (rc) 5349 goto cfail; 5350 5351 INIT_LIST_HEAD(&uctx->cq_list); 5352 mutex_init(&uctx->cq_lock); 5353 5354 return 0; 5355 cfail: 5356 free_page((u64)uctx->shpg); 5357 uctx->shpg = NULL; 5358 fail: 5359 return rc; 5360 } 5361 5362 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 5363 { 5364 struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, 5365 struct bnxt_re_ucontext, 5366 ibucontext); 5367 struct bnxt_re_dev *rdev = uctx->rdev; 5368 int rc = 0; 5369 5370 if (uctx->shpg) 5371 free_page((u64)uctx->shpg); 5372 5373 if (uctx->dpi.dbr) { 5374 /* Free DPI only if this is the first PD allocated by the 5375 * application and mark the context dpi as NULL 5376 */ 5377 if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) { 5378 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 5379 &uctx->wcdpi); 5380 if (rc) 5381 dev_err(rdev_to_dev(rdev), 5382 "dealloc push dp failed\n"); 5383 uctx->wcdpi.dbr = NULL; 5384 } 5385 5386 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 5387 &uctx->dpi); 5388 if (rc) 5389 dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n"); 5390 /* Don't fail, continue*/ 5391 uctx->dpi.dbr = NULL; 5392 } 5393 return; 5394 } 5395 5396 static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx, 5397 u64 pg_off) 5398 { 5399 struct bnxt_re_cq *cq = NULL, *tmp_cq; 5400 5401 if (!_is_chip_p7(uctx->rdev->chip_ctx)) 5402 return NULL; 5403 5404 mutex_lock(&uctx->cq_lock); 5405 list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) { 5406 if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) { 5407 cq = tmp_cq; 5408 break; 5409 } 5410 } 5411 mutex_unlock(&uctx->cq_lock); 5412 return cq; 5413 } 5414 5415 /* Helper function to mmap the virtual memory from user app */ 5416 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 5417 { 5418 struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, 5419 struct bnxt_re_ucontext, 5420 ibucontext); 5421 struct bnxt_re_dev *rdev = uctx->rdev; 5422 struct bnxt_re_cq *cq = NULL; 5423 int rc = 0; 5424 u64 pfn; 5425 5426 switch (vma->vm_pgoff) { 5427 case BNXT_RE_MAP_SH_PAGE: 5428 pfn = vtophys(uctx->shpg) >> PAGE_SHIFT; 5429 return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); 5430 dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n", 5431 __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn); 5432 if (rc) { 5433 dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n"); 5434 rc = -EAGAIN; 5435 } 5436 return rc; 5437 case BNXT_RE_MAP_WC: 5438 vma->vm_page_prot = 5439 pgprot_writecombine(vma->vm_page_prot); 5440 pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT); 5441 if (!pfn) 5442 return -EFAULT; 5443 break; 5444 case BNXT_RE_DBR_PAGE: 5445 /* Driver doesn't expect write access request */ 5446 if (vma->vm_flags & VM_WRITE) 5447 return -EFAULT; 5448 5449 pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT; 5450 if (!pfn) 5451 return -EFAULT; 5452 break; 5453 case BNXT_RE_MAP_DB_RECOVERY_PAGE: 5454 pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT; 5455 if (!pfn) 5456 return -EFAULT; 5457 break; 5458 default: 5459 cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff); 5460 if (cq) { 5461 pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT; 5462 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); 5463 if (rc) { 5464 dev_err(rdev_to_dev(rdev), 5465 "CQ page mapping failed!\n"); 5466 rc = -EAGAIN; 5467 } 5468 goto out; 5469 } else { 5470 vma->vm_page_prot = 5471 pgprot_noncached(vma->vm_page_prot); 5472 pfn = vma->vm_pgoff; 5473 } 5474 break; 5475 } 5476 5477 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); 5478 if (rc) { 5479 dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n"); 5480 return -EAGAIN; 5481 } 5482 rc = __bnxt_re_set_vma_data(uctx, vma); 5483 out: 5484 return rc; 5485 } 5486 5487 int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, 5488 const struct ib_wc *wc, const struct ib_grh *grh, 5489 const struct ib_mad_hdr *in_mad, size_t in_mad_size, 5490 struct ib_mad_hdr *out_mad, size_t *out_mad_size, 5491 u16 *out_mad_pkey_index) 5492 { 5493 return IB_MAD_RESULT_SUCCESS; 5494 } 5495 5496 void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx) 5497 { 5498 } 5499