1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: IB Verbs interpreter 29 */ 30 31 #include <linux/if_ether.h> 32 #include <linux/etherdevice.h> 33 #include <rdma/uverbs_ioctl.h> 34 35 #include "bnxt_re.h" 36 #include "ib_verbs.h" 37 38 static inline 39 struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap) 40 { 41 42 *nmap = umem->nmap; 43 return umem->sg_head.sgl; 44 } 45 46 static inline void bnxt_re_peer_mem_release(struct ib_umem *umem) 47 { 48 dev_dbg(NULL, "ib_umem_release getting invoked \n"); 49 ib_umem_release(umem); 50 } 51 52 static int __from_ib_access_flags(int iflags) 53 { 54 int qflags = 0; 55 56 if (iflags & IB_ACCESS_LOCAL_WRITE) 57 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 58 if (iflags & IB_ACCESS_REMOTE_READ) 59 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; 60 if (iflags & IB_ACCESS_REMOTE_WRITE) 61 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; 62 if (iflags & IB_ACCESS_REMOTE_ATOMIC) 63 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; 64 if (iflags & IB_ACCESS_MW_BIND) 65 qflags |= BNXT_QPLIB_ACCESS_MW_BIND; 66 if (iflags & IB_ZERO_BASED) 67 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; 68 if (iflags & IB_ACCESS_ON_DEMAND) 69 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; 70 return qflags; 71 }; 72 73 static enum ib_access_flags __to_ib_access_flags(int qflags) 74 { 75 enum ib_access_flags iflags = 0; 76 77 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) 78 iflags |= IB_ACCESS_LOCAL_WRITE; 79 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) 80 iflags |= IB_ACCESS_REMOTE_WRITE; 81 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) 82 iflags |= IB_ACCESS_REMOTE_READ; 83 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) 84 iflags |= IB_ACCESS_REMOTE_ATOMIC; 85 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) 86 iflags |= IB_ACCESS_MW_BIND; 87 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) 88 iflags |= IB_ZERO_BASED; 89 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) 90 iflags |= IB_ACCESS_ON_DEMAND; 91 return iflags; 92 }; 93 94 static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data, 95 int len, struct ib_udata *udata) 96 { 97 int rc; 98 99 rc = ib_copy_to_udata(udata, data, len); 100 if (rc) 101 dev_err(rdev_to_dev(rdev), 102 "ucontext copy failed from %ps rc %d\n", 103 __builtin_return_address(0), rc); 104 105 return rc; 106 } 107 108 struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, 109 u8 port_num) 110 { 111 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 112 struct ifnet *netdev = NULL; 113 114 rcu_read_lock(); 115 116 if (!rdev || !rdev->netdev) 117 goto end; 118 119 netdev = rdev->netdev; 120 121 /* In case of active-backup bond mode, return active slave */ 122 if (netdev) 123 dev_hold(netdev); 124 125 end: 126 rcu_read_unlock(); 127 return netdev; 128 } 129 130 int bnxt_re_query_device(struct ib_device *ibdev, 131 struct ib_device_attr *ib_attr, 132 struct ib_udata *udata) 133 { 134 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 135 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; 136 137 memset(ib_attr, 0, sizeof(*ib_attr)); 138 139 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4); 140 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid); 141 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; 142 ib_attr->page_size_cap = dev_attr->page_size_cap; 143 ib_attr->vendor_id = rdev->en_dev->pdev->vendor; 144 ib_attr->vendor_part_id = rdev->en_dev->pdev->device; 145 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; 146 ib_attr->max_qp = dev_attr->max_qp; 147 ib_attr->max_qp_wr = dev_attr->max_qp_wqes; 148 /* 149 * Read and set from the module param 'min_tx_depth' 150 * only once after the driver load 151 */ 152 if (rdev->min_tx_depth == 1 && 153 min_tx_depth < dev_attr->max_qp_wqes) 154 rdev->min_tx_depth = min_tx_depth; 155 ib_attr->device_cap_flags = 156 IB_DEVICE_CURR_QP_STATE_MOD 157 | IB_DEVICE_RC_RNR_NAK_GEN 158 | IB_DEVICE_SHUTDOWN_PORT 159 | IB_DEVICE_SYS_IMAGE_GUID 160 | IB_DEVICE_LOCAL_DMA_LKEY 161 | IB_DEVICE_RESIZE_MAX_WR 162 | IB_DEVICE_PORT_ACTIVE_EVENT 163 | IB_DEVICE_N_NOTIFY_CQ 164 | IB_DEVICE_MEM_WINDOW 165 | IB_DEVICE_MEM_WINDOW_TYPE_2B 166 | IB_DEVICE_MEM_MGT_EXTENSIONS; 167 ib_attr->max_send_sge = dev_attr->max_qp_sges; 168 ib_attr->max_recv_sge = dev_attr->max_qp_sges; 169 ib_attr->max_sge_rd = dev_attr->max_qp_sges; 170 ib_attr->max_cq = dev_attr->max_cq; 171 ib_attr->max_cqe = dev_attr->max_cq_wqes; 172 ib_attr->max_mr = dev_attr->max_mr; 173 ib_attr->max_pd = dev_attr->max_pd; 174 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; 175 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; 176 if (dev_attr->is_atomic) { 177 ib_attr->atomic_cap = IB_ATOMIC_GLOB; 178 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; 179 } 180 ib_attr->max_ee_rd_atom = 0; 181 ib_attr->max_res_rd_atom = 0; 182 ib_attr->max_ee_init_rd_atom = 0; 183 ib_attr->max_ee = 0; 184 ib_attr->max_rdd = 0; 185 ib_attr->max_mw = dev_attr->max_mw; 186 ib_attr->max_raw_ipv6_qp = 0; 187 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; 188 ib_attr->max_mcast_grp = 0; 189 ib_attr->max_mcast_qp_attach = 0; 190 ib_attr->max_total_mcast_qp_attach = 0; 191 ib_attr->max_ah = dev_attr->max_ah; 192 ib_attr->max_srq = dev_attr->max_srq; 193 ib_attr->max_srq_wr = dev_attr->max_srq_wqes; 194 ib_attr->max_srq_sge = dev_attr->max_srq_sges; 195 196 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; 197 ib_attr->max_pkeys = 1; 198 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; 199 ib_attr->sig_prot_cap = 0; 200 ib_attr->sig_guard_cap = 0; 201 ib_attr->odp_caps.general_caps = 0; 202 203 return 0; 204 } 205 206 int bnxt_re_modify_device(struct ib_device *ibdev, 207 int device_modify_mask, 208 struct ib_device_modify *device_modify) 209 { 210 dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n", 211 device_modify_mask); 212 213 switch (device_modify_mask) { 214 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: 215 /* Modify the GUID requires the modification of the GID table */ 216 /* GUID should be made as READ-ONLY */ 217 break; 218 case IB_DEVICE_MODIFY_NODE_DESC: 219 /* Node Desc should be made as READ-ONLY */ 220 break; 221 default: 222 break; 223 } 224 return 0; 225 } 226 227 static void __to_ib_speed_width(u32 espeed, u8 lanes, u16 *speed, u8 *width) 228 { 229 if (!lanes) { 230 switch (espeed) { 231 case SPEED_1000: 232 *speed = IB_SPEED_SDR; 233 *width = IB_WIDTH_1X; 234 break; 235 case SPEED_10000: 236 *speed = IB_SPEED_QDR; 237 *width = IB_WIDTH_1X; 238 break; 239 case SPEED_20000: 240 *speed = IB_SPEED_DDR; 241 *width = IB_WIDTH_4X; 242 break; 243 case SPEED_25000: 244 *speed = IB_SPEED_EDR; 245 *width = IB_WIDTH_1X; 246 break; 247 case SPEED_40000: 248 *speed = IB_SPEED_QDR; 249 *width = IB_WIDTH_4X; 250 break; 251 case SPEED_50000: 252 *speed = IB_SPEED_EDR; 253 *width = IB_WIDTH_2X; 254 break; 255 case SPEED_100000: 256 *speed = IB_SPEED_EDR; 257 *width = IB_WIDTH_4X; 258 break; 259 case SPEED_200000: 260 *speed = IB_SPEED_HDR; 261 *width = IB_WIDTH_4X; 262 break; 263 case SPEED_400000: 264 *speed = IB_SPEED_NDR; 265 *width = IB_WIDTH_4X; 266 break; 267 default: 268 *speed = IB_SPEED_SDR; 269 *width = IB_WIDTH_1X; 270 break; 271 } 272 return; 273 } 274 275 switch (lanes) { 276 case 1: 277 *width = IB_WIDTH_1X; 278 break; 279 case 2: 280 *width = IB_WIDTH_2X; 281 break; 282 case 4: 283 *width = IB_WIDTH_4X; 284 break; 285 case 8: 286 *width = IB_WIDTH_8X; 287 break; 288 case 12: 289 *width = IB_WIDTH_12X; 290 break; 291 default: 292 *width = IB_WIDTH_1X; 293 } 294 295 switch (espeed / lanes) { 296 case SPEED_2500: 297 *speed = IB_SPEED_SDR; 298 break; 299 case SPEED_5000: 300 *speed = IB_SPEED_DDR; 301 break; 302 case SPEED_10000: 303 *speed = IB_SPEED_FDR10; 304 break; 305 case SPEED_14000: 306 *speed = IB_SPEED_FDR; 307 break; 308 case SPEED_25000: 309 *speed = IB_SPEED_EDR; 310 break; 311 case SPEED_50000: 312 *speed = IB_SPEED_HDR; 313 break; 314 case SPEED_100000: 315 *speed = IB_SPEED_NDR; 316 break; 317 default: 318 *speed = IB_SPEED_SDR; 319 } 320 } 321 322 /* Port */ 323 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, 324 struct ib_port_attr *port_attr) 325 { 326 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 327 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; 328 u16 active_speed = 0; 329 u8 active_width = 0; 330 331 dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num); 332 memset(port_attr, 0, sizeof(*port_attr)); 333 334 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 335 port_attr->state = bnxt_re_get_link_state(rdev); 336 if (port_attr->state == IB_PORT_ACTIVE) 337 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 338 port_attr->max_mtu = IB_MTU_4096; 339 port_attr->active_mtu = iboe_get_mtu(if_getmtu(rdev->netdev)); 340 port_attr->gid_tbl_len = dev_attr->max_sgid; 341 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 342 IB_PORT_DEVICE_MGMT_SUP | 343 IB_PORT_VENDOR_CLASS_SUP | 344 IB_PORT_IP_BASED_GIDS; 345 346 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; 347 port_attr->bad_pkey_cntr = 0; 348 port_attr->qkey_viol_cntr = 0; 349 port_attr->pkey_tbl_len = dev_attr->max_pkey; 350 port_attr->lid = 0; 351 port_attr->sm_lid = 0; 352 port_attr->lmc = 0; 353 port_attr->max_vl_num = 4; 354 port_attr->sm_sl = 0; 355 port_attr->subnet_timeout = 0; 356 port_attr->init_type_reply = 0; 357 rdev->espeed = rdev->en_dev->espeed; 358 rdev->lanes = rdev->en_dev->lanes; 359 360 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 361 __to_ib_speed_width(rdev->espeed, rdev->lanes, &active_speed, 362 &active_width); 363 364 port_attr->active_speed = active_speed; 365 port_attr->active_width = active_width; 366 367 return 0; 368 } 369 370 int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, 371 int port_modify_mask, 372 struct ib_port_modify *port_modify) 373 { 374 dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n", 375 port_modify_mask); 376 377 switch (port_modify_mask) { 378 case IB_PORT_SHUTDOWN: 379 break; 380 case IB_PORT_INIT_TYPE: 381 break; 382 case IB_PORT_RESET_QKEY_CNTR: 383 break; 384 default: 385 break; 386 } 387 return 0; 388 } 389 390 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, 391 struct ib_port_immutable *immutable) 392 { 393 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 394 struct ib_port_attr port_attr; 395 396 if (bnxt_re_query_port(ibdev, port_num, &port_attr)) 397 return -EINVAL; 398 399 immutable->pkey_tbl_len = port_attr.pkey_tbl_len; 400 immutable->gid_tbl_len = port_attr.gid_tbl_len; 401 if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP) 402 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 403 else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP) 404 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 405 else 406 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | 407 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 408 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 409 return 0; 410 } 411 412 void bnxt_re_compat_qfwstr(void) 413 { 414 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 415 416 sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0], 417 rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2], 418 rdev->dev_attr->fw_ver[3]); 419 } 420 421 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, 422 u16 index, u16 *pkey) 423 { 424 if (index > 0) 425 return -EINVAL; 426 427 *pkey = IB_DEFAULT_PKEY_FULL; 428 429 return 0; 430 } 431 432 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, 433 int index, union ib_gid *gid) 434 { 435 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 436 int rc = 0; 437 438 /* Ignore port_num */ 439 memset(gid, 0, sizeof(*gid)); 440 rc = bnxt_qplib_get_sgid(&rdev->qplib_res, 441 &rdev->qplib_res.sgid_tbl, index, 442 (struct bnxt_qplib_gid *)gid); 443 return rc; 444 } 445 446 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context) 447 { 448 int rc = 0; 449 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 450 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 451 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 452 struct bnxt_qplib_gid *gid_to_del; 453 u16 vlan_id = 0xFFFF; 454 455 /* Delete the entry from the hardware */ 456 ctx = *context; 457 if (!ctx) { 458 dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n"); 459 return -EINVAL; 460 } 461 if (sgid_tbl && sgid_tbl->active) { 462 if (ctx->idx >= sgid_tbl->max) { 463 dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n"); 464 return -EINVAL; 465 } 466 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; 467 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; 468 ctx->refcnt--; 469 /* DEL_GID is called via WQ context(netdevice_event_work_handler) 470 * or via the ib_unregister_device path. In the former case QP1 471 * may not be destroyed yet, in which case just return as FW 472 * needs that entry to be present and will fail it's deletion. 473 * We could get invoked again after QP1 is destroyed OR get an 474 * ADD_GID call with a different GID value for the same index 475 * where we issue MODIFY_GID cmd to update the GID entry -- TBD 476 */ 477 if (ctx->idx == 0 && 478 rdma_link_local_addr((struct in6_addr *)gid_to_del) && 479 (rdev->gsi_ctx.gsi_sqp || 480 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) { 481 dev_dbg(rdev_to_dev(rdev), 482 "Trying to delete GID0 while QP1 is alive\n"); 483 if (!ctx->refcnt) { 484 rdev->gid_map[attr->index] = -1; 485 ctx_tbl = sgid_tbl->ctx; 486 ctx_tbl[ctx->idx] = NULL; 487 kfree(ctx); 488 } 489 return 0; 490 } 491 rdev->gid_map[attr->index] = -1; 492 if (!ctx->refcnt) { 493 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, 494 vlan_id, true); 495 if (!rc) { 496 dev_dbg(rdev_to_dev(rdev), "GID remove success\n"); 497 ctx_tbl = sgid_tbl->ctx; 498 ctx_tbl[ctx->idx] = NULL; 499 kfree(ctx); 500 } else { 501 dev_err(rdev_to_dev(rdev), 502 "Remove GID failed rc = 0x%x\n", rc); 503 } 504 } 505 } else { 506 dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n"); 507 return -EINVAL; 508 } 509 return rc; 510 } 511 512 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context) 513 { 514 int rc; 515 u32 tbl_idx = 0; 516 u16 vlan_id = 0xFFFF; 517 struct bnxt_re_gid_ctx *ctx, **ctx_tbl; 518 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev); 519 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 520 521 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL); 522 if (rc) 523 return rc; 524 525 rc = bnxt_qplib_add_sgid(sgid_tbl, &attr->gid, 526 rdev->dev_addr, 527 vlan_id, true, &tbl_idx); 528 if (rc == -EALREADY) { 529 dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", 530 &attr->gid); 531 ctx_tbl = sgid_tbl->ctx; 532 if (!ctx_tbl[tbl_idx]) { 533 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 534 if (!ctx) 535 return -ENOMEM; 536 ctx->idx = tbl_idx; 537 ctx->refcnt = 1; 538 ctx_tbl[tbl_idx] = ctx; 539 } else { 540 ctx_tbl[tbl_idx]->refcnt++; 541 } 542 *context = ctx_tbl[tbl_idx]; 543 /* tbl_idx is the HW table index and attr->index is the stack index */ 544 rdev->gid_map[attr->index] = tbl_idx; 545 return 0; 546 } else if (rc < 0) { 547 dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc); 548 return rc; 549 } else { 550 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 551 if (!ctx) { 552 dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n"); 553 return -ENOMEM; 554 } 555 ctx_tbl = sgid_tbl->ctx; 556 ctx->idx = tbl_idx; 557 ctx->refcnt = 1; 558 ctx_tbl[tbl_idx] = ctx; 559 /* tbl_idx is the HW table index and attr->index is the stack index */ 560 rdev->gid_map[attr->index] = tbl_idx; 561 *context = ctx; 562 } 563 return rc; 564 } 565 566 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, 567 u8 port_num) 568 { 569 return IB_LINK_LAYER_ETHERNET; 570 } 571 572 static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd) 573 { 574 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 575 struct ib_mr *ib_mr = &fence->mr->ib_mr; 576 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; 577 struct bnxt_re_dev *rdev = pd->rdev; 578 579 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 580 return; 581 582 memset(wqe, 0, sizeof(*wqe)); 583 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; 584 wqe->wr_id = BNXT_QPLIB_FENCE_WRID; 585 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 586 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 587 wqe->bind.zero_based = false; 588 wqe->bind.parent_l_key = ib_mr->lkey; 589 wqe->bind.va = (u64)fence->va; 590 wqe->bind.length = fence->size; 591 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); 592 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; 593 594 /* Save the initial rkey in fence structure for now; 595 * wqe->bind.r_key will be set at (re)bind time. 596 */ 597 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); 598 } 599 600 static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) 601 { 602 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, 603 qplib_qp); 604 struct ib_pd *ib_pd = qp->ib_qp.pd; 605 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 606 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 607 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; 608 struct bnxt_qplib_swqe wqe; 609 int rc; 610 611 /* TODO: Need SQ locking here when Fence WQE 612 * posting moves up into bnxt_re from bnxt_qplib. 613 */ 614 memcpy(&wqe, fence_wqe, sizeof(wqe)); 615 wqe.bind.r_key = fence->bind_rkey; 616 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); 617 618 dev_dbg(rdev_to_dev(qp->rdev), 619 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", 620 wqe.bind.r_key, qp->qplib_qp.id, pd); 621 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 622 if (rc) { 623 dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); 624 return rc; 625 } 626 bnxt_qplib_post_send_db(&qp->qplib_qp); 627 628 return rc; 629 } 630 631 static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd) 632 { 633 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; 634 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 635 struct bnxt_re_dev *rdev = pd->rdev; 636 struct bnxt_qplib_mrinfo mrinfo; 637 struct bnxt_re_mr *mr = NULL; 638 struct ib_mw *ib_mw = NULL; 639 dma_addr_t dma_addr = 0; 640 u32 max_mr_count; 641 u64 pbl_tbl; 642 int rc; 643 644 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 645 return 0; 646 647 memset(&mrinfo, 0, sizeof(mrinfo)); 648 /* Allocate a small chunk of memory and dma-map it */ 649 fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL); 650 if (!fence->va) 651 return -ENOMEM; 652 dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va, 653 BNXT_RE_LEGACY_FENCE_BYTES, 654 DMA_BIDIRECTIONAL); 655 rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr); 656 if (rc) { 657 dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); 658 rc = -EIO; 659 fence->dma_addr = 0; 660 goto free_va; 661 } 662 fence->dma_addr = dma_addr; 663 664 /* Allocate a MR */ 665 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 666 if (!mr) 667 goto free_dma_addr; 668 fence->mr = mr; 669 mr->rdev = rdev; 670 mr->qplib_mr.pd = &pd->qplib_pd; 671 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 672 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 673 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { 674 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 675 if (rc) { 676 dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); 677 goto free_mr; 678 } 679 /* Register MR */ 680 mr->ib_mr.lkey = mr->qplib_mr.lkey; 681 } 682 mr->qplib_mr.va = (u64)fence->va; 683 mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES; 684 pbl_tbl = dma_addr; 685 686 mrinfo.mrw = &mr->qplib_mr; 687 mrinfo.ptes = &pbl_tbl; 688 mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE; 689 690 mrinfo.sg.nmap = 0; 691 mrinfo.sg.sghead = 0; 692 mrinfo.sg.pgshft = PAGE_SHIFT; 693 mrinfo.sg.pgsize = PAGE_SIZE; 694 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 695 if (rc) { 696 dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); 697 goto free_mr; 698 } 699 mr->ib_mr.lkey = mr->qplib_mr.lkey; 700 mr->ib_mr.rkey = mr->qplib_mr.rkey; 701 atomic_inc(&rdev->stats.rsors.mr_count); 702 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 703 if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count))) 704 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 705 706 ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL); 707 /* Create a fence MW only for kernel consumers */ 708 if (!ib_mw) { 709 dev_err(rdev_to_dev(rdev), 710 "Failed to create fence-MW for PD: %p\n", pd); 711 rc = -EINVAL; 712 goto free_mr; 713 } 714 fence->mw = ib_mw; 715 716 bnxt_re_legacy_create_fence_wqe(pd); 717 return 0; 718 719 free_mr: 720 if (mr->ib_mr.lkey) { 721 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 722 atomic_dec(&rdev->stats.rsors.mr_count); 723 } 724 kfree(mr); 725 fence->mr = NULL; 726 727 free_dma_addr: 728 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, 729 BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); 730 fence->dma_addr = 0; 731 732 free_va: 733 kfree(fence->va); 734 fence->va = NULL; 735 return rc; 736 } 737 738 static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd) 739 { 740 struct bnxt_re_legacy_fence_data *fence = &pd->fence; 741 struct bnxt_re_dev *rdev = pd->rdev; 742 struct bnxt_re_mr *mr = fence->mr; 743 744 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 745 return; 746 747 if (fence->mw) { 748 bnxt_re_dealloc_mw(fence->mw); 749 fence->mw = NULL; 750 } 751 if (mr) { 752 if (mr->ib_mr.rkey) 753 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, 754 false); 755 if (mr->ib_mr.lkey) 756 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 757 kfree(mr); 758 fence->mr = NULL; 759 atomic_dec(&rdev->stats.rsors.mr_count); 760 } 761 if (fence->dma_addr) { 762 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, 763 BNXT_RE_LEGACY_FENCE_BYTES, 764 DMA_BIDIRECTIONAL); 765 fence->dma_addr = 0; 766 } 767 kfree(fence->va); 768 fence->va = NULL; 769 } 770 771 772 static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev, 773 struct bnxt_re_ucontext *cntx) 774 { 775 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; 776 int ret = 0; 777 u8 type; 778 /* Allocate DPI in alloc_pd or in create_cq to avoid failing of 779 * ibv_devinfo and family of application when DPIs are depleted. 780 */ 781 type = BNXT_QPLIB_DPI_TYPE_UC; 782 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type); 783 if (ret) { 784 dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n"); 785 goto out; 786 } 787 788 if (cctx->modes.db_push) { 789 type = BNXT_QPLIB_DPI_TYPE_WC; 790 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi, 791 cntx, type); 792 if (ret) 793 dev_err(rdev_to_dev(rdev), "push dp alloc failed\n"); 794 } 795 out: 796 return ret; 797 } 798 799 /* Protection Domains */ 800 void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) 801 { 802 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 803 struct bnxt_re_dev *rdev = pd->rdev; 804 int rc; 805 806 bnxt_re_legacy_destroy_fence_mr(pd); 807 808 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, 809 &rdev->qplib_res.pd_tbl, 810 &pd->qplib_pd); 811 if (rc) 812 dev_err_ratelimited(rdev_to_dev(rdev), 813 "%s failed rc = %d\n", __func__, rc); 814 atomic_dec(&rdev->stats.rsors.pd_count); 815 816 return; 817 } 818 819 int bnxt_re_alloc_pd(struct ib_pd *pd_in, 820 struct ib_udata *udata) 821 { 822 struct ib_pd *ibpd = pd_in; 823 struct ib_device *ibdev = ibpd->device; 824 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 825 struct bnxt_re_ucontext *ucntx = 826 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, 827 ibucontext); 828 u32 max_pd_count; 829 int rc; 830 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd); 831 832 pd->rdev = rdev; 833 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { 834 dev_err(rdev_to_dev(rdev), 835 "Allocate HW Protection Domain failed!\n"); 836 rc = -ENOMEM; 837 goto fail; 838 } 839 840 if (udata) { 841 struct bnxt_re_pd_resp resp = {}; 842 843 if (!ucntx->dpi.dbr) { 844 rc = bnxt_re_get_user_dpi(rdev, ucntx); 845 if (rc) 846 goto dbfail; 847 } 848 849 resp.pdid = pd->qplib_pd.id; 850 /* Still allow mapping this DBR to the new user PD. */ 851 resp.dpi = ucntx->dpi.dpi; 852 resp.dbr = (u64)ucntx->dpi.umdbr; 853 /* Copy only on a valid wcpdi */ 854 if (ucntx->wcdpi.dpi) { 855 resp.wcdpi = ucntx->wcdpi.dpi; 856 resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI; 857 } 858 if (rdev->dbr_pacing) { 859 WARN_ON(!rdev->dbr_bar_addr); 860 resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr; 861 resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR; 862 } 863 864 rc = bnxt_re_copy_to_udata(rdev, &resp, 865 min(udata->outlen, sizeof(resp)), 866 udata); 867 if (rc) 868 goto dbfail; 869 } 870 871 if (!udata) 872 if (bnxt_re_legacy_create_fence_mr(pd)) 873 dev_warn(rdev_to_dev(rdev), 874 "Failed to create Fence-MR\n"); 875 876 atomic_inc(&rdev->stats.rsors.pd_count); 877 max_pd_count = atomic_read(&rdev->stats.rsors.pd_count); 878 if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count)) 879 atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count); 880 881 return 0; 882 dbfail: 883 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, 884 &pd->qplib_pd); 885 fail: 886 return rc; 887 } 888 889 /* Address Handles */ 890 void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) 891 { 892 struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); 893 struct bnxt_re_dev *rdev = ah->rdev; 894 int rc = 0; 895 bool block = true; 896 897 block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); 898 899 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); 900 if (rc) 901 dev_err_ratelimited(rdev_to_dev(rdev), 902 "%s id = %d blocking %d failed rc = %d\n", 903 __func__, ah->qplib_ah.id, block, rc); 904 atomic_dec(&rdev->stats.rsors.ah_count); 905 906 return; 907 } 908 909 static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype) 910 { 911 u8 nw_type; 912 switch (ntype) { 913 case RDMA_NETWORK_IPV4: 914 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; 915 break; 916 case RDMA_NETWORK_IPV6: 917 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; 918 break; 919 default: 920 nw_type = CMDQ_CREATE_AH_TYPE_V1; 921 break; 922 } 923 return nw_type; 924 } 925 926 static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev, 927 struct rdma_ah_attr *ah_attr, 928 struct bnxt_re_ah_info *ah_info) 929 { 930 const struct ib_gid_attr *gattr; 931 enum rdma_network_type ib_ntype; 932 u8 ntype; 933 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 934 int rc = 0; 935 936 gattr = grh->sgid_attr; 937 ah_info->sgid_attr = *gattr; 938 ah_info->sgid = gattr->gid; 939 940 /* Get vlan tag */ 941 rc = rdma_read_gid_l2_fields(gattr, &ah_info->vlan_tag, NULL); 942 if (rc) 943 return rc; 944 945 /* Get network header type for this GID */ 946 ib_ntype = rdma_gid_attr_network_type(gattr); 947 ntype = _to_bnxt_re_nw_type(ib_ntype); 948 ah_info->nw_type = ntype; 949 950 return rc; 951 } 952 953 static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx) 954 { 955 gindx = rdev->gid_map[gindx]; 956 return gindx; 957 } 958 959 int bnxt_re_create_ah(struct ib_ah *ah_in, struct rdma_ah_attr *attr, 960 u32 flags, struct ib_udata *udata) 961 { 962 963 struct ib_ah *ib_ah = ah_in; 964 struct ib_pd *ib_pd = ib_ah->pd; 965 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah); 966 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd); 967 struct bnxt_re_dev *rdev = pd->rdev; 968 const struct ib_global_route *grh = rdma_ah_read_grh(attr); 969 struct bnxt_re_ah_info ah_info; 970 u32 max_ah_count; 971 int rc; 972 bool block = true; 973 struct rdma_ah_attr *ah_attr = attr; 974 block = !(flags & RDMA_CREATE_AH_SLEEPABLE); 975 976 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) 977 dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n"); 978 979 ah->rdev = rdev; 980 ah->qplib_ah.pd = &pd->qplib_pd; 981 982 /* Supply the configuration for the HW */ 983 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw, 984 sizeof(union ib_gid)); 985 ah->qplib_ah.sgid_index = _get_sgid_index(rdev, grh->sgid_index); 986 if (ah->qplib_ah.sgid_index == 0xFF) { 987 dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n"); 988 rc = -EINVAL; 989 goto fail; 990 } 991 ah->qplib_ah.host_sgid_index = grh->sgid_index; 992 ah->qplib_ah.traffic_class = grh->traffic_class; 993 ah->qplib_ah.flow_label = grh->flow_label; 994 ah->qplib_ah.hop_limit = grh->hop_limit; 995 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr); 996 rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info); 997 if (rc) 998 goto fail; 999 ah->qplib_ah.nw_type = ah_info.nw_type; 1000 memcpy(ah->qplib_ah.dmac, ROCE_DMAC(ah_attr), ETH_ALEN); 1001 1002 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block); 1003 if (rc) { 1004 dev_err(rdev_to_dev(rdev), 1005 "Allocate HW Address Handle failed!\n"); 1006 goto fail; 1007 } 1008 1009 /* Write AVID to shared page. */ 1010 if (ib_pd->uobject) { 1011 struct ib_ucontext *ib_uctx = ib_pd->uobject->context; 1012 struct bnxt_re_ucontext *uctx; 1013 unsigned long flag; 1014 u32 *wrptr; 1015 1016 uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); 1017 spin_lock_irqsave(&uctx->sh_lock, flag); 1018 wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT); 1019 *wrptr = ah->qplib_ah.id; 1020 wmb(); /* make sure cache is updated. */ 1021 spin_unlock_irqrestore(&uctx->sh_lock, flag); 1022 } 1023 atomic_inc(&rdev->stats.rsors.ah_count); 1024 max_ah_count = atomic_read(&rdev->stats.rsors.ah_count); 1025 if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count)) 1026 atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count); 1027 1028 return 0; 1029 fail: 1030 return rc; 1031 } 1032 1033 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 1034 { 1035 return 0; 1036 } 1037 1038 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) 1039 { 1040 struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); 1041 1042 ah_attr->type = ib_ah->type; 1043 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl); 1044 memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN); 1045 rdma_ah_set_grh(ah_attr, NULL, 0, 1046 ah->qplib_ah.host_sgid_index, 1047 0, ah->qplib_ah.traffic_class); 1048 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data); 1049 rdma_ah_set_port_num(ah_attr, 1); 1050 rdma_ah_set_static_rate(ah_attr, 0); 1051 return 0; 1052 } 1053 1054 /* Shared Receive Queues */ 1055 void bnxt_re_destroy_srq(struct ib_srq *ib_srq, 1056 struct ib_udata *udata) 1057 { 1058 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); 1059 struct bnxt_re_dev *rdev = srq->rdev; 1060 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; 1061 int rc = 0; 1062 1063 1064 rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); 1065 if (rc) 1066 dev_err_ratelimited(rdev_to_dev(rdev), 1067 "%s id = %d failed rc = %d\n", 1068 __func__, qplib_srq->id, rc); 1069 1070 if (srq->umem && !IS_ERR(srq->umem)) 1071 ib_umem_release(srq->umem); 1072 1073 atomic_dec(&rdev->stats.rsors.srq_count); 1074 1075 return; 1076 } 1077 1078 static u16 _max_rwqe_sz(int nsge) 1079 { 1080 return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge)); 1081 } 1082 1083 static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp, 1084 int rsge, int max) 1085 { 1086 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1087 rsge = max; 1088 1089 return _max_rwqe_sz(rsge); 1090 } 1091 1092 static inline 1093 struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, 1094 struct ib_ucontext *ucontext, 1095 struct ib_udata *udata, 1096 unsigned long addr, 1097 size_t size, int access, int dmasync) 1098 { 1099 return ib_umem_get(ucontext, addr, size, access, dmasync); 1100 } 1101 1102 static inline 1103 struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, 1104 struct ib_ucontext *ucontext, 1105 struct ib_udata *udata, 1106 unsigned long addr, 1107 size_t size, int access, int dmasync) 1108 { 1109 return ib_umem_get_compat(rdev, ucontext, udata, addr, size, 1110 access, 0); 1111 } 1112 1113 static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem) 1114 { 1115 return ib_umem_num_pages(umem); 1116 } 1117 1118 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, 1119 struct bnxt_re_pd *pd, 1120 struct bnxt_re_srq *srq, 1121 struct ib_udata *udata) 1122 { 1123 struct bnxt_qplib_sg_info *sginfo; 1124 struct bnxt_qplib_srq *qplib_srq; 1125 struct bnxt_re_ucontext *cntx; 1126 struct ib_ucontext *context; 1127 struct bnxt_re_srq_req ureq; 1128 struct ib_umem *umem; 1129 int rc, bytes = 0; 1130 1131 context = pd->ibpd.uobject->context; 1132 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 1133 qplib_srq = &srq->qplib_srq; 1134 sginfo = &qplib_srq->sginfo; 1135 1136 if (udata->inlen < sizeof(ureq)) 1137 dev_warn(rdev_to_dev(rdev), 1138 "Update the library ulen %d klen %d\n", 1139 (unsigned int)udata->inlen, 1140 (unsigned int)sizeof(ureq)); 1141 1142 rc = ib_copy_from_udata(&ureq, udata, 1143 min(udata->inlen, sizeof(ureq))); 1144 if (rc) 1145 return rc; 1146 1147 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); 1148 bytes = PAGE_ALIGN(bytes); 1149 umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes, 1150 IB_ACCESS_LOCAL_WRITE, 1); 1151 if (IS_ERR(umem)) { 1152 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", 1153 __func__, PTR_ERR(umem)); 1154 return PTR_ERR(umem); 1155 } 1156 1157 srq->umem = umem; 1158 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); 1159 sginfo->npages = ib_umem_num_pages_compat(umem); 1160 qplib_srq->srq_handle = ureq.srq_handle; 1161 qplib_srq->dpi = &cntx->dpi; 1162 qplib_srq->is_user = true; 1163 1164 return 0; 1165 } 1166 1167 int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr, 1168 struct ib_udata *udata) 1169 { 1170 struct bnxt_qplib_dev_attr *dev_attr; 1171 struct bnxt_re_ucontext *cntx = NULL; 1172 struct ib_ucontext *context; 1173 struct bnxt_re_dev *rdev; 1174 struct bnxt_re_pd *pd; 1175 int rc, entries; 1176 struct ib_srq *ib_srq = srq_in; 1177 struct ib_pd *ib_pd = ib_srq->pd; 1178 struct bnxt_re_srq *srq = 1179 container_of(ib_srq, struct bnxt_re_srq, ibsrq); 1180 u32 max_srq_count; 1181 1182 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 1183 rdev = pd->rdev; 1184 dev_attr = rdev->dev_attr; 1185 1186 if (rdev->mod_exit) { 1187 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 1188 rc = -EIO; 1189 goto exit; 1190 } 1191 1192 if (srq_init_attr->srq_type != IB_SRQT_BASIC) { 1193 dev_err(rdev_to_dev(rdev), "SRQ type not supported\n"); 1194 rc = -ENOTSUPP; 1195 goto exit; 1196 } 1197 1198 if (udata) { 1199 context = pd->ibpd.uobject->context; 1200 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 1201 } 1202 1203 if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) { 1204 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n"); 1205 rc = -EINVAL; 1206 goto exit; 1207 } 1208 1209 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { 1210 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n"); 1211 rc = -EINVAL; 1212 goto exit; 1213 } 1214 1215 srq->rdev = rdev; 1216 srq->qplib_srq.pd = &pd->qplib_pd; 1217 srq->qplib_srq.dpi = &rdev->dpi_privileged; 1218 1219 /* Allocate 1 more than what's provided so posting max doesn't 1220 mean empty */ 1221 entries = srq_init_attr->attr.max_wr + 1; 1222 entries = bnxt_re_init_depth(entries, cntx); 1223 if (entries > dev_attr->max_srq_wqes + 1) 1224 entries = dev_attr->max_srq_wqes + 1; 1225 1226 srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */ 1227 srq->qplib_srq.max_wqe = entries; 1228 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; 1229 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; 1230 srq->srq_limit = srq_init_attr->attr.srq_limit; 1231 srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id; 1232 srq->qplib_srq.sginfo.pgsize = PAGE_SIZE; 1233 srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT; 1234 1235 if (udata) { 1236 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); 1237 if (rc) 1238 goto fail; 1239 } 1240 1241 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); 1242 if (rc) { 1243 dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n"); 1244 goto fail; 1245 } 1246 1247 if (udata) { 1248 struct bnxt_re_srq_resp resp; 1249 1250 resp.srqid = srq->qplib_srq.id; 1251 rc = bnxt_re_copy_to_udata(rdev, &resp, 1252 min(udata->outlen, sizeof(resp)), 1253 udata); 1254 if (rc) { 1255 bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); 1256 goto fail; 1257 } 1258 } 1259 atomic_inc(&rdev->stats.rsors.srq_count); 1260 max_srq_count = atomic_read(&rdev->stats.rsors.srq_count); 1261 if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count)) 1262 atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count); 1263 spin_lock_init(&srq->lock); 1264 1265 return 0; 1266 fail: 1267 if (udata && srq->umem && !IS_ERR(srq->umem)) { 1268 ib_umem_release(srq->umem); 1269 srq->umem = NULL; 1270 } 1271 exit: 1272 return rc; 1273 } 1274 1275 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, 1276 enum ib_srq_attr_mask srq_attr_mask, 1277 struct ib_udata *udata) 1278 { 1279 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, 1280 ibsrq); 1281 struct bnxt_re_dev *rdev = srq->rdev; 1282 int rc; 1283 1284 switch (srq_attr_mask) { 1285 case IB_SRQ_MAX_WR: 1286 /* SRQ resize is not supported */ 1287 break; 1288 case IB_SRQ_LIMIT: 1289 /* Change the SRQ threshold */ 1290 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) 1291 return -EINVAL; 1292 1293 srq->qplib_srq.threshold = srq_attr->srq_limit; 1294 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); 1295 if (rc) { 1296 dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n"); 1297 return rc; 1298 } 1299 /* On success, update the shadow */ 1300 srq->srq_limit = srq_attr->srq_limit; 1301 1302 if (udata) { 1303 /* Build and send response back to udata */ 1304 rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata); 1305 if (rc) 1306 return rc; 1307 } 1308 break; 1309 default: 1310 dev_err(rdev_to_dev(rdev), 1311 "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask); 1312 return -EINVAL; 1313 } 1314 return 0; 1315 } 1316 1317 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) 1318 { 1319 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, 1320 ibsrq); 1321 struct bnxt_re_dev *rdev = srq->rdev; 1322 int rc; 1323 1324 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq); 1325 if (rc) { 1326 dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n", 1327 srq->qplib_srq.id, rc); 1328 return rc; 1329 } 1330 srq_attr->max_wr = srq->qplib_srq.max_wqe; 1331 srq_attr->max_sge = srq->qplib_srq.max_sge; 1332 srq_attr->srq_limit = srq->qplib_srq.threshold; 1333 1334 return 0; 1335 } 1336 1337 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, 1338 const struct ib_recv_wr **bad_wr) 1339 { 1340 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, 1341 ibsrq); 1342 struct bnxt_qplib_swqe wqe = {}; 1343 unsigned long flags; 1344 int rc = 0; 1345 1346 spin_lock_irqsave(&srq->lock, flags); 1347 while (wr) { 1348 /* Transcribe each ib_recv_wr to qplib_swqe */ 1349 wqe.num_sge = wr->num_sge; 1350 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 1351 wqe.wr_id = wr->wr_id; 1352 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 1353 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); 1354 if (rc) { 1355 *bad_wr = wr; 1356 break; 1357 } 1358 wr = wr->next; 1359 } 1360 spin_unlock_irqrestore(&srq->lock, flags); 1361 1362 return rc; 1363 } 1364 1365 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) 1366 { 1367 unsigned long flags; 1368 1369 spin_lock_irqsave(&qp->scq->cq_lock, flags); 1370 if (qp->rcq && qp->rcq != qp->scq) 1371 spin_lock(&qp->rcq->cq_lock); 1372 1373 return flags; 1374 } 1375 1376 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, 1377 unsigned long flags) 1378 { 1379 if (qp->rcq && qp->rcq != qp->scq) 1380 spin_unlock(&qp->rcq->cq_lock); 1381 spin_unlock_irqrestore(&qp->scq->cq_lock, flags); 1382 } 1383 1384 /* Queue Pairs */ 1385 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) 1386 { 1387 struct bnxt_re_qp *gsi_sqp; 1388 struct bnxt_re_ah *gsi_sah; 1389 struct bnxt_re_dev *rdev; 1390 unsigned long flags; 1391 int rc = 0; 1392 1393 rdev = qp->rdev; 1394 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 1395 gsi_sah = rdev->gsi_ctx.gsi_sah; 1396 1397 /* remove from active qp list */ 1398 mutex_lock(&rdev->qp_lock); 1399 list_del(&gsi_sqp->list); 1400 mutex_unlock(&rdev->qp_lock); 1401 1402 if (gsi_sah) { 1403 dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n"); 1404 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, 1405 true); 1406 if (rc) 1407 dev_err(rdev_to_dev(rdev), 1408 "Destroy HW AH for shadow QP failed!\n"); 1409 atomic_dec(&rdev->stats.rsors.ah_count); 1410 } 1411 1412 dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n"); 1413 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); 1414 if (rc) 1415 dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n"); 1416 1417 /* Clean the CQ for shadow QP completions */ 1418 flags = bnxt_re_lock_cqs(gsi_sqp); 1419 bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp); 1420 bnxt_re_unlock_cqs(gsi_sqp, flags); 1421 1422 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); 1423 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp); 1424 kfree(rdev->gsi_ctx.sqp_tbl); 1425 kfree(gsi_sah); 1426 kfree(gsi_sqp); 1427 rdev->gsi_ctx.gsi_sqp = NULL; 1428 rdev->gsi_ctx.gsi_sah = NULL; 1429 rdev->gsi_ctx.sqp_tbl = NULL; 1430 atomic_dec(&rdev->stats.rsors.qp_count); 1431 1432 return 0; 1433 } 1434 1435 static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps) 1436 { 1437 u32 total_qp = 0; 1438 u64 avg_time = 0; 1439 int i; 1440 1441 if (!rdev->rcfw.sp_perf_stats_enabled) 1442 return; 1443 1444 switch (active_qps) { 1445 case 1: 1446 /* Potential hint for Test Stop */ 1447 for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) { 1448 if (rdev->rcfw.qp_destroy_stats[i]) { 1449 total_qp++; 1450 avg_time += rdev->rcfw.qp_destroy_stats[i]; 1451 } 1452 } 1453 if (total_qp >= 0 || avg_time >= 0) 1454 dev_dbg(rdev_to_dev(rdev), 1455 "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n", 1456 __builtin_return_address(0), total_qp, 1457 (long)jiffies_to_msecs(avg_time)); 1458 break; 1459 case 2: 1460 /* Potential hint for Test Start */ 1461 dev_dbg(rdev_to_dev(rdev), 1462 "Perf Debug: %ps active_qps = %d\n", 1463 __builtin_return_address(0), active_qps); 1464 break; 1465 default: 1466 /* Potential hint to know latency of QP destroy. 1467 * Average time taken for 1K QP Destroy. 1468 */ 1469 if (active_qps > 1024 && !(active_qps % 1024)) 1470 dev_dbg(rdev_to_dev(rdev), 1471 "Perf Debug: %ps Active QP (%d) Watermark (%d)\n", 1472 __builtin_return_address(0), active_qps, 1473 atomic_read(&rdev->stats.rsors.max_qp_count)); 1474 break; 1475 } 1476 } 1477 1478 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) 1479 { 1480 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 1481 struct bnxt_re_dev *rdev = qp->rdev; 1482 unsigned long flags; 1483 u32 active_qps; 1484 int rc; 1485 1486 mutex_lock(&rdev->qp_lock); 1487 list_del(&qp->list); 1488 active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count); 1489 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) 1490 atomic_dec(&rdev->stats.rsors.rc_qp_count); 1491 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) 1492 atomic_dec(&rdev->stats.rsors.ud_qp_count); 1493 mutex_unlock(&rdev->qp_lock); 1494 1495 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 1496 if (rc) 1497 dev_err_ratelimited(rdev_to_dev(rdev), 1498 "%s id = %d failed rc = %d\n", 1499 __func__, qp->qplib_qp.id, rc); 1500 1501 if (!ib_qp->uobject) { 1502 flags = bnxt_re_lock_cqs(qp); 1503 bnxt_qplib_clean_qp(&qp->qplib_qp); 1504 bnxt_re_unlock_cqs(qp, flags); 1505 } 1506 1507 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); 1508 if (ib_qp->qp_type == IB_QPT_GSI && 1509 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 1510 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && 1511 rdev->gsi_ctx.gsi_sqp) { 1512 bnxt_re_destroy_gsi_sqp(qp); 1513 } 1514 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp); 1515 } 1516 1517 if (qp->rumem && !IS_ERR(qp->rumem)) 1518 ib_umem_release(qp->rumem); 1519 if (qp->sumem && !IS_ERR(qp->sumem)) 1520 ib_umem_release(qp->sumem); 1521 kfree(qp); 1522 1523 bnxt_re_dump_debug_stats(rdev, active_qps); 1524 1525 return 0; 1526 } 1527 1528 static u8 __from_ib_qp_type(enum ib_qp_type type) 1529 { 1530 switch (type) { 1531 case IB_QPT_GSI: 1532 return CMDQ_CREATE_QP1_TYPE_GSI; 1533 case IB_QPT_RC: 1534 return CMDQ_CREATE_QP_TYPE_RC; 1535 case IB_QPT_UD: 1536 return CMDQ_CREATE_QP_TYPE_UD; 1537 case IB_QPT_RAW_ETHERTYPE: 1538 return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE; 1539 default: 1540 return IB_QPT_MAX; 1541 } 1542 } 1543 1544 static u16 _get_swqe_sz(int nsge) 1545 { 1546 return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); 1547 } 1548 1549 static int bnxt_re_get_swqe_size(int ilsize, int nsge) 1550 { 1551 u16 wqe_size, calc_ils; 1552 1553 wqe_size = _get_swqe_sz(nsge); 1554 if (ilsize) { 1555 calc_ils = (sizeof(struct sq_send_hdr) + ilsize); 1556 wqe_size = max_t(int, calc_ils, wqe_size); 1557 wqe_size = ALIGN(wqe_size, 32); 1558 } 1559 return wqe_size; 1560 } 1561 1562 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, 1563 struct ib_qp_init_attr *init_attr) 1564 { 1565 struct bnxt_qplib_dev_attr *dev_attr; 1566 struct bnxt_qplib_qp *qplqp; 1567 struct bnxt_re_dev *rdev; 1568 struct bnxt_qplib_q *sq; 1569 int align, ilsize; 1570 1571 rdev = qp->rdev; 1572 qplqp = &qp->qplib_qp; 1573 sq = &qplqp->sq; 1574 dev_attr = rdev->dev_attr; 1575 1576 align = sizeof(struct sq_send_hdr); 1577 ilsize = ALIGN(init_attr->cap.max_inline_data, align); 1578 1579 /* For gen p4 and gen p5 fixed wqe compatibility mode 1580 * wqe size is fixed to 128 bytes - ie 6 SGEs 1581 */ 1582 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) { 1583 sq->wqe_size = _get_swqe_sz(BNXT_STATIC_MAX_SGE); 1584 sq->max_sge = BNXT_STATIC_MAX_SGE; 1585 } else { 1586 sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); 1587 if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) 1588 return -EINVAL; 1589 } 1590 1591 1592 if (init_attr->cap.max_inline_data) { 1593 qplqp->max_inline_data = sq->wqe_size - 1594 sizeof(struct sq_send_hdr); 1595 init_attr->cap.max_inline_data = qplqp->max_inline_data; 1596 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) 1597 sq->max_sge = qplqp->max_inline_data / 1598 sizeof(struct sq_sge); 1599 } 1600 1601 return 0; 1602 } 1603 1604 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, 1605 struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, 1606 struct ib_udata *udata) 1607 { 1608 struct bnxt_qplib_sg_info *sginfo; 1609 struct bnxt_qplib_qp *qplib_qp; 1610 struct bnxt_re_ucontext *cntx; 1611 struct ib_ucontext *context; 1612 struct bnxt_re_qp_req ureq; 1613 struct ib_umem *umem; 1614 int rc, bytes = 0; 1615 int psn_nume; 1616 int psn_sz; 1617 1618 qplib_qp = &qp->qplib_qp; 1619 context = pd->ibpd.uobject->context; 1620 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 1621 sginfo = &qplib_qp->sq.sginfo; 1622 1623 if (udata->inlen < sizeof(ureq)) 1624 dev_warn(rdev_to_dev(rdev), 1625 "Update the library ulen %d klen %d\n", 1626 (unsigned int)udata->inlen, 1627 (unsigned int)sizeof(ureq)); 1628 1629 rc = ib_copy_from_udata(&ureq, udata, 1630 min(udata->inlen, sizeof(ureq))); 1631 if (rc) 1632 return rc; 1633 1634 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); 1635 bytes = PAGE_ALIGN(bytes); 1636 /* Consider mapping PSN search memory only for RC QPs. */ 1637 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { 1638 psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? 1639 sizeof(struct sq_psn_search_ext) : 1640 sizeof(struct sq_psn_search); 1641 if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) 1642 psn_sz = sizeof(struct sq_msn_search); 1643 if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { 1644 psn_nume = ureq.sq_slots; 1645 } else { 1646 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? 1647 qplib_qp->sq.max_wqe : 1648 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / 1649 sizeof(struct bnxt_qplib_sge)); 1650 } 1651 if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) 1652 psn_nume = roundup_pow_of_two(psn_nume); 1653 1654 bytes += (psn_nume * psn_sz); 1655 bytes = PAGE_ALIGN(bytes); 1656 } 1657 umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, 1658 IB_ACCESS_LOCAL_WRITE, 1); 1659 if (IS_ERR(umem)) { 1660 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", 1661 __func__, PTR_ERR(umem)); 1662 return PTR_ERR(umem); 1663 } 1664 1665 qp->sumem = umem; 1666 /* pgsize and pgshft were initialize already. */ 1667 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); 1668 sginfo->npages = ib_umem_num_pages_compat(umem); 1669 qplib_qp->qp_handle = ureq.qp_handle; 1670 1671 if (!qp->qplib_qp.srq) { 1672 sginfo = &qplib_qp->rq.sginfo; 1673 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); 1674 bytes = PAGE_ALIGN(bytes); 1675 umem = ib_umem_get_compat(rdev, 1676 context, udata, ureq.qprva, bytes, 1677 IB_ACCESS_LOCAL_WRITE, 1); 1678 if (IS_ERR(umem)) { 1679 dev_err(rdev_to_dev(rdev), 1680 "%s: ib_umem_get failed ret =%ld\n", 1681 __func__, PTR_ERR(umem)); 1682 goto rqfail; 1683 } 1684 qp->rumem = umem; 1685 /* pgsize and pgshft were initialize already. */ 1686 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); 1687 sginfo->npages = ib_umem_num_pages_compat(umem); 1688 } 1689 1690 qplib_qp->dpi = &cntx->dpi; 1691 qplib_qp->is_user = true; 1692 1693 return 0; 1694 rqfail: 1695 ib_umem_release(qp->sumem); 1696 qp->sumem = NULL; 1697 qplib_qp->sq.sginfo.sghead = NULL; 1698 qplib_qp->sq.sginfo.nmap = 0; 1699 1700 return PTR_ERR(umem); 1701 } 1702 1703 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd, 1704 struct bnxt_qplib_res *qp1_res, 1705 struct bnxt_qplib_qp *qp1_qp) 1706 { 1707 struct bnxt_re_dev *rdev = pd->rdev; 1708 struct bnxt_re_ah *ah; 1709 union ib_gid sgid; 1710 int rc; 1711 1712 ah = kzalloc(sizeof(*ah), GFP_KERNEL); 1713 if (!ah) { 1714 dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n"); 1715 return NULL; 1716 } 1717 memset(ah, 0, sizeof(*ah)); 1718 ah->rdev = rdev; 1719 ah->qplib_ah.pd = &pd->qplib_pd; 1720 1721 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); 1722 if (rc) 1723 goto fail; 1724 1725 /* supply the dgid data same as sgid */ 1726 memcpy(ah->qplib_ah.dgid.data, &sgid.raw, 1727 sizeof(union ib_gid)); 1728 ah->qplib_ah.sgid_index = 0; 1729 1730 ah->qplib_ah.traffic_class = 0; 1731 ah->qplib_ah.flow_label = 0; 1732 ah->qplib_ah.hop_limit = 1; 1733 ah->qplib_ah.sl = 0; 1734 /* Have DMAC same as SMAC */ 1735 ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr); 1736 dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n", 1737 ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2], 1738 ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]); 1739 1740 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true); 1741 if (rc) { 1742 dev_err(rdev_to_dev(rdev), 1743 "Allocate HW AH for Shadow QP failed!\n"); 1744 goto fail; 1745 } 1746 dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id); 1747 atomic_inc(&rdev->stats.rsors.ah_count); 1748 1749 return ah; 1750 fail: 1751 kfree(ah); 1752 return NULL; 1753 } 1754 1755 void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev) 1756 { 1757 struct bnxt_re_qp *gsi_qp; 1758 struct bnxt_re_ah *sah; 1759 struct bnxt_re_pd *pd; 1760 struct ib_pd *ib_pd; 1761 int rc; 1762 1763 if (!rdev) 1764 return; 1765 1766 sah = rdev->gsi_ctx.gsi_sah; 1767 1768 dev_dbg(rdev_to_dev(rdev), "Updating the AH\n"); 1769 if (sah) { 1770 /* Check if the AH created with current mac address */ 1771 if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) { 1772 dev_dbg(rdev_to_dev(rdev), 1773 "Not modifying shadow AH during AH update\n"); 1774 return; 1775 } 1776 1777 gsi_qp = rdev->gsi_ctx.gsi_qp; 1778 ib_pd = gsi_qp->ib_qp.pd; 1779 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 1780 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, 1781 &sah->qplib_ah, false); 1782 if (rc) { 1783 dev_err(rdev_to_dev(rdev), 1784 "Failed to destroy shadow AH during AH update\n"); 1785 return; 1786 } 1787 atomic_dec(&rdev->stats.rsors.ah_count); 1788 kfree(sah); 1789 rdev->gsi_ctx.gsi_sah = NULL; 1790 1791 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 1792 &gsi_qp->qplib_qp); 1793 if (!sah) { 1794 dev_err(rdev_to_dev(rdev), 1795 "Failed to update AH for ShadowQP\n"); 1796 return; 1797 } 1798 rdev->gsi_ctx.gsi_sah = sah; 1799 atomic_inc(&rdev->stats.rsors.ah_count); 1800 } 1801 } 1802 1803 static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, 1804 struct bnxt_qplib_res *qp1_res, 1805 struct bnxt_qplib_qp *qp1_qp) 1806 { 1807 struct bnxt_re_dev *rdev = pd->rdev; 1808 struct bnxt_re_qp *qp; 1809 int rc; 1810 1811 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1812 if (!qp) { 1813 dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n"); 1814 return NULL; 1815 } 1816 memset(qp, 0, sizeof(*qp)); 1817 qp->rdev = rdev; 1818 1819 /* Initialize the shadow QP structure from the QP1 values */ 1820 ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr); 1821 qp->qplib_qp.pd = &pd->qplib_pd; 1822 qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp; 1823 qp->qplib_qp.type = IB_QPT_UD; 1824 1825 qp->qplib_qp.max_inline_data = 0; 1826 qp->qplib_qp.sig_type = true; 1827 1828 /* Shadow QP SQ depth should be same as QP1 RQ depth */ 1829 qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); 1830 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; 1831 qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe; 1832 qp->qplib_qp.sq.max_sge = 2; 1833 /* Q full delta can be 1 since it is internal QP */ 1834 qp->qplib_qp.sq.q_full_delta = 1; 1835 qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE; 1836 qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT; 1837 1838 qp->qplib_qp.scq = qp1_qp->scq; 1839 qp->qplib_qp.rcq = qp1_qp->rcq; 1840 1841 qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ 1842 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; 1843 qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe; 1844 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; 1845 qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; 1846 qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; 1847 /* Q full delta can be 1 since it is internal QP */ 1848 qp->qplib_qp.rq.q_full_delta = 1; 1849 qp->qplib_qp.mtu = qp1_qp->mtu; 1850 qp->qplib_qp.dpi = &rdev->dpi_privileged; 1851 1852 rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0, 1853 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6); 1854 if (rc) 1855 goto fail; 1856 1857 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); 1858 if (rc) { 1859 dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); 1860 goto qp_fail; 1861 } 1862 1863 dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n", 1864 qp->qplib_qp.id); 1865 spin_lock_init(&qp->sq_lock); 1866 INIT_LIST_HEAD(&qp->list); 1867 mutex_lock(&rdev->qp_lock); 1868 list_add_tail(&qp->list, &rdev->qp_list); 1869 atomic_inc(&rdev->stats.rsors.qp_count); 1870 mutex_unlock(&rdev->qp_lock); 1871 return qp; 1872 qp_fail: 1873 bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp); 1874 fail: 1875 kfree(qp); 1876 return NULL; 1877 } 1878 1879 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, 1880 struct ib_qp_init_attr *init_attr, void *cntx) 1881 { 1882 struct bnxt_qplib_dev_attr *dev_attr; 1883 struct bnxt_qplib_qp *qplqp; 1884 struct bnxt_re_dev *rdev; 1885 struct bnxt_qplib_q *rq; 1886 int entries; 1887 1888 rdev = qp->rdev; 1889 qplqp = &qp->qplib_qp; 1890 rq = &qplqp->rq; 1891 dev_attr = rdev->dev_attr; 1892 1893 if (init_attr->srq) { 1894 struct bnxt_re_srq *srq; 1895 1896 srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq); 1897 if (!srq) { 1898 dev_err(rdev_to_dev(rdev), "SRQ not found\n"); 1899 return -EINVAL; 1900 } 1901 qplqp->srq = &srq->qplib_srq; 1902 rq->max_wqe = 0; 1903 } else { 1904 rq->max_sge = init_attr->cap.max_recv_sge; 1905 if (rq->max_sge > dev_attr->max_qp_sges) 1906 rq->max_sge = dev_attr->max_qp_sges; 1907 init_attr->cap.max_recv_sge = rq->max_sge; 1908 rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge, 1909 dev_attr->max_qp_sges); 1910 1911 /* Allocate 1 more than what's provided so posting max doesn't 1912 mean empty */ 1913 entries = init_attr->cap.max_recv_wr + 1; 1914 entries = bnxt_re_init_depth(entries, cntx); 1915 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); 1916 rq->max_sw_wqe = rq->max_wqe; 1917 rq->q_full_delta = 0; 1918 rq->sginfo.pgsize = PAGE_SIZE; 1919 rq->sginfo.pgshft = PAGE_SHIFT; 1920 } 1921 1922 return 0; 1923 } 1924 1925 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) 1926 { 1927 struct bnxt_qplib_dev_attr *dev_attr; 1928 struct bnxt_qplib_qp *qplqp; 1929 struct bnxt_re_dev *rdev; 1930 1931 rdev = qp->rdev; 1932 qplqp = &qp->qplib_qp; 1933 dev_attr = rdev->dev_attr; 1934 1935 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) 1936 qplqp->rq.max_sge = dev_attr->max_qp_sges; 1937 } 1938 1939 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, 1940 struct ib_qp_init_attr *init_attr, 1941 void *cntx, struct ib_udata *udata) 1942 { 1943 struct bnxt_qplib_dev_attr *dev_attr; 1944 struct bnxt_qplib_qp *qplqp; 1945 struct bnxt_re_qp_req ureq; 1946 struct bnxt_re_dev *rdev; 1947 struct bnxt_qplib_q *sq; 1948 int diff = 0; 1949 int entries; 1950 int rc; 1951 1952 rdev = qp->rdev; 1953 qplqp = &qp->qplib_qp; 1954 sq = &qplqp->sq; 1955 dev_attr = rdev->dev_attr; 1956 1957 if (udata) { 1958 rc = ib_copy_from_udata(&ureq, udata, 1959 min(udata->inlen, sizeof(ureq))); 1960 if (rc) 1961 return -EINVAL; 1962 } 1963 1964 sq->max_sge = init_attr->cap.max_send_sge; 1965 entries = init_attr->cap.max_send_wr; 1966 if (cntx && udata && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) { 1967 sq->max_wqe = ureq.sq_slots; 1968 sq->max_sw_wqe = ureq.sq_slots; 1969 sq->wqe_size = sizeof(struct sq_sge); 1970 } else { 1971 if (sq->max_sge > dev_attr->max_qp_sges) { 1972 sq->max_sge = dev_attr->max_qp_sges; 1973 init_attr->cap.max_send_sge = sq->max_sge; 1974 } 1975 rc = bnxt_re_setup_swqe_size(qp, init_attr); 1976 if (rc) 1977 return rc; 1978 /* 1979 * Change the SQ depth if user has requested minimum using 1980 * configfs. Only supported for kernel consumers. Setting 1981 * min_tx_depth to 4096 to handle iser SQ full condition 1982 * in most of the newer OS distros 1983 */ 1984 1985 if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { 1986 /* 1987 * If users specify any value greater than 1 use min_tx_depth 1988 * provided by user for comparison. Else, compare it with the 1989 * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. 1990 */ 1991 if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) 1992 entries = rdev->min_tx_depth; 1993 else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) 1994 entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; 1995 } 1996 diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); 1997 entries = bnxt_re_init_depth(entries + diff + 1, cntx); 1998 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); 1999 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) 2000 sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true); 2001 else 2002 sq->max_sw_wqe = sq->max_wqe; 2003 } 2004 sq->q_full_delta = diff + 1; 2005 /* 2006 * Reserving one slot for Phantom WQE. Application can 2007 * post one extra entry in this case. But allowing this to avoid 2008 * unexpected Queue full condition 2009 */ 2010 sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */ 2011 sq->sginfo.pgsize = PAGE_SIZE; 2012 sq->sginfo.pgshft = PAGE_SHIFT; 2013 return 0; 2014 } 2015 2016 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, 2017 struct ib_qp_init_attr *init_attr, 2018 void *cntx) 2019 { 2020 struct bnxt_qplib_dev_attr *dev_attr; 2021 struct bnxt_qplib_qp *qplqp; 2022 struct bnxt_re_dev *rdev; 2023 int entries; 2024 2025 rdev = qp->rdev; 2026 qplqp = &qp->qplib_qp; 2027 dev_attr = rdev->dev_attr; 2028 2029 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 2030 entries = init_attr->cap.max_send_wr + 1; 2031 entries = bnxt_re_init_depth(entries, cntx); 2032 qplqp->sq.max_wqe = min_t(u32, entries, 2033 dev_attr->max_qp_wqes + 1); 2034 qplqp->sq.q_full_delta = qplqp->sq.max_wqe - 2035 init_attr->cap.max_send_wr; 2036 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ 2037 if (qplqp->sq.max_sge > dev_attr->max_qp_sges) 2038 qplqp->sq.max_sge = dev_attr->max_qp_sges; 2039 } 2040 } 2041 2042 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, 2043 struct ib_qp_init_attr *init_attr) 2044 { 2045 struct bnxt_qplib_chip_ctx *chip_ctx; 2046 struct bnxt_re_gsi_context *gsi_ctx; 2047 int qptype; 2048 2049 chip_ctx = rdev->chip_ctx; 2050 gsi_ctx = &rdev->gsi_ctx; 2051 2052 qptype = __from_ib_qp_type(init_attr->qp_type); 2053 if (qptype == IB_QPT_MAX) { 2054 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n", 2055 qptype); 2056 qptype = -EINVAL; 2057 goto out; 2058 } 2059 2060 if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) { 2061 /* For Thor always force UD mode. */ 2062 qptype = CMDQ_CREATE_QP_TYPE_GSI; 2063 gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD; 2064 } 2065 out: 2066 return qptype; 2067 } 2068 2069 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, 2070 struct ib_qp_init_attr *init_attr, 2071 struct ib_udata *udata) 2072 { 2073 struct bnxt_qplib_dev_attr *dev_attr; 2074 struct bnxt_re_ucontext *cntx = NULL; 2075 struct ib_ucontext *context; 2076 struct bnxt_qplib_qp *qplqp; 2077 struct bnxt_re_dev *rdev; 2078 struct bnxt_re_cq *cq; 2079 int rc = 0, qptype; 2080 2081 rdev = qp->rdev; 2082 qplqp = &qp->qplib_qp; 2083 dev_attr = rdev->dev_attr; 2084 2085 if (udata) { 2086 context = pd->ibpd.uobject->context; 2087 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); 2088 } 2089 2090 /* Setup misc params */ 2091 qplqp->is_user = false; 2092 qplqp->pd = &pd->qplib_pd; 2093 qplqp->qp_handle = (u64)qplqp; 2094 qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 2095 true : false); 2096 qptype = bnxt_re_init_qp_type(rdev, init_attr); 2097 if (qptype < 0) { 2098 rc = qptype; 2099 goto out; 2100 } 2101 qplqp->type = (u8)qptype; 2102 qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, cntx); 2103 ether_addr_copy(qplqp->smac, rdev->dev_addr); 2104 2105 if (init_attr->qp_type == IB_QPT_RC) { 2106 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; 2107 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; 2108 } 2109 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(if_getmtu(rdev->netdev))); 2110 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ 2111 if (init_attr->create_flags) { 2112 dev_dbg(rdev_to_dev(rdev), 2113 "QP create flags 0x%x not supported\n", 2114 init_attr->create_flags); 2115 return -EOPNOTSUPP; 2116 } 2117 2118 /* Setup CQs */ 2119 if (init_attr->send_cq) { 2120 cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq); 2121 if (!cq) { 2122 dev_err(rdev_to_dev(rdev), "Send CQ not found\n"); 2123 rc = -EINVAL; 2124 goto out; 2125 } 2126 qplqp->scq = &cq->qplib_cq; 2127 qp->scq = cq; 2128 } 2129 2130 if (init_attr->recv_cq) { 2131 cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq); 2132 if (!cq) { 2133 dev_err(rdev_to_dev(rdev), "Receive CQ not found\n"); 2134 rc = -EINVAL; 2135 goto out; 2136 } 2137 qplqp->rcq = &cq->qplib_cq; 2138 qp->rcq = cq; 2139 } 2140 2141 /* Setup RQ/SRQ */ 2142 rc = bnxt_re_init_rq_attr(qp, init_attr, cntx); 2143 if (rc) 2144 goto out; 2145 if (init_attr->qp_type == IB_QPT_GSI) 2146 bnxt_re_adjust_gsi_rq_attr(qp); 2147 2148 /* Setup SQ */ 2149 rc = bnxt_re_init_sq_attr(qp, init_attr, cntx, udata); 2150 if (rc) 2151 goto out; 2152 if (init_attr->qp_type == IB_QPT_GSI) 2153 bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx); 2154 2155 if (udata) /* This will update DPI and qp_handle */ 2156 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); 2157 out: 2158 return rc; 2159 } 2160 2161 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, 2162 struct bnxt_re_pd *pd) 2163 { 2164 struct bnxt_re_sqp_entries *sqp_tbl = NULL; 2165 struct bnxt_re_dev *rdev; 2166 struct bnxt_re_qp *sqp; 2167 struct bnxt_re_ah *sah; 2168 int rc = 0; 2169 2170 rdev = qp->rdev; 2171 /* Create a shadow QP to handle the QP1 traffic */ 2172 sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES, 2173 GFP_KERNEL); 2174 if (!sqp_tbl) 2175 return -ENOMEM; 2176 rdev->gsi_ctx.sqp_tbl = sqp_tbl; 2177 2178 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); 2179 if (!sqp) { 2180 rc = -ENODEV; 2181 dev_err(rdev_to_dev(rdev), 2182 "Failed to create Shadow QP for QP1\n"); 2183 goto out; 2184 } 2185 rdev->gsi_ctx.gsi_sqp = sqp; 2186 2187 sqp->rcq = qp->rcq; 2188 sqp->scq = qp->scq; 2189 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, 2190 &qp->qplib_qp); 2191 if (!sah) { 2192 bnxt_qplib_destroy_qp(&rdev->qplib_res, 2193 &sqp->qplib_qp); 2194 rc = -ENODEV; 2195 dev_err(rdev_to_dev(rdev), 2196 "Failed to create AH entry for ShadowQP\n"); 2197 goto out; 2198 } 2199 rdev->gsi_ctx.gsi_sah = sah; 2200 2201 return 0; 2202 out: 2203 kfree(sqp_tbl); 2204 return rc; 2205 } 2206 2207 static int __get_rq_hdr_buf_size(u8 gsi_mode) 2208 { 2209 return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ? 2210 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 : 2211 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE; 2212 } 2213 2214 static int __get_sq_hdr_buf_size(u8 gsi_mode) 2215 { 2216 return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ? 2217 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 : 2218 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE; 2219 } 2220 2221 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd) 2222 { 2223 struct bnxt_qplib_qp *qplqp; 2224 struct bnxt_qplib_res *res; 2225 struct bnxt_re_dev *rdev; 2226 u32 sstep, rstep; 2227 u8 gsi_mode; 2228 int rc = 0; 2229 2230 rdev = qp->rdev; 2231 qplqp = &qp->qplib_qp; 2232 res = &rdev->qplib_res; 2233 gsi_mode = rdev->gsi_ctx.gsi_qp_mode; 2234 2235 rstep = __get_rq_hdr_buf_size(gsi_mode); 2236 sstep = __get_sq_hdr_buf_size(gsi_mode); 2237 rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep); 2238 if (rc) 2239 goto out; 2240 2241 rc = bnxt_qplib_create_qp1(res, qplqp); 2242 if (rc) { 2243 dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n"); 2244 goto out; 2245 } 2246 2247 if (gsi_mode == BNXT_RE_GSI_MODE_ALL) 2248 rc = bnxt_re_create_shadow_gsi(qp, pd); 2249 out: 2250 return rc; 2251 } 2252 2253 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, 2254 struct ib_qp_init_attr *init_attr, 2255 struct bnxt_qplib_dev_attr *dev_attr) 2256 { 2257 bool rc = true; 2258 int ilsize; 2259 2260 ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge)); 2261 if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || 2262 (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || 2263 (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || 2264 (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || 2265 (ilsize > dev_attr->max_inline_data)) { 2266 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! " 2267 "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x " 2268 "0x%x/0x%x 0x%x/0x%x\n", 2269 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, 2270 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, 2271 init_attr->cap.max_send_sge, dev_attr->max_qp_sges, 2272 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, 2273 init_attr->cap.max_inline_data, 2274 dev_attr->max_inline_data); 2275 rc = false; 2276 } 2277 return rc; 2278 } 2279 2280 static inline struct 2281 bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, 2282 struct bnxt_re_dev *rdev) 2283 { 2284 struct bnxt_re_qp *qp; 2285 2286 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2287 if (!qp) 2288 dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n"); 2289 return qp; 2290 } 2291 2292 struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, 2293 struct ib_qp_init_attr *qp_init_attr, 2294 struct ib_udata *udata) 2295 { 2296 struct bnxt_re_pd *pd; 2297 struct ib_pd *ib_pd = qp_in; 2298 struct bnxt_qplib_dev_attr *dev_attr; 2299 struct bnxt_re_dev *rdev; 2300 u32 active_qps, tmp_qps; 2301 struct bnxt_re_qp *qp; 2302 int rc; 2303 2304 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 2305 rdev = pd->rdev; 2306 dev_attr = rdev->dev_attr; 2307 if (rdev->mod_exit) { 2308 rc = -EIO; 2309 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 2310 goto exit; 2311 } 2312 2313 if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) { 2314 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n", 2315 atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp); 2316 rc = -EINVAL; 2317 goto exit; 2318 } 2319 2320 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); 2321 if (!rc) { 2322 rc = -EINVAL; 2323 goto exit; 2324 } 2325 qp = __get_qp_from_qp_in(qp_in, rdev); 2326 if (!qp) { 2327 rc = -ENOMEM; 2328 goto exit; 2329 } 2330 qp->rdev = rdev; 2331 2332 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); 2333 if (rc) 2334 goto fail; 2335 2336 if (qp_init_attr->qp_type == IB_QPT_GSI && 2337 !_is_chip_gen_p5_p7(rdev->chip_ctx)) { 2338 rc = bnxt_re_create_gsi_qp(qp, pd); 2339 if (rc == -ENODEV) 2340 goto qp_destroy; 2341 if (rc) 2342 goto fail; 2343 } else { 2344 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); 2345 if (rc) { 2346 dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); 2347 goto free_umem; 2348 } 2349 2350 if (udata) { 2351 struct bnxt_re_qp_resp resp; 2352 2353 resp.qpid = qp->qplib_qp.id; 2354 rc = bnxt_re_copy_to_udata(rdev, &resp, 2355 min(udata->outlen, sizeof(resp)), 2356 udata); 2357 if (rc) 2358 goto qp_destroy; 2359 } 2360 } 2361 2362 qp->ib_qp.qp_num = qp->qplib_qp.id; 2363 if (qp_init_attr->qp_type == IB_QPT_GSI) 2364 rdev->gsi_ctx.gsi_qp = qp; 2365 spin_lock_init(&qp->sq_lock); 2366 spin_lock_init(&qp->rq_lock); 2367 INIT_LIST_HEAD(&qp->list); 2368 mutex_lock(&rdev->qp_lock); 2369 list_add_tail(&qp->list, &rdev->qp_list); 2370 mutex_unlock(&rdev->qp_lock); 2371 atomic_inc(&rdev->stats.rsors.qp_count); 2372 active_qps = atomic_read(&rdev->stats.rsors.qp_count); 2373 if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count)) 2374 atomic_set(&rdev->stats.rsors.max_qp_count, active_qps); 2375 2376 bnxt_re_dump_debug_stats(rdev, active_qps); 2377 2378 /* Get the counters for RC QPs and UD QPs */ 2379 if (qp_init_attr->qp_type == IB_QPT_RC) { 2380 tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count); 2381 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count)) 2382 atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps); 2383 } else if (qp_init_attr->qp_type == IB_QPT_UD) { 2384 tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count); 2385 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count)) 2386 atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps); 2387 } 2388 2389 return &qp->ib_qp; 2390 2391 qp_destroy: 2392 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); 2393 free_umem: 2394 if (udata) { 2395 if (qp->rumem && !IS_ERR(qp->rumem)) 2396 ib_umem_release(qp->rumem); 2397 if (qp->sumem && !IS_ERR(qp->sumem)) 2398 ib_umem_release(qp->sumem); 2399 } 2400 fail: 2401 kfree(qp); 2402 exit: 2403 return ERR_PTR(rc); 2404 } 2405 2406 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, 2407 struct bnxt_re_qp *qp1_qp, 2408 int qp_attr_mask) 2409 { 2410 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; 2411 int rc = 0; 2412 2413 if (qp_attr_mask & IB_QP_STATE) { 2414 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 2415 qp->qplib_qp.state = qp1_qp->qplib_qp.state; 2416 } 2417 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 2418 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 2419 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; 2420 } 2421 2422 if (qp_attr_mask & IB_QP_QKEY) { 2423 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 2424 /* Using a Random QKEY */ 2425 qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY; 2426 } 2427 if (qp_attr_mask & IB_QP_SQ_PSN) { 2428 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 2429 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; 2430 } 2431 2432 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2433 if (rc) 2434 dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n"); 2435 return rc; 2436 } 2437 2438 static u32 ipv4_from_gid(u8 *gid) 2439 { 2440 return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]); 2441 } 2442 2443 static u16 get_source_port(struct bnxt_re_dev *rdev, 2444 struct bnxt_re_qp *qp) 2445 { 2446 u8 ip_off, data[48], smac[ETH_ALEN]; 2447 u16 crc = 0, buf_len = 0, i; 2448 u8 addr_len; 2449 u32 qpn; 2450 2451 if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) { 2452 addr_len = 6; 2453 ip_off = 10; 2454 } else { 2455 addr_len = 4; 2456 ip_off = 12; 2457 } 2458 2459 memcpy(smac, qp->qplib_qp.smac, ETH_ALEN); 2460 2461 memset(data, 0, 48); 2462 memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN); 2463 buf_len += ETH_ALEN; 2464 2465 memcpy(data + buf_len, smac, ETH_ALEN); 2466 buf_len += ETH_ALEN; 2467 2468 memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len); 2469 buf_len += addr_len; 2470 2471 memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len); 2472 buf_len += addr_len; 2473 2474 qpn = htonl(qp->qplib_qp.dest_qpn); 2475 memcpy(data + buf_len, (u8 *)&qpn + 1, 3); 2476 buf_len += 3; 2477 2478 for (i = 0; i < buf_len; i++) 2479 crc = crc16(crc, (data + i), 1); 2480 2481 return crc; 2482 } 2483 2484 static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) 2485 { 2486 u16 type; 2487 2488 type = __from_hw_to_ib_qp_type(qp->qplib_qp.type); 2489 2490 /* User-space can extract ip address with sgid_index. */ 2491 if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) { 2492 qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw); 2493 qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data); 2494 } else { 2495 memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw, 2496 sizeof(qp->qp_info_entry.s_ip.ipv6_addr)); 2497 memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data, 2498 sizeof(qp->qp_info_entry.d_ip.ipv6_addr)); 2499 } 2500 2501 if (type == IB_QPT_RC && 2502 (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 || 2503 qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) { 2504 qp->qp_info_entry.s_port = get_source_port(rdev, qp); 2505 } 2506 qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT; 2507 } 2508 2509 static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp) 2510 { 2511 struct bnxt_qplib_q *rq, *sq; 2512 unsigned long flags; 2513 2514 if (qp->sumem) 2515 return; 2516 2517 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { 2518 rq = &qp->qplib_qp.rq; 2519 sq = &qp->qplib_qp.sq; 2520 2521 dev_dbg(rdev_to_dev(qp->rdev), 2522 "Move QP = %p to flush list\n", qp); 2523 flags = bnxt_re_lock_cqs(qp); 2524 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 2525 bnxt_re_unlock_cqs(qp, flags); 2526 2527 if (sq->hwq.prod != sq->hwq.cons) 2528 bnxt_re_handle_cqn(&qp->scq->qplib_cq); 2529 2530 if (qp->rcq && (qp->rcq != qp->scq) && 2531 (rq->hwq.prod != rq->hwq.cons)) 2532 bnxt_re_handle_cqn(&qp->rcq->qplib_cq); 2533 } 2534 2535 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { 2536 dev_dbg(rdev_to_dev(qp->rdev), 2537 "Move QP = %p out of flush list\n", qp); 2538 flags = bnxt_re_lock_cqs(qp); 2539 bnxt_qplib_clean_qp(&qp->qplib_qp); 2540 bnxt_re_unlock_cqs(qp, flags); 2541 } 2542 } 2543 2544 bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, 2545 enum ib_qp_state next_state, 2546 enum ib_qp_type type, 2547 enum ib_qp_attr_mask mask) 2548 { 2549 return (ib_modify_qp_is_ok(cur_state, next_state, 2550 type, mask)); 2551 } 2552 2553 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2554 int qp_attr_mask, struct ib_udata *udata) 2555 { 2556 enum ib_qp_state curr_qp_state, new_qp_state; 2557 struct bnxt_re_modify_qp_ex_resp resp = {}; 2558 struct bnxt_re_modify_qp_ex_req ureq = {}; 2559 struct bnxt_qplib_dev_attr *dev_attr; 2560 struct bnxt_qplib_ppp *ppp = NULL; 2561 struct bnxt_re_dev *rdev; 2562 struct bnxt_re_qp *qp; 2563 const struct ib_gid_attr *sgid_attr; 2564 const union ib_gid *gid_ptr = NULL; 2565 u8 nw_type; 2566 int rc, entries; 2567 bool is_copy_to_udata = false; 2568 bool is_qpmtu_high = false; 2569 2570 qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 2571 rdev = qp->rdev; 2572 dev_attr = rdev->dev_attr; 2573 2574 qp->qplib_qp.modify_flags = 0; 2575 ppp = &qp->qplib_qp.ppp; 2576 if (qp_attr_mask & IB_QP_STATE) { 2577 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); 2578 new_qp_state = qp_attr->qp_state; 2579 if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state, 2580 ib_qp->qp_type, qp_attr_mask)) { 2581 dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x" 2582 " specified for qpn=0x%x of type=0x%x" 2583 " current_qp_state=0x%x, new_qp_state=0x%x\n", 2584 qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, 2585 curr_qp_state, new_qp_state); 2586 return -EINVAL; 2587 } 2588 dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x " 2589 "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n", 2590 __func__, __LINE__, qp_attr_mask, ib_qp->qp_num, 2591 ib_qp->qp_type, curr_qp_state, new_qp_state); 2592 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; 2593 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); 2594 2595 if (udata && curr_qp_state == IB_QPS_RESET && 2596 new_qp_state == IB_QPS_INIT) { 2597 if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 2598 if (ureq.comp_mask & 2599 BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) { 2600 ppp->req = BNXT_QPLIB_PPP_REQ; 2601 ppp->dpi = ureq.dpi; 2602 } 2603 } 2604 } 2605 } 2606 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 2607 qp->qplib_qp.modify_flags |= 2608 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; 2609 qp->qplib_qp.en_sqd_async_notify = true; 2610 } 2611 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { 2612 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; 2613 qp->qplib_qp.access = 2614 __from_ib_access_flags(qp_attr->qp_access_flags); 2615 /* LOCAL_WRITE access must be set to allow RC receive */ 2616 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; 2617 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; 2618 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; 2619 } 2620 if (qp_attr_mask & IB_QP_PKEY_INDEX) { 2621 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; 2622 qp->qplib_qp.pkey_index = qp_attr->pkey_index; 2623 } 2624 if (qp_attr_mask & IB_QP_QKEY) { 2625 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; 2626 qp->qplib_qp.qkey = qp_attr->qkey; 2627 } 2628 if (qp_attr_mask & IB_QP_AV) { 2629 const struct ib_global_route *grh = 2630 rdma_ah_read_grh(&qp_attr->ah_attr); 2631 2632 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | 2633 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | 2634 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | 2635 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | 2636 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | 2637 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | 2638 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; 2639 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw, 2640 sizeof(qp->qplib_qp.ah.dgid.data)); 2641 qp->qplib_qp.ah.flow_label = grh->flow_label; 2642 qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev, 2643 grh->sgid_index); 2644 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index; 2645 qp->qplib_qp.ah.hop_limit = grh->hop_limit; 2646 qp->qplib_qp.ah.traffic_class = grh->traffic_class; 2647 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr); 2648 ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr)); 2649 sgid_attr = grh->sgid_attr; 2650 gid_ptr = &sgid_attr->gid; 2651 if (sgid_attr->ndev) { 2652 rc = rdma_read_gid_l2_fields(sgid_attr, NULL, 2653 &qp->qplib_qp.smac[0]); 2654 if (rc) 2655 return rc; 2656 2657 nw_type = rdma_gid_attr_network_type(sgid_attr); 2658 dev_dbg(rdev_to_dev(rdev), 2659 "Connection using the nw_type %d\n", nw_type); 2660 switch (nw_type) { 2661 case RDMA_NETWORK_IPV4: 2662 qp->qplib_qp.nw_type = 2663 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; 2664 break; 2665 case RDMA_NETWORK_IPV6: 2666 qp->qplib_qp.nw_type = 2667 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; 2668 break; 2669 default: 2670 qp->qplib_qp.nw_type = 2671 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; 2672 break; 2673 } 2674 } 2675 memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid)); 2676 } 2677 2678 /* MTU settings allowed only during INIT -> RTR */ 2679 if (qp_attr->qp_state == IB_QPS_RTR) { 2680 bnxt_re_init_qpmtu(qp, if_getmtu(rdev->netdev), qp_attr_mask, qp_attr, 2681 &is_qpmtu_high); 2682 if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { 2683 if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) { 2684 resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK; 2685 resp.path_mtu = qp->qplib_qp.mtu; 2686 is_copy_to_udata = true; 2687 } else if (is_qpmtu_high) { 2688 dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n", 2689 qp->qplib_qp.id); 2690 return -EINVAL; 2691 } 2692 } 2693 } 2694 2695 if (qp_attr_mask & IB_QP_TIMEOUT) { 2696 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; 2697 qp->qplib_qp.timeout = qp_attr->timeout; 2698 } 2699 if (qp_attr_mask & IB_QP_RETRY_CNT) { 2700 qp->qplib_qp.modify_flags |= 2701 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; 2702 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; 2703 } 2704 if (qp_attr_mask & IB_QP_RNR_RETRY) { 2705 qp->qplib_qp.modify_flags |= 2706 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; 2707 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; 2708 } 2709 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { 2710 qp->qplib_qp.modify_flags |= 2711 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; 2712 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; 2713 } 2714 if (qp_attr_mask & IB_QP_RQ_PSN) { 2715 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; 2716 qp->qplib_qp.rq.psn = qp_attr->rq_psn; 2717 } 2718 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2719 qp->qplib_qp.modify_flags |= 2720 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; 2721 /* Cap the max_rd_atomic to device max */ 2722 if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom) 2723 dev_dbg(rdev_to_dev(rdev), 2724 "max_rd_atomic requested %d is > device max %d\n", 2725 qp_attr->max_rd_atomic, 2726 dev_attr->max_qp_rd_atom); 2727 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, 2728 dev_attr->max_qp_rd_atom); 2729 } 2730 if (qp_attr_mask & IB_QP_SQ_PSN) { 2731 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; 2732 qp->qplib_qp.sq.psn = qp_attr->sq_psn; 2733 } 2734 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2735 if (qp_attr->max_dest_rd_atomic > 2736 dev_attr->max_qp_init_rd_atom) { 2737 dev_err(rdev_to_dev(rdev), 2738 "max_dest_rd_atomic requested %d is > device max %d\n", 2739 qp_attr->max_dest_rd_atomic, 2740 dev_attr->max_qp_init_rd_atom); 2741 return -EINVAL; 2742 } 2743 qp->qplib_qp.modify_flags |= 2744 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; 2745 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; 2746 } 2747 if (qp_attr_mask & IB_QP_CAP) { 2748 qp->qplib_qp.modify_flags |= 2749 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | 2750 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | 2751 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | 2752 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | 2753 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; 2754 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || 2755 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || 2756 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || 2757 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || 2758 (qp_attr->cap.max_inline_data >= 2759 dev_attr->max_inline_data)) { 2760 dev_err(rdev_to_dev(rdev), 2761 "Create QP failed - max exceeded\n"); 2762 return -EINVAL; 2763 } 2764 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); 2765 if (entries > dev_attr->max_qp_wqes) 2766 entries = dev_attr->max_qp_wqes; 2767 entries = min_t(u32, entries, dev_attr->max_qp_wqes); 2768 qp->qplib_qp.sq.max_wqe = entries; 2769 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - 2770 qp_attr->cap.max_send_wr; 2771 /* 2772 * Reserving one slot for Phantom WQE. Some application can 2773 * post one extra entry in this case. Allowing this to avoid 2774 * unexpected Queue full condition 2775 */ 2776 qp->qplib_qp.sq.q_full_delta -= 1; 2777 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; 2778 if (qp->qplib_qp.rq.max_wqe) { 2779 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); 2780 if (entries > dev_attr->max_qp_wqes) 2781 entries = dev_attr->max_qp_wqes; 2782 qp->qplib_qp.rq.max_wqe = entries; 2783 qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe; 2784 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - 2785 qp_attr->cap.max_recv_wr; 2786 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; 2787 } else { 2788 /* SRQ was used prior, just ignore the RQ caps */ 2789 } 2790 } 2791 if (qp_attr_mask & IB_QP_DEST_QPN) { 2792 qp->qplib_qp.modify_flags |= 2793 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; 2794 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; 2795 } 2796 2797 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 2798 if (rc) { 2799 dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n"); 2800 return rc; 2801 } 2802 if (qp_attr_mask & IB_QP_STATE) 2803 bnxt_qplib_manage_flush_qp(qp); 2804 if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK && 2805 ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) { 2806 resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN; 2807 resp.ppp_st_idx = ppp->st_idx_en >> 2808 BNXT_QPLIB_PPP_ST_IDX_SHIFT; 2809 is_copy_to_udata = true; 2810 } 2811 2812 if (is_copy_to_udata) { 2813 rc = bnxt_re_copy_to_udata(rdev, &resp, 2814 min(udata->outlen, sizeof(resp)), 2815 udata); 2816 if (rc) 2817 return rc; 2818 } 2819 2820 if (ib_qp->qp_type == IB_QPT_GSI && 2821 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && 2822 rdev->gsi_ctx.gsi_sqp) 2823 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); 2824 /* 2825 * Update info when qp_info_info 2826 */ 2827 bnxt_re_update_qp_info(rdev, qp); 2828 return rc; 2829 } 2830 2831 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, 2832 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 2833 { 2834 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 2835 struct bnxt_re_dev *rdev = qp->rdev; 2836 struct bnxt_qplib_qp *qplib_qp; 2837 int rc; 2838 2839 qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL); 2840 if (!qplib_qp) 2841 return -ENOMEM; 2842 2843 qplib_qp->id = qp->qplib_qp.id; 2844 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; 2845 2846 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); 2847 if (rc) { 2848 dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n", 2849 qplib_qp->id, rc); 2850 goto free_mem; 2851 } 2852 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); 2853 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); 2854 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; 2855 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); 2856 qp_attr->pkey_index = qplib_qp->pkey_index; 2857 qp_attr->qkey = qplib_qp->qkey; 2858 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2859 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label, 2860 qplib_qp->ah.host_sgid_index, 2861 qplib_qp->ah.hop_limit, 2862 qplib_qp->ah.traffic_class); 2863 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data); 2864 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl); 2865 ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac); 2866 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); 2867 qp_attr->timeout = qplib_qp->timeout; 2868 qp_attr->retry_cnt = qplib_qp->retry_cnt; 2869 qp_attr->rnr_retry = qplib_qp->rnr_retry; 2870 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; 2871 qp_attr->rq_psn = qplib_qp->rq.psn; 2872 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; 2873 qp_attr->sq_psn = qplib_qp->sq.psn; 2874 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; 2875 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : 2876 IB_SIGNAL_REQ_WR; 2877 qp_attr->dest_qp_num = qplib_qp->dest_qpn; 2878 2879 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; 2880 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; 2881 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; 2882 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; 2883 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; 2884 qp_init_attr->cap = qp_attr->cap; 2885 2886 free_mem: 2887 kfree(qplib_qp); 2888 return rc; 2889 } 2890 2891 /* Builders */ 2892 2893 /* For Raw, the application is responsible to build the entire packet */ 2894 static void bnxt_re_build_raw_send(const struct ib_send_wr *wr, 2895 struct bnxt_qplib_swqe *wqe) 2896 { 2897 switch (wr->send_flags) { 2898 case IB_SEND_IP_CSUM: 2899 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 2900 break; 2901 default: 2902 /* Pad HW RoCE iCRC */ 2903 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 2904 break; 2905 } 2906 } 2907 2908 /* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr 2909 * as according to the sgid and AV 2910 */ 2911 static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, 2912 struct bnxt_qplib_swqe *wqe, int payload_size) 2913 { 2914 struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, 2915 ibah); 2916 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; 2917 struct bnxt_qplib_sge sge; 2918 int i, rc = 0; 2919 union ib_gid sgid; 2920 u16 vlan_id; 2921 u8 *ptmac; 2922 void *buf; 2923 2924 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); 2925 2926 /* Get sgid */ 2927 rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid); 2928 if (rc) 2929 return rc; 2930 2931 /* ETH */ 2932 qp->qp1_hdr.eth_present = 1; 2933 ptmac = ah->qplib_ah.dmac; 2934 memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4); 2935 ptmac += 4; 2936 memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2); 2937 2938 ptmac = qp->qplib_qp.smac; 2939 memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2); 2940 ptmac += 2; 2941 memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4); 2942 2943 qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1); 2944 2945 /* For vlan, check the sgid for vlan existence */ 2946 vlan_id = rdma_get_vlan_id(&sgid); 2947 if (vlan_id && vlan_id < 0x1000) { 2948 qp->qp1_hdr.vlan_present = 1; 2949 qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q); 2950 } 2951 /* GRH */ 2952 qp->qp1_hdr.grh_present = 1; 2953 qp->qp1_hdr.grh.ip_version = 6; 2954 qp->qp1_hdr.grh.payload_length = 2955 cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7) 2956 & ~3); 2957 qp->qp1_hdr.grh.next_header = 0x1b; 2958 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); 2959 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, 2960 sizeof(sgid)); 2961 2962 /* BTH */ 2963 if (wr->opcode == IB_WR_SEND_WITH_IMM) { 2964 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; 2965 qp->qp1_hdr.immediate_present = 1; 2966 } else { 2967 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; 2968 } 2969 if (wr->send_flags & IB_SEND_SOLICITED) 2970 qp->qp1_hdr.bth.solicited_event = 1; 2971 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; 2972 /* P_key for QP1 is for all members */ 2973 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); 2974 qp->qp1_hdr.bth.destination_qpn = IB_QP1; 2975 qp->qp1_hdr.bth.ack_req = 0; 2976 qp->send_psn++; 2977 qp->send_psn &= BTH_PSN_MASK; 2978 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); 2979 /* DETH */ 2980 /* Use the priviledged Q_Key for QP1 */ 2981 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); 2982 qp->qp1_hdr.deth.source_qpn = IB_QP1; 2983 2984 /* Pack the QP1 to the transmit buffer */ 2985 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); 2986 if (!buf) { 2987 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); 2988 rc = -ENOMEM; 2989 } 2990 for (i = wqe->num_sge; i; i--) { 2991 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; 2992 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; 2993 wqe->sg_list[i].size = wqe->sg_list[i - 1].size; 2994 } 2995 wqe->sg_list[0].addr = sge.addr; 2996 wqe->sg_list[0].lkey = sge.lkey; 2997 wqe->sg_list[0].size = sge.size; 2998 wqe->num_sge++; 2999 3000 return rc; 3001 } 3002 3003 static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp, 3004 const struct ib_send_wr *wr, 3005 struct bnxt_qplib_swqe *wqe) 3006 { 3007 struct bnxt_re_dev *rdev; 3008 int rc, indx, len = 0; 3009 3010 rdev = qp->rdev; 3011 3012 /* Mode UD is applicable to Gen P5 only */ 3013 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) 3014 return 0; 3015 3016 for (indx = 0; indx < wr->num_sge; indx++) { 3017 wqe->sg_list[indx].addr = wr->sg_list[indx].addr; 3018 wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey; 3019 wqe->sg_list[indx].size = wr->sg_list[indx].length; 3020 len += wr->sg_list[indx].length; 3021 } 3022 rc = bnxt_re_build_qp1_send(qp, wr, wqe, len); 3023 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; 3024 3025 return rc; 3026 } 3027 3028 /* For the MAD layer, it only provides the recv SGE the size of 3029 ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, 3030 nor RoCE iCRC. The Cu+ solution must provide buffer for the entire 3031 receive packet (334 bytes) with no VLAN and then copy the GRH 3032 and the MAD datagram out to the provided SGE. 3033 */ 3034 3035 static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp, 3036 const struct ib_recv_wr *wr, 3037 struct bnxt_qplib_swqe *wqe) 3038 { 3039 struct bnxt_re_dev *rdev = qp->rdev; 3040 struct bnxt_qplib_sge ref, sge; 3041 u8 udp_hdr_size = 0; 3042 u8 ip_hdr_size = 0; 3043 int rc = 0; 3044 int size; 3045 3046 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { 3047 /* Create 5 SGEs as according to the following: 3048 * Ethernet header (14) 3049 * ib_grh (40) - as provided from the wr 3050 * ib_bth + ib_deth + UDP(RoCE v2 only) (28) 3051 * MAD (256) - as provided from the wr 3052 * iCRC (4) 3053 */ 3054 3055 /* Set RoCE v2 header size and offsets */ 3056 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4) 3057 ip_hdr_size = 20; 3058 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1) 3059 udp_hdr_size = 8; 3060 3061 /* Save the reference from ULP */ 3062 ref.addr = wr->sg_list[0].addr; 3063 ref.lkey = wr->sg_list[0].lkey; 3064 ref.size = wr->sg_list[0].length; 3065 3066 /* SGE 1 */ 3067 size = sge.size; 3068 wqe->sg_list[0].addr = sge.addr; 3069 wqe->sg_list[0].lkey = sge.lkey; 3070 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE; 3071 size -= wqe->sg_list[0].size; 3072 if (size <= 0) { 3073 dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n"); 3074 rc = -ENOMEM; 3075 goto done; 3076 } 3077 sge.size = (u32)size; 3078 sge.addr += wqe->sg_list[0].size; 3079 3080 /* SGE 2 */ 3081 /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */ 3082 wqe->sg_list[1].addr = ref.addr + ip_hdr_size; 3083 wqe->sg_list[1].lkey = ref.lkey; 3084 wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size; 3085 ref.size -= wqe->sg_list[1].size; 3086 if (ref.size <= 0) { 3087 dev_err(rdev_to_dev(qp->rdev), 3088 "QP1 ref buffer is empty!\n"); 3089 rc = -ENOMEM; 3090 goto done; 3091 } 3092 ref.addr += wqe->sg_list[1].size + ip_hdr_size; 3093 3094 /* SGE 3 */ 3095 wqe->sg_list[2].addr = sge.addr; 3096 wqe->sg_list[2].lkey = sge.lkey; 3097 wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE + 3098 udp_hdr_size; 3099 size -= wqe->sg_list[2].size; 3100 if (size <= 0) { 3101 dev_err(rdev_to_dev(qp->rdev), 3102 "QP1 rq buffer is empty!\n"); 3103 rc = -ENOMEM; 3104 goto done; 3105 } 3106 sge.size = (u32)size; 3107 sge.addr += wqe->sg_list[2].size; 3108 3109 /* SGE 4 */ 3110 wqe->sg_list[3].addr = ref.addr; 3111 wqe->sg_list[3].lkey = ref.lkey; 3112 wqe->sg_list[3].size = ref.size; 3113 ref.size -= wqe->sg_list[3].size; 3114 if (ref.size) { 3115 dev_err(rdev_to_dev(qp->rdev), 3116 "QP1 ref buffer is incorrect!\n"); 3117 rc = -ENOMEM; 3118 goto done; 3119 } 3120 /* SGE 5 */ 3121 wqe->sg_list[4].addr = sge.addr; 3122 wqe->sg_list[4].lkey = sge.lkey; 3123 wqe->sg_list[4].size = sge.size; 3124 size -= wqe->sg_list[4].size; 3125 if (size) { 3126 dev_err(rdev_to_dev(qp->rdev), 3127 "QP1 rq buffer is incorrect!\n"); 3128 rc = -ENOMEM; 3129 goto done; 3130 } 3131 sge.size = (u32)size; 3132 wqe->num_sge = 5; 3133 } else { 3134 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); 3135 rc = -ENOMEM; 3136 } 3137 done: 3138 return rc; 3139 } 3140 3141 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, 3142 const struct ib_recv_wr *wr, 3143 struct bnxt_qplib_swqe *wqe) 3144 { 3145 struct bnxt_re_sqp_entries *sqp_entry; 3146 struct bnxt_qplib_sge sge; 3147 struct bnxt_re_dev *rdev; 3148 u32 rq_prod_index; 3149 int rc = 0; 3150 3151 rdev = qp->rdev; 3152 3153 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); 3154 3155 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { 3156 /* Create 1 SGE to receive the entire 3157 * ethernet packet 3158 */ 3159 /* SGE 1 */ 3160 wqe->sg_list[0].addr = sge.addr; 3161 /* TODO check the lkey to be used */ 3162 wqe->sg_list[0].lkey = sge.lkey; 3163 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; 3164 if (sge.size < wqe->sg_list[0].size) { 3165 dev_err(rdev_to_dev(qp->rdev), 3166 "QP1 rq buffer is empty!\n"); 3167 rc = -ENOMEM; 3168 goto done; 3169 } 3170 3171 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; 3172 sqp_entry->sge.addr = wr->sg_list[0].addr; 3173 sqp_entry->sge.lkey = wr->sg_list[0].lkey; 3174 sqp_entry->sge.size = wr->sg_list[0].length; 3175 /* Store the wrid for reporting completion */ 3176 sqp_entry->wrid = wqe->wr_id; 3177 /* change the wqe->wrid to table index */ 3178 wqe->wr_id = rq_prod_index; 3179 } 3180 done: 3181 return rc; 3182 } 3183 3184 static bool is_ud_qp(struct bnxt_re_qp *qp) 3185 { 3186 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || 3187 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); 3188 } 3189 3190 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, 3191 const struct ib_send_wr *wr, 3192 struct bnxt_qplib_swqe *wqe) 3193 { 3194 struct bnxt_re_ah *ah = NULL; 3195 3196 if(is_ud_qp(qp)) { 3197 ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); 3198 wqe->send.q_key = ud_wr(wr)->remote_qkey; 3199 wqe->send.dst_qp = ud_wr(wr)->remote_qpn; 3200 wqe->send.avid = ah->qplib_ah.id; 3201 } 3202 switch (wr->opcode) { 3203 case IB_WR_SEND: 3204 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; 3205 break; 3206 case IB_WR_SEND_WITH_IMM: 3207 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; 3208 wqe->send.imm_data = wr->ex.imm_data; 3209 break; 3210 case IB_WR_SEND_WITH_INV: 3211 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; 3212 wqe->send.inv_key = wr->ex.invalidate_rkey; 3213 break; 3214 default: 3215 dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n", 3216 __func__, wr->opcode); 3217 return -EINVAL; 3218 } 3219 if (wr->send_flags & IB_SEND_SIGNALED) 3220 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3221 if (wr->send_flags & IB_SEND_FENCE) 3222 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3223 if (wr->send_flags & IB_SEND_SOLICITED) 3224 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3225 if (wr->send_flags & IB_SEND_INLINE) 3226 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 3227 3228 return 0; 3229 } 3230 3231 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, 3232 struct bnxt_qplib_swqe *wqe) 3233 { 3234 switch (wr->opcode) { 3235 case IB_WR_RDMA_WRITE: 3236 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; 3237 break; 3238 case IB_WR_RDMA_WRITE_WITH_IMM: 3239 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; 3240 wqe->rdma.imm_data = wr->ex.imm_data; 3241 break; 3242 case IB_WR_RDMA_READ: 3243 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; 3244 wqe->rdma.inv_key = wr->ex.invalidate_rkey; 3245 break; 3246 default: 3247 return -EINVAL; 3248 } 3249 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; 3250 wqe->rdma.r_key = rdma_wr(wr)->rkey; 3251 if (wr->send_flags & IB_SEND_SIGNALED) 3252 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3253 if (wr->send_flags & IB_SEND_FENCE) 3254 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3255 if (wr->send_flags & IB_SEND_SOLICITED) 3256 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3257 if (wr->send_flags & IB_SEND_INLINE) 3258 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; 3259 3260 return 0; 3261 } 3262 3263 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, 3264 struct bnxt_qplib_swqe *wqe) 3265 { 3266 switch (wr->opcode) { 3267 case IB_WR_ATOMIC_CMP_AND_SWP: 3268 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; 3269 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 3270 wqe->atomic.swap_data = atomic_wr(wr)->swap; 3271 break; 3272 case IB_WR_ATOMIC_FETCH_AND_ADD: 3273 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; 3274 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; 3275 break; 3276 default: 3277 return -EINVAL; 3278 } 3279 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; 3280 wqe->atomic.r_key = atomic_wr(wr)->rkey; 3281 if (wr->send_flags & IB_SEND_SIGNALED) 3282 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3283 if (wr->send_flags & IB_SEND_FENCE) 3284 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3285 if (wr->send_flags & IB_SEND_SOLICITED) 3286 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3287 return 0; 3288 } 3289 3290 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, 3291 struct bnxt_qplib_swqe *wqe) 3292 { 3293 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; 3294 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; 3295 if (wr->send_flags & IB_SEND_SIGNALED) 3296 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3297 if (wr->send_flags & IB_SEND_FENCE) 3298 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3299 if (wr->send_flags & IB_SEND_SOLICITED) 3300 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; 3301 3302 return 0; 3303 } 3304 3305 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, 3306 struct bnxt_qplib_swqe *wqe) 3307 { 3308 struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr); 3309 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; 3310 int reg_len, i, access = wr->access; 3311 3312 if (mr->npages > qplib_frpl->max_pg_ptrs) { 3313 dev_err_ratelimited(rdev_to_dev(mr->rdev), 3314 " %s: failed npages %d > %d\n", __func__, 3315 mr->npages, qplib_frpl->max_pg_ptrs); 3316 return -EINVAL; 3317 } 3318 3319 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; 3320 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; 3321 wqe->frmr.levels = qplib_frpl->hwq.level; 3322 wqe->frmr.page_list = mr->pages; 3323 wqe->frmr.page_list_len = mr->npages; 3324 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; 3325 3326 if (wr->wr.send_flags & IB_SEND_SIGNALED) 3327 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; 3328 if (access & IB_ACCESS_LOCAL_WRITE) 3329 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; 3330 if (access & IB_ACCESS_REMOTE_READ) 3331 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; 3332 if (access & IB_ACCESS_REMOTE_WRITE) 3333 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; 3334 if (access & IB_ACCESS_REMOTE_ATOMIC) 3335 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; 3336 if (access & IB_ACCESS_MW_BIND) 3337 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; 3338 3339 /* TODO: OFED provides the rkey of the MR instead of the lkey */ 3340 wqe->frmr.l_key = wr->key; 3341 wqe->frmr.length = wr->mr->length; 3342 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); 3343 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); 3344 wqe->frmr.va = wr->mr->iova; 3345 reg_len = wqe->frmr.page_list_len * wr->mr->page_size; 3346 3347 if (wqe->frmr.length > reg_len) { 3348 dev_err_ratelimited(rdev_to_dev(mr->rdev), 3349 "%s: bnxt_re_mr 0x%px len (%d > %d)\n", 3350 __func__, (void *)mr, wqe->frmr.length, 3351 reg_len); 3352 3353 for (i = 0; i < mr->npages; i++) 3354 dev_dbg(rdev_to_dev(mr->rdev), 3355 "%s: build_reg_wqe page[%d] = 0x%llx\n", 3356 __func__, i, mr->pages[i]); 3357 3358 return -EINVAL; 3359 } 3360 3361 return 0; 3362 } 3363 3364 static void bnxt_re_set_sg_list(const struct ib_send_wr *wr, 3365 struct bnxt_qplib_swqe *wqe) 3366 { 3367 wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 3368 wqe->num_sge = wr->num_sge; 3369 } 3370 3371 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) 3372 { 3373 if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI || 3374 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && 3375 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { 3376 int qp_attr_mask; 3377 struct ib_qp_attr qp_attr; 3378 3379 qp_attr_mask = IB_QP_STATE; 3380 qp_attr.qp_state = IB_QPS_RTS; 3381 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); 3382 qp->qplib_qp.wqe_cnt = 0; 3383 } 3384 } 3385 3386 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, 3387 struct bnxt_re_qp *qp, 3388 const struct ib_send_wr *wr) 3389 { 3390 struct bnxt_qplib_swqe wqe; 3391 unsigned long flags; 3392 int rc = 0; 3393 3394 spin_lock_irqsave(&qp->sq_lock, flags); 3395 while (wr) { 3396 /* House keeping */ 3397 memset(&wqe, 0, sizeof(wqe)); 3398 /* Common */ 3399 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 3400 dev_err(rdev_to_dev(rdev), 3401 "Limit exceeded for Send SGEs\n"); 3402 rc = -EINVAL; 3403 break; 3404 } 3405 3406 bnxt_re_set_sg_list(wr, &wqe); 3407 wqe.wr_id = wr->wr_id; 3408 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; 3409 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 3410 if (rc) 3411 break; 3412 3413 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 3414 if (rc) { 3415 dev_err(rdev_to_dev(rdev), 3416 "bad_wr seen with opcode = 0x%x rc = %d\n", 3417 wr->opcode, rc); 3418 break; 3419 } 3420 wr = wr->next; 3421 } 3422 bnxt_qplib_post_send_db(&qp->qplib_qp); 3423 bnxt_ud_qp_hw_stall_workaround(qp); 3424 spin_unlock_irqrestore(&qp->sq_lock, flags); 3425 return rc; 3426 } 3427 3428 static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe) 3429 { 3430 /* Need unconditional fence for non-wire memory opcode 3431 * to work as expected. 3432 */ 3433 if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || 3434 wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || 3435 wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || 3436 wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) 3437 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; 3438 } 3439 3440 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, 3441 const struct ib_send_wr **bad_wr) 3442 { 3443 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 3444 struct bnxt_qplib_sge sge[6]; 3445 struct bnxt_qplib_swqe wqe; 3446 struct bnxt_re_dev *rdev; 3447 unsigned long flags; 3448 int rc = 0; 3449 3450 rdev = qp->rdev; 3451 spin_lock_irqsave(&qp->sq_lock, flags); 3452 while (wr) { 3453 /* House keeping */ 3454 memset(&wqe, 0, sizeof(wqe)); 3455 /* Common */ 3456 if (wr->num_sge > qp->qplib_qp.sq.max_sge) { 3457 dev_err(rdev_to_dev(rdev), 3458 "Limit exceeded for Send SGEs\n"); 3459 rc = -EINVAL; 3460 goto bad; 3461 } 3462 3463 bnxt_re_set_sg_list(wr, &wqe); 3464 wqe.wr_id = wr->wr_id; 3465 3466 switch (wr->opcode) { 3467 case IB_WR_SEND: 3468 case IB_WR_SEND_WITH_IMM: 3469 if (ib_qp->qp_type == IB_QPT_GSI && 3470 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 3471 memset(sge, 0, sizeof(sge)); 3472 wqe.sg_list = sge; 3473 rc = bnxt_re_build_gsi_send(qp, wr, &wqe); 3474 if (rc) 3475 goto bad; 3476 } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) { 3477 bnxt_re_build_raw_send(wr, &wqe); 3478 } 3479 switch (wr->send_flags) { 3480 case IB_SEND_IP_CSUM: 3481 wqe.rawqp1.lflags |= 3482 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; 3483 break; 3484 default: 3485 break; 3486 } 3487 fallthrough; 3488 case IB_WR_SEND_WITH_INV: 3489 rc = bnxt_re_build_send_wqe(qp, wr, &wqe); 3490 break; 3491 case IB_WR_RDMA_WRITE: 3492 case IB_WR_RDMA_WRITE_WITH_IMM: 3493 case IB_WR_RDMA_READ: 3494 rc = bnxt_re_build_rdma_wqe(wr, &wqe); 3495 break; 3496 case IB_WR_ATOMIC_CMP_AND_SWP: 3497 case IB_WR_ATOMIC_FETCH_AND_ADD: 3498 rc = bnxt_re_build_atomic_wqe(wr, &wqe); 3499 break; 3500 case IB_WR_RDMA_READ_WITH_INV: 3501 dev_err(rdev_to_dev(rdev), 3502 "RDMA Read with Invalidate is not supported\n"); 3503 rc = -EINVAL; 3504 goto bad; 3505 case IB_WR_LOCAL_INV: 3506 rc = bnxt_re_build_inv_wqe(wr, &wqe); 3507 break; 3508 case IB_WR_REG_MR: 3509 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); 3510 break; 3511 default: 3512 /* Unsupported WRs */ 3513 dev_err(rdev_to_dev(rdev), 3514 "WR (0x%x) is not supported\n", wr->opcode); 3515 rc = -EINVAL; 3516 goto bad; 3517 } 3518 3519 if (likely(!rc)) { 3520 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 3521 bnxt_re_legacy_set_uc_fence(&wqe); 3522 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); 3523 } 3524 bad: 3525 if (unlikely(rc)) { 3526 dev_err(rdev_to_dev(rdev), 3527 "bad_wr seen with opcode = 0x%x\n", wr->opcode); 3528 *bad_wr = wr; 3529 break; 3530 } 3531 wr = wr->next; 3532 } 3533 bnxt_qplib_post_send_db(&qp->qplib_qp); 3534 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 3535 bnxt_ud_qp_hw_stall_workaround(qp); 3536 spin_unlock_irqrestore(&qp->sq_lock, flags); 3537 3538 return rc; 3539 } 3540 3541 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, 3542 struct bnxt_re_qp *qp, 3543 struct ib_recv_wr *wr) 3544 { 3545 struct bnxt_qplib_swqe wqe; 3546 int rc = 0; 3547 3548 /* rq lock can be pardoned here. */ 3549 while (wr) { 3550 /* House keeping */ 3551 memset(&wqe, 0, sizeof(wqe)); 3552 /* Common */ 3553 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 3554 dev_err(rdev_to_dev(rdev), 3555 "Limit exceeded for Receive SGEs\n"); 3556 rc = -EINVAL; 3557 goto bad; 3558 } 3559 3560 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 3561 wqe.num_sge = wr->num_sge; 3562 wqe.wr_id = wr->wr_id; 3563 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 3564 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 3565 bad: 3566 if (rc) { 3567 dev_err(rdev_to_dev(rdev), 3568 "bad_wr seen with RQ post\n"); 3569 break; 3570 } 3571 wr = wr->next; 3572 } 3573 bnxt_qplib_post_recv_db(&qp->qplib_qp); 3574 return rc; 3575 } 3576 3577 static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp, 3578 const struct ib_recv_wr *wr, 3579 struct bnxt_qplib_swqe *wqe) 3580 { 3581 struct bnxt_re_dev *rdev = qp->rdev; 3582 int rc = 0; 3583 3584 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) 3585 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe); 3586 else 3587 rc = bnxt_re_build_qp1_recv(qp, wr, wqe); 3588 3589 return rc; 3590 } 3591 3592 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, 3593 const struct ib_recv_wr **bad_wr) 3594 { 3595 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); 3596 struct bnxt_qplib_sge sge[6]; 3597 struct bnxt_qplib_swqe wqe; 3598 unsigned long flags; 3599 u32 count = 0; 3600 int rc = 0; 3601 3602 spin_lock_irqsave(&qp->rq_lock, flags); 3603 while (wr) { 3604 memset(&wqe, 0, sizeof(wqe)); 3605 if (wr->num_sge > qp->qplib_qp.rq.max_sge) { 3606 dev_err(rdev_to_dev(qp->rdev), 3607 "Limit exceeded for Receive SGEs\n"); 3608 rc = -EINVAL; 3609 goto bad; 3610 } 3611 wqe.num_sge = wr->num_sge; 3612 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; 3613 wqe.wr_id = wr->wr_id; 3614 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; 3615 3616 if (ib_qp->qp_type == IB_QPT_GSI && 3617 qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { 3618 memset(sge, 0, sizeof(sge)); 3619 wqe.sg_list = sge; 3620 rc = bnxt_re_build_gsi_recv(qp, wr, &wqe); 3621 if (rc) 3622 goto bad; 3623 } 3624 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); 3625 bad: 3626 if (rc) { 3627 dev_err(rdev_to_dev(qp->rdev), 3628 "bad_wr seen with RQ post\n"); 3629 *bad_wr = wr; 3630 break; 3631 } 3632 /* Ring DB if the RQEs posted reaches a threshold value */ 3633 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { 3634 bnxt_qplib_post_recv_db(&qp->qplib_qp); 3635 count = 0; 3636 } 3637 wr = wr->next; 3638 } 3639 3640 if (count) 3641 bnxt_qplib_post_recv_db(&qp->qplib_qp); 3642 spin_unlock_irqrestore(&qp->rq_lock, flags); 3643 3644 return rc; 3645 } 3646 3647 /* Completion Queues */ 3648 void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 3649 { 3650 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 3651 struct bnxt_re_dev *rdev = cq->rdev; 3652 int rc = 0; 3653 3654 if (cq->uctx_cq_page) { 3655 BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq); 3656 free_page((u64)cq->uctx_cq_page); 3657 cq->uctx_cq_page = NULL; 3658 } 3659 if (cq->is_dbr_soft_cq && cq->uctx) { 3660 void *dbr_page; 3661 3662 if (cq->uctx->dbr_recov_cq) { 3663 dbr_page = cq->uctx->dbr_recov_cq_page; 3664 cq->uctx->dbr_recov_cq_page = NULL; 3665 cq->uctx->dbr_recov_cq = NULL; 3666 free_page((unsigned long)dbr_page); 3667 } 3668 goto end; 3669 } 3670 /* CQ getting destroyed. Set this state for cqn handler */ 3671 spin_lock_bh(&cq->qplib_cq.compl_lock); 3672 cq->qplib_cq.destroyed = true; 3673 spin_unlock_bh(&cq->qplib_cq.compl_lock); 3674 if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE || 3675 ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE) 3676 cancel_work_sync(&ib_cq->work); 3677 3678 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); 3679 if (rc) 3680 dev_err_ratelimited(rdev_to_dev(rdev), 3681 "%s id = %d failed rc = %d\n", 3682 __func__, cq->qplib_cq.id, rc); 3683 3684 bnxt_re_put_nq(rdev, cq->qplib_cq.nq); 3685 if (cq->umem && !IS_ERR(cq->umem)) 3686 ib_umem_release(cq->umem); 3687 3688 kfree(cq->cql); 3689 atomic_dec(&rdev->stats.rsors.cq_count); 3690 end: 3691 return; 3692 } 3693 3694 static inline struct 3695 bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, 3696 struct bnxt_re_dev *rdev) 3697 { 3698 struct bnxt_re_cq *cq; 3699 cq = container_of(cq_in, struct bnxt_re_cq, ibcq); 3700 return cq; 3701 } 3702 3703 int bnxt_re_create_cq(struct ib_cq *cq_in, 3704 const struct ib_cq_init_attr *attr, 3705 struct ib_udata *udata) 3706 { 3707 struct bnxt_qplib_dev_attr *dev_attr; 3708 struct bnxt_re_ucontext *uctx = NULL; 3709 struct ib_ucontext *context = NULL; 3710 struct bnxt_qplib_cq *qplcq; 3711 struct bnxt_re_cq_req ureq; 3712 struct bnxt_re_dev *rdev; 3713 int rc, entries; 3714 struct bnxt_re_cq *cq; 3715 u32 max_active_cqs; 3716 int cqe = attr->cqe; 3717 3718 if (attr->flags) 3719 return -EOPNOTSUPP; 3720 3721 rdev = rdev_from_cq_in(cq_in); 3722 if (rdev->mod_exit) { 3723 rc = -EIO; 3724 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 3725 goto exit; 3726 } 3727 if (udata) { 3728 uctx = rdma_udata_to_drv_context(udata, 3729 struct bnxt_re_ucontext, 3730 ibucontext); 3731 context = &uctx->ibucontext; 3732 } 3733 dev_attr = rdev->dev_attr; 3734 3735 if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) { 3736 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n"); 3737 rc = -EINVAL; 3738 goto exit; 3739 } 3740 /* Validate CQ fields */ 3741 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 3742 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n"); 3743 rc = -EINVAL; 3744 goto exit; 3745 } 3746 3747 cq = __get_cq_from_cq_in(cq_in, rdev); 3748 if (!cq) { 3749 rc = -ENOMEM; 3750 goto exit; 3751 } 3752 cq->rdev = rdev; 3753 cq->uctx = uctx; 3754 qplcq = &cq->qplib_cq; 3755 qplcq->cq_handle = (u64)qplcq; 3756 /* 3757 * Since CQ is for QP1 is shared with Shadow CQ, the size 3758 * should be double the size. There is no way to identify 3759 * whether this CQ is for GSI QP. So assuming that the first 3760 * CQ created is for QP1 3761 */ 3762 if (!udata && !rdev->gsi_ctx.first_cq_created && 3763 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) { 3764 rdev->gsi_ctx.first_cq_created = true; 3765 /* 3766 * Total CQE required for the CQ = CQE for QP1 RQ + 3767 * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs. 3768 * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe 3769 */ 3770 cqe *= 3; 3771 } 3772 3773 entries = bnxt_re_init_depth(cqe + 1, uctx); 3774 if (entries > dev_attr->max_cq_wqes + 1) 3775 entries = dev_attr->max_cq_wqes + 1; 3776 3777 qplcq->sginfo.pgshft = PAGE_SHIFT; 3778 qplcq->sginfo.pgsize = PAGE_SIZE; 3779 if (udata) { 3780 if (udata->inlen < sizeof(ureq)) 3781 dev_warn(rdev_to_dev(rdev), 3782 "Update the library ulen %d klen %d\n", 3783 (unsigned int)udata->inlen, 3784 (unsigned int)sizeof(ureq)); 3785 3786 rc = ib_copy_from_udata(&ureq, udata, 3787 min(udata->inlen, sizeof(ureq))); 3788 if (rc) 3789 goto fail; 3790 3791 if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) { 3792 cq->is_dbr_soft_cq = true; 3793 goto success; 3794 } 3795 3796 if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) { 3797 void *dbr_page; 3798 u32 *epoch; 3799 3800 dbr_page = (void *)__get_free_page(GFP_KERNEL); 3801 if (!dbr_page) { 3802 dev_err(rdev_to_dev(rdev), 3803 "DBR recov CQ page allocation failed!"); 3804 rc = -ENOMEM; 3805 goto fail; 3806 } 3807 3808 /* memset the epoch and epoch_ack to 0 */ 3809 epoch = dbr_page; 3810 epoch[0] = 0x0; 3811 epoch[1] = 0x0; 3812 3813 uctx->dbr_recov_cq = cq; 3814 uctx->dbr_recov_cq_page = dbr_page; 3815 3816 cq->is_dbr_soft_cq = true; 3817 goto success; 3818 } 3819 3820 cq->umem = ib_umem_get_compat 3821 (rdev, context, udata, ureq.cq_va, 3822 entries * sizeof(struct cq_base), 3823 IB_ACCESS_LOCAL_WRITE, 1); 3824 if (IS_ERR(cq->umem)) { 3825 rc = PTR_ERR(cq->umem); 3826 dev_err(rdev_to_dev(rdev), 3827 "%s: ib_umem_get failed! rc = %d\n", 3828 __func__, rc); 3829 goto fail; 3830 } 3831 qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem, 3832 &qplcq->sginfo.nmap); 3833 qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem); 3834 if (!uctx->dpi.dbr) { 3835 rc = bnxt_re_get_user_dpi(rdev, uctx); 3836 if (rc) 3837 goto c2fail; 3838 } 3839 qplcq->dpi = &uctx->dpi; 3840 } else { 3841 cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries; 3842 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), 3843 GFP_KERNEL); 3844 if (!cq->cql) { 3845 dev_err(rdev_to_dev(rdev), 3846 "Allocate CQL for %d failed!\n", cq->max_cql); 3847 rc = -ENOMEM; 3848 goto fail; 3849 } 3850 qplcq->dpi = &rdev->dpi_privileged; 3851 } 3852 /* 3853 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a 3854 * used for getting the NQ index. 3855 */ 3856 qplcq->max_wqe = entries; 3857 qplcq->nq = bnxt_re_get_nq(rdev); 3858 qplcq->cnq_hw_ring_id = qplcq->nq->ring_id; 3859 3860 rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq); 3861 if (rc) { 3862 dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n"); 3863 goto fail; 3864 } 3865 3866 INIT_LIST_HEAD(&cq->cq_list); 3867 cq->ibcq.cqe = entries; 3868 cq->cq_period = qplcq->period; 3869 3870 atomic_inc(&rdev->stats.rsors.cq_count); 3871 max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count); 3872 if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count)) 3873 atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs); 3874 spin_lock_init(&cq->cq_lock); 3875 3876 if (udata) { 3877 struct bnxt_re_cq_resp resp; 3878 3879 resp.cqid = qplcq->id; 3880 resp.tail = qplcq->hwq.cons; 3881 resp.phase = qplcq->period; 3882 resp.comp_mask = 0; 3883 resp.dbr = (u64)uctx->dpi.umdbr; 3884 resp.dpi = uctx->dpi.dpi; 3885 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO; 3886 /* Copy only on a valid wcpdi */ 3887 if (uctx->wcdpi.dpi) { 3888 resp.wcdpi = uctx->wcdpi.dpi; 3889 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI; 3890 } 3891 3892 if (_is_chip_p7(rdev->chip_ctx)) { 3893 cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL); 3894 3895 if (!cq->uctx_cq_page) { 3896 dev_err(rdev_to_dev(rdev), 3897 "CQ page allocation failed!\n"); 3898 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); 3899 rc = -ENOMEM; 3900 goto c2fail; 3901 } 3902 3903 resp.uctx_cq_page = (u64)cq->uctx_cq_page; 3904 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE; 3905 } 3906 3907 rc = bnxt_re_copy_to_udata(rdev, &resp, 3908 min(udata->outlen, sizeof(resp)), 3909 udata); 3910 if (rc) { 3911 free_page((u64)cq->uctx_cq_page); 3912 cq->uctx_cq_page = NULL; 3913 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); 3914 goto c2fail; 3915 } 3916 3917 if (cq->uctx_cq_page) 3918 BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq); 3919 } 3920 3921 success: 3922 return 0; 3923 c2fail: 3924 if (udata && cq->umem && !IS_ERR(cq->umem)) 3925 ib_umem_release(cq->umem); 3926 fail: 3927 if (cq) { 3928 if (cq->cql) 3929 kfree(cq->cql); 3930 } 3931 exit: 3932 return rc; 3933 } 3934 3935 int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period) 3936 { 3937 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 3938 struct bnxt_re_dev *rdev = cq->rdev; 3939 int rc; 3940 3941 if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) { 3942 cq->qplib_cq.count = cq_count; 3943 cq->qplib_cq.period = cq_period; 3944 rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq); 3945 if (rc) { 3946 dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n", 3947 cq->qplib_cq.id); 3948 return rc; 3949 } 3950 /* On success, update the shadow */ 3951 cq->cq_count = cq_count; 3952 cq->cq_period = cq_period; 3953 } 3954 return 0; 3955 } 3956 3957 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) 3958 { 3959 struct bnxt_re_dev *rdev = cq->rdev; 3960 3961 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); 3962 3963 cq->qplib_cq.max_wqe = cq->resize_cqe; 3964 if (cq->resize_umem) { 3965 ib_umem_release(cq->umem); 3966 cq->umem = cq->resize_umem; 3967 cq->resize_umem = NULL; 3968 cq->resize_cqe = 0; 3969 } 3970 } 3971 3972 int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata) 3973 { 3974 struct bnxt_qplib_sg_info sginfo = {}; 3975 struct bnxt_qplib_dpi *orig_dpi = NULL; 3976 struct bnxt_qplib_dev_attr *dev_attr; 3977 struct bnxt_re_ucontext *uctx = NULL; 3978 struct bnxt_re_resize_cq_req ureq; 3979 struct ib_ucontext *context = NULL; 3980 struct bnxt_re_dev *rdev; 3981 struct bnxt_re_cq *cq; 3982 int rc, entries; 3983 3984 /* Don't allow more than one resize request at the same time. 3985 * TODO: need a mutex here when we support kernel consumers of resize. 3986 */ 3987 cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 3988 rdev = cq->rdev; 3989 dev_attr = rdev->dev_attr; 3990 if (ib_cq->uobject) { 3991 uctx = rdma_udata_to_drv_context(udata, 3992 struct bnxt_re_ucontext, 3993 ibucontext); 3994 context = &uctx->ibucontext; 3995 } 3996 3997 if (cq->resize_umem) { 3998 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n", 3999 cq->qplib_cq.id); 4000 return -EBUSY; 4001 } 4002 4003 /* Check the requested cq depth out of supported depth */ 4004 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { 4005 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n", 4006 cq->qplib_cq.id); 4007 return -EINVAL; 4008 } 4009 4010 entries = bnxt_re_init_depth(cqe + 1, uctx); 4011 entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1); 4012 4013 /* Check to see if the new requested size can be handled by already 4014 * existing CQ 4015 */ 4016 if (entries == cq->ibcq.cqe) { 4017 dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe); 4018 return 0; 4019 } 4020 4021 if (ib_cq->uobject && udata) { 4022 if (udata->inlen < sizeof(ureq)) 4023 dev_warn(rdev_to_dev(rdev), 4024 "Update the library ulen %d klen %d\n", 4025 (unsigned int)udata->inlen, 4026 (unsigned int)sizeof(ureq)); 4027 4028 rc = ib_copy_from_udata(&ureq, udata, 4029 min(udata->inlen, sizeof(ureq))); 4030 if (rc) 4031 goto fail; 4032 4033 dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__, 4034 (void *)ureq.cq_va); 4035 cq->resize_umem = ib_umem_get_compat 4036 (rdev, 4037 context, udata, ureq.cq_va, 4038 entries * sizeof(struct cq_base), 4039 IB_ACCESS_LOCAL_WRITE, 1); 4040 if (IS_ERR(cq->resize_umem)) { 4041 rc = PTR_ERR(cq->resize_umem); 4042 cq->resize_umem = NULL; 4043 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", 4044 __func__, rc); 4045 goto fail; 4046 } 4047 cq->resize_cqe = entries; 4048 dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n", 4049 __func__); 4050 memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo)); 4051 orig_dpi = cq->qplib_cq.dpi; 4052 4053 cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem, 4054 &cq->qplib_cq.sginfo.nmap); 4055 cq->qplib_cq.sginfo.npages = 4056 ib_umem_num_pages_compat(cq->resize_umem); 4057 cq->qplib_cq.sginfo.pgsize = PAGE_SIZE; 4058 cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT; 4059 cq->qplib_cq.dpi = &uctx->dpi; 4060 } else { 4061 /* TODO: kernel consumer */ 4062 } 4063 4064 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); 4065 if (rc) { 4066 dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n", 4067 cq->qplib_cq.id); 4068 goto fail; 4069 } 4070 4071 cq->ibcq.cqe = cq->resize_cqe; 4072 /* For kernel consumers complete resize here. For uverbs consumers, 4073 * we complete it in the context of ibv_poll_cq(). 4074 */ 4075 if (!cq->resize_umem) 4076 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); 4077 4078 atomic_inc(&rdev->stats.rsors.resize_count); 4079 return 0; 4080 4081 fail: 4082 if (cq->resize_umem) { 4083 ib_umem_release(cq->resize_umem); 4084 cq->resize_umem = NULL; 4085 cq->resize_cqe = 0; 4086 memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo)); 4087 cq->qplib_cq.dpi = orig_dpi; 4088 } 4089 return rc; 4090 } 4091 4092 static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus) 4093 { 4094 switch(qstatus) { 4095 case CQ_REQ_STATUS_OK: 4096 return IB_WC_SUCCESS; 4097 case CQ_REQ_STATUS_BAD_RESPONSE_ERR: 4098 return IB_WC_BAD_RESP_ERR; 4099 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: 4100 return IB_WC_LOC_LEN_ERR; 4101 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: 4102 return IB_WC_LOC_QP_OP_ERR; 4103 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: 4104 return IB_WC_LOC_PROT_ERR; 4105 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: 4106 return IB_WC_GENERAL_ERR; 4107 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: 4108 return IB_WC_REM_INV_REQ_ERR; 4109 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: 4110 return IB_WC_REM_ACCESS_ERR; 4111 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: 4112 return IB_WC_REM_OP_ERR; 4113 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: 4114 return IB_WC_RNR_RETRY_EXC_ERR; 4115 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: 4116 return IB_WC_RETRY_EXC_ERR; 4117 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: 4118 return IB_WC_WR_FLUSH_ERR; 4119 default: 4120 return IB_WC_GENERAL_ERR; 4121 } 4122 return 0; 4123 } 4124 4125 static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus) 4126 { 4127 switch(qstatus) { 4128 case CQ_RES_RAWETH_QP1_STATUS_OK: 4129 return IB_WC_SUCCESS; 4130 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: 4131 return IB_WC_LOC_ACCESS_ERR; 4132 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: 4133 return IB_WC_LOC_LEN_ERR; 4134 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: 4135 return IB_WC_LOC_PROT_ERR; 4136 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: 4137 return IB_WC_LOC_QP_OP_ERR; 4138 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: 4139 return IB_WC_GENERAL_ERR; 4140 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: 4141 return IB_WC_WR_FLUSH_ERR; 4142 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: 4143 return IB_WC_WR_FLUSH_ERR; 4144 default: 4145 return IB_WC_GENERAL_ERR; 4146 } 4147 } 4148 4149 static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus) 4150 { 4151 switch(qstatus) { 4152 case CQ_RES_RC_STATUS_OK: 4153 return IB_WC_SUCCESS; 4154 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: 4155 return IB_WC_LOC_ACCESS_ERR; 4156 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: 4157 return IB_WC_LOC_LEN_ERR; 4158 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: 4159 return IB_WC_LOC_PROT_ERR; 4160 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: 4161 return IB_WC_LOC_QP_OP_ERR; 4162 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: 4163 return IB_WC_GENERAL_ERR; 4164 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: 4165 return IB_WC_REM_INV_REQ_ERR; 4166 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: 4167 return IB_WC_WR_FLUSH_ERR; 4168 case CQ_RES_RC_STATUS_HW_FLUSH_ERR: 4169 return IB_WC_WR_FLUSH_ERR; 4170 default: 4171 return IB_WC_GENERAL_ERR; 4172 } 4173 } 4174 4175 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) 4176 { 4177 switch (cqe->type) { 4178 case BNXT_QPLIB_SWQE_TYPE_SEND: 4179 wc->opcode = IB_WC_SEND; 4180 break; 4181 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: 4182 wc->opcode = IB_WC_SEND; 4183 wc->wc_flags |= IB_WC_WITH_IMM; 4184 break; 4185 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: 4186 wc->opcode = IB_WC_SEND; 4187 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4188 break; 4189 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: 4190 wc->opcode = IB_WC_RDMA_WRITE; 4191 break; 4192 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: 4193 wc->opcode = IB_WC_RDMA_WRITE; 4194 wc->wc_flags |= IB_WC_WITH_IMM; 4195 break; 4196 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: 4197 wc->opcode = IB_WC_RDMA_READ; 4198 break; 4199 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: 4200 wc->opcode = IB_WC_COMP_SWAP; 4201 break; 4202 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: 4203 wc->opcode = IB_WC_FETCH_ADD; 4204 break; 4205 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: 4206 wc->opcode = IB_WC_LOCAL_INV; 4207 break; 4208 case BNXT_QPLIB_SWQE_TYPE_REG_MR: 4209 wc->opcode = IB_WC_REG_MR; 4210 break; 4211 default: 4212 wc->opcode = IB_WC_SEND; 4213 break; 4214 } 4215 4216 wc->status = __req_to_ib_wc_status(cqe->status); 4217 } 4218 4219 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2) 4220 { 4221 bool is_ipv6 = false, is_ipv4 = false; 4222 4223 /* raweth_qp1_flags Bit 9-6 indicates itype */ 4224 4225 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 4226 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) 4227 return -1; 4228 4229 if (raweth_qp1_flags2 & 4230 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && 4231 raweth_qp1_flags2 & 4232 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { 4233 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ 4234 (raweth_qp1_flags2 & 4235 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? 4236 (is_ipv6 = true) : (is_ipv4 = true); 4237 return ((is_ipv6) ? 4238 BNXT_RE_ROCEV2_IPV6_PACKET : 4239 BNXT_RE_ROCEV2_IPV4_PACKET); 4240 } else { 4241 return BNXT_RE_ROCE_V1_PACKET; 4242 } 4243 } 4244 4245 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, 4246 void *rq_hdr_buf) 4247 { 4248 u8 *tmp_buf = NULL; 4249 struct ethhdr *eth_hdr; 4250 u16 eth_type; 4251 bool rc = false; 4252 4253 tmp_buf = (u8 *)rq_hdr_buf; 4254 /* 4255 * If dest mac is not same as I/F mac, this could be a 4256 * loopback address or multicast address, check whether 4257 * it is a loopback packet 4258 */ 4259 if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) { 4260 tmp_buf += 4; 4261 /* Check the ether type */ 4262 eth_hdr = (struct ethhdr *)tmp_buf; 4263 eth_type = ntohs(eth_hdr->h_proto); 4264 switch (eth_type) { 4265 case BNXT_QPLIB_ETHTYPE_ROCEV1: 4266 rc = true; 4267 break; 4268 default: 4269 break; 4270 } 4271 } 4272 4273 return rc; 4274 } 4275 4276 static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev, 4277 void *rq_hdr_buf, 4278 struct bnxt_qplib_cqe *cqe) 4279 { 4280 struct vlan_hdr *vlan_hdr; 4281 struct ethhdr *eth_hdr; 4282 u8 *tmp_buf = NULL; 4283 u16 eth_type; 4284 4285 tmp_buf = (u8 *)rq_hdr_buf; 4286 /* Check the ether type */ 4287 eth_hdr = (struct ethhdr *)tmp_buf; 4288 eth_type = ntohs(eth_hdr->h_proto); 4289 if (eth_type == ETH_P_8021Q) { 4290 tmp_buf += sizeof(struct ethhdr); 4291 vlan_hdr = (struct vlan_hdr *)tmp_buf; 4292 cqe->raweth_qp1_metadata = 4293 ntohs(vlan_hdr->h_vlan_TCI) | 4294 (eth_type << 4295 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 4296 cqe->raweth_qp1_flags2 |= 4297 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN; 4298 return true; 4299 } 4300 4301 return false; 4302 } 4303 4304 static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp, 4305 struct bnxt_qplib_cqe *cqe) 4306 { 4307 struct bnxt_re_sqp_entries *sqp_entry = NULL; 4308 struct bnxt_qplib_hdrbuf *hdr_buf; 4309 dma_addr_t shrq_hdr_buf_map; 4310 struct ib_sge s_sge[2] = {}; 4311 struct ib_sge r_sge[2] = {}; 4312 struct ib_recv_wr rwr = {}; 4313 struct bnxt_re_ah *gsi_sah; 4314 struct bnxt_re_qp *gsi_sqp; 4315 dma_addr_t rq_hdr_buf_map; 4316 struct bnxt_re_dev *rdev; 4317 struct ib_send_wr *swr; 4318 u32 skip_bytes = 0; 4319 void *rq_hdr_buf; 4320 int pkt_type = 0; 4321 u32 offset = 0; 4322 u32 tbl_idx; 4323 int rc; 4324 struct ib_ud_wr udwr = {}; 4325 4326 swr = &udwr.wr; 4327 rdev = gsi_qp->rdev; 4328 gsi_sqp = rdev->gsi_ctx.gsi_sqp; 4329 tbl_idx = cqe->wr_id; 4330 4331 hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf; 4332 rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step; 4333 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, 4334 tbl_idx); 4335 /* Shadow QP header buffer */ 4336 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp, 4337 tbl_idx); 4338 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 4339 4340 /* Find packet type from the cqe */ 4341 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, 4342 cqe->raweth_qp1_flags2); 4343 if (pkt_type < 0) { 4344 dev_err(rdev_to_dev(rdev), "Not handling this packet\n"); 4345 return -EINVAL; 4346 } 4347 4348 /* Adjust the offset for the user buffer and post in the rq */ 4349 4350 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) 4351 offset = 20; 4352 4353 /* 4354 * QP1 loopback packet has 4 bytes of internal header before 4355 * ether header. Skip these four bytes. 4356 */ 4357 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) 4358 skip_bytes = 4; 4359 4360 if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe)) 4361 skip_bytes += VLAN_HLEN; 4362 4363 /* Store this cqe */ 4364 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); 4365 sqp_entry->qp1_qp = gsi_qp; 4366 4367 /* First send SGE . Skip the ether header*/ 4368 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 4369 + skip_bytes; 4370 s_sge[0].lkey = 0xFFFFFFFF; 4371 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : 4372 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; 4373 4374 /* Second Send SGE */ 4375 s_sge[1].addr = s_sge[0].addr + s_sge[0].length + 4376 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; 4377 if (pkt_type != BNXT_RE_ROCE_V1_PACKET) 4378 s_sge[1].addr += 8; 4379 s_sge[1].lkey = 0xFFFFFFFF; 4380 s_sge[1].length = 256; 4381 4382 /* First recv SGE */ 4383 r_sge[0].addr = shrq_hdr_buf_map; 4384 r_sge[0].lkey = 0xFFFFFFFF; 4385 r_sge[0].length = 40; 4386 4387 r_sge[1].addr = sqp_entry->sge.addr + offset; 4388 r_sge[1].lkey = sqp_entry->sge.lkey; 4389 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; 4390 4391 /* Create receive work request */ 4392 rwr.num_sge = 2; 4393 rwr.sg_list = r_sge; 4394 rwr.wr_id = tbl_idx; 4395 rwr.next = NULL; 4396 4397 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); 4398 if (rc) { 4399 dev_err(rdev_to_dev(rdev), 4400 "Failed to post Rx buffers to shadow QP\n"); 4401 return -ENOMEM; 4402 } 4403 4404 swr->num_sge = 2; 4405 swr->sg_list = s_sge; 4406 swr->wr_id = tbl_idx; 4407 swr->opcode = IB_WR_SEND; 4408 swr->next = NULL; 4409 4410 gsi_sah = rdev->gsi_ctx.gsi_sah; 4411 udwr.ah = &gsi_sah->ibah; 4412 udwr.remote_qpn = gsi_sqp->qplib_qp.id; 4413 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; 4414 /* post data received in the send queue */ 4415 rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); 4416 4417 return rc; 4418 } 4419 4420 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, 4421 struct bnxt_qplib_cqe *cqe) 4422 { 4423 wc->opcode = IB_WC_RECV; 4424 wc->status = __rawqp1_to_ib_wc_status(cqe->status); 4425 wc->wc_flags |= IB_WC_GRH; 4426 } 4427 4428 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, 4429 struct bnxt_qplib_cqe *cqe) 4430 { 4431 wc->opcode = IB_WC_RECV; 4432 wc->status = __rc_to_ib_wc_status(cqe->status); 4433 4434 if (cqe->flags & CQ_RES_RC_FLAGS_IMM) 4435 wc->wc_flags |= IB_WC_WITH_IMM; 4436 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 4437 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4438 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == 4439 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) 4440 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4441 } 4442 4443 /* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */ 4444 static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe, 4445 u16 *vid, u8 *sl) 4446 { 4447 u32 metadata; 4448 u16 tpid; 4449 bool ret = false; 4450 metadata = orig_cqe->raweth_qp1_metadata; 4451 if (orig_cqe->raweth_qp1_flags2 & 4452 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { 4453 tpid = ((metadata & 4454 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> 4455 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); 4456 if (tpid == ETH_P_8021Q) { 4457 *vid = metadata & 4458 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; 4459 *sl = (metadata & 4460 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> 4461 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; 4462 ret = !!(*vid); 4463 } 4464 } 4465 4466 return ret; 4467 } 4468 4469 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, 4470 struct ib_wc *wc, 4471 struct bnxt_qplib_cqe *cqe) 4472 { 4473 u32 tbl_idx; 4474 struct bnxt_re_dev *rdev = gsi_sqp->rdev; 4475 struct bnxt_re_qp *gsi_qp = NULL; 4476 struct bnxt_qplib_cqe *orig_cqe = NULL; 4477 struct bnxt_re_sqp_entries *sqp_entry = NULL; 4478 int nw_type; 4479 u16 vlan_id; 4480 u8 sl; 4481 4482 tbl_idx = cqe->wr_id; 4483 4484 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 4485 gsi_qp = sqp_entry->qp1_qp; 4486 orig_cqe = &sqp_entry->cqe; 4487 4488 wc->wr_id = sqp_entry->wrid; 4489 wc->byte_len = orig_cqe->length; 4490 wc->qp = &gsi_qp->ib_qp; 4491 4492 wc->ex.imm_data = orig_cqe->immdata; 4493 wc->src_qp = orig_cqe->src_qp; 4494 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); 4495 if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) { 4496 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 4497 wc->sl = sl; 4498 wc->vlan_id = vlan_id; 4499 wc->wc_flags |= IB_WC_WITH_VLAN; 4500 } 4501 } 4502 wc->port_num = 1; 4503 wc->vendor_err = orig_cqe->status; 4504 4505 wc->opcode = IB_WC_RECV; 4506 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); 4507 wc->wc_flags |= IB_WC_GRH; 4508 4509 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, 4510 orig_cqe->raweth_qp1_flags2); 4511 if(nw_type >= 0) 4512 dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type); 4513 } 4514 4515 static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev, 4516 struct bnxt_re_qp *qp, struct ib_wc *wc, 4517 struct bnxt_qplib_cqe *cqe) 4518 { 4519 u16 vlan_id = 0; 4520 4521 wc->opcode = IB_WC_RECV; 4522 wc->status = __rc_to_ib_wc_status(cqe->status); 4523 if (cqe->flags & CQ_RES_UD_FLAGS_IMM) 4524 wc->wc_flags |= IB_WC_WITH_IMM; 4525 if (cqe->flags & CQ_RES_RC_FLAGS_INV) 4526 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4527 /* report only on GSI QP for Thor */ 4528 if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id && 4529 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) { 4530 wc->wc_flags |= IB_WC_GRH; 4531 memcpy(wc->smac, cqe->smac, ETH_ALEN); 4532 wc->wc_flags |= IB_WC_WITH_SMAC; 4533 if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) { 4534 if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) { 4535 if (cqe->cfa_meta & 4536 BNXT_QPLIB_CQE_CFA_META1_VALID) 4537 vlan_id = (cqe->cfa_meta & 0xFFF); 4538 } 4539 } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { 4540 vlan_id = (cqe->cfa_meta & 0xFFF); 4541 } 4542 /* Mark only if vlan_id is non zero */ 4543 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { 4544 wc->vlan_id = vlan_id; 4545 wc->wc_flags |= IB_WC_WITH_VLAN; 4546 } 4547 } 4548 } 4549 4550 static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp) 4551 { 4552 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; 4553 unsigned long flags; 4554 int rc = 0; 4555 4556 spin_lock_irqsave(&qp->sq_lock, flags); 4557 4558 rc = bnxt_re_legacy_bind_fence_mw(lib_qp); 4559 if (!rc) { 4560 lib_qp->sq.phantom_wqe_cnt++; 4561 dev_dbg(&lib_qp->sq.hwq.pdev->dev, 4562 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", 4563 lib_qp->id, lib_qp->sq.hwq.prod, 4564 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), 4565 lib_qp->sq.phantom_wqe_cnt); 4566 } 4567 4568 spin_unlock_irqrestore(&qp->sq_lock, flags); 4569 return rc; 4570 } 4571 4572 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) 4573 { 4574 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 4575 struct bnxt_re_dev *rdev = cq->rdev; 4576 struct bnxt_re_qp *qp; 4577 struct bnxt_qplib_cqe *cqe; 4578 int i, ncqe, budget, init_budget; 4579 struct bnxt_qplib_q *sq; 4580 struct bnxt_qplib_qp *lib_qp; 4581 u32 tbl_idx; 4582 struct bnxt_re_sqp_entries *sqp_entry = NULL; 4583 unsigned long flags; 4584 u8 gsi_mode; 4585 4586 /* 4587 * DB recovery CQ; only process the door bell pacing alert from 4588 * the user lib 4589 */ 4590 if (cq->is_dbr_soft_cq) { 4591 bnxt_re_pacing_alert(rdev); 4592 return 0; 4593 } 4594 4595 /* User CQ; the only processing we do is to 4596 * complete any pending CQ resize operation. 4597 */ 4598 if (cq->umem) { 4599 if (cq->resize_umem) 4600 bnxt_re_resize_cq_complete(cq); 4601 return 0; 4602 } 4603 4604 spin_lock_irqsave(&cq->cq_lock, flags); 4605 4606 budget = min_t(u32, num_entries, cq->max_cql); 4607 init_budget = budget; 4608 if (!cq->cql) { 4609 dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n"); 4610 goto exit; 4611 } 4612 cqe = &cq->cql[0]; 4613 gsi_mode = rdev->gsi_ctx.gsi_qp_mode; 4614 while (budget) { 4615 lib_qp = NULL; 4616 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); 4617 if (lib_qp) { 4618 sq = &lib_qp->sq; 4619 if (sq->legacy_send_phantom == true) { 4620 qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp); 4621 if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM) 4622 dev_err(rdev_to_dev(rdev), 4623 "Phantom failed! Scheduled to send again\n"); 4624 else 4625 sq->legacy_send_phantom = false; 4626 } 4627 } 4628 if (ncqe < budget) 4629 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, 4630 cqe + ncqe, 4631 budget - ncqe); 4632 4633 if (!ncqe) 4634 break; 4635 4636 for (i = 0; i < ncqe; i++, cqe++) { 4637 /* Transcribe each qplib_wqe back to ib_wc */ 4638 memset(wc, 0, sizeof(*wc)); 4639 4640 wc->wr_id = cqe->wr_id; 4641 wc->byte_len = cqe->length; 4642 qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle, 4643 struct bnxt_re_qp, qplib_qp); 4644 if (!qp) { 4645 dev_err(rdev_to_dev(rdev), 4646 "POLL CQ bad QP handle\n"); 4647 continue; 4648 } 4649 wc->qp = &qp->ib_qp; 4650 wc->ex.imm_data = cqe->immdata; 4651 wc->src_qp = cqe->src_qp; 4652 memcpy(wc->smac, cqe->smac, ETH_ALEN); 4653 wc->port_num = 1; 4654 wc->vendor_err = cqe->status; 4655 4656 switch(cqe->opcode) { 4657 case CQ_BASE_CQE_TYPE_REQ: 4658 if (gsi_mode == BNXT_RE_GSI_MODE_ALL && 4659 qp->qplib_qp.id == 4660 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { 4661 /* Handle this completion with 4662 * the stored completion */ 4663 dev_dbg(rdev_to_dev(rdev), 4664 "Skipping this UD Send CQ\n"); 4665 memset(wc, 0, sizeof(*wc)); 4666 continue; 4667 } 4668 bnxt_re_process_req_wc(wc, cqe); 4669 break; 4670 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: 4671 if (gsi_mode == BNXT_RE_GSI_MODE_ALL) { 4672 if (!cqe->status) { 4673 int rc = 0; 4674 rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe); 4675 if (!rc) { 4676 memset(wc, 0, 4677 sizeof(*wc)); 4678 continue; 4679 } 4680 cqe->status = -1; 4681 } 4682 /* Errors need not be looped back. 4683 * But change the wr_id to the one 4684 * stored in the table 4685 */ 4686 tbl_idx = cqe->wr_id; 4687 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; 4688 wc->wr_id = sqp_entry->wrid; 4689 } 4690 4691 bnxt_re_process_res_rawqp1_wc(wc, cqe); 4692 break; 4693 case CQ_BASE_CQE_TYPE_RES_RC: 4694 bnxt_re_process_res_rc_wc(wc, cqe); 4695 break; 4696 case CQ_BASE_CQE_TYPE_RES_UD: 4697 if (gsi_mode == BNXT_RE_GSI_MODE_ALL && 4698 qp->qplib_qp.id == 4699 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { 4700 /* Handle this completion with 4701 * the stored completion 4702 */ 4703 dev_dbg(rdev_to_dev(rdev), 4704 "Handling the UD receive CQ\n"); 4705 if (cqe->status) { 4706 /* TODO handle this completion as a failure in 4707 * loopback porocedure 4708 */ 4709 continue; 4710 } else { 4711 bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe); 4712 break; 4713 } 4714 } 4715 bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe); 4716 break; 4717 default: 4718 dev_err(rdev_to_dev(cq->rdev), 4719 "POLL CQ type 0x%x not handled, skip!\n", 4720 cqe->opcode); 4721 continue; 4722 } 4723 wc++; 4724 budget--; 4725 } 4726 } 4727 exit: 4728 spin_unlock_irqrestore(&cq->cq_lock, flags); 4729 return init_budget - budget; 4730 } 4731 4732 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, 4733 enum ib_cq_notify_flags ib_cqn_flags) 4734 { 4735 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); 4736 int type = 0, rc = 0; 4737 unsigned long flags; 4738 4739 spin_lock_irqsave(&cq->cq_lock, flags); 4740 /* Trigger on the very next completion */ 4741 if (ib_cqn_flags & IB_CQ_NEXT_COMP) 4742 type = DBC_DBC_TYPE_CQ_ARMALL; 4743 /* Trigger on the next solicited completion */ 4744 else if (ib_cqn_flags & IB_CQ_SOLICITED) 4745 type = DBC_DBC_TYPE_CQ_ARMSE; 4746 4747 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); 4748 4749 /* Poll to see if there are missed events */ 4750 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && 4751 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) 4752 rc = 1; 4753 4754 spin_unlock_irqrestore(&cq->cq_lock, flags); 4755 4756 return rc; 4757 } 4758 4759 /* Memory Regions */ 4760 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 4761 { 4762 struct bnxt_qplib_mrinfo mrinfo; 4763 struct bnxt_re_dev *rdev; 4764 struct bnxt_re_mr *mr; 4765 struct bnxt_re_pd *pd; 4766 u32 max_mr_count; 4767 u64 pbl = 0; 4768 int rc; 4769 4770 memset(&mrinfo, 0, sizeof(mrinfo)); 4771 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 4772 rdev = pd->rdev; 4773 4774 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 4775 if (!mr) { 4776 dev_err(rdev_to_dev(rdev), 4777 "Allocate memory for DMA MR failed!\n"); 4778 return ERR_PTR(-ENOMEM); 4779 } 4780 mr->rdev = rdev; 4781 mr->qplib_mr.pd = &pd->qplib_pd; 4782 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 4783 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 4784 4785 /* Allocate and register 0 as the address */ 4786 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 4787 if (rc) { 4788 dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n"); 4789 goto fail; 4790 } 4791 mr->qplib_mr.total_size = -1; /* Infinite length */ 4792 mrinfo.ptes = &pbl; 4793 mrinfo.sg.npages = 0; 4794 mrinfo.sg.pgsize = PAGE_SIZE; 4795 mrinfo.sg.pgshft = PAGE_SHIFT; 4796 mrinfo.sg.pgsize = PAGE_SIZE; 4797 mrinfo.mrw = &mr->qplib_mr; 4798 mrinfo.is_dma = true; 4799 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 4800 if (rc) { 4801 dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n"); 4802 goto fail_mr; 4803 } 4804 mr->ib_mr.lkey = mr->qplib_mr.lkey; 4805 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | 4806 IB_ACCESS_REMOTE_ATOMIC)) 4807 mr->ib_mr.rkey = mr->ib_mr.lkey; 4808 atomic_inc(&rdev->stats.rsors.mr_count); 4809 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 4810 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) 4811 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 4812 4813 return &mr->ib_mr; 4814 4815 fail_mr: 4816 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4817 fail: 4818 kfree(mr); 4819 return ERR_PTR(rc); 4820 } 4821 4822 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 4823 { 4824 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 4825 struct bnxt_re_dev *rdev = mr->rdev; 4826 int rc = 0; 4827 4828 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4829 if (rc) 4830 dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n", 4831 mr->qplib_mr.lkey, rc); 4832 4833 if (mr->pages) { 4834 bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, 4835 &mr->qplib_frpl); 4836 kfree(mr->pages); 4837 mr->npages = 0; 4838 mr->pages = NULL; 4839 } 4840 if (!IS_ERR(mr->ib_umem) && mr->ib_umem) { 4841 mr->is_invalcb_active = false; 4842 bnxt_re_peer_mem_release(mr->ib_umem); 4843 } 4844 kfree(mr); 4845 atomic_dec(&rdev->stats.rsors.mr_count); 4846 return 0; 4847 } 4848 4849 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) 4850 { 4851 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 4852 4853 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) 4854 return -ENOMEM; 4855 4856 mr->pages[mr->npages++] = addr; 4857 dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n", 4858 ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1, 4859 mr->pages[mr->npages - 1]); 4860 return 0; 4861 } 4862 4863 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, 4864 int sg_nents, unsigned int *sg_offset) 4865 { 4866 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 4867 4868 mr->npages = 0; 4869 return ib_sg_to_pages(ib_mr, sg, sg_nents, 4870 sg_offset, bnxt_re_set_page); 4871 } 4872 4873 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 4874 u32 max_num_sg, struct ib_udata *udata) 4875 { 4876 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 4877 struct bnxt_re_dev *rdev = pd->rdev; 4878 struct bnxt_re_mr *mr; 4879 u32 max_mr_count; 4880 int rc; 4881 4882 dev_dbg(rdev_to_dev(rdev), "Alloc MR\n"); 4883 if (type != IB_MR_TYPE_MEM_REG) { 4884 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type); 4885 return ERR_PTR(-EINVAL); 4886 } 4887 if (max_num_sg > MAX_PBL_LVL_1_PGS) { 4888 dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n"); 4889 return ERR_PTR(-EINVAL); 4890 } 4891 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 4892 if (!mr) { 4893 dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n"); 4894 return ERR_PTR(-ENOMEM); 4895 } 4896 mr->rdev = rdev; 4897 mr->qplib_mr.pd = &pd->qplib_pd; 4898 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; 4899 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; 4900 4901 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 4902 if (rc) { 4903 dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); 4904 goto fail; 4905 } 4906 mr->ib_mr.lkey = mr->qplib_mr.lkey; 4907 mr->ib_mr.rkey = mr->ib_mr.lkey; 4908 mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL); 4909 if (!mr->pages) { 4910 dev_err(rdev_to_dev(rdev), 4911 "Allocate MR page list mem failed!\n"); 4912 rc = -ENOMEM; 4913 goto fail_mr; 4914 } 4915 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, 4916 &mr->qplib_frpl, max_num_sg); 4917 if (rc) { 4918 dev_err(rdev_to_dev(rdev), 4919 "Allocate HW Fast reg page list failed!\n"); 4920 goto free_page; 4921 } 4922 dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages); 4923 4924 atomic_inc(&rdev->stats.rsors.mr_count); 4925 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 4926 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) 4927 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 4928 return &mr->ib_mr; 4929 4930 free_page: 4931 kfree(mr->pages); 4932 fail_mr: 4933 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 4934 fail: 4935 kfree(mr); 4936 return ERR_PTR(rc); 4937 } 4938 4939 /* Memory Windows */ 4940 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 4941 struct ib_udata *udata) 4942 { 4943 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 4944 struct bnxt_re_dev *rdev = pd->rdev; 4945 struct bnxt_re_mw *mw; 4946 u32 max_mw_count; 4947 int rc; 4948 4949 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 4950 if (!mw) { 4951 dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); 4952 rc = -ENOMEM; 4953 goto exit; 4954 } 4955 mw->rdev = rdev; 4956 mw->qplib_mw.pd = &pd->qplib_pd; 4957 4958 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? 4959 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : 4960 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); 4961 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); 4962 if (rc) { 4963 dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); 4964 goto fail; 4965 } 4966 mw->ib_mw.rkey = mw->qplib_mw.rkey; 4967 atomic_inc(&rdev->stats.rsors.mw_count); 4968 max_mw_count = atomic_read(&rdev->stats.rsors.mw_count); 4969 if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count)) 4970 atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count); 4971 4972 return &mw->ib_mw; 4973 fail: 4974 kfree(mw); 4975 exit: 4976 return ERR_PTR(rc); 4977 } 4978 4979 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) 4980 { 4981 struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw); 4982 struct bnxt_re_dev *rdev = mw->rdev; 4983 int rc; 4984 4985 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); 4986 if (rc) { 4987 dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); 4988 return rc; 4989 } 4990 4991 kfree(mw); 4992 atomic_dec(&rdev->stats.rsors.mw_count); 4993 return rc; 4994 } 4995 4996 static int bnxt_re_page_size_ok(int page_shift) 4997 { 4998 switch (page_shift) { 4999 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: 5000 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: 5001 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: 5002 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: 5003 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: 5004 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: 5005 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: 5006 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB: 5007 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: 5008 return 1; 5009 default: 5010 return 0; 5011 } 5012 } 5013 5014 static int bnxt_re_get_page_shift(struct ib_umem *umem, 5015 u64 va, u64 st, u64 cmask) 5016 { 5017 return umem->page_shift; 5018 } 5019 5020 static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift) 5021 { 5022 int npages = 0; 5023 5024 if (page_shift == PAGE_SHIFT) { 5025 npages = ib_umem_num_pages_compat(umem); 5026 } else { 5027 npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift); 5028 if (start % BIT(page_shift)) 5029 npages++; 5030 } 5031 return npages; 5032 } 5033 5034 /* uverbs */ 5035 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 5036 u64 virt_addr, int mr_access_flags, 5037 struct ib_udata *udata) 5038 { 5039 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 5040 struct bnxt_re_dev *rdev = pd->rdev; 5041 struct bnxt_qplib_mrinfo mrinfo; 5042 int umem_pgs, page_shift, rc; 5043 struct bnxt_re_mr *mr; 5044 struct ib_umem *umem; 5045 u32 max_mr_count; 5046 int npages; 5047 5048 dev_dbg(rdev_to_dev(rdev), "Reg user MR\n"); 5049 5050 if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr) 5051 return ERR_PTR(-ENOMEM); 5052 5053 if (rdev->mod_exit) { 5054 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); 5055 return ERR_PTR(-EIO); 5056 } 5057 memset(&mrinfo, 0, sizeof(mrinfo)); 5058 if (length > BNXT_RE_MAX_MR_SIZE) { 5059 dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu " 5060 "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE); 5061 return ERR_PTR(-ENOMEM); 5062 } 5063 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 5064 if (!mr) { 5065 dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); 5066 return ERR_PTR (-ENOMEM); 5067 } 5068 mr->rdev = rdev; 5069 mr->qplib_mr.pd = &pd->qplib_pd; 5070 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 5071 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; 5072 5073 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { 5074 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); 5075 if (rc) { 5076 dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n"); 5077 goto fail; 5078 } 5079 /* The fixed portion of the rkey is the same as the lkey */ 5080 mr->ib_mr.rkey = mr->qplib_mr.rkey; 5081 } 5082 5083 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, 5084 udata, start, length, 5085 mr_access_flags, 0); 5086 if (IS_ERR(umem)) { 5087 rc = PTR_ERR(umem); 5088 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", 5089 __func__, rc); 5090 goto free_mr; 5091 } 5092 mr->ib_umem = umem; 5093 5094 mr->qplib_mr.va = virt_addr; 5095 umem_pgs = ib_umem_num_pages_compat(umem); 5096 if (!umem_pgs) { 5097 dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); 5098 rc = -EINVAL; 5099 goto free_umem; 5100 } 5101 mr->qplib_mr.total_size = length; 5102 page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, 5103 rdev->dev_attr->page_size_cap); 5104 if (!bnxt_re_page_size_ok(page_shift)) { 5105 dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n"); 5106 rc = -EFAULT; 5107 goto free_umem; 5108 } 5109 npages = bnxt_re_get_num_pages(umem, start, length, page_shift); 5110 5111 /* Map umem buf ptrs to the PBL */ 5112 mrinfo.sg.npages = npages; 5113 mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); 5114 mrinfo.sg.pgshft = page_shift; 5115 mrinfo.sg.pgsize = BIT(page_shift); 5116 5117 mrinfo.mrw = &mr->qplib_mr; 5118 5119 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 5120 if (rc) { 5121 dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n"); 5122 goto free_umem; 5123 } 5124 5125 mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey; 5126 atomic_inc(&rdev->stats.rsors.mr_count); 5127 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); 5128 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) 5129 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); 5130 5131 return &mr->ib_mr; 5132 5133 free_umem: 5134 bnxt_re_peer_mem_release(mr->ib_umem); 5135 free_mr: 5136 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) 5137 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); 5138 fail: 5139 kfree(mr); 5140 return ERR_PTR(rc); 5141 } 5142 5143 int 5144 bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, 5145 u64 virt_addr, int mr_access_flags, 5146 struct ib_pd *ib_pd, struct ib_udata *udata) 5147 { 5148 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); 5149 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); 5150 int umem_pgs = 0, page_shift = PAGE_SHIFT, rc; 5151 struct bnxt_re_dev *rdev = mr->rdev; 5152 struct bnxt_qplib_mrinfo mrinfo; 5153 struct ib_umem *umem; 5154 u32 npages; 5155 5156 /* TODO: Must decipher what to modify based on the flags */ 5157 memset(&mrinfo, 0, sizeof(mrinfo)); 5158 if (flags & IB_MR_REREG_TRANS) { 5159 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, 5160 udata, start, length, 5161 mr_access_flags, 0); 5162 if (IS_ERR(umem)) { 5163 rc = PTR_ERR(umem); 5164 dev_err(rdev_to_dev(rdev), 5165 "%s: ib_umem_get failed! ret = %d\n", 5166 __func__, rc); 5167 goto fail; 5168 } 5169 mr->ib_umem = umem; 5170 5171 mr->qplib_mr.va = virt_addr; 5172 umem_pgs = ib_umem_num_pages_compat(umem); 5173 if (!umem_pgs) { 5174 dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); 5175 rc = -EINVAL; 5176 goto fail_free_umem; 5177 } 5178 mr->qplib_mr.total_size = length; 5179 page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, 5180 rdev->dev_attr->page_size_cap); 5181 if (!bnxt_re_page_size_ok(page_shift)) { 5182 dev_err(rdev_to_dev(rdev), 5183 "umem page size unsupported!\n"); 5184 rc = -EFAULT; 5185 goto fail_free_umem; 5186 } 5187 npages = bnxt_re_get_num_pages(umem, start, length, page_shift); 5188 /* Map umem buf ptrs to the PBL */ 5189 mrinfo.sg.npages = npages; 5190 mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); 5191 mrinfo.sg.pgshft = page_shift; 5192 mrinfo.sg.pgsize = BIT(page_shift); 5193 } 5194 5195 mrinfo.mrw = &mr->qplib_mr; 5196 if (flags & IB_MR_REREG_PD) 5197 mr->qplib_mr.pd = &pd->qplib_pd; 5198 5199 if (flags & IB_MR_REREG_ACCESS) 5200 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); 5201 5202 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); 5203 if (rc) { 5204 dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n"); 5205 goto fail_free_umem; 5206 } 5207 mr->ib_mr.rkey = mr->qplib_mr.rkey; 5208 5209 return 0; 5210 5211 fail_free_umem: 5212 bnxt_re_peer_mem_release(mr->ib_umem); 5213 fail: 5214 return rc; 5215 } 5216 5217 static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev) 5218 { 5219 struct ib_device *ibdev = &rdev->ibdev; 5220 u32 uverbs_abi_ver; 5221 5222 uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev); 5223 dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n", 5224 uverbs_abi_ver); 5225 if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) { 5226 dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n", 5227 BNXT_RE_ABI_VERSION); 5228 return -EPERM; 5229 } 5230 return 0; 5231 } 5232 5233 int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, 5234 struct ib_udata *udata) 5235 { 5236 struct ib_ucontext *ctx = uctx_in; 5237 struct ib_device *ibdev = ctx->device; 5238 struct bnxt_re_ucontext *uctx = 5239 container_of(ctx, struct bnxt_re_ucontext, ibucontext); 5240 5241 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); 5242 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; 5243 struct bnxt_re_uctx_resp resp = {}; 5244 struct bnxt_re_uctx_req ureq = {}; 5245 struct bnxt_qplib_chip_ctx *cctx; 5246 u32 chip_met_rev_num; 5247 bool genp5 = false; 5248 int rc; 5249 5250 cctx = rdev->chip_ctx; 5251 rc = bnxt_re_check_abi_version(rdev); 5252 if (rc) 5253 goto fail; 5254 5255 uctx->rdev = rdev; 5256 uctx->shpg = (void *)__get_free_page(GFP_KERNEL); 5257 if (!uctx->shpg) { 5258 dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n"); 5259 rc = -ENOMEM; 5260 goto fail; 5261 } 5262 spin_lock_init(&uctx->sh_lock); 5263 if (BNXT_RE_ABI_VERSION >= 4) { 5264 chip_met_rev_num = cctx->chip_num; 5265 chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) << 5266 BNXT_RE_CHIP_ID0_CHIP_REV_SFT; 5267 chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) << 5268 BNXT_RE_CHIP_ID0_CHIP_MET_SFT; 5269 resp.chip_id0 = chip_met_rev_num; 5270 resp.chip_id1 = 0; /* future extension of chip info */ 5271 } 5272 5273 if (BNXT_RE_ABI_VERSION != 4) { 5274 /*Temp, Use idr_alloc instead*/ 5275 resp.dev_id = rdev->en_dev->pdev->devfn; 5276 resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max; 5277 } 5278 5279 genp5 = _is_chip_gen_p5_p7(cctx); 5280 resp.mode = genp5 ? cctx->modes.wqe_mode : 0; 5281 if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) 5282 resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; 5283 5284 resp.pg_size = PAGE_SIZE; 5285 resp.cqe_sz = sizeof(struct cq_base); 5286 resp.max_cqd = dev_attr->max_cq_wqes; 5287 if (genp5 && cctx->modes.db_push) { 5288 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; 5289 if (_is_chip_p7(cctx) && 5290 !(dev_attr->dev_cap_flags & 5291 CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE)) 5292 resp.comp_mask &= 5293 ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; 5294 } 5295 5296 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED; 5297 5298 if (rdev->dbr_pacing) 5299 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED; 5300 5301 if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov) 5302 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED; 5303 5304 if (udata->inlen >= sizeof(ureq)) { 5305 rc = ib_copy_from_udata(&ureq, udata, 5306 min(udata->inlen, sizeof(ureq))); 5307 if (rc) 5308 goto cfail; 5309 if (bnxt_re_init_pow2_flag(&ureq, &resp)) 5310 dev_warn(rdev_to_dev(rdev), 5311 "Enabled roundup logic. Library bug?\n"); 5312 if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5)) 5313 dev_warn(rdev_to_dev(rdev), 5314 "Rsvd wqe in use! Try the updated library.\n"); 5315 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) { 5316 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE; 5317 resp.mode = rdev->chip_ctx->modes.wqe_mode; 5318 if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE) 5319 resp.comp_mask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED; 5320 } 5321 } else { 5322 dev_warn(rdev_to_dev(rdev), 5323 "Enabled roundup logic. Update the library!\n"); 5324 resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; 5325 5326 dev_warn(rdev_to_dev(rdev), 5327 "Rsvd wqe in use. Update the library!\n"); 5328 resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; 5329 } 5330 5331 uctx->cmask = (uint64_t)resp.comp_mask; 5332 rc = bnxt_re_copy_to_udata(rdev, &resp, 5333 min(udata->outlen, sizeof(resp)), 5334 udata); 5335 if (rc) 5336 goto cfail; 5337 5338 INIT_LIST_HEAD(&uctx->cq_list); 5339 mutex_init(&uctx->cq_lock); 5340 5341 return 0; 5342 cfail: 5343 free_page((u64)uctx->shpg); 5344 uctx->shpg = NULL; 5345 fail: 5346 return rc; 5347 } 5348 5349 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) 5350 { 5351 struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, 5352 struct bnxt_re_ucontext, 5353 ibucontext); 5354 struct bnxt_re_dev *rdev = uctx->rdev; 5355 int rc = 0; 5356 5357 if (uctx->shpg) 5358 free_page((u64)uctx->shpg); 5359 5360 if (uctx->dpi.dbr) { 5361 /* Free DPI only if this is the first PD allocated by the 5362 * application and mark the context dpi as NULL 5363 */ 5364 if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) { 5365 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 5366 &uctx->wcdpi); 5367 if (rc) 5368 dev_err(rdev_to_dev(rdev), 5369 "dealloc push dp failed\n"); 5370 uctx->wcdpi.dbr = NULL; 5371 } 5372 5373 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, 5374 &uctx->dpi); 5375 if (rc) 5376 dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n"); 5377 /* Don't fail, continue*/ 5378 uctx->dpi.dbr = NULL; 5379 } 5380 return; 5381 } 5382 5383 static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx, 5384 u64 pg_off) 5385 { 5386 struct bnxt_re_cq *cq = NULL, *tmp_cq; 5387 5388 if (!_is_chip_p7(uctx->rdev->chip_ctx)) 5389 return NULL; 5390 5391 mutex_lock(&uctx->cq_lock); 5392 list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) { 5393 if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) { 5394 cq = tmp_cq; 5395 break; 5396 } 5397 } 5398 mutex_unlock(&uctx->cq_lock); 5399 return cq; 5400 } 5401 5402 /* Helper function to mmap the virtual memory from user app */ 5403 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) 5404 { 5405 struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, 5406 struct bnxt_re_ucontext, 5407 ibucontext); 5408 struct bnxt_re_dev *rdev = uctx->rdev; 5409 struct bnxt_re_cq *cq = NULL; 5410 int rc = 0; 5411 u64 pfn; 5412 5413 switch (vma->vm_pgoff) { 5414 case BNXT_RE_MAP_SH_PAGE: 5415 pfn = vtophys(uctx->shpg) >> PAGE_SHIFT; 5416 return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); 5417 dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n", 5418 __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn); 5419 if (rc) { 5420 dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n"); 5421 rc = -EAGAIN; 5422 } 5423 return rc; 5424 case BNXT_RE_MAP_WC: 5425 vma->vm_page_prot = 5426 pgprot_writecombine(vma->vm_page_prot); 5427 pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT); 5428 if (!pfn) 5429 return -EFAULT; 5430 break; 5431 case BNXT_RE_DBR_PAGE: 5432 /* Driver doesn't expect write access request */ 5433 if (vma->vm_flags & VM_WRITE) 5434 return -EFAULT; 5435 5436 pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT; 5437 if (!pfn) 5438 return -EFAULT; 5439 break; 5440 case BNXT_RE_MAP_DB_RECOVERY_PAGE: 5441 pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT; 5442 if (!pfn) 5443 return -EFAULT; 5444 break; 5445 default: 5446 cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff); 5447 if (cq) { 5448 pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT; 5449 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); 5450 if (rc) { 5451 dev_err(rdev_to_dev(rdev), 5452 "CQ page mapping failed!\n"); 5453 rc = -EAGAIN; 5454 } 5455 goto out; 5456 } else { 5457 vma->vm_page_prot = 5458 pgprot_noncached(vma->vm_page_prot); 5459 pfn = vma->vm_pgoff; 5460 } 5461 break; 5462 } 5463 5464 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); 5465 if (rc) { 5466 dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n"); 5467 return -EAGAIN; 5468 } 5469 rc = __bnxt_re_set_vma_data(uctx, vma); 5470 out: 5471 return rc; 5472 } 5473 5474 int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, 5475 const struct ib_wc *wc, const struct ib_grh *grh, 5476 const struct ib_mad_hdr *in_mad, size_t in_mad_size, 5477 struct ib_mad_hdr *out_mad, size_t *out_mad_size, 5478 u16 *out_mad_pkey_index) 5479 { 5480 return IB_MAD_RESULT_SUCCESS; 5481 } 5482 5483 void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx) 5484 { 5485 } 5486