1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <linux/module.h> 5 #include <linux/pci.h> 6 #include <linux/auxiliary_bus.h> 7 8 #include <rdma/ib_verbs.h> 9 10 #include "bng_res.h" 11 #include "bng_sp.h" 12 #include "bng_fw.h" 13 #include "bnge.h" 14 #include "bnge_auxr.h" 15 #include "bng_re.h" 16 #include "bnge_hwrm.h" 17 #include "bng_debugfs.h" 18 19 MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>"); 20 MODULE_DESCRIPTION(BNG_RE_DESC); 21 MODULE_LICENSE("Dual BSD/GPL"); 22 23 static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev, 24 struct bnge_auxr_dev *aux_dev) 25 { 26 struct bng_re_dev *rdev; 27 28 /* Allocate bng_re_dev instance */ 29 rdev = ib_alloc_device(bng_re_dev, ibdev); 30 if (!rdev) { 31 pr_err("%s: bng_re_dev allocation failure!", KBUILD_MODNAME); 32 return NULL; 33 } 34 35 /* Assign auxiliary device specific data */ 36 rdev->netdev = aux_dev->net; 37 rdev->aux_dev = aux_dev; 38 rdev->adev = adev; 39 rdev->fn_id = rdev->aux_dev->pdev->devfn; 40 41 return rdev; 42 } 43 44 45 static int bng_re_register_netdev(struct bng_re_dev *rdev) 46 { 47 struct bnge_auxr_dev *aux_dev; 48 49 aux_dev = rdev->aux_dev; 50 return bnge_register_dev(aux_dev, rdev->adev); 51 } 52 53 static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev) 54 { 55 struct bng_re_chip_ctx *chip_ctx; 56 57 kfree(rdev->dev_attr); 58 rdev->dev_attr = NULL; 59 60 chip_ctx = rdev->chip_ctx; 61 rdev->chip_ctx = NULL; 62 rdev->rcfw.res = NULL; 63 rdev->bng_res.cctx = NULL; 64 rdev->bng_res.pdev = NULL; 65 kfree(chip_ctx); 66 } 67 68 static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev) 69 { 70 struct bng_re_chip_ctx *chip_ctx; 71 struct bnge_auxr_dev *aux_dev; 72 int rc = -ENOMEM; 73 74 aux_dev = rdev->aux_dev; 75 rdev->bng_res.pdev = aux_dev->pdev; 76 rdev->rcfw.res = &rdev->bng_res; 77 chip_ctx = kzalloc_obj(*chip_ctx); 78 if (!chip_ctx) 79 return -ENOMEM; 80 chip_ctx->chip_num = aux_dev->chip_num; 81 chip_ctx->hw_stats_size = aux_dev->hw_ring_stats_size; 82 83 rdev->chip_ctx = chip_ctx; 84 rdev->bng_res.cctx = rdev->chip_ctx; 85 rdev->dev_attr = kzalloc_obj(*rdev->dev_attr); 86 if (!rdev->dev_attr) 87 goto free_chip_ctx; 88 rdev->bng_res.dattr = rdev->dev_attr; 89 90 return 0; 91 free_chip_ctx: 92 kfree(rdev->chip_ctx); 93 rdev->chip_ctx = NULL; 94 return rc; 95 } 96 97 static void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd) 98 { 99 hdr->req_type = cpu_to_le16(opcd); 100 hdr->cmpl_ring = cpu_to_le16(-1); 101 hdr->target_id = cpu_to_le16(-1); 102 } 103 104 static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg, 105 int msg_len, void *resp, int resp_max_len, 106 int timeout) 107 { 108 fw_msg->msg = msg; 109 fw_msg->msg_len = msg_len; 110 fw_msg->resp = resp; 111 fw_msg->resp_max_len = resp_max_len; 112 fw_msg->timeout = timeout; 113 } 114 115 static int bng_re_net_ring_free(struct bng_re_dev *rdev, 116 u16 fw_ring_id, int type) 117 { 118 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 119 struct hwrm_ring_free_input req = {}; 120 struct hwrm_ring_free_output resp; 121 struct bnge_fw_msg fw_msg = {}; 122 int rc = -EINVAL; 123 124 bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_FREE); 125 req.ring_type = type; 126 req.ring_id = cpu_to_le16(fw_ring_id); 127 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 128 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT); 129 rc = bnge_send_msg(aux_dev, &fw_msg); 130 if (rc) 131 ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x", 132 req.ring_id, rc); 133 return rc; 134 } 135 136 static int bng_re_net_ring_alloc(struct bng_re_dev *rdev, 137 struct bng_re_ring_attr *ring_attr, 138 u16 *fw_ring_id) 139 { 140 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 141 struct hwrm_ring_alloc_input req = {}; 142 struct hwrm_ring_alloc_output resp; 143 struct bnge_fw_msg fw_msg = {}; 144 int rc; 145 146 bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC); 147 req.enables = 0; 148 req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); 149 if (ring_attr->pages > 1) { 150 /* Page size is in log2 units */ 151 req.page_size = BNGE_PAGE_SHIFT; 152 req.page_tbl_depth = 1; 153 } 154 req.fbo = 0; 155 /* Association of ring index with doorbell index and MSIX number */ 156 req.logical_id = cpu_to_le16(ring_attr->lrid); 157 req.length = cpu_to_le32(ring_attr->depth + 1); 158 req.ring_type = ring_attr->type; 159 req.int_mode = ring_attr->mode; 160 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 161 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT); 162 rc = bnge_send_msg(aux_dev, &fw_msg); 163 if (!rc) 164 *fw_ring_id = le16_to_cpu(resp.ring_id); 165 166 return rc; 167 } 168 169 static int bng_re_stats_ctx_free(struct bng_re_dev *rdev) 170 { 171 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 172 struct hwrm_stat_ctx_free_input req = {}; 173 struct hwrm_stat_ctx_free_output resp = {}; 174 struct bnge_fw_msg fw_msg = {}; 175 int rc; 176 177 bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_FREE); 178 req.stat_ctx_id = cpu_to_le32(rdev->stats_ctx.fw_id); 179 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 180 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT); 181 rc = bnge_send_msg(aux_dev, &fw_msg); 182 if (rc) 183 ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x", 184 rc); 185 186 return rc; 187 } 188 189 static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev) 190 { 191 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 192 struct bng_re_stats *stats = &rdev->stats_ctx; 193 struct hwrm_stat_ctx_alloc_output resp = {}; 194 struct hwrm_stat_ctx_alloc_input req = {}; 195 struct bnge_fw_msg fw_msg = {}; 196 int rc; 197 198 stats->fw_id = BNGE_INVALID_STATS_CTX_ID; 199 200 bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_ALLOC); 201 req.update_period_ms = cpu_to_le32(1000); 202 req.stats_dma_addr = cpu_to_le64(stats->dma_map); 203 req.stats_dma_length = cpu_to_le16(rdev->chip_ctx->hw_stats_size); 204 req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE; 205 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 206 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT); 207 rc = bnge_send_msg(aux_dev, &fw_msg); 208 if (!rc) 209 stats->fw_id = le32_to_cpu(resp.stat_ctx_id); 210 return rc; 211 } 212 213 static int bng_re_query_hwrm_version(struct bng_re_dev *rdev) 214 { 215 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 216 struct hwrm_ver_get_output ver_get_resp = {}; 217 struct hwrm_ver_get_input ver_get_req = {}; 218 struct bng_re_chip_ctx *cctx; 219 struct bnge_fw_msg fw_msg = {}; 220 int rc; 221 222 bng_re_init_hwrm_hdr((void *)&ver_get_req, HWRM_VER_GET); 223 ver_get_req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 224 ver_get_req.hwrm_intf_min = HWRM_VERSION_MINOR; 225 ver_get_req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 226 bng_re_fill_fw_msg(&fw_msg, (void *)&ver_get_req, sizeof(ver_get_req), 227 (void *)&ver_get_resp, sizeof(ver_get_resp), 228 BNGE_DFLT_HWRM_CMD_TIMEOUT); 229 rc = bnge_send_msg(aux_dev, &fw_msg); 230 if (rc) { 231 ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x", 232 rc); 233 return rc; 234 } 235 236 cctx = rdev->chip_ctx; 237 cctx->hwrm_intf_ver = 238 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_major) << 48 | 239 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_minor) << 32 | 240 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_build) << 16 | 241 le16_to_cpu(ver_get_resp.hwrm_intf_patch); 242 243 cctx->hwrm_cmd_max_timeout = le16_to_cpu(ver_get_resp.max_req_timeout); 244 245 if (!cctx->hwrm_cmd_max_timeout) 246 cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT; 247 248 return 0; 249 } 250 251 static void bng_re_dev_uninit(struct bng_re_dev *rdev) 252 { 253 int rc; 254 bng_re_debugfs_rem_pdev(rdev); 255 256 if (test_and_clear_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { 257 rc = bng_re_deinit_rcfw(&rdev->rcfw); 258 if (rc) 259 ibdev_warn(&rdev->ibdev, 260 "Failed to deinitialize RCFW: %#x", rc); 261 bng_re_stats_ctx_free(rdev); 262 bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx); 263 bng_re_disable_rcfw_channel(&rdev->rcfw); 264 bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, 265 RING_ALLOC_REQ_RING_TYPE_NQ); 266 bng_re_free_rcfw_channel(&rdev->rcfw); 267 } 268 269 kfree(rdev->nqr); 270 rdev->nqr = NULL; 271 bng_re_destroy_chip_ctx(rdev); 272 if (test_and_clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) 273 bnge_unregister_dev(rdev->aux_dev); 274 } 275 276 static int bng_re_dev_init(struct bng_re_dev *rdev) 277 { 278 struct bng_re_ring_attr rattr = {}; 279 struct bng_re_creq_ctx *creq; 280 u32 db_offt; 281 int vid; 282 u8 type; 283 int rc; 284 285 /* Registered a new RoCE device instance to netdev */ 286 rc = bng_re_register_netdev(rdev); 287 if (rc) { 288 ibdev_err(&rdev->ibdev, 289 "Failed to register with netedev: %#x\n", rc); 290 goto reg_netdev_fail; 291 } 292 293 set_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 294 295 if (rdev->aux_dev->auxr_info->msix_requested < BNG_RE_MIN_MSIX) { 296 ibdev_err(&rdev->ibdev, 297 "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n", 298 rdev->aux_dev->auxr_info->msix_requested); 299 rc = -EINVAL; 300 goto msix_ctx_fail; 301 } 302 ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n", 303 rdev->aux_dev->auxr_info->msix_requested); 304 305 rc = bng_re_setup_chip_ctx(rdev); 306 if (rc) { 307 ibdev_err(&rdev->ibdev, "Failed to get chip context\n"); 308 goto msix_ctx_fail; 309 } 310 311 rc = bng_re_query_hwrm_version(rdev); 312 if (rc) 313 goto destroy_chip_ctx; 314 315 rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw); 316 if (rc) { 317 ibdev_err(&rdev->ibdev, 318 "Failed to allocate RCFW Channel: %#x\n", rc); 319 goto destroy_chip_ctx; 320 } 321 322 /* Allocate nq record memory */ 323 rdev->nqr = kzalloc_obj(*rdev->nqr); 324 if (!rdev->nqr) { 325 rc = -ENOMEM; 326 goto nq_alloc_fail; 327 } 328 329 rdev->nqr->num_msix = rdev->aux_dev->auxr_info->msix_requested; 330 memcpy(rdev->nqr->msix_entries, rdev->aux_dev->msix_info, 331 sizeof(struct bnge_msix_info) * rdev->nqr->num_msix); 332 333 type = RING_ALLOC_REQ_RING_TYPE_NQ; 334 creq = &rdev->rcfw.creq; 335 rattr.dma_arr = creq->hwq.pbl[BNG_PBL_LVL_0].pg_map_arr; 336 rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; 337 rattr.type = type; 338 rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX; 339 rattr.depth = BNG_FW_CREQE_MAX_CNT - 1; 340 rattr.lrid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].ring_idx; 341 rc = bng_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); 342 if (rc) { 343 ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc); 344 goto free_rcfw; 345 } 346 db_offt = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].db_offset; 347 vid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].vector; 348 349 rc = bng_re_enable_fw_channel(&rdev->rcfw, 350 vid, db_offt); 351 if (rc) { 352 ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n", 353 rc); 354 goto free_ring; 355 } 356 357 rc = bng_re_get_dev_attr(&rdev->rcfw); 358 if (rc) 359 goto disable_rcfw; 360 361 bng_re_debugfs_add_pdev(rdev); 362 rc = bng_re_alloc_stats_ctx_mem(rdev->bng_res.pdev, rdev->chip_ctx, 363 &rdev->stats_ctx); 364 if (rc) { 365 ibdev_err(&rdev->ibdev, 366 "Failed to allocate stats context: %#x\n", rc); 367 goto disable_rcfw; 368 } 369 370 rc = bng_re_stats_ctx_alloc(rdev); 371 if (rc) { 372 ibdev_err(&rdev->ibdev, 373 "Failed to allocate QPLIB context: %#x\n", rc); 374 goto free_stats_ctx; 375 } 376 377 rc = bng_re_init_rcfw(&rdev->rcfw, &rdev->stats_ctx); 378 if (rc) { 379 ibdev_err(&rdev->ibdev, 380 "Failed to initialize RCFW: %#x\n", rc); 381 goto free_sctx; 382 } 383 set_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); 384 385 return 0; 386 free_sctx: 387 bng_re_stats_ctx_free(rdev); 388 free_stats_ctx: 389 bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx); 390 disable_rcfw: 391 bng_re_disable_rcfw_channel(&rdev->rcfw); 392 free_ring: 393 bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type); 394 free_rcfw: 395 kfree(rdev->nqr); 396 nq_alloc_fail: 397 bng_re_free_rcfw_channel(&rdev->rcfw); 398 destroy_chip_ctx: 399 bng_re_destroy_chip_ctx(rdev); 400 msix_ctx_fail: 401 bnge_unregister_dev(rdev->aux_dev); 402 clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 403 reg_netdev_fail: 404 return rc; 405 } 406 407 static int bng_re_add_device(struct auxiliary_device *adev) 408 { 409 struct bnge_auxr_priv *auxr_priv = 410 container_of(adev, struct bnge_auxr_priv, aux_dev); 411 struct bng_re_en_dev_info *dev_info; 412 struct bng_re_dev *rdev; 413 int rc; 414 415 dev_info = auxiliary_get_drvdata(adev); 416 417 rdev = bng_re_dev_add(adev, auxr_priv->auxr_dev); 418 if (!rdev) { 419 rc = -ENOMEM; 420 goto exit; 421 } 422 423 dev_info->rdev = rdev; 424 425 rc = bng_re_dev_init(rdev); 426 if (rc) 427 goto re_dev_dealloc; 428 429 return 0; 430 431 re_dev_dealloc: 432 ib_dealloc_device(&rdev->ibdev); 433 exit: 434 return rc; 435 } 436 437 438 static void bng_re_remove_device(struct bng_re_dev *rdev, 439 struct auxiliary_device *aux_dev) 440 { 441 bng_re_dev_uninit(rdev); 442 ib_dealloc_device(&rdev->ibdev); 443 } 444 445 446 static int bng_re_probe(struct auxiliary_device *adev, 447 const struct auxiliary_device_id *id) 448 { 449 struct bnge_auxr_priv *aux_priv = 450 container_of(adev, struct bnge_auxr_priv, aux_dev); 451 struct bng_re_en_dev_info *en_info; 452 int rc; 453 454 en_info = kzalloc_obj(*en_info); 455 if (!en_info) 456 return -ENOMEM; 457 458 en_info->auxr_dev = aux_priv->auxr_dev; 459 460 auxiliary_set_drvdata(adev, en_info); 461 462 rc = bng_re_add_device(adev); 463 if (rc) 464 kfree(en_info); 465 466 return rc; 467 } 468 469 static void bng_re_remove(struct auxiliary_device *adev) 470 { 471 struct bng_re_en_dev_info *dev_info = auxiliary_get_drvdata(adev); 472 struct bng_re_dev *rdev; 473 474 rdev = dev_info->rdev; 475 476 bng_re_remove_device(rdev, adev); 477 kfree(dev_info); 478 } 479 480 static const struct auxiliary_device_id bng_re_id_table[] = { 481 { .name = BNG_RE_ADEV_NAME ".rdma", }, 482 {}, 483 }; 484 485 MODULE_DEVICE_TABLE(auxiliary, bng_re_id_table); 486 487 static struct auxiliary_driver bng_re_driver = { 488 .name = "rdma", 489 .probe = bng_re_probe, 490 .remove = bng_re_remove, 491 .id_table = bng_re_id_table, 492 }; 493 494 static int __init bng_re_mod_init(void) 495 { 496 int rc; 497 498 499 bng_re_register_debugfs(); 500 501 rc = auxiliary_driver_register(&bng_re_driver); 502 if (rc) { 503 pr_err("%s: Failed to register auxiliary driver\n", 504 KBUILD_MODNAME); 505 goto unreg_debugfs; 506 } 507 return 0; 508 unreg_debugfs: 509 bng_re_unregister_debugfs(); 510 return rc; 511 } 512 513 static void __exit bng_re_mod_exit(void) 514 { 515 auxiliary_driver_unregister(&bng_re_driver); 516 bng_re_unregister_debugfs(); 517 } 518 519 module_init(bng_re_mod_init); 520 module_exit(bng_re_mod_exit); 521