1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2025 Broadcom. 3 4 #include <linux/module.h> 5 #include <linux/pci.h> 6 #include <linux/auxiliary_bus.h> 7 8 #include <rdma/ib_verbs.h> 9 10 #include "bng_res.h" 11 #include "bng_sp.h" 12 #include "bng_fw.h" 13 #include "bnge.h" 14 #include "bnge_auxr.h" 15 #include "bng_re.h" 16 #include "bnge_hwrm.h" 17 #include "bng_debugfs.h" 18 19 MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>"); 20 MODULE_DESCRIPTION(BNG_RE_DESC); 21 MODULE_LICENSE("Dual BSD/GPL"); 22 23 static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev, 24 struct bnge_auxr_dev *aux_dev) 25 { 26 struct bng_re_dev *rdev; 27 28 /* Allocate bng_re_dev instance */ 29 rdev = ib_alloc_device(bng_re_dev, ibdev); 30 if (!rdev) { 31 pr_err("%s: bng_re_dev allocation failure!", KBUILD_MODNAME); 32 return NULL; 33 } 34 35 /* Assign auxiliary device specific data */ 36 rdev->netdev = aux_dev->net; 37 rdev->aux_dev = aux_dev; 38 rdev->adev = adev; 39 rdev->fn_id = rdev->aux_dev->pdev->devfn; 40 41 return rdev; 42 } 43 44 45 static int bng_re_register_netdev(struct bng_re_dev *rdev) 46 { 47 struct bnge_auxr_dev *aux_dev; 48 49 aux_dev = rdev->aux_dev; 50 return bnge_register_dev(aux_dev, rdev->adev); 51 } 52 53 static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev) 54 { 55 struct bng_re_chip_ctx *chip_ctx; 56 57 if (!rdev->chip_ctx) 58 return; 59 60 kfree(rdev->dev_attr); 61 rdev->dev_attr = NULL; 62 63 chip_ctx = rdev->chip_ctx; 64 rdev->chip_ctx = NULL; 65 rdev->rcfw.res = NULL; 66 rdev->bng_res.cctx = NULL; 67 rdev->bng_res.pdev = NULL; 68 kfree(chip_ctx); 69 } 70 71 static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev) 72 { 73 struct bng_re_chip_ctx *chip_ctx; 74 struct bnge_auxr_dev *aux_dev; 75 int rc = -ENOMEM; 76 77 aux_dev = rdev->aux_dev; 78 rdev->bng_res.pdev = aux_dev->pdev; 79 rdev->rcfw.res = &rdev->bng_res; 80 chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); 81 if (!chip_ctx) 82 return -ENOMEM; 83 chip_ctx->chip_num = aux_dev->chip_num; 84 chip_ctx->hw_stats_size = aux_dev->hw_ring_stats_size; 85 86 rdev->chip_ctx = chip_ctx; 87 rdev->bng_res.cctx = rdev->chip_ctx; 88 rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); 89 if (!rdev->dev_attr) 90 goto free_chip_ctx; 91 rdev->bng_res.dattr = rdev->dev_attr; 92 93 return 0; 94 free_chip_ctx: 95 kfree(rdev->chip_ctx); 96 rdev->chip_ctx = NULL; 97 return rc; 98 } 99 100 static void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd) 101 { 102 hdr->req_type = cpu_to_le16(opcd); 103 hdr->cmpl_ring = cpu_to_le16(-1); 104 hdr->target_id = cpu_to_le16(-1); 105 } 106 107 static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg, 108 int msg_len, void *resp, int resp_max_len, 109 int timeout) 110 { 111 fw_msg->msg = msg; 112 fw_msg->msg_len = msg_len; 113 fw_msg->resp = resp; 114 fw_msg->resp_max_len = resp_max_len; 115 fw_msg->timeout = timeout; 116 } 117 118 static int bng_re_net_ring_free(struct bng_re_dev *rdev, 119 u16 fw_ring_id, int type) 120 { 121 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 122 struct hwrm_ring_free_input req = {}; 123 struct hwrm_ring_free_output resp; 124 struct bnge_fw_msg fw_msg = {}; 125 int rc = -EINVAL; 126 127 if (!rdev) 128 return rc; 129 130 if (!aux_dev) 131 return rc; 132 133 bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_FREE); 134 req.ring_type = type; 135 req.ring_id = cpu_to_le16(fw_ring_id); 136 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 137 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT); 138 rc = bnge_send_msg(aux_dev, &fw_msg); 139 if (rc) 140 ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x", 141 req.ring_id, rc); 142 return rc; 143 } 144 145 static int bng_re_net_ring_alloc(struct bng_re_dev *rdev, 146 struct bng_re_ring_attr *ring_attr, 147 u16 *fw_ring_id) 148 { 149 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 150 struct hwrm_ring_alloc_input req = {}; 151 struct hwrm_ring_alloc_output resp; 152 struct bnge_fw_msg fw_msg = {}; 153 int rc = -EINVAL; 154 155 if (!aux_dev) 156 return rc; 157 158 bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC); 159 req.enables = 0; 160 req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); 161 if (ring_attr->pages > 1) { 162 /* Page size is in log2 units */ 163 req.page_size = BNGE_PAGE_SHIFT; 164 req.page_tbl_depth = 1; 165 } 166 req.fbo = 0; 167 /* Association of ring index with doorbell index and MSIX number */ 168 req.logical_id = cpu_to_le16(ring_attr->lrid); 169 req.length = cpu_to_le32(ring_attr->depth + 1); 170 req.ring_type = ring_attr->type; 171 req.int_mode = ring_attr->mode; 172 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 173 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT); 174 rc = bnge_send_msg(aux_dev, &fw_msg); 175 if (!rc) 176 *fw_ring_id = le16_to_cpu(resp.ring_id); 177 178 return rc; 179 } 180 181 static void bng_re_query_hwrm_version(struct bng_re_dev *rdev) 182 { 183 struct bnge_auxr_dev *aux_dev = rdev->aux_dev; 184 struct hwrm_ver_get_output ver_get_resp = {}; 185 struct hwrm_ver_get_input ver_get_req = {}; 186 struct bng_re_chip_ctx *cctx; 187 struct bnge_fw_msg fw_msg = {}; 188 int rc; 189 190 bng_re_init_hwrm_hdr((void *)&ver_get_req, HWRM_VER_GET); 191 ver_get_req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 192 ver_get_req.hwrm_intf_min = HWRM_VERSION_MINOR; 193 ver_get_req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 194 bng_re_fill_fw_msg(&fw_msg, (void *)&ver_get_req, sizeof(ver_get_req), 195 (void *)&ver_get_resp, sizeof(ver_get_resp), 196 BNGE_DFLT_HWRM_CMD_TIMEOUT); 197 rc = bnge_send_msg(aux_dev, &fw_msg); 198 if (rc) { 199 ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x", 200 rc); 201 return; 202 } 203 204 cctx = rdev->chip_ctx; 205 cctx->hwrm_intf_ver = 206 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_major) << 48 | 207 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_minor) << 32 | 208 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_build) << 16 | 209 le16_to_cpu(ver_get_resp.hwrm_intf_patch); 210 211 cctx->hwrm_cmd_max_timeout = le16_to_cpu(ver_get_resp.max_req_timeout); 212 213 if (!cctx->hwrm_cmd_max_timeout) 214 cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT; 215 } 216 217 static void bng_re_dev_uninit(struct bng_re_dev *rdev) 218 { 219 bng_re_debugfs_rem_pdev(rdev); 220 bng_re_disable_rcfw_channel(&rdev->rcfw); 221 bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, 222 RING_ALLOC_REQ_RING_TYPE_NQ); 223 bng_re_free_rcfw_channel(&rdev->rcfw); 224 225 kfree(rdev->nqr); 226 rdev->nqr = NULL; 227 bng_re_destroy_chip_ctx(rdev); 228 if (test_and_clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) 229 bnge_unregister_dev(rdev->aux_dev); 230 } 231 232 static int bng_re_dev_init(struct bng_re_dev *rdev) 233 { 234 struct bng_re_ring_attr rattr = {}; 235 struct bng_re_creq_ctx *creq; 236 u32 db_offt; 237 int vid; 238 u8 type; 239 int rc; 240 241 /* Registered a new RoCE device instance to netdev */ 242 rc = bng_re_register_netdev(rdev); 243 if (rc) { 244 ibdev_err(&rdev->ibdev, 245 "Failed to register with netedev: %#x\n", rc); 246 return -EINVAL; 247 } 248 249 set_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 250 251 if (rdev->aux_dev->auxr_info->msix_requested < BNG_RE_MIN_MSIX) { 252 ibdev_err(&rdev->ibdev, 253 "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n", 254 rdev->aux_dev->auxr_info->msix_requested); 255 bnge_unregister_dev(rdev->aux_dev); 256 clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 257 return -EINVAL; 258 } 259 ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n", 260 rdev->aux_dev->auxr_info->msix_requested); 261 262 rc = bng_re_setup_chip_ctx(rdev); 263 if (rc) { 264 bnge_unregister_dev(rdev->aux_dev); 265 clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 266 ibdev_err(&rdev->ibdev, "Failed to get chip context\n"); 267 return -EINVAL; 268 } 269 270 bng_re_query_hwrm_version(rdev); 271 272 rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw); 273 if (rc) { 274 ibdev_err(&rdev->ibdev, 275 "Failed to allocate RCFW Channel: %#x\n", rc); 276 goto fail; 277 } 278 279 /* Allocate nq record memory */ 280 rdev->nqr = kzalloc(sizeof(*rdev->nqr), GFP_KERNEL); 281 if (!rdev->nqr) { 282 bng_re_destroy_chip_ctx(rdev); 283 bnge_unregister_dev(rdev->aux_dev); 284 clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 285 return -ENOMEM; 286 } 287 288 rdev->nqr->num_msix = rdev->aux_dev->auxr_info->msix_requested; 289 memcpy(rdev->nqr->msix_entries, rdev->aux_dev->msix_info, 290 sizeof(struct bnge_msix_info) * rdev->nqr->num_msix); 291 292 type = RING_ALLOC_REQ_RING_TYPE_NQ; 293 creq = &rdev->rcfw.creq; 294 rattr.dma_arr = creq->hwq.pbl[BNG_PBL_LVL_0].pg_map_arr; 295 rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; 296 rattr.type = type; 297 rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX; 298 rattr.depth = BNG_FW_CREQE_MAX_CNT - 1; 299 rattr.lrid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].ring_idx; 300 rc = bng_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); 301 if (rc) { 302 ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc); 303 goto free_rcfw; 304 } 305 db_offt = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].db_offset; 306 vid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].vector; 307 308 rc = bng_re_enable_fw_channel(&rdev->rcfw, 309 vid, db_offt); 310 if (rc) { 311 ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n", 312 rc); 313 goto free_ring; 314 } 315 316 rc = bng_re_get_dev_attr(&rdev->rcfw); 317 if (rc) 318 goto disable_rcfw; 319 320 bng_re_debugfs_add_pdev(rdev); 321 322 return 0; 323 disable_rcfw: 324 bng_re_disable_rcfw_channel(&rdev->rcfw); 325 free_ring: 326 bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type); 327 free_rcfw: 328 bng_re_free_rcfw_channel(&rdev->rcfw); 329 fail: 330 bng_re_dev_uninit(rdev); 331 return rc; 332 } 333 334 static int bng_re_add_device(struct auxiliary_device *adev) 335 { 336 struct bnge_auxr_priv *auxr_priv = 337 container_of(adev, struct bnge_auxr_priv, aux_dev); 338 struct bng_re_en_dev_info *dev_info; 339 struct bng_re_dev *rdev; 340 int rc; 341 342 dev_info = auxiliary_get_drvdata(adev); 343 344 rdev = bng_re_dev_add(adev, auxr_priv->auxr_dev); 345 if (!rdev) { 346 rc = -ENOMEM; 347 goto exit; 348 } 349 350 dev_info->rdev = rdev; 351 352 rc = bng_re_dev_init(rdev); 353 if (rc) 354 goto re_dev_dealloc; 355 356 return 0; 357 358 re_dev_dealloc: 359 ib_dealloc_device(&rdev->ibdev); 360 exit: 361 return rc; 362 } 363 364 365 static void bng_re_remove_device(struct bng_re_dev *rdev, 366 struct auxiliary_device *aux_dev) 367 { 368 bng_re_dev_uninit(rdev); 369 ib_dealloc_device(&rdev->ibdev); 370 } 371 372 373 static int bng_re_probe(struct auxiliary_device *adev, 374 const struct auxiliary_device_id *id) 375 { 376 struct bnge_auxr_priv *aux_priv = 377 container_of(adev, struct bnge_auxr_priv, aux_dev); 378 struct bng_re_en_dev_info *en_info; 379 int rc; 380 381 en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); 382 if (!en_info) 383 return -ENOMEM; 384 385 en_info->auxr_dev = aux_priv->auxr_dev; 386 387 auxiliary_set_drvdata(adev, en_info); 388 389 rc = bng_re_add_device(adev); 390 if (rc) 391 kfree(en_info); 392 393 return rc; 394 } 395 396 static void bng_re_remove(struct auxiliary_device *adev) 397 { 398 struct bng_re_en_dev_info *dev_info = auxiliary_get_drvdata(adev); 399 struct bng_re_dev *rdev; 400 401 rdev = dev_info->rdev; 402 403 if (rdev) 404 bng_re_remove_device(rdev, adev); 405 kfree(dev_info); 406 } 407 408 static const struct auxiliary_device_id bng_re_id_table[] = { 409 { .name = BNG_RE_ADEV_NAME ".rdma", }, 410 {}, 411 }; 412 413 MODULE_DEVICE_TABLE(auxiliary, bng_re_id_table); 414 415 static struct auxiliary_driver bng_re_driver = { 416 .name = "rdma", 417 .probe = bng_re_probe, 418 .remove = bng_re_remove, 419 .id_table = bng_re_id_table, 420 }; 421 422 static int __init bng_re_mod_init(void) 423 { 424 int rc; 425 426 427 bng_re_register_debugfs(); 428 429 rc = auxiliary_driver_register(&bng_re_driver); 430 if (rc) { 431 pr_err("%s: Failed to register auxiliary driver\n", 432 KBUILD_MODNAME); 433 goto unreg_debugfs; 434 } 435 return 0; 436 unreg_debugfs: 437 bng_re_unregister_debugfs(); 438 return rc; 439 } 440 441 static void __exit bng_re_mod_exit(void) 442 { 443 auxiliary_driver_unregister(&bng_re_driver); 444 bng_re_unregister_debugfs(); 445 } 446 447 module_init(bng_re_mod_init); 448 module_exit(bng_re_mod_exit); 449