1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2025 Broadcom.
3
4 #include <linux/module.h>
5 #include <linux/pci.h>
6 #include <linux/auxiliary_bus.h>
7
8 #include <rdma/ib_verbs.h>
9
10 #include "bng_res.h"
11 #include "bng_sp.h"
12 #include "bng_fw.h"
13 #include "bnge.h"
14 #include "bnge_auxr.h"
15 #include "bng_re.h"
16 #include "bnge_hwrm.h"
17 #include "bng_debugfs.h"
18
19 MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>");
20 MODULE_DESCRIPTION(BNG_RE_DESC);
21 MODULE_LICENSE("Dual BSD/GPL");
22
bng_re_dev_add(struct auxiliary_device * adev,struct bnge_auxr_dev * aux_dev)23 static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev,
24 struct bnge_auxr_dev *aux_dev)
25 {
26 struct bng_re_dev *rdev;
27
28 /* Allocate bng_re_dev instance */
29 rdev = ib_alloc_device(bng_re_dev, ibdev);
30 if (!rdev) {
31 pr_err("%s: bng_re_dev allocation failure!", KBUILD_MODNAME);
32 return NULL;
33 }
34
35 /* Assign auxiliary device specific data */
36 rdev->netdev = aux_dev->net;
37 rdev->aux_dev = aux_dev;
38 rdev->adev = adev;
39 rdev->fn_id = rdev->aux_dev->pdev->devfn;
40
41 return rdev;
42 }
43
44
bng_re_register_netdev(struct bng_re_dev * rdev)45 static int bng_re_register_netdev(struct bng_re_dev *rdev)
46 {
47 struct bnge_auxr_dev *aux_dev;
48
49 aux_dev = rdev->aux_dev;
50 return bnge_register_dev(aux_dev, rdev->adev);
51 }
52
bng_re_destroy_chip_ctx(struct bng_re_dev * rdev)53 static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev)
54 {
55 struct bng_re_chip_ctx *chip_ctx;
56
57 kfree(rdev->dev_attr);
58 rdev->dev_attr = NULL;
59
60 chip_ctx = rdev->chip_ctx;
61 rdev->chip_ctx = NULL;
62 rdev->rcfw.res = NULL;
63 rdev->bng_res.cctx = NULL;
64 rdev->bng_res.pdev = NULL;
65 kfree(chip_ctx);
66 }
67
bng_re_setup_chip_ctx(struct bng_re_dev * rdev)68 static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev)
69 {
70 struct bng_re_chip_ctx *chip_ctx;
71 struct bnge_auxr_dev *aux_dev;
72 int rc = -ENOMEM;
73
74 aux_dev = rdev->aux_dev;
75 rdev->bng_res.pdev = aux_dev->pdev;
76 rdev->rcfw.res = &rdev->bng_res;
77 chip_ctx = kzalloc_obj(*chip_ctx);
78 if (!chip_ctx)
79 return -ENOMEM;
80 chip_ctx->chip_num = aux_dev->chip_num;
81 chip_ctx->hw_stats_size = aux_dev->hw_ring_stats_size;
82
83 rdev->chip_ctx = chip_ctx;
84 rdev->bng_res.cctx = rdev->chip_ctx;
85 rdev->dev_attr = kzalloc_obj(*rdev->dev_attr);
86 if (!rdev->dev_attr)
87 goto free_chip_ctx;
88 rdev->bng_res.dattr = rdev->dev_attr;
89
90 return 0;
91 free_chip_ctx:
92 kfree(rdev->chip_ctx);
93 rdev->chip_ctx = NULL;
94 return rc;
95 }
96
bng_re_init_hwrm_hdr(struct input * hdr,u16 opcd)97 static void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd)
98 {
99 hdr->req_type = cpu_to_le16(opcd);
100 hdr->cmpl_ring = cpu_to_le16(-1);
101 hdr->target_id = cpu_to_le16(-1);
102 }
103
bng_re_fill_fw_msg(struct bnge_fw_msg * fw_msg,void * msg,int msg_len,void * resp,int resp_max_len,int timeout)104 static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg,
105 int msg_len, void *resp, int resp_max_len,
106 int timeout)
107 {
108 fw_msg->msg = msg;
109 fw_msg->msg_len = msg_len;
110 fw_msg->resp = resp;
111 fw_msg->resp_max_len = resp_max_len;
112 fw_msg->timeout = timeout;
113 }
114
bng_re_net_ring_free(struct bng_re_dev * rdev,u16 fw_ring_id,int type)115 static int bng_re_net_ring_free(struct bng_re_dev *rdev,
116 u16 fw_ring_id, int type)
117 {
118 struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
119 struct hwrm_ring_free_input req = {};
120 struct hwrm_ring_free_output resp;
121 struct bnge_fw_msg fw_msg = {};
122 int rc = -EINVAL;
123
124 bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_FREE);
125 req.ring_type = type;
126 req.ring_id = cpu_to_le16(fw_ring_id);
127 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
128 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
129 rc = bnge_send_msg(aux_dev, &fw_msg);
130 if (rc)
131 ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
132 req.ring_id, rc);
133 return rc;
134 }
135
bng_re_net_ring_alloc(struct bng_re_dev * rdev,struct bng_re_ring_attr * ring_attr,u16 * fw_ring_id)136 static int bng_re_net_ring_alloc(struct bng_re_dev *rdev,
137 struct bng_re_ring_attr *ring_attr,
138 u16 *fw_ring_id)
139 {
140 struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
141 struct hwrm_ring_alloc_input req = {};
142 struct hwrm_ring_alloc_output resp;
143 struct bnge_fw_msg fw_msg = {};
144 int rc;
145
146 bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC);
147 req.enables = 0;
148 req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]);
149 if (ring_attr->pages > 1) {
150 /* Page size is in log2 units */
151 req.page_size = BNGE_PAGE_SHIFT;
152 req.page_tbl_depth = 1;
153 }
154 req.fbo = 0;
155 /* Association of ring index with doorbell index and MSIX number */
156 req.logical_id = cpu_to_le16(ring_attr->lrid);
157 req.length = cpu_to_le32(ring_attr->depth + 1);
158 req.ring_type = ring_attr->type;
159 req.int_mode = ring_attr->mode;
160 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
161 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
162 rc = bnge_send_msg(aux_dev, &fw_msg);
163 if (!rc)
164 *fw_ring_id = le16_to_cpu(resp.ring_id);
165
166 return rc;
167 }
168
bng_re_stats_ctx_free(struct bng_re_dev * rdev)169 static int bng_re_stats_ctx_free(struct bng_re_dev *rdev)
170 {
171 struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
172 struct hwrm_stat_ctx_free_input req = {};
173 struct hwrm_stat_ctx_free_output resp = {};
174 struct bnge_fw_msg fw_msg = {};
175 int rc;
176
177 bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_FREE);
178 req.stat_ctx_id = cpu_to_le32(rdev->stats_ctx.fw_id);
179 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
180 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
181 rc = bnge_send_msg(aux_dev, &fw_msg);
182 if (rc)
183 ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x",
184 rc);
185
186 return rc;
187 }
188
bng_re_stats_ctx_alloc(struct bng_re_dev * rdev)189 static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev)
190 {
191 struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
192 struct bng_re_stats *stats = &rdev->stats_ctx;
193 struct hwrm_stat_ctx_alloc_output resp = {};
194 struct hwrm_stat_ctx_alloc_input req = {};
195 struct bnge_fw_msg fw_msg = {};
196 int rc;
197
198 stats->fw_id = BNGE_INVALID_STATS_CTX_ID;
199
200 bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_ALLOC);
201 req.update_period_ms = cpu_to_le32(1000);
202 req.stats_dma_addr = cpu_to_le64(stats->dma_map);
203 req.stats_dma_length = cpu_to_le16(rdev->chip_ctx->hw_stats_size);
204 req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;
205 bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
206 sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
207 rc = bnge_send_msg(aux_dev, &fw_msg);
208 if (!rc)
209 stats->fw_id = le32_to_cpu(resp.stat_ctx_id);
210 return rc;
211 }
212
bng_re_query_hwrm_version(struct bng_re_dev * rdev)213 static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
214 {
215 struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
216 struct hwrm_ver_get_output ver_get_resp = {};
217 struct hwrm_ver_get_input ver_get_req = {};
218 struct bng_re_chip_ctx *cctx;
219 struct bnge_fw_msg fw_msg = {};
220 int rc;
221
222 bng_re_init_hwrm_hdr((void *)&ver_get_req, HWRM_VER_GET);
223 ver_get_req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
224 ver_get_req.hwrm_intf_min = HWRM_VERSION_MINOR;
225 ver_get_req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
226 bng_re_fill_fw_msg(&fw_msg, (void *)&ver_get_req, sizeof(ver_get_req),
227 (void *)&ver_get_resp, sizeof(ver_get_resp),
228 BNGE_DFLT_HWRM_CMD_TIMEOUT);
229 rc = bnge_send_msg(aux_dev, &fw_msg);
230 if (rc) {
231 ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
232 rc);
233 return rc;
234 }
235
236 cctx = rdev->chip_ctx;
237 cctx->hwrm_intf_ver =
238 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_major) << 48 |
239 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_minor) << 32 |
240 (u64)le16_to_cpu(ver_get_resp.hwrm_intf_build) << 16 |
241 le16_to_cpu(ver_get_resp.hwrm_intf_patch);
242
243 cctx->hwrm_cmd_max_timeout = le16_to_cpu(ver_get_resp.max_req_timeout);
244
245 if (!cctx->hwrm_cmd_max_timeout)
246 cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT;
247
248 return 0;
249 }
250
bng_re_dev_uninit(struct bng_re_dev * rdev)251 static void bng_re_dev_uninit(struct bng_re_dev *rdev)
252 {
253 int rc;
254 bng_re_debugfs_rem_pdev(rdev);
255
256 if (test_and_clear_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
257 rc = bng_re_deinit_rcfw(&rdev->rcfw);
258 if (rc)
259 ibdev_warn(&rdev->ibdev,
260 "Failed to deinitialize RCFW: %#x", rc);
261 bng_re_stats_ctx_free(rdev);
262 bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx);
263 bng_re_disable_rcfw_channel(&rdev->rcfw);
264 bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id,
265 RING_ALLOC_REQ_RING_TYPE_NQ);
266 bng_re_free_rcfw_channel(&rdev->rcfw);
267 }
268
269 kfree(rdev->nqr);
270 rdev->nqr = NULL;
271 bng_re_destroy_chip_ctx(rdev);
272 if (test_and_clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
273 bnge_unregister_dev(rdev->aux_dev);
274 }
275
bng_re_dev_init(struct bng_re_dev * rdev)276 static int bng_re_dev_init(struct bng_re_dev *rdev)
277 {
278 struct bng_re_ring_attr rattr = {};
279 struct bng_re_creq_ctx *creq;
280 u32 db_offt;
281 int vid;
282 u8 type;
283 int rc;
284
285 /* Registered a new RoCE device instance to netdev */
286 rc = bng_re_register_netdev(rdev);
287 if (rc) {
288 ibdev_err(&rdev->ibdev,
289 "Failed to register with netedev: %#x\n", rc);
290 goto reg_netdev_fail;
291 }
292
293 set_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
294
295 if (rdev->aux_dev->auxr_info->msix_requested < BNG_RE_MIN_MSIX) {
296 ibdev_err(&rdev->ibdev,
297 "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n",
298 rdev->aux_dev->auxr_info->msix_requested);
299 rc = -EINVAL;
300 goto msix_ctx_fail;
301 }
302 ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
303 rdev->aux_dev->auxr_info->msix_requested);
304
305 rc = bng_re_setup_chip_ctx(rdev);
306 if (rc) {
307 ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
308 goto msix_ctx_fail;
309 }
310
311 rc = bng_re_query_hwrm_version(rdev);
312 if (rc)
313 goto destroy_chip_ctx;
314
315 rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw);
316 if (rc) {
317 ibdev_err(&rdev->ibdev,
318 "Failed to allocate RCFW Channel: %#x\n", rc);
319 goto destroy_chip_ctx;
320 }
321
322 /* Allocate nq record memory */
323 rdev->nqr = kzalloc_obj(*rdev->nqr);
324 if (!rdev->nqr) {
325 rc = -ENOMEM;
326 goto nq_alloc_fail;
327 }
328
329 rdev->nqr->num_msix = rdev->aux_dev->auxr_info->msix_requested;
330 memcpy(rdev->nqr->msix_entries, rdev->aux_dev->msix_info,
331 sizeof(struct bnge_msix_info) * rdev->nqr->num_msix);
332
333 type = RING_ALLOC_REQ_RING_TYPE_NQ;
334 creq = &rdev->rcfw.creq;
335 rattr.dma_arr = creq->hwq.pbl[BNG_PBL_LVL_0].pg_map_arr;
336 rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
337 rattr.type = type;
338 rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
339 rattr.depth = BNG_FW_CREQE_MAX_CNT - 1;
340 rattr.lrid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].ring_idx;
341 rc = bng_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
342 if (rc) {
343 ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
344 goto free_rcfw;
345 }
346 db_offt = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].db_offset;
347 vid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].vector;
348
349 rc = bng_re_enable_fw_channel(&rdev->rcfw,
350 vid, db_offt);
351 if (rc) {
352 ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n",
353 rc);
354 goto free_ring;
355 }
356
357 rc = bng_re_get_dev_attr(&rdev->rcfw);
358 if (rc)
359 goto disable_rcfw;
360
361 bng_re_debugfs_add_pdev(rdev);
362 rc = bng_re_alloc_stats_ctx_mem(rdev->bng_res.pdev, rdev->chip_ctx,
363 &rdev->stats_ctx);
364 if (rc) {
365 ibdev_err(&rdev->ibdev,
366 "Failed to allocate stats context: %#x\n", rc);
367 goto disable_rcfw;
368 }
369
370 rc = bng_re_stats_ctx_alloc(rdev);
371 if (rc) {
372 ibdev_err(&rdev->ibdev,
373 "Failed to allocate QPLIB context: %#x\n", rc);
374 goto free_stats_ctx;
375 }
376
377 rc = bng_re_init_rcfw(&rdev->rcfw, &rdev->stats_ctx);
378 if (rc) {
379 ibdev_err(&rdev->ibdev,
380 "Failed to initialize RCFW: %#x\n", rc);
381 goto free_sctx;
382 }
383 set_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
384
385 return 0;
386 free_sctx:
387 bng_re_stats_ctx_free(rdev);
388 free_stats_ctx:
389 bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx);
390 disable_rcfw:
391 bng_re_disable_rcfw_channel(&rdev->rcfw);
392 free_ring:
393 bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
394 free_rcfw:
395 kfree(rdev->nqr);
396 nq_alloc_fail:
397 bng_re_free_rcfw_channel(&rdev->rcfw);
398 destroy_chip_ctx:
399 bng_re_destroy_chip_ctx(rdev);
400 msix_ctx_fail:
401 bnge_unregister_dev(rdev->aux_dev);
402 clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
403 reg_netdev_fail:
404 return rc;
405 }
406
bng_re_add_device(struct auxiliary_device * adev)407 static int bng_re_add_device(struct auxiliary_device *adev)
408 {
409 struct bnge_auxr_priv *auxr_priv =
410 container_of(adev, struct bnge_auxr_priv, aux_dev);
411 struct bng_re_en_dev_info *dev_info;
412 struct bng_re_dev *rdev;
413 int rc;
414
415 dev_info = auxiliary_get_drvdata(adev);
416
417 rdev = bng_re_dev_add(adev, auxr_priv->auxr_dev);
418 if (!rdev) {
419 rc = -ENOMEM;
420 goto exit;
421 }
422
423 dev_info->rdev = rdev;
424
425 rc = bng_re_dev_init(rdev);
426 if (rc)
427 goto re_dev_dealloc;
428
429 return 0;
430
431 re_dev_dealloc:
432 ib_dealloc_device(&rdev->ibdev);
433 exit:
434 return rc;
435 }
436
437
bng_re_remove_device(struct bng_re_dev * rdev,struct auxiliary_device * aux_dev)438 static void bng_re_remove_device(struct bng_re_dev *rdev,
439 struct auxiliary_device *aux_dev)
440 {
441 bng_re_dev_uninit(rdev);
442 ib_dealloc_device(&rdev->ibdev);
443 }
444
445
bng_re_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)446 static int bng_re_probe(struct auxiliary_device *adev,
447 const struct auxiliary_device_id *id)
448 {
449 struct bnge_auxr_priv *aux_priv =
450 container_of(adev, struct bnge_auxr_priv, aux_dev);
451 struct bng_re_en_dev_info *en_info;
452 int rc;
453
454 en_info = kzalloc_obj(*en_info);
455 if (!en_info)
456 return -ENOMEM;
457
458 en_info->auxr_dev = aux_priv->auxr_dev;
459
460 auxiliary_set_drvdata(adev, en_info);
461
462 rc = bng_re_add_device(adev);
463 if (rc)
464 kfree(en_info);
465
466 return rc;
467 }
468
bng_re_remove(struct auxiliary_device * adev)469 static void bng_re_remove(struct auxiliary_device *adev)
470 {
471 struct bng_re_en_dev_info *dev_info = auxiliary_get_drvdata(adev);
472 struct bng_re_dev *rdev;
473
474 rdev = dev_info->rdev;
475
476 bng_re_remove_device(rdev, adev);
477 kfree(dev_info);
478 }
479
480 static const struct auxiliary_device_id bng_re_id_table[] = {
481 { .name = BNG_RE_ADEV_NAME ".rdma", },
482 {},
483 };
484
485 MODULE_DEVICE_TABLE(auxiliary, bng_re_id_table);
486
487 static struct auxiliary_driver bng_re_driver = {
488 .name = "rdma",
489 .probe = bng_re_probe,
490 .remove = bng_re_remove,
491 .id_table = bng_re_id_table,
492 };
493
bng_re_mod_init(void)494 static int __init bng_re_mod_init(void)
495 {
496 int rc;
497
498
499 bng_re_register_debugfs();
500
501 rc = auxiliary_driver_register(&bng_re_driver);
502 if (rc) {
503 pr_err("%s: Failed to register auxiliary driver\n",
504 KBUILD_MODNAME);
505 goto unreg_debugfs;
506 }
507 return 0;
508 unreg_debugfs:
509 bng_re_unregister_debugfs();
510 return rc;
511 }
512
bng_re_mod_exit(void)513 static void __exit bng_re_mod_exit(void)
514 {
515 auxiliary_driver_unregister(&bng_re_driver);
516 bng_re_unregister_debugfs();
517 }
518
519 module_init(bng_re_mod_init);
520 module_exit(bng_re_mod_exit);
521