1 /*
2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: Main component of the bnxt_re driver
29 */
30
31 #include <linux/if_ether.h>
32 #include <linux/module.h>
33 #include <linux/errno.h>
34 #include <linux/pci.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/slab.h>
37 #include <linux/sched.h>
38 #include <linux/delay.h>
39 #include <linux/fs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/ib_addr.h>
42 #include <rdma/ib_cache.h>
43 #include <dev/mlx5/port.h>
44 #include <dev/mlx5/vport.h>
45 #include <linux/list.h>
46 #include <rdma/ib_smi.h>
47 #include <rdma/ib_umem.h>
48 #include <linux/in.h>
49 #include <linux/etherdevice.h>
50
51 #include "bnxt_re.h"
52 #include "ib_verbs.h"
53 #include "bnxt_re-abi.h"
54 #include "bnxt.h"
55 #include "bnxt_log.h"
56
57 static char drv_version[] =
58 "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \
59 " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n";
60
61 #define BNXT_RE_DESC "Broadcom NetXtreme RoCE"
62 #define BNXT_ADEV_NAME "if_bnxt"
63
64 MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver");
65 MODULE_LICENSE("Dual BSD/GPL");
66 MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1);
67 MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1);
68 MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1);
69 MODULE_VERSION(bnxt_re, 1);
70
71
72 DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */
73
74 static unsigned int restrict_mrs = 0;
75 module_param(restrict_mrs, uint, 0);
76 MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K");
77
78 unsigned int restrict_stats = 0;
79 module_param(restrict_stats, uint, 0);
80 MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default");
81
82 unsigned int enable_fc = 1;
83 module_param(enable_fc, uint, 0);
84 MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1");
85
86 unsigned int min_tx_depth = 1;
87 module_param(min_tx_depth, uint, 0);
88 MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1");
89
90 static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0};
91 static unsigned int max_msix_vec_argc;
92 module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444);
93 MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64");
94
95 unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD;
96 module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644);
97 MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64");
98
99
100 /* globals */
101 struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list);
102 static int bnxt_re_probe_count;
103
104 DEFINE_MUTEX(bnxt_re_dev_lock);
105 static u32 gmod_exit;
106 static u32 gadd_dev_inprogress;
107
108 static void bnxt_re_task(struct work_struct *work_task);
109 static struct workqueue_struct *bnxt_re_wq;
110 static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev);
111 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
112 u32 *offset);
113 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev);
114 static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev);
115 void _bnxt_re_remove(struct auxiliary_device *adev);
116
117 void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32);
118 u32 readl_fbsd(struct bnxt_softc *bp, u32, u8);
119 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev);
120
bnxt_re_register_netdevice_notifier(struct notifier_block * nb)121 int bnxt_re_register_netdevice_notifier(struct notifier_block *nb)
122 {
123 int rc;
124 rc = register_netdevice_notifier(nb);
125 return rc;
126 }
127
bnxt_re_unregister_netdevice_notifier(struct notifier_block * nb)128 int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb)
129 {
130 int rc;
131 rc = unregister_netdevice_notifier(nb);
132 return rc;
133 }
134
bnxt_re_set_dma_device(struct ib_device * ibdev,struct bnxt_re_dev * rdev)135 void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev)
136 {
137 ibdev->dma_device = &rdev->en_dev->pdev->dev;
138 }
139
readl_fbsd(struct bnxt_softc * bp,u32 reg_off,u8 bar_idx)140 u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)
141 {
142
143 if (bar_idx)
144 return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off);
145 else
146 return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off);
147 }
148
writel_fbsd(struct bnxt_softc * bp,u32 reg_off,u8 bar_idx,u32 val)149 void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val)
150 {
151 if (bar_idx)
152 bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val));
153 else
154 bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val));
155 }
156
bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev * rdev,u32 fifo_occup)157 static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev,
158 u32 fifo_occup)
159 {
160 if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark)
161 rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup;
162
163 if (fifo_occup > 8 * rdev->pacing_algo_th)
164 rdev->dbg_stats->dbq.fifo_occup_slab_4++;
165 else if (fifo_occup > 4 * rdev->pacing_algo_th)
166 rdev->dbg_stats->dbq.fifo_occup_slab_3++;
167 else if (fifo_occup > 2 * rdev->pacing_algo_th)
168 rdev->dbg_stats->dbq.fifo_occup_slab_2++;
169 else if (fifo_occup > rdev->pacing_algo_th)
170 rdev->dbg_stats->dbq.fifo_occup_slab_1++;
171 }
172
bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev * rdev)173 static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev)
174 {
175 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
176
177 if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark)
178 rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing;
179
180 if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing)
181 rdev->dbg_stats->dbq.do_pacing_slab_5++;
182 else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing)
183 rdev->dbg_stats->dbq.do_pacing_slab_4++;
184 else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing)
185 rdev->dbg_stats->dbq.do_pacing_slab_3++;
186 else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing)
187 rdev->dbg_stats->dbq.do_pacing_slab_2++;
188 else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing)
189 rdev->dbg_stats->dbq.do_pacing_slab_1++;
190 }
191
bnxt_re_is_qp1_qp(struct bnxt_re_qp * qp)192 static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp)
193 {
194 return qp->ib_qp.qp_type == IB_QPT_GSI;
195 }
196
bnxt_re_get_qp1_qp(struct bnxt_re_dev * rdev)197 static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev)
198 {
199 struct bnxt_re_qp *qp;
200
201 mutex_lock(&rdev->qp_lock);
202 list_for_each_entry(qp, &rdev->qp_list, list) {
203 if (bnxt_re_is_qp1_qp(qp)) {
204 mutex_unlock(&rdev->qp_lock);
205 return qp;
206 }
207 }
208 mutex_unlock(&rdev->qp_lock);
209 return NULL;
210 }
211
212 /* Set the maximum number of each resource that the driver actually wants
213 * to allocate. This may be up to the maximum number the firmware has
214 * reserved for the function. The driver may choose to allocate fewer
215 * resources than the firmware maximum.
216 */
bnxt_re_limit_pf_res(struct bnxt_re_dev * rdev)217 static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
218 {
219 struct bnxt_qplib_max_res dev_res = {};
220 struct bnxt_qplib_chip_ctx *cctx;
221 struct bnxt_qplib_dev_attr *attr;
222 struct bnxt_qplib_ctx *hctx;
223 int i;
224
225 attr = rdev->dev_attr;
226 hctx = rdev->qplib_res.hctx;
227 cctx = rdev->chip_ctx;
228
229 bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false);
230 if (!_is_chip_gen_p5_p7(cctx)) {
231 hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp);
232 hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr);
233 /* To accommodate 16k MRs and 16k AHs,
234 * driver has to allocate 32k backing store memory
235 */
236 hctx->mrw_ctx.max *= 2;
237 hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq);
238 hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq);
239 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
240 hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i];
241 } else {
242 hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp;
243 hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr;
244 hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq;
245 hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq;
246 }
247 }
248
bnxt_re_limit_vf_res(struct bnxt_re_dev * rdev,struct bnxt_qplib_vf_res * vf_res,u32 num_vf)249 static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev,
250 struct bnxt_qplib_vf_res *vf_res,
251 u32 num_vf)
252 {
253 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx;
254 struct bnxt_qplib_max_res dev_res = {};
255
256 bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true);
257 vf_res->max_qp = dev_res.max_qp / num_vf;
258 vf_res->max_srq = dev_res.max_srq / num_vf;
259 vf_res->max_cq = dev_res.max_cq / num_vf;
260 /*
261 * MR and AH shares the same backing store, the value specified
262 * for max_mrw is split into half by the FW for MR and AH
263 */
264 vf_res->max_mrw = dev_res.max_mr * 2 / num_vf;
265 vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF;
266 }
267
bnxt_re_set_resource_limits(struct bnxt_re_dev * rdev)268 static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
269 {
270 struct bnxt_qplib_ctx *hctx;
271
272 hctx = rdev->qplib_res.hctx;
273 memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
274 bnxt_re_limit_pf_res(rdev);
275
276 if (rdev->num_vfs)
277 bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs);
278 }
279
bnxt_re_dettach_irq(struct bnxt_re_dev * rdev)280 static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev)
281 {
282 struct bnxt_qplib_rcfw *rcfw = NULL;
283 struct bnxt_qplib_nq *nq;
284 int indx;
285
286 rcfw = &rdev->rcfw;
287 for (indx = 0; indx < rdev->nqr.max_init; indx++) {
288 nq = &rdev->nqr.nq[indx];
289 mutex_lock(&nq->lock);
290 bnxt_qplib_nq_stop_irq(nq, false);
291 mutex_unlock(&nq->lock);
292 }
293
294 bnxt_qplib_rcfw_stop_irq(rcfw, false);
295 }
296
bnxt_re_detach_err_device(struct bnxt_re_dev * rdev)297 static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev)
298 {
299 /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */
300 bnxt_re_dettach_irq(rdev);
301
302 /* Set the state as detached to prevent sending any more commands */
303 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
304 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
305 wake_up_all(&rdev->rcfw.cmdq.waitq);
306 }
307
308 #define MAX_DSCP_PRI_TUPLE 64
309
310 struct bnxt_re_dcb_work {
311 struct work_struct work;
312 struct bnxt_re_dev *rdev;
313 struct hwrm_async_event_cmpl cmpl;
314 };
315
bnxt_re_init_dcb_wq(struct bnxt_re_dev * rdev)316 static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
317 {
318 rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq");
319 }
320
bnxt_re_uninit_dcb_wq(struct bnxt_re_dev * rdev)321 static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
322 {
323 if (!rdev->dcb_wq)
324 return;
325 flush_workqueue(rdev->dcb_wq);
326 destroy_workqueue(rdev->dcb_wq);
327 rdev->dcb_wq = NULL;
328 }
329
bnxt_re_init_aer_wq(struct bnxt_re_dev * rdev)330 static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev)
331 {
332 rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq");
333 }
334
bnxt_re_uninit_aer_wq(struct bnxt_re_dev * rdev)335 static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev)
336 {
337 if (!rdev->aer_wq)
338 return;
339 flush_workqueue(rdev->aer_wq);
340 destroy_workqueue(rdev->aer_wq);
341 rdev->aer_wq = NULL;
342 }
343
bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev * rdev)344 static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev)
345 {
346 struct bnxt_re_qp *qp;
347
348 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
349 return 0;
350
351 qp = bnxt_re_get_qp1_qp(rdev);
352 if (!qp)
353 return 0;
354
355 qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP;
356 qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp;
357
358 return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
359 }
360
bnxt_re_reconfigure_dscp(struct bnxt_re_dev * rdev)361 static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev)
362 {
363 struct bnxt_qplib_cc_param *cc_param;
364 struct bnxt_re_tc_rec *tc_rec;
365 bool update_cc = false;
366 u8 dscp_user;
367 int rc;
368
369 cc_param = &rdev->cc_param;
370 tc_rec = &rdev->tc_rec[0];
371
372 if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user))
373 return;
374
375 if (cc_param->cnp_dscp_user) {
376 dscp_user = (cc_param->cnp_dscp_user & 0x3f);
377 if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) &&
378 (cc_param->alt_tos_dscp != dscp_user)) {
379 cc_param->alt_tos_dscp = dscp_user;
380 cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP;
381 update_cc = true;
382 }
383 }
384
385 if (cc_param->roce_dscp_user) {
386 dscp_user = (cc_param->roce_dscp_user & 0x3f);
387 if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) &&
388 (cc_param->tos_dscp != dscp_user)) {
389 cc_param->tos_dscp = dscp_user;
390 cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
391 update_cc = true;
392 }
393 }
394
395 if (update_cc) {
396 rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param);
397 if (rc)
398 dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n");
399 }
400 }
401
bnxt_re_dcb_wq_task(struct work_struct * work)402 static void bnxt_re_dcb_wq_task(struct work_struct *work)
403 {
404 struct bnxt_qplib_cc_param *cc_param;
405 struct bnxt_re_tc_rec *tc_rec;
406 struct bnxt_re_dev *rdev;
407 struct bnxt_re_dcb_work *dcb_work =
408 container_of(work, struct bnxt_re_dcb_work, work);
409 int rc;
410
411 rdev = dcb_work->rdev;
412 if (!rdev)
413 goto exit;
414
415 mutex_lock(&rdev->cc_lock);
416
417 cc_param = &rdev->cc_param;
418 rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param);
419 if (rc) {
420 dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc);
421 goto fail;
422 }
423 tc_rec = &rdev->tc_rec[0];
424 /*
425 * Upon the receival of DCB Async event:
426 * If roce_dscp or cnp_dscp or both (which user configured using configfs)
427 * is in the list, re-program the value using modify_roce_cc command
428 */
429 bnxt_re_reconfigure_dscp(rdev);
430
431 cc_param->roce_pri = tc_rec->roce_prio;
432 if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) {
433 cc_param->qp1_tos_dscp = cc_param->tos_dscp;
434 rc = bnxt_re_update_qp1_tos_dscp(rdev);
435 if (rc) {
436 dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d",
437 __func__, rc);
438 goto fail;
439 }
440 }
441
442 fail:
443 mutex_unlock(&rdev->cc_lock);
444 exit:
445 kfree(dcb_work);
446 }
447
bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev * rdev)448 static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev)
449 {
450 struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0};
451 struct hwrm_func_dbr_pacing_broadcast_event_input req = {0};
452 struct bnxt_en_dev *en_dev = rdev->en_dev;
453 struct bnxt_fw_msg fw_msg;
454 int rc;
455
456 memset(&fw_msg, 0, sizeof(fw_msg));
457 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
458 HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1);
459 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
460 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
461 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
462 if (rc) {
463 dev_dbg(rdev_to_dev(rdev),
464 "Failed to send dbr pacing broadcast event rc:%d", rc);
465 return rc;
466 }
467 return 0;
468 }
469
bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev * rdev)470 static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev)
471 {
472 struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0};
473 struct hwrm_func_dbr_pacing_nqlist_query_input req = {0};
474 struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list;
475 struct bnxt_en_dev *en_dev = rdev->en_dev;
476 bool primary_found = false;
477 struct bnxt_fw_msg fw_msg;
478 struct bnxt_qplib_nq *nq;
479 int rc, i, j = 1;
480 u16 *nql_ptr;
481
482 nq = &rdev->nqr.nq[0];
483
484 memset(&fw_msg, 0, sizeof(fw_msg));
485 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
486 HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1);
487 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
488 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
489 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
490 if (rc) {
491 dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc);
492 return rc;
493 }
494 nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs);
495 nql_ptr = &resp.nq_ring_id0;
496 /* populate the nq_list of the primary function with list received
497 * from FW. Fill the NQ IDs of secondary functions from index 1 to
498 * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the
499 * nq_id of the primary pf
500 */
501 for (i = 0; i < nq_list->num_nql_entries; i++) {
502 u16 nq_id = *nql_ptr;
503
504 dev_dbg(rdev_to_dev(rdev),
505 "nq_list->nq_id[%d] = %d\n", i, nq_id);
506 if (nq_id != nq->ring_id) {
507 nq_list->nq_id[j] = nq_id;
508 j++;
509 } else {
510 primary_found = true;
511 nq_list->nq_id[0] = nq->ring_id;
512 }
513 nql_ptr++;
514 }
515 if (primary_found)
516 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1);
517
518 return 0;
519 }
520
__wait_for_fifo_occupancy_below_th(struct bnxt_re_dev * rdev)521 static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev)
522 {
523 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
524 u32 read_val, fifo_occup;
525 bool first_read = true;
526
527 /* loop shouldn't run infintely as the occupancy usually goes
528 * below pacing algo threshold as soon as pacing kicks in.
529 */
530 while (1) {
531 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0);
532 fifo_occup = pacing_data->fifo_max_depth -
533 ((read_val & pacing_data->fifo_room_mask) >>
534 pacing_data->fifo_room_shift);
535 /* Fifo occupancy cannot be greater the MAX FIFO depth */
536 if (fifo_occup > pacing_data->fifo_max_depth)
537 break;
538
539 if (first_read) {
540 bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup);
541 first_read = false;
542 }
543 if (fifo_occup < pacing_data->pacing_th)
544 break;
545 }
546 }
547
bnxt_re_set_default_pacing_data(struct bnxt_re_dev * rdev)548 static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev)
549 {
550 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
551
552 pacing_data->do_pacing = rdev->dbr_def_do_pacing;
553 pacing_data->pacing_th = rdev->pacing_algo_th;
554 pacing_data->alarm_th =
555 pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
556 }
557
558 #define CAG_RING_MASK 0x7FF
559 #define CAG_RING_SHIFT 17
560 #define WATERMARK_MASK 0xFFF
561 #define WATERMARK_SHIFT 0
562
bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev * rdev)563 static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev)
564 {
565 u32 read_val;
566 int j;
567
568 for (j = 0; j < 10; j++) {
569 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0);
570 dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n",
571 read_val);
572 if (!read_val)
573 return true;
574 }
575 return false;
576 }
577
bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev * rdev,u16 nq_id,u32 throttle)578 int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle)
579 {
580 u32 cag_ring_water_mark = 0, read_val;
581 u32 throttle_val;
582
583 /* Convert throttle percentage to value */
584 throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100;
585
586 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
587 cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT |
588 (throttle_val & WATERMARK_MASK);
589 writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark);
590 read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0);
591 dev_dbg(rdev_to_dev(rdev),
592 "%s: dbr_throttling_reg_off read_val = 0x%x\n",
593 __func__, read_val);
594 if (read_val != cag_ring_water_mark) {
595 dev_dbg(rdev_to_dev(rdev),
596 "nq_id = %d write_val=0x%x read_val=0x%x\n",
597 nq_id, cag_ring_water_mark, read_val);
598 return 1;
599 }
600 }
601 writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1);
602 return 0;
603 }
604
bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev * rdev)605 static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev)
606 {
607 struct bnxt_dbq_nq_list *nq_list;
608 struct bnxt_qplib_nq *nq;
609 int i;
610
611 nq_list = &rdev->nq_list;
612 /* Run a loop for other Active functions if this is primary function */
613 if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
614 dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n",
615 __func__, nq_list->num_nql_entries);
616 nq = &rdev->nqr.nq[0];
617 for (i = nq_list->num_nql_entries - 1; i > 0; i--) {
618 u16 nq_id = nq_list->nq_id[i];
619 if (nq)
620 dev_dbg(rdev_to_dev(rdev),
621 "%s: nq_id = %d cur_fn_ring_id = %d\n",
622 __func__, nq_id, nq->ring_id);
623 if (bnxt_re_set_dbq_throttling_reg
624 (rdev, nq_id, 0))
625 break;
626 bnxt_re_check_if_dbq_intr_triggered(rdev);
627 }
628 }
629 }
630
bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev * rdev)631 static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev)
632 {
633 struct bnxt_qplib_nq *nq;
634 int rc = 0;
635
636 nq = &rdev->nqr.nq[0];
637
638 /* Query the NQ list*/
639 rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev);
640 if (rc) {
641 dev_err(rdev_to_dev(rdev),
642 "Failed to Query NQ list rc= %d", rc);
643 return;
644 }
645 /*Configure GRC access for Throttling and aeq_arm register */
646 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0,
647 rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK);
648
649 rdev->dbr_throttling_reg_off =
650 (rdev->chip_ctx->dbr_throttling_reg &
651 BNXT_GRC_OFFSET_MASK) + 0x8000;
652 rdev->dbr_aeq_arm_reg_off =
653 (rdev->chip_ctx->dbr_aeq_arm_reg &
654 BNXT_GRC_OFFSET_MASK) + 0x8000;
655
656 bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark);
657 }
658
bnxt_re_dbq_wq_task(struct work_struct * work)659 static void bnxt_re_dbq_wq_task(struct work_struct *work)
660 {
661 struct bnxt_re_dbq_work *dbq_work =
662 container_of(work, struct bnxt_re_dbq_work, work);
663 struct bnxt_re_dev *rdev;
664
665 rdev = dbq_work->rdev;
666
667 if (!rdev)
668 goto exit;
669 switch (dbq_work->event) {
670 case BNXT_RE_DBQ_EVENT_SCHED:
671 dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n",
672 __func__);
673 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx))
674 bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev);
675 else
676 bnxt_re_pacing_alert(rdev);
677 break;
678 case BNXT_RE_DBR_PACING_EVENT:
679 dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n",
680 __func__);
681 if (_is_chip_p7(rdev->chip_ctx))
682 bnxt_re_pacing_alert(rdev);
683 else if (!rdev->chip_ctx->modes.dbr_pacing_v0)
684 bnxt_re_hwrm_dbr_pacing_qcfg(rdev);
685 break;
686 case BNXT_RE_DBR_NQ_PACING_NOTIFICATION:
687 bnxt_re_handle_dbr_nq_pacing_notification(rdev);
688 /* Issue a broadcast event to notify other functions
689 * that primary changed
690 */
691 bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev);
692 break;
693 }
694 exit:
695 kfree(dbq_work);
696 }
697
bnxt_re_async_notifier(void * handle,struct hwrm_async_event_cmpl * cmpl)698 static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl)
699 {
700 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
701 struct bnxt_re_dcb_work *dcb_work;
702 struct bnxt_re_dbq_work *dbq_work;
703 struct bnxt_re_dev *rdev;
704 u16 event_id;
705 u32 data1;
706 u32 data2 = 0;
707
708 if (!cmpl) {
709 pr_err("Async event, bad completion\n");
710 return;
711 }
712
713 if (!en_info || !en_info->en_dev) {
714 pr_err("Async event, bad en_info or en_dev\n");
715 return;
716 }
717 rdev = en_info->rdev;
718
719 event_id = le16_to_cpu(cmpl->event_id);
720 data1 = le32_to_cpu(cmpl->event_data1);
721 data2 = le32_to_cpu(cmpl->event_data2);
722
723 if (!rdev || !rdev_to_dev(rdev)) {
724 dev_dbg(NULL, "Async event, bad rdev or netdev\n");
725 return;
726 }
727
728 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
729 !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
730 dev_dbg(NULL, "Async event, device already detached\n");
731 return;
732 }
733 if (data2 >= 0)
734 dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d",
735 event_id, data1, data2);
736
737 switch (event_id) {
738 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:
739 /* Not handling the event in older FWs */
740 if (!is_qport_service_type_supported(rdev))
741 break;
742 if (!rdev->dcb_wq)
743 break;
744 dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);
745 if (!dcb_work)
746 break;
747
748 dcb_work->rdev = rdev;
749 memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl));
750 INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task);
751 queue_work(rdev->dcb_wq, &dcb_work->work);
752 break;
753 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
754 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
755 /* Set rcfw flag to control commands send to Bono */
756 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
757 /* Set bnxt_re flag to control commands send via L2 driver */
758 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
759 wake_up_all(&rdev->rcfw.cmdq.waitq);
760 }
761 break;
762 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD:
763 if (!rdev->dbr_pacing)
764 break;
765 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
766 if (!dbq_work)
767 goto unlock;
768 dbq_work->rdev = rdev;
769 dbq_work->event = BNXT_RE_DBR_PACING_EVENT;
770 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
771 queue_work(rdev->dbq_wq, &dbq_work->work);
772 rdev->dbr_sw_stats->dbq_int_recv++;
773 break;
774 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE:
775 if (!rdev->dbr_pacing)
776 break;
777
778 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
779 if (!dbq_work)
780 goto unlock;
781 dbq_work->rdev = rdev;
782 dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION;
783 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
784 queue_work(rdev->dbq_wq, &dbq_work->work);
785 break;
786
787 default:
788 break;
789 }
790 unlock:
791 return;
792 }
793
bnxt_re_db_fifo_check(struct work_struct * work)794 static void bnxt_re_db_fifo_check(struct work_struct *work)
795 {
796 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
797 dbq_fifo_check_work);
798 struct bnxt_qplib_db_pacing_data *pacing_data;
799 u32 pacing_save;
800
801 if (!mutex_trylock(&rdev->dbq_lock))
802 return;
803 pacing_data = rdev->qplib_res.pacing_data;
804 pacing_save = rdev->do_pacing_save;
805 __wait_for_fifo_occupancy_below_th(rdev);
806 cancel_delayed_work_sync(&rdev->dbq_pacing_work);
807 if (rdev->dbr_recovery_on)
808 goto recovery_on;
809 if (pacing_save > rdev->dbr_def_do_pacing) {
810 /* Double the do_pacing value during the congestion */
811 pacing_save = pacing_save << 1;
812 } else {
813 /*
814 * when a new congestion is detected increase the do_pacing
815 * by 8 times. And also increase the pacing_th by 4 times. The
816 * reason to increase pacing_th is to give more space for the
817 * queue to oscillate down without getting empty, but also more
818 * room for the queue to increase without causing another alarm.
819 */
820 pacing_save = pacing_save << 3;
821 pacing_data->pacing_th = rdev->pacing_algo_th * 4;
822 }
823
824 if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING)
825 pacing_save = BNXT_RE_MAX_DBR_DO_PACING;
826
827 pacing_data->do_pacing = pacing_save;
828 rdev->do_pacing_save = pacing_data->do_pacing;
829 pacing_data->alarm_th =
830 pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
831 recovery_on:
832 schedule_delayed_work(&rdev->dbq_pacing_work,
833 msecs_to_jiffies(rdev->dbq_pacing_time));
834 rdev->dbr_sw_stats->dbq_pacing_alerts++;
835 mutex_unlock(&rdev->dbq_lock);
836 }
837
bnxt_re_pacing_timer_exp(struct work_struct * work)838 static void bnxt_re_pacing_timer_exp(struct work_struct *work)
839 {
840 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
841 dbq_pacing_work.work);
842 struct bnxt_qplib_db_pacing_data *pacing_data;
843 u32 read_val, fifo_occup;
844 struct bnxt_qplib_nq *nq;
845
846 if (!mutex_trylock(&rdev->dbq_lock))
847 return;
848
849 pacing_data = rdev->qplib_res.pacing_data;
850 read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0);
851 fifo_occup = pacing_data->fifo_max_depth -
852 ((read_val & pacing_data->fifo_room_mask) >>
853 pacing_data->fifo_room_shift);
854
855 if (fifo_occup > pacing_data->pacing_th)
856 goto restart_timer;
857
858 /*
859 * Instead of immediately going back to the default do_pacing
860 * reduce it by 1/8 times and restart the timer.
861 */
862 pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3);
863 pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing);
864 /*
865 * If the fifo_occup is less than the interrupt enable threshold
866 * enable the interrupt on the primary PF.
867 */
868 if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) {
869 if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
870 if (!rdev->chip_ctx->modes.dbr_pacing_v0) {
871 nq = &rdev->nqr.nq[0];
872 bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id,
873 rdev->dbq_watermark);
874 rdev->dbr_sw_stats->dbq_int_en++;
875 rdev->dbq_int_disable = false;
876 }
877 }
878 }
879 if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) {
880 bnxt_re_set_default_pacing_data(rdev);
881 rdev->dbr_sw_stats->dbq_pacing_complete++;
882 goto dbq_unlock;
883 }
884 restart_timer:
885 schedule_delayed_work(&rdev->dbq_pacing_work,
886 msecs_to_jiffies(rdev->dbq_pacing_time));
887 bnxt_re_update_do_pacing_slabs(rdev);
888 rdev->dbr_sw_stats->dbq_pacing_resched++;
889 dbq_unlock:
890 rdev->do_pacing_save = pacing_data->do_pacing;
891 mutex_unlock(&rdev->dbq_lock);
892 }
893
bnxt_re_pacing_alert(struct bnxt_re_dev * rdev)894 void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev)
895 {
896 struct bnxt_qplib_db_pacing_data *pacing_data;
897
898 if (!rdev->dbr_pacing)
899 return;
900 mutex_lock(&rdev->dbq_lock);
901 pacing_data = rdev->qplib_res.pacing_data;
902
903 /*
904 * Increase the alarm_th to max so that other user lib instances do not
905 * keep alerting the driver.
906 */
907 pacing_data->alarm_th = pacing_data->fifo_max_depth;
908 pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING;
909 cancel_work_sync(&rdev->dbq_fifo_check_work);
910 schedule_work(&rdev->dbq_fifo_check_work);
911 mutex_unlock(&rdev->dbq_lock);
912 }
913
bnxt_re_schedule_dbq_event(struct bnxt_qplib_res * res)914 void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res)
915 {
916 struct bnxt_re_dbq_work *dbq_work;
917 struct bnxt_re_dev *rdev;
918
919 rdev = container_of(res, struct bnxt_re_dev, qplib_res);
920
921 atomic_set(&rdev->dbq_intr_running, 1);
922
923 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
924 goto exit;
925 /* Run the loop to send dbq event to other functions
926 * for newer FW
927 */
928 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
929 !rdev->chip_ctx->modes.dbr_pacing_v0)
930 bnxt_re_set_dbq_throttling_for_non_primary(rdev);
931
932 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
933 if (!dbq_work)
934 goto exit;
935 dbq_work->rdev = rdev;
936 dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED;
937 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
938 queue_work(rdev->dbq_wq, &dbq_work->work);
939 rdev->dbr_sw_stats->dbq_int_recv++;
940 rdev->dbq_int_disable = true;
941 exit:
942 atomic_set(&rdev->dbq_intr_running, 0);
943 }
944
bnxt_re_free_msix(struct bnxt_re_dev * rdev)945 static void bnxt_re_free_msix(struct bnxt_re_dev *rdev)
946 {
947 struct bnxt_en_dev *en_dev = rdev->en_dev;
948 int rc;
949
950 rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
951 if (rc)
952 dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x",
953 rdev->netdev, rc);
954 }
955
bnxt_re_request_msix(struct bnxt_re_dev * rdev)956 static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
957 {
958 struct bnxt_en_dev *en_dev = rdev->en_dev;
959 int rc = 0, num_msix_want, num_msix_got;
960 struct bnxt_msix_entry *entry;
961
962 /*
963 * Request MSIx based on the function type. This is
964 * a temporory solution to enable max VFs when NPAR is
965 * enabled.
966 * TODO - change the scheme with an adapter specific check
967 * as the latest adapters can support more NQs. For now
968 * this change satisfy all adapter versions.
969 */
970
971 if (rdev->is_virtfn)
972 num_msix_want = BNXT_RE_MAX_MSIX_VF;
973 else if (BNXT_EN_NPAR(en_dev))
974 num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF;
975 else if (_is_chip_gen_p5_p7(rdev->chip_ctx))
976 num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF;
977 else
978 num_msix_want = BNXT_RE_MAX_MSIX_PF;
979
980 /*
981 * Since MSIX vectors are used for both NQs and CREQ, we should try to
982 * allocate num_online_cpus + 1 by taking into account the CREQ. This
983 * leaves the number of MSIX vectors for NQs match the number of CPUs
984 * and allows the system to be fully utilized
985 */
986 num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1);
987 num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX);
988 num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX);
989
990 entry = rdev->nqr.msix_entries;
991
992 num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
993 entry, num_msix_want);
994 if (num_msix_got < BNXT_RE_MIN_MSIX) {
995 rc = -EINVAL;
996 goto done;
997 }
998 if (num_msix_got != num_msix_want)
999 dev_warn(rdev_to_dev(rdev),
1000 "bnxt_request_msix: wanted %d vectors, got %d\n",
1001 num_msix_want, num_msix_got);
1002
1003 rdev->nqr.num_msix = num_msix_got;
1004 return 0;
1005 done:
1006 if (num_msix_got)
1007 bnxt_re_free_msix(rdev);
1008 return rc;
1009 }
1010
__wait_for_ib_unregister(struct bnxt_re_dev * rdev,struct bnxt_re_en_dev_info * en_info)1011 static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev,
1012 struct bnxt_re_en_dev_info *en_info)
1013 {
1014 u64 timeout = 0;
1015 u32 cur_prod = 0, cur_cons = 0;
1016 int retry = 0, rc = 0, ret = 0;
1017
1018 cur_prod = rdev->rcfw.cmdq.hwq.prod;
1019 cur_cons = rdev->rcfw.cmdq.hwq.cons;
1020 timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS);
1021 retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY;
1022 /* During module exit, increase timeout ten-fold to 100 mins to wait
1023 * as long as possible for ib_unregister() to complete
1024 */
1025 if (rdev->mod_exit)
1026 retry *= 10;
1027 do {
1028 /*
1029 * Since the caller of this function invokes with bnxt_re_mutex held,
1030 * release it to avoid holding a lock while in wait / sleep mode.
1031 */
1032 mutex_unlock(&bnxt_re_mutex);
1033 rc = wait_event_timeout(en_info->waitq,
1034 en_info->ib_uninit_done,
1035 timeout);
1036 mutex_lock(&bnxt_re_mutex);
1037
1038 if (!bnxt_re_is_rdev_valid(rdev))
1039 break;
1040
1041 if (rc)
1042 break;
1043
1044 if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) {
1045 /* No need to check for cmdq stall during module exit,
1046 * wait for ib unregister to complete
1047 */
1048 if (!rdev->mod_exit)
1049 ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons);
1050 if (ret || en_info->ib_uninit_done)
1051 break;
1052 }
1053 } while (retry--);
1054
1055 return rc;
1056 }
1057
bnxt_re_handle_start(struct auxiliary_device * adev)1058 static int bnxt_re_handle_start(struct auxiliary_device *adev)
1059 {
1060 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
1061 struct bnxt_re_dev *rdev = NULL;
1062 struct ifnet *real_dev;
1063 struct bnxt_en_dev *en_dev;
1064 struct ifnet *netdev;
1065 int rc = 0;
1066
1067 if (!en_info || !en_info->en_dev) {
1068 pr_err("Start, bad en_info or en_dev\n");
1069 return -EINVAL;
1070 }
1071 netdev = en_info->en_dev->net;
1072 if (en_info->rdev) {
1073 dev_info(rdev_to_dev(en_info->rdev),
1074 "%s: Device is already added adev %p rdev: %p\n",
1075 __func__, adev, en_info->rdev);
1076 return 0;
1077 }
1078
1079 en_dev = en_info->en_dev;
1080 real_dev = rdma_vlan_dev_real_dev(netdev);
1081 if (!real_dev)
1082 real_dev = netdev;
1083 rc = bnxt_re_add_device(&rdev, real_dev,
1084 en_info->gsi_mode,
1085 BNXT_RE_POST_RECOVERY_INIT,
1086 en_info->num_msix_requested, adev);
1087 if (rc) {
1088 /* Add device failed. Unregister the device.
1089 * This has to be done explicitly as
1090 * bnxt_re_stop would not have unregistered
1091 */
1092 rtnl_lock();
1093 en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP);
1094 rtnl_unlock();
1095 mutex_lock(&bnxt_re_dev_lock);
1096 gadd_dev_inprogress--;
1097 mutex_unlock(&bnxt_re_dev_lock);
1098 return rc;
1099 }
1100 rdev->adev = adev;
1101 rtnl_lock();
1102 bnxt_re_get_link_speed(rdev);
1103 rtnl_unlock();
1104 rc = bnxt_re_ib_init(rdev);
1105 if (rc) {
1106 dev_err(rdev_to_dev(rdev), "Failed ib_init\n");
1107 return rc;
1108 }
1109 bnxt_re_ib_init_2(rdev);
1110
1111 return rc;
1112 }
1113
bnxt_re_stop(void * handle)1114 static void bnxt_re_stop(void *handle)
1115 {
1116 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
1117 struct ifnet *netdev;
1118 struct bnxt_re_dev *rdev;
1119 struct bnxt_en_dev *en_dev;
1120 int rc = 0;
1121
1122 rtnl_unlock();
1123 mutex_lock(&bnxt_re_mutex);
1124 if (!en_info || !en_info->en_dev) {
1125 pr_err("Stop, bad en_info or en_dev\n");
1126 goto exit;
1127 }
1128 netdev = en_info->en_dev->net;
1129 rdev = en_info->rdev;
1130 if (!rdev)
1131 goto exit;
1132
1133 if (!bnxt_re_is_rdev_valid(rdev))
1134 goto exit;
1135
1136 /*
1137 * Check if fw has undergone reset or is in a fatal condition.
1138 * If so, set flags so that no further commands are sent down to FW
1139 */
1140 en_dev = rdev->en_dev;
1141 if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND ||
1142 en_dev->en_state & BNXT_STATE_FW_RESET_DET) {
1143 /* Set rcfw flag to control commands send to Bono */
1144 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
1145 /* Set bnxt_re flag to control commands send via L2 driver */
1146 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
1147 wake_up_all(&rdev->rcfw.cmdq.waitq);
1148 }
1149
1150 if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags))
1151 goto exit;
1152 set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags);
1153
1154 en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
1155 en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
1156 en_info->num_msix_requested = rdev->num_msix_requested;
1157 en_info->ib_uninit_done = false;
1158
1159 if (rdev->dbr_pacing)
1160 bnxt_re_set_pacing_dev_state(rdev);
1161
1162 dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop."
1163 "Attempting to stop and Dispatching event "
1164 "to inform the stack\n", __func__);
1165 init_waitqueue_head(&en_info->waitq);
1166 /* Schedule a work item to handle IB UNINIT for recovery */
1167 bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER,
1168 NULL, netdev, rdev->adev);
1169 rc = __wait_for_ib_unregister(rdev, en_info);
1170 if (!bnxt_re_is_rdev_valid(rdev))
1171 goto exit;
1172 if (!rc) {
1173 dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n",
1174 __func__);
1175 bnxt_re_detach_err_device(rdev);
1176 goto exit;
1177 }
1178 bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev);
1179 exit:
1180 mutex_unlock(&bnxt_re_mutex);
1181 /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */
1182 rtnl_lock();
1183
1184 return;
1185 }
1186
bnxt_re_start(void * handle)1187 static void bnxt_re_start(void *handle)
1188 {
1189 rtnl_unlock();
1190 mutex_lock(&bnxt_re_mutex);
1191 if (bnxt_re_handle_start((struct auxiliary_device *)handle))
1192 pr_err("Failed to start RoCE device");
1193 mutex_unlock(&bnxt_re_mutex);
1194 /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */
1195 rtnl_lock();
1196 return;
1197 }
1198
bnxt_re_shutdown(void * p)1199 static void bnxt_re_shutdown(void *p)
1200 {
1201 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p);
1202 struct bnxt_re_dev *rdev;
1203
1204 if (!en_info) {
1205 pr_err("Shutdown, bad en_info\n");
1206 return;
1207 }
1208 rtnl_unlock();
1209 mutex_lock(&bnxt_re_mutex);
1210 rdev = en_info->rdev;
1211 if (!rdev || !bnxt_re_is_rdev_valid(rdev))
1212 goto exit;
1213
1214 /* rtnl_lock held by L2 before coming here */
1215 bnxt_re_stopqps_and_ib_uninit(rdev);
1216 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev);
1217 exit:
1218 mutex_unlock(&bnxt_re_mutex);
1219 rtnl_lock();
1220 return;
1221 }
1222
bnxt_re_stop_irq(void * handle)1223 static void bnxt_re_stop_irq(void *handle)
1224 {
1225 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
1226 struct bnxt_qplib_rcfw *rcfw = NULL;
1227 struct bnxt_re_dev *rdev;
1228 struct bnxt_qplib_nq *nq;
1229 int indx;
1230
1231 if (!en_info) {
1232 pr_err("Stop irq, bad en_info\n");
1233 return;
1234 }
1235 rdev = en_info->rdev;
1236
1237 if (!rdev)
1238 return;
1239
1240 rcfw = &rdev->rcfw;
1241 for (indx = 0; indx < rdev->nqr.max_init; indx++) {
1242 nq = &rdev->nqr.nq[indx];
1243 mutex_lock(&nq->lock);
1244 bnxt_qplib_nq_stop_irq(nq, false);
1245 mutex_unlock(&nq->lock);
1246 }
1247
1248 if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags))
1249 bnxt_qplib_rcfw_stop_irq(rcfw, false);
1250 }
1251
bnxt_re_start_irq(void * handle,struct bnxt_msix_entry * ent)1252 static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
1253 {
1254 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
1255 struct bnxt_msix_entry *msix_ent = NULL;
1256 struct bnxt_qplib_rcfw *rcfw = NULL;
1257 struct bnxt_re_dev *rdev;
1258 struct bnxt_qplib_nq *nq;
1259 int indx, rc, vec;
1260
1261 if (!en_info) {
1262 pr_err("Start irq, bad en_info\n");
1263 return;
1264 }
1265 rdev = en_info->rdev;
1266 if (!rdev)
1267 return;
1268 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
1269 return;
1270 msix_ent = rdev->nqr.msix_entries;
1271 rcfw = &rdev->rcfw;
1272
1273 if (!ent) {
1274 /* Not setting the f/w timeout bit in rcfw.
1275 * During the driver unload the first command
1276 * to f/w will timeout and that will set the
1277 * timeout bit.
1278 */
1279 dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");
1280 return;
1281 }
1282
1283 /* Vectors may change after restart, so update with new vectors
1284 * in device structure.
1285 */
1286 for (indx = 0; indx < rdev->nqr.num_msix; indx++)
1287 rdev->nqr.msix_entries[indx].vector = ent[indx].vector;
1288
1289 if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) {
1290 rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
1291 false);
1292 if (rc) {
1293 dev_warn(rdev_to_dev(rdev),
1294 "Failed to reinit CREQ\n");
1295 return;
1296 }
1297 }
1298 for (indx = 0 ; indx < rdev->nqr.max_init; indx++) {
1299 nq = &rdev->nqr.nq[indx];
1300 vec = indx + 1;
1301 rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector,
1302 false);
1303 if (rc) {
1304 dev_warn(rdev_to_dev(rdev),
1305 "Failed to reinit NQ index %d\n", indx);
1306 return;
1307 }
1308 }
1309 }
1310
1311 /*
1312 * bnxt_re_create_snapdump_logs - Collect required debug data for snapdump.
1313 * @rdev - rdma device instance
1314 *
1315 * This function will use bnxt_ulp_log_live API and dump all
1316 * the required information for debugging.
1317 *
1318 * Returns: Nothing
1319 */
1320 static void
bnxt_re_create_snapdump_logs(struct bnxt_re_dev * rdev)1321 bnxt_re_create_snapdump_logs(struct bnxt_re_dev *rdev)
1322 {
1323 struct bnxt_re_ext_rstat *ext_s;
1324
1325 ext_s = &rdev->stats.dstat.ext_rstat[0];
1326
1327 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1328 "tx_atomic_req: %llu", ext_s->tx.atomic_req);
1329 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1330 "rx_atomic_req: %llu", ext_s->rx.atomic_req);
1331 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1332 "tx_read_req: %llu\n", ext_s->tx.read_req);
1333 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1334 "tx_read_resp: %llu\n", ext_s->tx.read_resp);
1335 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1336 "rx_read_req: %llu\n", ext_s->rx.read_req);
1337 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1338 "rx_read_resp: %llu\n", ext_s->rx.read_resp);
1339 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1340 "tx_write_req: %llu\n", ext_s->tx.write_req);
1341 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1342 "rx_write_req: %llu\n", ext_s->rx.write_req);
1343 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1344 "tx_send_req: %llu\n", ext_s->tx.send_req);
1345 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1346 "rx_send_req: %llu\n", ext_s->rx.send_req);
1347 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1348 "rx_good_pkts: %llu\n", ext_s->grx.rx_pkts);
1349 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE,
1350 "rx_good_bytes: %llu\n", ext_s->grx.rx_bytes);
1351 }
1352
1353 /*
1354 * bnxt_re_ulp_log_live - Callback from L2 driver to collect snapdump
1355 * @handle - en_dev information. L2 and RoCE device information
1356 *
1357 * This function is callback to support L2 and RoCE common API.
1358 * bnxt_ulp_ops.ulp_log_live().
1359 *
1360 * Returns: Nothing
1361 */
1362 static void
bnxt_re_ulp_log_live(void * handle)1363 bnxt_re_ulp_log_live(void *handle)
1364 {
1365 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
1366
1367 if (!en_info || !en_info->rdev)
1368 return;
1369
1370 bnxt_re_create_snapdump_logs(en_info->rdev);
1371 }
1372
1373 /*
1374 * Except for ulp_async_notifier, the remaining ulp_ops
1375 * below are called with rtnl_lock held
1376 */
1377 static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
1378 .ulp_async_notifier = bnxt_re_async_notifier,
1379 .ulp_stop = bnxt_re_stop,
1380 .ulp_start = bnxt_re_start,
1381 .ulp_shutdown = bnxt_re_shutdown,
1382 .ulp_irq_stop = bnxt_re_stop_irq,
1383 .ulp_irq_restart = bnxt_re_start_irq,
1384 .ulp_log_live = bnxt_re_ulp_log_live,
1385 };
1386
bnxt_re_netevent(unsigned long event)1387 static inline const char *bnxt_re_netevent(unsigned long event)
1388 {
1389 BNXT_RE_NETDEV_EVENT(event, NETDEV_UP);
1390 BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN);
1391 BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE);
1392 BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER);
1393 BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER);
1394 BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR);
1395 return "Unknown";
1396 }
1397
1398 /* RoCE -> Net driver */
1399
1400 /* Driver registration routines used to let the networking driver (bnxt_en)
1401 * to know that the RoCE driver is now installed */
bnxt_re_unregister_netdev(struct bnxt_re_dev * rdev)1402 static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev)
1403 {
1404 struct bnxt_en_dev *en_dev = rdev->en_dev;
1405 int rc;
1406
1407 rtnl_lock();
1408 rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev,
1409 BNXT_ROCE_ULP);
1410 rtnl_unlock();
1411 if (rc)
1412 dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x",
1413 rdev->en_dev->net, rc);
1414
1415 clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
1416 }
1417
bnxt_re_register_netdev(struct bnxt_re_dev * rdev)1418 static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
1419 {
1420 struct bnxt_en_dev *en_dev = rdev->en_dev;
1421 int rc = 0;
1422
1423 rtnl_lock();
1424 rc = en_dev->en_ops->bnxt_register_device(en_dev,
1425 BNXT_ROCE_ULP,
1426 &bnxt_re_ulp_ops,
1427 rdev->adev);
1428 rtnl_unlock();
1429 if (rc) {
1430 dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x",
1431 rdev->netdev, rc);
1432 return rc;
1433 }
1434
1435 return rc;
1436 }
1437
bnxt_re_set_db_offset(struct bnxt_re_dev * rdev)1438 static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
1439 {
1440 struct bnxt_qplib_chip_ctx *cctx;
1441 struct bnxt_en_dev *en_dev;
1442 struct bnxt_qplib_res *res;
1443 u32 l2db_len = 0;
1444 u32 offset = 0;
1445 u32 barlen;
1446 int rc;
1447
1448 res = &rdev->qplib_res;
1449 en_dev = rdev->en_dev;
1450 cctx = rdev->chip_ctx;
1451
1452 /* Issue qcfg */
1453 rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset);
1454 if (rc)
1455 dev_info(rdev_to_dev(rdev),
1456 "Couldn't get DB bar size, Low latency framework is disabled\n");
1457 /* set register offsets for both UC and WC */
1458 if (_is_chip_p7(cctx)) {
1459 res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset;
1460 res->dpi_tbl.wcreg.offset = en_dev->l2_db_size;
1461 } else {
1462 res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET :
1463 BNXT_QPLIB_DBR_PF_DB_OFFSET;
1464 res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset;
1465 }
1466
1467 /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size
1468 * is equal to the DB-Bar actual size. This indicates that L2
1469 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping
1470 * in such cases and DB-push will be disabled.
1471 */
1472 barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION);
1473 if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) {
1474 res->dpi_tbl.wcreg.offset = en_dev->l2_db_size;
1475 dev_info(rdev_to_dev(rdev),
1476 "Low latency framework is enabled\n");
1477 }
1478
1479 return;
1480 }
1481
bnxt_re_set_drv_mode(struct bnxt_re_dev * rdev)1482 static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev)
1483 {
1484 struct bnxt_qplib_chip_ctx *cctx;
1485 struct bnxt_en_dev *en_dev;
1486
1487 en_dev = rdev->en_dev;
1488 cctx = rdev->chip_ctx;
1489 cctx->modes.wqe_mode = _is_chip_p7(rdev->chip_ctx) ?
1490 BNXT_QPLIB_WQE_MODE_VARIABLE : BNXT_QPLIB_WQE_MODE_STATIC;
1491 cctx->modes.te_bypass = false;
1492 if (bnxt_re_hwrm_qcaps(rdev))
1493 dev_err(rdev_to_dev(rdev),
1494 "Failed to query hwrm qcaps\n");
1495 /*
1496 * TODO: Need a better mechanism for spreading of the
1497 * 512 extended PPP pages in the presence of VF and
1498 * NPAR, until then not enabling push
1499 */
1500 if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) {
1501 if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev))
1502 cctx->modes.db_push = false;
1503 }
1504
1505 rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP;
1506 dev_dbg(rdev_to_dev(rdev),
1507 "RoCE is supported on the device - caps:0x%x",
1508 rdev->roce_mode);
1509 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
1510 rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP;
1511 cctx->hw_stats_size = en_dev->hw_ring_stats_size;
1512 }
1513
bnxt_re_destroy_chip_ctx(struct bnxt_re_dev * rdev)1514 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
1515 {
1516 struct bnxt_qplib_chip_ctx *chip_ctx;
1517 struct bnxt_qplib_res *res;
1518
1519 if (!rdev->chip_ctx)
1520 return;
1521
1522 res = &rdev->qplib_res;
1523 bnxt_qplib_unmap_db_bar(res);
1524
1525 kfree(res->hctx);
1526 res->rcfw = NULL;
1527 kfree(rdev->dev_attr);
1528 rdev->dev_attr = NULL;
1529
1530 chip_ctx = rdev->chip_ctx;
1531 rdev->chip_ctx = NULL;
1532 res->cctx = NULL;
1533 res->hctx = NULL;
1534 res->pdev = NULL;
1535 res->netdev = NULL;
1536 kfree(chip_ctx);
1537 }
1538
bnxt_re_setup_chip_ctx(struct bnxt_re_dev * rdev)1539 static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev)
1540 {
1541 struct bnxt_qplib_chip_ctx *chip_ctx;
1542 struct bnxt_en_dev *en_dev;
1543 int rc;
1544
1545 en_dev = rdev->en_dev;
1546 /* Supply pci device to qplib */
1547 rdev->qplib_res.pdev = en_dev->pdev;
1548 rdev->qplib_res.netdev = rdev->netdev;
1549 rdev->qplib_res.en_dev = en_dev;
1550
1551 chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
1552 if (!chip_ctx)
1553 return -ENOMEM;
1554 rdev->chip_ctx = chip_ctx;
1555 rdev->qplib_res.cctx = chip_ctx;
1556 rc = bnxt_re_query_hwrm_intf_version(rdev);
1557 if (rc)
1558 goto fail;
1559 rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL);
1560 if (!rdev->dev_attr) {
1561 rc = -ENOMEM;
1562 goto fail;
1563 }
1564 rdev->qplib_res.dattr = rdev->dev_attr;
1565 rdev->qplib_res.rcfw = &rdev->rcfw;
1566 rdev->qplib_res.is_vf = rdev->is_virtfn;
1567
1568 rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx),
1569 GFP_KERNEL);
1570 if (!rdev->qplib_res.hctx) {
1571 rc = -ENOMEM;
1572 goto fail;
1573 }
1574 bnxt_re_set_drv_mode(rdev);
1575
1576 bnxt_re_set_db_offset(rdev);
1577 rc = bnxt_qplib_map_db_bar(&rdev->qplib_res);
1578 if (rc)
1579 goto fail;
1580
1581 rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev);
1582 if (rc)
1583 dev_dbg(rdev_to_dev(rdev),
1584 "platform doesn't support global atomics");
1585
1586 return 0;
1587 fail:
1588 kfree(rdev->chip_ctx);
1589 rdev->chip_ctx = NULL;
1590
1591 kfree(rdev->dev_attr);
1592 rdev->dev_attr = NULL;
1593
1594 kfree(rdev->qplib_res.hctx);
1595 rdev->qplib_res.hctx = NULL;
1596 return rc;
1597 }
1598
bnxt_re_get_rtype(struct bnxt_re_dev * rdev)1599 static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) {
1600 return _is_chip_gen_p5_p7(rdev->chip_ctx) ?
1601 HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ :
1602 HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL;
1603 }
1604
bnxt_re_net_ring_free(struct bnxt_re_dev * rdev,u16 fw_ring_id)1605 static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id)
1606 {
1607 int rc = -EINVAL;
1608 struct hwrm_ring_free_input req = {0};
1609 struct hwrm_ring_free_output resp;
1610 struct bnxt_en_dev *en_dev = rdev->en_dev;
1611 struct bnxt_fw_msg fw_msg;
1612
1613 if (!en_dev)
1614 return rc;
1615
1616 /* To avoid unnecessary error messages during recovery.
1617 * HW is anyway in error state. So dont send down the command */
1618 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
1619 return 0;
1620
1621 /* allocation had failed, no need to issue hwrm */
1622 if (fw_ring_id == 0xffff)
1623 return 0;
1624
1625 memset(&fw_msg, 0, sizeof(fw_msg));
1626
1627 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1);
1628 req.ring_type = bnxt_re_get_rtype(rdev);
1629 req.ring_id = cpu_to_le16(fw_ring_id);
1630 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1631 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1632 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1633 if (rc) {
1634 dev_err(rdev_to_dev(rdev),
1635 "Failed to free HW ring with rc = 0x%x", rc);
1636 return rc;
1637 }
1638 dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n",
1639 fw_ring_id);
1640
1641 return rc;
1642 }
1643
bnxt_re_net_ring_alloc(struct bnxt_re_dev * rdev,struct bnxt_re_ring_attr * ring_attr,u16 * fw_ring_id)1644 static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
1645 struct bnxt_re_ring_attr *ring_attr,
1646 u16 *fw_ring_id)
1647 {
1648 int rc = -EINVAL;
1649 struct hwrm_ring_alloc_input req = {0};
1650 struct hwrm_ring_alloc_output resp;
1651 struct bnxt_en_dev *en_dev = rdev->en_dev;
1652 struct bnxt_fw_msg fw_msg;
1653
1654 if (!en_dev)
1655 return rc;
1656
1657 memset(&fw_msg, 0, sizeof(fw_msg));
1658 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
1659 req.flags = cpu_to_le16(ring_attr->flags);
1660 req.enables = 0;
1661 req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]);
1662 if (ring_attr->pages > 1) {
1663 /* Page size is in log2 units */
1664 req.page_size = BNXT_PAGE_SHIFT;
1665 req.page_tbl_depth = 1;
1666 } else {
1667 req.page_size = 4;
1668 req.page_tbl_depth = 0;
1669 }
1670
1671 req.fbo = 0;
1672 /* Association of ring index with doorbell index and MSIX number */
1673 req.logical_id = cpu_to_le16(ring_attr->lrid);
1674 req.length = cpu_to_le32(ring_attr->depth + 1);
1675 req.ring_type = ring_attr->type;
1676 req.int_mode = ring_attr->mode;
1677 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1678 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1679 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1680 if (rc) {
1681 dev_err(rdev_to_dev(rdev),
1682 "Failed to allocate HW ring with rc = 0x%x", rc);
1683 return rc;
1684 }
1685 *fw_ring_id = le16_to_cpu(resp.ring_id);
1686 dev_dbg(rdev_to_dev(rdev),
1687 "HW ring allocated with id = 0x%x at slot 0x%x",
1688 resp.ring_id, ring_attr->lrid);
1689
1690 return rc;
1691 }
1692
bnxt_re_net_stats_ctx_free(struct bnxt_re_dev * rdev,u32 fw_stats_ctx_id,u16 tid)1693 static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
1694 u32 fw_stats_ctx_id, u16 tid)
1695 {
1696 struct bnxt_en_dev *en_dev = rdev->en_dev;
1697 struct hwrm_stat_ctx_free_input req = {0};
1698 struct hwrm_stat_ctx_free_output resp;
1699 struct bnxt_fw_msg fw_msg;
1700 int rc = -EINVAL;
1701
1702 if (!en_dev)
1703 return rc;
1704
1705 /* To avoid unnecessary error messages during recovery.
1706 * HW is anyway in error state. So dont send down the command */
1707 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
1708 return 0;
1709 memset(&fw_msg, 0, sizeof(fw_msg));
1710 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid);
1711 req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
1712 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1713 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1714 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1715 if (rc) {
1716 dev_err(rdev_to_dev(rdev),
1717 "Failed to free HW stats ctx with rc = 0x%x", rc);
1718 return rc;
1719 }
1720 dev_dbg(rdev_to_dev(rdev),
1721 "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id);
1722
1723 return rc;
1724 }
1725
bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev * rdev,u16 tid)1726 static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid)
1727 {
1728 struct hwrm_stat_ctx_alloc_output resp = {};
1729 struct hwrm_stat_ctx_alloc_input req = {};
1730 struct bnxt_en_dev *en_dev = rdev->en_dev;
1731 struct bnxt_qplib_stats *stat;
1732 struct bnxt_qplib_ctx *hctx;
1733 struct bnxt_fw_msg fw_msg;
1734 int rc = 0;
1735
1736 hctx = rdev->qplib_res.hctx;
1737 stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2;
1738 stat->fw_id = INVALID_STATS_CTX_ID;
1739
1740 if (!en_dev)
1741 return -EINVAL;
1742
1743 memset(&fw_msg, 0, sizeof(fw_msg));
1744 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
1745 HWRM_STAT_CTX_ALLOC, -1, tid);
1746 req.update_period_ms = cpu_to_le32(1000);
1747 req.stats_dma_length = rdev->chip_ctx->hw_stats_size;
1748 req.stats_dma_addr = cpu_to_le64(stat->dma_map);
1749 req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE;
1750 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1751 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1752 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1753 if (rc) {
1754 dev_err(rdev_to_dev(rdev),
1755 "Failed to allocate HW stats ctx, rc = 0x%x", rc);
1756 return rc;
1757 }
1758 stat->fw_id = le32_to_cpu(resp.stat_ctx_id);
1759 dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x",
1760 stat->fw_id);
1761
1762 return rc;
1763 }
1764
bnxt_re_net_unregister_async_event(struct bnxt_re_dev * rdev)1765 static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev)
1766 {
1767 const struct bnxt_en_ops *en_ops;
1768
1769 if (rdev->is_virtfn ||
1770 test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
1771 return;
1772
1773 memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap));
1774 en_ops = rdev->en_dev->en_ops;
1775 if (en_ops->bnxt_register_fw_async_events
1776 (rdev->en_dev, BNXT_ROCE_ULP,
1777 (unsigned long *)rdev->event_bitmap,
1778 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE))
1779 dev_err(rdev_to_dev(rdev),
1780 "Failed to unregister async event");
1781 }
1782
bnxt_re_net_register_async_event(struct bnxt_re_dev * rdev)1783 static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev)
1784 {
1785 const struct bnxt_en_ops *en_ops;
1786
1787 if (rdev->is_virtfn)
1788 return;
1789
1790 rdev->event_bitmap[0] |=
1791 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) |
1792 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY);
1793
1794 rdev->event_bitmap[2] |=
1795 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64);
1796 rdev->event_bitmap[2] |=
1797 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) |
1798 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64);
1799 en_ops = rdev->en_dev->en_ops;
1800 if (en_ops->bnxt_register_fw_async_events
1801 (rdev->en_dev, BNXT_ROCE_ULP,
1802 (unsigned long *)rdev->event_bitmap,
1803 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE))
1804 dev_err(rdev_to_dev(rdev),
1805 "Failed to reg Async event");
1806 }
1807
bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev * rdev)1808 static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
1809 {
1810 struct bnxt_en_dev *en_dev = rdev->en_dev;
1811 struct hwrm_ver_get_output resp = {0};
1812 struct hwrm_ver_get_input req = {0};
1813 struct bnxt_qplib_chip_ctx *cctx;
1814 struct bnxt_fw_msg fw_msg;
1815 int rc = 0;
1816
1817 memset(&fw_msg, 0, sizeof(fw_msg));
1818 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
1819 HWRM_VER_GET, -1, -1);
1820 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
1821 req.hwrm_intf_min = HWRM_VERSION_MINOR;
1822 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
1823 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1824 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1825 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1826 if (rc) {
1827 dev_err(rdev_to_dev(rdev),
1828 "Failed to query HW version, rc = 0x%x", rc);
1829 return rc;
1830 }
1831 cctx = rdev->chip_ctx;
1832 cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 |
1833 (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 |
1834 (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 |
1835 le16_to_cpu(resp.hwrm_intf_patch);
1836
1837 cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout);
1838
1839 if (!cctx->hwrm_cmd_max_timeout)
1840 cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT;
1841
1842 cctx->chip_num = le16_to_cpu(resp.chip_num);
1843 cctx->chip_rev = resp.chip_rev;
1844 cctx->chip_metal = resp.chip_metal;
1845 return 0;
1846 }
1847
1848 /* Query device config using common hwrm */
bnxt_re_hwrm_qcfg(struct bnxt_re_dev * rdev,u32 * db_len,u32 * offset)1849 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
1850 u32 *offset)
1851 {
1852 struct bnxt_en_dev *en_dev = rdev->en_dev;
1853 struct hwrm_func_qcfg_output resp = {0};
1854 struct hwrm_func_qcfg_input req = {0};
1855 struct bnxt_fw_msg fw_msg;
1856 int rc;
1857
1858 memset(&fw_msg, 0, sizeof(fw_msg));
1859 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
1860 HWRM_FUNC_QCFG, -1, -1);
1861 req.fid = cpu_to_le16(0xffff);
1862 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1863 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1864 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1865 if (rc) {
1866 dev_err(rdev_to_dev(rdev),
1867 "Failed to query config, rc = %#x", rc);
1868 return rc;
1869 }
1870
1871 *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024);
1872 *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024);
1873 return 0;
1874 }
1875
1876 /* Query function capabilities using common hwrm */
bnxt_re_hwrm_qcaps(struct bnxt_re_dev * rdev)1877 int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev)
1878 {
1879 struct bnxt_en_dev *en_dev = rdev->en_dev;
1880 struct hwrm_func_qcaps_output resp = {0};
1881 struct hwrm_func_qcaps_input req = {0};
1882 struct bnxt_qplib_chip_ctx *cctx;
1883 struct bnxt_fw_msg fw_msg;
1884 u8 push_enable = false;
1885 int rc;
1886
1887 cctx = rdev->chip_ctx;
1888 memset(&fw_msg, 0, sizeof(fw_msg));
1889 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
1890 HWRM_FUNC_QCAPS, -1, -1);
1891 req.fid = cpu_to_le16(0xffff);
1892 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1893 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1894 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1895 if (rc) {
1896 dev_err(rdev_to_dev(rdev),
1897 "Failed to query capabilities, rc = %#x", rc);
1898 return rc;
1899 }
1900 if (_is_chip_p7(rdev->chip_ctx))
1901 push_enable =
1902 (resp.flags_ext &
1903 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ?
1904 true : false;
1905 else
1906 push_enable =
1907 (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ?
1908 true : false;
1909 cctx->modes.db_push = push_enable;
1910
1911 cctx->modes.dbr_pacing =
1912 resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ?
1913 true : false;
1914 cctx->modes.dbr_pacing_ext =
1915 resp.flags_ext2 &
1916 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ?
1917 true : false;
1918 cctx->modes.dbr_drop_recov =
1919 (resp.flags_ext2 &
1920 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ?
1921 true : false;
1922 cctx->modes.dbr_pacing_v0 =
1923 (resp.flags_ext2 &
1924 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ?
1925 true : false;
1926 dev_dbg(rdev_to_dev(rdev),
1927 "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n",
1928 __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov);
1929
1930 return 0;
1931 }
1932
bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev * rdev)1933 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev)
1934 {
1935 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
1936 struct hwrm_func_dbr_pacing_qcfg_output resp = {0};
1937 struct hwrm_func_dbr_pacing_qcfg_input req = {0};
1938 struct bnxt_en_dev *en_dev = rdev->en_dev;
1939 struct bnxt_qplib_chip_ctx *cctx;
1940 struct bnxt_fw_msg fw_msg;
1941 u32 primary_nq_id;
1942 int rc;
1943
1944 cctx = rdev->chip_ctx;
1945 memset(&fw_msg, 0, sizeof(fw_msg));
1946 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
1947 HWRM_FUNC_DBR_PACING_QCFG, -1, -1);
1948 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1949 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
1950 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
1951 if (rc) {
1952 dev_dbg(rdev_to_dev(rdev),
1953 "Failed to query dbr pacing config, rc = %#x", rc);
1954 return rc;
1955 }
1956
1957 primary_nq_id = le32_to_cpu(resp.primary_nq_id);
1958 if (primary_nq_id == 0xffffffff &&
1959 !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
1960 dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n",
1961 __func__, __LINE__);
1962 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1);
1963 }
1964
1965 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
1966 struct bnxt_qplib_nq *nq;
1967
1968 nq = &rdev->nqr.nq[0];
1969 /* Reset the primary capability */
1970 if (nq->ring_id != primary_nq_id)
1971 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0);
1972 }
1973
1974 if ((resp.dbr_stat_db_fifo_reg &
1975 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) ==
1976 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC)
1977 cctx->dbr_stat_db_fifo =
1978 resp.dbr_stat_db_fifo_reg &
1979 ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
1980
1981 if ((resp.dbr_throttling_aeq_arm_reg &
1982 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK)
1983 == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) {
1984 cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg &
1985 ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
1986 cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4;
1987 }
1988 pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth);
1989 if (!pacing_data->fifo_max_depth)
1990 pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx);
1991 pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask);
1992 pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift;
1993 dev_dbg(rdev_to_dev(rdev),
1994 "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i"
1995 "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n",
1996 __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf,
1997 cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg,
1998 pacing_data->fifo_max_depth,
1999 le32_to_cpu(resp.dbr_stat_db_max_fifo_depth));
2000 return 0;
2001 }
2002
bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev * rdev,bool enable)2003 static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable)
2004 {
2005 struct hwrm_func_dbr_pacing_cfg_output resp = {0};
2006 struct hwrm_func_dbr_pacing_cfg_input req = {0};
2007 struct bnxt_en_dev *en_dev = rdev->en_dev;
2008 struct bnxt_fw_msg fw_msg;
2009 int rc;
2010
2011 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
2012 return 0;
2013
2014 memset(&fw_msg, 0, sizeof(fw_msg));
2015 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
2016 HWRM_FUNC_DBR_PACING_CFG, -1, -1);
2017 if (enable) {
2018 req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE;
2019 req.enables =
2020 cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID |
2021 HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID);
2022 } else {
2023 req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE;
2024 }
2025 req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id);
2026 req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark);
2027 dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x",
2028 __func__, req.primary_nq_id, req.pacing_threshold);
2029 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
2030 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
2031 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
2032 if (rc) {
2033 dev_dbg(rdev_to_dev(rdev),
2034 "Failed to set dbr pacing config, rc = %#x", rc);
2035 return rc;
2036 }
2037 return 0;
2038 }
2039
2040 /* Net -> RoCE driver */
2041
2042 /* Device */
bnxt_re_from_netdev(struct ifnet * netdev)2043 struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev)
2044 {
2045 struct bnxt_re_dev *rdev;
2046
2047 rcu_read_lock();
2048 list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
2049 if (rdev->netdev == netdev) {
2050 rcu_read_unlock();
2051 dev_dbg(rdev_to_dev(rdev),
2052 "netdev (%p) found, ref_count = 0x%x",
2053 netdev, atomic_read(&rdev->ref_count));
2054 return rdev;
2055 }
2056 }
2057 rcu_read_unlock();
2058 return NULL;
2059 }
2060
show_rev(struct device * device,struct device_attribute * attr,char * buf)2061 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
2062 char *buf)
2063 {
2064 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
2065
2066 return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
2067 }
2068
2069
show_hca(struct device * device,struct device_attribute * attr,char * buf)2070 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
2071 char *buf)
2072 {
2073 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
2074
2075 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
2076 }
2077
show_board_id(struct device * device,struct device_attribute * attr,char * buf)2078 static ssize_t show_board_id(struct device *device, struct device_attribute *attr,
2079 char *buf)
2080 {
2081 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
2082 char buffer[BNXT_VPD_PN_FLD_LEN] = {};
2083
2084 if (!rdev->is_virtfn)
2085 memcpy(buffer, rdev->en_dev->board_part_number,
2086 BNXT_VPD_PN_FLD_LEN - 1);
2087 else
2088 scnprintf(buffer, BNXT_VPD_PN_FLD_LEN,
2089 "0x%x-VF", rdev->en_dev->pdev->device);
2090
2091 return scnprintf(buf, PAGE_SIZE, "%s\n", buffer);
2092 }
2093
2094 static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
2095 static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
2096 static DEVICE_ATTR(board_id, 0444, show_board_id, NULL);
2097
2098 static struct device_attribute *bnxt_re_attributes[] = {
2099 &dev_attr_hw_rev,
2100 &dev_attr_hca_type,
2101 &dev_attr_board_id
2102 };
2103
ib_register_device_compat(struct bnxt_re_dev * rdev)2104 int ib_register_device_compat(struct bnxt_re_dev *rdev)
2105 {
2106 struct ib_device *ibdev = &rdev->ibdev;
2107 char name[IB_DEVICE_NAME_MAX];
2108
2109 memset(name, 0, IB_DEVICE_NAME_MAX);
2110 strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX);
2111
2112 strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
2113
2114 return ib_register_device(ibdev, NULL);
2115 }
2116
bnxt_re_register_ib(struct bnxt_re_dev * rdev)2117 static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
2118 {
2119 struct ib_device *ibdev = &rdev->ibdev;
2120 int ret = 0;
2121
2122 /* ib device init */
2123 ibdev->owner = THIS_MODULE;
2124 ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
2125 ibdev->node_type = RDMA_NODE_IB_CA;
2126 strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
2127 strlen(BNXT_RE_DESC) + 5);
2128 ibdev->phys_port_cnt = 1;
2129
2130 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid);
2131
2132 /* Data path irqs is one less than the max msix vectors */
2133 ibdev->num_comp_vectors = rdev->nqr.num_msix - 1;
2134 bnxt_re_set_dma_device(ibdev, rdev);
2135 ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
2136
2137 /* User space */
2138 ibdev->uverbs_cmd_mask =
2139 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
2140 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
2141 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
2142 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
2143 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
2144 (1ull << IB_USER_VERBS_CMD_REG_MR) |
2145 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
2146 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
2147 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
2148 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
2149 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
2150 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
2151 (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
2152 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
2153 (1ull << IB_USER_VERBS_CMD_REREG_MR) |
2154 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
2155 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
2156 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
2157 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
2158 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
2159 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
2160 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) |
2161 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
2162 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
2163 (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
2164 (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
2165
2166 ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP);
2167 ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ);
2168
2169 #define bnxt_re_ib_ah bnxt_re_ah
2170 #define bnxt_re_ib_cq bnxt_re_cq
2171 #define bnxt_re_ib_pd bnxt_re_pd
2172 #define bnxt_re_ib_srq bnxt_re_srq
2173 #define bnxt_re_ib_ucontext bnxt_re_ucontext
2174 INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE);
2175
2176 ibdev->query_device = bnxt_re_query_device;
2177 ibdev->modify_device = bnxt_re_modify_device;
2178 ibdev->query_port = bnxt_re_query_port;
2179 ibdev->modify_port = bnxt_re_modify_port;
2180 ibdev->get_port_immutable = bnxt_re_get_port_immutable;
2181 ibdev->query_pkey = bnxt_re_query_pkey;
2182 ibdev->get_netdev = bnxt_re_get_netdev;
2183 ibdev->add_gid = bnxt_re_add_gid;
2184 ibdev->del_gid = bnxt_re_del_gid;
2185 ibdev->get_link_layer = bnxt_re_get_link_layer;
2186 ibdev->alloc_pd = bnxt_re_alloc_pd;
2187 ibdev->dealloc_pd = bnxt_re_dealloc_pd;
2188 ibdev->create_ah = bnxt_re_create_ah;
2189 ibdev->modify_ah = bnxt_re_modify_ah;
2190 ibdev->query_ah = bnxt_re_query_ah;
2191 ibdev->destroy_ah = bnxt_re_destroy_ah;
2192 ibdev->create_srq = bnxt_re_create_srq;
2193 ibdev->modify_srq = bnxt_re_modify_srq;
2194 ibdev->query_srq = bnxt_re_query_srq;
2195 ibdev->destroy_srq = bnxt_re_destroy_srq;
2196 ibdev->post_srq_recv = bnxt_re_post_srq_recv;
2197 ibdev->create_qp = bnxt_re_create_qp;
2198 ibdev->modify_qp = bnxt_re_modify_qp;
2199 ibdev->query_qp = bnxt_re_query_qp;
2200 ibdev->destroy_qp = bnxt_re_destroy_qp;
2201 ibdev->post_send = bnxt_re_post_send;
2202 ibdev->post_recv = bnxt_re_post_recv;
2203 ibdev->create_cq = bnxt_re_create_cq;
2204 ibdev->modify_cq = bnxt_re_modify_cq;
2205 ibdev->destroy_cq = bnxt_re_destroy_cq;
2206 ibdev->resize_cq = bnxt_re_resize_cq;
2207 ibdev->poll_cq = bnxt_re_poll_cq;
2208 ibdev->req_notify_cq = bnxt_re_req_notify_cq;
2209 ibdev->get_dma_mr = bnxt_re_get_dma_mr;
2210 ibdev->get_hw_stats = bnxt_re_get_hw_stats;
2211 ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats;
2212 ibdev->dereg_mr = bnxt_re_dereg_mr;
2213 ibdev->alloc_mr = bnxt_re_alloc_mr;
2214 ibdev->map_mr_sg = bnxt_re_map_mr_sg;
2215 ibdev->alloc_mw = bnxt_re_alloc_mw;
2216 ibdev->dealloc_mw = bnxt_re_dealloc_mw;
2217 ibdev->reg_user_mr = bnxt_re_reg_user_mr;
2218 ibdev->rereg_user_mr = bnxt_re_rereg_user_mr;
2219 ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx;
2220 ibdev->alloc_ucontext = bnxt_re_alloc_ucontext;
2221 ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext;
2222 ibdev->mmap = bnxt_re_mmap;
2223 ibdev->process_mad = bnxt_re_process_mad;
2224
2225 ret = ib_register_device_compat(rdev);
2226 return ret;
2227 }
2228
bnxt_re_dev_dealloc(struct bnxt_re_dev * rdev)2229 static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev)
2230 {
2231 int i = BNXT_RE_REF_WAIT_COUNT;
2232
2233 dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev);
2234 /* Wait for rdev refcount to come down */
2235 while ((atomic_read(&rdev->ref_count) > 1) && i--)
2236 msleep(100);
2237
2238 if (atomic_read(&rdev->ref_count) > 1)
2239 dev_err(rdev_to_dev(rdev),
2240 "Failed waiting for ref count to deplete %d",
2241 atomic_read(&rdev->ref_count));
2242
2243 atomic_set(&rdev->ref_count, 0);
2244 if_rele(rdev->netdev);
2245 rdev->netdev = NULL;
2246 synchronize_rcu();
2247
2248 kfree(rdev->gid_map);
2249 kfree(rdev->dbg_stats);
2250 ib_dealloc_device(&rdev->ibdev);
2251 }
2252
bnxt_re_dev_alloc(struct ifnet * netdev,struct bnxt_en_dev * en_dev)2253 static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev,
2254 struct bnxt_en_dev *en_dev)
2255 {
2256 struct bnxt_re_dev *rdev;
2257 u32 count;
2258
2259 /* Allocate bnxt_re_dev instance here */
2260 rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev));
2261 if (!rdev) {
2262 pr_err("%s: bnxt_re_dev allocation failure!",
2263 ROCE_DRV_MODULE_NAME);
2264 return NULL;
2265 }
2266 /* Default values */
2267 atomic_set(&rdev->ref_count, 0);
2268 rdev->netdev = netdev;
2269 dev_hold(rdev->netdev);
2270 rdev->en_dev = en_dev;
2271 rdev->id = rdev->en_dev->pdev->devfn;
2272 INIT_LIST_HEAD(&rdev->qp_list);
2273 mutex_init(&rdev->qp_lock);
2274 mutex_init(&rdev->cc_lock);
2275 mutex_init(&rdev->dbq_lock);
2276 bnxt_re_clear_rsors_stat(&rdev->stats.rsors);
2277 rdev->cosq[0] = rdev->cosq[1] = 0xFFFF;
2278 rdev->min_tx_depth = 1;
2279 rdev->stats.stats_query_sec = 1;
2280 /* Disable priority vlan as the default mode is DSCP based PFC */
2281 rdev->cc_param.disable_prio_vlan_tx = 1;
2282
2283 /* Initialize worker for DBR Pacing */
2284 INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check);
2285 INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp);
2286 rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) *
2287 BNXT_RE_MAX_SGID_ENTRIES,
2288 GFP_KERNEL);
2289 if (!rdev->gid_map) {
2290 ib_dealloc_device(&rdev->ibdev);
2291 return NULL;
2292 }
2293 for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++)
2294 rdev->gid_map[count] = -1;
2295
2296 rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL);
2297 if (!rdev->dbg_stats) {
2298 ib_dealloc_device(&rdev->ibdev);
2299 return NULL;
2300 }
2301
2302 return rdev;
2303 }
2304
bnxt_re_handle_unaffi_async_event(struct creq_func_event * unaffi_async)2305 static int bnxt_re_handle_unaffi_async_event(
2306 struct creq_func_event *unaffi_async)
2307 {
2308 switch (unaffi_async->event) {
2309 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
2310 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
2311 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
2312 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
2313 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
2314 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
2315 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
2316 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
2317 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
2318 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
2319 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
2320 break;
2321 default:
2322 return -EINVAL;
2323 }
2324 return 0;
2325 }
2326
bnxt_re_handle_qp_async_event(void * qp_event,struct bnxt_re_qp * qp)2327 static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp)
2328 {
2329 struct creq_qp_error_notification *err_event;
2330 struct ib_event event;
2331 unsigned int flags;
2332
2333 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR &&
2334 !qp->qplib_qp.is_user) {
2335 flags = bnxt_re_lock_cqs(qp);
2336 bnxt_qplib_add_flush_qp(&qp->qplib_qp);
2337 bnxt_re_unlock_cqs(qp, flags);
2338 }
2339 memset(&event, 0, sizeof(event));
2340 event.device = &qp->rdev->ibdev;
2341 event.element.qp = &qp->ib_qp;
2342 event.event = IB_EVENT_QP_FATAL;
2343
2344 err_event = qp_event;
2345 switch(err_event->res_err_state_reason) {
2346 case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX:
2347 case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH:
2348 case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR:
2349 case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT:
2350 case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY:
2351 case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR:
2352 case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION:
2353 case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR:
2354 case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY:
2355 case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR:
2356 case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION:
2357 case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR:
2358 case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY:
2359 case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC:
2360 event.event = IB_EVENT_QP_ACCESS_ERR;
2361 break;
2362 case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE:
2363 case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR:
2364 case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR:
2365 case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE:
2366 case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE:
2367 event.event = IB_EVENT_QP_REQ_ERR;
2368 break;
2369 case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW:
2370 case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR:
2371 case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR:
2372 case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR:
2373 case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR:
2374 case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR:
2375 case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR:
2376 event.event = IB_EVENT_QP_FATAL;
2377 break;
2378 default:
2379 if (qp->qplib_qp.srq)
2380 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
2381 break;
2382 }
2383
2384 if (err_event->res_err_state_reason)
2385 dev_err(rdev_to_dev(qp->rdev),
2386 "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n",
2387 __func__, qp->qplib_qp.is_user ? "user" : "kernel",
2388 qp->qplib_qp.id,
2389 err_event->sq_cons_idx,
2390 err_event->rq_cons_idx,
2391 err_event->req_slow_path_state,
2392 err_event->req_err_state_reason,
2393 err_event->res_slow_path_state,
2394 err_event->res_err_state_reason);
2395
2396 if (event.device && qp->ib_qp.event_handler)
2397 qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
2398
2399 return 0;
2400 }
2401
bnxt_re_handle_cq_async_error(void * event,struct bnxt_re_cq * cq)2402 static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq)
2403 {
2404 struct creq_cq_error_notification *cqerr;
2405 bool send = false;
2406
2407 cqerr = event;
2408 switch (cqerr->cq_err_reason) {
2409 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR:
2410 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR:
2411 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR:
2412 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR:
2413 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR:
2414 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR:
2415 send = true;
2416 default:
2417 break;
2418 }
2419
2420 if (send && cq->ibcq.event_handler) {
2421 struct ib_event ibevent = {};
2422
2423 ibevent.event = IB_EVENT_CQ_ERR;
2424 ibevent.element.cq = &cq->ibcq;
2425 ibevent.device = &cq->rdev->ibdev;
2426
2427 dev_err(rdev_to_dev(cq->rdev),
2428 "%s err reason %d\n", __func__, cqerr->cq_err_reason);
2429 cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context);
2430 }
2431
2432 cq->qplib_cq.is_cq_err_event = true;
2433
2434 return 0;
2435 }
2436
bnxt_re_handle_affi_async_event(struct creq_qp_event * affi_async,void * obj)2437 static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
2438 void *obj)
2439 {
2440 struct bnxt_qplib_qp *qplqp;
2441 struct bnxt_qplib_cq *qplcq;
2442 struct bnxt_re_qp *qp;
2443 struct bnxt_re_cq *cq;
2444 int rc = 0;
2445 u8 event;
2446
2447 if (!obj)
2448 return rc; /* QP was already dead, still return success */
2449
2450 event = affi_async->event;
2451 switch (event) {
2452 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
2453 qplqp = obj;
2454 qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp);
2455 rc = bnxt_re_handle_qp_async_event(affi_async, qp);
2456 break;
2457 case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
2458 qplcq = obj;
2459 cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq);
2460 rc = bnxt_re_handle_cq_async_error(affi_async, cq);
2461 break;
2462 default:
2463 rc = -EINVAL;
2464 }
2465
2466 return rc;
2467 }
2468
bnxt_re_aeq_handler(struct bnxt_qplib_rcfw * rcfw,void * aeqe,void * obj)2469 static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
2470 void *aeqe, void *obj)
2471 {
2472 struct creq_func_event *unaffi_async;
2473 struct creq_qp_event *affi_async;
2474 u8 type;
2475 int rc;
2476
2477 type = ((struct creq_base *)aeqe)->type;
2478 if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
2479 unaffi_async = aeqe;
2480 rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
2481 } else {
2482 affi_async = aeqe;
2483 rc = bnxt_re_handle_affi_async_event(affi_async, obj);
2484 }
2485
2486 return rc;
2487 }
2488
bnxt_re_srqn_handler(struct bnxt_qplib_nq * nq,struct bnxt_qplib_srq * handle,u8 event)2489 static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
2490 struct bnxt_qplib_srq *handle, u8 event)
2491 {
2492 struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq,
2493 qplib_srq);
2494 struct ib_event ib_event;
2495
2496 if (srq == NULL) {
2497 pr_err("%s: SRQ is NULL, SRQN not handled",
2498 ROCE_DRV_MODULE_NAME);
2499 return -EINVAL;
2500 }
2501 ib_event.device = &srq->rdev->ibdev;
2502 ib_event.element.srq = &srq->ibsrq;
2503 if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
2504 ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
2505 else
2506 ib_event.event = IB_EVENT_SRQ_ERR;
2507
2508 if (srq->ibsrq.event_handler) {
2509 /* Lock event_handler? */
2510 (*srq->ibsrq.event_handler)(&ib_event,
2511 srq->ibsrq.srq_context);
2512 }
2513 return 0;
2514 }
2515
bnxt_re_cqn_handler(struct bnxt_qplib_nq * nq,struct bnxt_qplib_cq * handle)2516 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
2517 struct bnxt_qplib_cq *handle)
2518 {
2519 struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq,
2520 qplib_cq);
2521 u32 *cq_ptr;
2522
2523 if (cq == NULL) {
2524 pr_err("%s: CQ is NULL, CQN not handled",
2525 ROCE_DRV_MODULE_NAME);
2526 return -EINVAL;
2527 }
2528 /* CQ already in destroy path. Do not handle any more events */
2529 if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) {
2530 if (!handle->destroyed)
2531 dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled",
2532 ROCE_DRV_MODULE_NAME);
2533 return 0;
2534 }
2535
2536 if (cq->ibcq.comp_handler) {
2537 if (cq->uctx_cq_page) {
2538 cq_ptr = (u32 *)cq->uctx_cq_page;
2539 *cq_ptr = cq->qplib_cq.toggle;
2540 }
2541 /* Lock comp_handler? */
2542 (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context);
2543 }
2544
2545 return 0;
2546 }
2547
bnxt_re_get_nq(struct bnxt_re_dev * rdev)2548 struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
2549 {
2550 int min, indx;
2551
2552 mutex_lock(&rdev->nqr.load_lock);
2553 for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) {
2554 if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load)
2555 min = indx;
2556 }
2557 rdev->nqr.nq[min].load++;
2558 mutex_unlock(&rdev->nqr.load_lock);
2559
2560 return &rdev->nqr.nq[min];
2561 }
2562
bnxt_re_put_nq(struct bnxt_re_dev * rdev,struct bnxt_qplib_nq * nq)2563 void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
2564 {
2565 mutex_lock(&rdev->nqr.load_lock);
2566 nq->load--;
2567 mutex_unlock(&rdev->nqr.load_lock);
2568 }
2569
bnxt_re_check_min_attr(struct bnxt_re_dev * rdev)2570 static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev)
2571 {
2572 struct bnxt_qplib_dev_attr *attr;
2573 bool rc = true;
2574
2575 attr = rdev->dev_attr;
2576
2577 if (!attr->max_cq || !attr->max_qp ||
2578 !attr->max_sgid || !attr->max_mr) {
2579 dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources");
2580 dev_dbg(rdev_to_dev(rdev),
2581 "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d",
2582 attr->max_cq, attr->max_qp, attr->max_dpi,
2583 attr->max_sgid, attr->max_mr);
2584 rc = false;
2585 }
2586 return rc;
2587 }
2588
bnxt_re_dispatch_event(struct ib_device * ibdev,struct ib_qp * qp,u8 port_num,enum ib_event_type event)2589 static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
2590 u8 port_num, enum ib_event_type event)
2591 {
2592 struct ib_event ib_event;
2593
2594 ib_event.device = ibdev;
2595 if (qp) {
2596 ib_event.element.qp = qp;
2597 ib_event.event = event;
2598 if (qp->event_handler)
2599 qp->event_handler(&ib_event, qp->qp_context);
2600 } else {
2601 ib_event.element.port_num = port_num;
2602 ib_event.event = event;
2603 ib_dispatch_event(&ib_event);
2604 }
2605
2606 dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)),
2607 "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num);
2608 }
2609
bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp)2610 static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
2611 struct bnxt_re_qp *qp)
2612 {
2613 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL)
2614 return (qp->ib_qp.qp_type == IB_QPT_GSI) ||
2615 (qp == rdev->gsi_ctx.gsi_sqp);
2616 else
2617 return (qp->ib_qp.qp_type == IB_QPT_GSI);
2618 }
2619
bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev * rdev)2620 static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev)
2621 {
2622 struct bnxt_qplib_qp *qpl_qp;
2623 bool dev_detached = false;
2624 struct ib_qp_attr qp_attr;
2625 int num_qps_stopped = 0;
2626 int mask = IB_QP_STATE;
2627 struct bnxt_re_qp *qp;
2628 unsigned long flags;
2629
2630 if (!rdev)
2631 return;
2632
2633 restart:
2634 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
2635 dev_detached = true;
2636
2637 qp_attr.qp_state = IB_QPS_ERR;
2638 mutex_lock(&rdev->qp_lock);
2639 list_for_each_entry(qp, &rdev->qp_list, list) {
2640 qpl_qp = &qp->qplib_qp;
2641 if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
2642 if (qpl_qp->state !=
2643 CMDQ_MODIFY_QP_NEW_STATE_RESET &&
2644 qpl_qp->state !=
2645 CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2646 if (dev_detached) {
2647 /*
2648 * Cant actually send the command down,
2649 * marking the state for bookkeeping
2650 */
2651 qpl_qp->state =
2652 CMDQ_MODIFY_QP_NEW_STATE_ERR;
2653 qpl_qp->cur_qp_state = qpl_qp->state;
2654 if (!qpl_qp->is_user) {
2655 /* Add to flush list */
2656 flags = bnxt_re_lock_cqs(qp);
2657 bnxt_qplib_add_flush_qp(qpl_qp);
2658 bnxt_re_unlock_cqs(qp, flags);
2659 }
2660 } else {
2661 num_qps_stopped++;
2662 bnxt_re_modify_qp(&qp->ib_qp,
2663 &qp_attr, mask,
2664 NULL);
2665 }
2666
2667 bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp,
2668 1, IB_EVENT_QP_FATAL);
2669 /*
2670 * 1. Release qp_lock after a budget to unblock other verb
2671 * requests (like qp_destroy) from stack.
2672 * 2. Traverse through the qp_list freshly as addition / deletion
2673 * might have happened since qp_lock is getting released here.
2674 */
2675 if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) {
2676 mutex_unlock(&rdev->qp_lock);
2677 goto restart;
2678 }
2679 }
2680 }
2681 }
2682
2683 mutex_unlock(&rdev->qp_lock);
2684 }
2685
bnxt_re_update_gid(struct bnxt_re_dev * rdev)2686 static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
2687 {
2688 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
2689 struct bnxt_qplib_gid gid;
2690 u16 gid_idx, index;
2691 int rc = 0;
2692
2693 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
2694 return 0;
2695
2696 if (sgid_tbl == NULL) {
2697 dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");
2698 return -EINVAL;
2699 }
2700
2701 for (index = 0; index < sgid_tbl->active; index++) {
2702 gid_idx = sgid_tbl->hw_id[index];
2703
2704 if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
2705 sizeof(bnxt_qplib_gid_zero)))
2706 continue;
2707 /* Need to modify the VLAN enable setting of non VLAN GID only
2708 * as setting is done for VLAN GID while adding GID
2709 *
2710 * If disable_prio_vlan_tx is enable, then we'll need to remove the
2711 * vlan entry from the sgid_tbl.
2712 */
2713 if (sgid_tbl->vlan[index] == true)
2714 continue;
2715
2716 memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid));
2717
2718 rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
2719 rdev->dev_addr);
2720 }
2721
2722 return rc;
2723 }
2724
bnxt_re_clear_cc(struct bnxt_re_dev * rdev)2725 static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev)
2726 {
2727 struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param;
2728
2729 if (_is_chip_p7(rdev->chip_ctx)) {
2730 cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
2731 } else {
2732 cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
2733 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
2734 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
2735
2736 if (!is_qport_service_type_supported(rdev))
2737 cc_param->mask |=
2738 (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP |
2739 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP |
2740 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP);
2741 }
2742
2743 cc_param->cur_mask = cc_param->mask;
2744
2745 if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param))
2746 dev_err(rdev_to_dev(rdev), "Failed to modify cc\n");
2747 }
2748
bnxt_re_setup_cc(struct bnxt_re_dev * rdev)2749 static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev)
2750 {
2751 struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param;
2752 int rc;
2753
2754 if (_is_chip_p7(rdev->chip_ctx)) {
2755 cc_param->enable = 0x0;
2756 cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
2757 } else {
2758 cc_param->enable = 0x1;
2759 cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
2760 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
2761 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
2762
2763 if (!is_qport_service_type_supported(rdev))
2764 cc_param->mask |=
2765 (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP |
2766 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP |
2767 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP);
2768 }
2769
2770 cc_param->cur_mask = cc_param->mask;
2771
2772 rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param);
2773 if (rc) {
2774 dev_err(rdev_to_dev(rdev), "Failed to modify cc\n");
2775 return rc;
2776 }
2777 /* Reset the programming mask */
2778 cc_param->mask = 0;
2779 if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) {
2780 cc_param->qp1_tos_dscp = cc_param->tos_dscp;
2781 rc = bnxt_re_update_qp1_tos_dscp(rdev);
2782 if (rc) {
2783 dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d",
2784 __func__, rc);
2785 goto clear;
2786 }
2787 }
2788 return 0;
2789
2790 clear:
2791 bnxt_re_clear_cc(rdev);
2792 return rc;
2793 }
2794
bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev * rdev,struct bnxt_re_dscp2pri * d2p,u16 * count,u16 target_id)2795 int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
2796 struct bnxt_re_dscp2pri *d2p, u16 *count,
2797 u16 target_id)
2798 {
2799 struct bnxt_en_dev *en_dev = rdev->en_dev;
2800 struct hwrm_queue_dscp2pri_qcfg_input req;
2801 struct hwrm_queue_dscp2pri_qcfg_output resp;
2802 struct bnxt_re_dscp2pri *dscp2pri;
2803 struct bnxt_fw_msg fw_msg;
2804 u16 in_count = *count;
2805 dma_addr_t dma_handle;
2806 int rc = 0, i;
2807 u16 data_len;
2808 u8 *kmem;
2809
2810 data_len = *count * sizeof(*dscp2pri);
2811 memset(&fw_msg, 0, sizeof(fw_msg));
2812 memset(&req, 0, sizeof(req));
2813 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
2814 HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id);
2815 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
2816
2817 kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle,
2818 GFP_KERNEL);
2819 if (!kmem) {
2820 dev_err(rdev_to_dev(rdev),
2821 "dma_zalloc_coherent failure, length = %u\n",
2822 (unsigned)data_len);
2823 return -ENOMEM;
2824 }
2825 req.dest_data_addr = cpu_to_le64(dma_handle);
2826 req.dest_data_buffer_size = cpu_to_le16(data_len);
2827 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
2828 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
2829 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
2830 if (rc)
2831 goto out;
2832
2833 /* Upload the DSCP-MASK-PRI tuple(s) */
2834 dscp2pri = (struct bnxt_re_dscp2pri *)kmem;
2835 for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) {
2836 d2p[i].dscp = dscp2pri->dscp;
2837 d2p[i].mask = dscp2pri->mask;
2838 d2p[i].pri = dscp2pri->pri;
2839 dscp2pri++;
2840 }
2841 *count = le16_to_cpu(resp.entry_cnt);
2842 out:
2843 dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle);
2844 return rc;
2845 }
2846
bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev * rdev)2847 int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev)
2848 {
2849 /* Remove the VLAN from the GID entry */
2850 if (rdev->cc_param.disable_prio_vlan_tx)
2851 rdev->qplib_res.prio = false;
2852 else
2853 rdev->qplib_res.prio = true;
2854
2855 return bnxt_re_update_gid(rdev);
2856 }
2857
bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev * rdev,struct bnxt_re_dscp2pri * d2p,u16 count,u16 target_id)2858 int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
2859 struct bnxt_re_dscp2pri *d2p, u16 count,
2860 u16 target_id)
2861 {
2862 struct bnxt_en_dev *en_dev = rdev->en_dev;
2863 struct hwrm_queue_dscp2pri_cfg_input req;
2864 struct hwrm_queue_dscp2pri_cfg_output resp;
2865 struct bnxt_fw_msg fw_msg;
2866 struct bnxt_re_dscp2pri *dscp2pri;
2867 int i, rc, data_len = 3 * 256;
2868 dma_addr_t dma_handle;
2869 u8 *kmem;
2870
2871 memset(&req, 0, sizeof(req));
2872 memset(&fw_msg, 0, sizeof(fw_msg));
2873 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
2874 HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id);
2875 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
2876
2877 kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle,
2878 GFP_KERNEL);
2879 if (!kmem) {
2880 dev_err(rdev_to_dev(rdev),
2881 "dma_alloc_coherent failure, length = %u\n",
2882 (unsigned)data_len);
2883 return -ENOMEM;
2884 }
2885 req.src_data_addr = cpu_to_le64(dma_handle);
2886
2887 /* Download the DSCP-MASK-PRI tuple(s) */
2888 dscp2pri = (struct bnxt_re_dscp2pri *)kmem;
2889 for (i = 0; i < count; i++) {
2890 dscp2pri->dscp = d2p[i].dscp;
2891 dscp2pri->mask = d2p[i].mask;
2892 dscp2pri->pri = d2p[i].pri;
2893 dscp2pri++;
2894 }
2895
2896 req.entry_cnt = cpu_to_le16(count);
2897 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
2898 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
2899 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
2900 dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle);
2901 return rc;
2902 }
2903
bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev * rdev,struct bnxt_re_tc_rec * tc_rec,u16 tid)2904 int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev,
2905 struct bnxt_re_tc_rec *tc_rec, u16 tid)
2906 {
2907 u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1;
2908 struct hwrm_queue_qportcfg_output resp = {0};
2909 struct hwrm_queue_qportcfg_input req = {0};
2910 struct bnxt_en_dev *en_dev = rdev->en_dev;
2911 struct bnxt_fw_msg fw_msg;
2912 bool def_init = false;
2913 u8 *tmp_type;
2914 u8 cos_id;
2915 int rc;
2916
2917 memset(&fw_msg, 0, sizeof(fw_msg));
2918 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG,
2919 -1, tid);
2920 req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1;
2921 if (BNXT_EN_ASYM_Q(en_dev))
2922 req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX);
2923
2924 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
2925 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
2926 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
2927 if (rc)
2928 return rc;
2929
2930 if (!resp.max_configurable_queues)
2931 return -EINVAL;
2932
2933 max_tc = resp.max_configurable_queues;
2934 tc_rec->max_tc = max_tc;
2935
2936 if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE)
2937 tc_rec->serv_type_enabled = true;
2938
2939 qptr = &resp.queue_id0;
2940 type_ptr0 = &resp.queue_id0_service_profile_type;
2941 type_ptr1 = &resp.queue_id1_service_profile_type;
2942 for (tc = 0; tc < max_tc; tc++) {
2943 tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0;
2944
2945 cos_id = *qptr++;
2946 /* RoCE CoS queue is the first cos queue.
2947 * For MP12 and MP17 order is 405 and 141015.
2948 */
2949 if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) {
2950 tc_rec->cos_id_roce = cos_id;
2951 tc_rec->tc_roce = tc;
2952 } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) {
2953 tc_rec->cos_id_cnp = cos_id;
2954 tc_rec->tc_cnp = tc;
2955 } else if (!def_init) {
2956 def_init = true;
2957 tc_rec->tc_def = tc;
2958 tc_rec->cos_id_def = cos_id;
2959 }
2960 qptr++;
2961 }
2962
2963 return rc;
2964 }
2965
bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev * rdev,u16 target_id,struct bnxt_re_cos2bw_cfg * cfg)2966 int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id,
2967 struct bnxt_re_cos2bw_cfg *cfg)
2968 {
2969 struct bnxt_en_dev *en_dev = rdev->en_dev;
2970 struct hwrm_queue_cos2bw_qcfg_output resp;
2971 struct hwrm_queue_cos2bw_qcfg_input req = {0};
2972 struct bnxt_fw_msg fw_msg;
2973 int rc, indx;
2974 void *data;
2975
2976 memset(&fw_msg, 0, sizeof(fw_msg));
2977 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
2978 HWRM_QUEUE_COS2BW_QCFG, -1, target_id);
2979 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
2980
2981 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
2982 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
2983 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
2984 if (rc)
2985 return rc;
2986 data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg,
2987 queue_id);
2988 for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) {
2989 memcpy(&cfg->cfg, data, sizeof(cfg->cfg));
2990 if (indx == 0)
2991 cfg->queue_id = resp.queue_id0;
2992 cfg++;
2993 }
2994
2995 return rc;
2996 }
2997
bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev * rdev,u16 target_id,struct bnxt_re_cos2bw_cfg * cfg)2998 int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id,
2999 struct bnxt_re_cos2bw_cfg *cfg)
3000 {
3001 struct bnxt_en_dev *en_dev = rdev->en_dev;
3002 struct hwrm_queue_cos2bw_cfg_input req = {0};
3003 struct hwrm_queue_cos2bw_cfg_output resp = {0};
3004 struct bnxt_fw_msg fw_msg;
3005 void *data;
3006 int indx;
3007 int rc;
3008
3009 memset(&fw_msg, 0, sizeof(fw_msg));
3010 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
3011 HWRM_QUEUE_COS2BW_CFG, -1, target_id);
3012 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
3013
3014 /* Chimp wants enable bit to retain previous
3015 * config done by L2 driver
3016 */
3017 for (indx = 0; indx < 8; indx++) {
3018 if (cfg[indx].queue_id < 40) {
3019 req.enables |= cpu_to_le32(
3020 HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID <<
3021 indx);
3022 }
3023
3024 data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4);
3025 memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4);
3026 if (indx == 0) {
3027 req.queue_id0 = cfg[0].queue_id;
3028 req.unused_0 = 0;
3029 }
3030 }
3031
3032 memset(&resp, 0, sizeof(resp));
3033 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
3034 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
3035 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
3036 return rc;
3037 }
3038
bnxt_re_host_pf_id_query(struct bnxt_re_dev * rdev,struct bnxt_qplib_query_fn_info * fn_info,u32 * pf_mask,u32 * first_pf)3039 int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev,
3040 struct bnxt_qplib_query_fn_info *fn_info,
3041 u32 *pf_mask, u32 *first_pf)
3042 {
3043 struct hwrm_func_host_pf_ids_query_output resp = {0};
3044 struct hwrm_func_host_pf_ids_query_input req;
3045 struct bnxt_en_dev *en_dev = rdev->en_dev;
3046 struct bnxt_fw_msg fw_msg;
3047 int rc;
3048
3049 memset(&fw_msg, 0, sizeof(fw_msg));
3050 memset(&req, 0, sizeof(req));
3051 bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
3052 HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1);
3053 /* To query the info from the host EPs */
3054 switch (fn_info->host) {
3055 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC:
3056 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0:
3057 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1:
3058 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2:
3059 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3:
3060 req.host = fn_info->host;
3061 break;
3062 default:
3063 req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0;
3064 break;
3065 }
3066
3067 req.filter = fn_info->filter;
3068 if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE)
3069 req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL;
3070
3071 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
3072 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
3073 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
3074
3075
3076 *first_pf = le16_to_cpu(resp.first_pf_id);
3077 *pf_mask = le16_to_cpu(resp.pf_ordinal_mask);
3078
3079 return rc;
3080 }
3081
bnxt_re_put_stats_ctx(struct bnxt_re_dev * rdev)3082 static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev)
3083 {
3084 struct bnxt_qplib_ctx *hctx;
3085 struct bnxt_qplib_res *res;
3086 u16 tid = 0xffff;
3087
3088 res = &rdev->qplib_res;
3089 hctx = res->hctx;
3090
3091 if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) {
3092 bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid);
3093 bnxt_qplib_free_stat_mem(res, &hctx->stats);
3094 }
3095 }
3096
bnxt_re_put_stats2_ctx(struct bnxt_re_dev * rdev)3097 static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev)
3098 {
3099 test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags);
3100 }
3101
bnxt_re_get_stats_ctx(struct bnxt_re_dev * rdev)3102 static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev)
3103 {
3104 struct bnxt_qplib_ctx *hctx;
3105 struct bnxt_qplib_res *res;
3106 u16 tid = 0xffff;
3107 int rc;
3108
3109 res = &rdev->qplib_res;
3110 hctx = res->hctx;
3111
3112 rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats);
3113 if (rc)
3114 return -ENOMEM;
3115 rc = bnxt_re_net_stats_ctx_alloc(rdev, tid);
3116 if (rc)
3117 goto free_stat_mem;
3118 set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags);
3119
3120 return 0;
3121
3122 free_stat_mem:
3123 bnxt_qplib_free_stat_mem(res, &hctx->stats);
3124
3125 return rc;
3126 }
3127
bnxt_re_update_dev_attr(struct bnxt_re_dev * rdev)3128 static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev)
3129 {
3130 int rc;
3131
3132 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw);
3133 if (rc)
3134 return rc;
3135 if (!bnxt_re_check_min_attr(rdev))
3136 return -EINVAL;
3137 return 0;
3138 }
3139
bnxt_re_free_tbls(struct bnxt_re_dev * rdev)3140 static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev)
3141 {
3142 bnxt_qplib_clear_tbls(&rdev->qplib_res);
3143 bnxt_qplib_free_tbls(&rdev->qplib_res);
3144 }
3145
bnxt_re_alloc_init_tbls(struct bnxt_re_dev * rdev)3146 static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev)
3147 {
3148 struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx;
3149 u8 pppp_factor = 0;
3150 int rc;
3151
3152 /*
3153 * TODO: Need a better mechanism for spreading of the
3154 * 512 extended PPP pages. For now, spreading it
3155 * based on port_count
3156 */
3157 if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push)
3158 pppp_factor = rdev->en_dev->port_count;
3159 rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor);
3160 if (rc)
3161 return rc;
3162 bnxt_qplib_init_tbls(&rdev->qplib_res);
3163 set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags);
3164
3165 return 0;
3166 }
3167
bnxt_re_clean_nqs(struct bnxt_re_dev * rdev)3168 static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev)
3169 {
3170 struct bnxt_qplib_nq *nq;
3171 int i;
3172
3173 if (!rdev->nqr.max_init)
3174 return;
3175
3176 for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) {
3177 nq = &rdev->nqr.nq[i];
3178 bnxt_qplib_disable_nq(nq);
3179 bnxt_re_net_ring_free(rdev, nq->ring_id);
3180 bnxt_qplib_free_nq_mem(nq);
3181 }
3182 rdev->nqr.max_init = 0;
3183 }
3184
bnxt_re_setup_nqs(struct bnxt_re_dev * rdev)3185 static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev)
3186 {
3187 struct bnxt_re_ring_attr rattr = {};
3188 struct bnxt_qplib_nq *nq;
3189 int rc, i;
3190 int depth;
3191 u32 offt;
3192 u16 vec;
3193
3194 mutex_init(&rdev->nqr.load_lock);
3195 /*
3196 * TODO: Optimize the depth based on the
3197 * number of NQs.
3198 */
3199 depth = BNXT_QPLIB_NQE_MAX_CNT;
3200 for (i = 0; i < rdev->nqr.num_msix - 1; i++) {
3201 nq = &rdev->nqr.nq[i];
3202 vec = rdev->nqr.msix_entries[i + 1].vector;
3203 offt = rdev->nqr.msix_entries[i + 1].db_offset;
3204 nq->hwq.max_elements = depth;
3205 rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq);
3206 if (rc) {
3207 dev_err(rdev_to_dev(rdev),
3208 "Failed to get mem for NQ %d, rc = 0x%x",
3209 i, rc);
3210 goto fail_mem;
3211 }
3212
3213 rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
3214 rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count;
3215 rattr.type = bnxt_re_get_rtype(rdev);
3216 rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX;
3217 rattr.depth = nq->hwq.max_elements - 1;
3218 rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx;
3219
3220 /* Set DBR pacing capability on the first NQ ring only */
3221 if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx))
3222 rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING;
3223 else
3224 rattr.flags = 0;
3225
3226 rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
3227 if (rc) {
3228 nq->ring_id = 0xffff; /* Invalid ring-id */
3229 dev_err(rdev_to_dev(rdev),
3230 "Failed to get fw id for NQ %d, rc = 0x%x",
3231 i, rc);
3232 goto fail_ring;
3233 }
3234
3235 rc = bnxt_qplib_enable_nq(nq, i, vec, offt,
3236 &bnxt_re_cqn_handler,
3237 &bnxt_re_srqn_handler);
3238 if (rc) {
3239 dev_err(rdev_to_dev(rdev),
3240 "Failed to enable NQ %d, rc = 0x%x", i, rc);
3241 goto fail_en;
3242 }
3243 }
3244
3245 rdev->nqr.max_init = i;
3246 return 0;
3247 fail_en:
3248 /* *nq was i'th nq */
3249 bnxt_re_net_ring_free(rdev, nq->ring_id);
3250 fail_ring:
3251 bnxt_qplib_free_nq_mem(nq);
3252 fail_mem:
3253 rdev->nqr.max_init = i;
3254 return rc;
3255 }
3256
bnxt_re_sysfs_destroy_file(struct bnxt_re_dev * rdev)3257 static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev)
3258 {
3259 int i;
3260
3261 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++)
3262 device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]);
3263 }
3264
bnxt_re_sysfs_create_file(struct bnxt_re_dev * rdev)3265 static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev)
3266 {
3267 int i, j, rc = 0;
3268
3269 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) {
3270 rc = device_create_file(&rdev->ibdev.dev,
3271 bnxt_re_attributes[i]);
3272 if (rc) {
3273 dev_err(rdev_to_dev(rdev),
3274 "Failed to create IB sysfs with rc = 0x%x", rc);
3275 /* Must clean up all created device files */
3276 for (j = 0; j < i; j++)
3277 device_remove_file(&rdev->ibdev.dev,
3278 bnxt_re_attributes[j]);
3279 clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
3280 ib_unregister_device(&rdev->ibdev);
3281 return 1;
3282 }
3283 }
3284 return 0;
3285 }
3286
3287 /* worker thread for polling periodic events. Now used for QoS programming*/
bnxt_re_worker(struct work_struct * work)3288 static void bnxt_re_worker(struct work_struct *work)
3289 {
3290 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
3291 worker.work);
3292 int rc;
3293
3294 /* QoS is in 30s cadence for PFs*/
3295 if (!rdev->is_virtfn && !rdev->worker_30s--)
3296 rdev->worker_30s = 30;
3297 /* Use trylock for bnxt_re_dev_lock as this can be
3298 * held for long time by debugfs show path while issuing
3299 * HWRMS. If the debugfs name update is not done in this
3300 * iteration, the driver will check for the same in the
3301 * next schedule of the worker i.e after 1 sec.
3302 */
3303 if (mutex_trylock(&bnxt_re_dev_lock))
3304 mutex_unlock(&bnxt_re_dev_lock);
3305
3306 if (!rdev->stats.stats_query_sec)
3307 goto resched;
3308
3309 if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) &&
3310 (rdev->is_virtfn ||
3311 !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) {
3312 if (!(rdev->stats.stats_query_counter++ %
3313 rdev->stats.stats_query_sec)) {
3314 rc = bnxt_re_get_qos_stats(rdev);
3315 if (rc && rc != -ENOMEM)
3316 clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS,
3317 &rdev->flags);
3318 }
3319 }
3320
3321 resched:
3322 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000));
3323 }
3324
bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev * rdev)3325 static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev)
3326 {
3327 if (!(rdev->dbr_drop_recov || rdev->dbr_pacing))
3328 return 0;
3329
3330 rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL);
3331 if (!rdev->dbr_sw_stats)
3332 return -ENOMEM;
3333
3334 return 0;
3335 }
3336
bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev * rdev)3337 static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev)
3338 {
3339 kfree(rdev->dbr_sw_stats);
3340 rdev->dbr_sw_stats = NULL;
3341 }
3342
bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev * rdev)3343 static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev)
3344 {
3345 rdev->dbr_drop_recov_wq =
3346 create_singlethread_workqueue("bnxt_re_dbr_drop_recov");
3347 if (!rdev->dbr_drop_recov_wq) {
3348 dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!");
3349 return -EINVAL;
3350 }
3351 rdev->dbr_drop_recov = true;
3352
3353 /* Enable configfs setting dbr_drop_recov by default*/
3354 rdev->user_dbr_drop_recov = true;
3355
3356 rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT;
3357 return 0;
3358 }
3359
bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev * rdev)3360 static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev)
3361 {
3362 if (rdev->dbr_drop_recov_wq) {
3363 flush_workqueue(rdev->dbr_drop_recov_wq);
3364 destroy_workqueue(rdev->dbr_drop_recov_wq);
3365 rdev->dbr_drop_recov_wq = NULL;
3366 }
3367 rdev->dbr_drop_recov = false;
3368 }
3369
bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev * rdev)3370 static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev)
3371 {
3372 int rc;
3373
3374 /* Allocate a page for app use */
3375 rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL);
3376 if (!rdev->dbr_page) {
3377 dev_err(rdev_to_dev(rdev), "DBR page allocation failed!");
3378 return -ENOMEM;
3379 }
3380 memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE);
3381 rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page;
3382 rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev);
3383 if (rc) {
3384 dev_err(rdev_to_dev(rdev),
3385 "Failed to query dbr pacing config %d\n", rc);
3386 goto fail;
3387 }
3388 /* Create a work queue for scheduling dbq event */
3389 rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq");
3390 if (!rdev->dbq_wq) {
3391 dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!");
3392 rc = -ENOMEM;
3393 goto fail;
3394 }
3395 /* MAP grc window 2 for reading db fifo depth */
3396 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0,
3397 rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK);
3398 rdev->dbr_db_fifo_reg_off =
3399 (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) +
3400 0x2000;
3401 rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off;
3402
3403 rdev->dbr_bar_addr =
3404 pci_resource_start(rdev->qplib_res.pdev, 0) +
3405 rdev->dbr_db_fifo_reg_off;
3406
3407 /* Percentage of DB FIFO */
3408 rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD;
3409 rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD;
3410 rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD;
3411 rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME;
3412 rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION;
3413 rdev->do_pacing_save = rdev->dbr_def_do_pacing;
3414 bnxt_re_set_default_pacing_data(rdev);
3415 dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n");
3416
3417 return 0;
3418 fail:
3419 free_page((u64)rdev->dbr_page);
3420 rdev->dbr_page = NULL;
3421 return rc;
3422 }
3423
bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev * rdev)3424 static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev)
3425 {
3426 if (rdev->dbq_wq)
3427 flush_workqueue(rdev->dbq_wq);
3428
3429 cancel_work_sync(&rdev->dbq_fifo_check_work);
3430 cancel_delayed_work_sync(&rdev->dbq_pacing_work);
3431
3432 if (rdev->dbq_wq) {
3433 destroy_workqueue(rdev->dbq_wq);
3434 rdev->dbq_wq = NULL;
3435 }
3436
3437 if (rdev->dbr_page)
3438 free_page((u64)rdev->dbr_page);
3439 rdev->dbr_page = NULL;
3440 rdev->dbr_pacing = false;
3441 }
3442
3443 /* enable_dbr_pacing needs to be done only for older FWs
3444 * where host selects primary function. ie. pacing_ext
3445 * flags is not set.
3446 */
bnxt_re_enable_dbr_pacing(struct bnxt_re_dev * rdev)3447 int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev)
3448 {
3449 struct bnxt_qplib_nq *nq;
3450
3451 nq = &rdev->nqr.nq[0];
3452 rdev->dbq_nq_id = nq->ring_id;
3453
3454 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
3455 bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
3456 if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) {
3457 dev_err(rdev_to_dev(rdev),
3458 "Failed to set dbr pacing config\n");
3459 return -EIO;
3460 }
3461 /* MAP grc window 8 for ARMing the NQ DBQ */
3462 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0,
3463 rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK);
3464 rdev->dbr_aeq_arm_reg_off =
3465 (rdev->chip_ctx->dbr_aeq_arm_reg &
3466 BNXT_GRC_OFFSET_MASK) + 0x8000;
3467 writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1);
3468 }
3469
3470 return 0;
3471 }
3472
3473 /* disable_dbr_pacing needs to be done only for older FWs
3474 * where host selects primary function. ie. pacing_ext
3475 * flags is not set.
3476 */
3477
bnxt_re_disable_dbr_pacing(struct bnxt_re_dev * rdev)3478 int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev)
3479 {
3480 int rc = 0;
3481
3482 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
3483 bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx))
3484 rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false);
3485
3486 return rc;
3487 }
3488
bnxt_re_ib_uninit(struct bnxt_re_dev * rdev)3489 static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev)
3490 {
3491 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
3492 bnxt_re_sysfs_destroy_file(rdev);
3493 /* Cleanup ib dev */
3494 ib_unregister_device(&rdev->ibdev);
3495 clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
3496 return;
3497 }
3498 }
3499
bnxt_re_dev_uninit(struct bnxt_re_dev * rdev,u8 op_type)3500 static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)
3501 {
3502 struct bnxt_qplib_dpi *kdpi;
3503 int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT;
3504
3505 bnxt_re_net_unregister_async_event(rdev);
3506
3507 bnxt_re_put_stats2_ctx(rdev);
3508 if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED,
3509 &rdev->flags)) {
3510 /* did the caller hold the lock? */
3511 mutex_lock(&bnxt_re_dev_lock);
3512 list_del_rcu(&rdev->list);
3513 mutex_unlock(&bnxt_re_dev_lock);
3514 }
3515
3516 bnxt_re_uninit_dcb_wq(rdev);
3517 bnxt_re_uninit_aer_wq(rdev);
3518
3519 bnxt_re_deinitialize_dbr_drop_recov(rdev);
3520
3521 if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx))
3522 (void)bnxt_re_disable_dbr_pacing(rdev);
3523
3524 if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) {
3525 cancel_delayed_work_sync(&rdev->worker);
3526 }
3527
3528 /* Wait for ULPs to release references */
3529 while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count)
3530 usleep_range(500, 1000);
3531 if (!wait_count)
3532 dev_err(rdev_to_dev(rdev),
3533 "CQ resources not freed by stack, count = 0x%x",
3534 atomic_read(&rdev->stats.rsors.cq_count));
3535
3536 kdpi = &rdev->dpi_privileged;
3537 if (kdpi->umdbr) { /* kernel DPI was allocated with success */
3538 (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi);
3539 /*
3540 * Driver just need to know no command had failed
3541 * during driver load sequence and below command is
3542 * required indeed. Piggybacking dpi allocation status.
3543 */
3544 }
3545
3546 /* Protect the device uninitialization and start_irq/stop_irq L2
3547 * callbacks with rtnl lock to avoid race condition between these calls
3548 */
3549 rtnl_lock();
3550 if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags))
3551 bnxt_re_clean_nqs(rdev);
3552 rtnl_unlock();
3553
3554 if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags))
3555 bnxt_re_free_tbls(rdev);
3556 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) {
3557 rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
3558 if (rc)
3559 dev_warn(rdev_to_dev(rdev),
3560 "Failed to deinitialize fw, rc = 0x%x", rc);
3561 }
3562
3563 bnxt_re_put_stats_ctx(rdev);
3564
3565 if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags))
3566 bnxt_qplib_free_hwctx(&rdev->qplib_res);
3567
3568 rtnl_lock();
3569 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags))
3570 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
3571
3572 if (rdev->dbr_pacing)
3573 bnxt_re_deinitialize_dbr_pacing(rdev);
3574
3575 bnxt_re_free_dbr_sw_stats_mem(rdev);
3576
3577 if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags))
3578 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id);
3579
3580 if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags))
3581 bnxt_qplib_free_rcfw_channel(&rdev->qplib_res);
3582
3583 if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags))
3584 bnxt_re_free_msix(rdev);
3585 rtnl_unlock();
3586
3587 bnxt_re_destroy_chip_ctx(rdev);
3588
3589 if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) {
3590 if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED,
3591 &rdev->flags))
3592 bnxt_re_unregister_netdev(rdev);
3593 }
3594 }
3595
bnxt_re_dev_init(struct bnxt_re_dev * rdev,u8 op_type)3596 static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type)
3597 {
3598 struct bnxt_re_ring_attr rattr = {};
3599 struct bnxt_qplib_creq_ctx *creq;
3600 int vec, offset;
3601 int rc = 0;
3602
3603 if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
3604 /* Registered a new RoCE device instance to netdev */
3605 rc = bnxt_re_register_netdev(rdev);
3606 if (rc)
3607 return -EINVAL;
3608 }
3609 set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
3610
3611 rc = bnxt_re_setup_chip_ctx(rdev);
3612 if (rc) {
3613 dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc);
3614 bnxt_re_unregister_netdev(rdev);
3615 clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
3616 rc = -EINVAL;
3617 return rc;
3618 }
3619
3620 /* Protect the device initialization and start_irq/stop_irq L2 callbacks
3621 * with rtnl lock to avoid race condition between these calls
3622 */
3623 rtnl_lock();
3624 rc = bnxt_re_request_msix(rdev);
3625 if (rc) {
3626 dev_err(rdev_to_dev(rdev),
3627 "Requesting MSI-X vectors failed with rc = 0x%x", rc);
3628 rc = -EINVAL;
3629 goto release_rtnl;
3630 }
3631 set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
3632
3633 /* Establish RCFW Communication Channel to initialize the context
3634 memory for the function and all child VFs */
3635 rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res);
3636 if (rc) {
3637 dev_err(rdev_to_dev(rdev),
3638 "Failed to alloc mem for rcfw, rc = %#x\n", rc);
3639 goto release_rtnl;
3640 }
3641 set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags);
3642
3643 creq = &rdev->rcfw.creq;
3644 rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
3645 rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
3646 rattr.type = bnxt_re_get_rtype(rdev);
3647 rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX;
3648 rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
3649 rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
3650 rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
3651 if (rc) {
3652 creq->ring_id = 0xffff;
3653 dev_err(rdev_to_dev(rdev),
3654 "Failed to allocate CREQ fw id with rc = 0x%x", rc);
3655 goto release_rtnl;
3656 }
3657
3658 set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags);
3659
3660 if (!rdev->chip_ctx)
3661 goto release_rtnl;
3662
3663 if (!(_is_chip_p7(rdev->chip_ctx))) {
3664 /* Program the NQ ID for DBQ notification */
3665 if (rdev->chip_ctx->modes.dbr_pacing_v0 ||
3666 bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) ||
3667 bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
3668 rc = bnxt_re_initialize_dbr_pacing(rdev);
3669 if (!rc)
3670 rdev->dbr_pacing = true;
3671 else
3672 rdev->dbr_pacing = false;
3673 dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n",
3674 __func__, rc);
3675 }
3676 }
3677
3678 vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector;
3679 offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset;
3680 rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset,
3681 &bnxt_re_aeq_handler);
3682 if (rc) {
3683 dev_err(rdev_to_dev(rdev),
3684 "Failed to enable RCFW channel with rc = 0x%x", rc);
3685 goto release_rtnl;
3686 }
3687 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
3688
3689 rc = bnxt_re_update_dev_attr(rdev);
3690 if (rc)
3691 goto release_rtnl;
3692 bnxt_re_set_resource_limits(rdev);
3693 if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
3694 rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res);
3695 if (rc) {
3696 dev_err(rdev_to_dev(rdev),
3697 "Failed to alloc hw contexts, rc = 0x%x", rc);
3698 goto release_rtnl;
3699 }
3700 set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags);
3701 }
3702
3703 rc = bnxt_re_get_stats_ctx(rdev);
3704 if (rc)
3705 goto release_rtnl;
3706
3707 rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn);
3708 if (rc) {
3709 dev_err(rdev_to_dev(rdev),
3710 "Failed to initialize fw with rc = 0x%x", rc);
3711 goto release_rtnl;
3712 }
3713 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags);
3714
3715 /* Based resource count on the 'new' device caps */
3716 rc = bnxt_re_update_dev_attr(rdev);
3717 if (rc)
3718 goto release_rtnl;
3719 rc = bnxt_re_alloc_init_tbls(rdev);
3720 if (rc) {
3721 dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x",
3722 rc);
3723 goto release_rtnl;
3724 }
3725 rc = bnxt_re_setup_nqs(rdev);
3726 if (rc) {
3727 dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n",
3728 rc);
3729 if (rdev->nqr.max_init == 0)
3730 goto release_rtnl;
3731
3732 dev_warn(rdev_to_dev(rdev),
3733 "expected nqs %d available nqs %d\n",
3734 rdev->nqr.num_msix, rdev->nqr.max_init);
3735 }
3736 set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags);
3737 rtnl_unlock();
3738
3739 rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged,
3740 rdev, BNXT_QPLIB_DPI_TYPE_KERNEL);
3741 if (rc)
3742 goto fail;
3743
3744 if (rdev->dbr_pacing)
3745 bnxt_re_enable_dbr_pacing(rdev);
3746
3747 if (rdev->chip_ctx->modes.dbr_drop_recov)
3748 bnxt_re_initialize_dbr_drop_recov(rdev);
3749
3750 rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev);
3751 if (rc)
3752 goto fail;
3753
3754 /* This block of code is needed for error recovery support */
3755 if (!rdev->is_virtfn) {
3756 struct bnxt_re_tc_rec *tc_rec;
3757
3758 tc_rec = &rdev->tc_rec[0];
3759 rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF);
3760 if (rc) {
3761 dev_err(rdev_to_dev(rdev),
3762 "Failed to query port config rc:%d", rc);
3763 return rc;
3764 }
3765
3766 /* Query f/w defaults of CC params */
3767 rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param);
3768 if (rc)
3769 dev_warn(rdev_to_dev(rdev),
3770 "Failed to query CC defaults\n");
3771 if (1) {
3772 rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev);
3773 if (rdev->num_vfs) {
3774 bnxt_re_set_resource_limits(rdev);
3775 bnxt_qplib_set_func_resources(&rdev->qplib_res);
3776 }
3777 }
3778 }
3779 INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
3780 set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags);
3781 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000));
3782
3783 bnxt_re_init_dcb_wq(rdev);
3784 bnxt_re_init_aer_wq(rdev);
3785 mutex_lock(&bnxt_re_dev_lock);
3786 list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
3787 /* Added to the list, not in progress anymore */
3788 gadd_dev_inprogress--;
3789 set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags);
3790 mutex_unlock(&bnxt_re_dev_lock);
3791
3792
3793 return rc;
3794 release_rtnl:
3795 rtnl_unlock();
3796 fail:
3797 bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);
3798
3799 return rc;
3800 }
3801
bnxt_re_ib_init(struct bnxt_re_dev * rdev)3802 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
3803 {
3804 int rc = 0;
3805
3806 rc = bnxt_re_register_ib(rdev);
3807 if (rc) {
3808 dev_err(rdev_to_dev(rdev),
3809 "Register IB failed with rc = 0x%x", rc);
3810 goto fail;
3811 }
3812 if (bnxt_re_sysfs_create_file(rdev)) {
3813 bnxt_re_stopqps_and_ib_uninit(rdev);
3814 goto fail;
3815 }
3816
3817 set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
3818 set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
3819 set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags);
3820 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
3821 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
3822
3823 return rc;
3824 fail:
3825 bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);
3826 return rc;
3827 }
3828
3829 /* wrapper for ib_init funcs */
_bnxt_re_ib_init(struct bnxt_re_dev * rdev)3830 int _bnxt_re_ib_init(struct bnxt_re_dev *rdev)
3831 {
3832 return bnxt_re_ib_init(rdev);
3833 }
3834
3835 /* wrapper for aux init funcs */
_bnxt_re_ib_init2(struct bnxt_re_dev * rdev)3836 int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev)
3837 {
3838 bnxt_re_ib_init_2(rdev);
3839 return 0; /* add return for future proof */
3840 }
3841
bnxt_re_dev_unreg(struct bnxt_re_dev * rdev)3842 static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
3843 {
3844 bnxt_re_dev_dealloc(rdev);
3845 }
3846
3847
bnxt_re_dev_reg(struct bnxt_re_dev ** rdev,struct ifnet * netdev,struct bnxt_en_dev * en_dev)3848 static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev,
3849 struct bnxt_en_dev *en_dev)
3850 {
3851 struct ifnet *realdev = NULL;
3852
3853 realdev = netdev;
3854 if (realdev)
3855 dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__,
3856 realdev, netdev);
3857 /*
3858 * Note:
3859 * The first argument to bnxt_re_dev_alloc() is 'netdev' and
3860 * not 'realdev', since in the case of bonding we want to
3861 * register the bonded virtual netdev (master) to the ib stack.
3862 * And 'en_dev' (for L2/PCI communication) is the first slave
3863 * device (PF0 on the card).
3864 * In the case of a regular netdev, both netdev and the en_dev
3865 * correspond to the same device.
3866 */
3867 *rdev = bnxt_re_dev_alloc(netdev, en_dev);
3868 if (!*rdev) {
3869 pr_err("%s: netdev %p not handled",
3870 ROCE_DRV_MODULE_NAME, netdev);
3871 return -ENOMEM;
3872 }
3873 bnxt_re_hold(*rdev);
3874
3875 return 0;
3876 }
3877
bnxt_re_get_link_speed(struct bnxt_re_dev * rdev)3878 void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev)
3879 {
3880 rdev->espeed = rdev->en_dev->espeed;
3881 rdev->lanes = rdev->en_dev->lanes;
3882 return;
3883 }
3884
bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev * rdev)3885 void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev)
3886 {
3887 dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n",
3888 __func__, rdev);
3889 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
3890 bnxt_re_ib_uninit(rdev);
3891 }
3892
bnxt_re_remove_device(struct bnxt_re_dev * rdev,u8 op_type,struct auxiliary_device * aux_dev)3893 void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type,
3894 struct auxiliary_device *aux_dev)
3895 {
3896 struct bnxt_re_en_dev_info *en_info;
3897 struct bnxt_qplib_cmdq_ctx *cmdq;
3898 struct bnxt_qplib_rcfw *rcfw;
3899
3900 rcfw = &rdev->rcfw;
3901 cmdq = &rcfw->cmdq;
3902 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
3903 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
3904
3905 dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev);
3906 bnxt_re_dev_uninit(rdev, op_type);
3907 en_info = auxiliary_get_drvdata(aux_dev);
3908 if (en_info) {
3909 rtnl_lock();
3910 en_info->rdev = NULL;
3911 rtnl_unlock();
3912 if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) {
3913 clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags);
3914 clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags);
3915 clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags);
3916 }
3917 }
3918 bnxt_re_dev_unreg(rdev);
3919 }
3920
bnxt_re_add_device(struct bnxt_re_dev ** rdev,struct ifnet * netdev,u8 qp_mode,u8 op_type,u32 num_msix_requested,struct auxiliary_device * aux_dev)3921 int bnxt_re_add_device(struct bnxt_re_dev **rdev,
3922 struct ifnet *netdev,
3923 u8 qp_mode, u8 op_type,
3924 u32 num_msix_requested,
3925 struct auxiliary_device *aux_dev)
3926 {
3927 struct bnxt_re_en_dev_info *en_info;
3928 struct bnxt_en_dev *en_dev;
3929 int rc = 0;
3930
3931 en_info = auxiliary_get_drvdata(aux_dev);
3932 en_dev = en_info->en_dev;
3933
3934 mutex_lock(&bnxt_re_dev_lock);
3935 /* Check if driver already in mod exit and aux_dev is valid */
3936 if (gmod_exit || !aux_dev) {
3937 mutex_unlock(&bnxt_re_dev_lock);
3938 return -ENODEV;
3939 }
3940 /* Add device in progress */
3941 gadd_dev_inprogress++;
3942 mutex_unlock(&bnxt_re_dev_lock);
3943
3944 rc = bnxt_re_dev_reg(rdev, netdev, en_dev);
3945 if (rc) {
3946 dev_dbg(NULL, "Failed to create add device for netdev %p\n",
3947 netdev);
3948 /*
3949 * For BNXT_RE_POST_RECOVERY_INIT special case
3950 * called from bnxt_re_start, the work is
3951 * complete only after, bnxt_re_start completes
3952 * bnxt_unregister_device in case of failure.
3953 * So bnxt_re_start will decrement gadd_dev_inprogress
3954 * in case of failure.
3955 */
3956 if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
3957 mutex_lock(&bnxt_re_dev_lock);
3958 gadd_dev_inprogress--;
3959 mutex_unlock(&bnxt_re_dev_lock);
3960 }
3961 return rc;
3962 }
3963
3964 if (rc != 0)
3965 goto ref_error;
3966
3967 /*
3968 * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load.
3969 * Otherwaise, this invocation can be the result of lag create / destroy,
3970 * err revovery, hot fw upgrade, etc..
3971 */
3972 if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) {
3973 if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES)
3974 num_msix_requested = max_msix_vec[bnxt_re_probe_count++];
3975 else
3976 /* Consider as default when probe_count exceeds its limit */
3977 num_msix_requested = 0;
3978
3979 /* if user specifies only one value, use the same for all PFs */
3980 if (max_msix_vec_argc == 1)
3981 num_msix_requested = max_msix_vec[0];
3982 }
3983
3984 (*rdev)->num_msix_requested = num_msix_requested;
3985 (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode;
3986 (*rdev)->adev = aux_dev;
3987 (*rdev)->dev_addr = en_dev->softc->func.mac_addr;
3988 /* Before updating the rdev pointer in bnxt_re_en_dev_info structure,
3989 * take the rtnl lock to avoid accessing invalid rdev pointer from
3990 * L2 ULP callbacks. This is applicable in all the places where rdev
3991 * pointer is updated in bnxt_re_en_dev_info.
3992 */
3993 rtnl_lock();
3994 en_info->rdev = *rdev;
3995 rtnl_unlock();
3996 rc = bnxt_re_dev_init(*rdev, op_type);
3997 if (rc) {
3998 ref_error:
3999 bnxt_re_dev_unreg(*rdev);
4000 *rdev = NULL;
4001 /*
4002 * For BNXT_RE_POST_RECOVERY_INIT special case
4003 * called from bnxt_re_start, the work is
4004 * complete only after, bnxt_re_start completes
4005 * bnxt_unregister_device in case of failure.
4006 * So bnxt_re_start will decrement gadd_dev_inprogress
4007 * in case of failure.
4008 */
4009 if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
4010 mutex_lock(&bnxt_re_dev_lock);
4011 gadd_dev_inprogress--;
4012 mutex_unlock(&bnxt_re_dev_lock);
4013 }
4014 }
4015 dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev);
4016 if (!rc) {
4017 set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags);
4018 }
4019 return rc;
4020 }
4021
bnxt_re_get_peer_pf(struct bnxt_re_dev * rdev)4022 struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev)
4023 {
4024 struct pci_dev *pdev_in = rdev->en_dev->pdev;
4025 int tmp_bus_num, bus_num = pdev_in->bus->number;
4026 int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn);
4027 int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn);
4028 struct bnxt_re_dev *tmp_rdev;
4029
4030 rcu_read_lock();
4031 list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) {
4032 tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number;
4033 tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn);
4034 tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn);
4035
4036 if (bus_num == tmp_bus_num && dev_num == tmp_dev_num &&
4037 func_num != tmp_func_num) {
4038 rcu_read_unlock();
4039 return tmp_rdev;
4040 }
4041 }
4042 rcu_read_unlock();
4043 return NULL;
4044 }
4045
4046
bnxt_re_schedule_work(struct bnxt_re_dev * rdev,unsigned long event,struct ifnet * vlan_dev,struct ifnet * netdev,struct auxiliary_device * adev)4047 int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
4048 struct ifnet *vlan_dev,
4049 struct ifnet *netdev,
4050 struct auxiliary_device *adev)
4051 {
4052 struct bnxt_re_work *re_work;
4053
4054 /* Allocate for the deferred task */
4055 re_work = kzalloc(sizeof(*re_work), GFP_KERNEL);
4056 if (!re_work)
4057 return -ENOMEM;
4058
4059 re_work->rdev = rdev;
4060 re_work->event = event;
4061 re_work->vlan_dev = vlan_dev;
4062 re_work->adev = adev;
4063 INIT_WORK(&re_work->work, bnxt_re_task);
4064 if (rdev)
4065 atomic_inc(&rdev->sched_count);
4066 re_work->netdev = netdev;
4067 queue_work(bnxt_re_wq, &re_work->work);
4068
4069 return 0;
4070 }
4071
4072
bnxt_re_get_slot_pf_count(struct bnxt_re_dev * rdev)4073 int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev)
4074 {
4075 struct pci_dev *pdev_in = rdev->en_dev->pdev;
4076 int tmp_bus_num, bus_num = pdev_in->bus->number;
4077 int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn);
4078 struct bnxt_re_dev *tmp_rdev;
4079 int pf_cnt = 0;
4080
4081 rcu_read_lock();
4082 list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) {
4083 tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number;
4084 tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn);
4085
4086 if (bus_num == tmp_bus_num && dev_num == tmp_dev_num)
4087 pf_cnt++;
4088 }
4089 rcu_read_unlock();
4090 return pf_cnt;
4091 }
4092
4093 /* Handle all deferred netevents tasks */
bnxt_re_task(struct work_struct * work)4094 static void bnxt_re_task(struct work_struct *work)
4095 {
4096 struct bnxt_re_en_dev_info *en_info;
4097 struct auxiliary_device *aux_dev;
4098 struct bnxt_re_work *re_work;
4099 struct bnxt_re_dev *rdev;
4100
4101 re_work = container_of(work, struct bnxt_re_work, work);
4102
4103 mutex_lock(&bnxt_re_mutex);
4104 rdev = re_work->rdev;
4105
4106 /*
4107 * If the previous rdev is deleted due to bond creation
4108 * do not handle the event
4109 */
4110 if (!bnxt_re_is_rdev_valid(rdev))
4111 goto exit;
4112
4113 /* Ignore the event, if the device is not registred with IB stack. This
4114 * is to avoid handling any event while the device is added/removed.
4115 */
4116 if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
4117 dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx",
4118 __func__, re_work->event);
4119 goto done;
4120 }
4121
4122 /* Extra check to silence coverity. We shouldn't handle any event
4123 * when rdev is NULL.
4124 */
4125 if (!rdev)
4126 goto exit;
4127
4128 dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx",
4129 re_work->event);
4130
4131 switch (re_work->event) {
4132 case NETDEV_UP:
4133 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
4134 IB_EVENT_PORT_ACTIVE);
4135 bnxt_re_net_register_async_event(rdev);
4136 break;
4137
4138 case NETDEV_DOWN:
4139 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0);
4140 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
4141 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
4142 IB_EVENT_PORT_ERR);
4143 break;
4144
4145 case NETDEV_CHANGE:
4146 if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) {
4147 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
4148 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
4149 IB_EVENT_PORT_ERR);
4150 break;
4151 } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) {
4152 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
4153 IB_EVENT_PORT_ACTIVE);
4154 }
4155
4156 /* temporarily disable the check for SR2 */
4157 if (!bnxt_qplib_query_cc_param(&rdev->qplib_res,
4158 &rdev->cc_param) &&
4159 !_is_chip_p7(rdev->chip_ctx)) {
4160 /*
4161 * Disable CC for 10G speed
4162 * for non p5 devices
4163 */
4164 if (rdev->sl_espeed == SPEED_10000 &&
4165 !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
4166 if (rdev->cc_param.enable)
4167 bnxt_re_clear_cc(rdev);
4168 } else {
4169 if (!rdev->cc_param.enable &&
4170 rdev->cc_param.admin_enable)
4171 bnxt_re_setup_cc(rdev);
4172 }
4173 }
4174 break;
4175
4176 case NETDEV_UNREGISTER:
4177 bnxt_re_stopqps_and_ib_uninit(rdev);
4178 aux_dev = rdev->adev;
4179 if (re_work->adev)
4180 goto done;
4181
4182 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev);
4183
4184 break;
4185
4186 default:
4187 break;
4188 }
4189 done:
4190 if (rdev) {
4191 /* memory barrier to guarantee task completion
4192 * before decrementing sched count
4193 */
4194 mmiowb();
4195 atomic_dec(&rdev->sched_count);
4196 }
4197 exit:
4198 if (re_work->adev && re_work->event == NETDEV_UNREGISTER) {
4199 en_info = auxiliary_get_drvdata(re_work->adev);
4200 en_info->ib_uninit_done = true;
4201 wake_up(&en_info->waitq);
4202 }
4203 kfree(re_work);
4204 mutex_unlock(&bnxt_re_mutex);
4205 }
4206
4207 /*
4208 "Notifier chain callback can be invoked for the same chain from
4209 different CPUs at the same time".
4210
4211 For cases when the netdev is already present, our call to the
4212 register_netdevice_notifier() will actually get the rtnl_lock()
4213 before sending NETDEV_REGISTER and (if up) NETDEV_UP
4214 events.
4215
4216 But for cases when the netdev is not already present, the notifier
4217 chain is subjected to be invoked from different CPUs simultaneously.
4218
4219 This is protected by the netdev_mutex.
4220 */
bnxt_re_netdev_event(struct notifier_block * notifier,unsigned long event,void * ptr)4221 static int bnxt_re_netdev_event(struct notifier_block *notifier,
4222 unsigned long event, void *ptr)
4223 {
4224 struct ifnet *real_dev, *netdev;
4225 struct bnxt_re_dev *rdev = NULL;
4226
4227 netdev = netdev_notifier_info_to_ifp(ptr);
4228 real_dev = rdma_vlan_dev_real_dev(netdev);
4229 if (!real_dev)
4230 real_dev = netdev;
4231 /* In case of bonding,this will be bond's rdev */
4232 rdev = bnxt_re_from_netdev(real_dev);
4233
4234 if (!rdev)
4235 goto exit;
4236
4237 dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n",
4238 __func__, bnxt_re_netevent(event), event,
4239 rdev ? rdev->netdev ? if_getdname(rdev->netdev) : "->netdev = NULL" : "= NULL",
4240 (real_dev == netdev) ? "= netdev" : if_getdname(real_dev));
4241
4242 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
4243 goto exit;
4244
4245 bnxt_re_hold(rdev);
4246
4247 if (real_dev != netdev) {
4248 switch (event) {
4249 case NETDEV_UP:
4250 bnxt_re_schedule_work(rdev, event, netdev,
4251 NULL, NULL);
4252 break;
4253 case NETDEV_DOWN:
4254 break;
4255 default:
4256 break;
4257 }
4258 goto done;
4259 }
4260
4261 switch (event) {
4262 case NETDEV_CHANGEADDR:
4263 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
4264 bnxt_re_update_shadow_ah(rdev);
4265 bnxt_qplib_get_guid(rdev->dev_addr,
4266 (u8 *)&rdev->ibdev.node_guid);
4267 break;
4268
4269 case NETDEV_CHANGE:
4270 bnxt_re_get_link_speed(rdev);
4271 bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL);
4272 break;
4273 case NETDEV_UNREGISTER:
4274 /* netdev notifier will call NETDEV_UNREGISTER again later since
4275 * we are still holding the reference to the netdev
4276 */
4277
4278 /*
4279 * Workaround to avoid ib_unregister hang. Check for module
4280 * reference and dont free up the device if the reference
4281 * is non zero. Checking only for PF functions.
4282 */
4283
4284 if (rdev) {
4285 dev_info(rdev_to_dev(rdev),
4286 "bnxt_re:Unreg recvd when module refcnt > 0");
4287 dev_info(rdev_to_dev(rdev),
4288 "bnxt_re:Close all apps using bnxt_re devs");
4289 dev_info(rdev_to_dev(rdev),
4290 "bnxt_re:Remove the configfs entry created for the device");
4291 dev_info(rdev_to_dev(rdev),
4292 "bnxt_re:Refer documentation for details");
4293 goto done;
4294 }
4295
4296 if (atomic_read(&rdev->sched_count) > 0)
4297 goto done;
4298 if (!rdev->unreg_sched) {
4299 bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER,
4300 NULL, NULL, NULL);
4301 rdev->unreg_sched = true;
4302 goto done;
4303 }
4304
4305 break;
4306 default:
4307 break;
4308 }
4309 done:
4310 if (rdev)
4311 bnxt_re_put(rdev);
4312 exit:
4313 return NOTIFY_DONE;
4314 }
4315
4316 static struct notifier_block bnxt_re_netdev_notifier = {
4317 .notifier_call = bnxt_re_netdev_event
4318 };
4319
bnxt_re_remove_base_interface(struct bnxt_re_dev * rdev,struct auxiliary_device * adev)4320 static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev,
4321 struct auxiliary_device *adev)
4322 {
4323 bnxt_re_stopqps_and_ib_uninit(rdev);
4324 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev);
4325 auxiliary_set_drvdata(adev, NULL);
4326 }
4327
4328 /*
4329 * bnxt_re_remove - Removes the roce aux device
4330 * @adev - aux device pointer
4331 *
4332 * This function removes the roce device. This gets
4333 * called in the mod exit path and pci unbind path.
4334 * If the rdev is bond interace, destroys the lag
4335 * in module exit path, and in pci unbind case
4336 * destroys the lag and recreates other base interface.
4337 * If the device is already removed in error recovery
4338 * path, it just unregister with the L2.
4339 */
bnxt_re_remove(struct auxiliary_device * adev)4340 static void bnxt_re_remove(struct auxiliary_device *adev)
4341 {
4342 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
4343 struct bnxt_en_dev *en_dev;
4344 struct bnxt_re_dev *rdev;
4345 bool primary_dev = false;
4346 bool secondary_dev = false;
4347
4348 if (!en_info)
4349 return;
4350
4351 mutex_lock(&bnxt_re_mutex);
4352 en_dev = en_info->en_dev;
4353
4354 rdev = en_info->rdev;
4355
4356 if (rdev && bnxt_re_is_rdev_valid(rdev)) {
4357 if (pci_channel_offline(rdev->rcfw.pdev))
4358 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
4359
4360 if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags))
4361 primary_dev = true;
4362 if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags))
4363 secondary_dev = true;
4364
4365 /*
4366 * en_dev_info of primary device and secondary device have the
4367 * same rdev pointer when LAG is configured. This rdev pointer
4368 * is rdev of bond interface.
4369 */
4370 if (!primary_dev && !secondary_dev) {
4371 /* removal of non bond interface */
4372 bnxt_re_remove_base_interface(rdev, adev);
4373 } else {
4374 /*
4375 * removal of bond primary/secondary interface. In this
4376 * case bond device is already removed, so rdev->binfo
4377 * is NULL.
4378 */
4379 auxiliary_set_drvdata(adev, NULL);
4380 }
4381 } else {
4382 /* device is removed from ulp stop, unregister the net dev */
4383 if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) {
4384 rtnl_lock();
4385 en_dev->en_ops->bnxt_unregister_device(en_dev,
4386 BNXT_ROCE_ULP);
4387 rtnl_unlock();
4388 }
4389 }
4390 mutex_unlock(&bnxt_re_mutex);
4391 return;
4392 }
4393
4394 /* wrapper for all external user context callers */
_bnxt_re_remove(struct auxiliary_device * adev)4395 void _bnxt_re_remove(struct auxiliary_device *adev)
4396 {
4397 bnxt_re_remove(adev);
4398 }
4399
bnxt_re_ib_init_2(struct bnxt_re_dev * rdev)4400 static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev)
4401 {
4402 int rc;
4403
4404 rc = bnxt_re_get_device_stats(rdev);
4405 if (rc)
4406 dev_err(rdev_to_dev(rdev),
4407 "Failed initial device stat query");
4408
4409 bnxt_re_net_register_async_event(rdev);
4410 }
4411
bnxt_re_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)4412 static int bnxt_re_probe(struct auxiliary_device *adev,
4413 const struct auxiliary_device_id *id)
4414 {
4415 struct bnxt_aux_dev *aux_dev =
4416 container_of(adev, struct bnxt_aux_dev, aux_dev);
4417 struct bnxt_re_en_dev_info *en_info;
4418 struct bnxt_en_dev *en_dev = NULL;
4419 struct bnxt_re_dev *rdev;
4420 int rc = -ENODEV;
4421
4422 if (aux_dev)
4423 en_dev = aux_dev->edev;
4424
4425 if (!en_dev)
4426 return rc;
4427
4428 if (en_dev->ulp_version != BNXT_ULP_VERSION) {
4429 pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n",
4430 ROCE_DRV_MODULE_NAME, en_dev->ulp_version);
4431 return -EINVAL;
4432 }
4433
4434 en_info = kzalloc(sizeof(*en_info), GFP_KERNEL);
4435 if (!en_info)
4436 return -ENOMEM;
4437 memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info));
4438 en_info->en_dev = en_dev;
4439 auxiliary_set_drvdata(adev, en_info);
4440
4441 mutex_lock(&bnxt_re_mutex);
4442 rc = bnxt_re_add_device(&rdev, en_dev->net,
4443 BNXT_RE_GSI_MODE_ALL,
4444 BNXT_RE_COMPLETE_INIT,
4445 BNXT_RE_MSIX_FROM_MOD_PARAM, adev);
4446 if (rc) {
4447 mutex_unlock(&bnxt_re_mutex);
4448 return rc;
4449 }
4450
4451 rc = bnxt_re_ib_init(rdev);
4452 if (rc)
4453 goto err;
4454
4455 bnxt_re_ib_init_2(rdev);
4456
4457 dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev);
4458 rdev->adev = adev;
4459
4460 mutex_unlock(&bnxt_re_mutex);
4461
4462 return 0;
4463
4464 err:
4465 mutex_unlock(&bnxt_re_mutex);
4466 bnxt_re_remove(adev);
4467
4468 return rc;
4469 }
4470
4471 static const struct auxiliary_device_id bnxt_re_id_table[] = {
4472 { .name = BNXT_ADEV_NAME ".rdma", },
4473 {},
4474 };
4475
4476 MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
4477
4478 static struct auxiliary_driver bnxt_re_driver = {
4479 .name = "rdma",
4480 .probe = bnxt_re_probe,
4481 .remove = bnxt_re_remove,
4482 .id_table = bnxt_re_id_table,
4483 };
4484
bnxt_re_mod_init(void)4485 static int __init bnxt_re_mod_init(void)
4486 {
4487 int rc = 0;
4488
4489 pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version);
4490
4491 bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
4492 if (!bnxt_re_wq)
4493 return -ENOMEM;
4494
4495 rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier);
4496 if (rc) {
4497 pr_err("%s: Cannot register to netdevice_notifier",
4498 ROCE_DRV_MODULE_NAME);
4499 goto err_netdev;
4500 }
4501
4502 INIT_LIST_HEAD(&bnxt_re_dev_list);
4503
4504 rc = auxiliary_driver_register(&bnxt_re_driver);
4505 if (rc) {
4506 pr_err("%s: Failed to register auxiliary driver\n",
4507 ROCE_DRV_MODULE_NAME);
4508 goto err_auxdrv;
4509 }
4510
4511 return 0;
4512
4513 err_auxdrv:
4514 bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
4515
4516 err_netdev:
4517 destroy_workqueue(bnxt_re_wq);
4518
4519 return rc;
4520 }
4521
bnxt_re_mod_exit(void)4522 static void __exit bnxt_re_mod_exit(void)
4523 {
4524 gmod_exit = 1;
4525 auxiliary_driver_unregister(&bnxt_re_driver);
4526
4527 bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
4528
4529 if (bnxt_re_wq)
4530 destroy_workqueue(bnxt_re_wq);
4531 }
4532
4533 module_init(bnxt_re_mod_init);
4534 module_exit(bnxt_re_mod_exit);
4535