1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: Main component of the bnxt_re driver 29 */ 30 31 #include <linux/if_ether.h> 32 #include <linux/module.h> 33 #include <linux/errno.h> 34 #include <linux/pci.h> 35 #include <linux/dma-mapping.h> 36 #include <linux/slab.h> 37 #include <linux/sched.h> 38 #include <linux/delay.h> 39 #include <linux/fs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/ib_addr.h> 42 #include <rdma/ib_cache.h> 43 #include <dev/mlx5/port.h> 44 #include <dev/mlx5/vport.h> 45 #include <linux/list.h> 46 #include <rdma/ib_smi.h> 47 #include <rdma/ib_umem.h> 48 #include <linux/in.h> 49 #include <linux/etherdevice.h> 50 51 #include "bnxt_re.h" 52 #include "ib_verbs.h" 53 #include "bnxt_re-abi.h" 54 #include "bnxt.h" 55 #include "bnxt_log.h" 56 57 static char drv_version[] = 58 "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \ 59 " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n"; 60 61 #define BNXT_RE_DESC "Broadcom NetXtreme RoCE" 62 #define BNXT_ADEV_NAME "if_bnxt" 63 64 MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver"); 65 MODULE_LICENSE("Dual BSD/GPL"); 66 MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1); 67 MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1); 68 MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1); 69 MODULE_DEPEND(bnxt_re, iflib, 1, 1, 1); 70 MODULE_VERSION(bnxt_re, 1); 71 72 73 DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */ 74 75 static unsigned int restrict_mrs = 0; 76 module_param(restrict_mrs, uint, 0); 77 MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K"); 78 79 unsigned int restrict_stats = 0; 80 module_param(restrict_stats, uint, 0); 81 MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default"); 82 83 unsigned int enable_fc = 1; 84 module_param(enable_fc, uint, 0); 85 MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1"); 86 87 unsigned int min_tx_depth = 1; 88 module_param(min_tx_depth, uint, 0); 89 MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1"); 90 91 static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0}; 92 static unsigned int max_msix_vec_argc; 93 module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444); 94 MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64"); 95 96 unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD; 97 module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644); 98 MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64"); 99 100 101 /* globals */ 102 struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list); 103 static int bnxt_re_probe_count; 104 105 DEFINE_MUTEX(bnxt_re_dev_lock); 106 static u32 gmod_exit; 107 static u32 gadd_dev_inprogress; 108 109 static void bnxt_re_task(struct work_struct *work_task); 110 static struct workqueue_struct *bnxt_re_wq; 111 static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev); 112 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, 113 u32 *offset); 114 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev); 115 static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev); 116 void _bnxt_re_remove(struct auxiliary_device *adev); 117 118 void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); 119 u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); 120 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev); 121 122 int bnxt_re_register_netdevice_notifier(struct notifier_block *nb) 123 { 124 int rc; 125 rc = register_netdevice_notifier(nb); 126 return rc; 127 } 128 129 int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb) 130 { 131 int rc; 132 rc = unregister_netdevice_notifier(nb); 133 return rc; 134 } 135 136 void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev) 137 { 138 ibdev->dma_device = &rdev->en_dev->pdev->dev; 139 } 140 141 u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) 142 { 143 144 if (bar_idx) 145 return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off); 146 else 147 return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off); 148 } 149 150 void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val) 151 { 152 if (bar_idx) 153 bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val)); 154 else 155 bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val)); 156 } 157 158 static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev, 159 u32 fifo_occup) 160 { 161 if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark) 162 rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup; 163 164 if (fifo_occup > 8 * rdev->pacing_algo_th) 165 rdev->dbg_stats->dbq.fifo_occup_slab_4++; 166 else if (fifo_occup > 4 * rdev->pacing_algo_th) 167 rdev->dbg_stats->dbq.fifo_occup_slab_3++; 168 else if (fifo_occup > 2 * rdev->pacing_algo_th) 169 rdev->dbg_stats->dbq.fifo_occup_slab_2++; 170 else if (fifo_occup > rdev->pacing_algo_th) 171 rdev->dbg_stats->dbq.fifo_occup_slab_1++; 172 } 173 174 static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev) 175 { 176 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 177 178 if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark) 179 rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing; 180 181 if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing) 182 rdev->dbg_stats->dbq.do_pacing_slab_5++; 183 else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing) 184 rdev->dbg_stats->dbq.do_pacing_slab_4++; 185 else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing) 186 rdev->dbg_stats->dbq.do_pacing_slab_3++; 187 else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing) 188 rdev->dbg_stats->dbq.do_pacing_slab_2++; 189 else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing) 190 rdev->dbg_stats->dbq.do_pacing_slab_1++; 191 } 192 193 static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp) 194 { 195 return qp->ib_qp.qp_type == IB_QPT_GSI; 196 } 197 198 static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev) 199 { 200 struct bnxt_re_qp *qp; 201 202 mutex_lock(&rdev->qp_lock); 203 list_for_each_entry(qp, &rdev->qp_list, list) { 204 if (bnxt_re_is_qp1_qp(qp)) { 205 mutex_unlock(&rdev->qp_lock); 206 return qp; 207 } 208 } 209 mutex_unlock(&rdev->qp_lock); 210 return NULL; 211 } 212 213 /* Set the maximum number of each resource that the driver actually wants 214 * to allocate. This may be up to the maximum number the firmware has 215 * reserved for the function. The driver may choose to allocate fewer 216 * resources than the firmware maximum. 217 */ 218 static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev) 219 { 220 struct bnxt_qplib_max_res dev_res = {}; 221 struct bnxt_qplib_chip_ctx *cctx; 222 struct bnxt_qplib_dev_attr *attr; 223 struct bnxt_qplib_ctx *hctx; 224 int i; 225 226 attr = rdev->dev_attr; 227 hctx = rdev->qplib_res.hctx; 228 cctx = rdev->chip_ctx; 229 230 bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false); 231 if (!_is_chip_gen_p5_p7(cctx)) { 232 hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp); 233 hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr); 234 /* To accommodate 16k MRs and 16k AHs, 235 * driver has to allocate 32k backing store memory 236 */ 237 hctx->mrw_ctx.max *= 2; 238 hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq); 239 hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq); 240 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) 241 hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i]; 242 } else { 243 hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp; 244 hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr; 245 hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq; 246 hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq; 247 } 248 } 249 250 static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev, 251 struct bnxt_qplib_vf_res *vf_res, 252 u32 num_vf) 253 { 254 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; 255 struct bnxt_qplib_max_res dev_res = {}; 256 257 bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true); 258 vf_res->max_qp = dev_res.max_qp / num_vf; 259 vf_res->max_srq = dev_res.max_srq / num_vf; 260 vf_res->max_cq = dev_res.max_cq / num_vf; 261 /* 262 * MR and AH shares the same backing store, the value specified 263 * for max_mrw is split into half by the FW for MR and AH 264 */ 265 vf_res->max_mrw = dev_res.max_mr * 2 / num_vf; 266 vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF; 267 } 268 269 static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) 270 { 271 struct bnxt_qplib_ctx *hctx; 272 273 hctx = rdev->qplib_res.hctx; 274 memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res)); 275 bnxt_re_limit_pf_res(rdev); 276 277 if (rdev->num_vfs) 278 bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs); 279 } 280 281 static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev) 282 { 283 struct bnxt_qplib_rcfw *rcfw = NULL; 284 struct bnxt_qplib_nq *nq; 285 int indx; 286 287 rcfw = &rdev->rcfw; 288 for (indx = 0; indx < rdev->nqr.max_init; indx++) { 289 nq = &rdev->nqr.nq[indx]; 290 mutex_lock(&nq->lock); 291 bnxt_qplib_nq_stop_irq(nq, false); 292 mutex_unlock(&nq->lock); 293 } 294 295 bnxt_qplib_rcfw_stop_irq(rcfw, false); 296 } 297 298 static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev) 299 { 300 /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */ 301 bnxt_re_dettach_irq(rdev); 302 303 /* Set the state as detached to prevent sending any more commands */ 304 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 305 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 306 wake_up_all(&rdev->rcfw.cmdq.waitq); 307 } 308 309 #define MAX_DSCP_PRI_TUPLE 64 310 311 struct bnxt_re_dcb_work { 312 struct work_struct work; 313 struct bnxt_re_dev *rdev; 314 struct hwrm_async_event_cmpl cmpl; 315 }; 316 317 static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) 318 { 319 rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); 320 } 321 322 static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) 323 { 324 if (!rdev->dcb_wq) 325 return; 326 flush_workqueue(rdev->dcb_wq); 327 destroy_workqueue(rdev->dcb_wq); 328 rdev->dcb_wq = NULL; 329 } 330 331 static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev) 332 { 333 rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq"); 334 } 335 336 static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev) 337 { 338 if (!rdev->aer_wq) 339 return; 340 flush_workqueue(rdev->aer_wq); 341 destroy_workqueue(rdev->aer_wq); 342 rdev->aer_wq = NULL; 343 } 344 345 static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) 346 { 347 struct bnxt_re_qp *qp; 348 349 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 350 return 0; 351 352 qp = bnxt_re_get_qp1_qp(rdev); 353 if (!qp) 354 return 0; 355 356 qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP; 357 qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp; 358 359 return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 360 } 361 362 static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev) 363 { 364 struct bnxt_qplib_cc_param *cc_param; 365 struct bnxt_re_tc_rec *tc_rec; 366 bool update_cc = false; 367 u8 dscp_user; 368 int rc; 369 370 cc_param = &rdev->cc_param; 371 tc_rec = &rdev->tc_rec[0]; 372 373 if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user)) 374 return; 375 376 if (cc_param->cnp_dscp_user) { 377 dscp_user = (cc_param->cnp_dscp_user & 0x3f); 378 if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) && 379 (cc_param->alt_tos_dscp != dscp_user)) { 380 cc_param->alt_tos_dscp = dscp_user; 381 cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP; 382 update_cc = true; 383 } 384 } 385 386 if (cc_param->roce_dscp_user) { 387 dscp_user = (cc_param->roce_dscp_user & 0x3f); 388 if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) && 389 (cc_param->tos_dscp != dscp_user)) { 390 cc_param->tos_dscp = dscp_user; 391 cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; 392 update_cc = true; 393 } 394 } 395 396 if (update_cc) { 397 rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); 398 if (rc) 399 dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n"); 400 } 401 } 402 403 static void bnxt_re_dcb_wq_task(struct work_struct *work) 404 { 405 struct bnxt_qplib_cc_param *cc_param; 406 struct bnxt_re_tc_rec *tc_rec; 407 struct bnxt_re_dev *rdev; 408 struct bnxt_re_dcb_work *dcb_work = 409 container_of(work, struct bnxt_re_dcb_work, work); 410 int rc; 411 412 rdev = dcb_work->rdev; 413 if (!rdev) 414 goto exit; 415 416 mutex_lock(&rdev->cc_lock); 417 418 cc_param = &rdev->cc_param; 419 rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param); 420 if (rc) { 421 dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc); 422 goto fail; 423 } 424 tc_rec = &rdev->tc_rec[0]; 425 /* 426 * Upon the receival of DCB Async event: 427 * If roce_dscp or cnp_dscp or both (which user configured using configfs) 428 * is in the list, re-program the value using modify_roce_cc command 429 */ 430 bnxt_re_reconfigure_dscp(rdev); 431 432 cc_param->roce_pri = tc_rec->roce_prio; 433 if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { 434 cc_param->qp1_tos_dscp = cc_param->tos_dscp; 435 rc = bnxt_re_update_qp1_tos_dscp(rdev); 436 if (rc) { 437 dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d", 438 __func__, rc); 439 goto fail; 440 } 441 } 442 443 fail: 444 mutex_unlock(&rdev->cc_lock); 445 exit: 446 kfree(dcb_work); 447 } 448 449 static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev) 450 { 451 struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0}; 452 struct hwrm_func_dbr_pacing_broadcast_event_input req = {0}; 453 struct bnxt_en_dev *en_dev = rdev->en_dev; 454 struct bnxt_fw_msg fw_msg; 455 int rc; 456 457 memset(&fw_msg, 0, sizeof(fw_msg)); 458 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 459 HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1); 460 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 461 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 462 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 463 if (rc) { 464 dev_dbg(rdev_to_dev(rdev), 465 "Failed to send dbr pacing broadcast event rc:%d", rc); 466 return rc; 467 } 468 return 0; 469 } 470 471 static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev) 472 { 473 struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0}; 474 struct hwrm_func_dbr_pacing_nqlist_query_input req = {0}; 475 struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list; 476 struct bnxt_en_dev *en_dev = rdev->en_dev; 477 bool primary_found = false; 478 struct bnxt_fw_msg fw_msg; 479 struct bnxt_qplib_nq *nq; 480 int rc, i, j = 1; 481 u16 *nql_ptr; 482 483 nq = &rdev->nqr.nq[0]; 484 485 memset(&fw_msg, 0, sizeof(fw_msg)); 486 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 487 HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1); 488 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 489 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 490 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 491 if (rc) { 492 dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc); 493 return rc; 494 } 495 nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs); 496 nql_ptr = &resp.nq_ring_id0; 497 /* populate the nq_list of the primary function with list received 498 * from FW. Fill the NQ IDs of secondary functions from index 1 to 499 * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the 500 * nq_id of the primary pf 501 */ 502 for (i = 0; i < nq_list->num_nql_entries; i++) { 503 u16 nq_id = *nql_ptr; 504 505 dev_dbg(rdev_to_dev(rdev), 506 "nq_list->nq_id[%d] = %d\n", i, nq_id); 507 if (nq_id != nq->ring_id) { 508 nq_list->nq_id[j] = nq_id; 509 j++; 510 } else { 511 primary_found = true; 512 nq_list->nq_id[0] = nq->ring_id; 513 } 514 nql_ptr++; 515 } 516 if (primary_found) 517 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); 518 519 return 0; 520 } 521 522 static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) 523 { 524 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 525 u32 read_val, fifo_occup; 526 bool first_read = true; 527 528 /* loop shouldn't run infintely as the occupancy usually goes 529 * below pacing algo threshold as soon as pacing kicks in. 530 */ 531 while (1) { 532 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0); 533 fifo_occup = pacing_data->fifo_max_depth - 534 ((read_val & pacing_data->fifo_room_mask) >> 535 pacing_data->fifo_room_shift); 536 /* Fifo occupancy cannot be greater the MAX FIFO depth */ 537 if (fifo_occup > pacing_data->fifo_max_depth) 538 break; 539 540 if (first_read) { 541 bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup); 542 first_read = false; 543 } 544 if (fifo_occup < pacing_data->pacing_th) 545 break; 546 } 547 } 548 549 static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) 550 { 551 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 552 553 pacing_data->do_pacing = rdev->dbr_def_do_pacing; 554 pacing_data->pacing_th = rdev->pacing_algo_th; 555 pacing_data->alarm_th = 556 pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); 557 } 558 559 #define CAG_RING_MASK 0x7FF 560 #define CAG_RING_SHIFT 17 561 #define WATERMARK_MASK 0xFFF 562 #define WATERMARK_SHIFT 0 563 564 static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev) 565 { 566 u32 read_val; 567 int j; 568 569 for (j = 0; j < 10; j++) { 570 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0); 571 dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n", 572 read_val); 573 if (!read_val) 574 return true; 575 } 576 return false; 577 } 578 579 int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle) 580 { 581 u32 cag_ring_water_mark = 0, read_val; 582 u32 throttle_val; 583 584 /* Convert throttle percentage to value */ 585 throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100; 586 587 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 588 cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT | 589 (throttle_val & WATERMARK_MASK); 590 writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark); 591 read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0); 592 dev_dbg(rdev_to_dev(rdev), 593 "%s: dbr_throttling_reg_off read_val = 0x%x\n", 594 __func__, read_val); 595 if (read_val != cag_ring_water_mark) { 596 dev_dbg(rdev_to_dev(rdev), 597 "nq_id = %d write_val=0x%x read_val=0x%x\n", 598 nq_id, cag_ring_water_mark, read_val); 599 return 1; 600 } 601 } 602 writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1); 603 return 0; 604 } 605 606 static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev) 607 { 608 struct bnxt_dbq_nq_list *nq_list; 609 struct bnxt_qplib_nq *nq; 610 int i; 611 612 nq_list = &rdev->nq_list; 613 /* Run a loop for other Active functions if this is primary function */ 614 if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { 615 dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n", 616 __func__, nq_list->num_nql_entries); 617 nq = &rdev->nqr.nq[0]; 618 for (i = nq_list->num_nql_entries - 1; i > 0; i--) { 619 u16 nq_id = nq_list->nq_id[i]; 620 if (nq) 621 dev_dbg(rdev_to_dev(rdev), 622 "%s: nq_id = %d cur_fn_ring_id = %d\n", 623 __func__, nq_id, nq->ring_id); 624 if (bnxt_re_set_dbq_throttling_reg 625 (rdev, nq_id, 0)) 626 break; 627 bnxt_re_check_if_dbq_intr_triggered(rdev); 628 } 629 } 630 } 631 632 static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev) 633 { 634 struct bnxt_qplib_nq *nq; 635 int rc = 0; 636 637 nq = &rdev->nqr.nq[0]; 638 639 /* Query the NQ list*/ 640 rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev); 641 if (rc) { 642 dev_err(rdev_to_dev(rdev), 643 "Failed to Query NQ list rc= %d", rc); 644 return; 645 } 646 /*Configure GRC access for Throttling and aeq_arm register */ 647 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0, 648 rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); 649 650 rdev->dbr_throttling_reg_off = 651 (rdev->chip_ctx->dbr_throttling_reg & 652 BNXT_GRC_OFFSET_MASK) + 0x8000; 653 rdev->dbr_aeq_arm_reg_off = 654 (rdev->chip_ctx->dbr_aeq_arm_reg & 655 BNXT_GRC_OFFSET_MASK) + 0x8000; 656 657 bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); 658 } 659 660 static void bnxt_re_dbq_wq_task(struct work_struct *work) 661 { 662 struct bnxt_re_dbq_work *dbq_work = 663 container_of(work, struct bnxt_re_dbq_work, work); 664 struct bnxt_re_dev *rdev; 665 666 rdev = dbq_work->rdev; 667 668 if (!rdev) 669 goto exit; 670 switch (dbq_work->event) { 671 case BNXT_RE_DBQ_EVENT_SCHED: 672 dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n", 673 __func__); 674 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) 675 bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); 676 else 677 bnxt_re_pacing_alert(rdev); 678 break; 679 case BNXT_RE_DBR_PACING_EVENT: 680 dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n", 681 __func__); 682 if (_is_chip_p7(rdev->chip_ctx)) 683 bnxt_re_pacing_alert(rdev); 684 else if (!rdev->chip_ctx->modes.dbr_pacing_v0) 685 bnxt_re_hwrm_dbr_pacing_qcfg(rdev); 686 break; 687 case BNXT_RE_DBR_NQ_PACING_NOTIFICATION: 688 bnxt_re_handle_dbr_nq_pacing_notification(rdev); 689 /* Issue a broadcast event to notify other functions 690 * that primary changed 691 */ 692 bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); 693 break; 694 } 695 exit: 696 kfree(dbq_work); 697 } 698 699 static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl) 700 { 701 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 702 struct bnxt_re_dcb_work *dcb_work; 703 struct bnxt_re_dbq_work *dbq_work; 704 struct bnxt_re_dev *rdev; 705 u16 event_id; 706 u32 data1; 707 u32 data2 = 0; 708 709 if (!cmpl) { 710 pr_err("Async event, bad completion\n"); 711 return; 712 } 713 714 if (!en_info || !en_info->en_dev) { 715 pr_err("Async event, bad en_info or en_dev\n"); 716 return; 717 } 718 rdev = en_info->rdev; 719 720 event_id = le16_to_cpu(cmpl->event_id); 721 data1 = le32_to_cpu(cmpl->event_data1); 722 data2 = le32_to_cpu(cmpl->event_data2); 723 724 if (!rdev || !rdev_to_dev(rdev)) { 725 dev_dbg(NULL, "Async event, bad rdev or netdev\n"); 726 return; 727 } 728 729 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) || 730 !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { 731 dev_dbg(NULL, "Async event, device already detached\n"); 732 return; 733 } 734 if (data2 >= 0) 735 dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d", 736 event_id, data1, data2); 737 738 switch (event_id) { 739 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: 740 /* Not handling the event in older FWs */ 741 if (!is_qport_service_type_supported(rdev)) 742 break; 743 if (!rdev->dcb_wq) 744 break; 745 dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); 746 if (!dcb_work) 747 break; 748 749 dcb_work->rdev = rdev; 750 memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); 751 INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); 752 queue_work(rdev->dcb_wq, &dcb_work->work); 753 break; 754 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: 755 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 756 /* Set rcfw flag to control commands send to Bono */ 757 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 758 /* Set bnxt_re flag to control commands send via L2 driver */ 759 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 760 wake_up_all(&rdev->rcfw.cmdq.waitq); 761 } 762 break; 763 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: 764 if (!rdev->dbr_pacing) 765 break; 766 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); 767 if (!dbq_work) 768 goto unlock; 769 dbq_work->rdev = rdev; 770 dbq_work->event = BNXT_RE_DBR_PACING_EVENT; 771 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); 772 queue_work(rdev->dbq_wq, &dbq_work->work); 773 rdev->dbr_sw_stats->dbq_int_recv++; 774 break; 775 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: 776 if (!rdev->dbr_pacing) 777 break; 778 779 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); 780 if (!dbq_work) 781 goto unlock; 782 dbq_work->rdev = rdev; 783 dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION; 784 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); 785 queue_work(rdev->dbq_wq, &dbq_work->work); 786 break; 787 788 default: 789 break; 790 } 791 unlock: 792 return; 793 } 794 795 static void bnxt_re_db_fifo_check(struct work_struct *work) 796 { 797 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 798 dbq_fifo_check_work); 799 struct bnxt_qplib_db_pacing_data *pacing_data; 800 u32 pacing_save; 801 802 if (!mutex_trylock(&rdev->dbq_lock)) 803 return; 804 pacing_data = rdev->qplib_res.pacing_data; 805 pacing_save = rdev->do_pacing_save; 806 __wait_for_fifo_occupancy_below_th(rdev); 807 cancel_delayed_work_sync(&rdev->dbq_pacing_work); 808 if (rdev->dbr_recovery_on) 809 goto recovery_on; 810 if (pacing_save > rdev->dbr_def_do_pacing) { 811 /* Double the do_pacing value during the congestion */ 812 pacing_save = pacing_save << 1; 813 } else { 814 /* 815 * when a new congestion is detected increase the do_pacing 816 * by 8 times. And also increase the pacing_th by 4 times. The 817 * reason to increase pacing_th is to give more space for the 818 * queue to oscillate down without getting empty, but also more 819 * room for the queue to increase without causing another alarm. 820 */ 821 pacing_save = pacing_save << 3; 822 pacing_data->pacing_th = rdev->pacing_algo_th * 4; 823 } 824 825 if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) 826 pacing_save = BNXT_RE_MAX_DBR_DO_PACING; 827 828 pacing_data->do_pacing = pacing_save; 829 rdev->do_pacing_save = pacing_data->do_pacing; 830 pacing_data->alarm_th = 831 pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); 832 recovery_on: 833 schedule_delayed_work(&rdev->dbq_pacing_work, 834 msecs_to_jiffies(rdev->dbq_pacing_time)); 835 rdev->dbr_sw_stats->dbq_pacing_alerts++; 836 mutex_unlock(&rdev->dbq_lock); 837 } 838 839 static void bnxt_re_pacing_timer_exp(struct work_struct *work) 840 { 841 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 842 dbq_pacing_work.work); 843 struct bnxt_qplib_db_pacing_data *pacing_data; 844 u32 read_val, fifo_occup; 845 struct bnxt_qplib_nq *nq; 846 847 if (!mutex_trylock(&rdev->dbq_lock)) 848 return; 849 850 pacing_data = rdev->qplib_res.pacing_data; 851 read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0); 852 fifo_occup = pacing_data->fifo_max_depth - 853 ((read_val & pacing_data->fifo_room_mask) >> 854 pacing_data->fifo_room_shift); 855 856 if (fifo_occup > pacing_data->pacing_th) 857 goto restart_timer; 858 859 /* 860 * Instead of immediately going back to the default do_pacing 861 * reduce it by 1/8 times and restart the timer. 862 */ 863 pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); 864 pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing); 865 /* 866 * If the fifo_occup is less than the interrupt enable threshold 867 * enable the interrupt on the primary PF. 868 */ 869 if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) { 870 if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { 871 if (!rdev->chip_ctx->modes.dbr_pacing_v0) { 872 nq = &rdev->nqr.nq[0]; 873 bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, 874 rdev->dbq_watermark); 875 rdev->dbr_sw_stats->dbq_int_en++; 876 rdev->dbq_int_disable = false; 877 } 878 } 879 } 880 if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) { 881 bnxt_re_set_default_pacing_data(rdev); 882 rdev->dbr_sw_stats->dbq_pacing_complete++; 883 goto dbq_unlock; 884 } 885 restart_timer: 886 schedule_delayed_work(&rdev->dbq_pacing_work, 887 msecs_to_jiffies(rdev->dbq_pacing_time)); 888 bnxt_re_update_do_pacing_slabs(rdev); 889 rdev->dbr_sw_stats->dbq_pacing_resched++; 890 dbq_unlock: 891 rdev->do_pacing_save = pacing_data->do_pacing; 892 mutex_unlock(&rdev->dbq_lock); 893 } 894 895 void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) 896 { 897 struct bnxt_qplib_db_pacing_data *pacing_data; 898 899 if (!rdev->dbr_pacing) 900 return; 901 mutex_lock(&rdev->dbq_lock); 902 pacing_data = rdev->qplib_res.pacing_data; 903 904 /* 905 * Increase the alarm_th to max so that other user lib instances do not 906 * keep alerting the driver. 907 */ 908 pacing_data->alarm_th = pacing_data->fifo_max_depth; 909 pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; 910 cancel_work_sync(&rdev->dbq_fifo_check_work); 911 schedule_work(&rdev->dbq_fifo_check_work); 912 mutex_unlock(&rdev->dbq_lock); 913 } 914 915 void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res) 916 { 917 struct bnxt_re_dbq_work *dbq_work; 918 struct bnxt_re_dev *rdev; 919 920 rdev = container_of(res, struct bnxt_re_dev, qplib_res); 921 922 atomic_set(&rdev->dbq_intr_running, 1); 923 924 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 925 goto exit; 926 /* Run the loop to send dbq event to other functions 927 * for newer FW 928 */ 929 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && 930 !rdev->chip_ctx->modes.dbr_pacing_v0) 931 bnxt_re_set_dbq_throttling_for_non_primary(rdev); 932 933 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); 934 if (!dbq_work) 935 goto exit; 936 dbq_work->rdev = rdev; 937 dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED; 938 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); 939 queue_work(rdev->dbq_wq, &dbq_work->work); 940 rdev->dbr_sw_stats->dbq_int_recv++; 941 rdev->dbq_int_disable = true; 942 exit: 943 atomic_set(&rdev->dbq_intr_running, 0); 944 } 945 946 static void bnxt_re_free_msix(struct bnxt_re_dev *rdev) 947 { 948 struct bnxt_en_dev *en_dev = rdev->en_dev; 949 int rc; 950 951 rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); 952 if (rc) 953 dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x", 954 rdev->netdev, rc); 955 } 956 957 static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) 958 { 959 struct bnxt_en_dev *en_dev = rdev->en_dev; 960 int rc = 0, num_msix_want, num_msix_got; 961 struct bnxt_msix_entry *entry; 962 963 /* 964 * Request MSIx based on the function type. This is 965 * a temporory solution to enable max VFs when NPAR is 966 * enabled. 967 * TODO - change the scheme with an adapter specific check 968 * as the latest adapters can support more NQs. For now 969 * this change satisfy all adapter versions. 970 */ 971 972 if (rdev->is_virtfn) 973 num_msix_want = BNXT_RE_MAX_MSIX_VF; 974 else if (BNXT_EN_NPAR(en_dev)) 975 num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF; 976 else if (_is_chip_gen_p5_p7(rdev->chip_ctx)) 977 num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF; 978 else 979 num_msix_want = BNXT_RE_MAX_MSIX_PF; 980 981 /* 982 * Since MSIX vectors are used for both NQs and CREQ, we should try to 983 * allocate num_online_cpus + 1 by taking into account the CREQ. This 984 * leaves the number of MSIX vectors for NQs match the number of CPUs 985 * and allows the system to be fully utilized 986 */ 987 num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1); 988 num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX); 989 num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX); 990 991 entry = rdev->nqr.msix_entries; 992 993 num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, 994 entry, num_msix_want); 995 if (num_msix_got < BNXT_RE_MIN_MSIX) { 996 rc = -EINVAL; 997 goto done; 998 } 999 if (num_msix_got != num_msix_want) 1000 dev_warn(rdev_to_dev(rdev), 1001 "bnxt_request_msix: wanted %d vectors, got %d\n", 1002 num_msix_want, num_msix_got); 1003 1004 rdev->nqr.num_msix = num_msix_got; 1005 return 0; 1006 done: 1007 if (num_msix_got) 1008 bnxt_re_free_msix(rdev); 1009 return rc; 1010 } 1011 1012 static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev, 1013 struct bnxt_re_en_dev_info *en_info) 1014 { 1015 u64 timeout = 0; 1016 u32 cur_prod = 0, cur_cons = 0; 1017 int retry = 0, rc = 0, ret = 0; 1018 1019 cur_prod = rdev->rcfw.cmdq.hwq.prod; 1020 cur_cons = rdev->rcfw.cmdq.hwq.cons; 1021 timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS); 1022 retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY; 1023 /* During module exit, increase timeout ten-fold to 100 mins to wait 1024 * as long as possible for ib_unregister() to complete 1025 */ 1026 if (rdev->mod_exit) 1027 retry *= 10; 1028 do { 1029 /* 1030 * Since the caller of this function invokes with bnxt_re_mutex held, 1031 * release it to avoid holding a lock while in wait / sleep mode. 1032 */ 1033 mutex_unlock(&bnxt_re_mutex); 1034 rc = wait_event_timeout(en_info->waitq, 1035 en_info->ib_uninit_done, 1036 timeout); 1037 mutex_lock(&bnxt_re_mutex); 1038 1039 if (!bnxt_re_is_rdev_valid(rdev)) 1040 break; 1041 1042 if (rc) 1043 break; 1044 1045 if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) { 1046 /* No need to check for cmdq stall during module exit, 1047 * wait for ib unregister to complete 1048 */ 1049 if (!rdev->mod_exit) 1050 ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons); 1051 if (ret || en_info->ib_uninit_done) 1052 break; 1053 } 1054 } while (retry--); 1055 1056 return rc; 1057 } 1058 1059 static int bnxt_re_handle_start(struct auxiliary_device *adev) 1060 { 1061 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); 1062 struct bnxt_re_dev *rdev = NULL; 1063 struct ifnet *real_dev; 1064 struct bnxt_en_dev *en_dev; 1065 struct ifnet *netdev; 1066 int rc = 0; 1067 1068 if (!en_info || !en_info->en_dev) { 1069 pr_err("Start, bad en_info or en_dev\n"); 1070 return -EINVAL; 1071 } 1072 netdev = en_info->en_dev->net; 1073 if (en_info->rdev) { 1074 dev_info(rdev_to_dev(en_info->rdev), 1075 "%s: Device is already added adev %p rdev: %p\n", 1076 __func__, adev, en_info->rdev); 1077 return 0; 1078 } 1079 1080 en_dev = en_info->en_dev; 1081 real_dev = rdma_vlan_dev_real_dev(netdev); 1082 if (!real_dev) 1083 real_dev = netdev; 1084 rc = bnxt_re_add_device(&rdev, real_dev, 1085 en_info->gsi_mode, 1086 BNXT_RE_POST_RECOVERY_INIT, 1087 en_info->num_msix_requested, adev); 1088 if (rc) { 1089 /* Add device failed. Unregister the device. 1090 * This has to be done explicitly as 1091 * bnxt_re_stop would not have unregistered 1092 */ 1093 rtnl_lock(); 1094 en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); 1095 rtnl_unlock(); 1096 mutex_lock(&bnxt_re_dev_lock); 1097 gadd_dev_inprogress--; 1098 mutex_unlock(&bnxt_re_dev_lock); 1099 return rc; 1100 } 1101 rdev->adev = adev; 1102 rtnl_lock(); 1103 bnxt_re_get_link_speed(rdev); 1104 rtnl_unlock(); 1105 rc = bnxt_re_ib_init(rdev); 1106 if (rc) { 1107 dev_err(rdev_to_dev(rdev), "Failed ib_init\n"); 1108 return rc; 1109 } 1110 bnxt_re_ib_init_2(rdev); 1111 1112 return rc; 1113 } 1114 1115 static void bnxt_re_stop(void *handle) 1116 { 1117 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1118 struct ifnet *netdev; 1119 struct bnxt_re_dev *rdev; 1120 struct bnxt_en_dev *en_dev; 1121 int rc = 0; 1122 1123 rtnl_unlock(); 1124 mutex_lock(&bnxt_re_mutex); 1125 if (!en_info || !en_info->en_dev) { 1126 pr_err("Stop, bad en_info or en_dev\n"); 1127 goto exit; 1128 } 1129 netdev = en_info->en_dev->net; 1130 rdev = en_info->rdev; 1131 if (!rdev) 1132 goto exit; 1133 1134 if (!bnxt_re_is_rdev_valid(rdev)) 1135 goto exit; 1136 1137 /* 1138 * Check if fw has undergone reset or is in a fatal condition. 1139 * If so, set flags so that no further commands are sent down to FW 1140 */ 1141 en_dev = rdev->en_dev; 1142 if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND || 1143 en_dev->en_state & BNXT_STATE_FW_RESET_DET) { 1144 /* Set rcfw flag to control commands send to Bono */ 1145 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 1146 /* Set bnxt_re flag to control commands send via L2 driver */ 1147 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 1148 wake_up_all(&rdev->rcfw.cmdq.waitq); 1149 } 1150 1151 if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags)) 1152 goto exit; 1153 set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags); 1154 1155 en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode; 1156 en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode; 1157 en_info->num_msix_requested = rdev->num_msix_requested; 1158 en_info->ib_uninit_done = false; 1159 1160 if (rdev->dbr_pacing) 1161 bnxt_re_set_pacing_dev_state(rdev); 1162 1163 dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop." 1164 "Attempting to stop and Dispatching event " 1165 "to inform the stack\n", __func__); 1166 init_waitqueue_head(&en_info->waitq); 1167 /* Schedule a work item to handle IB UNINIT for recovery */ 1168 bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, 1169 NULL, netdev, rdev->adev); 1170 rc = __wait_for_ib_unregister(rdev, en_info); 1171 if (!bnxt_re_is_rdev_valid(rdev)) 1172 goto exit; 1173 if (!rc) { 1174 dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n", 1175 __func__); 1176 bnxt_re_detach_err_device(rdev); 1177 goto exit; 1178 } 1179 bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev); 1180 exit: 1181 mutex_unlock(&bnxt_re_mutex); 1182 /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */ 1183 rtnl_lock(); 1184 1185 return; 1186 } 1187 1188 static void bnxt_re_start(void *handle) 1189 { 1190 rtnl_unlock(); 1191 mutex_lock(&bnxt_re_mutex); 1192 if (bnxt_re_handle_start((struct auxiliary_device *)handle)) 1193 pr_err("Failed to start RoCE device"); 1194 mutex_unlock(&bnxt_re_mutex); 1195 /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */ 1196 rtnl_lock(); 1197 return; 1198 } 1199 1200 static void bnxt_re_shutdown(void *p) 1201 { 1202 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p); 1203 struct bnxt_re_dev *rdev; 1204 1205 if (!en_info) { 1206 pr_err("Shutdown, bad en_info\n"); 1207 return; 1208 } 1209 rtnl_unlock(); 1210 mutex_lock(&bnxt_re_mutex); 1211 rdev = en_info->rdev; 1212 if (!rdev || !bnxt_re_is_rdev_valid(rdev)) 1213 goto exit; 1214 1215 /* rtnl_lock held by L2 before coming here */ 1216 bnxt_re_stopqps_and_ib_uninit(rdev); 1217 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev); 1218 exit: 1219 mutex_unlock(&bnxt_re_mutex); 1220 rtnl_lock(); 1221 return; 1222 } 1223 1224 static void bnxt_re_stop_irq(void *handle) 1225 { 1226 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1227 struct bnxt_qplib_rcfw *rcfw = NULL; 1228 struct bnxt_re_dev *rdev; 1229 struct bnxt_qplib_nq *nq; 1230 int indx; 1231 1232 if (!en_info) { 1233 pr_err("Stop irq, bad en_info\n"); 1234 return; 1235 } 1236 rdev = en_info->rdev; 1237 1238 if (!rdev) 1239 return; 1240 1241 rcfw = &rdev->rcfw; 1242 for (indx = 0; indx < rdev->nqr.max_init; indx++) { 1243 nq = &rdev->nqr.nq[indx]; 1244 mutex_lock(&nq->lock); 1245 bnxt_qplib_nq_stop_irq(nq, false); 1246 mutex_unlock(&nq->lock); 1247 } 1248 1249 if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) 1250 bnxt_qplib_rcfw_stop_irq(rcfw, false); 1251 } 1252 1253 static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) 1254 { 1255 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1256 struct bnxt_msix_entry *msix_ent = NULL; 1257 struct bnxt_qplib_rcfw *rcfw = NULL; 1258 struct bnxt_re_dev *rdev; 1259 struct bnxt_qplib_nq *nq; 1260 int indx, rc, vec; 1261 1262 if (!en_info) { 1263 pr_err("Start irq, bad en_info\n"); 1264 return; 1265 } 1266 rdev = en_info->rdev; 1267 if (!rdev) 1268 return; 1269 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1270 return; 1271 msix_ent = rdev->nqr.msix_entries; 1272 rcfw = &rdev->rcfw; 1273 1274 if (!ent) { 1275 /* Not setting the f/w timeout bit in rcfw. 1276 * During the driver unload the first command 1277 * to f/w will timeout and that will set the 1278 * timeout bit. 1279 */ 1280 dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); 1281 return; 1282 } 1283 1284 /* Vectors may change after restart, so update with new vectors 1285 * in device structure. 1286 */ 1287 for (indx = 0; indx < rdev->nqr.num_msix; indx++) 1288 rdev->nqr.msix_entries[indx].vector = ent[indx].vector; 1289 1290 if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) { 1291 rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, 1292 false); 1293 if (rc) { 1294 dev_warn(rdev_to_dev(rdev), 1295 "Failed to reinit CREQ\n"); 1296 return; 1297 } 1298 } 1299 for (indx = 0 ; indx < rdev->nqr.max_init; indx++) { 1300 nq = &rdev->nqr.nq[indx]; 1301 vec = indx + 1; 1302 rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector, 1303 false); 1304 if (rc) { 1305 dev_warn(rdev_to_dev(rdev), 1306 "Failed to reinit NQ index %d\n", indx); 1307 return; 1308 } 1309 } 1310 } 1311 1312 /* 1313 * bnxt_re_create_snapdump_logs - Collect required debug data for snapdump. 1314 * @rdev - rdma device instance 1315 * 1316 * This function will use bnxt_ulp_log_live API and dump all 1317 * the required information for debugging. 1318 * 1319 * Returns: Nothing 1320 */ 1321 static void 1322 bnxt_re_create_snapdump_logs(struct bnxt_re_dev *rdev) 1323 { 1324 struct bnxt_re_ext_rstat *ext_s; 1325 1326 ext_s = &rdev->stats.dstat.ext_rstat[0]; 1327 1328 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1329 "tx_atomic_req: %llu", ext_s->tx.atomic_req); 1330 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1331 "rx_atomic_req: %llu", ext_s->rx.atomic_req); 1332 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1333 "tx_read_req: %llu\n", ext_s->tx.read_req); 1334 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1335 "tx_read_resp: %llu\n", ext_s->tx.read_resp); 1336 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1337 "rx_read_req: %llu\n", ext_s->rx.read_req); 1338 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1339 "rx_read_resp: %llu\n", ext_s->rx.read_resp); 1340 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1341 "tx_write_req: %llu\n", ext_s->tx.write_req); 1342 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1343 "rx_write_req: %llu\n", ext_s->rx.write_req); 1344 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1345 "tx_send_req: %llu\n", ext_s->tx.send_req); 1346 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1347 "rx_send_req: %llu\n", ext_s->rx.send_req); 1348 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1349 "rx_good_pkts: %llu\n", ext_s->grx.rx_pkts); 1350 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1351 "rx_good_bytes: %llu\n", ext_s->grx.rx_bytes); 1352 } 1353 1354 /* 1355 * bnxt_re_ulp_log_live - Callback from L2 driver to collect snapdump 1356 * @handle - en_dev information. L2 and RoCE device information 1357 * 1358 * This function is callback to support L2 and RoCE common API. 1359 * bnxt_ulp_ops.ulp_log_live(). 1360 * 1361 * Returns: Nothing 1362 */ 1363 static void 1364 bnxt_re_ulp_log_live(void *handle) 1365 { 1366 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1367 1368 if (!en_info || !en_info->rdev) 1369 return; 1370 1371 bnxt_re_create_snapdump_logs(en_info->rdev); 1372 } 1373 1374 /* 1375 * Except for ulp_async_notifier, the remaining ulp_ops 1376 * below are called with rtnl_lock held 1377 */ 1378 static struct bnxt_ulp_ops bnxt_re_ulp_ops = { 1379 .ulp_async_notifier = bnxt_re_async_notifier, 1380 .ulp_stop = bnxt_re_stop, 1381 .ulp_start = bnxt_re_start, 1382 .ulp_shutdown = bnxt_re_shutdown, 1383 .ulp_irq_stop = bnxt_re_stop_irq, 1384 .ulp_irq_restart = bnxt_re_start_irq, 1385 .ulp_log_live = bnxt_re_ulp_log_live, 1386 }; 1387 1388 static inline const char *bnxt_re_netevent(unsigned long event) 1389 { 1390 BNXT_RE_NETDEV_EVENT(event, NETDEV_UP); 1391 BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN); 1392 BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE); 1393 BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER); 1394 BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER); 1395 BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR); 1396 return "Unknown"; 1397 } 1398 1399 /* RoCE -> Net driver */ 1400 1401 /* Driver registration routines used to let the networking driver (bnxt_en) 1402 * to know that the RoCE driver is now installed */ 1403 static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) 1404 { 1405 struct bnxt_en_dev *en_dev = rdev->en_dev; 1406 int rc; 1407 1408 rtnl_lock(); 1409 rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, 1410 BNXT_ROCE_ULP); 1411 rtnl_unlock(); 1412 if (rc) 1413 dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x", 1414 rdev->en_dev->net, rc); 1415 1416 clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 1417 } 1418 1419 static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) 1420 { 1421 struct bnxt_en_dev *en_dev = rdev->en_dev; 1422 int rc = 0; 1423 1424 rtnl_lock(); 1425 rc = en_dev->en_ops->bnxt_register_device(en_dev, 1426 BNXT_ROCE_ULP, 1427 &bnxt_re_ulp_ops, 1428 rdev->adev); 1429 rtnl_unlock(); 1430 if (rc) { 1431 dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x", 1432 rdev->netdev, rc); 1433 return rc; 1434 } 1435 1436 return rc; 1437 } 1438 1439 static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) 1440 { 1441 struct bnxt_qplib_chip_ctx *cctx; 1442 struct bnxt_en_dev *en_dev; 1443 struct bnxt_qplib_res *res; 1444 u32 l2db_len = 0; 1445 u32 offset = 0; 1446 u32 barlen; 1447 int rc; 1448 1449 res = &rdev->qplib_res; 1450 en_dev = rdev->en_dev; 1451 cctx = rdev->chip_ctx; 1452 1453 /* Issue qcfg */ 1454 rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset); 1455 if (rc) 1456 dev_info(rdev_to_dev(rdev), 1457 "Couldn't get DB bar size, Low latency framework is disabled\n"); 1458 /* set register offsets for both UC and WC */ 1459 if (_is_chip_p7(cctx)) { 1460 res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset; 1461 res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; 1462 } else { 1463 res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : 1464 BNXT_QPLIB_DBR_PF_DB_OFFSET; 1465 res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; 1466 } 1467 1468 /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size 1469 * is equal to the DB-Bar actual size. This indicates that L2 1470 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping 1471 * in such cases and DB-push will be disabled. 1472 */ 1473 barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION); 1474 if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) { 1475 res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; 1476 dev_info(rdev_to_dev(rdev), 1477 "Low latency framework is enabled\n"); 1478 } 1479 1480 return; 1481 } 1482 1483 static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev) 1484 { 1485 struct bnxt_qplib_chip_ctx *cctx; 1486 struct bnxt_en_dev *en_dev; 1487 1488 en_dev = rdev->en_dev; 1489 cctx = rdev->chip_ctx; 1490 cctx->modes.wqe_mode = _is_chip_p7(rdev->chip_ctx) ? 1491 BNXT_QPLIB_WQE_MODE_VARIABLE : BNXT_QPLIB_WQE_MODE_STATIC; 1492 cctx->modes.te_bypass = false; 1493 if (bnxt_re_hwrm_qcaps(rdev)) 1494 dev_err(rdev_to_dev(rdev), 1495 "Failed to query hwrm qcaps\n"); 1496 /* 1497 * TODO: Need a better mechanism for spreading of the 1498 * 512 extended PPP pages in the presence of VF and 1499 * NPAR, until then not enabling push 1500 */ 1501 if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) { 1502 if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev)) 1503 cctx->modes.db_push = false; 1504 } 1505 1506 rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP; 1507 dev_dbg(rdev_to_dev(rdev), 1508 "RoCE is supported on the device - caps:0x%x", 1509 rdev->roce_mode); 1510 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 1511 rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP; 1512 cctx->hw_stats_size = en_dev->hw_ring_stats_size; 1513 } 1514 1515 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) 1516 { 1517 struct bnxt_qplib_chip_ctx *chip_ctx; 1518 struct bnxt_qplib_res *res; 1519 1520 if (!rdev->chip_ctx) 1521 return; 1522 1523 res = &rdev->qplib_res; 1524 bnxt_qplib_unmap_db_bar(res); 1525 1526 kfree(res->hctx); 1527 res->rcfw = NULL; 1528 kfree(rdev->dev_attr); 1529 rdev->dev_attr = NULL; 1530 1531 chip_ctx = rdev->chip_ctx; 1532 rdev->chip_ctx = NULL; 1533 res->cctx = NULL; 1534 res->hctx = NULL; 1535 res->pdev = NULL; 1536 res->netdev = NULL; 1537 kfree(chip_ctx); 1538 } 1539 1540 static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev) 1541 { 1542 struct bnxt_qplib_chip_ctx *chip_ctx; 1543 struct bnxt_en_dev *en_dev; 1544 int rc; 1545 1546 en_dev = rdev->en_dev; 1547 /* Supply pci device to qplib */ 1548 rdev->qplib_res.pdev = en_dev->pdev; 1549 rdev->qplib_res.netdev = rdev->netdev; 1550 rdev->qplib_res.en_dev = en_dev; 1551 1552 chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); 1553 if (!chip_ctx) 1554 return -ENOMEM; 1555 rdev->chip_ctx = chip_ctx; 1556 rdev->qplib_res.cctx = chip_ctx; 1557 rc = bnxt_re_query_hwrm_intf_version(rdev); 1558 if (rc) 1559 goto fail; 1560 rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); 1561 if (!rdev->dev_attr) { 1562 rc = -ENOMEM; 1563 goto fail; 1564 } 1565 rdev->qplib_res.dattr = rdev->dev_attr; 1566 rdev->qplib_res.rcfw = &rdev->rcfw; 1567 rdev->qplib_res.is_vf = rdev->is_virtfn; 1568 1569 rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx), 1570 GFP_KERNEL); 1571 if (!rdev->qplib_res.hctx) { 1572 rc = -ENOMEM; 1573 goto fail; 1574 } 1575 bnxt_re_set_drv_mode(rdev); 1576 1577 bnxt_re_set_db_offset(rdev); 1578 rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); 1579 if (rc) 1580 goto fail; 1581 1582 rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev); 1583 if (rc) 1584 dev_dbg(rdev_to_dev(rdev), 1585 "platform doesn't support global atomics"); 1586 1587 return 0; 1588 fail: 1589 kfree(rdev->chip_ctx); 1590 rdev->chip_ctx = NULL; 1591 1592 kfree(rdev->dev_attr); 1593 rdev->dev_attr = NULL; 1594 1595 kfree(rdev->qplib_res.hctx); 1596 rdev->qplib_res.hctx = NULL; 1597 return rc; 1598 } 1599 1600 static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) { 1601 return _is_chip_gen_p5_p7(rdev->chip_ctx) ? 1602 HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ : 1603 HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL; 1604 } 1605 1606 static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) 1607 { 1608 int rc = -EINVAL; 1609 struct hwrm_ring_free_input req = {0}; 1610 struct hwrm_ring_free_output resp; 1611 struct bnxt_en_dev *en_dev = rdev->en_dev; 1612 struct bnxt_fw_msg fw_msg; 1613 1614 if (!en_dev) 1615 return rc; 1616 1617 /* To avoid unnecessary error messages during recovery. 1618 * HW is anyway in error state. So dont send down the command */ 1619 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1620 return 0; 1621 1622 /* allocation had failed, no need to issue hwrm */ 1623 if (fw_ring_id == 0xffff) 1624 return 0; 1625 1626 memset(&fw_msg, 0, sizeof(fw_msg)); 1627 1628 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); 1629 req.ring_type = bnxt_re_get_rtype(rdev); 1630 req.ring_id = cpu_to_le16(fw_ring_id); 1631 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1632 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1633 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1634 if (rc) { 1635 dev_err(rdev_to_dev(rdev), 1636 "Failed to free HW ring with rc = 0x%x", rc); 1637 return rc; 1638 } 1639 dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n", 1640 fw_ring_id); 1641 1642 return rc; 1643 } 1644 1645 static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, 1646 struct bnxt_re_ring_attr *ring_attr, 1647 u16 *fw_ring_id) 1648 { 1649 int rc = -EINVAL; 1650 struct hwrm_ring_alloc_input req = {0}; 1651 struct hwrm_ring_alloc_output resp; 1652 struct bnxt_en_dev *en_dev = rdev->en_dev; 1653 struct bnxt_fw_msg fw_msg; 1654 1655 if (!en_dev) 1656 return rc; 1657 1658 memset(&fw_msg, 0, sizeof(fw_msg)); 1659 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); 1660 req.flags = cpu_to_le16(ring_attr->flags); 1661 req.enables = 0; 1662 req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); 1663 if (ring_attr->pages > 1) { 1664 /* Page size is in log2 units */ 1665 req.page_size = BNXT_PAGE_SHIFT; 1666 req.page_tbl_depth = 1; 1667 } else { 1668 req.page_size = 4; 1669 req.page_tbl_depth = 0; 1670 } 1671 1672 req.fbo = 0; 1673 /* Association of ring index with doorbell index and MSIX number */ 1674 req.logical_id = cpu_to_le16(ring_attr->lrid); 1675 req.length = cpu_to_le32(ring_attr->depth + 1); 1676 req.ring_type = ring_attr->type; 1677 req.int_mode = ring_attr->mode; 1678 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1679 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1680 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1681 if (rc) { 1682 dev_err(rdev_to_dev(rdev), 1683 "Failed to allocate HW ring with rc = 0x%x", rc); 1684 return rc; 1685 } 1686 *fw_ring_id = le16_to_cpu(resp.ring_id); 1687 dev_dbg(rdev_to_dev(rdev), 1688 "HW ring allocated with id = 0x%x at slot 0x%x", 1689 resp.ring_id, ring_attr->lrid); 1690 1691 return rc; 1692 } 1693 1694 static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, 1695 u32 fw_stats_ctx_id, u16 tid) 1696 { 1697 struct bnxt_en_dev *en_dev = rdev->en_dev; 1698 struct hwrm_stat_ctx_free_input req = {0}; 1699 struct hwrm_stat_ctx_free_output resp; 1700 struct bnxt_fw_msg fw_msg; 1701 int rc = -EINVAL; 1702 1703 if (!en_dev) 1704 return rc; 1705 1706 /* To avoid unnecessary error messages during recovery. 1707 * HW is anyway in error state. So dont send down the command */ 1708 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1709 return 0; 1710 memset(&fw_msg, 0, sizeof(fw_msg)); 1711 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid); 1712 req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); 1713 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1714 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1715 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1716 if (rc) { 1717 dev_err(rdev_to_dev(rdev), 1718 "Failed to free HW stats ctx with rc = 0x%x", rc); 1719 return rc; 1720 } 1721 dev_dbg(rdev_to_dev(rdev), 1722 "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id); 1723 1724 return rc; 1725 } 1726 1727 static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid) 1728 { 1729 struct hwrm_stat_ctx_alloc_output resp = {}; 1730 struct hwrm_stat_ctx_alloc_input req = {}; 1731 struct bnxt_en_dev *en_dev = rdev->en_dev; 1732 struct bnxt_qplib_stats *stat; 1733 struct bnxt_qplib_ctx *hctx; 1734 struct bnxt_fw_msg fw_msg; 1735 int rc = 0; 1736 1737 hctx = rdev->qplib_res.hctx; 1738 stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2; 1739 stat->fw_id = INVALID_STATS_CTX_ID; 1740 1741 if (!en_dev) 1742 return -EINVAL; 1743 1744 memset(&fw_msg, 0, sizeof(fw_msg)); 1745 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1746 HWRM_STAT_CTX_ALLOC, -1, tid); 1747 req.update_period_ms = cpu_to_le32(1000); 1748 req.stats_dma_length = rdev->chip_ctx->hw_stats_size; 1749 req.stats_dma_addr = cpu_to_le64(stat->dma_map); 1750 req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE; 1751 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1752 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1753 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1754 if (rc) { 1755 dev_err(rdev_to_dev(rdev), 1756 "Failed to allocate HW stats ctx, rc = 0x%x", rc); 1757 return rc; 1758 } 1759 stat->fw_id = le32_to_cpu(resp.stat_ctx_id); 1760 dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x", 1761 stat->fw_id); 1762 1763 return rc; 1764 } 1765 1766 static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev) 1767 { 1768 const struct bnxt_en_ops *en_ops; 1769 1770 if (rdev->is_virtfn || 1771 test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1772 return; 1773 1774 memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap)); 1775 en_ops = rdev->en_dev->en_ops; 1776 if (en_ops->bnxt_register_fw_async_events 1777 (rdev->en_dev, BNXT_ROCE_ULP, 1778 (unsigned long *)rdev->event_bitmap, 1779 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) 1780 dev_err(rdev_to_dev(rdev), 1781 "Failed to unregister async event"); 1782 } 1783 1784 static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev) 1785 { 1786 const struct bnxt_en_ops *en_ops; 1787 1788 if (rdev->is_virtfn) 1789 return; 1790 1791 rdev->event_bitmap[0] |= 1792 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) | 1793 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY); 1794 1795 rdev->event_bitmap[2] |= 1796 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64); 1797 rdev->event_bitmap[2] |= 1798 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) | 1799 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64); 1800 en_ops = rdev->en_dev->en_ops; 1801 if (en_ops->bnxt_register_fw_async_events 1802 (rdev->en_dev, BNXT_ROCE_ULP, 1803 (unsigned long *)rdev->event_bitmap, 1804 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) 1805 dev_err(rdev_to_dev(rdev), 1806 "Failed to reg Async event"); 1807 } 1808 1809 static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) 1810 { 1811 struct bnxt_en_dev *en_dev = rdev->en_dev; 1812 struct hwrm_ver_get_output resp = {0}; 1813 struct hwrm_ver_get_input req = {0}; 1814 struct bnxt_qplib_chip_ctx *cctx; 1815 struct bnxt_fw_msg fw_msg; 1816 int rc = 0; 1817 1818 memset(&fw_msg, 0, sizeof(fw_msg)); 1819 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1820 HWRM_VER_GET, -1, -1); 1821 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 1822 req.hwrm_intf_min = HWRM_VERSION_MINOR; 1823 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 1824 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1825 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1826 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1827 if (rc) { 1828 dev_err(rdev_to_dev(rdev), 1829 "Failed to query HW version, rc = 0x%x", rc); 1830 return rc; 1831 } 1832 cctx = rdev->chip_ctx; 1833 cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 | 1834 (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 | 1835 (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 | 1836 le16_to_cpu(resp.hwrm_intf_patch); 1837 1838 cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout); 1839 1840 if (!cctx->hwrm_cmd_max_timeout) 1841 cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT; 1842 1843 cctx->chip_num = le16_to_cpu(resp.chip_num); 1844 cctx->chip_rev = resp.chip_rev; 1845 cctx->chip_metal = resp.chip_metal; 1846 return 0; 1847 } 1848 1849 /* Query device config using common hwrm */ 1850 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, 1851 u32 *offset) 1852 { 1853 struct bnxt_en_dev *en_dev = rdev->en_dev; 1854 struct hwrm_func_qcfg_output resp = {0}; 1855 struct hwrm_func_qcfg_input req = {0}; 1856 struct bnxt_fw_msg fw_msg; 1857 int rc; 1858 1859 memset(&fw_msg, 0, sizeof(fw_msg)); 1860 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1861 HWRM_FUNC_QCFG, -1, -1); 1862 req.fid = cpu_to_le16(0xffff); 1863 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1864 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1865 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1866 if (rc) { 1867 dev_err(rdev_to_dev(rdev), 1868 "Failed to query config, rc = %#x", rc); 1869 return rc; 1870 } 1871 1872 *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024); 1873 *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024); 1874 return 0; 1875 } 1876 1877 /* Query function capabilities using common hwrm */ 1878 int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) 1879 { 1880 struct bnxt_en_dev *en_dev = rdev->en_dev; 1881 struct hwrm_func_qcaps_output resp = {0}; 1882 struct hwrm_func_qcaps_input req = {0}; 1883 struct bnxt_qplib_chip_ctx *cctx; 1884 struct bnxt_fw_msg fw_msg; 1885 u8 push_enable = false; 1886 int rc; 1887 1888 cctx = rdev->chip_ctx; 1889 memset(&fw_msg, 0, sizeof(fw_msg)); 1890 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1891 HWRM_FUNC_QCAPS, -1, -1); 1892 req.fid = cpu_to_le16(0xffff); 1893 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1894 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1895 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1896 if (rc) { 1897 dev_err(rdev_to_dev(rdev), 1898 "Failed to query capabilities, rc = %#x", rc); 1899 return rc; 1900 } 1901 if (_is_chip_p7(rdev->chip_ctx)) 1902 push_enable = 1903 (resp.flags_ext & 1904 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ? 1905 true : false; 1906 else 1907 push_enable = 1908 (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ? 1909 true : false; 1910 cctx->modes.db_push = push_enable; 1911 1912 cctx->modes.dbr_pacing = 1913 resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ? 1914 true : false; 1915 cctx->modes.dbr_pacing_ext = 1916 resp.flags_ext2 & 1917 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ? 1918 true : false; 1919 cctx->modes.dbr_drop_recov = 1920 (resp.flags_ext2 & 1921 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ? 1922 true : false; 1923 cctx->modes.dbr_pacing_v0 = 1924 (resp.flags_ext2 & 1925 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ? 1926 true : false; 1927 dev_dbg(rdev_to_dev(rdev), 1928 "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n", 1929 __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov); 1930 1931 return 0; 1932 } 1933 1934 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) 1935 { 1936 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 1937 struct hwrm_func_dbr_pacing_qcfg_output resp = {0}; 1938 struct hwrm_func_dbr_pacing_qcfg_input req = {0}; 1939 struct bnxt_en_dev *en_dev = rdev->en_dev; 1940 struct bnxt_qplib_chip_ctx *cctx; 1941 struct bnxt_fw_msg fw_msg; 1942 u32 primary_nq_id; 1943 int rc; 1944 1945 cctx = rdev->chip_ctx; 1946 memset(&fw_msg, 0, sizeof(fw_msg)); 1947 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1948 HWRM_FUNC_DBR_PACING_QCFG, -1, -1); 1949 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1950 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 1951 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1952 if (rc) { 1953 dev_dbg(rdev_to_dev(rdev), 1954 "Failed to query dbr pacing config, rc = %#x", rc); 1955 return rc; 1956 } 1957 1958 primary_nq_id = le32_to_cpu(resp.primary_nq_id); 1959 if (primary_nq_id == 0xffffffff && 1960 !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 1961 dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n", 1962 __func__, __LINE__); 1963 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); 1964 } 1965 1966 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 1967 struct bnxt_qplib_nq *nq; 1968 1969 nq = &rdev->nqr.nq[0]; 1970 /* Reset the primary capability */ 1971 if (nq->ring_id != primary_nq_id) 1972 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); 1973 } 1974 1975 if ((resp.dbr_stat_db_fifo_reg & 1976 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == 1977 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) 1978 cctx->dbr_stat_db_fifo = 1979 resp.dbr_stat_db_fifo_reg & 1980 ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; 1981 1982 if ((resp.dbr_throttling_aeq_arm_reg & 1983 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK) 1984 == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) { 1985 cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg & 1986 ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; 1987 cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4; 1988 } 1989 pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth); 1990 if (!pacing_data->fifo_max_depth) 1991 pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx); 1992 pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask); 1993 pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift; 1994 dev_dbg(rdev_to_dev(rdev), 1995 "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i" 1996 "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n", 1997 __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf, 1998 cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg, 1999 pacing_data->fifo_max_depth, 2000 le32_to_cpu(resp.dbr_stat_db_max_fifo_depth)); 2001 return 0; 2002 } 2003 2004 static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable) 2005 { 2006 struct hwrm_func_dbr_pacing_cfg_output resp = {0}; 2007 struct hwrm_func_dbr_pacing_cfg_input req = {0}; 2008 struct bnxt_en_dev *en_dev = rdev->en_dev; 2009 struct bnxt_fw_msg fw_msg; 2010 int rc; 2011 2012 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 2013 return 0; 2014 2015 memset(&fw_msg, 0, sizeof(fw_msg)); 2016 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2017 HWRM_FUNC_DBR_PACING_CFG, -1, -1); 2018 if (enable) { 2019 req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE; 2020 req.enables = 2021 cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID | 2022 HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID); 2023 } else { 2024 req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE; 2025 } 2026 req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id); 2027 req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark); 2028 dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x", 2029 __func__, req.primary_nq_id, req.pacing_threshold); 2030 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2031 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 2032 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2033 if (rc) { 2034 dev_dbg(rdev_to_dev(rdev), 2035 "Failed to set dbr pacing config, rc = %#x", rc); 2036 return rc; 2037 } 2038 return 0; 2039 } 2040 2041 /* Net -> RoCE driver */ 2042 2043 /* Device */ 2044 struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev) 2045 { 2046 struct bnxt_re_dev *rdev; 2047 2048 rcu_read_lock(); 2049 list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { 2050 if (rdev->netdev == netdev) { 2051 rcu_read_unlock(); 2052 dev_dbg(rdev_to_dev(rdev), 2053 "netdev (%p) found, ref_count = 0x%x", 2054 netdev, atomic_read(&rdev->ref_count)); 2055 return rdev; 2056 } 2057 } 2058 rcu_read_unlock(); 2059 return NULL; 2060 } 2061 2062 static ssize_t show_rev(struct device *device, struct device_attribute *attr, 2063 char *buf) 2064 { 2065 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 2066 2067 return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); 2068 } 2069 2070 2071 static ssize_t show_hca(struct device *device, struct device_attribute *attr, 2072 char *buf) 2073 { 2074 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 2075 2076 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); 2077 } 2078 2079 static ssize_t show_board_id(struct device *device, struct device_attribute *attr, 2080 char *buf) 2081 { 2082 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 2083 char buffer[BNXT_VPD_PN_FLD_LEN] = {}; 2084 2085 if (!rdev->is_virtfn) 2086 memcpy(buffer, rdev->en_dev->board_part_number, 2087 BNXT_VPD_PN_FLD_LEN - 1); 2088 else 2089 scnprintf(buffer, BNXT_VPD_PN_FLD_LEN, 2090 "0x%x-VF", rdev->en_dev->pdev->device); 2091 2092 return scnprintf(buf, PAGE_SIZE, "%s\n", buffer); 2093 } 2094 2095 static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); 2096 static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); 2097 static DEVICE_ATTR(board_id, 0444, show_board_id, NULL); 2098 2099 static struct device_attribute *bnxt_re_attributes[] = { 2100 &dev_attr_hw_rev, 2101 &dev_attr_hca_type, 2102 &dev_attr_board_id 2103 }; 2104 2105 int ib_register_device_compat(struct bnxt_re_dev *rdev) 2106 { 2107 struct ib_device *ibdev = &rdev->ibdev; 2108 char name[IB_DEVICE_NAME_MAX]; 2109 2110 memset(name, 0, IB_DEVICE_NAME_MAX); 2111 strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX); 2112 2113 strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 2114 2115 return ib_register_device(ibdev, NULL); 2116 } 2117 2118 static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) 2119 { 2120 struct ib_device *ibdev = &rdev->ibdev; 2121 int ret = 0; 2122 2123 /* ib device init */ 2124 ibdev->owner = THIS_MODULE; 2125 ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; 2126 ibdev->node_type = RDMA_NODE_IB_CA; 2127 strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", 2128 strlen(BNXT_RE_DESC) + 5); 2129 ibdev->phys_port_cnt = 1; 2130 2131 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid); 2132 2133 /* Data path irqs is one less than the max msix vectors */ 2134 ibdev->num_comp_vectors = rdev->nqr.num_msix - 1; 2135 bnxt_re_set_dma_device(ibdev, rdev); 2136 ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; 2137 2138 /* User space */ 2139 ibdev->uverbs_cmd_mask = 2140 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 2141 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 2142 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 2143 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 2144 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 2145 (1ull << IB_USER_VERBS_CMD_REG_MR) | 2146 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 2147 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 2148 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 2149 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 2150 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 2151 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 2152 (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 2153 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 2154 (1ull << IB_USER_VERBS_CMD_REREG_MR) | 2155 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 2156 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 2157 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 2158 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 2159 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 2160 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | 2161 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | 2162 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 2163 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | 2164 (1ull << IB_USER_VERBS_CMD_QUERY_AH) | 2165 (1ull << IB_USER_VERBS_CMD_DESTROY_AH); 2166 2167 ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP); 2168 ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ); 2169 2170 #define bnxt_re_ib_ah bnxt_re_ah 2171 #define bnxt_re_ib_cq bnxt_re_cq 2172 #define bnxt_re_ib_pd bnxt_re_pd 2173 #define bnxt_re_ib_srq bnxt_re_srq 2174 #define bnxt_re_ib_ucontext bnxt_re_ucontext 2175 INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE); 2176 2177 ibdev->query_device = bnxt_re_query_device; 2178 ibdev->modify_device = bnxt_re_modify_device; 2179 ibdev->query_port = bnxt_re_query_port; 2180 ibdev->modify_port = bnxt_re_modify_port; 2181 ibdev->get_port_immutable = bnxt_re_get_port_immutable; 2182 ibdev->query_pkey = bnxt_re_query_pkey; 2183 ibdev->get_netdev = bnxt_re_get_netdev; 2184 ibdev->add_gid = bnxt_re_add_gid; 2185 ibdev->del_gid = bnxt_re_del_gid; 2186 ibdev->get_link_layer = bnxt_re_get_link_layer; 2187 ibdev->alloc_pd = bnxt_re_alloc_pd; 2188 ibdev->dealloc_pd = bnxt_re_dealloc_pd; 2189 ibdev->create_ah = bnxt_re_create_ah; 2190 ibdev->modify_ah = bnxt_re_modify_ah; 2191 ibdev->query_ah = bnxt_re_query_ah; 2192 ibdev->destroy_ah = bnxt_re_destroy_ah; 2193 ibdev->create_srq = bnxt_re_create_srq; 2194 ibdev->modify_srq = bnxt_re_modify_srq; 2195 ibdev->query_srq = bnxt_re_query_srq; 2196 ibdev->destroy_srq = bnxt_re_destroy_srq; 2197 ibdev->post_srq_recv = bnxt_re_post_srq_recv; 2198 ibdev->create_qp = bnxt_re_create_qp; 2199 ibdev->modify_qp = bnxt_re_modify_qp; 2200 ibdev->query_qp = bnxt_re_query_qp; 2201 ibdev->destroy_qp = bnxt_re_destroy_qp; 2202 ibdev->post_send = bnxt_re_post_send; 2203 ibdev->post_recv = bnxt_re_post_recv; 2204 ibdev->create_cq = bnxt_re_create_cq; 2205 ibdev->modify_cq = bnxt_re_modify_cq; 2206 ibdev->destroy_cq = bnxt_re_destroy_cq; 2207 ibdev->resize_cq = bnxt_re_resize_cq; 2208 ibdev->poll_cq = bnxt_re_poll_cq; 2209 ibdev->req_notify_cq = bnxt_re_req_notify_cq; 2210 ibdev->get_dma_mr = bnxt_re_get_dma_mr; 2211 ibdev->get_hw_stats = bnxt_re_get_hw_stats; 2212 ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats; 2213 ibdev->dereg_mr = bnxt_re_dereg_mr; 2214 ibdev->alloc_mr = bnxt_re_alloc_mr; 2215 ibdev->map_mr_sg = bnxt_re_map_mr_sg; 2216 ibdev->alloc_mw = bnxt_re_alloc_mw; 2217 ibdev->dealloc_mw = bnxt_re_dealloc_mw; 2218 ibdev->reg_user_mr = bnxt_re_reg_user_mr; 2219 ibdev->rereg_user_mr = bnxt_re_rereg_user_mr; 2220 ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx; 2221 ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; 2222 ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; 2223 ibdev->mmap = bnxt_re_mmap; 2224 ibdev->process_mad = bnxt_re_process_mad; 2225 2226 ret = ib_register_device_compat(rdev); 2227 return ret; 2228 } 2229 2230 static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev) 2231 { 2232 int i = BNXT_RE_REF_WAIT_COUNT; 2233 2234 dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev); 2235 /* Wait for rdev refcount to come down */ 2236 while ((atomic_read(&rdev->ref_count) > 1) && i--) 2237 msleep(100); 2238 2239 if (atomic_read(&rdev->ref_count) > 1) 2240 dev_err(rdev_to_dev(rdev), 2241 "Failed waiting for ref count to deplete %d", 2242 atomic_read(&rdev->ref_count)); 2243 2244 atomic_set(&rdev->ref_count, 0); 2245 if_rele(rdev->netdev); 2246 rdev->netdev = NULL; 2247 synchronize_rcu(); 2248 2249 kfree(rdev->gid_map); 2250 kfree(rdev->dbg_stats); 2251 ib_dealloc_device(&rdev->ibdev); 2252 } 2253 2254 static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev, 2255 struct bnxt_en_dev *en_dev) 2256 { 2257 struct bnxt_re_dev *rdev; 2258 u32 count; 2259 2260 /* Allocate bnxt_re_dev instance here */ 2261 rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev)); 2262 if (!rdev) { 2263 pr_err("%s: bnxt_re_dev allocation failure!", 2264 ROCE_DRV_MODULE_NAME); 2265 return NULL; 2266 } 2267 /* Default values */ 2268 atomic_set(&rdev->ref_count, 0); 2269 rdev->netdev = netdev; 2270 dev_hold(rdev->netdev); 2271 rdev->en_dev = en_dev; 2272 rdev->id = rdev->en_dev->pdev->devfn; 2273 INIT_LIST_HEAD(&rdev->qp_list); 2274 mutex_init(&rdev->qp_lock); 2275 mutex_init(&rdev->cc_lock); 2276 mutex_init(&rdev->dbq_lock); 2277 bnxt_re_clear_rsors_stat(&rdev->stats.rsors); 2278 rdev->cosq[0] = rdev->cosq[1] = 0xFFFF; 2279 rdev->min_tx_depth = 1; 2280 rdev->stats.stats_query_sec = 1; 2281 /* Disable priority vlan as the default mode is DSCP based PFC */ 2282 rdev->cc_param.disable_prio_vlan_tx = 1; 2283 2284 /* Initialize worker for DBR Pacing */ 2285 INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); 2286 INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); 2287 rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) * 2288 BNXT_RE_MAX_SGID_ENTRIES, 2289 GFP_KERNEL); 2290 if (!rdev->gid_map) { 2291 ib_dealloc_device(&rdev->ibdev); 2292 return NULL; 2293 } 2294 for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++) 2295 rdev->gid_map[count] = -1; 2296 2297 rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL); 2298 if (!rdev->dbg_stats) { 2299 ib_dealloc_device(&rdev->ibdev); 2300 return NULL; 2301 } 2302 2303 return rdev; 2304 } 2305 2306 static int bnxt_re_handle_unaffi_async_event( 2307 struct creq_func_event *unaffi_async) 2308 { 2309 switch (unaffi_async->event) { 2310 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: 2311 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: 2312 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: 2313 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: 2314 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: 2315 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: 2316 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: 2317 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: 2318 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: 2319 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: 2320 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: 2321 break; 2322 default: 2323 return -EINVAL; 2324 } 2325 return 0; 2326 } 2327 2328 static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp) 2329 { 2330 struct creq_qp_error_notification *err_event; 2331 struct ib_event event; 2332 unsigned int flags; 2333 2334 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && 2335 !qp->qplib_qp.is_user) { 2336 flags = bnxt_re_lock_cqs(qp); 2337 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 2338 bnxt_re_unlock_cqs(qp, flags); 2339 } 2340 memset(&event, 0, sizeof(event)); 2341 event.device = &qp->rdev->ibdev; 2342 event.element.qp = &qp->ib_qp; 2343 event.event = IB_EVENT_QP_FATAL; 2344 2345 err_event = qp_event; 2346 switch(err_event->res_err_state_reason) { 2347 case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX: 2348 case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH: 2349 case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR: 2350 case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT: 2351 case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY: 2352 case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR: 2353 case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION: 2354 case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR: 2355 case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY: 2356 case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR: 2357 case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION: 2358 case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR: 2359 case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY: 2360 case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC: 2361 event.event = IB_EVENT_QP_ACCESS_ERR; 2362 break; 2363 case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE: 2364 case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR: 2365 case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR: 2366 case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE: 2367 case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE: 2368 event.event = IB_EVENT_QP_REQ_ERR; 2369 break; 2370 case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW: 2371 case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR: 2372 case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR: 2373 case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR: 2374 case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR: 2375 case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR: 2376 case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR: 2377 event.event = IB_EVENT_QP_FATAL; 2378 break; 2379 default: 2380 if (qp->qplib_qp.srq) 2381 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 2382 break; 2383 } 2384 2385 if (err_event->res_err_state_reason) 2386 dev_err(rdev_to_dev(qp->rdev), 2387 "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n", 2388 __func__, qp->qplib_qp.is_user ? "user" : "kernel", 2389 qp->qplib_qp.id, 2390 err_event->sq_cons_idx, 2391 err_event->rq_cons_idx, 2392 err_event->req_slow_path_state, 2393 err_event->req_err_state_reason, 2394 err_event->res_slow_path_state, 2395 err_event->res_err_state_reason); 2396 2397 if (event.device && qp->ib_qp.event_handler) 2398 qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); 2399 2400 return 0; 2401 } 2402 2403 static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq) 2404 { 2405 struct creq_cq_error_notification *cqerr; 2406 bool send = false; 2407 2408 cqerr = event; 2409 switch (cqerr->cq_err_reason) { 2410 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR: 2411 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR: 2412 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR: 2413 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR: 2414 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR: 2415 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR: 2416 send = true; 2417 default: 2418 break; 2419 } 2420 2421 if (send && cq->ibcq.event_handler) { 2422 struct ib_event ibevent = {}; 2423 2424 ibevent.event = IB_EVENT_CQ_ERR; 2425 ibevent.element.cq = &cq->ibcq; 2426 ibevent.device = &cq->rdev->ibdev; 2427 2428 dev_err(rdev_to_dev(cq->rdev), 2429 "%s err reason %d\n", __func__, cqerr->cq_err_reason); 2430 cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context); 2431 } 2432 2433 cq->qplib_cq.is_cq_err_event = true; 2434 2435 return 0; 2436 } 2437 2438 static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, 2439 void *obj) 2440 { 2441 struct bnxt_qplib_qp *qplqp; 2442 struct bnxt_qplib_cq *qplcq; 2443 struct bnxt_re_qp *qp; 2444 struct bnxt_re_cq *cq; 2445 int rc = 0; 2446 u8 event; 2447 2448 if (!obj) 2449 return rc; /* QP was already dead, still return success */ 2450 2451 event = affi_async->event; 2452 switch (event) { 2453 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: 2454 qplqp = obj; 2455 qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp); 2456 rc = bnxt_re_handle_qp_async_event(affi_async, qp); 2457 break; 2458 case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: 2459 qplcq = obj; 2460 cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq); 2461 rc = bnxt_re_handle_cq_async_error(affi_async, cq); 2462 break; 2463 default: 2464 rc = -EINVAL; 2465 } 2466 2467 return rc; 2468 } 2469 2470 static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, 2471 void *aeqe, void *obj) 2472 { 2473 struct creq_func_event *unaffi_async; 2474 struct creq_qp_event *affi_async; 2475 u8 type; 2476 int rc; 2477 2478 type = ((struct creq_base *)aeqe)->type; 2479 if (type == CREQ_BASE_TYPE_FUNC_EVENT) { 2480 unaffi_async = aeqe; 2481 rc = bnxt_re_handle_unaffi_async_event(unaffi_async); 2482 } else { 2483 affi_async = aeqe; 2484 rc = bnxt_re_handle_affi_async_event(affi_async, obj); 2485 } 2486 2487 return rc; 2488 } 2489 2490 static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, 2491 struct bnxt_qplib_srq *handle, u8 event) 2492 { 2493 struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq, 2494 qplib_srq); 2495 struct ib_event ib_event; 2496 2497 if (srq == NULL) { 2498 pr_err("%s: SRQ is NULL, SRQN not handled", 2499 ROCE_DRV_MODULE_NAME); 2500 return -EINVAL; 2501 } 2502 ib_event.device = &srq->rdev->ibdev; 2503 ib_event.element.srq = &srq->ibsrq; 2504 if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) 2505 ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; 2506 else 2507 ib_event.event = IB_EVENT_SRQ_ERR; 2508 2509 if (srq->ibsrq.event_handler) { 2510 /* Lock event_handler? */ 2511 (*srq->ibsrq.event_handler)(&ib_event, 2512 srq->ibsrq.srq_context); 2513 } 2514 return 0; 2515 } 2516 2517 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, 2518 struct bnxt_qplib_cq *handle) 2519 { 2520 struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq, 2521 qplib_cq); 2522 u32 *cq_ptr; 2523 2524 if (cq == NULL) { 2525 pr_err("%s: CQ is NULL, CQN not handled", 2526 ROCE_DRV_MODULE_NAME); 2527 return -EINVAL; 2528 } 2529 /* CQ already in destroy path. Do not handle any more events */ 2530 if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) { 2531 if (!handle->destroyed) 2532 dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled", 2533 ROCE_DRV_MODULE_NAME); 2534 return 0; 2535 } 2536 2537 if (cq->ibcq.comp_handler) { 2538 if (cq->uctx_cq_page) { 2539 cq_ptr = (u32 *)cq->uctx_cq_page; 2540 *cq_ptr = cq->qplib_cq.toggle; 2541 } 2542 /* Lock comp_handler? */ 2543 (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context); 2544 } 2545 2546 return 0; 2547 } 2548 2549 struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev) 2550 { 2551 int min, indx; 2552 2553 mutex_lock(&rdev->nqr.load_lock); 2554 for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) { 2555 if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load) 2556 min = indx; 2557 } 2558 rdev->nqr.nq[min].load++; 2559 mutex_unlock(&rdev->nqr.load_lock); 2560 2561 return &rdev->nqr.nq[min]; 2562 } 2563 2564 void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq) 2565 { 2566 mutex_lock(&rdev->nqr.load_lock); 2567 nq->load--; 2568 mutex_unlock(&rdev->nqr.load_lock); 2569 } 2570 2571 static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev) 2572 { 2573 struct bnxt_qplib_dev_attr *attr; 2574 bool rc = true; 2575 2576 attr = rdev->dev_attr; 2577 2578 if (!attr->max_cq || !attr->max_qp || 2579 !attr->max_sgid || !attr->max_mr) { 2580 dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources"); 2581 dev_dbg(rdev_to_dev(rdev), 2582 "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d", 2583 attr->max_cq, attr->max_qp, attr->max_dpi, 2584 attr->max_sgid, attr->max_mr); 2585 rc = false; 2586 } 2587 return rc; 2588 } 2589 2590 static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, 2591 u8 port_num, enum ib_event_type event) 2592 { 2593 struct ib_event ib_event; 2594 2595 ib_event.device = ibdev; 2596 if (qp) { 2597 ib_event.element.qp = qp; 2598 ib_event.event = event; 2599 if (qp->event_handler) 2600 qp->event_handler(&ib_event, qp->qp_context); 2601 } else { 2602 ib_event.element.port_num = port_num; 2603 ib_event.event = event; 2604 ib_dispatch_event(&ib_event); 2605 } 2606 2607 dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)), 2608 "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num); 2609 } 2610 2611 static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, 2612 struct bnxt_re_qp *qp) 2613 { 2614 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) 2615 return (qp->ib_qp.qp_type == IB_QPT_GSI) || 2616 (qp == rdev->gsi_ctx.gsi_sqp); 2617 else 2618 return (qp->ib_qp.qp_type == IB_QPT_GSI); 2619 } 2620 2621 static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev) 2622 { 2623 struct bnxt_qplib_qp *qpl_qp; 2624 bool dev_detached = false; 2625 struct ib_qp_attr qp_attr; 2626 int num_qps_stopped = 0; 2627 int mask = IB_QP_STATE; 2628 struct bnxt_re_qp *qp; 2629 unsigned long flags; 2630 2631 if (!rdev) 2632 return; 2633 2634 restart: 2635 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 2636 dev_detached = true; 2637 2638 qp_attr.qp_state = IB_QPS_ERR; 2639 mutex_lock(&rdev->qp_lock); 2640 list_for_each_entry(qp, &rdev->qp_list, list) { 2641 qpl_qp = &qp->qplib_qp; 2642 if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { 2643 if (qpl_qp->state != 2644 CMDQ_MODIFY_QP_NEW_STATE_RESET && 2645 qpl_qp->state != 2646 CMDQ_MODIFY_QP_NEW_STATE_ERR) { 2647 if (dev_detached) { 2648 /* 2649 * Cant actually send the command down, 2650 * marking the state for bookkeeping 2651 */ 2652 qpl_qp->state = 2653 CMDQ_MODIFY_QP_NEW_STATE_ERR; 2654 qpl_qp->cur_qp_state = qpl_qp->state; 2655 if (!qpl_qp->is_user) { 2656 /* Add to flush list */ 2657 flags = bnxt_re_lock_cqs(qp); 2658 bnxt_qplib_add_flush_qp(qpl_qp); 2659 bnxt_re_unlock_cqs(qp, flags); 2660 } 2661 } else { 2662 num_qps_stopped++; 2663 bnxt_re_modify_qp(&qp->ib_qp, 2664 &qp_attr, mask, 2665 NULL); 2666 } 2667 2668 bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, 2669 1, IB_EVENT_QP_FATAL); 2670 /* 2671 * 1. Release qp_lock after a budget to unblock other verb 2672 * requests (like qp_destroy) from stack. 2673 * 2. Traverse through the qp_list freshly as addition / deletion 2674 * might have happened since qp_lock is getting released here. 2675 */ 2676 if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) { 2677 mutex_unlock(&rdev->qp_lock); 2678 goto restart; 2679 } 2680 } 2681 } 2682 } 2683 2684 mutex_unlock(&rdev->qp_lock); 2685 } 2686 2687 static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) 2688 { 2689 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 2690 struct bnxt_qplib_gid gid; 2691 u16 gid_idx, index; 2692 int rc = 0; 2693 2694 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 2695 return 0; 2696 2697 if (sgid_tbl == NULL) { 2698 dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); 2699 return -EINVAL; 2700 } 2701 2702 for (index = 0; index < sgid_tbl->active; index++) { 2703 gid_idx = sgid_tbl->hw_id[index]; 2704 2705 if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, 2706 sizeof(bnxt_qplib_gid_zero))) 2707 continue; 2708 /* Need to modify the VLAN enable setting of non VLAN GID only 2709 * as setting is done for VLAN GID while adding GID 2710 * 2711 * If disable_prio_vlan_tx is enable, then we'll need to remove the 2712 * vlan entry from the sgid_tbl. 2713 */ 2714 if (sgid_tbl->vlan[index] == true) 2715 continue; 2716 2717 memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); 2718 2719 rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, 2720 rdev->dev_addr); 2721 } 2722 2723 return rc; 2724 } 2725 2726 static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev) 2727 { 2728 struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; 2729 2730 if (_is_chip_p7(rdev->chip_ctx)) { 2731 cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; 2732 } else { 2733 cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | 2734 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | 2735 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); 2736 2737 if (!is_qport_service_type_supported(rdev)) 2738 cc_param->mask |= 2739 (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | 2740 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | 2741 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); 2742 } 2743 2744 cc_param->cur_mask = cc_param->mask; 2745 2746 if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param)) 2747 dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); 2748 } 2749 2750 static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev) 2751 { 2752 struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; 2753 int rc; 2754 2755 if (_is_chip_p7(rdev->chip_ctx)) { 2756 cc_param->enable = 0x0; 2757 cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; 2758 } else { 2759 cc_param->enable = 0x1; 2760 cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | 2761 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | 2762 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); 2763 2764 if (!is_qport_service_type_supported(rdev)) 2765 cc_param->mask |= 2766 (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | 2767 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | 2768 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); 2769 } 2770 2771 cc_param->cur_mask = cc_param->mask; 2772 2773 rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); 2774 if (rc) { 2775 dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); 2776 return rc; 2777 } 2778 /* Reset the programming mask */ 2779 cc_param->mask = 0; 2780 if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { 2781 cc_param->qp1_tos_dscp = cc_param->tos_dscp; 2782 rc = bnxt_re_update_qp1_tos_dscp(rdev); 2783 if (rc) { 2784 dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d", 2785 __func__, rc); 2786 goto clear; 2787 } 2788 } 2789 return 0; 2790 2791 clear: 2792 bnxt_re_clear_cc(rdev); 2793 return rc; 2794 } 2795 2796 int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, 2797 struct bnxt_re_dscp2pri *d2p, u16 *count, 2798 u16 target_id) 2799 { 2800 struct bnxt_en_dev *en_dev = rdev->en_dev; 2801 struct hwrm_queue_dscp2pri_qcfg_input req; 2802 struct hwrm_queue_dscp2pri_qcfg_output resp; 2803 struct bnxt_re_dscp2pri *dscp2pri; 2804 struct bnxt_fw_msg fw_msg; 2805 u16 in_count = *count; 2806 dma_addr_t dma_handle; 2807 int rc = 0, i; 2808 u16 data_len; 2809 u8 *kmem; 2810 2811 data_len = *count * sizeof(*dscp2pri); 2812 memset(&fw_msg, 0, sizeof(fw_msg)); 2813 memset(&req, 0, sizeof(req)); 2814 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2815 HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id); 2816 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 2817 2818 kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, 2819 GFP_KERNEL); 2820 if (!kmem) { 2821 dev_err(rdev_to_dev(rdev), 2822 "dma_zalloc_coherent failure, length = %u\n", 2823 (unsigned)data_len); 2824 return -ENOMEM; 2825 } 2826 req.dest_data_addr = cpu_to_le64(dma_handle); 2827 req.dest_data_buffer_size = cpu_to_le16(data_len); 2828 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2829 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2830 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2831 if (rc) 2832 goto out; 2833 2834 /* Upload the DSCP-MASK-PRI tuple(s) */ 2835 dscp2pri = (struct bnxt_re_dscp2pri *)kmem; 2836 for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) { 2837 d2p[i].dscp = dscp2pri->dscp; 2838 d2p[i].mask = dscp2pri->mask; 2839 d2p[i].pri = dscp2pri->pri; 2840 dscp2pri++; 2841 } 2842 *count = le16_to_cpu(resp.entry_cnt); 2843 out: 2844 dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); 2845 return rc; 2846 } 2847 2848 int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev) 2849 { 2850 /* Remove the VLAN from the GID entry */ 2851 if (rdev->cc_param.disable_prio_vlan_tx) 2852 rdev->qplib_res.prio = false; 2853 else 2854 rdev->qplib_res.prio = true; 2855 2856 return bnxt_re_update_gid(rdev); 2857 } 2858 2859 int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, 2860 struct bnxt_re_dscp2pri *d2p, u16 count, 2861 u16 target_id) 2862 { 2863 struct bnxt_en_dev *en_dev = rdev->en_dev; 2864 struct hwrm_queue_dscp2pri_cfg_input req; 2865 struct hwrm_queue_dscp2pri_cfg_output resp; 2866 struct bnxt_fw_msg fw_msg; 2867 struct bnxt_re_dscp2pri *dscp2pri; 2868 int i, rc, data_len = 3 * 256; 2869 dma_addr_t dma_handle; 2870 u8 *kmem; 2871 2872 memset(&req, 0, sizeof(req)); 2873 memset(&fw_msg, 0, sizeof(fw_msg)); 2874 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2875 HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id); 2876 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 2877 2878 kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, 2879 GFP_KERNEL); 2880 if (!kmem) { 2881 dev_err(rdev_to_dev(rdev), 2882 "dma_alloc_coherent failure, length = %u\n", 2883 (unsigned)data_len); 2884 return -ENOMEM; 2885 } 2886 req.src_data_addr = cpu_to_le64(dma_handle); 2887 2888 /* Download the DSCP-MASK-PRI tuple(s) */ 2889 dscp2pri = (struct bnxt_re_dscp2pri *)kmem; 2890 for (i = 0; i < count; i++) { 2891 dscp2pri->dscp = d2p[i].dscp; 2892 dscp2pri->mask = d2p[i].mask; 2893 dscp2pri->pri = d2p[i].pri; 2894 dscp2pri++; 2895 } 2896 2897 req.entry_cnt = cpu_to_le16(count); 2898 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2899 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2900 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2901 dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); 2902 return rc; 2903 } 2904 2905 int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, 2906 struct bnxt_re_tc_rec *tc_rec, u16 tid) 2907 { 2908 u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1; 2909 struct hwrm_queue_qportcfg_output resp = {0}; 2910 struct hwrm_queue_qportcfg_input req = {0}; 2911 struct bnxt_en_dev *en_dev = rdev->en_dev; 2912 struct bnxt_fw_msg fw_msg; 2913 bool def_init = false; 2914 u8 *tmp_type; 2915 u8 cos_id; 2916 int rc; 2917 2918 memset(&fw_msg, 0, sizeof(fw_msg)); 2919 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG, 2920 -1, tid); 2921 req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1; 2922 if (BNXT_EN_ASYM_Q(en_dev)) 2923 req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); 2924 2925 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2926 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2927 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2928 if (rc) 2929 return rc; 2930 2931 if (!resp.max_configurable_queues) 2932 return -EINVAL; 2933 2934 max_tc = resp.max_configurable_queues; 2935 tc_rec->max_tc = max_tc; 2936 2937 if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE) 2938 tc_rec->serv_type_enabled = true; 2939 2940 qptr = &resp.queue_id0; 2941 type_ptr0 = &resp.queue_id0_service_profile_type; 2942 type_ptr1 = &resp.queue_id1_service_profile_type; 2943 for (tc = 0; tc < max_tc; tc++) { 2944 tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0; 2945 2946 cos_id = *qptr++; 2947 /* RoCE CoS queue is the first cos queue. 2948 * For MP12 and MP17 order is 405 and 141015. 2949 */ 2950 if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) { 2951 tc_rec->cos_id_roce = cos_id; 2952 tc_rec->tc_roce = tc; 2953 } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) { 2954 tc_rec->cos_id_cnp = cos_id; 2955 tc_rec->tc_cnp = tc; 2956 } else if (!def_init) { 2957 def_init = true; 2958 tc_rec->tc_def = tc; 2959 tc_rec->cos_id_def = cos_id; 2960 } 2961 qptr++; 2962 } 2963 2964 return rc; 2965 } 2966 2967 int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, 2968 struct bnxt_re_cos2bw_cfg *cfg) 2969 { 2970 struct bnxt_en_dev *en_dev = rdev->en_dev; 2971 struct hwrm_queue_cos2bw_qcfg_output resp; 2972 struct hwrm_queue_cos2bw_qcfg_input req = {0}; 2973 struct bnxt_fw_msg fw_msg; 2974 int rc, indx; 2975 void *data; 2976 2977 memset(&fw_msg, 0, sizeof(fw_msg)); 2978 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2979 HWRM_QUEUE_COS2BW_QCFG, -1, target_id); 2980 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 2981 2982 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2983 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2984 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2985 if (rc) 2986 return rc; 2987 data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg, 2988 queue_id); 2989 for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) { 2990 memcpy(&cfg->cfg, data, sizeof(cfg->cfg)); 2991 if (indx == 0) 2992 cfg->queue_id = resp.queue_id0; 2993 cfg++; 2994 } 2995 2996 return rc; 2997 } 2998 2999 int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, 3000 struct bnxt_re_cos2bw_cfg *cfg) 3001 { 3002 struct bnxt_en_dev *en_dev = rdev->en_dev; 3003 struct hwrm_queue_cos2bw_cfg_input req = {0}; 3004 struct hwrm_queue_cos2bw_cfg_output resp = {0}; 3005 struct bnxt_fw_msg fw_msg; 3006 void *data; 3007 int indx; 3008 int rc; 3009 3010 memset(&fw_msg, 0, sizeof(fw_msg)); 3011 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 3012 HWRM_QUEUE_COS2BW_CFG, -1, target_id); 3013 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 3014 3015 /* Chimp wants enable bit to retain previous 3016 * config done by L2 driver 3017 */ 3018 for (indx = 0; indx < 8; indx++) { 3019 if (cfg[indx].queue_id < 40) { 3020 req.enables |= cpu_to_le32( 3021 HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID << 3022 indx); 3023 } 3024 3025 data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4); 3026 memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4); 3027 if (indx == 0) { 3028 req.queue_id0 = cfg[0].queue_id; 3029 req.unused_0 = 0; 3030 } 3031 } 3032 3033 memset(&resp, 0, sizeof(resp)); 3034 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 3035 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 3036 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 3037 return rc; 3038 } 3039 3040 int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, 3041 struct bnxt_qplib_query_fn_info *fn_info, 3042 u32 *pf_mask, u32 *first_pf) 3043 { 3044 struct hwrm_func_host_pf_ids_query_output resp = {0}; 3045 struct hwrm_func_host_pf_ids_query_input req; 3046 struct bnxt_en_dev *en_dev = rdev->en_dev; 3047 struct bnxt_fw_msg fw_msg; 3048 int rc; 3049 3050 memset(&fw_msg, 0, sizeof(fw_msg)); 3051 memset(&req, 0, sizeof(req)); 3052 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 3053 HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1); 3054 /* To query the info from the host EPs */ 3055 switch (fn_info->host) { 3056 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC: 3057 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0: 3058 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1: 3059 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2: 3060 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3: 3061 req.host = fn_info->host; 3062 break; 3063 default: 3064 req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0; 3065 break; 3066 } 3067 3068 req.filter = fn_info->filter; 3069 if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE) 3070 req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL; 3071 3072 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 3073 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 3074 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 3075 3076 3077 *first_pf = le16_to_cpu(resp.first_pf_id); 3078 *pf_mask = le16_to_cpu(resp.pf_ordinal_mask); 3079 3080 return rc; 3081 } 3082 3083 static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev) 3084 { 3085 struct bnxt_qplib_ctx *hctx; 3086 struct bnxt_qplib_res *res; 3087 u16 tid = 0xffff; 3088 3089 res = &rdev->qplib_res; 3090 hctx = res->hctx; 3091 3092 if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) { 3093 bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid); 3094 bnxt_qplib_free_stat_mem(res, &hctx->stats); 3095 } 3096 } 3097 3098 static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev) 3099 { 3100 test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags); 3101 } 3102 3103 static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev) 3104 { 3105 struct bnxt_qplib_ctx *hctx; 3106 struct bnxt_qplib_res *res; 3107 u16 tid = 0xffff; 3108 int rc; 3109 3110 res = &rdev->qplib_res; 3111 hctx = res->hctx; 3112 3113 rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats); 3114 if (rc) 3115 return -ENOMEM; 3116 rc = bnxt_re_net_stats_ctx_alloc(rdev, tid); 3117 if (rc) 3118 goto free_stat_mem; 3119 set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags); 3120 3121 return 0; 3122 3123 free_stat_mem: 3124 bnxt_qplib_free_stat_mem(res, &hctx->stats); 3125 3126 return rc; 3127 } 3128 3129 static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev) 3130 { 3131 int rc; 3132 3133 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw); 3134 if (rc) 3135 return rc; 3136 if (!bnxt_re_check_min_attr(rdev)) 3137 return -EINVAL; 3138 return 0; 3139 } 3140 3141 static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev) 3142 { 3143 bnxt_qplib_clear_tbls(&rdev->qplib_res); 3144 bnxt_qplib_free_tbls(&rdev->qplib_res); 3145 } 3146 3147 static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev) 3148 { 3149 struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx; 3150 u8 pppp_factor = 0; 3151 int rc; 3152 3153 /* 3154 * TODO: Need a better mechanism for spreading of the 3155 * 512 extended PPP pages. For now, spreading it 3156 * based on port_count 3157 */ 3158 if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push) 3159 pppp_factor = rdev->en_dev->port_count; 3160 rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor); 3161 if (rc) 3162 return rc; 3163 bnxt_qplib_init_tbls(&rdev->qplib_res); 3164 set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags); 3165 3166 return 0; 3167 } 3168 3169 static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev) 3170 { 3171 struct bnxt_qplib_nq *nq; 3172 int i; 3173 3174 if (!rdev->nqr.max_init) 3175 return; 3176 3177 for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) { 3178 nq = &rdev->nqr.nq[i]; 3179 bnxt_qplib_disable_nq(nq); 3180 bnxt_re_net_ring_free(rdev, nq->ring_id); 3181 bnxt_qplib_free_nq_mem(nq); 3182 } 3183 rdev->nqr.max_init = 0; 3184 } 3185 3186 static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev) 3187 { 3188 struct bnxt_re_ring_attr rattr = {}; 3189 struct bnxt_qplib_nq *nq; 3190 int rc, i; 3191 int depth; 3192 u32 offt; 3193 u16 vec; 3194 3195 mutex_init(&rdev->nqr.load_lock); 3196 /* 3197 * TODO: Optimize the depth based on the 3198 * number of NQs. 3199 */ 3200 depth = BNXT_QPLIB_NQE_MAX_CNT; 3201 for (i = 0; i < rdev->nqr.num_msix - 1; i++) { 3202 nq = &rdev->nqr.nq[i]; 3203 vec = rdev->nqr.msix_entries[i + 1].vector; 3204 offt = rdev->nqr.msix_entries[i + 1].db_offset; 3205 nq->hwq.max_elements = depth; 3206 rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq); 3207 if (rc) { 3208 dev_err(rdev_to_dev(rdev), 3209 "Failed to get mem for NQ %d, rc = 0x%x", 3210 i, rc); 3211 goto fail_mem; 3212 } 3213 3214 rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr; 3215 rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count; 3216 rattr.type = bnxt_re_get_rtype(rdev); 3217 rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; 3218 rattr.depth = nq->hwq.max_elements - 1; 3219 rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx; 3220 3221 /* Set DBR pacing capability on the first NQ ring only */ 3222 if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) 3223 rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING; 3224 else 3225 rattr.flags = 0; 3226 3227 rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id); 3228 if (rc) { 3229 nq->ring_id = 0xffff; /* Invalid ring-id */ 3230 dev_err(rdev_to_dev(rdev), 3231 "Failed to get fw id for NQ %d, rc = 0x%x", 3232 i, rc); 3233 goto fail_ring; 3234 } 3235 3236 rc = bnxt_qplib_enable_nq(nq, i, vec, offt, 3237 &bnxt_re_cqn_handler, 3238 &bnxt_re_srqn_handler); 3239 if (rc) { 3240 dev_err(rdev_to_dev(rdev), 3241 "Failed to enable NQ %d, rc = 0x%x", i, rc); 3242 goto fail_en; 3243 } 3244 } 3245 3246 rdev->nqr.max_init = i; 3247 return 0; 3248 fail_en: 3249 /* *nq was i'th nq */ 3250 bnxt_re_net_ring_free(rdev, nq->ring_id); 3251 fail_ring: 3252 bnxt_qplib_free_nq_mem(nq); 3253 fail_mem: 3254 rdev->nqr.max_init = i; 3255 return rc; 3256 } 3257 3258 static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev) 3259 { 3260 int i; 3261 3262 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) 3263 device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); 3264 } 3265 3266 static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev) 3267 { 3268 int i, j, rc = 0; 3269 3270 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { 3271 rc = device_create_file(&rdev->ibdev.dev, 3272 bnxt_re_attributes[i]); 3273 if (rc) { 3274 dev_err(rdev_to_dev(rdev), 3275 "Failed to create IB sysfs with rc = 0x%x", rc); 3276 /* Must clean up all created device files */ 3277 for (j = 0; j < i; j++) 3278 device_remove_file(&rdev->ibdev.dev, 3279 bnxt_re_attributes[j]); 3280 clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 3281 ib_unregister_device(&rdev->ibdev); 3282 return 1; 3283 } 3284 } 3285 return 0; 3286 } 3287 3288 /* worker thread for polling periodic events. Now used for QoS programming*/ 3289 static void bnxt_re_worker(struct work_struct *work) 3290 { 3291 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 3292 worker.work); 3293 int rc; 3294 3295 /* QoS is in 30s cadence for PFs*/ 3296 if (!rdev->is_virtfn && !rdev->worker_30s--) 3297 rdev->worker_30s = 30; 3298 /* Use trylock for bnxt_re_dev_lock as this can be 3299 * held for long time by debugfs show path while issuing 3300 * HWRMS. If the debugfs name update is not done in this 3301 * iteration, the driver will check for the same in the 3302 * next schedule of the worker i.e after 1 sec. 3303 */ 3304 if (mutex_trylock(&bnxt_re_dev_lock)) 3305 mutex_unlock(&bnxt_re_dev_lock); 3306 3307 if (!rdev->stats.stats_query_sec) 3308 goto resched; 3309 3310 if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) && 3311 (rdev->is_virtfn || 3312 !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) { 3313 if (!(rdev->stats.stats_query_counter++ % 3314 rdev->stats.stats_query_sec)) { 3315 rc = bnxt_re_get_qos_stats(rdev); 3316 if (rc && rc != -ENOMEM) 3317 clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, 3318 &rdev->flags); 3319 } 3320 } 3321 3322 resched: 3323 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); 3324 } 3325 3326 static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) 3327 { 3328 if (!(rdev->dbr_drop_recov || rdev->dbr_pacing)) 3329 return 0; 3330 3331 rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL); 3332 if (!rdev->dbr_sw_stats) 3333 return -ENOMEM; 3334 3335 return 0; 3336 } 3337 3338 static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) 3339 { 3340 kfree(rdev->dbr_sw_stats); 3341 rdev->dbr_sw_stats = NULL; 3342 } 3343 3344 static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev) 3345 { 3346 rdev->dbr_drop_recov_wq = 3347 create_singlethread_workqueue("bnxt_re_dbr_drop_recov"); 3348 if (!rdev->dbr_drop_recov_wq) { 3349 dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!"); 3350 return -EINVAL; 3351 } 3352 rdev->dbr_drop_recov = true; 3353 3354 /* Enable configfs setting dbr_drop_recov by default*/ 3355 rdev->user_dbr_drop_recov = true; 3356 3357 rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT; 3358 return 0; 3359 } 3360 3361 static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev) 3362 { 3363 if (rdev->dbr_drop_recov_wq) { 3364 flush_workqueue(rdev->dbr_drop_recov_wq); 3365 destroy_workqueue(rdev->dbr_drop_recov_wq); 3366 rdev->dbr_drop_recov_wq = NULL; 3367 } 3368 rdev->dbr_drop_recov = false; 3369 } 3370 3371 static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) 3372 { 3373 int rc; 3374 3375 /* Allocate a page for app use */ 3376 rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL); 3377 if (!rdev->dbr_page) { 3378 dev_err(rdev_to_dev(rdev), "DBR page allocation failed!"); 3379 return -ENOMEM; 3380 } 3381 memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE); 3382 rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page; 3383 rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev); 3384 if (rc) { 3385 dev_err(rdev_to_dev(rdev), 3386 "Failed to query dbr pacing config %d\n", rc); 3387 goto fail; 3388 } 3389 /* Create a work queue for scheduling dbq event */ 3390 rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq"); 3391 if (!rdev->dbq_wq) { 3392 dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!"); 3393 rc = -ENOMEM; 3394 goto fail; 3395 } 3396 /* MAP grc window 2 for reading db fifo depth */ 3397 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0, 3398 rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK); 3399 rdev->dbr_db_fifo_reg_off = 3400 (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + 3401 0x2000; 3402 rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off; 3403 3404 rdev->dbr_bar_addr = 3405 pci_resource_start(rdev->qplib_res.pdev, 0) + 3406 rdev->dbr_db_fifo_reg_off; 3407 3408 /* Percentage of DB FIFO */ 3409 rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD; 3410 rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD; 3411 rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; 3412 rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME; 3413 rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; 3414 rdev->do_pacing_save = rdev->dbr_def_do_pacing; 3415 bnxt_re_set_default_pacing_data(rdev); 3416 dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n"); 3417 3418 return 0; 3419 fail: 3420 free_page((u64)rdev->dbr_page); 3421 rdev->dbr_page = NULL; 3422 return rc; 3423 } 3424 3425 static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) 3426 { 3427 if (rdev->dbq_wq) 3428 flush_workqueue(rdev->dbq_wq); 3429 3430 cancel_work_sync(&rdev->dbq_fifo_check_work); 3431 cancel_delayed_work_sync(&rdev->dbq_pacing_work); 3432 3433 if (rdev->dbq_wq) { 3434 destroy_workqueue(rdev->dbq_wq); 3435 rdev->dbq_wq = NULL; 3436 } 3437 3438 if (rdev->dbr_page) 3439 free_page((u64)rdev->dbr_page); 3440 rdev->dbr_page = NULL; 3441 rdev->dbr_pacing = false; 3442 } 3443 3444 /* enable_dbr_pacing needs to be done only for older FWs 3445 * where host selects primary function. ie. pacing_ext 3446 * flags is not set. 3447 */ 3448 int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev) 3449 { 3450 struct bnxt_qplib_nq *nq; 3451 3452 nq = &rdev->nqr.nq[0]; 3453 rdev->dbq_nq_id = nq->ring_id; 3454 3455 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && 3456 bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { 3457 if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) { 3458 dev_err(rdev_to_dev(rdev), 3459 "Failed to set dbr pacing config\n"); 3460 return -EIO; 3461 } 3462 /* MAP grc window 8 for ARMing the NQ DBQ */ 3463 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0, 3464 rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); 3465 rdev->dbr_aeq_arm_reg_off = 3466 (rdev->chip_ctx->dbr_aeq_arm_reg & 3467 BNXT_GRC_OFFSET_MASK) + 0x8000; 3468 writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1); 3469 } 3470 3471 return 0; 3472 } 3473 3474 /* disable_dbr_pacing needs to be done only for older FWs 3475 * where host selects primary function. ie. pacing_ext 3476 * flags is not set. 3477 */ 3478 3479 int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev) 3480 { 3481 int rc = 0; 3482 3483 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && 3484 bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) 3485 rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false); 3486 3487 return rc; 3488 } 3489 3490 static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) 3491 { 3492 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { 3493 bnxt_re_sysfs_destroy_file(rdev); 3494 /* Cleanup ib dev */ 3495 ib_unregister_device(&rdev->ibdev); 3496 clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 3497 return; 3498 } 3499 } 3500 3501 static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) 3502 { 3503 struct bnxt_qplib_dpi *kdpi; 3504 int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT; 3505 3506 bnxt_re_net_unregister_async_event(rdev); 3507 3508 bnxt_re_put_stats2_ctx(rdev); 3509 if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, 3510 &rdev->flags)) { 3511 /* did the caller hold the lock? */ 3512 mutex_lock(&bnxt_re_dev_lock); 3513 list_del_rcu(&rdev->list); 3514 mutex_unlock(&bnxt_re_dev_lock); 3515 } 3516 3517 bnxt_re_uninit_dcb_wq(rdev); 3518 bnxt_re_uninit_aer_wq(rdev); 3519 3520 bnxt_re_deinitialize_dbr_drop_recov(rdev); 3521 3522 if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) 3523 (void)bnxt_re_disable_dbr_pacing(rdev); 3524 3525 if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) { 3526 cancel_delayed_work_sync(&rdev->worker); 3527 } 3528 3529 /* Wait for ULPs to release references */ 3530 while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count) 3531 usleep_range(500, 1000); 3532 if (!wait_count) 3533 dev_err(rdev_to_dev(rdev), 3534 "CQ resources not freed by stack, count = 0x%x", 3535 atomic_read(&rdev->stats.rsors.cq_count)); 3536 3537 kdpi = &rdev->dpi_privileged; 3538 if (kdpi->umdbr) { /* kernel DPI was allocated with success */ 3539 (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi); 3540 /* 3541 * Driver just need to know no command had failed 3542 * during driver load sequence and below command is 3543 * required indeed. Piggybacking dpi allocation status. 3544 */ 3545 } 3546 3547 /* Protect the device uninitialization and start_irq/stop_irq L2 3548 * callbacks with rtnl lock to avoid race condition between these calls 3549 */ 3550 rtnl_lock(); 3551 if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags)) 3552 bnxt_re_clean_nqs(rdev); 3553 rtnl_unlock(); 3554 3555 if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags)) 3556 bnxt_re_free_tbls(rdev); 3557 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) { 3558 rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); 3559 if (rc) 3560 dev_warn(rdev_to_dev(rdev), 3561 "Failed to deinitialize fw, rc = 0x%x", rc); 3562 } 3563 3564 bnxt_re_put_stats_ctx(rdev); 3565 3566 if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags)) 3567 bnxt_qplib_free_hwctx(&rdev->qplib_res); 3568 3569 rtnl_lock(); 3570 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) 3571 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); 3572 3573 if (rdev->dbr_pacing) 3574 bnxt_re_deinitialize_dbr_pacing(rdev); 3575 3576 bnxt_re_free_dbr_sw_stats_mem(rdev); 3577 3578 if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags)) 3579 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id); 3580 3581 if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) 3582 bnxt_qplib_free_rcfw_channel(&rdev->qplib_res); 3583 3584 if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) 3585 bnxt_re_free_msix(rdev); 3586 rtnl_unlock(); 3587 3588 bnxt_re_destroy_chip_ctx(rdev); 3589 3590 if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { 3591 if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, 3592 &rdev->flags)) 3593 bnxt_re_unregister_netdev(rdev); 3594 } 3595 } 3596 3597 static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type) 3598 { 3599 struct bnxt_re_ring_attr rattr = {}; 3600 struct bnxt_qplib_creq_ctx *creq; 3601 int vec, offset; 3602 int rc = 0; 3603 3604 if (op_type != BNXT_RE_POST_RECOVERY_INIT) { 3605 /* Registered a new RoCE device instance to netdev */ 3606 rc = bnxt_re_register_netdev(rdev); 3607 if (rc) 3608 return -EINVAL; 3609 } 3610 set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 3611 3612 rc = bnxt_re_setup_chip_ctx(rdev); 3613 if (rc) { 3614 dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); 3615 bnxt_re_unregister_netdev(rdev); 3616 clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 3617 rc = -EINVAL; 3618 return rc; 3619 } 3620 3621 /* Protect the device initialization and start_irq/stop_irq L2 callbacks 3622 * with rtnl lock to avoid race condition between these calls 3623 */ 3624 rtnl_lock(); 3625 rc = bnxt_re_request_msix(rdev); 3626 if (rc) { 3627 dev_err(rdev_to_dev(rdev), 3628 "Requesting MSI-X vectors failed with rc = 0x%x", rc); 3629 rc = -EINVAL; 3630 goto release_rtnl; 3631 } 3632 set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); 3633 3634 /* Establish RCFW Communication Channel to initialize the context 3635 memory for the function and all child VFs */ 3636 rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res); 3637 if (rc) { 3638 dev_err(rdev_to_dev(rdev), 3639 "Failed to alloc mem for rcfw, rc = %#x\n", rc); 3640 goto release_rtnl; 3641 } 3642 set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags); 3643 3644 creq = &rdev->rcfw.creq; 3645 rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr; 3646 rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; 3647 rattr.type = bnxt_re_get_rtype(rdev); 3648 rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; 3649 rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1; 3650 rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx; 3651 rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); 3652 if (rc) { 3653 creq->ring_id = 0xffff; 3654 dev_err(rdev_to_dev(rdev), 3655 "Failed to allocate CREQ fw id with rc = 0x%x", rc); 3656 goto release_rtnl; 3657 } 3658 3659 set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags); 3660 3661 if (!rdev->chip_ctx) 3662 goto release_rtnl; 3663 3664 if (!(_is_chip_p7(rdev->chip_ctx))) { 3665 /* Program the NQ ID for DBQ notification */ 3666 if (rdev->chip_ctx->modes.dbr_pacing_v0 || 3667 bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || 3668 bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 3669 rc = bnxt_re_initialize_dbr_pacing(rdev); 3670 if (!rc) 3671 rdev->dbr_pacing = true; 3672 else 3673 rdev->dbr_pacing = false; 3674 dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", 3675 __func__, rc); 3676 } 3677 } 3678 3679 vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; 3680 offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset; 3681 rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset, 3682 &bnxt_re_aeq_handler); 3683 if (rc) { 3684 dev_err(rdev_to_dev(rdev), 3685 "Failed to enable RCFW channel with rc = 0x%x", rc); 3686 goto release_rtnl; 3687 } 3688 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); 3689 3690 rc = bnxt_re_update_dev_attr(rdev); 3691 if (rc) 3692 goto release_rtnl; 3693 bnxt_re_set_resource_limits(rdev); 3694 if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { 3695 rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res); 3696 if (rc) { 3697 dev_err(rdev_to_dev(rdev), 3698 "Failed to alloc hw contexts, rc = 0x%x", rc); 3699 goto release_rtnl; 3700 } 3701 set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags); 3702 } 3703 3704 rc = bnxt_re_get_stats_ctx(rdev); 3705 if (rc) 3706 goto release_rtnl; 3707 3708 rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn); 3709 if (rc) { 3710 dev_err(rdev_to_dev(rdev), 3711 "Failed to initialize fw with rc = 0x%x", rc); 3712 goto release_rtnl; 3713 } 3714 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags); 3715 3716 /* Based resource count on the 'new' device caps */ 3717 rc = bnxt_re_update_dev_attr(rdev); 3718 if (rc) 3719 goto release_rtnl; 3720 rc = bnxt_re_alloc_init_tbls(rdev); 3721 if (rc) { 3722 dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x", 3723 rc); 3724 goto release_rtnl; 3725 } 3726 rc = bnxt_re_setup_nqs(rdev); 3727 if (rc) { 3728 dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n", 3729 rc); 3730 if (rdev->nqr.max_init == 0) 3731 goto release_rtnl; 3732 3733 dev_warn(rdev_to_dev(rdev), 3734 "expected nqs %d available nqs %d\n", 3735 rdev->nqr.num_msix, rdev->nqr.max_init); 3736 } 3737 set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags); 3738 rtnl_unlock(); 3739 3740 rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged, 3741 rdev, BNXT_QPLIB_DPI_TYPE_KERNEL); 3742 if (rc) 3743 goto fail; 3744 3745 if (rdev->dbr_pacing) 3746 bnxt_re_enable_dbr_pacing(rdev); 3747 3748 if (rdev->chip_ctx->modes.dbr_drop_recov) 3749 bnxt_re_initialize_dbr_drop_recov(rdev); 3750 3751 rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev); 3752 if (rc) 3753 goto fail; 3754 3755 /* This block of code is needed for error recovery support */ 3756 if (!rdev->is_virtfn) { 3757 struct bnxt_re_tc_rec *tc_rec; 3758 3759 tc_rec = &rdev->tc_rec[0]; 3760 rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF); 3761 if (rc) { 3762 dev_err(rdev_to_dev(rdev), 3763 "Failed to query port config rc:%d", rc); 3764 return rc; 3765 } 3766 3767 /* Query f/w defaults of CC params */ 3768 rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param); 3769 if (rc) 3770 dev_warn(rdev_to_dev(rdev), 3771 "Failed to query CC defaults\n"); 3772 if (1) { 3773 rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev); 3774 if (rdev->num_vfs) { 3775 bnxt_re_set_resource_limits(rdev); 3776 bnxt_qplib_set_func_resources(&rdev->qplib_res); 3777 } 3778 } 3779 } 3780 INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); 3781 set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags); 3782 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); 3783 3784 bnxt_re_init_dcb_wq(rdev); 3785 bnxt_re_init_aer_wq(rdev); 3786 mutex_lock(&bnxt_re_dev_lock); 3787 list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); 3788 /* Added to the list, not in progress anymore */ 3789 gadd_dev_inprogress--; 3790 set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags); 3791 mutex_unlock(&bnxt_re_dev_lock); 3792 3793 3794 return rc; 3795 release_rtnl: 3796 rtnl_unlock(); 3797 fail: 3798 bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); 3799 3800 return rc; 3801 } 3802 3803 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) 3804 { 3805 int rc = 0; 3806 3807 rc = bnxt_re_register_ib(rdev); 3808 if (rc) { 3809 dev_err(rdev_to_dev(rdev), 3810 "Register IB failed with rc = 0x%x", rc); 3811 goto fail; 3812 } 3813 if (bnxt_re_sysfs_create_file(rdev)) { 3814 bnxt_re_stopqps_and_ib_uninit(rdev); 3815 goto fail; 3816 } 3817 3818 set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 3819 set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); 3820 set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); 3821 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); 3822 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); 3823 3824 return rc; 3825 fail: 3826 bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); 3827 return rc; 3828 } 3829 3830 /* wrapper for ib_init funcs */ 3831 int _bnxt_re_ib_init(struct bnxt_re_dev *rdev) 3832 { 3833 return bnxt_re_ib_init(rdev); 3834 } 3835 3836 /* wrapper for aux init funcs */ 3837 int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev) 3838 { 3839 bnxt_re_ib_init_2(rdev); 3840 return 0; /* add return for future proof */ 3841 } 3842 3843 static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) 3844 { 3845 bnxt_re_dev_dealloc(rdev); 3846 } 3847 3848 3849 static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, 3850 struct bnxt_en_dev *en_dev) 3851 { 3852 struct ifnet *realdev = NULL; 3853 3854 realdev = netdev; 3855 if (realdev) 3856 dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__, 3857 realdev, netdev); 3858 /* 3859 * Note: 3860 * The first argument to bnxt_re_dev_alloc() is 'netdev' and 3861 * not 'realdev', since in the case of bonding we want to 3862 * register the bonded virtual netdev (master) to the ib stack. 3863 * And 'en_dev' (for L2/PCI communication) is the first slave 3864 * device (PF0 on the card). 3865 * In the case of a regular netdev, both netdev and the en_dev 3866 * correspond to the same device. 3867 */ 3868 *rdev = bnxt_re_dev_alloc(netdev, en_dev); 3869 if (!*rdev) { 3870 pr_err("%s: netdev %p not handled", 3871 ROCE_DRV_MODULE_NAME, netdev); 3872 return -ENOMEM; 3873 } 3874 bnxt_re_hold(*rdev); 3875 3876 return 0; 3877 } 3878 3879 void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) 3880 { 3881 rdev->espeed = rdev->en_dev->espeed; 3882 rdev->lanes = rdev->en_dev->lanes; 3883 return; 3884 } 3885 3886 void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev) 3887 { 3888 dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n", 3889 __func__, rdev); 3890 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); 3891 bnxt_re_ib_uninit(rdev); 3892 } 3893 3894 void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, 3895 struct auxiliary_device *aux_dev) 3896 { 3897 struct bnxt_re_en_dev_info *en_info; 3898 struct bnxt_qplib_cmdq_ctx *cmdq; 3899 struct bnxt_qplib_rcfw *rcfw; 3900 3901 rcfw = &rdev->rcfw; 3902 cmdq = &rcfw->cmdq; 3903 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) 3904 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 3905 3906 dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev); 3907 bnxt_re_dev_uninit(rdev, op_type); 3908 en_info = auxiliary_get_drvdata(aux_dev); 3909 if (en_info) { 3910 rtnl_lock(); 3911 en_info->rdev = NULL; 3912 rtnl_unlock(); 3913 if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { 3914 clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags); 3915 clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags); 3916 clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); 3917 } 3918 } 3919 bnxt_re_dev_unreg(rdev); 3920 } 3921 3922 int bnxt_re_add_device(struct bnxt_re_dev **rdev, 3923 struct ifnet *netdev, 3924 u8 qp_mode, u8 op_type, 3925 u32 num_msix_requested, 3926 struct auxiliary_device *aux_dev) 3927 { 3928 struct bnxt_re_en_dev_info *en_info; 3929 struct bnxt_en_dev *en_dev; 3930 int rc = 0; 3931 3932 en_info = auxiliary_get_drvdata(aux_dev); 3933 en_dev = en_info->en_dev; 3934 3935 mutex_lock(&bnxt_re_dev_lock); 3936 /* Check if driver already in mod exit and aux_dev is valid */ 3937 if (gmod_exit || !aux_dev) { 3938 mutex_unlock(&bnxt_re_dev_lock); 3939 return -ENODEV; 3940 } 3941 /* Add device in progress */ 3942 gadd_dev_inprogress++; 3943 mutex_unlock(&bnxt_re_dev_lock); 3944 3945 rc = bnxt_re_dev_reg(rdev, netdev, en_dev); 3946 if (rc) { 3947 dev_dbg(NULL, "Failed to create add device for netdev %p\n", 3948 netdev); 3949 /* 3950 * For BNXT_RE_POST_RECOVERY_INIT special case 3951 * called from bnxt_re_start, the work is 3952 * complete only after, bnxt_re_start completes 3953 * bnxt_unregister_device in case of failure. 3954 * So bnxt_re_start will decrement gadd_dev_inprogress 3955 * in case of failure. 3956 */ 3957 if (op_type != BNXT_RE_POST_RECOVERY_INIT) { 3958 mutex_lock(&bnxt_re_dev_lock); 3959 gadd_dev_inprogress--; 3960 mutex_unlock(&bnxt_re_dev_lock); 3961 } 3962 return rc; 3963 } 3964 3965 if (rc != 0) 3966 goto ref_error; 3967 3968 /* 3969 * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load. 3970 * Otherwaise, this invocation can be the result of lag create / destroy, 3971 * err revovery, hot fw upgrade, etc.. 3972 */ 3973 if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) { 3974 if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES) 3975 num_msix_requested = max_msix_vec[bnxt_re_probe_count++]; 3976 else 3977 /* Consider as default when probe_count exceeds its limit */ 3978 num_msix_requested = 0; 3979 3980 /* if user specifies only one value, use the same for all PFs */ 3981 if (max_msix_vec_argc == 1) 3982 num_msix_requested = max_msix_vec[0]; 3983 } 3984 3985 (*rdev)->num_msix_requested = num_msix_requested; 3986 (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode; 3987 (*rdev)->adev = aux_dev; 3988 (*rdev)->dev_addr = en_dev->softc->func.mac_addr; 3989 /* Before updating the rdev pointer in bnxt_re_en_dev_info structure, 3990 * take the rtnl lock to avoid accessing invalid rdev pointer from 3991 * L2 ULP callbacks. This is applicable in all the places where rdev 3992 * pointer is updated in bnxt_re_en_dev_info. 3993 */ 3994 rtnl_lock(); 3995 en_info->rdev = *rdev; 3996 rtnl_unlock(); 3997 rc = bnxt_re_dev_init(*rdev, op_type); 3998 if (rc) { 3999 ref_error: 4000 bnxt_re_dev_unreg(*rdev); 4001 *rdev = NULL; 4002 /* 4003 * For BNXT_RE_POST_RECOVERY_INIT special case 4004 * called from bnxt_re_start, the work is 4005 * complete only after, bnxt_re_start completes 4006 * bnxt_unregister_device in case of failure. 4007 * So bnxt_re_start will decrement gadd_dev_inprogress 4008 * in case of failure. 4009 */ 4010 if (op_type != BNXT_RE_POST_RECOVERY_INIT) { 4011 mutex_lock(&bnxt_re_dev_lock); 4012 gadd_dev_inprogress--; 4013 mutex_unlock(&bnxt_re_dev_lock); 4014 } 4015 } 4016 dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev); 4017 if (!rc) { 4018 set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); 4019 } 4020 return rc; 4021 } 4022 4023 struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev) 4024 { 4025 struct pci_dev *pdev_in = rdev->en_dev->pdev; 4026 int tmp_bus_num, bus_num = pdev_in->bus->number; 4027 int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); 4028 int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn); 4029 struct bnxt_re_dev *tmp_rdev; 4030 4031 rcu_read_lock(); 4032 list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { 4033 tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; 4034 tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); 4035 tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn); 4036 4037 if (bus_num == tmp_bus_num && dev_num == tmp_dev_num && 4038 func_num != tmp_func_num) { 4039 rcu_read_unlock(); 4040 return tmp_rdev; 4041 } 4042 } 4043 rcu_read_unlock(); 4044 return NULL; 4045 } 4046 4047 4048 int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, 4049 struct ifnet *vlan_dev, 4050 struct ifnet *netdev, 4051 struct auxiliary_device *adev) 4052 { 4053 struct bnxt_re_work *re_work; 4054 4055 /* Allocate for the deferred task */ 4056 re_work = kzalloc(sizeof(*re_work), GFP_KERNEL); 4057 if (!re_work) 4058 return -ENOMEM; 4059 4060 re_work->rdev = rdev; 4061 re_work->event = event; 4062 re_work->vlan_dev = vlan_dev; 4063 re_work->adev = adev; 4064 INIT_WORK(&re_work->work, bnxt_re_task); 4065 if (rdev) 4066 atomic_inc(&rdev->sched_count); 4067 re_work->netdev = netdev; 4068 queue_work(bnxt_re_wq, &re_work->work); 4069 4070 return 0; 4071 } 4072 4073 4074 int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev) 4075 { 4076 struct pci_dev *pdev_in = rdev->en_dev->pdev; 4077 int tmp_bus_num, bus_num = pdev_in->bus->number; 4078 int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); 4079 struct bnxt_re_dev *tmp_rdev; 4080 int pf_cnt = 0; 4081 4082 rcu_read_lock(); 4083 list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { 4084 tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; 4085 tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); 4086 4087 if (bus_num == tmp_bus_num && dev_num == tmp_dev_num) 4088 pf_cnt++; 4089 } 4090 rcu_read_unlock(); 4091 return pf_cnt; 4092 } 4093 4094 /* Handle all deferred netevents tasks */ 4095 static void bnxt_re_task(struct work_struct *work) 4096 { 4097 struct bnxt_re_en_dev_info *en_info; 4098 struct auxiliary_device *aux_dev; 4099 struct bnxt_re_work *re_work; 4100 struct bnxt_re_dev *rdev; 4101 4102 re_work = container_of(work, struct bnxt_re_work, work); 4103 4104 mutex_lock(&bnxt_re_mutex); 4105 rdev = re_work->rdev; 4106 4107 /* 4108 * If the previous rdev is deleted due to bond creation 4109 * do not handle the event 4110 */ 4111 if (!bnxt_re_is_rdev_valid(rdev)) 4112 goto exit; 4113 4114 /* Ignore the event, if the device is not registred with IB stack. This 4115 * is to avoid handling any event while the device is added/removed. 4116 */ 4117 if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { 4118 dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx", 4119 __func__, re_work->event); 4120 goto done; 4121 } 4122 4123 /* Extra check to silence coverity. We shouldn't handle any event 4124 * when rdev is NULL. 4125 */ 4126 if (!rdev) 4127 goto exit; 4128 4129 dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx", 4130 re_work->event); 4131 4132 switch (re_work->event) { 4133 case NETDEV_UP: 4134 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4135 IB_EVENT_PORT_ACTIVE); 4136 bnxt_re_net_register_async_event(rdev); 4137 break; 4138 4139 case NETDEV_DOWN: 4140 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); 4141 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); 4142 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4143 IB_EVENT_PORT_ERR); 4144 break; 4145 4146 case NETDEV_CHANGE: 4147 if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) { 4148 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); 4149 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4150 IB_EVENT_PORT_ERR); 4151 break; 4152 } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) { 4153 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4154 IB_EVENT_PORT_ACTIVE); 4155 } 4156 4157 /* temporarily disable the check for SR2 */ 4158 if (!bnxt_qplib_query_cc_param(&rdev->qplib_res, 4159 &rdev->cc_param) && 4160 !_is_chip_p7(rdev->chip_ctx)) { 4161 /* 4162 * Disable CC for 10G speed 4163 * for non p5 devices 4164 */ 4165 if (rdev->sl_espeed == SPEED_10000 && 4166 !_is_chip_gen_p5_p7(rdev->chip_ctx)) { 4167 if (rdev->cc_param.enable) 4168 bnxt_re_clear_cc(rdev); 4169 } else { 4170 if (!rdev->cc_param.enable && 4171 rdev->cc_param.admin_enable) 4172 bnxt_re_setup_cc(rdev); 4173 } 4174 } 4175 break; 4176 4177 case NETDEV_UNREGISTER: 4178 bnxt_re_stopqps_and_ib_uninit(rdev); 4179 aux_dev = rdev->adev; 4180 if (re_work->adev) 4181 goto done; 4182 4183 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev); 4184 4185 break; 4186 4187 default: 4188 break; 4189 } 4190 done: 4191 if (rdev) { 4192 /* memory barrier to guarantee task completion 4193 * before decrementing sched count 4194 */ 4195 mmiowb(); 4196 atomic_dec(&rdev->sched_count); 4197 } 4198 exit: 4199 if (re_work->adev && re_work->event == NETDEV_UNREGISTER) { 4200 en_info = auxiliary_get_drvdata(re_work->adev); 4201 en_info->ib_uninit_done = true; 4202 wake_up(&en_info->waitq); 4203 } 4204 kfree(re_work); 4205 mutex_unlock(&bnxt_re_mutex); 4206 } 4207 4208 /* 4209 "Notifier chain callback can be invoked for the same chain from 4210 different CPUs at the same time". 4211 4212 For cases when the netdev is already present, our call to the 4213 register_netdevice_notifier() will actually get the rtnl_lock() 4214 before sending NETDEV_REGISTER and (if up) NETDEV_UP 4215 events. 4216 4217 But for cases when the netdev is not already present, the notifier 4218 chain is subjected to be invoked from different CPUs simultaneously. 4219 4220 This is protected by the netdev_mutex. 4221 */ 4222 static int bnxt_re_netdev_event(struct notifier_block *notifier, 4223 unsigned long event, void *ptr) 4224 { 4225 struct ifnet *real_dev, *netdev; 4226 struct bnxt_re_dev *rdev = NULL; 4227 4228 netdev = netdev_notifier_info_to_ifp(ptr); 4229 real_dev = rdma_vlan_dev_real_dev(netdev); 4230 if (!real_dev) 4231 real_dev = netdev; 4232 /* In case of bonding,this will be bond's rdev */ 4233 rdev = bnxt_re_from_netdev(real_dev); 4234 4235 if (!rdev) 4236 goto exit; 4237 4238 dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n", 4239 __func__, bnxt_re_netevent(event), event, 4240 rdev ? rdev->netdev ? if_getdname(rdev->netdev) : "->netdev = NULL" : "= NULL", 4241 (real_dev == netdev) ? "= netdev" : if_getdname(real_dev)); 4242 4243 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 4244 goto exit; 4245 4246 bnxt_re_hold(rdev); 4247 4248 if (real_dev != netdev) { 4249 switch (event) { 4250 case NETDEV_UP: 4251 bnxt_re_schedule_work(rdev, event, netdev, 4252 NULL, NULL); 4253 break; 4254 case NETDEV_DOWN: 4255 break; 4256 default: 4257 break; 4258 } 4259 goto done; 4260 } 4261 4262 switch (event) { 4263 case NETDEV_CHANGEADDR: 4264 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 4265 bnxt_re_update_shadow_ah(rdev); 4266 bnxt_qplib_get_guid(rdev->dev_addr, 4267 (u8 *)&rdev->ibdev.node_guid); 4268 break; 4269 4270 case NETDEV_CHANGE: 4271 bnxt_re_get_link_speed(rdev); 4272 bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL); 4273 break; 4274 case NETDEV_UNREGISTER: 4275 /* netdev notifier will call NETDEV_UNREGISTER again later since 4276 * we are still holding the reference to the netdev 4277 */ 4278 4279 /* 4280 * Workaround to avoid ib_unregister hang. Check for module 4281 * reference and dont free up the device if the reference 4282 * is non zero. Checking only for PF functions. 4283 */ 4284 4285 if (rdev) { 4286 dev_info(rdev_to_dev(rdev), 4287 "bnxt_re:Unreg recvd when module refcnt > 0"); 4288 dev_info(rdev_to_dev(rdev), 4289 "bnxt_re:Close all apps using bnxt_re devs"); 4290 dev_info(rdev_to_dev(rdev), 4291 "bnxt_re:Remove the configfs entry created for the device"); 4292 dev_info(rdev_to_dev(rdev), 4293 "bnxt_re:Refer documentation for details"); 4294 goto done; 4295 } 4296 4297 if (atomic_read(&rdev->sched_count) > 0) 4298 goto done; 4299 if (!rdev->unreg_sched) { 4300 bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, 4301 NULL, NULL, NULL); 4302 rdev->unreg_sched = true; 4303 goto done; 4304 } 4305 4306 break; 4307 default: 4308 break; 4309 } 4310 done: 4311 if (rdev) 4312 bnxt_re_put(rdev); 4313 exit: 4314 return NOTIFY_DONE; 4315 } 4316 4317 static struct notifier_block bnxt_re_netdev_notifier = { 4318 .notifier_call = bnxt_re_netdev_event 4319 }; 4320 4321 static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev, 4322 struct auxiliary_device *adev) 4323 { 4324 bnxt_re_stopqps_and_ib_uninit(rdev); 4325 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev); 4326 auxiliary_set_drvdata(adev, NULL); 4327 } 4328 4329 /* 4330 * bnxt_re_remove - Removes the roce aux device 4331 * @adev - aux device pointer 4332 * 4333 * This function removes the roce device. This gets 4334 * called in the mod exit path and pci unbind path. 4335 * If the rdev is bond interace, destroys the lag 4336 * in module exit path, and in pci unbind case 4337 * destroys the lag and recreates other base interface. 4338 * If the device is already removed in error recovery 4339 * path, it just unregister with the L2. 4340 */ 4341 static void bnxt_re_remove(struct auxiliary_device *adev) 4342 { 4343 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); 4344 struct bnxt_en_dev *en_dev; 4345 struct bnxt_re_dev *rdev; 4346 bool primary_dev = false; 4347 bool secondary_dev = false; 4348 4349 if (!en_info) 4350 return; 4351 4352 mutex_lock(&bnxt_re_mutex); 4353 en_dev = en_info->en_dev; 4354 4355 rdev = en_info->rdev; 4356 4357 if (rdev && bnxt_re_is_rdev_valid(rdev)) { 4358 if (pci_channel_offline(rdev->rcfw.pdev)) 4359 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 4360 4361 if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags)) 4362 primary_dev = true; 4363 if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags)) 4364 secondary_dev = true; 4365 4366 /* 4367 * en_dev_info of primary device and secondary device have the 4368 * same rdev pointer when LAG is configured. This rdev pointer 4369 * is rdev of bond interface. 4370 */ 4371 if (!primary_dev && !secondary_dev) { 4372 /* removal of non bond interface */ 4373 bnxt_re_remove_base_interface(rdev, adev); 4374 } else { 4375 /* 4376 * removal of bond primary/secondary interface. In this 4377 * case bond device is already removed, so rdev->binfo 4378 * is NULL. 4379 */ 4380 auxiliary_set_drvdata(adev, NULL); 4381 } 4382 } else { 4383 /* device is removed from ulp stop, unregister the net dev */ 4384 if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) { 4385 rtnl_lock(); 4386 en_dev->en_ops->bnxt_unregister_device(en_dev, 4387 BNXT_ROCE_ULP); 4388 rtnl_unlock(); 4389 } 4390 } 4391 mutex_unlock(&bnxt_re_mutex); 4392 return; 4393 } 4394 4395 /* wrapper for all external user context callers */ 4396 void _bnxt_re_remove(struct auxiliary_device *adev) 4397 { 4398 bnxt_re_remove(adev); 4399 } 4400 4401 static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev) 4402 { 4403 int rc; 4404 4405 rc = bnxt_re_get_device_stats(rdev); 4406 if (rc) 4407 dev_err(rdev_to_dev(rdev), 4408 "Failed initial device stat query"); 4409 4410 bnxt_re_net_register_async_event(rdev); 4411 } 4412 4413 static int bnxt_re_probe(struct auxiliary_device *adev, 4414 const struct auxiliary_device_id *id) 4415 { 4416 struct bnxt_aux_dev *aux_dev = 4417 container_of(adev, struct bnxt_aux_dev, aux_dev); 4418 struct bnxt_re_en_dev_info *en_info; 4419 struct bnxt_en_dev *en_dev = NULL; 4420 struct bnxt_re_dev *rdev; 4421 int rc = -ENODEV; 4422 4423 if (aux_dev) 4424 en_dev = aux_dev->edev; 4425 4426 if (!en_dev) 4427 return rc; 4428 4429 if (en_dev->ulp_version != BNXT_ULP_VERSION) { 4430 pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n", 4431 ROCE_DRV_MODULE_NAME, en_dev->ulp_version); 4432 return -EINVAL; 4433 } 4434 4435 en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); 4436 if (!en_info) 4437 return -ENOMEM; 4438 memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info)); 4439 en_info->en_dev = en_dev; 4440 auxiliary_set_drvdata(adev, en_info); 4441 4442 mutex_lock(&bnxt_re_mutex); 4443 rc = bnxt_re_add_device(&rdev, en_dev->net, 4444 BNXT_RE_GSI_MODE_ALL, 4445 BNXT_RE_COMPLETE_INIT, 4446 BNXT_RE_MSIX_FROM_MOD_PARAM, adev); 4447 if (rc) { 4448 mutex_unlock(&bnxt_re_mutex); 4449 return rc; 4450 } 4451 4452 rc = bnxt_re_ib_init(rdev); 4453 if (rc) 4454 goto err; 4455 4456 bnxt_re_ib_init_2(rdev); 4457 4458 dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev); 4459 rdev->adev = adev; 4460 4461 mutex_unlock(&bnxt_re_mutex); 4462 4463 return 0; 4464 4465 err: 4466 mutex_unlock(&bnxt_re_mutex); 4467 bnxt_re_remove(adev); 4468 4469 return rc; 4470 } 4471 4472 static const struct auxiliary_device_id bnxt_re_id_table[] = { 4473 { .name = BNXT_ADEV_NAME ".rdma", }, 4474 {}, 4475 }; 4476 4477 MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table); 4478 4479 static struct auxiliary_driver bnxt_re_driver = { 4480 .name = "rdma", 4481 .probe = bnxt_re_probe, 4482 .remove = bnxt_re_remove, 4483 .id_table = bnxt_re_id_table, 4484 }; 4485 4486 static int __init bnxt_re_mod_init(void) 4487 { 4488 int rc = 0; 4489 4490 pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version); 4491 4492 bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); 4493 if (!bnxt_re_wq) 4494 return -ENOMEM; 4495 4496 rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier); 4497 if (rc) { 4498 pr_err("%s: Cannot register to netdevice_notifier", 4499 ROCE_DRV_MODULE_NAME); 4500 goto err_netdev; 4501 } 4502 4503 INIT_LIST_HEAD(&bnxt_re_dev_list); 4504 4505 rc = auxiliary_driver_register(&bnxt_re_driver); 4506 if (rc) { 4507 pr_err("%s: Failed to register auxiliary driver\n", 4508 ROCE_DRV_MODULE_NAME); 4509 goto err_auxdrv; 4510 } 4511 4512 return 0; 4513 4514 err_auxdrv: 4515 bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); 4516 4517 err_netdev: 4518 destroy_workqueue(bnxt_re_wq); 4519 4520 return rc; 4521 } 4522 4523 static void __exit bnxt_re_mod_exit(void) 4524 { 4525 gmod_exit = 1; 4526 auxiliary_driver_unregister(&bnxt_re_driver); 4527 4528 bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); 4529 4530 if (bnxt_re_wq) 4531 destroy_workqueue(bnxt_re_wq); 4532 } 4533 4534 module_init(bnxt_re_mod_init); 4535 module_exit(bnxt_re_mod_exit); 4536