1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: Main component of the bnxt_re driver 29 */ 30 31 #include <linux/if_ether.h> 32 #include <linux/module.h> 33 #include <linux/errno.h> 34 #include <linux/pci.h> 35 #include <linux/dma-mapping.h> 36 #include <linux/slab.h> 37 #include <linux/sched.h> 38 #include <linux/delay.h> 39 #include <linux/fs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/ib_addr.h> 42 #include <rdma/ib_cache.h> 43 #include <dev/mlx5/port.h> 44 #include <dev/mlx5/vport.h> 45 #include <linux/list.h> 46 #include <rdma/ib_smi.h> 47 #include <rdma/ib_umem.h> 48 #include <linux/in.h> 49 #include <linux/etherdevice.h> 50 51 #include "bnxt_re.h" 52 #include "ib_verbs.h" 53 #include "bnxt_re-abi.h" 54 #include "bnxt.h" 55 #include "bnxt_log.h" 56 57 static char drv_version[] = 58 "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \ 59 " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n"; 60 61 #define BNXT_RE_DESC "Broadcom NetXtreme RoCE" 62 #define BNXT_ADEV_NAME "if_bnxt" 63 64 MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver"); 65 MODULE_LICENSE("Dual BSD/GPL"); 66 MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1); 67 MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1); 68 MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1); 69 MODULE_DEPEND(bnxt_re, iflib, 1, 1, 1); 70 MODULE_VERSION(bnxt_re, 1); 71 72 73 DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */ 74 75 static unsigned int restrict_mrs = 0; 76 module_param(restrict_mrs, uint, 0); 77 MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K"); 78 79 unsigned int restrict_stats = 0; 80 module_param(restrict_stats, uint, 0); 81 MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default"); 82 83 unsigned int enable_fc = 1; 84 module_param(enable_fc, uint, 0); 85 MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1"); 86 87 unsigned int min_tx_depth = 1; 88 module_param(min_tx_depth, uint, 0); 89 MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1"); 90 91 static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0}; 92 static unsigned int max_msix_vec_argc; 93 module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444); 94 MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64"); 95 96 unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD; 97 module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644); 98 MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64"); 99 100 101 /* globals */ 102 struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list); 103 static int bnxt_re_probe_count; 104 105 DEFINE_MUTEX(bnxt_re_dev_lock); 106 static u32 gmod_exit; 107 static u32 gadd_dev_inprogress; 108 109 static void bnxt_re_task(struct work_struct *work_task); 110 static struct workqueue_struct *bnxt_re_wq; 111 static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev); 112 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, 113 u32 *offset); 114 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev); 115 static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev); 116 void _bnxt_re_remove(struct auxiliary_device *adev); 117 118 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev); 119 120 int bnxt_re_register_netdevice_notifier(struct notifier_block *nb) 121 { 122 int rc; 123 rc = register_netdevice_notifier(nb); 124 return rc; 125 } 126 127 int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb) 128 { 129 int rc; 130 rc = unregister_netdevice_notifier(nb); 131 return rc; 132 } 133 134 void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev) 135 { 136 ibdev->dma_device = &rdev->en_dev->pdev->dev; 137 } 138 139 static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev, 140 u32 fifo_occup) 141 { 142 if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark) 143 rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup; 144 145 if (fifo_occup > 8 * rdev->pacing_algo_th) 146 rdev->dbg_stats->dbq.fifo_occup_slab_4++; 147 else if (fifo_occup > 4 * rdev->pacing_algo_th) 148 rdev->dbg_stats->dbq.fifo_occup_slab_3++; 149 else if (fifo_occup > 2 * rdev->pacing_algo_th) 150 rdev->dbg_stats->dbq.fifo_occup_slab_2++; 151 else if (fifo_occup > rdev->pacing_algo_th) 152 rdev->dbg_stats->dbq.fifo_occup_slab_1++; 153 } 154 155 static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev) 156 { 157 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 158 159 if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark) 160 rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing; 161 162 if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing) 163 rdev->dbg_stats->dbq.do_pacing_slab_5++; 164 else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing) 165 rdev->dbg_stats->dbq.do_pacing_slab_4++; 166 else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing) 167 rdev->dbg_stats->dbq.do_pacing_slab_3++; 168 else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing) 169 rdev->dbg_stats->dbq.do_pacing_slab_2++; 170 else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing) 171 rdev->dbg_stats->dbq.do_pacing_slab_1++; 172 } 173 174 static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp) 175 { 176 return qp->ib_qp.qp_type == IB_QPT_GSI; 177 } 178 179 static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev) 180 { 181 struct bnxt_re_qp *qp; 182 183 mutex_lock(&rdev->qp_lock); 184 list_for_each_entry(qp, &rdev->qp_list, list) { 185 if (bnxt_re_is_qp1_qp(qp)) { 186 mutex_unlock(&rdev->qp_lock); 187 return qp; 188 } 189 } 190 mutex_unlock(&rdev->qp_lock); 191 return NULL; 192 } 193 194 /* Set the maximum number of each resource that the driver actually wants 195 * to allocate. This may be up to the maximum number the firmware has 196 * reserved for the function. The driver may choose to allocate fewer 197 * resources than the firmware maximum. 198 */ 199 static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev) 200 { 201 struct bnxt_qplib_max_res dev_res = {}; 202 struct bnxt_qplib_chip_ctx *cctx; 203 struct bnxt_qplib_dev_attr *attr; 204 struct bnxt_qplib_ctx *hctx; 205 int i; 206 207 attr = rdev->dev_attr; 208 hctx = rdev->qplib_res.hctx; 209 cctx = rdev->chip_ctx; 210 211 bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false); 212 if (!_is_chip_gen_p5_p7(cctx)) { 213 hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp); 214 hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr); 215 /* To accommodate 16k MRs and 16k AHs, 216 * driver has to allocate 32k backing store memory 217 */ 218 hctx->mrw_ctx.max *= 2; 219 hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq); 220 hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq); 221 for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) 222 hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i]; 223 } else { 224 hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp; 225 hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr; 226 hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq; 227 hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq; 228 } 229 } 230 231 static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev, 232 struct bnxt_qplib_vf_res *vf_res, 233 u32 num_vf) 234 { 235 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; 236 struct bnxt_qplib_max_res dev_res = {}; 237 238 bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true); 239 vf_res->max_qp = dev_res.max_qp / num_vf; 240 vf_res->max_srq = dev_res.max_srq / num_vf; 241 vf_res->max_cq = dev_res.max_cq / num_vf; 242 /* 243 * MR and AH shares the same backing store, the value specified 244 * for max_mrw is split into half by the FW for MR and AH 245 */ 246 vf_res->max_mrw = dev_res.max_mr * 2 / num_vf; 247 vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF; 248 } 249 250 static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) 251 { 252 struct bnxt_qplib_ctx *hctx; 253 254 hctx = rdev->qplib_res.hctx; 255 memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res)); 256 bnxt_re_limit_pf_res(rdev); 257 258 if (rdev->num_vfs) 259 bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs); 260 } 261 262 static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev) 263 { 264 struct bnxt_qplib_rcfw *rcfw = NULL; 265 struct bnxt_qplib_nq *nq; 266 int indx; 267 268 rcfw = &rdev->rcfw; 269 for (indx = 0; indx < rdev->nqr.max_init; indx++) { 270 nq = &rdev->nqr.nq[indx]; 271 mutex_lock(&nq->lock); 272 bnxt_qplib_nq_stop_irq(nq, false); 273 mutex_unlock(&nq->lock); 274 } 275 276 bnxt_qplib_rcfw_stop_irq(rcfw, false); 277 } 278 279 static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev) 280 { 281 /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */ 282 bnxt_re_dettach_irq(rdev); 283 284 /* Set the state as detached to prevent sending any more commands */ 285 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 286 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 287 wake_up_all(&rdev->rcfw.cmdq.waitq); 288 } 289 290 #define MAX_DSCP_PRI_TUPLE 64 291 292 struct bnxt_re_dcb_work { 293 struct work_struct work; 294 struct bnxt_re_dev *rdev; 295 struct hwrm_async_event_cmpl cmpl; 296 }; 297 298 static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) 299 { 300 rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); 301 } 302 303 static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) 304 { 305 if (!rdev->dcb_wq) 306 return; 307 flush_workqueue(rdev->dcb_wq); 308 destroy_workqueue(rdev->dcb_wq); 309 rdev->dcb_wq = NULL; 310 } 311 312 static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev) 313 { 314 rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq"); 315 } 316 317 static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev) 318 { 319 if (!rdev->aer_wq) 320 return; 321 flush_workqueue(rdev->aer_wq); 322 destroy_workqueue(rdev->aer_wq); 323 rdev->aer_wq = NULL; 324 } 325 326 static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) 327 { 328 struct bnxt_re_qp *qp; 329 330 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 331 return 0; 332 333 qp = bnxt_re_get_qp1_qp(rdev); 334 if (!qp) 335 return 0; 336 337 qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP; 338 qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp; 339 340 return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); 341 } 342 343 static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev) 344 { 345 struct bnxt_qplib_cc_param *cc_param; 346 struct bnxt_re_tc_rec *tc_rec; 347 bool update_cc = false; 348 u8 dscp_user; 349 int rc; 350 351 cc_param = &rdev->cc_param; 352 tc_rec = &rdev->tc_rec[0]; 353 354 if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user)) 355 return; 356 357 if (cc_param->cnp_dscp_user) { 358 dscp_user = (cc_param->cnp_dscp_user & 0x3f); 359 if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) && 360 (cc_param->alt_tos_dscp != dscp_user)) { 361 cc_param->alt_tos_dscp = dscp_user; 362 cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP; 363 update_cc = true; 364 } 365 } 366 367 if (cc_param->roce_dscp_user) { 368 dscp_user = (cc_param->roce_dscp_user & 0x3f); 369 if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) && 370 (cc_param->tos_dscp != dscp_user)) { 371 cc_param->tos_dscp = dscp_user; 372 cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; 373 update_cc = true; 374 } 375 } 376 377 if (update_cc) { 378 rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); 379 if (rc) 380 dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n"); 381 } 382 } 383 384 static void bnxt_re_dcb_wq_task(struct work_struct *work) 385 { 386 struct bnxt_qplib_cc_param *cc_param; 387 struct bnxt_re_tc_rec *tc_rec; 388 struct bnxt_re_dev *rdev; 389 struct bnxt_re_dcb_work *dcb_work = 390 container_of(work, struct bnxt_re_dcb_work, work); 391 int rc; 392 393 rdev = dcb_work->rdev; 394 if (!rdev) 395 goto exit; 396 397 mutex_lock(&rdev->cc_lock); 398 399 cc_param = &rdev->cc_param; 400 rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param); 401 if (rc) { 402 dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc); 403 goto fail; 404 } 405 tc_rec = &rdev->tc_rec[0]; 406 /* 407 * Upon the receival of DCB Async event: 408 * If roce_dscp or cnp_dscp or both (which user configured using configfs) 409 * is in the list, re-program the value using modify_roce_cc command 410 */ 411 bnxt_re_reconfigure_dscp(rdev); 412 413 cc_param->roce_pri = tc_rec->roce_prio; 414 if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { 415 cc_param->qp1_tos_dscp = cc_param->tos_dscp; 416 rc = bnxt_re_update_qp1_tos_dscp(rdev); 417 if (rc) { 418 dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d", 419 __func__, rc); 420 goto fail; 421 } 422 } 423 424 fail: 425 mutex_unlock(&rdev->cc_lock); 426 exit: 427 kfree(dcb_work); 428 } 429 430 static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev) 431 { 432 struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0}; 433 struct hwrm_func_dbr_pacing_broadcast_event_input req = {0}; 434 struct bnxt_en_dev *en_dev = rdev->en_dev; 435 struct bnxt_fw_msg fw_msg; 436 int rc; 437 438 memset(&fw_msg, 0, sizeof(fw_msg)); 439 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 440 HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1); 441 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 442 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 443 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 444 if (rc) { 445 dev_dbg(rdev_to_dev(rdev), 446 "Failed to send dbr pacing broadcast event rc:%d", rc); 447 return rc; 448 } 449 return 0; 450 } 451 452 static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev) 453 { 454 struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0}; 455 struct hwrm_func_dbr_pacing_nqlist_query_input req = {0}; 456 struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list; 457 struct bnxt_en_dev *en_dev = rdev->en_dev; 458 bool primary_found = false; 459 struct bnxt_fw_msg fw_msg; 460 struct bnxt_qplib_nq *nq; 461 int rc, i, j = 1; 462 u16 *nql_ptr; 463 464 nq = &rdev->nqr.nq[0]; 465 466 memset(&fw_msg, 0, sizeof(fw_msg)); 467 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 468 HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1); 469 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 470 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 471 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 472 if (rc) { 473 dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc); 474 return rc; 475 } 476 nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs); 477 nql_ptr = &resp.nq_ring_id0; 478 /* populate the nq_list of the primary function with list received 479 * from FW. Fill the NQ IDs of secondary functions from index 1 to 480 * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the 481 * nq_id of the primary pf 482 */ 483 for (i = 0; i < nq_list->num_nql_entries; i++) { 484 u16 nq_id = *nql_ptr; 485 486 dev_dbg(rdev_to_dev(rdev), 487 "nq_list->nq_id[%d] = %d\n", i, nq_id); 488 if (nq_id != nq->ring_id) { 489 nq_list->nq_id[j] = nq_id; 490 j++; 491 } else { 492 primary_found = true; 493 nq_list->nq_id[0] = nq->ring_id; 494 } 495 nql_ptr++; 496 } 497 if (primary_found) 498 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); 499 500 return 0; 501 } 502 503 static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) 504 { 505 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 506 u32 read_val, fifo_occup; 507 bool first_read = true; 508 509 /* loop shouldn't run infintely as the occupancy usually goes 510 * below pacing algo threshold as soon as pacing kicks in. 511 */ 512 while (1) { 513 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 514 BNXT_HWRM_BAR_IDX); 515 fifo_occup = pacing_data->fifo_max_depth - 516 ((read_val & pacing_data->fifo_room_mask) >> 517 pacing_data->fifo_room_shift); 518 /* Fifo occupancy cannot be greater the MAX FIFO depth */ 519 if (fifo_occup > pacing_data->fifo_max_depth) 520 break; 521 522 if (first_read) { 523 bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup); 524 first_read = false; 525 } 526 if (fifo_occup < pacing_data->pacing_th) 527 break; 528 } 529 } 530 531 static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) 532 { 533 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 534 535 pacing_data->do_pacing = rdev->dbr_def_do_pacing; 536 pacing_data->pacing_th = rdev->pacing_algo_th; 537 pacing_data->alarm_th = 538 pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); 539 } 540 541 #define CAG_RING_MASK 0x7FF 542 #define CAG_RING_SHIFT 17 543 #define WATERMARK_MASK 0xFFF 544 #define WATERMARK_SHIFT 0 545 546 static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev) 547 { 548 u32 read_val; 549 int j; 550 551 for (j = 0; j < 10; j++) { 552 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 553 BNXT_HWRM_BAR_IDX); 554 dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n", 555 read_val); 556 if (!read_val) 557 return true; 558 } 559 return false; 560 } 561 562 int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle) 563 { 564 u32 cag_ring_water_mark = 0, read_val; 565 u32 throttle_val; 566 567 /* Convert throttle percentage to value */ 568 throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100; 569 570 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 571 cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT | 572 (throttle_val & WATERMARK_MASK); 573 writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 574 BNXT_HWRM_BAR_IDX, cag_ring_water_mark); 575 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 576 BNXT_HWRM_BAR_IDX); 577 dev_dbg(rdev_to_dev(rdev), 578 "%s: dbr_throttling_reg_off read_val = 0x%x\n", 579 __func__, read_val); 580 if (read_val != cag_ring_water_mark) { 581 dev_dbg(rdev_to_dev(rdev), 582 "nq_id = %d write_val=0x%x read_val=0x%x\n", 583 nq_id, cag_ring_water_mark, read_val); 584 return 1; 585 } 586 } 587 writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, BNXT_HWRM_BAR_IDX, 1); 588 return 0; 589 } 590 591 static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev) 592 { 593 struct bnxt_dbq_nq_list *nq_list; 594 struct bnxt_qplib_nq *nq; 595 int i; 596 597 nq_list = &rdev->nq_list; 598 /* Run a loop for other Active functions if this is primary function */ 599 if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { 600 dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n", 601 __func__, nq_list->num_nql_entries); 602 nq = &rdev->nqr.nq[0]; 603 for (i = nq_list->num_nql_entries - 1; i > 0; i--) { 604 u16 nq_id = nq_list->nq_id[i]; 605 if (nq) 606 dev_dbg(rdev_to_dev(rdev), 607 "%s: nq_id = %d cur_fn_ring_id = %d\n", 608 __func__, nq_id, nq->ring_id); 609 if (bnxt_re_set_dbq_throttling_reg 610 (rdev, nq_id, 0)) 611 break; 612 bnxt_re_check_if_dbq_intr_triggered(rdev); 613 } 614 } 615 } 616 617 static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev) 618 { 619 struct bnxt_qplib_nq *nq; 620 int rc = 0; 621 622 nq = &rdev->nqr.nq[0]; 623 624 /* Query the NQ list*/ 625 rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev); 626 if (rc) { 627 dev_err(rdev_to_dev(rdev), 628 "Failed to Query NQ list rc= %d", rc); 629 return; 630 } 631 /*Configure GRC access for Throttling and aeq_arm register */ 632 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 633 BNXT_HWRM_BAR_IDX, 634 rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); 635 636 rdev->dbr_throttling_reg_off = 637 (rdev->chip_ctx->dbr_throttling_reg & 638 BNXT_GRC_OFFSET_MASK) + 0x8000; 639 rdev->dbr_aeq_arm_reg_off = 640 (rdev->chip_ctx->dbr_aeq_arm_reg & 641 BNXT_GRC_OFFSET_MASK) + 0x8000; 642 643 bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); 644 } 645 646 static void bnxt_re_dbq_wq_task(struct work_struct *work) 647 { 648 struct bnxt_re_dbq_work *dbq_work = 649 container_of(work, struct bnxt_re_dbq_work, work); 650 struct bnxt_re_dev *rdev; 651 652 rdev = dbq_work->rdev; 653 654 if (!rdev) 655 goto exit; 656 switch (dbq_work->event) { 657 case BNXT_RE_DBQ_EVENT_SCHED: 658 dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n", 659 __func__); 660 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) 661 bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); 662 else 663 bnxt_re_pacing_alert(rdev); 664 break; 665 case BNXT_RE_DBR_PACING_EVENT: 666 dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n", 667 __func__); 668 if (_is_chip_p7(rdev->chip_ctx)) 669 bnxt_re_pacing_alert(rdev); 670 else if (!rdev->chip_ctx->modes.dbr_pacing_v0) 671 bnxt_re_hwrm_dbr_pacing_qcfg(rdev); 672 break; 673 case BNXT_RE_DBR_NQ_PACING_NOTIFICATION: 674 bnxt_re_handle_dbr_nq_pacing_notification(rdev); 675 /* Issue a broadcast event to notify other functions 676 * that primary changed 677 */ 678 bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); 679 break; 680 } 681 exit: 682 kfree(dbq_work); 683 } 684 685 static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl) 686 { 687 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 688 struct bnxt_re_dcb_work *dcb_work; 689 struct bnxt_re_dbq_work *dbq_work; 690 struct bnxt_re_dev *rdev; 691 u16 event_id; 692 u32 data1; 693 u32 data2 = 0; 694 695 if (!cmpl) { 696 pr_err("Async event, bad completion\n"); 697 return; 698 } 699 700 if (!en_info || !en_info->en_dev) { 701 pr_err("Async event, bad en_info or en_dev\n"); 702 return; 703 } 704 rdev = en_info->rdev; 705 706 event_id = le16_to_cpu(cmpl->event_id); 707 data1 = le32_to_cpu(cmpl->event_data1); 708 data2 = le32_to_cpu(cmpl->event_data2); 709 710 if (!rdev || !rdev_to_dev(rdev)) { 711 dev_dbg(NULL, "Async event, bad rdev or netdev\n"); 712 return; 713 } 714 715 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) || 716 !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { 717 dev_dbg(NULL, "Async event, device already detached\n"); 718 return; 719 } 720 if (data2 >= 0) 721 dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d", 722 event_id, data1, data2); 723 724 switch (event_id) { 725 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: 726 /* Not handling the event in older FWs */ 727 if (!is_qport_service_type_supported(rdev)) 728 break; 729 if (!rdev->dcb_wq) 730 break; 731 dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); 732 if (!dcb_work) 733 break; 734 735 dcb_work->rdev = rdev; 736 memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); 737 INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); 738 queue_work(rdev->dcb_wq, &dcb_work->work); 739 break; 740 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: 741 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 742 /* Set rcfw flag to control commands send to Bono */ 743 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 744 /* Set bnxt_re flag to control commands send via L2 driver */ 745 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 746 wake_up_all(&rdev->rcfw.cmdq.waitq); 747 } 748 break; 749 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: 750 if (!rdev->dbr_pacing) 751 break; 752 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); 753 if (!dbq_work) 754 goto unlock; 755 dbq_work->rdev = rdev; 756 dbq_work->event = BNXT_RE_DBR_PACING_EVENT; 757 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); 758 queue_work(rdev->dbq_wq, &dbq_work->work); 759 rdev->dbr_sw_stats->dbq_int_recv++; 760 break; 761 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: 762 if (!rdev->dbr_pacing) 763 break; 764 765 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); 766 if (!dbq_work) 767 goto unlock; 768 dbq_work->rdev = rdev; 769 dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION; 770 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); 771 queue_work(rdev->dbq_wq, &dbq_work->work); 772 break; 773 774 default: 775 break; 776 } 777 unlock: 778 return; 779 } 780 781 static void bnxt_re_db_fifo_check(struct work_struct *work) 782 { 783 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 784 dbq_fifo_check_work); 785 struct bnxt_qplib_db_pacing_data *pacing_data; 786 u32 pacing_save; 787 788 if (!mutex_trylock(&rdev->dbq_lock)) 789 return; 790 pacing_data = rdev->qplib_res.pacing_data; 791 pacing_save = rdev->do_pacing_save; 792 __wait_for_fifo_occupancy_below_th(rdev); 793 cancel_delayed_work_sync(&rdev->dbq_pacing_work); 794 if (rdev->dbr_recovery_on) 795 goto recovery_on; 796 if (pacing_save > rdev->dbr_def_do_pacing) { 797 /* Double the do_pacing value during the congestion */ 798 pacing_save = pacing_save << 1; 799 } else { 800 /* 801 * when a new congestion is detected increase the do_pacing 802 * by 8 times. And also increase the pacing_th by 4 times. The 803 * reason to increase pacing_th is to give more space for the 804 * queue to oscillate down without getting empty, but also more 805 * room for the queue to increase without causing another alarm. 806 */ 807 pacing_save = pacing_save << 3; 808 pacing_data->pacing_th = rdev->pacing_algo_th * 4; 809 } 810 811 if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) 812 pacing_save = BNXT_RE_MAX_DBR_DO_PACING; 813 814 pacing_data->do_pacing = pacing_save; 815 rdev->do_pacing_save = pacing_data->do_pacing; 816 pacing_data->alarm_th = 817 pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); 818 recovery_on: 819 schedule_delayed_work(&rdev->dbq_pacing_work, 820 msecs_to_jiffies(rdev->dbq_pacing_time)); 821 rdev->dbr_sw_stats->dbq_pacing_alerts++; 822 mutex_unlock(&rdev->dbq_lock); 823 } 824 825 static void bnxt_re_pacing_timer_exp(struct work_struct *work) 826 { 827 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 828 dbq_pacing_work.work); 829 struct bnxt_qplib_db_pacing_data *pacing_data; 830 u32 read_val, fifo_occup; 831 struct bnxt_qplib_nq *nq; 832 833 if (!mutex_trylock(&rdev->dbq_lock)) 834 return; 835 836 pacing_data = rdev->qplib_res.pacing_data; 837 read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 838 BNXT_HWRM_BAR_IDX); 839 fifo_occup = pacing_data->fifo_max_depth - 840 ((read_val & pacing_data->fifo_room_mask) >> 841 pacing_data->fifo_room_shift); 842 843 if (fifo_occup > pacing_data->pacing_th) 844 goto restart_timer; 845 846 /* 847 * Instead of immediately going back to the default do_pacing 848 * reduce it by 1/8 times and restart the timer. 849 */ 850 pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); 851 pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing); 852 /* 853 * If the fifo_occup is less than the interrupt enable threshold 854 * enable the interrupt on the primary PF. 855 */ 856 if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) { 857 if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { 858 if (!rdev->chip_ctx->modes.dbr_pacing_v0) { 859 nq = &rdev->nqr.nq[0]; 860 bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, 861 rdev->dbq_watermark); 862 rdev->dbr_sw_stats->dbq_int_en++; 863 rdev->dbq_int_disable = false; 864 } 865 } 866 } 867 if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) { 868 bnxt_re_set_default_pacing_data(rdev); 869 rdev->dbr_sw_stats->dbq_pacing_complete++; 870 goto dbq_unlock; 871 } 872 restart_timer: 873 schedule_delayed_work(&rdev->dbq_pacing_work, 874 msecs_to_jiffies(rdev->dbq_pacing_time)); 875 bnxt_re_update_do_pacing_slabs(rdev); 876 rdev->dbr_sw_stats->dbq_pacing_resched++; 877 dbq_unlock: 878 rdev->do_pacing_save = pacing_data->do_pacing; 879 mutex_unlock(&rdev->dbq_lock); 880 } 881 882 void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) 883 { 884 struct bnxt_qplib_db_pacing_data *pacing_data; 885 886 if (!rdev->dbr_pacing) 887 return; 888 mutex_lock(&rdev->dbq_lock); 889 pacing_data = rdev->qplib_res.pacing_data; 890 891 /* 892 * Increase the alarm_th to max so that other user lib instances do not 893 * keep alerting the driver. 894 */ 895 pacing_data->alarm_th = pacing_data->fifo_max_depth; 896 pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; 897 cancel_work_sync(&rdev->dbq_fifo_check_work); 898 schedule_work(&rdev->dbq_fifo_check_work); 899 mutex_unlock(&rdev->dbq_lock); 900 } 901 902 void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res) 903 { 904 struct bnxt_re_dbq_work *dbq_work; 905 struct bnxt_re_dev *rdev; 906 907 rdev = container_of(res, struct bnxt_re_dev, qplib_res); 908 909 atomic_set(&rdev->dbq_intr_running, 1); 910 911 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 912 goto exit; 913 /* Run the loop to send dbq event to other functions 914 * for newer FW 915 */ 916 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && 917 !rdev->chip_ctx->modes.dbr_pacing_v0) 918 bnxt_re_set_dbq_throttling_for_non_primary(rdev); 919 920 dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); 921 if (!dbq_work) 922 goto exit; 923 dbq_work->rdev = rdev; 924 dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED; 925 INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); 926 queue_work(rdev->dbq_wq, &dbq_work->work); 927 rdev->dbr_sw_stats->dbq_int_recv++; 928 rdev->dbq_int_disable = true; 929 exit: 930 atomic_set(&rdev->dbq_intr_running, 0); 931 } 932 933 static void bnxt_re_free_msix(struct bnxt_re_dev *rdev) 934 { 935 struct bnxt_en_dev *en_dev = rdev->en_dev; 936 int rc; 937 938 rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); 939 if (rc) 940 dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x", 941 rdev->netdev, rc); 942 } 943 944 static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) 945 { 946 struct bnxt_en_dev *en_dev = rdev->en_dev; 947 int rc = 0, num_msix_want, num_msix_got; 948 struct bnxt_msix_entry *entry; 949 950 /* 951 * Request MSIx based on the function type. This is 952 * a temporory solution to enable max VFs when NPAR is 953 * enabled. 954 * TODO - change the scheme with an adapter specific check 955 * as the latest adapters can support more NQs. For now 956 * this change satisfy all adapter versions. 957 */ 958 959 if (rdev->is_virtfn) 960 num_msix_want = BNXT_RE_MAX_MSIX_VF; 961 else if (BNXT_EN_NPAR(en_dev)) 962 num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF; 963 else if (_is_chip_gen_p5_p7(rdev->chip_ctx)) 964 num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF; 965 else 966 num_msix_want = BNXT_RE_MAX_MSIX_PF; 967 968 /* 969 * Since MSIX vectors are used for both NQs and CREQ, we should try to 970 * allocate num_online_cpus + 1 by taking into account the CREQ. This 971 * leaves the number of MSIX vectors for NQs match the number of CPUs 972 * and allows the system to be fully utilized 973 */ 974 num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1); 975 num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX); 976 num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX); 977 978 entry = rdev->nqr.msix_entries; 979 980 num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, 981 entry, num_msix_want); 982 if (num_msix_got < BNXT_RE_MIN_MSIX) { 983 rc = -EINVAL; 984 goto done; 985 } 986 if (num_msix_got != num_msix_want) 987 dev_warn(rdev_to_dev(rdev), 988 "bnxt_request_msix: wanted %d vectors, got %d\n", 989 num_msix_want, num_msix_got); 990 991 rdev->nqr.num_msix = num_msix_got; 992 return 0; 993 done: 994 if (num_msix_got) 995 bnxt_re_free_msix(rdev); 996 return rc; 997 } 998 999 static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev, 1000 struct bnxt_re_en_dev_info *en_info) 1001 { 1002 u64 timeout = 0; 1003 u32 cur_prod = 0, cur_cons = 0; 1004 int retry = 0, rc = 0, ret = 0; 1005 1006 cur_prod = rdev->rcfw.cmdq.hwq.prod; 1007 cur_cons = rdev->rcfw.cmdq.hwq.cons; 1008 timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS); 1009 retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY; 1010 /* During module exit, increase timeout ten-fold to 100 mins to wait 1011 * as long as possible for ib_unregister() to complete 1012 */ 1013 if (rdev->mod_exit) 1014 retry *= 10; 1015 do { 1016 /* 1017 * Since the caller of this function invokes with bnxt_re_mutex held, 1018 * release it to avoid holding a lock while in wait / sleep mode. 1019 */ 1020 mutex_unlock(&bnxt_re_mutex); 1021 rc = wait_event_timeout(en_info->waitq, 1022 en_info->ib_uninit_done, 1023 timeout); 1024 mutex_lock(&bnxt_re_mutex); 1025 1026 if (!bnxt_re_is_rdev_valid(rdev)) 1027 break; 1028 1029 if (rc) 1030 break; 1031 1032 if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) { 1033 /* No need to check for cmdq stall during module exit, 1034 * wait for ib unregister to complete 1035 */ 1036 if (!rdev->mod_exit) 1037 ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons); 1038 if (ret || en_info->ib_uninit_done) 1039 break; 1040 } 1041 } while (retry--); 1042 1043 return rc; 1044 } 1045 1046 static int bnxt_re_handle_start(struct auxiliary_device *adev) 1047 { 1048 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); 1049 struct bnxt_re_dev *rdev = NULL; 1050 struct ifnet *real_dev; 1051 struct bnxt_en_dev *en_dev; 1052 struct ifnet *netdev; 1053 int rc = 0; 1054 1055 if (!en_info || !en_info->en_dev) { 1056 pr_err("Start, bad en_info or en_dev\n"); 1057 return -EINVAL; 1058 } 1059 netdev = en_info->en_dev->net; 1060 if (en_info->rdev) { 1061 dev_info(rdev_to_dev(en_info->rdev), 1062 "%s: Device is already added adev %p rdev: %p\n", 1063 __func__, adev, en_info->rdev); 1064 return 0; 1065 } 1066 1067 en_dev = en_info->en_dev; 1068 real_dev = rdma_vlan_dev_real_dev(netdev); 1069 if (!real_dev) 1070 real_dev = netdev; 1071 rc = bnxt_re_add_device(&rdev, real_dev, 1072 en_info->gsi_mode, 1073 BNXT_RE_POST_RECOVERY_INIT, 1074 en_info->num_msix_requested, adev); 1075 if (rc) { 1076 /* Add device failed. Unregister the device. 1077 * This has to be done explicitly as 1078 * bnxt_re_stop would not have unregistered 1079 */ 1080 rtnl_lock(); 1081 en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); 1082 rtnl_unlock(); 1083 mutex_lock(&bnxt_re_dev_lock); 1084 gadd_dev_inprogress--; 1085 mutex_unlock(&bnxt_re_dev_lock); 1086 return rc; 1087 } 1088 rdev->adev = adev; 1089 rtnl_lock(); 1090 bnxt_re_get_link_speed(rdev); 1091 rtnl_unlock(); 1092 rc = bnxt_re_ib_init(rdev); 1093 if (rc) { 1094 dev_err(rdev_to_dev(rdev), "Failed ib_init\n"); 1095 return rc; 1096 } 1097 bnxt_re_ib_init_2(rdev); 1098 1099 return rc; 1100 } 1101 1102 static void bnxt_re_stop(void *handle) 1103 { 1104 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1105 struct ifnet *netdev; 1106 struct bnxt_re_dev *rdev; 1107 struct bnxt_en_dev *en_dev; 1108 int rc = 0; 1109 1110 rtnl_unlock(); 1111 mutex_lock(&bnxt_re_mutex); 1112 if (!en_info || !en_info->en_dev) { 1113 pr_err("Stop, bad en_info or en_dev\n"); 1114 goto exit; 1115 } 1116 netdev = en_info->en_dev->net; 1117 rdev = en_info->rdev; 1118 if (!rdev) 1119 goto exit; 1120 1121 if (!bnxt_re_is_rdev_valid(rdev)) 1122 goto exit; 1123 1124 /* 1125 * Check if fw has undergone reset or is in a fatal condition. 1126 * If so, set flags so that no further commands are sent down to FW 1127 */ 1128 en_dev = rdev->en_dev; 1129 if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND || 1130 en_dev->en_state & BNXT_STATE_FW_RESET_DET) { 1131 /* Set rcfw flag to control commands send to Bono */ 1132 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 1133 /* Set bnxt_re flag to control commands send via L2 driver */ 1134 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 1135 wake_up_all(&rdev->rcfw.cmdq.waitq); 1136 } 1137 1138 if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags)) 1139 goto exit; 1140 set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags); 1141 1142 en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode; 1143 en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode; 1144 en_info->num_msix_requested = rdev->num_msix_requested; 1145 en_info->ib_uninit_done = false; 1146 1147 if (rdev->dbr_pacing) 1148 bnxt_re_set_pacing_dev_state(rdev); 1149 1150 dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop." 1151 "Attempting to stop and Dispatching event " 1152 "to inform the stack\n", __func__); 1153 init_waitqueue_head(&en_info->waitq); 1154 /* Schedule a work item to handle IB UNINIT for recovery */ 1155 bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, 1156 NULL, netdev, rdev->adev); 1157 rc = __wait_for_ib_unregister(rdev, en_info); 1158 if (!bnxt_re_is_rdev_valid(rdev)) 1159 goto exit; 1160 if (!rc) { 1161 dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n", 1162 __func__); 1163 bnxt_re_detach_err_device(rdev); 1164 goto exit; 1165 } 1166 bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev); 1167 exit: 1168 mutex_unlock(&bnxt_re_mutex); 1169 /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */ 1170 rtnl_lock(); 1171 1172 return; 1173 } 1174 1175 static void bnxt_re_start(void *handle) 1176 { 1177 rtnl_unlock(); 1178 mutex_lock(&bnxt_re_mutex); 1179 if (bnxt_re_handle_start((struct auxiliary_device *)handle)) 1180 pr_err("Failed to start RoCE device"); 1181 mutex_unlock(&bnxt_re_mutex); 1182 /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */ 1183 rtnl_lock(); 1184 return; 1185 } 1186 1187 static void bnxt_re_shutdown(void *p) 1188 { 1189 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p); 1190 struct bnxt_re_dev *rdev; 1191 1192 if (!en_info) { 1193 pr_err("Shutdown, bad en_info\n"); 1194 return; 1195 } 1196 rtnl_unlock(); 1197 mutex_lock(&bnxt_re_mutex); 1198 rdev = en_info->rdev; 1199 if (!rdev || !bnxt_re_is_rdev_valid(rdev)) 1200 goto exit; 1201 1202 /* rtnl_lock held by L2 before coming here */ 1203 bnxt_re_stopqps_and_ib_uninit(rdev); 1204 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev); 1205 exit: 1206 mutex_unlock(&bnxt_re_mutex); 1207 rtnl_lock(); 1208 return; 1209 } 1210 1211 static void bnxt_re_stop_irq(void *handle) 1212 { 1213 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1214 struct bnxt_qplib_rcfw *rcfw = NULL; 1215 struct bnxt_re_dev *rdev; 1216 struct bnxt_qplib_nq *nq; 1217 int indx; 1218 1219 if (!en_info) { 1220 pr_err("Stop irq, bad en_info\n"); 1221 return; 1222 } 1223 rdev = en_info->rdev; 1224 1225 if (!rdev) 1226 return; 1227 1228 rcfw = &rdev->rcfw; 1229 for (indx = 0; indx < rdev->nqr.max_init; indx++) { 1230 nq = &rdev->nqr.nq[indx]; 1231 mutex_lock(&nq->lock); 1232 bnxt_qplib_nq_stop_irq(nq, false); 1233 mutex_unlock(&nq->lock); 1234 } 1235 1236 if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) 1237 bnxt_qplib_rcfw_stop_irq(rcfw, false); 1238 } 1239 1240 static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) 1241 { 1242 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1243 struct bnxt_msix_entry *msix_ent = NULL; 1244 struct bnxt_qplib_rcfw *rcfw = NULL; 1245 struct bnxt_re_dev *rdev; 1246 struct bnxt_qplib_nq *nq; 1247 int indx, rc, vec; 1248 1249 if (!en_info) { 1250 pr_err("Start irq, bad en_info\n"); 1251 return; 1252 } 1253 rdev = en_info->rdev; 1254 if (!rdev) 1255 return; 1256 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1257 return; 1258 msix_ent = rdev->nqr.msix_entries; 1259 rcfw = &rdev->rcfw; 1260 1261 if (!ent) { 1262 /* Not setting the f/w timeout bit in rcfw. 1263 * During the driver unload the first command 1264 * to f/w will timeout and that will set the 1265 * timeout bit. 1266 */ 1267 dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); 1268 return; 1269 } 1270 1271 /* Vectors may change after restart, so update with new vectors 1272 * in device structure. 1273 */ 1274 for (indx = 0; indx < rdev->nqr.num_msix; indx++) 1275 rdev->nqr.msix_entries[indx].vector = ent[indx].vector; 1276 1277 if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) { 1278 rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, 1279 false); 1280 if (rc) { 1281 dev_warn(rdev_to_dev(rdev), 1282 "Failed to reinit CREQ\n"); 1283 return; 1284 } 1285 } 1286 for (indx = 0 ; indx < rdev->nqr.max_init; indx++) { 1287 nq = &rdev->nqr.nq[indx]; 1288 vec = indx + 1; 1289 rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector, 1290 false); 1291 if (rc) { 1292 dev_warn(rdev_to_dev(rdev), 1293 "Failed to reinit NQ index %d\n", indx); 1294 return; 1295 } 1296 } 1297 } 1298 1299 /* 1300 * bnxt_re_create_snapdump_logs - Collect required debug data for snapdump. 1301 * @rdev - rdma device instance 1302 * 1303 * This function will use bnxt_ulp_log_live API and dump all 1304 * the required information for debugging. 1305 * 1306 * Returns: Nothing 1307 */ 1308 static void 1309 bnxt_re_create_snapdump_logs(struct bnxt_re_dev *rdev) 1310 { 1311 struct bnxt_re_ext_rstat *ext_s; 1312 1313 ext_s = &rdev->stats.dstat.ext_rstat[0]; 1314 1315 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1316 "tx_atomic_req: %llu", ext_s->tx.atomic_req); 1317 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1318 "rx_atomic_req: %llu", ext_s->rx.atomic_req); 1319 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1320 "tx_read_req: %llu\n", ext_s->tx.read_req); 1321 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1322 "tx_read_resp: %llu\n", ext_s->tx.read_resp); 1323 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1324 "rx_read_req: %llu\n", ext_s->rx.read_req); 1325 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1326 "rx_read_resp: %llu\n", ext_s->rx.read_resp); 1327 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1328 "tx_write_req: %llu\n", ext_s->tx.write_req); 1329 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1330 "rx_write_req: %llu\n", ext_s->rx.write_req); 1331 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1332 "tx_send_req: %llu\n", ext_s->tx.send_req); 1333 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1334 "rx_send_req: %llu\n", ext_s->rx.send_req); 1335 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1336 "rx_good_pkts: %llu\n", ext_s->grx.rx_pkts); 1337 bnxt_ulp_log_live(rdev->en_dev, BNXT_LOGGER_ROCE, 1338 "rx_good_bytes: %llu\n", ext_s->grx.rx_bytes); 1339 } 1340 1341 /* 1342 * bnxt_re_ulp_log_live - Callback from L2 driver to collect snapdump 1343 * @handle - en_dev information. L2 and RoCE device information 1344 * 1345 * This function is callback to support L2 and RoCE common API. 1346 * bnxt_ulp_ops.ulp_log_live(). 1347 * 1348 * Returns: Nothing 1349 */ 1350 static void 1351 bnxt_re_ulp_log_live(void *handle) 1352 { 1353 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); 1354 1355 if (!en_info || !en_info->rdev) 1356 return; 1357 1358 bnxt_re_create_snapdump_logs(en_info->rdev); 1359 } 1360 1361 /* 1362 * Except for ulp_async_notifier, the remaining ulp_ops 1363 * below are called with rtnl_lock held 1364 */ 1365 static struct bnxt_ulp_ops bnxt_re_ulp_ops = { 1366 .ulp_async_notifier = bnxt_re_async_notifier, 1367 .ulp_stop = bnxt_re_stop, 1368 .ulp_start = bnxt_re_start, 1369 .ulp_shutdown = bnxt_re_shutdown, 1370 .ulp_irq_stop = bnxt_re_stop_irq, 1371 .ulp_irq_restart = bnxt_re_start_irq, 1372 .ulp_log_live = bnxt_re_ulp_log_live, 1373 }; 1374 1375 static inline const char *bnxt_re_netevent(unsigned long event) 1376 { 1377 BNXT_RE_NETDEV_EVENT(event, NETDEV_UP); 1378 BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN); 1379 BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE); 1380 BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER); 1381 BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER); 1382 BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR); 1383 return "Unknown"; 1384 } 1385 1386 /* RoCE -> Net driver */ 1387 1388 /* Driver registration routines used to let the networking driver (bnxt_en) 1389 * to know that the RoCE driver is now installed */ 1390 static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) 1391 { 1392 struct bnxt_en_dev *en_dev = rdev->en_dev; 1393 int rc; 1394 1395 rtnl_lock(); 1396 rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, 1397 BNXT_ROCE_ULP); 1398 rtnl_unlock(); 1399 if (rc) 1400 dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x", 1401 rdev->en_dev->net, rc); 1402 1403 clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 1404 } 1405 1406 static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) 1407 { 1408 struct bnxt_en_dev *en_dev = rdev->en_dev; 1409 int rc = 0; 1410 1411 rtnl_lock(); 1412 rc = en_dev->en_ops->bnxt_register_device(en_dev, 1413 BNXT_ROCE_ULP, 1414 &bnxt_re_ulp_ops, 1415 rdev->adev); 1416 rtnl_unlock(); 1417 if (rc) { 1418 dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x", 1419 rdev->netdev, rc); 1420 return rc; 1421 } 1422 1423 return rc; 1424 } 1425 1426 static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) 1427 { 1428 struct bnxt_qplib_chip_ctx *cctx; 1429 struct bnxt_en_dev *en_dev; 1430 struct bnxt_qplib_res *res; 1431 u32 l2db_len = 0; 1432 u32 offset = 0; 1433 u32 barlen; 1434 int rc; 1435 1436 res = &rdev->qplib_res; 1437 en_dev = rdev->en_dev; 1438 cctx = rdev->chip_ctx; 1439 1440 /* Issue qcfg */ 1441 rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset); 1442 if (rc) 1443 dev_info(rdev_to_dev(rdev), 1444 "Couldn't get DB bar size, Low latency framework is disabled\n"); 1445 /* set register offsets for both UC and WC */ 1446 if (_is_chip_p7(cctx)) { 1447 res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset; 1448 res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; 1449 } else { 1450 res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : 1451 BNXT_QPLIB_DBR_PF_DB_OFFSET; 1452 res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; 1453 } 1454 1455 /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size 1456 * is equal to the DB-Bar actual size. This indicates that L2 1457 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping 1458 * in such cases and DB-push will be disabled. 1459 */ 1460 barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION); 1461 if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) { 1462 res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; 1463 dev_info(rdev_to_dev(rdev), 1464 "Low latency framework is enabled\n"); 1465 } 1466 1467 return; 1468 } 1469 1470 static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev) 1471 { 1472 struct bnxt_qplib_chip_ctx *cctx; 1473 struct bnxt_en_dev *en_dev; 1474 1475 en_dev = rdev->en_dev; 1476 cctx = rdev->chip_ctx; 1477 cctx->modes.wqe_mode = _is_chip_p7(rdev->chip_ctx) ? 1478 BNXT_QPLIB_WQE_MODE_VARIABLE : BNXT_QPLIB_WQE_MODE_STATIC; 1479 cctx->modes.te_bypass = false; 1480 if (bnxt_re_hwrm_qcaps(rdev)) 1481 dev_err(rdev_to_dev(rdev), 1482 "Failed to query hwrm qcaps\n"); 1483 /* 1484 * TODO: Need a better mechanism for spreading of the 1485 * 512 extended PPP pages in the presence of VF and 1486 * NPAR, until then not enabling push 1487 */ 1488 if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) { 1489 if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev)) 1490 cctx->modes.db_push = false; 1491 } 1492 1493 rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP; 1494 dev_dbg(rdev_to_dev(rdev), 1495 "RoCE is supported on the device - caps:0x%x", 1496 rdev->roce_mode); 1497 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 1498 rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP; 1499 cctx->hw_stats_size = en_dev->hw_ring_stats_size; 1500 } 1501 1502 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) 1503 { 1504 struct bnxt_qplib_chip_ctx *chip_ctx; 1505 struct bnxt_qplib_res *res; 1506 1507 if (!rdev->chip_ctx) 1508 return; 1509 1510 res = &rdev->qplib_res; 1511 bnxt_qplib_unmap_db_bar(res); 1512 1513 kfree(res->hctx); 1514 res->rcfw = NULL; 1515 kfree(rdev->dev_attr); 1516 rdev->dev_attr = NULL; 1517 1518 chip_ctx = rdev->chip_ctx; 1519 rdev->chip_ctx = NULL; 1520 res->cctx = NULL; 1521 res->hctx = NULL; 1522 res->pdev = NULL; 1523 res->netdev = NULL; 1524 kfree(chip_ctx); 1525 } 1526 1527 static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev) 1528 { 1529 struct bnxt_qplib_chip_ctx *chip_ctx; 1530 struct bnxt_en_dev *en_dev; 1531 int rc; 1532 1533 en_dev = rdev->en_dev; 1534 /* Supply pci device to qplib */ 1535 rdev->qplib_res.pdev = en_dev->pdev; 1536 rdev->qplib_res.netdev = rdev->netdev; 1537 rdev->qplib_res.en_dev = en_dev; 1538 1539 chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); 1540 if (!chip_ctx) 1541 return -ENOMEM; 1542 rdev->chip_ctx = chip_ctx; 1543 rdev->qplib_res.cctx = chip_ctx; 1544 rc = bnxt_re_query_hwrm_intf_version(rdev); 1545 if (rc) 1546 goto fail; 1547 rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); 1548 if (!rdev->dev_attr) { 1549 rc = -ENOMEM; 1550 goto fail; 1551 } 1552 rdev->qplib_res.dattr = rdev->dev_attr; 1553 rdev->qplib_res.rcfw = &rdev->rcfw; 1554 rdev->qplib_res.is_vf = rdev->is_virtfn; 1555 1556 rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx), 1557 GFP_KERNEL); 1558 if (!rdev->qplib_res.hctx) { 1559 rc = -ENOMEM; 1560 goto fail; 1561 } 1562 bnxt_re_set_drv_mode(rdev); 1563 1564 bnxt_re_set_db_offset(rdev); 1565 rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); 1566 if (rc) 1567 goto fail; 1568 1569 rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev); 1570 if (rc) 1571 dev_dbg(rdev_to_dev(rdev), 1572 "platform doesn't support global atomics"); 1573 1574 return 0; 1575 fail: 1576 kfree(rdev->chip_ctx); 1577 rdev->chip_ctx = NULL; 1578 1579 kfree(rdev->dev_attr); 1580 rdev->dev_attr = NULL; 1581 1582 kfree(rdev->qplib_res.hctx); 1583 rdev->qplib_res.hctx = NULL; 1584 return rc; 1585 } 1586 1587 static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) { 1588 return _is_chip_gen_p5_p7(rdev->chip_ctx) ? 1589 HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ : 1590 HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL; 1591 } 1592 1593 static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) 1594 { 1595 int rc = -EINVAL; 1596 struct hwrm_ring_free_input req = {0}; 1597 struct hwrm_ring_free_output resp; 1598 struct bnxt_en_dev *en_dev = rdev->en_dev; 1599 struct bnxt_fw_msg fw_msg; 1600 1601 if (!en_dev) 1602 return rc; 1603 1604 /* To avoid unnecessary error messages during recovery. 1605 * HW is anyway in error state. So dont send down the command */ 1606 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1607 return 0; 1608 1609 /* allocation had failed, no need to issue hwrm */ 1610 if (fw_ring_id == 0xffff) 1611 return 0; 1612 1613 memset(&fw_msg, 0, sizeof(fw_msg)); 1614 1615 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); 1616 req.ring_type = bnxt_re_get_rtype(rdev); 1617 req.ring_id = cpu_to_le16(fw_ring_id); 1618 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1619 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1620 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1621 if (rc) { 1622 dev_err(rdev_to_dev(rdev), 1623 "Failed to free HW ring with rc = 0x%x", rc); 1624 return rc; 1625 } 1626 dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n", 1627 fw_ring_id); 1628 1629 return rc; 1630 } 1631 1632 static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, 1633 struct bnxt_re_ring_attr *ring_attr, 1634 u16 *fw_ring_id) 1635 { 1636 int rc = -EINVAL; 1637 struct hwrm_ring_alloc_input req = {0}; 1638 struct hwrm_ring_alloc_output resp; 1639 struct bnxt_en_dev *en_dev = rdev->en_dev; 1640 struct bnxt_fw_msg fw_msg; 1641 1642 if (!en_dev) 1643 return rc; 1644 1645 memset(&fw_msg, 0, sizeof(fw_msg)); 1646 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); 1647 req.flags = cpu_to_le16(ring_attr->flags); 1648 req.enables = 0; 1649 req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); 1650 if (ring_attr->pages > 1) { 1651 /* Page size is in log2 units */ 1652 req.page_size = BNXT_PAGE_SHIFT; 1653 req.page_tbl_depth = 1; 1654 } else { 1655 req.page_size = 4; 1656 req.page_tbl_depth = 0; 1657 } 1658 1659 req.fbo = 0; 1660 /* Association of ring index with doorbell index and MSIX number */ 1661 req.logical_id = cpu_to_le16(ring_attr->lrid); 1662 req.length = cpu_to_le32(ring_attr->depth + 1); 1663 req.ring_type = ring_attr->type; 1664 req.int_mode = ring_attr->mode; 1665 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1666 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1667 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1668 if (rc) { 1669 dev_err(rdev_to_dev(rdev), 1670 "Failed to allocate HW ring with rc = 0x%x", rc); 1671 return rc; 1672 } 1673 *fw_ring_id = le16_to_cpu(resp.ring_id); 1674 dev_dbg(rdev_to_dev(rdev), 1675 "HW ring allocated with id = 0x%x at slot 0x%x", 1676 resp.ring_id, ring_attr->lrid); 1677 1678 return rc; 1679 } 1680 1681 static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, 1682 u32 fw_stats_ctx_id, u16 tid) 1683 { 1684 struct bnxt_en_dev *en_dev = rdev->en_dev; 1685 struct hwrm_stat_ctx_free_input req = {0}; 1686 struct hwrm_stat_ctx_free_output resp; 1687 struct bnxt_fw_msg fw_msg; 1688 int rc = -EINVAL; 1689 1690 if (!en_dev) 1691 return rc; 1692 1693 /* To avoid unnecessary error messages during recovery. 1694 * HW is anyway in error state. So dont send down the command */ 1695 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1696 return 0; 1697 memset(&fw_msg, 0, sizeof(fw_msg)); 1698 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid); 1699 req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); 1700 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1701 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1702 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1703 if (rc) { 1704 dev_err(rdev_to_dev(rdev), 1705 "Failed to free HW stats ctx with rc = 0x%x", rc); 1706 return rc; 1707 } 1708 dev_dbg(rdev_to_dev(rdev), 1709 "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id); 1710 1711 return rc; 1712 } 1713 1714 static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid) 1715 { 1716 struct hwrm_stat_ctx_alloc_output resp = {}; 1717 struct hwrm_stat_ctx_alloc_input req = {}; 1718 struct bnxt_en_dev *en_dev = rdev->en_dev; 1719 struct bnxt_qplib_stats *stat; 1720 struct bnxt_qplib_ctx *hctx; 1721 struct bnxt_fw_msg fw_msg; 1722 int rc = 0; 1723 1724 hctx = rdev->qplib_res.hctx; 1725 stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2; 1726 stat->fw_id = INVALID_STATS_CTX_ID; 1727 1728 if (!en_dev) 1729 return -EINVAL; 1730 1731 memset(&fw_msg, 0, sizeof(fw_msg)); 1732 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1733 HWRM_STAT_CTX_ALLOC, -1, tid); 1734 req.update_period_ms = cpu_to_le32(1000); 1735 req.stats_dma_length = rdev->chip_ctx->hw_stats_size; 1736 req.stats_dma_addr = cpu_to_le64(stat->dma_map); 1737 req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE; 1738 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1739 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1740 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1741 if (rc) { 1742 dev_err(rdev_to_dev(rdev), 1743 "Failed to allocate HW stats ctx, rc = 0x%x", rc); 1744 return rc; 1745 } 1746 stat->fw_id = le32_to_cpu(resp.stat_ctx_id); 1747 dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x", 1748 stat->fw_id); 1749 1750 return rc; 1751 } 1752 1753 static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev) 1754 { 1755 const struct bnxt_en_ops *en_ops; 1756 1757 if (rdev->is_virtfn || 1758 test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 1759 return; 1760 1761 memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap)); 1762 en_ops = rdev->en_dev->en_ops; 1763 if (en_ops->bnxt_register_fw_async_events 1764 (rdev->en_dev, BNXT_ROCE_ULP, 1765 (unsigned long *)rdev->event_bitmap, 1766 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) 1767 dev_err(rdev_to_dev(rdev), 1768 "Failed to unregister async event"); 1769 } 1770 1771 static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev) 1772 { 1773 const struct bnxt_en_ops *en_ops; 1774 1775 if (rdev->is_virtfn) 1776 return; 1777 1778 rdev->event_bitmap[0] |= 1779 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) | 1780 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY); 1781 1782 rdev->event_bitmap[2] |= 1783 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64); 1784 rdev->event_bitmap[2] |= 1785 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) | 1786 BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64); 1787 en_ops = rdev->en_dev->en_ops; 1788 if (en_ops->bnxt_register_fw_async_events 1789 (rdev->en_dev, BNXT_ROCE_ULP, 1790 (unsigned long *)rdev->event_bitmap, 1791 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) 1792 dev_err(rdev_to_dev(rdev), 1793 "Failed to reg Async event"); 1794 } 1795 1796 static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) 1797 { 1798 struct bnxt_en_dev *en_dev = rdev->en_dev; 1799 struct hwrm_ver_get_output resp = {0}; 1800 struct hwrm_ver_get_input req = {0}; 1801 struct bnxt_qplib_chip_ctx *cctx; 1802 struct bnxt_fw_msg fw_msg; 1803 int rc = 0; 1804 1805 memset(&fw_msg, 0, sizeof(fw_msg)); 1806 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1807 HWRM_VER_GET, -1, -1); 1808 req.hwrm_intf_maj = HWRM_VERSION_MAJOR; 1809 req.hwrm_intf_min = HWRM_VERSION_MINOR; 1810 req.hwrm_intf_upd = HWRM_VERSION_UPDATE; 1811 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1812 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1813 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1814 if (rc) { 1815 dev_err(rdev_to_dev(rdev), 1816 "Failed to query HW version, rc = 0x%x", rc); 1817 return rc; 1818 } 1819 cctx = rdev->chip_ctx; 1820 cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 | 1821 (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 | 1822 (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 | 1823 le16_to_cpu(resp.hwrm_intf_patch); 1824 1825 cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout); 1826 1827 if (!cctx->hwrm_cmd_max_timeout) 1828 cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT; 1829 1830 cctx->chip_num = le16_to_cpu(resp.chip_num); 1831 cctx->chip_rev = resp.chip_rev; 1832 cctx->chip_metal = resp.chip_metal; 1833 return 0; 1834 } 1835 1836 /* Query device config using common hwrm */ 1837 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, 1838 u32 *offset) 1839 { 1840 struct bnxt_en_dev *en_dev = rdev->en_dev; 1841 struct hwrm_func_qcfg_output resp = {0}; 1842 struct hwrm_func_qcfg_input req = {0}; 1843 struct bnxt_fw_msg fw_msg; 1844 int rc; 1845 1846 memset(&fw_msg, 0, sizeof(fw_msg)); 1847 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1848 HWRM_FUNC_QCFG, -1, -1); 1849 req.fid = cpu_to_le16(0xffff); 1850 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1851 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1852 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1853 if (rc) { 1854 dev_err(rdev_to_dev(rdev), 1855 "Failed to query config, rc = %#x", rc); 1856 return rc; 1857 } 1858 1859 *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024); 1860 *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024); 1861 return 0; 1862 } 1863 1864 /* Query function capabilities using common hwrm */ 1865 int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) 1866 { 1867 struct bnxt_en_dev *en_dev = rdev->en_dev; 1868 struct hwrm_func_qcaps_output resp = {0}; 1869 struct hwrm_func_qcaps_input req = {0}; 1870 struct bnxt_qplib_chip_ctx *cctx; 1871 struct bnxt_fw_msg fw_msg; 1872 u8 push_enable = false; 1873 int rc; 1874 1875 cctx = rdev->chip_ctx; 1876 memset(&fw_msg, 0, sizeof(fw_msg)); 1877 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1878 HWRM_FUNC_QCAPS, -1, -1); 1879 req.fid = cpu_to_le16(0xffff); 1880 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1881 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 1882 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1883 if (rc) { 1884 dev_err(rdev_to_dev(rdev), 1885 "Failed to query capabilities, rc = %#x", rc); 1886 return rc; 1887 } 1888 if (_is_chip_p7(rdev->chip_ctx)) 1889 push_enable = 1890 (resp.flags_ext & 1891 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ? 1892 true : false; 1893 else 1894 push_enable = 1895 (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ? 1896 true : false; 1897 cctx->modes.db_push = push_enable; 1898 1899 cctx->modes.dbr_pacing = 1900 resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ? 1901 true : false; 1902 cctx->modes.dbr_pacing_ext = 1903 resp.flags_ext2 & 1904 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ? 1905 true : false; 1906 cctx->modes.dbr_drop_recov = 1907 (resp.flags_ext2 & 1908 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ? 1909 true : false; 1910 cctx->modes.dbr_pacing_v0 = 1911 (resp.flags_ext2 & 1912 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ? 1913 true : false; 1914 dev_dbg(rdev_to_dev(rdev), 1915 "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n", 1916 __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov); 1917 1918 return 0; 1919 } 1920 1921 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) 1922 { 1923 struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; 1924 struct hwrm_func_dbr_pacing_qcfg_output resp = {0}; 1925 struct hwrm_func_dbr_pacing_qcfg_input req = {0}; 1926 struct bnxt_en_dev *en_dev = rdev->en_dev; 1927 struct bnxt_qplib_chip_ctx *cctx; 1928 struct bnxt_fw_msg fw_msg; 1929 u32 primary_nq_id; 1930 int rc; 1931 1932 cctx = rdev->chip_ctx; 1933 memset(&fw_msg, 0, sizeof(fw_msg)); 1934 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 1935 HWRM_FUNC_DBR_PACING_QCFG, -1, -1); 1936 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 1937 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 1938 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 1939 if (rc) { 1940 dev_dbg(rdev_to_dev(rdev), 1941 "Failed to query dbr pacing config, rc = %#x", rc); 1942 return rc; 1943 } 1944 1945 primary_nq_id = le32_to_cpu(resp.primary_nq_id); 1946 if (primary_nq_id == 0xffffffff && 1947 !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 1948 dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n", 1949 __func__, __LINE__); 1950 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); 1951 } 1952 1953 if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 1954 struct bnxt_qplib_nq *nq; 1955 1956 nq = &rdev->nqr.nq[0]; 1957 /* Reset the primary capability */ 1958 if (nq->ring_id != primary_nq_id) 1959 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); 1960 } 1961 1962 if ((resp.dbr_stat_db_fifo_reg & 1963 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == 1964 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) 1965 cctx->dbr_stat_db_fifo = 1966 resp.dbr_stat_db_fifo_reg & 1967 ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; 1968 1969 if ((resp.dbr_throttling_aeq_arm_reg & 1970 HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK) 1971 == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) { 1972 cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg & 1973 ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; 1974 cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4; 1975 } 1976 pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth); 1977 if (!pacing_data->fifo_max_depth) 1978 pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx); 1979 pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask); 1980 pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift; 1981 dev_dbg(rdev_to_dev(rdev), 1982 "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i" 1983 "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n", 1984 __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf, 1985 cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg, 1986 pacing_data->fifo_max_depth, 1987 le32_to_cpu(resp.dbr_stat_db_max_fifo_depth)); 1988 return 0; 1989 } 1990 1991 static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable) 1992 { 1993 struct hwrm_func_dbr_pacing_cfg_output resp = {0}; 1994 struct hwrm_func_dbr_pacing_cfg_input req = {0}; 1995 struct bnxt_en_dev *en_dev = rdev->en_dev; 1996 struct bnxt_fw_msg fw_msg; 1997 int rc; 1998 1999 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 2000 return 0; 2001 2002 memset(&fw_msg, 0, sizeof(fw_msg)); 2003 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2004 HWRM_FUNC_DBR_PACING_CFG, -1, -1); 2005 if (enable) { 2006 req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE; 2007 req.enables = 2008 cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID | 2009 HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID); 2010 } else { 2011 req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE; 2012 } 2013 req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id); 2014 req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark); 2015 dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x", 2016 __func__, req.primary_nq_id, req.pacing_threshold); 2017 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2018 sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); 2019 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2020 if (rc) { 2021 dev_dbg(rdev_to_dev(rdev), 2022 "Failed to set dbr pacing config, rc = %#x", rc); 2023 return rc; 2024 } 2025 return 0; 2026 } 2027 2028 /* Net -> RoCE driver */ 2029 2030 /* Device */ 2031 struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev) 2032 { 2033 struct bnxt_re_dev *rdev; 2034 2035 rcu_read_lock(); 2036 list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { 2037 if (rdev->netdev == netdev) { 2038 rcu_read_unlock(); 2039 dev_dbg(rdev_to_dev(rdev), 2040 "netdev (%p) found, ref_count = 0x%x", 2041 netdev, atomic_read(&rdev->ref_count)); 2042 return rdev; 2043 } 2044 } 2045 rcu_read_unlock(); 2046 return NULL; 2047 } 2048 2049 static ssize_t show_rev(struct device *device, struct device_attribute *attr, 2050 char *buf) 2051 { 2052 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 2053 2054 return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); 2055 } 2056 2057 2058 static ssize_t show_hca(struct device *device, struct device_attribute *attr, 2059 char *buf) 2060 { 2061 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 2062 2063 return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); 2064 } 2065 2066 static ssize_t show_board_id(struct device *device, struct device_attribute *attr, 2067 char *buf) 2068 { 2069 struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); 2070 char buffer[BNXT_VPD_PN_FLD_LEN] = {}; 2071 2072 if (!rdev->is_virtfn) 2073 memcpy(buffer, rdev->en_dev->board_part_number, 2074 BNXT_VPD_PN_FLD_LEN - 1); 2075 else 2076 scnprintf(buffer, BNXT_VPD_PN_FLD_LEN, 2077 "0x%x-VF", rdev->en_dev->pdev->device); 2078 2079 return scnprintf(buf, PAGE_SIZE, "%s\n", buffer); 2080 } 2081 2082 static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); 2083 static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); 2084 static DEVICE_ATTR(board_id, 0444, show_board_id, NULL); 2085 2086 static struct device_attribute *bnxt_re_attributes[] = { 2087 &dev_attr_hw_rev, 2088 &dev_attr_hca_type, 2089 &dev_attr_board_id 2090 }; 2091 2092 int ib_register_device_compat(struct bnxt_re_dev *rdev) 2093 { 2094 struct ib_device *ibdev = &rdev->ibdev; 2095 char name[IB_DEVICE_NAME_MAX]; 2096 2097 memset(name, 0, IB_DEVICE_NAME_MAX); 2098 strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX); 2099 2100 strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); 2101 2102 return ib_register_device(ibdev, NULL); 2103 } 2104 2105 static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) 2106 { 2107 struct ib_device *ibdev = &rdev->ibdev; 2108 int ret = 0; 2109 2110 /* ib device init */ 2111 ibdev->owner = THIS_MODULE; 2112 ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; 2113 ibdev->node_type = RDMA_NODE_IB_CA; 2114 strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", 2115 strlen(BNXT_RE_DESC) + 5); 2116 ibdev->phys_port_cnt = 1; 2117 2118 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid); 2119 2120 /* Data path irqs is one less than the max msix vectors */ 2121 ibdev->num_comp_vectors = rdev->nqr.num_msix - 1; 2122 bnxt_re_set_dma_device(ibdev, rdev); 2123 ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; 2124 2125 /* User space */ 2126 ibdev->uverbs_cmd_mask = 2127 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | 2128 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | 2129 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | 2130 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | 2131 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | 2132 (1ull << IB_USER_VERBS_CMD_REG_MR) | 2133 (1ull << IB_USER_VERBS_CMD_DEREG_MR) | 2134 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 2135 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | 2136 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | 2137 (1ull << IB_USER_VERBS_CMD_CREATE_QP) | 2138 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | 2139 (1ull << IB_USER_VERBS_CMD_QUERY_QP) | 2140 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | 2141 (1ull << IB_USER_VERBS_CMD_REREG_MR) | 2142 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | 2143 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | 2144 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | 2145 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | 2146 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | 2147 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | 2148 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | 2149 (1ull << IB_USER_VERBS_CMD_CREATE_AH) | 2150 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | 2151 (1ull << IB_USER_VERBS_CMD_QUERY_AH) | 2152 (1ull << IB_USER_VERBS_CMD_DESTROY_AH); 2153 2154 ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP); 2155 ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ); 2156 2157 #define bnxt_re_ib_ah bnxt_re_ah 2158 #define bnxt_re_ib_cq bnxt_re_cq 2159 #define bnxt_re_ib_pd bnxt_re_pd 2160 #define bnxt_re_ib_srq bnxt_re_srq 2161 #define bnxt_re_ib_ucontext bnxt_re_ucontext 2162 INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE); 2163 2164 ibdev->query_device = bnxt_re_query_device; 2165 ibdev->modify_device = bnxt_re_modify_device; 2166 ibdev->query_port = bnxt_re_query_port; 2167 ibdev->modify_port = bnxt_re_modify_port; 2168 ibdev->get_port_immutable = bnxt_re_get_port_immutable; 2169 ibdev->query_pkey = bnxt_re_query_pkey; 2170 ibdev->get_netdev = bnxt_re_get_netdev; 2171 ibdev->add_gid = bnxt_re_add_gid; 2172 ibdev->del_gid = bnxt_re_del_gid; 2173 ibdev->get_link_layer = bnxt_re_get_link_layer; 2174 ibdev->alloc_pd = bnxt_re_alloc_pd; 2175 ibdev->dealloc_pd = bnxt_re_dealloc_pd; 2176 ibdev->create_ah = bnxt_re_create_ah; 2177 ibdev->modify_ah = bnxt_re_modify_ah; 2178 ibdev->query_ah = bnxt_re_query_ah; 2179 ibdev->destroy_ah = bnxt_re_destroy_ah; 2180 ibdev->create_srq = bnxt_re_create_srq; 2181 ibdev->modify_srq = bnxt_re_modify_srq; 2182 ibdev->query_srq = bnxt_re_query_srq; 2183 ibdev->destroy_srq = bnxt_re_destroy_srq; 2184 ibdev->post_srq_recv = bnxt_re_post_srq_recv; 2185 ibdev->create_qp = bnxt_re_create_qp; 2186 ibdev->modify_qp = bnxt_re_modify_qp; 2187 ibdev->query_qp = bnxt_re_query_qp; 2188 ibdev->destroy_qp = bnxt_re_destroy_qp; 2189 ibdev->post_send = bnxt_re_post_send; 2190 ibdev->post_recv = bnxt_re_post_recv; 2191 ibdev->create_cq = bnxt_re_create_cq; 2192 ibdev->modify_cq = bnxt_re_modify_cq; 2193 ibdev->destroy_cq = bnxt_re_destroy_cq; 2194 ibdev->resize_cq = bnxt_re_resize_cq; 2195 ibdev->poll_cq = bnxt_re_poll_cq; 2196 ibdev->req_notify_cq = bnxt_re_req_notify_cq; 2197 ibdev->get_dma_mr = bnxt_re_get_dma_mr; 2198 ibdev->get_hw_stats = bnxt_re_get_hw_stats; 2199 ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats; 2200 ibdev->dereg_mr = bnxt_re_dereg_mr; 2201 ibdev->alloc_mr = bnxt_re_alloc_mr; 2202 ibdev->map_mr_sg = bnxt_re_map_mr_sg; 2203 ibdev->alloc_mw = bnxt_re_alloc_mw; 2204 ibdev->dealloc_mw = bnxt_re_dealloc_mw; 2205 ibdev->reg_user_mr = bnxt_re_reg_user_mr; 2206 ibdev->rereg_user_mr = bnxt_re_rereg_user_mr; 2207 ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx; 2208 ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; 2209 ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; 2210 ibdev->mmap = bnxt_re_mmap; 2211 ibdev->process_mad = bnxt_re_process_mad; 2212 2213 ret = ib_register_device_compat(rdev); 2214 return ret; 2215 } 2216 2217 static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev) 2218 { 2219 int i = BNXT_RE_REF_WAIT_COUNT; 2220 2221 dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev); 2222 /* Wait for rdev refcount to come down */ 2223 while ((atomic_read(&rdev->ref_count) > 1) && i--) 2224 msleep(100); 2225 2226 if (atomic_read(&rdev->ref_count) > 1) 2227 dev_err(rdev_to_dev(rdev), 2228 "Failed waiting for ref count to deplete %d", 2229 atomic_read(&rdev->ref_count)); 2230 2231 atomic_set(&rdev->ref_count, 0); 2232 if_rele(rdev->netdev); 2233 rdev->netdev = NULL; 2234 synchronize_rcu(); 2235 2236 kfree(rdev->gid_map); 2237 kfree(rdev->dbg_stats); 2238 ib_dealloc_device(&rdev->ibdev); 2239 } 2240 2241 static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev, 2242 struct bnxt_en_dev *en_dev) 2243 { 2244 struct bnxt_re_dev *rdev; 2245 u32 count; 2246 2247 /* Allocate bnxt_re_dev instance here */ 2248 rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev)); 2249 if (!rdev) { 2250 pr_err("%s: bnxt_re_dev allocation failure!", 2251 ROCE_DRV_MODULE_NAME); 2252 return NULL; 2253 } 2254 /* Default values */ 2255 atomic_set(&rdev->ref_count, 0); 2256 rdev->netdev = netdev; 2257 dev_hold(rdev->netdev); 2258 rdev->en_dev = en_dev; 2259 rdev->id = rdev->en_dev->pdev->devfn; 2260 INIT_LIST_HEAD(&rdev->qp_list); 2261 mutex_init(&rdev->qp_lock); 2262 mutex_init(&rdev->cc_lock); 2263 mutex_init(&rdev->dbq_lock); 2264 bnxt_re_clear_rsors_stat(&rdev->stats.rsors); 2265 rdev->cosq[0] = rdev->cosq[1] = 0xFFFF; 2266 rdev->min_tx_depth = 1; 2267 rdev->stats.stats_query_sec = 1; 2268 /* Disable priority vlan as the default mode is DSCP based PFC */ 2269 rdev->cc_param.disable_prio_vlan_tx = 1; 2270 2271 /* Initialize worker for DBR Pacing */ 2272 INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); 2273 INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); 2274 rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) * 2275 BNXT_RE_MAX_SGID_ENTRIES, 2276 GFP_KERNEL); 2277 if (!rdev->gid_map) { 2278 ib_dealloc_device(&rdev->ibdev); 2279 return NULL; 2280 } 2281 for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++) 2282 rdev->gid_map[count] = -1; 2283 2284 rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL); 2285 if (!rdev->dbg_stats) { 2286 ib_dealloc_device(&rdev->ibdev); 2287 return NULL; 2288 } 2289 2290 return rdev; 2291 } 2292 2293 static int bnxt_re_handle_unaffi_async_event( 2294 struct creq_func_event *unaffi_async) 2295 { 2296 switch (unaffi_async->event) { 2297 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: 2298 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: 2299 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: 2300 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: 2301 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: 2302 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: 2303 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: 2304 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: 2305 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: 2306 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: 2307 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: 2308 break; 2309 default: 2310 return -EINVAL; 2311 } 2312 return 0; 2313 } 2314 2315 static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp) 2316 { 2317 struct creq_qp_error_notification *err_event; 2318 struct ib_event event; 2319 unsigned int flags; 2320 2321 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && 2322 !qp->qplib_qp.is_user) { 2323 flags = bnxt_re_lock_cqs(qp); 2324 bnxt_qplib_add_flush_qp(&qp->qplib_qp); 2325 bnxt_re_unlock_cqs(qp, flags); 2326 } 2327 memset(&event, 0, sizeof(event)); 2328 event.device = &qp->rdev->ibdev; 2329 event.element.qp = &qp->ib_qp; 2330 event.event = IB_EVENT_QP_FATAL; 2331 2332 err_event = qp_event; 2333 switch(err_event->res_err_state_reason) { 2334 case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX: 2335 case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH: 2336 case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR: 2337 case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT: 2338 case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY: 2339 case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR: 2340 case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION: 2341 case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR: 2342 case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY: 2343 case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR: 2344 case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION: 2345 case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR: 2346 case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY: 2347 case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC: 2348 event.event = IB_EVENT_QP_ACCESS_ERR; 2349 break; 2350 case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE: 2351 case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR: 2352 case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR: 2353 case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE: 2354 case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE: 2355 event.event = IB_EVENT_QP_REQ_ERR; 2356 break; 2357 case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW: 2358 case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR: 2359 case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR: 2360 case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR: 2361 case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR: 2362 case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR: 2363 case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR: 2364 event.event = IB_EVENT_QP_FATAL; 2365 break; 2366 default: 2367 if (qp->qplib_qp.srq) 2368 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 2369 break; 2370 } 2371 2372 if (err_event->res_err_state_reason) 2373 dev_err(rdev_to_dev(qp->rdev), 2374 "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n", 2375 __func__, qp->qplib_qp.is_user ? "user" : "kernel", 2376 qp->qplib_qp.id, 2377 err_event->sq_cons_idx, 2378 err_event->rq_cons_idx, 2379 err_event->req_slow_path_state, 2380 err_event->req_err_state_reason, 2381 err_event->res_slow_path_state, 2382 err_event->res_err_state_reason); 2383 2384 if (event.device && qp->ib_qp.event_handler) 2385 qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); 2386 2387 return 0; 2388 } 2389 2390 static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq) 2391 { 2392 struct creq_cq_error_notification *cqerr; 2393 bool send = false; 2394 2395 cqerr = event; 2396 switch (cqerr->cq_err_reason) { 2397 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR: 2398 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR: 2399 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR: 2400 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR: 2401 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR: 2402 case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR: 2403 send = true; 2404 default: 2405 break; 2406 } 2407 2408 if (send && cq->ibcq.event_handler) { 2409 struct ib_event ibevent = {}; 2410 2411 ibevent.event = IB_EVENT_CQ_ERR; 2412 ibevent.element.cq = &cq->ibcq; 2413 ibevent.device = &cq->rdev->ibdev; 2414 2415 dev_err(rdev_to_dev(cq->rdev), 2416 "%s err reason %d\n", __func__, cqerr->cq_err_reason); 2417 cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context); 2418 } 2419 2420 cq->qplib_cq.is_cq_err_event = true; 2421 2422 return 0; 2423 } 2424 2425 static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, 2426 void *obj) 2427 { 2428 struct bnxt_qplib_qp *qplqp; 2429 struct bnxt_qplib_cq *qplcq; 2430 struct bnxt_re_qp *qp; 2431 struct bnxt_re_cq *cq; 2432 int rc = 0; 2433 u8 event; 2434 2435 if (!obj) 2436 return rc; /* QP was already dead, still return success */ 2437 2438 event = affi_async->event; 2439 switch (event) { 2440 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: 2441 qplqp = obj; 2442 qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp); 2443 rc = bnxt_re_handle_qp_async_event(affi_async, qp); 2444 break; 2445 case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: 2446 qplcq = obj; 2447 cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq); 2448 rc = bnxt_re_handle_cq_async_error(affi_async, cq); 2449 break; 2450 default: 2451 rc = -EINVAL; 2452 } 2453 2454 return rc; 2455 } 2456 2457 static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, 2458 void *aeqe, void *obj) 2459 { 2460 struct creq_func_event *unaffi_async; 2461 struct creq_qp_event *affi_async; 2462 u8 type; 2463 int rc; 2464 2465 type = ((struct creq_base *)aeqe)->type; 2466 if (type == CREQ_BASE_TYPE_FUNC_EVENT) { 2467 unaffi_async = aeqe; 2468 rc = bnxt_re_handle_unaffi_async_event(unaffi_async); 2469 } else { 2470 affi_async = aeqe; 2471 rc = bnxt_re_handle_affi_async_event(affi_async, obj); 2472 } 2473 2474 return rc; 2475 } 2476 2477 static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, 2478 struct bnxt_qplib_srq *handle, u8 event) 2479 { 2480 struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq, 2481 qplib_srq); 2482 struct ib_event ib_event; 2483 2484 if (srq == NULL) { 2485 pr_err("%s: SRQ is NULL, SRQN not handled", 2486 ROCE_DRV_MODULE_NAME); 2487 return -EINVAL; 2488 } 2489 ib_event.device = &srq->rdev->ibdev; 2490 ib_event.element.srq = &srq->ibsrq; 2491 if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) 2492 ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; 2493 else 2494 ib_event.event = IB_EVENT_SRQ_ERR; 2495 2496 if (srq->ibsrq.event_handler) { 2497 /* Lock event_handler? */ 2498 (*srq->ibsrq.event_handler)(&ib_event, 2499 srq->ibsrq.srq_context); 2500 } 2501 return 0; 2502 } 2503 2504 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, 2505 struct bnxt_qplib_cq *handle) 2506 { 2507 struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq, 2508 qplib_cq); 2509 u32 *cq_ptr; 2510 2511 if (cq == NULL) { 2512 pr_err("%s: CQ is NULL, CQN not handled", 2513 ROCE_DRV_MODULE_NAME); 2514 return -EINVAL; 2515 } 2516 /* CQ already in destroy path. Do not handle any more events */ 2517 if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) { 2518 if (!handle->destroyed) 2519 dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled", 2520 ROCE_DRV_MODULE_NAME); 2521 return 0; 2522 } 2523 2524 if (cq->ibcq.comp_handler) { 2525 if (cq->uctx_cq_page) { 2526 cq_ptr = (u32 *)cq->uctx_cq_page; 2527 *cq_ptr = cq->qplib_cq.toggle; 2528 } 2529 /* Lock comp_handler? */ 2530 (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context); 2531 } 2532 2533 return 0; 2534 } 2535 2536 struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev) 2537 { 2538 int min, indx; 2539 2540 mutex_lock(&rdev->nqr.load_lock); 2541 for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) { 2542 if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load) 2543 min = indx; 2544 } 2545 rdev->nqr.nq[min].load++; 2546 mutex_unlock(&rdev->nqr.load_lock); 2547 2548 return &rdev->nqr.nq[min]; 2549 } 2550 2551 void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq) 2552 { 2553 mutex_lock(&rdev->nqr.load_lock); 2554 nq->load--; 2555 mutex_unlock(&rdev->nqr.load_lock); 2556 } 2557 2558 static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev) 2559 { 2560 struct bnxt_qplib_dev_attr *attr; 2561 bool rc = true; 2562 2563 attr = rdev->dev_attr; 2564 2565 if (!attr->max_cq || !attr->max_qp || 2566 !attr->max_sgid || !attr->max_mr) { 2567 dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources"); 2568 dev_dbg(rdev_to_dev(rdev), 2569 "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d", 2570 attr->max_cq, attr->max_qp, attr->max_dpi, 2571 attr->max_sgid, attr->max_mr); 2572 rc = false; 2573 } 2574 return rc; 2575 } 2576 2577 static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, 2578 u8 port_num, enum ib_event_type event) 2579 { 2580 struct ib_event ib_event; 2581 2582 ib_event.device = ibdev; 2583 if (qp) { 2584 ib_event.element.qp = qp; 2585 ib_event.event = event; 2586 if (qp->event_handler) 2587 qp->event_handler(&ib_event, qp->qp_context); 2588 } else { 2589 ib_event.element.port_num = port_num; 2590 ib_event.event = event; 2591 ib_dispatch_event(&ib_event); 2592 } 2593 2594 dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)), 2595 "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num); 2596 } 2597 2598 static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, 2599 struct bnxt_re_qp *qp) 2600 { 2601 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) 2602 return (qp->ib_qp.qp_type == IB_QPT_GSI) || 2603 (qp == rdev->gsi_ctx.gsi_sqp); 2604 else 2605 return (qp->ib_qp.qp_type == IB_QPT_GSI); 2606 } 2607 2608 static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev) 2609 { 2610 struct bnxt_qplib_qp *qpl_qp; 2611 bool dev_detached = false; 2612 struct ib_qp_attr qp_attr; 2613 int num_qps_stopped = 0; 2614 int mask = IB_QP_STATE; 2615 struct bnxt_re_qp *qp; 2616 unsigned long flags; 2617 2618 if (!rdev) 2619 return; 2620 2621 restart: 2622 if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) 2623 dev_detached = true; 2624 2625 qp_attr.qp_state = IB_QPS_ERR; 2626 mutex_lock(&rdev->qp_lock); 2627 list_for_each_entry(qp, &rdev->qp_list, list) { 2628 qpl_qp = &qp->qplib_qp; 2629 if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { 2630 if (qpl_qp->state != 2631 CMDQ_MODIFY_QP_NEW_STATE_RESET && 2632 qpl_qp->state != 2633 CMDQ_MODIFY_QP_NEW_STATE_ERR) { 2634 if (dev_detached) { 2635 /* 2636 * Cant actually send the command down, 2637 * marking the state for bookkeeping 2638 */ 2639 qpl_qp->state = 2640 CMDQ_MODIFY_QP_NEW_STATE_ERR; 2641 qpl_qp->cur_qp_state = qpl_qp->state; 2642 if (!qpl_qp->is_user) { 2643 /* Add to flush list */ 2644 flags = bnxt_re_lock_cqs(qp); 2645 bnxt_qplib_add_flush_qp(qpl_qp); 2646 bnxt_re_unlock_cqs(qp, flags); 2647 } 2648 } else { 2649 num_qps_stopped++; 2650 bnxt_re_modify_qp(&qp->ib_qp, 2651 &qp_attr, mask, 2652 NULL); 2653 } 2654 2655 bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, 2656 1, IB_EVENT_QP_FATAL); 2657 /* 2658 * 1. Release qp_lock after a budget to unblock other verb 2659 * requests (like qp_destroy) from stack. 2660 * 2. Traverse through the qp_list freshly as addition / deletion 2661 * might have happened since qp_lock is getting released here. 2662 */ 2663 if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) { 2664 mutex_unlock(&rdev->qp_lock); 2665 goto restart; 2666 } 2667 } 2668 } 2669 } 2670 2671 mutex_unlock(&rdev->qp_lock); 2672 } 2673 2674 static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) 2675 { 2676 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; 2677 struct bnxt_qplib_gid gid; 2678 u16 gid_idx, index; 2679 int rc = 0; 2680 2681 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 2682 return 0; 2683 2684 if (sgid_tbl == NULL) { 2685 dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); 2686 return -EINVAL; 2687 } 2688 2689 for (index = 0; index < sgid_tbl->active; index++) { 2690 gid_idx = sgid_tbl->hw_id[index]; 2691 2692 if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, 2693 sizeof(bnxt_qplib_gid_zero))) 2694 continue; 2695 /* Need to modify the VLAN enable setting of non VLAN GID only 2696 * as setting is done for VLAN GID while adding GID 2697 * 2698 * If disable_prio_vlan_tx is enable, then we'll need to remove the 2699 * vlan entry from the sgid_tbl. 2700 */ 2701 if (sgid_tbl->vlan[index] == true) 2702 continue; 2703 2704 memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); 2705 2706 rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, 2707 rdev->dev_addr); 2708 } 2709 2710 return rc; 2711 } 2712 2713 static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev) 2714 { 2715 struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; 2716 2717 if (_is_chip_p7(rdev->chip_ctx)) { 2718 cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; 2719 } else { 2720 cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | 2721 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | 2722 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); 2723 2724 if (!is_qport_service_type_supported(rdev)) 2725 cc_param->mask |= 2726 (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | 2727 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | 2728 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); 2729 } 2730 2731 cc_param->cur_mask = cc_param->mask; 2732 2733 if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param)) 2734 dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); 2735 } 2736 2737 static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev) 2738 { 2739 struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; 2740 int rc; 2741 2742 if (_is_chip_p7(rdev->chip_ctx)) { 2743 cc_param->enable = 0x0; 2744 cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; 2745 } else { 2746 cc_param->enable = 0x1; 2747 cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | 2748 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | 2749 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); 2750 2751 if (!is_qport_service_type_supported(rdev)) 2752 cc_param->mask |= 2753 (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | 2754 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | 2755 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); 2756 } 2757 2758 cc_param->cur_mask = cc_param->mask; 2759 2760 rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); 2761 if (rc) { 2762 dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); 2763 return rc; 2764 } 2765 /* Reset the programming mask */ 2766 cc_param->mask = 0; 2767 if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { 2768 cc_param->qp1_tos_dscp = cc_param->tos_dscp; 2769 rc = bnxt_re_update_qp1_tos_dscp(rdev); 2770 if (rc) { 2771 dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d", 2772 __func__, rc); 2773 goto clear; 2774 } 2775 } 2776 return 0; 2777 2778 clear: 2779 bnxt_re_clear_cc(rdev); 2780 return rc; 2781 } 2782 2783 int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, 2784 struct bnxt_re_dscp2pri *d2p, u16 *count, 2785 u16 target_id) 2786 { 2787 struct bnxt_en_dev *en_dev = rdev->en_dev; 2788 struct hwrm_queue_dscp2pri_qcfg_input req; 2789 struct hwrm_queue_dscp2pri_qcfg_output resp; 2790 struct bnxt_re_dscp2pri *dscp2pri; 2791 struct bnxt_fw_msg fw_msg; 2792 u16 in_count = *count; 2793 dma_addr_t dma_handle; 2794 int rc = 0, i; 2795 u16 data_len; 2796 u8 *kmem; 2797 2798 data_len = *count * sizeof(*dscp2pri); 2799 memset(&fw_msg, 0, sizeof(fw_msg)); 2800 memset(&req, 0, sizeof(req)); 2801 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2802 HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id); 2803 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 2804 2805 kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, 2806 GFP_KERNEL); 2807 if (!kmem) { 2808 dev_err(rdev_to_dev(rdev), 2809 "dma_zalloc_coherent failure, length = %u\n", 2810 (unsigned)data_len); 2811 return -ENOMEM; 2812 } 2813 req.dest_data_addr = cpu_to_le64(dma_handle); 2814 req.dest_data_buffer_size = cpu_to_le16(data_len); 2815 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2816 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2817 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2818 if (rc) 2819 goto out; 2820 2821 /* Upload the DSCP-MASK-PRI tuple(s) */ 2822 dscp2pri = (struct bnxt_re_dscp2pri *)kmem; 2823 for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) { 2824 d2p[i].dscp = dscp2pri->dscp; 2825 d2p[i].mask = dscp2pri->mask; 2826 d2p[i].pri = dscp2pri->pri; 2827 dscp2pri++; 2828 } 2829 *count = le16_to_cpu(resp.entry_cnt); 2830 out: 2831 dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); 2832 return rc; 2833 } 2834 2835 int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev) 2836 { 2837 /* Remove the VLAN from the GID entry */ 2838 if (rdev->cc_param.disable_prio_vlan_tx) 2839 rdev->qplib_res.prio = false; 2840 else 2841 rdev->qplib_res.prio = true; 2842 2843 return bnxt_re_update_gid(rdev); 2844 } 2845 2846 int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, 2847 struct bnxt_re_dscp2pri *d2p, u16 count, 2848 u16 target_id) 2849 { 2850 struct bnxt_en_dev *en_dev = rdev->en_dev; 2851 struct hwrm_queue_dscp2pri_cfg_input req; 2852 struct hwrm_queue_dscp2pri_cfg_output resp; 2853 struct bnxt_fw_msg fw_msg; 2854 struct bnxt_re_dscp2pri *dscp2pri; 2855 int i, rc, data_len = 3 * 256; 2856 dma_addr_t dma_handle; 2857 u8 *kmem; 2858 2859 memset(&req, 0, sizeof(req)); 2860 memset(&fw_msg, 0, sizeof(fw_msg)); 2861 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2862 HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id); 2863 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 2864 2865 kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, 2866 GFP_KERNEL); 2867 if (!kmem) { 2868 dev_err(rdev_to_dev(rdev), 2869 "dma_alloc_coherent failure, length = %u\n", 2870 (unsigned)data_len); 2871 return -ENOMEM; 2872 } 2873 req.src_data_addr = cpu_to_le64(dma_handle); 2874 2875 /* Download the DSCP-MASK-PRI tuple(s) */ 2876 dscp2pri = (struct bnxt_re_dscp2pri *)kmem; 2877 for (i = 0; i < count; i++) { 2878 dscp2pri->dscp = d2p[i].dscp; 2879 dscp2pri->mask = d2p[i].mask; 2880 dscp2pri->pri = d2p[i].pri; 2881 dscp2pri++; 2882 } 2883 2884 req.entry_cnt = cpu_to_le16(count); 2885 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2886 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2887 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2888 dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); 2889 return rc; 2890 } 2891 2892 int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, 2893 struct bnxt_re_tc_rec *tc_rec, u16 tid) 2894 { 2895 u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1; 2896 struct hwrm_queue_qportcfg_output resp = {0}; 2897 struct hwrm_queue_qportcfg_input req = {0}; 2898 struct bnxt_en_dev *en_dev = rdev->en_dev; 2899 struct bnxt_fw_msg fw_msg; 2900 bool def_init = false; 2901 u8 *tmp_type; 2902 u8 cos_id; 2903 int rc; 2904 2905 memset(&fw_msg, 0, sizeof(fw_msg)); 2906 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG, 2907 -1, tid); 2908 req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1; 2909 if (BNXT_EN_ASYM_Q(en_dev)) 2910 req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); 2911 2912 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2913 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2914 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2915 if (rc) 2916 return rc; 2917 2918 if (!resp.max_configurable_queues) 2919 return -EINVAL; 2920 2921 max_tc = resp.max_configurable_queues; 2922 tc_rec->max_tc = max_tc; 2923 2924 if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE) 2925 tc_rec->serv_type_enabled = true; 2926 2927 qptr = &resp.queue_id0; 2928 type_ptr0 = &resp.queue_id0_service_profile_type; 2929 type_ptr1 = &resp.queue_id1_service_profile_type; 2930 for (tc = 0; tc < max_tc; tc++) { 2931 tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0; 2932 2933 cos_id = *qptr++; 2934 /* RoCE CoS queue is the first cos queue. 2935 * For MP12 and MP17 order is 405 and 141015. 2936 */ 2937 if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) { 2938 tc_rec->cos_id_roce = cos_id; 2939 tc_rec->tc_roce = tc; 2940 } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) { 2941 tc_rec->cos_id_cnp = cos_id; 2942 tc_rec->tc_cnp = tc; 2943 } else if (!def_init) { 2944 def_init = true; 2945 tc_rec->tc_def = tc; 2946 tc_rec->cos_id_def = cos_id; 2947 } 2948 qptr++; 2949 } 2950 2951 return rc; 2952 } 2953 2954 int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, 2955 struct bnxt_re_cos2bw_cfg *cfg) 2956 { 2957 struct bnxt_en_dev *en_dev = rdev->en_dev; 2958 struct hwrm_queue_cos2bw_qcfg_output resp; 2959 struct hwrm_queue_cos2bw_qcfg_input req = {0}; 2960 struct bnxt_fw_msg fw_msg; 2961 int rc, indx; 2962 void *data; 2963 2964 memset(&fw_msg, 0, sizeof(fw_msg)); 2965 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2966 HWRM_QUEUE_COS2BW_QCFG, -1, target_id); 2967 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 2968 2969 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 2970 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 2971 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 2972 if (rc) 2973 return rc; 2974 data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg, 2975 queue_id); 2976 for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) { 2977 memcpy(&cfg->cfg, data, sizeof(cfg->cfg)); 2978 if (indx == 0) 2979 cfg->queue_id = resp.queue_id0; 2980 cfg++; 2981 } 2982 2983 return rc; 2984 } 2985 2986 int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, 2987 struct bnxt_re_cos2bw_cfg *cfg) 2988 { 2989 struct bnxt_en_dev *en_dev = rdev->en_dev; 2990 struct hwrm_queue_cos2bw_cfg_input req = {0}; 2991 struct hwrm_queue_cos2bw_cfg_output resp = {0}; 2992 struct bnxt_fw_msg fw_msg; 2993 void *data; 2994 int indx; 2995 int rc; 2996 2997 memset(&fw_msg, 0, sizeof(fw_msg)); 2998 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 2999 HWRM_QUEUE_COS2BW_CFG, -1, target_id); 3000 req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; 3001 3002 /* Chimp wants enable bit to retain previous 3003 * config done by L2 driver 3004 */ 3005 for (indx = 0; indx < 8; indx++) { 3006 if (cfg[indx].queue_id < 40) { 3007 req.enables |= cpu_to_le32( 3008 HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID << 3009 indx); 3010 } 3011 3012 data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4); 3013 memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4); 3014 if (indx == 0) { 3015 req.queue_id0 = cfg[0].queue_id; 3016 req.unused_0 = 0; 3017 } 3018 } 3019 3020 memset(&resp, 0, sizeof(resp)); 3021 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 3022 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 3023 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 3024 return rc; 3025 } 3026 3027 int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, 3028 struct bnxt_qplib_query_fn_info *fn_info, 3029 u32 *pf_mask, u32 *first_pf) 3030 { 3031 struct hwrm_func_host_pf_ids_query_output resp = {0}; 3032 struct hwrm_func_host_pf_ids_query_input req; 3033 struct bnxt_en_dev *en_dev = rdev->en_dev; 3034 struct bnxt_fw_msg fw_msg; 3035 int rc; 3036 3037 memset(&fw_msg, 0, sizeof(fw_msg)); 3038 memset(&req, 0, sizeof(req)); 3039 bnxt_re_init_hwrm_hdr(rdev, (void *)&req, 3040 HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1); 3041 /* To query the info from the host EPs */ 3042 switch (fn_info->host) { 3043 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC: 3044 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0: 3045 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1: 3046 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2: 3047 case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3: 3048 req.host = fn_info->host; 3049 break; 3050 default: 3051 req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0; 3052 break; 3053 } 3054 3055 req.filter = fn_info->filter; 3056 if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE) 3057 req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL; 3058 3059 bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, 3060 sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); 3061 rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); 3062 3063 3064 *first_pf = le16_to_cpu(resp.first_pf_id); 3065 *pf_mask = le16_to_cpu(resp.pf_ordinal_mask); 3066 3067 return rc; 3068 } 3069 3070 static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev) 3071 { 3072 struct bnxt_qplib_ctx *hctx; 3073 struct bnxt_qplib_res *res; 3074 u16 tid = 0xffff; 3075 3076 res = &rdev->qplib_res; 3077 hctx = res->hctx; 3078 3079 if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) { 3080 bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid); 3081 bnxt_qplib_free_stat_mem(res, &hctx->stats); 3082 } 3083 } 3084 3085 static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev) 3086 { 3087 test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags); 3088 } 3089 3090 static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev) 3091 { 3092 struct bnxt_qplib_ctx *hctx; 3093 struct bnxt_qplib_res *res; 3094 u16 tid = 0xffff; 3095 int rc; 3096 3097 res = &rdev->qplib_res; 3098 hctx = res->hctx; 3099 3100 rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats); 3101 if (rc) 3102 return -ENOMEM; 3103 rc = bnxt_re_net_stats_ctx_alloc(rdev, tid); 3104 if (rc) 3105 goto free_stat_mem; 3106 set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags); 3107 3108 return 0; 3109 3110 free_stat_mem: 3111 bnxt_qplib_free_stat_mem(res, &hctx->stats); 3112 3113 return rc; 3114 } 3115 3116 static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev) 3117 { 3118 int rc; 3119 3120 rc = bnxt_qplib_get_dev_attr(&rdev->rcfw); 3121 if (rc) 3122 return rc; 3123 if (!bnxt_re_check_min_attr(rdev)) 3124 return -EINVAL; 3125 return 0; 3126 } 3127 3128 static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev) 3129 { 3130 bnxt_qplib_clear_tbls(&rdev->qplib_res); 3131 bnxt_qplib_free_tbls(&rdev->qplib_res); 3132 } 3133 3134 static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev) 3135 { 3136 struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx; 3137 u8 pppp_factor = 0; 3138 int rc; 3139 3140 /* 3141 * TODO: Need a better mechanism for spreading of the 3142 * 512 extended PPP pages. For now, spreading it 3143 * based on port_count 3144 */ 3145 if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push) 3146 pppp_factor = rdev->en_dev->port_count; 3147 rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor); 3148 if (rc) 3149 return rc; 3150 bnxt_qplib_init_tbls(&rdev->qplib_res); 3151 set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags); 3152 3153 return 0; 3154 } 3155 3156 static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev) 3157 { 3158 struct bnxt_qplib_nq *nq; 3159 int i; 3160 3161 if (!rdev->nqr.max_init) 3162 return; 3163 3164 for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) { 3165 nq = &rdev->nqr.nq[i]; 3166 bnxt_qplib_disable_nq(nq); 3167 bnxt_re_net_ring_free(rdev, nq->ring_id); 3168 bnxt_qplib_free_nq_mem(nq); 3169 } 3170 rdev->nqr.max_init = 0; 3171 } 3172 3173 static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev) 3174 { 3175 struct bnxt_re_ring_attr rattr = {}; 3176 struct bnxt_qplib_nq *nq; 3177 int rc, i; 3178 int depth; 3179 u32 offt; 3180 u16 vec; 3181 3182 mutex_init(&rdev->nqr.load_lock); 3183 /* 3184 * TODO: Optimize the depth based on the 3185 * number of NQs. 3186 */ 3187 depth = BNXT_QPLIB_NQE_MAX_CNT; 3188 for (i = 0; i < rdev->nqr.num_msix - 1; i++) { 3189 nq = &rdev->nqr.nq[i]; 3190 vec = rdev->nqr.msix_entries[i + 1].vector; 3191 offt = rdev->nqr.msix_entries[i + 1].db_offset; 3192 nq->hwq.max_elements = depth; 3193 rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq); 3194 if (rc) { 3195 dev_err(rdev_to_dev(rdev), 3196 "Failed to get mem for NQ %d, rc = 0x%x", 3197 i, rc); 3198 goto fail_mem; 3199 } 3200 3201 rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr; 3202 rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count; 3203 rattr.type = bnxt_re_get_rtype(rdev); 3204 rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; 3205 rattr.depth = nq->hwq.max_elements - 1; 3206 rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx; 3207 3208 /* Set DBR pacing capability on the first NQ ring only */ 3209 if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) 3210 rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING; 3211 else 3212 rattr.flags = 0; 3213 3214 rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id); 3215 if (rc) { 3216 nq->ring_id = 0xffff; /* Invalid ring-id */ 3217 dev_err(rdev_to_dev(rdev), 3218 "Failed to get fw id for NQ %d, rc = 0x%x", 3219 i, rc); 3220 goto fail_ring; 3221 } 3222 3223 rc = bnxt_qplib_enable_nq(nq, i, vec, offt, 3224 &bnxt_re_cqn_handler, 3225 &bnxt_re_srqn_handler); 3226 if (rc) { 3227 dev_err(rdev_to_dev(rdev), 3228 "Failed to enable NQ %d, rc = 0x%x", i, rc); 3229 goto fail_en; 3230 } 3231 } 3232 3233 rdev->nqr.max_init = i; 3234 return 0; 3235 fail_en: 3236 /* *nq was i'th nq */ 3237 bnxt_re_net_ring_free(rdev, nq->ring_id); 3238 fail_ring: 3239 bnxt_qplib_free_nq_mem(nq); 3240 fail_mem: 3241 rdev->nqr.max_init = i; 3242 return rc; 3243 } 3244 3245 static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev) 3246 { 3247 int i; 3248 3249 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) 3250 device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); 3251 } 3252 3253 static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev) 3254 { 3255 int i, j, rc = 0; 3256 3257 for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { 3258 rc = device_create_file(&rdev->ibdev.dev, 3259 bnxt_re_attributes[i]); 3260 if (rc) { 3261 dev_err(rdev_to_dev(rdev), 3262 "Failed to create IB sysfs with rc = 0x%x", rc); 3263 /* Must clean up all created device files */ 3264 for (j = 0; j < i; j++) 3265 device_remove_file(&rdev->ibdev.dev, 3266 bnxt_re_attributes[j]); 3267 clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 3268 ib_unregister_device(&rdev->ibdev); 3269 return 1; 3270 } 3271 } 3272 return 0; 3273 } 3274 3275 /* worker thread for polling periodic events. Now used for QoS programming*/ 3276 static void bnxt_re_worker(struct work_struct *work) 3277 { 3278 struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, 3279 worker.work); 3280 int rc; 3281 3282 /* QoS is in 30s cadence for PFs*/ 3283 if (!rdev->is_virtfn && !rdev->worker_30s--) 3284 rdev->worker_30s = 30; 3285 /* Use trylock for bnxt_re_dev_lock as this can be 3286 * held for long time by debugfs show path while issuing 3287 * HWRMS. If the debugfs name update is not done in this 3288 * iteration, the driver will check for the same in the 3289 * next schedule of the worker i.e after 1 sec. 3290 */ 3291 if (mutex_trylock(&bnxt_re_dev_lock)) 3292 mutex_unlock(&bnxt_re_dev_lock); 3293 3294 if (!rdev->stats.stats_query_sec) 3295 goto resched; 3296 3297 if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) && 3298 (rdev->is_virtfn || 3299 !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) { 3300 if (!(rdev->stats.stats_query_counter++ % 3301 rdev->stats.stats_query_sec)) { 3302 rc = bnxt_re_get_qos_stats(rdev); 3303 if (rc && rc != -ENOMEM) 3304 clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, 3305 &rdev->flags); 3306 } 3307 } 3308 3309 resched: 3310 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); 3311 } 3312 3313 static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) 3314 { 3315 if (!(rdev->dbr_drop_recov || rdev->dbr_pacing)) 3316 return 0; 3317 3318 rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL); 3319 if (!rdev->dbr_sw_stats) 3320 return -ENOMEM; 3321 3322 return 0; 3323 } 3324 3325 static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) 3326 { 3327 kfree(rdev->dbr_sw_stats); 3328 rdev->dbr_sw_stats = NULL; 3329 } 3330 3331 static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev) 3332 { 3333 rdev->dbr_drop_recov_wq = 3334 create_singlethread_workqueue("bnxt_re_dbr_drop_recov"); 3335 if (!rdev->dbr_drop_recov_wq) { 3336 dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!"); 3337 return -EINVAL; 3338 } 3339 rdev->dbr_drop_recov = true; 3340 3341 /* Enable configfs setting dbr_drop_recov by default*/ 3342 rdev->user_dbr_drop_recov = true; 3343 3344 rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT; 3345 return 0; 3346 } 3347 3348 static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev) 3349 { 3350 if (rdev->dbr_drop_recov_wq) { 3351 flush_workqueue(rdev->dbr_drop_recov_wq); 3352 destroy_workqueue(rdev->dbr_drop_recov_wq); 3353 rdev->dbr_drop_recov_wq = NULL; 3354 } 3355 rdev->dbr_drop_recov = false; 3356 } 3357 3358 static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) 3359 { 3360 int rc; 3361 3362 /* Allocate a page for app use */ 3363 rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL); 3364 if (!rdev->dbr_page) { 3365 dev_err(rdev_to_dev(rdev), "DBR page allocation failed!"); 3366 return -ENOMEM; 3367 } 3368 memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE); 3369 rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page; 3370 rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev); 3371 if (rc) { 3372 dev_err(rdev_to_dev(rdev), 3373 "Failed to query dbr pacing config %d\n", rc); 3374 goto fail; 3375 } 3376 /* Create a work queue for scheduling dbq event */ 3377 rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq"); 3378 if (!rdev->dbq_wq) { 3379 dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!"); 3380 rc = -ENOMEM; 3381 goto fail; 3382 } 3383 /* MAP grc window 2 for reading db fifo depth */ 3384 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 3385 BNXT_HWRM_BAR_IDX, 3386 rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK); 3387 rdev->dbr_db_fifo_reg_off = 3388 (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + 3389 0x2000; 3390 rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off; 3391 3392 rdev->dbr_bar_addr = 3393 pci_resource_start(rdev->qplib_res.pdev, 0) + 3394 rdev->dbr_db_fifo_reg_off; 3395 3396 /* Percentage of DB FIFO */ 3397 rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD; 3398 rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD; 3399 rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; 3400 rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME; 3401 rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; 3402 rdev->do_pacing_save = rdev->dbr_def_do_pacing; 3403 bnxt_re_set_default_pacing_data(rdev); 3404 dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n"); 3405 3406 return 0; 3407 fail: 3408 free_page((u64)rdev->dbr_page); 3409 rdev->dbr_page = NULL; 3410 return rc; 3411 } 3412 3413 static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) 3414 { 3415 if (rdev->dbq_wq) 3416 flush_workqueue(rdev->dbq_wq); 3417 3418 cancel_work_sync(&rdev->dbq_fifo_check_work); 3419 cancel_delayed_work_sync(&rdev->dbq_pacing_work); 3420 3421 if (rdev->dbq_wq) { 3422 destroy_workqueue(rdev->dbq_wq); 3423 rdev->dbq_wq = NULL; 3424 } 3425 3426 if (rdev->dbr_page) 3427 free_page((u64)rdev->dbr_page); 3428 rdev->dbr_page = NULL; 3429 rdev->dbr_pacing = false; 3430 } 3431 3432 /* enable_dbr_pacing needs to be done only for older FWs 3433 * where host selects primary function. ie. pacing_ext 3434 * flags is not set. 3435 */ 3436 int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev) 3437 { 3438 struct bnxt_qplib_nq *nq; 3439 3440 nq = &rdev->nqr.nq[0]; 3441 rdev->dbq_nq_id = nq->ring_id; 3442 3443 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && 3444 bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { 3445 if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) { 3446 dev_err(rdev_to_dev(rdev), 3447 "Failed to set dbr pacing config\n"); 3448 return -EIO; 3449 } 3450 /* MAP grc window 8 for ARMing the NQ DBQ */ 3451 writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 3452 BNXT_HWRM_BAR_IDX, 3453 rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); 3454 rdev->dbr_aeq_arm_reg_off = 3455 (rdev->chip_ctx->dbr_aeq_arm_reg & 3456 BNXT_GRC_OFFSET_MASK) + 0x8000; 3457 writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 3458 BNXT_HWRM_BAR_IDX, 1); 3459 } 3460 3461 return 0; 3462 } 3463 3464 /* disable_dbr_pacing needs to be done only for older FWs 3465 * where host selects primary function. ie. pacing_ext 3466 * flags is not set. 3467 */ 3468 3469 int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev) 3470 { 3471 int rc = 0; 3472 3473 if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && 3474 bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) 3475 rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false); 3476 3477 return rc; 3478 } 3479 3480 static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) 3481 { 3482 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { 3483 bnxt_re_sysfs_destroy_file(rdev); 3484 /* Cleanup ib dev */ 3485 ib_unregister_device(&rdev->ibdev); 3486 clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 3487 return; 3488 } 3489 } 3490 3491 static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) 3492 { 3493 struct bnxt_qplib_dpi *kdpi; 3494 int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT; 3495 3496 bnxt_re_net_unregister_async_event(rdev); 3497 3498 bnxt_re_put_stats2_ctx(rdev); 3499 if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, 3500 &rdev->flags)) { 3501 /* did the caller hold the lock? */ 3502 mutex_lock(&bnxt_re_dev_lock); 3503 list_del_rcu(&rdev->list); 3504 mutex_unlock(&bnxt_re_dev_lock); 3505 } 3506 3507 bnxt_re_uninit_dcb_wq(rdev); 3508 bnxt_re_uninit_aer_wq(rdev); 3509 3510 bnxt_re_deinitialize_dbr_drop_recov(rdev); 3511 3512 if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) 3513 (void)bnxt_re_disable_dbr_pacing(rdev); 3514 3515 if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) { 3516 cancel_delayed_work_sync(&rdev->worker); 3517 } 3518 3519 /* Wait for ULPs to release references */ 3520 while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count) 3521 usleep_range(500, 1000); 3522 if (!wait_count) 3523 dev_err(rdev_to_dev(rdev), 3524 "CQ resources not freed by stack, count = 0x%x", 3525 atomic_read(&rdev->stats.rsors.cq_count)); 3526 3527 kdpi = &rdev->dpi_privileged; 3528 if (kdpi->umdbr) { /* kernel DPI was allocated with success */ 3529 (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi); 3530 /* 3531 * Driver just need to know no command had failed 3532 * during driver load sequence and below command is 3533 * required indeed. Piggybacking dpi allocation status. 3534 */ 3535 } 3536 3537 /* Protect the device uninitialization and start_irq/stop_irq L2 3538 * callbacks with rtnl lock to avoid race condition between these calls 3539 */ 3540 rtnl_lock(); 3541 if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags)) 3542 bnxt_re_clean_nqs(rdev); 3543 rtnl_unlock(); 3544 3545 if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags)) 3546 bnxt_re_free_tbls(rdev); 3547 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) { 3548 rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); 3549 if (rc) 3550 dev_warn(rdev_to_dev(rdev), 3551 "Failed to deinitialize fw, rc = 0x%x", rc); 3552 } 3553 3554 bnxt_re_put_stats_ctx(rdev); 3555 3556 if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags)) 3557 bnxt_qplib_free_hwctx(&rdev->qplib_res); 3558 3559 rtnl_lock(); 3560 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) 3561 bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); 3562 3563 if (rdev->dbr_pacing) 3564 bnxt_re_deinitialize_dbr_pacing(rdev); 3565 3566 bnxt_re_free_dbr_sw_stats_mem(rdev); 3567 3568 if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags)) 3569 bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id); 3570 3571 if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) 3572 bnxt_qplib_free_rcfw_channel(&rdev->qplib_res); 3573 3574 if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) 3575 bnxt_re_free_msix(rdev); 3576 rtnl_unlock(); 3577 3578 bnxt_re_destroy_chip_ctx(rdev); 3579 3580 if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { 3581 if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, 3582 &rdev->flags)) 3583 bnxt_re_unregister_netdev(rdev); 3584 } 3585 } 3586 3587 static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type) 3588 { 3589 struct bnxt_re_ring_attr rattr = {}; 3590 struct bnxt_qplib_creq_ctx *creq; 3591 int vec, offset; 3592 int rc = 0; 3593 3594 if (op_type != BNXT_RE_POST_RECOVERY_INIT) { 3595 /* Registered a new RoCE device instance to netdev */ 3596 rc = bnxt_re_register_netdev(rdev); 3597 if (rc) 3598 return -EINVAL; 3599 } 3600 set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 3601 3602 rc = bnxt_re_setup_chip_ctx(rdev); 3603 if (rc) { 3604 dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); 3605 bnxt_re_unregister_netdev(rdev); 3606 clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); 3607 rc = -EINVAL; 3608 return rc; 3609 } 3610 3611 /* Protect the device initialization and start_irq/stop_irq L2 callbacks 3612 * with rtnl lock to avoid race condition between these calls 3613 */ 3614 rtnl_lock(); 3615 rc = bnxt_re_request_msix(rdev); 3616 if (rc) { 3617 dev_err(rdev_to_dev(rdev), 3618 "Requesting MSI-X vectors failed with rc = 0x%x", rc); 3619 rc = -EINVAL; 3620 goto release_rtnl; 3621 } 3622 set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); 3623 3624 /* Establish RCFW Communication Channel to initialize the context 3625 memory for the function and all child VFs */ 3626 rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res); 3627 if (rc) { 3628 dev_err(rdev_to_dev(rdev), 3629 "Failed to alloc mem for rcfw, rc = %#x\n", rc); 3630 goto release_rtnl; 3631 } 3632 set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags); 3633 3634 creq = &rdev->rcfw.creq; 3635 rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr; 3636 rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; 3637 rattr.type = bnxt_re_get_rtype(rdev); 3638 rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; 3639 rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1; 3640 rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx; 3641 rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); 3642 if (rc) { 3643 creq->ring_id = 0xffff; 3644 dev_err(rdev_to_dev(rdev), 3645 "Failed to allocate CREQ fw id with rc = 0x%x", rc); 3646 goto release_rtnl; 3647 } 3648 3649 set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags); 3650 3651 if (!rdev->chip_ctx) 3652 goto release_rtnl; 3653 3654 if (!(_is_chip_p7(rdev->chip_ctx))) { 3655 /* Program the NQ ID for DBQ notification */ 3656 if (rdev->chip_ctx->modes.dbr_pacing_v0 || 3657 bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || 3658 bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { 3659 rc = bnxt_re_initialize_dbr_pacing(rdev); 3660 if (!rc) 3661 rdev->dbr_pacing = true; 3662 else 3663 rdev->dbr_pacing = false; 3664 dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", 3665 __func__, rc); 3666 } 3667 } 3668 3669 vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; 3670 offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset; 3671 rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset, 3672 &bnxt_re_aeq_handler); 3673 if (rc) { 3674 dev_err(rdev_to_dev(rdev), 3675 "Failed to enable RCFW channel with rc = 0x%x", rc); 3676 goto release_rtnl; 3677 } 3678 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); 3679 3680 rc = bnxt_re_update_dev_attr(rdev); 3681 if (rc) 3682 goto release_rtnl; 3683 bnxt_re_set_resource_limits(rdev); 3684 if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { 3685 rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res); 3686 if (rc) { 3687 dev_err(rdev_to_dev(rdev), 3688 "Failed to alloc hw contexts, rc = 0x%x", rc); 3689 goto release_rtnl; 3690 } 3691 set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags); 3692 } 3693 3694 rc = bnxt_re_get_stats_ctx(rdev); 3695 if (rc) 3696 goto release_rtnl; 3697 3698 rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn); 3699 if (rc) { 3700 dev_err(rdev_to_dev(rdev), 3701 "Failed to initialize fw with rc = 0x%x", rc); 3702 goto release_rtnl; 3703 } 3704 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags); 3705 3706 /* Based resource count on the 'new' device caps */ 3707 rc = bnxt_re_update_dev_attr(rdev); 3708 if (rc) 3709 goto release_rtnl; 3710 rc = bnxt_re_alloc_init_tbls(rdev); 3711 if (rc) { 3712 dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x", 3713 rc); 3714 goto release_rtnl; 3715 } 3716 rc = bnxt_re_setup_nqs(rdev); 3717 if (rc) { 3718 dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n", 3719 rc); 3720 if (rdev->nqr.max_init == 0) 3721 goto release_rtnl; 3722 3723 dev_warn(rdev_to_dev(rdev), 3724 "expected nqs %d available nqs %d\n", 3725 rdev->nqr.num_msix, rdev->nqr.max_init); 3726 } 3727 set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags); 3728 rtnl_unlock(); 3729 3730 rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged, 3731 rdev, BNXT_QPLIB_DPI_TYPE_KERNEL); 3732 if (rc) 3733 goto fail; 3734 3735 if (rdev->dbr_pacing) 3736 bnxt_re_enable_dbr_pacing(rdev); 3737 3738 if (rdev->chip_ctx->modes.dbr_drop_recov) 3739 bnxt_re_initialize_dbr_drop_recov(rdev); 3740 3741 rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev); 3742 if (rc) 3743 goto fail; 3744 3745 /* This block of code is needed for error recovery support */ 3746 if (!rdev->is_virtfn) { 3747 struct bnxt_re_tc_rec *tc_rec; 3748 3749 tc_rec = &rdev->tc_rec[0]; 3750 rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF); 3751 if (rc) { 3752 dev_err(rdev_to_dev(rdev), 3753 "Failed to query port config rc:%d", rc); 3754 return rc; 3755 } 3756 3757 /* Query f/w defaults of CC params */ 3758 rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param); 3759 if (rc) 3760 dev_warn(rdev_to_dev(rdev), 3761 "Failed to query CC defaults\n"); 3762 if (1) { 3763 rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev); 3764 if (rdev->num_vfs) { 3765 bnxt_re_set_resource_limits(rdev); 3766 bnxt_qplib_set_func_resources(&rdev->qplib_res); 3767 } 3768 } 3769 } 3770 INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); 3771 set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags); 3772 schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); 3773 3774 bnxt_re_init_dcb_wq(rdev); 3775 bnxt_re_init_aer_wq(rdev); 3776 mutex_lock(&bnxt_re_dev_lock); 3777 list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); 3778 /* Added to the list, not in progress anymore */ 3779 gadd_dev_inprogress--; 3780 set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags); 3781 mutex_unlock(&bnxt_re_dev_lock); 3782 3783 3784 return rc; 3785 release_rtnl: 3786 rtnl_unlock(); 3787 fail: 3788 bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); 3789 3790 return rc; 3791 } 3792 3793 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) 3794 { 3795 int rc = 0; 3796 3797 rc = bnxt_re_register_ib(rdev); 3798 if (rc) { 3799 dev_err(rdev_to_dev(rdev), 3800 "Register IB failed with rc = 0x%x", rc); 3801 goto fail; 3802 } 3803 if (bnxt_re_sysfs_create_file(rdev)) { 3804 bnxt_re_stopqps_and_ib_uninit(rdev); 3805 goto fail; 3806 } 3807 3808 set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); 3809 set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); 3810 set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); 3811 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); 3812 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); 3813 3814 return rc; 3815 fail: 3816 bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); 3817 return rc; 3818 } 3819 3820 /* wrapper for ib_init funcs */ 3821 int _bnxt_re_ib_init(struct bnxt_re_dev *rdev) 3822 { 3823 return bnxt_re_ib_init(rdev); 3824 } 3825 3826 /* wrapper for aux init funcs */ 3827 int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev) 3828 { 3829 bnxt_re_ib_init_2(rdev); 3830 return 0; /* add return for future proof */ 3831 } 3832 3833 static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) 3834 { 3835 bnxt_re_dev_dealloc(rdev); 3836 } 3837 3838 3839 static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, 3840 struct bnxt_en_dev *en_dev) 3841 { 3842 struct ifnet *realdev = NULL; 3843 3844 realdev = netdev; 3845 if (realdev) 3846 dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__, 3847 realdev, netdev); 3848 /* 3849 * Note: 3850 * The first argument to bnxt_re_dev_alloc() is 'netdev' and 3851 * not 'realdev', since in the case of bonding we want to 3852 * register the bonded virtual netdev (master) to the ib stack. 3853 * And 'en_dev' (for L2/PCI communication) is the first slave 3854 * device (PF0 on the card). 3855 * In the case of a regular netdev, both netdev and the en_dev 3856 * correspond to the same device. 3857 */ 3858 *rdev = bnxt_re_dev_alloc(netdev, en_dev); 3859 if (!*rdev) { 3860 pr_err("%s: netdev %p not handled", 3861 ROCE_DRV_MODULE_NAME, netdev); 3862 return -ENOMEM; 3863 } 3864 bnxt_re_hold(*rdev); 3865 3866 return 0; 3867 } 3868 3869 void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) 3870 { 3871 rdev->espeed = rdev->en_dev->espeed; 3872 rdev->lanes = rdev->en_dev->lanes; 3873 return; 3874 } 3875 3876 void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev) 3877 { 3878 dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n", 3879 __func__, rdev); 3880 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); 3881 bnxt_re_ib_uninit(rdev); 3882 } 3883 3884 void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, 3885 struct auxiliary_device *aux_dev) 3886 { 3887 struct bnxt_re_en_dev_info *en_info; 3888 struct bnxt_qplib_cmdq_ctx *cmdq; 3889 struct bnxt_qplib_rcfw *rcfw; 3890 3891 rcfw = &rdev->rcfw; 3892 cmdq = &rcfw->cmdq; 3893 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) 3894 set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); 3895 3896 dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev); 3897 bnxt_re_dev_uninit(rdev, op_type); 3898 en_info = auxiliary_get_drvdata(aux_dev); 3899 if (en_info) { 3900 rtnl_lock(); 3901 en_info->rdev = NULL; 3902 rtnl_unlock(); 3903 if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { 3904 clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags); 3905 clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags); 3906 clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); 3907 } 3908 } 3909 bnxt_re_dev_unreg(rdev); 3910 } 3911 3912 int bnxt_re_add_device(struct bnxt_re_dev **rdev, 3913 struct ifnet *netdev, 3914 u8 qp_mode, u8 op_type, 3915 u32 num_msix_requested, 3916 struct auxiliary_device *aux_dev) 3917 { 3918 struct bnxt_re_en_dev_info *en_info; 3919 struct bnxt_en_dev *en_dev; 3920 int rc = 0; 3921 3922 en_info = auxiliary_get_drvdata(aux_dev); 3923 en_dev = en_info->en_dev; 3924 3925 mutex_lock(&bnxt_re_dev_lock); 3926 /* Check if driver already in mod exit and aux_dev is valid */ 3927 if (gmod_exit || !aux_dev) { 3928 mutex_unlock(&bnxt_re_dev_lock); 3929 return -ENODEV; 3930 } 3931 /* Add device in progress */ 3932 gadd_dev_inprogress++; 3933 mutex_unlock(&bnxt_re_dev_lock); 3934 3935 rc = bnxt_re_dev_reg(rdev, netdev, en_dev); 3936 if (rc) { 3937 dev_dbg(NULL, "Failed to create add device for netdev %p\n", 3938 netdev); 3939 /* 3940 * For BNXT_RE_POST_RECOVERY_INIT special case 3941 * called from bnxt_re_start, the work is 3942 * complete only after, bnxt_re_start completes 3943 * bnxt_unregister_device in case of failure. 3944 * So bnxt_re_start will decrement gadd_dev_inprogress 3945 * in case of failure. 3946 */ 3947 if (op_type != BNXT_RE_POST_RECOVERY_INIT) { 3948 mutex_lock(&bnxt_re_dev_lock); 3949 gadd_dev_inprogress--; 3950 mutex_unlock(&bnxt_re_dev_lock); 3951 } 3952 return rc; 3953 } 3954 3955 if (rc != 0) 3956 goto ref_error; 3957 3958 /* 3959 * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load. 3960 * Otherwaise, this invocation can be the result of lag create / destroy, 3961 * err revovery, hot fw upgrade, etc.. 3962 */ 3963 if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) { 3964 if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES) 3965 num_msix_requested = max_msix_vec[bnxt_re_probe_count++]; 3966 else 3967 /* Consider as default when probe_count exceeds its limit */ 3968 num_msix_requested = 0; 3969 3970 /* if user specifies only one value, use the same for all PFs */ 3971 if (max_msix_vec_argc == 1) 3972 num_msix_requested = max_msix_vec[0]; 3973 } 3974 3975 (*rdev)->num_msix_requested = num_msix_requested; 3976 (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode; 3977 (*rdev)->adev = aux_dev; 3978 (*rdev)->dev_addr = en_dev->softc->func.mac_addr; 3979 /* Before updating the rdev pointer in bnxt_re_en_dev_info structure, 3980 * take the rtnl lock to avoid accessing invalid rdev pointer from 3981 * L2 ULP callbacks. This is applicable in all the places where rdev 3982 * pointer is updated in bnxt_re_en_dev_info. 3983 */ 3984 rtnl_lock(); 3985 en_info->rdev = *rdev; 3986 rtnl_unlock(); 3987 rc = bnxt_re_dev_init(*rdev, op_type); 3988 if (rc) { 3989 ref_error: 3990 bnxt_re_dev_unreg(*rdev); 3991 *rdev = NULL; 3992 /* 3993 * For BNXT_RE_POST_RECOVERY_INIT special case 3994 * called from bnxt_re_start, the work is 3995 * complete only after, bnxt_re_start completes 3996 * bnxt_unregister_device in case of failure. 3997 * So bnxt_re_start will decrement gadd_dev_inprogress 3998 * in case of failure. 3999 */ 4000 if (op_type != BNXT_RE_POST_RECOVERY_INIT) { 4001 mutex_lock(&bnxt_re_dev_lock); 4002 gadd_dev_inprogress--; 4003 mutex_unlock(&bnxt_re_dev_lock); 4004 } 4005 } 4006 dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev); 4007 if (!rc) { 4008 set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); 4009 } 4010 return rc; 4011 } 4012 4013 struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev) 4014 { 4015 struct pci_dev *pdev_in = rdev->en_dev->pdev; 4016 int tmp_bus_num, bus_num = pdev_in->bus->number; 4017 int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); 4018 int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn); 4019 struct bnxt_re_dev *tmp_rdev; 4020 4021 rcu_read_lock(); 4022 list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { 4023 tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; 4024 tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); 4025 tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn); 4026 4027 if (bus_num == tmp_bus_num && dev_num == tmp_dev_num && 4028 func_num != tmp_func_num) { 4029 rcu_read_unlock(); 4030 return tmp_rdev; 4031 } 4032 } 4033 rcu_read_unlock(); 4034 return NULL; 4035 } 4036 4037 4038 int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, 4039 struct ifnet *vlan_dev, 4040 struct ifnet *netdev, 4041 struct auxiliary_device *adev) 4042 { 4043 struct bnxt_re_work *re_work; 4044 4045 /* Allocate for the deferred task */ 4046 re_work = kzalloc(sizeof(*re_work), GFP_KERNEL); 4047 if (!re_work) 4048 return -ENOMEM; 4049 4050 re_work->rdev = rdev; 4051 re_work->event = event; 4052 re_work->vlan_dev = vlan_dev; 4053 re_work->adev = adev; 4054 INIT_WORK(&re_work->work, bnxt_re_task); 4055 if (rdev) 4056 atomic_inc(&rdev->sched_count); 4057 re_work->netdev = netdev; 4058 queue_work(bnxt_re_wq, &re_work->work); 4059 4060 return 0; 4061 } 4062 4063 4064 int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev) 4065 { 4066 struct pci_dev *pdev_in = rdev->en_dev->pdev; 4067 int tmp_bus_num, bus_num = pdev_in->bus->number; 4068 int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); 4069 struct bnxt_re_dev *tmp_rdev; 4070 int pf_cnt = 0; 4071 4072 rcu_read_lock(); 4073 list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { 4074 tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; 4075 tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); 4076 4077 if (bus_num == tmp_bus_num && dev_num == tmp_dev_num) 4078 pf_cnt++; 4079 } 4080 rcu_read_unlock(); 4081 return pf_cnt; 4082 } 4083 4084 /* Handle all deferred netevents tasks */ 4085 static void bnxt_re_task(struct work_struct *work) 4086 { 4087 struct bnxt_re_en_dev_info *en_info; 4088 struct auxiliary_device *aux_dev; 4089 struct bnxt_re_work *re_work; 4090 struct bnxt_re_dev *rdev; 4091 4092 re_work = container_of(work, struct bnxt_re_work, work); 4093 4094 mutex_lock(&bnxt_re_mutex); 4095 rdev = re_work->rdev; 4096 4097 /* 4098 * If the previous rdev is deleted due to bond creation 4099 * do not handle the event 4100 */ 4101 if (!bnxt_re_is_rdev_valid(rdev)) 4102 goto exit; 4103 4104 /* Ignore the event, if the device is not registred with IB stack. This 4105 * is to avoid handling any event while the device is added/removed. 4106 */ 4107 if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { 4108 dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx", 4109 __func__, re_work->event); 4110 goto done; 4111 } 4112 4113 /* Extra check to silence coverity. We shouldn't handle any event 4114 * when rdev is NULL. 4115 */ 4116 if (!rdev) 4117 goto exit; 4118 4119 dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx", 4120 re_work->event); 4121 4122 switch (re_work->event) { 4123 case NETDEV_UP: 4124 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4125 IB_EVENT_PORT_ACTIVE); 4126 bnxt_re_net_register_async_event(rdev); 4127 break; 4128 4129 case NETDEV_DOWN: 4130 bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); 4131 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); 4132 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4133 IB_EVENT_PORT_ERR); 4134 break; 4135 4136 case NETDEV_CHANGE: 4137 if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) { 4138 bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); 4139 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4140 IB_EVENT_PORT_ERR); 4141 break; 4142 } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) { 4143 bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, 4144 IB_EVENT_PORT_ACTIVE); 4145 } 4146 4147 /* temporarily disable the check for SR2 */ 4148 if (!bnxt_qplib_query_cc_param(&rdev->qplib_res, 4149 &rdev->cc_param) && 4150 !_is_chip_p7(rdev->chip_ctx)) { 4151 /* 4152 * Disable CC for 10G speed 4153 * for non p5 devices 4154 */ 4155 if (rdev->sl_espeed == SPEED_10000 && 4156 !_is_chip_gen_p5_p7(rdev->chip_ctx)) { 4157 if (rdev->cc_param.enable) 4158 bnxt_re_clear_cc(rdev); 4159 } else { 4160 if (!rdev->cc_param.enable && 4161 rdev->cc_param.admin_enable) 4162 bnxt_re_setup_cc(rdev); 4163 } 4164 } 4165 break; 4166 4167 case NETDEV_UNREGISTER: 4168 bnxt_re_stopqps_and_ib_uninit(rdev); 4169 aux_dev = rdev->adev; 4170 if (re_work->adev) 4171 goto done; 4172 4173 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev); 4174 4175 break; 4176 4177 default: 4178 break; 4179 } 4180 done: 4181 if (rdev) { 4182 /* memory barrier to guarantee task completion 4183 * before decrementing sched count 4184 */ 4185 mmiowb(); 4186 atomic_dec(&rdev->sched_count); 4187 } 4188 exit: 4189 if (re_work->adev && re_work->event == NETDEV_UNREGISTER) { 4190 en_info = auxiliary_get_drvdata(re_work->adev); 4191 en_info->ib_uninit_done = true; 4192 wake_up(&en_info->waitq); 4193 } 4194 kfree(re_work); 4195 mutex_unlock(&bnxt_re_mutex); 4196 } 4197 4198 /* 4199 "Notifier chain callback can be invoked for the same chain from 4200 different CPUs at the same time". 4201 4202 For cases when the netdev is already present, our call to the 4203 register_netdevice_notifier() will actually get the rtnl_lock() 4204 before sending NETDEV_REGISTER and (if up) NETDEV_UP 4205 events. 4206 4207 But for cases when the netdev is not already present, the notifier 4208 chain is subjected to be invoked from different CPUs simultaneously. 4209 4210 This is protected by the netdev_mutex. 4211 */ 4212 static int bnxt_re_netdev_event(struct notifier_block *notifier, 4213 unsigned long event, void *ptr) 4214 { 4215 struct ifnet *real_dev, *netdev; 4216 struct bnxt_re_dev *rdev = NULL; 4217 4218 netdev = netdev_notifier_info_to_ifp(ptr); 4219 real_dev = rdma_vlan_dev_real_dev(netdev); 4220 if (!real_dev) 4221 real_dev = netdev; 4222 /* In case of bonding,this will be bond's rdev */ 4223 rdev = bnxt_re_from_netdev(real_dev); 4224 4225 if (!rdev) 4226 goto exit; 4227 4228 dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n", 4229 __func__, bnxt_re_netevent(event), event, 4230 rdev ? rdev->netdev ? if_getdname(rdev->netdev) : "->netdev = NULL" : "= NULL", 4231 (real_dev == netdev) ? "= netdev" : if_getdname(real_dev)); 4232 4233 if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) 4234 goto exit; 4235 4236 bnxt_re_hold(rdev); 4237 4238 if (real_dev != netdev) { 4239 switch (event) { 4240 case NETDEV_UP: 4241 bnxt_re_schedule_work(rdev, event, netdev, 4242 NULL, NULL); 4243 break; 4244 case NETDEV_DOWN: 4245 break; 4246 default: 4247 break; 4248 } 4249 goto done; 4250 } 4251 4252 switch (event) { 4253 case NETDEV_CHANGEADDR: 4254 if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) 4255 bnxt_re_update_shadow_ah(rdev); 4256 bnxt_qplib_get_guid(rdev->dev_addr, 4257 (u8 *)&rdev->ibdev.node_guid); 4258 break; 4259 4260 case NETDEV_CHANGE: 4261 bnxt_re_get_link_speed(rdev); 4262 bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL); 4263 break; 4264 case NETDEV_UNREGISTER: 4265 /* netdev notifier will call NETDEV_UNREGISTER again later since 4266 * we are still holding the reference to the netdev 4267 */ 4268 4269 /* 4270 * Workaround to avoid ib_unregister hang. Check for module 4271 * reference and dont free up the device if the reference 4272 * is non zero. Checking only for PF functions. 4273 */ 4274 4275 if (rdev) { 4276 dev_info(rdev_to_dev(rdev), 4277 "bnxt_re:Unreg recvd when module refcnt > 0"); 4278 dev_info(rdev_to_dev(rdev), 4279 "bnxt_re:Close all apps using bnxt_re devs"); 4280 dev_info(rdev_to_dev(rdev), 4281 "bnxt_re:Remove the configfs entry created for the device"); 4282 dev_info(rdev_to_dev(rdev), 4283 "bnxt_re:Refer documentation for details"); 4284 goto done; 4285 } 4286 4287 if (atomic_read(&rdev->sched_count) > 0) 4288 goto done; 4289 if (!rdev->unreg_sched) { 4290 bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, 4291 NULL, NULL, NULL); 4292 rdev->unreg_sched = true; 4293 goto done; 4294 } 4295 4296 break; 4297 default: 4298 break; 4299 } 4300 done: 4301 if (rdev) 4302 bnxt_re_put(rdev); 4303 exit: 4304 return NOTIFY_DONE; 4305 } 4306 4307 static struct notifier_block bnxt_re_netdev_notifier = { 4308 .notifier_call = bnxt_re_netdev_event 4309 }; 4310 4311 static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev, 4312 struct auxiliary_device *adev) 4313 { 4314 bnxt_re_stopqps_and_ib_uninit(rdev); 4315 bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev); 4316 auxiliary_set_drvdata(adev, NULL); 4317 } 4318 4319 /* 4320 * bnxt_re_remove - Removes the roce aux device 4321 * @adev - aux device pointer 4322 * 4323 * This function removes the roce device. This gets 4324 * called in the mod exit path and pci unbind path. 4325 * If the rdev is bond interace, destroys the lag 4326 * in module exit path, and in pci unbind case 4327 * destroys the lag and recreates other base interface. 4328 * If the device is already removed in error recovery 4329 * path, it just unregister with the L2. 4330 */ 4331 static void bnxt_re_remove(struct auxiliary_device *adev) 4332 { 4333 struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); 4334 struct bnxt_en_dev *en_dev; 4335 struct bnxt_re_dev *rdev; 4336 bool primary_dev = false; 4337 bool secondary_dev = false; 4338 4339 if (!en_info) 4340 return; 4341 4342 mutex_lock(&bnxt_re_mutex); 4343 en_dev = en_info->en_dev; 4344 4345 rdev = en_info->rdev; 4346 4347 if (rdev && bnxt_re_is_rdev_valid(rdev)) { 4348 if (pci_channel_offline(rdev->rcfw.pdev)) 4349 set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); 4350 4351 if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags)) 4352 primary_dev = true; 4353 if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags)) 4354 secondary_dev = true; 4355 4356 /* 4357 * en_dev_info of primary device and secondary device have the 4358 * same rdev pointer when LAG is configured. This rdev pointer 4359 * is rdev of bond interface. 4360 */ 4361 if (!primary_dev && !secondary_dev) { 4362 /* removal of non bond interface */ 4363 bnxt_re_remove_base_interface(rdev, adev); 4364 } else { 4365 /* 4366 * removal of bond primary/secondary interface. In this 4367 * case bond device is already removed, so rdev->binfo 4368 * is NULL. 4369 */ 4370 auxiliary_set_drvdata(adev, NULL); 4371 } 4372 } else { 4373 /* device is removed from ulp stop, unregister the net dev */ 4374 if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) { 4375 rtnl_lock(); 4376 en_dev->en_ops->bnxt_unregister_device(en_dev, 4377 BNXT_ROCE_ULP); 4378 rtnl_unlock(); 4379 } 4380 } 4381 mutex_unlock(&bnxt_re_mutex); 4382 return; 4383 } 4384 4385 /* wrapper for all external user context callers */ 4386 void _bnxt_re_remove(struct auxiliary_device *adev) 4387 { 4388 bnxt_re_remove(adev); 4389 } 4390 4391 static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev) 4392 { 4393 int rc; 4394 4395 rc = bnxt_re_get_device_stats(rdev); 4396 if (rc) 4397 dev_err(rdev_to_dev(rdev), 4398 "Failed initial device stat query"); 4399 4400 bnxt_re_net_register_async_event(rdev); 4401 } 4402 4403 static int bnxt_re_probe(struct auxiliary_device *adev, 4404 const struct auxiliary_device_id *id) 4405 { 4406 struct bnxt_aux_dev *aux_dev = 4407 container_of(adev, struct bnxt_aux_dev, aux_dev); 4408 struct bnxt_re_en_dev_info *en_info; 4409 struct bnxt_en_dev *en_dev = NULL; 4410 struct bnxt_re_dev *rdev; 4411 int rc = -ENODEV; 4412 4413 if (aux_dev) 4414 en_dev = aux_dev->edev; 4415 4416 if (!en_dev) 4417 return rc; 4418 4419 if (en_dev->ulp_version != BNXT_ULP_VERSION) { 4420 pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n", 4421 ROCE_DRV_MODULE_NAME, en_dev->ulp_version); 4422 return -EINVAL; 4423 } 4424 4425 en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); 4426 if (!en_info) 4427 return -ENOMEM; 4428 memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info)); 4429 en_info->en_dev = en_dev; 4430 auxiliary_set_drvdata(adev, en_info); 4431 4432 mutex_lock(&bnxt_re_mutex); 4433 rc = bnxt_re_add_device(&rdev, en_dev->net, 4434 BNXT_RE_GSI_MODE_ALL, 4435 BNXT_RE_COMPLETE_INIT, 4436 BNXT_RE_MSIX_FROM_MOD_PARAM, adev); 4437 if (rc) { 4438 mutex_unlock(&bnxt_re_mutex); 4439 return rc; 4440 } 4441 4442 rc = bnxt_re_ib_init(rdev); 4443 if (rc) 4444 goto err; 4445 4446 bnxt_re_ib_init_2(rdev); 4447 4448 dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev); 4449 rdev->adev = adev; 4450 4451 mutex_unlock(&bnxt_re_mutex); 4452 4453 return 0; 4454 4455 err: 4456 mutex_unlock(&bnxt_re_mutex); 4457 bnxt_re_remove(adev); 4458 4459 return rc; 4460 } 4461 4462 static const struct auxiliary_device_id bnxt_re_id_table[] = { 4463 { .name = BNXT_ADEV_NAME ".rdma", }, 4464 {}, 4465 }; 4466 4467 MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table); 4468 4469 static struct auxiliary_driver bnxt_re_driver = { 4470 .name = "rdma", 4471 .probe = bnxt_re_probe, 4472 .remove = bnxt_re_remove, 4473 .id_table = bnxt_re_id_table, 4474 }; 4475 4476 static int __init bnxt_re_mod_init(void) 4477 { 4478 int rc = 0; 4479 4480 pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version); 4481 4482 bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); 4483 if (!bnxt_re_wq) 4484 return -ENOMEM; 4485 4486 rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier); 4487 if (rc) { 4488 pr_err("%s: Cannot register to netdevice_notifier", 4489 ROCE_DRV_MODULE_NAME); 4490 goto err_netdev; 4491 } 4492 4493 INIT_LIST_HEAD(&bnxt_re_dev_list); 4494 4495 rc = auxiliary_driver_register(&bnxt_re_driver); 4496 if (rc) { 4497 pr_err("%s: Failed to register auxiliary driver\n", 4498 ROCE_DRV_MODULE_NAME); 4499 goto err_auxdrv; 4500 } 4501 4502 return 0; 4503 4504 err_auxdrv: 4505 bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); 4506 4507 err_netdev: 4508 destroy_workqueue(bnxt_re_wq); 4509 4510 return rc; 4511 } 4512 4513 static void __exit bnxt_re_mod_exit(void) 4514 { 4515 gmod_exit = 1; 4516 auxiliary_driver_unregister(&bnxt_re_driver); 4517 4518 bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); 4519 4520 if (bnxt_re_wq) 4521 destroy_workqueue(bnxt_re_wq); 4522 } 4523 4524 module_init(bnxt_re_mod_init); 4525 module_exit(bnxt_re_mod_exit); 4526