1 /*- 2 * Broadcom NetXtreme-C/E network driver. 3 * 4 * Copyright (c) 2024 Broadcom, All Rights Reserved. 5 * The term Broadcom refers to Broadcom Limited and/or its subsidiaries 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 * THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <linux/module.h> 30 #include <linux/kernel.h> 31 #include <linux/errno.h> 32 #include <linux/interrupt.h> 33 #include <linux/pci.h> 34 #include <linux/netdevice.h> 35 #include <linux/bitops.h> 36 #include <linux/delay.h> 37 #include <asm/byteorder.h> 38 #include <linux/bitmap.h> 39 #include <linux/rcupdate.h> 40 #include <net/if.h> 41 #include <net/if_dl.h> 42 #include <net/if_media.h> 43 #include <net/if_var.h> 44 #include <net/ethernet.h> 45 #include <net/iflib.h> 46 47 #include "hsi_struct_def.h" 48 #include "bnxt.h" 49 #include "bnxt_hwrm.h" 50 #include "bnxt_ulp.h" 51 #include "bnxt_log.h" 52 53 void bnxt_destroy_irq(struct bnxt_softc *softc); 54 55 static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id, 56 struct bnxt_ulp_ops *ulp_ops, void *handle) 57 { 58 struct bnxt_softc *bp = edev->softc; 59 struct bnxt_ulp *ulp; 60 int rc = 0; 61 62 if (ulp_id >= BNXT_MAX_ULP) 63 return -EINVAL; 64 65 mtx_lock(&bp->en_ops_lock); 66 ulp = &edev->ulp_tbl[ulp_id]; 67 if (rcu_access_pointer(ulp->ulp_ops)) { 68 device_printf(bp->dev, "ulp id %d already registered\n", ulp_id); 69 rc = -EBUSY; 70 goto exit; 71 } 72 73 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 74 atomic_set(&ulp->ref_count, 0); 75 ulp->handle = handle; 76 rcu_assign_pointer(ulp->ulp_ops, ulp_ops); 77 78 if (ulp_id == BNXT_ROCE_ULP) { 79 if (test_bit(BNXT_STATE_OPEN, &bp->state) && bp->is_dev_init) 80 bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info); 81 } 82 83 exit: 84 mtx_unlock(&bp->en_ops_lock); 85 return rc; 86 } 87 88 static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id) 89 { 90 struct bnxt_softc *bp = edev->softc; 91 struct bnxt_ulp *ulp; 92 int i = 0; 93 94 if (ulp_id >= BNXT_MAX_ULP) 95 return -EINVAL; 96 97 ulp = &edev->ulp_tbl[ulp_id]; 98 if (!rcu_access_pointer(ulp->ulp_ops)) { 99 device_printf(bp->dev, "ulp id %d not registered\n", ulp_id); 100 return -EINVAL; 101 } 102 if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested) 103 edev->en_ops->bnxt_free_msix(edev, ulp_id); 104 105 mtx_lock(&bp->en_ops_lock); 106 RCU_INIT_POINTER(ulp->ulp_ops, NULL); 107 synchronize_rcu(); 108 ulp->max_async_event_id = 0; 109 ulp->async_events_bmap = NULL; 110 while (atomic_read(&ulp->ref_count) != 0 && i < 10) { 111 msleep(100); 112 i++; 113 } 114 mtx_unlock(&bp->en_ops_lock); 115 return 0; 116 } 117 118 static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *ent) 119 { 120 struct bnxt_en_dev *edev = bp->edev; 121 int num_msix, idx, i; 122 123 num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; 124 idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; 125 for (i = 0; i < num_msix; i++) { 126 ent[i].vector = bp->irq_tbl[idx + i].vector; 127 ent[i].ring_idx = idx + i; 128 if (BNXT_CHIP_P5_PLUS(bp)) 129 ent[i].db_offset = bp->db_offset; 130 else 131 ent[i].db_offset = (idx + i) * 0x80; 132 133 } 134 } 135 136 static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, 137 struct bnxt_msix_entry *ent, int num_msix) 138 { 139 struct bnxt_softc *bp = edev->softc; 140 int avail_msix, idx; 141 142 if (ulp_id != BNXT_ROCE_ULP) 143 return -EINVAL; 144 145 if (edev->ulp_tbl[ulp_id].msix_requested) 146 return -EAGAIN; 147 148 idx = bp->total_irqs - BNXT_ROCE_IRQ_COUNT; 149 avail_msix = BNXT_ROCE_IRQ_COUNT; 150 151 mtx_lock(&bp->en_ops_lock); 152 edev->ulp_tbl[ulp_id].msix_base = idx; 153 edev->ulp_tbl[ulp_id].msix_requested = avail_msix; 154 155 bnxt_fill_msix_vecs(bp, ent); 156 edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED; 157 mtx_unlock(&bp->en_ops_lock); 158 return avail_msix; 159 } 160 161 static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id) 162 { 163 struct bnxt_softc *bp = edev->softc; 164 165 if (ulp_id != BNXT_ROCE_ULP) 166 return -EINVAL; 167 168 if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) 169 return 0; 170 171 mtx_lock(&bp->en_ops_lock); 172 edev->ulp_tbl[ulp_id].msix_requested = 0; 173 edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED; 174 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) 175 goto stopped; 176 177 stopped: 178 mtx_unlock(&bp->en_ops_lock); 179 180 return 0; 181 } 182 183 int bnxt_get_ulp_msix_num(struct bnxt_softc *bp) 184 { 185 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 186 struct bnxt_en_dev *edev = bp->edev; 187 188 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; 189 } 190 return 0; 191 } 192 193 int bnxt_get_ulp_msix_base(struct bnxt_softc *bp) 194 { 195 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 196 struct bnxt_en_dev *edev = bp->edev; 197 198 if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested) 199 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; 200 } 201 return 0; 202 } 203 204 static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id, 205 struct bnxt_fw_msg *fw_msg) 206 { 207 struct bnxt_softc *softc = edev->softc; 208 int rc; 209 210 if ((ulp_id != BNXT_ROCE_ULP) && softc->fw_reset_state) 211 return -EBUSY; 212 213 rc = bnxt_hwrm_passthrough(softc, fw_msg->msg, fw_msg->msg_len, fw_msg->resp, 214 fw_msg->resp_max_len, fw_msg->timeout); 215 return rc; 216 } 217 218 static void bnxt_ulp_get(struct bnxt_ulp *ulp) 219 { 220 atomic_inc(&ulp->ref_count); 221 } 222 223 static void bnxt_ulp_put(struct bnxt_ulp *ulp) 224 { 225 atomic_dec(&ulp->ref_count); 226 } 227 228 void bnxt_ulp_stop(struct bnxt_softc *bp) 229 { 230 struct bnxt_en_dev *edev = bp->edev; 231 struct bnxt_ulp_ops *ops; 232 int i; 233 234 if (!edev) 235 return; 236 237 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; 238 edev->en_state = bp->state; 239 for (i = 0; i < BNXT_MAX_ULP; i++) { 240 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 241 242 ops = ulp->ulp_ops; 243 if (!ops || !ops->ulp_stop) 244 continue; 245 ops->ulp_stop(ulp->handle); 246 } 247 } 248 249 void bnxt_ulp_start(struct bnxt_softc *bp, int err) 250 { 251 struct bnxt_en_dev *edev = bp->edev; 252 struct bnxt_ulp_ops *ops; 253 int i; 254 255 if (!edev) 256 return; 257 258 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 259 edev->en_state = bp->state; 260 261 if (err) 262 return; 263 264 for (i = 0; i < BNXT_MAX_ULP; i++) { 265 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 266 267 ops = ulp->ulp_ops; 268 if (!ops || !ops->ulp_start) 269 continue; 270 ops->ulp_start(ulp->handle); 271 } 272 } 273 274 void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs) 275 { 276 struct bnxt_en_dev *edev = bp->edev; 277 struct bnxt_ulp_ops *ops; 278 int i; 279 280 if (!edev) 281 return; 282 283 for (i = 0; i < BNXT_MAX_ULP; i++) { 284 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 285 286 rcu_read_lock(); 287 ops = rcu_dereference(ulp->ulp_ops); 288 if (!ops || !ops->ulp_sriov_config) { 289 rcu_read_unlock(); 290 continue; 291 } 292 bnxt_ulp_get(ulp); 293 rcu_read_unlock(); 294 ops->ulp_sriov_config(ulp->handle, num_vfs); 295 bnxt_ulp_put(ulp); 296 } 297 } 298 299 void bnxt_ulp_shutdown(struct bnxt_softc *bp) 300 { 301 struct bnxt_en_dev *edev = bp->edev; 302 struct bnxt_ulp_ops *ops; 303 int i; 304 305 if (!edev) 306 return; 307 308 for (i = 0; i < BNXT_MAX_ULP; i++) { 309 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 310 311 ops = ulp->ulp_ops; 312 if (!ops || !ops->ulp_shutdown) 313 continue; 314 ops->ulp_shutdown(ulp->handle); 315 } 316 } 317 318 void bnxt_ulp_irq_stop(struct bnxt_softc *bp) 319 { 320 struct bnxt_en_dev *edev = bp->edev; 321 struct bnxt_ulp_ops *ops; 322 323 if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) 324 return; 325 326 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 327 struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP]; 328 329 if (!ulp->msix_requested) 330 return; 331 332 ops = ulp->ulp_ops; 333 if (!ops || !ops->ulp_irq_stop) 334 return; 335 ops->ulp_irq_stop(ulp->handle); 336 } 337 } 338 339 void 340 bnxt_logger_ulp_live_data(void *d) 341 { 342 struct bnxt_en_dev *edev; 343 struct bnxt_softc *bp; 344 345 bp = d; 346 edev = bp->edev; 347 348 if (!edev) 349 return; 350 351 if (bnxt_ulp_registered(edev, BNXT_ROCE_ULP)) { 352 struct bnxt_ulp_ops *ops; 353 struct bnxt_ulp *ulp; 354 355 ulp = edev->ulp_tbl; 356 //ops = rtnl_dereference(ulp->ulp_ops); 357 ops = ulp->ulp_ops; 358 if (!ops || !ops->ulp_log_live) 359 return; 360 361 ops->ulp_log_live(ulp->handle); 362 } 363 } 364 365 void 366 bnxt_ulp_log_live(struct bnxt_en_dev *edev, u16 logger_id, 367 const char *format, ...) 368 { 369 bnxt_log_live(if_getsoftc(edev->net), logger_id, format); 370 } 371 EXPORT_SYMBOL(bnxt_ulp_log_live); 372 373 void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl) 374 { 375 u16 event_id = le16_to_cpu(cmpl->event_id); 376 struct bnxt_en_dev *edev = bp->edev; 377 struct bnxt_ulp_ops *ops; 378 int i; 379 380 if (!edev) 381 return; 382 383 rcu_read_lock(); 384 for (i = 0; i < BNXT_MAX_ULP; i++) { 385 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 386 387 ops = rcu_dereference(ulp->ulp_ops); 388 if (!ops || !ops->ulp_async_notifier) 389 continue; 390 if (!ulp->async_events_bmap || 391 event_id > ulp->max_async_event_id) 392 continue; 393 394 /* Read max_async_event_id first before testing the bitmap. */ 395 rmb(); 396 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) 397 continue; 398 399 if (test_bit(event_id, ulp->async_events_bmap)) 400 ops->ulp_async_notifier(ulp->handle, cmpl); 401 } 402 rcu_read_unlock(); 403 } 404 405 static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id, 406 unsigned long *events_bmap, u16 max_id) 407 { 408 struct bnxt_softc *bp = edev->softc; 409 struct bnxt_ulp *ulp; 410 411 if (ulp_id >= BNXT_MAX_ULP) 412 return -EINVAL; 413 414 mtx_lock(&bp->en_ops_lock); 415 ulp = &edev->ulp_tbl[ulp_id]; 416 ulp->async_events_bmap = events_bmap; 417 wmb(); 418 ulp->max_async_event_id = max_id; 419 bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true); 420 mtx_unlock(&bp->en_ops_lock); 421 return 0; 422 } 423 424 void bnxt_destroy_irq(struct bnxt_softc *softc) 425 { 426 kfree(softc->irq_tbl); 427 } 428 429 static int bnxt_populate_irq(struct bnxt_softc *softc) 430 { 431 struct resource_list *rl = NULL; 432 struct resource_list_entry *rle = NULL; 433 struct bnxt_msix_tbl *irq_tbl = NULL; 434 struct pci_devinfo *dinfo = NULL; 435 int i; 436 437 softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT; 438 irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); 439 440 if (!irq_tbl) { 441 device_printf(softc->dev, "Failed to allocate IRQ table\n"); 442 return -1; 443 } 444 dinfo = device_get_ivars(softc->pdev->dev.bsddev); 445 rl = &dinfo->resources; 446 rle = resource_list_find(rl, SYS_RES_IRQ, 1); 447 softc->pdev->dev.irq_start = rle->start; 448 softc->pdev->dev.irq_end = rle->start + softc->total_irqs; 449 450 for (i = 0; i < softc->total_irqs; i++) { 451 irq_tbl[i].entry = i; 452 irq_tbl[i].vector = softc->pdev->dev.irq_start + i; 453 } 454 455 softc->irq_tbl = irq_tbl; 456 457 return 0; 458 } 459 460 static const struct bnxt_en_ops bnxt_en_ops_tbl = { 461 .bnxt_register_device = bnxt_register_dev, 462 .bnxt_unregister_device = bnxt_unregister_dev, 463 .bnxt_request_msix = bnxt_req_msix_vecs, 464 .bnxt_free_msix = bnxt_free_msix_vecs, 465 .bnxt_send_fw_msg = bnxt_send_msg, 466 .bnxt_register_fw_async_events = bnxt_register_async_events, 467 }; 468 469 void bnxt_aux_dev_release(struct device *dev) 470 { 471 struct bnxt_aux_dev *bnxt_adev = 472 container_of(dev, struct bnxt_aux_dev, aux_dev.dev); 473 struct bnxt_softc *bp = bnxt_adev->edev->softc; 474 475 kfree(bnxt_adev->edev); 476 bnxt_adev->edev = NULL; 477 bp->edev = NULL; 478 } 479 480 static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_softc *bp) 481 { 482 edev->en_ops = &bnxt_en_ops_tbl; 483 edev->net = bp->ifp; 484 edev->pdev = bp->pdev; 485 edev->softc = bp; 486 edev->l2_db_size = bp->db_size; 487 edev->l2_db_offset = bp->db_offset; 488 mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); 489 490 if (bp->flags & BNXT_FLAG_ROCEV1_CAP) 491 edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; 492 if (bp->flags & BNXT_FLAG_ROCEV2_CAP) 493 edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; 494 if (bp->is_asym_q) 495 edev->flags |= BNXT_EN_FLAG_ASYM_Q; 496 if (BNXT_SW_RES_LMT(bp)) 497 edev->flags |= BNXT_EN_FLAG_SW_RES_LMT; 498 edev->hwrm_bar = bp->hwrm_bar; 499 edev->port_partition_type = bp->port_partition_type; 500 edev->ulp_version = BNXT_ULP_VERSION; 501 memcpy(edev->board_part_number, bp->board_partno, BNXT_VPD_PN_FLD_LEN - 1); 502 } 503 504 int bnxt_rdma_aux_device_del(struct bnxt_softc *softc) 505 { 506 struct bnxt_aux_dev *bnxt_adev = softc->aux_dev; 507 struct auxiliary_device *adev; 508 509 adev = &bnxt_adev->aux_dev; 510 auxiliary_device_delete(adev); 511 auxiliary_device_uninit(adev); 512 bnxt_destroy_irq(softc); 513 514 return 0; 515 } 516 517 int bnxt_rdma_aux_device_add(struct bnxt_softc *bp) 518 { 519 struct bnxt_aux_dev *bnxt_adev = bp->aux_dev; 520 struct bnxt_en_dev *edev = bnxt_adev->edev; 521 struct auxiliary_device *aux_dev; 522 int ret = -1; 523 524 if (bnxt_populate_irq(bp)) 525 return ret; 526 527 device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, " 528 "devfn 0x%x, cla 0x%x, rev 0x%x, msi_en 0x%x\n", 529 bp->pdev->vendor, bp->pdev->device, bp->pdev->subsystem_vendor, 530 bp->pdev->subsystem_device, bp->pdev->irq, bp->pdev->devfn, 531 bp->pdev->class, bp->pdev->revision, bp->pdev->msi_enabled); 532 533 aux_dev = &bnxt_adev->aux_dev; 534 aux_dev->id = bnxt_adev->id; 535 aux_dev->name = "rdma"; 536 aux_dev->dev.parent = &bp->pdev->dev; 537 aux_dev->dev.release = bnxt_aux_dev_release; 538 539 if (!edev) { 540 edev = kzalloc(sizeof(*edev), GFP_KERNEL); 541 if (!edev) 542 return -ENOMEM; 543 } 544 545 bnxt_set_edev_info(edev, bp); 546 bnxt_adev->edev = edev; 547 bp->edev = edev; 548 549 ret = auxiliary_device_init(aux_dev); 550 if (ret) 551 goto err_free_edev; 552 553 ret = auxiliary_device_add(aux_dev); 554 if (ret) 555 goto err_dev_uninit; 556 557 return 0; 558 err_dev_uninit: 559 auxiliary_device_uninit(aux_dev); 560 err_free_edev: 561 kfree(edev); 562 bnxt_adev->edev = NULL; 563 bp->edev = NULL; 564 return ret; 565 } 566