1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/init.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/rtnetlink.h> 8 #include <linux/types.h> 9 #include <net/devlink.h> 10 11 #include "fbnic.h" 12 #include "fbnic_drvinfo.h" 13 #include "fbnic_hw_stats.h" 14 #include "fbnic_netdev.h" 15 16 char fbnic_driver_name[] = DRV_NAME; 17 18 MODULE_DESCRIPTION(DRV_SUMMARY); 19 MODULE_LICENSE("GPL"); 20 21 static const struct fbnic_info fbnic_asic_info = { 22 .max_num_queues = FBNIC_MAX_QUEUES, 23 .bar_mask = BIT(0) | BIT(4) 24 }; 25 26 static const struct fbnic_info *fbnic_info_tbl[] = { 27 [fbnic_board_asic] = &fbnic_asic_info, 28 }; 29 30 static const struct pci_device_id fbnic_pci_tbl[] = { 31 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) }, 32 /* Required last entry */ 33 {0, } 34 }; 35 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl); 36 37 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg) 38 { 39 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0); 40 u32 value; 41 42 if (!csr) 43 return ~0U; 44 45 value = readl(csr + reg); 46 47 /* If any bits are 0 value should be valid */ 48 if (~value) 49 return value; 50 51 /* All 1's may be valid if ZEROs register still works */ 52 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0)) 53 return value; 54 55 /* Hardware is giving us all 1's reads, assume it is gone */ 56 WRITE_ONCE(fbd->uc_addr0, NULL); 57 WRITE_ONCE(fbd->uc_addr4, NULL); 58 59 dev_err(fbd->dev, 60 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 61 reg, reg << 2); 62 63 /* Notify stack that device has lost (PCIe) link */ 64 if (!fbnic_init_failure(fbd)) 65 netif_device_detach(fbd->netdev); 66 67 return ~0U; 68 } 69 70 bool fbnic_fw_present(struct fbnic_dev *fbd) 71 { 72 return !!READ_ONCE(fbd->uc_addr4); 73 } 74 75 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val) 76 { 77 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 78 79 if (csr) 80 writel(val, csr + reg); 81 } 82 83 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg) 84 { 85 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 86 u32 value; 87 88 if (!csr) 89 return ~0U; 90 91 value = readl(csr + reg); 92 93 /* If any bits are 0 value should be valid */ 94 if (~value) 95 return value; 96 97 /* All 1's may be valid if ZEROs register still works */ 98 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG)) 99 return value; 100 101 /* Hardware is giving us all 1's reads, assume it is gone */ 102 WRITE_ONCE(fbd->uc_addr0, NULL); 103 WRITE_ONCE(fbd->uc_addr4, NULL); 104 105 dev_err(fbd->dev, 106 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 107 reg, reg << 2); 108 109 /* Notify stack that device has lost (PCIe) link */ 110 if (!fbnic_init_failure(fbd)) 111 netif_device_detach(fbd->netdev); 112 113 return ~0U; 114 } 115 116 static void fbnic_service_task_start(struct fbnic_net *fbn) 117 { 118 struct fbnic_dev *fbd = fbn->fbd; 119 120 schedule_delayed_work(&fbd->service_task, HZ); 121 } 122 123 static void fbnic_service_task_stop(struct fbnic_net *fbn) 124 { 125 struct fbnic_dev *fbd = fbn->fbd; 126 127 cancel_delayed_work(&fbd->service_task); 128 } 129 130 void fbnic_up(struct fbnic_net *fbn) 131 { 132 fbnic_enable(fbn); 133 134 fbnic_fill(fbn); 135 136 fbnic_rss_reinit_hw(fbn->fbd, fbn); 137 138 netif_addr_lock_bh(fbn->netdev); 139 __fbnic_set_rx_mode(fbn->fbd, &fbn->netdev->uc, &fbn->netdev->mc); 140 netif_addr_unlock_bh(fbn->netdev); 141 142 /* Enable Tx/Rx processing */ 143 fbnic_napi_enable(fbn); 144 netif_tx_wake_all_queues(fbn->netdev); 145 146 fbnic_service_task_start(fbn); 147 148 fbnic_dbg_up(fbn); 149 } 150 151 void fbnic_down_noidle(struct fbnic_net *fbn) 152 { 153 fbnic_dbg_down(fbn); 154 155 fbnic_service_task_stop(fbn); 156 157 /* Disable Tx/Rx Processing */ 158 fbnic_napi_disable(fbn); 159 netif_tx_disable(fbn->netdev); 160 161 fbnic_clear_rx_mode(fbn->fbd); 162 fbnic_clear_rules(fbn->fbd); 163 fbnic_rss_disable_hw(fbn->fbd); 164 fbnic_disable(fbn); 165 } 166 167 void fbnic_down(struct fbnic_net *fbn) 168 { 169 fbnic_down_noidle(fbn); 170 171 fbnic_wait_all_queues_idle(fbn->fbd, false); 172 173 fbnic_flush(fbn); 174 } 175 176 static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd) 177 { 178 if (fbnic_fw_xmit_ownership_msg(fbd, true)) { 179 dev_err(fbd->dev, "NIC failed to take ownership\n"); 180 181 return -1; 182 } 183 184 fbnic_rpc_reset_valid_entries(fbd); 185 netif_addr_lock_bh(fbd->netdev); 186 __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc); 187 netif_addr_unlock_bh(fbd->netdev); 188 189 return 0; 190 } 191 192 static void fbnic_health_check(struct fbnic_dev *fbd) 193 { 194 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; 195 196 /* As long as the heart is beating the FW is healthy */ 197 if (fbd->fw_heartbeat_enabled) 198 return; 199 200 /* If the Tx mailbox still has messages sitting in it then there likely 201 * isn't anything we can do. We will wait until the mailbox is empty to 202 * report the fault so we can collect the crashlog. 203 */ 204 if (tx_mbx->head != tx_mbx->tail) 205 return; 206 207 fbnic_devlink_fw_report(fbd, "Firmware crash detected!"); 208 fbnic_devlink_otp_check(fbd, "error detected after firmware recovery"); 209 210 if (fbnic_fw_config_after_crash(fbd)) 211 dev_err(fbd->dev, "Firmware recovery failed after crash\n"); 212 } 213 214 static void fbnic_service_task(struct work_struct *work) 215 { 216 struct fbnic_dev *fbd = container_of(to_delayed_work(work), 217 struct fbnic_dev, service_task); 218 struct net_device *netdev = fbd->netdev; 219 220 if (netif_running(netdev)) 221 fbnic_phylink_pmd_training_complete_notify(netdev); 222 223 rtnl_lock(); 224 225 fbnic_get_hw_stats32(fbd); 226 227 if (fbd->ps_timeout && fbnic_mac_check_tx_pause(fbd)) 228 fbnic_mac_ps_protect_handler(fbd); 229 230 fbnic_fw_check_heartbeat(fbd); 231 232 fbnic_health_check(fbd); 233 234 fbnic_bmc_rpc_check(fbd); 235 236 if (netif_carrier_ok(fbd->netdev)) { 237 netdev_lock(fbd->netdev); 238 fbnic_napi_depletion_check(fbd->netdev); 239 netdev_unlock(fbd->netdev); 240 } 241 242 if (netif_running(netdev)) 243 schedule_delayed_work(&fbd->service_task, HZ); 244 245 rtnl_unlock(); 246 } 247 248 /** 249 * fbnic_probe - Device Initialization Routine 250 * @pdev: PCI device information struct 251 * @ent: entry in fbnic_pci_tbl 252 * 253 * Initializes a PCI device identified by a pci_dev structure. 254 * The OS initialization, configuring of the adapter private structure, 255 * and a hardware reset occur. 256 * 257 * Return: 0 on success, negative on failure 258 **/ 259 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 260 { 261 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data]; 262 struct net_device *netdev; 263 struct fbnic_dev *fbd; 264 int err; 265 266 if (pdev->error_state != pci_channel_io_normal) { 267 dev_err(&pdev->dev, 268 "PCI device still in an error state. Unable to load...\n"); 269 return -EIO; 270 } 271 272 err = pcim_enable_device(pdev); 273 if (err) { 274 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err); 275 return err; 276 } 277 278 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); 279 if (err) 280 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 281 if (err) { 282 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); 283 return err; 284 } 285 286 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name); 287 if (err) { 288 dev_err(&pdev->dev, 289 "pci_request_selected_regions failed: %d\n", err); 290 return err; 291 } 292 293 fbd = fbnic_devlink_alloc(pdev); 294 if (!fbd) { 295 dev_err(&pdev->dev, "Devlink allocation failed\n"); 296 return -ENOMEM; 297 } 298 299 err = fbnic_devlink_health_create(fbd); 300 if (err) 301 goto free_fbd; 302 303 /* Populate driver with hardware-specific info and handlers */ 304 fbd->max_num_queues = info->max_num_queues; 305 306 fbd->ps_timeout = FBNIC_MAC_PS_TO_DEFAULT_MS; 307 308 pci_set_master(pdev); 309 pci_save_state(pdev); 310 311 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task); 312 313 err = fbnic_alloc_irqs(fbd); 314 if (err) 315 goto err_destroy_health; 316 317 err = fbnic_mac_init(fbd); 318 if (err) { 319 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err); 320 goto free_irqs; 321 } 322 323 err = fbnic_fw_log_init(fbd); 324 if (err) 325 dev_warn(fbd->dev, 326 "Unable to initialize firmware log buffer: %d\n", 327 err); 328 329 err = fbnic_fw_request_mbx(fbd); 330 if (err) { 331 dev_err(&pdev->dev, 332 "Firmware mailbox initialization failure\n"); 333 goto free_fw_log; 334 } 335 336 /* Send the request to enable the FW logging to host. Note if this 337 * fails we ignore the error and just display a message as it is 338 * possible the FW is just too old to support the logging and needs 339 * to be updated. 340 */ 341 fbnic_fw_log_enable(fbd, true); 342 343 fbnic_devlink_register(fbd); 344 fbnic_devlink_otp_check(fbd, "error detected during probe"); 345 fbnic_dbg_fbd_init(fbd); 346 347 /* Capture snapshot of hardware stats so netdev can calculate delta */ 348 fbnic_init_hw_stats(fbd); 349 350 fbnic_hwmon_register(fbd); 351 352 if (!fbd->dsn) { 353 dev_warn(&pdev->dev, "Reading serial number failed\n"); 354 goto init_failure_mode; 355 } 356 357 if (fbnic_mdiobus_create(fbd)) 358 goto init_failure_mode; 359 360 netdev = fbnic_netdev_alloc(fbd); 361 if (!netdev) { 362 dev_err(&pdev->dev, "Netdev allocation failed\n"); 363 goto init_failure_mode; 364 } 365 366 err = fbnic_ptp_setup(fbd); 367 if (err) 368 goto ifm_free_netdev; 369 370 err = fbnic_netdev_register(netdev); 371 if (err) { 372 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); 373 goto ifm_destroy_ptp; 374 } 375 376 return 0; 377 378 ifm_destroy_ptp: 379 fbnic_ptp_destroy(fbd); 380 ifm_free_netdev: 381 fbnic_netdev_free(fbd); 382 init_failure_mode: 383 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n"); 384 /* Always return 0 even on error so devlink is registered to allow 385 * firmware updates for fixes. 386 */ 387 return 0; 388 free_fw_log: 389 fbnic_fw_log_free(fbd); 390 free_irqs: 391 fbnic_free_irqs(fbd); 392 err_destroy_health: 393 fbnic_devlink_health_destroy(fbd); 394 free_fbd: 395 fbnic_devlink_free(fbd); 396 397 return err; 398 } 399 400 /** 401 * fbnic_remove - Device Removal Routine 402 * @pdev: PCI device information struct 403 * 404 * Called by the PCI subsystem to alert the driver that it should release 405 * a PCI device. This could be caused by a Hot-Plug event, or because the 406 * driver is going to be removed from memory. 407 **/ 408 static void fbnic_remove(struct pci_dev *pdev) 409 { 410 struct fbnic_dev *fbd = pci_get_drvdata(pdev); 411 412 if (!fbnic_init_failure(fbd)) { 413 struct net_device *netdev = fbd->netdev; 414 415 fbnic_netdev_unregister(netdev); 416 cancel_delayed_work_sync(&fbd->service_task); 417 fbnic_ptp_destroy(fbd); 418 fbnic_netdev_free(fbd); 419 } 420 421 fbnic_hwmon_unregister(fbd); 422 fbnic_dbg_fbd_exit(fbd); 423 fbnic_devlink_unregister(fbd); 424 fbnic_fw_log_disable(fbd); 425 fbnic_fw_free_mbx(fbd); 426 fbnic_fw_log_free(fbd); 427 fbnic_free_irqs(fbd); 428 429 fbnic_devlink_health_destroy(fbd); 430 fbnic_devlink_free(fbd); 431 } 432 433 static int fbnic_pm_suspend(struct device *dev) 434 { 435 struct fbnic_dev *fbd = dev_get_drvdata(dev); 436 struct net_device *netdev = fbd->netdev; 437 struct fbnic_net *fbn; 438 439 if (fbnic_init_failure(fbd)) 440 goto null_uc_addr; 441 442 rtnl_lock(); 443 netdev_lock(netdev); 444 445 fbn = netdev_priv(netdev); 446 447 netif_device_detach(netdev); 448 449 if (netif_running(netdev)) 450 netdev->netdev_ops->ndo_stop(netdev); 451 452 /* The IRQs are about to be freed, so drop the napi vector count */ 453 fbn->num_napi = 0; 454 455 netdev_unlock(netdev); 456 rtnl_unlock(); 457 458 null_uc_addr: 459 fbnic_fw_log_disable(fbd); 460 461 devl_lock(priv_to_devlink(fbd)); 462 463 fbnic_fw_free_mbx(fbd); 464 465 devl_unlock(priv_to_devlink(fbd)); 466 467 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */ 468 fbnic_free_irqs(fbd); 469 470 /* Hardware is about to go away, so switch off MMIO access internally */ 471 WRITE_ONCE(fbd->uc_addr0, NULL); 472 WRITE_ONCE(fbd->uc_addr4, NULL); 473 474 return 0; 475 } 476 477 static int __fbnic_pm_resume(struct device *dev) 478 { 479 struct fbnic_dev *fbd = dev_get_drvdata(dev); 480 struct net_device *netdev = fbd->netdev; 481 void __iomem * const *iomap_table; 482 struct fbnic_net *fbn; 483 int err; 484 485 /* Restore MMIO access */ 486 iomap_table = pcim_iomap_table(to_pci_dev(dev)); 487 fbd->uc_addr0 = iomap_table[0]; 488 fbd->uc_addr4 = iomap_table[4]; 489 490 /* Rerequest the IRQs */ 491 err = fbnic_alloc_irqs(fbd); 492 if (err) 493 goto err_invalidate_uc_addr; 494 495 fbd->mac->init_regs(fbd); 496 497 devl_lock(priv_to_devlink(fbd)); 498 499 /* Re-enable mailbox */ 500 err = fbnic_fw_request_mbx(fbd); 501 devl_unlock(priv_to_devlink(fbd)); 502 if (err) 503 goto err_free_irqs; 504 505 /* Only send log history if log buffer is empty to prevent duplicate 506 * log entries. 507 */ 508 fbnic_fw_log_enable(fbd, list_empty(&fbd->fw_log.entries)); 509 510 /* Since the FW should be up, check if it reported OTP errors */ 511 fbnic_devlink_otp_check(fbd, "error detected after PM resume"); 512 513 /* No netdev means there isn't a network interface to bring up */ 514 if (fbnic_init_failure(fbd)) 515 return 0; 516 517 rtnl_lock(); 518 netdev_lock(netdev); 519 520 fbn = netdev_priv(netdev); 521 522 /* Reset the queues if needed */ 523 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); 524 525 if (netif_running(netdev)) { 526 err = __fbnic_open(fbn); 527 /* On failure the vectors are freed, so drop the count */ 528 if (err) 529 fbn->num_napi = 0; 530 } 531 532 netdev_unlock(netdev); 533 rtnl_unlock(); 534 if (err) 535 goto err_free_mbx; 536 537 return 0; 538 err_free_mbx: 539 fbnic_fw_log_disable(fbd); 540 541 devl_lock(priv_to_devlink(fbd)); 542 fbnic_fw_free_mbx(fbd); 543 devl_unlock(priv_to_devlink(fbd)); 544 err_free_irqs: 545 fbnic_free_irqs(fbd); 546 err_invalidate_uc_addr: 547 WRITE_ONCE(fbd->uc_addr0, NULL); 548 WRITE_ONCE(fbd->uc_addr4, NULL); 549 return err; 550 } 551 552 static void __fbnic_pm_attach(struct device *dev) 553 { 554 struct fbnic_dev *fbd = dev_get_drvdata(dev); 555 struct net_device *netdev = fbd->netdev; 556 struct fbnic_net *fbn; 557 558 rtnl_lock(); 559 fbnic_reset_hw_stats(fbd); 560 rtnl_unlock(); 561 562 if (fbnic_init_failure(fbd)) 563 return; 564 565 fbn = netdev_priv(netdev); 566 567 if (netif_running(netdev)) 568 fbnic_up(fbn); 569 570 netif_device_attach(netdev); 571 } 572 573 static int __maybe_unused fbnic_pm_resume(struct device *dev) 574 { 575 int err; 576 577 err = __fbnic_pm_resume(dev); 578 if (!err) 579 __fbnic_pm_attach(dev); 580 581 return err; 582 } 583 584 static const struct dev_pm_ops fbnic_pm_ops = { 585 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume) 586 }; 587 588 static void fbnic_shutdown(struct pci_dev *pdev) 589 { 590 fbnic_pm_suspend(&pdev->dev); 591 } 592 593 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev, 594 pci_channel_state_t state) 595 { 596 /* Disconnect device if failure is not recoverable via reset */ 597 if (state == pci_channel_io_perm_failure) 598 return PCI_ERS_RESULT_DISCONNECT; 599 600 fbnic_pm_suspend(&pdev->dev); 601 602 /* Request a slot reset */ 603 return PCI_ERS_RESULT_NEED_RESET; 604 } 605 606 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev) 607 { 608 int err; 609 610 pci_set_power_state(pdev, PCI_D0); 611 pci_restore_state(pdev); 612 613 if (pci_enable_device_mem(pdev)) { 614 dev_err(&pdev->dev, 615 "Cannot re-enable PCI device after reset.\n"); 616 return PCI_ERS_RESULT_DISCONNECT; 617 } 618 619 /* Restore device to previous state */ 620 err = __fbnic_pm_resume(&pdev->dev); 621 622 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 623 } 624 625 static void fbnic_err_resume(struct pci_dev *pdev) 626 { 627 __fbnic_pm_attach(&pdev->dev); 628 } 629 630 static const struct pci_error_handlers fbnic_err_handler = { 631 .error_detected = fbnic_err_error_detected, 632 .slot_reset = fbnic_err_slot_reset, 633 .resume = fbnic_err_resume, 634 }; 635 636 static struct pci_driver fbnic_driver = { 637 .name = fbnic_driver_name, 638 .id_table = fbnic_pci_tbl, 639 .probe = fbnic_probe, 640 .remove = fbnic_remove, 641 .driver.pm = &fbnic_pm_ops, 642 .shutdown = fbnic_shutdown, 643 .err_handler = &fbnic_err_handler, 644 }; 645 646 /** 647 * fbnic_init_module - Driver Registration Routine 648 * 649 * The first routine called when the driver is loaded. All it does is 650 * register with the PCI subsystem. 651 * 652 * Return: 0 on success, negative on failure 653 **/ 654 static int __init fbnic_init_module(void) 655 { 656 int err; 657 658 fbnic_dbg_init(); 659 660 err = pci_register_driver(&fbnic_driver); 661 if (err) { 662 fbnic_dbg_exit(); 663 goto out; 664 } 665 666 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name); 667 out: 668 return err; 669 } 670 module_init(fbnic_init_module); 671 672 /** 673 * fbnic_exit_module - Driver Exit Cleanup Routine 674 * 675 * Called just before the driver is removed from memory. 676 **/ 677 static void __exit fbnic_exit_module(void) 678 { 679 pci_unregister_driver(&fbnic_driver); 680 681 fbnic_dbg_exit(); 682 } 683 module_exit(fbnic_exit_module); 684