1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/init.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/rtnetlink.h> 8 #include <linux/types.h> 9 #include <net/devlink.h> 10 11 #include "fbnic.h" 12 #include "fbnic_drvinfo.h" 13 #include "fbnic_hw_stats.h" 14 #include "fbnic_netdev.h" 15 16 char fbnic_driver_name[] = DRV_NAME; 17 18 MODULE_DESCRIPTION(DRV_SUMMARY); 19 MODULE_LICENSE("GPL"); 20 21 static const struct fbnic_info fbnic_asic_info = { 22 .max_num_queues = FBNIC_MAX_QUEUES, 23 .bar_mask = BIT(0) | BIT(4) 24 }; 25 26 static const struct fbnic_info *fbnic_info_tbl[] = { 27 [fbnic_board_asic] = &fbnic_asic_info, 28 }; 29 30 static const struct pci_device_id fbnic_pci_tbl[] = { 31 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) }, 32 /* Required last entry */ 33 {0, } 34 }; 35 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl); 36 37 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg) 38 { 39 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0); 40 u32 value; 41 42 if (!csr) 43 return ~0U; 44 45 value = readl(csr + reg); 46 47 /* If any bits are 0 value should be valid */ 48 if (~value) 49 return value; 50 51 /* All 1's may be valid if ZEROs register still works */ 52 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0)) 53 return value; 54 55 /* Hardware is giving us all 1's reads, assume it is gone */ 56 WRITE_ONCE(fbd->uc_addr0, NULL); 57 WRITE_ONCE(fbd->uc_addr4, NULL); 58 59 dev_err(fbd->dev, 60 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 61 reg, reg << 2); 62 63 /* Notify stack that device has lost (PCIe) link */ 64 if (!fbnic_init_failure(fbd)) 65 netif_device_detach(fbd->netdev); 66 67 return ~0U; 68 } 69 70 bool fbnic_fw_present(struct fbnic_dev *fbd) 71 { 72 return !!READ_ONCE(fbd->uc_addr4); 73 } 74 75 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val) 76 { 77 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 78 79 if (csr) 80 writel(val, csr + reg); 81 } 82 83 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg) 84 { 85 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 86 u32 value; 87 88 if (!csr) 89 return ~0U; 90 91 value = readl(csr + reg); 92 93 /* If any bits are 0 value should be valid */ 94 if (~value) 95 return value; 96 97 /* All 1's may be valid if ZEROs register still works */ 98 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG)) 99 return value; 100 101 /* Hardware is giving us all 1's reads, assume it is gone */ 102 WRITE_ONCE(fbd->uc_addr0, NULL); 103 WRITE_ONCE(fbd->uc_addr4, NULL); 104 105 dev_err(fbd->dev, 106 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 107 reg, reg << 2); 108 109 /* Notify stack that device has lost (PCIe) link */ 110 if (!fbnic_init_failure(fbd)) 111 netif_device_detach(fbd->netdev); 112 113 return ~0U; 114 } 115 116 static void fbnic_service_task_start(struct fbnic_net *fbn) 117 { 118 struct fbnic_dev *fbd = fbn->fbd; 119 120 schedule_delayed_work(&fbd->service_task, HZ); 121 phylink_resume(fbn->phylink); 122 } 123 124 static void fbnic_service_task_stop(struct fbnic_net *fbn) 125 { 126 struct fbnic_dev *fbd = fbn->fbd; 127 128 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd)); 129 cancel_delayed_work(&fbd->service_task); 130 } 131 132 void fbnic_up(struct fbnic_net *fbn) 133 { 134 fbnic_enable(fbn); 135 136 fbnic_fill(fbn); 137 138 fbnic_rss_reinit_hw(fbn->fbd, fbn); 139 140 __fbnic_set_rx_mode(fbn->netdev); 141 142 /* Enable Tx/Rx processing */ 143 fbnic_napi_enable(fbn); 144 netif_tx_start_all_queues(fbn->netdev); 145 146 fbnic_service_task_start(fbn); 147 } 148 149 void fbnic_down_noidle(struct fbnic_net *fbn) 150 { 151 fbnic_service_task_stop(fbn); 152 153 /* Disable Tx/Rx Processing */ 154 fbnic_napi_disable(fbn); 155 netif_tx_disable(fbn->netdev); 156 157 fbnic_clear_rx_mode(fbn->netdev); 158 fbnic_clear_rules(fbn->fbd); 159 fbnic_rss_disable_hw(fbn->fbd); 160 fbnic_disable(fbn); 161 } 162 163 void fbnic_down(struct fbnic_net *fbn) 164 { 165 fbnic_down_noidle(fbn); 166 167 fbnic_wait_all_queues_idle(fbn->fbd, false); 168 169 fbnic_flush(fbn); 170 } 171 172 static void fbnic_health_check(struct fbnic_dev *fbd) 173 { 174 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; 175 176 /* As long as the heart is beating the FW is healty */ 177 if (fbd->fw_heartbeat_enabled) 178 return; 179 180 /* If the Tx mailbox still has messages sitting in it then there likely 181 * isn't anything we can do. We will wait until the mailbox is empty to 182 * report the fault so we can collect the crashlog. 183 */ 184 if (tx_mbx->head != tx_mbx->tail) 185 return; 186 187 /* TBD: Need to add a more thorough recovery here. 188 * Specifically I need to verify what all the firmware will have 189 * changed since we had setup and it rebooted. May just need to 190 * perform a down/up. For now we will just reclaim ownership so 191 * the heartbeat can catch the next fault. 192 */ 193 fbnic_fw_xmit_ownership_msg(fbd, true); 194 } 195 196 static void fbnic_service_task(struct work_struct *work) 197 { 198 struct fbnic_dev *fbd = container_of(to_delayed_work(work), 199 struct fbnic_dev, service_task); 200 201 rtnl_lock(); 202 203 fbnic_get_hw_stats32(fbd); 204 205 fbnic_fw_check_heartbeat(fbd); 206 207 fbnic_health_check(fbd); 208 209 if (netif_carrier_ok(fbd->netdev)) 210 fbnic_napi_depletion_check(fbd->netdev); 211 212 if (netif_running(fbd->netdev)) 213 schedule_delayed_work(&fbd->service_task, HZ); 214 215 rtnl_unlock(); 216 } 217 218 /** 219 * fbnic_probe - Device Initialization Routine 220 * @pdev: PCI device information struct 221 * @ent: entry in fbnic_pci_tbl 222 * 223 * Initializes a PCI device identified by a pci_dev structure. 224 * The OS initialization, configuring of the adapter private structure, 225 * and a hardware reset occur. 226 * 227 * Return: 0 on success, negative on failure 228 **/ 229 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 230 { 231 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data]; 232 struct net_device *netdev; 233 struct fbnic_dev *fbd; 234 int err; 235 236 if (pdev->error_state != pci_channel_io_normal) { 237 dev_err(&pdev->dev, 238 "PCI device still in an error state. Unable to load...\n"); 239 return -EIO; 240 } 241 242 err = pcim_enable_device(pdev); 243 if (err) { 244 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err); 245 return err; 246 } 247 248 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); 249 if (err) 250 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 251 if (err) { 252 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); 253 return err; 254 } 255 256 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name); 257 if (err) { 258 dev_err(&pdev->dev, 259 "pci_request_selected_regions failed: %d\n", err); 260 return err; 261 } 262 263 fbd = fbnic_devlink_alloc(pdev); 264 if (!fbd) { 265 dev_err(&pdev->dev, "Devlink allocation failed\n"); 266 return -ENOMEM; 267 } 268 269 /* Populate driver with hardware-specific info and handlers */ 270 fbd->max_num_queues = info->max_num_queues; 271 272 pci_set_master(pdev); 273 pci_save_state(pdev); 274 275 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task); 276 277 err = fbnic_alloc_irqs(fbd); 278 if (err) 279 goto free_fbd; 280 281 err = fbnic_mac_init(fbd); 282 if (err) { 283 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err); 284 goto free_irqs; 285 } 286 287 err = fbnic_fw_request_mbx(fbd); 288 if (err) { 289 dev_err(&pdev->dev, 290 "Firmware mailbox initialization failure\n"); 291 goto free_irqs; 292 } 293 294 fbnic_devlink_register(fbd); 295 fbnic_dbg_fbd_init(fbd); 296 spin_lock_init(&fbd->hw_stats_lock); 297 298 /* Capture snapshot of hardware stats so netdev can calculate delta */ 299 fbnic_reset_hw_stats(fbd); 300 301 fbnic_hwmon_register(fbd); 302 303 if (!fbd->dsn) { 304 dev_warn(&pdev->dev, "Reading serial number failed\n"); 305 goto init_failure_mode; 306 } 307 308 netdev = fbnic_netdev_alloc(fbd); 309 if (!netdev) { 310 dev_err(&pdev->dev, "Netdev allocation failed\n"); 311 goto init_failure_mode; 312 } 313 314 err = fbnic_ptp_setup(fbd); 315 if (err) 316 goto ifm_free_netdev; 317 318 err = fbnic_netdev_register(netdev); 319 if (err) { 320 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); 321 goto ifm_destroy_ptp; 322 } 323 324 return 0; 325 326 ifm_destroy_ptp: 327 fbnic_ptp_destroy(fbd); 328 ifm_free_netdev: 329 fbnic_netdev_free(fbd); 330 init_failure_mode: 331 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n"); 332 /* Always return 0 even on error so devlink is registered to allow 333 * firmware updates for fixes. 334 */ 335 return 0; 336 free_irqs: 337 fbnic_free_irqs(fbd); 338 free_fbd: 339 fbnic_devlink_free(fbd); 340 341 return err; 342 } 343 344 /** 345 * fbnic_remove - Device Removal Routine 346 * @pdev: PCI device information struct 347 * 348 * Called by the PCI subsystem to alert the driver that it should release 349 * a PCI device. The could be caused by a Hot-Plug event, or because the 350 * driver is going to be removed from memory. 351 **/ 352 static void fbnic_remove(struct pci_dev *pdev) 353 { 354 struct fbnic_dev *fbd = pci_get_drvdata(pdev); 355 356 if (!fbnic_init_failure(fbd)) { 357 struct net_device *netdev = fbd->netdev; 358 359 fbnic_netdev_unregister(netdev); 360 cancel_delayed_work_sync(&fbd->service_task); 361 fbnic_ptp_destroy(fbd); 362 fbnic_netdev_free(fbd); 363 } 364 365 fbnic_hwmon_unregister(fbd); 366 fbnic_dbg_fbd_exit(fbd); 367 fbnic_devlink_unregister(fbd); 368 fbnic_fw_free_mbx(fbd); 369 fbnic_free_irqs(fbd); 370 371 fbnic_devlink_free(fbd); 372 } 373 374 static int fbnic_pm_suspend(struct device *dev) 375 { 376 struct fbnic_dev *fbd = dev_get_drvdata(dev); 377 struct net_device *netdev = fbd->netdev; 378 379 if (fbnic_init_failure(fbd)) 380 goto null_uc_addr; 381 382 rtnl_lock(); 383 384 netif_device_detach(netdev); 385 386 if (netif_running(netdev)) 387 netdev->netdev_ops->ndo_stop(netdev); 388 389 rtnl_unlock(); 390 391 null_uc_addr: 392 devl_lock(priv_to_devlink(fbd)); 393 394 fbnic_fw_free_mbx(fbd); 395 396 devl_unlock(priv_to_devlink(fbd)); 397 398 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */ 399 fbnic_free_irqs(fbd); 400 401 /* Hardware is about to go away, so switch off MMIO access internally */ 402 WRITE_ONCE(fbd->uc_addr0, NULL); 403 WRITE_ONCE(fbd->uc_addr4, NULL); 404 405 return 0; 406 } 407 408 static int __fbnic_pm_resume(struct device *dev) 409 { 410 struct fbnic_dev *fbd = dev_get_drvdata(dev); 411 struct net_device *netdev = fbd->netdev; 412 void __iomem * const *iomap_table; 413 struct fbnic_net *fbn; 414 int err; 415 416 /* Restore MMIO access */ 417 iomap_table = pcim_iomap_table(to_pci_dev(dev)); 418 fbd->uc_addr0 = iomap_table[0]; 419 fbd->uc_addr4 = iomap_table[4]; 420 421 /* Rerequest the IRQs */ 422 err = fbnic_alloc_irqs(fbd); 423 if (err) 424 goto err_invalidate_uc_addr; 425 426 fbd->mac->init_regs(fbd); 427 428 devl_lock(priv_to_devlink(fbd)); 429 430 /* Re-enable mailbox */ 431 err = fbnic_fw_request_mbx(fbd); 432 if (err) 433 goto err_free_irqs; 434 435 devl_unlock(priv_to_devlink(fbd)); 436 437 /* No netdev means there isn't a network interface to bring up */ 438 if (fbnic_init_failure(fbd)) 439 return 0; 440 441 fbn = netdev_priv(netdev); 442 443 /* Reset the queues if needed */ 444 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); 445 446 rtnl_lock(); 447 448 if (netif_running(netdev)) { 449 err = __fbnic_open(fbn); 450 if (err) 451 goto err_free_mbx; 452 } 453 454 rtnl_unlock(); 455 456 return 0; 457 err_free_mbx: 458 rtnl_unlock(); 459 fbnic_fw_free_mbx(fbd); 460 err_free_irqs: 461 fbnic_free_irqs(fbd); 462 err_invalidate_uc_addr: 463 WRITE_ONCE(fbd->uc_addr0, NULL); 464 WRITE_ONCE(fbd->uc_addr4, NULL); 465 return err; 466 } 467 468 static void __fbnic_pm_attach(struct device *dev) 469 { 470 struct fbnic_dev *fbd = dev_get_drvdata(dev); 471 struct net_device *netdev = fbd->netdev; 472 struct fbnic_net *fbn; 473 474 if (fbnic_init_failure(fbd)) 475 return; 476 477 fbn = netdev_priv(netdev); 478 479 if (netif_running(netdev)) 480 fbnic_up(fbn); 481 482 netif_device_attach(netdev); 483 } 484 485 static int __maybe_unused fbnic_pm_resume(struct device *dev) 486 { 487 int err; 488 489 err = __fbnic_pm_resume(dev); 490 if (!err) 491 __fbnic_pm_attach(dev); 492 493 return err; 494 } 495 496 static const struct dev_pm_ops fbnic_pm_ops = { 497 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume) 498 }; 499 500 static void fbnic_shutdown(struct pci_dev *pdev) 501 { 502 fbnic_pm_suspend(&pdev->dev); 503 } 504 505 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev, 506 pci_channel_state_t state) 507 { 508 /* Disconnect device if failure is not recoverable via reset */ 509 if (state == pci_channel_io_perm_failure) 510 return PCI_ERS_RESULT_DISCONNECT; 511 512 fbnic_pm_suspend(&pdev->dev); 513 514 /* Request a slot reset */ 515 return PCI_ERS_RESULT_NEED_RESET; 516 } 517 518 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev) 519 { 520 int err; 521 522 pci_set_power_state(pdev, PCI_D0); 523 pci_restore_state(pdev); 524 pci_save_state(pdev); 525 526 if (pci_enable_device_mem(pdev)) { 527 dev_err(&pdev->dev, 528 "Cannot re-enable PCI device after reset.\n"); 529 return PCI_ERS_RESULT_DISCONNECT; 530 } 531 532 /* Restore device to previous state */ 533 err = __fbnic_pm_resume(&pdev->dev); 534 535 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 536 } 537 538 static void fbnic_err_resume(struct pci_dev *pdev) 539 { 540 __fbnic_pm_attach(&pdev->dev); 541 } 542 543 static const struct pci_error_handlers fbnic_err_handler = { 544 .error_detected = fbnic_err_error_detected, 545 .slot_reset = fbnic_err_slot_reset, 546 .resume = fbnic_err_resume, 547 }; 548 549 static struct pci_driver fbnic_driver = { 550 .name = fbnic_driver_name, 551 .id_table = fbnic_pci_tbl, 552 .probe = fbnic_probe, 553 .remove = fbnic_remove, 554 .driver.pm = &fbnic_pm_ops, 555 .shutdown = fbnic_shutdown, 556 .err_handler = &fbnic_err_handler, 557 }; 558 559 /** 560 * fbnic_init_module - Driver Registration Routine 561 * 562 * The first routine called when the driver is loaded. All it does is 563 * register with the PCI subsystem. 564 * 565 * Return: 0 on success, negative on failure 566 **/ 567 static int __init fbnic_init_module(void) 568 { 569 int err; 570 571 fbnic_dbg_init(); 572 573 err = pci_register_driver(&fbnic_driver); 574 if (err) { 575 fbnic_dbg_exit(); 576 goto out; 577 } 578 579 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name); 580 out: 581 return err; 582 } 583 module_init(fbnic_init_module); 584 585 /** 586 * fbnic_exit_module - Driver Exit Cleanup Routine 587 * 588 * Called just before the driver is removed from memory. 589 **/ 590 static void __exit fbnic_exit_module(void) 591 { 592 pci_unregister_driver(&fbnic_driver); 593 594 fbnic_dbg_exit(); 595 } 596 module_exit(fbnic_exit_module); 597