1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/init.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/rtnetlink.h> 8 #include <linux/types.h> 9 10 #include "fbnic.h" 11 #include "fbnic_drvinfo.h" 12 #include "fbnic_hw_stats.h" 13 #include "fbnic_netdev.h" 14 15 char fbnic_driver_name[] = DRV_NAME; 16 17 MODULE_DESCRIPTION(DRV_SUMMARY); 18 MODULE_LICENSE("GPL"); 19 20 static const struct fbnic_info fbnic_asic_info = { 21 .max_num_queues = FBNIC_MAX_QUEUES, 22 .bar_mask = BIT(0) | BIT(4) 23 }; 24 25 static const struct fbnic_info *fbnic_info_tbl[] = { 26 [fbnic_board_asic] = &fbnic_asic_info, 27 }; 28 29 static const struct pci_device_id fbnic_pci_tbl[] = { 30 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) }, 31 /* Required last entry */ 32 {0, } 33 }; 34 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl); 35 36 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg) 37 { 38 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0); 39 u32 value; 40 41 if (!csr) 42 return ~0U; 43 44 value = readl(csr + reg); 45 46 /* If any bits are 0 value should be valid */ 47 if (~value) 48 return value; 49 50 /* All 1's may be valid if ZEROs register still works */ 51 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0)) 52 return value; 53 54 /* Hardware is giving us all 1's reads, assume it is gone */ 55 WRITE_ONCE(fbd->uc_addr0, NULL); 56 WRITE_ONCE(fbd->uc_addr4, NULL); 57 58 dev_err(fbd->dev, 59 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 60 reg, reg << 2); 61 62 /* Notify stack that device has lost (PCIe) link */ 63 if (!fbnic_init_failure(fbd)) 64 netif_device_detach(fbd->netdev); 65 66 return ~0U; 67 } 68 69 bool fbnic_fw_present(struct fbnic_dev *fbd) 70 { 71 return !!READ_ONCE(fbd->uc_addr4); 72 } 73 74 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val) 75 { 76 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 77 78 if (csr) 79 writel(val, csr + reg); 80 } 81 82 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg) 83 { 84 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 85 u32 value; 86 87 if (!csr) 88 return ~0U; 89 90 value = readl(csr + reg); 91 92 /* If any bits are 0 value should be valid */ 93 if (~value) 94 return value; 95 96 /* All 1's may be valid if ZEROs register still works */ 97 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG)) 98 return value; 99 100 /* Hardware is giving us all 1's reads, assume it is gone */ 101 WRITE_ONCE(fbd->uc_addr0, NULL); 102 WRITE_ONCE(fbd->uc_addr4, NULL); 103 104 dev_err(fbd->dev, 105 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 106 reg, reg << 2); 107 108 /* Notify stack that device has lost (PCIe) link */ 109 if (!fbnic_init_failure(fbd)) 110 netif_device_detach(fbd->netdev); 111 112 return ~0U; 113 } 114 115 static void fbnic_service_task_start(struct fbnic_net *fbn) 116 { 117 struct fbnic_dev *fbd = fbn->fbd; 118 119 schedule_delayed_work(&fbd->service_task, HZ); 120 phylink_resume(fbn->phylink); 121 } 122 123 static void fbnic_service_task_stop(struct fbnic_net *fbn) 124 { 125 struct fbnic_dev *fbd = fbn->fbd; 126 127 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd)); 128 cancel_delayed_work(&fbd->service_task); 129 } 130 131 void fbnic_up(struct fbnic_net *fbn) 132 { 133 fbnic_enable(fbn); 134 135 fbnic_fill(fbn); 136 137 fbnic_rss_reinit_hw(fbn->fbd, fbn); 138 139 __fbnic_set_rx_mode(fbn->netdev); 140 141 /* Enable Tx/Rx processing */ 142 fbnic_napi_enable(fbn); 143 netif_tx_start_all_queues(fbn->netdev); 144 145 fbnic_service_task_start(fbn); 146 } 147 148 static void fbnic_down_noidle(struct fbnic_net *fbn) 149 { 150 fbnic_service_task_stop(fbn); 151 152 /* Disable Tx/Rx Processing */ 153 fbnic_napi_disable(fbn); 154 netif_tx_disable(fbn->netdev); 155 156 fbnic_clear_rx_mode(fbn->netdev); 157 fbnic_clear_rules(fbn->fbd); 158 fbnic_rss_disable_hw(fbn->fbd); 159 fbnic_disable(fbn); 160 } 161 162 void fbnic_down(struct fbnic_net *fbn) 163 { 164 fbnic_down_noidle(fbn); 165 166 fbnic_wait_all_queues_idle(fbn->fbd, false); 167 168 fbnic_flush(fbn); 169 } 170 171 static void fbnic_health_check(struct fbnic_dev *fbd) 172 { 173 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; 174 175 /* As long as the heart is beating the FW is healty */ 176 if (fbd->fw_heartbeat_enabled) 177 return; 178 179 /* If the Tx mailbox still has messages sitting in it then there likely 180 * isn't anything we can do. We will wait until the mailbox is empty to 181 * report the fault so we can collect the crashlog. 182 */ 183 if (tx_mbx->head != tx_mbx->tail) 184 return; 185 186 /* TBD: Need to add a more thorough recovery here. 187 * Specifically I need to verify what all the firmware will have 188 * changed since we had setup and it rebooted. May just need to 189 * perform a down/up. For now we will just reclaim ownership so 190 * the heartbeat can catch the next fault. 191 */ 192 fbnic_fw_xmit_ownership_msg(fbd, true); 193 } 194 195 static void fbnic_service_task(struct work_struct *work) 196 { 197 struct fbnic_dev *fbd = container_of(to_delayed_work(work), 198 struct fbnic_dev, service_task); 199 200 rtnl_lock(); 201 202 fbnic_get_hw_stats32(fbd); 203 204 fbnic_fw_check_heartbeat(fbd); 205 206 fbnic_health_check(fbd); 207 208 if (netif_carrier_ok(fbd->netdev)) 209 fbnic_napi_depletion_check(fbd->netdev); 210 211 if (netif_running(fbd->netdev)) 212 schedule_delayed_work(&fbd->service_task, HZ); 213 214 rtnl_unlock(); 215 } 216 217 /** 218 * fbnic_probe - Device Initialization Routine 219 * @pdev: PCI device information struct 220 * @ent: entry in fbnic_pci_tbl 221 * 222 * Initializes a PCI device identified by a pci_dev structure. 223 * The OS initialization, configuring of the adapter private structure, 224 * and a hardware reset occur. 225 * 226 * Return: 0 on success, negative on failure 227 **/ 228 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 229 { 230 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data]; 231 struct net_device *netdev; 232 struct fbnic_dev *fbd; 233 int err; 234 235 if (pdev->error_state != pci_channel_io_normal) { 236 dev_err(&pdev->dev, 237 "PCI device still in an error state. Unable to load...\n"); 238 return -EIO; 239 } 240 241 err = pcim_enable_device(pdev); 242 if (err) { 243 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err); 244 return err; 245 } 246 247 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); 248 if (err) 249 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 250 if (err) { 251 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); 252 return err; 253 } 254 255 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name); 256 if (err) { 257 dev_err(&pdev->dev, 258 "pci_request_selected_regions failed: %d\n", err); 259 return err; 260 } 261 262 fbd = fbnic_devlink_alloc(pdev); 263 if (!fbd) { 264 dev_err(&pdev->dev, "Devlink allocation failed\n"); 265 return -ENOMEM; 266 } 267 268 /* Populate driver with hardware-specific info and handlers */ 269 fbd->max_num_queues = info->max_num_queues; 270 271 pci_set_master(pdev); 272 pci_save_state(pdev); 273 274 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task); 275 276 err = fbnic_alloc_irqs(fbd); 277 if (err) 278 goto free_fbd; 279 280 err = fbnic_mac_init(fbd); 281 if (err) { 282 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err); 283 goto free_irqs; 284 } 285 286 err = fbnic_fw_enable_mbx(fbd); 287 if (err) { 288 dev_err(&pdev->dev, 289 "Firmware mailbox initialization failure\n"); 290 goto free_irqs; 291 } 292 293 fbnic_devlink_register(fbd); 294 fbnic_dbg_fbd_init(fbd); 295 296 /* Capture snapshot of hardware stats so netdev can calculate delta */ 297 fbnic_reset_hw_stats(fbd); 298 299 fbnic_hwmon_register(fbd); 300 301 if (!fbd->dsn) { 302 dev_warn(&pdev->dev, "Reading serial number failed\n"); 303 goto init_failure_mode; 304 } 305 306 netdev = fbnic_netdev_alloc(fbd); 307 if (!netdev) { 308 dev_err(&pdev->dev, "Netdev allocation failed\n"); 309 goto init_failure_mode; 310 } 311 312 err = fbnic_ptp_setup(fbd); 313 if (err) 314 goto ifm_free_netdev; 315 316 err = fbnic_netdev_register(netdev); 317 if (err) { 318 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); 319 goto ifm_destroy_ptp; 320 } 321 322 return 0; 323 324 ifm_destroy_ptp: 325 fbnic_ptp_destroy(fbd); 326 ifm_free_netdev: 327 fbnic_netdev_free(fbd); 328 init_failure_mode: 329 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n"); 330 /* Always return 0 even on error so devlink is registered to allow 331 * firmware updates for fixes. 332 */ 333 return 0; 334 free_irqs: 335 fbnic_free_irqs(fbd); 336 free_fbd: 337 pci_disable_device(pdev); 338 fbnic_devlink_free(fbd); 339 340 return err; 341 } 342 343 /** 344 * fbnic_remove - Device Removal Routine 345 * @pdev: PCI device information struct 346 * 347 * Called by the PCI subsystem to alert the driver that it should release 348 * a PCI device. The could be caused by a Hot-Plug event, or because the 349 * driver is going to be removed from memory. 350 **/ 351 static void fbnic_remove(struct pci_dev *pdev) 352 { 353 struct fbnic_dev *fbd = pci_get_drvdata(pdev); 354 355 if (!fbnic_init_failure(fbd)) { 356 struct net_device *netdev = fbd->netdev; 357 358 fbnic_netdev_unregister(netdev); 359 cancel_delayed_work_sync(&fbd->service_task); 360 fbnic_ptp_destroy(fbd); 361 fbnic_netdev_free(fbd); 362 } 363 364 fbnic_hwmon_unregister(fbd); 365 fbnic_dbg_fbd_exit(fbd); 366 fbnic_devlink_unregister(fbd); 367 fbnic_fw_disable_mbx(fbd); 368 fbnic_free_irqs(fbd); 369 370 pci_disable_device(pdev); 371 fbnic_devlink_free(fbd); 372 } 373 374 static int fbnic_pm_suspend(struct device *dev) 375 { 376 struct fbnic_dev *fbd = dev_get_drvdata(dev); 377 struct net_device *netdev = fbd->netdev; 378 379 if (fbnic_init_failure(fbd)) 380 goto null_uc_addr; 381 382 rtnl_lock(); 383 384 netif_device_detach(netdev); 385 386 if (netif_running(netdev)) 387 netdev->netdev_ops->ndo_stop(netdev); 388 389 rtnl_unlock(); 390 391 null_uc_addr: 392 fbnic_fw_disable_mbx(fbd); 393 394 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */ 395 fbnic_free_irqs(fbd); 396 397 /* Hardware is about to go away, so switch off MMIO access internally */ 398 WRITE_ONCE(fbd->uc_addr0, NULL); 399 WRITE_ONCE(fbd->uc_addr4, NULL); 400 401 return 0; 402 } 403 404 static int __fbnic_pm_resume(struct device *dev) 405 { 406 struct fbnic_dev *fbd = dev_get_drvdata(dev); 407 struct net_device *netdev = fbd->netdev; 408 void __iomem * const *iomap_table; 409 struct fbnic_net *fbn; 410 int err; 411 412 /* Restore MMIO access */ 413 iomap_table = pcim_iomap_table(to_pci_dev(dev)); 414 fbd->uc_addr0 = iomap_table[0]; 415 fbd->uc_addr4 = iomap_table[4]; 416 417 /* Rerequest the IRQs */ 418 err = fbnic_alloc_irqs(fbd); 419 if (err) 420 goto err_invalidate_uc_addr; 421 422 fbd->mac->init_regs(fbd); 423 424 /* Re-enable mailbox */ 425 err = fbnic_fw_enable_mbx(fbd); 426 if (err) 427 goto err_free_irqs; 428 429 /* No netdev means there isn't a network interface to bring up */ 430 if (fbnic_init_failure(fbd)) 431 return 0; 432 433 fbn = netdev_priv(netdev); 434 435 /* Reset the queues if needed */ 436 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); 437 438 rtnl_lock(); 439 440 if (netif_running(netdev)) { 441 err = __fbnic_open(fbn); 442 if (err) 443 goto err_disable_mbx; 444 } 445 446 rtnl_unlock(); 447 448 return 0; 449 err_disable_mbx: 450 rtnl_unlock(); 451 fbnic_fw_disable_mbx(fbd); 452 err_free_irqs: 453 fbnic_free_irqs(fbd); 454 err_invalidate_uc_addr: 455 WRITE_ONCE(fbd->uc_addr0, NULL); 456 WRITE_ONCE(fbd->uc_addr4, NULL); 457 return err; 458 } 459 460 static void __fbnic_pm_attach(struct device *dev) 461 { 462 struct fbnic_dev *fbd = dev_get_drvdata(dev); 463 struct net_device *netdev = fbd->netdev; 464 struct fbnic_net *fbn; 465 466 if (fbnic_init_failure(fbd)) 467 return; 468 469 fbn = netdev_priv(netdev); 470 471 if (netif_running(netdev)) 472 fbnic_up(fbn); 473 474 netif_device_attach(netdev); 475 } 476 477 static int __maybe_unused fbnic_pm_resume(struct device *dev) 478 { 479 int err; 480 481 err = __fbnic_pm_resume(dev); 482 if (!err) 483 __fbnic_pm_attach(dev); 484 485 return err; 486 } 487 488 static const struct dev_pm_ops fbnic_pm_ops = { 489 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume) 490 }; 491 492 static void fbnic_shutdown(struct pci_dev *pdev) 493 { 494 fbnic_pm_suspend(&pdev->dev); 495 } 496 497 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev, 498 pci_channel_state_t state) 499 { 500 /* Disconnect device if failure is not recoverable via reset */ 501 if (state == pci_channel_io_perm_failure) 502 return PCI_ERS_RESULT_DISCONNECT; 503 504 fbnic_pm_suspend(&pdev->dev); 505 506 /* Request a slot reset */ 507 return PCI_ERS_RESULT_NEED_RESET; 508 } 509 510 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev) 511 { 512 int err; 513 514 pci_set_power_state(pdev, PCI_D0); 515 pci_restore_state(pdev); 516 pci_save_state(pdev); 517 518 if (pci_enable_device_mem(pdev)) { 519 dev_err(&pdev->dev, 520 "Cannot re-enable PCI device after reset.\n"); 521 return PCI_ERS_RESULT_DISCONNECT; 522 } 523 524 /* Restore device to previous state */ 525 err = __fbnic_pm_resume(&pdev->dev); 526 527 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 528 } 529 530 static void fbnic_err_resume(struct pci_dev *pdev) 531 { 532 __fbnic_pm_attach(&pdev->dev); 533 } 534 535 static const struct pci_error_handlers fbnic_err_handler = { 536 .error_detected = fbnic_err_error_detected, 537 .slot_reset = fbnic_err_slot_reset, 538 .resume = fbnic_err_resume, 539 }; 540 541 static struct pci_driver fbnic_driver = { 542 .name = fbnic_driver_name, 543 .id_table = fbnic_pci_tbl, 544 .probe = fbnic_probe, 545 .remove = fbnic_remove, 546 .driver.pm = &fbnic_pm_ops, 547 .shutdown = fbnic_shutdown, 548 .err_handler = &fbnic_err_handler, 549 }; 550 551 /** 552 * fbnic_init_module - Driver Registration Routine 553 * 554 * The first routine called when the driver is loaded. All it does is 555 * register with the PCI subsystem. 556 * 557 * Return: 0 on success, negative on failure 558 **/ 559 static int __init fbnic_init_module(void) 560 { 561 int err; 562 563 fbnic_dbg_init(); 564 565 err = pci_register_driver(&fbnic_driver); 566 if (err) { 567 fbnic_dbg_exit(); 568 goto out; 569 } 570 571 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name); 572 out: 573 return err; 574 } 575 module_init(fbnic_init_module); 576 577 /** 578 * fbnic_exit_module - Driver Exit Cleanup Routine 579 * 580 * Called just before the driver is removed from memory. 581 **/ 582 static void __exit fbnic_exit_module(void) 583 { 584 pci_unregister_driver(&fbnic_driver); 585 586 fbnic_dbg_exit(); 587 } 588 module_exit(fbnic_exit_module); 589