1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/init.h> 5 #include <linux/module.h> 6 #include <linux/pci.h> 7 #include <linux/rtnetlink.h> 8 #include <linux/types.h> 9 10 #include "fbnic.h" 11 #include "fbnic_drvinfo.h" 12 #include "fbnic_netdev.h" 13 14 char fbnic_driver_name[] = DRV_NAME; 15 16 MODULE_DESCRIPTION(DRV_SUMMARY); 17 MODULE_LICENSE("GPL"); 18 19 static const struct fbnic_info fbnic_asic_info = { 20 .max_num_queues = FBNIC_MAX_QUEUES, 21 .bar_mask = BIT(0) | BIT(4) 22 }; 23 24 static const struct fbnic_info *fbnic_info_tbl[] = { 25 [fbnic_board_asic] = &fbnic_asic_info, 26 }; 27 28 static const struct pci_device_id fbnic_pci_tbl[] = { 29 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) }, 30 /* Required last entry */ 31 {0, } 32 }; 33 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl); 34 35 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg) 36 { 37 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0); 38 u32 value; 39 40 if (!csr) 41 return ~0U; 42 43 value = readl(csr + reg); 44 45 /* If any bits are 0 value should be valid */ 46 if (~value) 47 return value; 48 49 /* All 1's may be valid if ZEROs register still works */ 50 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0)) 51 return value; 52 53 /* Hardware is giving us all 1's reads, assume it is gone */ 54 WRITE_ONCE(fbd->uc_addr0, NULL); 55 WRITE_ONCE(fbd->uc_addr4, NULL); 56 57 dev_err(fbd->dev, 58 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 59 reg, reg << 2); 60 61 /* Notify stack that device has lost (PCIe) link */ 62 if (!fbnic_init_failure(fbd)) 63 netif_device_detach(fbd->netdev); 64 65 return ~0U; 66 } 67 68 bool fbnic_fw_present(struct fbnic_dev *fbd) 69 { 70 return !!READ_ONCE(fbd->uc_addr4); 71 } 72 73 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val) 74 { 75 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 76 77 if (csr) 78 writel(val, csr + reg); 79 } 80 81 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg) 82 { 83 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); 84 u32 value; 85 86 if (!csr) 87 return ~0U; 88 89 value = readl(csr + reg); 90 91 /* If any bits are 0 value should be valid */ 92 if (~value) 93 return value; 94 95 /* All 1's may be valid if ZEROs register still works */ 96 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG)) 97 return value; 98 99 /* Hardware is giving us all 1's reads, assume it is gone */ 100 WRITE_ONCE(fbd->uc_addr0, NULL); 101 WRITE_ONCE(fbd->uc_addr4, NULL); 102 103 dev_err(fbd->dev, 104 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", 105 reg, reg << 2); 106 107 /* Notify stack that device has lost (PCIe) link */ 108 if (!fbnic_init_failure(fbd)) 109 netif_device_detach(fbd->netdev); 110 111 return ~0U; 112 } 113 114 static void fbnic_service_task_start(struct fbnic_net *fbn) 115 { 116 struct fbnic_dev *fbd = fbn->fbd; 117 118 schedule_delayed_work(&fbd->service_task, HZ); 119 phylink_resume(fbn->phylink); 120 } 121 122 static void fbnic_service_task_stop(struct fbnic_net *fbn) 123 { 124 struct fbnic_dev *fbd = fbn->fbd; 125 126 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd)); 127 cancel_delayed_work(&fbd->service_task); 128 } 129 130 void fbnic_up(struct fbnic_net *fbn) 131 { 132 fbnic_enable(fbn); 133 134 fbnic_fill(fbn); 135 136 fbnic_rss_reinit_hw(fbn->fbd, fbn); 137 138 __fbnic_set_rx_mode(fbn->netdev); 139 140 /* Enable Tx/Rx processing */ 141 fbnic_napi_enable(fbn); 142 netif_tx_start_all_queues(fbn->netdev); 143 144 fbnic_service_task_start(fbn); 145 } 146 147 static void fbnic_down_noidle(struct fbnic_net *fbn) 148 { 149 fbnic_service_task_stop(fbn); 150 151 /* Disable Tx/Rx Processing */ 152 fbnic_napi_disable(fbn); 153 netif_tx_disable(fbn->netdev); 154 155 fbnic_clear_rx_mode(fbn->netdev); 156 fbnic_clear_rules(fbn->fbd); 157 fbnic_rss_disable_hw(fbn->fbd); 158 fbnic_disable(fbn); 159 } 160 161 void fbnic_down(struct fbnic_net *fbn) 162 { 163 fbnic_down_noidle(fbn); 164 165 fbnic_wait_all_queues_idle(fbn->fbd, false); 166 167 fbnic_flush(fbn); 168 } 169 170 static void fbnic_health_check(struct fbnic_dev *fbd) 171 { 172 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; 173 174 /* As long as the heart is beating the FW is healty */ 175 if (fbd->fw_heartbeat_enabled) 176 return; 177 178 /* If the Tx mailbox still has messages sitting in it then there likely 179 * isn't anything we can do. We will wait until the mailbox is empty to 180 * report the fault so we can collect the crashlog. 181 */ 182 if (tx_mbx->head != tx_mbx->tail) 183 return; 184 185 /* TBD: Need to add a more thorough recovery here. 186 * Specifically I need to verify what all the firmware will have 187 * changed since we had setup and it rebooted. May just need to 188 * perform a down/up. For now we will just reclaim ownership so 189 * the heartbeat can catch the next fault. 190 */ 191 fbnic_fw_xmit_ownership_msg(fbd, true); 192 } 193 194 static void fbnic_service_task(struct work_struct *work) 195 { 196 struct fbnic_dev *fbd = container_of(to_delayed_work(work), 197 struct fbnic_dev, service_task); 198 199 rtnl_lock(); 200 201 fbnic_fw_check_heartbeat(fbd); 202 203 fbnic_health_check(fbd); 204 205 if (netif_carrier_ok(fbd->netdev)) 206 fbnic_napi_depletion_check(fbd->netdev); 207 208 if (netif_running(fbd->netdev)) 209 schedule_delayed_work(&fbd->service_task, HZ); 210 211 rtnl_unlock(); 212 } 213 214 /** 215 * fbnic_probe - Device Initialization Routine 216 * @pdev: PCI device information struct 217 * @ent: entry in fbnic_pci_tbl 218 * 219 * Initializes a PCI device identified by a pci_dev structure. 220 * The OS initialization, configuring of the adapter private structure, 221 * and a hardware reset occur. 222 * 223 * Return: 0 on success, negative on failure 224 **/ 225 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 226 { 227 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data]; 228 struct net_device *netdev; 229 struct fbnic_dev *fbd; 230 int err; 231 232 if (pdev->error_state != pci_channel_io_normal) { 233 dev_err(&pdev->dev, 234 "PCI device still in an error state. Unable to load...\n"); 235 return -EIO; 236 } 237 238 err = pcim_enable_device(pdev); 239 if (err) { 240 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err); 241 return err; 242 } 243 244 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); 245 if (err) 246 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 247 if (err) { 248 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); 249 return err; 250 } 251 252 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name); 253 if (err) { 254 dev_err(&pdev->dev, 255 "pci_request_selected_regions failed: %d\n", err); 256 return err; 257 } 258 259 fbd = fbnic_devlink_alloc(pdev); 260 if (!fbd) { 261 dev_err(&pdev->dev, "Devlink allocation failed\n"); 262 return -ENOMEM; 263 } 264 265 /* Populate driver with hardware-specific info and handlers */ 266 fbd->max_num_queues = info->max_num_queues; 267 268 pci_set_master(pdev); 269 pci_save_state(pdev); 270 271 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task); 272 273 err = fbnic_alloc_irqs(fbd); 274 if (err) 275 goto free_fbd; 276 277 err = fbnic_mac_init(fbd); 278 if (err) { 279 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err); 280 goto free_irqs; 281 } 282 283 err = fbnic_fw_enable_mbx(fbd); 284 if (err) { 285 dev_err(&pdev->dev, 286 "Firmware mailbox initialization failure\n"); 287 goto free_irqs; 288 } 289 290 fbnic_devlink_register(fbd); 291 292 fbnic_hwmon_register(fbd); 293 294 if (!fbd->dsn) { 295 dev_warn(&pdev->dev, "Reading serial number failed\n"); 296 goto init_failure_mode; 297 } 298 299 netdev = fbnic_netdev_alloc(fbd); 300 if (!netdev) { 301 dev_err(&pdev->dev, "Netdev allocation failed\n"); 302 goto init_failure_mode; 303 } 304 305 err = fbnic_ptp_setup(fbd); 306 if (err) 307 goto ifm_free_netdev; 308 309 err = fbnic_netdev_register(netdev); 310 if (err) { 311 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); 312 goto ifm_destroy_ptp; 313 } 314 315 return 0; 316 317 ifm_destroy_ptp: 318 fbnic_ptp_destroy(fbd); 319 ifm_free_netdev: 320 fbnic_netdev_free(fbd); 321 init_failure_mode: 322 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n"); 323 /* Always return 0 even on error so devlink is registered to allow 324 * firmware updates for fixes. 325 */ 326 return 0; 327 free_irqs: 328 fbnic_free_irqs(fbd); 329 free_fbd: 330 pci_disable_device(pdev); 331 fbnic_devlink_free(fbd); 332 333 return err; 334 } 335 336 /** 337 * fbnic_remove - Device Removal Routine 338 * @pdev: PCI device information struct 339 * 340 * Called by the PCI subsystem to alert the driver that it should release 341 * a PCI device. The could be caused by a Hot-Plug event, or because the 342 * driver is going to be removed from memory. 343 **/ 344 static void fbnic_remove(struct pci_dev *pdev) 345 { 346 struct fbnic_dev *fbd = pci_get_drvdata(pdev); 347 348 if (!fbnic_init_failure(fbd)) { 349 struct net_device *netdev = fbd->netdev; 350 351 fbnic_netdev_unregister(netdev); 352 cancel_delayed_work_sync(&fbd->service_task); 353 fbnic_ptp_destroy(fbd); 354 fbnic_netdev_free(fbd); 355 } 356 357 fbnic_hwmon_unregister(fbd); 358 fbnic_devlink_unregister(fbd); 359 fbnic_fw_disable_mbx(fbd); 360 fbnic_free_irqs(fbd); 361 362 pci_disable_device(pdev); 363 fbnic_devlink_free(fbd); 364 } 365 366 static int fbnic_pm_suspend(struct device *dev) 367 { 368 struct fbnic_dev *fbd = dev_get_drvdata(dev); 369 struct net_device *netdev = fbd->netdev; 370 371 if (fbnic_init_failure(fbd)) 372 goto null_uc_addr; 373 374 rtnl_lock(); 375 376 netif_device_detach(netdev); 377 378 if (netif_running(netdev)) 379 netdev->netdev_ops->ndo_stop(netdev); 380 381 rtnl_unlock(); 382 383 null_uc_addr: 384 fbnic_fw_disable_mbx(fbd); 385 386 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */ 387 fbnic_free_irqs(fbd); 388 389 /* Hardware is about to go away, so switch off MMIO access internally */ 390 WRITE_ONCE(fbd->uc_addr0, NULL); 391 WRITE_ONCE(fbd->uc_addr4, NULL); 392 393 return 0; 394 } 395 396 static int __fbnic_pm_resume(struct device *dev) 397 { 398 struct fbnic_dev *fbd = dev_get_drvdata(dev); 399 struct net_device *netdev = fbd->netdev; 400 void __iomem * const *iomap_table; 401 struct fbnic_net *fbn; 402 int err; 403 404 /* Restore MMIO access */ 405 iomap_table = pcim_iomap_table(to_pci_dev(dev)); 406 fbd->uc_addr0 = iomap_table[0]; 407 fbd->uc_addr4 = iomap_table[4]; 408 409 /* Rerequest the IRQs */ 410 err = fbnic_alloc_irqs(fbd); 411 if (err) 412 goto err_invalidate_uc_addr; 413 414 fbd->mac->init_regs(fbd); 415 416 /* Re-enable mailbox */ 417 err = fbnic_fw_enable_mbx(fbd); 418 if (err) 419 goto err_free_irqs; 420 421 /* No netdev means there isn't a network interface to bring up */ 422 if (fbnic_init_failure(fbd)) 423 return 0; 424 425 fbn = netdev_priv(netdev); 426 427 /* Reset the queues if needed */ 428 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); 429 430 rtnl_lock(); 431 432 if (netif_running(netdev)) { 433 err = __fbnic_open(fbn); 434 if (err) 435 goto err_disable_mbx; 436 } 437 438 rtnl_unlock(); 439 440 return 0; 441 err_disable_mbx: 442 rtnl_unlock(); 443 fbnic_fw_disable_mbx(fbd); 444 err_free_irqs: 445 fbnic_free_irqs(fbd); 446 err_invalidate_uc_addr: 447 WRITE_ONCE(fbd->uc_addr0, NULL); 448 WRITE_ONCE(fbd->uc_addr4, NULL); 449 return err; 450 } 451 452 static void __fbnic_pm_attach(struct device *dev) 453 { 454 struct fbnic_dev *fbd = dev_get_drvdata(dev); 455 struct net_device *netdev = fbd->netdev; 456 struct fbnic_net *fbn; 457 458 if (fbnic_init_failure(fbd)) 459 return; 460 461 fbn = netdev_priv(netdev); 462 463 if (netif_running(netdev)) 464 fbnic_up(fbn); 465 466 netif_device_attach(netdev); 467 } 468 469 static int __maybe_unused fbnic_pm_resume(struct device *dev) 470 { 471 int err; 472 473 err = __fbnic_pm_resume(dev); 474 if (!err) 475 __fbnic_pm_attach(dev); 476 477 return err; 478 } 479 480 static const struct dev_pm_ops fbnic_pm_ops = { 481 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume) 482 }; 483 484 static void fbnic_shutdown(struct pci_dev *pdev) 485 { 486 fbnic_pm_suspend(&pdev->dev); 487 } 488 489 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev, 490 pci_channel_state_t state) 491 { 492 /* Disconnect device if failure is not recoverable via reset */ 493 if (state == pci_channel_io_perm_failure) 494 return PCI_ERS_RESULT_DISCONNECT; 495 496 fbnic_pm_suspend(&pdev->dev); 497 498 /* Request a slot reset */ 499 return PCI_ERS_RESULT_NEED_RESET; 500 } 501 502 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev) 503 { 504 int err; 505 506 pci_set_power_state(pdev, PCI_D0); 507 pci_restore_state(pdev); 508 pci_save_state(pdev); 509 510 if (pci_enable_device_mem(pdev)) { 511 dev_err(&pdev->dev, 512 "Cannot re-enable PCI device after reset.\n"); 513 return PCI_ERS_RESULT_DISCONNECT; 514 } 515 516 /* Restore device to previous state */ 517 err = __fbnic_pm_resume(&pdev->dev); 518 519 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 520 } 521 522 static void fbnic_err_resume(struct pci_dev *pdev) 523 { 524 __fbnic_pm_attach(&pdev->dev); 525 } 526 527 static const struct pci_error_handlers fbnic_err_handler = { 528 .error_detected = fbnic_err_error_detected, 529 .slot_reset = fbnic_err_slot_reset, 530 .resume = fbnic_err_resume, 531 }; 532 533 static struct pci_driver fbnic_driver = { 534 .name = fbnic_driver_name, 535 .id_table = fbnic_pci_tbl, 536 .probe = fbnic_probe, 537 .remove = fbnic_remove, 538 .driver.pm = &fbnic_pm_ops, 539 .shutdown = fbnic_shutdown, 540 .err_handler = &fbnic_err_handler, 541 }; 542 543 /** 544 * fbnic_init_module - Driver Registration Routine 545 * 546 * The first routine called when the driver is loaded. All it does is 547 * register with the PCI subsystem. 548 * 549 * Return: 0 on success, negative on failure 550 **/ 551 static int __init fbnic_init_module(void) 552 { 553 int err; 554 555 err = pci_register_driver(&fbnic_driver); 556 if (err) 557 goto out; 558 559 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name); 560 out: 561 return err; 562 } 563 module_init(fbnic_init_module); 564 565 /** 566 * fbnic_exit_module - Driver Exit Cleanup Routine 567 * 568 * Called just before the driver is removed from memory. 569 **/ 570 static void __exit fbnic_exit_module(void) 571 { 572 pci_unregister_driver(&fbnic_driver); 573 } 574 module_exit(fbnic_exit_module); 575