1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
3
4 #include <linux/init.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/rtnetlink.h>
8 #include <linux/types.h>
9 #include <net/devlink.h>
10
11 #include "fbnic.h"
12 #include "fbnic_drvinfo.h"
13 #include "fbnic_hw_stats.h"
14 #include "fbnic_netdev.h"
15
16 char fbnic_driver_name[] = DRV_NAME;
17
18 MODULE_DESCRIPTION(DRV_SUMMARY);
19 MODULE_LICENSE("GPL");
20
21 static const struct fbnic_info fbnic_asic_info = {
22 .max_num_queues = FBNIC_MAX_QUEUES,
23 .bar_mask = BIT(0) | BIT(4)
24 };
25
26 static const struct fbnic_info *fbnic_info_tbl[] = {
27 [fbnic_board_asic] = &fbnic_asic_info,
28 };
29
30 static const struct pci_device_id fbnic_pci_tbl[] = {
31 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },
32 /* Required last entry */
33 {0, }
34 };
35 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
36
fbnic_rd32(struct fbnic_dev * fbd,u32 reg)37 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
38 {
39 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
40 u32 value;
41
42 if (!csr)
43 return ~0U;
44
45 value = readl(csr + reg);
46
47 /* If any bits are 0 value should be valid */
48 if (~value)
49 return value;
50
51 /* All 1's may be valid if ZEROs register still works */
52 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
53 return value;
54
55 /* Hardware is giving us all 1's reads, assume it is gone */
56 WRITE_ONCE(fbd->uc_addr0, NULL);
57 WRITE_ONCE(fbd->uc_addr4, NULL);
58
59 dev_err(fbd->dev,
60 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
61 reg, reg << 2);
62
63 /* Notify stack that device has lost (PCIe) link */
64 if (!fbnic_init_failure(fbd))
65 netif_device_detach(fbd->netdev);
66
67 return ~0U;
68 }
69
fbnic_fw_present(struct fbnic_dev * fbd)70 bool fbnic_fw_present(struct fbnic_dev *fbd)
71 {
72 return !!READ_ONCE(fbd->uc_addr4);
73 }
74
fbnic_fw_wr32(struct fbnic_dev * fbd,u32 reg,u32 val)75 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
76 {
77 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
78
79 if (csr)
80 writel(val, csr + reg);
81 }
82
fbnic_fw_rd32(struct fbnic_dev * fbd,u32 reg)83 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)
84 {
85 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
86 u32 value;
87
88 if (!csr)
89 return ~0U;
90
91 value = readl(csr + reg);
92
93 /* If any bits are 0 value should be valid */
94 if (~value)
95 return value;
96
97 /* All 1's may be valid if ZEROs register still works */
98 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))
99 return value;
100
101 /* Hardware is giving us all 1's reads, assume it is gone */
102 WRITE_ONCE(fbd->uc_addr0, NULL);
103 WRITE_ONCE(fbd->uc_addr4, NULL);
104
105 dev_err(fbd->dev,
106 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
107 reg, reg << 2);
108
109 /* Notify stack that device has lost (PCIe) link */
110 if (!fbnic_init_failure(fbd))
111 netif_device_detach(fbd->netdev);
112
113 return ~0U;
114 }
115
fbnic_service_task_start(struct fbnic_net * fbn)116 static void fbnic_service_task_start(struct fbnic_net *fbn)
117 {
118 struct fbnic_dev *fbd = fbn->fbd;
119
120 schedule_delayed_work(&fbd->service_task, HZ);
121 }
122
fbnic_service_task_stop(struct fbnic_net * fbn)123 static void fbnic_service_task_stop(struct fbnic_net *fbn)
124 {
125 struct fbnic_dev *fbd = fbn->fbd;
126
127 cancel_delayed_work(&fbd->service_task);
128 }
129
fbnic_up(struct fbnic_net * fbn)130 void fbnic_up(struct fbnic_net *fbn)
131 {
132 fbnic_enable(fbn);
133
134 fbnic_fill(fbn);
135
136 fbnic_rss_reinit_hw(fbn->fbd, fbn);
137
138 netif_addr_lock_bh(fbn->netdev);
139 __fbnic_set_rx_mode(fbn->fbd, &fbn->netdev->uc, &fbn->netdev->mc);
140 netif_addr_unlock_bh(fbn->netdev);
141
142 /* Enable Tx/Rx processing */
143 fbnic_napi_enable(fbn);
144 netif_tx_wake_all_queues(fbn->netdev);
145
146 fbnic_service_task_start(fbn);
147
148 fbnic_dbg_up(fbn);
149 }
150
fbnic_down_noidle(struct fbnic_net * fbn)151 void fbnic_down_noidle(struct fbnic_net *fbn)
152 {
153 fbnic_dbg_down(fbn);
154
155 fbnic_service_task_stop(fbn);
156
157 /* Disable Tx/Rx Processing */
158 fbnic_napi_disable(fbn);
159 netif_tx_disable(fbn->netdev);
160
161 fbnic_clear_rx_mode(fbn->fbd);
162 fbnic_clear_rules(fbn->fbd);
163 fbnic_rss_disable_hw(fbn->fbd);
164 fbnic_disable(fbn);
165 }
166
fbnic_down(struct fbnic_net * fbn)167 void fbnic_down(struct fbnic_net *fbn)
168 {
169 fbnic_down_noidle(fbn);
170
171 fbnic_wait_all_queues_idle(fbn->fbd, false);
172
173 fbnic_flush(fbn);
174 }
175
fbnic_fw_config_after_crash(struct fbnic_dev * fbd)176 static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd)
177 {
178 if (fbnic_fw_xmit_ownership_msg(fbd, true)) {
179 dev_err(fbd->dev, "NIC failed to take ownership\n");
180
181 return -1;
182 }
183
184 fbnic_rpc_reset_valid_entries(fbd);
185 netif_addr_lock_bh(fbd->netdev);
186 __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc);
187 netif_addr_unlock_bh(fbd->netdev);
188
189 return 0;
190 }
191
fbnic_health_check(struct fbnic_dev * fbd)192 static void fbnic_health_check(struct fbnic_dev *fbd)
193 {
194 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
195
196 /* As long as the heart is beating the FW is healthy */
197 if (fbd->fw_heartbeat_enabled)
198 return;
199
200 /* If the Tx mailbox still has messages sitting in it then there likely
201 * isn't anything we can do. We will wait until the mailbox is empty to
202 * report the fault so we can collect the crashlog.
203 */
204 if (tx_mbx->head != tx_mbx->tail)
205 return;
206
207 fbnic_devlink_fw_report(fbd, "Firmware crash detected!");
208 fbnic_devlink_otp_check(fbd, "error detected after firmware recovery");
209
210 if (fbnic_fw_config_after_crash(fbd))
211 dev_err(fbd->dev, "Firmware recovery failed after crash\n");
212 }
213
fbnic_service_task(struct work_struct * work)214 static void fbnic_service_task(struct work_struct *work)
215 {
216 struct fbnic_dev *fbd = container_of(to_delayed_work(work),
217 struct fbnic_dev, service_task);
218 struct net_device *netdev = fbd->netdev;
219
220 if (netif_running(netdev))
221 fbnic_phylink_pmd_training_complete_notify(netdev);
222
223 rtnl_lock();
224
225 fbnic_get_hw_stats32(fbd);
226
227 if (fbd->ps_timeout && fbnic_mac_check_tx_pause(fbd))
228 fbnic_mac_ps_protect_handler(fbd);
229
230 fbnic_fw_check_heartbeat(fbd);
231
232 fbnic_health_check(fbd);
233
234 fbnic_bmc_rpc_check(fbd);
235
236 if (netif_carrier_ok(fbd->netdev)) {
237 netdev_lock(fbd->netdev);
238 fbnic_napi_depletion_check(fbd->netdev);
239 netdev_unlock(fbd->netdev);
240 }
241
242 if (netif_running(netdev))
243 schedule_delayed_work(&fbd->service_task, HZ);
244
245 rtnl_unlock();
246 }
247
248 /**
249 * fbnic_probe - Device Initialization Routine
250 * @pdev: PCI device information struct
251 * @ent: entry in fbnic_pci_tbl
252 *
253 * Initializes a PCI device identified by a pci_dev structure.
254 * The OS initialization, configuring of the adapter private structure,
255 * and a hardware reset occur.
256 *
257 * Return: 0 on success, negative on failure
258 **/
fbnic_probe(struct pci_dev * pdev,const struct pci_device_id * ent)259 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
260 {
261 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];
262 struct net_device *netdev;
263 struct fbnic_dev *fbd;
264 int err;
265
266 if (pdev->error_state != pci_channel_io_normal) {
267 dev_err(&pdev->dev,
268 "PCI device still in an error state. Unable to load...\n");
269 return -EIO;
270 }
271
272 err = pcim_enable_device(pdev);
273 if (err) {
274 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
275 return err;
276 }
277
278 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
279 if (err)
280 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
281 if (err) {
282 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
283 return err;
284 }
285
286 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);
287 if (err) {
288 dev_err(&pdev->dev,
289 "pci_request_selected_regions failed: %d\n", err);
290 return err;
291 }
292
293 fbd = fbnic_devlink_alloc(pdev);
294 if (!fbd) {
295 dev_err(&pdev->dev, "Devlink allocation failed\n");
296 return -ENOMEM;
297 }
298
299 err = fbnic_devlink_health_create(fbd);
300 if (err)
301 goto free_fbd;
302
303 /* Populate driver with hardware-specific info and handlers */
304 fbd->max_num_queues = info->max_num_queues;
305
306 fbd->ps_timeout = FBNIC_MAC_PS_TO_DEFAULT_MS;
307
308 pci_set_master(pdev);
309 pci_save_state(pdev);
310
311 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);
312
313 err = fbnic_alloc_irqs(fbd);
314 if (err)
315 goto err_destroy_health;
316
317 err = fbnic_mac_init(fbd);
318 if (err) {
319 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
320 goto free_irqs;
321 }
322
323 err = fbnic_fw_log_init(fbd);
324 if (err)
325 dev_warn(fbd->dev,
326 "Unable to initialize firmware log buffer: %d\n",
327 err);
328
329 err = fbnic_fw_request_mbx(fbd);
330 if (err) {
331 dev_err(&pdev->dev,
332 "Firmware mailbox initialization failure\n");
333 goto free_fw_log;
334 }
335
336 /* Send the request to enable the FW logging to host. Note if this
337 * fails we ignore the error and just display a message as it is
338 * possible the FW is just too old to support the logging and needs
339 * to be updated.
340 */
341 fbnic_fw_log_enable(fbd, true);
342
343 fbnic_devlink_register(fbd);
344 fbnic_devlink_otp_check(fbd, "error detected during probe");
345 fbnic_dbg_fbd_init(fbd);
346
347 /* Capture snapshot of hardware stats so netdev can calculate delta */
348 fbnic_init_hw_stats(fbd);
349
350 fbnic_hwmon_register(fbd);
351
352 if (!fbd->dsn) {
353 dev_warn(&pdev->dev, "Reading serial number failed\n");
354 goto init_failure_mode;
355 }
356
357 if (fbnic_mdiobus_create(fbd))
358 goto init_failure_mode;
359
360 netdev = fbnic_netdev_alloc(fbd);
361 if (!netdev) {
362 dev_err(&pdev->dev, "Netdev allocation failed\n");
363 goto init_failure_mode;
364 }
365
366 err = fbnic_ptp_setup(fbd);
367 if (err)
368 goto ifm_free_netdev;
369
370 err = fbnic_netdev_register(netdev);
371 if (err) {
372 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
373 goto ifm_destroy_ptp;
374 }
375
376 return 0;
377
378 ifm_destroy_ptp:
379 fbnic_ptp_destroy(fbd);
380 ifm_free_netdev:
381 fbnic_netdev_free(fbd);
382 init_failure_mode:
383 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");
384 /* Always return 0 even on error so devlink is registered to allow
385 * firmware updates for fixes.
386 */
387 return 0;
388 free_fw_log:
389 fbnic_fw_log_free(fbd);
390 free_irqs:
391 fbnic_free_irqs(fbd);
392 err_destroy_health:
393 fbnic_devlink_health_destroy(fbd);
394 free_fbd:
395 fbnic_devlink_free(fbd);
396
397 return err;
398 }
399
400 /**
401 * fbnic_remove - Device Removal Routine
402 * @pdev: PCI device information struct
403 *
404 * Called by the PCI subsystem to alert the driver that it should release
405 * a PCI device. This could be caused by a Hot-Plug event, or because the
406 * driver is going to be removed from memory.
407 **/
fbnic_remove(struct pci_dev * pdev)408 static void fbnic_remove(struct pci_dev *pdev)
409 {
410 struct fbnic_dev *fbd = pci_get_drvdata(pdev);
411
412 if (!fbnic_init_failure(fbd)) {
413 struct net_device *netdev = fbd->netdev;
414
415 fbnic_netdev_unregister(netdev);
416 cancel_delayed_work_sync(&fbd->service_task);
417 fbnic_ptp_destroy(fbd);
418 fbnic_netdev_free(fbd);
419 }
420
421 fbnic_hwmon_unregister(fbd);
422 fbnic_dbg_fbd_exit(fbd);
423 fbnic_devlink_unregister(fbd);
424 fbnic_fw_log_disable(fbd);
425 fbnic_fw_free_mbx(fbd);
426 fbnic_fw_log_free(fbd);
427 fbnic_free_irqs(fbd);
428
429 fbnic_devlink_health_destroy(fbd);
430 fbnic_devlink_free(fbd);
431 }
432
fbnic_pm_suspend(struct device * dev)433 static int fbnic_pm_suspend(struct device *dev)
434 {
435 struct fbnic_dev *fbd = dev_get_drvdata(dev);
436 struct net_device *netdev = fbd->netdev;
437
438 if (fbnic_init_failure(fbd))
439 goto null_uc_addr;
440
441 rtnl_lock();
442 netdev_lock(netdev);
443
444 netif_device_detach(netdev);
445
446 if (netif_running(netdev))
447 netdev->netdev_ops->ndo_stop(netdev);
448
449 netdev_unlock(netdev);
450 rtnl_unlock();
451
452 null_uc_addr:
453 fbnic_fw_log_disable(fbd);
454
455 devl_lock(priv_to_devlink(fbd));
456
457 fbnic_fw_free_mbx(fbd);
458
459 devl_unlock(priv_to_devlink(fbd));
460
461 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */
462 fbnic_free_irqs(fbd);
463
464 /* Hardware is about to go away, so switch off MMIO access internally */
465 WRITE_ONCE(fbd->uc_addr0, NULL);
466 WRITE_ONCE(fbd->uc_addr4, NULL);
467
468 return 0;
469 }
470
__fbnic_pm_resume(struct device * dev)471 static int __fbnic_pm_resume(struct device *dev)
472 {
473 struct fbnic_dev *fbd = dev_get_drvdata(dev);
474 struct net_device *netdev = fbd->netdev;
475 void __iomem * const *iomap_table;
476 struct fbnic_net *fbn;
477 int err;
478
479 /* Restore MMIO access */
480 iomap_table = pcim_iomap_table(to_pci_dev(dev));
481 fbd->uc_addr0 = iomap_table[0];
482 fbd->uc_addr4 = iomap_table[4];
483
484 /* Rerequest the IRQs */
485 err = fbnic_alloc_irqs(fbd);
486 if (err)
487 goto err_invalidate_uc_addr;
488
489 fbd->mac->init_regs(fbd);
490
491 devl_lock(priv_to_devlink(fbd));
492
493 /* Re-enable mailbox */
494 err = fbnic_fw_request_mbx(fbd);
495 devl_unlock(priv_to_devlink(fbd));
496 if (err)
497 goto err_free_irqs;
498
499 /* Only send log history if log buffer is empty to prevent duplicate
500 * log entries.
501 */
502 fbnic_fw_log_enable(fbd, list_empty(&fbd->fw_log.entries));
503
504 /* Since the FW should be up, check if it reported OTP errors */
505 fbnic_devlink_otp_check(fbd, "error detected after PM resume");
506
507 /* No netdev means there isn't a network interface to bring up */
508 if (fbnic_init_failure(fbd))
509 return 0;
510
511 fbn = netdev_priv(netdev);
512
513 /* Reset the queues if needed */
514 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
515
516 rtnl_lock();
517 netdev_lock(netdev);
518
519 if (netif_running(netdev))
520 err = __fbnic_open(fbn);
521
522 netdev_unlock(netdev);
523 rtnl_unlock();
524 if (err)
525 goto err_free_mbx;
526
527 return 0;
528 err_free_mbx:
529 fbnic_fw_log_disable(fbd);
530
531 devl_lock(priv_to_devlink(fbd));
532 fbnic_fw_free_mbx(fbd);
533 devl_unlock(priv_to_devlink(fbd));
534 err_free_irqs:
535 fbnic_free_irqs(fbd);
536 err_invalidate_uc_addr:
537 WRITE_ONCE(fbd->uc_addr0, NULL);
538 WRITE_ONCE(fbd->uc_addr4, NULL);
539 return err;
540 }
541
__fbnic_pm_attach(struct device * dev)542 static void __fbnic_pm_attach(struct device *dev)
543 {
544 struct fbnic_dev *fbd = dev_get_drvdata(dev);
545 struct net_device *netdev = fbd->netdev;
546 struct fbnic_net *fbn;
547
548 rtnl_lock();
549 fbnic_reset_hw_stats(fbd);
550 rtnl_unlock();
551
552 if (fbnic_init_failure(fbd))
553 return;
554
555 fbn = netdev_priv(netdev);
556
557 if (netif_running(netdev))
558 fbnic_up(fbn);
559
560 netif_device_attach(netdev);
561 }
562
fbnic_pm_resume(struct device * dev)563 static int __maybe_unused fbnic_pm_resume(struct device *dev)
564 {
565 int err;
566
567 err = __fbnic_pm_resume(dev);
568 if (!err)
569 __fbnic_pm_attach(dev);
570
571 return err;
572 }
573
574 static const struct dev_pm_ops fbnic_pm_ops = {
575 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)
576 };
577
fbnic_shutdown(struct pci_dev * pdev)578 static void fbnic_shutdown(struct pci_dev *pdev)
579 {
580 fbnic_pm_suspend(&pdev->dev);
581 }
582
fbnic_err_error_detected(struct pci_dev * pdev,pci_channel_state_t state)583 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,
584 pci_channel_state_t state)
585 {
586 /* Disconnect device if failure is not recoverable via reset */
587 if (state == pci_channel_io_perm_failure)
588 return PCI_ERS_RESULT_DISCONNECT;
589
590 fbnic_pm_suspend(&pdev->dev);
591
592 /* Request a slot reset */
593 return PCI_ERS_RESULT_NEED_RESET;
594 }
595
fbnic_err_slot_reset(struct pci_dev * pdev)596 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)
597 {
598 int err;
599
600 pci_set_power_state(pdev, PCI_D0);
601 pci_restore_state(pdev);
602
603 if (pci_enable_device_mem(pdev)) {
604 dev_err(&pdev->dev,
605 "Cannot re-enable PCI device after reset.\n");
606 return PCI_ERS_RESULT_DISCONNECT;
607 }
608
609 /* Restore device to previous state */
610 err = __fbnic_pm_resume(&pdev->dev);
611
612 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
613 }
614
fbnic_err_resume(struct pci_dev * pdev)615 static void fbnic_err_resume(struct pci_dev *pdev)
616 {
617 __fbnic_pm_attach(&pdev->dev);
618 }
619
620 static const struct pci_error_handlers fbnic_err_handler = {
621 .error_detected = fbnic_err_error_detected,
622 .slot_reset = fbnic_err_slot_reset,
623 .resume = fbnic_err_resume,
624 };
625
626 static struct pci_driver fbnic_driver = {
627 .name = fbnic_driver_name,
628 .id_table = fbnic_pci_tbl,
629 .probe = fbnic_probe,
630 .remove = fbnic_remove,
631 .driver.pm = &fbnic_pm_ops,
632 .shutdown = fbnic_shutdown,
633 .err_handler = &fbnic_err_handler,
634 };
635
636 /**
637 * fbnic_init_module - Driver Registration Routine
638 *
639 * The first routine called when the driver is loaded. All it does is
640 * register with the PCI subsystem.
641 *
642 * Return: 0 on success, negative on failure
643 **/
fbnic_init_module(void)644 static int __init fbnic_init_module(void)
645 {
646 int err;
647
648 fbnic_dbg_init();
649
650 err = pci_register_driver(&fbnic_driver);
651 if (err) {
652 fbnic_dbg_exit();
653 goto out;
654 }
655
656 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);
657 out:
658 return err;
659 }
660 module_init(fbnic_init_module);
661
662 /**
663 * fbnic_exit_module - Driver Exit Cleanup Routine
664 *
665 * Called just before the driver is removed from memory.
666 **/
fbnic_exit_module(void)667 static void __exit fbnic_exit_module(void)
668 {
669 pci_unregister_driver(&fbnic_driver);
670
671 fbnic_dbg_exit();
672 }
673 module_exit(fbnic_exit_module);
674