1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
3
4 #include <linux/init.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/rtnetlink.h>
8 #include <linux/types.h>
9 #include <net/devlink.h>
10
11 #include "fbnic.h"
12 #include "fbnic_drvinfo.h"
13 #include "fbnic_hw_stats.h"
14 #include "fbnic_netdev.h"
15
16 char fbnic_driver_name[] = DRV_NAME;
17
18 MODULE_DESCRIPTION(DRV_SUMMARY);
19 MODULE_LICENSE("GPL");
20
21 static const struct fbnic_info fbnic_asic_info = {
22 .max_num_queues = FBNIC_MAX_QUEUES,
23 .bar_mask = BIT(0) | BIT(4)
24 };
25
26 static const struct fbnic_info *fbnic_info_tbl[] = {
27 [fbnic_board_asic] = &fbnic_asic_info,
28 };
29
30 static const struct pci_device_id fbnic_pci_tbl[] = {
31 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },
32 /* Required last entry */
33 {0, }
34 };
35 MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
36
fbnic_rd32(struct fbnic_dev * fbd,u32 reg)37 u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
38 {
39 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
40 u32 value;
41
42 if (!csr)
43 return ~0U;
44
45 value = readl(csr + reg);
46
47 /* If any bits are 0 value should be valid */
48 if (~value)
49 return value;
50
51 /* All 1's may be valid if ZEROs register still works */
52 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
53 return value;
54
55 /* Hardware is giving us all 1's reads, assume it is gone */
56 WRITE_ONCE(fbd->uc_addr0, NULL);
57 WRITE_ONCE(fbd->uc_addr4, NULL);
58
59 dev_err(fbd->dev,
60 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
61 reg, reg << 2);
62
63 /* Notify stack that device has lost (PCIe) link */
64 if (!fbnic_init_failure(fbd))
65 netif_device_detach(fbd->netdev);
66
67 return ~0U;
68 }
69
fbnic_fw_present(struct fbnic_dev * fbd)70 bool fbnic_fw_present(struct fbnic_dev *fbd)
71 {
72 return !!READ_ONCE(fbd->uc_addr4);
73 }
74
fbnic_fw_wr32(struct fbnic_dev * fbd,u32 reg,u32 val)75 void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
76 {
77 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
78
79 if (csr)
80 writel(val, csr + reg);
81 }
82
fbnic_fw_rd32(struct fbnic_dev * fbd,u32 reg)83 u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)
84 {
85 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
86 u32 value;
87
88 if (!csr)
89 return ~0U;
90
91 value = readl(csr + reg);
92
93 /* If any bits are 0 value should be valid */
94 if (~value)
95 return value;
96
97 /* All 1's may be valid if ZEROs register still works */
98 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))
99 return value;
100
101 /* Hardware is giving us all 1's reads, assume it is gone */
102 WRITE_ONCE(fbd->uc_addr0, NULL);
103 WRITE_ONCE(fbd->uc_addr4, NULL);
104
105 dev_err(fbd->dev,
106 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
107 reg, reg << 2);
108
109 /* Notify stack that device has lost (PCIe) link */
110 if (!fbnic_init_failure(fbd))
111 netif_device_detach(fbd->netdev);
112
113 return ~0U;
114 }
115
fbnic_service_task_start(struct fbnic_net * fbn)116 static void fbnic_service_task_start(struct fbnic_net *fbn)
117 {
118 struct fbnic_dev *fbd = fbn->fbd;
119
120 schedule_delayed_work(&fbd->service_task, HZ);
121 }
122
fbnic_service_task_stop(struct fbnic_net * fbn)123 static void fbnic_service_task_stop(struct fbnic_net *fbn)
124 {
125 struct fbnic_dev *fbd = fbn->fbd;
126
127 cancel_delayed_work(&fbd->service_task);
128 }
129
fbnic_up(struct fbnic_net * fbn)130 void fbnic_up(struct fbnic_net *fbn)
131 {
132 fbnic_enable(fbn);
133
134 fbnic_fill(fbn);
135
136 fbnic_rss_reinit_hw(fbn->fbd, fbn);
137
138 __fbnic_set_rx_mode(fbn->fbd);
139
140 /* Enable Tx/Rx processing */
141 fbnic_napi_enable(fbn);
142 netif_tx_start_all_queues(fbn->netdev);
143
144 fbnic_service_task_start(fbn);
145 }
146
fbnic_down_noidle(struct fbnic_net * fbn)147 void fbnic_down_noidle(struct fbnic_net *fbn)
148 {
149 fbnic_service_task_stop(fbn);
150
151 /* Disable Tx/Rx Processing */
152 fbnic_napi_disable(fbn);
153 netif_tx_disable(fbn->netdev);
154
155 fbnic_clear_rx_mode(fbn->fbd);
156 fbnic_clear_rules(fbn->fbd);
157 fbnic_rss_disable_hw(fbn->fbd);
158 fbnic_disable(fbn);
159 }
160
fbnic_down(struct fbnic_net * fbn)161 void fbnic_down(struct fbnic_net *fbn)
162 {
163 fbnic_down_noidle(fbn);
164
165 fbnic_wait_all_queues_idle(fbn->fbd, false);
166
167 fbnic_flush(fbn);
168 }
169
fbnic_fw_config_after_crash(struct fbnic_dev * fbd)170 static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd)
171 {
172 if (fbnic_fw_xmit_ownership_msg(fbd, true)) {
173 dev_err(fbd->dev, "NIC failed to take ownership\n");
174
175 return -1;
176 }
177
178 fbnic_rpc_reset_valid_entries(fbd);
179 __fbnic_set_rx_mode(fbd);
180
181 return 0;
182 }
183
fbnic_health_check(struct fbnic_dev * fbd)184 static void fbnic_health_check(struct fbnic_dev *fbd)
185 {
186 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
187
188 /* As long as the heart is beating the FW is healty */
189 if (fbd->fw_heartbeat_enabled)
190 return;
191
192 /* If the Tx mailbox still has messages sitting in it then there likely
193 * isn't anything we can do. We will wait until the mailbox is empty to
194 * report the fault so we can collect the crashlog.
195 */
196 if (tx_mbx->head != tx_mbx->tail)
197 return;
198
199 fbnic_devlink_fw_report(fbd, "Firmware crashed detected!");
200 fbnic_devlink_otp_check(fbd, "error detected after firmware recovery");
201
202 if (fbnic_fw_config_after_crash(fbd))
203 dev_err(fbd->dev, "Firmware recovery failed after crash\n");
204 }
205
fbnic_service_task(struct work_struct * work)206 static void fbnic_service_task(struct work_struct *work)
207 {
208 struct fbnic_dev *fbd = container_of(to_delayed_work(work),
209 struct fbnic_dev, service_task);
210
211 rtnl_lock();
212
213 fbnic_get_hw_stats32(fbd);
214
215 fbnic_fw_check_heartbeat(fbd);
216
217 fbnic_health_check(fbd);
218
219 fbnic_bmc_rpc_check(fbd);
220
221 if (netif_carrier_ok(fbd->netdev)) {
222 netdev_lock(fbd->netdev);
223 fbnic_napi_depletion_check(fbd->netdev);
224 netdev_unlock(fbd->netdev);
225 }
226
227 if (netif_running(fbd->netdev))
228 schedule_delayed_work(&fbd->service_task, HZ);
229
230 rtnl_unlock();
231 }
232
233 /**
234 * fbnic_probe - Device Initialization Routine
235 * @pdev: PCI device information struct
236 * @ent: entry in fbnic_pci_tbl
237 *
238 * Initializes a PCI device identified by a pci_dev structure.
239 * The OS initialization, configuring of the adapter private structure,
240 * and a hardware reset occur.
241 *
242 * Return: 0 on success, negative on failure
243 **/
fbnic_probe(struct pci_dev * pdev,const struct pci_device_id * ent)244 static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
245 {
246 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];
247 struct net_device *netdev;
248 struct fbnic_dev *fbd;
249 int err;
250
251 if (pdev->error_state != pci_channel_io_normal) {
252 dev_err(&pdev->dev,
253 "PCI device still in an error state. Unable to load...\n");
254 return -EIO;
255 }
256
257 err = pcim_enable_device(pdev);
258 if (err) {
259 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
260 return err;
261 }
262
263 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
264 if (err)
265 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
266 if (err) {
267 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
268 return err;
269 }
270
271 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);
272 if (err) {
273 dev_err(&pdev->dev,
274 "pci_request_selected_regions failed: %d\n", err);
275 return err;
276 }
277
278 fbd = fbnic_devlink_alloc(pdev);
279 if (!fbd) {
280 dev_err(&pdev->dev, "Devlink allocation failed\n");
281 return -ENOMEM;
282 }
283
284 err = fbnic_devlink_health_create(fbd);
285 if (err)
286 goto free_fbd;
287
288 /* Populate driver with hardware-specific info and handlers */
289 fbd->max_num_queues = info->max_num_queues;
290
291 pci_set_master(pdev);
292 pci_save_state(pdev);
293
294 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);
295
296 err = fbnic_alloc_irqs(fbd);
297 if (err)
298 goto err_destroy_health;
299
300 err = fbnic_mac_init(fbd);
301 if (err) {
302 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
303 goto free_irqs;
304 }
305
306 err = fbnic_fw_request_mbx(fbd);
307 if (err) {
308 dev_err(&pdev->dev,
309 "Firmware mailbox initialization failure\n");
310 goto free_irqs;
311 }
312
313 /* Send the request to enable the FW logging to host. Note if this
314 * fails we ignore the error and just display a message as it is
315 * possible the FW is just too old to support the logging and needs
316 * to be updated.
317 */
318 err = fbnic_fw_log_init(fbd);
319 if (err)
320 dev_warn(fbd->dev,
321 "Unable to initialize firmware log buffer: %d\n",
322 err);
323
324 fbnic_devlink_register(fbd);
325 fbnic_devlink_otp_check(fbd, "error detected during probe");
326 fbnic_dbg_fbd_init(fbd);
327
328 /* Capture snapshot of hardware stats so netdev can calculate delta */
329 fbnic_init_hw_stats(fbd);
330
331 fbnic_hwmon_register(fbd);
332
333 if (!fbd->dsn) {
334 dev_warn(&pdev->dev, "Reading serial number failed\n");
335 goto init_failure_mode;
336 }
337
338 netdev = fbnic_netdev_alloc(fbd);
339 if (!netdev) {
340 dev_err(&pdev->dev, "Netdev allocation failed\n");
341 goto init_failure_mode;
342 }
343
344 err = fbnic_ptp_setup(fbd);
345 if (err)
346 goto ifm_free_netdev;
347
348 err = fbnic_netdev_register(netdev);
349 if (err) {
350 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
351 goto ifm_destroy_ptp;
352 }
353
354 return 0;
355
356 ifm_destroy_ptp:
357 fbnic_ptp_destroy(fbd);
358 ifm_free_netdev:
359 fbnic_netdev_free(fbd);
360 init_failure_mode:
361 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");
362 /* Always return 0 even on error so devlink is registered to allow
363 * firmware updates for fixes.
364 */
365 return 0;
366 free_irqs:
367 fbnic_free_irqs(fbd);
368 err_destroy_health:
369 fbnic_devlink_health_destroy(fbd);
370 free_fbd:
371 fbnic_devlink_free(fbd);
372
373 return err;
374 }
375
376 /**
377 * fbnic_remove - Device Removal Routine
378 * @pdev: PCI device information struct
379 *
380 * Called by the PCI subsystem to alert the driver that it should release
381 * a PCI device. The could be caused by a Hot-Plug event, or because the
382 * driver is going to be removed from memory.
383 **/
fbnic_remove(struct pci_dev * pdev)384 static void fbnic_remove(struct pci_dev *pdev)
385 {
386 struct fbnic_dev *fbd = pci_get_drvdata(pdev);
387
388 if (!fbnic_init_failure(fbd)) {
389 struct net_device *netdev = fbd->netdev;
390
391 fbnic_netdev_unregister(netdev);
392 cancel_delayed_work_sync(&fbd->service_task);
393 fbnic_ptp_destroy(fbd);
394 fbnic_netdev_free(fbd);
395 }
396
397 fbnic_hwmon_unregister(fbd);
398 fbnic_dbg_fbd_exit(fbd);
399 fbnic_devlink_unregister(fbd);
400 fbnic_fw_log_free(fbd);
401 fbnic_fw_free_mbx(fbd);
402 fbnic_free_irqs(fbd);
403
404 fbnic_devlink_health_destroy(fbd);
405 fbnic_devlink_free(fbd);
406 }
407
fbnic_pm_suspend(struct device * dev)408 static int fbnic_pm_suspend(struct device *dev)
409 {
410 struct fbnic_dev *fbd = dev_get_drvdata(dev);
411 struct net_device *netdev = fbd->netdev;
412
413 if (fbnic_init_failure(fbd))
414 goto null_uc_addr;
415
416 rtnl_lock();
417 netdev_lock(netdev);
418
419 netif_device_detach(netdev);
420
421 if (netif_running(netdev))
422 netdev->netdev_ops->ndo_stop(netdev);
423
424 netdev_unlock(netdev);
425 rtnl_unlock();
426
427 null_uc_addr:
428 fbnic_fw_log_disable(fbd);
429
430 devl_lock(priv_to_devlink(fbd));
431
432 fbnic_fw_free_mbx(fbd);
433
434 devl_unlock(priv_to_devlink(fbd));
435
436 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */
437 fbnic_free_irqs(fbd);
438
439 /* Hardware is about to go away, so switch off MMIO access internally */
440 WRITE_ONCE(fbd->uc_addr0, NULL);
441 WRITE_ONCE(fbd->uc_addr4, NULL);
442
443 return 0;
444 }
445
__fbnic_pm_resume(struct device * dev)446 static int __fbnic_pm_resume(struct device *dev)
447 {
448 struct fbnic_dev *fbd = dev_get_drvdata(dev);
449 struct net_device *netdev = fbd->netdev;
450 void __iomem * const *iomap_table;
451 struct fbnic_net *fbn;
452 int err;
453
454 /* Restore MMIO access */
455 iomap_table = pcim_iomap_table(to_pci_dev(dev));
456 fbd->uc_addr0 = iomap_table[0];
457 fbd->uc_addr4 = iomap_table[4];
458
459 /* Rerequest the IRQs */
460 err = fbnic_alloc_irqs(fbd);
461 if (err)
462 goto err_invalidate_uc_addr;
463
464 fbd->mac->init_regs(fbd);
465
466 devl_lock(priv_to_devlink(fbd));
467
468 /* Re-enable mailbox */
469 err = fbnic_fw_request_mbx(fbd);
470 devl_unlock(priv_to_devlink(fbd));
471 if (err)
472 goto err_free_irqs;
473
474 /* Only send log history if log buffer is empty to prevent duplicate
475 * log entries.
476 */
477 fbnic_fw_log_enable(fbd, list_empty(&fbd->fw_log.entries));
478
479 /* Since the FW should be up, check if it reported OTP errors */
480 fbnic_devlink_otp_check(fbd, "error detected after PM resume");
481
482 /* No netdev means there isn't a network interface to bring up */
483 if (fbnic_init_failure(fbd))
484 return 0;
485
486 fbn = netdev_priv(netdev);
487
488 /* Reset the queues if needed */
489 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
490
491 rtnl_lock();
492 netdev_lock(netdev);
493
494 if (netif_running(netdev))
495 err = __fbnic_open(fbn);
496
497 netdev_unlock(netdev);
498 rtnl_unlock();
499 if (err)
500 goto err_free_mbx;
501
502 return 0;
503 err_free_mbx:
504 fbnic_fw_log_disable(fbd);
505
506 devl_lock(priv_to_devlink(fbd));
507 fbnic_fw_free_mbx(fbd);
508 devl_unlock(priv_to_devlink(fbd));
509 err_free_irqs:
510 fbnic_free_irqs(fbd);
511 err_invalidate_uc_addr:
512 WRITE_ONCE(fbd->uc_addr0, NULL);
513 WRITE_ONCE(fbd->uc_addr4, NULL);
514 return err;
515 }
516
__fbnic_pm_attach(struct device * dev)517 static void __fbnic_pm_attach(struct device *dev)
518 {
519 struct fbnic_dev *fbd = dev_get_drvdata(dev);
520 struct net_device *netdev = fbd->netdev;
521 struct fbnic_net *fbn;
522
523 rtnl_lock();
524 fbnic_reset_hw_stats(fbd);
525 rtnl_unlock();
526
527 if (fbnic_init_failure(fbd))
528 return;
529
530 fbn = netdev_priv(netdev);
531
532 if (netif_running(netdev))
533 fbnic_up(fbn);
534
535 netif_device_attach(netdev);
536 }
537
fbnic_pm_resume(struct device * dev)538 static int __maybe_unused fbnic_pm_resume(struct device *dev)
539 {
540 int err;
541
542 err = __fbnic_pm_resume(dev);
543 if (!err)
544 __fbnic_pm_attach(dev);
545
546 return err;
547 }
548
549 static const struct dev_pm_ops fbnic_pm_ops = {
550 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)
551 };
552
fbnic_shutdown(struct pci_dev * pdev)553 static void fbnic_shutdown(struct pci_dev *pdev)
554 {
555 fbnic_pm_suspend(&pdev->dev);
556 }
557
fbnic_err_error_detected(struct pci_dev * pdev,pci_channel_state_t state)558 static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,
559 pci_channel_state_t state)
560 {
561 /* Disconnect device if failure is not recoverable via reset */
562 if (state == pci_channel_io_perm_failure)
563 return PCI_ERS_RESULT_DISCONNECT;
564
565 fbnic_pm_suspend(&pdev->dev);
566
567 /* Request a slot reset */
568 return PCI_ERS_RESULT_NEED_RESET;
569 }
570
fbnic_err_slot_reset(struct pci_dev * pdev)571 static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)
572 {
573 int err;
574
575 pci_set_power_state(pdev, PCI_D0);
576 pci_restore_state(pdev);
577 pci_save_state(pdev);
578
579 if (pci_enable_device_mem(pdev)) {
580 dev_err(&pdev->dev,
581 "Cannot re-enable PCI device after reset.\n");
582 return PCI_ERS_RESULT_DISCONNECT;
583 }
584
585 /* Restore device to previous state */
586 err = __fbnic_pm_resume(&pdev->dev);
587
588 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
589 }
590
fbnic_err_resume(struct pci_dev * pdev)591 static void fbnic_err_resume(struct pci_dev *pdev)
592 {
593 __fbnic_pm_attach(&pdev->dev);
594 }
595
596 static const struct pci_error_handlers fbnic_err_handler = {
597 .error_detected = fbnic_err_error_detected,
598 .slot_reset = fbnic_err_slot_reset,
599 .resume = fbnic_err_resume,
600 };
601
602 static struct pci_driver fbnic_driver = {
603 .name = fbnic_driver_name,
604 .id_table = fbnic_pci_tbl,
605 .probe = fbnic_probe,
606 .remove = fbnic_remove,
607 .driver.pm = &fbnic_pm_ops,
608 .shutdown = fbnic_shutdown,
609 .err_handler = &fbnic_err_handler,
610 };
611
612 /**
613 * fbnic_init_module - Driver Registration Routine
614 *
615 * The first routine called when the driver is loaded. All it does is
616 * register with the PCI subsystem.
617 *
618 * Return: 0 on success, negative on failure
619 **/
fbnic_init_module(void)620 static int __init fbnic_init_module(void)
621 {
622 int err;
623
624 fbnic_dbg_init();
625
626 err = pci_register_driver(&fbnic_driver);
627 if (err) {
628 fbnic_dbg_exit();
629 goto out;
630 }
631
632 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);
633 out:
634 return err;
635 }
636 module_init(fbnic_init_module);
637
638 /**
639 * fbnic_exit_module - Driver Exit Cleanup Routine
640 *
641 * Called just before the driver is removed from memory.
642 **/
fbnic_exit_module(void)643 static void __exit fbnic_exit_module(void)
644 {
645 pci_unregister_driver(&fbnic_driver);
646
647 fbnic_dbg_exit();
648 }
649 module_exit(fbnic_exit_module);
650