1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2015 - 2026 Beijing WangXun Technology Co., Ltd. */
3 /* Copyright (c) 1999 - 2026 Intel Corporation. */
4
5 #include <linux/netdevice.h>
6 #include <linux/pci.h>
7 #include <linux/aer.h>
8
9 #include "wx_type.h"
10 #include "wx_lib.h"
11 #include "wx_err.h"
12
13 /**
14 * wx_io_error_detected - called when PCI error is detected
15 * @pdev: Pointer to PCI device
16 * @state: The current pci connection state
17 *
18 * Return: pci_ers_result_t.
19 *
20 * This function is called after a PCI bus error affecting
21 * this device has been detected.
22 */
wx_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)23 static pci_ers_result_t wx_io_error_detected(struct pci_dev *pdev,
24 pci_channel_state_t state)
25 {
26 struct wx *wx = pci_get_drvdata(pdev);
27 struct net_device *netdev;
28
29 if (!wx)
30 return PCI_ERS_RESULT_DISCONNECT;
31
32 netdev = wx->netdev;
33 if (!netif_device_present(netdev))
34 return PCI_ERS_RESULT_DISCONNECT;
35
36 rtnl_lock();
37 netif_device_detach(netdev);
38 set_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags);
39 wx_soft_quiesce(wx);
40
41 if (state == pci_channel_io_perm_failure) {
42 rtnl_unlock();
43 return PCI_ERS_RESULT_DISCONNECT;
44 }
45
46 if (!test_and_set_bit(WX_STATE_DISABLED, wx->state))
47 pci_disable_device(pdev);
48 rtnl_unlock();
49
50 /* Request a slot reset. */
51 return PCI_ERS_RESULT_NEED_RESET;
52 }
53
54 /**
55 * wx_io_slot_reset - called after the pci bus has been reset.
56 * @pdev: Pointer to PCI device
57 *
58 * Return: pci_ers_result_t.
59 *
60 * Restart the card from scratch, as if from a cold-boot.
61 */
wx_io_slot_reset(struct pci_dev * pdev)62 static pci_ers_result_t wx_io_slot_reset(struct pci_dev *pdev)
63 {
64 struct wx *wx = pci_get_drvdata(pdev);
65
66 if (pci_enable_device_mem(pdev)) {
67 wx_err(wx, "Cannot re-enable PCI device after reset.\n");
68 return PCI_ERS_RESULT_DISCONNECT;
69 }
70
71 /* make all memory operations done before clearing the flag */
72 smp_mb__before_atomic();
73 clear_bit(WX_STATE_DISABLED, wx->state);
74 clear_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags);
75 pci_set_master(pdev);
76 pci_restore_state(pdev);
77 pci_wake_from_d3(pdev, false);
78
79 rtnl_lock();
80 if (netif_running(wx->netdev) && wx->down_suspend)
81 wx->down_suspend(wx);
82 if (wx->do_reset)
83 wx->do_reset(wx->netdev, false);
84 rtnl_unlock();
85
86 return PCI_ERS_RESULT_RECOVERED;
87 }
88
89 /**
90 * wx_io_resume - called when traffic can start flowing again.
91 * @pdev: Pointer to PCI device
92 *
93 * This callback is called when the error recovery driver tells us that
94 * its OK to resume normal operation.
95 */
wx_io_resume(struct pci_dev * pdev)96 static void wx_io_resume(struct pci_dev *pdev)
97 {
98 struct wx *wx = pci_get_drvdata(pdev);
99 struct net_device *netdev;
100 int err;
101
102 netdev = wx->netdev;
103 rtnl_lock();
104 if (netif_running(netdev)) {
105 err = netdev->netdev_ops->ndo_open(netdev);
106 if (err) {
107 wx_err(wx, "Failed to open netdev after reset\n");
108 goto out;
109 }
110 }
111 netif_device_attach(netdev);
112 out:
113 rtnl_unlock();
114 }
115
116 const struct pci_error_handlers wx_err_handler = {
117 .error_detected = wx_io_error_detected,
118 .slot_reset = wx_io_slot_reset,
119 .resume = wx_io_resume,
120 };
121 EXPORT_SYMBOL(wx_err_handler);
122
wx_check_pcie_error(struct wx * wx)123 static bool wx_check_pcie_error(struct wx *wx)
124 {
125 u16 vid, pci_cmd;
126
127 pci_read_config_word(wx->pdev, PCI_VENDOR_ID, &vid);
128 pci_read_config_word(wx->pdev, PCI_COMMAND, &pci_cmd);
129
130 /* PCIe link loss or memory space can't access */
131 if (vid == U16_MAX || !(pci_cmd & PCI_COMMAND_MEMORY))
132 return true;
133
134 return false;
135 }
136
wx_pf_reset_subtask(struct wx * wx)137 static void wx_pf_reset_subtask(struct wx *wx)
138 {
139 if (!test_and_clear_bit(WX_FLAG_NEED_DO_RESET, wx->flags))
140 return;
141
142 wx_warn(wx, "Reset adapter.\n");
143 if (wx->do_reset)
144 wx->do_reset(wx->netdev, true);
145 }
146
wx_reset_task(struct work_struct * work)147 static void wx_reset_task(struct work_struct *work)
148 {
149 struct wx *wx = container_of(work, struct wx, reset_task);
150
151 rtnl_lock();
152
153 /* If the device has been detached (e.g., due to AER error handling),
154 * abort the reset task to prevent operating on a dead or unmanaged
155 * hardware.
156 */
157 if (!netif_device_present(wx->netdev))
158 goto out;
159
160 if (test_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags)) {
161 /* Double check: Verify if the PCIe error is still present. */
162 if (wx_check_pcie_error(wx))
163 wx_soft_quiesce(wx);
164 else
165 clear_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags);
166 goto out;
167 }
168
169 if (test_bit(WX_STATE_DOWN, wx->state) ||
170 test_bit(WX_STATE_RESETTING, wx->state))
171 goto out;
172
173 wx_pf_reset_subtask(wx);
174
175 out:
176 rtnl_unlock();
177 }
178
wx_check_err_subtask(struct wx * wx)179 void wx_check_err_subtask(struct wx *wx)
180 {
181 if (test_bit(WX_FLAG_NEED_DO_RESET, wx->flags))
182 queue_work(wx->reset_wq, &wx->reset_task);
183 }
184 EXPORT_SYMBOL(wx_check_err_subtask);
185
wx_init_err_task(struct wx * wx)186 int wx_init_err_task(struct wx *wx)
187 {
188 wx->reset_wq = alloc_workqueue("%s_reset_wq_%x", WQ_UNBOUND | WQ_HIGHPRI,
189 1, wx->driver_name, pci_dev_id(wx->pdev));
190 if (!wx->reset_wq) {
191 wx_err(wx, "Failed to create wx_reset_wq workqueue\n");
192 return -ENOMEM;
193 }
194
195 INIT_WORK(&wx->reset_task, wx_reset_task);
196 return 0;
197 }
198 EXPORT_SYMBOL(wx_init_err_task);
199
wx_ring_tx_pending(struct wx * wx)200 static bool wx_ring_tx_pending(struct wx *wx)
201 {
202 int i;
203
204 for (i = 0; i < wx->num_tx_queues; i++) {
205 struct wx_ring *tx_ring = wx->tx_ring[i];
206
207 if (tx_ring->next_to_use != tx_ring->next_to_clean)
208 return true;
209 }
210
211 return false;
212 }
213
wx_vf_tx_pending(struct wx * wx)214 static bool wx_vf_tx_pending(struct wx *wx)
215 {
216 struct wx_ring_feature *vmdq = &wx->ring_feature[RING_F_VMDQ];
217 u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask);
218 u32 i, j;
219
220 if (!wx->num_vfs)
221 return false;
222
223 for (i = 0; i < wx->num_vfs; i++) {
224 for (j = 0; j < q_per_pool; j++) {
225 u32 h, t;
226
227 h = rd32(wx, WX_PX_TR_RP_PV(q_per_pool, i, j));
228 t = rd32(wx, WX_PX_TR_WP_PV(q_per_pool, i, j));
229
230 if (h != t)
231 return true;
232 }
233 }
234
235 return false;
236 }
237
wx_watchdog_flush_tx(struct wx * wx)238 static void wx_watchdog_flush_tx(struct wx *wx)
239 {
240 if (!netif_running(wx->netdev))
241 return;
242 if (netif_carrier_ok(wx->netdev))
243 return;
244
245 if (wx_ring_tx_pending(wx) || wx_vf_tx_pending(wx)) {
246 /* We've lost link, so the controller stops DMA,
247 * but we've got queued Tx work that's never going
248 * to get done, so reset controller to flush Tx.
249 * (Do the reset outside of interrupt context).
250 */
251 wx_warn(wx, "initiating reset due to lost link with pending Tx work\n");
252 set_bit(WX_FLAG_NEED_DO_RESET, wx->flags);
253 }
254 }
255
wx_detect_tx_hang(struct wx * wx)256 static void wx_detect_tx_hang(struct wx *wx)
257 {
258 int i;
259
260 /* If we're down or resetting, just bail */
261 if (!netif_running(wx->netdev) ||
262 test_bit(WX_STATE_RESETTING, wx->state))
263 return;
264
265 /* Force detection of hung controller */
266 if (netif_carrier_ok(wx->netdev)) {
267 for (i = 0; i < wx->num_tx_queues; i++)
268 set_bit(WX_TX_DETECT_HANG, wx->tx_ring[i]->state);
269 }
270 }
271
wx_check_hang_subtask(struct wx * wx)272 void wx_check_hang_subtask(struct wx *wx)
273 {
274 if (test_bit(WX_STATE_DOWN, wx->state) ||
275 test_bit(WX_STATE_RESETTING, wx->state))
276 return;
277
278 wx_watchdog_flush_tx(wx);
279 wx_detect_tx_hang(wx);
280 }
281 EXPORT_SYMBOL(wx_check_hang_subtask);
282
wx_tx_timeout_recovery(struct wx * wx)283 static void wx_tx_timeout_recovery(struct wx *wx)
284 {
285 /*
286 * When a PCIe hardware error occurs, the driver should initiate a PCIe
287 * recovery mechanism. However, this recovery flow relies on the AER
288 * driver for current kernel policy. Therefore, a self-contained
289 * recovery mechanism is not implemented yet.
290 */
291 set_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags);
292 wx_err(wx, "PCIe error detected during tx timeout\n");
293 queue_work(wx->reset_wq, &wx->reset_task);
294 }
295
wx_tx_timeout_reset(struct wx * wx)296 static void wx_tx_timeout_reset(struct wx *wx)
297 {
298 if (test_bit(WX_STATE_DOWN, wx->state))
299 return;
300
301 set_bit(WX_FLAG_NEED_DO_RESET, wx->flags);
302 wx_warn(wx, "initiating reset due to tx timeout\n");
303 wx_service_event_schedule(wx);
304 }
305
wx_tx_timeout(struct net_device * netdev,unsigned int __always_unused txqueue)306 void wx_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
307 {
308 struct wx *wx = netdev_priv(netdev);
309
310 if (wx_check_pcie_error(wx))
311 wx_tx_timeout_recovery(wx);
312 else
313 wx_tx_timeout_reset(wx);
314 }
315 EXPORT_SYMBOL(wx_tx_timeout);
316
wx_handle_tx_hang(struct wx_ring * tx_ring,unsigned int next)317 void wx_handle_tx_hang(struct wx_ring *tx_ring, unsigned int next)
318 {
319 struct wx *wx = netdev_priv(tx_ring->netdev);
320
321 wx_warn(wx,
322 "Detected Tx Unit Hang: Queue %d, TDH %x, TDT %x, ntu %x, ntc %x, ntc.time_stamp %lx, jiffies %lx\n",
323 tx_ring->queue_index,
324 rd32(wx, WX_PX_TR_RP(tx_ring->reg_idx)),
325 rd32(wx, WX_PX_TR_WP(tx_ring->reg_idx)),
326 tx_ring->next_to_use, next,
327 tx_ring->tx_buffer_info[next].time_stamp, jiffies);
328
329 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
330
331 wx_tx_timeout_reset(wx);
332 }
333