1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2015 - 2026 Beijing WangXun Technology Co., Ltd. */ 3 /* Copyright (c) 1999 - 2026 Intel Corporation. */ 4 5 #include <linux/netdevice.h> 6 #include <linux/pci.h> 7 #include <linux/aer.h> 8 9 #include "wx_type.h" 10 #include "wx_lib.h" 11 #include "wx_err.h" 12 13 /** 14 * wx_io_error_detected - called when PCI error is detected 15 * @pdev: Pointer to PCI device 16 * @state: The current pci connection state 17 * 18 * Return: pci_ers_result_t. 19 * 20 * This function is called after a PCI bus error affecting 21 * this device has been detected. 22 */ 23 static pci_ers_result_t wx_io_error_detected(struct pci_dev *pdev, 24 pci_channel_state_t state) 25 { 26 struct wx *wx = pci_get_drvdata(pdev); 27 struct net_device *netdev; 28 29 if (!wx) 30 return PCI_ERS_RESULT_DISCONNECT; 31 32 netdev = wx->netdev; 33 if (!netif_device_present(netdev)) 34 return PCI_ERS_RESULT_DISCONNECT; 35 36 rtnl_lock(); 37 netif_device_detach(netdev); 38 set_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags); 39 wx_soft_quiesce(wx); 40 41 if (state == pci_channel_io_perm_failure) { 42 rtnl_unlock(); 43 return PCI_ERS_RESULT_DISCONNECT; 44 } 45 46 if (!test_and_set_bit(WX_STATE_DISABLED, wx->state)) 47 pci_disable_device(pdev); 48 rtnl_unlock(); 49 50 /* Request a slot reset. */ 51 return PCI_ERS_RESULT_NEED_RESET; 52 } 53 54 /** 55 * wx_io_slot_reset - called after the pci bus has been reset. 56 * @pdev: Pointer to PCI device 57 * 58 * Return: pci_ers_result_t. 59 * 60 * Restart the card from scratch, as if from a cold-boot. 61 */ 62 static pci_ers_result_t wx_io_slot_reset(struct pci_dev *pdev) 63 { 64 struct wx *wx = pci_get_drvdata(pdev); 65 66 if (pci_enable_device_mem(pdev)) { 67 wx_err(wx, "Cannot re-enable PCI device after reset.\n"); 68 return PCI_ERS_RESULT_DISCONNECT; 69 } 70 71 /* make all memory operations done before clearing the flag */ 72 smp_mb__before_atomic(); 73 clear_bit(WX_STATE_DISABLED, wx->state); 74 clear_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags); 75 pci_set_master(pdev); 76 pci_restore_state(pdev); 77 pci_wake_from_d3(pdev, false); 78 79 rtnl_lock(); 80 if (netif_running(wx->netdev) && wx->down_suspend) 81 wx->down_suspend(wx); 82 if (wx->do_reset) 83 wx->do_reset(wx->netdev, false); 84 rtnl_unlock(); 85 86 return PCI_ERS_RESULT_RECOVERED; 87 } 88 89 /** 90 * wx_io_resume - called when traffic can start flowing again. 91 * @pdev: Pointer to PCI device 92 * 93 * This callback is called when the error recovery driver tells us that 94 * its OK to resume normal operation. 95 */ 96 static void wx_io_resume(struct pci_dev *pdev) 97 { 98 struct wx *wx = pci_get_drvdata(pdev); 99 struct net_device *netdev; 100 int err; 101 102 netdev = wx->netdev; 103 rtnl_lock(); 104 if (netif_running(netdev)) { 105 err = netdev->netdev_ops->ndo_open(netdev); 106 if (err) { 107 wx_err(wx, "Failed to open netdev after reset\n"); 108 goto out; 109 } 110 } 111 netif_device_attach(netdev); 112 out: 113 rtnl_unlock(); 114 } 115 116 const struct pci_error_handlers wx_err_handler = { 117 .error_detected = wx_io_error_detected, 118 .slot_reset = wx_io_slot_reset, 119 .resume = wx_io_resume, 120 }; 121 EXPORT_SYMBOL(wx_err_handler); 122 123 static bool wx_check_pcie_error(struct wx *wx) 124 { 125 u16 vid, pci_cmd; 126 127 pci_read_config_word(wx->pdev, PCI_VENDOR_ID, &vid); 128 pci_read_config_word(wx->pdev, PCI_COMMAND, &pci_cmd); 129 130 /* PCIe link loss or memory space can't access */ 131 if (vid == U16_MAX || !(pci_cmd & PCI_COMMAND_MEMORY)) 132 return true; 133 134 return false; 135 } 136 137 static void wx_pf_reset_subtask(struct wx *wx) 138 { 139 if (!test_and_clear_bit(WX_FLAG_NEED_DO_RESET, wx->flags)) 140 return; 141 142 wx_warn(wx, "Reset adapter.\n"); 143 if (wx->do_reset) 144 wx->do_reset(wx->netdev, true); 145 } 146 147 static void wx_reset_task(struct work_struct *work) 148 { 149 struct wx *wx = container_of(work, struct wx, reset_task); 150 151 rtnl_lock(); 152 153 /* If the device has been detached (e.g., due to AER error handling), 154 * abort the reset task to prevent operating on a dead or unmanaged 155 * hardware. 156 */ 157 if (!netif_device_present(wx->netdev)) 158 goto out; 159 160 if (test_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags)) { 161 /* Double check: Verify if the PCIe error is still present. */ 162 if (wx_check_pcie_error(wx)) 163 wx_soft_quiesce(wx); 164 else 165 clear_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags); 166 goto out; 167 } 168 169 if (test_bit(WX_STATE_DOWN, wx->state) || 170 test_bit(WX_STATE_RESETTING, wx->state)) 171 goto out; 172 173 wx_pf_reset_subtask(wx); 174 175 out: 176 rtnl_unlock(); 177 } 178 179 void wx_check_err_subtask(struct wx *wx) 180 { 181 if (test_bit(WX_FLAG_NEED_DO_RESET, wx->flags)) 182 queue_work(wx->reset_wq, &wx->reset_task); 183 } 184 EXPORT_SYMBOL(wx_check_err_subtask); 185 186 int wx_init_err_task(struct wx *wx) 187 { 188 wx->reset_wq = alloc_workqueue("%s_reset_wq_%x", WQ_UNBOUND | WQ_HIGHPRI, 189 1, wx->driver_name, pci_dev_id(wx->pdev)); 190 if (!wx->reset_wq) { 191 wx_err(wx, "Failed to create wx_reset_wq workqueue\n"); 192 return -ENOMEM; 193 } 194 195 INIT_WORK(&wx->reset_task, wx_reset_task); 196 return 0; 197 } 198 EXPORT_SYMBOL(wx_init_err_task); 199 200 static bool wx_ring_tx_pending(struct wx *wx) 201 { 202 int i; 203 204 for (i = 0; i < wx->num_tx_queues; i++) { 205 struct wx_ring *tx_ring = wx->tx_ring[i]; 206 207 if (tx_ring->next_to_use != tx_ring->next_to_clean) 208 return true; 209 } 210 211 return false; 212 } 213 214 static bool wx_vf_tx_pending(struct wx *wx) 215 { 216 struct wx_ring_feature *vmdq = &wx->ring_feature[RING_F_VMDQ]; 217 u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask); 218 u32 i, j; 219 220 if (!wx->num_vfs) 221 return false; 222 223 for (i = 0; i < wx->num_vfs; i++) { 224 for (j = 0; j < q_per_pool; j++) { 225 u32 h, t; 226 227 h = rd32(wx, WX_PX_TR_RP_PV(q_per_pool, i, j)); 228 t = rd32(wx, WX_PX_TR_WP_PV(q_per_pool, i, j)); 229 230 if (h != t) 231 return true; 232 } 233 } 234 235 return false; 236 } 237 238 static void wx_watchdog_flush_tx(struct wx *wx) 239 { 240 if (!netif_running(wx->netdev)) 241 return; 242 if (netif_carrier_ok(wx->netdev)) 243 return; 244 245 if (wx_ring_tx_pending(wx) || wx_vf_tx_pending(wx)) { 246 /* We've lost link, so the controller stops DMA, 247 * but we've got queued Tx work that's never going 248 * to get done, so reset controller to flush Tx. 249 * (Do the reset outside of interrupt context). 250 */ 251 wx_warn(wx, "initiating reset due to lost link with pending Tx work\n"); 252 set_bit(WX_FLAG_NEED_DO_RESET, wx->flags); 253 } 254 } 255 256 static void wx_detect_tx_hang(struct wx *wx) 257 { 258 int i; 259 260 /* If we're down or resetting, just bail */ 261 if (!netif_running(wx->netdev) || 262 test_bit(WX_STATE_RESETTING, wx->state)) 263 return; 264 265 /* Force detection of hung controller */ 266 if (netif_carrier_ok(wx->netdev)) { 267 for (i = 0; i < wx->num_tx_queues; i++) 268 set_bit(WX_TX_DETECT_HANG, wx->tx_ring[i]->state); 269 } 270 } 271 272 void wx_check_hang_subtask(struct wx *wx) 273 { 274 if (test_bit(WX_STATE_DOWN, wx->state) || 275 test_bit(WX_STATE_RESETTING, wx->state)) 276 return; 277 278 wx_watchdog_flush_tx(wx); 279 wx_detect_tx_hang(wx); 280 } 281 EXPORT_SYMBOL(wx_check_hang_subtask); 282 283 static void wx_tx_timeout_recovery(struct wx *wx) 284 { 285 /* 286 * When a PCIe hardware error occurs, the driver should initiate a PCIe 287 * recovery mechanism. However, this recovery flow relies on the AER 288 * driver for current kernel policy. Therefore, a self-contained 289 * recovery mechanism is not implemented yet. 290 */ 291 set_bit(WX_FLAG_NEED_PCIE_RECOVERY, wx->flags); 292 wx_err(wx, "PCIe error detected during tx timeout\n"); 293 queue_work(wx->reset_wq, &wx->reset_task); 294 } 295 296 static void wx_tx_timeout_reset(struct wx *wx) 297 { 298 if (test_bit(WX_STATE_DOWN, wx->state)) 299 return; 300 301 set_bit(WX_FLAG_NEED_DO_RESET, wx->flags); 302 wx_warn(wx, "initiating reset due to tx timeout\n"); 303 wx_service_event_schedule(wx); 304 } 305 306 void wx_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) 307 { 308 struct wx *wx = netdev_priv(netdev); 309 310 if (wx_check_pcie_error(wx)) 311 wx_tx_timeout_recovery(wx); 312 else 313 wx_tx_timeout_reset(wx); 314 } 315 EXPORT_SYMBOL(wx_tx_timeout); 316 317 void wx_handle_tx_hang(struct wx_ring *tx_ring, unsigned int next) 318 { 319 struct wx *wx = netdev_priv(tx_ring->netdev); 320 321 wx_warn(wx, 322 "Detected Tx Unit Hang: Queue %d, TDH %x, TDT %x, ntu %x, ntc %x, ntc.time_stamp %lx, jiffies %lx\n", 323 tx_ring->queue_index, 324 rd32(wx, WX_PX_TR_RP(tx_ring->reg_idx)), 325 rd32(wx, WX_PX_TR_WP(tx_ring->reg_idx)), 326 tx_ring->next_to_use, next, 327 tx_ring->tx_buffer_info[next].time_stamp, jiffies); 328 329 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 330 331 wx_tx_timeout_reset(wx); 332 } 333