1 /******************************************************************************* 2 3 Intel 10 Gigabit PCI Express Linux driver 4 Copyright(c) 1999 - 2016 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 Linux NICS <linux.nics@intel.com> 24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 26 27 *******************************************************************************/ 28 29 #include <linux/types.h> 30 #include <linux/module.h> 31 #include <linux/pci.h> 32 #include <linux/netdevice.h> 33 #include <linux/vmalloc.h> 34 #include <linux/string.h> 35 #include <linux/in.h> 36 #include <linux/interrupt.h> 37 #include <linux/ip.h> 38 #include <linux/tcp.h> 39 #include <linux/sctp.h> 40 #include <linux/pkt_sched.h> 41 #include <linux/ipv6.h> 42 #include <linux/slab.h> 43 #include <net/checksum.h> 44 #include <net/ip6_checksum.h> 45 #include <linux/etherdevice.h> 46 #include <linux/ethtool.h> 47 #include <linux/if.h> 48 #include <linux/if_vlan.h> 49 #include <linux/if_macvlan.h> 50 #include <linux/if_bridge.h> 51 #include <linux/prefetch.h> 52 #include <scsi/fc/fc_fcoe.h> 53 #include <net/udp_tunnel.h> 54 #include <net/pkt_cls.h> 55 #include <net/tc_act/tc_gact.h> 56 #include <net/tc_act/tc_mirred.h> 57 #include <net/vxlan.h> 58 59 #include "ixgbe.h" 60 #include "ixgbe_common.h" 61 #include "ixgbe_dcb_82599.h" 62 #include "ixgbe_sriov.h" 63 #include "ixgbe_model.h" 64 65 char ixgbe_driver_name[] = "ixgbe"; 66 static const char ixgbe_driver_string[] = 67 "Intel(R) 10 Gigabit PCI Express Network Driver"; 68 #ifdef IXGBE_FCOE 69 char ixgbe_default_device_descr[] = 70 "Intel(R) 10 Gigabit Network Connection"; 71 #else 72 static char ixgbe_default_device_descr[] = 73 "Intel(R) 10 Gigabit Network Connection"; 74 #endif 75 #define DRV_VERSION "4.4.0-k" 76 const char ixgbe_driver_version[] = DRV_VERSION; 77 static const char ixgbe_copyright[] = 78 "Copyright (c) 1999-2016 Intel Corporation."; 79 80 static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter"; 81 82 static const struct ixgbe_info *ixgbe_info_tbl[] = { 83 [board_82598] = &ixgbe_82598_info, 84 [board_82599] = &ixgbe_82599_info, 85 [board_X540] = &ixgbe_X540_info, 86 [board_X550] = &ixgbe_X550_info, 87 [board_X550EM_x] = &ixgbe_X550EM_x_info, 88 [board_x550em_a] = &ixgbe_x550em_a_info, 89 [board_x550em_a_fw] = &ixgbe_x550em_a_fw_info, 90 }; 91 92 /* ixgbe_pci_tbl - PCI Device ID Table 93 * 94 * Wildcard entries (PCI_ANY_ID) should come last 95 * Last entry must be all 0s 96 * 97 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 98 * Class, Class Mask, private data (not used) } 99 */ 100 static const struct pci_device_id ixgbe_pci_tbl[] = { 101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 }, 102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 }, 103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 }, 104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 }, 105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 }, 106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 }, 107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 }, 108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 }, 109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 }, 110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 }, 111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 }, 112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 }, 113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 }, 114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 }, 115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 }, 116 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 }, 117 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 }, 118 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 }, 119 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 }, 120 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 }, 121 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 }, 122 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 }, 123 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 }, 124 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 }, 125 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 }, 126 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 }, 127 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 }, 128 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 }, 129 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 }, 130 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 }, 131 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550}, 132 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550}, 133 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x}, 134 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x}, 135 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x}, 136 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x}, 137 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a }, 138 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a }, 139 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a }, 140 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a }, 141 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a }, 142 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a}, 143 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a }, 144 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw }, 145 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw }, 146 /* required last entry */ 147 {0, } 148 }; 149 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); 150 151 #ifdef CONFIG_IXGBE_DCA 152 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, 153 void *p); 154 static struct notifier_block dca_notifier = { 155 .notifier_call = ixgbe_notify_dca, 156 .next = NULL, 157 .priority = 0 158 }; 159 #endif 160 161 #ifdef CONFIG_PCI_IOV 162 static unsigned int max_vfs; 163 module_param(max_vfs, uint, 0); 164 MODULE_PARM_DESC(max_vfs, 165 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)"); 166 #endif /* CONFIG_PCI_IOV */ 167 168 static unsigned int allow_unsupported_sfp; 169 module_param(allow_unsupported_sfp, uint, 0); 170 MODULE_PARM_DESC(allow_unsupported_sfp, 171 "Allow unsupported and untested SFP+ modules on 82599-based adapters"); 172 173 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 174 static int debug = -1; 175 module_param(debug, int, 0); 176 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 177 178 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 179 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); 180 MODULE_LICENSE("GPL"); 181 MODULE_VERSION(DRV_VERSION); 182 183 static struct workqueue_struct *ixgbe_wq; 184 185 static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev); 186 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *); 187 188 static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter, 189 u32 reg, u16 *value) 190 { 191 struct pci_dev *parent_dev; 192 struct pci_bus *parent_bus; 193 194 parent_bus = adapter->pdev->bus->parent; 195 if (!parent_bus) 196 return -1; 197 198 parent_dev = parent_bus->self; 199 if (!parent_dev) 200 return -1; 201 202 if (!pci_is_pcie(parent_dev)) 203 return -1; 204 205 pcie_capability_read_word(parent_dev, reg, value); 206 if (*value == IXGBE_FAILED_READ_CFG_WORD && 207 ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) 208 return -1; 209 return 0; 210 } 211 212 static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter) 213 { 214 struct ixgbe_hw *hw = &adapter->hw; 215 u16 link_status = 0; 216 int err; 217 218 hw->bus.type = ixgbe_bus_type_pci_express; 219 220 /* Get the negotiated link width and speed from PCI config space of the 221 * parent, as this device is behind a switch 222 */ 223 err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status); 224 225 /* assume caller will handle error case */ 226 if (err) 227 return err; 228 229 hw->bus.width = ixgbe_convert_bus_width(link_status); 230 hw->bus.speed = ixgbe_convert_bus_speed(link_status); 231 232 return 0; 233 } 234 235 /** 236 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent 237 * @hw: hw specific details 238 * 239 * This function is used by probe to determine whether a device's PCI-Express 240 * bandwidth details should be gathered from the parent bus instead of from the 241 * device. Used to ensure that various locations all have the correct device ID 242 * checks. 243 */ 244 static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw) 245 { 246 switch (hw->device_id) { 247 case IXGBE_DEV_ID_82599_SFP_SF_QP: 248 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 249 return true; 250 default: 251 return false; 252 } 253 } 254 255 static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, 256 int expected_gts) 257 { 258 struct ixgbe_hw *hw = &adapter->hw; 259 int max_gts = 0; 260 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN; 261 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN; 262 struct pci_dev *pdev; 263 264 /* Some devices are not connected over PCIe and thus do not negotiate 265 * speed. These devices do not have valid bus info, and thus any report 266 * we generate may not be correct. 267 */ 268 if (hw->bus.type == ixgbe_bus_type_internal) 269 return; 270 271 /* determine whether to use the parent device */ 272 if (ixgbe_pcie_from_parent(&adapter->hw)) 273 pdev = adapter->pdev->bus->parent->self; 274 else 275 pdev = adapter->pdev; 276 277 if (pcie_get_minimum_link(pdev, &speed, &width) || 278 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 279 e_dev_warn("Unable to determine PCI Express bandwidth.\n"); 280 return; 281 } 282 283 switch (speed) { 284 case PCIE_SPEED_2_5GT: 285 /* 8b/10b encoding reduces max throughput by 20% */ 286 max_gts = 2 * width; 287 break; 288 case PCIE_SPEED_5_0GT: 289 /* 8b/10b encoding reduces max throughput by 20% */ 290 max_gts = 4 * width; 291 break; 292 case PCIE_SPEED_8_0GT: 293 /* 128b/130b encoding reduces throughput by less than 2% */ 294 max_gts = 8 * width; 295 break; 296 default: 297 e_dev_warn("Unable to determine PCI Express bandwidth.\n"); 298 return; 299 } 300 301 e_dev_info("PCI Express bandwidth of %dGT/s available\n", 302 max_gts); 303 e_dev_info("(Speed:%s, Width: x%d, Encoding Loss:%s)\n", 304 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : 305 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : 306 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : 307 "Unknown"), 308 width, 309 (speed == PCIE_SPEED_2_5GT ? "20%" : 310 speed == PCIE_SPEED_5_0GT ? "20%" : 311 speed == PCIE_SPEED_8_0GT ? "<2%" : 312 "Unknown")); 313 314 if (max_gts < expected_gts) { 315 e_dev_warn("This is not sufficient for optimal performance of this card.\n"); 316 e_dev_warn("For optimal performance, at least %dGT/s of bandwidth is required.\n", 317 expected_gts); 318 e_dev_warn("A slot with more lanes and/or higher speed is suggested.\n"); 319 } 320 } 321 322 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) 323 { 324 if (!test_bit(__IXGBE_DOWN, &adapter->state) && 325 !test_bit(__IXGBE_REMOVING, &adapter->state) && 326 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) 327 queue_work(ixgbe_wq, &adapter->service_task); 328 } 329 330 static void ixgbe_remove_adapter(struct ixgbe_hw *hw) 331 { 332 struct ixgbe_adapter *adapter = hw->back; 333 334 if (!hw->hw_addr) 335 return; 336 hw->hw_addr = NULL; 337 e_dev_err("Adapter removed\n"); 338 if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state)) 339 ixgbe_service_event_schedule(adapter); 340 } 341 342 static void ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg) 343 { 344 u32 value; 345 346 /* The following check not only optimizes a bit by not 347 * performing a read on the status register when the 348 * register just read was a status register read that 349 * returned IXGBE_FAILED_READ_REG. It also blocks any 350 * potential recursion. 351 */ 352 if (reg == IXGBE_STATUS) { 353 ixgbe_remove_adapter(hw); 354 return; 355 } 356 value = ixgbe_read_reg(hw, IXGBE_STATUS); 357 if (value == IXGBE_FAILED_READ_REG) 358 ixgbe_remove_adapter(hw); 359 } 360 361 /** 362 * ixgbe_read_reg - Read from device register 363 * @hw: hw specific details 364 * @reg: offset of register to read 365 * 366 * Returns : value read or IXGBE_FAILED_READ_REG if removed 367 * 368 * This function is used to read device registers. It checks for device 369 * removal by confirming any read that returns all ones by checking the 370 * status register value for all ones. This function avoids reading from 371 * the hardware if a removal was previously detected in which case it 372 * returns IXGBE_FAILED_READ_REG (all ones). 373 */ 374 u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg) 375 { 376 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 377 u32 value; 378 379 if (ixgbe_removed(reg_addr)) 380 return IXGBE_FAILED_READ_REG; 381 if (unlikely(hw->phy.nw_mng_if_sel & 382 IXGBE_NW_MNG_IF_SEL_ENABLE_10_100M)) { 383 struct ixgbe_adapter *adapter; 384 int i; 385 386 for (i = 0; i < 200; ++i) { 387 value = readl(reg_addr + IXGBE_MAC_SGMII_BUSY); 388 if (likely(!value)) 389 goto writes_completed; 390 if (value == IXGBE_FAILED_READ_REG) { 391 ixgbe_remove_adapter(hw); 392 return IXGBE_FAILED_READ_REG; 393 } 394 udelay(5); 395 } 396 397 adapter = hw->back; 398 e_warn(hw, "register writes incomplete %08x\n", value); 399 } 400 401 writes_completed: 402 value = readl(reg_addr + reg); 403 if (unlikely(value == IXGBE_FAILED_READ_REG)) 404 ixgbe_check_remove(hw, reg); 405 return value; 406 } 407 408 static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev) 409 { 410 u16 value; 411 412 pci_read_config_word(pdev, PCI_VENDOR_ID, &value); 413 if (value == IXGBE_FAILED_READ_CFG_WORD) { 414 ixgbe_remove_adapter(hw); 415 return true; 416 } 417 return false; 418 } 419 420 u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg) 421 { 422 struct ixgbe_adapter *adapter = hw->back; 423 u16 value; 424 425 if (ixgbe_removed(hw->hw_addr)) 426 return IXGBE_FAILED_READ_CFG_WORD; 427 pci_read_config_word(adapter->pdev, reg, &value); 428 if (value == IXGBE_FAILED_READ_CFG_WORD && 429 ixgbe_check_cfg_remove(hw, adapter->pdev)) 430 return IXGBE_FAILED_READ_CFG_WORD; 431 return value; 432 } 433 434 #ifdef CONFIG_PCI_IOV 435 static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg) 436 { 437 struct ixgbe_adapter *adapter = hw->back; 438 u32 value; 439 440 if (ixgbe_removed(hw->hw_addr)) 441 return IXGBE_FAILED_READ_CFG_DWORD; 442 pci_read_config_dword(adapter->pdev, reg, &value); 443 if (value == IXGBE_FAILED_READ_CFG_DWORD && 444 ixgbe_check_cfg_remove(hw, adapter->pdev)) 445 return IXGBE_FAILED_READ_CFG_DWORD; 446 return value; 447 } 448 #endif /* CONFIG_PCI_IOV */ 449 450 void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value) 451 { 452 struct ixgbe_adapter *adapter = hw->back; 453 454 if (ixgbe_removed(hw->hw_addr)) 455 return; 456 pci_write_config_word(adapter->pdev, reg, value); 457 } 458 459 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter) 460 { 461 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state)); 462 463 /* flush memory to make sure state is correct before next watchdog */ 464 smp_mb__before_atomic(); 465 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 466 } 467 468 struct ixgbe_reg_info { 469 u32 ofs; 470 char *name; 471 }; 472 473 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = { 474 475 /* General Registers */ 476 {IXGBE_CTRL, "CTRL"}, 477 {IXGBE_STATUS, "STATUS"}, 478 {IXGBE_CTRL_EXT, "CTRL_EXT"}, 479 480 /* Interrupt Registers */ 481 {IXGBE_EICR, "EICR"}, 482 483 /* RX Registers */ 484 {IXGBE_SRRCTL(0), "SRRCTL"}, 485 {IXGBE_DCA_RXCTRL(0), "DRXCTL"}, 486 {IXGBE_RDLEN(0), "RDLEN"}, 487 {IXGBE_RDH(0), "RDH"}, 488 {IXGBE_RDT(0), "RDT"}, 489 {IXGBE_RXDCTL(0), "RXDCTL"}, 490 {IXGBE_RDBAL(0), "RDBAL"}, 491 {IXGBE_RDBAH(0), "RDBAH"}, 492 493 /* TX Registers */ 494 {IXGBE_TDBAL(0), "TDBAL"}, 495 {IXGBE_TDBAH(0), "TDBAH"}, 496 {IXGBE_TDLEN(0), "TDLEN"}, 497 {IXGBE_TDH(0), "TDH"}, 498 {IXGBE_TDT(0), "TDT"}, 499 {IXGBE_TXDCTL(0), "TXDCTL"}, 500 501 /* List Terminator */ 502 { .name = NULL } 503 }; 504 505 506 /* 507 * ixgbe_regdump - register printout routine 508 */ 509 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo) 510 { 511 int i = 0, j = 0; 512 char rname[16]; 513 u32 regs[64]; 514 515 switch (reginfo->ofs) { 516 case IXGBE_SRRCTL(0): 517 for (i = 0; i < 64; i++) 518 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 519 break; 520 case IXGBE_DCA_RXCTRL(0): 521 for (i = 0; i < 64; i++) 522 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i)); 523 break; 524 case IXGBE_RDLEN(0): 525 for (i = 0; i < 64; i++) 526 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i)); 527 break; 528 case IXGBE_RDH(0): 529 for (i = 0; i < 64; i++) 530 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i)); 531 break; 532 case IXGBE_RDT(0): 533 for (i = 0; i < 64; i++) 534 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i)); 535 break; 536 case IXGBE_RXDCTL(0): 537 for (i = 0; i < 64; i++) 538 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 539 break; 540 case IXGBE_RDBAL(0): 541 for (i = 0; i < 64; i++) 542 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i)); 543 break; 544 case IXGBE_RDBAH(0): 545 for (i = 0; i < 64; i++) 546 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i)); 547 break; 548 case IXGBE_TDBAL(0): 549 for (i = 0; i < 64; i++) 550 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i)); 551 break; 552 case IXGBE_TDBAH(0): 553 for (i = 0; i < 64; i++) 554 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i)); 555 break; 556 case IXGBE_TDLEN(0): 557 for (i = 0; i < 64; i++) 558 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i)); 559 break; 560 case IXGBE_TDH(0): 561 for (i = 0; i < 64; i++) 562 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i)); 563 break; 564 case IXGBE_TDT(0): 565 for (i = 0; i < 64; i++) 566 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i)); 567 break; 568 case IXGBE_TXDCTL(0): 569 for (i = 0; i < 64; i++) 570 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 571 break; 572 default: 573 pr_info("%-15s %08x\n", reginfo->name, 574 IXGBE_READ_REG(hw, reginfo->ofs)); 575 return; 576 } 577 578 for (i = 0; i < 8; i++) { 579 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7); 580 pr_err("%-15s", rname); 581 for (j = 0; j < 8; j++) 582 pr_cont(" %08x", regs[i*8+j]); 583 pr_cont("\n"); 584 } 585 586 } 587 588 /* 589 * ixgbe_dump - Print registers, tx-rings and rx-rings 590 */ 591 static void ixgbe_dump(struct ixgbe_adapter *adapter) 592 { 593 struct net_device *netdev = adapter->netdev; 594 struct ixgbe_hw *hw = &adapter->hw; 595 struct ixgbe_reg_info *reginfo; 596 int n = 0; 597 struct ixgbe_ring *tx_ring; 598 struct ixgbe_tx_buffer *tx_buffer; 599 union ixgbe_adv_tx_desc *tx_desc; 600 struct my_u0 { u64 a; u64 b; } *u0; 601 struct ixgbe_ring *rx_ring; 602 union ixgbe_adv_rx_desc *rx_desc; 603 struct ixgbe_rx_buffer *rx_buffer_info; 604 u32 staterr; 605 int i = 0; 606 607 if (!netif_msg_hw(adapter)) 608 return; 609 610 /* Print netdevice Info */ 611 if (netdev) { 612 dev_info(&adapter->pdev->dev, "Net device Info\n"); 613 pr_info("Device Name state " 614 "trans_start\n"); 615 pr_info("%-15s %016lX %016lX\n", 616 netdev->name, 617 netdev->state, 618 dev_trans_start(netdev)); 619 } 620 621 /* Print Registers */ 622 dev_info(&adapter->pdev->dev, "Register Dump\n"); 623 pr_info(" Register Name Value\n"); 624 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; 625 reginfo->name; reginfo++) { 626 ixgbe_regdump(hw, reginfo); 627 } 628 629 /* Print TX Ring Summary */ 630 if (!netdev || !netif_running(netdev)) 631 return; 632 633 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 634 pr_info(" %s %s %s %s\n", 635 "Queue [NTU] [NTC] [bi(ntc)->dma ]", 636 "leng", "ntw", "timestamp"); 637 for (n = 0; n < adapter->num_tx_queues; n++) { 638 tx_ring = adapter->tx_ring[n]; 639 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 640 pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n", 641 n, tx_ring->next_to_use, tx_ring->next_to_clean, 642 (u64)dma_unmap_addr(tx_buffer, dma), 643 dma_unmap_len(tx_buffer, len), 644 tx_buffer->next_to_watch, 645 (u64)tx_buffer->time_stamp); 646 } 647 648 /* Print TX Rings */ 649 if (!netif_msg_tx_done(adapter)) 650 goto rx_ring_summary; 651 652 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 653 654 /* Transmit Descriptor Formats 655 * 656 * 82598 Advanced Transmit Descriptor 657 * +--------------------------------------------------------------+ 658 * 0 | Buffer Address [63:0] | 659 * +--------------------------------------------------------------+ 660 * 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN | 661 * +--------------------------------------------------------------+ 662 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0 663 * 664 * 82598 Advanced Transmit Descriptor (Write-Back Format) 665 * +--------------------------------------------------------------+ 666 * 0 | RSV [63:0] | 667 * +--------------------------------------------------------------+ 668 * 8 | RSV | STA | NXTSEQ | 669 * +--------------------------------------------------------------+ 670 * 63 36 35 32 31 0 671 * 672 * 82599+ Advanced Transmit Descriptor 673 * +--------------------------------------------------------------+ 674 * 0 | Buffer Address [63:0] | 675 * +--------------------------------------------------------------+ 676 * 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN | 677 * +--------------------------------------------------------------+ 678 * 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0 679 * 680 * 82599+ Advanced Transmit Descriptor (Write-Back Format) 681 * +--------------------------------------------------------------+ 682 * 0 | RSV [63:0] | 683 * +--------------------------------------------------------------+ 684 * 8 | RSV | STA | RSV | 685 * +--------------------------------------------------------------+ 686 * 63 36 35 32 31 0 687 */ 688 689 for (n = 0; n < adapter->num_tx_queues; n++) { 690 tx_ring = adapter->tx_ring[n]; 691 pr_info("------------------------------------\n"); 692 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 693 pr_info("------------------------------------\n"); 694 pr_info("%s%s %s %s %s %s\n", 695 "T [desc] [address 63:0 ] ", 696 "[PlPOIdStDDt Ln] [bi->dma ] ", 697 "leng", "ntw", "timestamp", "bi->skb"); 698 699 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 700 tx_desc = IXGBE_TX_DESC(tx_ring, i); 701 tx_buffer = &tx_ring->tx_buffer_info[i]; 702 u0 = (struct my_u0 *)tx_desc; 703 if (dma_unmap_len(tx_buffer, len) > 0) { 704 pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p", 705 i, 706 le64_to_cpu(u0->a), 707 le64_to_cpu(u0->b), 708 (u64)dma_unmap_addr(tx_buffer, dma), 709 dma_unmap_len(tx_buffer, len), 710 tx_buffer->next_to_watch, 711 (u64)tx_buffer->time_stamp, 712 tx_buffer->skb); 713 if (i == tx_ring->next_to_use && 714 i == tx_ring->next_to_clean) 715 pr_cont(" NTC/U\n"); 716 else if (i == tx_ring->next_to_use) 717 pr_cont(" NTU\n"); 718 else if (i == tx_ring->next_to_clean) 719 pr_cont(" NTC\n"); 720 else 721 pr_cont("\n"); 722 723 if (netif_msg_pktdata(adapter) && 724 tx_buffer->skb) 725 print_hex_dump(KERN_INFO, "", 726 DUMP_PREFIX_ADDRESS, 16, 1, 727 tx_buffer->skb->data, 728 dma_unmap_len(tx_buffer, len), 729 true); 730 } 731 } 732 } 733 734 /* Print RX Rings Summary */ 735 rx_ring_summary: 736 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 737 pr_info("Queue [NTU] [NTC]\n"); 738 for (n = 0; n < adapter->num_rx_queues; n++) { 739 rx_ring = adapter->rx_ring[n]; 740 pr_info("%5d %5X %5X\n", 741 n, rx_ring->next_to_use, rx_ring->next_to_clean); 742 } 743 744 /* Print RX Rings */ 745 if (!netif_msg_rx_status(adapter)) 746 return; 747 748 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 749 750 /* Receive Descriptor Formats 751 * 752 * 82598 Advanced Receive Descriptor (Read) Format 753 * 63 1 0 754 * +-----------------------------------------------------+ 755 * 0 | Packet Buffer Address [63:1] |A0/NSE| 756 * +----------------------------------------------+------+ 757 * 8 | Header Buffer Address [63:1] | DD | 758 * +-----------------------------------------------------+ 759 * 760 * 761 * 82598 Advanced Receive Descriptor (Write-Back) Format 762 * 763 * 63 48 47 32 31 30 21 20 16 15 4 3 0 764 * +------------------------------------------------------+ 765 * 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS | 766 * | Packet | IP | | | | Type | Type | 767 * | Checksum | Ident | | | | | | 768 * +------------------------------------------------------+ 769 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 770 * +------------------------------------------------------+ 771 * 63 48 47 32 31 20 19 0 772 * 773 * 82599+ Advanced Receive Descriptor (Read) Format 774 * 63 1 0 775 * +-----------------------------------------------------+ 776 * 0 | Packet Buffer Address [63:1] |A0/NSE| 777 * +----------------------------------------------+------+ 778 * 8 | Header Buffer Address [63:1] | DD | 779 * +-----------------------------------------------------+ 780 * 781 * 782 * 82599+ Advanced Receive Descriptor (Write-Back) Format 783 * 784 * 63 48 47 32 31 30 21 20 17 16 4 3 0 785 * +------------------------------------------------------+ 786 * 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS | 787 * |/ RTT / PCoE_PARAM | | | CNT | Type | Type | 788 * |/ Flow Dir Flt ID | | | | | | 789 * +------------------------------------------------------+ 790 * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP | 791 * +------------------------------------------------------+ 792 * 63 48 47 32 31 20 19 0 793 */ 794 795 for (n = 0; n < adapter->num_rx_queues; n++) { 796 rx_ring = adapter->rx_ring[n]; 797 pr_info("------------------------------------\n"); 798 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 799 pr_info("------------------------------------\n"); 800 pr_info("%s%s%s", 801 "R [desc] [ PktBuf A0] ", 802 "[ HeadBuf DD] [bi->dma ] [bi->skb ] ", 803 "<-- Adv Rx Read format\n"); 804 pr_info("%s%s%s", 805 "RWB[desc] [PcsmIpSHl PtRs] ", 806 "[vl er S cks ln] ---------------- [bi->skb ] ", 807 "<-- Adv Rx Write-Back format\n"); 808 809 for (i = 0; i < rx_ring->count; i++) { 810 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 811 rx_desc = IXGBE_RX_DESC(rx_ring, i); 812 u0 = (struct my_u0 *)rx_desc; 813 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 814 if (staterr & IXGBE_RXD_STAT_DD) { 815 /* Descriptor Done */ 816 pr_info("RWB[0x%03X] %016llX " 817 "%016llX ---------------- %p", i, 818 le64_to_cpu(u0->a), 819 le64_to_cpu(u0->b), 820 rx_buffer_info->skb); 821 } else { 822 pr_info("R [0x%03X] %016llX " 823 "%016llX %016llX %p", i, 824 le64_to_cpu(u0->a), 825 le64_to_cpu(u0->b), 826 (u64)rx_buffer_info->dma, 827 rx_buffer_info->skb); 828 829 if (netif_msg_pktdata(adapter) && 830 rx_buffer_info->dma) { 831 print_hex_dump(KERN_INFO, "", 832 DUMP_PREFIX_ADDRESS, 16, 1, 833 page_address(rx_buffer_info->page) + 834 rx_buffer_info->page_offset, 835 ixgbe_rx_bufsz(rx_ring), true); 836 } 837 } 838 839 if (i == rx_ring->next_to_use) 840 pr_cont(" NTU\n"); 841 else if (i == rx_ring->next_to_clean) 842 pr_cont(" NTC\n"); 843 else 844 pr_cont("\n"); 845 846 } 847 } 848 } 849 850 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter) 851 { 852 u32 ctrl_ext; 853 854 /* Let firmware take over control of h/w */ 855 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 856 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 857 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD); 858 } 859 860 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter) 861 { 862 u32 ctrl_ext; 863 864 /* Let firmware know the driver has taken over */ 865 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 866 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 867 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD); 868 } 869 870 /** 871 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors 872 * @adapter: pointer to adapter struct 873 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 874 * @queue: queue to map the corresponding interrupt to 875 * @msix_vector: the vector to map to the corresponding queue 876 * 877 */ 878 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, 879 u8 queue, u8 msix_vector) 880 { 881 u32 ivar, index; 882 struct ixgbe_hw *hw = &adapter->hw; 883 switch (hw->mac.type) { 884 case ixgbe_mac_82598EB: 885 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 886 if (direction == -1) 887 direction = 0; 888 index = (((direction * 64) + queue) >> 2) & 0x1F; 889 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 890 ivar &= ~(0xFF << (8 * (queue & 0x3))); 891 ivar |= (msix_vector << (8 * (queue & 0x3))); 892 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 893 break; 894 case ixgbe_mac_82599EB: 895 case ixgbe_mac_X540: 896 case ixgbe_mac_X550: 897 case ixgbe_mac_X550EM_x: 898 case ixgbe_mac_x550em_a: 899 if (direction == -1) { 900 /* other causes */ 901 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 902 index = ((queue & 1) * 8); 903 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC); 904 ivar &= ~(0xFF << index); 905 ivar |= (msix_vector << index); 906 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar); 907 break; 908 } else { 909 /* tx or rx causes */ 910 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 911 index = ((16 * (queue & 1)) + (8 * direction)); 912 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 913 ivar &= ~(0xFF << index); 914 ivar |= (msix_vector << index); 915 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar); 916 break; 917 } 918 default: 919 break; 920 } 921 } 922 923 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, 924 u64 qmask) 925 { 926 u32 mask; 927 928 switch (adapter->hw.mac.type) { 929 case ixgbe_mac_82598EB: 930 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 931 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 932 break; 933 case ixgbe_mac_82599EB: 934 case ixgbe_mac_X540: 935 case ixgbe_mac_X550: 936 case ixgbe_mac_X550EM_x: 937 case ixgbe_mac_x550em_a: 938 mask = (qmask & 0xFFFFFFFF); 939 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 940 mask = (qmask >> 32); 941 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 942 break; 943 default: 944 break; 945 } 946 } 947 948 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring, 949 struct ixgbe_tx_buffer *tx_buffer) 950 { 951 if (tx_buffer->skb) { 952 dev_kfree_skb_any(tx_buffer->skb); 953 if (dma_unmap_len(tx_buffer, len)) 954 dma_unmap_single(ring->dev, 955 dma_unmap_addr(tx_buffer, dma), 956 dma_unmap_len(tx_buffer, len), 957 DMA_TO_DEVICE); 958 } else if (dma_unmap_len(tx_buffer, len)) { 959 dma_unmap_page(ring->dev, 960 dma_unmap_addr(tx_buffer, dma), 961 dma_unmap_len(tx_buffer, len), 962 DMA_TO_DEVICE); 963 } 964 tx_buffer->next_to_watch = NULL; 965 tx_buffer->skb = NULL; 966 dma_unmap_len_set(tx_buffer, len, 0); 967 /* tx_buffer must be completely set up in the transmit path */ 968 } 969 970 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) 971 { 972 struct ixgbe_hw *hw = &adapter->hw; 973 struct ixgbe_hw_stats *hwstats = &adapter->stats; 974 int i; 975 u32 data; 976 977 if ((hw->fc.current_mode != ixgbe_fc_full) && 978 (hw->fc.current_mode != ixgbe_fc_rx_pause)) 979 return; 980 981 switch (hw->mac.type) { 982 case ixgbe_mac_82598EB: 983 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 984 break; 985 default: 986 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 987 } 988 hwstats->lxoffrxc += data; 989 990 /* refill credits (no tx hang) if we received xoff */ 991 if (!data) 992 return; 993 994 for (i = 0; i < adapter->num_tx_queues; i++) 995 clear_bit(__IXGBE_HANG_CHECK_ARMED, 996 &adapter->tx_ring[i]->state); 997 } 998 999 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) 1000 { 1001 struct ixgbe_hw *hw = &adapter->hw; 1002 struct ixgbe_hw_stats *hwstats = &adapter->stats; 1003 u32 xoff[8] = {0}; 1004 u8 tc; 1005 int i; 1006 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 1007 1008 if (adapter->ixgbe_ieee_pfc) 1009 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 1010 1011 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) { 1012 ixgbe_update_xoff_rx_lfc(adapter); 1013 return; 1014 } 1015 1016 /* update stats for each tc, only valid with PFC enabled */ 1017 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) { 1018 u32 pxoffrxc; 1019 1020 switch (hw->mac.type) { 1021 case ixgbe_mac_82598EB: 1022 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 1023 break; 1024 default: 1025 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 1026 } 1027 hwstats->pxoffrxc[i] += pxoffrxc; 1028 /* Get the TC for given UP */ 1029 tc = netdev_get_prio_tc_map(adapter->netdev, i); 1030 xoff[tc] += pxoffrxc; 1031 } 1032 1033 /* disarm tx queues that have received xoff frames */ 1034 for (i = 0; i < adapter->num_tx_queues; i++) { 1035 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 1036 1037 tc = tx_ring->dcb_tc; 1038 if (xoff[tc]) 1039 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 1040 } 1041 } 1042 1043 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring) 1044 { 1045 return ring->stats.packets; 1046 } 1047 1048 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring) 1049 { 1050 struct ixgbe_adapter *adapter; 1051 struct ixgbe_hw *hw; 1052 u32 head, tail; 1053 1054 if (ring->l2_accel_priv) 1055 adapter = ring->l2_accel_priv->real_adapter; 1056 else 1057 adapter = netdev_priv(ring->netdev); 1058 1059 hw = &adapter->hw; 1060 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx)); 1061 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx)); 1062 1063 if (head != tail) 1064 return (head < tail) ? 1065 tail - head : (tail + ring->count - head); 1066 1067 return 0; 1068 } 1069 1070 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring) 1071 { 1072 u32 tx_done = ixgbe_get_tx_completed(tx_ring); 1073 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 1074 u32 tx_pending = ixgbe_get_tx_pending(tx_ring); 1075 1076 clear_check_for_tx_hang(tx_ring); 1077 1078 /* 1079 * Check for a hung queue, but be thorough. This verifies 1080 * that a transmit has been completed since the previous 1081 * check AND there is at least one packet pending. The 1082 * ARMED bit is set to indicate a potential hang. The 1083 * bit is cleared if a pause frame is received to remove 1084 * false hang detection due to PFC or 802.3x frames. By 1085 * requiring this to fail twice we avoid races with 1086 * pfc clearing the ARMED bit and conditions where we 1087 * run the check_tx_hang logic with a transmit completion 1088 * pending but without time to complete it yet. 1089 */ 1090 if (tx_done_old == tx_done && tx_pending) 1091 /* make sure it is true for two checks in a row */ 1092 return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED, 1093 &tx_ring->state); 1094 /* update completed stats and continue */ 1095 tx_ring->tx_stats.tx_done_old = tx_done; 1096 /* reset the countdown */ 1097 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 1098 1099 return false; 1100 } 1101 1102 /** 1103 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout 1104 * @adapter: driver private struct 1105 **/ 1106 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) 1107 { 1108 1109 /* Do the reset outside of interrupt context */ 1110 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 1111 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state); 1112 e_warn(drv, "initiating reset due to tx timeout\n"); 1113 ixgbe_service_event_schedule(adapter); 1114 } 1115 } 1116 1117 /** 1118 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate 1119 **/ 1120 static int ixgbe_tx_maxrate(struct net_device *netdev, 1121 int queue_index, u32 maxrate) 1122 { 1123 struct ixgbe_adapter *adapter = netdev_priv(netdev); 1124 struct ixgbe_hw *hw = &adapter->hw; 1125 u32 bcnrc_val = ixgbe_link_mbps(adapter); 1126 1127 if (!maxrate) 1128 return 0; 1129 1130 /* Calculate the rate factor values to set */ 1131 bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT; 1132 bcnrc_val /= maxrate; 1133 1134 /* clear everything but the rate factor */ 1135 bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK | 1136 IXGBE_RTTBCNRC_RF_DEC_MASK; 1137 1138 /* enable the rate scheduler */ 1139 bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA; 1140 1141 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index); 1142 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val); 1143 1144 return 0; 1145 } 1146 1147 /** 1148 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes 1149 * @q_vector: structure containing interrupt and ring information 1150 * @tx_ring: tx ring to clean 1151 * @napi_budget: Used to determine if we are in netpoll 1152 **/ 1153 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, 1154 struct ixgbe_ring *tx_ring, int napi_budget) 1155 { 1156 struct ixgbe_adapter *adapter = q_vector->adapter; 1157 struct ixgbe_tx_buffer *tx_buffer; 1158 union ixgbe_adv_tx_desc *tx_desc; 1159 unsigned int total_bytes = 0, total_packets = 0; 1160 unsigned int budget = q_vector->tx.work_limit; 1161 unsigned int i = tx_ring->next_to_clean; 1162 1163 if (test_bit(__IXGBE_DOWN, &adapter->state)) 1164 return true; 1165 1166 tx_buffer = &tx_ring->tx_buffer_info[i]; 1167 tx_desc = IXGBE_TX_DESC(tx_ring, i); 1168 i -= tx_ring->count; 1169 1170 do { 1171 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 1172 1173 /* if next_to_watch is not set then there is no work pending */ 1174 if (!eop_desc) 1175 break; 1176 1177 /* prevent any other reads prior to eop_desc */ 1178 read_barrier_depends(); 1179 1180 /* if DD is not set pending work has not been completed */ 1181 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 1182 break; 1183 1184 /* clear next_to_watch to prevent false hangs */ 1185 tx_buffer->next_to_watch = NULL; 1186 1187 /* update the statistics for this packet */ 1188 total_bytes += tx_buffer->bytecount; 1189 total_packets += tx_buffer->gso_segs; 1190 1191 /* free the skb */ 1192 napi_consume_skb(tx_buffer->skb, napi_budget); 1193 1194 /* unmap skb header data */ 1195 dma_unmap_single(tx_ring->dev, 1196 dma_unmap_addr(tx_buffer, dma), 1197 dma_unmap_len(tx_buffer, len), 1198 DMA_TO_DEVICE); 1199 1200 /* clear tx_buffer data */ 1201 tx_buffer->skb = NULL; 1202 dma_unmap_len_set(tx_buffer, len, 0); 1203 1204 /* unmap remaining buffers */ 1205 while (tx_desc != eop_desc) { 1206 tx_buffer++; 1207 tx_desc++; 1208 i++; 1209 if (unlikely(!i)) { 1210 i -= tx_ring->count; 1211 tx_buffer = tx_ring->tx_buffer_info; 1212 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 1213 } 1214 1215 /* unmap any remaining paged data */ 1216 if (dma_unmap_len(tx_buffer, len)) { 1217 dma_unmap_page(tx_ring->dev, 1218 dma_unmap_addr(tx_buffer, dma), 1219 dma_unmap_len(tx_buffer, len), 1220 DMA_TO_DEVICE); 1221 dma_unmap_len_set(tx_buffer, len, 0); 1222 } 1223 } 1224 1225 /* move us one more past the eop_desc for start of next pkt */ 1226 tx_buffer++; 1227 tx_desc++; 1228 i++; 1229 if (unlikely(!i)) { 1230 i -= tx_ring->count; 1231 tx_buffer = tx_ring->tx_buffer_info; 1232 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 1233 } 1234 1235 /* issue prefetch for next Tx descriptor */ 1236 prefetch(tx_desc); 1237 1238 /* update budget accounting */ 1239 budget--; 1240 } while (likely(budget)); 1241 1242 i += tx_ring->count; 1243 tx_ring->next_to_clean = i; 1244 u64_stats_update_begin(&tx_ring->syncp); 1245 tx_ring->stats.bytes += total_bytes; 1246 tx_ring->stats.packets += total_packets; 1247 u64_stats_update_end(&tx_ring->syncp); 1248 q_vector->tx.total_bytes += total_bytes; 1249 q_vector->tx.total_packets += total_packets; 1250 1251 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { 1252 /* schedule immediate reset if we believe we hung */ 1253 struct ixgbe_hw *hw = &adapter->hw; 1254 e_err(drv, "Detected Tx Unit Hang\n" 1255 " Tx Queue <%d>\n" 1256 " TDH, TDT <%x>, <%x>\n" 1257 " next_to_use <%x>\n" 1258 " next_to_clean <%x>\n" 1259 "tx_buffer_info[next_to_clean]\n" 1260 " time_stamp <%lx>\n" 1261 " jiffies <%lx>\n", 1262 tx_ring->queue_index, 1263 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), 1264 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), 1265 tx_ring->next_to_use, i, 1266 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 1267 1268 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 1269 1270 e_info(probe, 1271 "tx hang %d detected on queue %d, resetting adapter\n", 1272 adapter->tx_timeout_count + 1, tx_ring->queue_index); 1273 1274 /* schedule immediate reset if we believe we hung */ 1275 ixgbe_tx_timeout_reset(adapter); 1276 1277 /* the adapter is about to reset, no point in enabling stuff */ 1278 return true; 1279 } 1280 1281 netdev_tx_completed_queue(txring_txq(tx_ring), 1282 total_packets, total_bytes); 1283 1284 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 1285 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 1286 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 1287 /* Make sure that anybody stopping the queue after this 1288 * sees the new next_to_clean. 1289 */ 1290 smp_mb(); 1291 if (__netif_subqueue_stopped(tx_ring->netdev, 1292 tx_ring->queue_index) 1293 && !test_bit(__IXGBE_DOWN, &adapter->state)) { 1294 netif_wake_subqueue(tx_ring->netdev, 1295 tx_ring->queue_index); 1296 ++tx_ring->tx_stats.restart_queue; 1297 } 1298 } 1299 1300 return !!budget; 1301 } 1302 1303 #ifdef CONFIG_IXGBE_DCA 1304 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, 1305 struct ixgbe_ring *tx_ring, 1306 int cpu) 1307 { 1308 struct ixgbe_hw *hw = &adapter->hw; 1309 u32 txctrl = 0; 1310 u16 reg_offset; 1311 1312 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1313 txctrl = dca3_get_tag(tx_ring->dev, cpu); 1314 1315 switch (hw->mac.type) { 1316 case ixgbe_mac_82598EB: 1317 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx); 1318 break; 1319 case ixgbe_mac_82599EB: 1320 case ixgbe_mac_X540: 1321 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx); 1322 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599; 1323 break; 1324 default: 1325 /* for unknown hardware do not write register */ 1326 return; 1327 } 1328 1329 /* 1330 * We can enable relaxed ordering for reads, but not writes when 1331 * DCA is enabled. This is due to a known issue in some chipsets 1332 * which will cause the DCA tag to be cleared. 1333 */ 1334 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1335 IXGBE_DCA_TXCTRL_DATA_RRO_EN | 1336 IXGBE_DCA_TXCTRL_DESC_DCA_EN; 1337 1338 IXGBE_WRITE_REG(hw, reg_offset, txctrl); 1339 } 1340 1341 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, 1342 struct ixgbe_ring *rx_ring, 1343 int cpu) 1344 { 1345 struct ixgbe_hw *hw = &adapter->hw; 1346 u32 rxctrl = 0; 1347 u8 reg_idx = rx_ring->reg_idx; 1348 1349 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1350 rxctrl = dca3_get_tag(rx_ring->dev, cpu); 1351 1352 switch (hw->mac.type) { 1353 case ixgbe_mac_82599EB: 1354 case ixgbe_mac_X540: 1355 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599; 1356 break; 1357 default: 1358 break; 1359 } 1360 1361 /* 1362 * We can enable relaxed ordering for reads, but not writes when 1363 * DCA is enabled. This is due to a known issue in some chipsets 1364 * which will cause the DCA tag to be cleared. 1365 */ 1366 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN | 1367 IXGBE_DCA_RXCTRL_DATA_DCA_EN | 1368 IXGBE_DCA_RXCTRL_DESC_DCA_EN; 1369 1370 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl); 1371 } 1372 1373 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector) 1374 { 1375 struct ixgbe_adapter *adapter = q_vector->adapter; 1376 struct ixgbe_ring *ring; 1377 int cpu = get_cpu(); 1378 1379 if (q_vector->cpu == cpu) 1380 goto out_no_update; 1381 1382 ixgbe_for_each_ring(ring, q_vector->tx) 1383 ixgbe_update_tx_dca(adapter, ring, cpu); 1384 1385 ixgbe_for_each_ring(ring, q_vector->rx) 1386 ixgbe_update_rx_dca(adapter, ring, cpu); 1387 1388 q_vector->cpu = cpu; 1389 out_no_update: 1390 put_cpu(); 1391 } 1392 1393 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) 1394 { 1395 int i; 1396 1397 /* always use CB2 mode, difference is masked in the CB driver */ 1398 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1399 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1400 IXGBE_DCA_CTRL_DCA_MODE_CB2); 1401 else 1402 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1403 IXGBE_DCA_CTRL_DCA_DISABLE); 1404 1405 for (i = 0; i < adapter->num_q_vectors; i++) { 1406 adapter->q_vector[i]->cpu = -1; 1407 ixgbe_update_dca(adapter->q_vector[i]); 1408 } 1409 } 1410 1411 static int __ixgbe_notify_dca(struct device *dev, void *data) 1412 { 1413 struct ixgbe_adapter *adapter = dev_get_drvdata(dev); 1414 unsigned long event = *(unsigned long *)data; 1415 1416 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE)) 1417 return 0; 1418 1419 switch (event) { 1420 case DCA_PROVIDER_ADD: 1421 /* if we're already enabled, don't do it again */ 1422 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1423 break; 1424 if (dca_add_requester(dev) == 0) { 1425 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 1426 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1427 IXGBE_DCA_CTRL_DCA_MODE_CB2); 1428 break; 1429 } 1430 /* Fall Through since DCA is disabled. */ 1431 case DCA_PROVIDER_REMOVE: 1432 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 1433 dca_remove_requester(dev); 1434 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 1435 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1436 IXGBE_DCA_CTRL_DCA_DISABLE); 1437 } 1438 break; 1439 } 1440 1441 return 0; 1442 } 1443 1444 #endif /* CONFIG_IXGBE_DCA */ 1445 1446 #define IXGBE_RSS_L4_TYPES_MASK \ 1447 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 1448 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 1449 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 1450 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 1451 1452 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, 1453 union ixgbe_adv_rx_desc *rx_desc, 1454 struct sk_buff *skb) 1455 { 1456 u16 rss_type; 1457 1458 if (!(ring->netdev->features & NETIF_F_RXHASH)) 1459 return; 1460 1461 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 1462 IXGBE_RXDADV_RSSTYPE_MASK; 1463 1464 if (!rss_type) 1465 return; 1466 1467 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1468 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 1469 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 1470 } 1471 1472 #ifdef IXGBE_FCOE 1473 /** 1474 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type 1475 * @ring: structure containing ring specific data 1476 * @rx_desc: advanced rx descriptor 1477 * 1478 * Returns : true if it is FCoE pkt 1479 */ 1480 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, 1481 union ixgbe_adv_rx_desc *rx_desc) 1482 { 1483 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1484 1485 return test_bit(__IXGBE_RX_FCOE, &ring->state) && 1486 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == 1487 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << 1488 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); 1489 } 1490 1491 #endif /* IXGBE_FCOE */ 1492 /** 1493 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum 1494 * @ring: structure containing ring specific data 1495 * @rx_desc: current Rx descriptor being processed 1496 * @skb: skb currently being received and modified 1497 **/ 1498 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, 1499 union ixgbe_adv_rx_desc *rx_desc, 1500 struct sk_buff *skb) 1501 { 1502 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1503 bool encap_pkt = false; 1504 1505 skb_checksum_none_assert(skb); 1506 1507 /* Rx csum disabled */ 1508 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1509 return; 1510 1511 /* check for VXLAN and Geneve packets */ 1512 if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) { 1513 encap_pkt = true; 1514 skb->encapsulation = 1; 1515 } 1516 1517 /* if IP and error */ 1518 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 1519 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 1520 ring->rx_stats.csum_err++; 1521 return; 1522 } 1523 1524 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 1525 return; 1526 1527 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 1528 /* 1529 * 82599 errata, UDP frames with a 0 checksum can be marked as 1530 * checksum errors. 1531 */ 1532 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && 1533 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) 1534 return; 1535 1536 ring->rx_stats.csum_err++; 1537 return; 1538 } 1539 1540 /* It must be a TCP or UDP packet with a valid checksum */ 1541 skb->ip_summed = CHECKSUM_UNNECESSARY; 1542 if (encap_pkt) { 1543 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS)) 1544 return; 1545 1546 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) { 1547 skb->ip_summed = CHECKSUM_NONE; 1548 return; 1549 } 1550 /* If we checked the outer header let the stack know */ 1551 skb->csum_level = 1; 1552 } 1553 } 1554 1555 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, 1556 struct ixgbe_rx_buffer *bi) 1557 { 1558 struct page *page = bi->page; 1559 dma_addr_t dma; 1560 1561 /* since we are recycling buffers we should seldom need to alloc */ 1562 if (likely(page)) 1563 return true; 1564 1565 /* alloc new page for storage */ 1566 page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring)); 1567 if (unlikely(!page)) { 1568 rx_ring->rx_stats.alloc_rx_page_failed++; 1569 return false; 1570 } 1571 1572 /* map page for use */ 1573 dma = dma_map_page(rx_ring->dev, page, 0, 1574 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1575 1576 /* 1577 * if mapping failed free memory back to system since 1578 * there isn't much point in holding memory we can't use 1579 */ 1580 if (dma_mapping_error(rx_ring->dev, dma)) { 1581 __free_pages(page, ixgbe_rx_pg_order(rx_ring)); 1582 1583 rx_ring->rx_stats.alloc_rx_page_failed++; 1584 return false; 1585 } 1586 1587 bi->dma = dma; 1588 bi->page = page; 1589 bi->page_offset = 0; 1590 1591 return true; 1592 } 1593 1594 /** 1595 * ixgbe_alloc_rx_buffers - Replace used receive buffers 1596 * @rx_ring: ring to place buffers on 1597 * @cleaned_count: number of buffers to replace 1598 **/ 1599 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) 1600 { 1601 union ixgbe_adv_rx_desc *rx_desc; 1602 struct ixgbe_rx_buffer *bi; 1603 u16 i = rx_ring->next_to_use; 1604 1605 /* nothing to do */ 1606 if (!cleaned_count) 1607 return; 1608 1609 rx_desc = IXGBE_RX_DESC(rx_ring, i); 1610 bi = &rx_ring->rx_buffer_info[i]; 1611 i -= rx_ring->count; 1612 1613 do { 1614 if (!ixgbe_alloc_mapped_page(rx_ring, bi)) 1615 break; 1616 1617 /* 1618 * Refresh the desc even if buffer_addrs didn't change 1619 * because each write-back erases this info. 1620 */ 1621 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1622 1623 rx_desc++; 1624 bi++; 1625 i++; 1626 if (unlikely(!i)) { 1627 rx_desc = IXGBE_RX_DESC(rx_ring, 0); 1628 bi = rx_ring->rx_buffer_info; 1629 i -= rx_ring->count; 1630 } 1631 1632 /* clear the status bits for the next_to_use descriptor */ 1633 rx_desc->wb.upper.status_error = 0; 1634 1635 cleaned_count--; 1636 } while (cleaned_count); 1637 1638 i += rx_ring->count; 1639 1640 if (rx_ring->next_to_use != i) { 1641 rx_ring->next_to_use = i; 1642 1643 /* update next to alloc since we have filled the ring */ 1644 rx_ring->next_to_alloc = i; 1645 1646 /* Force memory writes to complete before letting h/w 1647 * know there are new descriptors to fetch. (Only 1648 * applicable for weak-ordered memory model archs, 1649 * such as IA-64). 1650 */ 1651 wmb(); 1652 writel(i, rx_ring->tail); 1653 } 1654 } 1655 1656 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring, 1657 struct sk_buff *skb) 1658 { 1659 u16 hdr_len = skb_headlen(skb); 1660 1661 /* set gso_size to avoid messing up TCP MSS */ 1662 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len), 1663 IXGBE_CB(skb)->append_cnt); 1664 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 1665 } 1666 1667 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, 1668 struct sk_buff *skb) 1669 { 1670 /* if append_cnt is 0 then frame is not RSC */ 1671 if (!IXGBE_CB(skb)->append_cnt) 1672 return; 1673 1674 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt; 1675 rx_ring->rx_stats.rsc_flush++; 1676 1677 ixgbe_set_rsc_gso_size(rx_ring, skb); 1678 1679 /* gso_size is computed using append_cnt so always clear it last */ 1680 IXGBE_CB(skb)->append_cnt = 0; 1681 } 1682 1683 /** 1684 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor 1685 * @rx_ring: rx descriptor ring packet is being transacted on 1686 * @rx_desc: pointer to the EOP Rx descriptor 1687 * @skb: pointer to current skb being populated 1688 * 1689 * This function checks the ring, descriptor, and packet information in 1690 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1691 * other fields within the skb. 1692 **/ 1693 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, 1694 union ixgbe_adv_rx_desc *rx_desc, 1695 struct sk_buff *skb) 1696 { 1697 struct net_device *dev = rx_ring->netdev; 1698 u32 flags = rx_ring->q_vector->adapter->flags; 1699 1700 ixgbe_update_rsc_stats(rx_ring, skb); 1701 1702 ixgbe_rx_hash(rx_ring, rx_desc, skb); 1703 1704 ixgbe_rx_checksum(rx_ring, rx_desc, skb); 1705 1706 if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED)) 1707 ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); 1708 1709 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1710 ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 1711 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 1712 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 1713 } 1714 1715 skb_record_rx_queue(skb, rx_ring->queue_index); 1716 1717 skb->protocol = eth_type_trans(skb, dev); 1718 } 1719 1720 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, 1721 struct sk_buff *skb) 1722 { 1723 napi_gro_receive(&q_vector->napi, skb); 1724 } 1725 1726 /** 1727 * ixgbe_is_non_eop - process handling of non-EOP buffers 1728 * @rx_ring: Rx ring being processed 1729 * @rx_desc: Rx descriptor for current buffer 1730 * @skb: Current socket buffer containing buffer in progress 1731 * 1732 * This function updates next to clean. If the buffer is an EOP buffer 1733 * this function exits returning false, otherwise it will place the 1734 * sk_buff in the next buffer to be chained and return true indicating 1735 * that this is in fact a non-EOP buffer. 1736 **/ 1737 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, 1738 union ixgbe_adv_rx_desc *rx_desc, 1739 struct sk_buff *skb) 1740 { 1741 u32 ntc = rx_ring->next_to_clean + 1; 1742 1743 /* fetch, update, and store next to clean */ 1744 ntc = (ntc < rx_ring->count) ? ntc : 0; 1745 rx_ring->next_to_clean = ntc; 1746 1747 prefetch(IXGBE_RX_DESC(rx_ring, ntc)); 1748 1749 /* update RSC append count if present */ 1750 if (ring_is_rsc_enabled(rx_ring)) { 1751 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & 1752 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); 1753 1754 if (unlikely(rsc_enabled)) { 1755 u32 rsc_cnt = le32_to_cpu(rsc_enabled); 1756 1757 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 1758 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; 1759 1760 /* update ntc based on RSC value */ 1761 ntc = le32_to_cpu(rx_desc->wb.upper.status_error); 1762 ntc &= IXGBE_RXDADV_NEXTP_MASK; 1763 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT; 1764 } 1765 } 1766 1767 /* if we are the last buffer then there is nothing else to do */ 1768 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 1769 return false; 1770 1771 /* place skb in next buffer to be received */ 1772 rx_ring->rx_buffer_info[ntc].skb = skb; 1773 rx_ring->rx_stats.non_eop_descs++; 1774 1775 return true; 1776 } 1777 1778 /** 1779 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail 1780 * @rx_ring: rx descriptor ring packet is being transacted on 1781 * @skb: pointer to current skb being adjusted 1782 * 1783 * This function is an ixgbe specific version of __pskb_pull_tail. The 1784 * main difference between this version and the original function is that 1785 * this function can make several assumptions about the state of things 1786 * that allow for significant optimizations versus the standard function. 1787 * As a result we can do things like drop a frag and maintain an accurate 1788 * truesize for the skb. 1789 */ 1790 static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, 1791 struct sk_buff *skb) 1792 { 1793 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1794 unsigned char *va; 1795 unsigned int pull_len; 1796 1797 /* 1798 * it is valid to use page_address instead of kmap since we are 1799 * working with pages allocated out of the lomem pool per 1800 * alloc_page(GFP_ATOMIC) 1801 */ 1802 va = skb_frag_address(frag); 1803 1804 /* 1805 * we need the header to contain the greater of either ETH_HLEN or 1806 * 60 bytes if the skb->len is less than 60 for skb_pad. 1807 */ 1808 pull_len = eth_get_headlen(va, IXGBE_RX_HDR_SIZE); 1809 1810 /* align pull length to size of long to optimize memcpy performance */ 1811 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 1812 1813 /* update all of the pointers */ 1814 skb_frag_size_sub(frag, pull_len); 1815 frag->page_offset += pull_len; 1816 skb->data_len -= pull_len; 1817 skb->tail += pull_len; 1818 } 1819 1820 /** 1821 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB 1822 * @rx_ring: rx descriptor ring packet is being transacted on 1823 * @skb: pointer to current skb being updated 1824 * 1825 * This function provides a basic DMA sync up for the first fragment of an 1826 * skb. The reason for doing this is that the first fragment cannot be 1827 * unmapped until we have reached the end of packet descriptor for a buffer 1828 * chain. 1829 */ 1830 static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, 1831 struct sk_buff *skb) 1832 { 1833 /* if the page was released unmap it, else just sync our portion */ 1834 if (unlikely(IXGBE_CB(skb)->page_released)) { 1835 dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma, 1836 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1837 IXGBE_CB(skb)->page_released = false; 1838 } else { 1839 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1840 1841 dma_sync_single_range_for_cpu(rx_ring->dev, 1842 IXGBE_CB(skb)->dma, 1843 frag->page_offset, 1844 ixgbe_rx_bufsz(rx_ring), 1845 DMA_FROM_DEVICE); 1846 } 1847 IXGBE_CB(skb)->dma = 0; 1848 } 1849 1850 /** 1851 * ixgbe_cleanup_headers - Correct corrupted or empty headers 1852 * @rx_ring: rx descriptor ring packet is being transacted on 1853 * @rx_desc: pointer to the EOP Rx descriptor 1854 * @skb: pointer to current skb being fixed 1855 * 1856 * Check for corrupted packet headers caused by senders on the local L2 1857 * embedded NIC switch not setting up their Tx Descriptors right. These 1858 * should be very rare. 1859 * 1860 * Also address the case where we are pulling data in on pages only 1861 * and as such no data is present in the skb header. 1862 * 1863 * In addition if skb is not at least 60 bytes we need to pad it so that 1864 * it is large enough to qualify as a valid Ethernet frame. 1865 * 1866 * Returns true if an error was encountered and skb was freed. 1867 **/ 1868 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, 1869 union ixgbe_adv_rx_desc *rx_desc, 1870 struct sk_buff *skb) 1871 { 1872 struct net_device *netdev = rx_ring->netdev; 1873 1874 /* verify that the packet does not have any known errors */ 1875 if (unlikely(ixgbe_test_staterr(rx_desc, 1876 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && 1877 !(netdev->features & NETIF_F_RXALL))) { 1878 dev_kfree_skb_any(skb); 1879 return true; 1880 } 1881 1882 /* place header in linear portion of buffer */ 1883 if (skb_is_nonlinear(skb)) 1884 ixgbe_pull_tail(rx_ring, skb); 1885 1886 #ifdef IXGBE_FCOE 1887 /* do not attempt to pad FCoE Frames as this will disrupt DDP */ 1888 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) 1889 return false; 1890 1891 #endif 1892 /* if eth_skb_pad returns an error the skb was freed */ 1893 if (eth_skb_pad(skb)) 1894 return true; 1895 1896 return false; 1897 } 1898 1899 /** 1900 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring 1901 * @rx_ring: rx descriptor ring to store buffers on 1902 * @old_buff: donor buffer to have page reused 1903 * 1904 * Synchronizes page for reuse by the adapter 1905 **/ 1906 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, 1907 struct ixgbe_rx_buffer *old_buff) 1908 { 1909 struct ixgbe_rx_buffer *new_buff; 1910 u16 nta = rx_ring->next_to_alloc; 1911 1912 new_buff = &rx_ring->rx_buffer_info[nta]; 1913 1914 /* update, and store next to alloc */ 1915 nta++; 1916 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1917 1918 /* transfer page from old buffer to new buffer */ 1919 *new_buff = *old_buff; 1920 1921 /* sync the buffer for use by the device */ 1922 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 1923 new_buff->page_offset, 1924 ixgbe_rx_bufsz(rx_ring), 1925 DMA_FROM_DEVICE); 1926 } 1927 1928 static inline bool ixgbe_page_is_reserved(struct page *page) 1929 { 1930 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1931 } 1932 1933 /** 1934 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff 1935 * @rx_ring: rx descriptor ring to transact packets on 1936 * @rx_buffer: buffer containing page to add 1937 * @rx_desc: descriptor containing length of buffer written by hardware 1938 * @skb: sk_buff to place the data into 1939 * 1940 * This function will add the data contained in rx_buffer->page to the skb. 1941 * This is done either through a direct copy if the data in the buffer is 1942 * less than the skb header size, otherwise it will just attach the page as 1943 * a frag to the skb. 1944 * 1945 * The function will then update the page offset if necessary and return 1946 * true if the buffer can be reused by the adapter. 1947 **/ 1948 static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, 1949 struct ixgbe_rx_buffer *rx_buffer, 1950 union ixgbe_adv_rx_desc *rx_desc, 1951 struct sk_buff *skb) 1952 { 1953 struct page *page = rx_buffer->page; 1954 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 1955 #if (PAGE_SIZE < 8192) 1956 unsigned int truesize = ixgbe_rx_bufsz(rx_ring); 1957 #else 1958 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 1959 unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) - 1960 ixgbe_rx_bufsz(rx_ring); 1961 #endif 1962 1963 if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) { 1964 unsigned char *va = page_address(page) + rx_buffer->page_offset; 1965 1966 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 1967 1968 /* page is not reserved, we can reuse buffer as-is */ 1969 if (likely(!ixgbe_page_is_reserved(page))) 1970 return true; 1971 1972 /* this page cannot be reused so discard it */ 1973 __free_pages(page, ixgbe_rx_pg_order(rx_ring)); 1974 return false; 1975 } 1976 1977 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 1978 rx_buffer->page_offset, size, truesize); 1979 1980 /* avoid re-using remote pages */ 1981 if (unlikely(ixgbe_page_is_reserved(page))) 1982 return false; 1983 1984 #if (PAGE_SIZE < 8192) 1985 /* if we are only owner of page we can reuse it */ 1986 if (unlikely(page_count(page) != 1)) 1987 return false; 1988 1989 /* flip page offset to other buffer */ 1990 rx_buffer->page_offset ^= truesize; 1991 #else 1992 /* move offset up to the next cache line */ 1993 rx_buffer->page_offset += truesize; 1994 1995 if (rx_buffer->page_offset > last_offset) 1996 return false; 1997 #endif 1998 1999 /* Even if we own the page, we are not allowed to use atomic_set() 2000 * This would break get_page_unless_zero() users. 2001 */ 2002 page_ref_inc(page); 2003 2004 return true; 2005 } 2006 2007 static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring, 2008 union ixgbe_adv_rx_desc *rx_desc) 2009 { 2010 struct ixgbe_rx_buffer *rx_buffer; 2011 struct sk_buff *skb; 2012 struct page *page; 2013 2014 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 2015 page = rx_buffer->page; 2016 prefetchw(page); 2017 2018 skb = rx_buffer->skb; 2019 2020 if (likely(!skb)) { 2021 void *page_addr = page_address(page) + 2022 rx_buffer->page_offset; 2023 2024 /* prefetch first cache line of first page */ 2025 prefetch(page_addr); 2026 #if L1_CACHE_BYTES < 128 2027 prefetch(page_addr + L1_CACHE_BYTES); 2028 #endif 2029 2030 /* allocate a skb to store the frags */ 2031 skb = napi_alloc_skb(&rx_ring->q_vector->napi, 2032 IXGBE_RX_HDR_SIZE); 2033 if (unlikely(!skb)) { 2034 rx_ring->rx_stats.alloc_rx_buff_failed++; 2035 return NULL; 2036 } 2037 2038 /* 2039 * we will be copying header into skb->data in 2040 * pskb_may_pull so it is in our interest to prefetch 2041 * it now to avoid a possible cache miss 2042 */ 2043 prefetchw(skb->data); 2044 2045 /* 2046 * Delay unmapping of the first packet. It carries the 2047 * header information, HW may still access the header 2048 * after the writeback. Only unmap it when EOP is 2049 * reached 2050 */ 2051 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 2052 goto dma_sync; 2053 2054 IXGBE_CB(skb)->dma = rx_buffer->dma; 2055 } else { 2056 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) 2057 ixgbe_dma_sync_frag(rx_ring, skb); 2058 2059 dma_sync: 2060 /* we are reusing so sync this buffer for CPU use */ 2061 dma_sync_single_range_for_cpu(rx_ring->dev, 2062 rx_buffer->dma, 2063 rx_buffer->page_offset, 2064 ixgbe_rx_bufsz(rx_ring), 2065 DMA_FROM_DEVICE); 2066 2067 rx_buffer->skb = NULL; 2068 } 2069 2070 /* pull page into skb */ 2071 if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 2072 /* hand second half of page back to the ring */ 2073 ixgbe_reuse_rx_page(rx_ring, rx_buffer); 2074 } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) { 2075 /* the page has been released from the ring */ 2076 IXGBE_CB(skb)->page_released = true; 2077 } else { 2078 /* we are not reusing the buffer so unmap it */ 2079 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 2080 ixgbe_rx_pg_size(rx_ring), 2081 DMA_FROM_DEVICE); 2082 } 2083 2084 /* clear contents of buffer_info */ 2085 rx_buffer->page = NULL; 2086 2087 return skb; 2088 } 2089 2090 /** 2091 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 2092 * @q_vector: structure containing interrupt and ring information 2093 * @rx_ring: rx descriptor ring to transact packets on 2094 * @budget: Total limit on number of packets to process 2095 * 2096 * This function provides a "bounce buffer" approach to Rx interrupt 2097 * processing. The advantage to this is that on systems that have 2098 * expensive overhead for IOMMU access this provides a means of avoiding 2099 * it by maintaining the mapping of the page to the syste. 2100 * 2101 * Returns amount of work completed 2102 **/ 2103 static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, 2104 struct ixgbe_ring *rx_ring, 2105 const int budget) 2106 { 2107 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 2108 #ifdef IXGBE_FCOE 2109 struct ixgbe_adapter *adapter = q_vector->adapter; 2110 int ddp_bytes; 2111 unsigned int mss = 0; 2112 #endif /* IXGBE_FCOE */ 2113 u16 cleaned_count = ixgbe_desc_unused(rx_ring); 2114 2115 while (likely(total_rx_packets < budget)) { 2116 union ixgbe_adv_rx_desc *rx_desc; 2117 struct sk_buff *skb; 2118 2119 /* return some buffers to hardware, one at a time is too slow */ 2120 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { 2121 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 2122 cleaned_count = 0; 2123 } 2124 2125 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); 2126 2127 if (!rx_desc->wb.upper.status_error) 2128 break; 2129 2130 /* This memory barrier is needed to keep us from reading 2131 * any other fields out of the rx_desc until we know the 2132 * descriptor has been written back 2133 */ 2134 dma_rmb(); 2135 2136 /* retrieve a buffer from the ring */ 2137 skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc); 2138 2139 /* exit if we failed to retrieve a buffer */ 2140 if (!skb) 2141 break; 2142 2143 cleaned_count++; 2144 2145 /* place incomplete frames back on ring for completion */ 2146 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) 2147 continue; 2148 2149 /* verify the packet layout is correct */ 2150 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) 2151 continue; 2152 2153 /* probably a little skewed due to removing CRC */ 2154 total_rx_bytes += skb->len; 2155 2156 /* populate checksum, timestamp, VLAN, and protocol */ 2157 ixgbe_process_skb_fields(rx_ring, rx_desc, skb); 2158 2159 #ifdef IXGBE_FCOE 2160 /* if ddp, not passing to ULD unless for FCP_RSP or error */ 2161 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) { 2162 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb); 2163 /* include DDPed FCoE data */ 2164 if (ddp_bytes > 0) { 2165 if (!mss) { 2166 mss = rx_ring->netdev->mtu - 2167 sizeof(struct fcoe_hdr) - 2168 sizeof(struct fc_frame_header) - 2169 sizeof(struct fcoe_crc_eof); 2170 if (mss > 512) 2171 mss &= ~511; 2172 } 2173 total_rx_bytes += ddp_bytes; 2174 total_rx_packets += DIV_ROUND_UP(ddp_bytes, 2175 mss); 2176 } 2177 if (!ddp_bytes) { 2178 dev_kfree_skb_any(skb); 2179 continue; 2180 } 2181 } 2182 2183 #endif /* IXGBE_FCOE */ 2184 ixgbe_rx_skb(q_vector, skb); 2185 2186 /* update budget accounting */ 2187 total_rx_packets++; 2188 } 2189 2190 u64_stats_update_begin(&rx_ring->syncp); 2191 rx_ring->stats.packets += total_rx_packets; 2192 rx_ring->stats.bytes += total_rx_bytes; 2193 u64_stats_update_end(&rx_ring->syncp); 2194 q_vector->rx.total_packets += total_rx_packets; 2195 q_vector->rx.total_bytes += total_rx_bytes; 2196 2197 return total_rx_packets; 2198 } 2199 2200 /** 2201 * ixgbe_configure_msix - Configure MSI-X hardware 2202 * @adapter: board private structure 2203 * 2204 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X 2205 * interrupts. 2206 **/ 2207 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) 2208 { 2209 struct ixgbe_q_vector *q_vector; 2210 int v_idx; 2211 u32 mask; 2212 2213 /* Populate MSIX to EITR Select */ 2214 if (adapter->num_vfs > 32) { 2215 u32 eitrsel = BIT(adapter->num_vfs - 32) - 1; 2216 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel); 2217 } 2218 2219 /* 2220 * Populate the IVAR table and set the ITR values to the 2221 * corresponding register. 2222 */ 2223 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { 2224 struct ixgbe_ring *ring; 2225 q_vector = adapter->q_vector[v_idx]; 2226 2227 ixgbe_for_each_ring(ring, q_vector->rx) 2228 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx); 2229 2230 ixgbe_for_each_ring(ring, q_vector->tx) 2231 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx); 2232 2233 ixgbe_write_eitr(q_vector); 2234 } 2235 2236 switch (adapter->hw.mac.type) { 2237 case ixgbe_mac_82598EB: 2238 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, 2239 v_idx); 2240 break; 2241 case ixgbe_mac_82599EB: 2242 case ixgbe_mac_X540: 2243 case ixgbe_mac_X550: 2244 case ixgbe_mac_X550EM_x: 2245 case ixgbe_mac_x550em_a: 2246 ixgbe_set_ivar(adapter, -1, 1, v_idx); 2247 break; 2248 default: 2249 break; 2250 } 2251 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950); 2252 2253 /* set up to autoclear timer, and the vectors */ 2254 mask = IXGBE_EIMS_ENABLE_MASK; 2255 mask &= ~(IXGBE_EIMS_OTHER | 2256 IXGBE_EIMS_MAILBOX | 2257 IXGBE_EIMS_LSC); 2258 2259 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); 2260 } 2261 2262 enum latency_range { 2263 lowest_latency = 0, 2264 low_latency = 1, 2265 bulk_latency = 2, 2266 latency_invalid = 255 2267 }; 2268 2269 /** 2270 * ixgbe_update_itr - update the dynamic ITR value based on statistics 2271 * @q_vector: structure containing interrupt and ring information 2272 * @ring_container: structure containing ring performance data 2273 * 2274 * Stores a new ITR value based on packets and byte 2275 * counts during the last interrupt. The advantage of per interrupt 2276 * computation is faster updates and more accurate ITR for the current 2277 * traffic pattern. Constants in this function were computed 2278 * based on theoretical maximum wire speed and thresholds were set based 2279 * on testing data as well as attempting to minimize response time 2280 * while increasing bulk throughput. 2281 * this functionality is controlled by the InterruptThrottleRate module 2282 * parameter (see ixgbe_param.c) 2283 **/ 2284 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, 2285 struct ixgbe_ring_container *ring_container) 2286 { 2287 int bytes = ring_container->total_bytes; 2288 int packets = ring_container->total_packets; 2289 u32 timepassed_us; 2290 u64 bytes_perint; 2291 u8 itr_setting = ring_container->itr; 2292 2293 if (packets == 0) 2294 return; 2295 2296 /* simple throttlerate management 2297 * 0-10MB/s lowest (100000 ints/s) 2298 * 10-20MB/s low (20000 ints/s) 2299 * 20-1249MB/s bulk (12000 ints/s) 2300 */ 2301 /* what was last interrupt timeslice? */ 2302 timepassed_us = q_vector->itr >> 2; 2303 if (timepassed_us == 0) 2304 return; 2305 2306 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 2307 2308 switch (itr_setting) { 2309 case lowest_latency: 2310 if (bytes_perint > 10) 2311 itr_setting = low_latency; 2312 break; 2313 case low_latency: 2314 if (bytes_perint > 20) 2315 itr_setting = bulk_latency; 2316 else if (bytes_perint <= 10) 2317 itr_setting = lowest_latency; 2318 break; 2319 case bulk_latency: 2320 if (bytes_perint <= 20) 2321 itr_setting = low_latency; 2322 break; 2323 } 2324 2325 /* clear work counters since we have the values we need */ 2326 ring_container->total_bytes = 0; 2327 ring_container->total_packets = 0; 2328 2329 /* write updated itr to ring container */ 2330 ring_container->itr = itr_setting; 2331 } 2332 2333 /** 2334 * ixgbe_write_eitr - write EITR register in hardware specific way 2335 * @q_vector: structure containing interrupt and ring information 2336 * 2337 * This function is made to be called by ethtool and by the driver 2338 * when it needs to update EITR registers at runtime. Hardware 2339 * specific quirks/differences are taken care of here. 2340 */ 2341 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector) 2342 { 2343 struct ixgbe_adapter *adapter = q_vector->adapter; 2344 struct ixgbe_hw *hw = &adapter->hw; 2345 int v_idx = q_vector->v_idx; 2346 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 2347 2348 switch (adapter->hw.mac.type) { 2349 case ixgbe_mac_82598EB: 2350 /* must write high and low 16 bits to reset counter */ 2351 itr_reg |= (itr_reg << 16); 2352 break; 2353 case ixgbe_mac_82599EB: 2354 case ixgbe_mac_X540: 2355 case ixgbe_mac_X550: 2356 case ixgbe_mac_X550EM_x: 2357 case ixgbe_mac_x550em_a: 2358 /* 2359 * set the WDIS bit to not clear the timer bits and cause an 2360 * immediate assertion of the interrupt 2361 */ 2362 itr_reg |= IXGBE_EITR_CNT_WDIS; 2363 break; 2364 default: 2365 break; 2366 } 2367 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg); 2368 } 2369 2370 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector) 2371 { 2372 u32 new_itr = q_vector->itr; 2373 u8 current_itr; 2374 2375 ixgbe_update_itr(q_vector, &q_vector->tx); 2376 ixgbe_update_itr(q_vector, &q_vector->rx); 2377 2378 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 2379 2380 switch (current_itr) { 2381 /* counts and packets in update_itr are dependent on these numbers */ 2382 case lowest_latency: 2383 new_itr = IXGBE_100K_ITR; 2384 break; 2385 case low_latency: 2386 new_itr = IXGBE_20K_ITR; 2387 break; 2388 case bulk_latency: 2389 new_itr = IXGBE_12K_ITR; 2390 break; 2391 default: 2392 break; 2393 } 2394 2395 if (new_itr != q_vector->itr) { 2396 /* do an exponential smoothing */ 2397 new_itr = (10 * new_itr * q_vector->itr) / 2398 ((9 * new_itr) + q_vector->itr); 2399 2400 /* save the algorithm value here */ 2401 q_vector->itr = new_itr; 2402 2403 ixgbe_write_eitr(q_vector); 2404 } 2405 } 2406 2407 /** 2408 * ixgbe_check_overtemp_subtask - check for over temperature 2409 * @adapter: pointer to adapter 2410 **/ 2411 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) 2412 { 2413 struct ixgbe_hw *hw = &adapter->hw; 2414 u32 eicr = adapter->interrupt_event; 2415 s32 rc; 2416 2417 if (test_bit(__IXGBE_DOWN, &adapter->state)) 2418 return; 2419 2420 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) && 2421 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT)) 2422 return; 2423 2424 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2425 2426 switch (hw->device_id) { 2427 case IXGBE_DEV_ID_82599_T3_LOM: 2428 /* 2429 * Since the warning interrupt is for both ports 2430 * we don't have to check if: 2431 * - This interrupt wasn't for our port. 2432 * - We may have missed the interrupt so always have to 2433 * check if we got a LSC 2434 */ 2435 if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) && 2436 !(eicr & IXGBE_EICR_LSC)) 2437 return; 2438 2439 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { 2440 u32 speed; 2441 bool link_up = false; 2442 2443 hw->mac.ops.check_link(hw, &speed, &link_up, false); 2444 2445 if (link_up) 2446 return; 2447 } 2448 2449 /* Check if this is not due to overtemp */ 2450 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP) 2451 return; 2452 2453 break; 2454 case IXGBE_DEV_ID_X550EM_A_1G_T: 2455 case IXGBE_DEV_ID_X550EM_A_1G_T_L: 2456 rc = hw->phy.ops.check_overtemp(hw); 2457 if (rc != IXGBE_ERR_OVERTEMP) 2458 return; 2459 break; 2460 default: 2461 if (adapter->hw.mac.type >= ixgbe_mac_X540) 2462 return; 2463 if (!(eicr & IXGBE_EICR_GPI_SDP0(hw))) 2464 return; 2465 break; 2466 } 2467 e_crit(drv, "%s\n", ixgbe_overheat_msg); 2468 2469 adapter->interrupt_event = 0; 2470 } 2471 2472 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) 2473 { 2474 struct ixgbe_hw *hw = &adapter->hw; 2475 2476 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 2477 (eicr & IXGBE_EICR_GPI_SDP1(hw))) { 2478 e_crit(probe, "Fan has stopped, replace the adapter\n"); 2479 /* write to clear the interrupt */ 2480 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); 2481 } 2482 } 2483 2484 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) 2485 { 2486 struct ixgbe_hw *hw = &adapter->hw; 2487 2488 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)) 2489 return; 2490 2491 switch (adapter->hw.mac.type) { 2492 case ixgbe_mac_82599EB: 2493 /* 2494 * Need to check link state so complete overtemp check 2495 * on service task 2496 */ 2497 if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) || 2498 (eicr & IXGBE_EICR_LSC)) && 2499 (!test_bit(__IXGBE_DOWN, &adapter->state))) { 2500 adapter->interrupt_event = eicr; 2501 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2502 ixgbe_service_event_schedule(adapter); 2503 return; 2504 } 2505 return; 2506 case ixgbe_mac_x550em_a: 2507 if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) { 2508 adapter->interrupt_event = eicr; 2509 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2510 ixgbe_service_event_schedule(adapter); 2511 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 2512 IXGBE_EICR_GPI_SDP0_X550EM_a); 2513 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR, 2514 IXGBE_EICR_GPI_SDP0_X550EM_a); 2515 } 2516 return; 2517 case ixgbe_mac_X550: 2518 case ixgbe_mac_X540: 2519 if (!(eicr & IXGBE_EICR_TS)) 2520 return; 2521 break; 2522 default: 2523 return; 2524 } 2525 2526 e_crit(drv, "%s\n", ixgbe_overheat_msg); 2527 } 2528 2529 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw) 2530 { 2531 switch (hw->mac.type) { 2532 case ixgbe_mac_82598EB: 2533 if (hw->phy.type == ixgbe_phy_nl) 2534 return true; 2535 return false; 2536 case ixgbe_mac_82599EB: 2537 case ixgbe_mac_X550EM_x: 2538 case ixgbe_mac_x550em_a: 2539 switch (hw->mac.ops.get_media_type(hw)) { 2540 case ixgbe_media_type_fiber: 2541 case ixgbe_media_type_fiber_qsfp: 2542 return true; 2543 default: 2544 return false; 2545 } 2546 default: 2547 return false; 2548 } 2549 } 2550 2551 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) 2552 { 2553 struct ixgbe_hw *hw = &adapter->hw; 2554 u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw); 2555 2556 if (!ixgbe_is_sfp(hw)) 2557 return; 2558 2559 /* Later MAC's use different SDP */ 2560 if (hw->mac.type >= ixgbe_mac_X540) 2561 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 2562 2563 if (eicr & eicr_mask) { 2564 /* Clear the interrupt */ 2565 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask); 2566 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2567 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 2568 adapter->sfp_poll_time = 0; 2569 ixgbe_service_event_schedule(adapter); 2570 } 2571 } 2572 2573 if (adapter->hw.mac.type == ixgbe_mac_82599EB && 2574 (eicr & IXGBE_EICR_GPI_SDP1(hw))) { 2575 /* Clear the interrupt */ 2576 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); 2577 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2578 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 2579 ixgbe_service_event_schedule(adapter); 2580 } 2581 } 2582 } 2583 2584 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) 2585 { 2586 struct ixgbe_hw *hw = &adapter->hw; 2587 2588 adapter->lsc_int++; 2589 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2590 adapter->link_check_timeout = jiffies; 2591 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2592 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 2593 IXGBE_WRITE_FLUSH(hw); 2594 ixgbe_service_event_schedule(adapter); 2595 } 2596 } 2597 2598 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, 2599 u64 qmask) 2600 { 2601 u32 mask; 2602 struct ixgbe_hw *hw = &adapter->hw; 2603 2604 switch (hw->mac.type) { 2605 case ixgbe_mac_82598EB: 2606 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2607 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2608 break; 2609 case ixgbe_mac_82599EB: 2610 case ixgbe_mac_X540: 2611 case ixgbe_mac_X550: 2612 case ixgbe_mac_X550EM_x: 2613 case ixgbe_mac_x550em_a: 2614 mask = (qmask & 0xFFFFFFFF); 2615 if (mask) 2616 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2617 mask = (qmask >> 32); 2618 if (mask) 2619 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2620 break; 2621 default: 2622 break; 2623 } 2624 /* skip the flush */ 2625 } 2626 2627 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter, 2628 u64 qmask) 2629 { 2630 u32 mask; 2631 struct ixgbe_hw *hw = &adapter->hw; 2632 2633 switch (hw->mac.type) { 2634 case ixgbe_mac_82598EB: 2635 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2636 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2637 break; 2638 case ixgbe_mac_82599EB: 2639 case ixgbe_mac_X540: 2640 case ixgbe_mac_X550: 2641 case ixgbe_mac_X550EM_x: 2642 case ixgbe_mac_x550em_a: 2643 mask = (qmask & 0xFFFFFFFF); 2644 if (mask) 2645 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2646 mask = (qmask >> 32); 2647 if (mask) 2648 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2649 break; 2650 default: 2651 break; 2652 } 2653 /* skip the flush */ 2654 } 2655 2656 /** 2657 * ixgbe_irq_enable - Enable default interrupt generation settings 2658 * @adapter: board private structure 2659 **/ 2660 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, 2661 bool flush) 2662 { 2663 struct ixgbe_hw *hw = &adapter->hw; 2664 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2665 2666 /* don't reenable LSC while waiting for link */ 2667 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 2668 mask &= ~IXGBE_EIMS_LSC; 2669 2670 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) 2671 switch (adapter->hw.mac.type) { 2672 case ixgbe_mac_82599EB: 2673 mask |= IXGBE_EIMS_GPI_SDP0(hw); 2674 break; 2675 case ixgbe_mac_X540: 2676 case ixgbe_mac_X550: 2677 case ixgbe_mac_X550EM_x: 2678 case ixgbe_mac_x550em_a: 2679 mask |= IXGBE_EIMS_TS; 2680 break; 2681 default: 2682 break; 2683 } 2684 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 2685 mask |= IXGBE_EIMS_GPI_SDP1(hw); 2686 switch (adapter->hw.mac.type) { 2687 case ixgbe_mac_82599EB: 2688 mask |= IXGBE_EIMS_GPI_SDP1(hw); 2689 mask |= IXGBE_EIMS_GPI_SDP2(hw); 2690 /* fall through */ 2691 case ixgbe_mac_X540: 2692 case ixgbe_mac_X550: 2693 case ixgbe_mac_X550EM_x: 2694 case ixgbe_mac_x550em_a: 2695 if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP || 2696 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || 2697 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) 2698 mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw); 2699 if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t) 2700 mask |= IXGBE_EICR_GPI_SDP0_X540; 2701 mask |= IXGBE_EIMS_ECC; 2702 mask |= IXGBE_EIMS_MAILBOX; 2703 break; 2704 default: 2705 break; 2706 } 2707 2708 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && 2709 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 2710 mask |= IXGBE_EIMS_FLOW_DIR; 2711 2712 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); 2713 if (queues) 2714 ixgbe_irq_enable_queues(adapter, ~0); 2715 if (flush) 2716 IXGBE_WRITE_FLUSH(&adapter->hw); 2717 } 2718 2719 static irqreturn_t ixgbe_msix_other(int irq, void *data) 2720 { 2721 struct ixgbe_adapter *adapter = data; 2722 struct ixgbe_hw *hw = &adapter->hw; 2723 u32 eicr; 2724 2725 /* 2726 * Workaround for Silicon errata. Use clear-by-write instead 2727 * of clear-by-read. Reading with EICS will return the 2728 * interrupt causes without clearing, which later be done 2729 * with the write to EICR. 2730 */ 2731 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 2732 2733 /* The lower 16bits of the EICR register are for the queue interrupts 2734 * which should be masked here in order to not accidentally clear them if 2735 * the bits are high when ixgbe_msix_other is called. There is a race 2736 * condition otherwise which results in possible performance loss 2737 * especially if the ixgbe_msix_other interrupt is triggering 2738 * consistently (as it would when PPS is turned on for the X540 device) 2739 */ 2740 eicr &= 0xFFFF0000; 2741 2742 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 2743 2744 if (eicr & IXGBE_EICR_LSC) 2745 ixgbe_check_lsc(adapter); 2746 2747 if (eicr & IXGBE_EICR_MAILBOX) 2748 ixgbe_msg_task(adapter); 2749 2750 switch (hw->mac.type) { 2751 case ixgbe_mac_82599EB: 2752 case ixgbe_mac_X540: 2753 case ixgbe_mac_X550: 2754 case ixgbe_mac_X550EM_x: 2755 case ixgbe_mac_x550em_a: 2756 if (hw->phy.type == ixgbe_phy_x550em_ext_t && 2757 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 2758 adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT; 2759 ixgbe_service_event_schedule(adapter); 2760 IXGBE_WRITE_REG(hw, IXGBE_EICR, 2761 IXGBE_EICR_GPI_SDP0_X540); 2762 } 2763 if (eicr & IXGBE_EICR_ECC) { 2764 e_info(link, "Received ECC Err, initiating reset\n"); 2765 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state); 2766 ixgbe_service_event_schedule(adapter); 2767 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 2768 } 2769 /* Handle Flow Director Full threshold interrupt */ 2770 if (eicr & IXGBE_EICR_FLOW_DIR) { 2771 int reinit_count = 0; 2772 int i; 2773 for (i = 0; i < adapter->num_tx_queues; i++) { 2774 struct ixgbe_ring *ring = adapter->tx_ring[i]; 2775 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, 2776 &ring->state)) 2777 reinit_count++; 2778 } 2779 if (reinit_count) { 2780 /* no more flow director interrupts until after init */ 2781 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR); 2782 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 2783 ixgbe_service_event_schedule(adapter); 2784 } 2785 } 2786 ixgbe_check_sfp_event(adapter, eicr); 2787 ixgbe_check_overtemp_event(adapter, eicr); 2788 break; 2789 default: 2790 break; 2791 } 2792 2793 ixgbe_check_fan_failure(adapter, eicr); 2794 2795 if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) 2796 ixgbe_ptp_check_pps_event(adapter); 2797 2798 /* re-enable the original interrupt state, no lsc, no queues */ 2799 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2800 ixgbe_irq_enable(adapter, false, false); 2801 2802 return IRQ_HANDLED; 2803 } 2804 2805 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data) 2806 { 2807 struct ixgbe_q_vector *q_vector = data; 2808 2809 /* EIAM disabled interrupts (on this vector) for us */ 2810 2811 if (q_vector->rx.ring || q_vector->tx.ring) 2812 napi_schedule_irqoff(&q_vector->napi); 2813 2814 return IRQ_HANDLED; 2815 } 2816 2817 /** 2818 * ixgbe_poll - NAPI Rx polling callback 2819 * @napi: structure for representing this polling device 2820 * @budget: how many packets driver is allowed to clean 2821 * 2822 * This function is used for legacy and MSI, NAPI mode 2823 **/ 2824 int ixgbe_poll(struct napi_struct *napi, int budget) 2825 { 2826 struct ixgbe_q_vector *q_vector = 2827 container_of(napi, struct ixgbe_q_vector, napi); 2828 struct ixgbe_adapter *adapter = q_vector->adapter; 2829 struct ixgbe_ring *ring; 2830 int per_ring_budget, work_done = 0; 2831 bool clean_complete = true; 2832 2833 #ifdef CONFIG_IXGBE_DCA 2834 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 2835 ixgbe_update_dca(q_vector); 2836 #endif 2837 2838 ixgbe_for_each_ring(ring, q_vector->tx) { 2839 if (!ixgbe_clean_tx_irq(q_vector, ring, budget)) 2840 clean_complete = false; 2841 } 2842 2843 /* Exit if we are called by netpoll */ 2844 if (budget <= 0) 2845 return budget; 2846 2847 /* attempt to distribute budget to each queue fairly, but don't allow 2848 * the budget to go below 1 because we'll exit polling */ 2849 if (q_vector->rx.count > 1) 2850 per_ring_budget = max(budget/q_vector->rx.count, 1); 2851 else 2852 per_ring_budget = budget; 2853 2854 ixgbe_for_each_ring(ring, q_vector->rx) { 2855 int cleaned = ixgbe_clean_rx_irq(q_vector, ring, 2856 per_ring_budget); 2857 2858 work_done += cleaned; 2859 if (cleaned >= per_ring_budget) 2860 clean_complete = false; 2861 } 2862 2863 /* If all work not completed, return budget and keep polling */ 2864 if (!clean_complete) 2865 return budget; 2866 2867 /* all work done, exit the polling mode */ 2868 napi_complete_done(napi, work_done); 2869 if (adapter->rx_itr_setting & 1) 2870 ixgbe_set_itr(q_vector); 2871 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2872 ixgbe_irq_enable_queues(adapter, BIT_ULL(q_vector->v_idx)); 2873 2874 return min(work_done, budget - 1); 2875 } 2876 2877 /** 2878 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts 2879 * @adapter: board private structure 2880 * 2881 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests 2882 * interrupts from the kernel. 2883 **/ 2884 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) 2885 { 2886 struct net_device *netdev = adapter->netdev; 2887 int vector, err; 2888 int ri = 0, ti = 0; 2889 2890 for (vector = 0; vector < adapter->num_q_vectors; vector++) { 2891 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 2892 struct msix_entry *entry = &adapter->msix_entries[vector]; 2893 2894 if (q_vector->tx.ring && q_vector->rx.ring) { 2895 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2896 "%s-%s-%d", netdev->name, "TxRx", ri++); 2897 ti++; 2898 } else if (q_vector->rx.ring) { 2899 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2900 "%s-%s-%d", netdev->name, "rx", ri++); 2901 } else if (q_vector->tx.ring) { 2902 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2903 "%s-%s-%d", netdev->name, "tx", ti++); 2904 } else { 2905 /* skip this unused q_vector */ 2906 continue; 2907 } 2908 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0, 2909 q_vector->name, q_vector); 2910 if (err) { 2911 e_err(probe, "request_irq failed for MSIX interrupt " 2912 "Error: %d\n", err); 2913 goto free_queue_irqs; 2914 } 2915 /* If Flow Director is enabled, set interrupt affinity */ 2916 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 2917 /* assign the mask for this irq */ 2918 irq_set_affinity_hint(entry->vector, 2919 &q_vector->affinity_mask); 2920 } 2921 } 2922 2923 err = request_irq(adapter->msix_entries[vector].vector, 2924 ixgbe_msix_other, 0, netdev->name, adapter); 2925 if (err) { 2926 e_err(probe, "request_irq for msix_other failed: %d\n", err); 2927 goto free_queue_irqs; 2928 } 2929 2930 return 0; 2931 2932 free_queue_irqs: 2933 while (vector) { 2934 vector--; 2935 irq_set_affinity_hint(adapter->msix_entries[vector].vector, 2936 NULL); 2937 free_irq(adapter->msix_entries[vector].vector, 2938 adapter->q_vector[vector]); 2939 } 2940 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; 2941 pci_disable_msix(adapter->pdev); 2942 kfree(adapter->msix_entries); 2943 adapter->msix_entries = NULL; 2944 return err; 2945 } 2946 2947 /** 2948 * ixgbe_intr - legacy mode Interrupt Handler 2949 * @irq: interrupt number 2950 * @data: pointer to a network interface device structure 2951 **/ 2952 static irqreturn_t ixgbe_intr(int irq, void *data) 2953 { 2954 struct ixgbe_adapter *adapter = data; 2955 struct ixgbe_hw *hw = &adapter->hw; 2956 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2957 u32 eicr; 2958 2959 /* 2960 * Workaround for silicon errata #26 on 82598. Mask the interrupt 2961 * before the read of EICR. 2962 */ 2963 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 2964 2965 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read 2966 * therefore no explicit interrupt disable is necessary */ 2967 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 2968 if (!eicr) { 2969 /* 2970 * shared interrupt alert! 2971 * make sure interrupts are enabled because the read will 2972 * have disabled interrupts due to EIAM 2973 * finish the workaround of silicon errata on 82598. Unmask 2974 * the interrupt that we masked before the EICR read. 2975 */ 2976 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2977 ixgbe_irq_enable(adapter, true, true); 2978 return IRQ_NONE; /* Not our interrupt */ 2979 } 2980 2981 if (eicr & IXGBE_EICR_LSC) 2982 ixgbe_check_lsc(adapter); 2983 2984 switch (hw->mac.type) { 2985 case ixgbe_mac_82599EB: 2986 ixgbe_check_sfp_event(adapter, eicr); 2987 /* Fall through */ 2988 case ixgbe_mac_X540: 2989 case ixgbe_mac_X550: 2990 case ixgbe_mac_X550EM_x: 2991 case ixgbe_mac_x550em_a: 2992 if (eicr & IXGBE_EICR_ECC) { 2993 e_info(link, "Received ECC Err, initiating reset\n"); 2994 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state); 2995 ixgbe_service_event_schedule(adapter); 2996 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 2997 } 2998 ixgbe_check_overtemp_event(adapter, eicr); 2999 break; 3000 default: 3001 break; 3002 } 3003 3004 ixgbe_check_fan_failure(adapter, eicr); 3005 if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) 3006 ixgbe_ptp_check_pps_event(adapter); 3007 3008 /* would disable interrupts here but EIAM disabled it */ 3009 napi_schedule_irqoff(&q_vector->napi); 3010 3011 /* 3012 * re-enable link(maybe) and non-queue interrupts, no flush. 3013 * ixgbe_poll will re-enable the queue interrupts 3014 */ 3015 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 3016 ixgbe_irq_enable(adapter, false, false); 3017 3018 return IRQ_HANDLED; 3019 } 3020 3021 /** 3022 * ixgbe_request_irq - initialize interrupts 3023 * @adapter: board private structure 3024 * 3025 * Attempts to configure interrupts using the best available 3026 * capabilities of the hardware and kernel. 3027 **/ 3028 static int ixgbe_request_irq(struct ixgbe_adapter *adapter) 3029 { 3030 struct net_device *netdev = adapter->netdev; 3031 int err; 3032 3033 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 3034 err = ixgbe_request_msix_irqs(adapter); 3035 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) 3036 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0, 3037 netdev->name, adapter); 3038 else 3039 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED, 3040 netdev->name, adapter); 3041 3042 if (err) 3043 e_err(probe, "request_irq failed, Error %d\n", err); 3044 3045 return err; 3046 } 3047 3048 static void ixgbe_free_irq(struct ixgbe_adapter *adapter) 3049 { 3050 int vector; 3051 3052 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 3053 free_irq(adapter->pdev->irq, adapter); 3054 return; 3055 } 3056 3057 if (!adapter->msix_entries) 3058 return; 3059 3060 for (vector = 0; vector < adapter->num_q_vectors; vector++) { 3061 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 3062 struct msix_entry *entry = &adapter->msix_entries[vector]; 3063 3064 /* free only the irqs that were actually requested */ 3065 if (!q_vector->rx.ring && !q_vector->tx.ring) 3066 continue; 3067 3068 /* clear the affinity_mask in the IRQ descriptor */ 3069 irq_set_affinity_hint(entry->vector, NULL); 3070 3071 free_irq(entry->vector, q_vector); 3072 } 3073 3074 free_irq(adapter->msix_entries[vector].vector, adapter); 3075 } 3076 3077 /** 3078 * ixgbe_irq_disable - Mask off interrupt generation on the NIC 3079 * @adapter: board private structure 3080 **/ 3081 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter) 3082 { 3083 switch (adapter->hw.mac.type) { 3084 case ixgbe_mac_82598EB: 3085 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 3086 break; 3087 case ixgbe_mac_82599EB: 3088 case ixgbe_mac_X540: 3089 case ixgbe_mac_X550: 3090 case ixgbe_mac_X550EM_x: 3091 case ixgbe_mac_x550em_a: 3092 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 3093 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 3094 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 3095 break; 3096 default: 3097 break; 3098 } 3099 IXGBE_WRITE_FLUSH(&adapter->hw); 3100 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 3101 int vector; 3102 3103 for (vector = 0; vector < adapter->num_q_vectors; vector++) 3104 synchronize_irq(adapter->msix_entries[vector].vector); 3105 3106 synchronize_irq(adapter->msix_entries[vector++].vector); 3107 } else { 3108 synchronize_irq(adapter->pdev->irq); 3109 } 3110 } 3111 3112 /** 3113 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts 3114 * 3115 **/ 3116 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) 3117 { 3118 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 3119 3120 ixgbe_write_eitr(q_vector); 3121 3122 ixgbe_set_ivar(adapter, 0, 0, 0); 3123 ixgbe_set_ivar(adapter, 1, 0, 0); 3124 3125 e_info(hw, "Legacy interrupt IVAR setup done\n"); 3126 } 3127 3128 /** 3129 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset 3130 * @adapter: board private structure 3131 * @ring: structure containing ring specific data 3132 * 3133 * Configure the Tx descriptor ring after a reset. 3134 **/ 3135 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, 3136 struct ixgbe_ring *ring) 3137 { 3138 struct ixgbe_hw *hw = &adapter->hw; 3139 u64 tdba = ring->dma; 3140 int wait_loop = 10; 3141 u32 txdctl = IXGBE_TXDCTL_ENABLE; 3142 u8 reg_idx = ring->reg_idx; 3143 3144 /* disable queue to avoid issues while updating state */ 3145 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); 3146 IXGBE_WRITE_FLUSH(hw); 3147 3148 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx), 3149 (tdba & DMA_BIT_MASK(32))); 3150 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32)); 3151 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx), 3152 ring->count * sizeof(union ixgbe_adv_tx_desc)); 3153 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0); 3154 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0); 3155 ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx); 3156 3157 /* 3158 * set WTHRESH to encourage burst writeback, it should not be set 3159 * higher than 1 when: 3160 * - ITR is 0 as it could cause false TX hangs 3161 * - ITR is set to > 100k int/sec and BQL is enabled 3162 * 3163 * In order to avoid issues WTHRESH + PTHRESH should always be equal 3164 * to or less than the number of on chip descriptors, which is 3165 * currently 40. 3166 */ 3167 if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR)) 3168 txdctl |= 1u << 16; /* WTHRESH = 1 */ 3169 else 3170 txdctl |= 8u << 16; /* WTHRESH = 8 */ 3171 3172 /* 3173 * Setting PTHRESH to 32 both improves performance 3174 * and avoids a TX hang with DFP enabled 3175 */ 3176 txdctl |= (1u << 8) | /* HTHRESH = 1 */ 3177 32; /* PTHRESH = 32 */ 3178 3179 /* reinitialize flowdirector state */ 3180 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 3181 ring->atr_sample_rate = adapter->atr_sample_rate; 3182 ring->atr_count = 0; 3183 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state); 3184 } else { 3185 ring->atr_sample_rate = 0; 3186 } 3187 3188 /* initialize XPS */ 3189 if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) { 3190 struct ixgbe_q_vector *q_vector = ring->q_vector; 3191 3192 if (q_vector) 3193 netif_set_xps_queue(ring->netdev, 3194 &q_vector->affinity_mask, 3195 ring->queue_index); 3196 } 3197 3198 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); 3199 3200 /* enable queue */ 3201 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); 3202 3203 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 3204 if (hw->mac.type == ixgbe_mac_82598EB && 3205 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3206 return; 3207 3208 /* poll to verify queue is enabled */ 3209 do { 3210 usleep_range(1000, 2000); 3211 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx)); 3212 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 3213 if (!wait_loop) 3214 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx); 3215 } 3216 3217 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter) 3218 { 3219 struct ixgbe_hw *hw = &adapter->hw; 3220 u32 rttdcs, mtqc; 3221 u8 tcs = netdev_get_num_tc(adapter->netdev); 3222 3223 if (hw->mac.type == ixgbe_mac_82598EB) 3224 return; 3225 3226 /* disable the arbiter while setting MTQC */ 3227 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3228 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3229 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3230 3231 /* set transmit pool layout */ 3232 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 3233 mtqc = IXGBE_MTQC_VT_ENA; 3234 if (tcs > 4) 3235 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 3236 else if (tcs > 1) 3237 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 3238 else if (adapter->ring_feature[RING_F_VMDQ].mask == 3239 IXGBE_82599_VMDQ_4Q_MASK) 3240 mtqc |= IXGBE_MTQC_32VF; 3241 else 3242 mtqc |= IXGBE_MTQC_64VF; 3243 } else { 3244 if (tcs > 4) 3245 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 3246 else if (tcs > 1) 3247 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 3248 else 3249 mtqc = IXGBE_MTQC_64Q_1PB; 3250 } 3251 3252 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 3253 3254 /* Enable Security TX Buffer IFG for multiple pb */ 3255 if (tcs) { 3256 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 3257 sectx |= IXGBE_SECTX_DCB; 3258 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx); 3259 } 3260 3261 /* re-enable the arbiter */ 3262 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3263 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3264 } 3265 3266 /** 3267 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset 3268 * @adapter: board private structure 3269 * 3270 * Configure the Tx unit of the MAC after a reset. 3271 **/ 3272 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter) 3273 { 3274 struct ixgbe_hw *hw = &adapter->hw; 3275 u32 dmatxctl; 3276 u32 i; 3277 3278 ixgbe_setup_mtqc(adapter); 3279 3280 if (hw->mac.type != ixgbe_mac_82598EB) { 3281 /* DMATXCTL.EN must be before Tx queues are enabled */ 3282 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3283 dmatxctl |= IXGBE_DMATXCTL_TE; 3284 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3285 } 3286 3287 /* Setup the HW Tx Head and Tail descriptor pointers */ 3288 for (i = 0; i < adapter->num_tx_queues; i++) 3289 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]); 3290 } 3291 3292 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter, 3293 struct ixgbe_ring *ring) 3294 { 3295 struct ixgbe_hw *hw = &adapter->hw; 3296 u8 reg_idx = ring->reg_idx; 3297 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 3298 3299 srrctl |= IXGBE_SRRCTL_DROP_EN; 3300 3301 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 3302 } 3303 3304 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter, 3305 struct ixgbe_ring *ring) 3306 { 3307 struct ixgbe_hw *hw = &adapter->hw; 3308 u8 reg_idx = ring->reg_idx; 3309 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 3310 3311 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 3312 3313 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 3314 } 3315 3316 #ifdef CONFIG_IXGBE_DCB 3317 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 3318 #else 3319 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 3320 #endif 3321 { 3322 int i; 3323 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 3324 3325 if (adapter->ixgbe_ieee_pfc) 3326 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 3327 3328 /* 3329 * We should set the drop enable bit if: 3330 * SR-IOV is enabled 3331 * or 3332 * Number of Rx queues > 1 and flow control is disabled 3333 * 3334 * This allows us to avoid head of line blocking for security 3335 * and performance reasons. 3336 */ 3337 if (adapter->num_vfs || (adapter->num_rx_queues > 1 && 3338 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { 3339 for (i = 0; i < adapter->num_rx_queues; i++) 3340 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); 3341 } else { 3342 for (i = 0; i < adapter->num_rx_queues; i++) 3343 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]); 3344 } 3345 } 3346 3347 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3348 3349 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, 3350 struct ixgbe_ring *rx_ring) 3351 { 3352 struct ixgbe_hw *hw = &adapter->hw; 3353 u32 srrctl; 3354 u8 reg_idx = rx_ring->reg_idx; 3355 3356 if (hw->mac.type == ixgbe_mac_82598EB) { 3357 u16 mask = adapter->ring_feature[RING_F_RSS].mask; 3358 3359 /* 3360 * if VMDq is not active we must program one srrctl register 3361 * per RSS queue since we have enabled RDRXCTL.MVMEN 3362 */ 3363 reg_idx &= mask; 3364 } 3365 3366 /* configure header buffer length, needed for RSC */ 3367 srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 3368 3369 /* configure the packet buffer length */ 3370 srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3371 3372 /* configure descriptor type */ 3373 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3374 3375 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 3376 } 3377 3378 /** 3379 * ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries 3380 * @adapter: device handle 3381 * 3382 * - 82598/82599/X540: 128 3383 * - X550(non-SRIOV mode): 512 3384 * - X550(SRIOV mode): 64 3385 */ 3386 u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter) 3387 { 3388 if (adapter->hw.mac.type < ixgbe_mac_X550) 3389 return 128; 3390 else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 3391 return 64; 3392 else 3393 return 512; 3394 } 3395 3396 /** 3397 * ixgbe_store_reta - Write the RETA table to HW 3398 * @adapter: device handle 3399 * 3400 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW. 3401 */ 3402 void ixgbe_store_reta(struct ixgbe_adapter *adapter) 3403 { 3404 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter); 3405 struct ixgbe_hw *hw = &adapter->hw; 3406 u32 reta = 0; 3407 u32 indices_multi; 3408 u8 *indir_tbl = adapter->rss_indir_tbl; 3409 3410 /* Fill out the redirection table as follows: 3411 * - 82598: 8 bit wide entries containing pair of 4 bit RSS 3412 * indices. 3413 * - 82599/X540: 8 bit wide entries containing 4 bit RSS index 3414 * - X550: 8 bit wide entries containing 6 bit RSS index 3415 */ 3416 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3417 indices_multi = 0x11; 3418 else 3419 indices_multi = 0x1; 3420 3421 /* Write redirection table to HW */ 3422 for (i = 0; i < reta_entries; i++) { 3423 reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8; 3424 if ((i & 3) == 3) { 3425 if (i < 128) 3426 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3427 else 3428 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3429 reta); 3430 reta = 0; 3431 } 3432 } 3433 } 3434 3435 /** 3436 * ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode) 3437 * @adapter: device handle 3438 * 3439 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW. 3440 */ 3441 static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter) 3442 { 3443 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter); 3444 struct ixgbe_hw *hw = &adapter->hw; 3445 u32 vfreta = 0; 3446 unsigned int pf_pool = adapter->num_vfs; 3447 3448 /* Write redirection table to HW */ 3449 for (i = 0; i < reta_entries; i++) { 3450 vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8; 3451 if ((i & 3) == 3) { 3452 IXGBE_WRITE_REG(hw, IXGBE_PFVFRETA(i >> 2, pf_pool), 3453 vfreta); 3454 vfreta = 0; 3455 } 3456 } 3457 } 3458 3459 static void ixgbe_setup_reta(struct ixgbe_adapter *adapter) 3460 { 3461 struct ixgbe_hw *hw = &adapter->hw; 3462 u32 i, j; 3463 u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter); 3464 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; 3465 3466 /* Program table for at least 4 queues w/ SR-IOV so that VFs can 3467 * make full use of any rings they may have. We will use the 3468 * PSRTYPE register to control how many rings we use within the PF. 3469 */ 3470 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4)) 3471 rss_i = 4; 3472 3473 /* Fill out hash function seeds */ 3474 for (i = 0; i < 10; i++) 3475 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]); 3476 3477 /* Fill out redirection table */ 3478 memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl)); 3479 3480 for (i = 0, j = 0; i < reta_entries; i++, j++) { 3481 if (j == rss_i) 3482 j = 0; 3483 3484 adapter->rss_indir_tbl[i] = j; 3485 } 3486 3487 ixgbe_store_reta(adapter); 3488 } 3489 3490 static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter) 3491 { 3492 struct ixgbe_hw *hw = &adapter->hw; 3493 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; 3494 unsigned int pf_pool = adapter->num_vfs; 3495 int i, j; 3496 3497 /* Fill out hash function seeds */ 3498 for (i = 0; i < 10; i++) 3499 IXGBE_WRITE_REG(hw, IXGBE_PFVFRSSRK(i, pf_pool), 3500 adapter->rss_key[i]); 3501 3502 /* Fill out the redirection table */ 3503 for (i = 0, j = 0; i < 64; i++, j++) { 3504 if (j == rss_i) 3505 j = 0; 3506 3507 adapter->rss_indir_tbl[i] = j; 3508 } 3509 3510 ixgbe_store_vfreta(adapter); 3511 } 3512 3513 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) 3514 { 3515 struct ixgbe_hw *hw = &adapter->hw; 3516 u32 mrqc = 0, rss_field = 0, vfmrqc = 0; 3517 u32 rxcsum; 3518 3519 /* Disable indicating checksum in descriptor, enables RSS hash */ 3520 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3521 rxcsum |= IXGBE_RXCSUM_PCSD; 3522 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3523 3524 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 3525 if (adapter->ring_feature[RING_F_RSS].mask) 3526 mrqc = IXGBE_MRQC_RSSEN; 3527 } else { 3528 u8 tcs = netdev_get_num_tc(adapter->netdev); 3529 3530 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 3531 if (tcs > 4) 3532 mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */ 3533 else if (tcs > 1) 3534 mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */ 3535 else if (adapter->ring_feature[RING_F_VMDQ].mask == 3536 IXGBE_82599_VMDQ_4Q_MASK) 3537 mrqc = IXGBE_MRQC_VMDQRSS32EN; 3538 else 3539 mrqc = IXGBE_MRQC_VMDQRSS64EN; 3540 } else { 3541 if (tcs > 4) 3542 mrqc = IXGBE_MRQC_RTRSS8TCEN; 3543 else if (tcs > 1) 3544 mrqc = IXGBE_MRQC_RTRSS4TCEN; 3545 else 3546 mrqc = IXGBE_MRQC_RSSEN; 3547 } 3548 } 3549 3550 /* Perform hash on these packet types */ 3551 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 | 3552 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 3553 IXGBE_MRQC_RSS_FIELD_IPV6 | 3554 IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 3555 3556 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) 3557 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 3558 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP) 3559 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 3560 3561 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key)); 3562 if ((hw->mac.type >= ixgbe_mac_X550) && 3563 (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { 3564 unsigned int pf_pool = adapter->num_vfs; 3565 3566 /* Enable VF RSS mode */ 3567 mrqc |= IXGBE_MRQC_MULTIPLE_RSS; 3568 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3569 3570 /* Setup RSS through the VF registers */ 3571 ixgbe_setup_vfreta(adapter); 3572 vfmrqc = IXGBE_MRQC_RSSEN; 3573 vfmrqc |= rss_field; 3574 IXGBE_WRITE_REG(hw, IXGBE_PFVFMRQC(pf_pool), vfmrqc); 3575 } else { 3576 ixgbe_setup_reta(adapter); 3577 mrqc |= rss_field; 3578 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3579 } 3580 } 3581 3582 /** 3583 * ixgbe_configure_rscctl - enable RSC for the indicated ring 3584 * @adapter: address of board private structure 3585 * @index: index of ring to set 3586 **/ 3587 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, 3588 struct ixgbe_ring *ring) 3589 { 3590 struct ixgbe_hw *hw = &adapter->hw; 3591 u32 rscctrl; 3592 u8 reg_idx = ring->reg_idx; 3593 3594 if (!ring_is_rsc_enabled(ring)) 3595 return; 3596 3597 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx)); 3598 rscctrl |= IXGBE_RSCCTL_RSCEN; 3599 /* 3600 * we must limit the number of descriptors so that the 3601 * total size of max desc * buf_len is not greater 3602 * than 65536 3603 */ 3604 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3605 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); 3606 } 3607 3608 #define IXGBE_MAX_RX_DESC_POLL 10 3609 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, 3610 struct ixgbe_ring *ring) 3611 { 3612 struct ixgbe_hw *hw = &adapter->hw; 3613 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3614 u32 rxdctl; 3615 u8 reg_idx = ring->reg_idx; 3616 3617 if (ixgbe_removed(hw->hw_addr)) 3618 return; 3619 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 3620 if (hw->mac.type == ixgbe_mac_82598EB && 3621 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3622 return; 3623 3624 do { 3625 usleep_range(1000, 2000); 3626 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3627 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 3628 3629 if (!wait_loop) { 3630 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " 3631 "the polling period\n", reg_idx); 3632 } 3633 } 3634 3635 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter, 3636 struct ixgbe_ring *ring) 3637 { 3638 struct ixgbe_hw *hw = &adapter->hw; 3639 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3640 u32 rxdctl; 3641 u8 reg_idx = ring->reg_idx; 3642 3643 if (ixgbe_removed(hw->hw_addr)) 3644 return; 3645 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3646 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 3647 3648 /* write value back with RXDCTL.ENABLE bit cleared */ 3649 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3650 3651 if (hw->mac.type == ixgbe_mac_82598EB && 3652 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3653 return; 3654 3655 /* the hardware may take up to 100us to really disable the rx queue */ 3656 do { 3657 udelay(10); 3658 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3659 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 3660 3661 if (!wait_loop) { 3662 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within " 3663 "the polling period\n", reg_idx); 3664 } 3665 } 3666 3667 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 3668 struct ixgbe_ring *ring) 3669 { 3670 struct ixgbe_hw *hw = &adapter->hw; 3671 u64 rdba = ring->dma; 3672 u32 rxdctl; 3673 u8 reg_idx = ring->reg_idx; 3674 3675 /* disable queue to avoid issues while updating state */ 3676 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3677 ixgbe_disable_rx_queue(adapter, ring); 3678 3679 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32))); 3680 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32)); 3681 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx), 3682 ring->count * sizeof(union ixgbe_adv_rx_desc)); 3683 /* Force flushing of IXGBE_RDLEN to prevent MDD */ 3684 IXGBE_WRITE_FLUSH(hw); 3685 3686 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0); 3687 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); 3688 ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx); 3689 3690 ixgbe_configure_srrctl(adapter, ring); 3691 ixgbe_configure_rscctl(adapter, ring); 3692 3693 if (hw->mac.type == ixgbe_mac_82598EB) { 3694 /* 3695 * enable cache line friendly hardware writes: 3696 * PTHRESH=32 descriptors (half the internal cache), 3697 * this also removes ugly rx_no_buffer_count increment 3698 * HTHRESH=4 descriptors (to minimize latency on fetch) 3699 * WTHRESH=8 burst writeback up to two cache lines 3700 */ 3701 rxdctl &= ~0x3FFFFF; 3702 rxdctl |= 0x080420; 3703 } 3704 3705 /* enable receive descriptor ring */ 3706 rxdctl |= IXGBE_RXDCTL_ENABLE; 3707 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3708 3709 ixgbe_rx_desc_queue_enable(adapter, ring); 3710 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); 3711 } 3712 3713 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) 3714 { 3715 struct ixgbe_hw *hw = &adapter->hw; 3716 int rss_i = adapter->ring_feature[RING_F_RSS].indices; 3717 u16 pool; 3718 3719 /* PSRTYPE must be initialized in non 82598 adapters */ 3720 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 3721 IXGBE_PSRTYPE_UDPHDR | 3722 IXGBE_PSRTYPE_IPV4HDR | 3723 IXGBE_PSRTYPE_L2HDR | 3724 IXGBE_PSRTYPE_IPV6HDR; 3725 3726 if (hw->mac.type == ixgbe_mac_82598EB) 3727 return; 3728 3729 if (rss_i > 3) 3730 psrtype |= 2u << 29; 3731 else if (rss_i > 1) 3732 psrtype |= 1u << 29; 3733 3734 for_each_set_bit(pool, &adapter->fwd_bitmask, 32) 3735 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype); 3736 } 3737 3738 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) 3739 { 3740 struct ixgbe_hw *hw = &adapter->hw; 3741 u32 reg_offset, vf_shift; 3742 u32 gcr_ext, vmdctl; 3743 int i; 3744 3745 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 3746 return; 3747 3748 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 3749 vmdctl |= IXGBE_VMD_CTL_VMDQ_EN; 3750 vmdctl &= ~IXGBE_VT_CTL_POOL_MASK; 3751 vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT; 3752 vmdctl |= IXGBE_VT_CTL_REPLEN; 3753 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl); 3754 3755 vf_shift = VMDQ_P(0) % 32; 3756 reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0; 3757 3758 /* Enable only the PF's pool for Tx/Rx */ 3759 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), GENMASK(31, vf_shift)); 3760 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1); 3761 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), GENMASK(31, vf_shift)); 3762 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1); 3763 if (adapter->bridge_mode == BRIDGE_MODE_VEB) 3764 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 3765 3766 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */ 3767 hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0)); 3768 3769 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3770 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC; 3771 3772 /* 3773 * Set up VF register offsets for selected VT Mode, 3774 * i.e. 32 or 64 VFs for SR-IOV 3775 */ 3776 switch (adapter->ring_feature[RING_F_VMDQ].mask) { 3777 case IXGBE_82599_VMDQ_8Q_MASK: 3778 gcr_ext = IXGBE_GCR_EXT_VT_MODE_16; 3779 break; 3780 case IXGBE_82599_VMDQ_4Q_MASK: 3781 gcr_ext = IXGBE_GCR_EXT_VT_MODE_32; 3782 break; 3783 default: 3784 gcr_ext = IXGBE_GCR_EXT_VT_MODE_64; 3785 break; 3786 } 3787 3788 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext); 3789 3790 for (i = 0; i < adapter->num_vfs; i++) { 3791 /* configure spoof checking */ 3792 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, 3793 adapter->vfinfo[i].spoofchk_enabled); 3794 3795 /* Enable/Disable RSS query feature */ 3796 ixgbe_ndo_set_vf_rss_query_en(adapter->netdev, i, 3797 adapter->vfinfo[i].rss_query_enabled); 3798 } 3799 } 3800 3801 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) 3802 { 3803 struct ixgbe_hw *hw = &adapter->hw; 3804 struct net_device *netdev = adapter->netdev; 3805 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3806 struct ixgbe_ring *rx_ring; 3807 int i; 3808 u32 mhadd, hlreg0; 3809 3810 #ifdef IXGBE_FCOE 3811 /* adjust max frame to be able to do baby jumbo for FCoE */ 3812 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && 3813 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) 3814 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3815 3816 #endif /* IXGBE_FCOE */ 3817 3818 /* adjust max frame to be at least the size of a standard frame */ 3819 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3820 max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); 3821 3822 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 3823 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { 3824 mhadd &= ~IXGBE_MHADD_MFS_MASK; 3825 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; 3826 3827 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 3828 } 3829 3830 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3831 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ 3832 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 3833 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 3834 3835 /* 3836 * Setup the HW Rx Head and Tail Descriptor Pointers and 3837 * the Base and Length of the Rx Descriptor Ring 3838 */ 3839 for (i = 0; i < adapter->num_rx_queues; i++) { 3840 rx_ring = adapter->rx_ring[i]; 3841 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 3842 set_ring_rsc_enabled(rx_ring); 3843 else 3844 clear_ring_rsc_enabled(rx_ring); 3845 } 3846 } 3847 3848 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) 3849 { 3850 struct ixgbe_hw *hw = &adapter->hw; 3851 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3852 3853 switch (hw->mac.type) { 3854 case ixgbe_mac_82598EB: 3855 /* 3856 * For VMDq support of different descriptor types or 3857 * buffer sizes through the use of multiple SRRCTL 3858 * registers, RDRXCTL.MVMEN must be set to 1 3859 * 3860 * also, the manual doesn't mention it clearly but DCA hints 3861 * will only use queue 0's tags unless this bit is set. Side 3862 * effects of setting this bit are only that SRRCTL must be 3863 * fully programmed [0..15] 3864 */ 3865 rdrxctl |= IXGBE_RDRXCTL_MVMEN; 3866 break; 3867 case ixgbe_mac_X550: 3868 case ixgbe_mac_X550EM_x: 3869 case ixgbe_mac_x550em_a: 3870 if (adapter->num_vfs) 3871 rdrxctl |= IXGBE_RDRXCTL_PSP; 3872 /* fall through for older HW */ 3873 case ixgbe_mac_82599EB: 3874 case ixgbe_mac_X540: 3875 /* Disable RSC for ACK packets */ 3876 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3877 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3878 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3879 /* hardware requires some bits to be set by default */ 3880 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX); 3881 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3882 break; 3883 default: 3884 /* We should do nothing since we don't know this hardware */ 3885 return; 3886 } 3887 3888 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3889 } 3890 3891 /** 3892 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset 3893 * @adapter: board private structure 3894 * 3895 * Configure the Rx unit of the MAC after a reset. 3896 **/ 3897 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) 3898 { 3899 struct ixgbe_hw *hw = &adapter->hw; 3900 int i; 3901 u32 rxctrl, rfctl; 3902 3903 /* disable receives while setting up the descriptors */ 3904 hw->mac.ops.disable_rx(hw); 3905 3906 ixgbe_setup_psrtype(adapter); 3907 ixgbe_setup_rdrxctl(adapter); 3908 3909 /* RSC Setup */ 3910 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL); 3911 rfctl &= ~IXGBE_RFCTL_RSC_DIS; 3912 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) 3913 rfctl |= IXGBE_RFCTL_RSC_DIS; 3914 3915 /* disable NFS filtering */ 3916 rfctl |= (IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS); 3917 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl); 3918 3919 /* Program registers for the distribution of queues */ 3920 ixgbe_setup_mrqc(adapter); 3921 3922 /* set_rx_buffer_len must be called before ring initialization */ 3923 ixgbe_set_rx_buffer_len(adapter); 3924 3925 /* 3926 * Setup the HW Rx Head and Tail Descriptor Pointers and 3927 * the Base and Length of the Rx Descriptor Ring 3928 */ 3929 for (i = 0; i < adapter->num_rx_queues; i++) 3930 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); 3931 3932 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3933 /* disable drop enable for 82598 parts */ 3934 if (hw->mac.type == ixgbe_mac_82598EB) 3935 rxctrl |= IXGBE_RXCTRL_DMBYPS; 3936 3937 /* enable all receives */ 3938 rxctrl |= IXGBE_RXCTRL_RXEN; 3939 hw->mac.ops.enable_rx_dma(hw, rxctrl); 3940 } 3941 3942 static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, 3943 __be16 proto, u16 vid) 3944 { 3945 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3946 struct ixgbe_hw *hw = &adapter->hw; 3947 3948 /* add VID to filter table */ 3949 if (!vid || !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)) 3950 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true, !!vid); 3951 3952 set_bit(vid, adapter->active_vlans); 3953 3954 return 0; 3955 } 3956 3957 static int ixgbe_find_vlvf_entry(struct ixgbe_hw *hw, u32 vlan) 3958 { 3959 u32 vlvf; 3960 int idx; 3961 3962 /* short cut the special case */ 3963 if (vlan == 0) 3964 return 0; 3965 3966 /* Search for the vlan id in the VLVF entries */ 3967 for (idx = IXGBE_VLVF_ENTRIES; --idx;) { 3968 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(idx)); 3969 if ((vlvf & VLAN_VID_MASK) == vlan) 3970 break; 3971 } 3972 3973 return idx; 3974 } 3975 3976 void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid) 3977 { 3978 struct ixgbe_hw *hw = &adapter->hw; 3979 u32 bits, word; 3980 int idx; 3981 3982 idx = ixgbe_find_vlvf_entry(hw, vid); 3983 if (!idx) 3984 return; 3985 3986 /* See if any other pools are set for this VLAN filter 3987 * entry other than the PF. 3988 */ 3989 word = idx * 2 + (VMDQ_P(0) / 32); 3990 bits = ~BIT(VMDQ_P(0) % 32); 3991 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word)); 3992 3993 /* Disable the filter so this falls into the default pool. */ 3994 if (!bits && !IXGBE_READ_REG(hw, IXGBE_VLVFB(word ^ 1))) { 3995 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)) 3996 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), 0); 3997 IXGBE_WRITE_REG(hw, IXGBE_VLVF(idx), 0); 3998 } 3999 } 4000 4001 static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, 4002 __be16 proto, u16 vid) 4003 { 4004 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4005 struct ixgbe_hw *hw = &adapter->hw; 4006 4007 /* remove VID from filter table */ 4008 if (vid && !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)) 4009 hw->mac.ops.set_vfta(hw, vid, VMDQ_P(0), false, true); 4010 4011 clear_bit(vid, adapter->active_vlans); 4012 4013 return 0; 4014 } 4015 4016 /** 4017 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping 4018 * @adapter: driver data 4019 */ 4020 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter) 4021 { 4022 struct ixgbe_hw *hw = &adapter->hw; 4023 u32 vlnctrl; 4024 int i, j; 4025 4026 switch (hw->mac.type) { 4027 case ixgbe_mac_82598EB: 4028 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4029 vlnctrl &= ~IXGBE_VLNCTRL_VME; 4030 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 4031 break; 4032 case ixgbe_mac_82599EB: 4033 case ixgbe_mac_X540: 4034 case ixgbe_mac_X550: 4035 case ixgbe_mac_X550EM_x: 4036 case ixgbe_mac_x550em_a: 4037 for (i = 0; i < adapter->num_rx_queues; i++) { 4038 struct ixgbe_ring *ring = adapter->rx_ring[i]; 4039 4040 if (ring->l2_accel_priv) 4041 continue; 4042 j = ring->reg_idx; 4043 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 4044 vlnctrl &= ~IXGBE_RXDCTL_VME; 4045 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 4046 } 4047 break; 4048 default: 4049 break; 4050 } 4051 } 4052 4053 /** 4054 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping 4055 * @adapter: driver data 4056 */ 4057 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter) 4058 { 4059 struct ixgbe_hw *hw = &adapter->hw; 4060 u32 vlnctrl; 4061 int i, j; 4062 4063 switch (hw->mac.type) { 4064 case ixgbe_mac_82598EB: 4065 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4066 vlnctrl |= IXGBE_VLNCTRL_VME; 4067 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 4068 break; 4069 case ixgbe_mac_82599EB: 4070 case ixgbe_mac_X540: 4071 case ixgbe_mac_X550: 4072 case ixgbe_mac_X550EM_x: 4073 case ixgbe_mac_x550em_a: 4074 for (i = 0; i < adapter->num_rx_queues; i++) { 4075 struct ixgbe_ring *ring = adapter->rx_ring[i]; 4076 4077 if (ring->l2_accel_priv) 4078 continue; 4079 j = ring->reg_idx; 4080 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 4081 vlnctrl |= IXGBE_RXDCTL_VME; 4082 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 4083 } 4084 break; 4085 default: 4086 break; 4087 } 4088 } 4089 4090 static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter) 4091 { 4092 struct ixgbe_hw *hw = &adapter->hw; 4093 u32 vlnctrl, i; 4094 4095 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4096 4097 if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) { 4098 /* For VMDq and SR-IOV we must leave VLAN filtering enabled */ 4099 vlnctrl |= IXGBE_VLNCTRL_VFE; 4100 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 4101 } else { 4102 vlnctrl &= ~IXGBE_VLNCTRL_VFE; 4103 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 4104 return; 4105 } 4106 4107 /* Nothing to do for 82598 */ 4108 if (hw->mac.type == ixgbe_mac_82598EB) 4109 return; 4110 4111 /* We are already in VLAN promisc, nothing to do */ 4112 if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC) 4113 return; 4114 4115 /* Set flag so we don't redo unnecessary work */ 4116 adapter->flags2 |= IXGBE_FLAG2_VLAN_PROMISC; 4117 4118 /* Add PF to all active pools */ 4119 for (i = IXGBE_VLVF_ENTRIES; --i;) { 4120 u32 reg_offset = IXGBE_VLVFB(i * 2 + VMDQ_P(0) / 32); 4121 u32 vlvfb = IXGBE_READ_REG(hw, reg_offset); 4122 4123 vlvfb |= BIT(VMDQ_P(0) % 32); 4124 IXGBE_WRITE_REG(hw, reg_offset, vlvfb); 4125 } 4126 4127 /* Set all bits in the VLAN filter table array */ 4128 for (i = hw->mac.vft_size; i--;) 4129 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ~0U); 4130 } 4131 4132 #define VFTA_BLOCK_SIZE 8 4133 static void ixgbe_scrub_vfta(struct ixgbe_adapter *adapter, u32 vfta_offset) 4134 { 4135 struct ixgbe_hw *hw = &adapter->hw; 4136 u32 vfta[VFTA_BLOCK_SIZE] = { 0 }; 4137 u32 vid_start = vfta_offset * 32; 4138 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32); 4139 u32 i, vid, word, bits; 4140 4141 for (i = IXGBE_VLVF_ENTRIES; --i;) { 4142 u32 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(i)); 4143 4144 /* pull VLAN ID from VLVF */ 4145 vid = vlvf & VLAN_VID_MASK; 4146 4147 /* only concern outselves with a certain range */ 4148 if (vid < vid_start || vid >= vid_end) 4149 continue; 4150 4151 if (vlvf) { 4152 /* record VLAN ID in VFTA */ 4153 vfta[(vid - vid_start) / 32] |= BIT(vid % 32); 4154 4155 /* if PF is part of this then continue */ 4156 if (test_bit(vid, adapter->active_vlans)) 4157 continue; 4158 } 4159 4160 /* remove PF from the pool */ 4161 word = i * 2 + VMDQ_P(0) / 32; 4162 bits = ~BIT(VMDQ_P(0) % 32); 4163 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word)); 4164 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), bits); 4165 } 4166 4167 /* extract values from active_vlans and write back to VFTA */ 4168 for (i = VFTA_BLOCK_SIZE; i--;) { 4169 vid = (vfta_offset + i) * 32; 4170 word = vid / BITS_PER_LONG; 4171 bits = vid % BITS_PER_LONG; 4172 4173 vfta[i] |= adapter->active_vlans[word] >> bits; 4174 4175 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vfta_offset + i), vfta[i]); 4176 } 4177 } 4178 4179 static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter) 4180 { 4181 struct ixgbe_hw *hw = &adapter->hw; 4182 u32 vlnctrl, i; 4183 4184 /* Set VLAN filtering to enabled */ 4185 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4186 vlnctrl |= IXGBE_VLNCTRL_VFE; 4187 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 4188 4189 if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) || 4190 hw->mac.type == ixgbe_mac_82598EB) 4191 return; 4192 4193 /* We are not in VLAN promisc, nothing to do */ 4194 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)) 4195 return; 4196 4197 /* Set flag so we don't redo unnecessary work */ 4198 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC; 4199 4200 for (i = 0; i < hw->mac.vft_size; i += VFTA_BLOCK_SIZE) 4201 ixgbe_scrub_vfta(adapter, i); 4202 } 4203 4204 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter) 4205 { 4206 u16 vid = 1; 4207 4208 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 4209 4210 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID) 4211 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 4212 } 4213 4214 /** 4215 * ixgbe_write_mc_addr_list - write multicast addresses to MTA 4216 * @netdev: network interface device structure 4217 * 4218 * Writes multicast address list to the MTA hash table. 4219 * Returns: -ENOMEM on failure 4220 * 0 on no addresses written 4221 * X on writing X addresses to MTA 4222 **/ 4223 static int ixgbe_write_mc_addr_list(struct net_device *netdev) 4224 { 4225 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4226 struct ixgbe_hw *hw = &adapter->hw; 4227 4228 if (!netif_running(netdev)) 4229 return 0; 4230 4231 if (hw->mac.ops.update_mc_addr_list) 4232 hw->mac.ops.update_mc_addr_list(hw, netdev); 4233 else 4234 return -ENOMEM; 4235 4236 #ifdef CONFIG_PCI_IOV 4237 ixgbe_restore_vf_multicasts(adapter); 4238 #endif 4239 4240 return netdev_mc_count(netdev); 4241 } 4242 4243 #ifdef CONFIG_PCI_IOV 4244 void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter) 4245 { 4246 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4247 struct ixgbe_hw *hw = &adapter->hw; 4248 int i; 4249 4250 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) { 4251 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED; 4252 4253 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) 4254 hw->mac.ops.set_rar(hw, i, 4255 mac_table->addr, 4256 mac_table->pool, 4257 IXGBE_RAH_AV); 4258 else 4259 hw->mac.ops.clear_rar(hw, i); 4260 } 4261 } 4262 4263 #endif 4264 static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter) 4265 { 4266 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4267 struct ixgbe_hw *hw = &adapter->hw; 4268 int i; 4269 4270 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) { 4271 if (!(mac_table->state & IXGBE_MAC_STATE_MODIFIED)) 4272 continue; 4273 4274 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED; 4275 4276 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) 4277 hw->mac.ops.set_rar(hw, i, 4278 mac_table->addr, 4279 mac_table->pool, 4280 IXGBE_RAH_AV); 4281 else 4282 hw->mac.ops.clear_rar(hw, i); 4283 } 4284 } 4285 4286 static void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter) 4287 { 4288 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4289 struct ixgbe_hw *hw = &adapter->hw; 4290 int i; 4291 4292 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) { 4293 mac_table->state |= IXGBE_MAC_STATE_MODIFIED; 4294 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE; 4295 } 4296 4297 ixgbe_sync_mac_table(adapter); 4298 } 4299 4300 static int ixgbe_available_rars(struct ixgbe_adapter *adapter, u16 pool) 4301 { 4302 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4303 struct ixgbe_hw *hw = &adapter->hw; 4304 int i, count = 0; 4305 4306 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) { 4307 /* do not count default RAR as available */ 4308 if (mac_table->state & IXGBE_MAC_STATE_DEFAULT) 4309 continue; 4310 4311 /* only count unused and addresses that belong to us */ 4312 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) { 4313 if (mac_table->pool != pool) 4314 continue; 4315 } 4316 4317 count++; 4318 } 4319 4320 return count; 4321 } 4322 4323 /* this function destroys the first RAR entry */ 4324 static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter) 4325 { 4326 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4327 struct ixgbe_hw *hw = &adapter->hw; 4328 4329 memcpy(&mac_table->addr, hw->mac.addr, ETH_ALEN); 4330 mac_table->pool = VMDQ_P(0); 4331 4332 mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE; 4333 4334 hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool, 4335 IXGBE_RAH_AV); 4336 } 4337 4338 int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, 4339 const u8 *addr, u16 pool) 4340 { 4341 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4342 struct ixgbe_hw *hw = &adapter->hw; 4343 int i; 4344 4345 if (is_zero_ether_addr(addr)) 4346 return -EINVAL; 4347 4348 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) { 4349 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) 4350 continue; 4351 4352 ether_addr_copy(mac_table->addr, addr); 4353 mac_table->pool = pool; 4354 4355 mac_table->state |= IXGBE_MAC_STATE_MODIFIED | 4356 IXGBE_MAC_STATE_IN_USE; 4357 4358 ixgbe_sync_mac_table(adapter); 4359 4360 return i; 4361 } 4362 4363 return -ENOMEM; 4364 } 4365 4366 int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter, 4367 const u8 *addr, u16 pool) 4368 { 4369 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0]; 4370 struct ixgbe_hw *hw = &adapter->hw; 4371 int i; 4372 4373 if (is_zero_ether_addr(addr)) 4374 return -EINVAL; 4375 4376 /* search table for addr, if found clear IN_USE flag and sync */ 4377 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) { 4378 /* we can only delete an entry if it is in use */ 4379 if (!(mac_table->state & IXGBE_MAC_STATE_IN_USE)) 4380 continue; 4381 /* we only care about entries that belong to the given pool */ 4382 if (mac_table->pool != pool) 4383 continue; 4384 /* we only care about a specific MAC address */ 4385 if (!ether_addr_equal(addr, mac_table->addr)) 4386 continue; 4387 4388 mac_table->state |= IXGBE_MAC_STATE_MODIFIED; 4389 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE; 4390 4391 ixgbe_sync_mac_table(adapter); 4392 4393 return 0; 4394 } 4395 4396 return -ENOMEM; 4397 } 4398 /** 4399 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table 4400 * @netdev: network interface device structure 4401 * 4402 * Writes unicast address list to the RAR table. 4403 * Returns: -ENOMEM on failure/insufficient address space 4404 * 0 on no addresses written 4405 * X on writing X addresses to the RAR table 4406 **/ 4407 static int ixgbe_write_uc_addr_list(struct net_device *netdev, int vfn) 4408 { 4409 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4410 int count = 0; 4411 4412 /* return ENOMEM indicating insufficient memory for addresses */ 4413 if (netdev_uc_count(netdev) > ixgbe_available_rars(adapter, vfn)) 4414 return -ENOMEM; 4415 4416 if (!netdev_uc_empty(netdev)) { 4417 struct netdev_hw_addr *ha; 4418 netdev_for_each_uc_addr(ha, netdev) { 4419 ixgbe_del_mac_filter(adapter, ha->addr, vfn); 4420 ixgbe_add_mac_filter(adapter, ha->addr, vfn); 4421 count++; 4422 } 4423 } 4424 return count; 4425 } 4426 4427 static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr) 4428 { 4429 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4430 int ret; 4431 4432 ret = ixgbe_add_mac_filter(adapter, addr, VMDQ_P(0)); 4433 4434 return min_t(int, ret, 0); 4435 } 4436 4437 static int ixgbe_uc_unsync(struct net_device *netdev, const unsigned char *addr) 4438 { 4439 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4440 4441 ixgbe_del_mac_filter(adapter, addr, VMDQ_P(0)); 4442 4443 return 0; 4444 } 4445 4446 /** 4447 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set 4448 * @netdev: network interface device structure 4449 * 4450 * The set_rx_method entry point is called whenever the unicast/multicast 4451 * address list or the network interface flags are updated. This routine is 4452 * responsible for configuring the hardware for proper unicast, multicast and 4453 * promiscuous mode. 4454 **/ 4455 void ixgbe_set_rx_mode(struct net_device *netdev) 4456 { 4457 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4458 struct ixgbe_hw *hw = &adapter->hw; 4459 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE; 4460 netdev_features_t features = netdev->features; 4461 int count; 4462 4463 /* Check for Promiscuous and All Multicast modes */ 4464 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4465 4466 /* set all bits that we expect to always be set */ 4467 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */ 4468 fctrl |= IXGBE_FCTRL_BAM; 4469 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */ 4470 fctrl |= IXGBE_FCTRL_PMCF; 4471 4472 /* clear the bits we are changing the status of */ 4473 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4474 if (netdev->flags & IFF_PROMISC) { 4475 hw->addr_ctrl.user_set_promisc = true; 4476 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4477 vmolr |= IXGBE_VMOLR_MPE; 4478 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4479 } else { 4480 if (netdev->flags & IFF_ALLMULTI) { 4481 fctrl |= IXGBE_FCTRL_MPE; 4482 vmolr |= IXGBE_VMOLR_MPE; 4483 } 4484 hw->addr_ctrl.user_set_promisc = false; 4485 } 4486 4487 /* 4488 * Write addresses to available RAR registers, if there is not 4489 * sufficient space to store all the addresses then enable 4490 * unicast promiscuous mode 4491 */ 4492 if (__dev_uc_sync(netdev, ixgbe_uc_sync, ixgbe_uc_unsync)) { 4493 fctrl |= IXGBE_FCTRL_UPE; 4494 vmolr |= IXGBE_VMOLR_ROPE; 4495 } 4496 4497 /* Write addresses to the MTA, if the attempt fails 4498 * then we should just turn on promiscuous mode so 4499 * that we can at least receive multicast traffic 4500 */ 4501 count = ixgbe_write_mc_addr_list(netdev); 4502 if (count < 0) { 4503 fctrl |= IXGBE_FCTRL_MPE; 4504 vmolr |= IXGBE_VMOLR_MPE; 4505 } else if (count) { 4506 vmolr |= IXGBE_VMOLR_ROMPE; 4507 } 4508 4509 if (hw->mac.type != ixgbe_mac_82598EB) { 4510 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) & 4511 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE | 4512 IXGBE_VMOLR_ROPE); 4513 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr); 4514 } 4515 4516 /* This is useful for sniffing bad packets. */ 4517 if (features & NETIF_F_RXALL) { 4518 /* UPE and MPE will be handled by normal PROMISC logic 4519 * in e1000e_set_rx_mode */ 4520 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */ 4521 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */ 4522 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */ 4523 4524 fctrl &= ~(IXGBE_FCTRL_DPF); 4525 /* NOTE: VLAN filtering is disabled by setting PROMISC */ 4526 } 4527 4528 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4529 4530 if (features & NETIF_F_HW_VLAN_CTAG_RX) 4531 ixgbe_vlan_strip_enable(adapter); 4532 else 4533 ixgbe_vlan_strip_disable(adapter); 4534 4535 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) 4536 ixgbe_vlan_promisc_disable(adapter); 4537 else 4538 ixgbe_vlan_promisc_enable(adapter); 4539 } 4540 4541 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter) 4542 { 4543 int q_idx; 4544 4545 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) 4546 napi_enable(&adapter->q_vector[q_idx]->napi); 4547 } 4548 4549 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) 4550 { 4551 int q_idx; 4552 4553 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) 4554 napi_disable(&adapter->q_vector[q_idx]->napi); 4555 } 4556 4557 static void ixgbe_clear_udp_tunnel_port(struct ixgbe_adapter *adapter, u32 mask) 4558 { 4559 struct ixgbe_hw *hw = &adapter->hw; 4560 u32 vxlanctrl; 4561 4562 if (!(adapter->flags & (IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE | 4563 IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE))) 4564 return; 4565 4566 vxlanctrl = IXGBE_READ_REG(hw, IXGBE_VXLANCTRL) && ~mask; 4567 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, vxlanctrl); 4568 4569 if (mask & IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK) 4570 adapter->vxlan_port = 0; 4571 4572 if (mask & IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK) 4573 adapter->geneve_port = 0; 4574 } 4575 4576 #ifdef CONFIG_IXGBE_DCB 4577 /** 4578 * ixgbe_configure_dcb - Configure DCB hardware 4579 * @adapter: ixgbe adapter struct 4580 * 4581 * This is called by the driver on open to configure the DCB hardware. 4582 * This is also called by the gennetlink interface when reconfiguring 4583 * the DCB state. 4584 */ 4585 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) 4586 { 4587 struct ixgbe_hw *hw = &adapter->hw; 4588 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 4589 4590 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) { 4591 if (hw->mac.type == ixgbe_mac_82598EB) 4592 netif_set_gso_max_size(adapter->netdev, 65536); 4593 return; 4594 } 4595 4596 if (hw->mac.type == ixgbe_mac_82598EB) 4597 netif_set_gso_max_size(adapter->netdev, 32768); 4598 4599 #ifdef IXGBE_FCOE 4600 if (adapter->netdev->features & NETIF_F_FCOE_MTU) 4601 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); 4602 #endif 4603 4604 /* reconfigure the hardware */ 4605 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) { 4606 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 4607 DCB_TX_CONFIG); 4608 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 4609 DCB_RX_CONFIG); 4610 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg); 4611 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) { 4612 ixgbe_dcb_hw_ets(&adapter->hw, 4613 adapter->ixgbe_ieee_ets, 4614 max_frame); 4615 ixgbe_dcb_hw_pfc_config(&adapter->hw, 4616 adapter->ixgbe_ieee_pfc->pfc_en, 4617 adapter->ixgbe_ieee_ets->prio_tc); 4618 } 4619 4620 /* Enable RSS Hash per TC */ 4621 if (hw->mac.type != ixgbe_mac_82598EB) { 4622 u32 msb = 0; 4623 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1; 4624 4625 while (rss_i) { 4626 msb++; 4627 rss_i >>= 1; 4628 } 4629 4630 /* write msb to all 8 TCs in one write */ 4631 IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111); 4632 } 4633 } 4634 #endif 4635 4636 /* Additional bittime to account for IXGBE framing */ 4637 #define IXGBE_ETH_FRAMING 20 4638 4639 /** 4640 * ixgbe_hpbthresh - calculate high water mark for flow control 4641 * 4642 * @adapter: board private structure to calculate for 4643 * @pb: packet buffer to calculate 4644 */ 4645 static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) 4646 { 4647 struct ixgbe_hw *hw = &adapter->hw; 4648 struct net_device *dev = adapter->netdev; 4649 int link, tc, kb, marker; 4650 u32 dv_id, rx_pba; 4651 4652 /* Calculate max LAN frame size */ 4653 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING; 4654 4655 #ifdef IXGBE_FCOE 4656 /* FCoE traffic class uses FCOE jumbo frames */ 4657 if ((dev->features & NETIF_F_FCOE_MTU) && 4658 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && 4659 (pb == ixgbe_fcoe_get_tc(adapter))) 4660 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; 4661 #endif 4662 4663 /* Calculate delay value for device */ 4664 switch (hw->mac.type) { 4665 case ixgbe_mac_X540: 4666 case ixgbe_mac_X550: 4667 case ixgbe_mac_X550EM_x: 4668 case ixgbe_mac_x550em_a: 4669 dv_id = IXGBE_DV_X540(link, tc); 4670 break; 4671 default: 4672 dv_id = IXGBE_DV(link, tc); 4673 break; 4674 } 4675 4676 /* Loopback switch introduces additional latency */ 4677 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 4678 dv_id += IXGBE_B2BT(tc); 4679 4680 /* Delay value is calculated in bit times convert to KB */ 4681 kb = IXGBE_BT2KB(dv_id); 4682 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10; 4683 4684 marker = rx_pba - kb; 4685 4686 /* It is possible that the packet buffer is not large enough 4687 * to provide required headroom. In this case throw an error 4688 * to user and a do the best we can. 4689 */ 4690 if (marker < 0) { 4691 e_warn(drv, "Packet Buffer(%i) can not provide enough" 4692 "headroom to support flow control." 4693 "Decrease MTU or number of traffic classes\n", pb); 4694 marker = tc + 1; 4695 } 4696 4697 return marker; 4698 } 4699 4700 /** 4701 * ixgbe_lpbthresh - calculate low water mark for for flow control 4702 * 4703 * @adapter: board private structure to calculate for 4704 * @pb: packet buffer to calculate 4705 */ 4706 static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) 4707 { 4708 struct ixgbe_hw *hw = &adapter->hw; 4709 struct net_device *dev = adapter->netdev; 4710 int tc; 4711 u32 dv_id; 4712 4713 /* Calculate max LAN frame size */ 4714 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN; 4715 4716 #ifdef IXGBE_FCOE 4717 /* FCoE traffic class uses FCOE jumbo frames */ 4718 if ((dev->features & NETIF_F_FCOE_MTU) && 4719 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && 4720 (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) 4721 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; 4722 #endif 4723 4724 /* Calculate delay value for device */ 4725 switch (hw->mac.type) { 4726 case ixgbe_mac_X540: 4727 case ixgbe_mac_X550: 4728 case ixgbe_mac_X550EM_x: 4729 case ixgbe_mac_x550em_a: 4730 dv_id = IXGBE_LOW_DV_X540(tc); 4731 break; 4732 default: 4733 dv_id = IXGBE_LOW_DV(tc); 4734 break; 4735 } 4736 4737 /* Delay value is calculated in bit times convert to KB */ 4738 return IXGBE_BT2KB(dv_id); 4739 } 4740 4741 /* 4742 * ixgbe_pbthresh_setup - calculate and setup high low water marks 4743 */ 4744 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter) 4745 { 4746 struct ixgbe_hw *hw = &adapter->hw; 4747 int num_tc = netdev_get_num_tc(adapter->netdev); 4748 int i; 4749 4750 if (!num_tc) 4751 num_tc = 1; 4752 4753 for (i = 0; i < num_tc; i++) { 4754 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i); 4755 hw->fc.low_water[i] = ixgbe_lpbthresh(adapter, i); 4756 4757 /* Low water marks must not be larger than high water marks */ 4758 if (hw->fc.low_water[i] > hw->fc.high_water[i]) 4759 hw->fc.low_water[i] = 0; 4760 } 4761 4762 for (; i < MAX_TRAFFIC_CLASS; i++) 4763 hw->fc.high_water[i] = 0; 4764 } 4765 4766 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) 4767 { 4768 struct ixgbe_hw *hw = &adapter->hw; 4769 int hdrm; 4770 u8 tc = netdev_get_num_tc(adapter->netdev); 4771 4772 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || 4773 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 4774 hdrm = 32 << adapter->fdir_pballoc; 4775 else 4776 hdrm = 0; 4777 4778 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); 4779 ixgbe_pbthresh_setup(adapter); 4780 } 4781 4782 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) 4783 { 4784 struct ixgbe_hw *hw = &adapter->hw; 4785 struct hlist_node *node2; 4786 struct ixgbe_fdir_filter *filter; 4787 4788 spin_lock(&adapter->fdir_perfect_lock); 4789 4790 if (!hlist_empty(&adapter->fdir_filter_list)) 4791 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask); 4792 4793 hlist_for_each_entry_safe(filter, node2, 4794 &adapter->fdir_filter_list, fdir_node) { 4795 ixgbe_fdir_write_perfect_filter_82599(hw, 4796 &filter->filter, 4797 filter->sw_idx, 4798 (filter->action == IXGBE_FDIR_DROP_QUEUE) ? 4799 IXGBE_FDIR_DROP_QUEUE : 4800 adapter->rx_ring[filter->action]->reg_idx); 4801 } 4802 4803 spin_unlock(&adapter->fdir_perfect_lock); 4804 } 4805 4806 static void ixgbe_macvlan_set_rx_mode(struct net_device *dev, unsigned int pool, 4807 struct ixgbe_adapter *adapter) 4808 { 4809 struct ixgbe_hw *hw = &adapter->hw; 4810 u32 vmolr; 4811 4812 /* No unicast promiscuous support for VMDQ devices. */ 4813 vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool)); 4814 vmolr |= (IXGBE_VMOLR_ROMPE | IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE); 4815 4816 /* clear the affected bit */ 4817 vmolr &= ~IXGBE_VMOLR_MPE; 4818 4819 if (dev->flags & IFF_ALLMULTI) { 4820 vmolr |= IXGBE_VMOLR_MPE; 4821 } else { 4822 vmolr |= IXGBE_VMOLR_ROMPE; 4823 hw->mac.ops.update_mc_addr_list(hw, dev); 4824 } 4825 ixgbe_write_uc_addr_list(adapter->netdev, pool); 4826 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr); 4827 } 4828 4829 static void ixgbe_fwd_psrtype(struct ixgbe_fwd_adapter *vadapter) 4830 { 4831 struct ixgbe_adapter *adapter = vadapter->real_adapter; 4832 int rss_i = adapter->num_rx_queues_per_pool; 4833 struct ixgbe_hw *hw = &adapter->hw; 4834 u16 pool = vadapter->pool; 4835 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4836 IXGBE_PSRTYPE_UDPHDR | 4837 IXGBE_PSRTYPE_IPV4HDR | 4838 IXGBE_PSRTYPE_L2HDR | 4839 IXGBE_PSRTYPE_IPV6HDR; 4840 4841 if (hw->mac.type == ixgbe_mac_82598EB) 4842 return; 4843 4844 if (rss_i > 3) 4845 psrtype |= 2u << 29; 4846 else if (rss_i > 1) 4847 psrtype |= 1u << 29; 4848 4849 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype); 4850 } 4851 4852 /** 4853 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue 4854 * @rx_ring: ring to free buffers from 4855 **/ 4856 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) 4857 { 4858 struct device *dev = rx_ring->dev; 4859 unsigned long size; 4860 u16 i; 4861 4862 /* ring already cleared, nothing to do */ 4863 if (!rx_ring->rx_buffer_info) 4864 return; 4865 4866 /* Free all the Rx ring sk_buffs */ 4867 for (i = 0; i < rx_ring->count; i++) { 4868 struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i]; 4869 4870 if (rx_buffer->skb) { 4871 struct sk_buff *skb = rx_buffer->skb; 4872 if (IXGBE_CB(skb)->page_released) 4873 dma_unmap_page(dev, 4874 IXGBE_CB(skb)->dma, 4875 ixgbe_rx_bufsz(rx_ring), 4876 DMA_FROM_DEVICE); 4877 dev_kfree_skb(skb); 4878 rx_buffer->skb = NULL; 4879 } 4880 4881 if (!rx_buffer->page) 4882 continue; 4883 4884 dma_unmap_page(dev, rx_buffer->dma, 4885 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 4886 __free_pages(rx_buffer->page, ixgbe_rx_pg_order(rx_ring)); 4887 4888 rx_buffer->page = NULL; 4889 } 4890 4891 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4892 memset(rx_ring->rx_buffer_info, 0, size); 4893 4894 /* Zero out the descriptor ring */ 4895 memset(rx_ring->desc, 0, rx_ring->size); 4896 4897 rx_ring->next_to_alloc = 0; 4898 rx_ring->next_to_clean = 0; 4899 rx_ring->next_to_use = 0; 4900 } 4901 4902 static void ixgbe_disable_fwd_ring(struct ixgbe_fwd_adapter *vadapter, 4903 struct ixgbe_ring *rx_ring) 4904 { 4905 struct ixgbe_adapter *adapter = vadapter->real_adapter; 4906 int index = rx_ring->queue_index + vadapter->rx_base_queue; 4907 4908 /* shutdown specific queue receive and wait for dma to settle */ 4909 ixgbe_disable_rx_queue(adapter, rx_ring); 4910 usleep_range(10000, 20000); 4911 ixgbe_irq_disable_queues(adapter, BIT_ULL(index)); 4912 ixgbe_clean_rx_ring(rx_ring); 4913 rx_ring->l2_accel_priv = NULL; 4914 } 4915 4916 static int ixgbe_fwd_ring_down(struct net_device *vdev, 4917 struct ixgbe_fwd_adapter *accel) 4918 { 4919 struct ixgbe_adapter *adapter = accel->real_adapter; 4920 unsigned int rxbase = accel->rx_base_queue; 4921 unsigned int txbase = accel->tx_base_queue; 4922 int i; 4923 4924 netif_tx_stop_all_queues(vdev); 4925 4926 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) { 4927 ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]); 4928 adapter->rx_ring[rxbase + i]->netdev = adapter->netdev; 4929 } 4930 4931 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) { 4932 adapter->tx_ring[txbase + i]->l2_accel_priv = NULL; 4933 adapter->tx_ring[txbase + i]->netdev = adapter->netdev; 4934 } 4935 4936 4937 return 0; 4938 } 4939 4940 static int ixgbe_fwd_ring_up(struct net_device *vdev, 4941 struct ixgbe_fwd_adapter *accel) 4942 { 4943 struct ixgbe_adapter *adapter = accel->real_adapter; 4944 unsigned int rxbase, txbase, queues; 4945 int i, baseq, err = 0; 4946 4947 if (!test_bit(accel->pool, &adapter->fwd_bitmask)) 4948 return 0; 4949 4950 baseq = accel->pool * adapter->num_rx_queues_per_pool; 4951 netdev_dbg(vdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n", 4952 accel->pool, adapter->num_rx_pools, 4953 baseq, baseq + adapter->num_rx_queues_per_pool, 4954 adapter->fwd_bitmask); 4955 4956 accel->netdev = vdev; 4957 accel->rx_base_queue = rxbase = baseq; 4958 accel->tx_base_queue = txbase = baseq; 4959 4960 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) 4961 ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]); 4962 4963 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) { 4964 adapter->rx_ring[rxbase + i]->netdev = vdev; 4965 adapter->rx_ring[rxbase + i]->l2_accel_priv = accel; 4966 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[rxbase + i]); 4967 } 4968 4969 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) { 4970 adapter->tx_ring[txbase + i]->netdev = vdev; 4971 adapter->tx_ring[txbase + i]->l2_accel_priv = accel; 4972 } 4973 4974 queues = min_t(unsigned int, 4975 adapter->num_rx_queues_per_pool, vdev->num_tx_queues); 4976 err = netif_set_real_num_tx_queues(vdev, queues); 4977 if (err) 4978 goto fwd_queue_err; 4979 4980 err = netif_set_real_num_rx_queues(vdev, queues); 4981 if (err) 4982 goto fwd_queue_err; 4983 4984 if (is_valid_ether_addr(vdev->dev_addr)) 4985 ixgbe_add_mac_filter(adapter, vdev->dev_addr, accel->pool); 4986 4987 ixgbe_fwd_psrtype(accel); 4988 ixgbe_macvlan_set_rx_mode(vdev, accel->pool, adapter); 4989 return err; 4990 fwd_queue_err: 4991 ixgbe_fwd_ring_down(vdev, accel); 4992 return err; 4993 } 4994 4995 static int ixgbe_upper_dev_walk(struct net_device *upper, void *data) 4996 { 4997 if (netif_is_macvlan(upper)) { 4998 struct macvlan_dev *dfwd = netdev_priv(upper); 4999 struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv; 5000 5001 if (dfwd->fwd_priv) 5002 ixgbe_fwd_ring_up(upper, vadapter); 5003 } 5004 5005 return 0; 5006 } 5007 5008 static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter) 5009 { 5010 netdev_walk_all_upper_dev_rcu(adapter->netdev, 5011 ixgbe_upper_dev_walk, NULL); 5012 } 5013 5014 static void ixgbe_configure(struct ixgbe_adapter *adapter) 5015 { 5016 struct ixgbe_hw *hw = &adapter->hw; 5017 5018 ixgbe_configure_pb(adapter); 5019 #ifdef CONFIG_IXGBE_DCB 5020 ixgbe_configure_dcb(adapter); 5021 #endif 5022 /* 5023 * We must restore virtualization before VLANs or else 5024 * the VLVF registers will not be populated 5025 */ 5026 ixgbe_configure_virtualization(adapter); 5027 5028 ixgbe_set_rx_mode(adapter->netdev); 5029 ixgbe_restore_vlan(adapter); 5030 5031 switch (hw->mac.type) { 5032 case ixgbe_mac_82599EB: 5033 case ixgbe_mac_X540: 5034 hw->mac.ops.disable_rx_buff(hw); 5035 break; 5036 default: 5037 break; 5038 } 5039 5040 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 5041 ixgbe_init_fdir_signature_82599(&adapter->hw, 5042 adapter->fdir_pballoc); 5043 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { 5044 ixgbe_init_fdir_perfect_82599(&adapter->hw, 5045 adapter->fdir_pballoc); 5046 ixgbe_fdir_filter_restore(adapter); 5047 } 5048 5049 switch (hw->mac.type) { 5050 case ixgbe_mac_82599EB: 5051 case ixgbe_mac_X540: 5052 hw->mac.ops.enable_rx_buff(hw); 5053 break; 5054 default: 5055 break; 5056 } 5057 5058 #ifdef CONFIG_IXGBE_DCA 5059 /* configure DCA */ 5060 if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE) 5061 ixgbe_setup_dca(adapter); 5062 #endif /* CONFIG_IXGBE_DCA */ 5063 5064 #ifdef IXGBE_FCOE 5065 /* configure FCoE L2 filters, redirection table, and Rx control */ 5066 ixgbe_configure_fcoe(adapter); 5067 5068 #endif /* IXGBE_FCOE */ 5069 ixgbe_configure_tx(adapter); 5070 ixgbe_configure_rx(adapter); 5071 ixgbe_configure_dfwd(adapter); 5072 } 5073 5074 /** 5075 * ixgbe_sfp_link_config - set up SFP+ link 5076 * @adapter: pointer to private adapter struct 5077 **/ 5078 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) 5079 { 5080 /* 5081 * We are assuming the worst case scenario here, and that 5082 * is that an SFP was inserted/removed after the reset 5083 * but before SFP detection was enabled. As such the best 5084 * solution is to just start searching as soon as we start 5085 */ 5086 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 5087 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 5088 5089 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 5090 adapter->sfp_poll_time = 0; 5091 } 5092 5093 /** 5094 * ixgbe_non_sfp_link_config - set up non-SFP+ link 5095 * @hw: pointer to private hardware struct 5096 * 5097 * Returns 0 on success, negative on failure 5098 **/ 5099 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) 5100 { 5101 u32 speed; 5102 bool autoneg, link_up = false; 5103 int ret = IXGBE_ERR_LINK_SETUP; 5104 5105 if (hw->mac.ops.check_link) 5106 ret = hw->mac.ops.check_link(hw, &speed, &link_up, false); 5107 5108 if (ret) 5109 return ret; 5110 5111 speed = hw->phy.autoneg_advertised; 5112 if ((!speed) && (hw->mac.ops.get_link_capabilities)) 5113 ret = hw->mac.ops.get_link_capabilities(hw, &speed, 5114 &autoneg); 5115 if (ret) 5116 return ret; 5117 5118 if (hw->mac.ops.setup_link) 5119 ret = hw->mac.ops.setup_link(hw, speed, link_up); 5120 5121 return ret; 5122 } 5123 5124 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) 5125 { 5126 struct ixgbe_hw *hw = &adapter->hw; 5127 u32 gpie = 0; 5128 5129 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 5130 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 5131 IXGBE_GPIE_OCD; 5132 gpie |= IXGBE_GPIE_EIAME; 5133 /* 5134 * use EIAM to auto-mask when MSI-X interrupt is asserted 5135 * this saves a register write for every interrupt 5136 */ 5137 switch (hw->mac.type) { 5138 case ixgbe_mac_82598EB: 5139 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 5140 break; 5141 case ixgbe_mac_82599EB: 5142 case ixgbe_mac_X540: 5143 case ixgbe_mac_X550: 5144 case ixgbe_mac_X550EM_x: 5145 case ixgbe_mac_x550em_a: 5146 default: 5147 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 5148 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 5149 break; 5150 } 5151 } else { 5152 /* legacy interrupts, use EIAM to auto-mask when reading EICR, 5153 * specifically only auto mask tx and rx interrupts */ 5154 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 5155 } 5156 5157 /* XXX: to interrupt immediately for EICS writes, enable this */ 5158 /* gpie |= IXGBE_GPIE_EIMEN; */ 5159 5160 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 5161 gpie &= ~IXGBE_GPIE_VTMODE_MASK; 5162 5163 switch (adapter->ring_feature[RING_F_VMDQ].mask) { 5164 case IXGBE_82599_VMDQ_8Q_MASK: 5165 gpie |= IXGBE_GPIE_VTMODE_16; 5166 break; 5167 case IXGBE_82599_VMDQ_4Q_MASK: 5168 gpie |= IXGBE_GPIE_VTMODE_32; 5169 break; 5170 default: 5171 gpie |= IXGBE_GPIE_VTMODE_64; 5172 break; 5173 } 5174 } 5175 5176 /* Enable Thermal over heat sensor interrupt */ 5177 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) { 5178 switch (adapter->hw.mac.type) { 5179 case ixgbe_mac_82599EB: 5180 gpie |= IXGBE_SDP0_GPIEN_8259X; 5181 break; 5182 default: 5183 break; 5184 } 5185 } 5186 5187 /* Enable fan failure interrupt */ 5188 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 5189 gpie |= IXGBE_SDP1_GPIEN(hw); 5190 5191 switch (hw->mac.type) { 5192 case ixgbe_mac_82599EB: 5193 gpie |= IXGBE_SDP1_GPIEN_8259X | IXGBE_SDP2_GPIEN_8259X; 5194 break; 5195 case ixgbe_mac_X550EM_x: 5196 case ixgbe_mac_x550em_a: 5197 gpie |= IXGBE_SDP0_GPIEN_X540; 5198 break; 5199 default: 5200 break; 5201 } 5202 5203 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 5204 } 5205 5206 static void ixgbe_up_complete(struct ixgbe_adapter *adapter) 5207 { 5208 struct ixgbe_hw *hw = &adapter->hw; 5209 int err; 5210 u32 ctrl_ext; 5211 5212 ixgbe_get_hw_control(adapter); 5213 ixgbe_setup_gpie(adapter); 5214 5215 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 5216 ixgbe_configure_msix(adapter); 5217 else 5218 ixgbe_configure_msi_and_legacy(adapter); 5219 5220 /* enable the optics for 82599 SFP+ fiber */ 5221 if (hw->mac.ops.enable_tx_laser) 5222 hw->mac.ops.enable_tx_laser(hw); 5223 5224 if (hw->phy.ops.set_phy_power) 5225 hw->phy.ops.set_phy_power(hw, true); 5226 5227 smp_mb__before_atomic(); 5228 clear_bit(__IXGBE_DOWN, &adapter->state); 5229 ixgbe_napi_enable_all(adapter); 5230 5231 if (ixgbe_is_sfp(hw)) { 5232 ixgbe_sfp_link_config(adapter); 5233 } else { 5234 err = ixgbe_non_sfp_link_config(hw); 5235 if (err) 5236 e_err(probe, "link_config FAILED %d\n", err); 5237 } 5238 5239 /* clear any pending interrupts, may auto mask */ 5240 IXGBE_READ_REG(hw, IXGBE_EICR); 5241 ixgbe_irq_enable(adapter, true, true); 5242 5243 /* 5244 * If this adapter has a fan, check to see if we had a failure 5245 * before we enabled the interrupt. 5246 */ 5247 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 5248 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 5249 if (esdp & IXGBE_ESDP_SDP1) 5250 e_crit(drv, "Fan has stopped, replace the adapter\n"); 5251 } 5252 5253 /* bring the link up in the watchdog, this could race with our first 5254 * link up interrupt but shouldn't be a problem */ 5255 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 5256 adapter->link_check_timeout = jiffies; 5257 mod_timer(&adapter->service_timer, jiffies); 5258 5259 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 5260 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5261 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 5262 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5263 } 5264 5265 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) 5266 { 5267 WARN_ON(in_interrupt()); 5268 /* put off any impending NetWatchDogTimeout */ 5269 netif_trans_update(adapter->netdev); 5270 5271 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state)) 5272 usleep_range(1000, 2000); 5273 if (adapter->hw.phy.type == ixgbe_phy_fw) 5274 ixgbe_watchdog_link_is_down(adapter); 5275 ixgbe_down(adapter); 5276 /* 5277 * If SR-IOV enabled then wait a bit before bringing the adapter 5278 * back up to give the VFs time to respond to the reset. The 5279 * two second wait is based upon the watchdog timer cycle in 5280 * the VF driver. 5281 */ 5282 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 5283 msleep(2000); 5284 ixgbe_up(adapter); 5285 clear_bit(__IXGBE_RESETTING, &adapter->state); 5286 } 5287 5288 void ixgbe_up(struct ixgbe_adapter *adapter) 5289 { 5290 /* hardware has been reset, we need to reload some things */ 5291 ixgbe_configure(adapter); 5292 5293 ixgbe_up_complete(adapter); 5294 } 5295 5296 void ixgbe_reset(struct ixgbe_adapter *adapter) 5297 { 5298 struct ixgbe_hw *hw = &adapter->hw; 5299 struct net_device *netdev = adapter->netdev; 5300 int err; 5301 5302 if (ixgbe_removed(hw->hw_addr)) 5303 return; 5304 /* lock SFP init bit to prevent race conditions with the watchdog */ 5305 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5306 usleep_range(1000, 2000); 5307 5308 /* clear all SFP and link config related flags while holding SFP_INIT */ 5309 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | 5310 IXGBE_FLAG2_SFP_NEEDS_RESET); 5311 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 5312 5313 err = hw->mac.ops.init_hw(hw); 5314 switch (err) { 5315 case 0: 5316 case IXGBE_ERR_SFP_NOT_PRESENT: 5317 case IXGBE_ERR_SFP_NOT_SUPPORTED: 5318 break; 5319 case IXGBE_ERR_MASTER_REQUESTS_PENDING: 5320 e_dev_err("master disable timed out\n"); 5321 break; 5322 case IXGBE_ERR_EEPROM_VERSION: 5323 /* We are running on a pre-production device, log a warning */ 5324 e_dev_warn("This device is a pre-production adapter/LOM. " 5325 "Please be aware there may be issues associated with " 5326 "your hardware. If you are experiencing problems " 5327 "please contact your Intel or hardware " 5328 "representative who provided you with this " 5329 "hardware.\n"); 5330 break; 5331 default: 5332 e_dev_err("Hardware Error: %d\n", err); 5333 } 5334 5335 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 5336 5337 /* flush entries out of MAC table */ 5338 ixgbe_flush_sw_mac_table(adapter); 5339 __dev_uc_unsync(netdev, NULL); 5340 5341 /* do not flush user set addresses */ 5342 ixgbe_mac_set_default_filter(adapter); 5343 5344 /* update SAN MAC vmdq pool selection */ 5345 if (hw->mac.san_mac_rar_index) 5346 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0)); 5347 5348 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) 5349 ixgbe_ptp_reset(adapter); 5350 5351 if (hw->phy.ops.set_phy_power) { 5352 if (!netif_running(adapter->netdev) && !adapter->wol) 5353 hw->phy.ops.set_phy_power(hw, false); 5354 else 5355 hw->phy.ops.set_phy_power(hw, true); 5356 } 5357 } 5358 5359 /** 5360 * ixgbe_clean_tx_ring - Free Tx Buffers 5361 * @tx_ring: ring to be cleaned 5362 **/ 5363 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) 5364 { 5365 struct ixgbe_tx_buffer *tx_buffer_info; 5366 unsigned long size; 5367 u16 i; 5368 5369 /* ring already cleared, nothing to do */ 5370 if (!tx_ring->tx_buffer_info) 5371 return; 5372 5373 /* Free all the Tx ring sk_buffs */ 5374 for (i = 0; i < tx_ring->count; i++) { 5375 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 5376 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 5377 } 5378 5379 netdev_tx_reset_queue(txring_txq(tx_ring)); 5380 5381 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 5382 memset(tx_ring->tx_buffer_info, 0, size); 5383 5384 /* Zero out the descriptor ring */ 5385 memset(tx_ring->desc, 0, tx_ring->size); 5386 5387 tx_ring->next_to_use = 0; 5388 tx_ring->next_to_clean = 0; 5389 } 5390 5391 /** 5392 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues 5393 * @adapter: board private structure 5394 **/ 5395 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) 5396 { 5397 int i; 5398 5399 for (i = 0; i < adapter->num_rx_queues; i++) 5400 ixgbe_clean_rx_ring(adapter->rx_ring[i]); 5401 } 5402 5403 /** 5404 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues 5405 * @adapter: board private structure 5406 **/ 5407 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter) 5408 { 5409 int i; 5410 5411 for (i = 0; i < adapter->num_tx_queues; i++) 5412 ixgbe_clean_tx_ring(adapter->tx_ring[i]); 5413 } 5414 5415 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter) 5416 { 5417 struct hlist_node *node2; 5418 struct ixgbe_fdir_filter *filter; 5419 5420 spin_lock(&adapter->fdir_perfect_lock); 5421 5422 hlist_for_each_entry_safe(filter, node2, 5423 &adapter->fdir_filter_list, fdir_node) { 5424 hlist_del(&filter->fdir_node); 5425 kfree(filter); 5426 } 5427 adapter->fdir_filter_count = 0; 5428 5429 spin_unlock(&adapter->fdir_perfect_lock); 5430 } 5431 5432 static int ixgbe_disable_macvlan(struct net_device *upper, void *data) 5433 { 5434 if (netif_is_macvlan(upper)) { 5435 struct macvlan_dev *vlan = netdev_priv(upper); 5436 5437 if (vlan->fwd_priv) { 5438 netif_tx_stop_all_queues(upper); 5439 netif_carrier_off(upper); 5440 netif_tx_disable(upper); 5441 } 5442 } 5443 5444 return 0; 5445 } 5446 5447 void ixgbe_down(struct ixgbe_adapter *adapter) 5448 { 5449 struct net_device *netdev = adapter->netdev; 5450 struct ixgbe_hw *hw = &adapter->hw; 5451 int i; 5452 5453 /* signal that we are down to the interrupt handler */ 5454 if (test_and_set_bit(__IXGBE_DOWN, &adapter->state)) 5455 return; /* do nothing if already down */ 5456 5457 /* disable receives */ 5458 hw->mac.ops.disable_rx(hw); 5459 5460 /* disable all enabled rx queues */ 5461 for (i = 0; i < adapter->num_rx_queues; i++) 5462 /* this call also flushes the previous write */ 5463 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]); 5464 5465 usleep_range(10000, 20000); 5466 5467 netif_tx_stop_all_queues(netdev); 5468 5469 /* call carrier off first to avoid false dev_watchdog timeouts */ 5470 netif_carrier_off(netdev); 5471 netif_tx_disable(netdev); 5472 5473 /* disable any upper devices */ 5474 netdev_walk_all_upper_dev_rcu(adapter->netdev, 5475 ixgbe_disable_macvlan, NULL); 5476 5477 ixgbe_irq_disable(adapter); 5478 5479 ixgbe_napi_disable_all(adapter); 5480 5481 clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state); 5482 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 5483 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 5484 5485 del_timer_sync(&adapter->service_timer); 5486 5487 if (adapter->num_vfs) { 5488 /* Clear EITR Select mapping */ 5489 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0); 5490 5491 /* Mark all the VFs as inactive */ 5492 for (i = 0 ; i < adapter->num_vfs; i++) 5493 adapter->vfinfo[i].clear_to_send = false; 5494 5495 /* ping all the active vfs to let them know we are going down */ 5496 ixgbe_ping_all_vfs(adapter); 5497 5498 /* Disable all VFTE/VFRE TX/RX */ 5499 ixgbe_disable_tx_rx(adapter); 5500 } 5501 5502 /* disable transmits in the hardware now that interrupts are off */ 5503 for (i = 0; i < adapter->num_tx_queues; i++) { 5504 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 5505 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 5506 } 5507 5508 /* Disable the Tx DMA engine on 82599 and later MAC */ 5509 switch (hw->mac.type) { 5510 case ixgbe_mac_82599EB: 5511 case ixgbe_mac_X540: 5512 case ixgbe_mac_X550: 5513 case ixgbe_mac_X550EM_x: 5514 case ixgbe_mac_x550em_a: 5515 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, 5516 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) & 5517 ~IXGBE_DMATXCTL_TE)); 5518 break; 5519 default: 5520 break; 5521 } 5522 5523 if (!pci_channel_offline(adapter->pdev)) 5524 ixgbe_reset(adapter); 5525 5526 /* power down the optics for 82599 SFP+ fiber */ 5527 if (hw->mac.ops.disable_tx_laser) 5528 hw->mac.ops.disable_tx_laser(hw); 5529 5530 ixgbe_clean_all_tx_rings(adapter); 5531 ixgbe_clean_all_rx_rings(adapter); 5532 } 5533 5534 /** 5535 * ixgbe_eee_capable - helper function to determine EEE support on X550 5536 * @adapter: board private structure 5537 */ 5538 static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter) 5539 { 5540 struct ixgbe_hw *hw = &adapter->hw; 5541 5542 switch (hw->device_id) { 5543 case IXGBE_DEV_ID_X550EM_A_1G_T: 5544 case IXGBE_DEV_ID_X550EM_A_1G_T_L: 5545 if (!hw->phy.eee_speeds_supported) 5546 break; 5547 adapter->flags2 |= IXGBE_FLAG2_EEE_CAPABLE; 5548 if (!hw->phy.eee_speeds_advertised) 5549 break; 5550 adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED; 5551 break; 5552 default: 5553 adapter->flags2 &= ~IXGBE_FLAG2_EEE_CAPABLE; 5554 adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED; 5555 break; 5556 } 5557 } 5558 5559 /** 5560 * ixgbe_tx_timeout - Respond to a Tx Hang 5561 * @netdev: network interface device structure 5562 **/ 5563 static void ixgbe_tx_timeout(struct net_device *netdev) 5564 { 5565 struct ixgbe_adapter *adapter = netdev_priv(netdev); 5566 5567 /* Do the reset outside of interrupt context */ 5568 ixgbe_tx_timeout_reset(adapter); 5569 } 5570 5571 #ifdef CONFIG_IXGBE_DCB 5572 static void ixgbe_init_dcb(struct ixgbe_adapter *adapter) 5573 { 5574 struct ixgbe_hw *hw = &adapter->hw; 5575 struct tc_configuration *tc; 5576 int j; 5577 5578 switch (hw->mac.type) { 5579 case ixgbe_mac_82598EB: 5580 case ixgbe_mac_82599EB: 5581 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS; 5582 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS; 5583 break; 5584 case ixgbe_mac_X540: 5585 case ixgbe_mac_X550: 5586 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS; 5587 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS; 5588 break; 5589 case ixgbe_mac_X550EM_x: 5590 case ixgbe_mac_x550em_a: 5591 default: 5592 adapter->dcb_cfg.num_tcs.pg_tcs = DEF_TRAFFIC_CLASS; 5593 adapter->dcb_cfg.num_tcs.pfc_tcs = DEF_TRAFFIC_CLASS; 5594 break; 5595 } 5596 5597 /* Configure DCB traffic classes */ 5598 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) { 5599 tc = &adapter->dcb_cfg.tc_config[j]; 5600 tc->path[DCB_TX_CONFIG].bwg_id = 0; 5601 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1); 5602 tc->path[DCB_RX_CONFIG].bwg_id = 0; 5603 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1); 5604 tc->dcb_pfc = pfc_disabled; 5605 } 5606 5607 /* Initialize default user to priority mapping, UPx->TC0 */ 5608 tc = &adapter->dcb_cfg.tc_config[0]; 5609 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 5610 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 5611 5612 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100; 5613 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100; 5614 adapter->dcb_cfg.pfc_mode_enable = false; 5615 adapter->dcb_set_bitmap = 0x00; 5616 if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) 5617 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; 5618 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, 5619 sizeof(adapter->temp_dcb_cfg)); 5620 } 5621 #endif 5622 5623 /** 5624 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter) 5625 * @adapter: board private structure to initialize 5626 * 5627 * ixgbe_sw_init initializes the Adapter private data structure. 5628 * Fields are initialized based on PCI device information and 5629 * OS network device settings (MTU size). 5630 **/ 5631 static int ixgbe_sw_init(struct ixgbe_adapter *adapter, 5632 const struct ixgbe_info *ii) 5633 { 5634 struct ixgbe_hw *hw = &adapter->hw; 5635 struct pci_dev *pdev = adapter->pdev; 5636 unsigned int rss, fdir; 5637 u32 fwsm; 5638 int i; 5639 5640 /* PCI config space info */ 5641 5642 hw->vendor_id = pdev->vendor; 5643 hw->device_id = pdev->device; 5644 hw->revision_id = pdev->revision; 5645 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5646 hw->subsystem_device_id = pdev->subsystem_device; 5647 5648 /* get_invariants needs the device IDs */ 5649 ii->get_invariants(hw); 5650 5651 /* Set common capability flags and settings */ 5652 rss = min_t(int, ixgbe_max_rss_indices(adapter), num_online_cpus()); 5653 adapter->ring_feature[RING_F_RSS].limit = rss; 5654 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE; 5655 adapter->max_q_vectors = MAX_Q_VECTORS_82599; 5656 adapter->atr_sample_rate = 20; 5657 fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus()); 5658 adapter->ring_feature[RING_F_FDIR].limit = fdir; 5659 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K; 5660 #ifdef CONFIG_IXGBE_DCA 5661 adapter->flags |= IXGBE_FLAG_DCA_CAPABLE; 5662 #endif 5663 #ifdef CONFIG_IXGBE_DCB 5664 adapter->flags |= IXGBE_FLAG_DCB_CAPABLE; 5665 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; 5666 #endif 5667 #ifdef IXGBE_FCOE 5668 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE; 5669 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 5670 #ifdef CONFIG_IXGBE_DCB 5671 /* Default traffic class to use for FCoE */ 5672 adapter->fcoe.up = IXGBE_FCOE_DEFTC; 5673 #endif /* CONFIG_IXGBE_DCB */ 5674 #endif /* IXGBE_FCOE */ 5675 5676 /* initialize static ixgbe jump table entries */ 5677 adapter->jump_tables[0] = kzalloc(sizeof(*adapter->jump_tables[0]), 5678 GFP_KERNEL); 5679 if (!adapter->jump_tables[0]) 5680 return -ENOMEM; 5681 adapter->jump_tables[0]->mat = ixgbe_ipv4_fields; 5682 5683 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) 5684 adapter->jump_tables[i] = NULL; 5685 5686 adapter->mac_table = kzalloc(sizeof(struct ixgbe_mac_addr) * 5687 hw->mac.num_rar_entries, 5688 GFP_ATOMIC); 5689 if (!adapter->mac_table) 5690 return -ENOMEM; 5691 5692 /* Set MAC specific capability flags and exceptions */ 5693 switch (hw->mac.type) { 5694 case ixgbe_mac_82598EB: 5695 adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE; 5696 5697 if (hw->device_id == IXGBE_DEV_ID_82598AT) 5698 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 5699 5700 adapter->max_q_vectors = MAX_Q_VECTORS_82598; 5701 adapter->ring_feature[RING_F_FDIR].limit = 0; 5702 adapter->atr_sample_rate = 0; 5703 adapter->fdir_pballoc = 0; 5704 #ifdef IXGBE_FCOE 5705 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 5706 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 5707 #ifdef CONFIG_IXGBE_DCB 5708 adapter->fcoe.up = 0; 5709 #endif /* IXGBE_DCB */ 5710 #endif /* IXGBE_FCOE */ 5711 break; 5712 case ixgbe_mac_82599EB: 5713 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) 5714 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 5715 break; 5716 case ixgbe_mac_X540: 5717 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw)); 5718 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5719 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 5720 break; 5721 case ixgbe_mac_x550em_a: 5722 adapter->flags |= IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE; 5723 switch (hw->device_id) { 5724 case IXGBE_DEV_ID_X550EM_A_1G_T: 5725 case IXGBE_DEV_ID_X550EM_A_1G_T_L: 5726 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 5727 break; 5728 default: 5729 break; 5730 } 5731 /* fall through */ 5732 case ixgbe_mac_X550EM_x: 5733 #ifdef CONFIG_IXGBE_DCB 5734 adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE; 5735 #endif 5736 #ifdef IXGBE_FCOE 5737 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 5738 #ifdef CONFIG_IXGBE_DCB 5739 adapter->fcoe.up = 0; 5740 #endif /* IXGBE_DCB */ 5741 #endif /* IXGBE_FCOE */ 5742 /* Fall Through */ 5743 case ixgbe_mac_X550: 5744 if (hw->mac.type == ixgbe_mac_X550) 5745 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 5746 #ifdef CONFIG_IXGBE_DCA 5747 adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE; 5748 #endif 5749 adapter->flags |= IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE; 5750 break; 5751 default: 5752 break; 5753 } 5754 5755 #ifdef IXGBE_FCOE 5756 /* FCoE support exists, always init the FCoE lock */ 5757 spin_lock_init(&adapter->fcoe.lock); 5758 5759 #endif 5760 /* n-tuple support exists, always init our spinlock */ 5761 spin_lock_init(&adapter->fdir_perfect_lock); 5762 5763 #ifdef CONFIG_IXGBE_DCB 5764 ixgbe_init_dcb(adapter); 5765 #endif 5766 5767 /* default flow control settings */ 5768 hw->fc.requested_mode = ixgbe_fc_full; 5769 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */ 5770 ixgbe_pbthresh_setup(adapter); 5771 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE; 5772 hw->fc.send_xon = true; 5773 hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw); 5774 5775 #ifdef CONFIG_PCI_IOV 5776 if (max_vfs > 0) 5777 e_dev_warn("Enabling SR-IOV VFs using the max_vfs module parameter is deprecated - please use the pci sysfs interface instead.\n"); 5778 5779 /* assign number of SR-IOV VFs */ 5780 if (hw->mac.type != ixgbe_mac_82598EB) { 5781 if (max_vfs > IXGBE_MAX_VFS_DRV_LIMIT) { 5782 adapter->num_vfs = 0; 5783 e_dev_warn("max_vfs parameter out of range. Not assigning any SR-IOV VFs\n"); 5784 } else { 5785 adapter->num_vfs = max_vfs; 5786 } 5787 } 5788 #endif /* CONFIG_PCI_IOV */ 5789 5790 /* enable itr by default in dynamic mode */ 5791 adapter->rx_itr_setting = 1; 5792 adapter->tx_itr_setting = 1; 5793 5794 /* set default ring sizes */ 5795 adapter->tx_ring_count = IXGBE_DEFAULT_TXD; 5796 adapter->rx_ring_count = IXGBE_DEFAULT_RXD; 5797 5798 /* set default work limits */ 5799 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK; 5800 5801 /* initialize eeprom parameters */ 5802 if (ixgbe_init_eeprom_params_generic(hw)) { 5803 e_dev_err("EEPROM initialization failed\n"); 5804 return -EIO; 5805 } 5806 5807 /* PF holds first pool slot */ 5808 set_bit(0, &adapter->fwd_bitmask); 5809 set_bit(__IXGBE_DOWN, &adapter->state); 5810 5811 return 0; 5812 } 5813 5814 /** 5815 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors) 5816 * @tx_ring: tx descriptor ring (for a specific queue) to setup 5817 * 5818 * Return 0 on success, negative on failure 5819 **/ 5820 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) 5821 { 5822 struct device *dev = tx_ring->dev; 5823 int orig_node = dev_to_node(dev); 5824 int ring_node = -1; 5825 int size; 5826 5827 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 5828 5829 if (tx_ring->q_vector) 5830 ring_node = tx_ring->q_vector->numa_node; 5831 5832 tx_ring->tx_buffer_info = vzalloc_node(size, ring_node); 5833 if (!tx_ring->tx_buffer_info) 5834 tx_ring->tx_buffer_info = vzalloc(size); 5835 if (!tx_ring->tx_buffer_info) 5836 goto err; 5837 5838 u64_stats_init(&tx_ring->syncp); 5839 5840 /* round up to nearest 4K */ 5841 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 5842 tx_ring->size = ALIGN(tx_ring->size, 4096); 5843 5844 set_dev_node(dev, ring_node); 5845 tx_ring->desc = dma_alloc_coherent(dev, 5846 tx_ring->size, 5847 &tx_ring->dma, 5848 GFP_KERNEL); 5849 set_dev_node(dev, orig_node); 5850 if (!tx_ring->desc) 5851 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 5852 &tx_ring->dma, GFP_KERNEL); 5853 if (!tx_ring->desc) 5854 goto err; 5855 5856 tx_ring->next_to_use = 0; 5857 tx_ring->next_to_clean = 0; 5858 return 0; 5859 5860 err: 5861 vfree(tx_ring->tx_buffer_info); 5862 tx_ring->tx_buffer_info = NULL; 5863 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); 5864 return -ENOMEM; 5865 } 5866 5867 /** 5868 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources 5869 * @adapter: board private structure 5870 * 5871 * If this function returns with an error, then it's possible one or 5872 * more of the rings is populated (while the rest are not). It is the 5873 * callers duty to clean those orphaned rings. 5874 * 5875 * Return 0 on success, negative on failure 5876 **/ 5877 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) 5878 { 5879 int i, err = 0; 5880 5881 for (i = 0; i < adapter->num_tx_queues; i++) { 5882 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]); 5883 if (!err) 5884 continue; 5885 5886 e_err(probe, "Allocation for Tx Queue %u failed\n", i); 5887 goto err_setup_tx; 5888 } 5889 5890 return 0; 5891 err_setup_tx: 5892 /* rewind the index freeing the rings as we go */ 5893 while (i--) 5894 ixgbe_free_tx_resources(adapter->tx_ring[i]); 5895 return err; 5896 } 5897 5898 /** 5899 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) 5900 * @rx_ring: rx descriptor ring (for a specific queue) to setup 5901 * 5902 * Returns 0 on success, negative on failure 5903 **/ 5904 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) 5905 { 5906 struct device *dev = rx_ring->dev; 5907 int orig_node = dev_to_node(dev); 5908 int ring_node = -1; 5909 int size; 5910 5911 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 5912 5913 if (rx_ring->q_vector) 5914 ring_node = rx_ring->q_vector->numa_node; 5915 5916 rx_ring->rx_buffer_info = vzalloc_node(size, ring_node); 5917 if (!rx_ring->rx_buffer_info) 5918 rx_ring->rx_buffer_info = vzalloc(size); 5919 if (!rx_ring->rx_buffer_info) 5920 goto err; 5921 5922 u64_stats_init(&rx_ring->syncp); 5923 5924 /* Round up to nearest 4K */ 5925 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 5926 rx_ring->size = ALIGN(rx_ring->size, 4096); 5927 5928 set_dev_node(dev, ring_node); 5929 rx_ring->desc = dma_alloc_coherent(dev, 5930 rx_ring->size, 5931 &rx_ring->dma, 5932 GFP_KERNEL); 5933 set_dev_node(dev, orig_node); 5934 if (!rx_ring->desc) 5935 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 5936 &rx_ring->dma, GFP_KERNEL); 5937 if (!rx_ring->desc) 5938 goto err; 5939 5940 rx_ring->next_to_clean = 0; 5941 rx_ring->next_to_use = 0; 5942 5943 return 0; 5944 err: 5945 vfree(rx_ring->rx_buffer_info); 5946 rx_ring->rx_buffer_info = NULL; 5947 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); 5948 return -ENOMEM; 5949 } 5950 5951 /** 5952 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources 5953 * @adapter: board private structure 5954 * 5955 * If this function returns with an error, then it's possible one or 5956 * more of the rings is populated (while the rest are not). It is the 5957 * callers duty to clean those orphaned rings. 5958 * 5959 * Return 0 on success, negative on failure 5960 **/ 5961 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) 5962 { 5963 int i, err = 0; 5964 5965 for (i = 0; i < adapter->num_rx_queues; i++) { 5966 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]); 5967 if (!err) 5968 continue; 5969 5970 e_err(probe, "Allocation for Rx Queue %u failed\n", i); 5971 goto err_setup_rx; 5972 } 5973 5974 #ifdef IXGBE_FCOE 5975 err = ixgbe_setup_fcoe_ddp_resources(adapter); 5976 if (!err) 5977 #endif 5978 return 0; 5979 err_setup_rx: 5980 /* rewind the index freeing the rings as we go */ 5981 while (i--) 5982 ixgbe_free_rx_resources(adapter->rx_ring[i]); 5983 return err; 5984 } 5985 5986 /** 5987 * ixgbe_free_tx_resources - Free Tx Resources per Queue 5988 * @tx_ring: Tx descriptor ring for a specific queue 5989 * 5990 * Free all transmit software resources 5991 **/ 5992 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring) 5993 { 5994 ixgbe_clean_tx_ring(tx_ring); 5995 5996 vfree(tx_ring->tx_buffer_info); 5997 tx_ring->tx_buffer_info = NULL; 5998 5999 /* if not set, then don't free */ 6000 if (!tx_ring->desc) 6001 return; 6002 6003 dma_free_coherent(tx_ring->dev, tx_ring->size, 6004 tx_ring->desc, tx_ring->dma); 6005 6006 tx_ring->desc = NULL; 6007 } 6008 6009 /** 6010 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues 6011 * @adapter: board private structure 6012 * 6013 * Free all transmit software resources 6014 **/ 6015 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) 6016 { 6017 int i; 6018 6019 for (i = 0; i < adapter->num_tx_queues; i++) 6020 if (adapter->tx_ring[i]->desc) 6021 ixgbe_free_tx_resources(adapter->tx_ring[i]); 6022 } 6023 6024 /** 6025 * ixgbe_free_rx_resources - Free Rx Resources 6026 * @rx_ring: ring to clean the resources from 6027 * 6028 * Free all receive software resources 6029 **/ 6030 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) 6031 { 6032 ixgbe_clean_rx_ring(rx_ring); 6033 6034 vfree(rx_ring->rx_buffer_info); 6035 rx_ring->rx_buffer_info = NULL; 6036 6037 /* if not set, then don't free */ 6038 if (!rx_ring->desc) 6039 return; 6040 6041 dma_free_coherent(rx_ring->dev, rx_ring->size, 6042 rx_ring->desc, rx_ring->dma); 6043 6044 rx_ring->desc = NULL; 6045 } 6046 6047 /** 6048 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues 6049 * @adapter: board private structure 6050 * 6051 * Free all receive software resources 6052 **/ 6053 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) 6054 { 6055 int i; 6056 6057 #ifdef IXGBE_FCOE 6058 ixgbe_free_fcoe_ddp_resources(adapter); 6059 6060 #endif 6061 for (i = 0; i < adapter->num_rx_queues; i++) 6062 if (adapter->rx_ring[i]->desc) 6063 ixgbe_free_rx_resources(adapter->rx_ring[i]); 6064 } 6065 6066 /** 6067 * ixgbe_change_mtu - Change the Maximum Transfer Unit 6068 * @netdev: network interface device structure 6069 * @new_mtu: new value for maximum frame size 6070 * 6071 * Returns 0 on success, negative on failure 6072 **/ 6073 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) 6074 { 6075 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6076 6077 /* 6078 * For 82599EB we cannot allow legacy VFs to enable their receive 6079 * paths when MTU greater than 1500 is configured. So display a 6080 * warning that legacy VFs will be disabled. 6081 */ 6082 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && 6083 (adapter->hw.mac.type == ixgbe_mac_82599EB) && 6084 (new_mtu > ETH_DATA_LEN)) 6085 e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n"); 6086 6087 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 6088 6089 /* must set new MTU before calling down or up */ 6090 netdev->mtu = new_mtu; 6091 6092 if (netif_running(netdev)) 6093 ixgbe_reinit_locked(adapter); 6094 6095 return 0; 6096 } 6097 6098 /** 6099 * ixgbe_open - Called when a network interface is made active 6100 * @netdev: network interface device structure 6101 * 6102 * Returns 0 on success, negative value on failure 6103 * 6104 * The open entry point is called when a network interface is made 6105 * active by the system (IFF_UP). At this point all resources needed 6106 * for transmit and receive operations are allocated, the interrupt 6107 * handler is registered with the OS, the watchdog timer is started, 6108 * and the stack is notified that the interface is ready. 6109 **/ 6110 int ixgbe_open(struct net_device *netdev) 6111 { 6112 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6113 struct ixgbe_hw *hw = &adapter->hw; 6114 int err, queues; 6115 6116 /* disallow open during test */ 6117 if (test_bit(__IXGBE_TESTING, &adapter->state)) 6118 return -EBUSY; 6119 6120 netif_carrier_off(netdev); 6121 6122 /* allocate transmit descriptors */ 6123 err = ixgbe_setup_all_tx_resources(adapter); 6124 if (err) 6125 goto err_setup_tx; 6126 6127 /* allocate receive descriptors */ 6128 err = ixgbe_setup_all_rx_resources(adapter); 6129 if (err) 6130 goto err_setup_rx; 6131 6132 ixgbe_configure(adapter); 6133 6134 err = ixgbe_request_irq(adapter); 6135 if (err) 6136 goto err_req_irq; 6137 6138 /* Notify the stack of the actual queue counts. */ 6139 if (adapter->num_rx_pools > 1) 6140 queues = adapter->num_rx_queues_per_pool; 6141 else 6142 queues = adapter->num_tx_queues; 6143 6144 err = netif_set_real_num_tx_queues(netdev, queues); 6145 if (err) 6146 goto err_set_queues; 6147 6148 if (adapter->num_rx_pools > 1 && 6149 adapter->num_rx_queues > IXGBE_MAX_L2A_QUEUES) 6150 queues = IXGBE_MAX_L2A_QUEUES; 6151 else 6152 queues = adapter->num_rx_queues; 6153 err = netif_set_real_num_rx_queues(netdev, queues); 6154 if (err) 6155 goto err_set_queues; 6156 6157 ixgbe_ptp_init(adapter); 6158 6159 ixgbe_up_complete(adapter); 6160 6161 ixgbe_clear_udp_tunnel_port(adapter, IXGBE_VXLANCTRL_ALL_UDPPORT_MASK); 6162 udp_tunnel_get_rx_info(netdev); 6163 6164 return 0; 6165 6166 err_set_queues: 6167 ixgbe_free_irq(adapter); 6168 err_req_irq: 6169 ixgbe_free_all_rx_resources(adapter); 6170 if (hw->phy.ops.set_phy_power && !adapter->wol) 6171 hw->phy.ops.set_phy_power(&adapter->hw, false); 6172 err_setup_rx: 6173 ixgbe_free_all_tx_resources(adapter); 6174 err_setup_tx: 6175 ixgbe_reset(adapter); 6176 6177 return err; 6178 } 6179 6180 static void ixgbe_close_suspend(struct ixgbe_adapter *adapter) 6181 { 6182 ixgbe_ptp_suspend(adapter); 6183 6184 if (adapter->hw.phy.ops.enter_lplu) { 6185 adapter->hw.phy.reset_disable = true; 6186 ixgbe_down(adapter); 6187 adapter->hw.phy.ops.enter_lplu(&adapter->hw); 6188 adapter->hw.phy.reset_disable = false; 6189 } else { 6190 ixgbe_down(adapter); 6191 } 6192 6193 ixgbe_free_irq(adapter); 6194 6195 ixgbe_free_all_tx_resources(adapter); 6196 ixgbe_free_all_rx_resources(adapter); 6197 } 6198 6199 /** 6200 * ixgbe_close - Disables a network interface 6201 * @netdev: network interface device structure 6202 * 6203 * Returns 0, this is not allowed to fail 6204 * 6205 * The close entry point is called when an interface is de-activated 6206 * by the OS. The hardware is still under the drivers control, but 6207 * needs to be disabled. A global MAC reset is issued to stop the 6208 * hardware, and all transmit and receive resources are freed. 6209 **/ 6210 int ixgbe_close(struct net_device *netdev) 6211 { 6212 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6213 6214 ixgbe_ptp_stop(adapter); 6215 6216 if (netif_device_present(netdev)) 6217 ixgbe_close_suspend(adapter); 6218 6219 ixgbe_fdir_filter_exit(adapter); 6220 6221 ixgbe_release_hw_control(adapter); 6222 6223 return 0; 6224 } 6225 6226 #ifdef CONFIG_PM 6227 static int ixgbe_resume(struct pci_dev *pdev) 6228 { 6229 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 6230 struct net_device *netdev = adapter->netdev; 6231 u32 err; 6232 6233 adapter->hw.hw_addr = adapter->io_addr; 6234 pci_set_power_state(pdev, PCI_D0); 6235 pci_restore_state(pdev); 6236 /* 6237 * pci_restore_state clears dev->state_saved so call 6238 * pci_save_state to restore it. 6239 */ 6240 pci_save_state(pdev); 6241 6242 err = pci_enable_device_mem(pdev); 6243 if (err) { 6244 e_dev_err("Cannot enable PCI device from suspend\n"); 6245 return err; 6246 } 6247 smp_mb__before_atomic(); 6248 clear_bit(__IXGBE_DISABLED, &adapter->state); 6249 pci_set_master(pdev); 6250 6251 pci_wake_from_d3(pdev, false); 6252 6253 ixgbe_reset(adapter); 6254 6255 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 6256 6257 rtnl_lock(); 6258 err = ixgbe_init_interrupt_scheme(adapter); 6259 if (!err && netif_running(netdev)) 6260 err = ixgbe_open(netdev); 6261 6262 6263 if (!err) 6264 netif_device_attach(netdev); 6265 rtnl_unlock(); 6266 6267 return err; 6268 } 6269 #endif /* CONFIG_PM */ 6270 6271 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) 6272 { 6273 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 6274 struct net_device *netdev = adapter->netdev; 6275 struct ixgbe_hw *hw = &adapter->hw; 6276 u32 ctrl, fctrl; 6277 u32 wufc = adapter->wol; 6278 #ifdef CONFIG_PM 6279 int retval = 0; 6280 #endif 6281 6282 rtnl_lock(); 6283 netif_device_detach(netdev); 6284 6285 if (netif_running(netdev)) 6286 ixgbe_close_suspend(adapter); 6287 6288 ixgbe_clear_interrupt_scheme(adapter); 6289 rtnl_unlock(); 6290 6291 #ifdef CONFIG_PM 6292 retval = pci_save_state(pdev); 6293 if (retval) 6294 return retval; 6295 6296 #endif 6297 if (hw->mac.ops.stop_link_on_d3) 6298 hw->mac.ops.stop_link_on_d3(hw); 6299 6300 if (wufc) { 6301 ixgbe_set_rx_mode(netdev); 6302 6303 /* enable the optics for 82599 SFP+ fiber as we can WoL */ 6304 if (hw->mac.ops.enable_tx_laser) 6305 hw->mac.ops.enable_tx_laser(hw); 6306 6307 /* turn on all-multi mode if wake on multicast is enabled */ 6308 if (wufc & IXGBE_WUFC_MC) { 6309 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 6310 fctrl |= IXGBE_FCTRL_MPE; 6311 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 6312 } 6313 6314 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL); 6315 ctrl |= IXGBE_CTRL_GIO_DIS; 6316 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); 6317 6318 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc); 6319 } else { 6320 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 6321 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 6322 } 6323 6324 switch (hw->mac.type) { 6325 case ixgbe_mac_82598EB: 6326 pci_wake_from_d3(pdev, false); 6327 break; 6328 case ixgbe_mac_82599EB: 6329 case ixgbe_mac_X540: 6330 case ixgbe_mac_X550: 6331 case ixgbe_mac_X550EM_x: 6332 case ixgbe_mac_x550em_a: 6333 pci_wake_from_d3(pdev, !!wufc); 6334 break; 6335 default: 6336 break; 6337 } 6338 6339 *enable_wake = !!wufc; 6340 if (hw->phy.ops.set_phy_power && !*enable_wake) 6341 hw->phy.ops.set_phy_power(hw, false); 6342 6343 ixgbe_release_hw_control(adapter); 6344 6345 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state)) 6346 pci_disable_device(pdev); 6347 6348 return 0; 6349 } 6350 6351 #ifdef CONFIG_PM 6352 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state) 6353 { 6354 int retval; 6355 bool wake; 6356 6357 retval = __ixgbe_shutdown(pdev, &wake); 6358 if (retval) 6359 return retval; 6360 6361 if (wake) { 6362 pci_prepare_to_sleep(pdev); 6363 } else { 6364 pci_wake_from_d3(pdev, false); 6365 pci_set_power_state(pdev, PCI_D3hot); 6366 } 6367 6368 return 0; 6369 } 6370 #endif /* CONFIG_PM */ 6371 6372 static void ixgbe_shutdown(struct pci_dev *pdev) 6373 { 6374 bool wake; 6375 6376 __ixgbe_shutdown(pdev, &wake); 6377 6378 if (system_state == SYSTEM_POWER_OFF) { 6379 pci_wake_from_d3(pdev, wake); 6380 pci_set_power_state(pdev, PCI_D3hot); 6381 } 6382 } 6383 6384 /** 6385 * ixgbe_update_stats - Update the board statistics counters. 6386 * @adapter: board private structure 6387 **/ 6388 void ixgbe_update_stats(struct ixgbe_adapter *adapter) 6389 { 6390 struct net_device *netdev = adapter->netdev; 6391 struct ixgbe_hw *hw = &adapter->hw; 6392 struct ixgbe_hw_stats *hwstats = &adapter->stats; 6393 u64 total_mpc = 0; 6394 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; 6395 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0; 6396 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 6397 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; 6398 6399 if (test_bit(__IXGBE_DOWN, &adapter->state) || 6400 test_bit(__IXGBE_RESETTING, &adapter->state)) 6401 return; 6402 6403 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { 6404 u64 rsc_count = 0; 6405 u64 rsc_flush = 0; 6406 for (i = 0; i < adapter->num_rx_queues; i++) { 6407 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count; 6408 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush; 6409 } 6410 adapter->rsc_total_count = rsc_count; 6411 adapter->rsc_total_flush = rsc_flush; 6412 } 6413 6414 for (i = 0; i < adapter->num_rx_queues; i++) { 6415 struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; 6416 non_eop_descs += rx_ring->rx_stats.non_eop_descs; 6417 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 6418 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 6419 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 6420 bytes += rx_ring->stats.bytes; 6421 packets += rx_ring->stats.packets; 6422 } 6423 adapter->non_eop_descs = non_eop_descs; 6424 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 6425 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 6426 adapter->hw_csum_rx_error = hw_csum_rx_error; 6427 netdev->stats.rx_bytes = bytes; 6428 netdev->stats.rx_packets = packets; 6429 6430 bytes = 0; 6431 packets = 0; 6432 /* gather some stats to the adapter struct that are per queue */ 6433 for (i = 0; i < adapter->num_tx_queues; i++) { 6434 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 6435 restart_queue += tx_ring->tx_stats.restart_queue; 6436 tx_busy += tx_ring->tx_stats.tx_busy; 6437 bytes += tx_ring->stats.bytes; 6438 packets += tx_ring->stats.packets; 6439 } 6440 adapter->restart_queue = restart_queue; 6441 adapter->tx_busy = tx_busy; 6442 netdev->stats.tx_bytes = bytes; 6443 netdev->stats.tx_packets = packets; 6444 6445 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 6446 6447 /* 8 register reads */ 6448 for (i = 0; i < 8; i++) { 6449 /* for packet buffers not used, the register should read 0 */ 6450 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 6451 missed_rx += mpc; 6452 hwstats->mpc[i] += mpc; 6453 total_mpc += hwstats->mpc[i]; 6454 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 6455 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 6456 switch (hw->mac.type) { 6457 case ixgbe_mac_82598EB: 6458 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 6459 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 6460 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 6461 hwstats->pxonrxc[i] += 6462 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 6463 break; 6464 case ixgbe_mac_82599EB: 6465 case ixgbe_mac_X540: 6466 case ixgbe_mac_X550: 6467 case ixgbe_mac_X550EM_x: 6468 case ixgbe_mac_x550em_a: 6469 hwstats->pxonrxc[i] += 6470 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 6471 break; 6472 default: 6473 break; 6474 } 6475 } 6476 6477 /*16 register reads */ 6478 for (i = 0; i < 16; i++) { 6479 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 6480 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 6481 if ((hw->mac.type == ixgbe_mac_82599EB) || 6482 (hw->mac.type == ixgbe_mac_X540) || 6483 (hw->mac.type == ixgbe_mac_X550) || 6484 (hw->mac.type == ixgbe_mac_X550EM_x) || 6485 (hw->mac.type == ixgbe_mac_x550em_a)) { 6486 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 6487 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ 6488 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 6489 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */ 6490 } 6491 } 6492 6493 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 6494 /* work around hardware counting issue */ 6495 hwstats->gprc -= missed_rx; 6496 6497 ixgbe_update_xoff_received(adapter); 6498 6499 /* 82598 hardware only has a 32 bit counter in the high register */ 6500 switch (hw->mac.type) { 6501 case ixgbe_mac_82598EB: 6502 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 6503 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 6504 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 6505 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 6506 break; 6507 case ixgbe_mac_X540: 6508 case ixgbe_mac_X550: 6509 case ixgbe_mac_X550EM_x: 6510 case ixgbe_mac_x550em_a: 6511 /* OS2BMC stats are X540 and later */ 6512 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC); 6513 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC); 6514 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC); 6515 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC); 6516 case ixgbe_mac_82599EB: 6517 for (i = 0; i < 16; i++) 6518 adapter->hw_rx_no_dma_resources += 6519 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 6520 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 6521 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ 6522 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 6523 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */ 6524 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 6525 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */ 6526 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 6527 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 6528 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 6529 #ifdef IXGBE_FCOE 6530 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 6531 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 6532 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 6533 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 6534 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 6535 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 6536 /* Add up per cpu counters for total ddp aloc fail */ 6537 if (adapter->fcoe.ddp_pool) { 6538 struct ixgbe_fcoe *fcoe = &adapter->fcoe; 6539 struct ixgbe_fcoe_ddp_pool *ddp_pool; 6540 unsigned int cpu; 6541 u64 noddp = 0, noddp_ext_buff = 0; 6542 for_each_possible_cpu(cpu) { 6543 ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu); 6544 noddp += ddp_pool->noddp; 6545 noddp_ext_buff += ddp_pool->noddp_ext_buff; 6546 } 6547 hwstats->fcoe_noddp = noddp; 6548 hwstats->fcoe_noddp_ext_buff = noddp_ext_buff; 6549 } 6550 #endif /* IXGBE_FCOE */ 6551 break; 6552 default: 6553 break; 6554 } 6555 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 6556 hwstats->bprc += bprc; 6557 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 6558 if (hw->mac.type == ixgbe_mac_82598EB) 6559 hwstats->mprc -= bprc; 6560 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 6561 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 6562 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 6563 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 6564 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 6565 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 6566 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 6567 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 6568 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 6569 hwstats->lxontxc += lxon; 6570 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 6571 hwstats->lxofftxc += lxoff; 6572 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 6573 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 6574 /* 6575 * 82598 errata - tx of flow control packets is included in tx counters 6576 */ 6577 xon_off_tot = lxon + lxoff; 6578 hwstats->gptc -= xon_off_tot; 6579 hwstats->mptc -= xon_off_tot; 6580 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN)); 6581 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 6582 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 6583 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 6584 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 6585 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 6586 hwstats->ptc64 -= xon_off_tot; 6587 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 6588 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 6589 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 6590 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 6591 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 6592 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 6593 6594 /* Fill out the OS statistics structure */ 6595 netdev->stats.multicast = hwstats->mprc; 6596 6597 /* Rx Errors */ 6598 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec; 6599 netdev->stats.rx_dropped = 0; 6600 netdev->stats.rx_length_errors = hwstats->rlec; 6601 netdev->stats.rx_crc_errors = hwstats->crcerrs; 6602 netdev->stats.rx_missed_errors = total_mpc; 6603 } 6604 6605 /** 6606 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table 6607 * @adapter: pointer to the device adapter structure 6608 **/ 6609 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) 6610 { 6611 struct ixgbe_hw *hw = &adapter->hw; 6612 int i; 6613 6614 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 6615 return; 6616 6617 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 6618 6619 /* if interface is down do nothing */ 6620 if (test_bit(__IXGBE_DOWN, &adapter->state)) 6621 return; 6622 6623 /* do nothing if we are not using signature filters */ 6624 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) 6625 return; 6626 6627 adapter->fdir_overflow++; 6628 6629 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) { 6630 for (i = 0; i < adapter->num_tx_queues; i++) 6631 set_bit(__IXGBE_TX_FDIR_INIT_DONE, 6632 &(adapter->tx_ring[i]->state)); 6633 /* re-enable flow director interrupts */ 6634 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); 6635 } else { 6636 e_err(probe, "failed to finish FDIR re-initialization, " 6637 "ignored adding FDIR ATR filters\n"); 6638 } 6639 } 6640 6641 /** 6642 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts 6643 * @adapter: pointer to the device adapter structure 6644 * 6645 * This function serves two purposes. First it strobes the interrupt lines 6646 * in order to make certain interrupts are occurring. Secondly it sets the 6647 * bits needed to check for TX hangs. As a result we should immediately 6648 * determine if a hang has occurred. 6649 */ 6650 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) 6651 { 6652 struct ixgbe_hw *hw = &adapter->hw; 6653 u64 eics = 0; 6654 int i; 6655 6656 /* If we're down, removing or resetting, just bail */ 6657 if (test_bit(__IXGBE_DOWN, &adapter->state) || 6658 test_bit(__IXGBE_REMOVING, &adapter->state) || 6659 test_bit(__IXGBE_RESETTING, &adapter->state)) 6660 return; 6661 6662 /* Force detection of hung controller */ 6663 if (netif_carrier_ok(adapter->netdev)) { 6664 for (i = 0; i < adapter->num_tx_queues; i++) 6665 set_check_for_tx_hang(adapter->tx_ring[i]); 6666 } 6667 6668 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 6669 /* 6670 * for legacy and MSI interrupts don't set any bits 6671 * that are enabled for EIAM, because this operation 6672 * would set *both* EIMS and EICS for any bit in EIAM 6673 */ 6674 IXGBE_WRITE_REG(hw, IXGBE_EICS, 6675 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER)); 6676 } else { 6677 /* get one bit for every active tx/rx interrupt vector */ 6678 for (i = 0; i < adapter->num_q_vectors; i++) { 6679 struct ixgbe_q_vector *qv = adapter->q_vector[i]; 6680 if (qv->rx.ring || qv->tx.ring) 6681 eics |= BIT_ULL(i); 6682 } 6683 } 6684 6685 /* Cause software interrupt to ensure rings are cleaned */ 6686 ixgbe_irq_rearm_queues(adapter, eics); 6687 } 6688 6689 /** 6690 * ixgbe_watchdog_update_link - update the link status 6691 * @adapter: pointer to the device adapter structure 6692 * @link_speed: pointer to a u32 to store the link_speed 6693 **/ 6694 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) 6695 { 6696 struct ixgbe_hw *hw = &adapter->hw; 6697 u32 link_speed = adapter->link_speed; 6698 bool link_up = adapter->link_up; 6699 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 6700 6701 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 6702 return; 6703 6704 if (hw->mac.ops.check_link) { 6705 hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 6706 } else { 6707 /* always assume link is up, if no check link function */ 6708 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 6709 link_up = true; 6710 } 6711 6712 if (adapter->ixgbe_ieee_pfc) 6713 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 6714 6715 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) { 6716 hw->mac.ops.fc_enable(hw); 6717 ixgbe_set_rx_drop_en(adapter); 6718 } 6719 6720 if (link_up || 6721 time_after(jiffies, (adapter->link_check_timeout + 6722 IXGBE_TRY_LINK_TIMEOUT))) { 6723 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 6724 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); 6725 IXGBE_WRITE_FLUSH(hw); 6726 } 6727 6728 adapter->link_up = link_up; 6729 adapter->link_speed = link_speed; 6730 } 6731 6732 static void ixgbe_update_default_up(struct ixgbe_adapter *adapter) 6733 { 6734 #ifdef CONFIG_IXGBE_DCB 6735 struct net_device *netdev = adapter->netdev; 6736 struct dcb_app app = { 6737 .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, 6738 .protocol = 0, 6739 }; 6740 u8 up = 0; 6741 6742 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) 6743 up = dcb_ieee_getapp_mask(netdev, &app); 6744 6745 adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0; 6746 #endif 6747 } 6748 6749 static int ixgbe_enable_macvlan(struct net_device *upper, void *data) 6750 { 6751 if (netif_is_macvlan(upper)) { 6752 struct macvlan_dev *vlan = netdev_priv(upper); 6753 6754 if (vlan->fwd_priv) 6755 netif_tx_wake_all_queues(upper); 6756 } 6757 6758 return 0; 6759 } 6760 6761 /** 6762 * ixgbe_watchdog_link_is_up - update netif_carrier status and 6763 * print link up message 6764 * @adapter: pointer to the device adapter structure 6765 **/ 6766 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) 6767 { 6768 struct net_device *netdev = adapter->netdev; 6769 struct ixgbe_hw *hw = &adapter->hw; 6770 u32 link_speed = adapter->link_speed; 6771 const char *speed_str; 6772 bool flow_rx, flow_tx; 6773 6774 /* only continue if link was previously down */ 6775 if (netif_carrier_ok(netdev)) 6776 return; 6777 6778 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 6779 6780 switch (hw->mac.type) { 6781 case ixgbe_mac_82598EB: { 6782 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 6783 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS); 6784 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE); 6785 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X); 6786 } 6787 break; 6788 case ixgbe_mac_X540: 6789 case ixgbe_mac_X550: 6790 case ixgbe_mac_X550EM_x: 6791 case ixgbe_mac_x550em_a: 6792 case ixgbe_mac_82599EB: { 6793 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 6794 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 6795 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE); 6796 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X); 6797 } 6798 break; 6799 default: 6800 flow_tx = false; 6801 flow_rx = false; 6802 break; 6803 } 6804 6805 adapter->last_rx_ptp_check = jiffies; 6806 6807 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) 6808 ixgbe_ptp_start_cyclecounter(adapter); 6809 6810 switch (link_speed) { 6811 case IXGBE_LINK_SPEED_10GB_FULL: 6812 speed_str = "10 Gbps"; 6813 break; 6814 case IXGBE_LINK_SPEED_2_5GB_FULL: 6815 speed_str = "2.5 Gbps"; 6816 break; 6817 case IXGBE_LINK_SPEED_1GB_FULL: 6818 speed_str = "1 Gbps"; 6819 break; 6820 case IXGBE_LINK_SPEED_100_FULL: 6821 speed_str = "100 Mbps"; 6822 break; 6823 case IXGBE_LINK_SPEED_10_FULL: 6824 speed_str = "10 Mbps"; 6825 break; 6826 default: 6827 speed_str = "unknown speed"; 6828 break; 6829 } 6830 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str, 6831 ((flow_rx && flow_tx) ? "RX/TX" : 6832 (flow_rx ? "RX" : 6833 (flow_tx ? "TX" : "None")))); 6834 6835 netif_carrier_on(netdev); 6836 ixgbe_check_vf_rate_limit(adapter); 6837 6838 /* enable transmits */ 6839 netif_tx_wake_all_queues(adapter->netdev); 6840 6841 /* enable any upper devices */ 6842 rtnl_lock(); 6843 netdev_walk_all_upper_dev_rcu(adapter->netdev, 6844 ixgbe_enable_macvlan, NULL); 6845 rtnl_unlock(); 6846 6847 /* update the default user priority for VFs */ 6848 ixgbe_update_default_up(adapter); 6849 6850 /* ping all the active vfs to let them know link has changed */ 6851 ixgbe_ping_all_vfs(adapter); 6852 } 6853 6854 /** 6855 * ixgbe_watchdog_link_is_down - update netif_carrier status and 6856 * print link down message 6857 * @adapter: pointer to the adapter structure 6858 **/ 6859 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter) 6860 { 6861 struct net_device *netdev = adapter->netdev; 6862 struct ixgbe_hw *hw = &adapter->hw; 6863 6864 adapter->link_up = false; 6865 adapter->link_speed = 0; 6866 6867 /* only continue if link was up previously */ 6868 if (!netif_carrier_ok(netdev)) 6869 return; 6870 6871 /* poll for SFP+ cable when link is down */ 6872 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB) 6873 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 6874 6875 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) 6876 ixgbe_ptp_start_cyclecounter(adapter); 6877 6878 e_info(drv, "NIC Link is Down\n"); 6879 netif_carrier_off(netdev); 6880 6881 /* ping all the active vfs to let them know link has changed */ 6882 ixgbe_ping_all_vfs(adapter); 6883 } 6884 6885 static bool ixgbe_ring_tx_pending(struct ixgbe_adapter *adapter) 6886 { 6887 int i; 6888 6889 for (i = 0; i < adapter->num_tx_queues; i++) { 6890 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 6891 6892 if (tx_ring->next_to_use != tx_ring->next_to_clean) 6893 return true; 6894 } 6895 6896 return false; 6897 } 6898 6899 static bool ixgbe_vf_tx_pending(struct ixgbe_adapter *adapter) 6900 { 6901 struct ixgbe_hw *hw = &adapter->hw; 6902 struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ]; 6903 u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask); 6904 6905 int i, j; 6906 6907 if (!adapter->num_vfs) 6908 return false; 6909 6910 /* resetting the PF is only needed for MAC before X550 */ 6911 if (hw->mac.type >= ixgbe_mac_X550) 6912 return false; 6913 6914 for (i = 0; i < adapter->num_vfs; i++) { 6915 for (j = 0; j < q_per_pool; j++) { 6916 u32 h, t; 6917 6918 h = IXGBE_READ_REG(hw, IXGBE_PVFTDHN(q_per_pool, i, j)); 6919 t = IXGBE_READ_REG(hw, IXGBE_PVFTDTN(q_per_pool, i, j)); 6920 6921 if (h != t) 6922 return true; 6923 } 6924 } 6925 6926 return false; 6927 } 6928 6929 /** 6930 * ixgbe_watchdog_flush_tx - flush queues on link down 6931 * @adapter: pointer to the device adapter structure 6932 **/ 6933 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) 6934 { 6935 if (!netif_carrier_ok(adapter->netdev)) { 6936 if (ixgbe_ring_tx_pending(adapter) || 6937 ixgbe_vf_tx_pending(adapter)) { 6938 /* We've lost link, so the controller stops DMA, 6939 * but we've got queued Tx work that's never going 6940 * to get done, so reset controller to flush Tx. 6941 * (Do the reset outside of interrupt context). 6942 */ 6943 e_warn(drv, "initiating reset to clear Tx work after link loss\n"); 6944 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state); 6945 } 6946 } 6947 } 6948 6949 #ifdef CONFIG_PCI_IOV 6950 static inline void ixgbe_issue_vf_flr(struct ixgbe_adapter *adapter, 6951 struct pci_dev *vfdev) 6952 { 6953 if (!pci_wait_for_pending_transaction(vfdev)) 6954 e_dev_warn("Issuing VFLR with pending transactions\n"); 6955 6956 e_dev_err("Issuing VFLR for VF %s\n", pci_name(vfdev)); 6957 pcie_capability_set_word(vfdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 6958 6959 msleep(100); 6960 } 6961 6962 static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) 6963 { 6964 struct ixgbe_hw *hw = &adapter->hw; 6965 struct pci_dev *pdev = adapter->pdev; 6966 unsigned int vf; 6967 u32 gpc; 6968 6969 if (!(netif_carrier_ok(adapter->netdev))) 6970 return; 6971 6972 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC); 6973 if (gpc) /* If incrementing then no need for the check below */ 6974 return; 6975 /* Check to see if a bad DMA write target from an errant or 6976 * malicious VF has caused a PCIe error. If so then we can 6977 * issue a VFLR to the offending VF(s) and then resume without 6978 * requesting a full slot reset. 6979 */ 6980 6981 if (!pdev) 6982 return; 6983 6984 /* check status reg for all VFs owned by this PF */ 6985 for (vf = 0; vf < adapter->num_vfs; ++vf) { 6986 struct pci_dev *vfdev = adapter->vfinfo[vf].vfdev; 6987 u16 status_reg; 6988 6989 if (!vfdev) 6990 continue; 6991 pci_read_config_word(vfdev, PCI_STATUS, &status_reg); 6992 if (status_reg != IXGBE_FAILED_READ_CFG_WORD && 6993 status_reg & PCI_STATUS_REC_MASTER_ABORT) 6994 ixgbe_issue_vf_flr(adapter, vfdev); 6995 } 6996 } 6997 6998 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) 6999 { 7000 u32 ssvpc; 7001 7002 /* Do not perform spoof check for 82598 or if not in IOV mode */ 7003 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 7004 adapter->num_vfs == 0) 7005 return; 7006 7007 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); 7008 7009 /* 7010 * ssvpc register is cleared on read, if zero then no 7011 * spoofed packets in the last interval. 7012 */ 7013 if (!ssvpc) 7014 return; 7015 7016 e_warn(drv, "%u Spoofed packets detected\n", ssvpc); 7017 } 7018 #else 7019 static void ixgbe_spoof_check(struct ixgbe_adapter __always_unused *adapter) 7020 { 7021 } 7022 7023 static void 7024 ixgbe_check_for_bad_vf(struct ixgbe_adapter __always_unused *adapter) 7025 { 7026 } 7027 #endif /* CONFIG_PCI_IOV */ 7028 7029 7030 /** 7031 * ixgbe_watchdog_subtask - check and bring link up 7032 * @adapter: pointer to the device adapter structure 7033 **/ 7034 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) 7035 { 7036 /* if interface is down, removing or resetting, do nothing */ 7037 if (test_bit(__IXGBE_DOWN, &adapter->state) || 7038 test_bit(__IXGBE_REMOVING, &adapter->state) || 7039 test_bit(__IXGBE_RESETTING, &adapter->state)) 7040 return; 7041 7042 ixgbe_watchdog_update_link(adapter); 7043 7044 if (adapter->link_up) 7045 ixgbe_watchdog_link_is_up(adapter); 7046 else 7047 ixgbe_watchdog_link_is_down(adapter); 7048 7049 ixgbe_check_for_bad_vf(adapter); 7050 ixgbe_spoof_check(adapter); 7051 ixgbe_update_stats(adapter); 7052 7053 ixgbe_watchdog_flush_tx(adapter); 7054 } 7055 7056 /** 7057 * ixgbe_sfp_detection_subtask - poll for SFP+ cable 7058 * @adapter: the ixgbe adapter structure 7059 **/ 7060 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) 7061 { 7062 struct ixgbe_hw *hw = &adapter->hw; 7063 s32 err; 7064 7065 /* not searching for SFP so there is nothing to do here */ 7066 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && 7067 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 7068 return; 7069 7070 if (adapter->sfp_poll_time && 7071 time_after(adapter->sfp_poll_time, jiffies)) 7072 return; /* If not yet time to poll for SFP */ 7073 7074 /* someone else is in init, wait until next service event */ 7075 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 7076 return; 7077 7078 adapter->sfp_poll_time = jiffies + IXGBE_SFP_POLL_JIFFIES - 1; 7079 7080 err = hw->phy.ops.identify_sfp(hw); 7081 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 7082 goto sfp_out; 7083 7084 if (err == IXGBE_ERR_SFP_NOT_PRESENT) { 7085 /* If no cable is present, then we need to reset 7086 * the next time we find a good cable. */ 7087 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 7088 } 7089 7090 /* exit on error */ 7091 if (err) 7092 goto sfp_out; 7093 7094 /* exit if reset not needed */ 7095 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 7096 goto sfp_out; 7097 7098 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET; 7099 7100 /* 7101 * A module may be identified correctly, but the EEPROM may not have 7102 * support for that module. setup_sfp() will fail in that case, so 7103 * we should not allow that module to load. 7104 */ 7105 if (hw->mac.type == ixgbe_mac_82598EB) 7106 err = hw->phy.ops.reset(hw); 7107 else 7108 err = hw->mac.ops.setup_sfp(hw); 7109 7110 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 7111 goto sfp_out; 7112 7113 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 7114 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type); 7115 7116 sfp_out: 7117 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 7118 7119 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) && 7120 (adapter->netdev->reg_state == NETREG_REGISTERED)) { 7121 e_dev_err("failed to initialize because an unsupported " 7122 "SFP+ module type was detected.\n"); 7123 e_dev_err("Reload the driver after installing a " 7124 "supported module.\n"); 7125 unregister_netdev(adapter->netdev); 7126 } 7127 } 7128 7129 /** 7130 * ixgbe_sfp_link_config_subtask - set up link SFP after module install 7131 * @adapter: the ixgbe adapter structure 7132 **/ 7133 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) 7134 { 7135 struct ixgbe_hw *hw = &adapter->hw; 7136 u32 speed; 7137 bool autoneg = false; 7138 7139 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG)) 7140 return; 7141 7142 /* someone else is in init, wait until next service event */ 7143 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 7144 return; 7145 7146 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 7147 7148 speed = hw->phy.autoneg_advertised; 7149 if ((!speed) && (hw->mac.ops.get_link_capabilities)) { 7150 hw->mac.ops.get_link_capabilities(hw, &speed, &autoneg); 7151 7152 /* setup the highest link when no autoneg */ 7153 if (!autoneg) { 7154 if (speed & IXGBE_LINK_SPEED_10GB_FULL) 7155 speed = IXGBE_LINK_SPEED_10GB_FULL; 7156 } 7157 } 7158 7159 if (hw->mac.ops.setup_link) 7160 hw->mac.ops.setup_link(hw, speed, true); 7161 7162 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 7163 adapter->link_check_timeout = jiffies; 7164 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 7165 } 7166 7167 /** 7168 * ixgbe_service_timer - Timer Call-back 7169 * @data: pointer to adapter cast into an unsigned long 7170 **/ 7171 static void ixgbe_service_timer(unsigned long data) 7172 { 7173 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data; 7174 unsigned long next_event_offset; 7175 7176 /* poll faster when waiting for link */ 7177 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 7178 next_event_offset = HZ / 10; 7179 else 7180 next_event_offset = HZ * 2; 7181 7182 /* Reset the timer */ 7183 mod_timer(&adapter->service_timer, next_event_offset + jiffies); 7184 7185 ixgbe_service_event_schedule(adapter); 7186 } 7187 7188 static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter) 7189 { 7190 struct ixgbe_hw *hw = &adapter->hw; 7191 u32 status; 7192 7193 if (!(adapter->flags2 & IXGBE_FLAG2_PHY_INTERRUPT)) 7194 return; 7195 7196 adapter->flags2 &= ~IXGBE_FLAG2_PHY_INTERRUPT; 7197 7198 if (!hw->phy.ops.handle_lasi) 7199 return; 7200 7201 status = hw->phy.ops.handle_lasi(&adapter->hw); 7202 if (status != IXGBE_ERR_OVERTEMP) 7203 return; 7204 7205 e_crit(drv, "%s\n", ixgbe_overheat_msg); 7206 } 7207 7208 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) 7209 { 7210 if (!test_and_clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state)) 7211 return; 7212 7213 /* If we're already down, removing or resetting, just bail */ 7214 if (test_bit(__IXGBE_DOWN, &adapter->state) || 7215 test_bit(__IXGBE_REMOVING, &adapter->state) || 7216 test_bit(__IXGBE_RESETTING, &adapter->state)) 7217 return; 7218 7219 ixgbe_dump(adapter); 7220 netdev_err(adapter->netdev, "Reset adapter\n"); 7221 adapter->tx_timeout_count++; 7222 7223 rtnl_lock(); 7224 ixgbe_reinit_locked(adapter); 7225 rtnl_unlock(); 7226 } 7227 7228 /** 7229 * ixgbe_service_task - manages and runs subtasks 7230 * @work: pointer to work_struct containing our data 7231 **/ 7232 static void ixgbe_service_task(struct work_struct *work) 7233 { 7234 struct ixgbe_adapter *adapter = container_of(work, 7235 struct ixgbe_adapter, 7236 service_task); 7237 if (ixgbe_removed(adapter->hw.hw_addr)) { 7238 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 7239 rtnl_lock(); 7240 ixgbe_down(adapter); 7241 rtnl_unlock(); 7242 } 7243 ixgbe_service_event_complete(adapter); 7244 return; 7245 } 7246 if (adapter->flags2 & IXGBE_FLAG2_UDP_TUN_REREG_NEEDED) { 7247 rtnl_lock(); 7248 adapter->flags2 &= ~IXGBE_FLAG2_UDP_TUN_REREG_NEEDED; 7249 udp_tunnel_get_rx_info(adapter->netdev); 7250 rtnl_unlock(); 7251 } 7252 ixgbe_reset_subtask(adapter); 7253 ixgbe_phy_interrupt_subtask(adapter); 7254 ixgbe_sfp_detection_subtask(adapter); 7255 ixgbe_sfp_link_config_subtask(adapter); 7256 ixgbe_check_overtemp_subtask(adapter); 7257 ixgbe_watchdog_subtask(adapter); 7258 ixgbe_fdir_reinit_subtask(adapter); 7259 ixgbe_check_hang_subtask(adapter); 7260 7261 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) { 7262 ixgbe_ptp_overflow_check(adapter); 7263 ixgbe_ptp_rx_hang(adapter); 7264 } 7265 7266 ixgbe_service_event_complete(adapter); 7267 } 7268 7269 static int ixgbe_tso(struct ixgbe_ring *tx_ring, 7270 struct ixgbe_tx_buffer *first, 7271 u8 *hdr_len) 7272 { 7273 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 7274 struct sk_buff *skb = first->skb; 7275 union { 7276 struct iphdr *v4; 7277 struct ipv6hdr *v6; 7278 unsigned char *hdr; 7279 } ip; 7280 union { 7281 struct tcphdr *tcp; 7282 unsigned char *hdr; 7283 } l4; 7284 u32 paylen, l4_offset; 7285 int err; 7286 7287 if (skb->ip_summed != CHECKSUM_PARTIAL) 7288 return 0; 7289 7290 if (!skb_is_gso(skb)) 7291 return 0; 7292 7293 err = skb_cow_head(skb, 0); 7294 if (err < 0) 7295 return err; 7296 7297 ip.hdr = skb_network_header(skb); 7298 l4.hdr = skb_checksum_start(skb); 7299 7300 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 7301 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 7302 7303 /* initialize outer IP header fields */ 7304 if (ip.v4->version == 4) { 7305 unsigned char *csum_start = skb_checksum_start(skb); 7306 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 7307 7308 /* IP header will have to cancel out any data that 7309 * is not a part of the outer IP header 7310 */ 7311 ip.v4->check = csum_fold(csum_partial(trans_start, 7312 csum_start - trans_start, 7313 0)); 7314 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 7315 7316 ip.v4->tot_len = 0; 7317 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 7318 IXGBE_TX_FLAGS_CSUM | 7319 IXGBE_TX_FLAGS_IPV4; 7320 } else { 7321 ip.v6->payload_len = 0; 7322 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 7323 IXGBE_TX_FLAGS_CSUM; 7324 } 7325 7326 /* determine offset of inner transport header */ 7327 l4_offset = l4.hdr - skb->data; 7328 7329 /* compute length of segmentation header */ 7330 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 7331 7332 /* remove payload length from inner checksum */ 7333 paylen = skb->len - l4_offset; 7334 csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); 7335 7336 /* update gso size and bytecount with header size */ 7337 first->gso_segs = skb_shinfo(skb)->gso_segs; 7338 first->bytecount += (first->gso_segs - 1) * *hdr_len; 7339 7340 /* mss_l4len_id: use 0 as index for TSO */ 7341 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT; 7342 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 7343 7344 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 7345 vlan_macip_lens = l4.hdr - ip.hdr; 7346 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT; 7347 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 7348 7349 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 7350 mss_l4len_idx); 7351 7352 return 1; 7353 } 7354 7355 static inline bool ixgbe_ipv6_csum_is_sctp(struct sk_buff *skb) 7356 { 7357 unsigned int offset = 0; 7358 7359 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 7360 7361 return offset == skb_checksum_start_offset(skb); 7362 } 7363 7364 static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, 7365 struct ixgbe_tx_buffer *first) 7366 { 7367 struct sk_buff *skb = first->skb; 7368 u32 vlan_macip_lens = 0; 7369 u32 type_tucmd = 0; 7370 7371 if (skb->ip_summed != CHECKSUM_PARTIAL) { 7372 csum_failed: 7373 if (!(first->tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | 7374 IXGBE_TX_FLAGS_CC))) 7375 return; 7376 goto no_csum; 7377 } 7378 7379 switch (skb->csum_offset) { 7380 case offsetof(struct tcphdr, check): 7381 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 7382 /* fall through */ 7383 case offsetof(struct udphdr, check): 7384 break; 7385 case offsetof(struct sctphdr, checksum): 7386 /* validate that this is actually an SCTP request */ 7387 if (((first->protocol == htons(ETH_P_IP)) && 7388 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 7389 ((first->protocol == htons(ETH_P_IPV6)) && 7390 ixgbe_ipv6_csum_is_sctp(skb))) { 7391 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP; 7392 break; 7393 } 7394 /* fall through */ 7395 default: 7396 skb_checksum_help(skb); 7397 goto csum_failed; 7398 } 7399 7400 /* update TX checksum flag */ 7401 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 7402 vlan_macip_lens = skb_checksum_start_offset(skb) - 7403 skb_network_offset(skb); 7404 no_csum: 7405 /* vlan_macip_lens: MACLEN, VLAN tag */ 7406 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 7407 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 7408 7409 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 0); 7410 } 7411 7412 #define IXGBE_SET_FLAG(_input, _flag, _result) \ 7413 ((_flag <= _result) ? \ 7414 ((u32)(_input & _flag) * (_result / _flag)) : \ 7415 ((u32)(_input & _flag) / (_flag / _result))) 7416 7417 static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 7418 { 7419 /* set type for advanced descriptor with frame checksum insertion */ 7420 u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | 7421 IXGBE_ADVTXD_DCMD_DEXT | 7422 IXGBE_ADVTXD_DCMD_IFCS; 7423 7424 /* set HW vlan bit if vlan is present */ 7425 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN, 7426 IXGBE_ADVTXD_DCMD_VLE); 7427 7428 /* set segmentation enable bits for TSO/FSO */ 7429 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO, 7430 IXGBE_ADVTXD_DCMD_TSE); 7431 7432 /* set timestamp bit if present */ 7433 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP, 7434 IXGBE_ADVTXD_MAC_TSTAMP); 7435 7436 /* insert frame checksum */ 7437 cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS); 7438 7439 return cmd_type; 7440 } 7441 7442 static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 7443 u32 tx_flags, unsigned int paylen) 7444 { 7445 u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 7446 7447 /* enable L4 checksum for TSO and TX checksum offload */ 7448 olinfo_status |= IXGBE_SET_FLAG(tx_flags, 7449 IXGBE_TX_FLAGS_CSUM, 7450 IXGBE_ADVTXD_POPTS_TXSM); 7451 7452 /* enble IPv4 checksum for TSO */ 7453 olinfo_status |= IXGBE_SET_FLAG(tx_flags, 7454 IXGBE_TX_FLAGS_IPV4, 7455 IXGBE_ADVTXD_POPTS_IXSM); 7456 7457 /* 7458 * Check Context must be set if Tx switch is enabled, which it 7459 * always is for case where virtual functions are running 7460 */ 7461 olinfo_status |= IXGBE_SET_FLAG(tx_flags, 7462 IXGBE_TX_FLAGS_CC, 7463 IXGBE_ADVTXD_CC); 7464 7465 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 7466 } 7467 7468 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 7469 { 7470 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 7471 7472 /* Herbert's original patch had: 7473 * smp_mb__after_netif_stop_queue(); 7474 * but since that doesn't exist yet, just open code it. 7475 */ 7476 smp_mb(); 7477 7478 /* We need to check again in a case another CPU has just 7479 * made room available. 7480 */ 7481 if (likely(ixgbe_desc_unused(tx_ring) < size)) 7482 return -EBUSY; 7483 7484 /* A reprieve! - use start_queue because it doesn't call schedule */ 7485 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 7486 ++tx_ring->tx_stats.restart_queue; 7487 return 0; 7488 } 7489 7490 static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 7491 { 7492 if (likely(ixgbe_desc_unused(tx_ring) >= size)) 7493 return 0; 7494 7495 return __ixgbe_maybe_stop_tx(tx_ring, size); 7496 } 7497 7498 #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ 7499 IXGBE_TXD_CMD_RS) 7500 7501 static void ixgbe_tx_map(struct ixgbe_ring *tx_ring, 7502 struct ixgbe_tx_buffer *first, 7503 const u8 hdr_len) 7504 { 7505 struct sk_buff *skb = first->skb; 7506 struct ixgbe_tx_buffer *tx_buffer; 7507 union ixgbe_adv_tx_desc *tx_desc; 7508 struct skb_frag_struct *frag; 7509 dma_addr_t dma; 7510 unsigned int data_len, size; 7511 u32 tx_flags = first->tx_flags; 7512 u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags); 7513 u16 i = tx_ring->next_to_use; 7514 7515 tx_desc = IXGBE_TX_DESC(tx_ring, i); 7516 7517 ixgbe_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len); 7518 7519 size = skb_headlen(skb); 7520 data_len = skb->data_len; 7521 7522 #ifdef IXGBE_FCOE 7523 if (tx_flags & IXGBE_TX_FLAGS_FCOE) { 7524 if (data_len < sizeof(struct fcoe_crc_eof)) { 7525 size -= sizeof(struct fcoe_crc_eof) - data_len; 7526 data_len = 0; 7527 } else { 7528 data_len -= sizeof(struct fcoe_crc_eof); 7529 } 7530 } 7531 7532 #endif 7533 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 7534 7535 tx_buffer = first; 7536 7537 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 7538 if (dma_mapping_error(tx_ring->dev, dma)) 7539 goto dma_error; 7540 7541 /* record length, and DMA address */ 7542 dma_unmap_len_set(tx_buffer, len, size); 7543 dma_unmap_addr_set(tx_buffer, dma, dma); 7544 7545 tx_desc->read.buffer_addr = cpu_to_le64(dma); 7546 7547 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 7548 tx_desc->read.cmd_type_len = 7549 cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD); 7550 7551 i++; 7552 tx_desc++; 7553 if (i == tx_ring->count) { 7554 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 7555 i = 0; 7556 } 7557 tx_desc->read.olinfo_status = 0; 7558 7559 dma += IXGBE_MAX_DATA_PER_TXD; 7560 size -= IXGBE_MAX_DATA_PER_TXD; 7561 7562 tx_desc->read.buffer_addr = cpu_to_le64(dma); 7563 } 7564 7565 if (likely(!data_len)) 7566 break; 7567 7568 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 7569 7570 i++; 7571 tx_desc++; 7572 if (i == tx_ring->count) { 7573 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 7574 i = 0; 7575 } 7576 tx_desc->read.olinfo_status = 0; 7577 7578 #ifdef IXGBE_FCOE 7579 size = min_t(unsigned int, data_len, skb_frag_size(frag)); 7580 #else 7581 size = skb_frag_size(frag); 7582 #endif 7583 data_len -= size; 7584 7585 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 7586 DMA_TO_DEVICE); 7587 7588 tx_buffer = &tx_ring->tx_buffer_info[i]; 7589 } 7590 7591 /* write last descriptor with RS and EOP bits */ 7592 cmd_type |= size | IXGBE_TXD_CMD; 7593 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 7594 7595 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 7596 7597 /* set the timestamp */ 7598 first->time_stamp = jiffies; 7599 7600 /* 7601 * Force memory writes to complete before letting h/w know there 7602 * are new descriptors to fetch. (Only applicable for weak-ordered 7603 * memory model archs, such as IA-64). 7604 * 7605 * We also need this memory barrier to make certain all of the 7606 * status bits have been updated before next_to_watch is written. 7607 */ 7608 wmb(); 7609 7610 /* set next_to_watch value indicating a packet is present */ 7611 first->next_to_watch = tx_desc; 7612 7613 i++; 7614 if (i == tx_ring->count) 7615 i = 0; 7616 7617 tx_ring->next_to_use = i; 7618 7619 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED); 7620 7621 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 7622 writel(i, tx_ring->tail); 7623 7624 /* we need this if more than one processor can write to our tail 7625 * at a time, it synchronizes IO on IA64/Altix systems 7626 */ 7627 mmiowb(); 7628 } 7629 7630 return; 7631 dma_error: 7632 dev_err(tx_ring->dev, "TX DMA map failed\n"); 7633 7634 /* clear dma mappings for failed tx_buffer_info map */ 7635 for (;;) { 7636 tx_buffer = &tx_ring->tx_buffer_info[i]; 7637 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer); 7638 if (tx_buffer == first) 7639 break; 7640 if (i == 0) 7641 i = tx_ring->count; 7642 i--; 7643 } 7644 7645 tx_ring->next_to_use = i; 7646 } 7647 7648 static void ixgbe_atr(struct ixgbe_ring *ring, 7649 struct ixgbe_tx_buffer *first) 7650 { 7651 struct ixgbe_q_vector *q_vector = ring->q_vector; 7652 union ixgbe_atr_hash_dword input = { .dword = 0 }; 7653 union ixgbe_atr_hash_dword common = { .dword = 0 }; 7654 union { 7655 unsigned char *network; 7656 struct iphdr *ipv4; 7657 struct ipv6hdr *ipv6; 7658 } hdr; 7659 struct tcphdr *th; 7660 unsigned int hlen; 7661 struct sk_buff *skb; 7662 __be16 vlan_id; 7663 int l4_proto; 7664 7665 /* if ring doesn't have a interrupt vector, cannot perform ATR */ 7666 if (!q_vector) 7667 return; 7668 7669 /* do nothing if sampling is disabled */ 7670 if (!ring->atr_sample_rate) 7671 return; 7672 7673 ring->atr_count++; 7674 7675 /* currently only IPv4/IPv6 with TCP is supported */ 7676 if ((first->protocol != htons(ETH_P_IP)) && 7677 (first->protocol != htons(ETH_P_IPV6))) 7678 return; 7679 7680 /* snag network header to get L4 type and address */ 7681 skb = first->skb; 7682 hdr.network = skb_network_header(skb); 7683 if (unlikely(hdr.network <= skb->data)) 7684 return; 7685 if (skb->encapsulation && 7686 first->protocol == htons(ETH_P_IP) && 7687 hdr.ipv4->protocol == IPPROTO_UDP) { 7688 struct ixgbe_adapter *adapter = q_vector->adapter; 7689 7690 if (unlikely(skb_tail_pointer(skb) < hdr.network + 7691 VXLAN_HEADROOM)) 7692 return; 7693 7694 /* verify the port is recognized as VXLAN */ 7695 if (adapter->vxlan_port && 7696 udp_hdr(skb)->dest == adapter->vxlan_port) 7697 hdr.network = skb_inner_network_header(skb); 7698 7699 if (adapter->geneve_port && 7700 udp_hdr(skb)->dest == adapter->geneve_port) 7701 hdr.network = skb_inner_network_header(skb); 7702 } 7703 7704 /* Make sure we have at least [minimum IPv4 header + TCP] 7705 * or [IPv6 header] bytes 7706 */ 7707 if (unlikely(skb_tail_pointer(skb) < hdr.network + 40)) 7708 return; 7709 7710 /* Currently only IPv4/IPv6 with TCP is supported */ 7711 switch (hdr.ipv4->version) { 7712 case IPVERSION: 7713 /* access ihl as u8 to avoid unaligned access on ia64 */ 7714 hlen = (hdr.network[0] & 0x0F) << 2; 7715 l4_proto = hdr.ipv4->protocol; 7716 break; 7717 case 6: 7718 hlen = hdr.network - skb->data; 7719 l4_proto = ipv6_find_hdr(skb, &hlen, IPPROTO_TCP, NULL, NULL); 7720 hlen -= hdr.network - skb->data; 7721 break; 7722 default: 7723 return; 7724 } 7725 7726 if (l4_proto != IPPROTO_TCP) 7727 return; 7728 7729 if (unlikely(skb_tail_pointer(skb) < hdr.network + 7730 hlen + sizeof(struct tcphdr))) 7731 return; 7732 7733 th = (struct tcphdr *)(hdr.network + hlen); 7734 7735 /* skip this packet since the socket is closing */ 7736 if (th->fin) 7737 return; 7738 7739 /* sample on all syn packets or once every atr sample count */ 7740 if (!th->syn && (ring->atr_count < ring->atr_sample_rate)) 7741 return; 7742 7743 /* reset sample count */ 7744 ring->atr_count = 0; 7745 7746 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT); 7747 7748 /* 7749 * src and dst are inverted, think how the receiver sees them 7750 * 7751 * The input is broken into two sections, a non-compressed section 7752 * containing vm_pool, vlan_id, and flow_type. The rest of the data 7753 * is XORed together and stored in the compressed dword. 7754 */ 7755 input.formatted.vlan_id = vlan_id; 7756 7757 /* 7758 * since src port and flex bytes occupy the same word XOR them together 7759 * and write the value to source port portion of compressed dword 7760 */ 7761 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN)) 7762 common.port.src ^= th->dest ^ htons(ETH_P_8021Q); 7763 else 7764 common.port.src ^= th->dest ^ first->protocol; 7765 common.port.dst ^= th->source; 7766 7767 switch (hdr.ipv4->version) { 7768 case IPVERSION: 7769 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4; 7770 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr; 7771 break; 7772 case 6: 7773 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; 7774 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ 7775 hdr.ipv6->saddr.s6_addr32[1] ^ 7776 hdr.ipv6->saddr.s6_addr32[2] ^ 7777 hdr.ipv6->saddr.s6_addr32[3] ^ 7778 hdr.ipv6->daddr.s6_addr32[0] ^ 7779 hdr.ipv6->daddr.s6_addr32[1] ^ 7780 hdr.ipv6->daddr.s6_addr32[2] ^ 7781 hdr.ipv6->daddr.s6_addr32[3]; 7782 break; 7783 default: 7784 break; 7785 } 7786 7787 if (hdr.network != skb_network_header(skb)) 7788 input.formatted.flow_type |= IXGBE_ATR_L4TYPE_TUNNEL_MASK; 7789 7790 /* This assumes the Rx queue and Tx queue are bound to the same CPU */ 7791 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, 7792 input, common, ring->queue_index); 7793 } 7794 7795 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, 7796 void *accel_priv, select_queue_fallback_t fallback) 7797 { 7798 struct ixgbe_fwd_adapter *fwd_adapter = accel_priv; 7799 #ifdef IXGBE_FCOE 7800 struct ixgbe_adapter *adapter; 7801 struct ixgbe_ring_feature *f; 7802 int txq; 7803 #endif 7804 7805 if (fwd_adapter) 7806 return skb->queue_mapping + fwd_adapter->tx_base_queue; 7807 7808 #ifdef IXGBE_FCOE 7809 7810 /* 7811 * only execute the code below if protocol is FCoE 7812 * or FIP and we have FCoE enabled on the adapter 7813 */ 7814 switch (vlan_get_protocol(skb)) { 7815 case htons(ETH_P_FCOE): 7816 case htons(ETH_P_FIP): 7817 adapter = netdev_priv(dev); 7818 7819 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) 7820 break; 7821 default: 7822 return fallback(dev, skb); 7823 } 7824 7825 f = &adapter->ring_feature[RING_F_FCOE]; 7826 7827 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 7828 smp_processor_id(); 7829 7830 while (txq >= f->indices) 7831 txq -= f->indices; 7832 7833 return txq + f->offset; 7834 #else 7835 return fallback(dev, skb); 7836 #endif 7837 } 7838 7839 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, 7840 struct ixgbe_adapter *adapter, 7841 struct ixgbe_ring *tx_ring) 7842 { 7843 struct ixgbe_tx_buffer *first; 7844 int tso; 7845 u32 tx_flags = 0; 7846 unsigned short f; 7847 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 7848 __be16 protocol = skb->protocol; 7849 u8 hdr_len = 0; 7850 7851 /* 7852 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 7853 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 7854 * + 2 desc gap to keep tail from touching head, 7855 * + 1 desc for context descriptor, 7856 * otherwise try next time 7857 */ 7858 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 7859 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 7860 7861 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) { 7862 tx_ring->tx_stats.tx_busy++; 7863 return NETDEV_TX_BUSY; 7864 } 7865 7866 /* record the location of the first descriptor for this packet */ 7867 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 7868 first->skb = skb; 7869 first->bytecount = skb->len; 7870 first->gso_segs = 1; 7871 7872 /* if we have a HW VLAN tag being added default to the HW one */ 7873 if (skb_vlan_tag_present(skb)) { 7874 tx_flags |= skb_vlan_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT; 7875 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 7876 /* else if it is a SW VLAN check the next protocol and store the tag */ 7877 } else if (protocol == htons(ETH_P_8021Q)) { 7878 struct vlan_hdr *vhdr, _vhdr; 7879 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 7880 if (!vhdr) 7881 goto out_drop; 7882 7883 tx_flags |= ntohs(vhdr->h_vlan_TCI) << 7884 IXGBE_TX_FLAGS_VLAN_SHIFT; 7885 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN; 7886 } 7887 protocol = vlan_get_protocol(skb); 7888 7889 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 7890 adapter->ptp_clock && 7891 !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, 7892 &adapter->state)) { 7893 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 7894 tx_flags |= IXGBE_TX_FLAGS_TSTAMP; 7895 7896 /* schedule check for Tx timestamp */ 7897 adapter->ptp_tx_skb = skb_get(skb); 7898 adapter->ptp_tx_start = jiffies; 7899 schedule_work(&adapter->ptp_tx_work); 7900 } 7901 7902 skb_tx_timestamp(skb); 7903 7904 #ifdef CONFIG_PCI_IOV 7905 /* 7906 * Use the l2switch_enable flag - would be false if the DMA 7907 * Tx switch had been disabled. 7908 */ 7909 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 7910 tx_flags |= IXGBE_TX_FLAGS_CC; 7911 7912 #endif 7913 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ 7914 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && 7915 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || 7916 (skb->priority != TC_PRIO_CONTROL))) { 7917 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; 7918 tx_flags |= (skb->priority & 0x7) << 7919 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; 7920 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) { 7921 struct vlan_ethhdr *vhdr; 7922 7923 if (skb_cow_head(skb, 0)) 7924 goto out_drop; 7925 vhdr = (struct vlan_ethhdr *)skb->data; 7926 vhdr->h_vlan_TCI = htons(tx_flags >> 7927 IXGBE_TX_FLAGS_VLAN_SHIFT); 7928 } else { 7929 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 7930 } 7931 } 7932 7933 /* record initial flags and protocol */ 7934 first->tx_flags = tx_flags; 7935 first->protocol = protocol; 7936 7937 #ifdef IXGBE_FCOE 7938 /* setup tx offload for FCoE */ 7939 if ((protocol == htons(ETH_P_FCOE)) && 7940 (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { 7941 tso = ixgbe_fso(tx_ring, first, &hdr_len); 7942 if (tso < 0) 7943 goto out_drop; 7944 7945 goto xmit_fcoe; 7946 } 7947 7948 #endif /* IXGBE_FCOE */ 7949 tso = ixgbe_tso(tx_ring, first, &hdr_len); 7950 if (tso < 0) 7951 goto out_drop; 7952 else if (!tso) 7953 ixgbe_tx_csum(tx_ring, first); 7954 7955 /* add the ATR filter if ATR is on */ 7956 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state)) 7957 ixgbe_atr(tx_ring, first); 7958 7959 #ifdef IXGBE_FCOE 7960 xmit_fcoe: 7961 #endif /* IXGBE_FCOE */ 7962 ixgbe_tx_map(tx_ring, first, hdr_len); 7963 7964 return NETDEV_TX_OK; 7965 7966 out_drop: 7967 dev_kfree_skb_any(first->skb); 7968 first->skb = NULL; 7969 7970 return NETDEV_TX_OK; 7971 } 7972 7973 static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, 7974 struct net_device *netdev, 7975 struct ixgbe_ring *ring) 7976 { 7977 struct ixgbe_adapter *adapter = netdev_priv(netdev); 7978 struct ixgbe_ring *tx_ring; 7979 7980 /* 7981 * The minimum packet size for olinfo paylen is 17 so pad the skb 7982 * in order to meet this minimum size requirement. 7983 */ 7984 if (skb_put_padto(skb, 17)) 7985 return NETDEV_TX_OK; 7986 7987 tx_ring = ring ? ring : adapter->tx_ring[skb->queue_mapping]; 7988 7989 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); 7990 } 7991 7992 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, 7993 struct net_device *netdev) 7994 { 7995 return __ixgbe_xmit_frame(skb, netdev, NULL); 7996 } 7997 7998 /** 7999 * ixgbe_set_mac - Change the Ethernet Address of the NIC 8000 * @netdev: network interface device structure 8001 * @p: pointer to an address structure 8002 * 8003 * Returns 0 on success, negative on failure 8004 **/ 8005 static int ixgbe_set_mac(struct net_device *netdev, void *p) 8006 { 8007 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8008 struct ixgbe_hw *hw = &adapter->hw; 8009 struct sockaddr *addr = p; 8010 8011 if (!is_valid_ether_addr(addr->sa_data)) 8012 return -EADDRNOTAVAIL; 8013 8014 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 8015 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 8016 8017 ixgbe_mac_set_default_filter(adapter); 8018 8019 return 0; 8020 } 8021 8022 static int 8023 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) 8024 { 8025 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8026 struct ixgbe_hw *hw = &adapter->hw; 8027 u16 value; 8028 int rc; 8029 8030 if (prtad != hw->phy.mdio.prtad) 8031 return -EINVAL; 8032 rc = hw->phy.ops.read_reg(hw, addr, devad, &value); 8033 if (!rc) 8034 rc = value; 8035 return rc; 8036 } 8037 8038 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, 8039 u16 addr, u16 value) 8040 { 8041 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8042 struct ixgbe_hw *hw = &adapter->hw; 8043 8044 if (prtad != hw->phy.mdio.prtad) 8045 return -EINVAL; 8046 return hw->phy.ops.write_reg(hw, addr, devad, value); 8047 } 8048 8049 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) 8050 { 8051 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8052 8053 switch (cmd) { 8054 case SIOCSHWTSTAMP: 8055 return ixgbe_ptp_set_ts_config(adapter, req); 8056 case SIOCGHWTSTAMP: 8057 return ixgbe_ptp_get_ts_config(adapter, req); 8058 default: 8059 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd); 8060 } 8061 } 8062 8063 /** 8064 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding 8065 * netdev->dev_addrs 8066 * @netdev: network interface device structure 8067 * 8068 * Returns non-zero on failure 8069 **/ 8070 static int ixgbe_add_sanmac_netdev(struct net_device *dev) 8071 { 8072 int err = 0; 8073 struct ixgbe_adapter *adapter = netdev_priv(dev); 8074 struct ixgbe_hw *hw = &adapter->hw; 8075 8076 if (is_valid_ether_addr(hw->mac.san_addr)) { 8077 rtnl_lock(); 8078 err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN); 8079 rtnl_unlock(); 8080 8081 /* update SAN MAC vmdq pool selection */ 8082 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0)); 8083 } 8084 return err; 8085 } 8086 8087 /** 8088 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding 8089 * netdev->dev_addrs 8090 * @netdev: network interface device structure 8091 * 8092 * Returns non-zero on failure 8093 **/ 8094 static int ixgbe_del_sanmac_netdev(struct net_device *dev) 8095 { 8096 int err = 0; 8097 struct ixgbe_adapter *adapter = netdev_priv(dev); 8098 struct ixgbe_mac_info *mac = &adapter->hw.mac; 8099 8100 if (is_valid_ether_addr(mac->san_addr)) { 8101 rtnl_lock(); 8102 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); 8103 rtnl_unlock(); 8104 } 8105 return err; 8106 } 8107 8108 #ifdef CONFIG_NET_POLL_CONTROLLER 8109 /* 8110 * Polling 'interrupt' - used by things like netconsole to send skbs 8111 * without having to re-enable interrupts. It's not called while 8112 * the interrupt routine is executing. 8113 */ 8114 static void ixgbe_netpoll(struct net_device *netdev) 8115 { 8116 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8117 int i; 8118 8119 /* if interface is down do nothing */ 8120 if (test_bit(__IXGBE_DOWN, &adapter->state)) 8121 return; 8122 8123 /* loop through and schedule all active queues */ 8124 for (i = 0; i < adapter->num_q_vectors; i++) 8125 ixgbe_msix_clean_rings(0, adapter->q_vector[i]); 8126 } 8127 8128 #endif 8129 8130 static void ixgbe_get_stats64(struct net_device *netdev, 8131 struct rtnl_link_stats64 *stats) 8132 { 8133 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8134 int i; 8135 8136 rcu_read_lock(); 8137 for (i = 0; i < adapter->num_rx_queues; i++) { 8138 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]); 8139 u64 bytes, packets; 8140 unsigned int start; 8141 8142 if (ring) { 8143 do { 8144 start = u64_stats_fetch_begin_irq(&ring->syncp); 8145 packets = ring->stats.packets; 8146 bytes = ring->stats.bytes; 8147 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 8148 stats->rx_packets += packets; 8149 stats->rx_bytes += bytes; 8150 } 8151 } 8152 8153 for (i = 0; i < adapter->num_tx_queues; i++) { 8154 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]); 8155 u64 bytes, packets; 8156 unsigned int start; 8157 8158 if (ring) { 8159 do { 8160 start = u64_stats_fetch_begin_irq(&ring->syncp); 8161 packets = ring->stats.packets; 8162 bytes = ring->stats.bytes; 8163 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 8164 stats->tx_packets += packets; 8165 stats->tx_bytes += bytes; 8166 } 8167 } 8168 rcu_read_unlock(); 8169 8170 /* following stats updated by ixgbe_watchdog_task() */ 8171 stats->multicast = netdev->stats.multicast; 8172 stats->rx_errors = netdev->stats.rx_errors; 8173 stats->rx_length_errors = netdev->stats.rx_length_errors; 8174 stats->rx_crc_errors = netdev->stats.rx_crc_errors; 8175 stats->rx_missed_errors = netdev->stats.rx_missed_errors; 8176 } 8177 8178 #ifdef CONFIG_IXGBE_DCB 8179 /** 8180 * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid. 8181 * @adapter: pointer to ixgbe_adapter 8182 * @tc: number of traffic classes currently enabled 8183 * 8184 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm 8185 * 802.1Q priority maps to a packet buffer that exists. 8186 */ 8187 static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc) 8188 { 8189 struct ixgbe_hw *hw = &adapter->hw; 8190 u32 reg, rsave; 8191 int i; 8192 8193 /* 82598 have a static priority to TC mapping that can not 8194 * be changed so no validation is needed. 8195 */ 8196 if (hw->mac.type == ixgbe_mac_82598EB) 8197 return; 8198 8199 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC); 8200 rsave = reg; 8201 8202 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) { 8203 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT); 8204 8205 /* If up2tc is out of bounds default to zero */ 8206 if (up2tc > tc) 8207 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT); 8208 } 8209 8210 if (reg != rsave) 8211 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg); 8212 8213 return; 8214 } 8215 8216 /** 8217 * ixgbe_set_prio_tc_map - Configure netdev prio tc map 8218 * @adapter: Pointer to adapter struct 8219 * 8220 * Populate the netdev user priority to tc map 8221 */ 8222 static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) 8223 { 8224 struct net_device *dev = adapter->netdev; 8225 struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg; 8226 struct ieee_ets *ets = adapter->ixgbe_ieee_ets; 8227 u8 prio; 8228 8229 for (prio = 0; prio < MAX_USER_PRIORITY; prio++) { 8230 u8 tc = 0; 8231 8232 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) 8233 tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio); 8234 else if (ets) 8235 tc = ets->prio_tc[prio]; 8236 8237 netdev_set_prio_tc_map(dev, prio, tc); 8238 } 8239 } 8240 8241 #endif /* CONFIG_IXGBE_DCB */ 8242 /** 8243 * ixgbe_setup_tc - configure net_device for multiple traffic classes 8244 * 8245 * @netdev: net device to configure 8246 * @tc: number of traffic classes to enable 8247 */ 8248 int ixgbe_setup_tc(struct net_device *dev, u8 tc) 8249 { 8250 struct ixgbe_adapter *adapter = netdev_priv(dev); 8251 struct ixgbe_hw *hw = &adapter->hw; 8252 bool pools; 8253 8254 /* Hardware supports up to 8 traffic classes */ 8255 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs) 8256 return -EINVAL; 8257 8258 if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS) 8259 return -EINVAL; 8260 8261 pools = (find_first_zero_bit(&adapter->fwd_bitmask, 32) > 1); 8262 if (tc && pools && adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS) 8263 return -EBUSY; 8264 8265 /* Hardware has to reinitialize queues and interrupts to 8266 * match packet buffer alignment. Unfortunately, the 8267 * hardware is not flexible enough to do this dynamically. 8268 */ 8269 if (netif_running(dev)) 8270 ixgbe_close(dev); 8271 else 8272 ixgbe_reset(adapter); 8273 8274 ixgbe_clear_interrupt_scheme(adapter); 8275 8276 #ifdef CONFIG_IXGBE_DCB 8277 if (tc) { 8278 netdev_set_num_tc(dev, tc); 8279 ixgbe_set_prio_tc_map(adapter); 8280 8281 adapter->flags |= IXGBE_FLAG_DCB_ENABLED; 8282 8283 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 8284 adapter->last_lfc_mode = adapter->hw.fc.requested_mode; 8285 adapter->hw.fc.requested_mode = ixgbe_fc_none; 8286 } 8287 } else { 8288 netdev_reset_tc(dev); 8289 8290 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 8291 adapter->hw.fc.requested_mode = adapter->last_lfc_mode; 8292 8293 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; 8294 8295 adapter->temp_dcb_cfg.pfc_mode_enable = false; 8296 adapter->dcb_cfg.pfc_mode_enable = false; 8297 } 8298 8299 ixgbe_validate_rtr(adapter, tc); 8300 8301 #endif /* CONFIG_IXGBE_DCB */ 8302 ixgbe_init_interrupt_scheme(adapter); 8303 8304 if (netif_running(dev)) 8305 return ixgbe_open(dev); 8306 8307 return 0; 8308 } 8309 8310 static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, 8311 struct tc_cls_u32_offload *cls) 8312 { 8313 u32 hdl = cls->knode.handle; 8314 u32 uhtid = TC_U32_USERHTID(cls->knode.handle); 8315 u32 loc = cls->knode.handle & 0xfffff; 8316 int err = 0, i, j; 8317 struct ixgbe_jump_table *jump = NULL; 8318 8319 if (loc > IXGBE_MAX_HW_ENTRIES) 8320 return -EINVAL; 8321 8322 if ((uhtid != 0x800) && (uhtid >= IXGBE_MAX_LINK_HANDLE)) 8323 return -EINVAL; 8324 8325 /* Clear this filter in the link data it is associated with */ 8326 if (uhtid != 0x800) { 8327 jump = adapter->jump_tables[uhtid]; 8328 if (!jump) 8329 return -EINVAL; 8330 if (!test_bit(loc - 1, jump->child_loc_map)) 8331 return -EINVAL; 8332 clear_bit(loc - 1, jump->child_loc_map); 8333 } 8334 8335 /* Check if the filter being deleted is a link */ 8336 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) { 8337 jump = adapter->jump_tables[i]; 8338 if (jump && jump->link_hdl == hdl) { 8339 /* Delete filters in the hardware in the child hash 8340 * table associated with this link 8341 */ 8342 for (j = 0; j < IXGBE_MAX_HW_ENTRIES; j++) { 8343 if (!test_bit(j, jump->child_loc_map)) 8344 continue; 8345 spin_lock(&adapter->fdir_perfect_lock); 8346 err = ixgbe_update_ethtool_fdir_entry(adapter, 8347 NULL, 8348 j + 1); 8349 spin_unlock(&adapter->fdir_perfect_lock); 8350 clear_bit(j, jump->child_loc_map); 8351 } 8352 /* Remove resources for this link */ 8353 kfree(jump->input); 8354 kfree(jump->mask); 8355 kfree(jump); 8356 adapter->jump_tables[i] = NULL; 8357 return err; 8358 } 8359 } 8360 8361 spin_lock(&adapter->fdir_perfect_lock); 8362 err = ixgbe_update_ethtool_fdir_entry(adapter, NULL, loc); 8363 spin_unlock(&adapter->fdir_perfect_lock); 8364 return err; 8365 } 8366 8367 static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, 8368 __be16 protocol, 8369 struct tc_cls_u32_offload *cls) 8370 { 8371 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); 8372 8373 if (uhtid >= IXGBE_MAX_LINK_HANDLE) 8374 return -EINVAL; 8375 8376 /* This ixgbe devices do not support hash tables at the moment 8377 * so abort when given hash tables. 8378 */ 8379 if (cls->hnode.divisor > 0) 8380 return -EINVAL; 8381 8382 set_bit(uhtid - 1, &adapter->tables); 8383 return 0; 8384 } 8385 8386 static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter, 8387 struct tc_cls_u32_offload *cls) 8388 { 8389 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); 8390 8391 if (uhtid >= IXGBE_MAX_LINK_HANDLE) 8392 return -EINVAL; 8393 8394 clear_bit(uhtid - 1, &adapter->tables); 8395 return 0; 8396 } 8397 8398 #ifdef CONFIG_NET_CLS_ACT 8399 struct upper_walk_data { 8400 struct ixgbe_adapter *adapter; 8401 u64 action; 8402 int ifindex; 8403 u8 queue; 8404 }; 8405 8406 static int get_macvlan_queue(struct net_device *upper, void *_data) 8407 { 8408 if (netif_is_macvlan(upper)) { 8409 struct macvlan_dev *dfwd = netdev_priv(upper); 8410 struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv; 8411 struct upper_walk_data *data = _data; 8412 struct ixgbe_adapter *adapter = data->adapter; 8413 int ifindex = data->ifindex; 8414 8415 if (vadapter && vadapter->netdev->ifindex == ifindex) { 8416 data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx; 8417 data->action = data->queue; 8418 return 1; 8419 } 8420 } 8421 8422 return 0; 8423 } 8424 8425 static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex, 8426 u8 *queue, u64 *action) 8427 { 8428 unsigned int num_vfs = adapter->num_vfs, vf; 8429 struct upper_walk_data data; 8430 struct net_device *upper; 8431 8432 /* redirect to a SRIOV VF */ 8433 for (vf = 0; vf < num_vfs; ++vf) { 8434 upper = pci_get_drvdata(adapter->vfinfo[vf].vfdev); 8435 if (upper->ifindex == ifindex) { 8436 if (adapter->num_rx_pools > 1) 8437 *queue = vf * 2; 8438 else 8439 *queue = vf * adapter->num_rx_queues_per_pool; 8440 8441 *action = vf + 1; 8442 *action <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF; 8443 return 0; 8444 } 8445 } 8446 8447 /* redirect to a offloaded macvlan netdev */ 8448 data.adapter = adapter; 8449 data.ifindex = ifindex; 8450 data.action = 0; 8451 data.queue = 0; 8452 if (netdev_walk_all_upper_dev_rcu(adapter->netdev, 8453 get_macvlan_queue, &data)) { 8454 *action = data.action; 8455 *queue = data.queue; 8456 8457 return 0; 8458 } 8459 8460 return -EINVAL; 8461 } 8462 8463 static int parse_tc_actions(struct ixgbe_adapter *adapter, 8464 struct tcf_exts *exts, u64 *action, u8 *queue) 8465 { 8466 const struct tc_action *a; 8467 LIST_HEAD(actions); 8468 int err; 8469 8470 if (tc_no_actions(exts)) 8471 return -EINVAL; 8472 8473 tcf_exts_to_list(exts, &actions); 8474 list_for_each_entry(a, &actions, list) { 8475 8476 /* Drop action */ 8477 if (is_tcf_gact_shot(a)) { 8478 *action = IXGBE_FDIR_DROP_QUEUE; 8479 *queue = IXGBE_FDIR_DROP_QUEUE; 8480 return 0; 8481 } 8482 8483 /* Redirect to a VF or a offloaded macvlan */ 8484 if (is_tcf_mirred_egress_redirect(a)) { 8485 int ifindex = tcf_mirred_ifindex(a); 8486 8487 err = handle_redirect_action(adapter, ifindex, queue, 8488 action); 8489 if (err == 0) 8490 return err; 8491 } 8492 } 8493 8494 return -EINVAL; 8495 } 8496 #else 8497 static int parse_tc_actions(struct ixgbe_adapter *adapter, 8498 struct tcf_exts *exts, u64 *action, u8 *queue) 8499 { 8500 return -EINVAL; 8501 } 8502 #endif /* CONFIG_NET_CLS_ACT */ 8503 8504 static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, 8505 union ixgbe_atr_input *mask, 8506 struct tc_cls_u32_offload *cls, 8507 struct ixgbe_mat_field *field_ptr, 8508 struct ixgbe_nexthdr *nexthdr) 8509 { 8510 int i, j, off; 8511 __be32 val, m; 8512 bool found_entry = false, found_jump_field = false; 8513 8514 for (i = 0; i < cls->knode.sel->nkeys; i++) { 8515 off = cls->knode.sel->keys[i].off; 8516 val = cls->knode.sel->keys[i].val; 8517 m = cls->knode.sel->keys[i].mask; 8518 8519 for (j = 0; field_ptr[j].val; j++) { 8520 if (field_ptr[j].off == off) { 8521 field_ptr[j].val(input, mask, val, m); 8522 input->filter.formatted.flow_type |= 8523 field_ptr[j].type; 8524 found_entry = true; 8525 break; 8526 } 8527 } 8528 if (nexthdr) { 8529 if (nexthdr->off == cls->knode.sel->keys[i].off && 8530 nexthdr->val == cls->knode.sel->keys[i].val && 8531 nexthdr->mask == cls->knode.sel->keys[i].mask) 8532 found_jump_field = true; 8533 else 8534 continue; 8535 } 8536 } 8537 8538 if (nexthdr && !found_jump_field) 8539 return -EINVAL; 8540 8541 if (!found_entry) 8542 return 0; 8543 8544 mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK | 8545 IXGBE_ATR_L4TYPE_MASK; 8546 8547 if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4) 8548 mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK; 8549 8550 return 0; 8551 } 8552 8553 static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, 8554 __be16 protocol, 8555 struct tc_cls_u32_offload *cls) 8556 { 8557 u32 loc = cls->knode.handle & 0xfffff; 8558 struct ixgbe_hw *hw = &adapter->hw; 8559 struct ixgbe_mat_field *field_ptr; 8560 struct ixgbe_fdir_filter *input = NULL; 8561 union ixgbe_atr_input *mask = NULL; 8562 struct ixgbe_jump_table *jump = NULL; 8563 int i, err = -EINVAL; 8564 u8 queue; 8565 u32 uhtid, link_uhtid; 8566 8567 uhtid = TC_U32_USERHTID(cls->knode.handle); 8568 link_uhtid = TC_U32_USERHTID(cls->knode.link_handle); 8569 8570 /* At the moment cls_u32 jumps to network layer and skips past 8571 * L2 headers. The canonical method to match L2 frames is to use 8572 * negative values. However this is error prone at best but really 8573 * just broken because there is no way to "know" what sort of hdr 8574 * is in front of the network layer. Fix cls_u32 to support L2 8575 * headers when needed. 8576 */ 8577 if (protocol != htons(ETH_P_IP)) 8578 return err; 8579 8580 if (loc >= ((1024 << adapter->fdir_pballoc) - 2)) { 8581 e_err(drv, "Location out of range\n"); 8582 return err; 8583 } 8584 8585 /* cls u32 is a graph starting at root node 0x800. The driver tracks 8586 * links and also the fields used to advance the parser across each 8587 * link (e.g. nexthdr/eat parameters from 'tc'). This way we can map 8588 * the u32 graph onto the hardware parse graph denoted in ixgbe_model.h 8589 * To add support for new nodes update ixgbe_model.h parse structures 8590 * this function _should_ be generic try not to hardcode values here. 8591 */ 8592 if (uhtid == 0x800) { 8593 field_ptr = (adapter->jump_tables[0])->mat; 8594 } else { 8595 if (uhtid >= IXGBE_MAX_LINK_HANDLE) 8596 return err; 8597 if (!adapter->jump_tables[uhtid]) 8598 return err; 8599 field_ptr = (adapter->jump_tables[uhtid])->mat; 8600 } 8601 8602 if (!field_ptr) 8603 return err; 8604 8605 /* At this point we know the field_ptr is valid and need to either 8606 * build cls_u32 link or attach filter. Because adding a link to 8607 * a handle that does not exist is invalid and the same for adding 8608 * rules to handles that don't exist. 8609 */ 8610 8611 if (link_uhtid) { 8612 struct ixgbe_nexthdr *nexthdr = ixgbe_ipv4_jumps; 8613 8614 if (link_uhtid >= IXGBE_MAX_LINK_HANDLE) 8615 return err; 8616 8617 if (!test_bit(link_uhtid - 1, &adapter->tables)) 8618 return err; 8619 8620 /* Multiple filters as links to the same hash table are not 8621 * supported. To add a new filter with the same next header 8622 * but different match/jump conditions, create a new hash table 8623 * and link to it. 8624 */ 8625 if (adapter->jump_tables[link_uhtid] && 8626 (adapter->jump_tables[link_uhtid])->link_hdl) { 8627 e_err(drv, "Link filter exists for link: %x\n", 8628 link_uhtid); 8629 return err; 8630 } 8631 8632 for (i = 0; nexthdr[i].jump; i++) { 8633 if (nexthdr[i].o != cls->knode.sel->offoff || 8634 nexthdr[i].s != cls->knode.sel->offshift || 8635 nexthdr[i].m != cls->knode.sel->offmask) 8636 return err; 8637 8638 jump = kzalloc(sizeof(*jump), GFP_KERNEL); 8639 if (!jump) 8640 return -ENOMEM; 8641 input = kzalloc(sizeof(*input), GFP_KERNEL); 8642 if (!input) { 8643 err = -ENOMEM; 8644 goto free_jump; 8645 } 8646 mask = kzalloc(sizeof(*mask), GFP_KERNEL); 8647 if (!mask) { 8648 err = -ENOMEM; 8649 goto free_input; 8650 } 8651 jump->input = input; 8652 jump->mask = mask; 8653 jump->link_hdl = cls->knode.handle; 8654 8655 err = ixgbe_clsu32_build_input(input, mask, cls, 8656 field_ptr, &nexthdr[i]); 8657 if (!err) { 8658 jump->mat = nexthdr[i].jump; 8659 adapter->jump_tables[link_uhtid] = jump; 8660 break; 8661 } 8662 } 8663 return 0; 8664 } 8665 8666 input = kzalloc(sizeof(*input), GFP_KERNEL); 8667 if (!input) 8668 return -ENOMEM; 8669 mask = kzalloc(sizeof(*mask), GFP_KERNEL); 8670 if (!mask) { 8671 err = -ENOMEM; 8672 goto free_input; 8673 } 8674 8675 if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) { 8676 if ((adapter->jump_tables[uhtid])->input) 8677 memcpy(input, (adapter->jump_tables[uhtid])->input, 8678 sizeof(*input)); 8679 if ((adapter->jump_tables[uhtid])->mask) 8680 memcpy(mask, (adapter->jump_tables[uhtid])->mask, 8681 sizeof(*mask)); 8682 8683 /* Lookup in all child hash tables if this location is already 8684 * filled with a filter 8685 */ 8686 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) { 8687 struct ixgbe_jump_table *link = adapter->jump_tables[i]; 8688 8689 if (link && (test_bit(loc - 1, link->child_loc_map))) { 8690 e_err(drv, "Filter exists in location: %x\n", 8691 loc); 8692 err = -EINVAL; 8693 goto err_out; 8694 } 8695 } 8696 } 8697 err = ixgbe_clsu32_build_input(input, mask, cls, field_ptr, NULL); 8698 if (err) 8699 goto err_out; 8700 8701 err = parse_tc_actions(adapter, cls->knode.exts, &input->action, 8702 &queue); 8703 if (err < 0) 8704 goto err_out; 8705 8706 input->sw_idx = loc; 8707 8708 spin_lock(&adapter->fdir_perfect_lock); 8709 8710 if (hlist_empty(&adapter->fdir_filter_list)) { 8711 memcpy(&adapter->fdir_mask, mask, sizeof(*mask)); 8712 err = ixgbe_fdir_set_input_mask_82599(hw, mask); 8713 if (err) 8714 goto err_out_w_lock; 8715 } else if (memcmp(&adapter->fdir_mask, mask, sizeof(*mask))) { 8716 err = -EINVAL; 8717 goto err_out_w_lock; 8718 } 8719 8720 ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask); 8721 err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter, 8722 input->sw_idx, queue); 8723 if (!err) 8724 ixgbe_update_ethtool_fdir_entry(adapter, input, input->sw_idx); 8725 spin_unlock(&adapter->fdir_perfect_lock); 8726 8727 if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) 8728 set_bit(loc - 1, (adapter->jump_tables[uhtid])->child_loc_map); 8729 8730 kfree(mask); 8731 return err; 8732 err_out_w_lock: 8733 spin_unlock(&adapter->fdir_perfect_lock); 8734 err_out: 8735 kfree(mask); 8736 free_input: 8737 kfree(input); 8738 free_jump: 8739 kfree(jump); 8740 return err; 8741 } 8742 8743 static int __ixgbe_setup_tc(struct net_device *dev, u32 handle, __be16 proto, 8744 struct tc_to_netdev *tc) 8745 { 8746 struct ixgbe_adapter *adapter = netdev_priv(dev); 8747 8748 if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) && 8749 tc->type == TC_SETUP_CLSU32) { 8750 switch (tc->cls_u32->command) { 8751 case TC_CLSU32_NEW_KNODE: 8752 case TC_CLSU32_REPLACE_KNODE: 8753 return ixgbe_configure_clsu32(adapter, 8754 proto, tc->cls_u32); 8755 case TC_CLSU32_DELETE_KNODE: 8756 return ixgbe_delete_clsu32(adapter, tc->cls_u32); 8757 case TC_CLSU32_NEW_HNODE: 8758 case TC_CLSU32_REPLACE_HNODE: 8759 return ixgbe_configure_clsu32_add_hnode(adapter, proto, 8760 tc->cls_u32); 8761 case TC_CLSU32_DELETE_HNODE: 8762 return ixgbe_configure_clsu32_del_hnode(adapter, 8763 tc->cls_u32); 8764 default: 8765 return -EINVAL; 8766 } 8767 } 8768 8769 if (tc->type != TC_SETUP_MQPRIO) 8770 return -EINVAL; 8771 8772 return ixgbe_setup_tc(dev, tc->tc); 8773 } 8774 8775 #ifdef CONFIG_PCI_IOV 8776 void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter) 8777 { 8778 struct net_device *netdev = adapter->netdev; 8779 8780 rtnl_lock(); 8781 ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev)); 8782 rtnl_unlock(); 8783 } 8784 8785 #endif 8786 void ixgbe_do_reset(struct net_device *netdev) 8787 { 8788 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8789 8790 if (netif_running(netdev)) 8791 ixgbe_reinit_locked(adapter); 8792 else 8793 ixgbe_reset(adapter); 8794 } 8795 8796 static netdev_features_t ixgbe_fix_features(struct net_device *netdev, 8797 netdev_features_t features) 8798 { 8799 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8800 8801 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ 8802 if (!(features & NETIF_F_RXCSUM)) 8803 features &= ~NETIF_F_LRO; 8804 8805 /* Turn off LRO if not RSC capable */ 8806 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) 8807 features &= ~NETIF_F_LRO; 8808 8809 return features; 8810 } 8811 8812 static int ixgbe_set_features(struct net_device *netdev, 8813 netdev_features_t features) 8814 { 8815 struct ixgbe_adapter *adapter = netdev_priv(netdev); 8816 netdev_features_t changed = netdev->features ^ features; 8817 bool need_reset = false; 8818 8819 /* Make sure RSC matches LRO, reset if change */ 8820 if (!(features & NETIF_F_LRO)) { 8821 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 8822 need_reset = true; 8823 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED; 8824 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && 8825 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { 8826 if (adapter->rx_itr_setting == 1 || 8827 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { 8828 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 8829 need_reset = true; 8830 } else if ((changed ^ features) & NETIF_F_LRO) { 8831 e_info(probe, "rx-usecs set too low, " 8832 "disabling RSC\n"); 8833 } 8834 } 8835 8836 /* 8837 * Check if Flow Director n-tuple support or hw_tc support was 8838 * enabled or disabled. If the state changed, we need to reset. 8839 */ 8840 if ((features & NETIF_F_NTUPLE) || (features & NETIF_F_HW_TC)) { 8841 /* turn off ATR, enable perfect filters and reset */ 8842 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) 8843 need_reset = true; 8844 8845 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 8846 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 8847 } else { 8848 /* turn off perfect filters, enable ATR and reset */ 8849 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 8850 need_reset = true; 8851 8852 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 8853 8854 /* We cannot enable ATR if SR-IOV is enabled */ 8855 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED || 8856 /* We cannot enable ATR if we have 2 or more tcs */ 8857 (netdev_get_num_tc(netdev) > 1) || 8858 /* We cannot enable ATR if RSS is disabled */ 8859 (adapter->ring_feature[RING_F_RSS].limit <= 1) || 8860 /* A sample rate of 0 indicates ATR disabled */ 8861 (!adapter->atr_sample_rate)) 8862 ; /* do nothing not supported */ 8863 else /* otherwise supported and set the flag */ 8864 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 8865 } 8866 8867 if (changed & NETIF_F_RXALL) 8868 need_reset = true; 8869 8870 netdev->features = features; 8871 8872 if ((adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) { 8873 if (features & NETIF_F_RXCSUM) { 8874 adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED; 8875 } else { 8876 u32 port_mask = IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK; 8877 8878 ixgbe_clear_udp_tunnel_port(adapter, port_mask); 8879 } 8880 } 8881 8882 if ((adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)) { 8883 if (features & NETIF_F_RXCSUM) { 8884 adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED; 8885 } else { 8886 u32 port_mask = IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK; 8887 8888 ixgbe_clear_udp_tunnel_port(adapter, port_mask); 8889 } 8890 } 8891 8892 if (need_reset) 8893 ixgbe_do_reset(netdev); 8894 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | 8895 NETIF_F_HW_VLAN_CTAG_FILTER)) 8896 ixgbe_set_rx_mode(netdev); 8897 8898 return 0; 8899 } 8900 8901 /** 8902 * ixgbe_add_udp_tunnel_port - Get notifications about adding UDP tunnel ports 8903 * @dev: The port's netdev 8904 * @ti: Tunnel endpoint information 8905 **/ 8906 static void ixgbe_add_udp_tunnel_port(struct net_device *dev, 8907 struct udp_tunnel_info *ti) 8908 { 8909 struct ixgbe_adapter *adapter = netdev_priv(dev); 8910 struct ixgbe_hw *hw = &adapter->hw; 8911 __be16 port = ti->port; 8912 u32 port_shift = 0; 8913 u32 reg; 8914 8915 if (ti->sa_family != AF_INET) 8916 return; 8917 8918 switch (ti->type) { 8919 case UDP_TUNNEL_TYPE_VXLAN: 8920 if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) 8921 return; 8922 8923 if (adapter->vxlan_port == port) 8924 return; 8925 8926 if (adapter->vxlan_port) { 8927 netdev_info(dev, 8928 "VXLAN port %d set, not adding port %d\n", 8929 ntohs(adapter->vxlan_port), 8930 ntohs(port)); 8931 return; 8932 } 8933 8934 adapter->vxlan_port = port; 8935 break; 8936 case UDP_TUNNEL_TYPE_GENEVE: 8937 if (!(adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)) 8938 return; 8939 8940 if (adapter->geneve_port == port) 8941 return; 8942 8943 if (adapter->geneve_port) { 8944 netdev_info(dev, 8945 "GENEVE port %d set, not adding port %d\n", 8946 ntohs(adapter->geneve_port), 8947 ntohs(port)); 8948 return; 8949 } 8950 8951 port_shift = IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT; 8952 adapter->geneve_port = port; 8953 break; 8954 default: 8955 return; 8956 } 8957 8958 reg = IXGBE_READ_REG(hw, IXGBE_VXLANCTRL) | ntohs(port) << port_shift; 8959 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, reg); 8960 } 8961 8962 /** 8963 * ixgbe_del_udp_tunnel_port - Get notifications about removing UDP tunnel ports 8964 * @dev: The port's netdev 8965 * @ti: Tunnel endpoint information 8966 **/ 8967 static void ixgbe_del_udp_tunnel_port(struct net_device *dev, 8968 struct udp_tunnel_info *ti) 8969 { 8970 struct ixgbe_adapter *adapter = netdev_priv(dev); 8971 u32 port_mask; 8972 8973 if (ti->type != UDP_TUNNEL_TYPE_VXLAN && 8974 ti->type != UDP_TUNNEL_TYPE_GENEVE) 8975 return; 8976 8977 if (ti->sa_family != AF_INET) 8978 return; 8979 8980 switch (ti->type) { 8981 case UDP_TUNNEL_TYPE_VXLAN: 8982 if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) 8983 return; 8984 8985 if (adapter->vxlan_port != ti->port) { 8986 netdev_info(dev, "VXLAN port %d not found\n", 8987 ntohs(ti->port)); 8988 return; 8989 } 8990 8991 port_mask = IXGBE_VXLANCTRL_VXLAN_UDPPORT_MASK; 8992 break; 8993 case UDP_TUNNEL_TYPE_GENEVE: 8994 if (!(adapter->flags & IXGBE_FLAG_GENEVE_OFFLOAD_CAPABLE)) 8995 return; 8996 8997 if (adapter->geneve_port != ti->port) { 8998 netdev_info(dev, "GENEVE port %d not found\n", 8999 ntohs(ti->port)); 9000 return; 9001 } 9002 9003 port_mask = IXGBE_VXLANCTRL_GENEVE_UDPPORT_MASK; 9004 break; 9005 default: 9006 return; 9007 } 9008 9009 ixgbe_clear_udp_tunnel_port(adapter, port_mask); 9010 adapter->flags2 |= IXGBE_FLAG2_UDP_TUN_REREG_NEEDED; 9011 } 9012 9013 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 9014 struct net_device *dev, 9015 const unsigned char *addr, u16 vid, 9016 u16 flags) 9017 { 9018 /* guarantee we can provide a unique filter for the unicast address */ 9019 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { 9020 struct ixgbe_adapter *adapter = netdev_priv(dev); 9021 u16 pool = VMDQ_P(0); 9022 9023 if (netdev_uc_count(dev) >= ixgbe_available_rars(adapter, pool)) 9024 return -ENOMEM; 9025 } 9026 9027 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags); 9028 } 9029 9030 /** 9031 * ixgbe_configure_bridge_mode - set various bridge modes 9032 * @adapter - the private structure 9033 * @mode - requested bridge mode 9034 * 9035 * Configure some settings require for various bridge modes. 9036 **/ 9037 static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, 9038 __u16 mode) 9039 { 9040 struct ixgbe_hw *hw = &adapter->hw; 9041 unsigned int p, num_pools; 9042 u32 vmdctl; 9043 9044 switch (mode) { 9045 case BRIDGE_MODE_VEPA: 9046 /* disable Tx loopback, rely on switch hairpin mode */ 9047 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 0); 9048 9049 /* must enable Rx switching replication to allow multicast 9050 * packet reception on all VFs, and to enable source address 9051 * pruning. 9052 */ 9053 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL); 9054 vmdctl |= IXGBE_VT_CTL_REPLEN; 9055 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 9056 9057 /* enable Rx source address pruning. Note, this requires 9058 * replication to be enabled or else it does nothing. 9059 */ 9060 num_pools = adapter->num_vfs + adapter->num_rx_pools; 9061 for (p = 0; p < num_pools; p++) { 9062 if (hw->mac.ops.set_source_address_pruning) 9063 hw->mac.ops.set_source_address_pruning(hw, 9064 true, 9065 p); 9066 } 9067 break; 9068 case BRIDGE_MODE_VEB: 9069 /* enable Tx loopback for internal VF/PF communication */ 9070 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 9071 IXGBE_PFDTXGSWC_VT_LBEN); 9072 9073 /* disable Rx switching replication unless we have SR-IOV 9074 * virtual functions 9075 */ 9076 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL); 9077 if (!adapter->num_vfs) 9078 vmdctl &= ~IXGBE_VT_CTL_REPLEN; 9079 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 9080 9081 /* disable Rx source address pruning, since we don't expect to 9082 * be receiving external loopback of our transmitted frames. 9083 */ 9084 num_pools = adapter->num_vfs + adapter->num_rx_pools; 9085 for (p = 0; p < num_pools; p++) { 9086 if (hw->mac.ops.set_source_address_pruning) 9087 hw->mac.ops.set_source_address_pruning(hw, 9088 false, 9089 p); 9090 } 9091 break; 9092 default: 9093 return -EINVAL; 9094 } 9095 9096 adapter->bridge_mode = mode; 9097 9098 e_info(drv, "enabling bridge mode: %s\n", 9099 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 9100 9101 return 0; 9102 } 9103 9104 static int ixgbe_ndo_bridge_setlink(struct net_device *dev, 9105 struct nlmsghdr *nlh, u16 flags) 9106 { 9107 struct ixgbe_adapter *adapter = netdev_priv(dev); 9108 struct nlattr *attr, *br_spec; 9109 int rem; 9110 9111 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 9112 return -EOPNOTSUPP; 9113 9114 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 9115 if (!br_spec) 9116 return -EINVAL; 9117 9118 nla_for_each_nested(attr, br_spec, rem) { 9119 int status; 9120 __u16 mode; 9121 9122 if (nla_type(attr) != IFLA_BRIDGE_MODE) 9123 continue; 9124 9125 if (nla_len(attr) < sizeof(mode)) 9126 return -EINVAL; 9127 9128 mode = nla_get_u16(attr); 9129 status = ixgbe_configure_bridge_mode(adapter, mode); 9130 if (status) 9131 return status; 9132 9133 break; 9134 } 9135 9136 return 0; 9137 } 9138 9139 static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 9140 struct net_device *dev, 9141 u32 filter_mask, int nlflags) 9142 { 9143 struct ixgbe_adapter *adapter = netdev_priv(dev); 9144 9145 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 9146 return 0; 9147 9148 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 9149 adapter->bridge_mode, 0, 0, nlflags, 9150 filter_mask, NULL); 9151 } 9152 9153 static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) 9154 { 9155 struct ixgbe_fwd_adapter *fwd_adapter = NULL; 9156 struct ixgbe_adapter *adapter = netdev_priv(pdev); 9157 int used_pools = adapter->num_vfs + adapter->num_rx_pools; 9158 unsigned int limit; 9159 int pool, err; 9160 9161 /* Hardware has a limited number of available pools. Each VF, and the 9162 * PF require a pool. Check to ensure we don't attempt to use more 9163 * then the available number of pools. 9164 */ 9165 if (used_pools >= IXGBE_MAX_VF_FUNCTIONS) 9166 return ERR_PTR(-EINVAL); 9167 9168 #ifdef CONFIG_RPS 9169 if (vdev->num_rx_queues != vdev->num_tx_queues) { 9170 netdev_info(pdev, "%s: Only supports a single queue count for TX and RX\n", 9171 vdev->name); 9172 return ERR_PTR(-EINVAL); 9173 } 9174 #endif 9175 /* Check for hardware restriction on number of rx/tx queues */ 9176 if (vdev->num_tx_queues > IXGBE_MAX_L2A_QUEUES || 9177 vdev->num_tx_queues == IXGBE_BAD_L2A_QUEUE) { 9178 netdev_info(pdev, 9179 "%s: Supports RX/TX Queue counts 1,2, and 4\n", 9180 pdev->name); 9181 return ERR_PTR(-EINVAL); 9182 } 9183 9184 if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && 9185 adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS - 1) || 9186 (adapter->num_rx_pools > IXGBE_MAX_MACVLANS)) 9187 return ERR_PTR(-EBUSY); 9188 9189 fwd_adapter = kzalloc(sizeof(*fwd_adapter), GFP_KERNEL); 9190 if (!fwd_adapter) 9191 return ERR_PTR(-ENOMEM); 9192 9193 pool = find_first_zero_bit(&adapter->fwd_bitmask, 32); 9194 adapter->num_rx_pools++; 9195 set_bit(pool, &adapter->fwd_bitmask); 9196 limit = find_last_bit(&adapter->fwd_bitmask, 32); 9197 9198 /* Enable VMDq flag so device will be set in VM mode */ 9199 adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED; 9200 adapter->ring_feature[RING_F_VMDQ].limit = limit + 1; 9201 adapter->ring_feature[RING_F_RSS].limit = vdev->num_tx_queues; 9202 9203 /* Force reinit of ring allocation with VMDQ enabled */ 9204 err = ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev)); 9205 if (err) 9206 goto fwd_add_err; 9207 fwd_adapter->pool = pool; 9208 fwd_adapter->real_adapter = adapter; 9209 9210 if (netif_running(pdev)) { 9211 err = ixgbe_fwd_ring_up(vdev, fwd_adapter); 9212 if (err) 9213 goto fwd_add_err; 9214 netif_tx_start_all_queues(vdev); 9215 } 9216 9217 return fwd_adapter; 9218 fwd_add_err: 9219 /* unwind counter and free adapter struct */ 9220 netdev_info(pdev, 9221 "%s: dfwd hardware acceleration failed\n", vdev->name); 9222 clear_bit(pool, &adapter->fwd_bitmask); 9223 adapter->num_rx_pools--; 9224 kfree(fwd_adapter); 9225 return ERR_PTR(err); 9226 } 9227 9228 static void ixgbe_fwd_del(struct net_device *pdev, void *priv) 9229 { 9230 struct ixgbe_fwd_adapter *fwd_adapter = priv; 9231 struct ixgbe_adapter *adapter = fwd_adapter->real_adapter; 9232 unsigned int limit; 9233 9234 clear_bit(fwd_adapter->pool, &adapter->fwd_bitmask); 9235 adapter->num_rx_pools--; 9236 9237 limit = find_last_bit(&adapter->fwd_bitmask, 32); 9238 adapter->ring_feature[RING_F_VMDQ].limit = limit + 1; 9239 ixgbe_fwd_ring_down(fwd_adapter->netdev, fwd_adapter); 9240 ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev)); 9241 netdev_dbg(pdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n", 9242 fwd_adapter->pool, adapter->num_rx_pools, 9243 fwd_adapter->rx_base_queue, 9244 fwd_adapter->rx_base_queue + adapter->num_rx_queues_per_pool, 9245 adapter->fwd_bitmask); 9246 kfree(fwd_adapter); 9247 } 9248 9249 #define IXGBE_MAX_MAC_HDR_LEN 127 9250 #define IXGBE_MAX_NETWORK_HDR_LEN 511 9251 9252 static netdev_features_t 9253 ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, 9254 netdev_features_t features) 9255 { 9256 unsigned int network_hdr_len, mac_hdr_len; 9257 9258 /* Make certain the headers can be described by a context descriptor */ 9259 mac_hdr_len = skb_network_header(skb) - skb->data; 9260 if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN)) 9261 return features & ~(NETIF_F_HW_CSUM | 9262 NETIF_F_SCTP_CRC | 9263 NETIF_F_HW_VLAN_CTAG_TX | 9264 NETIF_F_TSO | 9265 NETIF_F_TSO6); 9266 9267 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 9268 if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN)) 9269 return features & ~(NETIF_F_HW_CSUM | 9270 NETIF_F_SCTP_CRC | 9271 NETIF_F_TSO | 9272 NETIF_F_TSO6); 9273 9274 /* We can only support IPV4 TSO in tunnels if we can mangle the 9275 * inner IP ID field, so strip TSO if MANGLEID is not supported. 9276 */ 9277 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 9278 features &= ~NETIF_F_TSO; 9279 9280 return features; 9281 } 9282 9283 static const struct net_device_ops ixgbe_netdev_ops = { 9284 .ndo_open = ixgbe_open, 9285 .ndo_stop = ixgbe_close, 9286 .ndo_start_xmit = ixgbe_xmit_frame, 9287 .ndo_select_queue = ixgbe_select_queue, 9288 .ndo_set_rx_mode = ixgbe_set_rx_mode, 9289 .ndo_validate_addr = eth_validate_addr, 9290 .ndo_set_mac_address = ixgbe_set_mac, 9291 .ndo_change_mtu = ixgbe_change_mtu, 9292 .ndo_tx_timeout = ixgbe_tx_timeout, 9293 .ndo_set_tx_maxrate = ixgbe_tx_maxrate, 9294 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid, 9295 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid, 9296 .ndo_do_ioctl = ixgbe_ioctl, 9297 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac, 9298 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan, 9299 .ndo_set_vf_rate = ixgbe_ndo_set_vf_bw, 9300 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk, 9301 .ndo_set_vf_rss_query_en = ixgbe_ndo_set_vf_rss_query_en, 9302 .ndo_set_vf_trust = ixgbe_ndo_set_vf_trust, 9303 .ndo_get_vf_config = ixgbe_ndo_get_vf_config, 9304 .ndo_get_stats64 = ixgbe_get_stats64, 9305 .ndo_setup_tc = __ixgbe_setup_tc, 9306 #ifdef CONFIG_NET_POLL_CONTROLLER 9307 .ndo_poll_controller = ixgbe_netpoll, 9308 #endif 9309 #ifdef IXGBE_FCOE 9310 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get, 9311 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target, 9312 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put, 9313 .ndo_fcoe_enable = ixgbe_fcoe_enable, 9314 .ndo_fcoe_disable = ixgbe_fcoe_disable, 9315 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn, 9316 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo, 9317 #endif /* IXGBE_FCOE */ 9318 .ndo_set_features = ixgbe_set_features, 9319 .ndo_fix_features = ixgbe_fix_features, 9320 .ndo_fdb_add = ixgbe_ndo_fdb_add, 9321 .ndo_bridge_setlink = ixgbe_ndo_bridge_setlink, 9322 .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink, 9323 .ndo_dfwd_add_station = ixgbe_fwd_add, 9324 .ndo_dfwd_del_station = ixgbe_fwd_del, 9325 .ndo_udp_tunnel_add = ixgbe_add_udp_tunnel_port, 9326 .ndo_udp_tunnel_del = ixgbe_del_udp_tunnel_port, 9327 .ndo_features_check = ixgbe_features_check, 9328 }; 9329 9330 /** 9331 * ixgbe_enumerate_functions - Get the number of ports this device has 9332 * @adapter: adapter structure 9333 * 9334 * This function enumerates the phsyical functions co-located on a single slot, 9335 * in order to determine how many ports a device has. This is most useful in 9336 * determining the required GT/s of PCIe bandwidth necessary for optimal 9337 * performance. 9338 **/ 9339 static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter) 9340 { 9341 struct pci_dev *entry, *pdev = adapter->pdev; 9342 int physfns = 0; 9343 9344 /* Some cards can not use the generic count PCIe functions method, 9345 * because they are behind a parent switch, so we hardcode these with 9346 * the correct number of functions. 9347 */ 9348 if (ixgbe_pcie_from_parent(&adapter->hw)) 9349 physfns = 4; 9350 9351 list_for_each_entry(entry, &adapter->pdev->bus->devices, bus_list) { 9352 /* don't count virtual functions */ 9353 if (entry->is_virtfn) 9354 continue; 9355 9356 /* When the devices on the bus don't all match our device ID, 9357 * we can't reliably determine the correct number of 9358 * functions. This can occur if a function has been direct 9359 * attached to a virtual machine using VT-d, for example. In 9360 * this case, simply return -1 to indicate this. 9361 */ 9362 if ((entry->vendor != pdev->vendor) || 9363 (entry->device != pdev->device)) 9364 return -1; 9365 9366 physfns++; 9367 } 9368 9369 return physfns; 9370 } 9371 9372 /** 9373 * ixgbe_wol_supported - Check whether device supports WoL 9374 * @adapter: the adapter private structure 9375 * @device_id: the device ID 9376 * @subdev_id: the subsystem device ID 9377 * 9378 * This function is used by probe and ethtool to determine 9379 * which devices have WoL support 9380 * 9381 **/ 9382 bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, 9383 u16 subdevice_id) 9384 { 9385 struct ixgbe_hw *hw = &adapter->hw; 9386 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK; 9387 9388 /* WOL not supported on 82598 */ 9389 if (hw->mac.type == ixgbe_mac_82598EB) 9390 return false; 9391 9392 /* check eeprom to see if WOL is enabled for X540 and newer */ 9393 if (hw->mac.type >= ixgbe_mac_X540) { 9394 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 9395 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && 9396 (hw->bus.func == 0))) 9397 return true; 9398 } 9399 9400 /* WOL is determined based on device IDs for 82599 MACs */ 9401 switch (device_id) { 9402 case IXGBE_DEV_ID_82599_SFP: 9403 /* Only these subdevices could supports WOL */ 9404 switch (subdevice_id) { 9405 case IXGBE_SUBDEV_ID_82599_560FLR: 9406 case IXGBE_SUBDEV_ID_82599_LOM_SNAP6: 9407 case IXGBE_SUBDEV_ID_82599_SFP_WOL0: 9408 case IXGBE_SUBDEV_ID_82599_SFP_2OCP: 9409 /* only support first port */ 9410 if (hw->bus.func != 0) 9411 break; 9412 case IXGBE_SUBDEV_ID_82599_SP_560FLR: 9413 case IXGBE_SUBDEV_ID_82599_SFP: 9414 case IXGBE_SUBDEV_ID_82599_RNDC: 9415 case IXGBE_SUBDEV_ID_82599_ECNA_DP: 9416 case IXGBE_SUBDEV_ID_82599_SFP_1OCP: 9417 case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM1: 9418 case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM2: 9419 return true; 9420 } 9421 break; 9422 case IXGBE_DEV_ID_82599EN_SFP: 9423 /* Only these subdevices support WOL */ 9424 switch (subdevice_id) { 9425 case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1: 9426 return true; 9427 } 9428 break; 9429 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: 9430 /* All except this subdevice support WOL */ 9431 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) 9432 return true; 9433 break; 9434 case IXGBE_DEV_ID_82599_KX4: 9435 return true; 9436 default: 9437 break; 9438 } 9439 9440 return false; 9441 } 9442 9443 /** 9444 * ixgbe_probe - Device Initialization Routine 9445 * @pdev: PCI device information struct 9446 * @ent: entry in ixgbe_pci_tbl 9447 * 9448 * Returns 0 on success, negative on failure 9449 * 9450 * ixgbe_probe initializes an adapter identified by a pci_dev structure. 9451 * The OS initialization, configuring of the adapter private structure, 9452 * and a hardware reset occur. 9453 **/ 9454 static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 9455 { 9456 struct net_device *netdev; 9457 struct ixgbe_adapter *adapter = NULL; 9458 struct ixgbe_hw *hw; 9459 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data]; 9460 int i, err, pci_using_dac, expected_gts; 9461 unsigned int indices = MAX_TX_QUEUES; 9462 u8 part_str[IXGBE_PBANUM_LENGTH]; 9463 bool disable_dev = false; 9464 #ifdef IXGBE_FCOE 9465 u16 device_caps; 9466 #endif 9467 u32 eec; 9468 9469 /* Catch broken hardware that put the wrong VF device ID in 9470 * the PCIe SR-IOV capability. 9471 */ 9472 if (pdev->is_virtfn) { 9473 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", 9474 pci_name(pdev), pdev->vendor, pdev->device); 9475 return -EINVAL; 9476 } 9477 9478 err = pci_enable_device_mem(pdev); 9479 if (err) 9480 return err; 9481 9482 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 9483 pci_using_dac = 1; 9484 } else { 9485 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 9486 if (err) { 9487 dev_err(&pdev->dev, 9488 "No usable DMA configuration, aborting\n"); 9489 goto err_dma; 9490 } 9491 pci_using_dac = 0; 9492 } 9493 9494 err = pci_request_mem_regions(pdev, ixgbe_driver_name); 9495 if (err) { 9496 dev_err(&pdev->dev, 9497 "pci_request_selected_regions failed 0x%x\n", err); 9498 goto err_pci_reg; 9499 } 9500 9501 pci_enable_pcie_error_reporting(pdev); 9502 9503 pci_set_master(pdev); 9504 pci_save_state(pdev); 9505 9506 if (ii->mac == ixgbe_mac_82598EB) { 9507 #ifdef CONFIG_IXGBE_DCB 9508 /* 8 TC w/ 4 queues per TC */ 9509 indices = 4 * MAX_TRAFFIC_CLASS; 9510 #else 9511 indices = IXGBE_MAX_RSS_INDICES; 9512 #endif 9513 } 9514 9515 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices); 9516 if (!netdev) { 9517 err = -ENOMEM; 9518 goto err_alloc_etherdev; 9519 } 9520 9521 SET_NETDEV_DEV(netdev, &pdev->dev); 9522 9523 adapter = netdev_priv(netdev); 9524 9525 adapter->netdev = netdev; 9526 adapter->pdev = pdev; 9527 hw = &adapter->hw; 9528 hw->back = adapter; 9529 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 9530 9531 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 9532 pci_resource_len(pdev, 0)); 9533 adapter->io_addr = hw->hw_addr; 9534 if (!hw->hw_addr) { 9535 err = -EIO; 9536 goto err_ioremap; 9537 } 9538 9539 netdev->netdev_ops = &ixgbe_netdev_ops; 9540 ixgbe_set_ethtool_ops(netdev); 9541 netdev->watchdog_timeo = 5 * HZ; 9542 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); 9543 9544 /* Setup hw api */ 9545 hw->mac.ops = *ii->mac_ops; 9546 hw->mac.type = ii->mac; 9547 hw->mvals = ii->mvals; 9548 if (ii->link_ops) 9549 hw->link.ops = *ii->link_ops; 9550 9551 /* EEPROM */ 9552 hw->eeprom.ops = *ii->eeprom_ops; 9553 eec = IXGBE_READ_REG(hw, IXGBE_EEC(hw)); 9554 if (ixgbe_removed(hw->hw_addr)) { 9555 err = -EIO; 9556 goto err_ioremap; 9557 } 9558 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */ 9559 if (!(eec & BIT(8))) 9560 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic; 9561 9562 /* PHY */ 9563 hw->phy.ops = *ii->phy_ops; 9564 hw->phy.sfp_type = ixgbe_sfp_type_unknown; 9565 /* ixgbe_identify_phy_generic will set prtad and mmds properly */ 9566 hw->phy.mdio.prtad = MDIO_PRTAD_NONE; 9567 hw->phy.mdio.mmds = 0; 9568 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22; 9569 hw->phy.mdio.dev = netdev; 9570 hw->phy.mdio.mdio_read = ixgbe_mdio_read; 9571 hw->phy.mdio.mdio_write = ixgbe_mdio_write; 9572 9573 /* setup the private structure */ 9574 err = ixgbe_sw_init(adapter, ii); 9575 if (err) 9576 goto err_sw_init; 9577 9578 /* Make sure the SWFW semaphore is in a valid state */ 9579 if (hw->mac.ops.init_swfw_sync) 9580 hw->mac.ops.init_swfw_sync(hw); 9581 9582 /* Make it possible the adapter to be woken up via WOL */ 9583 switch (adapter->hw.mac.type) { 9584 case ixgbe_mac_82599EB: 9585 case ixgbe_mac_X540: 9586 case ixgbe_mac_X550: 9587 case ixgbe_mac_X550EM_x: 9588 case ixgbe_mac_x550em_a: 9589 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 9590 break; 9591 default: 9592 break; 9593 } 9594 9595 /* 9596 * If there is a fan on this device and it has failed log the 9597 * failure. 9598 */ 9599 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 9600 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 9601 if (esdp & IXGBE_ESDP_SDP1) 9602 e_crit(probe, "Fan has stopped, replace the adapter\n"); 9603 } 9604 9605 if (allow_unsupported_sfp) 9606 hw->allow_unsupported_sfp = allow_unsupported_sfp; 9607 9608 /* reset_hw fills in the perm_addr as well */ 9609 hw->phy.reset_if_overtemp = true; 9610 err = hw->mac.ops.reset_hw(hw); 9611 hw->phy.reset_if_overtemp = false; 9612 ixgbe_set_eee_capable(adapter); 9613 if (err == IXGBE_ERR_SFP_NOT_PRESENT) { 9614 err = 0; 9615 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 9616 e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n"); 9617 e_dev_err("Reload the driver after installing a supported module.\n"); 9618 goto err_sw_init; 9619 } else if (err) { 9620 e_dev_err("HW Init failed: %d\n", err); 9621 goto err_sw_init; 9622 } 9623 9624 #ifdef CONFIG_PCI_IOV 9625 /* SR-IOV not supported on the 82598 */ 9626 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 9627 goto skip_sriov; 9628 /* Mailbox */ 9629 ixgbe_init_mbx_params_pf(hw); 9630 hw->mbx.ops = ii->mbx_ops; 9631 pci_sriov_set_totalvfs(pdev, IXGBE_MAX_VFS_DRV_LIMIT); 9632 ixgbe_enable_sriov(adapter); 9633 skip_sriov: 9634 9635 #endif 9636 netdev->features = NETIF_F_SG | 9637 NETIF_F_TSO | 9638 NETIF_F_TSO6 | 9639 NETIF_F_RXHASH | 9640 NETIF_F_RXCSUM | 9641 NETIF_F_HW_CSUM; 9642 9643 #define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 9644 NETIF_F_GSO_GRE_CSUM | \ 9645 NETIF_F_GSO_IPXIP4 | \ 9646 NETIF_F_GSO_IPXIP6 | \ 9647 NETIF_F_GSO_UDP_TUNNEL | \ 9648 NETIF_F_GSO_UDP_TUNNEL_CSUM) 9649 9650 netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES; 9651 netdev->features |= NETIF_F_GSO_PARTIAL | 9652 IXGBE_GSO_PARTIAL_FEATURES; 9653 9654 if (hw->mac.type >= ixgbe_mac_82599EB) 9655 netdev->features |= NETIF_F_SCTP_CRC; 9656 9657 /* copy netdev features into list of user selectable features */ 9658 netdev->hw_features |= netdev->features | 9659 NETIF_F_HW_VLAN_CTAG_FILTER | 9660 NETIF_F_HW_VLAN_CTAG_RX | 9661 NETIF_F_HW_VLAN_CTAG_TX | 9662 NETIF_F_RXALL | 9663 NETIF_F_HW_L2FW_DOFFLOAD; 9664 9665 if (hw->mac.type >= ixgbe_mac_82599EB) 9666 netdev->hw_features |= NETIF_F_NTUPLE | 9667 NETIF_F_HW_TC; 9668 9669 if (pci_using_dac) 9670 netdev->features |= NETIF_F_HIGHDMA; 9671 9672 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 9673 netdev->hw_enc_features |= netdev->vlan_features; 9674 netdev->mpls_features |= NETIF_F_HW_CSUM; 9675 9676 /* set this bit last since it cannot be part of vlan_features */ 9677 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 9678 NETIF_F_HW_VLAN_CTAG_RX | 9679 NETIF_F_HW_VLAN_CTAG_TX; 9680 9681 netdev->priv_flags |= IFF_UNICAST_FLT; 9682 netdev->priv_flags |= IFF_SUPP_NOFCS; 9683 9684 /* MTU range: 68 - 9710 */ 9685 netdev->min_mtu = ETH_MIN_MTU; 9686 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); 9687 9688 #ifdef CONFIG_IXGBE_DCB 9689 if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) 9690 netdev->dcbnl_ops = &dcbnl_ops; 9691 #endif 9692 9693 #ifdef IXGBE_FCOE 9694 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) { 9695 unsigned int fcoe_l; 9696 9697 if (hw->mac.ops.get_device_caps) { 9698 hw->mac.ops.get_device_caps(hw, &device_caps); 9699 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS) 9700 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 9701 } 9702 9703 9704 fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus()); 9705 adapter->ring_feature[RING_F_FCOE].limit = fcoe_l; 9706 9707 netdev->features |= NETIF_F_FSO | 9708 NETIF_F_FCOE_CRC; 9709 9710 netdev->vlan_features |= NETIF_F_FSO | 9711 NETIF_F_FCOE_CRC | 9712 NETIF_F_FCOE_MTU; 9713 } 9714 #endif /* IXGBE_FCOE */ 9715 9716 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) 9717 netdev->hw_features |= NETIF_F_LRO; 9718 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 9719 netdev->features |= NETIF_F_LRO; 9720 9721 /* make sure the EEPROM is good */ 9722 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) { 9723 e_dev_err("The EEPROM Checksum Is Not Valid\n"); 9724 err = -EIO; 9725 goto err_sw_init; 9726 } 9727 9728 eth_platform_get_mac_address(&adapter->pdev->dev, 9729 adapter->hw.mac.perm_addr); 9730 9731 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len); 9732 9733 if (!is_valid_ether_addr(netdev->dev_addr)) { 9734 e_dev_err("invalid MAC address\n"); 9735 err = -EIO; 9736 goto err_sw_init; 9737 } 9738 9739 /* Set hw->mac.addr to permanent MAC address */ 9740 ether_addr_copy(hw->mac.addr, hw->mac.perm_addr); 9741 ixgbe_mac_set_default_filter(adapter); 9742 9743 setup_timer(&adapter->service_timer, &ixgbe_service_timer, 9744 (unsigned long) adapter); 9745 9746 if (ixgbe_removed(hw->hw_addr)) { 9747 err = -EIO; 9748 goto err_sw_init; 9749 } 9750 INIT_WORK(&adapter->service_task, ixgbe_service_task); 9751 set_bit(__IXGBE_SERVICE_INITED, &adapter->state); 9752 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 9753 9754 err = ixgbe_init_interrupt_scheme(adapter); 9755 if (err) 9756 goto err_sw_init; 9757 9758 /* WOL not supported for all devices */ 9759 adapter->wol = 0; 9760 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap); 9761 hw->wol_enabled = ixgbe_wol_supported(adapter, pdev->device, 9762 pdev->subsystem_device); 9763 if (hw->wol_enabled) 9764 adapter->wol = IXGBE_WUFC_MAG; 9765 9766 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); 9767 9768 /* save off EEPROM version number */ 9769 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh); 9770 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl); 9771 9772 /* pick up the PCI bus settings for reporting later */ 9773 if (ixgbe_pcie_from_parent(hw)) 9774 ixgbe_get_parent_bus_info(adapter); 9775 else 9776 hw->mac.ops.get_bus_info(hw); 9777 9778 /* calculate the expected PCIe bandwidth required for optimal 9779 * performance. Note that some older parts will never have enough 9780 * bandwidth due to being older generation PCIe parts. We clamp these 9781 * parts to ensure no warning is displayed if it can't be fixed. 9782 */ 9783 switch (hw->mac.type) { 9784 case ixgbe_mac_82598EB: 9785 expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16); 9786 break; 9787 default: 9788 expected_gts = ixgbe_enumerate_functions(adapter) * 10; 9789 break; 9790 } 9791 9792 /* don't check link if we failed to enumerate functions */ 9793 if (expected_gts > 0) 9794 ixgbe_check_minimum_link(adapter, expected_gts); 9795 9796 err = ixgbe_read_pba_string_generic(hw, part_str, sizeof(part_str)); 9797 if (err) 9798 strlcpy(part_str, "Unknown", sizeof(part_str)); 9799 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 9800 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n", 9801 hw->mac.type, hw->phy.type, hw->phy.sfp_type, 9802 part_str); 9803 else 9804 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n", 9805 hw->mac.type, hw->phy.type, part_str); 9806 9807 e_dev_info("%pM\n", netdev->dev_addr); 9808 9809 /* reset the hardware with the new settings */ 9810 err = hw->mac.ops.start_hw(hw); 9811 if (err == IXGBE_ERR_EEPROM_VERSION) { 9812 /* We are running on a pre-production device, log a warning */ 9813 e_dev_warn("This device is a pre-production adapter/LOM. " 9814 "Please be aware there may be issues associated " 9815 "with your hardware. If you are experiencing " 9816 "problems please contact your Intel or hardware " 9817 "representative who provided you with this " 9818 "hardware.\n"); 9819 } 9820 strcpy(netdev->name, "eth%d"); 9821 err = register_netdev(netdev); 9822 if (err) 9823 goto err_register; 9824 9825 pci_set_drvdata(pdev, adapter); 9826 9827 /* power down the optics for 82599 SFP+ fiber */ 9828 if (hw->mac.ops.disable_tx_laser) 9829 hw->mac.ops.disable_tx_laser(hw); 9830 9831 /* carrier off reporting is important to ethtool even BEFORE open */ 9832 netif_carrier_off(netdev); 9833 9834 #ifdef CONFIG_IXGBE_DCA 9835 if (dca_add_requester(&pdev->dev) == 0) { 9836 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 9837 ixgbe_setup_dca(adapter); 9838 } 9839 #endif 9840 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 9841 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs); 9842 for (i = 0; i < adapter->num_vfs; i++) 9843 ixgbe_vf_configuration(pdev, (i | 0x10000000)); 9844 } 9845 9846 /* firmware requires driver version to be 0xFFFFFFFF 9847 * since os does not support feature 9848 */ 9849 if (hw->mac.ops.set_fw_drv_ver) 9850 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 0xFF, 9851 sizeof(ixgbe_driver_version) - 1, 9852 ixgbe_driver_version); 9853 9854 /* add san mac addr to netdev */ 9855 ixgbe_add_sanmac_netdev(netdev); 9856 9857 e_dev_info("%s\n", ixgbe_default_device_descr); 9858 9859 #ifdef CONFIG_IXGBE_HWMON 9860 if (ixgbe_sysfs_init(adapter)) 9861 e_err(probe, "failed to allocate sysfs resources\n"); 9862 #endif /* CONFIG_IXGBE_HWMON */ 9863 9864 ixgbe_dbg_adapter_init(adapter); 9865 9866 /* setup link for SFP devices with MNG FW, else wait for IXGBE_UP */ 9867 if (ixgbe_mng_enabled(hw) && ixgbe_is_sfp(hw) && hw->mac.ops.setup_link) 9868 hw->mac.ops.setup_link(hw, 9869 IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL, 9870 true); 9871 9872 return 0; 9873 9874 err_register: 9875 ixgbe_release_hw_control(adapter); 9876 ixgbe_clear_interrupt_scheme(adapter); 9877 err_sw_init: 9878 ixgbe_disable_sriov(adapter); 9879 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 9880 iounmap(adapter->io_addr); 9881 kfree(adapter->jump_tables[0]); 9882 kfree(adapter->mac_table); 9883 err_ioremap: 9884 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state); 9885 free_netdev(netdev); 9886 err_alloc_etherdev: 9887 pci_release_mem_regions(pdev); 9888 err_pci_reg: 9889 err_dma: 9890 if (!adapter || disable_dev) 9891 pci_disable_device(pdev); 9892 return err; 9893 } 9894 9895 /** 9896 * ixgbe_remove - Device Removal Routine 9897 * @pdev: PCI device information struct 9898 * 9899 * ixgbe_remove is called by the PCI subsystem to alert the driver 9900 * that it should release a PCI device. The could be caused by a 9901 * Hot-Plug event, or because the driver is going to be removed from 9902 * memory. 9903 **/ 9904 static void ixgbe_remove(struct pci_dev *pdev) 9905 { 9906 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 9907 struct net_device *netdev; 9908 bool disable_dev; 9909 int i; 9910 9911 /* if !adapter then we already cleaned up in probe */ 9912 if (!adapter) 9913 return; 9914 9915 netdev = adapter->netdev; 9916 ixgbe_dbg_adapter_exit(adapter); 9917 9918 set_bit(__IXGBE_REMOVING, &adapter->state); 9919 cancel_work_sync(&adapter->service_task); 9920 9921 9922 #ifdef CONFIG_IXGBE_DCA 9923 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 9924 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 9925 dca_remove_requester(&pdev->dev); 9926 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 9927 IXGBE_DCA_CTRL_DCA_DISABLE); 9928 } 9929 9930 #endif 9931 #ifdef CONFIG_IXGBE_HWMON 9932 ixgbe_sysfs_exit(adapter); 9933 #endif /* CONFIG_IXGBE_HWMON */ 9934 9935 /* remove the added san mac */ 9936 ixgbe_del_sanmac_netdev(netdev); 9937 9938 #ifdef CONFIG_PCI_IOV 9939 ixgbe_disable_sriov(adapter); 9940 #endif 9941 if (netdev->reg_state == NETREG_REGISTERED) 9942 unregister_netdev(netdev); 9943 9944 ixgbe_clear_interrupt_scheme(adapter); 9945 9946 ixgbe_release_hw_control(adapter); 9947 9948 #ifdef CONFIG_DCB 9949 kfree(adapter->ixgbe_ieee_pfc); 9950 kfree(adapter->ixgbe_ieee_ets); 9951 9952 #endif 9953 iounmap(adapter->io_addr); 9954 pci_release_mem_regions(pdev); 9955 9956 e_dev_info("complete\n"); 9957 9958 for (i = 0; i < IXGBE_MAX_LINK_HANDLE; i++) { 9959 if (adapter->jump_tables[i]) { 9960 kfree(adapter->jump_tables[i]->input); 9961 kfree(adapter->jump_tables[i]->mask); 9962 } 9963 kfree(adapter->jump_tables[i]); 9964 } 9965 9966 kfree(adapter->mac_table); 9967 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state); 9968 free_netdev(netdev); 9969 9970 pci_disable_pcie_error_reporting(pdev); 9971 9972 if (disable_dev) 9973 pci_disable_device(pdev); 9974 } 9975 9976 /** 9977 * ixgbe_io_error_detected - called when PCI error is detected 9978 * @pdev: Pointer to PCI device 9979 * @state: The current pci connection state 9980 * 9981 * This function is called after a PCI bus error affecting 9982 * this device has been detected. 9983 */ 9984 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, 9985 pci_channel_state_t state) 9986 { 9987 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 9988 struct net_device *netdev = adapter->netdev; 9989 9990 #ifdef CONFIG_PCI_IOV 9991 struct ixgbe_hw *hw = &adapter->hw; 9992 struct pci_dev *bdev, *vfdev; 9993 u32 dw0, dw1, dw2, dw3; 9994 int vf, pos; 9995 u16 req_id, pf_func; 9996 9997 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 9998 adapter->num_vfs == 0) 9999 goto skip_bad_vf_detection; 10000 10001 bdev = pdev->bus->self; 10002 while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT)) 10003 bdev = bdev->bus->self; 10004 10005 if (!bdev) 10006 goto skip_bad_vf_detection; 10007 10008 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR); 10009 if (!pos) 10010 goto skip_bad_vf_detection; 10011 10012 dw0 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG); 10013 dw1 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 4); 10014 dw2 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 8); 10015 dw3 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 12); 10016 if (ixgbe_removed(hw->hw_addr)) 10017 goto skip_bad_vf_detection; 10018 10019 req_id = dw1 >> 16; 10020 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */ 10021 if (!(req_id & 0x0080)) 10022 goto skip_bad_vf_detection; 10023 10024 pf_func = req_id & 0x01; 10025 if ((pf_func & 1) == (pdev->devfn & 1)) { 10026 unsigned int device_id; 10027 10028 vf = (req_id & 0x7F) >> 1; 10029 e_dev_err("VF %d has caused a PCIe error\n", vf); 10030 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: " 10031 "%8.8x\tdw3: %8.8x\n", 10032 dw0, dw1, dw2, dw3); 10033 switch (adapter->hw.mac.type) { 10034 case ixgbe_mac_82599EB: 10035 device_id = IXGBE_82599_VF_DEVICE_ID; 10036 break; 10037 case ixgbe_mac_X540: 10038 device_id = IXGBE_X540_VF_DEVICE_ID; 10039 break; 10040 case ixgbe_mac_X550: 10041 device_id = IXGBE_DEV_ID_X550_VF; 10042 break; 10043 case ixgbe_mac_X550EM_x: 10044 device_id = IXGBE_DEV_ID_X550EM_X_VF; 10045 break; 10046 case ixgbe_mac_x550em_a: 10047 device_id = IXGBE_DEV_ID_X550EM_A_VF; 10048 break; 10049 default: 10050 device_id = 0; 10051 break; 10052 } 10053 10054 /* Find the pci device of the offending VF */ 10055 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL); 10056 while (vfdev) { 10057 if (vfdev->devfn == (req_id & 0xFF)) 10058 break; 10059 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, 10060 device_id, vfdev); 10061 } 10062 /* 10063 * There's a slim chance the VF could have been hot plugged, 10064 * so if it is no longer present we don't need to issue the 10065 * VFLR. Just clean up the AER in that case. 10066 */ 10067 if (vfdev) { 10068 ixgbe_issue_vf_flr(adapter, vfdev); 10069 /* Free device reference count */ 10070 pci_dev_put(vfdev); 10071 } 10072 10073 pci_cleanup_aer_uncorrect_error_status(pdev); 10074 } 10075 10076 /* 10077 * Even though the error may have occurred on the other port 10078 * we still need to increment the vf error reference count for 10079 * both ports because the I/O resume function will be called 10080 * for both of them. 10081 */ 10082 adapter->vferr_refcount++; 10083 10084 return PCI_ERS_RESULT_RECOVERED; 10085 10086 skip_bad_vf_detection: 10087 #endif /* CONFIG_PCI_IOV */ 10088 if (!test_bit(__IXGBE_SERVICE_INITED, &adapter->state)) 10089 return PCI_ERS_RESULT_DISCONNECT; 10090 10091 rtnl_lock(); 10092 netif_device_detach(netdev); 10093 10094 if (state == pci_channel_io_perm_failure) { 10095 rtnl_unlock(); 10096 return PCI_ERS_RESULT_DISCONNECT; 10097 } 10098 10099 if (netif_running(netdev)) 10100 ixgbe_close_suspend(adapter); 10101 10102 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state)) 10103 pci_disable_device(pdev); 10104 rtnl_unlock(); 10105 10106 /* Request a slot reset. */ 10107 return PCI_ERS_RESULT_NEED_RESET; 10108 } 10109 10110 /** 10111 * ixgbe_io_slot_reset - called after the pci bus has been reset. 10112 * @pdev: Pointer to PCI device 10113 * 10114 * Restart the card from scratch, as if from a cold-boot. 10115 */ 10116 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) 10117 { 10118 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 10119 pci_ers_result_t result; 10120 int err; 10121 10122 if (pci_enable_device_mem(pdev)) { 10123 e_err(probe, "Cannot re-enable PCI device after reset.\n"); 10124 result = PCI_ERS_RESULT_DISCONNECT; 10125 } else { 10126 smp_mb__before_atomic(); 10127 clear_bit(__IXGBE_DISABLED, &adapter->state); 10128 adapter->hw.hw_addr = adapter->io_addr; 10129 pci_set_master(pdev); 10130 pci_restore_state(pdev); 10131 pci_save_state(pdev); 10132 10133 pci_wake_from_d3(pdev, false); 10134 10135 ixgbe_reset(adapter); 10136 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 10137 result = PCI_ERS_RESULT_RECOVERED; 10138 } 10139 10140 err = pci_cleanup_aer_uncorrect_error_status(pdev); 10141 if (err) { 10142 e_dev_err("pci_cleanup_aer_uncorrect_error_status " 10143 "failed 0x%0x\n", err); 10144 /* non-fatal, continue */ 10145 } 10146 10147 return result; 10148 } 10149 10150 /** 10151 * ixgbe_io_resume - called when traffic can start flowing again. 10152 * @pdev: Pointer to PCI device 10153 * 10154 * This callback is called when the error recovery driver tells us that 10155 * its OK to resume normal operation. 10156 */ 10157 static void ixgbe_io_resume(struct pci_dev *pdev) 10158 { 10159 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 10160 struct net_device *netdev = adapter->netdev; 10161 10162 #ifdef CONFIG_PCI_IOV 10163 if (adapter->vferr_refcount) { 10164 e_info(drv, "Resuming after VF err\n"); 10165 adapter->vferr_refcount--; 10166 return; 10167 } 10168 10169 #endif 10170 rtnl_lock(); 10171 if (netif_running(netdev)) 10172 ixgbe_open(netdev); 10173 10174 netif_device_attach(netdev); 10175 rtnl_unlock(); 10176 } 10177 10178 static const struct pci_error_handlers ixgbe_err_handler = { 10179 .error_detected = ixgbe_io_error_detected, 10180 .slot_reset = ixgbe_io_slot_reset, 10181 .resume = ixgbe_io_resume, 10182 }; 10183 10184 static struct pci_driver ixgbe_driver = { 10185 .name = ixgbe_driver_name, 10186 .id_table = ixgbe_pci_tbl, 10187 .probe = ixgbe_probe, 10188 .remove = ixgbe_remove, 10189 #ifdef CONFIG_PM 10190 .suspend = ixgbe_suspend, 10191 .resume = ixgbe_resume, 10192 #endif 10193 .shutdown = ixgbe_shutdown, 10194 .sriov_configure = ixgbe_pci_sriov_configure, 10195 .err_handler = &ixgbe_err_handler 10196 }; 10197 10198 /** 10199 * ixgbe_init_module - Driver Registration Routine 10200 * 10201 * ixgbe_init_module is the first routine called when the driver is 10202 * loaded. All it does is register with the PCI subsystem. 10203 **/ 10204 static int __init ixgbe_init_module(void) 10205 { 10206 int ret; 10207 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version); 10208 pr_info("%s\n", ixgbe_copyright); 10209 10210 ixgbe_wq = create_singlethread_workqueue(ixgbe_driver_name); 10211 if (!ixgbe_wq) { 10212 pr_err("%s: Failed to create workqueue\n", ixgbe_driver_name); 10213 return -ENOMEM; 10214 } 10215 10216 ixgbe_dbg_init(); 10217 10218 ret = pci_register_driver(&ixgbe_driver); 10219 if (ret) { 10220 destroy_workqueue(ixgbe_wq); 10221 ixgbe_dbg_exit(); 10222 return ret; 10223 } 10224 10225 #ifdef CONFIG_IXGBE_DCA 10226 dca_register_notify(&dca_notifier); 10227 #endif 10228 10229 return 0; 10230 } 10231 10232 module_init(ixgbe_init_module); 10233 10234 /** 10235 * ixgbe_exit_module - Driver Exit Cleanup Routine 10236 * 10237 * ixgbe_exit_module is called just before the driver is removed 10238 * from memory. 10239 **/ 10240 static void __exit ixgbe_exit_module(void) 10241 { 10242 #ifdef CONFIG_IXGBE_DCA 10243 dca_unregister_notify(&dca_notifier); 10244 #endif 10245 pci_unregister_driver(&ixgbe_driver); 10246 10247 ixgbe_dbg_exit(); 10248 if (ixgbe_wq) { 10249 destroy_workqueue(ixgbe_wq); 10250 ixgbe_wq = NULL; 10251 } 10252 } 10253 10254 #ifdef CONFIG_IXGBE_DCA 10255 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, 10256 void *p) 10257 { 10258 int ret_val; 10259 10260 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event, 10261 __ixgbe_notify_dca); 10262 10263 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 10264 } 10265 10266 #endif /* CONFIG_IXGBE_DCA */ 10267 10268 module_exit(ixgbe_exit_module); 10269 10270 /* ixgbe_main.c */ 10271