1 // SPDX-License-Identifier: GPL-2.0 2 /* Intel(R) Gigabit Ethernet Linux driver 3 * Copyright(c) 2007-2014 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, see <http://www.gnu.org/licenses/>. 16 * 17 * The full GNU General Public License is included in this distribution in 18 * the file called "COPYING". 19 * 20 * Contact Information: 21 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 22 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 23 */ 24 25 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 26 27 #include <linux/module.h> 28 #include <linux/types.h> 29 #include <linux/init.h> 30 #include <linux/bitops.h> 31 #include <linux/vmalloc.h> 32 #include <linux/pagemap.h> 33 #include <linux/netdevice.h> 34 #include <linux/ipv6.h> 35 #include <linux/slab.h> 36 #include <net/checksum.h> 37 #include <net/ip6_checksum.h> 38 #include <net/pkt_sched.h> 39 #include <net/pkt_cls.h> 40 #include <linux/net_tstamp.h> 41 #include <linux/mii.h> 42 #include <linux/ethtool.h> 43 #include <linux/if.h> 44 #include <linux/if_vlan.h> 45 #include <linux/pci.h> 46 #include <linux/pci-aspm.h> 47 #include <linux/delay.h> 48 #include <linux/interrupt.h> 49 #include <linux/ip.h> 50 #include <linux/tcp.h> 51 #include <linux/sctp.h> 52 #include <linux/if_ether.h> 53 #include <linux/aer.h> 54 #include <linux/prefetch.h> 55 #include <linux/pm_runtime.h> 56 #include <linux/etherdevice.h> 57 #ifdef CONFIG_IGB_DCA 58 #include <linux/dca.h> 59 #endif 60 #include <linux/i2c.h> 61 #include "igb.h" 62 63 #define MAJ 5 64 #define MIN 4 65 #define BUILD 0 66 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ 67 __stringify(BUILD) "-k" 68 69 enum queue_mode { 70 QUEUE_MODE_STRICT_PRIORITY, 71 QUEUE_MODE_STREAM_RESERVATION, 72 }; 73 74 enum tx_queue_prio { 75 TX_QUEUE_PRIO_HIGH, 76 TX_QUEUE_PRIO_LOW, 77 }; 78 79 char igb_driver_name[] = "igb"; 80 char igb_driver_version[] = DRV_VERSION; 81 static const char igb_driver_string[] = 82 "Intel(R) Gigabit Ethernet Network Driver"; 83 static const char igb_copyright[] = 84 "Copyright (c) 2007-2014 Intel Corporation."; 85 86 static const struct e1000_info *igb_info_tbl[] = { 87 [board_82575] = &e1000_82575_info, 88 }; 89 90 static const struct pci_device_id igb_pci_tbl[] = { 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) }, 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 }, 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 }, 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 }, 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 }, 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 }, 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 }, 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 }, 101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 }, 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 }, 103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 }, 104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 }, 105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 }, 106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 }, 107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 }, 108 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 }, 109 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 }, 110 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 }, 111 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 }, 112 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 }, 113 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 }, 114 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 }, 115 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 }, 116 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 }, 117 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 }, 118 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 }, 119 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 }, 120 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 }, 121 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 }, 122 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 }, 123 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 }, 124 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 }, 125 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 }, 126 /* required last entry */ 127 {0, } 128 }; 129 130 MODULE_DEVICE_TABLE(pci, igb_pci_tbl); 131 132 static int igb_setup_all_tx_resources(struct igb_adapter *); 133 static int igb_setup_all_rx_resources(struct igb_adapter *); 134 static void igb_free_all_tx_resources(struct igb_adapter *); 135 static void igb_free_all_rx_resources(struct igb_adapter *); 136 static void igb_setup_mrqc(struct igb_adapter *); 137 static int igb_probe(struct pci_dev *, const struct pci_device_id *); 138 static void igb_remove(struct pci_dev *pdev); 139 static int igb_sw_init(struct igb_adapter *); 140 int igb_open(struct net_device *); 141 int igb_close(struct net_device *); 142 static void igb_configure(struct igb_adapter *); 143 static void igb_configure_tx(struct igb_adapter *); 144 static void igb_configure_rx(struct igb_adapter *); 145 static void igb_clean_all_tx_rings(struct igb_adapter *); 146 static void igb_clean_all_rx_rings(struct igb_adapter *); 147 static void igb_clean_tx_ring(struct igb_ring *); 148 static void igb_clean_rx_ring(struct igb_ring *); 149 static void igb_set_rx_mode(struct net_device *); 150 static void igb_update_phy_info(struct timer_list *); 151 static void igb_watchdog(struct timer_list *); 152 static void igb_watchdog_task(struct work_struct *); 153 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *); 154 static void igb_get_stats64(struct net_device *dev, 155 struct rtnl_link_stats64 *stats); 156 static int igb_change_mtu(struct net_device *, int); 157 static int igb_set_mac(struct net_device *, void *); 158 static void igb_set_uta(struct igb_adapter *adapter, bool set); 159 static irqreturn_t igb_intr(int irq, void *); 160 static irqreturn_t igb_intr_msi(int irq, void *); 161 static irqreturn_t igb_msix_other(int irq, void *); 162 static irqreturn_t igb_msix_ring(int irq, void *); 163 #ifdef CONFIG_IGB_DCA 164 static void igb_update_dca(struct igb_q_vector *); 165 static void igb_setup_dca(struct igb_adapter *); 166 #endif /* CONFIG_IGB_DCA */ 167 static int igb_poll(struct napi_struct *, int); 168 static bool igb_clean_tx_irq(struct igb_q_vector *, int); 169 static int igb_clean_rx_irq(struct igb_q_vector *, int); 170 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); 171 static void igb_tx_timeout(struct net_device *); 172 static void igb_reset_task(struct work_struct *); 173 static void igb_vlan_mode(struct net_device *netdev, 174 netdev_features_t features); 175 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16); 176 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16); 177 static void igb_restore_vlan(struct igb_adapter *); 178 static void igb_rar_set_index(struct igb_adapter *, u32); 179 static void igb_ping_all_vfs(struct igb_adapter *); 180 static void igb_msg_task(struct igb_adapter *); 181 static void igb_vmm_control(struct igb_adapter *); 182 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *); 183 static void igb_flush_mac_table(struct igb_adapter *); 184 static int igb_available_rars(struct igb_adapter *, u8); 185 static void igb_set_default_mac_filter(struct igb_adapter *); 186 static int igb_uc_sync(struct net_device *, const unsigned char *); 187 static int igb_uc_unsync(struct net_device *, const unsigned char *); 188 static void igb_restore_vf_multicasts(struct igb_adapter *adapter); 189 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac); 190 static int igb_ndo_set_vf_vlan(struct net_device *netdev, 191 int vf, u16 vlan, u8 qos, __be16 vlan_proto); 192 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int); 193 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, 194 bool setting); 195 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, 196 bool setting); 197 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf, 198 struct ifla_vf_info *ivi); 199 static void igb_check_vf_rate_limit(struct igb_adapter *); 200 static void igb_nfc_filter_exit(struct igb_adapter *adapter); 201 static void igb_nfc_filter_restore(struct igb_adapter *adapter); 202 203 #ifdef CONFIG_PCI_IOV 204 static int igb_vf_configure(struct igb_adapter *adapter, int vf); 205 static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs); 206 static int igb_disable_sriov(struct pci_dev *dev); 207 static int igb_pci_disable_sriov(struct pci_dev *dev); 208 #endif 209 210 static int igb_suspend(struct device *); 211 static int igb_resume(struct device *); 212 static int igb_runtime_suspend(struct device *dev); 213 static int igb_runtime_resume(struct device *dev); 214 static int igb_runtime_idle(struct device *dev); 215 static const struct dev_pm_ops igb_pm_ops = { 216 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume) 217 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume, 218 igb_runtime_idle) 219 }; 220 static void igb_shutdown(struct pci_dev *); 221 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs); 222 #ifdef CONFIG_IGB_DCA 223 static int igb_notify_dca(struct notifier_block *, unsigned long, void *); 224 static struct notifier_block dca_notifier = { 225 .notifier_call = igb_notify_dca, 226 .next = NULL, 227 .priority = 0 228 }; 229 #endif 230 #ifdef CONFIG_NET_POLL_CONTROLLER 231 /* for netdump / net console */ 232 static void igb_netpoll(struct net_device *); 233 #endif 234 #ifdef CONFIG_PCI_IOV 235 static unsigned int max_vfs; 236 module_param(max_vfs, uint, 0); 237 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function"); 238 #endif /* CONFIG_PCI_IOV */ 239 240 static pci_ers_result_t igb_io_error_detected(struct pci_dev *, 241 pci_channel_state_t); 242 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *); 243 static void igb_io_resume(struct pci_dev *); 244 245 static const struct pci_error_handlers igb_err_handler = { 246 .error_detected = igb_io_error_detected, 247 .slot_reset = igb_io_slot_reset, 248 .resume = igb_io_resume, 249 }; 250 251 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba); 252 253 static struct pci_driver igb_driver = { 254 .name = igb_driver_name, 255 .id_table = igb_pci_tbl, 256 .probe = igb_probe, 257 .remove = igb_remove, 258 #ifdef CONFIG_PM 259 .driver.pm = &igb_pm_ops, 260 #endif 261 .shutdown = igb_shutdown, 262 .sriov_configure = igb_pci_sriov_configure, 263 .err_handler = &igb_err_handler 264 }; 265 266 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 267 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver"); 268 MODULE_LICENSE("GPL"); 269 MODULE_VERSION(DRV_VERSION); 270 271 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 272 static int debug = -1; 273 module_param(debug, int, 0); 274 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 275 276 struct igb_reg_info { 277 u32 ofs; 278 char *name; 279 }; 280 281 static const struct igb_reg_info igb_reg_info_tbl[] = { 282 283 /* General Registers */ 284 {E1000_CTRL, "CTRL"}, 285 {E1000_STATUS, "STATUS"}, 286 {E1000_CTRL_EXT, "CTRL_EXT"}, 287 288 /* Interrupt Registers */ 289 {E1000_ICR, "ICR"}, 290 291 /* RX Registers */ 292 {E1000_RCTL, "RCTL"}, 293 {E1000_RDLEN(0), "RDLEN"}, 294 {E1000_RDH(0), "RDH"}, 295 {E1000_RDT(0), "RDT"}, 296 {E1000_RXDCTL(0), "RXDCTL"}, 297 {E1000_RDBAL(0), "RDBAL"}, 298 {E1000_RDBAH(0), "RDBAH"}, 299 300 /* TX Registers */ 301 {E1000_TCTL, "TCTL"}, 302 {E1000_TDBAL(0), "TDBAL"}, 303 {E1000_TDBAH(0), "TDBAH"}, 304 {E1000_TDLEN(0), "TDLEN"}, 305 {E1000_TDH(0), "TDH"}, 306 {E1000_TDT(0), "TDT"}, 307 {E1000_TXDCTL(0), "TXDCTL"}, 308 {E1000_TDFH, "TDFH"}, 309 {E1000_TDFT, "TDFT"}, 310 {E1000_TDFHS, "TDFHS"}, 311 {E1000_TDFPC, "TDFPC"}, 312 313 /* List Terminator */ 314 {} 315 }; 316 317 /* igb_regdump - register printout routine */ 318 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo) 319 { 320 int n = 0; 321 char rname[16]; 322 u32 regs[8]; 323 324 switch (reginfo->ofs) { 325 case E1000_RDLEN(0): 326 for (n = 0; n < 4; n++) 327 regs[n] = rd32(E1000_RDLEN(n)); 328 break; 329 case E1000_RDH(0): 330 for (n = 0; n < 4; n++) 331 regs[n] = rd32(E1000_RDH(n)); 332 break; 333 case E1000_RDT(0): 334 for (n = 0; n < 4; n++) 335 regs[n] = rd32(E1000_RDT(n)); 336 break; 337 case E1000_RXDCTL(0): 338 for (n = 0; n < 4; n++) 339 regs[n] = rd32(E1000_RXDCTL(n)); 340 break; 341 case E1000_RDBAL(0): 342 for (n = 0; n < 4; n++) 343 regs[n] = rd32(E1000_RDBAL(n)); 344 break; 345 case E1000_RDBAH(0): 346 for (n = 0; n < 4; n++) 347 regs[n] = rd32(E1000_RDBAH(n)); 348 break; 349 case E1000_TDBAL(0): 350 for (n = 0; n < 4; n++) 351 regs[n] = rd32(E1000_RDBAL(n)); 352 break; 353 case E1000_TDBAH(0): 354 for (n = 0; n < 4; n++) 355 regs[n] = rd32(E1000_TDBAH(n)); 356 break; 357 case E1000_TDLEN(0): 358 for (n = 0; n < 4; n++) 359 regs[n] = rd32(E1000_TDLEN(n)); 360 break; 361 case E1000_TDH(0): 362 for (n = 0; n < 4; n++) 363 regs[n] = rd32(E1000_TDH(n)); 364 break; 365 case E1000_TDT(0): 366 for (n = 0; n < 4; n++) 367 regs[n] = rd32(E1000_TDT(n)); 368 break; 369 case E1000_TXDCTL(0): 370 for (n = 0; n < 4; n++) 371 regs[n] = rd32(E1000_TXDCTL(n)); 372 break; 373 default: 374 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs)); 375 return; 376 } 377 378 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]"); 379 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], 380 regs[2], regs[3]); 381 } 382 383 /* igb_dump - Print registers, Tx-rings and Rx-rings */ 384 static void igb_dump(struct igb_adapter *adapter) 385 { 386 struct net_device *netdev = adapter->netdev; 387 struct e1000_hw *hw = &adapter->hw; 388 struct igb_reg_info *reginfo; 389 struct igb_ring *tx_ring; 390 union e1000_adv_tx_desc *tx_desc; 391 struct my_u0 { u64 a; u64 b; } *u0; 392 struct igb_ring *rx_ring; 393 union e1000_adv_rx_desc *rx_desc; 394 u32 staterr; 395 u16 i, n; 396 397 if (!netif_msg_hw(adapter)) 398 return; 399 400 /* Print netdevice Info */ 401 if (netdev) { 402 dev_info(&adapter->pdev->dev, "Net device Info\n"); 403 pr_info("Device Name state trans_start\n"); 404 pr_info("%-15s %016lX %016lX\n", netdev->name, 405 netdev->state, dev_trans_start(netdev)); 406 } 407 408 /* Print Registers */ 409 dev_info(&adapter->pdev->dev, "Register Dump\n"); 410 pr_info(" Register Name Value\n"); 411 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl; 412 reginfo->name; reginfo++) { 413 igb_regdump(hw, reginfo); 414 } 415 416 /* Print TX Ring Summary */ 417 if (!netdev || !netif_running(netdev)) 418 goto exit; 419 420 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 421 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 422 for (n = 0; n < adapter->num_tx_queues; n++) { 423 struct igb_tx_buffer *buffer_info; 424 tx_ring = adapter->tx_ring[n]; 425 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 426 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", 427 n, tx_ring->next_to_use, tx_ring->next_to_clean, 428 (u64)dma_unmap_addr(buffer_info, dma), 429 dma_unmap_len(buffer_info, len), 430 buffer_info->next_to_watch, 431 (u64)buffer_info->time_stamp); 432 } 433 434 /* Print TX Rings */ 435 if (!netif_msg_tx_done(adapter)) 436 goto rx_ring_summary; 437 438 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 439 440 /* Transmit Descriptor Formats 441 * 442 * Advanced Transmit Descriptor 443 * +--------------------------------------------------------------+ 444 * 0 | Buffer Address [63:0] | 445 * +--------------------------------------------------------------+ 446 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN | 447 * +--------------------------------------------------------------+ 448 * 63 46 45 40 39 38 36 35 32 31 24 15 0 449 */ 450 451 for (n = 0; n < adapter->num_tx_queues; n++) { 452 tx_ring = adapter->tx_ring[n]; 453 pr_info("------------------------------------\n"); 454 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 455 pr_info("------------------------------------\n"); 456 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n"); 457 458 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 459 const char *next_desc; 460 struct igb_tx_buffer *buffer_info; 461 tx_desc = IGB_TX_DESC(tx_ring, i); 462 buffer_info = &tx_ring->tx_buffer_info[i]; 463 u0 = (struct my_u0 *)tx_desc; 464 if (i == tx_ring->next_to_use && 465 i == tx_ring->next_to_clean) 466 next_desc = " NTC/U"; 467 else if (i == tx_ring->next_to_use) 468 next_desc = " NTU"; 469 else if (i == tx_ring->next_to_clean) 470 next_desc = " NTC"; 471 else 472 next_desc = ""; 473 474 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n", 475 i, le64_to_cpu(u0->a), 476 le64_to_cpu(u0->b), 477 (u64)dma_unmap_addr(buffer_info, dma), 478 dma_unmap_len(buffer_info, len), 479 buffer_info->next_to_watch, 480 (u64)buffer_info->time_stamp, 481 buffer_info->skb, next_desc); 482 483 if (netif_msg_pktdata(adapter) && buffer_info->skb) 484 print_hex_dump(KERN_INFO, "", 485 DUMP_PREFIX_ADDRESS, 486 16, 1, buffer_info->skb->data, 487 dma_unmap_len(buffer_info, len), 488 true); 489 } 490 } 491 492 /* Print RX Rings Summary */ 493 rx_ring_summary: 494 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 495 pr_info("Queue [NTU] [NTC]\n"); 496 for (n = 0; n < adapter->num_rx_queues; n++) { 497 rx_ring = adapter->rx_ring[n]; 498 pr_info(" %5d %5X %5X\n", 499 n, rx_ring->next_to_use, rx_ring->next_to_clean); 500 } 501 502 /* Print RX Rings */ 503 if (!netif_msg_rx_status(adapter)) 504 goto exit; 505 506 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 507 508 /* Advanced Receive Descriptor (Read) Format 509 * 63 1 0 510 * +-----------------------------------------------------+ 511 * 0 | Packet Buffer Address [63:1] |A0/NSE| 512 * +----------------------------------------------+------+ 513 * 8 | Header Buffer Address [63:1] | DD | 514 * +-----------------------------------------------------+ 515 * 516 * 517 * Advanced Receive Descriptor (Write-Back) Format 518 * 519 * 63 48 47 32 31 30 21 20 17 16 4 3 0 520 * +------------------------------------------------------+ 521 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | 522 * | Checksum Ident | | | | Type | Type | 523 * +------------------------------------------------------+ 524 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 525 * +------------------------------------------------------+ 526 * 63 48 47 32 31 20 19 0 527 */ 528 529 for (n = 0; n < adapter->num_rx_queues; n++) { 530 rx_ring = adapter->rx_ring[n]; 531 pr_info("------------------------------------\n"); 532 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 533 pr_info("------------------------------------\n"); 534 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); 535 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n"); 536 537 for (i = 0; i < rx_ring->count; i++) { 538 const char *next_desc; 539 struct igb_rx_buffer *buffer_info; 540 buffer_info = &rx_ring->rx_buffer_info[i]; 541 rx_desc = IGB_RX_DESC(rx_ring, i); 542 u0 = (struct my_u0 *)rx_desc; 543 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 544 545 if (i == rx_ring->next_to_use) 546 next_desc = " NTU"; 547 else if (i == rx_ring->next_to_clean) 548 next_desc = " NTC"; 549 else 550 next_desc = ""; 551 552 if (staterr & E1000_RXD_STAT_DD) { 553 /* Descriptor Done */ 554 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n", 555 "RWB", i, 556 le64_to_cpu(u0->a), 557 le64_to_cpu(u0->b), 558 next_desc); 559 } else { 560 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n", 561 "R ", i, 562 le64_to_cpu(u0->a), 563 le64_to_cpu(u0->b), 564 (u64)buffer_info->dma, 565 next_desc); 566 567 if (netif_msg_pktdata(adapter) && 568 buffer_info->dma && buffer_info->page) { 569 print_hex_dump(KERN_INFO, "", 570 DUMP_PREFIX_ADDRESS, 571 16, 1, 572 page_address(buffer_info->page) + 573 buffer_info->page_offset, 574 igb_rx_bufsz(rx_ring), true); 575 } 576 } 577 } 578 } 579 580 exit: 581 return; 582 } 583 584 /** 585 * igb_get_i2c_data - Reads the I2C SDA data bit 586 * @hw: pointer to hardware structure 587 * @i2cctl: Current value of I2CCTL register 588 * 589 * Returns the I2C data bit value 590 **/ 591 static int igb_get_i2c_data(void *data) 592 { 593 struct igb_adapter *adapter = (struct igb_adapter *)data; 594 struct e1000_hw *hw = &adapter->hw; 595 s32 i2cctl = rd32(E1000_I2CPARAMS); 596 597 return !!(i2cctl & E1000_I2C_DATA_IN); 598 } 599 600 /** 601 * igb_set_i2c_data - Sets the I2C data bit 602 * @data: pointer to hardware structure 603 * @state: I2C data value (0 or 1) to set 604 * 605 * Sets the I2C data bit 606 **/ 607 static void igb_set_i2c_data(void *data, int state) 608 { 609 struct igb_adapter *adapter = (struct igb_adapter *)data; 610 struct e1000_hw *hw = &adapter->hw; 611 s32 i2cctl = rd32(E1000_I2CPARAMS); 612 613 if (state) 614 i2cctl |= E1000_I2C_DATA_OUT; 615 else 616 i2cctl &= ~E1000_I2C_DATA_OUT; 617 618 i2cctl &= ~E1000_I2C_DATA_OE_N; 619 i2cctl |= E1000_I2C_CLK_OE_N; 620 wr32(E1000_I2CPARAMS, i2cctl); 621 wrfl(); 622 623 } 624 625 /** 626 * igb_set_i2c_clk - Sets the I2C SCL clock 627 * @data: pointer to hardware structure 628 * @state: state to set clock 629 * 630 * Sets the I2C clock line to state 631 **/ 632 static void igb_set_i2c_clk(void *data, int state) 633 { 634 struct igb_adapter *adapter = (struct igb_adapter *)data; 635 struct e1000_hw *hw = &adapter->hw; 636 s32 i2cctl = rd32(E1000_I2CPARAMS); 637 638 if (state) { 639 i2cctl |= E1000_I2C_CLK_OUT; 640 i2cctl &= ~E1000_I2C_CLK_OE_N; 641 } else { 642 i2cctl &= ~E1000_I2C_CLK_OUT; 643 i2cctl &= ~E1000_I2C_CLK_OE_N; 644 } 645 wr32(E1000_I2CPARAMS, i2cctl); 646 wrfl(); 647 } 648 649 /** 650 * igb_get_i2c_clk - Gets the I2C SCL clock state 651 * @data: pointer to hardware structure 652 * 653 * Gets the I2C clock state 654 **/ 655 static int igb_get_i2c_clk(void *data) 656 { 657 struct igb_adapter *adapter = (struct igb_adapter *)data; 658 struct e1000_hw *hw = &adapter->hw; 659 s32 i2cctl = rd32(E1000_I2CPARAMS); 660 661 return !!(i2cctl & E1000_I2C_CLK_IN); 662 } 663 664 static const struct i2c_algo_bit_data igb_i2c_algo = { 665 .setsda = igb_set_i2c_data, 666 .setscl = igb_set_i2c_clk, 667 .getsda = igb_get_i2c_data, 668 .getscl = igb_get_i2c_clk, 669 .udelay = 5, 670 .timeout = 20, 671 }; 672 673 /** 674 * igb_get_hw_dev - return device 675 * @hw: pointer to hardware structure 676 * 677 * used by hardware layer to print debugging information 678 **/ 679 struct net_device *igb_get_hw_dev(struct e1000_hw *hw) 680 { 681 struct igb_adapter *adapter = hw->back; 682 return adapter->netdev; 683 } 684 685 /** 686 * igb_init_module - Driver Registration Routine 687 * 688 * igb_init_module is the first routine called when the driver is 689 * loaded. All it does is register with the PCI subsystem. 690 **/ 691 static int __init igb_init_module(void) 692 { 693 int ret; 694 695 pr_info("%s - version %s\n", 696 igb_driver_string, igb_driver_version); 697 pr_info("%s\n", igb_copyright); 698 699 #ifdef CONFIG_IGB_DCA 700 dca_register_notify(&dca_notifier); 701 #endif 702 ret = pci_register_driver(&igb_driver); 703 return ret; 704 } 705 706 module_init(igb_init_module); 707 708 /** 709 * igb_exit_module - Driver Exit Cleanup Routine 710 * 711 * igb_exit_module is called just before the driver is removed 712 * from memory. 713 **/ 714 static void __exit igb_exit_module(void) 715 { 716 #ifdef CONFIG_IGB_DCA 717 dca_unregister_notify(&dca_notifier); 718 #endif 719 pci_unregister_driver(&igb_driver); 720 } 721 722 module_exit(igb_exit_module); 723 724 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1)) 725 /** 726 * igb_cache_ring_register - Descriptor ring to register mapping 727 * @adapter: board private structure to initialize 728 * 729 * Once we know the feature-set enabled for the device, we'll cache 730 * the register offset the descriptor ring is assigned to. 731 **/ 732 static void igb_cache_ring_register(struct igb_adapter *adapter) 733 { 734 int i = 0, j = 0; 735 u32 rbase_offset = adapter->vfs_allocated_count; 736 737 switch (adapter->hw.mac.type) { 738 case e1000_82576: 739 /* The queues are allocated for virtualization such that VF 0 740 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc. 741 * In order to avoid collision we start at the first free queue 742 * and continue consuming queues in the same sequence 743 */ 744 if (adapter->vfs_allocated_count) { 745 for (; i < adapter->rss_queues; i++) 746 adapter->rx_ring[i]->reg_idx = rbase_offset + 747 Q_IDX_82576(i); 748 } 749 /* Fall through */ 750 case e1000_82575: 751 case e1000_82580: 752 case e1000_i350: 753 case e1000_i354: 754 case e1000_i210: 755 case e1000_i211: 756 /* Fall through */ 757 default: 758 for (; i < adapter->num_rx_queues; i++) 759 adapter->rx_ring[i]->reg_idx = rbase_offset + i; 760 for (; j < adapter->num_tx_queues; j++) 761 adapter->tx_ring[j]->reg_idx = rbase_offset + j; 762 break; 763 } 764 } 765 766 u32 igb_rd32(struct e1000_hw *hw, u32 reg) 767 { 768 struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw); 769 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 770 u32 value = 0; 771 772 if (E1000_REMOVED(hw_addr)) 773 return ~value; 774 775 value = readl(&hw_addr[reg]); 776 777 /* reads should not return all F's */ 778 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 779 struct net_device *netdev = igb->netdev; 780 hw->hw_addr = NULL; 781 netdev_err(netdev, "PCIe link lost\n"); 782 } 783 784 return value; 785 } 786 787 /** 788 * igb_write_ivar - configure ivar for given MSI-X vector 789 * @hw: pointer to the HW structure 790 * @msix_vector: vector number we are allocating to a given ring 791 * @index: row index of IVAR register to write within IVAR table 792 * @offset: column offset of in IVAR, should be multiple of 8 793 * 794 * This function is intended to handle the writing of the IVAR register 795 * for adapters 82576 and newer. The IVAR table consists of 2 columns, 796 * each containing an cause allocation for an Rx and Tx ring, and a 797 * variable number of rows depending on the number of queues supported. 798 **/ 799 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector, 800 int index, int offset) 801 { 802 u32 ivar = array_rd32(E1000_IVAR0, index); 803 804 /* clear any bits that are currently set */ 805 ivar &= ~((u32)0xFF << offset); 806 807 /* write vector and valid bit */ 808 ivar |= (msix_vector | E1000_IVAR_VALID) << offset; 809 810 array_wr32(E1000_IVAR0, index, ivar); 811 } 812 813 #define IGB_N0_QUEUE -1 814 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector) 815 { 816 struct igb_adapter *adapter = q_vector->adapter; 817 struct e1000_hw *hw = &adapter->hw; 818 int rx_queue = IGB_N0_QUEUE; 819 int tx_queue = IGB_N0_QUEUE; 820 u32 msixbm = 0; 821 822 if (q_vector->rx.ring) 823 rx_queue = q_vector->rx.ring->reg_idx; 824 if (q_vector->tx.ring) 825 tx_queue = q_vector->tx.ring->reg_idx; 826 827 switch (hw->mac.type) { 828 case e1000_82575: 829 /* The 82575 assigns vectors using a bitmask, which matches the 830 * bitmask for the EICR/EIMS/EIMC registers. To assign one 831 * or more queues to a vector, we write the appropriate bits 832 * into the MSIXBM register for that vector. 833 */ 834 if (rx_queue > IGB_N0_QUEUE) 835 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue; 836 if (tx_queue > IGB_N0_QUEUE) 837 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue; 838 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0) 839 msixbm |= E1000_EIMS_OTHER; 840 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm); 841 q_vector->eims_value = msixbm; 842 break; 843 case e1000_82576: 844 /* 82576 uses a table that essentially consists of 2 columns 845 * with 8 rows. The ordering is column-major so we use the 846 * lower 3 bits as the row index, and the 4th bit as the 847 * column offset. 848 */ 849 if (rx_queue > IGB_N0_QUEUE) 850 igb_write_ivar(hw, msix_vector, 851 rx_queue & 0x7, 852 (rx_queue & 0x8) << 1); 853 if (tx_queue > IGB_N0_QUEUE) 854 igb_write_ivar(hw, msix_vector, 855 tx_queue & 0x7, 856 ((tx_queue & 0x8) << 1) + 8); 857 q_vector->eims_value = BIT(msix_vector); 858 break; 859 case e1000_82580: 860 case e1000_i350: 861 case e1000_i354: 862 case e1000_i210: 863 case e1000_i211: 864 /* On 82580 and newer adapters the scheme is similar to 82576 865 * however instead of ordering column-major we have things 866 * ordered row-major. So we traverse the table by using 867 * bit 0 as the column offset, and the remaining bits as the 868 * row index. 869 */ 870 if (rx_queue > IGB_N0_QUEUE) 871 igb_write_ivar(hw, msix_vector, 872 rx_queue >> 1, 873 (rx_queue & 0x1) << 4); 874 if (tx_queue > IGB_N0_QUEUE) 875 igb_write_ivar(hw, msix_vector, 876 tx_queue >> 1, 877 ((tx_queue & 0x1) << 4) + 8); 878 q_vector->eims_value = BIT(msix_vector); 879 break; 880 default: 881 BUG(); 882 break; 883 } 884 885 /* add q_vector eims value to global eims_enable_mask */ 886 adapter->eims_enable_mask |= q_vector->eims_value; 887 888 /* configure q_vector to set itr on first interrupt */ 889 q_vector->set_itr = 1; 890 } 891 892 /** 893 * igb_configure_msix - Configure MSI-X hardware 894 * @adapter: board private structure to initialize 895 * 896 * igb_configure_msix sets up the hardware to properly 897 * generate MSI-X interrupts. 898 **/ 899 static void igb_configure_msix(struct igb_adapter *adapter) 900 { 901 u32 tmp; 902 int i, vector = 0; 903 struct e1000_hw *hw = &adapter->hw; 904 905 adapter->eims_enable_mask = 0; 906 907 /* set vector for other causes, i.e. link changes */ 908 switch (hw->mac.type) { 909 case e1000_82575: 910 tmp = rd32(E1000_CTRL_EXT); 911 /* enable MSI-X PBA support*/ 912 tmp |= E1000_CTRL_EXT_PBA_CLR; 913 914 /* Auto-Mask interrupts upon ICR read. */ 915 tmp |= E1000_CTRL_EXT_EIAME; 916 tmp |= E1000_CTRL_EXT_IRCA; 917 918 wr32(E1000_CTRL_EXT, tmp); 919 920 /* enable msix_other interrupt */ 921 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER); 922 adapter->eims_other = E1000_EIMS_OTHER; 923 924 break; 925 926 case e1000_82576: 927 case e1000_82580: 928 case e1000_i350: 929 case e1000_i354: 930 case e1000_i210: 931 case e1000_i211: 932 /* Turn on MSI-X capability first, or our settings 933 * won't stick. And it will take days to debug. 934 */ 935 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | 936 E1000_GPIE_PBA | E1000_GPIE_EIAME | 937 E1000_GPIE_NSICR); 938 939 /* enable msix_other interrupt */ 940 adapter->eims_other = BIT(vector); 941 tmp = (vector++ | E1000_IVAR_VALID) << 8; 942 943 wr32(E1000_IVAR_MISC, tmp); 944 break; 945 default: 946 /* do nothing, since nothing else supports MSI-X */ 947 break; 948 } /* switch (hw->mac.type) */ 949 950 adapter->eims_enable_mask |= adapter->eims_other; 951 952 for (i = 0; i < adapter->num_q_vectors; i++) 953 igb_assign_vector(adapter->q_vector[i], vector++); 954 955 wrfl(); 956 } 957 958 /** 959 * igb_request_msix - Initialize MSI-X interrupts 960 * @adapter: board private structure to initialize 961 * 962 * igb_request_msix allocates MSI-X vectors and requests interrupts from the 963 * kernel. 964 **/ 965 static int igb_request_msix(struct igb_adapter *adapter) 966 { 967 struct net_device *netdev = adapter->netdev; 968 int i, err = 0, vector = 0, free_vector = 0; 969 970 err = request_irq(adapter->msix_entries[vector].vector, 971 igb_msix_other, 0, netdev->name, adapter); 972 if (err) 973 goto err_out; 974 975 for (i = 0; i < adapter->num_q_vectors; i++) { 976 struct igb_q_vector *q_vector = adapter->q_vector[i]; 977 978 vector++; 979 980 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector); 981 982 if (q_vector->rx.ring && q_vector->tx.ring) 983 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 984 q_vector->rx.ring->queue_index); 985 else if (q_vector->tx.ring) 986 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 987 q_vector->tx.ring->queue_index); 988 else if (q_vector->rx.ring) 989 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 990 q_vector->rx.ring->queue_index); 991 else 992 sprintf(q_vector->name, "%s-unused", netdev->name); 993 994 err = request_irq(adapter->msix_entries[vector].vector, 995 igb_msix_ring, 0, q_vector->name, 996 q_vector); 997 if (err) 998 goto err_free; 999 } 1000 1001 igb_configure_msix(adapter); 1002 return 0; 1003 1004 err_free: 1005 /* free already assigned IRQs */ 1006 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 1007 1008 vector--; 1009 for (i = 0; i < vector; i++) { 1010 free_irq(adapter->msix_entries[free_vector++].vector, 1011 adapter->q_vector[i]); 1012 } 1013 err_out: 1014 return err; 1015 } 1016 1017 /** 1018 * igb_free_q_vector - Free memory allocated for specific interrupt vector 1019 * @adapter: board private structure to initialize 1020 * @v_idx: Index of vector to be freed 1021 * 1022 * This function frees the memory allocated to the q_vector. 1023 **/ 1024 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx) 1025 { 1026 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1027 1028 adapter->q_vector[v_idx] = NULL; 1029 1030 /* igb_get_stats64() might access the rings on this vector, 1031 * we must wait a grace period before freeing it. 1032 */ 1033 if (q_vector) 1034 kfree_rcu(q_vector, rcu); 1035 } 1036 1037 /** 1038 * igb_reset_q_vector - Reset config for interrupt vector 1039 * @adapter: board private structure to initialize 1040 * @v_idx: Index of vector to be reset 1041 * 1042 * If NAPI is enabled it will delete any references to the 1043 * NAPI struct. This is preparation for igb_free_q_vector. 1044 **/ 1045 static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx) 1046 { 1047 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1048 1049 /* Coming from igb_set_interrupt_capability, the vectors are not yet 1050 * allocated. So, q_vector is NULL so we should stop here. 1051 */ 1052 if (!q_vector) 1053 return; 1054 1055 if (q_vector->tx.ring) 1056 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 1057 1058 if (q_vector->rx.ring) 1059 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 1060 1061 netif_napi_del(&q_vector->napi); 1062 1063 } 1064 1065 static void igb_reset_interrupt_capability(struct igb_adapter *adapter) 1066 { 1067 int v_idx = adapter->num_q_vectors; 1068 1069 if (adapter->flags & IGB_FLAG_HAS_MSIX) 1070 pci_disable_msix(adapter->pdev); 1071 else if (adapter->flags & IGB_FLAG_HAS_MSI) 1072 pci_disable_msi(adapter->pdev); 1073 1074 while (v_idx--) 1075 igb_reset_q_vector(adapter, v_idx); 1076 } 1077 1078 /** 1079 * igb_free_q_vectors - Free memory allocated for interrupt vectors 1080 * @adapter: board private structure to initialize 1081 * 1082 * This function frees the memory allocated to the q_vectors. In addition if 1083 * NAPI is enabled it will delete any references to the NAPI struct prior 1084 * to freeing the q_vector. 1085 **/ 1086 static void igb_free_q_vectors(struct igb_adapter *adapter) 1087 { 1088 int v_idx = adapter->num_q_vectors; 1089 1090 adapter->num_tx_queues = 0; 1091 adapter->num_rx_queues = 0; 1092 adapter->num_q_vectors = 0; 1093 1094 while (v_idx--) { 1095 igb_reset_q_vector(adapter, v_idx); 1096 igb_free_q_vector(adapter, v_idx); 1097 } 1098 } 1099 1100 /** 1101 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts 1102 * @adapter: board private structure to initialize 1103 * 1104 * This function resets the device so that it has 0 Rx queues, Tx queues, and 1105 * MSI-X interrupts allocated. 1106 */ 1107 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter) 1108 { 1109 igb_free_q_vectors(adapter); 1110 igb_reset_interrupt_capability(adapter); 1111 } 1112 1113 /** 1114 * igb_set_interrupt_capability - set MSI or MSI-X if supported 1115 * @adapter: board private structure to initialize 1116 * @msix: boolean value of MSIX capability 1117 * 1118 * Attempt to configure interrupts using the best available 1119 * capabilities of the hardware and kernel. 1120 **/ 1121 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix) 1122 { 1123 int err; 1124 int numvecs, i; 1125 1126 if (!msix) 1127 goto msi_only; 1128 adapter->flags |= IGB_FLAG_HAS_MSIX; 1129 1130 /* Number of supported queues. */ 1131 adapter->num_rx_queues = adapter->rss_queues; 1132 if (adapter->vfs_allocated_count) 1133 adapter->num_tx_queues = 1; 1134 else 1135 adapter->num_tx_queues = adapter->rss_queues; 1136 1137 /* start with one vector for every Rx queue */ 1138 numvecs = adapter->num_rx_queues; 1139 1140 /* if Tx handler is separate add 1 for every Tx queue */ 1141 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS)) 1142 numvecs += adapter->num_tx_queues; 1143 1144 /* store the number of vectors reserved for queues */ 1145 adapter->num_q_vectors = numvecs; 1146 1147 /* add 1 vector for link status interrupts */ 1148 numvecs++; 1149 for (i = 0; i < numvecs; i++) 1150 adapter->msix_entries[i].entry = i; 1151 1152 err = pci_enable_msix_range(adapter->pdev, 1153 adapter->msix_entries, 1154 numvecs, 1155 numvecs); 1156 if (err > 0) 1157 return; 1158 1159 igb_reset_interrupt_capability(adapter); 1160 1161 /* If we can't do MSI-X, try MSI */ 1162 msi_only: 1163 adapter->flags &= ~IGB_FLAG_HAS_MSIX; 1164 #ifdef CONFIG_PCI_IOV 1165 /* disable SR-IOV for non MSI-X configurations */ 1166 if (adapter->vf_data) { 1167 struct e1000_hw *hw = &adapter->hw; 1168 /* disable iov and allow time for transactions to clear */ 1169 pci_disable_sriov(adapter->pdev); 1170 msleep(500); 1171 1172 kfree(adapter->vf_mac_list); 1173 adapter->vf_mac_list = NULL; 1174 kfree(adapter->vf_data); 1175 adapter->vf_data = NULL; 1176 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 1177 wrfl(); 1178 msleep(100); 1179 dev_info(&adapter->pdev->dev, "IOV Disabled\n"); 1180 } 1181 #endif 1182 adapter->vfs_allocated_count = 0; 1183 adapter->rss_queues = 1; 1184 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 1185 adapter->num_rx_queues = 1; 1186 adapter->num_tx_queues = 1; 1187 adapter->num_q_vectors = 1; 1188 if (!pci_enable_msi(adapter->pdev)) 1189 adapter->flags |= IGB_FLAG_HAS_MSI; 1190 } 1191 1192 static void igb_add_ring(struct igb_ring *ring, 1193 struct igb_ring_container *head) 1194 { 1195 head->ring = ring; 1196 head->count++; 1197 } 1198 1199 /** 1200 * igb_alloc_q_vector - Allocate memory for a single interrupt vector 1201 * @adapter: board private structure to initialize 1202 * @v_count: q_vectors allocated on adapter, used for ring interleaving 1203 * @v_idx: index of vector in adapter struct 1204 * @txr_count: total number of Tx rings to allocate 1205 * @txr_idx: index of first Tx ring to allocate 1206 * @rxr_count: total number of Rx rings to allocate 1207 * @rxr_idx: index of first Rx ring to allocate 1208 * 1209 * We allocate one q_vector. If allocation fails we return -ENOMEM. 1210 **/ 1211 static int igb_alloc_q_vector(struct igb_adapter *adapter, 1212 int v_count, int v_idx, 1213 int txr_count, int txr_idx, 1214 int rxr_count, int rxr_idx) 1215 { 1216 struct igb_q_vector *q_vector; 1217 struct igb_ring *ring; 1218 int ring_count, size; 1219 1220 /* igb only supports 1 Tx and/or 1 Rx queue per vector */ 1221 if (txr_count > 1 || rxr_count > 1) 1222 return -ENOMEM; 1223 1224 ring_count = txr_count + rxr_count; 1225 size = sizeof(struct igb_q_vector) + 1226 (sizeof(struct igb_ring) * ring_count); 1227 1228 /* allocate q_vector and rings */ 1229 q_vector = adapter->q_vector[v_idx]; 1230 if (!q_vector) { 1231 q_vector = kzalloc(size, GFP_KERNEL); 1232 } else if (size > ksize(q_vector)) { 1233 kfree_rcu(q_vector, rcu); 1234 q_vector = kzalloc(size, GFP_KERNEL); 1235 } else { 1236 memset(q_vector, 0, size); 1237 } 1238 if (!q_vector) 1239 return -ENOMEM; 1240 1241 /* initialize NAPI */ 1242 netif_napi_add(adapter->netdev, &q_vector->napi, 1243 igb_poll, 64); 1244 1245 /* tie q_vector and adapter together */ 1246 adapter->q_vector[v_idx] = q_vector; 1247 q_vector->adapter = adapter; 1248 1249 /* initialize work limits */ 1250 q_vector->tx.work_limit = adapter->tx_work_limit; 1251 1252 /* initialize ITR configuration */ 1253 q_vector->itr_register = adapter->io_addr + E1000_EITR(0); 1254 q_vector->itr_val = IGB_START_ITR; 1255 1256 /* initialize pointer to rings */ 1257 ring = q_vector->ring; 1258 1259 /* intialize ITR */ 1260 if (rxr_count) { 1261 /* rx or rx/tx vector */ 1262 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 1263 q_vector->itr_val = adapter->rx_itr_setting; 1264 } else { 1265 /* tx only vector */ 1266 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 1267 q_vector->itr_val = adapter->tx_itr_setting; 1268 } 1269 1270 if (txr_count) { 1271 /* assign generic ring traits */ 1272 ring->dev = &adapter->pdev->dev; 1273 ring->netdev = adapter->netdev; 1274 1275 /* configure backlink on ring */ 1276 ring->q_vector = q_vector; 1277 1278 /* update q_vector Tx values */ 1279 igb_add_ring(ring, &q_vector->tx); 1280 1281 /* For 82575, context index must be unique per ring. */ 1282 if (adapter->hw.mac.type == e1000_82575) 1283 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags); 1284 1285 /* apply Tx specific ring traits */ 1286 ring->count = adapter->tx_ring_count; 1287 ring->queue_index = txr_idx; 1288 1289 ring->cbs_enable = false; 1290 ring->idleslope = 0; 1291 ring->sendslope = 0; 1292 ring->hicredit = 0; 1293 ring->locredit = 0; 1294 1295 u64_stats_init(&ring->tx_syncp); 1296 u64_stats_init(&ring->tx_syncp2); 1297 1298 /* assign ring to adapter */ 1299 adapter->tx_ring[txr_idx] = ring; 1300 1301 /* push pointer to next ring */ 1302 ring++; 1303 } 1304 1305 if (rxr_count) { 1306 /* assign generic ring traits */ 1307 ring->dev = &adapter->pdev->dev; 1308 ring->netdev = adapter->netdev; 1309 1310 /* configure backlink on ring */ 1311 ring->q_vector = q_vector; 1312 1313 /* update q_vector Rx values */ 1314 igb_add_ring(ring, &q_vector->rx); 1315 1316 /* set flag indicating ring supports SCTP checksum offload */ 1317 if (adapter->hw.mac.type >= e1000_82576) 1318 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags); 1319 1320 /* On i350, i354, i210, and i211, loopback VLAN packets 1321 * have the tag byte-swapped. 1322 */ 1323 if (adapter->hw.mac.type >= e1000_i350) 1324 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags); 1325 1326 /* apply Rx specific ring traits */ 1327 ring->count = adapter->rx_ring_count; 1328 ring->queue_index = rxr_idx; 1329 1330 u64_stats_init(&ring->rx_syncp); 1331 1332 /* assign ring to adapter */ 1333 adapter->rx_ring[rxr_idx] = ring; 1334 } 1335 1336 return 0; 1337 } 1338 1339 1340 /** 1341 * igb_alloc_q_vectors - Allocate memory for interrupt vectors 1342 * @adapter: board private structure to initialize 1343 * 1344 * We allocate one q_vector per queue interrupt. If allocation fails we 1345 * return -ENOMEM. 1346 **/ 1347 static int igb_alloc_q_vectors(struct igb_adapter *adapter) 1348 { 1349 int q_vectors = adapter->num_q_vectors; 1350 int rxr_remaining = adapter->num_rx_queues; 1351 int txr_remaining = adapter->num_tx_queues; 1352 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 1353 int err; 1354 1355 if (q_vectors >= (rxr_remaining + txr_remaining)) { 1356 for (; rxr_remaining; v_idx++) { 1357 err = igb_alloc_q_vector(adapter, q_vectors, v_idx, 1358 0, 0, 1, rxr_idx); 1359 1360 if (err) 1361 goto err_out; 1362 1363 /* update counts and index */ 1364 rxr_remaining--; 1365 rxr_idx++; 1366 } 1367 } 1368 1369 for (; v_idx < q_vectors; v_idx++) { 1370 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 1371 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 1372 1373 err = igb_alloc_q_vector(adapter, q_vectors, v_idx, 1374 tqpv, txr_idx, rqpv, rxr_idx); 1375 1376 if (err) 1377 goto err_out; 1378 1379 /* update counts and index */ 1380 rxr_remaining -= rqpv; 1381 txr_remaining -= tqpv; 1382 rxr_idx++; 1383 txr_idx++; 1384 } 1385 1386 return 0; 1387 1388 err_out: 1389 adapter->num_tx_queues = 0; 1390 adapter->num_rx_queues = 0; 1391 adapter->num_q_vectors = 0; 1392 1393 while (v_idx--) 1394 igb_free_q_vector(adapter, v_idx); 1395 1396 return -ENOMEM; 1397 } 1398 1399 /** 1400 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 1401 * @adapter: board private structure to initialize 1402 * @msix: boolean value of MSIX capability 1403 * 1404 * This function initializes the interrupts and allocates all of the queues. 1405 **/ 1406 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix) 1407 { 1408 struct pci_dev *pdev = adapter->pdev; 1409 int err; 1410 1411 igb_set_interrupt_capability(adapter, msix); 1412 1413 err = igb_alloc_q_vectors(adapter); 1414 if (err) { 1415 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 1416 goto err_alloc_q_vectors; 1417 } 1418 1419 igb_cache_ring_register(adapter); 1420 1421 return 0; 1422 1423 err_alloc_q_vectors: 1424 igb_reset_interrupt_capability(adapter); 1425 return err; 1426 } 1427 1428 /** 1429 * igb_request_irq - initialize interrupts 1430 * @adapter: board private structure to initialize 1431 * 1432 * Attempts to configure interrupts using the best available 1433 * capabilities of the hardware and kernel. 1434 **/ 1435 static int igb_request_irq(struct igb_adapter *adapter) 1436 { 1437 struct net_device *netdev = adapter->netdev; 1438 struct pci_dev *pdev = adapter->pdev; 1439 int err = 0; 1440 1441 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1442 err = igb_request_msix(adapter); 1443 if (!err) 1444 goto request_done; 1445 /* fall back to MSI */ 1446 igb_free_all_tx_resources(adapter); 1447 igb_free_all_rx_resources(adapter); 1448 1449 igb_clear_interrupt_scheme(adapter); 1450 err = igb_init_interrupt_scheme(adapter, false); 1451 if (err) 1452 goto request_done; 1453 1454 igb_setup_all_tx_resources(adapter); 1455 igb_setup_all_rx_resources(adapter); 1456 igb_configure(adapter); 1457 } 1458 1459 igb_assign_vector(adapter->q_vector[0], 0); 1460 1461 if (adapter->flags & IGB_FLAG_HAS_MSI) { 1462 err = request_irq(pdev->irq, igb_intr_msi, 0, 1463 netdev->name, adapter); 1464 if (!err) 1465 goto request_done; 1466 1467 /* fall back to legacy interrupts */ 1468 igb_reset_interrupt_capability(adapter); 1469 adapter->flags &= ~IGB_FLAG_HAS_MSI; 1470 } 1471 1472 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED, 1473 netdev->name, adapter); 1474 1475 if (err) 1476 dev_err(&pdev->dev, "Error %d getting interrupt\n", 1477 err); 1478 1479 request_done: 1480 return err; 1481 } 1482 1483 static void igb_free_irq(struct igb_adapter *adapter) 1484 { 1485 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1486 int vector = 0, i; 1487 1488 free_irq(adapter->msix_entries[vector++].vector, adapter); 1489 1490 for (i = 0; i < adapter->num_q_vectors; i++) 1491 free_irq(adapter->msix_entries[vector++].vector, 1492 adapter->q_vector[i]); 1493 } else { 1494 free_irq(adapter->pdev->irq, adapter); 1495 } 1496 } 1497 1498 /** 1499 * igb_irq_disable - Mask off interrupt generation on the NIC 1500 * @adapter: board private structure 1501 **/ 1502 static void igb_irq_disable(struct igb_adapter *adapter) 1503 { 1504 struct e1000_hw *hw = &adapter->hw; 1505 1506 /* we need to be careful when disabling interrupts. The VFs are also 1507 * mapped into these registers and so clearing the bits can cause 1508 * issues on the VF drivers so we only need to clear what we set 1509 */ 1510 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1511 u32 regval = rd32(E1000_EIAM); 1512 1513 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask); 1514 wr32(E1000_EIMC, adapter->eims_enable_mask); 1515 regval = rd32(E1000_EIAC); 1516 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask); 1517 } 1518 1519 wr32(E1000_IAM, 0); 1520 wr32(E1000_IMC, ~0); 1521 wrfl(); 1522 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1523 int i; 1524 1525 for (i = 0; i < adapter->num_q_vectors; i++) 1526 synchronize_irq(adapter->msix_entries[i].vector); 1527 } else { 1528 synchronize_irq(adapter->pdev->irq); 1529 } 1530 } 1531 1532 /** 1533 * igb_irq_enable - Enable default interrupt generation settings 1534 * @adapter: board private structure 1535 **/ 1536 static void igb_irq_enable(struct igb_adapter *adapter) 1537 { 1538 struct e1000_hw *hw = &adapter->hw; 1539 1540 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 1541 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA; 1542 u32 regval = rd32(E1000_EIAC); 1543 1544 wr32(E1000_EIAC, regval | adapter->eims_enable_mask); 1545 regval = rd32(E1000_EIAM); 1546 wr32(E1000_EIAM, regval | adapter->eims_enable_mask); 1547 wr32(E1000_EIMS, adapter->eims_enable_mask); 1548 if (adapter->vfs_allocated_count) { 1549 wr32(E1000_MBVFIMR, 0xFF); 1550 ims |= E1000_IMS_VMMB; 1551 } 1552 wr32(E1000_IMS, ims); 1553 } else { 1554 wr32(E1000_IMS, IMS_ENABLE_MASK | 1555 E1000_IMS_DRSTA); 1556 wr32(E1000_IAM, IMS_ENABLE_MASK | 1557 E1000_IMS_DRSTA); 1558 } 1559 } 1560 1561 static void igb_update_mng_vlan(struct igb_adapter *adapter) 1562 { 1563 struct e1000_hw *hw = &adapter->hw; 1564 u16 pf_id = adapter->vfs_allocated_count; 1565 u16 vid = adapter->hw.mng_cookie.vlan_id; 1566 u16 old_vid = adapter->mng_vlan_id; 1567 1568 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 1569 /* add VID to filter table */ 1570 igb_vfta_set(hw, vid, pf_id, true, true); 1571 adapter->mng_vlan_id = vid; 1572 } else { 1573 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; 1574 } 1575 1576 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && 1577 (vid != old_vid) && 1578 !test_bit(old_vid, adapter->active_vlans)) { 1579 /* remove VID from filter table */ 1580 igb_vfta_set(hw, vid, pf_id, false, true); 1581 } 1582 } 1583 1584 /** 1585 * igb_release_hw_control - release control of the h/w to f/w 1586 * @adapter: address of board private structure 1587 * 1588 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 1589 * For ASF and Pass Through versions of f/w this means that the 1590 * driver is no longer loaded. 1591 **/ 1592 static void igb_release_hw_control(struct igb_adapter *adapter) 1593 { 1594 struct e1000_hw *hw = &adapter->hw; 1595 u32 ctrl_ext; 1596 1597 /* Let firmware take over control of h/w */ 1598 ctrl_ext = rd32(E1000_CTRL_EXT); 1599 wr32(E1000_CTRL_EXT, 1600 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 1601 } 1602 1603 /** 1604 * igb_get_hw_control - get control of the h/w from f/w 1605 * @adapter: address of board private structure 1606 * 1607 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 1608 * For ASF and Pass Through versions of f/w this means that 1609 * the driver is loaded. 1610 **/ 1611 static void igb_get_hw_control(struct igb_adapter *adapter) 1612 { 1613 struct e1000_hw *hw = &adapter->hw; 1614 u32 ctrl_ext; 1615 1616 /* Let firmware know the driver has taken over */ 1617 ctrl_ext = rd32(E1000_CTRL_EXT); 1618 wr32(E1000_CTRL_EXT, 1619 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 1620 } 1621 1622 static void enable_fqtss(struct igb_adapter *adapter, bool enable) 1623 { 1624 struct net_device *netdev = adapter->netdev; 1625 struct e1000_hw *hw = &adapter->hw; 1626 1627 WARN_ON(hw->mac.type != e1000_i210); 1628 1629 if (enable) 1630 adapter->flags |= IGB_FLAG_FQTSS; 1631 else 1632 adapter->flags &= ~IGB_FLAG_FQTSS; 1633 1634 if (netif_running(netdev)) 1635 schedule_work(&adapter->reset_task); 1636 } 1637 1638 static bool is_fqtss_enabled(struct igb_adapter *adapter) 1639 { 1640 return (adapter->flags & IGB_FLAG_FQTSS) ? true : false; 1641 } 1642 1643 static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue, 1644 enum tx_queue_prio prio) 1645 { 1646 u32 val; 1647 1648 WARN_ON(hw->mac.type != e1000_i210); 1649 WARN_ON(queue < 0 || queue > 4); 1650 1651 val = rd32(E1000_I210_TXDCTL(queue)); 1652 1653 if (prio == TX_QUEUE_PRIO_HIGH) 1654 val |= E1000_TXDCTL_PRIORITY; 1655 else 1656 val &= ~E1000_TXDCTL_PRIORITY; 1657 1658 wr32(E1000_I210_TXDCTL(queue), val); 1659 } 1660 1661 static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode) 1662 { 1663 u32 val; 1664 1665 WARN_ON(hw->mac.type != e1000_i210); 1666 WARN_ON(queue < 0 || queue > 1); 1667 1668 val = rd32(E1000_I210_TQAVCC(queue)); 1669 1670 if (mode == QUEUE_MODE_STREAM_RESERVATION) 1671 val |= E1000_TQAVCC_QUEUEMODE; 1672 else 1673 val &= ~E1000_TQAVCC_QUEUEMODE; 1674 1675 wr32(E1000_I210_TQAVCC(queue), val); 1676 } 1677 1678 /** 1679 * igb_configure_cbs - Configure Credit-Based Shaper (CBS) 1680 * @adapter: pointer to adapter struct 1681 * @queue: queue number 1682 * @enable: true = enable CBS, false = disable CBS 1683 * @idleslope: idleSlope in kbps 1684 * @sendslope: sendSlope in kbps 1685 * @hicredit: hiCredit in bytes 1686 * @locredit: loCredit in bytes 1687 * 1688 * Configure CBS for a given hardware queue. When disabling, idleslope, 1689 * sendslope, hicredit, locredit arguments are ignored. Returns 0 if 1690 * success. Negative otherwise. 1691 **/ 1692 static void igb_configure_cbs(struct igb_adapter *adapter, int queue, 1693 bool enable, int idleslope, int sendslope, 1694 int hicredit, int locredit) 1695 { 1696 struct net_device *netdev = adapter->netdev; 1697 struct e1000_hw *hw = &adapter->hw; 1698 u32 tqavcc; 1699 u16 value; 1700 1701 WARN_ON(hw->mac.type != e1000_i210); 1702 WARN_ON(queue < 0 || queue > 1); 1703 1704 if (enable || queue == 0) { 1705 /* i210 does not allow the queue 0 to be in the Strict 1706 * Priority mode while the Qav mode is enabled, so, 1707 * instead of disabling strict priority mode, we give 1708 * queue 0 the maximum of credits possible. 1709 * 1710 * See section 8.12.19 of the i210 datasheet, "Note: 1711 * Queue0 QueueMode must be set to 1b when 1712 * TransmitMode is set to Qav." 1713 */ 1714 if (queue == 0 && !enable) { 1715 /* max "linkspeed" idleslope in kbps */ 1716 idleslope = 1000000; 1717 hicredit = ETH_FRAME_LEN; 1718 } 1719 1720 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH); 1721 set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION); 1722 1723 /* According to i210 datasheet section 7.2.7.7, we should set 1724 * the 'idleSlope' field from TQAVCC register following the 1725 * equation: 1726 * 1727 * For 100 Mbps link speed: 1728 * 1729 * value = BW * 0x7735 * 0.2 (E1) 1730 * 1731 * For 1000Mbps link speed: 1732 * 1733 * value = BW * 0x7735 * 2 (E2) 1734 * 1735 * E1 and E2 can be merged into one equation as shown below. 1736 * Note that 'link-speed' is in Mbps. 1737 * 1738 * value = BW * 0x7735 * 2 * link-speed 1739 * -------------- (E3) 1740 * 1000 1741 * 1742 * 'BW' is the percentage bandwidth out of full link speed 1743 * which can be found with the following equation. Note that 1744 * idleSlope here is the parameter from this function which 1745 * is in kbps. 1746 * 1747 * BW = idleSlope 1748 * ----------------- (E4) 1749 * link-speed * 1000 1750 * 1751 * That said, we can come up with a generic equation to 1752 * calculate the value we should set it TQAVCC register by 1753 * replacing 'BW' in E3 by E4. The resulting equation is: 1754 * 1755 * value = idleSlope * 0x7735 * 2 * link-speed 1756 * ----------------- -------------- (E5) 1757 * link-speed * 1000 1000 1758 * 1759 * 'link-speed' is present in both sides of the fraction so 1760 * it is canceled out. The final equation is the following: 1761 * 1762 * value = idleSlope * 61034 1763 * ----------------- (E6) 1764 * 1000000 1765 * 1766 * NOTE: For i210, given the above, we can see that idleslope 1767 * is represented in 16.38431 kbps units by the value at 1768 * the TQAVCC register (1Gbps / 61034), which reduces 1769 * the granularity for idleslope increments. 1770 * For instance, if you want to configure a 2576kbps 1771 * idleslope, the value to be written on the register 1772 * would have to be 157.23. If rounded down, you end 1773 * up with less bandwidth available than originally 1774 * required (~2572 kbps). If rounded up, you end up 1775 * with a higher bandwidth (~2589 kbps). Below the 1776 * approach we take is to always round up the 1777 * calculated value, so the resulting bandwidth might 1778 * be slightly higher for some configurations. 1779 */ 1780 value = DIV_ROUND_UP_ULL(idleslope * 61034ULL, 1000000); 1781 1782 tqavcc = rd32(E1000_I210_TQAVCC(queue)); 1783 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK; 1784 tqavcc |= value; 1785 wr32(E1000_I210_TQAVCC(queue), tqavcc); 1786 1787 wr32(E1000_I210_TQAVHC(queue), 0x80000000 + hicredit * 0x7735); 1788 } else { 1789 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW); 1790 set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY); 1791 1792 /* Set idleSlope to zero. */ 1793 tqavcc = rd32(E1000_I210_TQAVCC(queue)); 1794 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK; 1795 wr32(E1000_I210_TQAVCC(queue), tqavcc); 1796 1797 /* Set hiCredit to zero. */ 1798 wr32(E1000_I210_TQAVHC(queue), 0); 1799 } 1800 1801 /* XXX: In i210 controller the sendSlope and loCredit parameters from 1802 * CBS are not configurable by software so we don't do any 'controller 1803 * configuration' in respect to these parameters. 1804 */ 1805 1806 netdev_dbg(netdev, "CBS %s: queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n", 1807 (enable) ? "enabled" : "disabled", queue, 1808 idleslope, sendslope, hicredit, locredit); 1809 } 1810 1811 static int igb_save_cbs_params(struct igb_adapter *adapter, int queue, 1812 bool enable, int idleslope, int sendslope, 1813 int hicredit, int locredit) 1814 { 1815 struct igb_ring *ring; 1816 1817 if (queue < 0 || queue > adapter->num_tx_queues) 1818 return -EINVAL; 1819 1820 ring = adapter->tx_ring[queue]; 1821 1822 ring->cbs_enable = enable; 1823 ring->idleslope = idleslope; 1824 ring->sendslope = sendslope; 1825 ring->hicredit = hicredit; 1826 ring->locredit = locredit; 1827 1828 return 0; 1829 } 1830 1831 static bool is_any_cbs_enabled(struct igb_adapter *adapter) 1832 { 1833 struct igb_ring *ring; 1834 int i; 1835 1836 for (i = 0; i < adapter->num_tx_queues; i++) { 1837 ring = adapter->tx_ring[i]; 1838 1839 if (ring->cbs_enable) 1840 return true; 1841 } 1842 1843 return false; 1844 } 1845 1846 static void igb_setup_tx_mode(struct igb_adapter *adapter) 1847 { 1848 struct net_device *netdev = adapter->netdev; 1849 struct e1000_hw *hw = &adapter->hw; 1850 u32 val; 1851 1852 /* Only i210 controller supports changing the transmission mode. */ 1853 if (hw->mac.type != e1000_i210) 1854 return; 1855 1856 if (is_fqtss_enabled(adapter)) { 1857 int i, max_queue; 1858 1859 /* Configure TQAVCTRL register: set transmit mode to 'Qav', 1860 * set data fetch arbitration to 'round robin' and set data 1861 * transfer arbitration to 'credit shaper algorithm. 1862 */ 1863 val = rd32(E1000_I210_TQAVCTRL); 1864 val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_DATATRANARB; 1865 val &= ~E1000_TQAVCTRL_DATAFETCHARB; 1866 wr32(E1000_I210_TQAVCTRL, val); 1867 1868 /* Configure Tx and Rx packet buffers sizes as described in 1869 * i210 datasheet section 7.2.7.7. 1870 */ 1871 val = rd32(E1000_TXPBS); 1872 val &= ~I210_TXPBSIZE_MASK; 1873 val |= I210_TXPBSIZE_PB0_8KB | I210_TXPBSIZE_PB1_8KB | 1874 I210_TXPBSIZE_PB2_4KB | I210_TXPBSIZE_PB3_4KB; 1875 wr32(E1000_TXPBS, val); 1876 1877 val = rd32(E1000_RXPBS); 1878 val &= ~I210_RXPBSIZE_MASK; 1879 val |= I210_RXPBSIZE_PB_32KB; 1880 wr32(E1000_RXPBS, val); 1881 1882 /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ 1883 * register should not exceed the buffer size programmed in 1884 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB 1885 * so according to the datasheet we should set MAX_TPKT_SIZE to 1886 * 4kB / 64. 1887 * 1888 * However, when we do so, no frame from queue 2 and 3 are 1889 * transmitted. It seems the MAX_TPKT_SIZE should not be great 1890 * or _equal_ to the buffer size programmed in TXPBS. For this 1891 * reason, we set set MAX_ TPKT_SIZE to (4kB - 1) / 64. 1892 */ 1893 val = (4096 - 1) / 64; 1894 wr32(E1000_I210_DTXMXPKTSZ, val); 1895 1896 /* Since FQTSS mode is enabled, apply any CBS configuration 1897 * previously set. If no previous CBS configuration has been 1898 * done, then the initial configuration is applied, which means 1899 * CBS is disabled. 1900 */ 1901 max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ? 1902 adapter->num_tx_queues : I210_SR_QUEUES_NUM; 1903 1904 for (i = 0; i < max_queue; i++) { 1905 struct igb_ring *ring = adapter->tx_ring[i]; 1906 1907 igb_configure_cbs(adapter, i, ring->cbs_enable, 1908 ring->idleslope, ring->sendslope, 1909 ring->hicredit, ring->locredit); 1910 } 1911 } else { 1912 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT); 1913 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT); 1914 wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT); 1915 1916 val = rd32(E1000_I210_TQAVCTRL); 1917 /* According to Section 8.12.21, the other flags we've set when 1918 * enabling FQTSS are not relevant when disabling FQTSS so we 1919 * don't set they here. 1920 */ 1921 val &= ~E1000_TQAVCTRL_XMIT_MODE; 1922 wr32(E1000_I210_TQAVCTRL, val); 1923 } 1924 1925 netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ? 1926 "enabled" : "disabled"); 1927 } 1928 1929 /** 1930 * igb_configure - configure the hardware for RX and TX 1931 * @adapter: private board structure 1932 **/ 1933 static void igb_configure(struct igb_adapter *adapter) 1934 { 1935 struct net_device *netdev = adapter->netdev; 1936 int i; 1937 1938 igb_get_hw_control(adapter); 1939 igb_set_rx_mode(netdev); 1940 igb_setup_tx_mode(adapter); 1941 1942 igb_restore_vlan(adapter); 1943 1944 igb_setup_tctl(adapter); 1945 igb_setup_mrqc(adapter); 1946 igb_setup_rctl(adapter); 1947 1948 igb_nfc_filter_restore(adapter); 1949 igb_configure_tx(adapter); 1950 igb_configure_rx(adapter); 1951 1952 igb_rx_fifo_flush_82575(&adapter->hw); 1953 1954 /* call igb_desc_unused which always leaves 1955 * at least 1 descriptor unused to make sure 1956 * next_to_use != next_to_clean 1957 */ 1958 for (i = 0; i < adapter->num_rx_queues; i++) { 1959 struct igb_ring *ring = adapter->rx_ring[i]; 1960 igb_alloc_rx_buffers(ring, igb_desc_unused(ring)); 1961 } 1962 } 1963 1964 /** 1965 * igb_power_up_link - Power up the phy/serdes link 1966 * @adapter: address of board private structure 1967 **/ 1968 void igb_power_up_link(struct igb_adapter *adapter) 1969 { 1970 igb_reset_phy(&adapter->hw); 1971 1972 if (adapter->hw.phy.media_type == e1000_media_type_copper) 1973 igb_power_up_phy_copper(&adapter->hw); 1974 else 1975 igb_power_up_serdes_link_82575(&adapter->hw); 1976 1977 igb_setup_link(&adapter->hw); 1978 } 1979 1980 /** 1981 * igb_power_down_link - Power down the phy/serdes link 1982 * @adapter: address of board private structure 1983 */ 1984 static void igb_power_down_link(struct igb_adapter *adapter) 1985 { 1986 if (adapter->hw.phy.media_type == e1000_media_type_copper) 1987 igb_power_down_phy_copper_82575(&adapter->hw); 1988 else 1989 igb_shutdown_serdes_link_82575(&adapter->hw); 1990 } 1991 1992 /** 1993 * Detect and switch function for Media Auto Sense 1994 * @adapter: address of the board private structure 1995 **/ 1996 static void igb_check_swap_media(struct igb_adapter *adapter) 1997 { 1998 struct e1000_hw *hw = &adapter->hw; 1999 u32 ctrl_ext, connsw; 2000 bool swap_now = false; 2001 2002 ctrl_ext = rd32(E1000_CTRL_EXT); 2003 connsw = rd32(E1000_CONNSW); 2004 2005 /* need to live swap if current media is copper and we have fiber/serdes 2006 * to go to. 2007 */ 2008 2009 if ((hw->phy.media_type == e1000_media_type_copper) && 2010 (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) { 2011 swap_now = true; 2012 } else if (!(connsw & E1000_CONNSW_SERDESD)) { 2013 /* copper signal takes time to appear */ 2014 if (adapter->copper_tries < 4) { 2015 adapter->copper_tries++; 2016 connsw |= E1000_CONNSW_AUTOSENSE_CONF; 2017 wr32(E1000_CONNSW, connsw); 2018 return; 2019 } else { 2020 adapter->copper_tries = 0; 2021 if ((connsw & E1000_CONNSW_PHYSD) && 2022 (!(connsw & E1000_CONNSW_PHY_PDN))) { 2023 swap_now = true; 2024 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF; 2025 wr32(E1000_CONNSW, connsw); 2026 } 2027 } 2028 } 2029 2030 if (!swap_now) 2031 return; 2032 2033 switch (hw->phy.media_type) { 2034 case e1000_media_type_copper: 2035 netdev_info(adapter->netdev, 2036 "MAS: changing media to fiber/serdes\n"); 2037 ctrl_ext |= 2038 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 2039 adapter->flags |= IGB_FLAG_MEDIA_RESET; 2040 adapter->copper_tries = 0; 2041 break; 2042 case e1000_media_type_internal_serdes: 2043 case e1000_media_type_fiber: 2044 netdev_info(adapter->netdev, 2045 "MAS: changing media to copper\n"); 2046 ctrl_ext &= 2047 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 2048 adapter->flags |= IGB_FLAG_MEDIA_RESET; 2049 break; 2050 default: 2051 /* shouldn't get here during regular operation */ 2052 netdev_err(adapter->netdev, 2053 "AMS: Invalid media type found, returning\n"); 2054 break; 2055 } 2056 wr32(E1000_CTRL_EXT, ctrl_ext); 2057 } 2058 2059 /** 2060 * igb_up - Open the interface and prepare it to handle traffic 2061 * @adapter: board private structure 2062 **/ 2063 int igb_up(struct igb_adapter *adapter) 2064 { 2065 struct e1000_hw *hw = &adapter->hw; 2066 int i; 2067 2068 /* hardware has been reset, we need to reload some things */ 2069 igb_configure(adapter); 2070 2071 clear_bit(__IGB_DOWN, &adapter->state); 2072 2073 for (i = 0; i < adapter->num_q_vectors; i++) 2074 napi_enable(&(adapter->q_vector[i]->napi)); 2075 2076 if (adapter->flags & IGB_FLAG_HAS_MSIX) 2077 igb_configure_msix(adapter); 2078 else 2079 igb_assign_vector(adapter->q_vector[0], 0); 2080 2081 /* Clear any pending interrupts. */ 2082 rd32(E1000_ICR); 2083 igb_irq_enable(adapter); 2084 2085 /* notify VFs that reset has been completed */ 2086 if (adapter->vfs_allocated_count) { 2087 u32 reg_data = rd32(E1000_CTRL_EXT); 2088 2089 reg_data |= E1000_CTRL_EXT_PFRSTD; 2090 wr32(E1000_CTRL_EXT, reg_data); 2091 } 2092 2093 netif_tx_start_all_queues(adapter->netdev); 2094 2095 /* start the watchdog. */ 2096 hw->mac.get_link_status = 1; 2097 schedule_work(&adapter->watchdog_task); 2098 2099 if ((adapter->flags & IGB_FLAG_EEE) && 2100 (!hw->dev_spec._82575.eee_disable)) 2101 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; 2102 2103 return 0; 2104 } 2105 2106 void igb_down(struct igb_adapter *adapter) 2107 { 2108 struct net_device *netdev = adapter->netdev; 2109 struct e1000_hw *hw = &adapter->hw; 2110 u32 tctl, rctl; 2111 int i; 2112 2113 /* signal that we're down so the interrupt handler does not 2114 * reschedule our watchdog timer 2115 */ 2116 set_bit(__IGB_DOWN, &adapter->state); 2117 2118 /* disable receives in the hardware */ 2119 rctl = rd32(E1000_RCTL); 2120 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN); 2121 /* flush and sleep below */ 2122 2123 igb_nfc_filter_exit(adapter); 2124 2125 netif_carrier_off(netdev); 2126 netif_tx_stop_all_queues(netdev); 2127 2128 /* disable transmits in the hardware */ 2129 tctl = rd32(E1000_TCTL); 2130 tctl &= ~E1000_TCTL_EN; 2131 wr32(E1000_TCTL, tctl); 2132 /* flush both disables and wait for them to finish */ 2133 wrfl(); 2134 usleep_range(10000, 11000); 2135 2136 igb_irq_disable(adapter); 2137 2138 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; 2139 2140 for (i = 0; i < adapter->num_q_vectors; i++) { 2141 if (adapter->q_vector[i]) { 2142 napi_synchronize(&adapter->q_vector[i]->napi); 2143 napi_disable(&adapter->q_vector[i]->napi); 2144 } 2145 } 2146 2147 del_timer_sync(&adapter->watchdog_timer); 2148 del_timer_sync(&adapter->phy_info_timer); 2149 2150 /* record the stats before reset*/ 2151 spin_lock(&adapter->stats64_lock); 2152 igb_update_stats(adapter); 2153 spin_unlock(&adapter->stats64_lock); 2154 2155 adapter->link_speed = 0; 2156 adapter->link_duplex = 0; 2157 2158 if (!pci_channel_offline(adapter->pdev)) 2159 igb_reset(adapter); 2160 2161 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 2162 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC; 2163 2164 igb_clean_all_tx_rings(adapter); 2165 igb_clean_all_rx_rings(adapter); 2166 #ifdef CONFIG_IGB_DCA 2167 2168 /* since we reset the hardware DCA settings were cleared */ 2169 igb_setup_dca(adapter); 2170 #endif 2171 } 2172 2173 void igb_reinit_locked(struct igb_adapter *adapter) 2174 { 2175 WARN_ON(in_interrupt()); 2176 while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) 2177 usleep_range(1000, 2000); 2178 igb_down(adapter); 2179 igb_up(adapter); 2180 clear_bit(__IGB_RESETTING, &adapter->state); 2181 } 2182 2183 /** igb_enable_mas - Media Autosense re-enable after swap 2184 * 2185 * @adapter: adapter struct 2186 **/ 2187 static void igb_enable_mas(struct igb_adapter *adapter) 2188 { 2189 struct e1000_hw *hw = &adapter->hw; 2190 u32 connsw = rd32(E1000_CONNSW); 2191 2192 /* configure for SerDes media detect */ 2193 if ((hw->phy.media_type == e1000_media_type_copper) && 2194 (!(connsw & E1000_CONNSW_SERDESD))) { 2195 connsw |= E1000_CONNSW_ENRGSRC; 2196 connsw |= E1000_CONNSW_AUTOSENSE_EN; 2197 wr32(E1000_CONNSW, connsw); 2198 wrfl(); 2199 } 2200 } 2201 2202 void igb_reset(struct igb_adapter *adapter) 2203 { 2204 struct pci_dev *pdev = adapter->pdev; 2205 struct e1000_hw *hw = &adapter->hw; 2206 struct e1000_mac_info *mac = &hw->mac; 2207 struct e1000_fc_info *fc = &hw->fc; 2208 u32 pba, hwm; 2209 2210 /* Repartition Pba for greater than 9k mtu 2211 * To take effect CTRL.RST is required. 2212 */ 2213 switch (mac->type) { 2214 case e1000_i350: 2215 case e1000_i354: 2216 case e1000_82580: 2217 pba = rd32(E1000_RXPBS); 2218 pba = igb_rxpbs_adjust_82580(pba); 2219 break; 2220 case e1000_82576: 2221 pba = rd32(E1000_RXPBS); 2222 pba &= E1000_RXPBS_SIZE_MASK_82576; 2223 break; 2224 case e1000_82575: 2225 case e1000_i210: 2226 case e1000_i211: 2227 default: 2228 pba = E1000_PBA_34K; 2229 break; 2230 } 2231 2232 if (mac->type == e1000_82575) { 2233 u32 min_rx_space, min_tx_space, needed_tx_space; 2234 2235 /* write Rx PBA so that hardware can report correct Tx PBA */ 2236 wr32(E1000_PBA, pba); 2237 2238 /* To maintain wire speed transmits, the Tx FIFO should be 2239 * large enough to accommodate two full transmit packets, 2240 * rounded up to the next 1KB and expressed in KB. Likewise, 2241 * the Rx FIFO should be large enough to accommodate at least 2242 * one full receive packet and is similarly rounded up and 2243 * expressed in KB. 2244 */ 2245 min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024); 2246 2247 /* The Tx FIFO also stores 16 bytes of information about the Tx 2248 * but don't include Ethernet FCS because hardware appends it. 2249 * We only need to round down to the nearest 512 byte block 2250 * count since the value we care about is 2 frames, not 1. 2251 */ 2252 min_tx_space = adapter->max_frame_size; 2253 min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN; 2254 min_tx_space = DIV_ROUND_UP(min_tx_space, 512); 2255 2256 /* upper 16 bits has Tx packet buffer allocation size in KB */ 2257 needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16); 2258 2259 /* If current Tx allocation is less than the min Tx FIFO size, 2260 * and the min Tx FIFO size is less than the current Rx FIFO 2261 * allocation, take space away from current Rx allocation. 2262 */ 2263 if (needed_tx_space < pba) { 2264 pba -= needed_tx_space; 2265 2266 /* if short on Rx space, Rx wins and must trump Tx 2267 * adjustment 2268 */ 2269 if (pba < min_rx_space) 2270 pba = min_rx_space; 2271 } 2272 2273 /* adjust PBA for jumbo frames */ 2274 wr32(E1000_PBA, pba); 2275 } 2276 2277 /* flow control settings 2278 * The high water mark must be low enough to fit one full frame 2279 * after transmitting the pause frame. As such we must have enough 2280 * space to allow for us to complete our current transmit and then 2281 * receive the frame that is in progress from the link partner. 2282 * Set it to: 2283 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 2284 */ 2285 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 2286 2287 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 2288 fc->low_water = fc->high_water - 16; 2289 fc->pause_time = 0xFFFF; 2290 fc->send_xon = 1; 2291 fc->current_mode = fc->requested_mode; 2292 2293 /* disable receive for all VFs and wait one second */ 2294 if (adapter->vfs_allocated_count) { 2295 int i; 2296 2297 for (i = 0 ; i < adapter->vfs_allocated_count; i++) 2298 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC; 2299 2300 /* ping all the active vfs to let them know we are going down */ 2301 igb_ping_all_vfs(adapter); 2302 2303 /* disable transmits and receives */ 2304 wr32(E1000_VFRE, 0); 2305 wr32(E1000_VFTE, 0); 2306 } 2307 2308 /* Allow time for pending master requests to run */ 2309 hw->mac.ops.reset_hw(hw); 2310 wr32(E1000_WUC, 0); 2311 2312 if (adapter->flags & IGB_FLAG_MEDIA_RESET) { 2313 /* need to resetup here after media swap */ 2314 adapter->ei.get_invariants(hw); 2315 adapter->flags &= ~IGB_FLAG_MEDIA_RESET; 2316 } 2317 if ((mac->type == e1000_82575) && 2318 (adapter->flags & IGB_FLAG_MAS_ENABLE)) { 2319 igb_enable_mas(adapter); 2320 } 2321 if (hw->mac.ops.init_hw(hw)) 2322 dev_err(&pdev->dev, "Hardware Error\n"); 2323 2324 /* RAR registers were cleared during init_hw, clear mac table */ 2325 igb_flush_mac_table(adapter); 2326 __dev_uc_unsync(adapter->netdev, NULL); 2327 2328 /* Recover default RAR entry */ 2329 igb_set_default_mac_filter(adapter); 2330 2331 /* Flow control settings reset on hardware reset, so guarantee flow 2332 * control is off when forcing speed. 2333 */ 2334 if (!hw->mac.autoneg) 2335 igb_force_mac_fc(hw); 2336 2337 igb_init_dmac(adapter, pba); 2338 #ifdef CONFIG_IGB_HWMON 2339 /* Re-initialize the thermal sensor on i350 devices. */ 2340 if (!test_bit(__IGB_DOWN, &adapter->state)) { 2341 if (mac->type == e1000_i350 && hw->bus.func == 0) { 2342 /* If present, re-initialize the external thermal sensor 2343 * interface. 2344 */ 2345 if (adapter->ets) 2346 mac->ops.init_thermal_sensor_thresh(hw); 2347 } 2348 } 2349 #endif 2350 /* Re-establish EEE setting */ 2351 if (hw->phy.media_type == e1000_media_type_copper) { 2352 switch (mac->type) { 2353 case e1000_i350: 2354 case e1000_i210: 2355 case e1000_i211: 2356 igb_set_eee_i350(hw, true, true); 2357 break; 2358 case e1000_i354: 2359 igb_set_eee_i354(hw, true, true); 2360 break; 2361 default: 2362 break; 2363 } 2364 } 2365 if (!netif_running(adapter->netdev)) 2366 igb_power_down_link(adapter); 2367 2368 igb_update_mng_vlan(adapter); 2369 2370 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 2371 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE); 2372 2373 /* Re-enable PTP, where applicable. */ 2374 if (adapter->ptp_flags & IGB_PTP_ENABLED) 2375 igb_ptp_reset(adapter); 2376 2377 igb_get_phy_info(hw); 2378 } 2379 2380 static netdev_features_t igb_fix_features(struct net_device *netdev, 2381 netdev_features_t features) 2382 { 2383 /* Since there is no support for separate Rx/Tx vlan accel 2384 * enable/disable make sure Tx flag is always in same state as Rx. 2385 */ 2386 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2387 features |= NETIF_F_HW_VLAN_CTAG_TX; 2388 else 2389 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 2390 2391 return features; 2392 } 2393 2394 static int igb_set_features(struct net_device *netdev, 2395 netdev_features_t features) 2396 { 2397 netdev_features_t changed = netdev->features ^ features; 2398 struct igb_adapter *adapter = netdev_priv(netdev); 2399 2400 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 2401 igb_vlan_mode(netdev, features); 2402 2403 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 2404 return 0; 2405 2406 if (!(features & NETIF_F_NTUPLE)) { 2407 struct hlist_node *node2; 2408 struct igb_nfc_filter *rule; 2409 2410 spin_lock(&adapter->nfc_lock); 2411 hlist_for_each_entry_safe(rule, node2, 2412 &adapter->nfc_filter_list, nfc_node) { 2413 igb_erase_filter(adapter, rule); 2414 hlist_del(&rule->nfc_node); 2415 kfree(rule); 2416 } 2417 spin_unlock(&adapter->nfc_lock); 2418 adapter->nfc_filter_count = 0; 2419 } 2420 2421 netdev->features = features; 2422 2423 if (netif_running(netdev)) 2424 igb_reinit_locked(adapter); 2425 else 2426 igb_reset(adapter); 2427 2428 return 0; 2429 } 2430 2431 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 2432 struct net_device *dev, 2433 const unsigned char *addr, u16 vid, 2434 u16 flags) 2435 { 2436 /* guarantee we can provide a unique filter for the unicast address */ 2437 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { 2438 struct igb_adapter *adapter = netdev_priv(dev); 2439 int vfn = adapter->vfs_allocated_count; 2440 2441 if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn)) 2442 return -ENOMEM; 2443 } 2444 2445 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags); 2446 } 2447 2448 #define IGB_MAX_MAC_HDR_LEN 127 2449 #define IGB_MAX_NETWORK_HDR_LEN 511 2450 2451 static netdev_features_t 2452 igb_features_check(struct sk_buff *skb, struct net_device *dev, 2453 netdev_features_t features) 2454 { 2455 unsigned int network_hdr_len, mac_hdr_len; 2456 2457 /* Make certain the headers can be described by a context descriptor */ 2458 mac_hdr_len = skb_network_header(skb) - skb->data; 2459 if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN)) 2460 return features & ~(NETIF_F_HW_CSUM | 2461 NETIF_F_SCTP_CRC | 2462 NETIF_F_HW_VLAN_CTAG_TX | 2463 NETIF_F_TSO | 2464 NETIF_F_TSO6); 2465 2466 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 2467 if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN)) 2468 return features & ~(NETIF_F_HW_CSUM | 2469 NETIF_F_SCTP_CRC | 2470 NETIF_F_TSO | 2471 NETIF_F_TSO6); 2472 2473 /* We can only support IPV4 TSO in tunnels if we can mangle the 2474 * inner IP ID field, so strip TSO if MANGLEID is not supported. 2475 */ 2476 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 2477 features &= ~NETIF_F_TSO; 2478 2479 return features; 2480 } 2481 2482 static int igb_offload_cbs(struct igb_adapter *adapter, 2483 struct tc_cbs_qopt_offload *qopt) 2484 { 2485 struct e1000_hw *hw = &adapter->hw; 2486 int err; 2487 2488 /* CBS offloading is only supported by i210 controller. */ 2489 if (hw->mac.type != e1000_i210) 2490 return -EOPNOTSUPP; 2491 2492 /* CBS offloading is only supported by queue 0 and queue 1. */ 2493 if (qopt->queue < 0 || qopt->queue > 1) 2494 return -EINVAL; 2495 2496 err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable, 2497 qopt->idleslope, qopt->sendslope, 2498 qopt->hicredit, qopt->locredit); 2499 if (err) 2500 return err; 2501 2502 if (is_fqtss_enabled(adapter)) { 2503 igb_configure_cbs(adapter, qopt->queue, qopt->enable, 2504 qopt->idleslope, qopt->sendslope, 2505 qopt->hicredit, qopt->locredit); 2506 2507 if (!is_any_cbs_enabled(adapter)) 2508 enable_fqtss(adapter, false); 2509 2510 } else { 2511 enable_fqtss(adapter, true); 2512 } 2513 2514 return 0; 2515 } 2516 2517 #define ETHER_TYPE_FULL_MASK ((__force __be16)~0) 2518 #define VLAN_PRIO_FULL_MASK (0x07) 2519 2520 static int igb_parse_cls_flower(struct igb_adapter *adapter, 2521 struct tc_cls_flower_offload *f, 2522 int traffic_class, 2523 struct igb_nfc_filter *input) 2524 { 2525 struct netlink_ext_ack *extack = f->common.extack; 2526 2527 if (f->dissector->used_keys & 2528 ~(BIT(FLOW_DISSECTOR_KEY_BASIC) | 2529 BIT(FLOW_DISSECTOR_KEY_CONTROL) | 2530 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 2531 BIT(FLOW_DISSECTOR_KEY_VLAN))) { 2532 NL_SET_ERR_MSG_MOD(extack, 2533 "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported"); 2534 return -EOPNOTSUPP; 2535 } 2536 2537 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 2538 struct flow_dissector_key_eth_addrs *key, *mask; 2539 2540 key = skb_flow_dissector_target(f->dissector, 2541 FLOW_DISSECTOR_KEY_ETH_ADDRS, 2542 f->key); 2543 mask = skb_flow_dissector_target(f->dissector, 2544 FLOW_DISSECTOR_KEY_ETH_ADDRS, 2545 f->mask); 2546 2547 if (!is_zero_ether_addr(mask->dst)) { 2548 if (!is_broadcast_ether_addr(mask->dst)) { 2549 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address"); 2550 return -EINVAL; 2551 } 2552 2553 input->filter.match_flags |= 2554 IGB_FILTER_FLAG_DST_MAC_ADDR; 2555 ether_addr_copy(input->filter.dst_addr, key->dst); 2556 } 2557 2558 if (!is_zero_ether_addr(mask->src)) { 2559 if (!is_broadcast_ether_addr(mask->src)) { 2560 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address"); 2561 return -EINVAL; 2562 } 2563 2564 input->filter.match_flags |= 2565 IGB_FILTER_FLAG_SRC_MAC_ADDR; 2566 ether_addr_copy(input->filter.src_addr, key->src); 2567 } 2568 } 2569 2570 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 2571 struct flow_dissector_key_basic *key, *mask; 2572 2573 key = skb_flow_dissector_target(f->dissector, 2574 FLOW_DISSECTOR_KEY_BASIC, 2575 f->key); 2576 mask = skb_flow_dissector_target(f->dissector, 2577 FLOW_DISSECTOR_KEY_BASIC, 2578 f->mask); 2579 2580 if (mask->n_proto) { 2581 if (mask->n_proto != ETHER_TYPE_FULL_MASK) { 2582 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter"); 2583 return -EINVAL; 2584 } 2585 2586 input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE; 2587 input->filter.etype = key->n_proto; 2588 } 2589 } 2590 2591 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) { 2592 struct flow_dissector_key_vlan *key, *mask; 2593 2594 key = skb_flow_dissector_target(f->dissector, 2595 FLOW_DISSECTOR_KEY_VLAN, 2596 f->key); 2597 mask = skb_flow_dissector_target(f->dissector, 2598 FLOW_DISSECTOR_KEY_VLAN, 2599 f->mask); 2600 2601 if (mask->vlan_priority) { 2602 if (mask->vlan_priority != VLAN_PRIO_FULL_MASK) { 2603 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority"); 2604 return -EINVAL; 2605 } 2606 2607 input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI; 2608 input->filter.vlan_tci = key->vlan_priority; 2609 } 2610 } 2611 2612 input->action = traffic_class; 2613 input->cookie = f->cookie; 2614 2615 return 0; 2616 } 2617 2618 static int igb_configure_clsflower(struct igb_adapter *adapter, 2619 struct tc_cls_flower_offload *cls_flower) 2620 { 2621 struct netlink_ext_ack *extack = cls_flower->common.extack; 2622 struct igb_nfc_filter *filter, *f; 2623 int err, tc; 2624 2625 tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 2626 if (tc < 0) { 2627 NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class"); 2628 return -EINVAL; 2629 } 2630 2631 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 2632 if (!filter) 2633 return -ENOMEM; 2634 2635 err = igb_parse_cls_flower(adapter, cls_flower, tc, filter); 2636 if (err < 0) 2637 goto err_parse; 2638 2639 spin_lock(&adapter->nfc_lock); 2640 2641 hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) { 2642 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) { 2643 err = -EEXIST; 2644 NL_SET_ERR_MSG_MOD(extack, 2645 "This filter is already set in ethtool"); 2646 goto err_locked; 2647 } 2648 } 2649 2650 hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) { 2651 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) { 2652 err = -EEXIST; 2653 NL_SET_ERR_MSG_MOD(extack, 2654 "This filter is already set in cls_flower"); 2655 goto err_locked; 2656 } 2657 } 2658 2659 err = igb_add_filter(adapter, filter); 2660 if (err < 0) { 2661 NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter"); 2662 goto err_locked; 2663 } 2664 2665 hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list); 2666 2667 spin_unlock(&adapter->nfc_lock); 2668 2669 return 0; 2670 2671 err_locked: 2672 spin_unlock(&adapter->nfc_lock); 2673 2674 err_parse: 2675 kfree(filter); 2676 2677 return err; 2678 } 2679 2680 static int igb_delete_clsflower(struct igb_adapter *adapter, 2681 struct tc_cls_flower_offload *cls_flower) 2682 { 2683 struct igb_nfc_filter *filter; 2684 int err; 2685 2686 spin_lock(&adapter->nfc_lock); 2687 2688 hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node) 2689 if (filter->cookie == cls_flower->cookie) 2690 break; 2691 2692 if (!filter) { 2693 err = -ENOENT; 2694 goto out; 2695 } 2696 2697 err = igb_erase_filter(adapter, filter); 2698 if (err < 0) 2699 goto out; 2700 2701 hlist_del(&filter->nfc_node); 2702 kfree(filter); 2703 2704 out: 2705 spin_unlock(&adapter->nfc_lock); 2706 2707 return err; 2708 } 2709 2710 static int igb_setup_tc_cls_flower(struct igb_adapter *adapter, 2711 struct tc_cls_flower_offload *cls_flower) 2712 { 2713 switch (cls_flower->command) { 2714 case TC_CLSFLOWER_REPLACE: 2715 return igb_configure_clsflower(adapter, cls_flower); 2716 case TC_CLSFLOWER_DESTROY: 2717 return igb_delete_clsflower(adapter, cls_flower); 2718 case TC_CLSFLOWER_STATS: 2719 return -EOPNOTSUPP; 2720 default: 2721 return -EINVAL; 2722 } 2723 } 2724 2725 static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 2726 void *cb_priv) 2727 { 2728 struct igb_adapter *adapter = cb_priv; 2729 2730 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) 2731 return -EOPNOTSUPP; 2732 2733 switch (type) { 2734 case TC_SETUP_CLSFLOWER: 2735 return igb_setup_tc_cls_flower(adapter, type_data); 2736 2737 default: 2738 return -EOPNOTSUPP; 2739 } 2740 } 2741 2742 static int igb_setup_tc_block(struct igb_adapter *adapter, 2743 struct tc_block_offload *f) 2744 { 2745 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 2746 return -EOPNOTSUPP; 2747 2748 switch (f->command) { 2749 case TC_BLOCK_BIND: 2750 return tcf_block_cb_register(f->block, igb_setup_tc_block_cb, 2751 adapter, adapter); 2752 case TC_BLOCK_UNBIND: 2753 tcf_block_cb_unregister(f->block, igb_setup_tc_block_cb, 2754 adapter); 2755 return 0; 2756 default: 2757 return -EOPNOTSUPP; 2758 } 2759 } 2760 2761 static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type, 2762 void *type_data) 2763 { 2764 struct igb_adapter *adapter = netdev_priv(dev); 2765 2766 switch (type) { 2767 case TC_SETUP_QDISC_CBS: 2768 return igb_offload_cbs(adapter, type_data); 2769 case TC_SETUP_BLOCK: 2770 return igb_setup_tc_block(adapter, type_data); 2771 2772 default: 2773 return -EOPNOTSUPP; 2774 } 2775 } 2776 2777 static const struct net_device_ops igb_netdev_ops = { 2778 .ndo_open = igb_open, 2779 .ndo_stop = igb_close, 2780 .ndo_start_xmit = igb_xmit_frame, 2781 .ndo_get_stats64 = igb_get_stats64, 2782 .ndo_set_rx_mode = igb_set_rx_mode, 2783 .ndo_set_mac_address = igb_set_mac, 2784 .ndo_change_mtu = igb_change_mtu, 2785 .ndo_do_ioctl = igb_ioctl, 2786 .ndo_tx_timeout = igb_tx_timeout, 2787 .ndo_validate_addr = eth_validate_addr, 2788 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid, 2789 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid, 2790 .ndo_set_vf_mac = igb_ndo_set_vf_mac, 2791 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan, 2792 .ndo_set_vf_rate = igb_ndo_set_vf_bw, 2793 .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk, 2794 .ndo_set_vf_trust = igb_ndo_set_vf_trust, 2795 .ndo_get_vf_config = igb_ndo_get_vf_config, 2796 #ifdef CONFIG_NET_POLL_CONTROLLER 2797 .ndo_poll_controller = igb_netpoll, 2798 #endif 2799 .ndo_fix_features = igb_fix_features, 2800 .ndo_set_features = igb_set_features, 2801 .ndo_fdb_add = igb_ndo_fdb_add, 2802 .ndo_features_check = igb_features_check, 2803 .ndo_setup_tc = igb_setup_tc, 2804 }; 2805 2806 /** 2807 * igb_set_fw_version - Configure version string for ethtool 2808 * @adapter: adapter struct 2809 **/ 2810 void igb_set_fw_version(struct igb_adapter *adapter) 2811 { 2812 struct e1000_hw *hw = &adapter->hw; 2813 struct e1000_fw_version fw; 2814 2815 igb_get_fw_version(hw, &fw); 2816 2817 switch (hw->mac.type) { 2818 case e1000_i210: 2819 case e1000_i211: 2820 if (!(igb_get_flash_presence_i210(hw))) { 2821 snprintf(adapter->fw_version, 2822 sizeof(adapter->fw_version), 2823 "%2d.%2d-%d", 2824 fw.invm_major, fw.invm_minor, 2825 fw.invm_img_type); 2826 break; 2827 } 2828 /* fall through */ 2829 default: 2830 /* if option is rom valid, display its version too */ 2831 if (fw.or_valid) { 2832 snprintf(adapter->fw_version, 2833 sizeof(adapter->fw_version), 2834 "%d.%d, 0x%08x, %d.%d.%d", 2835 fw.eep_major, fw.eep_minor, fw.etrack_id, 2836 fw.or_major, fw.or_build, fw.or_patch); 2837 /* no option rom */ 2838 } else if (fw.etrack_id != 0X0000) { 2839 snprintf(adapter->fw_version, 2840 sizeof(adapter->fw_version), 2841 "%d.%d, 0x%08x", 2842 fw.eep_major, fw.eep_minor, fw.etrack_id); 2843 } else { 2844 snprintf(adapter->fw_version, 2845 sizeof(adapter->fw_version), 2846 "%d.%d.%d", 2847 fw.eep_major, fw.eep_minor, fw.eep_build); 2848 } 2849 break; 2850 } 2851 } 2852 2853 /** 2854 * igb_init_mas - init Media Autosense feature if enabled in the NVM 2855 * 2856 * @adapter: adapter struct 2857 **/ 2858 static void igb_init_mas(struct igb_adapter *adapter) 2859 { 2860 struct e1000_hw *hw = &adapter->hw; 2861 u16 eeprom_data; 2862 2863 hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data); 2864 switch (hw->bus.func) { 2865 case E1000_FUNC_0: 2866 if (eeprom_data & IGB_MAS_ENABLE_0) { 2867 adapter->flags |= IGB_FLAG_MAS_ENABLE; 2868 netdev_info(adapter->netdev, 2869 "MAS: Enabling Media Autosense for port %d\n", 2870 hw->bus.func); 2871 } 2872 break; 2873 case E1000_FUNC_1: 2874 if (eeprom_data & IGB_MAS_ENABLE_1) { 2875 adapter->flags |= IGB_FLAG_MAS_ENABLE; 2876 netdev_info(adapter->netdev, 2877 "MAS: Enabling Media Autosense for port %d\n", 2878 hw->bus.func); 2879 } 2880 break; 2881 case E1000_FUNC_2: 2882 if (eeprom_data & IGB_MAS_ENABLE_2) { 2883 adapter->flags |= IGB_FLAG_MAS_ENABLE; 2884 netdev_info(adapter->netdev, 2885 "MAS: Enabling Media Autosense for port %d\n", 2886 hw->bus.func); 2887 } 2888 break; 2889 case E1000_FUNC_3: 2890 if (eeprom_data & IGB_MAS_ENABLE_3) { 2891 adapter->flags |= IGB_FLAG_MAS_ENABLE; 2892 netdev_info(adapter->netdev, 2893 "MAS: Enabling Media Autosense for port %d\n", 2894 hw->bus.func); 2895 } 2896 break; 2897 default: 2898 /* Shouldn't get here */ 2899 netdev_err(adapter->netdev, 2900 "MAS: Invalid port configuration, returning\n"); 2901 break; 2902 } 2903 } 2904 2905 /** 2906 * igb_init_i2c - Init I2C interface 2907 * @adapter: pointer to adapter structure 2908 **/ 2909 static s32 igb_init_i2c(struct igb_adapter *adapter) 2910 { 2911 s32 status = 0; 2912 2913 /* I2C interface supported on i350 devices */ 2914 if (adapter->hw.mac.type != e1000_i350) 2915 return 0; 2916 2917 /* Initialize the i2c bus which is controlled by the registers. 2918 * This bus will use the i2c_algo_bit structue that implements 2919 * the protocol through toggling of the 4 bits in the register. 2920 */ 2921 adapter->i2c_adap.owner = THIS_MODULE; 2922 adapter->i2c_algo = igb_i2c_algo; 2923 adapter->i2c_algo.data = adapter; 2924 adapter->i2c_adap.algo_data = &adapter->i2c_algo; 2925 adapter->i2c_adap.dev.parent = &adapter->pdev->dev; 2926 strlcpy(adapter->i2c_adap.name, "igb BB", 2927 sizeof(adapter->i2c_adap.name)); 2928 status = i2c_bit_add_bus(&adapter->i2c_adap); 2929 return status; 2930 } 2931 2932 /** 2933 * igb_probe - Device Initialization Routine 2934 * @pdev: PCI device information struct 2935 * @ent: entry in igb_pci_tbl 2936 * 2937 * Returns 0 on success, negative on failure 2938 * 2939 * igb_probe initializes an adapter identified by a pci_dev structure. 2940 * The OS initialization, configuring of the adapter private structure, 2941 * and a hardware reset occur. 2942 **/ 2943 static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 2944 { 2945 struct net_device *netdev; 2946 struct igb_adapter *adapter; 2947 struct e1000_hw *hw; 2948 u16 eeprom_data = 0; 2949 s32 ret_val; 2950 static int global_quad_port_a; /* global quad port a indication */ 2951 const struct e1000_info *ei = igb_info_tbl[ent->driver_data]; 2952 int err, pci_using_dac; 2953 u8 part_str[E1000_PBANUM_LENGTH]; 2954 2955 /* Catch broken hardware that put the wrong VF device ID in 2956 * the PCIe SR-IOV capability. 2957 */ 2958 if (pdev->is_virtfn) { 2959 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", 2960 pci_name(pdev), pdev->vendor, pdev->device); 2961 return -EINVAL; 2962 } 2963 2964 err = pci_enable_device_mem(pdev); 2965 if (err) 2966 return err; 2967 2968 pci_using_dac = 0; 2969 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2970 if (!err) { 2971 pci_using_dac = 1; 2972 } else { 2973 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 2974 if (err) { 2975 dev_err(&pdev->dev, 2976 "No usable DMA configuration, aborting\n"); 2977 goto err_dma; 2978 } 2979 } 2980 2981 err = pci_request_mem_regions(pdev, igb_driver_name); 2982 if (err) 2983 goto err_pci_reg; 2984 2985 pci_enable_pcie_error_reporting(pdev); 2986 2987 pci_set_master(pdev); 2988 pci_save_state(pdev); 2989 2990 err = -ENOMEM; 2991 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter), 2992 IGB_MAX_TX_QUEUES); 2993 if (!netdev) 2994 goto err_alloc_etherdev; 2995 2996 SET_NETDEV_DEV(netdev, &pdev->dev); 2997 2998 pci_set_drvdata(pdev, netdev); 2999 adapter = netdev_priv(netdev); 3000 adapter->netdev = netdev; 3001 adapter->pdev = pdev; 3002 hw = &adapter->hw; 3003 hw->back = adapter; 3004 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3005 3006 err = -EIO; 3007 adapter->io_addr = pci_iomap(pdev, 0, 0); 3008 if (!adapter->io_addr) 3009 goto err_ioremap; 3010 /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */ 3011 hw->hw_addr = adapter->io_addr; 3012 3013 netdev->netdev_ops = &igb_netdev_ops; 3014 igb_set_ethtool_ops(netdev); 3015 netdev->watchdog_timeo = 5 * HZ; 3016 3017 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 3018 3019 netdev->mem_start = pci_resource_start(pdev, 0); 3020 netdev->mem_end = pci_resource_end(pdev, 0); 3021 3022 /* PCI config space info */ 3023 hw->vendor_id = pdev->vendor; 3024 hw->device_id = pdev->device; 3025 hw->revision_id = pdev->revision; 3026 hw->subsystem_vendor_id = pdev->subsystem_vendor; 3027 hw->subsystem_device_id = pdev->subsystem_device; 3028 3029 /* Copy the default MAC, PHY and NVM function pointers */ 3030 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 3031 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 3032 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 3033 /* Initialize skew-specific constants */ 3034 err = ei->get_invariants(hw); 3035 if (err) 3036 goto err_sw_init; 3037 3038 /* setup the private structure */ 3039 err = igb_sw_init(adapter); 3040 if (err) 3041 goto err_sw_init; 3042 3043 igb_get_bus_info_pcie(hw); 3044 3045 hw->phy.autoneg_wait_to_complete = false; 3046 3047 /* Copper options */ 3048 if (hw->phy.media_type == e1000_media_type_copper) { 3049 hw->phy.mdix = AUTO_ALL_MODES; 3050 hw->phy.disable_polarity_correction = false; 3051 hw->phy.ms_type = e1000_ms_hw_default; 3052 } 3053 3054 if (igb_check_reset_block(hw)) 3055 dev_info(&pdev->dev, 3056 "PHY reset is blocked due to SOL/IDER session.\n"); 3057 3058 /* features is initialized to 0 in allocation, it might have bits 3059 * set by igb_sw_init so we should use an or instead of an 3060 * assignment. 3061 */ 3062 netdev->features |= NETIF_F_SG | 3063 NETIF_F_TSO | 3064 NETIF_F_TSO6 | 3065 NETIF_F_RXHASH | 3066 NETIF_F_RXCSUM | 3067 NETIF_F_HW_CSUM; 3068 3069 if (hw->mac.type >= e1000_82576) 3070 netdev->features |= NETIF_F_SCTP_CRC; 3071 3072 if (hw->mac.type >= e1000_i350) 3073 netdev->features |= NETIF_F_HW_TC; 3074 3075 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 3076 NETIF_F_GSO_GRE_CSUM | \ 3077 NETIF_F_GSO_IPXIP4 | \ 3078 NETIF_F_GSO_IPXIP6 | \ 3079 NETIF_F_GSO_UDP_TUNNEL | \ 3080 NETIF_F_GSO_UDP_TUNNEL_CSUM) 3081 3082 netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES; 3083 netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES; 3084 3085 /* copy netdev features into list of user selectable features */ 3086 netdev->hw_features |= netdev->features | 3087 NETIF_F_HW_VLAN_CTAG_RX | 3088 NETIF_F_HW_VLAN_CTAG_TX | 3089 NETIF_F_RXALL; 3090 3091 if (hw->mac.type >= e1000_i350) 3092 netdev->hw_features |= NETIF_F_NTUPLE; 3093 3094 if (pci_using_dac) 3095 netdev->features |= NETIF_F_HIGHDMA; 3096 3097 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 3098 netdev->mpls_features |= NETIF_F_HW_CSUM; 3099 netdev->hw_enc_features |= netdev->vlan_features; 3100 3101 /* set this bit last since it cannot be part of vlan_features */ 3102 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 3103 NETIF_F_HW_VLAN_CTAG_RX | 3104 NETIF_F_HW_VLAN_CTAG_TX; 3105 3106 netdev->priv_flags |= IFF_SUPP_NOFCS; 3107 3108 netdev->priv_flags |= IFF_UNICAST_FLT; 3109 3110 /* MTU range: 68 - 9216 */ 3111 netdev->min_mtu = ETH_MIN_MTU; 3112 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 3113 3114 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw); 3115 3116 /* before reading the NVM, reset the controller to put the device in a 3117 * known good starting state 3118 */ 3119 hw->mac.ops.reset_hw(hw); 3120 3121 /* make sure the NVM is good , i211/i210 parts can have special NVM 3122 * that doesn't contain a checksum 3123 */ 3124 switch (hw->mac.type) { 3125 case e1000_i210: 3126 case e1000_i211: 3127 if (igb_get_flash_presence_i210(hw)) { 3128 if (hw->nvm.ops.validate(hw) < 0) { 3129 dev_err(&pdev->dev, 3130 "The NVM Checksum Is Not Valid\n"); 3131 err = -EIO; 3132 goto err_eeprom; 3133 } 3134 } 3135 break; 3136 default: 3137 if (hw->nvm.ops.validate(hw) < 0) { 3138 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 3139 err = -EIO; 3140 goto err_eeprom; 3141 } 3142 break; 3143 } 3144 3145 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 3146 /* copy the MAC address out of the NVM */ 3147 if (hw->mac.ops.read_mac_addr(hw)) 3148 dev_err(&pdev->dev, "NVM Read Error\n"); 3149 } 3150 3151 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 3152 3153 if (!is_valid_ether_addr(netdev->dev_addr)) { 3154 dev_err(&pdev->dev, "Invalid MAC Address\n"); 3155 err = -EIO; 3156 goto err_eeprom; 3157 } 3158 3159 igb_set_default_mac_filter(adapter); 3160 3161 /* get firmware version for ethtool -i */ 3162 igb_set_fw_version(adapter); 3163 3164 /* configure RXPBSIZE and TXPBSIZE */ 3165 if (hw->mac.type == e1000_i210) { 3166 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT); 3167 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT); 3168 } 3169 3170 timer_setup(&adapter->watchdog_timer, igb_watchdog, 0); 3171 timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0); 3172 3173 INIT_WORK(&adapter->reset_task, igb_reset_task); 3174 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task); 3175 3176 /* Initialize link properties that are user-changeable */ 3177 adapter->fc_autoneg = true; 3178 hw->mac.autoneg = true; 3179 hw->phy.autoneg_advertised = 0x2f; 3180 3181 hw->fc.requested_mode = e1000_fc_default; 3182 hw->fc.current_mode = e1000_fc_default; 3183 3184 igb_validate_mdi_setting(hw); 3185 3186 /* By default, support wake on port A */ 3187 if (hw->bus.func == 0) 3188 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3189 3190 /* Check the NVM for wake support on non-port A ports */ 3191 if (hw->mac.type >= e1000_82580) 3192 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A + 3193 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1, 3194 &eeprom_data); 3195 else if (hw->bus.func == 1) 3196 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 3197 3198 if (eeprom_data & IGB_EEPROM_APME) 3199 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3200 3201 /* now that we have the eeprom settings, apply the special cases where 3202 * the eeprom may be wrong or the board simply won't support wake on 3203 * lan on a particular port 3204 */ 3205 switch (pdev->device) { 3206 case E1000_DEV_ID_82575GB_QUAD_COPPER: 3207 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3208 break; 3209 case E1000_DEV_ID_82575EB_FIBER_SERDES: 3210 case E1000_DEV_ID_82576_FIBER: 3211 case E1000_DEV_ID_82576_SERDES: 3212 /* Wake events only supported on port A for dual fiber 3213 * regardless of eeprom setting 3214 */ 3215 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1) 3216 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3217 break; 3218 case E1000_DEV_ID_82576_QUAD_COPPER: 3219 case E1000_DEV_ID_82576_QUAD_COPPER_ET2: 3220 /* if quad port adapter, disable WoL on all but port A */ 3221 if (global_quad_port_a != 0) 3222 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3223 else 3224 adapter->flags |= IGB_FLAG_QUAD_PORT_A; 3225 /* Reset for multiple quad port adapters */ 3226 if (++global_quad_port_a == 4) 3227 global_quad_port_a = 0; 3228 break; 3229 default: 3230 /* If the device can't wake, don't set software support */ 3231 if (!device_can_wakeup(&adapter->pdev->dev)) 3232 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; 3233 } 3234 3235 /* initialize the wol settings based on the eeprom settings */ 3236 if (adapter->flags & IGB_FLAG_WOL_SUPPORTED) 3237 adapter->wol |= E1000_WUFC_MAG; 3238 3239 /* Some vendors want WoL disabled by default, but still supported */ 3240 if ((hw->mac.type == e1000_i350) && 3241 (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) { 3242 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3243 adapter->wol = 0; 3244 } 3245 3246 /* Some vendors want the ability to Use the EEPROM setting as 3247 * enable/disable only, and not for capability 3248 */ 3249 if (((hw->mac.type == e1000_i350) || 3250 (hw->mac.type == e1000_i354)) && 3251 (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) { 3252 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3253 adapter->wol = 0; 3254 } 3255 if (hw->mac.type == e1000_i350) { 3256 if (((pdev->subsystem_device == 0x5001) || 3257 (pdev->subsystem_device == 0x5002)) && 3258 (hw->bus.func == 0)) { 3259 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3260 adapter->wol = 0; 3261 } 3262 if (pdev->subsystem_device == 0x1F52) 3263 adapter->flags |= IGB_FLAG_WOL_SUPPORTED; 3264 } 3265 3266 device_set_wakeup_enable(&adapter->pdev->dev, 3267 adapter->flags & IGB_FLAG_WOL_SUPPORTED); 3268 3269 /* reset the hardware with the new settings */ 3270 igb_reset(adapter); 3271 3272 /* Init the I2C interface */ 3273 err = igb_init_i2c(adapter); 3274 if (err) { 3275 dev_err(&pdev->dev, "failed to init i2c interface\n"); 3276 goto err_eeprom; 3277 } 3278 3279 /* let the f/w know that the h/w is now under the control of the 3280 * driver. 3281 */ 3282 igb_get_hw_control(adapter); 3283 3284 strcpy(netdev->name, "eth%d"); 3285 err = register_netdev(netdev); 3286 if (err) 3287 goto err_register; 3288 3289 /* carrier off reporting is important to ethtool even BEFORE open */ 3290 netif_carrier_off(netdev); 3291 3292 #ifdef CONFIG_IGB_DCA 3293 if (dca_add_requester(&pdev->dev) == 0) { 3294 adapter->flags |= IGB_FLAG_DCA_ENABLED; 3295 dev_info(&pdev->dev, "DCA enabled\n"); 3296 igb_setup_dca(adapter); 3297 } 3298 3299 #endif 3300 #ifdef CONFIG_IGB_HWMON 3301 /* Initialize the thermal sensor on i350 devices. */ 3302 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) { 3303 u16 ets_word; 3304 3305 /* Read the NVM to determine if this i350 device supports an 3306 * external thermal sensor. 3307 */ 3308 hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word); 3309 if (ets_word != 0x0000 && ets_word != 0xFFFF) 3310 adapter->ets = true; 3311 else 3312 adapter->ets = false; 3313 if (igb_sysfs_init(adapter)) 3314 dev_err(&pdev->dev, 3315 "failed to allocate sysfs resources\n"); 3316 } else { 3317 adapter->ets = false; 3318 } 3319 #endif 3320 /* Check if Media Autosense is enabled */ 3321 adapter->ei = *ei; 3322 if (hw->dev_spec._82575.mas_capable) 3323 igb_init_mas(adapter); 3324 3325 /* do hw tstamp init after resetting */ 3326 igb_ptp_init(adapter); 3327 3328 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n"); 3329 /* print bus type/speed/width info, not applicable to i354 */ 3330 if (hw->mac.type != e1000_i354) { 3331 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n", 3332 netdev->name, 3333 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" : 3334 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" : 3335 "unknown"), 3336 ((hw->bus.width == e1000_bus_width_pcie_x4) ? 3337 "Width x4" : 3338 (hw->bus.width == e1000_bus_width_pcie_x2) ? 3339 "Width x2" : 3340 (hw->bus.width == e1000_bus_width_pcie_x1) ? 3341 "Width x1" : "unknown"), netdev->dev_addr); 3342 } 3343 3344 if ((hw->mac.type >= e1000_i210 || 3345 igb_get_flash_presence_i210(hw))) { 3346 ret_val = igb_read_part_string(hw, part_str, 3347 E1000_PBANUM_LENGTH); 3348 } else { 3349 ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND; 3350 } 3351 3352 if (ret_val) 3353 strcpy(part_str, "Unknown"); 3354 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str); 3355 dev_info(&pdev->dev, 3356 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n", 3357 (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" : 3358 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy", 3359 adapter->num_rx_queues, adapter->num_tx_queues); 3360 if (hw->phy.media_type == e1000_media_type_copper) { 3361 switch (hw->mac.type) { 3362 case e1000_i350: 3363 case e1000_i210: 3364 case e1000_i211: 3365 /* Enable EEE for internal copper PHY devices */ 3366 err = igb_set_eee_i350(hw, true, true); 3367 if ((!err) && 3368 (!hw->dev_spec._82575.eee_disable)) { 3369 adapter->eee_advert = 3370 MDIO_EEE_100TX | MDIO_EEE_1000T; 3371 adapter->flags |= IGB_FLAG_EEE; 3372 } 3373 break; 3374 case e1000_i354: 3375 if ((rd32(E1000_CTRL_EXT) & 3376 E1000_CTRL_EXT_LINK_MODE_SGMII)) { 3377 err = igb_set_eee_i354(hw, true, true); 3378 if ((!err) && 3379 (!hw->dev_spec._82575.eee_disable)) { 3380 adapter->eee_advert = 3381 MDIO_EEE_100TX | MDIO_EEE_1000T; 3382 adapter->flags |= IGB_FLAG_EEE; 3383 } 3384 } 3385 break; 3386 default: 3387 break; 3388 } 3389 } 3390 pm_runtime_put_noidle(&pdev->dev); 3391 return 0; 3392 3393 err_register: 3394 igb_release_hw_control(adapter); 3395 memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap)); 3396 err_eeprom: 3397 if (!igb_check_reset_block(hw)) 3398 igb_reset_phy(hw); 3399 3400 if (hw->flash_address) 3401 iounmap(hw->flash_address); 3402 err_sw_init: 3403 kfree(adapter->mac_table); 3404 kfree(adapter->shadow_vfta); 3405 igb_clear_interrupt_scheme(adapter); 3406 #ifdef CONFIG_PCI_IOV 3407 igb_disable_sriov(pdev); 3408 #endif 3409 pci_iounmap(pdev, adapter->io_addr); 3410 err_ioremap: 3411 free_netdev(netdev); 3412 err_alloc_etherdev: 3413 pci_release_mem_regions(pdev); 3414 err_pci_reg: 3415 err_dma: 3416 pci_disable_device(pdev); 3417 return err; 3418 } 3419 3420 #ifdef CONFIG_PCI_IOV 3421 static int igb_disable_sriov(struct pci_dev *pdev) 3422 { 3423 struct net_device *netdev = pci_get_drvdata(pdev); 3424 struct igb_adapter *adapter = netdev_priv(netdev); 3425 struct e1000_hw *hw = &adapter->hw; 3426 3427 /* reclaim resources allocated to VFs */ 3428 if (adapter->vf_data) { 3429 /* disable iov and allow time for transactions to clear */ 3430 if (pci_vfs_assigned(pdev)) { 3431 dev_warn(&pdev->dev, 3432 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n"); 3433 return -EPERM; 3434 } else { 3435 pci_disable_sriov(pdev); 3436 msleep(500); 3437 } 3438 3439 kfree(adapter->vf_mac_list); 3440 adapter->vf_mac_list = NULL; 3441 kfree(adapter->vf_data); 3442 adapter->vf_data = NULL; 3443 adapter->vfs_allocated_count = 0; 3444 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 3445 wrfl(); 3446 msleep(100); 3447 dev_info(&pdev->dev, "IOV Disabled\n"); 3448 3449 /* Re-enable DMA Coalescing flag since IOV is turned off */ 3450 adapter->flags |= IGB_FLAG_DMAC; 3451 } 3452 3453 return 0; 3454 } 3455 3456 static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs) 3457 { 3458 struct net_device *netdev = pci_get_drvdata(pdev); 3459 struct igb_adapter *adapter = netdev_priv(netdev); 3460 int old_vfs = pci_num_vf(pdev); 3461 struct vf_mac_filter *mac_list; 3462 int err = 0; 3463 int num_vf_mac_filters, i; 3464 3465 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) { 3466 err = -EPERM; 3467 goto out; 3468 } 3469 if (!num_vfs) 3470 goto out; 3471 3472 if (old_vfs) { 3473 dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n", 3474 old_vfs, max_vfs); 3475 adapter->vfs_allocated_count = old_vfs; 3476 } else 3477 adapter->vfs_allocated_count = num_vfs; 3478 3479 adapter->vf_data = kcalloc(adapter->vfs_allocated_count, 3480 sizeof(struct vf_data_storage), GFP_KERNEL); 3481 3482 /* if allocation failed then we do not support SR-IOV */ 3483 if (!adapter->vf_data) { 3484 adapter->vfs_allocated_count = 0; 3485 err = -ENOMEM; 3486 goto out; 3487 } 3488 3489 /* Due to the limited number of RAR entries calculate potential 3490 * number of MAC filters available for the VFs. Reserve entries 3491 * for PF default MAC, PF MAC filters and at least one RAR entry 3492 * for each VF for VF MAC. 3493 */ 3494 num_vf_mac_filters = adapter->hw.mac.rar_entry_count - 3495 (1 + IGB_PF_MAC_FILTERS_RESERVED + 3496 adapter->vfs_allocated_count); 3497 3498 adapter->vf_mac_list = kcalloc(num_vf_mac_filters, 3499 sizeof(struct vf_mac_filter), 3500 GFP_KERNEL); 3501 3502 mac_list = adapter->vf_mac_list; 3503 INIT_LIST_HEAD(&adapter->vf_macs.l); 3504 3505 if (adapter->vf_mac_list) { 3506 /* Initialize list of VF MAC filters */ 3507 for (i = 0; i < num_vf_mac_filters; i++) { 3508 mac_list->vf = -1; 3509 mac_list->free = true; 3510 list_add(&mac_list->l, &adapter->vf_macs.l); 3511 mac_list++; 3512 } 3513 } else { 3514 /* If we could not allocate memory for the VF MAC filters 3515 * we can continue without this feature but warn user. 3516 */ 3517 dev_err(&pdev->dev, 3518 "Unable to allocate memory for VF MAC filter list\n"); 3519 } 3520 3521 /* only call pci_enable_sriov() if no VFs are allocated already */ 3522 if (!old_vfs) { 3523 err = pci_enable_sriov(pdev, adapter->vfs_allocated_count); 3524 if (err) 3525 goto err_out; 3526 } 3527 dev_info(&pdev->dev, "%d VFs allocated\n", 3528 adapter->vfs_allocated_count); 3529 for (i = 0; i < adapter->vfs_allocated_count; i++) 3530 igb_vf_configure(adapter, i); 3531 3532 /* DMA Coalescing is not supported in IOV mode. */ 3533 adapter->flags &= ~IGB_FLAG_DMAC; 3534 goto out; 3535 3536 err_out: 3537 kfree(adapter->vf_mac_list); 3538 adapter->vf_mac_list = NULL; 3539 kfree(adapter->vf_data); 3540 adapter->vf_data = NULL; 3541 adapter->vfs_allocated_count = 0; 3542 out: 3543 return err; 3544 } 3545 3546 #endif 3547 /** 3548 * igb_remove_i2c - Cleanup I2C interface 3549 * @adapter: pointer to adapter structure 3550 **/ 3551 static void igb_remove_i2c(struct igb_adapter *adapter) 3552 { 3553 /* free the adapter bus structure */ 3554 i2c_del_adapter(&adapter->i2c_adap); 3555 } 3556 3557 /** 3558 * igb_remove - Device Removal Routine 3559 * @pdev: PCI device information struct 3560 * 3561 * igb_remove is called by the PCI subsystem to alert the driver 3562 * that it should release a PCI device. The could be caused by a 3563 * Hot-Plug event, or because the driver is going to be removed from 3564 * memory. 3565 **/ 3566 static void igb_remove(struct pci_dev *pdev) 3567 { 3568 struct net_device *netdev = pci_get_drvdata(pdev); 3569 struct igb_adapter *adapter = netdev_priv(netdev); 3570 struct e1000_hw *hw = &adapter->hw; 3571 3572 pm_runtime_get_noresume(&pdev->dev); 3573 #ifdef CONFIG_IGB_HWMON 3574 igb_sysfs_exit(adapter); 3575 #endif 3576 igb_remove_i2c(adapter); 3577 igb_ptp_stop(adapter); 3578 /* The watchdog timer may be rescheduled, so explicitly 3579 * disable watchdog from being rescheduled. 3580 */ 3581 set_bit(__IGB_DOWN, &adapter->state); 3582 del_timer_sync(&adapter->watchdog_timer); 3583 del_timer_sync(&adapter->phy_info_timer); 3584 3585 cancel_work_sync(&adapter->reset_task); 3586 cancel_work_sync(&adapter->watchdog_task); 3587 3588 #ifdef CONFIG_IGB_DCA 3589 if (adapter->flags & IGB_FLAG_DCA_ENABLED) { 3590 dev_info(&pdev->dev, "DCA disabled\n"); 3591 dca_remove_requester(&pdev->dev); 3592 adapter->flags &= ~IGB_FLAG_DCA_ENABLED; 3593 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); 3594 } 3595 #endif 3596 3597 /* Release control of h/w to f/w. If f/w is AMT enabled, this 3598 * would have already happened in close and is redundant. 3599 */ 3600 igb_release_hw_control(adapter); 3601 3602 #ifdef CONFIG_PCI_IOV 3603 igb_disable_sriov(pdev); 3604 #endif 3605 3606 unregister_netdev(netdev); 3607 3608 igb_clear_interrupt_scheme(adapter); 3609 3610 pci_iounmap(pdev, adapter->io_addr); 3611 if (hw->flash_address) 3612 iounmap(hw->flash_address); 3613 pci_release_mem_regions(pdev); 3614 3615 kfree(adapter->mac_table); 3616 kfree(adapter->shadow_vfta); 3617 free_netdev(netdev); 3618 3619 pci_disable_pcie_error_reporting(pdev); 3620 3621 pci_disable_device(pdev); 3622 } 3623 3624 /** 3625 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space 3626 * @adapter: board private structure to initialize 3627 * 3628 * This function initializes the vf specific data storage and then attempts to 3629 * allocate the VFs. The reason for ordering it this way is because it is much 3630 * mor expensive time wise to disable SR-IOV than it is to allocate and free 3631 * the memory for the VFs. 3632 **/ 3633 static void igb_probe_vfs(struct igb_adapter *adapter) 3634 { 3635 #ifdef CONFIG_PCI_IOV 3636 struct pci_dev *pdev = adapter->pdev; 3637 struct e1000_hw *hw = &adapter->hw; 3638 3639 /* Virtualization features not supported on i210 family. */ 3640 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211)) 3641 return; 3642 3643 /* Of the below we really only want the effect of getting 3644 * IGB_FLAG_HAS_MSIX set (if available), without which 3645 * igb_enable_sriov() has no effect. 3646 */ 3647 igb_set_interrupt_capability(adapter, true); 3648 igb_reset_interrupt_capability(adapter); 3649 3650 pci_sriov_set_totalvfs(pdev, 7); 3651 igb_enable_sriov(pdev, max_vfs); 3652 3653 #endif /* CONFIG_PCI_IOV */ 3654 } 3655 3656 unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter) 3657 { 3658 struct e1000_hw *hw = &adapter->hw; 3659 unsigned int max_rss_queues; 3660 3661 /* Determine the maximum number of RSS queues supported. */ 3662 switch (hw->mac.type) { 3663 case e1000_i211: 3664 max_rss_queues = IGB_MAX_RX_QUEUES_I211; 3665 break; 3666 case e1000_82575: 3667 case e1000_i210: 3668 max_rss_queues = IGB_MAX_RX_QUEUES_82575; 3669 break; 3670 case e1000_i350: 3671 /* I350 cannot do RSS and SR-IOV at the same time */ 3672 if (!!adapter->vfs_allocated_count) { 3673 max_rss_queues = 1; 3674 break; 3675 } 3676 /* fall through */ 3677 case e1000_82576: 3678 if (!!adapter->vfs_allocated_count) { 3679 max_rss_queues = 2; 3680 break; 3681 } 3682 /* fall through */ 3683 case e1000_82580: 3684 case e1000_i354: 3685 default: 3686 max_rss_queues = IGB_MAX_RX_QUEUES; 3687 break; 3688 } 3689 3690 return max_rss_queues; 3691 } 3692 3693 static void igb_init_queue_configuration(struct igb_adapter *adapter) 3694 { 3695 u32 max_rss_queues; 3696 3697 max_rss_queues = igb_get_max_rss_queues(adapter); 3698 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 3699 3700 igb_set_flag_queue_pairs(adapter, max_rss_queues); 3701 } 3702 3703 void igb_set_flag_queue_pairs(struct igb_adapter *adapter, 3704 const u32 max_rss_queues) 3705 { 3706 struct e1000_hw *hw = &adapter->hw; 3707 3708 /* Determine if we need to pair queues. */ 3709 switch (hw->mac.type) { 3710 case e1000_82575: 3711 case e1000_i211: 3712 /* Device supports enough interrupts without queue pairing. */ 3713 break; 3714 case e1000_82576: 3715 case e1000_82580: 3716 case e1000_i350: 3717 case e1000_i354: 3718 case e1000_i210: 3719 default: 3720 /* If rss_queues > half of max_rss_queues, pair the queues in 3721 * order to conserve interrupts due to limited supply. 3722 */ 3723 if (adapter->rss_queues > (max_rss_queues / 2)) 3724 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 3725 else 3726 adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS; 3727 break; 3728 } 3729 } 3730 3731 /** 3732 * igb_sw_init - Initialize general software structures (struct igb_adapter) 3733 * @adapter: board private structure to initialize 3734 * 3735 * igb_sw_init initializes the Adapter private data structure. 3736 * Fields are initialized based on PCI device information and 3737 * OS network device settings (MTU size). 3738 **/ 3739 static int igb_sw_init(struct igb_adapter *adapter) 3740 { 3741 struct e1000_hw *hw = &adapter->hw; 3742 struct net_device *netdev = adapter->netdev; 3743 struct pci_dev *pdev = adapter->pdev; 3744 3745 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3746 3747 /* set default ring sizes */ 3748 adapter->tx_ring_count = IGB_DEFAULT_TXD; 3749 adapter->rx_ring_count = IGB_DEFAULT_RXD; 3750 3751 /* set default ITR values */ 3752 adapter->rx_itr_setting = IGB_DEFAULT_ITR; 3753 adapter->tx_itr_setting = IGB_DEFAULT_ITR; 3754 3755 /* set default work limits */ 3756 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK; 3757 3758 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3759 VLAN_HLEN; 3760 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3761 3762 spin_lock_init(&adapter->nfc_lock); 3763 spin_lock_init(&adapter->stats64_lock); 3764 #ifdef CONFIG_PCI_IOV 3765 switch (hw->mac.type) { 3766 case e1000_82576: 3767 case e1000_i350: 3768 if (max_vfs > 7) { 3769 dev_warn(&pdev->dev, 3770 "Maximum of 7 VFs per PF, using max\n"); 3771 max_vfs = adapter->vfs_allocated_count = 7; 3772 } else 3773 adapter->vfs_allocated_count = max_vfs; 3774 if (adapter->vfs_allocated_count) 3775 dev_warn(&pdev->dev, 3776 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n"); 3777 break; 3778 default: 3779 break; 3780 } 3781 #endif /* CONFIG_PCI_IOV */ 3782 3783 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3784 adapter->flags |= IGB_FLAG_HAS_MSIX; 3785 3786 adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) * 3787 hw->mac.rar_entry_count, GFP_ATOMIC); 3788 if (!adapter->mac_table) 3789 return -ENOMEM; 3790 3791 igb_probe_vfs(adapter); 3792 3793 igb_init_queue_configuration(adapter); 3794 3795 /* Setup and initialize a copy of the hw vlan table array */ 3796 adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32), 3797 GFP_ATOMIC); 3798 if (!adapter->shadow_vfta) 3799 return -ENOMEM; 3800 3801 /* This call may decrease the number of queues */ 3802 if (igb_init_interrupt_scheme(adapter, true)) { 3803 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 3804 return -ENOMEM; 3805 } 3806 3807 /* Explicitly disable IRQ since the NIC can be in any state. */ 3808 igb_irq_disable(adapter); 3809 3810 if (hw->mac.type >= e1000_i350) 3811 adapter->flags &= ~IGB_FLAG_DMAC; 3812 3813 set_bit(__IGB_DOWN, &adapter->state); 3814 return 0; 3815 } 3816 3817 /** 3818 * igb_open - Called when a network interface is made active 3819 * @netdev: network interface device structure 3820 * 3821 * Returns 0 on success, negative value on failure 3822 * 3823 * The open entry point is called when a network interface is made 3824 * active by the system (IFF_UP). At this point all resources needed 3825 * for transmit and receive operations are allocated, the interrupt 3826 * handler is registered with the OS, the watchdog timer is started, 3827 * and the stack is notified that the interface is ready. 3828 **/ 3829 static int __igb_open(struct net_device *netdev, bool resuming) 3830 { 3831 struct igb_adapter *adapter = netdev_priv(netdev); 3832 struct e1000_hw *hw = &adapter->hw; 3833 struct pci_dev *pdev = adapter->pdev; 3834 int err; 3835 int i; 3836 3837 /* disallow open during test */ 3838 if (test_bit(__IGB_TESTING, &adapter->state)) { 3839 WARN_ON(resuming); 3840 return -EBUSY; 3841 } 3842 3843 if (!resuming) 3844 pm_runtime_get_sync(&pdev->dev); 3845 3846 netif_carrier_off(netdev); 3847 3848 /* allocate transmit descriptors */ 3849 err = igb_setup_all_tx_resources(adapter); 3850 if (err) 3851 goto err_setup_tx; 3852 3853 /* allocate receive descriptors */ 3854 err = igb_setup_all_rx_resources(adapter); 3855 if (err) 3856 goto err_setup_rx; 3857 3858 igb_power_up_link(adapter); 3859 3860 /* before we allocate an interrupt, we must be ready to handle it. 3861 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 3862 * as soon as we call pci_request_irq, so we have to setup our 3863 * clean_rx handler before we do so. 3864 */ 3865 igb_configure(adapter); 3866 3867 err = igb_request_irq(adapter); 3868 if (err) 3869 goto err_req_irq; 3870 3871 /* Notify the stack of the actual queue counts. */ 3872 err = netif_set_real_num_tx_queues(adapter->netdev, 3873 adapter->num_tx_queues); 3874 if (err) 3875 goto err_set_queues; 3876 3877 err = netif_set_real_num_rx_queues(adapter->netdev, 3878 adapter->num_rx_queues); 3879 if (err) 3880 goto err_set_queues; 3881 3882 /* From here on the code is the same as igb_up() */ 3883 clear_bit(__IGB_DOWN, &adapter->state); 3884 3885 for (i = 0; i < adapter->num_q_vectors; i++) 3886 napi_enable(&(adapter->q_vector[i]->napi)); 3887 3888 /* Clear any pending interrupts. */ 3889 rd32(E1000_ICR); 3890 3891 igb_irq_enable(adapter); 3892 3893 /* notify VFs that reset has been completed */ 3894 if (adapter->vfs_allocated_count) { 3895 u32 reg_data = rd32(E1000_CTRL_EXT); 3896 3897 reg_data |= E1000_CTRL_EXT_PFRSTD; 3898 wr32(E1000_CTRL_EXT, reg_data); 3899 } 3900 3901 netif_tx_start_all_queues(netdev); 3902 3903 if (!resuming) 3904 pm_runtime_put(&pdev->dev); 3905 3906 /* start the watchdog. */ 3907 hw->mac.get_link_status = 1; 3908 schedule_work(&adapter->watchdog_task); 3909 3910 return 0; 3911 3912 err_set_queues: 3913 igb_free_irq(adapter); 3914 err_req_irq: 3915 igb_release_hw_control(adapter); 3916 igb_power_down_link(adapter); 3917 igb_free_all_rx_resources(adapter); 3918 err_setup_rx: 3919 igb_free_all_tx_resources(adapter); 3920 err_setup_tx: 3921 igb_reset(adapter); 3922 if (!resuming) 3923 pm_runtime_put(&pdev->dev); 3924 3925 return err; 3926 } 3927 3928 int igb_open(struct net_device *netdev) 3929 { 3930 return __igb_open(netdev, false); 3931 } 3932 3933 /** 3934 * igb_close - Disables a network interface 3935 * @netdev: network interface device structure 3936 * 3937 * Returns 0, this is not allowed to fail 3938 * 3939 * The close entry point is called when an interface is de-activated 3940 * by the OS. The hardware is still under the driver's control, but 3941 * needs to be disabled. A global MAC reset is issued to stop the 3942 * hardware, and all transmit and receive resources are freed. 3943 **/ 3944 static int __igb_close(struct net_device *netdev, bool suspending) 3945 { 3946 struct igb_adapter *adapter = netdev_priv(netdev); 3947 struct pci_dev *pdev = adapter->pdev; 3948 3949 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state)); 3950 3951 if (!suspending) 3952 pm_runtime_get_sync(&pdev->dev); 3953 3954 igb_down(adapter); 3955 igb_free_irq(adapter); 3956 3957 igb_free_all_tx_resources(adapter); 3958 igb_free_all_rx_resources(adapter); 3959 3960 if (!suspending) 3961 pm_runtime_put_sync(&pdev->dev); 3962 return 0; 3963 } 3964 3965 int igb_close(struct net_device *netdev) 3966 { 3967 if (netif_device_present(netdev) || netdev->dismantle) 3968 return __igb_close(netdev, false); 3969 return 0; 3970 } 3971 3972 /** 3973 * igb_setup_tx_resources - allocate Tx resources (Descriptors) 3974 * @tx_ring: tx descriptor ring (for a specific queue) to setup 3975 * 3976 * Return 0 on success, negative on failure 3977 **/ 3978 int igb_setup_tx_resources(struct igb_ring *tx_ring) 3979 { 3980 struct device *dev = tx_ring->dev; 3981 int size; 3982 3983 size = sizeof(struct igb_tx_buffer) * tx_ring->count; 3984 3985 tx_ring->tx_buffer_info = vmalloc(size); 3986 if (!tx_ring->tx_buffer_info) 3987 goto err; 3988 3989 /* round up to nearest 4K */ 3990 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc); 3991 tx_ring->size = ALIGN(tx_ring->size, 4096); 3992 3993 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 3994 &tx_ring->dma, GFP_KERNEL); 3995 if (!tx_ring->desc) 3996 goto err; 3997 3998 tx_ring->next_to_use = 0; 3999 tx_ring->next_to_clean = 0; 4000 4001 return 0; 4002 4003 err: 4004 vfree(tx_ring->tx_buffer_info); 4005 tx_ring->tx_buffer_info = NULL; 4006 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); 4007 return -ENOMEM; 4008 } 4009 4010 /** 4011 * igb_setup_all_tx_resources - wrapper to allocate Tx resources 4012 * (Descriptors) for all queues 4013 * @adapter: board private structure 4014 * 4015 * Return 0 on success, negative on failure 4016 **/ 4017 static int igb_setup_all_tx_resources(struct igb_adapter *adapter) 4018 { 4019 struct pci_dev *pdev = adapter->pdev; 4020 int i, err = 0; 4021 4022 for (i = 0; i < adapter->num_tx_queues; i++) { 4023 err = igb_setup_tx_resources(adapter->tx_ring[i]); 4024 if (err) { 4025 dev_err(&pdev->dev, 4026 "Allocation for Tx Queue %u failed\n", i); 4027 for (i--; i >= 0; i--) 4028 igb_free_tx_resources(adapter->tx_ring[i]); 4029 break; 4030 } 4031 } 4032 4033 return err; 4034 } 4035 4036 /** 4037 * igb_setup_tctl - configure the transmit control registers 4038 * @adapter: Board private structure 4039 **/ 4040 void igb_setup_tctl(struct igb_adapter *adapter) 4041 { 4042 struct e1000_hw *hw = &adapter->hw; 4043 u32 tctl; 4044 4045 /* disable queue 0 which is enabled by default on 82575 and 82576 */ 4046 wr32(E1000_TXDCTL(0), 0); 4047 4048 /* Program the Transmit Control Register */ 4049 tctl = rd32(E1000_TCTL); 4050 tctl &= ~E1000_TCTL_CT; 4051 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 4052 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 4053 4054 igb_config_collision_dist(hw); 4055 4056 /* Enable transmits */ 4057 tctl |= E1000_TCTL_EN; 4058 4059 wr32(E1000_TCTL, tctl); 4060 } 4061 4062 /** 4063 * igb_configure_tx_ring - Configure transmit ring after Reset 4064 * @adapter: board private structure 4065 * @ring: tx ring to configure 4066 * 4067 * Configure a transmit ring after a reset. 4068 **/ 4069 void igb_configure_tx_ring(struct igb_adapter *adapter, 4070 struct igb_ring *ring) 4071 { 4072 struct e1000_hw *hw = &adapter->hw; 4073 u32 txdctl = 0; 4074 u64 tdba = ring->dma; 4075 int reg_idx = ring->reg_idx; 4076 4077 /* disable the queue */ 4078 wr32(E1000_TXDCTL(reg_idx), 0); 4079 wrfl(); 4080 mdelay(10); 4081 4082 wr32(E1000_TDLEN(reg_idx), 4083 ring->count * sizeof(union e1000_adv_tx_desc)); 4084 wr32(E1000_TDBAL(reg_idx), 4085 tdba & 0x00000000ffffffffULL); 4086 wr32(E1000_TDBAH(reg_idx), tdba >> 32); 4087 4088 ring->tail = adapter->io_addr + E1000_TDT(reg_idx); 4089 wr32(E1000_TDH(reg_idx), 0); 4090 writel(0, ring->tail); 4091 4092 txdctl |= IGB_TX_PTHRESH; 4093 txdctl |= IGB_TX_HTHRESH << 8; 4094 txdctl |= IGB_TX_WTHRESH << 16; 4095 4096 /* reinitialize tx_buffer_info */ 4097 memset(ring->tx_buffer_info, 0, 4098 sizeof(struct igb_tx_buffer) * ring->count); 4099 4100 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 4101 wr32(E1000_TXDCTL(reg_idx), txdctl); 4102 } 4103 4104 /** 4105 * igb_configure_tx - Configure transmit Unit after Reset 4106 * @adapter: board private structure 4107 * 4108 * Configure the Tx unit of the MAC after a reset. 4109 **/ 4110 static void igb_configure_tx(struct igb_adapter *adapter) 4111 { 4112 int i; 4113 4114 for (i = 0; i < adapter->num_tx_queues; i++) 4115 igb_configure_tx_ring(adapter, adapter->tx_ring[i]); 4116 } 4117 4118 /** 4119 * igb_setup_rx_resources - allocate Rx resources (Descriptors) 4120 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 4121 * 4122 * Returns 0 on success, negative on failure 4123 **/ 4124 int igb_setup_rx_resources(struct igb_ring *rx_ring) 4125 { 4126 struct device *dev = rx_ring->dev; 4127 int size; 4128 4129 size = sizeof(struct igb_rx_buffer) * rx_ring->count; 4130 4131 rx_ring->rx_buffer_info = vmalloc(size); 4132 if (!rx_ring->rx_buffer_info) 4133 goto err; 4134 4135 /* Round up to nearest 4K */ 4136 rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc); 4137 rx_ring->size = ALIGN(rx_ring->size, 4096); 4138 4139 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 4140 &rx_ring->dma, GFP_KERNEL); 4141 if (!rx_ring->desc) 4142 goto err; 4143 4144 rx_ring->next_to_alloc = 0; 4145 rx_ring->next_to_clean = 0; 4146 rx_ring->next_to_use = 0; 4147 4148 return 0; 4149 4150 err: 4151 vfree(rx_ring->rx_buffer_info); 4152 rx_ring->rx_buffer_info = NULL; 4153 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); 4154 return -ENOMEM; 4155 } 4156 4157 /** 4158 * igb_setup_all_rx_resources - wrapper to allocate Rx resources 4159 * (Descriptors) for all queues 4160 * @adapter: board private structure 4161 * 4162 * Return 0 on success, negative on failure 4163 **/ 4164 static int igb_setup_all_rx_resources(struct igb_adapter *adapter) 4165 { 4166 struct pci_dev *pdev = adapter->pdev; 4167 int i, err = 0; 4168 4169 for (i = 0; i < adapter->num_rx_queues; i++) { 4170 err = igb_setup_rx_resources(adapter->rx_ring[i]); 4171 if (err) { 4172 dev_err(&pdev->dev, 4173 "Allocation for Rx Queue %u failed\n", i); 4174 for (i--; i >= 0; i--) 4175 igb_free_rx_resources(adapter->rx_ring[i]); 4176 break; 4177 } 4178 } 4179 4180 return err; 4181 } 4182 4183 /** 4184 * igb_setup_mrqc - configure the multiple receive queue control registers 4185 * @adapter: Board private structure 4186 **/ 4187 static void igb_setup_mrqc(struct igb_adapter *adapter) 4188 { 4189 struct e1000_hw *hw = &adapter->hw; 4190 u32 mrqc, rxcsum; 4191 u32 j, num_rx_queues; 4192 u32 rss_key[10]; 4193 4194 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 4195 for (j = 0; j < 10; j++) 4196 wr32(E1000_RSSRK(j), rss_key[j]); 4197 4198 num_rx_queues = adapter->rss_queues; 4199 4200 switch (hw->mac.type) { 4201 case e1000_82576: 4202 /* 82576 supports 2 RSS queues for SR-IOV */ 4203 if (adapter->vfs_allocated_count) 4204 num_rx_queues = 2; 4205 break; 4206 default: 4207 break; 4208 } 4209 4210 if (adapter->rss_indir_tbl_init != num_rx_queues) { 4211 for (j = 0; j < IGB_RETA_SIZE; j++) 4212 adapter->rss_indir_tbl[j] = 4213 (j * num_rx_queues) / IGB_RETA_SIZE; 4214 adapter->rss_indir_tbl_init = num_rx_queues; 4215 } 4216 igb_write_rss_indir_tbl(adapter); 4217 4218 /* Disable raw packet checksumming so that RSS hash is placed in 4219 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 4220 * offloads as they are enabled by default 4221 */ 4222 rxcsum = rd32(E1000_RXCSUM); 4223 rxcsum |= E1000_RXCSUM_PCSD; 4224 4225 if (adapter->hw.mac.type >= e1000_82576) 4226 /* Enable Receive Checksum Offload for SCTP */ 4227 rxcsum |= E1000_RXCSUM_CRCOFL; 4228 4229 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 4230 wr32(E1000_RXCSUM, rxcsum); 4231 4232 /* Generate RSS hash based on packet types, TCP/UDP 4233 * port numbers and/or IPv4/v6 src and dst addresses 4234 */ 4235 mrqc = E1000_MRQC_RSS_FIELD_IPV4 | 4236 E1000_MRQC_RSS_FIELD_IPV4_TCP | 4237 E1000_MRQC_RSS_FIELD_IPV6 | 4238 E1000_MRQC_RSS_FIELD_IPV6_TCP | 4239 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX; 4240 4241 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP) 4242 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP; 4243 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP) 4244 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP; 4245 4246 /* If VMDq is enabled then we set the appropriate mode for that, else 4247 * we default to RSS so that an RSS hash is calculated per packet even 4248 * if we are only using one queue 4249 */ 4250 if (adapter->vfs_allocated_count) { 4251 if (hw->mac.type > e1000_82575) { 4252 /* Set the default pool for the PF's first queue */ 4253 u32 vtctl = rd32(E1000_VT_CTL); 4254 4255 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK | 4256 E1000_VT_CTL_DISABLE_DEF_POOL); 4257 vtctl |= adapter->vfs_allocated_count << 4258 E1000_VT_CTL_DEFAULT_POOL_SHIFT; 4259 wr32(E1000_VT_CTL, vtctl); 4260 } 4261 if (adapter->rss_queues > 1) 4262 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ; 4263 else 4264 mrqc |= E1000_MRQC_ENABLE_VMDQ; 4265 } else { 4266 if (hw->mac.type != e1000_i211) 4267 mrqc |= E1000_MRQC_ENABLE_RSS_MQ; 4268 } 4269 igb_vmm_control(adapter); 4270 4271 wr32(E1000_MRQC, mrqc); 4272 } 4273 4274 /** 4275 * igb_setup_rctl - configure the receive control registers 4276 * @adapter: Board private structure 4277 **/ 4278 void igb_setup_rctl(struct igb_adapter *adapter) 4279 { 4280 struct e1000_hw *hw = &adapter->hw; 4281 u32 rctl; 4282 4283 rctl = rd32(E1000_RCTL); 4284 4285 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4286 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 4287 4288 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF | 4289 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4290 4291 /* enable stripping of CRC. It's unlikely this will break BMC 4292 * redirection as it did with e1000. Newer features require 4293 * that the HW strips the CRC. 4294 */ 4295 rctl |= E1000_RCTL_SECRC; 4296 4297 /* disable store bad packets and clear size bits. */ 4298 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256); 4299 4300 /* enable LPE to allow for reception of jumbo frames */ 4301 rctl |= E1000_RCTL_LPE; 4302 4303 /* disable queue 0 to prevent tail write w/o re-config */ 4304 wr32(E1000_RXDCTL(0), 0); 4305 4306 /* Attention!!! For SR-IOV PF driver operations you must enable 4307 * queue drop for all VF and PF queues to prevent head of line blocking 4308 * if an un-trusted VF does not provide descriptors to hardware. 4309 */ 4310 if (adapter->vfs_allocated_count) { 4311 /* set all queue drop enable bits */ 4312 wr32(E1000_QDE, ALL_QUEUES); 4313 } 4314 4315 /* This is useful for sniffing bad packets. */ 4316 if (adapter->netdev->features & NETIF_F_RXALL) { 4317 /* UPE and MPE will be handled by normal PROMISC logic 4318 * in e1000e_set_rx_mode 4319 */ 4320 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ 4321 E1000_RCTL_BAM | /* RX All Bcast Pkts */ 4322 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 4323 4324 rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */ 4325 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ 4326 /* Do not mess with E1000_CTRL_VME, it affects transmit as well, 4327 * and that breaks VLANs. 4328 */ 4329 } 4330 4331 wr32(E1000_RCTL, rctl); 4332 } 4333 4334 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size, 4335 int vfn) 4336 { 4337 struct e1000_hw *hw = &adapter->hw; 4338 u32 vmolr; 4339 4340 if (size > MAX_JUMBO_FRAME_SIZE) 4341 size = MAX_JUMBO_FRAME_SIZE; 4342 4343 vmolr = rd32(E1000_VMOLR(vfn)); 4344 vmolr &= ~E1000_VMOLR_RLPML_MASK; 4345 vmolr |= size | E1000_VMOLR_LPE; 4346 wr32(E1000_VMOLR(vfn), vmolr); 4347 4348 return 0; 4349 } 4350 4351 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter, 4352 int vfn, bool enable) 4353 { 4354 struct e1000_hw *hw = &adapter->hw; 4355 u32 val, reg; 4356 4357 if (hw->mac.type < e1000_82576) 4358 return; 4359 4360 if (hw->mac.type == e1000_i350) 4361 reg = E1000_DVMOLR(vfn); 4362 else 4363 reg = E1000_VMOLR(vfn); 4364 4365 val = rd32(reg); 4366 if (enable) 4367 val |= E1000_VMOLR_STRVLAN; 4368 else 4369 val &= ~(E1000_VMOLR_STRVLAN); 4370 wr32(reg, val); 4371 } 4372 4373 static inline void igb_set_vmolr(struct igb_adapter *adapter, 4374 int vfn, bool aupe) 4375 { 4376 struct e1000_hw *hw = &adapter->hw; 4377 u32 vmolr; 4378 4379 /* This register exists only on 82576 and newer so if we are older then 4380 * we should exit and do nothing 4381 */ 4382 if (hw->mac.type < e1000_82576) 4383 return; 4384 4385 vmolr = rd32(E1000_VMOLR(vfn)); 4386 if (aupe) 4387 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */ 4388 else 4389 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */ 4390 4391 /* clear all bits that might not be set */ 4392 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE); 4393 4394 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count) 4395 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */ 4396 /* for VMDq only allow the VFs and pool 0 to accept broadcast and 4397 * multicast packets 4398 */ 4399 if (vfn <= adapter->vfs_allocated_count) 4400 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */ 4401 4402 wr32(E1000_VMOLR(vfn), vmolr); 4403 } 4404 4405 /** 4406 * igb_configure_rx_ring - Configure a receive ring after Reset 4407 * @adapter: board private structure 4408 * @ring: receive ring to be configured 4409 * 4410 * Configure the Rx unit of the MAC after a reset. 4411 **/ 4412 void igb_configure_rx_ring(struct igb_adapter *adapter, 4413 struct igb_ring *ring) 4414 { 4415 struct e1000_hw *hw = &adapter->hw; 4416 union e1000_adv_rx_desc *rx_desc; 4417 u64 rdba = ring->dma; 4418 int reg_idx = ring->reg_idx; 4419 u32 srrctl = 0, rxdctl = 0; 4420 4421 /* disable the queue */ 4422 wr32(E1000_RXDCTL(reg_idx), 0); 4423 4424 /* Set DMA base address registers */ 4425 wr32(E1000_RDBAL(reg_idx), 4426 rdba & 0x00000000ffffffffULL); 4427 wr32(E1000_RDBAH(reg_idx), rdba >> 32); 4428 wr32(E1000_RDLEN(reg_idx), 4429 ring->count * sizeof(union e1000_adv_rx_desc)); 4430 4431 /* initialize head and tail */ 4432 ring->tail = adapter->io_addr + E1000_RDT(reg_idx); 4433 wr32(E1000_RDH(reg_idx), 0); 4434 writel(0, ring->tail); 4435 4436 /* set descriptor configuration */ 4437 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 4438 if (ring_uses_large_buffer(ring)) 4439 srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4440 else 4441 srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 4442 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 4443 if (hw->mac.type >= e1000_82580) 4444 srrctl |= E1000_SRRCTL_TIMESTAMP; 4445 /* Only set Drop Enable if we are supporting multiple queues */ 4446 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1) 4447 srrctl |= E1000_SRRCTL_DROP_EN; 4448 4449 wr32(E1000_SRRCTL(reg_idx), srrctl); 4450 4451 /* set filtering for VMDQ pools */ 4452 igb_set_vmolr(adapter, reg_idx & 0x7, true); 4453 4454 rxdctl |= IGB_RX_PTHRESH; 4455 rxdctl |= IGB_RX_HTHRESH << 8; 4456 rxdctl |= IGB_RX_WTHRESH << 16; 4457 4458 /* initialize rx_buffer_info */ 4459 memset(ring->rx_buffer_info, 0, 4460 sizeof(struct igb_rx_buffer) * ring->count); 4461 4462 /* initialize Rx descriptor 0 */ 4463 rx_desc = IGB_RX_DESC(ring, 0); 4464 rx_desc->wb.upper.length = 0; 4465 4466 /* enable receive descriptor fetching */ 4467 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 4468 wr32(E1000_RXDCTL(reg_idx), rxdctl); 4469 } 4470 4471 static void igb_set_rx_buffer_len(struct igb_adapter *adapter, 4472 struct igb_ring *rx_ring) 4473 { 4474 /* set build_skb and buffer size flags */ 4475 clear_ring_build_skb_enabled(rx_ring); 4476 clear_ring_uses_large_buffer(rx_ring); 4477 4478 if (adapter->flags & IGB_FLAG_RX_LEGACY) 4479 return; 4480 4481 set_ring_build_skb_enabled(rx_ring); 4482 4483 #if (PAGE_SIZE < 8192) 4484 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) 4485 return; 4486 4487 set_ring_uses_large_buffer(rx_ring); 4488 #endif 4489 } 4490 4491 /** 4492 * igb_configure_rx - Configure receive Unit after Reset 4493 * @adapter: board private structure 4494 * 4495 * Configure the Rx unit of the MAC after a reset. 4496 **/ 4497 static void igb_configure_rx(struct igb_adapter *adapter) 4498 { 4499 int i; 4500 4501 /* set the correct pool for the PF default MAC address in entry 0 */ 4502 igb_set_default_mac_filter(adapter); 4503 4504 /* Setup the HW Rx Head and Tail Descriptor Pointers and 4505 * the Base and Length of the Rx Descriptor Ring 4506 */ 4507 for (i = 0; i < adapter->num_rx_queues; i++) { 4508 struct igb_ring *rx_ring = adapter->rx_ring[i]; 4509 4510 igb_set_rx_buffer_len(adapter, rx_ring); 4511 igb_configure_rx_ring(adapter, rx_ring); 4512 } 4513 } 4514 4515 /** 4516 * igb_free_tx_resources - Free Tx Resources per Queue 4517 * @tx_ring: Tx descriptor ring for a specific queue 4518 * 4519 * Free all transmit software resources 4520 **/ 4521 void igb_free_tx_resources(struct igb_ring *tx_ring) 4522 { 4523 igb_clean_tx_ring(tx_ring); 4524 4525 vfree(tx_ring->tx_buffer_info); 4526 tx_ring->tx_buffer_info = NULL; 4527 4528 /* if not set, then don't free */ 4529 if (!tx_ring->desc) 4530 return; 4531 4532 dma_free_coherent(tx_ring->dev, tx_ring->size, 4533 tx_ring->desc, tx_ring->dma); 4534 4535 tx_ring->desc = NULL; 4536 } 4537 4538 /** 4539 * igb_free_all_tx_resources - Free Tx Resources for All Queues 4540 * @adapter: board private structure 4541 * 4542 * Free all transmit software resources 4543 **/ 4544 static void igb_free_all_tx_resources(struct igb_adapter *adapter) 4545 { 4546 int i; 4547 4548 for (i = 0; i < adapter->num_tx_queues; i++) 4549 if (adapter->tx_ring[i]) 4550 igb_free_tx_resources(adapter->tx_ring[i]); 4551 } 4552 4553 /** 4554 * igb_clean_tx_ring - Free Tx Buffers 4555 * @tx_ring: ring to be cleaned 4556 **/ 4557 static void igb_clean_tx_ring(struct igb_ring *tx_ring) 4558 { 4559 u16 i = tx_ring->next_to_clean; 4560 struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 4561 4562 while (i != tx_ring->next_to_use) { 4563 union e1000_adv_tx_desc *eop_desc, *tx_desc; 4564 4565 /* Free all the Tx ring sk_buffs */ 4566 dev_kfree_skb_any(tx_buffer->skb); 4567 4568 /* unmap skb header data */ 4569 dma_unmap_single(tx_ring->dev, 4570 dma_unmap_addr(tx_buffer, dma), 4571 dma_unmap_len(tx_buffer, len), 4572 DMA_TO_DEVICE); 4573 4574 /* check for eop_desc to determine the end of the packet */ 4575 eop_desc = tx_buffer->next_to_watch; 4576 tx_desc = IGB_TX_DESC(tx_ring, i); 4577 4578 /* unmap remaining buffers */ 4579 while (tx_desc != eop_desc) { 4580 tx_buffer++; 4581 tx_desc++; 4582 i++; 4583 if (unlikely(i == tx_ring->count)) { 4584 i = 0; 4585 tx_buffer = tx_ring->tx_buffer_info; 4586 tx_desc = IGB_TX_DESC(tx_ring, 0); 4587 } 4588 4589 /* unmap any remaining paged data */ 4590 if (dma_unmap_len(tx_buffer, len)) 4591 dma_unmap_page(tx_ring->dev, 4592 dma_unmap_addr(tx_buffer, dma), 4593 dma_unmap_len(tx_buffer, len), 4594 DMA_TO_DEVICE); 4595 } 4596 4597 /* move us one more past the eop_desc for start of next pkt */ 4598 tx_buffer++; 4599 i++; 4600 if (unlikely(i == tx_ring->count)) { 4601 i = 0; 4602 tx_buffer = tx_ring->tx_buffer_info; 4603 } 4604 } 4605 4606 /* reset BQL for queue */ 4607 netdev_tx_reset_queue(txring_txq(tx_ring)); 4608 4609 /* reset next_to_use and next_to_clean */ 4610 tx_ring->next_to_use = 0; 4611 tx_ring->next_to_clean = 0; 4612 } 4613 4614 /** 4615 * igb_clean_all_tx_rings - Free Tx Buffers for all queues 4616 * @adapter: board private structure 4617 **/ 4618 static void igb_clean_all_tx_rings(struct igb_adapter *adapter) 4619 { 4620 int i; 4621 4622 for (i = 0; i < adapter->num_tx_queues; i++) 4623 if (adapter->tx_ring[i]) 4624 igb_clean_tx_ring(adapter->tx_ring[i]); 4625 } 4626 4627 /** 4628 * igb_free_rx_resources - Free Rx Resources 4629 * @rx_ring: ring to clean the resources from 4630 * 4631 * Free all receive software resources 4632 **/ 4633 void igb_free_rx_resources(struct igb_ring *rx_ring) 4634 { 4635 igb_clean_rx_ring(rx_ring); 4636 4637 vfree(rx_ring->rx_buffer_info); 4638 rx_ring->rx_buffer_info = NULL; 4639 4640 /* if not set, then don't free */ 4641 if (!rx_ring->desc) 4642 return; 4643 4644 dma_free_coherent(rx_ring->dev, rx_ring->size, 4645 rx_ring->desc, rx_ring->dma); 4646 4647 rx_ring->desc = NULL; 4648 } 4649 4650 /** 4651 * igb_free_all_rx_resources - Free Rx Resources for All Queues 4652 * @adapter: board private structure 4653 * 4654 * Free all receive software resources 4655 **/ 4656 static void igb_free_all_rx_resources(struct igb_adapter *adapter) 4657 { 4658 int i; 4659 4660 for (i = 0; i < adapter->num_rx_queues; i++) 4661 if (adapter->rx_ring[i]) 4662 igb_free_rx_resources(adapter->rx_ring[i]); 4663 } 4664 4665 /** 4666 * igb_clean_rx_ring - Free Rx Buffers per Queue 4667 * @rx_ring: ring to free buffers from 4668 **/ 4669 static void igb_clean_rx_ring(struct igb_ring *rx_ring) 4670 { 4671 u16 i = rx_ring->next_to_clean; 4672 4673 if (rx_ring->skb) 4674 dev_kfree_skb(rx_ring->skb); 4675 rx_ring->skb = NULL; 4676 4677 /* Free all the Rx ring sk_buffs */ 4678 while (i != rx_ring->next_to_alloc) { 4679 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 4680 4681 /* Invalidate cache lines that may have been written to by 4682 * device so that we avoid corrupting memory. 4683 */ 4684 dma_sync_single_range_for_cpu(rx_ring->dev, 4685 buffer_info->dma, 4686 buffer_info->page_offset, 4687 igb_rx_bufsz(rx_ring), 4688 DMA_FROM_DEVICE); 4689 4690 /* free resources associated with mapping */ 4691 dma_unmap_page_attrs(rx_ring->dev, 4692 buffer_info->dma, 4693 igb_rx_pg_size(rx_ring), 4694 DMA_FROM_DEVICE, 4695 IGB_RX_DMA_ATTR); 4696 __page_frag_cache_drain(buffer_info->page, 4697 buffer_info->pagecnt_bias); 4698 4699 i++; 4700 if (i == rx_ring->count) 4701 i = 0; 4702 } 4703 4704 rx_ring->next_to_alloc = 0; 4705 rx_ring->next_to_clean = 0; 4706 rx_ring->next_to_use = 0; 4707 } 4708 4709 /** 4710 * igb_clean_all_rx_rings - Free Rx Buffers for all queues 4711 * @adapter: board private structure 4712 **/ 4713 static void igb_clean_all_rx_rings(struct igb_adapter *adapter) 4714 { 4715 int i; 4716 4717 for (i = 0; i < adapter->num_rx_queues; i++) 4718 if (adapter->rx_ring[i]) 4719 igb_clean_rx_ring(adapter->rx_ring[i]); 4720 } 4721 4722 /** 4723 * igb_set_mac - Change the Ethernet Address of the NIC 4724 * @netdev: network interface device structure 4725 * @p: pointer to an address structure 4726 * 4727 * Returns 0 on success, negative on failure 4728 **/ 4729 static int igb_set_mac(struct net_device *netdev, void *p) 4730 { 4731 struct igb_adapter *adapter = netdev_priv(netdev); 4732 struct e1000_hw *hw = &adapter->hw; 4733 struct sockaddr *addr = p; 4734 4735 if (!is_valid_ether_addr(addr->sa_data)) 4736 return -EADDRNOTAVAIL; 4737 4738 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 4739 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 4740 4741 /* set the correct pool for the new PF MAC address in entry 0 */ 4742 igb_set_default_mac_filter(adapter); 4743 4744 return 0; 4745 } 4746 4747 /** 4748 * igb_write_mc_addr_list - write multicast addresses to MTA 4749 * @netdev: network interface device structure 4750 * 4751 * Writes multicast address list to the MTA hash table. 4752 * Returns: -ENOMEM on failure 4753 * 0 on no addresses written 4754 * X on writing X addresses to MTA 4755 **/ 4756 static int igb_write_mc_addr_list(struct net_device *netdev) 4757 { 4758 struct igb_adapter *adapter = netdev_priv(netdev); 4759 struct e1000_hw *hw = &adapter->hw; 4760 struct netdev_hw_addr *ha; 4761 u8 *mta_list; 4762 int i; 4763 4764 if (netdev_mc_empty(netdev)) { 4765 /* nothing to program, so clear mc list */ 4766 igb_update_mc_addr_list(hw, NULL, 0); 4767 igb_restore_vf_multicasts(adapter); 4768 return 0; 4769 } 4770 4771 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC); 4772 if (!mta_list) 4773 return -ENOMEM; 4774 4775 /* The shared function expects a packed array of only addresses. */ 4776 i = 0; 4777 netdev_for_each_mc_addr(ha, netdev) 4778 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 4779 4780 igb_update_mc_addr_list(hw, mta_list, i); 4781 kfree(mta_list); 4782 4783 return netdev_mc_count(netdev); 4784 } 4785 4786 static int igb_vlan_promisc_enable(struct igb_adapter *adapter) 4787 { 4788 struct e1000_hw *hw = &adapter->hw; 4789 u32 i, pf_id; 4790 4791 switch (hw->mac.type) { 4792 case e1000_i210: 4793 case e1000_i211: 4794 case e1000_i350: 4795 /* VLAN filtering needed for VLAN prio filter */ 4796 if (adapter->netdev->features & NETIF_F_NTUPLE) 4797 break; 4798 /* fall through */ 4799 case e1000_82576: 4800 case e1000_82580: 4801 case e1000_i354: 4802 /* VLAN filtering needed for pool filtering */ 4803 if (adapter->vfs_allocated_count) 4804 break; 4805 /* fall through */ 4806 default: 4807 return 1; 4808 } 4809 4810 /* We are already in VLAN promisc, nothing to do */ 4811 if (adapter->flags & IGB_FLAG_VLAN_PROMISC) 4812 return 0; 4813 4814 if (!adapter->vfs_allocated_count) 4815 goto set_vfta; 4816 4817 /* Add PF to all active pools */ 4818 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; 4819 4820 for (i = E1000_VLVF_ARRAY_SIZE; --i;) { 4821 u32 vlvf = rd32(E1000_VLVF(i)); 4822 4823 vlvf |= BIT(pf_id); 4824 wr32(E1000_VLVF(i), vlvf); 4825 } 4826 4827 set_vfta: 4828 /* Set all bits in the VLAN filter table array */ 4829 for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;) 4830 hw->mac.ops.write_vfta(hw, i, ~0U); 4831 4832 /* Set flag so we don't redo unnecessary work */ 4833 adapter->flags |= IGB_FLAG_VLAN_PROMISC; 4834 4835 return 0; 4836 } 4837 4838 #define VFTA_BLOCK_SIZE 8 4839 static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset) 4840 { 4841 struct e1000_hw *hw = &adapter->hw; 4842 u32 vfta[VFTA_BLOCK_SIZE] = { 0 }; 4843 u32 vid_start = vfta_offset * 32; 4844 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32); 4845 u32 i, vid, word, bits, pf_id; 4846 4847 /* guarantee that we don't scrub out management VLAN */ 4848 vid = adapter->mng_vlan_id; 4849 if (vid >= vid_start && vid < vid_end) 4850 vfta[(vid - vid_start) / 32] |= BIT(vid % 32); 4851 4852 if (!adapter->vfs_allocated_count) 4853 goto set_vfta; 4854 4855 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; 4856 4857 for (i = E1000_VLVF_ARRAY_SIZE; --i;) { 4858 u32 vlvf = rd32(E1000_VLVF(i)); 4859 4860 /* pull VLAN ID from VLVF */ 4861 vid = vlvf & VLAN_VID_MASK; 4862 4863 /* only concern ourselves with a certain range */ 4864 if (vid < vid_start || vid >= vid_end) 4865 continue; 4866 4867 if (vlvf & E1000_VLVF_VLANID_ENABLE) { 4868 /* record VLAN ID in VFTA */ 4869 vfta[(vid - vid_start) / 32] |= BIT(vid % 32); 4870 4871 /* if PF is part of this then continue */ 4872 if (test_bit(vid, adapter->active_vlans)) 4873 continue; 4874 } 4875 4876 /* remove PF from the pool */ 4877 bits = ~BIT(pf_id); 4878 bits &= rd32(E1000_VLVF(i)); 4879 wr32(E1000_VLVF(i), bits); 4880 } 4881 4882 set_vfta: 4883 /* extract values from active_vlans and write back to VFTA */ 4884 for (i = VFTA_BLOCK_SIZE; i--;) { 4885 vid = (vfta_offset + i) * 32; 4886 word = vid / BITS_PER_LONG; 4887 bits = vid % BITS_PER_LONG; 4888 4889 vfta[i] |= adapter->active_vlans[word] >> bits; 4890 4891 hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]); 4892 } 4893 } 4894 4895 static void igb_vlan_promisc_disable(struct igb_adapter *adapter) 4896 { 4897 u32 i; 4898 4899 /* We are not in VLAN promisc, nothing to do */ 4900 if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC)) 4901 return; 4902 4903 /* Set flag so we don't redo unnecessary work */ 4904 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC; 4905 4906 for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE) 4907 igb_scrub_vfta(adapter, i); 4908 } 4909 4910 /** 4911 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 4912 * @netdev: network interface device structure 4913 * 4914 * The set_rx_mode entry point is called whenever the unicast or multicast 4915 * address lists or the network interface flags are updated. This routine is 4916 * responsible for configuring the hardware for proper unicast, multicast, 4917 * promiscuous mode, and all-multi behavior. 4918 **/ 4919 static void igb_set_rx_mode(struct net_device *netdev) 4920 { 4921 struct igb_adapter *adapter = netdev_priv(netdev); 4922 struct e1000_hw *hw = &adapter->hw; 4923 unsigned int vfn = adapter->vfs_allocated_count; 4924 u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 4925 int count; 4926 4927 /* Check for Promiscuous and All Multicast modes */ 4928 if (netdev->flags & IFF_PROMISC) { 4929 rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE; 4930 vmolr |= E1000_VMOLR_MPME; 4931 4932 /* enable use of UTA filter to force packets to default pool */ 4933 if (hw->mac.type == e1000_82576) 4934 vmolr |= E1000_VMOLR_ROPE; 4935 } else { 4936 if (netdev->flags & IFF_ALLMULTI) { 4937 rctl |= E1000_RCTL_MPE; 4938 vmolr |= E1000_VMOLR_MPME; 4939 } else { 4940 /* Write addresses to the MTA, if the attempt fails 4941 * then we should just turn on promiscuous mode so 4942 * that we can at least receive multicast traffic 4943 */ 4944 count = igb_write_mc_addr_list(netdev); 4945 if (count < 0) { 4946 rctl |= E1000_RCTL_MPE; 4947 vmolr |= E1000_VMOLR_MPME; 4948 } else if (count) { 4949 vmolr |= E1000_VMOLR_ROMPE; 4950 } 4951 } 4952 } 4953 4954 /* Write addresses to available RAR registers, if there is not 4955 * sufficient space to store all the addresses then enable 4956 * unicast promiscuous mode 4957 */ 4958 if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) { 4959 rctl |= E1000_RCTL_UPE; 4960 vmolr |= E1000_VMOLR_ROPE; 4961 } 4962 4963 /* enable VLAN filtering by default */ 4964 rctl |= E1000_RCTL_VFE; 4965 4966 /* disable VLAN filtering for modes that require it */ 4967 if ((netdev->flags & IFF_PROMISC) || 4968 (netdev->features & NETIF_F_RXALL)) { 4969 /* if we fail to set all rules then just clear VFE */ 4970 if (igb_vlan_promisc_enable(adapter)) 4971 rctl &= ~E1000_RCTL_VFE; 4972 } else { 4973 igb_vlan_promisc_disable(adapter); 4974 } 4975 4976 /* update state of unicast, multicast, and VLAN filtering modes */ 4977 rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE | 4978 E1000_RCTL_VFE); 4979 wr32(E1000_RCTL, rctl); 4980 4981 #if (PAGE_SIZE < 8192) 4982 if (!adapter->vfs_allocated_count) { 4983 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) 4984 rlpml = IGB_MAX_FRAME_BUILD_SKB; 4985 } 4986 #endif 4987 wr32(E1000_RLPML, rlpml); 4988 4989 /* In order to support SR-IOV and eventually VMDq it is necessary to set 4990 * the VMOLR to enable the appropriate modes. Without this workaround 4991 * we will have issues with VLAN tag stripping not being done for frames 4992 * that are only arriving because we are the default pool 4993 */ 4994 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350)) 4995 return; 4996 4997 /* set UTA to appropriate mode */ 4998 igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE)); 4999 5000 vmolr |= rd32(E1000_VMOLR(vfn)) & 5001 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE); 5002 5003 /* enable Rx jumbo frames, restrict as needed to support build_skb */ 5004 vmolr &= ~E1000_VMOLR_RLPML_MASK; 5005 #if (PAGE_SIZE < 8192) 5006 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) 5007 vmolr |= IGB_MAX_FRAME_BUILD_SKB; 5008 else 5009 #endif 5010 vmolr |= MAX_JUMBO_FRAME_SIZE; 5011 vmolr |= E1000_VMOLR_LPE; 5012 5013 wr32(E1000_VMOLR(vfn), vmolr); 5014 5015 igb_restore_vf_multicasts(adapter); 5016 } 5017 5018 static void igb_check_wvbr(struct igb_adapter *adapter) 5019 { 5020 struct e1000_hw *hw = &adapter->hw; 5021 u32 wvbr = 0; 5022 5023 switch (hw->mac.type) { 5024 case e1000_82576: 5025 case e1000_i350: 5026 wvbr = rd32(E1000_WVBR); 5027 if (!wvbr) 5028 return; 5029 break; 5030 default: 5031 break; 5032 } 5033 5034 adapter->wvbr |= wvbr; 5035 } 5036 5037 #define IGB_STAGGERED_QUEUE_OFFSET 8 5038 5039 static void igb_spoof_check(struct igb_adapter *adapter) 5040 { 5041 int j; 5042 5043 if (!adapter->wvbr) 5044 return; 5045 5046 for (j = 0; j < adapter->vfs_allocated_count; j++) { 5047 if (adapter->wvbr & BIT(j) || 5048 adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) { 5049 dev_warn(&adapter->pdev->dev, 5050 "Spoof event(s) detected on VF %d\n", j); 5051 adapter->wvbr &= 5052 ~(BIT(j) | 5053 BIT(j + IGB_STAGGERED_QUEUE_OFFSET)); 5054 } 5055 } 5056 } 5057 5058 /* Need to wait a few seconds after link up to get diagnostic information from 5059 * the phy 5060 */ 5061 static void igb_update_phy_info(struct timer_list *t) 5062 { 5063 struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer); 5064 igb_get_phy_info(&adapter->hw); 5065 } 5066 5067 /** 5068 * igb_has_link - check shared code for link and determine up/down 5069 * @adapter: pointer to driver private info 5070 **/ 5071 bool igb_has_link(struct igb_adapter *adapter) 5072 { 5073 struct e1000_hw *hw = &adapter->hw; 5074 bool link_active = false; 5075 5076 /* get_link_status is set on LSC (link status) interrupt or 5077 * rx sequence error interrupt. get_link_status will stay 5078 * false until the e1000_check_for_link establishes link 5079 * for copper adapters ONLY 5080 */ 5081 switch (hw->phy.media_type) { 5082 case e1000_media_type_copper: 5083 if (!hw->mac.get_link_status) 5084 return true; 5085 case e1000_media_type_internal_serdes: 5086 hw->mac.ops.check_for_link(hw); 5087 link_active = !hw->mac.get_link_status; 5088 break; 5089 default: 5090 case e1000_media_type_unknown: 5091 break; 5092 } 5093 5094 if (((hw->mac.type == e1000_i210) || 5095 (hw->mac.type == e1000_i211)) && 5096 (hw->phy.id == I210_I_PHY_ID)) { 5097 if (!netif_carrier_ok(adapter->netdev)) { 5098 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; 5099 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) { 5100 adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE; 5101 adapter->link_check_timeout = jiffies; 5102 } 5103 } 5104 5105 return link_active; 5106 } 5107 5108 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event) 5109 { 5110 bool ret = false; 5111 u32 ctrl_ext, thstat; 5112 5113 /* check for thermal sensor event on i350 copper only */ 5114 if (hw->mac.type == e1000_i350) { 5115 thstat = rd32(E1000_THSTAT); 5116 ctrl_ext = rd32(E1000_CTRL_EXT); 5117 5118 if ((hw->phy.media_type == e1000_media_type_copper) && 5119 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) 5120 ret = !!(thstat & event); 5121 } 5122 5123 return ret; 5124 } 5125 5126 /** 5127 * igb_check_lvmmc - check for malformed packets received 5128 * and indicated in LVMMC register 5129 * @adapter: pointer to adapter 5130 **/ 5131 static void igb_check_lvmmc(struct igb_adapter *adapter) 5132 { 5133 struct e1000_hw *hw = &adapter->hw; 5134 u32 lvmmc; 5135 5136 lvmmc = rd32(E1000_LVMMC); 5137 if (lvmmc) { 5138 if (unlikely(net_ratelimit())) { 5139 netdev_warn(adapter->netdev, 5140 "malformed Tx packet detected and dropped, LVMMC:0x%08x\n", 5141 lvmmc); 5142 } 5143 } 5144 } 5145 5146 /** 5147 * igb_watchdog - Timer Call-back 5148 * @data: pointer to adapter cast into an unsigned long 5149 **/ 5150 static void igb_watchdog(struct timer_list *t) 5151 { 5152 struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer); 5153 /* Do the rest outside of interrupt context */ 5154 schedule_work(&adapter->watchdog_task); 5155 } 5156 5157 static void igb_watchdog_task(struct work_struct *work) 5158 { 5159 struct igb_adapter *adapter = container_of(work, 5160 struct igb_adapter, 5161 watchdog_task); 5162 struct e1000_hw *hw = &adapter->hw; 5163 struct e1000_phy_info *phy = &hw->phy; 5164 struct net_device *netdev = adapter->netdev; 5165 u32 link; 5166 int i; 5167 u32 connsw; 5168 u16 phy_data, retry_count = 20; 5169 5170 link = igb_has_link(adapter); 5171 5172 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) { 5173 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 5174 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; 5175 else 5176 link = false; 5177 } 5178 5179 /* Force link down if we have fiber to swap to */ 5180 if (adapter->flags & IGB_FLAG_MAS_ENABLE) { 5181 if (hw->phy.media_type == e1000_media_type_copper) { 5182 connsw = rd32(E1000_CONNSW); 5183 if (!(connsw & E1000_CONNSW_AUTOSENSE_EN)) 5184 link = 0; 5185 } 5186 } 5187 if (link) { 5188 /* Perform a reset if the media type changed. */ 5189 if (hw->dev_spec._82575.media_changed) { 5190 hw->dev_spec._82575.media_changed = false; 5191 adapter->flags |= IGB_FLAG_MEDIA_RESET; 5192 igb_reset(adapter); 5193 } 5194 /* Cancel scheduled suspend requests. */ 5195 pm_runtime_resume(netdev->dev.parent); 5196 5197 if (!netif_carrier_ok(netdev)) { 5198 u32 ctrl; 5199 5200 hw->mac.ops.get_speed_and_duplex(hw, 5201 &adapter->link_speed, 5202 &adapter->link_duplex); 5203 5204 ctrl = rd32(E1000_CTRL); 5205 /* Links status message must follow this format */ 5206 netdev_info(netdev, 5207 "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5208 netdev->name, 5209 adapter->link_speed, 5210 adapter->link_duplex == FULL_DUPLEX ? 5211 "Full" : "Half", 5212 (ctrl & E1000_CTRL_TFCE) && 5213 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" : 5214 (ctrl & E1000_CTRL_RFCE) ? "RX" : 5215 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None"); 5216 5217 /* disable EEE if enabled */ 5218 if ((adapter->flags & IGB_FLAG_EEE) && 5219 (adapter->link_duplex == HALF_DUPLEX)) { 5220 dev_info(&adapter->pdev->dev, 5221 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n"); 5222 adapter->hw.dev_spec._82575.eee_disable = true; 5223 adapter->flags &= ~IGB_FLAG_EEE; 5224 } 5225 5226 /* check if SmartSpeed worked */ 5227 igb_check_downshift(hw); 5228 if (phy->speed_downgraded) 5229 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 5230 5231 /* check for thermal sensor event */ 5232 if (igb_thermal_sensor_event(hw, 5233 E1000_THSTAT_LINK_THROTTLE)) 5234 netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n"); 5235 5236 /* adjust timeout factor according to speed/duplex */ 5237 adapter->tx_timeout_factor = 1; 5238 switch (adapter->link_speed) { 5239 case SPEED_10: 5240 adapter->tx_timeout_factor = 14; 5241 break; 5242 case SPEED_100: 5243 /* maybe add some timeout factor ? */ 5244 break; 5245 } 5246 5247 if (adapter->link_speed != SPEED_1000) 5248 goto no_wait; 5249 5250 /* wait for Remote receiver status OK */ 5251 retry_read_status: 5252 if (!igb_read_phy_reg(hw, PHY_1000T_STATUS, 5253 &phy_data)) { 5254 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 5255 retry_count) { 5256 msleep(100); 5257 retry_count--; 5258 goto retry_read_status; 5259 } else if (!retry_count) { 5260 dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); 5261 } 5262 } else { 5263 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); 5264 } 5265 no_wait: 5266 netif_carrier_on(netdev); 5267 5268 igb_ping_all_vfs(adapter); 5269 igb_check_vf_rate_limit(adapter); 5270 5271 /* link state has changed, schedule phy info update */ 5272 if (!test_bit(__IGB_DOWN, &adapter->state)) 5273 mod_timer(&adapter->phy_info_timer, 5274 round_jiffies(jiffies + 2 * HZ)); 5275 } 5276 } else { 5277 if (netif_carrier_ok(netdev)) { 5278 adapter->link_speed = 0; 5279 adapter->link_duplex = 0; 5280 5281 /* check for thermal sensor event */ 5282 if (igb_thermal_sensor_event(hw, 5283 E1000_THSTAT_PWR_DOWN)) { 5284 netdev_err(netdev, "The network adapter was stopped because it overheated\n"); 5285 } 5286 5287 /* Links status message must follow this format */ 5288 netdev_info(netdev, "igb: %s NIC Link is Down\n", 5289 netdev->name); 5290 netif_carrier_off(netdev); 5291 5292 igb_ping_all_vfs(adapter); 5293 5294 /* link state has changed, schedule phy info update */ 5295 if (!test_bit(__IGB_DOWN, &adapter->state)) 5296 mod_timer(&adapter->phy_info_timer, 5297 round_jiffies(jiffies + 2 * HZ)); 5298 5299 /* link is down, time to check for alternate media */ 5300 if (adapter->flags & IGB_FLAG_MAS_ENABLE) { 5301 igb_check_swap_media(adapter); 5302 if (adapter->flags & IGB_FLAG_MEDIA_RESET) { 5303 schedule_work(&adapter->reset_task); 5304 /* return immediately */ 5305 return; 5306 } 5307 } 5308 pm_schedule_suspend(netdev->dev.parent, 5309 MSEC_PER_SEC * 5); 5310 5311 /* also check for alternate media here */ 5312 } else if (!netif_carrier_ok(netdev) && 5313 (adapter->flags & IGB_FLAG_MAS_ENABLE)) { 5314 igb_check_swap_media(adapter); 5315 if (adapter->flags & IGB_FLAG_MEDIA_RESET) { 5316 schedule_work(&adapter->reset_task); 5317 /* return immediately */ 5318 return; 5319 } 5320 } 5321 } 5322 5323 spin_lock(&adapter->stats64_lock); 5324 igb_update_stats(adapter); 5325 spin_unlock(&adapter->stats64_lock); 5326 5327 for (i = 0; i < adapter->num_tx_queues; i++) { 5328 struct igb_ring *tx_ring = adapter->tx_ring[i]; 5329 if (!netif_carrier_ok(netdev)) { 5330 /* We've lost link, so the controller stops DMA, 5331 * but we've got queued Tx work that's never going 5332 * to get done, so reset controller to flush Tx. 5333 * (Do the reset outside of interrupt context). 5334 */ 5335 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) { 5336 adapter->tx_timeout_count++; 5337 schedule_work(&adapter->reset_task); 5338 /* return immediately since reset is imminent */ 5339 return; 5340 } 5341 } 5342 5343 /* Force detection of hung controller every watchdog period */ 5344 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 5345 } 5346 5347 /* Cause software interrupt to ensure Rx ring is cleaned */ 5348 if (adapter->flags & IGB_FLAG_HAS_MSIX) { 5349 u32 eics = 0; 5350 5351 for (i = 0; i < adapter->num_q_vectors; i++) 5352 eics |= adapter->q_vector[i]->eims_value; 5353 wr32(E1000_EICS, eics); 5354 } else { 5355 wr32(E1000_ICS, E1000_ICS_RXDMT0); 5356 } 5357 5358 igb_spoof_check(adapter); 5359 igb_ptp_rx_hang(adapter); 5360 igb_ptp_tx_hang(adapter); 5361 5362 /* Check LVMMC register on i350/i354 only */ 5363 if ((adapter->hw.mac.type == e1000_i350) || 5364 (adapter->hw.mac.type == e1000_i354)) 5365 igb_check_lvmmc(adapter); 5366 5367 /* Reset the timer */ 5368 if (!test_bit(__IGB_DOWN, &adapter->state)) { 5369 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) 5370 mod_timer(&adapter->watchdog_timer, 5371 round_jiffies(jiffies + HZ)); 5372 else 5373 mod_timer(&adapter->watchdog_timer, 5374 round_jiffies(jiffies + 2 * HZ)); 5375 } 5376 } 5377 5378 enum latency_range { 5379 lowest_latency = 0, 5380 low_latency = 1, 5381 bulk_latency = 2, 5382 latency_invalid = 255 5383 }; 5384 5385 /** 5386 * igb_update_ring_itr - update the dynamic ITR value based on packet size 5387 * @q_vector: pointer to q_vector 5388 * 5389 * Stores a new ITR value based on strictly on packet size. This 5390 * algorithm is less sophisticated than that used in igb_update_itr, 5391 * due to the difficulty of synchronizing statistics across multiple 5392 * receive rings. The divisors and thresholds used by this function 5393 * were determined based on theoretical maximum wire speed and testing 5394 * data, in order to minimize response time while increasing bulk 5395 * throughput. 5396 * This functionality is controlled by ethtool's coalescing settings. 5397 * NOTE: This function is called only when operating in a multiqueue 5398 * receive environment. 5399 **/ 5400 static void igb_update_ring_itr(struct igb_q_vector *q_vector) 5401 { 5402 int new_val = q_vector->itr_val; 5403 int avg_wire_size = 0; 5404 struct igb_adapter *adapter = q_vector->adapter; 5405 unsigned int packets; 5406 5407 /* For non-gigabit speeds, just fix the interrupt rate at 4000 5408 * ints/sec - ITR timer value of 120 ticks. 5409 */ 5410 if (adapter->link_speed != SPEED_1000) { 5411 new_val = IGB_4K_ITR; 5412 goto set_itr_val; 5413 } 5414 5415 packets = q_vector->rx.total_packets; 5416 if (packets) 5417 avg_wire_size = q_vector->rx.total_bytes / packets; 5418 5419 packets = q_vector->tx.total_packets; 5420 if (packets) 5421 avg_wire_size = max_t(u32, avg_wire_size, 5422 q_vector->tx.total_bytes / packets); 5423 5424 /* if avg_wire_size isn't set no work was done */ 5425 if (!avg_wire_size) 5426 goto clear_counts; 5427 5428 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 5429 avg_wire_size += 24; 5430 5431 /* Don't starve jumbo frames */ 5432 avg_wire_size = min(avg_wire_size, 3000); 5433 5434 /* Give a little boost to mid-size frames */ 5435 if ((avg_wire_size > 300) && (avg_wire_size < 1200)) 5436 new_val = avg_wire_size / 3; 5437 else 5438 new_val = avg_wire_size / 2; 5439 5440 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 5441 if (new_val < IGB_20K_ITR && 5442 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 5443 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 5444 new_val = IGB_20K_ITR; 5445 5446 set_itr_val: 5447 if (new_val != q_vector->itr_val) { 5448 q_vector->itr_val = new_val; 5449 q_vector->set_itr = 1; 5450 } 5451 clear_counts: 5452 q_vector->rx.total_bytes = 0; 5453 q_vector->rx.total_packets = 0; 5454 q_vector->tx.total_bytes = 0; 5455 q_vector->tx.total_packets = 0; 5456 } 5457 5458 /** 5459 * igb_update_itr - update the dynamic ITR value based on statistics 5460 * @q_vector: pointer to q_vector 5461 * @ring_container: ring info to update the itr for 5462 * 5463 * Stores a new ITR value based on packets and byte 5464 * counts during the last interrupt. The advantage of per interrupt 5465 * computation is faster updates and more accurate ITR for the current 5466 * traffic pattern. Constants in this function were computed 5467 * based on theoretical maximum wire speed and thresholds were set based 5468 * on testing data as well as attempting to minimize response time 5469 * while increasing bulk throughput. 5470 * This functionality is controlled by ethtool's coalescing settings. 5471 * NOTE: These calculations are only valid when operating in a single- 5472 * queue environment. 5473 **/ 5474 static void igb_update_itr(struct igb_q_vector *q_vector, 5475 struct igb_ring_container *ring_container) 5476 { 5477 unsigned int packets = ring_container->total_packets; 5478 unsigned int bytes = ring_container->total_bytes; 5479 u8 itrval = ring_container->itr; 5480 5481 /* no packets, exit with status unchanged */ 5482 if (packets == 0) 5483 return; 5484 5485 switch (itrval) { 5486 case lowest_latency: 5487 /* handle TSO and jumbo frames */ 5488 if (bytes/packets > 8000) 5489 itrval = bulk_latency; 5490 else if ((packets < 5) && (bytes > 512)) 5491 itrval = low_latency; 5492 break; 5493 case low_latency: /* 50 usec aka 20000 ints/s */ 5494 if (bytes > 10000) { 5495 /* this if handles the TSO accounting */ 5496 if (bytes/packets > 8000) 5497 itrval = bulk_latency; 5498 else if ((packets < 10) || ((bytes/packets) > 1200)) 5499 itrval = bulk_latency; 5500 else if ((packets > 35)) 5501 itrval = lowest_latency; 5502 } else if (bytes/packets > 2000) { 5503 itrval = bulk_latency; 5504 } else if (packets <= 2 && bytes < 512) { 5505 itrval = lowest_latency; 5506 } 5507 break; 5508 case bulk_latency: /* 250 usec aka 4000 ints/s */ 5509 if (bytes > 25000) { 5510 if (packets > 35) 5511 itrval = low_latency; 5512 } else if (bytes < 1500) { 5513 itrval = low_latency; 5514 } 5515 break; 5516 } 5517 5518 /* clear work counters since we have the values we need */ 5519 ring_container->total_bytes = 0; 5520 ring_container->total_packets = 0; 5521 5522 /* write updated itr to ring container */ 5523 ring_container->itr = itrval; 5524 } 5525 5526 static void igb_set_itr(struct igb_q_vector *q_vector) 5527 { 5528 struct igb_adapter *adapter = q_vector->adapter; 5529 u32 new_itr = q_vector->itr_val; 5530 u8 current_itr = 0; 5531 5532 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 5533 if (adapter->link_speed != SPEED_1000) { 5534 current_itr = 0; 5535 new_itr = IGB_4K_ITR; 5536 goto set_itr_now; 5537 } 5538 5539 igb_update_itr(q_vector, &q_vector->tx); 5540 igb_update_itr(q_vector, &q_vector->rx); 5541 5542 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 5543 5544 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 5545 if (current_itr == lowest_latency && 5546 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 5547 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 5548 current_itr = low_latency; 5549 5550 switch (current_itr) { 5551 /* counts and packets in update_itr are dependent on these numbers */ 5552 case lowest_latency: 5553 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */ 5554 break; 5555 case low_latency: 5556 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */ 5557 break; 5558 case bulk_latency: 5559 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */ 5560 break; 5561 default: 5562 break; 5563 } 5564 5565 set_itr_now: 5566 if (new_itr != q_vector->itr_val) { 5567 /* this attempts to bias the interrupt rate towards Bulk 5568 * by adding intermediate steps when interrupt rate is 5569 * increasing 5570 */ 5571 new_itr = new_itr > q_vector->itr_val ? 5572 max((new_itr * q_vector->itr_val) / 5573 (new_itr + (q_vector->itr_val >> 2)), 5574 new_itr) : new_itr; 5575 /* Don't write the value here; it resets the adapter's 5576 * internal timer, and causes us to delay far longer than 5577 * we should between interrupts. Instead, we write the ITR 5578 * value at the beginning of the next interrupt so the timing 5579 * ends up being correct. 5580 */ 5581 q_vector->itr_val = new_itr; 5582 q_vector->set_itr = 1; 5583 } 5584 } 5585 5586 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens, 5587 u32 type_tucmd, u32 mss_l4len_idx) 5588 { 5589 struct e1000_adv_tx_context_desc *context_desc; 5590 u16 i = tx_ring->next_to_use; 5591 5592 context_desc = IGB_TX_CTXTDESC(tx_ring, i); 5593 5594 i++; 5595 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 5596 5597 /* set bits to identify this as an advanced context descriptor */ 5598 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 5599 5600 /* For 82575, context index must be unique per ring. */ 5601 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 5602 mss_l4len_idx |= tx_ring->reg_idx << 4; 5603 5604 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 5605 context_desc->seqnum_seed = 0; 5606 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 5607 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 5608 } 5609 5610 static int igb_tso(struct igb_ring *tx_ring, 5611 struct igb_tx_buffer *first, 5612 u8 *hdr_len) 5613 { 5614 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 5615 struct sk_buff *skb = first->skb; 5616 union { 5617 struct iphdr *v4; 5618 struct ipv6hdr *v6; 5619 unsigned char *hdr; 5620 } ip; 5621 union { 5622 struct tcphdr *tcp; 5623 unsigned char *hdr; 5624 } l4; 5625 u32 paylen, l4_offset; 5626 int err; 5627 5628 if (skb->ip_summed != CHECKSUM_PARTIAL) 5629 return 0; 5630 5631 if (!skb_is_gso(skb)) 5632 return 0; 5633 5634 err = skb_cow_head(skb, 0); 5635 if (err < 0) 5636 return err; 5637 5638 ip.hdr = skb_network_header(skb); 5639 l4.hdr = skb_checksum_start(skb); 5640 5641 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 5642 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP; 5643 5644 /* initialize outer IP header fields */ 5645 if (ip.v4->version == 4) { 5646 unsigned char *csum_start = skb_checksum_start(skb); 5647 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 5648 5649 /* IP header will have to cancel out any data that 5650 * is not a part of the outer IP header 5651 */ 5652 ip.v4->check = csum_fold(csum_partial(trans_start, 5653 csum_start - trans_start, 5654 0)); 5655 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4; 5656 5657 ip.v4->tot_len = 0; 5658 first->tx_flags |= IGB_TX_FLAGS_TSO | 5659 IGB_TX_FLAGS_CSUM | 5660 IGB_TX_FLAGS_IPV4; 5661 } else { 5662 ip.v6->payload_len = 0; 5663 first->tx_flags |= IGB_TX_FLAGS_TSO | 5664 IGB_TX_FLAGS_CSUM; 5665 } 5666 5667 /* determine offset of inner transport header */ 5668 l4_offset = l4.hdr - skb->data; 5669 5670 /* compute length of segmentation header */ 5671 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 5672 5673 /* remove payload length from inner checksum */ 5674 paylen = skb->len - l4_offset; 5675 csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); 5676 5677 /* update gso size and bytecount with header size */ 5678 first->gso_segs = skb_shinfo(skb)->gso_segs; 5679 first->bytecount += (first->gso_segs - 1) * *hdr_len; 5680 5681 /* MSS L4LEN IDX */ 5682 mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT; 5683 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT; 5684 5685 /* VLAN MACLEN IPLEN */ 5686 vlan_macip_lens = l4.hdr - ip.hdr; 5687 vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT; 5688 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; 5689 5690 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx); 5691 5692 return 1; 5693 } 5694 5695 static inline bool igb_ipv6_csum_is_sctp(struct sk_buff *skb) 5696 { 5697 unsigned int offset = 0; 5698 5699 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 5700 5701 return offset == skb_checksum_start_offset(skb); 5702 } 5703 5704 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first) 5705 { 5706 struct sk_buff *skb = first->skb; 5707 u32 vlan_macip_lens = 0; 5708 u32 type_tucmd = 0; 5709 5710 if (skb->ip_summed != CHECKSUM_PARTIAL) { 5711 csum_failed: 5712 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN)) 5713 return; 5714 goto no_csum; 5715 } 5716 5717 switch (skb->csum_offset) { 5718 case offsetof(struct tcphdr, check): 5719 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP; 5720 /* fall through */ 5721 case offsetof(struct udphdr, check): 5722 break; 5723 case offsetof(struct sctphdr, checksum): 5724 /* validate that this is actually an SCTP request */ 5725 if (((first->protocol == htons(ETH_P_IP)) && 5726 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 5727 ((first->protocol == htons(ETH_P_IPV6)) && 5728 igb_ipv6_csum_is_sctp(skb))) { 5729 type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP; 5730 break; 5731 } 5732 default: 5733 skb_checksum_help(skb); 5734 goto csum_failed; 5735 } 5736 5737 /* update TX checksum flag */ 5738 first->tx_flags |= IGB_TX_FLAGS_CSUM; 5739 vlan_macip_lens = skb_checksum_start_offset(skb) - 5740 skb_network_offset(skb); 5741 no_csum: 5742 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT; 5743 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; 5744 5745 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0); 5746 } 5747 5748 #define IGB_SET_FLAG(_input, _flag, _result) \ 5749 ((_flag <= _result) ? \ 5750 ((u32)(_input & _flag) * (_result / _flag)) : \ 5751 ((u32)(_input & _flag) / (_flag / _result))) 5752 5753 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 5754 { 5755 /* set type for advanced descriptor with frame checksum insertion */ 5756 u32 cmd_type = E1000_ADVTXD_DTYP_DATA | 5757 E1000_ADVTXD_DCMD_DEXT | 5758 E1000_ADVTXD_DCMD_IFCS; 5759 5760 /* set HW vlan bit if vlan is present */ 5761 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN, 5762 (E1000_ADVTXD_DCMD_VLE)); 5763 5764 /* set segmentation bits for TSO */ 5765 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO, 5766 (E1000_ADVTXD_DCMD_TSE)); 5767 5768 /* set timestamp bit if present */ 5769 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP, 5770 (E1000_ADVTXD_MAC_TSTAMP)); 5771 5772 /* insert frame checksum */ 5773 cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS); 5774 5775 return cmd_type; 5776 } 5777 5778 static void igb_tx_olinfo_status(struct igb_ring *tx_ring, 5779 union e1000_adv_tx_desc *tx_desc, 5780 u32 tx_flags, unsigned int paylen) 5781 { 5782 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT; 5783 5784 /* 82575 requires a unique index per ring */ 5785 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 5786 olinfo_status |= tx_ring->reg_idx << 4; 5787 5788 /* insert L4 checksum */ 5789 olinfo_status |= IGB_SET_FLAG(tx_flags, 5790 IGB_TX_FLAGS_CSUM, 5791 (E1000_TXD_POPTS_TXSM << 8)); 5792 5793 /* insert IPv4 checksum */ 5794 olinfo_status |= IGB_SET_FLAG(tx_flags, 5795 IGB_TX_FLAGS_IPV4, 5796 (E1000_TXD_POPTS_IXSM << 8)); 5797 5798 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 5799 } 5800 5801 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) 5802 { 5803 struct net_device *netdev = tx_ring->netdev; 5804 5805 netif_stop_subqueue(netdev, tx_ring->queue_index); 5806 5807 /* Herbert's original patch had: 5808 * smp_mb__after_netif_stop_queue(); 5809 * but since that doesn't exist yet, just open code it. 5810 */ 5811 smp_mb(); 5812 5813 /* We need to check again in a case another CPU has just 5814 * made room available. 5815 */ 5816 if (igb_desc_unused(tx_ring) < size) 5817 return -EBUSY; 5818 5819 /* A reprieve! */ 5820 netif_wake_subqueue(netdev, tx_ring->queue_index); 5821 5822 u64_stats_update_begin(&tx_ring->tx_syncp2); 5823 tx_ring->tx_stats.restart_queue2++; 5824 u64_stats_update_end(&tx_ring->tx_syncp2); 5825 5826 return 0; 5827 } 5828 5829 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) 5830 { 5831 if (igb_desc_unused(tx_ring) >= size) 5832 return 0; 5833 return __igb_maybe_stop_tx(tx_ring, size); 5834 } 5835 5836 static int igb_tx_map(struct igb_ring *tx_ring, 5837 struct igb_tx_buffer *first, 5838 const u8 hdr_len) 5839 { 5840 struct sk_buff *skb = first->skb; 5841 struct igb_tx_buffer *tx_buffer; 5842 union e1000_adv_tx_desc *tx_desc; 5843 struct skb_frag_struct *frag; 5844 dma_addr_t dma; 5845 unsigned int data_len, size; 5846 u32 tx_flags = first->tx_flags; 5847 u32 cmd_type = igb_tx_cmd_type(skb, tx_flags); 5848 u16 i = tx_ring->next_to_use; 5849 5850 tx_desc = IGB_TX_DESC(tx_ring, i); 5851 5852 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 5853 5854 size = skb_headlen(skb); 5855 data_len = skb->data_len; 5856 5857 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 5858 5859 tx_buffer = first; 5860 5861 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 5862 if (dma_mapping_error(tx_ring->dev, dma)) 5863 goto dma_error; 5864 5865 /* record length, and DMA address */ 5866 dma_unmap_len_set(tx_buffer, len, size); 5867 dma_unmap_addr_set(tx_buffer, dma, dma); 5868 5869 tx_desc->read.buffer_addr = cpu_to_le64(dma); 5870 5871 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) { 5872 tx_desc->read.cmd_type_len = 5873 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD); 5874 5875 i++; 5876 tx_desc++; 5877 if (i == tx_ring->count) { 5878 tx_desc = IGB_TX_DESC(tx_ring, 0); 5879 i = 0; 5880 } 5881 tx_desc->read.olinfo_status = 0; 5882 5883 dma += IGB_MAX_DATA_PER_TXD; 5884 size -= IGB_MAX_DATA_PER_TXD; 5885 5886 tx_desc->read.buffer_addr = cpu_to_le64(dma); 5887 } 5888 5889 if (likely(!data_len)) 5890 break; 5891 5892 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 5893 5894 i++; 5895 tx_desc++; 5896 if (i == tx_ring->count) { 5897 tx_desc = IGB_TX_DESC(tx_ring, 0); 5898 i = 0; 5899 } 5900 tx_desc->read.olinfo_status = 0; 5901 5902 size = skb_frag_size(frag); 5903 data_len -= size; 5904 5905 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 5906 size, DMA_TO_DEVICE); 5907 5908 tx_buffer = &tx_ring->tx_buffer_info[i]; 5909 } 5910 5911 /* write last descriptor with RS and EOP bits */ 5912 cmd_type |= size | IGB_TXD_DCMD; 5913 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 5914 5915 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 5916 5917 /* set the timestamp */ 5918 first->time_stamp = jiffies; 5919 5920 /* Force memory writes to complete before letting h/w know there 5921 * are new descriptors to fetch. (Only applicable for weak-ordered 5922 * memory model archs, such as IA-64). 5923 * 5924 * We also need this memory barrier to make certain all of the 5925 * status bits have been updated before next_to_watch is written. 5926 */ 5927 wmb(); 5928 5929 /* set next_to_watch value indicating a packet is present */ 5930 first->next_to_watch = tx_desc; 5931 5932 i++; 5933 if (i == tx_ring->count) 5934 i = 0; 5935 5936 tx_ring->next_to_use = i; 5937 5938 /* Make sure there is space in the ring for the next send. */ 5939 igb_maybe_stop_tx(tx_ring, DESC_NEEDED); 5940 5941 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) { 5942 writel(i, tx_ring->tail); 5943 5944 /* we need this if more than one processor can write to our tail 5945 * at a time, it synchronizes IO on IA64/Altix systems 5946 */ 5947 mmiowb(); 5948 } 5949 return 0; 5950 5951 dma_error: 5952 dev_err(tx_ring->dev, "TX DMA map failed\n"); 5953 tx_buffer = &tx_ring->tx_buffer_info[i]; 5954 5955 /* clear dma mappings for failed tx_buffer_info map */ 5956 while (tx_buffer != first) { 5957 if (dma_unmap_len(tx_buffer, len)) 5958 dma_unmap_page(tx_ring->dev, 5959 dma_unmap_addr(tx_buffer, dma), 5960 dma_unmap_len(tx_buffer, len), 5961 DMA_TO_DEVICE); 5962 dma_unmap_len_set(tx_buffer, len, 0); 5963 5964 if (i-- == 0) 5965 i += tx_ring->count; 5966 tx_buffer = &tx_ring->tx_buffer_info[i]; 5967 } 5968 5969 if (dma_unmap_len(tx_buffer, len)) 5970 dma_unmap_single(tx_ring->dev, 5971 dma_unmap_addr(tx_buffer, dma), 5972 dma_unmap_len(tx_buffer, len), 5973 DMA_TO_DEVICE); 5974 dma_unmap_len_set(tx_buffer, len, 0); 5975 5976 dev_kfree_skb_any(tx_buffer->skb); 5977 tx_buffer->skb = NULL; 5978 5979 tx_ring->next_to_use = i; 5980 5981 return -1; 5982 } 5983 5984 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb, 5985 struct igb_ring *tx_ring) 5986 { 5987 struct igb_tx_buffer *first; 5988 int tso; 5989 u32 tx_flags = 0; 5990 unsigned short f; 5991 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 5992 __be16 protocol = vlan_get_protocol(skb); 5993 u8 hdr_len = 0; 5994 5995 /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD, 5996 * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD, 5997 * + 2 desc gap to keep tail from touching head, 5998 * + 1 desc for context descriptor, 5999 * otherwise try next time 6000 */ 6001 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 6002 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 6003 6004 if (igb_maybe_stop_tx(tx_ring, count + 3)) { 6005 /* this is a hard error */ 6006 return NETDEV_TX_BUSY; 6007 } 6008 6009 /* record the location of the first descriptor for this packet */ 6010 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 6011 first->skb = skb; 6012 first->bytecount = skb->len; 6013 first->gso_segs = 1; 6014 6015 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 6016 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev); 6017 6018 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 6019 !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS, 6020 &adapter->state)) { 6021 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 6022 tx_flags |= IGB_TX_FLAGS_TSTAMP; 6023 6024 adapter->ptp_tx_skb = skb_get(skb); 6025 adapter->ptp_tx_start = jiffies; 6026 if (adapter->hw.mac.type == e1000_82576) 6027 schedule_work(&adapter->ptp_tx_work); 6028 } else { 6029 adapter->tx_hwtstamp_skipped++; 6030 } 6031 } 6032 6033 skb_tx_timestamp(skb); 6034 6035 if (skb_vlan_tag_present(skb)) { 6036 tx_flags |= IGB_TX_FLAGS_VLAN; 6037 tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT); 6038 } 6039 6040 /* record initial flags and protocol */ 6041 first->tx_flags = tx_flags; 6042 first->protocol = protocol; 6043 6044 tso = igb_tso(tx_ring, first, &hdr_len); 6045 if (tso < 0) 6046 goto out_drop; 6047 else if (!tso) 6048 igb_tx_csum(tx_ring, first); 6049 6050 if (igb_tx_map(tx_ring, first, hdr_len)) 6051 goto cleanup_tx_tstamp; 6052 6053 return NETDEV_TX_OK; 6054 6055 out_drop: 6056 dev_kfree_skb_any(first->skb); 6057 first->skb = NULL; 6058 cleanup_tx_tstamp: 6059 if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) { 6060 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev); 6061 6062 dev_kfree_skb_any(adapter->ptp_tx_skb); 6063 adapter->ptp_tx_skb = NULL; 6064 if (adapter->hw.mac.type == e1000_82576) 6065 cancel_work_sync(&adapter->ptp_tx_work); 6066 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state); 6067 } 6068 6069 return NETDEV_TX_OK; 6070 } 6071 6072 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter, 6073 struct sk_buff *skb) 6074 { 6075 unsigned int r_idx = skb->queue_mapping; 6076 6077 if (r_idx >= adapter->num_tx_queues) 6078 r_idx = r_idx % adapter->num_tx_queues; 6079 6080 return adapter->tx_ring[r_idx]; 6081 } 6082 6083 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, 6084 struct net_device *netdev) 6085 { 6086 struct igb_adapter *adapter = netdev_priv(netdev); 6087 6088 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 6089 * in order to meet this minimum size requirement. 6090 */ 6091 if (skb_put_padto(skb, 17)) 6092 return NETDEV_TX_OK; 6093 6094 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb)); 6095 } 6096 6097 /** 6098 * igb_tx_timeout - Respond to a Tx Hang 6099 * @netdev: network interface device structure 6100 **/ 6101 static void igb_tx_timeout(struct net_device *netdev) 6102 { 6103 struct igb_adapter *adapter = netdev_priv(netdev); 6104 struct e1000_hw *hw = &adapter->hw; 6105 6106 /* Do the reset outside of interrupt context */ 6107 adapter->tx_timeout_count++; 6108 6109 if (hw->mac.type >= e1000_82580) 6110 hw->dev_spec._82575.global_device_reset = true; 6111 6112 schedule_work(&adapter->reset_task); 6113 wr32(E1000_EICS, 6114 (adapter->eims_enable_mask & ~adapter->eims_other)); 6115 } 6116 6117 static void igb_reset_task(struct work_struct *work) 6118 { 6119 struct igb_adapter *adapter; 6120 adapter = container_of(work, struct igb_adapter, reset_task); 6121 6122 igb_dump(adapter); 6123 netdev_err(adapter->netdev, "Reset adapter\n"); 6124 igb_reinit_locked(adapter); 6125 } 6126 6127 /** 6128 * igb_get_stats64 - Get System Network Statistics 6129 * @netdev: network interface device structure 6130 * @stats: rtnl_link_stats64 pointer 6131 **/ 6132 static void igb_get_stats64(struct net_device *netdev, 6133 struct rtnl_link_stats64 *stats) 6134 { 6135 struct igb_adapter *adapter = netdev_priv(netdev); 6136 6137 spin_lock(&adapter->stats64_lock); 6138 igb_update_stats(adapter); 6139 memcpy(stats, &adapter->stats64, sizeof(*stats)); 6140 spin_unlock(&adapter->stats64_lock); 6141 } 6142 6143 /** 6144 * igb_change_mtu - Change the Maximum Transfer Unit 6145 * @netdev: network interface device structure 6146 * @new_mtu: new value for maximum frame size 6147 * 6148 * Returns 0 on success, negative on failure 6149 **/ 6150 static int igb_change_mtu(struct net_device *netdev, int new_mtu) 6151 { 6152 struct igb_adapter *adapter = netdev_priv(netdev); 6153 struct pci_dev *pdev = adapter->pdev; 6154 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 6155 6156 /* adjust max frame to be at least the size of a standard frame */ 6157 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 6158 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 6159 6160 while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) 6161 usleep_range(1000, 2000); 6162 6163 /* igb_down has a dependency on max_frame_size */ 6164 adapter->max_frame_size = max_frame; 6165 6166 if (netif_running(netdev)) 6167 igb_down(adapter); 6168 6169 dev_info(&pdev->dev, "changing MTU from %d to %d\n", 6170 netdev->mtu, new_mtu); 6171 netdev->mtu = new_mtu; 6172 6173 if (netif_running(netdev)) 6174 igb_up(adapter); 6175 else 6176 igb_reset(adapter); 6177 6178 clear_bit(__IGB_RESETTING, &adapter->state); 6179 6180 return 0; 6181 } 6182 6183 /** 6184 * igb_update_stats - Update the board statistics counters 6185 * @adapter: board private structure 6186 **/ 6187 void igb_update_stats(struct igb_adapter *adapter) 6188 { 6189 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 6190 struct e1000_hw *hw = &adapter->hw; 6191 struct pci_dev *pdev = adapter->pdev; 6192 u32 reg, mpc; 6193 int i; 6194 u64 bytes, packets; 6195 unsigned int start; 6196 u64 _bytes, _packets; 6197 6198 /* Prevent stats update while adapter is being reset, or if the pci 6199 * connection is down. 6200 */ 6201 if (adapter->link_speed == 0) 6202 return; 6203 if (pci_channel_offline(pdev)) 6204 return; 6205 6206 bytes = 0; 6207 packets = 0; 6208 6209 rcu_read_lock(); 6210 for (i = 0; i < adapter->num_rx_queues; i++) { 6211 struct igb_ring *ring = adapter->rx_ring[i]; 6212 u32 rqdpc = rd32(E1000_RQDPC(i)); 6213 if (hw->mac.type >= e1000_i210) 6214 wr32(E1000_RQDPC(i), 0); 6215 6216 if (rqdpc) { 6217 ring->rx_stats.drops += rqdpc; 6218 net_stats->rx_fifo_errors += rqdpc; 6219 } 6220 6221 do { 6222 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 6223 _bytes = ring->rx_stats.bytes; 6224 _packets = ring->rx_stats.packets; 6225 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 6226 bytes += _bytes; 6227 packets += _packets; 6228 } 6229 6230 net_stats->rx_bytes = bytes; 6231 net_stats->rx_packets = packets; 6232 6233 bytes = 0; 6234 packets = 0; 6235 for (i = 0; i < adapter->num_tx_queues; i++) { 6236 struct igb_ring *ring = adapter->tx_ring[i]; 6237 do { 6238 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 6239 _bytes = ring->tx_stats.bytes; 6240 _packets = ring->tx_stats.packets; 6241 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 6242 bytes += _bytes; 6243 packets += _packets; 6244 } 6245 net_stats->tx_bytes = bytes; 6246 net_stats->tx_packets = packets; 6247 rcu_read_unlock(); 6248 6249 /* read stats registers */ 6250 adapter->stats.crcerrs += rd32(E1000_CRCERRS); 6251 adapter->stats.gprc += rd32(E1000_GPRC); 6252 adapter->stats.gorc += rd32(E1000_GORCL); 6253 rd32(E1000_GORCH); /* clear GORCL */ 6254 adapter->stats.bprc += rd32(E1000_BPRC); 6255 adapter->stats.mprc += rd32(E1000_MPRC); 6256 adapter->stats.roc += rd32(E1000_ROC); 6257 6258 adapter->stats.prc64 += rd32(E1000_PRC64); 6259 adapter->stats.prc127 += rd32(E1000_PRC127); 6260 adapter->stats.prc255 += rd32(E1000_PRC255); 6261 adapter->stats.prc511 += rd32(E1000_PRC511); 6262 adapter->stats.prc1023 += rd32(E1000_PRC1023); 6263 adapter->stats.prc1522 += rd32(E1000_PRC1522); 6264 adapter->stats.symerrs += rd32(E1000_SYMERRS); 6265 adapter->stats.sec += rd32(E1000_SEC); 6266 6267 mpc = rd32(E1000_MPC); 6268 adapter->stats.mpc += mpc; 6269 net_stats->rx_fifo_errors += mpc; 6270 adapter->stats.scc += rd32(E1000_SCC); 6271 adapter->stats.ecol += rd32(E1000_ECOL); 6272 adapter->stats.mcc += rd32(E1000_MCC); 6273 adapter->stats.latecol += rd32(E1000_LATECOL); 6274 adapter->stats.dc += rd32(E1000_DC); 6275 adapter->stats.rlec += rd32(E1000_RLEC); 6276 adapter->stats.xonrxc += rd32(E1000_XONRXC); 6277 adapter->stats.xontxc += rd32(E1000_XONTXC); 6278 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC); 6279 adapter->stats.xofftxc += rd32(E1000_XOFFTXC); 6280 adapter->stats.fcruc += rd32(E1000_FCRUC); 6281 adapter->stats.gptc += rd32(E1000_GPTC); 6282 adapter->stats.gotc += rd32(E1000_GOTCL); 6283 rd32(E1000_GOTCH); /* clear GOTCL */ 6284 adapter->stats.rnbc += rd32(E1000_RNBC); 6285 adapter->stats.ruc += rd32(E1000_RUC); 6286 adapter->stats.rfc += rd32(E1000_RFC); 6287 adapter->stats.rjc += rd32(E1000_RJC); 6288 adapter->stats.tor += rd32(E1000_TORH); 6289 adapter->stats.tot += rd32(E1000_TOTH); 6290 adapter->stats.tpr += rd32(E1000_TPR); 6291 6292 adapter->stats.ptc64 += rd32(E1000_PTC64); 6293 adapter->stats.ptc127 += rd32(E1000_PTC127); 6294 adapter->stats.ptc255 += rd32(E1000_PTC255); 6295 adapter->stats.ptc511 += rd32(E1000_PTC511); 6296 adapter->stats.ptc1023 += rd32(E1000_PTC1023); 6297 adapter->stats.ptc1522 += rd32(E1000_PTC1522); 6298 6299 adapter->stats.mptc += rd32(E1000_MPTC); 6300 adapter->stats.bptc += rd32(E1000_BPTC); 6301 6302 adapter->stats.tpt += rd32(E1000_TPT); 6303 adapter->stats.colc += rd32(E1000_COLC); 6304 6305 adapter->stats.algnerrc += rd32(E1000_ALGNERRC); 6306 /* read internal phy specific stats */ 6307 reg = rd32(E1000_CTRL_EXT); 6308 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) { 6309 adapter->stats.rxerrc += rd32(E1000_RXERRC); 6310 6311 /* this stat has invalid values on i210/i211 */ 6312 if ((hw->mac.type != e1000_i210) && 6313 (hw->mac.type != e1000_i211)) 6314 adapter->stats.tncrs += rd32(E1000_TNCRS); 6315 } 6316 6317 adapter->stats.tsctc += rd32(E1000_TSCTC); 6318 adapter->stats.tsctfc += rd32(E1000_TSCTFC); 6319 6320 adapter->stats.iac += rd32(E1000_IAC); 6321 adapter->stats.icrxoc += rd32(E1000_ICRXOC); 6322 adapter->stats.icrxptc += rd32(E1000_ICRXPTC); 6323 adapter->stats.icrxatc += rd32(E1000_ICRXATC); 6324 adapter->stats.ictxptc += rd32(E1000_ICTXPTC); 6325 adapter->stats.ictxatc += rd32(E1000_ICTXATC); 6326 adapter->stats.ictxqec += rd32(E1000_ICTXQEC); 6327 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC); 6328 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC); 6329 6330 /* Fill out the OS statistics structure */ 6331 net_stats->multicast = adapter->stats.mprc; 6332 net_stats->collisions = adapter->stats.colc; 6333 6334 /* Rx Errors */ 6335 6336 /* RLEC on some newer hardware can be incorrect so build 6337 * our own version based on RUC and ROC 6338 */ 6339 net_stats->rx_errors = adapter->stats.rxerrc + 6340 adapter->stats.crcerrs + adapter->stats.algnerrc + 6341 adapter->stats.ruc + adapter->stats.roc + 6342 adapter->stats.cexterr; 6343 net_stats->rx_length_errors = adapter->stats.ruc + 6344 adapter->stats.roc; 6345 net_stats->rx_crc_errors = adapter->stats.crcerrs; 6346 net_stats->rx_frame_errors = adapter->stats.algnerrc; 6347 net_stats->rx_missed_errors = adapter->stats.mpc; 6348 6349 /* Tx Errors */ 6350 net_stats->tx_errors = adapter->stats.ecol + 6351 adapter->stats.latecol; 6352 net_stats->tx_aborted_errors = adapter->stats.ecol; 6353 net_stats->tx_window_errors = adapter->stats.latecol; 6354 net_stats->tx_carrier_errors = adapter->stats.tncrs; 6355 6356 /* Tx Dropped needs to be maintained elsewhere */ 6357 6358 /* Management Stats */ 6359 adapter->stats.mgptc += rd32(E1000_MGTPTC); 6360 adapter->stats.mgprc += rd32(E1000_MGTPRC); 6361 adapter->stats.mgpdc += rd32(E1000_MGTPDC); 6362 6363 /* OS2BMC Stats */ 6364 reg = rd32(E1000_MANC); 6365 if (reg & E1000_MANC_EN_BMC2OS) { 6366 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC); 6367 adapter->stats.o2bspc += rd32(E1000_O2BSPC); 6368 adapter->stats.b2ospc += rd32(E1000_B2OSPC); 6369 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC); 6370 } 6371 } 6372 6373 static void igb_tsync_interrupt(struct igb_adapter *adapter) 6374 { 6375 struct e1000_hw *hw = &adapter->hw; 6376 struct ptp_clock_event event; 6377 struct timespec64 ts; 6378 u32 ack = 0, tsauxc, sec, nsec, tsicr = rd32(E1000_TSICR); 6379 6380 if (tsicr & TSINTR_SYS_WRAP) { 6381 event.type = PTP_CLOCK_PPS; 6382 if (adapter->ptp_caps.pps) 6383 ptp_clock_event(adapter->ptp_clock, &event); 6384 ack |= TSINTR_SYS_WRAP; 6385 } 6386 6387 if (tsicr & E1000_TSICR_TXTS) { 6388 /* retrieve hardware timestamp */ 6389 schedule_work(&adapter->ptp_tx_work); 6390 ack |= E1000_TSICR_TXTS; 6391 } 6392 6393 if (tsicr & TSINTR_TT0) { 6394 spin_lock(&adapter->tmreg_lock); 6395 ts = timespec64_add(adapter->perout[0].start, 6396 adapter->perout[0].period); 6397 /* u32 conversion of tv_sec is safe until y2106 */ 6398 wr32(E1000_TRGTTIML0, ts.tv_nsec); 6399 wr32(E1000_TRGTTIMH0, (u32)ts.tv_sec); 6400 tsauxc = rd32(E1000_TSAUXC); 6401 tsauxc |= TSAUXC_EN_TT0; 6402 wr32(E1000_TSAUXC, tsauxc); 6403 adapter->perout[0].start = ts; 6404 spin_unlock(&adapter->tmreg_lock); 6405 ack |= TSINTR_TT0; 6406 } 6407 6408 if (tsicr & TSINTR_TT1) { 6409 spin_lock(&adapter->tmreg_lock); 6410 ts = timespec64_add(adapter->perout[1].start, 6411 adapter->perout[1].period); 6412 wr32(E1000_TRGTTIML1, ts.tv_nsec); 6413 wr32(E1000_TRGTTIMH1, (u32)ts.tv_sec); 6414 tsauxc = rd32(E1000_TSAUXC); 6415 tsauxc |= TSAUXC_EN_TT1; 6416 wr32(E1000_TSAUXC, tsauxc); 6417 adapter->perout[1].start = ts; 6418 spin_unlock(&adapter->tmreg_lock); 6419 ack |= TSINTR_TT1; 6420 } 6421 6422 if (tsicr & TSINTR_AUTT0) { 6423 nsec = rd32(E1000_AUXSTMPL0); 6424 sec = rd32(E1000_AUXSTMPH0); 6425 event.type = PTP_CLOCK_EXTTS; 6426 event.index = 0; 6427 event.timestamp = sec * 1000000000ULL + nsec; 6428 ptp_clock_event(adapter->ptp_clock, &event); 6429 ack |= TSINTR_AUTT0; 6430 } 6431 6432 if (tsicr & TSINTR_AUTT1) { 6433 nsec = rd32(E1000_AUXSTMPL1); 6434 sec = rd32(E1000_AUXSTMPH1); 6435 event.type = PTP_CLOCK_EXTTS; 6436 event.index = 1; 6437 event.timestamp = sec * 1000000000ULL + nsec; 6438 ptp_clock_event(adapter->ptp_clock, &event); 6439 ack |= TSINTR_AUTT1; 6440 } 6441 6442 /* acknowledge the interrupts */ 6443 wr32(E1000_TSICR, ack); 6444 } 6445 6446 static irqreturn_t igb_msix_other(int irq, void *data) 6447 { 6448 struct igb_adapter *adapter = data; 6449 struct e1000_hw *hw = &adapter->hw; 6450 u32 icr = rd32(E1000_ICR); 6451 /* reading ICR causes bit 31 of EICR to be cleared */ 6452 6453 if (icr & E1000_ICR_DRSTA) 6454 schedule_work(&adapter->reset_task); 6455 6456 if (icr & E1000_ICR_DOUTSYNC) { 6457 /* HW is reporting DMA is out of sync */ 6458 adapter->stats.doosync++; 6459 /* The DMA Out of Sync is also indication of a spoof event 6460 * in IOV mode. Check the Wrong VM Behavior register to 6461 * see if it is really a spoof event. 6462 */ 6463 igb_check_wvbr(adapter); 6464 } 6465 6466 /* Check for a mailbox event */ 6467 if (icr & E1000_ICR_VMMB) 6468 igb_msg_task(adapter); 6469 6470 if (icr & E1000_ICR_LSC) { 6471 hw->mac.get_link_status = 1; 6472 /* guard against interrupt when we're going down */ 6473 if (!test_bit(__IGB_DOWN, &adapter->state)) 6474 mod_timer(&adapter->watchdog_timer, jiffies + 1); 6475 } 6476 6477 if (icr & E1000_ICR_TS) 6478 igb_tsync_interrupt(adapter); 6479 6480 wr32(E1000_EIMS, adapter->eims_other); 6481 6482 return IRQ_HANDLED; 6483 } 6484 6485 static void igb_write_itr(struct igb_q_vector *q_vector) 6486 { 6487 struct igb_adapter *adapter = q_vector->adapter; 6488 u32 itr_val = q_vector->itr_val & 0x7FFC; 6489 6490 if (!q_vector->set_itr) 6491 return; 6492 6493 if (!itr_val) 6494 itr_val = 0x4; 6495 6496 if (adapter->hw.mac.type == e1000_82575) 6497 itr_val |= itr_val << 16; 6498 else 6499 itr_val |= E1000_EITR_CNT_IGNR; 6500 6501 writel(itr_val, q_vector->itr_register); 6502 q_vector->set_itr = 0; 6503 } 6504 6505 static irqreturn_t igb_msix_ring(int irq, void *data) 6506 { 6507 struct igb_q_vector *q_vector = data; 6508 6509 /* Write the ITR value calculated from the previous interrupt. */ 6510 igb_write_itr(q_vector); 6511 6512 napi_schedule(&q_vector->napi); 6513 6514 return IRQ_HANDLED; 6515 } 6516 6517 #ifdef CONFIG_IGB_DCA 6518 static void igb_update_tx_dca(struct igb_adapter *adapter, 6519 struct igb_ring *tx_ring, 6520 int cpu) 6521 { 6522 struct e1000_hw *hw = &adapter->hw; 6523 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu); 6524 6525 if (hw->mac.type != e1000_82575) 6526 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT; 6527 6528 /* We can enable relaxed ordering for reads, but not writes when 6529 * DCA is enabled. This is due to a known issue in some chipsets 6530 * which will cause the DCA tag to be cleared. 6531 */ 6532 txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN | 6533 E1000_DCA_TXCTRL_DATA_RRO_EN | 6534 E1000_DCA_TXCTRL_DESC_DCA_EN; 6535 6536 wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl); 6537 } 6538 6539 static void igb_update_rx_dca(struct igb_adapter *adapter, 6540 struct igb_ring *rx_ring, 6541 int cpu) 6542 { 6543 struct e1000_hw *hw = &adapter->hw; 6544 u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu); 6545 6546 if (hw->mac.type != e1000_82575) 6547 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT; 6548 6549 /* We can enable relaxed ordering for reads, but not writes when 6550 * DCA is enabled. This is due to a known issue in some chipsets 6551 * which will cause the DCA tag to be cleared. 6552 */ 6553 rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN | 6554 E1000_DCA_RXCTRL_DESC_DCA_EN; 6555 6556 wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl); 6557 } 6558 6559 static void igb_update_dca(struct igb_q_vector *q_vector) 6560 { 6561 struct igb_adapter *adapter = q_vector->adapter; 6562 int cpu = get_cpu(); 6563 6564 if (q_vector->cpu == cpu) 6565 goto out_no_update; 6566 6567 if (q_vector->tx.ring) 6568 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu); 6569 6570 if (q_vector->rx.ring) 6571 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu); 6572 6573 q_vector->cpu = cpu; 6574 out_no_update: 6575 put_cpu(); 6576 } 6577 6578 static void igb_setup_dca(struct igb_adapter *adapter) 6579 { 6580 struct e1000_hw *hw = &adapter->hw; 6581 int i; 6582 6583 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED)) 6584 return; 6585 6586 /* Always use CB2 mode, difference is masked in the CB driver. */ 6587 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2); 6588 6589 for (i = 0; i < adapter->num_q_vectors; i++) { 6590 adapter->q_vector[i]->cpu = -1; 6591 igb_update_dca(adapter->q_vector[i]); 6592 } 6593 } 6594 6595 static int __igb_notify_dca(struct device *dev, void *data) 6596 { 6597 struct net_device *netdev = dev_get_drvdata(dev); 6598 struct igb_adapter *adapter = netdev_priv(netdev); 6599 struct pci_dev *pdev = adapter->pdev; 6600 struct e1000_hw *hw = &adapter->hw; 6601 unsigned long event = *(unsigned long *)data; 6602 6603 switch (event) { 6604 case DCA_PROVIDER_ADD: 6605 /* if already enabled, don't do it again */ 6606 if (adapter->flags & IGB_FLAG_DCA_ENABLED) 6607 break; 6608 if (dca_add_requester(dev) == 0) { 6609 adapter->flags |= IGB_FLAG_DCA_ENABLED; 6610 dev_info(&pdev->dev, "DCA enabled\n"); 6611 igb_setup_dca(adapter); 6612 break; 6613 } 6614 /* Fall Through since DCA is disabled. */ 6615 case DCA_PROVIDER_REMOVE: 6616 if (adapter->flags & IGB_FLAG_DCA_ENABLED) { 6617 /* without this a class_device is left 6618 * hanging around in the sysfs model 6619 */ 6620 dca_remove_requester(dev); 6621 dev_info(&pdev->dev, "DCA disabled\n"); 6622 adapter->flags &= ~IGB_FLAG_DCA_ENABLED; 6623 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); 6624 } 6625 break; 6626 } 6627 6628 return 0; 6629 } 6630 6631 static int igb_notify_dca(struct notifier_block *nb, unsigned long event, 6632 void *p) 6633 { 6634 int ret_val; 6635 6636 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event, 6637 __igb_notify_dca); 6638 6639 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 6640 } 6641 #endif /* CONFIG_IGB_DCA */ 6642 6643 #ifdef CONFIG_PCI_IOV 6644 static int igb_vf_configure(struct igb_adapter *adapter, int vf) 6645 { 6646 unsigned char mac_addr[ETH_ALEN]; 6647 6648 eth_zero_addr(mac_addr); 6649 igb_set_vf_mac(adapter, vf, mac_addr); 6650 6651 /* By default spoof check is enabled for all VFs */ 6652 adapter->vf_data[vf].spoofchk_enabled = true; 6653 6654 /* By default VFs are not trusted */ 6655 adapter->vf_data[vf].trusted = false; 6656 6657 return 0; 6658 } 6659 6660 #endif 6661 static void igb_ping_all_vfs(struct igb_adapter *adapter) 6662 { 6663 struct e1000_hw *hw = &adapter->hw; 6664 u32 ping; 6665 int i; 6666 6667 for (i = 0 ; i < adapter->vfs_allocated_count; i++) { 6668 ping = E1000_PF_CONTROL_MSG; 6669 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS) 6670 ping |= E1000_VT_MSGTYPE_CTS; 6671 igb_write_mbx(hw, &ping, 1, i); 6672 } 6673 } 6674 6675 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) 6676 { 6677 struct e1000_hw *hw = &adapter->hw; 6678 u32 vmolr = rd32(E1000_VMOLR(vf)); 6679 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 6680 6681 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC | 6682 IGB_VF_FLAG_MULTI_PROMISC); 6683 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); 6684 6685 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) { 6686 vmolr |= E1000_VMOLR_MPME; 6687 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC; 6688 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST; 6689 } else { 6690 /* if we have hashes and we are clearing a multicast promisc 6691 * flag we need to write the hashes to the MTA as this step 6692 * was previously skipped 6693 */ 6694 if (vf_data->num_vf_mc_hashes > 30) { 6695 vmolr |= E1000_VMOLR_MPME; 6696 } else if (vf_data->num_vf_mc_hashes) { 6697 int j; 6698 6699 vmolr |= E1000_VMOLR_ROMPE; 6700 for (j = 0; j < vf_data->num_vf_mc_hashes; j++) 6701 igb_mta_set(hw, vf_data->vf_mc_hashes[j]); 6702 } 6703 } 6704 6705 wr32(E1000_VMOLR(vf), vmolr); 6706 6707 /* there are flags left unprocessed, likely not supported */ 6708 if (*msgbuf & E1000_VT_MSGINFO_MASK) 6709 return -EINVAL; 6710 6711 return 0; 6712 } 6713 6714 static int igb_set_vf_multicasts(struct igb_adapter *adapter, 6715 u32 *msgbuf, u32 vf) 6716 { 6717 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT; 6718 u16 *hash_list = (u16 *)&msgbuf[1]; 6719 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 6720 int i; 6721 6722 /* salt away the number of multicast addresses assigned 6723 * to this VF for later use to restore when the PF multi cast 6724 * list changes 6725 */ 6726 vf_data->num_vf_mc_hashes = n; 6727 6728 /* only up to 30 hash values supported */ 6729 if (n > 30) 6730 n = 30; 6731 6732 /* store the hashes for later use */ 6733 for (i = 0; i < n; i++) 6734 vf_data->vf_mc_hashes[i] = hash_list[i]; 6735 6736 /* Flush and reset the mta with the new values */ 6737 igb_set_rx_mode(adapter->netdev); 6738 6739 return 0; 6740 } 6741 6742 static void igb_restore_vf_multicasts(struct igb_adapter *adapter) 6743 { 6744 struct e1000_hw *hw = &adapter->hw; 6745 struct vf_data_storage *vf_data; 6746 int i, j; 6747 6748 for (i = 0; i < adapter->vfs_allocated_count; i++) { 6749 u32 vmolr = rd32(E1000_VMOLR(i)); 6750 6751 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); 6752 6753 vf_data = &adapter->vf_data[i]; 6754 6755 if ((vf_data->num_vf_mc_hashes > 30) || 6756 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) { 6757 vmolr |= E1000_VMOLR_MPME; 6758 } else if (vf_data->num_vf_mc_hashes) { 6759 vmolr |= E1000_VMOLR_ROMPE; 6760 for (j = 0; j < vf_data->num_vf_mc_hashes; j++) 6761 igb_mta_set(hw, vf_data->vf_mc_hashes[j]); 6762 } 6763 wr32(E1000_VMOLR(i), vmolr); 6764 } 6765 } 6766 6767 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf) 6768 { 6769 struct e1000_hw *hw = &adapter->hw; 6770 u32 pool_mask, vlvf_mask, i; 6771 6772 /* create mask for VF and other pools */ 6773 pool_mask = E1000_VLVF_POOLSEL_MASK; 6774 vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf); 6775 6776 /* drop PF from pool bits */ 6777 pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT + 6778 adapter->vfs_allocated_count); 6779 6780 /* Find the vlan filter for this id */ 6781 for (i = E1000_VLVF_ARRAY_SIZE; i--;) { 6782 u32 vlvf = rd32(E1000_VLVF(i)); 6783 u32 vfta_mask, vid, vfta; 6784 6785 /* remove the vf from the pool */ 6786 if (!(vlvf & vlvf_mask)) 6787 continue; 6788 6789 /* clear out bit from VLVF */ 6790 vlvf ^= vlvf_mask; 6791 6792 /* if other pools are present, just remove ourselves */ 6793 if (vlvf & pool_mask) 6794 goto update_vlvfb; 6795 6796 /* if PF is present, leave VFTA */ 6797 if (vlvf & E1000_VLVF_POOLSEL_MASK) 6798 goto update_vlvf; 6799 6800 vid = vlvf & E1000_VLVF_VLANID_MASK; 6801 vfta_mask = BIT(vid % 32); 6802 6803 /* clear bit from VFTA */ 6804 vfta = adapter->shadow_vfta[vid / 32]; 6805 if (vfta & vfta_mask) 6806 hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask); 6807 update_vlvf: 6808 /* clear pool selection enable */ 6809 if (adapter->flags & IGB_FLAG_VLAN_PROMISC) 6810 vlvf &= E1000_VLVF_POOLSEL_MASK; 6811 else 6812 vlvf = 0; 6813 update_vlvfb: 6814 /* clear pool bits */ 6815 wr32(E1000_VLVF(i), vlvf); 6816 } 6817 } 6818 6819 static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan) 6820 { 6821 u32 vlvf; 6822 int idx; 6823 6824 /* short cut the special case */ 6825 if (vlan == 0) 6826 return 0; 6827 6828 /* Search for the VLAN id in the VLVF entries */ 6829 for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) { 6830 vlvf = rd32(E1000_VLVF(idx)); 6831 if ((vlvf & VLAN_VID_MASK) == vlan) 6832 break; 6833 } 6834 6835 return idx; 6836 } 6837 6838 static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid) 6839 { 6840 struct e1000_hw *hw = &adapter->hw; 6841 u32 bits, pf_id; 6842 int idx; 6843 6844 idx = igb_find_vlvf_entry(hw, vid); 6845 if (!idx) 6846 return; 6847 6848 /* See if any other pools are set for this VLAN filter 6849 * entry other than the PF. 6850 */ 6851 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; 6852 bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK; 6853 bits &= rd32(E1000_VLVF(idx)); 6854 6855 /* Disable the filter so this falls into the default pool. */ 6856 if (!bits) { 6857 if (adapter->flags & IGB_FLAG_VLAN_PROMISC) 6858 wr32(E1000_VLVF(idx), BIT(pf_id)); 6859 else 6860 wr32(E1000_VLVF(idx), 0); 6861 } 6862 } 6863 6864 static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid, 6865 bool add, u32 vf) 6866 { 6867 int pf_id = adapter->vfs_allocated_count; 6868 struct e1000_hw *hw = &adapter->hw; 6869 int err; 6870 6871 /* If VLAN overlaps with one the PF is currently monitoring make 6872 * sure that we are able to allocate a VLVF entry. This may be 6873 * redundant but it guarantees PF will maintain visibility to 6874 * the VLAN. 6875 */ 6876 if (add && test_bit(vid, adapter->active_vlans)) { 6877 err = igb_vfta_set(hw, vid, pf_id, true, false); 6878 if (err) 6879 return err; 6880 } 6881 6882 err = igb_vfta_set(hw, vid, vf, add, false); 6883 6884 if (add && !err) 6885 return err; 6886 6887 /* If we failed to add the VF VLAN or we are removing the VF VLAN 6888 * we may need to drop the PF pool bit in order to allow us to free 6889 * up the VLVF resources. 6890 */ 6891 if (test_bit(vid, adapter->active_vlans) || 6892 (adapter->flags & IGB_FLAG_VLAN_PROMISC)) 6893 igb_update_pf_vlvf(adapter, vid); 6894 6895 return err; 6896 } 6897 6898 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf) 6899 { 6900 struct e1000_hw *hw = &adapter->hw; 6901 6902 if (vid) 6903 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT)); 6904 else 6905 wr32(E1000_VMVIR(vf), 0); 6906 } 6907 6908 static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf, 6909 u16 vlan, u8 qos) 6910 { 6911 int err; 6912 6913 err = igb_set_vf_vlan(adapter, vlan, true, vf); 6914 if (err) 6915 return err; 6916 6917 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf); 6918 igb_set_vmolr(adapter, vf, !vlan); 6919 6920 /* revoke access to previous VLAN */ 6921 if (vlan != adapter->vf_data[vf].pf_vlan) 6922 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan, 6923 false, vf); 6924 6925 adapter->vf_data[vf].pf_vlan = vlan; 6926 adapter->vf_data[vf].pf_qos = qos; 6927 igb_set_vf_vlan_strip(adapter, vf, true); 6928 dev_info(&adapter->pdev->dev, 6929 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf); 6930 if (test_bit(__IGB_DOWN, &adapter->state)) { 6931 dev_warn(&adapter->pdev->dev, 6932 "The VF VLAN has been set, but the PF device is not up.\n"); 6933 dev_warn(&adapter->pdev->dev, 6934 "Bring the PF device up before attempting to use the VF device.\n"); 6935 } 6936 6937 return err; 6938 } 6939 6940 static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf) 6941 { 6942 /* Restore tagless access via VLAN 0 */ 6943 igb_set_vf_vlan(adapter, 0, true, vf); 6944 6945 igb_set_vmvir(adapter, 0, vf); 6946 igb_set_vmolr(adapter, vf, true); 6947 6948 /* Remove any PF assigned VLAN */ 6949 if (adapter->vf_data[vf].pf_vlan) 6950 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan, 6951 false, vf); 6952 6953 adapter->vf_data[vf].pf_vlan = 0; 6954 adapter->vf_data[vf].pf_qos = 0; 6955 igb_set_vf_vlan_strip(adapter, vf, false); 6956 6957 return 0; 6958 } 6959 6960 static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf, 6961 u16 vlan, u8 qos, __be16 vlan_proto) 6962 { 6963 struct igb_adapter *adapter = netdev_priv(netdev); 6964 6965 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7)) 6966 return -EINVAL; 6967 6968 if (vlan_proto != htons(ETH_P_8021Q)) 6969 return -EPROTONOSUPPORT; 6970 6971 return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) : 6972 igb_disable_port_vlan(adapter, vf); 6973 } 6974 6975 static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) 6976 { 6977 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT; 6978 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK); 6979 int ret; 6980 6981 if (adapter->vf_data[vf].pf_vlan) 6982 return -1; 6983 6984 /* VLAN 0 is a special case, don't allow it to be removed */ 6985 if (!vid && !add) 6986 return 0; 6987 6988 ret = igb_set_vf_vlan(adapter, vid, !!add, vf); 6989 if (!ret) 6990 igb_set_vf_vlan_strip(adapter, vf, !!vid); 6991 return ret; 6992 } 6993 6994 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf) 6995 { 6996 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 6997 6998 /* clear flags - except flag that indicates PF has set the MAC */ 6999 vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC; 7000 vf_data->last_nack = jiffies; 7001 7002 /* reset vlans for device */ 7003 igb_clear_vf_vfta(adapter, vf); 7004 igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf); 7005 igb_set_vmvir(adapter, vf_data->pf_vlan | 7006 (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf); 7007 igb_set_vmolr(adapter, vf, !vf_data->pf_vlan); 7008 igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan)); 7009 7010 /* reset multicast table array for vf */ 7011 adapter->vf_data[vf].num_vf_mc_hashes = 0; 7012 7013 /* Flush and reset the mta with the new values */ 7014 igb_set_rx_mode(adapter->netdev); 7015 } 7016 7017 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf) 7018 { 7019 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; 7020 7021 /* clear mac address as we were hotplug removed/added */ 7022 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC)) 7023 eth_zero_addr(vf_mac); 7024 7025 /* process remaining reset events */ 7026 igb_vf_reset(adapter, vf); 7027 } 7028 7029 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf) 7030 { 7031 struct e1000_hw *hw = &adapter->hw; 7032 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; 7033 u32 reg, msgbuf[3]; 7034 u8 *addr = (u8 *)(&msgbuf[1]); 7035 7036 /* process all the same items cleared in a function level reset */ 7037 igb_vf_reset(adapter, vf); 7038 7039 /* set vf mac address */ 7040 igb_set_vf_mac(adapter, vf, vf_mac); 7041 7042 /* enable transmit and receive for vf */ 7043 reg = rd32(E1000_VFTE); 7044 wr32(E1000_VFTE, reg | BIT(vf)); 7045 reg = rd32(E1000_VFRE); 7046 wr32(E1000_VFRE, reg | BIT(vf)); 7047 7048 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS; 7049 7050 /* reply to reset with ack and vf mac address */ 7051 if (!is_zero_ether_addr(vf_mac)) { 7052 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK; 7053 memcpy(addr, vf_mac, ETH_ALEN); 7054 } else { 7055 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK; 7056 } 7057 igb_write_mbx(hw, msgbuf, 3, vf); 7058 } 7059 7060 static void igb_flush_mac_table(struct igb_adapter *adapter) 7061 { 7062 struct e1000_hw *hw = &adapter->hw; 7063 int i; 7064 7065 for (i = 0; i < hw->mac.rar_entry_count; i++) { 7066 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE; 7067 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 7068 adapter->mac_table[i].queue = 0; 7069 igb_rar_set_index(adapter, i); 7070 } 7071 } 7072 7073 static int igb_available_rars(struct igb_adapter *adapter, u8 queue) 7074 { 7075 struct e1000_hw *hw = &adapter->hw; 7076 /* do not count rar entries reserved for VFs MAC addresses */ 7077 int rar_entries = hw->mac.rar_entry_count - 7078 adapter->vfs_allocated_count; 7079 int i, count = 0; 7080 7081 for (i = 0; i < rar_entries; i++) { 7082 /* do not count default entries */ 7083 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) 7084 continue; 7085 7086 /* do not count "in use" entries for different queues */ 7087 if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) && 7088 (adapter->mac_table[i].queue != queue)) 7089 continue; 7090 7091 count++; 7092 } 7093 7094 return count; 7095 } 7096 7097 /* Set default MAC address for the PF in the first RAR entry */ 7098 static void igb_set_default_mac_filter(struct igb_adapter *adapter) 7099 { 7100 struct igb_mac_addr *mac_table = &adapter->mac_table[0]; 7101 7102 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); 7103 mac_table->queue = adapter->vfs_allocated_count; 7104 mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE; 7105 7106 igb_rar_set_index(adapter, 0); 7107 } 7108 7109 /* If the filter to be added and an already existing filter express 7110 * the same address and address type, it should be possible to only 7111 * override the other configurations, for example the queue to steer 7112 * traffic. 7113 */ 7114 static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry, 7115 const u8 *addr, const u8 flags) 7116 { 7117 if (!(entry->state & IGB_MAC_STATE_IN_USE)) 7118 return true; 7119 7120 if ((entry->state & IGB_MAC_STATE_SRC_ADDR) != 7121 (flags & IGB_MAC_STATE_SRC_ADDR)) 7122 return false; 7123 7124 if (!ether_addr_equal(addr, entry->addr)) 7125 return false; 7126 7127 return true; 7128 } 7129 7130 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 7131 * 'flags' is used to indicate what kind of match is made, match is by 7132 * default for the destination address, if matching by source address 7133 * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used. 7134 */ 7135 static int igb_add_mac_filter_flags(struct igb_adapter *adapter, 7136 const u8 *addr, const u8 queue, 7137 const u8 flags) 7138 { 7139 struct e1000_hw *hw = &adapter->hw; 7140 int rar_entries = hw->mac.rar_entry_count - 7141 adapter->vfs_allocated_count; 7142 int i; 7143 7144 if (is_zero_ether_addr(addr)) 7145 return -EINVAL; 7146 7147 /* Search for the first empty entry in the MAC table. 7148 * Do not touch entries at the end of the table reserved for the VF MAC 7149 * addresses. 7150 */ 7151 for (i = 0; i < rar_entries; i++) { 7152 if (!igb_mac_entry_can_be_used(&adapter->mac_table[i], 7153 addr, flags)) 7154 continue; 7155 7156 ether_addr_copy(adapter->mac_table[i].addr, addr); 7157 adapter->mac_table[i].queue = queue; 7158 adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags; 7159 7160 igb_rar_set_index(adapter, i); 7161 return i; 7162 } 7163 7164 return -ENOSPC; 7165 } 7166 7167 static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr, 7168 const u8 queue) 7169 { 7170 return igb_add_mac_filter_flags(adapter, addr, queue, 0); 7171 } 7172 7173 /* Remove a MAC filter for 'addr' directing matching traffic to 7174 * 'queue', 'flags' is used to indicate what kind of match need to be 7175 * removed, match is by default for the destination address, if 7176 * matching by source address is to be removed the flag 7177 * IGB_MAC_STATE_SRC_ADDR can be used. 7178 */ 7179 static int igb_del_mac_filter_flags(struct igb_adapter *adapter, 7180 const u8 *addr, const u8 queue, 7181 const u8 flags) 7182 { 7183 struct e1000_hw *hw = &adapter->hw; 7184 int rar_entries = hw->mac.rar_entry_count - 7185 adapter->vfs_allocated_count; 7186 int i; 7187 7188 if (is_zero_ether_addr(addr)) 7189 return -EINVAL; 7190 7191 /* Search for matching entry in the MAC table based on given address 7192 * and queue. Do not touch entries at the end of the table reserved 7193 * for the VF MAC addresses. 7194 */ 7195 for (i = 0; i < rar_entries; i++) { 7196 if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)) 7197 continue; 7198 if ((adapter->mac_table[i].state & flags) != flags) 7199 continue; 7200 if (adapter->mac_table[i].queue != queue) 7201 continue; 7202 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 7203 continue; 7204 7205 /* When a filter for the default address is "deleted", 7206 * we return it to its initial configuration 7207 */ 7208 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) { 7209 adapter->mac_table[i].state = 7210 IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE; 7211 adapter->mac_table[i].queue = 7212 adapter->vfs_allocated_count; 7213 } else { 7214 adapter->mac_table[i].state = 0; 7215 adapter->mac_table[i].queue = 0; 7216 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 7217 } 7218 7219 igb_rar_set_index(adapter, i); 7220 return 0; 7221 } 7222 7223 return -ENOENT; 7224 } 7225 7226 static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr, 7227 const u8 queue) 7228 { 7229 return igb_del_mac_filter_flags(adapter, addr, queue, 0); 7230 } 7231 7232 int igb_add_mac_steering_filter(struct igb_adapter *adapter, 7233 const u8 *addr, u8 queue, u8 flags) 7234 { 7235 struct e1000_hw *hw = &adapter->hw; 7236 7237 /* In theory, this should be supported on 82575 as well, but 7238 * that part wasn't easily accessible during development. 7239 */ 7240 if (hw->mac.type != e1000_i210) 7241 return -EOPNOTSUPP; 7242 7243 return igb_add_mac_filter_flags(adapter, addr, queue, 7244 IGB_MAC_STATE_QUEUE_STEERING | flags); 7245 } 7246 7247 int igb_del_mac_steering_filter(struct igb_adapter *adapter, 7248 const u8 *addr, u8 queue, u8 flags) 7249 { 7250 return igb_del_mac_filter_flags(adapter, addr, queue, 7251 IGB_MAC_STATE_QUEUE_STEERING | flags); 7252 } 7253 7254 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr) 7255 { 7256 struct igb_adapter *adapter = netdev_priv(netdev); 7257 int ret; 7258 7259 ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count); 7260 7261 return min_t(int, ret, 0); 7262 } 7263 7264 static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr) 7265 { 7266 struct igb_adapter *adapter = netdev_priv(netdev); 7267 7268 igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count); 7269 7270 return 0; 7271 } 7272 7273 static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf, 7274 const u32 info, const u8 *addr) 7275 { 7276 struct pci_dev *pdev = adapter->pdev; 7277 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7278 struct list_head *pos; 7279 struct vf_mac_filter *entry = NULL; 7280 int ret = 0; 7281 7282 switch (info) { 7283 case E1000_VF_MAC_FILTER_CLR: 7284 /* remove all unicast MAC filters related to the current VF */ 7285 list_for_each(pos, &adapter->vf_macs.l) { 7286 entry = list_entry(pos, struct vf_mac_filter, l); 7287 if (entry->vf == vf) { 7288 entry->vf = -1; 7289 entry->free = true; 7290 igb_del_mac_filter(adapter, entry->vf_mac, vf); 7291 } 7292 } 7293 break; 7294 case E1000_VF_MAC_FILTER_ADD: 7295 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) && 7296 !vf_data->trusted) { 7297 dev_warn(&pdev->dev, 7298 "VF %d requested MAC filter but is administratively denied\n", 7299 vf); 7300 return -EINVAL; 7301 } 7302 if (!is_valid_ether_addr(addr)) { 7303 dev_warn(&pdev->dev, 7304 "VF %d attempted to set invalid MAC filter\n", 7305 vf); 7306 return -EINVAL; 7307 } 7308 7309 /* try to find empty slot in the list */ 7310 list_for_each(pos, &adapter->vf_macs.l) { 7311 entry = list_entry(pos, struct vf_mac_filter, l); 7312 if (entry->free) 7313 break; 7314 } 7315 7316 if (entry && entry->free) { 7317 entry->free = false; 7318 entry->vf = vf; 7319 ether_addr_copy(entry->vf_mac, addr); 7320 7321 ret = igb_add_mac_filter(adapter, addr, vf); 7322 ret = min_t(int, ret, 0); 7323 } else { 7324 ret = -ENOSPC; 7325 } 7326 7327 if (ret == -ENOSPC) 7328 dev_warn(&pdev->dev, 7329 "VF %d has requested MAC filter but there is no space for it\n", 7330 vf); 7331 break; 7332 default: 7333 ret = -EINVAL; 7334 break; 7335 } 7336 7337 return ret; 7338 } 7339 7340 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf) 7341 { 7342 struct pci_dev *pdev = adapter->pdev; 7343 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7344 u32 info = msg[0] & E1000_VT_MSGINFO_MASK; 7345 7346 /* The VF MAC Address is stored in a packed array of bytes 7347 * starting at the second 32 bit word of the msg array 7348 */ 7349 unsigned char *addr = (unsigned char *)&msg[1]; 7350 int ret = 0; 7351 7352 if (!info) { 7353 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) && 7354 !vf_data->trusted) { 7355 dev_warn(&pdev->dev, 7356 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n", 7357 vf); 7358 return -EINVAL; 7359 } 7360 7361 if (!is_valid_ether_addr(addr)) { 7362 dev_warn(&pdev->dev, 7363 "VF %d attempted to set invalid MAC\n", 7364 vf); 7365 return -EINVAL; 7366 } 7367 7368 ret = igb_set_vf_mac(adapter, vf, addr); 7369 } else { 7370 ret = igb_set_vf_mac_filter(adapter, vf, info, addr); 7371 } 7372 7373 return ret; 7374 } 7375 7376 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf) 7377 { 7378 struct e1000_hw *hw = &adapter->hw; 7379 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7380 u32 msg = E1000_VT_MSGTYPE_NACK; 7381 7382 /* if device isn't clear to send it shouldn't be reading either */ 7383 if (!(vf_data->flags & IGB_VF_FLAG_CTS) && 7384 time_after(jiffies, vf_data->last_nack + (2 * HZ))) { 7385 igb_write_mbx(hw, &msg, 1, vf); 7386 vf_data->last_nack = jiffies; 7387 } 7388 } 7389 7390 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf) 7391 { 7392 struct pci_dev *pdev = adapter->pdev; 7393 u32 msgbuf[E1000_VFMAILBOX_SIZE]; 7394 struct e1000_hw *hw = &adapter->hw; 7395 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 7396 s32 retval; 7397 7398 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false); 7399 7400 if (retval) { 7401 /* if receive failed revoke VF CTS stats and restart init */ 7402 dev_err(&pdev->dev, "Error receiving message from VF\n"); 7403 vf_data->flags &= ~IGB_VF_FLAG_CTS; 7404 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) 7405 goto unlock; 7406 goto out; 7407 } 7408 7409 /* this is a message we already processed, do nothing */ 7410 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) 7411 goto unlock; 7412 7413 /* until the vf completes a reset it should not be 7414 * allowed to start any configuration. 7415 */ 7416 if (msgbuf[0] == E1000_VF_RESET) { 7417 /* unlocks mailbox */ 7418 igb_vf_reset_msg(adapter, vf); 7419 return; 7420 } 7421 7422 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) { 7423 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) 7424 goto unlock; 7425 retval = -1; 7426 goto out; 7427 } 7428 7429 switch ((msgbuf[0] & 0xFFFF)) { 7430 case E1000_VF_SET_MAC_ADDR: 7431 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf); 7432 break; 7433 case E1000_VF_SET_PROMISC: 7434 retval = igb_set_vf_promisc(adapter, msgbuf, vf); 7435 break; 7436 case E1000_VF_SET_MULTICAST: 7437 retval = igb_set_vf_multicasts(adapter, msgbuf, vf); 7438 break; 7439 case E1000_VF_SET_LPE: 7440 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf); 7441 break; 7442 case E1000_VF_SET_VLAN: 7443 retval = -1; 7444 if (vf_data->pf_vlan) 7445 dev_warn(&pdev->dev, 7446 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n", 7447 vf); 7448 else 7449 retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf); 7450 break; 7451 default: 7452 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]); 7453 retval = -1; 7454 break; 7455 } 7456 7457 msgbuf[0] |= E1000_VT_MSGTYPE_CTS; 7458 out: 7459 /* notify the VF of the results of what it sent us */ 7460 if (retval) 7461 msgbuf[0] |= E1000_VT_MSGTYPE_NACK; 7462 else 7463 msgbuf[0] |= E1000_VT_MSGTYPE_ACK; 7464 7465 /* unlocks mailbox */ 7466 igb_write_mbx(hw, msgbuf, 1, vf); 7467 return; 7468 7469 unlock: 7470 igb_unlock_mbx(hw, vf); 7471 } 7472 7473 static void igb_msg_task(struct igb_adapter *adapter) 7474 { 7475 struct e1000_hw *hw = &adapter->hw; 7476 u32 vf; 7477 7478 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) { 7479 /* process any reset requests */ 7480 if (!igb_check_for_rst(hw, vf)) 7481 igb_vf_reset_event(adapter, vf); 7482 7483 /* process any messages pending */ 7484 if (!igb_check_for_msg(hw, vf)) 7485 igb_rcv_msg_from_vf(adapter, vf); 7486 7487 /* process any acks */ 7488 if (!igb_check_for_ack(hw, vf)) 7489 igb_rcv_ack_from_vf(adapter, vf); 7490 } 7491 } 7492 7493 /** 7494 * igb_set_uta - Set unicast filter table address 7495 * @adapter: board private structure 7496 * @set: boolean indicating if we are setting or clearing bits 7497 * 7498 * The unicast table address is a register array of 32-bit registers. 7499 * The table is meant to be used in a way similar to how the MTA is used 7500 * however due to certain limitations in the hardware it is necessary to 7501 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous 7502 * enable bit to allow vlan tag stripping when promiscuous mode is enabled 7503 **/ 7504 static void igb_set_uta(struct igb_adapter *adapter, bool set) 7505 { 7506 struct e1000_hw *hw = &adapter->hw; 7507 u32 uta = set ? ~0 : 0; 7508 int i; 7509 7510 /* we only need to do this if VMDq is enabled */ 7511 if (!adapter->vfs_allocated_count) 7512 return; 7513 7514 for (i = hw->mac.uta_reg_count; i--;) 7515 array_wr32(E1000_UTA, i, uta); 7516 } 7517 7518 /** 7519 * igb_intr_msi - Interrupt Handler 7520 * @irq: interrupt number 7521 * @data: pointer to a network interface device structure 7522 **/ 7523 static irqreturn_t igb_intr_msi(int irq, void *data) 7524 { 7525 struct igb_adapter *adapter = data; 7526 struct igb_q_vector *q_vector = adapter->q_vector[0]; 7527 struct e1000_hw *hw = &adapter->hw; 7528 /* read ICR disables interrupts using IAM */ 7529 u32 icr = rd32(E1000_ICR); 7530 7531 igb_write_itr(q_vector); 7532 7533 if (icr & E1000_ICR_DRSTA) 7534 schedule_work(&adapter->reset_task); 7535 7536 if (icr & E1000_ICR_DOUTSYNC) { 7537 /* HW is reporting DMA is out of sync */ 7538 adapter->stats.doosync++; 7539 } 7540 7541 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 7542 hw->mac.get_link_status = 1; 7543 if (!test_bit(__IGB_DOWN, &adapter->state)) 7544 mod_timer(&adapter->watchdog_timer, jiffies + 1); 7545 } 7546 7547 if (icr & E1000_ICR_TS) 7548 igb_tsync_interrupt(adapter); 7549 7550 napi_schedule(&q_vector->napi); 7551 7552 return IRQ_HANDLED; 7553 } 7554 7555 /** 7556 * igb_intr - Legacy Interrupt Handler 7557 * @irq: interrupt number 7558 * @data: pointer to a network interface device structure 7559 **/ 7560 static irqreturn_t igb_intr(int irq, void *data) 7561 { 7562 struct igb_adapter *adapter = data; 7563 struct igb_q_vector *q_vector = adapter->q_vector[0]; 7564 struct e1000_hw *hw = &adapter->hw; 7565 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 7566 * need for the IMC write 7567 */ 7568 u32 icr = rd32(E1000_ICR); 7569 7570 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 7571 * not set, then the adapter didn't send an interrupt 7572 */ 7573 if (!(icr & E1000_ICR_INT_ASSERTED)) 7574 return IRQ_NONE; 7575 7576 igb_write_itr(q_vector); 7577 7578 if (icr & E1000_ICR_DRSTA) 7579 schedule_work(&adapter->reset_task); 7580 7581 if (icr & E1000_ICR_DOUTSYNC) { 7582 /* HW is reporting DMA is out of sync */ 7583 adapter->stats.doosync++; 7584 } 7585 7586 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 7587 hw->mac.get_link_status = 1; 7588 /* guard against interrupt when we're going down */ 7589 if (!test_bit(__IGB_DOWN, &adapter->state)) 7590 mod_timer(&adapter->watchdog_timer, jiffies + 1); 7591 } 7592 7593 if (icr & E1000_ICR_TS) 7594 igb_tsync_interrupt(adapter); 7595 7596 napi_schedule(&q_vector->napi); 7597 7598 return IRQ_HANDLED; 7599 } 7600 7601 static void igb_ring_irq_enable(struct igb_q_vector *q_vector) 7602 { 7603 struct igb_adapter *adapter = q_vector->adapter; 7604 struct e1000_hw *hw = &adapter->hw; 7605 7606 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 7607 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 7608 if ((adapter->num_q_vectors == 1) && !adapter->vf_data) 7609 igb_set_itr(q_vector); 7610 else 7611 igb_update_ring_itr(q_vector); 7612 } 7613 7614 if (!test_bit(__IGB_DOWN, &adapter->state)) { 7615 if (adapter->flags & IGB_FLAG_HAS_MSIX) 7616 wr32(E1000_EIMS, q_vector->eims_value); 7617 else 7618 igb_irq_enable(adapter); 7619 } 7620 } 7621 7622 /** 7623 * igb_poll - NAPI Rx polling callback 7624 * @napi: napi polling structure 7625 * @budget: count of how many packets we should handle 7626 **/ 7627 static int igb_poll(struct napi_struct *napi, int budget) 7628 { 7629 struct igb_q_vector *q_vector = container_of(napi, 7630 struct igb_q_vector, 7631 napi); 7632 bool clean_complete = true; 7633 int work_done = 0; 7634 7635 #ifdef CONFIG_IGB_DCA 7636 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED) 7637 igb_update_dca(q_vector); 7638 #endif 7639 if (q_vector->tx.ring) 7640 clean_complete = igb_clean_tx_irq(q_vector, budget); 7641 7642 if (q_vector->rx.ring) { 7643 int cleaned = igb_clean_rx_irq(q_vector, budget); 7644 7645 work_done += cleaned; 7646 if (cleaned >= budget) 7647 clean_complete = false; 7648 } 7649 7650 /* If all work not completed, return budget and keep polling */ 7651 if (!clean_complete) 7652 return budget; 7653 7654 /* If not enough Rx work done, exit the polling mode */ 7655 napi_complete_done(napi, work_done); 7656 igb_ring_irq_enable(q_vector); 7657 7658 return 0; 7659 } 7660 7661 /** 7662 * igb_clean_tx_irq - Reclaim resources after transmit completes 7663 * @q_vector: pointer to q_vector containing needed info 7664 * @napi_budget: Used to determine if we are in netpoll 7665 * 7666 * returns true if ring is completely cleaned 7667 **/ 7668 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget) 7669 { 7670 struct igb_adapter *adapter = q_vector->adapter; 7671 struct igb_ring *tx_ring = q_vector->tx.ring; 7672 struct igb_tx_buffer *tx_buffer; 7673 union e1000_adv_tx_desc *tx_desc; 7674 unsigned int total_bytes = 0, total_packets = 0; 7675 unsigned int budget = q_vector->tx.work_limit; 7676 unsigned int i = tx_ring->next_to_clean; 7677 7678 if (test_bit(__IGB_DOWN, &adapter->state)) 7679 return true; 7680 7681 tx_buffer = &tx_ring->tx_buffer_info[i]; 7682 tx_desc = IGB_TX_DESC(tx_ring, i); 7683 i -= tx_ring->count; 7684 7685 do { 7686 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 7687 7688 /* if next_to_watch is not set then there is no work pending */ 7689 if (!eop_desc) 7690 break; 7691 7692 /* prevent any other reads prior to eop_desc */ 7693 smp_rmb(); 7694 7695 /* if DD is not set pending work has not been completed */ 7696 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD))) 7697 break; 7698 7699 /* clear next_to_watch to prevent false hangs */ 7700 tx_buffer->next_to_watch = NULL; 7701 7702 /* update the statistics for this packet */ 7703 total_bytes += tx_buffer->bytecount; 7704 total_packets += tx_buffer->gso_segs; 7705 7706 /* free the skb */ 7707 napi_consume_skb(tx_buffer->skb, napi_budget); 7708 7709 /* unmap skb header data */ 7710 dma_unmap_single(tx_ring->dev, 7711 dma_unmap_addr(tx_buffer, dma), 7712 dma_unmap_len(tx_buffer, len), 7713 DMA_TO_DEVICE); 7714 7715 /* clear tx_buffer data */ 7716 dma_unmap_len_set(tx_buffer, len, 0); 7717 7718 /* clear last DMA location and unmap remaining buffers */ 7719 while (tx_desc != eop_desc) { 7720 tx_buffer++; 7721 tx_desc++; 7722 i++; 7723 if (unlikely(!i)) { 7724 i -= tx_ring->count; 7725 tx_buffer = tx_ring->tx_buffer_info; 7726 tx_desc = IGB_TX_DESC(tx_ring, 0); 7727 } 7728 7729 /* unmap any remaining paged data */ 7730 if (dma_unmap_len(tx_buffer, len)) { 7731 dma_unmap_page(tx_ring->dev, 7732 dma_unmap_addr(tx_buffer, dma), 7733 dma_unmap_len(tx_buffer, len), 7734 DMA_TO_DEVICE); 7735 dma_unmap_len_set(tx_buffer, len, 0); 7736 } 7737 } 7738 7739 /* move us one more past the eop_desc for start of next pkt */ 7740 tx_buffer++; 7741 tx_desc++; 7742 i++; 7743 if (unlikely(!i)) { 7744 i -= tx_ring->count; 7745 tx_buffer = tx_ring->tx_buffer_info; 7746 tx_desc = IGB_TX_DESC(tx_ring, 0); 7747 } 7748 7749 /* issue prefetch for next Tx descriptor */ 7750 prefetch(tx_desc); 7751 7752 /* update budget accounting */ 7753 budget--; 7754 } while (likely(budget)); 7755 7756 netdev_tx_completed_queue(txring_txq(tx_ring), 7757 total_packets, total_bytes); 7758 i += tx_ring->count; 7759 tx_ring->next_to_clean = i; 7760 u64_stats_update_begin(&tx_ring->tx_syncp); 7761 tx_ring->tx_stats.bytes += total_bytes; 7762 tx_ring->tx_stats.packets += total_packets; 7763 u64_stats_update_end(&tx_ring->tx_syncp); 7764 q_vector->tx.total_bytes += total_bytes; 7765 q_vector->tx.total_packets += total_packets; 7766 7767 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 7768 struct e1000_hw *hw = &adapter->hw; 7769 7770 /* Detect a transmit hang in hardware, this serializes the 7771 * check with the clearing of time_stamp and movement of i 7772 */ 7773 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 7774 if (tx_buffer->next_to_watch && 7775 time_after(jiffies, tx_buffer->time_stamp + 7776 (adapter->tx_timeout_factor * HZ)) && 7777 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) { 7778 7779 /* detected Tx unit hang */ 7780 dev_err(tx_ring->dev, 7781 "Detected Tx Unit Hang\n" 7782 " Tx Queue <%d>\n" 7783 " TDH <%x>\n" 7784 " TDT <%x>\n" 7785 " next_to_use <%x>\n" 7786 " next_to_clean <%x>\n" 7787 "buffer_info[next_to_clean]\n" 7788 " time_stamp <%lx>\n" 7789 " next_to_watch <%p>\n" 7790 " jiffies <%lx>\n" 7791 " desc.status <%x>\n", 7792 tx_ring->queue_index, 7793 rd32(E1000_TDH(tx_ring->reg_idx)), 7794 readl(tx_ring->tail), 7795 tx_ring->next_to_use, 7796 tx_ring->next_to_clean, 7797 tx_buffer->time_stamp, 7798 tx_buffer->next_to_watch, 7799 jiffies, 7800 tx_buffer->next_to_watch->wb.status); 7801 netif_stop_subqueue(tx_ring->netdev, 7802 tx_ring->queue_index); 7803 7804 /* we are about to reset, no point in enabling stuff */ 7805 return true; 7806 } 7807 } 7808 7809 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 7810 if (unlikely(total_packets && 7811 netif_carrier_ok(tx_ring->netdev) && 7812 igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 7813 /* Make sure that anybody stopping the queue after this 7814 * sees the new next_to_clean. 7815 */ 7816 smp_mb(); 7817 if (__netif_subqueue_stopped(tx_ring->netdev, 7818 tx_ring->queue_index) && 7819 !(test_bit(__IGB_DOWN, &adapter->state))) { 7820 netif_wake_subqueue(tx_ring->netdev, 7821 tx_ring->queue_index); 7822 7823 u64_stats_update_begin(&tx_ring->tx_syncp); 7824 tx_ring->tx_stats.restart_queue++; 7825 u64_stats_update_end(&tx_ring->tx_syncp); 7826 } 7827 } 7828 7829 return !!budget; 7830 } 7831 7832 /** 7833 * igb_reuse_rx_page - page flip buffer and store it back on the ring 7834 * @rx_ring: rx descriptor ring to store buffers on 7835 * @old_buff: donor buffer to have page reused 7836 * 7837 * Synchronizes page for reuse by the adapter 7838 **/ 7839 static void igb_reuse_rx_page(struct igb_ring *rx_ring, 7840 struct igb_rx_buffer *old_buff) 7841 { 7842 struct igb_rx_buffer *new_buff; 7843 u16 nta = rx_ring->next_to_alloc; 7844 7845 new_buff = &rx_ring->rx_buffer_info[nta]; 7846 7847 /* update, and store next to alloc */ 7848 nta++; 7849 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 7850 7851 /* Transfer page from old buffer to new buffer. 7852 * Move each member individually to avoid possible store 7853 * forwarding stalls. 7854 */ 7855 new_buff->dma = old_buff->dma; 7856 new_buff->page = old_buff->page; 7857 new_buff->page_offset = old_buff->page_offset; 7858 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 7859 } 7860 7861 static inline bool igb_page_is_reserved(struct page *page) 7862 { 7863 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 7864 } 7865 7866 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer) 7867 { 7868 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 7869 struct page *page = rx_buffer->page; 7870 7871 /* avoid re-using remote pages */ 7872 if (unlikely(igb_page_is_reserved(page))) 7873 return false; 7874 7875 #if (PAGE_SIZE < 8192) 7876 /* if we are only owner of page we can reuse it */ 7877 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 7878 return false; 7879 #else 7880 #define IGB_LAST_OFFSET \ 7881 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048) 7882 7883 if (rx_buffer->page_offset > IGB_LAST_OFFSET) 7884 return false; 7885 #endif 7886 7887 /* If we have drained the page fragment pool we need to update 7888 * the pagecnt_bias and page count so that we fully restock the 7889 * number of references the driver holds. 7890 */ 7891 if (unlikely(!pagecnt_bias)) { 7892 page_ref_add(page, USHRT_MAX); 7893 rx_buffer->pagecnt_bias = USHRT_MAX; 7894 } 7895 7896 return true; 7897 } 7898 7899 /** 7900 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff 7901 * @rx_ring: rx descriptor ring to transact packets on 7902 * @rx_buffer: buffer containing page to add 7903 * @skb: sk_buff to place the data into 7904 * @size: size of buffer to be added 7905 * 7906 * This function will add the data contained in rx_buffer->page to the skb. 7907 **/ 7908 static void igb_add_rx_frag(struct igb_ring *rx_ring, 7909 struct igb_rx_buffer *rx_buffer, 7910 struct sk_buff *skb, 7911 unsigned int size) 7912 { 7913 #if (PAGE_SIZE < 8192) 7914 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; 7915 #else 7916 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 7917 SKB_DATA_ALIGN(IGB_SKB_PAD + size) : 7918 SKB_DATA_ALIGN(size); 7919 #endif 7920 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 7921 rx_buffer->page_offset, size, truesize); 7922 #if (PAGE_SIZE < 8192) 7923 rx_buffer->page_offset ^= truesize; 7924 #else 7925 rx_buffer->page_offset += truesize; 7926 #endif 7927 } 7928 7929 static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, 7930 struct igb_rx_buffer *rx_buffer, 7931 union e1000_adv_rx_desc *rx_desc, 7932 unsigned int size) 7933 { 7934 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 7935 #if (PAGE_SIZE < 8192) 7936 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; 7937 #else 7938 unsigned int truesize = SKB_DATA_ALIGN(size); 7939 #endif 7940 unsigned int headlen; 7941 struct sk_buff *skb; 7942 7943 /* prefetch first cache line of first page */ 7944 prefetch(va); 7945 #if L1_CACHE_BYTES < 128 7946 prefetch(va + L1_CACHE_BYTES); 7947 #endif 7948 7949 /* allocate a skb to store the frags */ 7950 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN); 7951 if (unlikely(!skb)) 7952 return NULL; 7953 7954 if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) { 7955 igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 7956 va += IGB_TS_HDR_LEN; 7957 size -= IGB_TS_HDR_LEN; 7958 } 7959 7960 /* Determine available headroom for copy */ 7961 headlen = size; 7962 if (headlen > IGB_RX_HDR_LEN) 7963 headlen = eth_get_headlen(va, IGB_RX_HDR_LEN); 7964 7965 /* align pull length to size of long to optimize memcpy performance */ 7966 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 7967 7968 /* update all of the pointers */ 7969 size -= headlen; 7970 if (size) { 7971 skb_add_rx_frag(skb, 0, rx_buffer->page, 7972 (va + headlen) - page_address(rx_buffer->page), 7973 size, truesize); 7974 #if (PAGE_SIZE < 8192) 7975 rx_buffer->page_offset ^= truesize; 7976 #else 7977 rx_buffer->page_offset += truesize; 7978 #endif 7979 } else { 7980 rx_buffer->pagecnt_bias++; 7981 } 7982 7983 return skb; 7984 } 7985 7986 static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, 7987 struct igb_rx_buffer *rx_buffer, 7988 union e1000_adv_rx_desc *rx_desc, 7989 unsigned int size) 7990 { 7991 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 7992 #if (PAGE_SIZE < 8192) 7993 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; 7994 #else 7995 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 7996 SKB_DATA_ALIGN(IGB_SKB_PAD + size); 7997 #endif 7998 struct sk_buff *skb; 7999 8000 /* prefetch first cache line of first page */ 8001 prefetch(va); 8002 #if L1_CACHE_BYTES < 128 8003 prefetch(va + L1_CACHE_BYTES); 8004 #endif 8005 8006 /* build an skb around the page buffer */ 8007 skb = build_skb(va - IGB_SKB_PAD, truesize); 8008 if (unlikely(!skb)) 8009 return NULL; 8010 8011 /* update pointers within the skb to store the data */ 8012 skb_reserve(skb, IGB_SKB_PAD); 8013 __skb_put(skb, size); 8014 8015 /* pull timestamp out of packet data */ 8016 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { 8017 igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb); 8018 __skb_pull(skb, IGB_TS_HDR_LEN); 8019 } 8020 8021 /* update buffer offset */ 8022 #if (PAGE_SIZE < 8192) 8023 rx_buffer->page_offset ^= truesize; 8024 #else 8025 rx_buffer->page_offset += truesize; 8026 #endif 8027 8028 return skb; 8029 } 8030 8031 static inline void igb_rx_checksum(struct igb_ring *ring, 8032 union e1000_adv_rx_desc *rx_desc, 8033 struct sk_buff *skb) 8034 { 8035 skb_checksum_none_assert(skb); 8036 8037 /* Ignore Checksum bit is set */ 8038 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM)) 8039 return; 8040 8041 /* Rx checksum disabled via ethtool */ 8042 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 8043 return; 8044 8045 /* TCP/UDP checksum error bit is set */ 8046 if (igb_test_staterr(rx_desc, 8047 E1000_RXDEXT_STATERR_TCPE | 8048 E1000_RXDEXT_STATERR_IPE)) { 8049 /* work around errata with sctp packets where the TCPE aka 8050 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 8051 * packets, (aka let the stack check the crc32c) 8052 */ 8053 if (!((skb->len == 60) && 8054 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 8055 u64_stats_update_begin(&ring->rx_syncp); 8056 ring->rx_stats.csum_err++; 8057 u64_stats_update_end(&ring->rx_syncp); 8058 } 8059 /* let the stack verify checksum errors */ 8060 return; 8061 } 8062 /* It must be a TCP or UDP packet with a valid checksum */ 8063 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS | 8064 E1000_RXD_STAT_UDPCS)) 8065 skb->ip_summed = CHECKSUM_UNNECESSARY; 8066 8067 dev_dbg(ring->dev, "cksum success: bits %08X\n", 8068 le32_to_cpu(rx_desc->wb.upper.status_error)); 8069 } 8070 8071 static inline void igb_rx_hash(struct igb_ring *ring, 8072 union e1000_adv_rx_desc *rx_desc, 8073 struct sk_buff *skb) 8074 { 8075 if (ring->netdev->features & NETIF_F_RXHASH) 8076 skb_set_hash(skb, 8077 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 8078 PKT_HASH_TYPE_L3); 8079 } 8080 8081 /** 8082 * igb_is_non_eop - process handling of non-EOP buffers 8083 * @rx_ring: Rx ring being processed 8084 * @rx_desc: Rx descriptor for current buffer 8085 * @skb: current socket buffer containing buffer in progress 8086 * 8087 * This function updates next to clean. If the buffer is an EOP buffer 8088 * this function exits returning false, otherwise it will place the 8089 * sk_buff in the next buffer to be chained and return true indicating 8090 * that this is in fact a non-EOP buffer. 8091 **/ 8092 static bool igb_is_non_eop(struct igb_ring *rx_ring, 8093 union e1000_adv_rx_desc *rx_desc) 8094 { 8095 u32 ntc = rx_ring->next_to_clean + 1; 8096 8097 /* fetch, update, and store next to clean */ 8098 ntc = (ntc < rx_ring->count) ? ntc : 0; 8099 rx_ring->next_to_clean = ntc; 8100 8101 prefetch(IGB_RX_DESC(rx_ring, ntc)); 8102 8103 if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP))) 8104 return false; 8105 8106 return true; 8107 } 8108 8109 /** 8110 * igb_cleanup_headers - Correct corrupted or empty headers 8111 * @rx_ring: rx descriptor ring packet is being transacted on 8112 * @rx_desc: pointer to the EOP Rx descriptor 8113 * @skb: pointer to current skb being fixed 8114 * 8115 * Address the case where we are pulling data in on pages only 8116 * and as such no data is present in the skb header. 8117 * 8118 * In addition if skb is not at least 60 bytes we need to pad it so that 8119 * it is large enough to qualify as a valid Ethernet frame. 8120 * 8121 * Returns true if an error was encountered and skb was freed. 8122 **/ 8123 static bool igb_cleanup_headers(struct igb_ring *rx_ring, 8124 union e1000_adv_rx_desc *rx_desc, 8125 struct sk_buff *skb) 8126 { 8127 if (unlikely((igb_test_staterr(rx_desc, 8128 E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) { 8129 struct net_device *netdev = rx_ring->netdev; 8130 if (!(netdev->features & NETIF_F_RXALL)) { 8131 dev_kfree_skb_any(skb); 8132 return true; 8133 } 8134 } 8135 8136 /* if eth_skb_pad returns an error the skb was freed */ 8137 if (eth_skb_pad(skb)) 8138 return true; 8139 8140 return false; 8141 } 8142 8143 /** 8144 * igb_process_skb_fields - Populate skb header fields from Rx descriptor 8145 * @rx_ring: rx descriptor ring packet is being transacted on 8146 * @rx_desc: pointer to the EOP Rx descriptor 8147 * @skb: pointer to current skb being populated 8148 * 8149 * This function checks the ring, descriptor, and packet information in 8150 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 8151 * other fields within the skb. 8152 **/ 8153 static void igb_process_skb_fields(struct igb_ring *rx_ring, 8154 union e1000_adv_rx_desc *rx_desc, 8155 struct sk_buff *skb) 8156 { 8157 struct net_device *dev = rx_ring->netdev; 8158 8159 igb_rx_hash(rx_ring, rx_desc, skb); 8160 8161 igb_rx_checksum(rx_ring, rx_desc, skb); 8162 8163 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) && 8164 !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) 8165 igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb); 8166 8167 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && 8168 igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) { 8169 u16 vid; 8170 8171 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) && 8172 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) 8173 vid = be16_to_cpu(rx_desc->wb.upper.vlan); 8174 else 8175 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 8176 8177 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 8178 } 8179 8180 skb_record_rx_queue(skb, rx_ring->queue_index); 8181 8182 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 8183 } 8184 8185 static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring, 8186 const unsigned int size) 8187 { 8188 struct igb_rx_buffer *rx_buffer; 8189 8190 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 8191 prefetchw(rx_buffer->page); 8192 8193 /* we are reusing so sync this buffer for CPU use */ 8194 dma_sync_single_range_for_cpu(rx_ring->dev, 8195 rx_buffer->dma, 8196 rx_buffer->page_offset, 8197 size, 8198 DMA_FROM_DEVICE); 8199 8200 rx_buffer->pagecnt_bias--; 8201 8202 return rx_buffer; 8203 } 8204 8205 static void igb_put_rx_buffer(struct igb_ring *rx_ring, 8206 struct igb_rx_buffer *rx_buffer) 8207 { 8208 if (igb_can_reuse_rx_page(rx_buffer)) { 8209 /* hand second half of page back to the ring */ 8210 igb_reuse_rx_page(rx_ring, rx_buffer); 8211 } else { 8212 /* We are not reusing the buffer so unmap it and free 8213 * any references we are holding to it 8214 */ 8215 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 8216 igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 8217 IGB_RX_DMA_ATTR); 8218 __page_frag_cache_drain(rx_buffer->page, 8219 rx_buffer->pagecnt_bias); 8220 } 8221 8222 /* clear contents of rx_buffer */ 8223 rx_buffer->page = NULL; 8224 } 8225 8226 static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) 8227 { 8228 struct igb_ring *rx_ring = q_vector->rx.ring; 8229 struct sk_buff *skb = rx_ring->skb; 8230 unsigned int total_bytes = 0, total_packets = 0; 8231 u16 cleaned_count = igb_desc_unused(rx_ring); 8232 8233 while (likely(total_packets < budget)) { 8234 union e1000_adv_rx_desc *rx_desc; 8235 struct igb_rx_buffer *rx_buffer; 8236 unsigned int size; 8237 8238 /* return some buffers to hardware, one at a time is too slow */ 8239 if (cleaned_count >= IGB_RX_BUFFER_WRITE) { 8240 igb_alloc_rx_buffers(rx_ring, cleaned_count); 8241 cleaned_count = 0; 8242 } 8243 8244 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean); 8245 size = le16_to_cpu(rx_desc->wb.upper.length); 8246 if (!size) 8247 break; 8248 8249 /* This memory barrier is needed to keep us from reading 8250 * any other fields out of the rx_desc until we know the 8251 * descriptor has been written back 8252 */ 8253 dma_rmb(); 8254 8255 rx_buffer = igb_get_rx_buffer(rx_ring, size); 8256 8257 /* retrieve a buffer from the ring */ 8258 if (skb) 8259 igb_add_rx_frag(rx_ring, rx_buffer, skb, size); 8260 else if (ring_uses_build_skb(rx_ring)) 8261 skb = igb_build_skb(rx_ring, rx_buffer, rx_desc, size); 8262 else 8263 skb = igb_construct_skb(rx_ring, rx_buffer, 8264 rx_desc, size); 8265 8266 /* exit if we failed to retrieve a buffer */ 8267 if (!skb) { 8268 rx_ring->rx_stats.alloc_failed++; 8269 rx_buffer->pagecnt_bias++; 8270 break; 8271 } 8272 8273 igb_put_rx_buffer(rx_ring, rx_buffer); 8274 cleaned_count++; 8275 8276 /* fetch next buffer in frame if non-eop */ 8277 if (igb_is_non_eop(rx_ring, rx_desc)) 8278 continue; 8279 8280 /* verify the packet layout is correct */ 8281 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) { 8282 skb = NULL; 8283 continue; 8284 } 8285 8286 /* probably a little skewed due to removing CRC */ 8287 total_bytes += skb->len; 8288 8289 /* populate checksum, timestamp, VLAN, and protocol */ 8290 igb_process_skb_fields(rx_ring, rx_desc, skb); 8291 8292 napi_gro_receive(&q_vector->napi, skb); 8293 8294 /* reset skb pointer */ 8295 skb = NULL; 8296 8297 /* update budget accounting */ 8298 total_packets++; 8299 } 8300 8301 /* place incomplete frames back on ring for completion */ 8302 rx_ring->skb = skb; 8303 8304 u64_stats_update_begin(&rx_ring->rx_syncp); 8305 rx_ring->rx_stats.packets += total_packets; 8306 rx_ring->rx_stats.bytes += total_bytes; 8307 u64_stats_update_end(&rx_ring->rx_syncp); 8308 q_vector->rx.total_packets += total_packets; 8309 q_vector->rx.total_bytes += total_bytes; 8310 8311 if (cleaned_count) 8312 igb_alloc_rx_buffers(rx_ring, cleaned_count); 8313 8314 return total_packets; 8315 } 8316 8317 static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring) 8318 { 8319 return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0; 8320 } 8321 8322 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, 8323 struct igb_rx_buffer *bi) 8324 { 8325 struct page *page = bi->page; 8326 dma_addr_t dma; 8327 8328 /* since we are recycling buffers we should seldom need to alloc */ 8329 if (likely(page)) 8330 return true; 8331 8332 /* alloc new page for storage */ 8333 page = dev_alloc_pages(igb_rx_pg_order(rx_ring)); 8334 if (unlikely(!page)) { 8335 rx_ring->rx_stats.alloc_failed++; 8336 return false; 8337 } 8338 8339 /* map page for use */ 8340 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 8341 igb_rx_pg_size(rx_ring), 8342 DMA_FROM_DEVICE, 8343 IGB_RX_DMA_ATTR); 8344 8345 /* if mapping failed free memory back to system since 8346 * there isn't much point in holding memory we can't use 8347 */ 8348 if (dma_mapping_error(rx_ring->dev, dma)) { 8349 __free_pages(page, igb_rx_pg_order(rx_ring)); 8350 8351 rx_ring->rx_stats.alloc_failed++; 8352 return false; 8353 } 8354 8355 bi->dma = dma; 8356 bi->page = page; 8357 bi->page_offset = igb_rx_offset(rx_ring); 8358 bi->pagecnt_bias = 1; 8359 8360 return true; 8361 } 8362 8363 /** 8364 * igb_alloc_rx_buffers - Replace used receive buffers; packet split 8365 * @adapter: address of board private structure 8366 **/ 8367 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count) 8368 { 8369 union e1000_adv_rx_desc *rx_desc; 8370 struct igb_rx_buffer *bi; 8371 u16 i = rx_ring->next_to_use; 8372 u16 bufsz; 8373 8374 /* nothing to do */ 8375 if (!cleaned_count) 8376 return; 8377 8378 rx_desc = IGB_RX_DESC(rx_ring, i); 8379 bi = &rx_ring->rx_buffer_info[i]; 8380 i -= rx_ring->count; 8381 8382 bufsz = igb_rx_bufsz(rx_ring); 8383 8384 do { 8385 if (!igb_alloc_mapped_page(rx_ring, bi)) 8386 break; 8387 8388 /* sync the buffer for use by the device */ 8389 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 8390 bi->page_offset, bufsz, 8391 DMA_FROM_DEVICE); 8392 8393 /* Refresh the desc even if buffer_addrs didn't change 8394 * because each write-back erases this info. 8395 */ 8396 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 8397 8398 rx_desc++; 8399 bi++; 8400 i++; 8401 if (unlikely(!i)) { 8402 rx_desc = IGB_RX_DESC(rx_ring, 0); 8403 bi = rx_ring->rx_buffer_info; 8404 i -= rx_ring->count; 8405 } 8406 8407 /* clear the length for the next_to_use descriptor */ 8408 rx_desc->wb.upper.length = 0; 8409 8410 cleaned_count--; 8411 } while (cleaned_count); 8412 8413 i += rx_ring->count; 8414 8415 if (rx_ring->next_to_use != i) { 8416 /* record the next descriptor to use */ 8417 rx_ring->next_to_use = i; 8418 8419 /* update next to alloc since we have filled the ring */ 8420 rx_ring->next_to_alloc = i; 8421 8422 /* Force memory writes to complete before letting h/w 8423 * know there are new descriptors to fetch. (Only 8424 * applicable for weak-ordered memory model archs, 8425 * such as IA-64). 8426 */ 8427 wmb(); 8428 writel(i, rx_ring->tail); 8429 } 8430 } 8431 8432 /** 8433 * igb_mii_ioctl - 8434 * @netdev: 8435 * @ifreq: 8436 * @cmd: 8437 **/ 8438 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 8439 { 8440 struct igb_adapter *adapter = netdev_priv(netdev); 8441 struct mii_ioctl_data *data = if_mii(ifr); 8442 8443 if (adapter->hw.phy.media_type != e1000_media_type_copper) 8444 return -EOPNOTSUPP; 8445 8446 switch (cmd) { 8447 case SIOCGMIIPHY: 8448 data->phy_id = adapter->hw.phy.addr; 8449 break; 8450 case SIOCGMIIREG: 8451 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F, 8452 &data->val_out)) 8453 return -EIO; 8454 break; 8455 case SIOCSMIIREG: 8456 default: 8457 return -EOPNOTSUPP; 8458 } 8459 return 0; 8460 } 8461 8462 /** 8463 * igb_ioctl - 8464 * @netdev: 8465 * @ifreq: 8466 * @cmd: 8467 **/ 8468 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 8469 { 8470 switch (cmd) { 8471 case SIOCGMIIPHY: 8472 case SIOCGMIIREG: 8473 case SIOCSMIIREG: 8474 return igb_mii_ioctl(netdev, ifr, cmd); 8475 case SIOCGHWTSTAMP: 8476 return igb_ptp_get_ts_config(netdev, ifr); 8477 case SIOCSHWTSTAMP: 8478 return igb_ptp_set_ts_config(netdev, ifr); 8479 default: 8480 return -EOPNOTSUPP; 8481 } 8482 } 8483 8484 void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value) 8485 { 8486 struct igb_adapter *adapter = hw->back; 8487 8488 pci_read_config_word(adapter->pdev, reg, value); 8489 } 8490 8491 void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value) 8492 { 8493 struct igb_adapter *adapter = hw->back; 8494 8495 pci_write_config_word(adapter->pdev, reg, *value); 8496 } 8497 8498 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) 8499 { 8500 struct igb_adapter *adapter = hw->back; 8501 8502 if (pcie_capability_read_word(adapter->pdev, reg, value)) 8503 return -E1000_ERR_CONFIG; 8504 8505 return 0; 8506 } 8507 8508 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) 8509 { 8510 struct igb_adapter *adapter = hw->back; 8511 8512 if (pcie_capability_write_word(adapter->pdev, reg, *value)) 8513 return -E1000_ERR_CONFIG; 8514 8515 return 0; 8516 } 8517 8518 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features) 8519 { 8520 struct igb_adapter *adapter = netdev_priv(netdev); 8521 struct e1000_hw *hw = &adapter->hw; 8522 u32 ctrl, rctl; 8523 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); 8524 8525 if (enable) { 8526 /* enable VLAN tag insert/strip */ 8527 ctrl = rd32(E1000_CTRL); 8528 ctrl |= E1000_CTRL_VME; 8529 wr32(E1000_CTRL, ctrl); 8530 8531 /* Disable CFI check */ 8532 rctl = rd32(E1000_RCTL); 8533 rctl &= ~E1000_RCTL_CFIEN; 8534 wr32(E1000_RCTL, rctl); 8535 } else { 8536 /* disable VLAN tag insert/strip */ 8537 ctrl = rd32(E1000_CTRL); 8538 ctrl &= ~E1000_CTRL_VME; 8539 wr32(E1000_CTRL, ctrl); 8540 } 8541 8542 igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable); 8543 } 8544 8545 static int igb_vlan_rx_add_vid(struct net_device *netdev, 8546 __be16 proto, u16 vid) 8547 { 8548 struct igb_adapter *adapter = netdev_priv(netdev); 8549 struct e1000_hw *hw = &adapter->hw; 8550 int pf_id = adapter->vfs_allocated_count; 8551 8552 /* add the filter since PF can receive vlans w/o entry in vlvf */ 8553 if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC)) 8554 igb_vfta_set(hw, vid, pf_id, true, !!vid); 8555 8556 set_bit(vid, adapter->active_vlans); 8557 8558 return 0; 8559 } 8560 8561 static int igb_vlan_rx_kill_vid(struct net_device *netdev, 8562 __be16 proto, u16 vid) 8563 { 8564 struct igb_adapter *adapter = netdev_priv(netdev); 8565 int pf_id = adapter->vfs_allocated_count; 8566 struct e1000_hw *hw = &adapter->hw; 8567 8568 /* remove VID from filter table */ 8569 if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC)) 8570 igb_vfta_set(hw, vid, pf_id, false, true); 8571 8572 clear_bit(vid, adapter->active_vlans); 8573 8574 return 0; 8575 } 8576 8577 static void igb_restore_vlan(struct igb_adapter *adapter) 8578 { 8579 u16 vid = 1; 8580 8581 igb_vlan_mode(adapter->netdev, adapter->netdev->features); 8582 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 8583 8584 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID) 8585 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 8586 } 8587 8588 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx) 8589 { 8590 struct pci_dev *pdev = adapter->pdev; 8591 struct e1000_mac_info *mac = &adapter->hw.mac; 8592 8593 mac->autoneg = 0; 8594 8595 /* Make sure dplx is at most 1 bit and lsb of speed is not set 8596 * for the switch() below to work 8597 */ 8598 if ((spd & 1) || (dplx & ~1)) 8599 goto err_inval; 8600 8601 /* Fiber NIC's only allow 1000 gbps Full duplex 8602 * and 100Mbps Full duplex for 100baseFx sfp 8603 */ 8604 if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 8605 switch (spd + dplx) { 8606 case SPEED_10 + DUPLEX_HALF: 8607 case SPEED_10 + DUPLEX_FULL: 8608 case SPEED_100 + DUPLEX_HALF: 8609 goto err_inval; 8610 default: 8611 break; 8612 } 8613 } 8614 8615 switch (spd + dplx) { 8616 case SPEED_10 + DUPLEX_HALF: 8617 mac->forced_speed_duplex = ADVERTISE_10_HALF; 8618 break; 8619 case SPEED_10 + DUPLEX_FULL: 8620 mac->forced_speed_duplex = ADVERTISE_10_FULL; 8621 break; 8622 case SPEED_100 + DUPLEX_HALF: 8623 mac->forced_speed_duplex = ADVERTISE_100_HALF; 8624 break; 8625 case SPEED_100 + DUPLEX_FULL: 8626 mac->forced_speed_duplex = ADVERTISE_100_FULL; 8627 break; 8628 case SPEED_1000 + DUPLEX_FULL: 8629 mac->autoneg = 1; 8630 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 8631 break; 8632 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 8633 default: 8634 goto err_inval; 8635 } 8636 8637 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 8638 adapter->hw.phy.mdix = AUTO_ALL_MODES; 8639 8640 return 0; 8641 8642 err_inval: 8643 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 8644 return -EINVAL; 8645 } 8646 8647 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake, 8648 bool runtime) 8649 { 8650 struct net_device *netdev = pci_get_drvdata(pdev); 8651 struct igb_adapter *adapter = netdev_priv(netdev); 8652 struct e1000_hw *hw = &adapter->hw; 8653 u32 ctrl, rctl, status; 8654 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol; 8655 #ifdef CONFIG_PM 8656 int retval = 0; 8657 #endif 8658 8659 rtnl_lock(); 8660 netif_device_detach(netdev); 8661 8662 if (netif_running(netdev)) 8663 __igb_close(netdev, true); 8664 8665 igb_ptp_suspend(adapter); 8666 8667 igb_clear_interrupt_scheme(adapter); 8668 rtnl_unlock(); 8669 8670 #ifdef CONFIG_PM 8671 retval = pci_save_state(pdev); 8672 if (retval) 8673 return retval; 8674 #endif 8675 8676 status = rd32(E1000_STATUS); 8677 if (status & E1000_STATUS_LU) 8678 wufc &= ~E1000_WUFC_LNKC; 8679 8680 if (wufc) { 8681 igb_setup_rctl(adapter); 8682 igb_set_rx_mode(netdev); 8683 8684 /* turn on all-multi mode if wake on multicast is enabled */ 8685 if (wufc & E1000_WUFC_MC) { 8686 rctl = rd32(E1000_RCTL); 8687 rctl |= E1000_RCTL_MPE; 8688 wr32(E1000_RCTL, rctl); 8689 } 8690 8691 ctrl = rd32(E1000_CTRL); 8692 /* advertise wake from D3Cold */ 8693 #define E1000_CTRL_ADVD3WUC 0x00100000 8694 /* phy power management enable */ 8695 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000 8696 ctrl |= E1000_CTRL_ADVD3WUC; 8697 wr32(E1000_CTRL, ctrl); 8698 8699 /* Allow time for pending master requests to run */ 8700 igb_disable_pcie_master(hw); 8701 8702 wr32(E1000_WUC, E1000_WUC_PME_EN); 8703 wr32(E1000_WUFC, wufc); 8704 } else { 8705 wr32(E1000_WUC, 0); 8706 wr32(E1000_WUFC, 0); 8707 } 8708 8709 *enable_wake = wufc || adapter->en_mng_pt; 8710 if (!*enable_wake) 8711 igb_power_down_link(adapter); 8712 else 8713 igb_power_up_link(adapter); 8714 8715 /* Release control of h/w to f/w. If f/w is AMT enabled, this 8716 * would have already happened in close and is redundant. 8717 */ 8718 igb_release_hw_control(adapter); 8719 8720 pci_disable_device(pdev); 8721 8722 return 0; 8723 } 8724 8725 static void igb_deliver_wake_packet(struct net_device *netdev) 8726 { 8727 struct igb_adapter *adapter = netdev_priv(netdev); 8728 struct e1000_hw *hw = &adapter->hw; 8729 struct sk_buff *skb; 8730 u32 wupl; 8731 8732 wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK; 8733 8734 /* WUPM stores only the first 128 bytes of the wake packet. 8735 * Read the packet only if we have the whole thing. 8736 */ 8737 if ((wupl == 0) || (wupl > E1000_WUPM_BYTES)) 8738 return; 8739 8740 skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES); 8741 if (!skb) 8742 return; 8743 8744 skb_put(skb, wupl); 8745 8746 /* Ensure reads are 32-bit aligned */ 8747 wupl = roundup(wupl, 4); 8748 8749 memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl); 8750 8751 skb->protocol = eth_type_trans(skb, netdev); 8752 netif_rx(skb); 8753 } 8754 8755 static int __maybe_unused igb_suspend(struct device *dev) 8756 { 8757 int retval; 8758 bool wake; 8759 struct pci_dev *pdev = to_pci_dev(dev); 8760 8761 retval = __igb_shutdown(pdev, &wake, 0); 8762 if (retval) 8763 return retval; 8764 8765 if (wake) { 8766 pci_prepare_to_sleep(pdev); 8767 } else { 8768 pci_wake_from_d3(pdev, false); 8769 pci_set_power_state(pdev, PCI_D3hot); 8770 } 8771 8772 return 0; 8773 } 8774 8775 static int __maybe_unused igb_resume(struct device *dev) 8776 { 8777 struct pci_dev *pdev = to_pci_dev(dev); 8778 struct net_device *netdev = pci_get_drvdata(pdev); 8779 struct igb_adapter *adapter = netdev_priv(netdev); 8780 struct e1000_hw *hw = &adapter->hw; 8781 u32 err, val; 8782 8783 pci_set_power_state(pdev, PCI_D0); 8784 pci_restore_state(pdev); 8785 pci_save_state(pdev); 8786 8787 if (!pci_device_is_present(pdev)) 8788 return -ENODEV; 8789 err = pci_enable_device_mem(pdev); 8790 if (err) { 8791 dev_err(&pdev->dev, 8792 "igb: Cannot enable PCI device from suspend\n"); 8793 return err; 8794 } 8795 pci_set_master(pdev); 8796 8797 pci_enable_wake(pdev, PCI_D3hot, 0); 8798 pci_enable_wake(pdev, PCI_D3cold, 0); 8799 8800 if (igb_init_interrupt_scheme(adapter, true)) { 8801 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 8802 return -ENOMEM; 8803 } 8804 8805 igb_reset(adapter); 8806 8807 /* let the f/w know that the h/w is now under the control of the 8808 * driver. 8809 */ 8810 igb_get_hw_control(adapter); 8811 8812 val = rd32(E1000_WUS); 8813 if (val & WAKE_PKT_WUS) 8814 igb_deliver_wake_packet(netdev); 8815 8816 wr32(E1000_WUS, ~0); 8817 8818 rtnl_lock(); 8819 if (!err && netif_running(netdev)) 8820 err = __igb_open(netdev, true); 8821 8822 if (!err) 8823 netif_device_attach(netdev); 8824 rtnl_unlock(); 8825 8826 return err; 8827 } 8828 8829 static int __maybe_unused igb_runtime_idle(struct device *dev) 8830 { 8831 struct pci_dev *pdev = to_pci_dev(dev); 8832 struct net_device *netdev = pci_get_drvdata(pdev); 8833 struct igb_adapter *adapter = netdev_priv(netdev); 8834 8835 if (!igb_has_link(adapter)) 8836 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 8837 8838 return -EBUSY; 8839 } 8840 8841 static int __maybe_unused igb_runtime_suspend(struct device *dev) 8842 { 8843 struct pci_dev *pdev = to_pci_dev(dev); 8844 int retval; 8845 bool wake; 8846 8847 retval = __igb_shutdown(pdev, &wake, 1); 8848 if (retval) 8849 return retval; 8850 8851 if (wake) { 8852 pci_prepare_to_sleep(pdev); 8853 } else { 8854 pci_wake_from_d3(pdev, false); 8855 pci_set_power_state(pdev, PCI_D3hot); 8856 } 8857 8858 return 0; 8859 } 8860 8861 static int __maybe_unused igb_runtime_resume(struct device *dev) 8862 { 8863 return igb_resume(dev); 8864 } 8865 8866 static void igb_shutdown(struct pci_dev *pdev) 8867 { 8868 bool wake; 8869 8870 __igb_shutdown(pdev, &wake, 0); 8871 8872 if (system_state == SYSTEM_POWER_OFF) { 8873 pci_wake_from_d3(pdev, wake); 8874 pci_set_power_state(pdev, PCI_D3hot); 8875 } 8876 } 8877 8878 #ifdef CONFIG_PCI_IOV 8879 static int igb_sriov_reinit(struct pci_dev *dev) 8880 { 8881 struct net_device *netdev = pci_get_drvdata(dev); 8882 struct igb_adapter *adapter = netdev_priv(netdev); 8883 struct pci_dev *pdev = adapter->pdev; 8884 8885 rtnl_lock(); 8886 8887 if (netif_running(netdev)) 8888 igb_close(netdev); 8889 else 8890 igb_reset(adapter); 8891 8892 igb_clear_interrupt_scheme(adapter); 8893 8894 igb_init_queue_configuration(adapter); 8895 8896 if (igb_init_interrupt_scheme(adapter, true)) { 8897 rtnl_unlock(); 8898 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 8899 return -ENOMEM; 8900 } 8901 8902 if (netif_running(netdev)) 8903 igb_open(netdev); 8904 8905 rtnl_unlock(); 8906 8907 return 0; 8908 } 8909 8910 static int igb_pci_disable_sriov(struct pci_dev *dev) 8911 { 8912 int err = igb_disable_sriov(dev); 8913 8914 if (!err) 8915 err = igb_sriov_reinit(dev); 8916 8917 return err; 8918 } 8919 8920 static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs) 8921 { 8922 int err = igb_enable_sriov(dev, num_vfs); 8923 8924 if (err) 8925 goto out; 8926 8927 err = igb_sriov_reinit(dev); 8928 if (!err) 8929 return num_vfs; 8930 8931 out: 8932 return err; 8933 } 8934 8935 #endif 8936 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 8937 { 8938 #ifdef CONFIG_PCI_IOV 8939 if (num_vfs == 0) 8940 return igb_pci_disable_sriov(dev); 8941 else 8942 return igb_pci_enable_sriov(dev, num_vfs); 8943 #endif 8944 return 0; 8945 } 8946 8947 #ifdef CONFIG_NET_POLL_CONTROLLER 8948 /* Polling 'interrupt' - used by things like netconsole to send skbs 8949 * without having to re-enable interrupts. It's not called while 8950 * the interrupt routine is executing. 8951 */ 8952 static void igb_netpoll(struct net_device *netdev) 8953 { 8954 struct igb_adapter *adapter = netdev_priv(netdev); 8955 struct e1000_hw *hw = &adapter->hw; 8956 struct igb_q_vector *q_vector; 8957 int i; 8958 8959 for (i = 0; i < adapter->num_q_vectors; i++) { 8960 q_vector = adapter->q_vector[i]; 8961 if (adapter->flags & IGB_FLAG_HAS_MSIX) 8962 wr32(E1000_EIMC, q_vector->eims_value); 8963 else 8964 igb_irq_disable(adapter); 8965 napi_schedule(&q_vector->napi); 8966 } 8967 } 8968 #endif /* CONFIG_NET_POLL_CONTROLLER */ 8969 8970 /** 8971 * igb_io_error_detected - called when PCI error is detected 8972 * @pdev: Pointer to PCI device 8973 * @state: The current pci connection state 8974 * 8975 * This function is called after a PCI bus error affecting 8976 * this device has been detected. 8977 **/ 8978 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev, 8979 pci_channel_state_t state) 8980 { 8981 struct net_device *netdev = pci_get_drvdata(pdev); 8982 struct igb_adapter *adapter = netdev_priv(netdev); 8983 8984 netif_device_detach(netdev); 8985 8986 if (state == pci_channel_io_perm_failure) 8987 return PCI_ERS_RESULT_DISCONNECT; 8988 8989 if (netif_running(netdev)) 8990 igb_down(adapter); 8991 pci_disable_device(pdev); 8992 8993 /* Request a slot slot reset. */ 8994 return PCI_ERS_RESULT_NEED_RESET; 8995 } 8996 8997 /** 8998 * igb_io_slot_reset - called after the pci bus has been reset. 8999 * @pdev: Pointer to PCI device 9000 * 9001 * Restart the card from scratch, as if from a cold-boot. Implementation 9002 * resembles the first-half of the igb_resume routine. 9003 **/ 9004 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev) 9005 { 9006 struct net_device *netdev = pci_get_drvdata(pdev); 9007 struct igb_adapter *adapter = netdev_priv(netdev); 9008 struct e1000_hw *hw = &adapter->hw; 9009 pci_ers_result_t result; 9010 int err; 9011 9012 if (pci_enable_device_mem(pdev)) { 9013 dev_err(&pdev->dev, 9014 "Cannot re-enable PCI device after reset.\n"); 9015 result = PCI_ERS_RESULT_DISCONNECT; 9016 } else { 9017 pci_set_master(pdev); 9018 pci_restore_state(pdev); 9019 pci_save_state(pdev); 9020 9021 pci_enable_wake(pdev, PCI_D3hot, 0); 9022 pci_enable_wake(pdev, PCI_D3cold, 0); 9023 9024 /* In case of PCI error, adapter lose its HW address 9025 * so we should re-assign it here. 9026 */ 9027 hw->hw_addr = adapter->io_addr; 9028 9029 igb_reset(adapter); 9030 wr32(E1000_WUS, ~0); 9031 result = PCI_ERS_RESULT_RECOVERED; 9032 } 9033 9034 err = pci_cleanup_aer_uncorrect_error_status(pdev); 9035 if (err) { 9036 dev_err(&pdev->dev, 9037 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 9038 err); 9039 /* non-fatal, continue */ 9040 } 9041 9042 return result; 9043 } 9044 9045 /** 9046 * igb_io_resume - called when traffic can start flowing again. 9047 * @pdev: Pointer to PCI device 9048 * 9049 * This callback is called when the error recovery driver tells us that 9050 * its OK to resume normal operation. Implementation resembles the 9051 * second-half of the igb_resume routine. 9052 */ 9053 static void igb_io_resume(struct pci_dev *pdev) 9054 { 9055 struct net_device *netdev = pci_get_drvdata(pdev); 9056 struct igb_adapter *adapter = netdev_priv(netdev); 9057 9058 if (netif_running(netdev)) { 9059 if (igb_up(adapter)) { 9060 dev_err(&pdev->dev, "igb_up failed after reset\n"); 9061 return; 9062 } 9063 } 9064 9065 netif_device_attach(netdev); 9066 9067 /* let the f/w know that the h/w is now under the control of the 9068 * driver. 9069 */ 9070 igb_get_hw_control(adapter); 9071 } 9072 9073 /** 9074 * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table 9075 * @adapter: Pointer to adapter structure 9076 * @index: Index of the RAR entry which need to be synced with MAC table 9077 **/ 9078 static void igb_rar_set_index(struct igb_adapter *adapter, u32 index) 9079 { 9080 struct e1000_hw *hw = &adapter->hw; 9081 u32 rar_low, rar_high; 9082 u8 *addr = adapter->mac_table[index].addr; 9083 9084 /* HW expects these to be in network order when they are plugged 9085 * into the registers which are little endian. In order to guarantee 9086 * that ordering we need to do an leXX_to_cpup here in order to be 9087 * ready for the byteswap that occurs with writel 9088 */ 9089 rar_low = le32_to_cpup((__le32 *)(addr)); 9090 rar_high = le16_to_cpup((__le16 *)(addr + 4)); 9091 9092 /* Indicate to hardware the Address is Valid. */ 9093 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) { 9094 if (is_valid_ether_addr(addr)) 9095 rar_high |= E1000_RAH_AV; 9096 9097 if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR) 9098 rar_high |= E1000_RAH_ASEL_SRC_ADDR; 9099 9100 switch (hw->mac.type) { 9101 case e1000_82575: 9102 case e1000_i210: 9103 if (adapter->mac_table[index].state & 9104 IGB_MAC_STATE_QUEUE_STEERING) 9105 rar_high |= E1000_RAH_QSEL_ENABLE; 9106 9107 rar_high |= E1000_RAH_POOL_1 * 9108 adapter->mac_table[index].queue; 9109 break; 9110 default: 9111 rar_high |= E1000_RAH_POOL_1 << 9112 adapter->mac_table[index].queue; 9113 break; 9114 } 9115 } 9116 9117 wr32(E1000_RAL(index), rar_low); 9118 wrfl(); 9119 wr32(E1000_RAH(index), rar_high); 9120 wrfl(); 9121 } 9122 9123 static int igb_set_vf_mac(struct igb_adapter *adapter, 9124 int vf, unsigned char *mac_addr) 9125 { 9126 struct e1000_hw *hw = &adapter->hw; 9127 /* VF MAC addresses start at end of receive addresses and moves 9128 * towards the first, as a result a collision should not be possible 9129 */ 9130 int rar_entry = hw->mac.rar_entry_count - (vf + 1); 9131 unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses; 9132 9133 ether_addr_copy(vf_mac_addr, mac_addr); 9134 ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr); 9135 adapter->mac_table[rar_entry].queue = vf; 9136 adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE; 9137 igb_rar_set_index(adapter, rar_entry); 9138 9139 return 0; 9140 } 9141 9142 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 9143 { 9144 struct igb_adapter *adapter = netdev_priv(netdev); 9145 9146 if (vf >= adapter->vfs_allocated_count) 9147 return -EINVAL; 9148 9149 /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC 9150 * flag and allows to overwrite the MAC via VF netdev. This 9151 * is necessary to allow libvirt a way to restore the original 9152 * MAC after unbinding vfio-pci and reloading igbvf after shutting 9153 * down a VM. 9154 */ 9155 if (is_zero_ether_addr(mac)) { 9156 adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC; 9157 dev_info(&adapter->pdev->dev, 9158 "remove administratively set MAC on VF %d\n", 9159 vf); 9160 } else if (is_valid_ether_addr(mac)) { 9161 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC; 9162 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", 9163 mac, vf); 9164 dev_info(&adapter->pdev->dev, 9165 "Reload the VF driver to make this change effective."); 9166 /* Generate additional warning if PF is down */ 9167 if (test_bit(__IGB_DOWN, &adapter->state)) { 9168 dev_warn(&adapter->pdev->dev, 9169 "The VF MAC address has been set, but the PF device is not up.\n"); 9170 dev_warn(&adapter->pdev->dev, 9171 "Bring the PF device up before attempting to use the VF device.\n"); 9172 } 9173 } else { 9174 return -EINVAL; 9175 } 9176 return igb_set_vf_mac(adapter, vf, mac); 9177 } 9178 9179 static int igb_link_mbps(int internal_link_speed) 9180 { 9181 switch (internal_link_speed) { 9182 case SPEED_100: 9183 return 100; 9184 case SPEED_1000: 9185 return 1000; 9186 default: 9187 return 0; 9188 } 9189 } 9190 9191 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate, 9192 int link_speed) 9193 { 9194 int rf_dec, rf_int; 9195 u32 bcnrc_val; 9196 9197 if (tx_rate != 0) { 9198 /* Calculate the rate factor values to set */ 9199 rf_int = link_speed / tx_rate; 9200 rf_dec = (link_speed - (rf_int * tx_rate)); 9201 rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) / 9202 tx_rate; 9203 9204 bcnrc_val = E1000_RTTBCNRC_RS_ENA; 9205 bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) & 9206 E1000_RTTBCNRC_RF_INT_MASK); 9207 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK); 9208 } else { 9209 bcnrc_val = 0; 9210 } 9211 9212 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */ 9213 /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM 9214 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported. 9215 */ 9216 wr32(E1000_RTTBCNRM, 0x14); 9217 wr32(E1000_RTTBCNRC, bcnrc_val); 9218 } 9219 9220 static void igb_check_vf_rate_limit(struct igb_adapter *adapter) 9221 { 9222 int actual_link_speed, i; 9223 bool reset_rate = false; 9224 9225 /* VF TX rate limit was not set or not supported */ 9226 if ((adapter->vf_rate_link_speed == 0) || 9227 (adapter->hw.mac.type != e1000_82576)) 9228 return; 9229 9230 actual_link_speed = igb_link_mbps(adapter->link_speed); 9231 if (actual_link_speed != adapter->vf_rate_link_speed) { 9232 reset_rate = true; 9233 adapter->vf_rate_link_speed = 0; 9234 dev_info(&adapter->pdev->dev, 9235 "Link speed has been changed. VF Transmit rate is disabled\n"); 9236 } 9237 9238 for (i = 0; i < adapter->vfs_allocated_count; i++) { 9239 if (reset_rate) 9240 adapter->vf_data[i].tx_rate = 0; 9241 9242 igb_set_vf_rate_limit(&adapter->hw, i, 9243 adapter->vf_data[i].tx_rate, 9244 actual_link_speed); 9245 } 9246 } 9247 9248 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, 9249 int min_tx_rate, int max_tx_rate) 9250 { 9251 struct igb_adapter *adapter = netdev_priv(netdev); 9252 struct e1000_hw *hw = &adapter->hw; 9253 int actual_link_speed; 9254 9255 if (hw->mac.type != e1000_82576) 9256 return -EOPNOTSUPP; 9257 9258 if (min_tx_rate) 9259 return -EINVAL; 9260 9261 actual_link_speed = igb_link_mbps(adapter->link_speed); 9262 if ((vf >= adapter->vfs_allocated_count) || 9263 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) || 9264 (max_tx_rate < 0) || 9265 (max_tx_rate > actual_link_speed)) 9266 return -EINVAL; 9267 9268 adapter->vf_rate_link_speed = actual_link_speed; 9269 adapter->vf_data[vf].tx_rate = (u16)max_tx_rate; 9270 igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed); 9271 9272 return 0; 9273 } 9274 9275 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, 9276 bool setting) 9277 { 9278 struct igb_adapter *adapter = netdev_priv(netdev); 9279 struct e1000_hw *hw = &adapter->hw; 9280 u32 reg_val, reg_offset; 9281 9282 if (!adapter->vfs_allocated_count) 9283 return -EOPNOTSUPP; 9284 9285 if (vf >= adapter->vfs_allocated_count) 9286 return -EINVAL; 9287 9288 reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC; 9289 reg_val = rd32(reg_offset); 9290 if (setting) 9291 reg_val |= (BIT(vf) | 9292 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)); 9293 else 9294 reg_val &= ~(BIT(vf) | 9295 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)); 9296 wr32(reg_offset, reg_val); 9297 9298 adapter->vf_data[vf].spoofchk_enabled = setting; 9299 return 0; 9300 } 9301 9302 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting) 9303 { 9304 struct igb_adapter *adapter = netdev_priv(netdev); 9305 9306 if (vf >= adapter->vfs_allocated_count) 9307 return -EINVAL; 9308 if (adapter->vf_data[vf].trusted == setting) 9309 return 0; 9310 9311 adapter->vf_data[vf].trusted = setting; 9312 9313 dev_info(&adapter->pdev->dev, "VF %u is %strusted\n", 9314 vf, setting ? "" : "not "); 9315 return 0; 9316 } 9317 9318 static int igb_ndo_get_vf_config(struct net_device *netdev, 9319 int vf, struct ifla_vf_info *ivi) 9320 { 9321 struct igb_adapter *adapter = netdev_priv(netdev); 9322 if (vf >= adapter->vfs_allocated_count) 9323 return -EINVAL; 9324 ivi->vf = vf; 9325 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN); 9326 ivi->max_tx_rate = adapter->vf_data[vf].tx_rate; 9327 ivi->min_tx_rate = 0; 9328 ivi->vlan = adapter->vf_data[vf].pf_vlan; 9329 ivi->qos = adapter->vf_data[vf].pf_qos; 9330 ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled; 9331 ivi->trusted = adapter->vf_data[vf].trusted; 9332 return 0; 9333 } 9334 9335 static void igb_vmm_control(struct igb_adapter *adapter) 9336 { 9337 struct e1000_hw *hw = &adapter->hw; 9338 u32 reg; 9339 9340 switch (hw->mac.type) { 9341 case e1000_82575: 9342 case e1000_i210: 9343 case e1000_i211: 9344 case e1000_i354: 9345 default: 9346 /* replication is not supported for 82575 */ 9347 return; 9348 case e1000_82576: 9349 /* notify HW that the MAC is adding vlan tags */ 9350 reg = rd32(E1000_DTXCTL); 9351 reg |= E1000_DTXCTL_VLAN_ADDED; 9352 wr32(E1000_DTXCTL, reg); 9353 /* Fall through */ 9354 case e1000_82580: 9355 /* enable replication vlan tag stripping */ 9356 reg = rd32(E1000_RPLOLR); 9357 reg |= E1000_RPLOLR_STRVLAN; 9358 wr32(E1000_RPLOLR, reg); 9359 /* Fall through */ 9360 case e1000_i350: 9361 /* none of the above registers are supported by i350 */ 9362 break; 9363 } 9364 9365 if (adapter->vfs_allocated_count) { 9366 igb_vmdq_set_loopback_pf(hw, true); 9367 igb_vmdq_set_replication_pf(hw, true); 9368 igb_vmdq_set_anti_spoofing_pf(hw, true, 9369 adapter->vfs_allocated_count); 9370 } else { 9371 igb_vmdq_set_loopback_pf(hw, false); 9372 igb_vmdq_set_replication_pf(hw, false); 9373 } 9374 } 9375 9376 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba) 9377 { 9378 struct e1000_hw *hw = &adapter->hw; 9379 u32 dmac_thr; 9380 u16 hwm; 9381 9382 if (hw->mac.type > e1000_82580) { 9383 if (adapter->flags & IGB_FLAG_DMAC) { 9384 u32 reg; 9385 9386 /* force threshold to 0. */ 9387 wr32(E1000_DMCTXTH, 0); 9388 9389 /* DMA Coalescing high water mark needs to be greater 9390 * than the Rx threshold. Set hwm to PBA - max frame 9391 * size in 16B units, capping it at PBA - 6KB. 9392 */ 9393 hwm = 64 * (pba - 6); 9394 reg = rd32(E1000_FCRTC); 9395 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 9396 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 9397 & E1000_FCRTC_RTH_COAL_MASK); 9398 wr32(E1000_FCRTC, reg); 9399 9400 /* Set the DMA Coalescing Rx threshold to PBA - 2 * max 9401 * frame size, capping it at PBA - 10KB. 9402 */ 9403 dmac_thr = pba - 10; 9404 reg = rd32(E1000_DMACR); 9405 reg &= ~E1000_DMACR_DMACTHR_MASK; 9406 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT) 9407 & E1000_DMACR_DMACTHR_MASK); 9408 9409 /* transition to L0x or L1 if available..*/ 9410 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 9411 9412 /* watchdog timer= +-1000 usec in 32usec intervals */ 9413 reg |= (1000 >> 5); 9414 9415 /* Disable BMC-to-OS Watchdog Enable */ 9416 if (hw->mac.type != e1000_i354) 9417 reg &= ~E1000_DMACR_DC_BMC2OSW_EN; 9418 9419 wr32(E1000_DMACR, reg); 9420 9421 /* no lower threshold to disable 9422 * coalescing(smart fifb)-UTRESH=0 9423 */ 9424 wr32(E1000_DMCRTRH, 0); 9425 9426 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4); 9427 9428 wr32(E1000_DMCTLX, reg); 9429 9430 /* free space in tx packet buffer to wake from 9431 * DMA coal 9432 */ 9433 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE - 9434 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6); 9435 9436 /* make low power state decision controlled 9437 * by DMA coal 9438 */ 9439 reg = rd32(E1000_PCIEMISC); 9440 reg &= ~E1000_PCIEMISC_LX_DECISION; 9441 wr32(E1000_PCIEMISC, reg); 9442 } /* endif adapter->dmac is not disabled */ 9443 } else if (hw->mac.type == e1000_82580) { 9444 u32 reg = rd32(E1000_PCIEMISC); 9445 9446 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION); 9447 wr32(E1000_DMACR, 0); 9448 } 9449 } 9450 9451 /** 9452 * igb_read_i2c_byte - Reads 8 bit word over I2C 9453 * @hw: pointer to hardware structure 9454 * @byte_offset: byte offset to read 9455 * @dev_addr: device address 9456 * @data: value read 9457 * 9458 * Performs byte read operation over I2C interface at 9459 * a specified device address. 9460 **/ 9461 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset, 9462 u8 dev_addr, u8 *data) 9463 { 9464 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw); 9465 struct i2c_client *this_client = adapter->i2c_client; 9466 s32 status; 9467 u16 swfw_mask = 0; 9468 9469 if (!this_client) 9470 return E1000_ERR_I2C; 9471 9472 swfw_mask = E1000_SWFW_PHY0_SM; 9473 9474 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 9475 return E1000_ERR_SWFW_SYNC; 9476 9477 status = i2c_smbus_read_byte_data(this_client, byte_offset); 9478 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 9479 9480 if (status < 0) 9481 return E1000_ERR_I2C; 9482 else { 9483 *data = status; 9484 return 0; 9485 } 9486 } 9487 9488 /** 9489 * igb_write_i2c_byte - Writes 8 bit word over I2C 9490 * @hw: pointer to hardware structure 9491 * @byte_offset: byte offset to write 9492 * @dev_addr: device address 9493 * @data: value to write 9494 * 9495 * Performs byte write operation over I2C interface at 9496 * a specified device address. 9497 **/ 9498 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset, 9499 u8 dev_addr, u8 data) 9500 { 9501 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw); 9502 struct i2c_client *this_client = adapter->i2c_client; 9503 s32 status; 9504 u16 swfw_mask = E1000_SWFW_PHY0_SM; 9505 9506 if (!this_client) 9507 return E1000_ERR_I2C; 9508 9509 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 9510 return E1000_ERR_SWFW_SYNC; 9511 status = i2c_smbus_write_byte_data(this_client, byte_offset, data); 9512 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 9513 9514 if (status) 9515 return E1000_ERR_I2C; 9516 else 9517 return 0; 9518 9519 } 9520 9521 int igb_reinit_queues(struct igb_adapter *adapter) 9522 { 9523 struct net_device *netdev = adapter->netdev; 9524 struct pci_dev *pdev = adapter->pdev; 9525 int err = 0; 9526 9527 if (netif_running(netdev)) 9528 igb_close(netdev); 9529 9530 igb_reset_interrupt_capability(adapter); 9531 9532 if (igb_init_interrupt_scheme(adapter, true)) { 9533 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 9534 return -ENOMEM; 9535 } 9536 9537 if (netif_running(netdev)) 9538 err = igb_open(netdev); 9539 9540 return err; 9541 } 9542 9543 static void igb_nfc_filter_exit(struct igb_adapter *adapter) 9544 { 9545 struct igb_nfc_filter *rule; 9546 9547 spin_lock(&adapter->nfc_lock); 9548 9549 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 9550 igb_erase_filter(adapter, rule); 9551 9552 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) 9553 igb_erase_filter(adapter, rule); 9554 9555 spin_unlock(&adapter->nfc_lock); 9556 } 9557 9558 static void igb_nfc_filter_restore(struct igb_adapter *adapter) 9559 { 9560 struct igb_nfc_filter *rule; 9561 9562 spin_lock(&adapter->nfc_lock); 9563 9564 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 9565 igb_add_filter(adapter, rule); 9566 9567 spin_unlock(&adapter->nfc_lock); 9568 } 9569 /* igb_main.c */ 9570