1 /******************************************************************************* 2 3 Intel(R) Gigabit Ethernet Linux driver 4 Copyright(c) 2007-2011 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #include <linux/module.h> 31 #include <linux/types.h> 32 #include <linux/init.h> 33 #include <linux/bitops.h> 34 #include <linux/vmalloc.h> 35 #include <linux/pagemap.h> 36 #include <linux/netdevice.h> 37 #include <linux/ipv6.h> 38 #include <linux/slab.h> 39 #include <net/checksum.h> 40 #include <net/ip6_checksum.h> 41 #include <linux/net_tstamp.h> 42 #include <linux/mii.h> 43 #include <linux/ethtool.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/pci.h> 47 #include <linux/pci-aspm.h> 48 #include <linux/delay.h> 49 #include <linux/interrupt.h> 50 #include <linux/ip.h> 51 #include <linux/tcp.h> 52 #include <linux/sctp.h> 53 #include <linux/if_ether.h> 54 #include <linux/aer.h> 55 #include <linux/prefetch.h> 56 #include <linux/pm_runtime.h> 57 #ifdef CONFIG_IGB_DCA 58 #include <linux/dca.h> 59 #endif 60 #include "igb.h" 61 62 #define MAJ 3 63 #define MIN 2 64 #define BUILD 10 65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ 66 __stringify(BUILD) "-k" 67 char igb_driver_name[] = "igb"; 68 char igb_driver_version[] = DRV_VERSION; 69 static const char igb_driver_string[] = 70 "Intel(R) Gigabit Ethernet Network Driver"; 71 static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation."; 72 73 static const struct e1000_info *igb_info_tbl[] = { 74 [board_82575] = &e1000_82575_info, 75 }; 76 77 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = { 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 }, 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 }, 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 }, 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 }, 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 }, 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 }, 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 }, 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 }, 86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 }, 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 }, 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 }, 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 }, 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 }, 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 }, 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 }, 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 }, 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 }, 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 }, 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 }, 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 }, 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 }, 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 }, 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 }, 101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 }, 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 }, 103 /* required last entry */ 104 {0, } 105 }; 106 107 MODULE_DEVICE_TABLE(pci, igb_pci_tbl); 108 109 void igb_reset(struct igb_adapter *); 110 static int igb_setup_all_tx_resources(struct igb_adapter *); 111 static int igb_setup_all_rx_resources(struct igb_adapter *); 112 static void igb_free_all_tx_resources(struct igb_adapter *); 113 static void igb_free_all_rx_resources(struct igb_adapter *); 114 static void igb_setup_mrqc(struct igb_adapter *); 115 static int igb_probe(struct pci_dev *, const struct pci_device_id *); 116 static void __devexit igb_remove(struct pci_dev *pdev); 117 static void igb_init_hw_timer(struct igb_adapter *adapter); 118 static int igb_sw_init(struct igb_adapter *); 119 static int igb_open(struct net_device *); 120 static int igb_close(struct net_device *); 121 static void igb_configure_tx(struct igb_adapter *); 122 static void igb_configure_rx(struct igb_adapter *); 123 static void igb_clean_all_tx_rings(struct igb_adapter *); 124 static void igb_clean_all_rx_rings(struct igb_adapter *); 125 static void igb_clean_tx_ring(struct igb_ring *); 126 static void igb_clean_rx_ring(struct igb_ring *); 127 static void igb_set_rx_mode(struct net_device *); 128 static void igb_update_phy_info(unsigned long); 129 static void igb_watchdog(unsigned long); 130 static void igb_watchdog_task(struct work_struct *); 131 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *); 132 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev, 133 struct rtnl_link_stats64 *stats); 134 static int igb_change_mtu(struct net_device *, int); 135 static int igb_set_mac(struct net_device *, void *); 136 static void igb_set_uta(struct igb_adapter *adapter); 137 static irqreturn_t igb_intr(int irq, void *); 138 static irqreturn_t igb_intr_msi(int irq, void *); 139 static irqreturn_t igb_msix_other(int irq, void *); 140 static irqreturn_t igb_msix_ring(int irq, void *); 141 #ifdef CONFIG_IGB_DCA 142 static void igb_update_dca(struct igb_q_vector *); 143 static void igb_setup_dca(struct igb_adapter *); 144 #endif /* CONFIG_IGB_DCA */ 145 static int igb_poll(struct napi_struct *, int); 146 static bool igb_clean_tx_irq(struct igb_q_vector *); 147 static bool igb_clean_rx_irq(struct igb_q_vector *, int); 148 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); 149 static void igb_tx_timeout(struct net_device *); 150 static void igb_reset_task(struct work_struct *); 151 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features); 152 static int igb_vlan_rx_add_vid(struct net_device *, u16); 153 static int igb_vlan_rx_kill_vid(struct net_device *, u16); 154 static void igb_restore_vlan(struct igb_adapter *); 155 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8); 156 static void igb_ping_all_vfs(struct igb_adapter *); 157 static void igb_msg_task(struct igb_adapter *); 158 static void igb_vmm_control(struct igb_adapter *); 159 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *); 160 static void igb_restore_vf_multicasts(struct igb_adapter *adapter); 161 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac); 162 static int igb_ndo_set_vf_vlan(struct net_device *netdev, 163 int vf, u16 vlan, u8 qos); 164 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate); 165 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf, 166 struct ifla_vf_info *ivi); 167 static void igb_check_vf_rate_limit(struct igb_adapter *); 168 169 #ifdef CONFIG_PCI_IOV 170 static int igb_vf_configure(struct igb_adapter *adapter, int vf); 171 static int igb_find_enabled_vfs(struct igb_adapter *adapter); 172 static int igb_check_vf_assignment(struct igb_adapter *adapter); 173 #endif 174 175 #ifdef CONFIG_PM 176 static int igb_suspend(struct device *); 177 static int igb_resume(struct device *); 178 #ifdef CONFIG_PM_RUNTIME 179 static int igb_runtime_suspend(struct device *dev); 180 static int igb_runtime_resume(struct device *dev); 181 static int igb_runtime_idle(struct device *dev); 182 #endif 183 static const struct dev_pm_ops igb_pm_ops = { 184 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume) 185 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume, 186 igb_runtime_idle) 187 }; 188 #endif 189 static void igb_shutdown(struct pci_dev *); 190 #ifdef CONFIG_IGB_DCA 191 static int igb_notify_dca(struct notifier_block *, unsigned long, void *); 192 static struct notifier_block dca_notifier = { 193 .notifier_call = igb_notify_dca, 194 .next = NULL, 195 .priority = 0 196 }; 197 #endif 198 #ifdef CONFIG_NET_POLL_CONTROLLER 199 /* for netdump / net console */ 200 static void igb_netpoll(struct net_device *); 201 #endif 202 #ifdef CONFIG_PCI_IOV 203 static unsigned int max_vfs = 0; 204 module_param(max_vfs, uint, 0); 205 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate " 206 "per physical function"); 207 #endif /* CONFIG_PCI_IOV */ 208 209 static pci_ers_result_t igb_io_error_detected(struct pci_dev *, 210 pci_channel_state_t); 211 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *); 212 static void igb_io_resume(struct pci_dev *); 213 214 static struct pci_error_handlers igb_err_handler = { 215 .error_detected = igb_io_error_detected, 216 .slot_reset = igb_io_slot_reset, 217 .resume = igb_io_resume, 218 }; 219 220 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba); 221 222 static struct pci_driver igb_driver = { 223 .name = igb_driver_name, 224 .id_table = igb_pci_tbl, 225 .probe = igb_probe, 226 .remove = __devexit_p(igb_remove), 227 #ifdef CONFIG_PM 228 .driver.pm = &igb_pm_ops, 229 #endif 230 .shutdown = igb_shutdown, 231 .err_handler = &igb_err_handler 232 }; 233 234 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 235 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver"); 236 MODULE_LICENSE("GPL"); 237 MODULE_VERSION(DRV_VERSION); 238 239 struct igb_reg_info { 240 u32 ofs; 241 char *name; 242 }; 243 244 static const struct igb_reg_info igb_reg_info_tbl[] = { 245 246 /* General Registers */ 247 {E1000_CTRL, "CTRL"}, 248 {E1000_STATUS, "STATUS"}, 249 {E1000_CTRL_EXT, "CTRL_EXT"}, 250 251 /* Interrupt Registers */ 252 {E1000_ICR, "ICR"}, 253 254 /* RX Registers */ 255 {E1000_RCTL, "RCTL"}, 256 {E1000_RDLEN(0), "RDLEN"}, 257 {E1000_RDH(0), "RDH"}, 258 {E1000_RDT(0), "RDT"}, 259 {E1000_RXDCTL(0), "RXDCTL"}, 260 {E1000_RDBAL(0), "RDBAL"}, 261 {E1000_RDBAH(0), "RDBAH"}, 262 263 /* TX Registers */ 264 {E1000_TCTL, "TCTL"}, 265 {E1000_TDBAL(0), "TDBAL"}, 266 {E1000_TDBAH(0), "TDBAH"}, 267 {E1000_TDLEN(0), "TDLEN"}, 268 {E1000_TDH(0), "TDH"}, 269 {E1000_TDT(0), "TDT"}, 270 {E1000_TXDCTL(0), "TXDCTL"}, 271 {E1000_TDFH, "TDFH"}, 272 {E1000_TDFT, "TDFT"}, 273 {E1000_TDFHS, "TDFHS"}, 274 {E1000_TDFPC, "TDFPC"}, 275 276 /* List Terminator */ 277 {} 278 }; 279 280 /* 281 * igb_regdump - register printout routine 282 */ 283 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo) 284 { 285 int n = 0; 286 char rname[16]; 287 u32 regs[8]; 288 289 switch (reginfo->ofs) { 290 case E1000_RDLEN(0): 291 for (n = 0; n < 4; n++) 292 regs[n] = rd32(E1000_RDLEN(n)); 293 break; 294 case E1000_RDH(0): 295 for (n = 0; n < 4; n++) 296 regs[n] = rd32(E1000_RDH(n)); 297 break; 298 case E1000_RDT(0): 299 for (n = 0; n < 4; n++) 300 regs[n] = rd32(E1000_RDT(n)); 301 break; 302 case E1000_RXDCTL(0): 303 for (n = 0; n < 4; n++) 304 regs[n] = rd32(E1000_RXDCTL(n)); 305 break; 306 case E1000_RDBAL(0): 307 for (n = 0; n < 4; n++) 308 regs[n] = rd32(E1000_RDBAL(n)); 309 break; 310 case E1000_RDBAH(0): 311 for (n = 0; n < 4; n++) 312 regs[n] = rd32(E1000_RDBAH(n)); 313 break; 314 case E1000_TDBAL(0): 315 for (n = 0; n < 4; n++) 316 regs[n] = rd32(E1000_RDBAL(n)); 317 break; 318 case E1000_TDBAH(0): 319 for (n = 0; n < 4; n++) 320 regs[n] = rd32(E1000_TDBAH(n)); 321 break; 322 case E1000_TDLEN(0): 323 for (n = 0; n < 4; n++) 324 regs[n] = rd32(E1000_TDLEN(n)); 325 break; 326 case E1000_TDH(0): 327 for (n = 0; n < 4; n++) 328 regs[n] = rd32(E1000_TDH(n)); 329 break; 330 case E1000_TDT(0): 331 for (n = 0; n < 4; n++) 332 regs[n] = rd32(E1000_TDT(n)); 333 break; 334 case E1000_TXDCTL(0): 335 for (n = 0; n < 4; n++) 336 regs[n] = rd32(E1000_TXDCTL(n)); 337 break; 338 default: 339 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs)); 340 return; 341 } 342 343 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]"); 344 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1], 345 regs[2], regs[3]); 346 } 347 348 /* 349 * igb_dump - Print registers, tx-rings and rx-rings 350 */ 351 static void igb_dump(struct igb_adapter *adapter) 352 { 353 struct net_device *netdev = adapter->netdev; 354 struct e1000_hw *hw = &adapter->hw; 355 struct igb_reg_info *reginfo; 356 struct igb_ring *tx_ring; 357 union e1000_adv_tx_desc *tx_desc; 358 struct my_u0 { u64 a; u64 b; } *u0; 359 struct igb_ring *rx_ring; 360 union e1000_adv_rx_desc *rx_desc; 361 u32 staterr; 362 u16 i, n; 363 364 if (!netif_msg_hw(adapter)) 365 return; 366 367 /* Print netdevice Info */ 368 if (netdev) { 369 dev_info(&adapter->pdev->dev, "Net device Info\n"); 370 pr_info("Device Name state trans_start " 371 "last_rx\n"); 372 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name, 373 netdev->state, netdev->trans_start, netdev->last_rx); 374 } 375 376 /* Print Registers */ 377 dev_info(&adapter->pdev->dev, "Register Dump\n"); 378 pr_info(" Register Name Value\n"); 379 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl; 380 reginfo->name; reginfo++) { 381 igb_regdump(hw, reginfo); 382 } 383 384 /* Print TX Ring Summary */ 385 if (!netdev || !netif_running(netdev)) 386 goto exit; 387 388 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 389 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 390 for (n = 0; n < adapter->num_tx_queues; n++) { 391 struct igb_tx_buffer *buffer_info; 392 tx_ring = adapter->tx_ring[n]; 393 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 394 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", 395 n, tx_ring->next_to_use, tx_ring->next_to_clean, 396 (u64)buffer_info->dma, 397 buffer_info->length, 398 buffer_info->next_to_watch, 399 (u64)buffer_info->time_stamp); 400 } 401 402 /* Print TX Rings */ 403 if (!netif_msg_tx_done(adapter)) 404 goto rx_ring_summary; 405 406 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 407 408 /* Transmit Descriptor Formats 409 * 410 * Advanced Transmit Descriptor 411 * +--------------------------------------------------------------+ 412 * 0 | Buffer Address [63:0] | 413 * +--------------------------------------------------------------+ 414 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN | 415 * +--------------------------------------------------------------+ 416 * 63 46 45 40 39 38 36 35 32 31 24 15 0 417 */ 418 419 for (n = 0; n < adapter->num_tx_queues; n++) { 420 tx_ring = adapter->tx_ring[n]; 421 pr_info("------------------------------------\n"); 422 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 423 pr_info("------------------------------------\n"); 424 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] " 425 "[bi->dma ] leng ntw timestamp " 426 "bi->skb\n"); 427 428 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 429 const char *next_desc; 430 struct igb_tx_buffer *buffer_info; 431 tx_desc = IGB_TX_DESC(tx_ring, i); 432 buffer_info = &tx_ring->tx_buffer_info[i]; 433 u0 = (struct my_u0 *)tx_desc; 434 if (i == tx_ring->next_to_use && 435 i == tx_ring->next_to_clean) 436 next_desc = " NTC/U"; 437 else if (i == tx_ring->next_to_use) 438 next_desc = " NTU"; 439 else if (i == tx_ring->next_to_clean) 440 next_desc = " NTC"; 441 else 442 next_desc = ""; 443 444 pr_info("T [0x%03X] %016llX %016llX %016llX" 445 " %04X %p %016llX %p%s\n", i, 446 le64_to_cpu(u0->a), 447 le64_to_cpu(u0->b), 448 (u64)buffer_info->dma, 449 buffer_info->length, 450 buffer_info->next_to_watch, 451 (u64)buffer_info->time_stamp, 452 buffer_info->skb, next_desc); 453 454 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0) 455 print_hex_dump(KERN_INFO, "", 456 DUMP_PREFIX_ADDRESS, 457 16, 1, phys_to_virt(buffer_info->dma), 458 buffer_info->length, true); 459 } 460 } 461 462 /* Print RX Rings Summary */ 463 rx_ring_summary: 464 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 465 pr_info("Queue [NTU] [NTC]\n"); 466 for (n = 0; n < adapter->num_rx_queues; n++) { 467 rx_ring = adapter->rx_ring[n]; 468 pr_info(" %5d %5X %5X\n", 469 n, rx_ring->next_to_use, rx_ring->next_to_clean); 470 } 471 472 /* Print RX Rings */ 473 if (!netif_msg_rx_status(adapter)) 474 goto exit; 475 476 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 477 478 /* Advanced Receive Descriptor (Read) Format 479 * 63 1 0 480 * +-----------------------------------------------------+ 481 * 0 | Packet Buffer Address [63:1] |A0/NSE| 482 * +----------------------------------------------+------+ 483 * 8 | Header Buffer Address [63:1] | DD | 484 * +-----------------------------------------------------+ 485 * 486 * 487 * Advanced Receive Descriptor (Write-Back) Format 488 * 489 * 63 48 47 32 31 30 21 20 17 16 4 3 0 490 * +------------------------------------------------------+ 491 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | 492 * | Checksum Ident | | | | Type | Type | 493 * +------------------------------------------------------+ 494 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 495 * +------------------------------------------------------+ 496 * 63 48 47 32 31 20 19 0 497 */ 498 499 for (n = 0; n < adapter->num_rx_queues; n++) { 500 rx_ring = adapter->rx_ring[n]; 501 pr_info("------------------------------------\n"); 502 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 503 pr_info("------------------------------------\n"); 504 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] " 505 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n"); 506 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----" 507 "----------- [bi->skb] <-- Adv Rx Write-Back format\n"); 508 509 for (i = 0; i < rx_ring->count; i++) { 510 const char *next_desc; 511 struct igb_rx_buffer *buffer_info; 512 buffer_info = &rx_ring->rx_buffer_info[i]; 513 rx_desc = IGB_RX_DESC(rx_ring, i); 514 u0 = (struct my_u0 *)rx_desc; 515 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 516 517 if (i == rx_ring->next_to_use) 518 next_desc = " NTU"; 519 else if (i == rx_ring->next_to_clean) 520 next_desc = " NTC"; 521 else 522 next_desc = ""; 523 524 if (staterr & E1000_RXD_STAT_DD) { 525 /* Descriptor Done */ 526 pr_info("%s[0x%03X] %016llX %016llX -------" 527 "--------- %p%s\n", "RWB", i, 528 le64_to_cpu(u0->a), 529 le64_to_cpu(u0->b), 530 buffer_info->skb, next_desc); 531 } else { 532 pr_info("%s[0x%03X] %016llX %016llX %016llX" 533 " %p%s\n", "R ", i, 534 le64_to_cpu(u0->a), 535 le64_to_cpu(u0->b), 536 (u64)buffer_info->dma, 537 buffer_info->skb, next_desc); 538 539 if (netif_msg_pktdata(adapter)) { 540 print_hex_dump(KERN_INFO, "", 541 DUMP_PREFIX_ADDRESS, 542 16, 1, 543 phys_to_virt(buffer_info->dma), 544 IGB_RX_HDR_LEN, true); 545 print_hex_dump(KERN_INFO, "", 546 DUMP_PREFIX_ADDRESS, 547 16, 1, 548 phys_to_virt( 549 buffer_info->page_dma + 550 buffer_info->page_offset), 551 PAGE_SIZE/2, true); 552 } 553 } 554 } 555 } 556 557 exit: 558 return; 559 } 560 561 562 /** 563 * igb_read_clock - read raw cycle counter (to be used by time counter) 564 */ 565 static cycle_t igb_read_clock(const struct cyclecounter *tc) 566 { 567 struct igb_adapter *adapter = 568 container_of(tc, struct igb_adapter, cycles); 569 struct e1000_hw *hw = &adapter->hw; 570 u64 stamp = 0; 571 int shift = 0; 572 573 /* 574 * The timestamp latches on lowest register read. For the 82580 575 * the lowest register is SYSTIMR instead of SYSTIML. However we never 576 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it. 577 */ 578 if (hw->mac.type >= e1000_82580) { 579 stamp = rd32(E1000_SYSTIMR) >> 8; 580 shift = IGB_82580_TSYNC_SHIFT; 581 } 582 583 stamp |= (u64)rd32(E1000_SYSTIML) << shift; 584 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32); 585 return stamp; 586 } 587 588 /** 589 * igb_get_hw_dev - return device 590 * used by hardware layer to print debugging information 591 **/ 592 struct net_device *igb_get_hw_dev(struct e1000_hw *hw) 593 { 594 struct igb_adapter *adapter = hw->back; 595 return adapter->netdev; 596 } 597 598 /** 599 * igb_init_module - Driver Registration Routine 600 * 601 * igb_init_module is the first routine called when the driver is 602 * loaded. All it does is register with the PCI subsystem. 603 **/ 604 static int __init igb_init_module(void) 605 { 606 int ret; 607 pr_info("%s - version %s\n", 608 igb_driver_string, igb_driver_version); 609 610 pr_info("%s\n", igb_copyright); 611 612 #ifdef CONFIG_IGB_DCA 613 dca_register_notify(&dca_notifier); 614 #endif 615 ret = pci_register_driver(&igb_driver); 616 return ret; 617 } 618 619 module_init(igb_init_module); 620 621 /** 622 * igb_exit_module - Driver Exit Cleanup Routine 623 * 624 * igb_exit_module is called just before the driver is removed 625 * from memory. 626 **/ 627 static void __exit igb_exit_module(void) 628 { 629 #ifdef CONFIG_IGB_DCA 630 dca_unregister_notify(&dca_notifier); 631 #endif 632 pci_unregister_driver(&igb_driver); 633 } 634 635 module_exit(igb_exit_module); 636 637 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1)) 638 /** 639 * igb_cache_ring_register - Descriptor ring to register mapping 640 * @adapter: board private structure to initialize 641 * 642 * Once we know the feature-set enabled for the device, we'll cache 643 * the register offset the descriptor ring is assigned to. 644 **/ 645 static void igb_cache_ring_register(struct igb_adapter *adapter) 646 { 647 int i = 0, j = 0; 648 u32 rbase_offset = adapter->vfs_allocated_count; 649 650 switch (adapter->hw.mac.type) { 651 case e1000_82576: 652 /* The queues are allocated for virtualization such that VF 0 653 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc. 654 * In order to avoid collision we start at the first free queue 655 * and continue consuming queues in the same sequence 656 */ 657 if (adapter->vfs_allocated_count) { 658 for (; i < adapter->rss_queues; i++) 659 adapter->rx_ring[i]->reg_idx = rbase_offset + 660 Q_IDX_82576(i); 661 } 662 case e1000_82575: 663 case e1000_82580: 664 case e1000_i350: 665 default: 666 for (; i < adapter->num_rx_queues; i++) 667 adapter->rx_ring[i]->reg_idx = rbase_offset + i; 668 for (; j < adapter->num_tx_queues; j++) 669 adapter->tx_ring[j]->reg_idx = rbase_offset + j; 670 break; 671 } 672 } 673 674 static void igb_free_queues(struct igb_adapter *adapter) 675 { 676 int i; 677 678 for (i = 0; i < adapter->num_tx_queues; i++) { 679 kfree(adapter->tx_ring[i]); 680 adapter->tx_ring[i] = NULL; 681 } 682 for (i = 0; i < adapter->num_rx_queues; i++) { 683 kfree(adapter->rx_ring[i]); 684 adapter->rx_ring[i] = NULL; 685 } 686 adapter->num_rx_queues = 0; 687 adapter->num_tx_queues = 0; 688 } 689 690 /** 691 * igb_alloc_queues - Allocate memory for all rings 692 * @adapter: board private structure to initialize 693 * 694 * We allocate one ring per queue at run-time since we don't know the 695 * number of queues at compile-time. 696 **/ 697 static int igb_alloc_queues(struct igb_adapter *adapter) 698 { 699 struct igb_ring *ring; 700 int i; 701 int orig_node = adapter->node; 702 703 for (i = 0; i < adapter->num_tx_queues; i++) { 704 if (orig_node == -1) { 705 int cur_node = next_online_node(adapter->node); 706 if (cur_node == MAX_NUMNODES) 707 cur_node = first_online_node; 708 adapter->node = cur_node; 709 } 710 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL, 711 adapter->node); 712 if (!ring) 713 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL); 714 if (!ring) 715 goto err; 716 ring->count = adapter->tx_ring_count; 717 ring->queue_index = i; 718 ring->dev = &adapter->pdev->dev; 719 ring->netdev = adapter->netdev; 720 ring->numa_node = adapter->node; 721 /* For 82575, context index must be unique per ring. */ 722 if (adapter->hw.mac.type == e1000_82575) 723 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags); 724 adapter->tx_ring[i] = ring; 725 } 726 /* Restore the adapter's original node */ 727 adapter->node = orig_node; 728 729 for (i = 0; i < adapter->num_rx_queues; i++) { 730 if (orig_node == -1) { 731 int cur_node = next_online_node(adapter->node); 732 if (cur_node == MAX_NUMNODES) 733 cur_node = first_online_node; 734 adapter->node = cur_node; 735 } 736 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL, 737 adapter->node); 738 if (!ring) 739 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL); 740 if (!ring) 741 goto err; 742 ring->count = adapter->rx_ring_count; 743 ring->queue_index = i; 744 ring->dev = &adapter->pdev->dev; 745 ring->netdev = adapter->netdev; 746 ring->numa_node = adapter->node; 747 /* set flag indicating ring supports SCTP checksum offload */ 748 if (adapter->hw.mac.type >= e1000_82576) 749 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags); 750 751 /* On i350, loopback VLAN packets have the tag byte-swapped. */ 752 if (adapter->hw.mac.type == e1000_i350) 753 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags); 754 755 adapter->rx_ring[i] = ring; 756 } 757 /* Restore the adapter's original node */ 758 adapter->node = orig_node; 759 760 igb_cache_ring_register(adapter); 761 762 return 0; 763 764 err: 765 /* Restore the adapter's original node */ 766 adapter->node = orig_node; 767 igb_free_queues(adapter); 768 769 return -ENOMEM; 770 } 771 772 /** 773 * igb_write_ivar - configure ivar for given MSI-X vector 774 * @hw: pointer to the HW structure 775 * @msix_vector: vector number we are allocating to a given ring 776 * @index: row index of IVAR register to write within IVAR table 777 * @offset: column offset of in IVAR, should be multiple of 8 778 * 779 * This function is intended to handle the writing of the IVAR register 780 * for adapters 82576 and newer. The IVAR table consists of 2 columns, 781 * each containing an cause allocation for an Rx and Tx ring, and a 782 * variable number of rows depending on the number of queues supported. 783 **/ 784 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector, 785 int index, int offset) 786 { 787 u32 ivar = array_rd32(E1000_IVAR0, index); 788 789 /* clear any bits that are currently set */ 790 ivar &= ~((u32)0xFF << offset); 791 792 /* write vector and valid bit */ 793 ivar |= (msix_vector | E1000_IVAR_VALID) << offset; 794 795 array_wr32(E1000_IVAR0, index, ivar); 796 } 797 798 #define IGB_N0_QUEUE -1 799 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector) 800 { 801 struct igb_adapter *adapter = q_vector->adapter; 802 struct e1000_hw *hw = &adapter->hw; 803 int rx_queue = IGB_N0_QUEUE; 804 int tx_queue = IGB_N0_QUEUE; 805 u32 msixbm = 0; 806 807 if (q_vector->rx.ring) 808 rx_queue = q_vector->rx.ring->reg_idx; 809 if (q_vector->tx.ring) 810 tx_queue = q_vector->tx.ring->reg_idx; 811 812 switch (hw->mac.type) { 813 case e1000_82575: 814 /* The 82575 assigns vectors using a bitmask, which matches the 815 bitmask for the EICR/EIMS/EIMC registers. To assign one 816 or more queues to a vector, we write the appropriate bits 817 into the MSIXBM register for that vector. */ 818 if (rx_queue > IGB_N0_QUEUE) 819 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue; 820 if (tx_queue > IGB_N0_QUEUE) 821 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue; 822 if (!adapter->msix_entries && msix_vector == 0) 823 msixbm |= E1000_EIMS_OTHER; 824 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm); 825 q_vector->eims_value = msixbm; 826 break; 827 case e1000_82576: 828 /* 829 * 82576 uses a table that essentially consists of 2 columns 830 * with 8 rows. The ordering is column-major so we use the 831 * lower 3 bits as the row index, and the 4th bit as the 832 * column offset. 833 */ 834 if (rx_queue > IGB_N0_QUEUE) 835 igb_write_ivar(hw, msix_vector, 836 rx_queue & 0x7, 837 (rx_queue & 0x8) << 1); 838 if (tx_queue > IGB_N0_QUEUE) 839 igb_write_ivar(hw, msix_vector, 840 tx_queue & 0x7, 841 ((tx_queue & 0x8) << 1) + 8); 842 q_vector->eims_value = 1 << msix_vector; 843 break; 844 case e1000_82580: 845 case e1000_i350: 846 /* 847 * On 82580 and newer adapters the scheme is similar to 82576 848 * however instead of ordering column-major we have things 849 * ordered row-major. So we traverse the table by using 850 * bit 0 as the column offset, and the remaining bits as the 851 * row index. 852 */ 853 if (rx_queue > IGB_N0_QUEUE) 854 igb_write_ivar(hw, msix_vector, 855 rx_queue >> 1, 856 (rx_queue & 0x1) << 4); 857 if (tx_queue > IGB_N0_QUEUE) 858 igb_write_ivar(hw, msix_vector, 859 tx_queue >> 1, 860 ((tx_queue & 0x1) << 4) + 8); 861 q_vector->eims_value = 1 << msix_vector; 862 break; 863 default: 864 BUG(); 865 break; 866 } 867 868 /* add q_vector eims value to global eims_enable_mask */ 869 adapter->eims_enable_mask |= q_vector->eims_value; 870 871 /* configure q_vector to set itr on first interrupt */ 872 q_vector->set_itr = 1; 873 } 874 875 /** 876 * igb_configure_msix - Configure MSI-X hardware 877 * 878 * igb_configure_msix sets up the hardware to properly 879 * generate MSI-X interrupts. 880 **/ 881 static void igb_configure_msix(struct igb_adapter *adapter) 882 { 883 u32 tmp; 884 int i, vector = 0; 885 struct e1000_hw *hw = &adapter->hw; 886 887 adapter->eims_enable_mask = 0; 888 889 /* set vector for other causes, i.e. link changes */ 890 switch (hw->mac.type) { 891 case e1000_82575: 892 tmp = rd32(E1000_CTRL_EXT); 893 /* enable MSI-X PBA support*/ 894 tmp |= E1000_CTRL_EXT_PBA_CLR; 895 896 /* Auto-Mask interrupts upon ICR read. */ 897 tmp |= E1000_CTRL_EXT_EIAME; 898 tmp |= E1000_CTRL_EXT_IRCA; 899 900 wr32(E1000_CTRL_EXT, tmp); 901 902 /* enable msix_other interrupt */ 903 array_wr32(E1000_MSIXBM(0), vector++, 904 E1000_EIMS_OTHER); 905 adapter->eims_other = E1000_EIMS_OTHER; 906 907 break; 908 909 case e1000_82576: 910 case e1000_82580: 911 case e1000_i350: 912 /* Turn on MSI-X capability first, or our settings 913 * won't stick. And it will take days to debug. */ 914 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | 915 E1000_GPIE_PBA | E1000_GPIE_EIAME | 916 E1000_GPIE_NSICR); 917 918 /* enable msix_other interrupt */ 919 adapter->eims_other = 1 << vector; 920 tmp = (vector++ | E1000_IVAR_VALID) << 8; 921 922 wr32(E1000_IVAR_MISC, tmp); 923 break; 924 default: 925 /* do nothing, since nothing else supports MSI-X */ 926 break; 927 } /* switch (hw->mac.type) */ 928 929 adapter->eims_enable_mask |= adapter->eims_other; 930 931 for (i = 0; i < adapter->num_q_vectors; i++) 932 igb_assign_vector(adapter->q_vector[i], vector++); 933 934 wrfl(); 935 } 936 937 /** 938 * igb_request_msix - Initialize MSI-X interrupts 939 * 940 * igb_request_msix allocates MSI-X vectors and requests interrupts from the 941 * kernel. 942 **/ 943 static int igb_request_msix(struct igb_adapter *adapter) 944 { 945 struct net_device *netdev = adapter->netdev; 946 struct e1000_hw *hw = &adapter->hw; 947 int i, err = 0, vector = 0; 948 949 err = request_irq(adapter->msix_entries[vector].vector, 950 igb_msix_other, 0, netdev->name, adapter); 951 if (err) 952 goto out; 953 vector++; 954 955 for (i = 0; i < adapter->num_q_vectors; i++) { 956 struct igb_q_vector *q_vector = adapter->q_vector[i]; 957 958 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector); 959 960 if (q_vector->rx.ring && q_vector->tx.ring) 961 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 962 q_vector->rx.ring->queue_index); 963 else if (q_vector->tx.ring) 964 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 965 q_vector->tx.ring->queue_index); 966 else if (q_vector->rx.ring) 967 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 968 q_vector->rx.ring->queue_index); 969 else 970 sprintf(q_vector->name, "%s-unused", netdev->name); 971 972 err = request_irq(adapter->msix_entries[vector].vector, 973 igb_msix_ring, 0, q_vector->name, 974 q_vector); 975 if (err) 976 goto out; 977 vector++; 978 } 979 980 igb_configure_msix(adapter); 981 return 0; 982 out: 983 return err; 984 } 985 986 static void igb_reset_interrupt_capability(struct igb_adapter *adapter) 987 { 988 if (adapter->msix_entries) { 989 pci_disable_msix(adapter->pdev); 990 kfree(adapter->msix_entries); 991 adapter->msix_entries = NULL; 992 } else if (adapter->flags & IGB_FLAG_HAS_MSI) { 993 pci_disable_msi(adapter->pdev); 994 } 995 } 996 997 /** 998 * igb_free_q_vectors - Free memory allocated for interrupt vectors 999 * @adapter: board private structure to initialize 1000 * 1001 * This function frees the memory allocated to the q_vectors. In addition if 1002 * NAPI is enabled it will delete any references to the NAPI struct prior 1003 * to freeing the q_vector. 1004 **/ 1005 static void igb_free_q_vectors(struct igb_adapter *adapter) 1006 { 1007 int v_idx; 1008 1009 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { 1010 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1011 adapter->q_vector[v_idx] = NULL; 1012 if (!q_vector) 1013 continue; 1014 netif_napi_del(&q_vector->napi); 1015 kfree(q_vector); 1016 } 1017 adapter->num_q_vectors = 0; 1018 } 1019 1020 /** 1021 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts 1022 * 1023 * This function resets the device so that it has 0 rx queues, tx queues, and 1024 * MSI-X interrupts allocated. 1025 */ 1026 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter) 1027 { 1028 igb_free_queues(adapter); 1029 igb_free_q_vectors(adapter); 1030 igb_reset_interrupt_capability(adapter); 1031 } 1032 1033 /** 1034 * igb_set_interrupt_capability - set MSI or MSI-X if supported 1035 * 1036 * Attempt to configure interrupts using the best available 1037 * capabilities of the hardware and kernel. 1038 **/ 1039 static int igb_set_interrupt_capability(struct igb_adapter *adapter) 1040 { 1041 int err; 1042 int numvecs, i; 1043 1044 /* Number of supported queues. */ 1045 adapter->num_rx_queues = adapter->rss_queues; 1046 if (adapter->vfs_allocated_count) 1047 adapter->num_tx_queues = 1; 1048 else 1049 adapter->num_tx_queues = adapter->rss_queues; 1050 1051 /* start with one vector for every rx queue */ 1052 numvecs = adapter->num_rx_queues; 1053 1054 /* if tx handler is separate add 1 for every tx queue */ 1055 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS)) 1056 numvecs += adapter->num_tx_queues; 1057 1058 /* store the number of vectors reserved for queues */ 1059 adapter->num_q_vectors = numvecs; 1060 1061 /* add 1 vector for link status interrupts */ 1062 numvecs++; 1063 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 1064 GFP_KERNEL); 1065 if (!adapter->msix_entries) 1066 goto msi_only; 1067 1068 for (i = 0; i < numvecs; i++) 1069 adapter->msix_entries[i].entry = i; 1070 1071 err = pci_enable_msix(adapter->pdev, 1072 adapter->msix_entries, 1073 numvecs); 1074 if (err == 0) 1075 goto out; 1076 1077 igb_reset_interrupt_capability(adapter); 1078 1079 /* If we can't do MSI-X, try MSI */ 1080 msi_only: 1081 #ifdef CONFIG_PCI_IOV 1082 /* disable SR-IOV for non MSI-X configurations */ 1083 if (adapter->vf_data) { 1084 struct e1000_hw *hw = &adapter->hw; 1085 /* disable iov and allow time for transactions to clear */ 1086 pci_disable_sriov(adapter->pdev); 1087 msleep(500); 1088 1089 kfree(adapter->vf_data); 1090 adapter->vf_data = NULL; 1091 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 1092 wrfl(); 1093 msleep(100); 1094 dev_info(&adapter->pdev->dev, "IOV Disabled\n"); 1095 } 1096 #endif 1097 adapter->vfs_allocated_count = 0; 1098 adapter->rss_queues = 1; 1099 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 1100 adapter->num_rx_queues = 1; 1101 adapter->num_tx_queues = 1; 1102 adapter->num_q_vectors = 1; 1103 if (!pci_enable_msi(adapter->pdev)) 1104 adapter->flags |= IGB_FLAG_HAS_MSI; 1105 out: 1106 /* Notify the stack of the (possibly) reduced queue counts. */ 1107 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 1108 return netif_set_real_num_rx_queues(adapter->netdev, 1109 adapter->num_rx_queues); 1110 } 1111 1112 /** 1113 * igb_alloc_q_vectors - Allocate memory for interrupt vectors 1114 * @adapter: board private structure to initialize 1115 * 1116 * We allocate one q_vector per queue interrupt. If allocation fails we 1117 * return -ENOMEM. 1118 **/ 1119 static int igb_alloc_q_vectors(struct igb_adapter *adapter) 1120 { 1121 struct igb_q_vector *q_vector; 1122 struct e1000_hw *hw = &adapter->hw; 1123 int v_idx; 1124 int orig_node = adapter->node; 1125 1126 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { 1127 if ((adapter->num_q_vectors == (adapter->num_rx_queues + 1128 adapter->num_tx_queues)) && 1129 (adapter->num_rx_queues == v_idx)) 1130 adapter->node = orig_node; 1131 if (orig_node == -1) { 1132 int cur_node = next_online_node(adapter->node); 1133 if (cur_node == MAX_NUMNODES) 1134 cur_node = first_online_node; 1135 adapter->node = cur_node; 1136 } 1137 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL, 1138 adapter->node); 1139 if (!q_vector) 1140 q_vector = kzalloc(sizeof(struct igb_q_vector), 1141 GFP_KERNEL); 1142 if (!q_vector) 1143 goto err_out; 1144 q_vector->adapter = adapter; 1145 q_vector->itr_register = hw->hw_addr + E1000_EITR(0); 1146 q_vector->itr_val = IGB_START_ITR; 1147 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64); 1148 adapter->q_vector[v_idx] = q_vector; 1149 } 1150 /* Restore the adapter's original node */ 1151 adapter->node = orig_node; 1152 1153 return 0; 1154 1155 err_out: 1156 /* Restore the adapter's original node */ 1157 adapter->node = orig_node; 1158 igb_free_q_vectors(adapter); 1159 return -ENOMEM; 1160 } 1161 1162 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter, 1163 int ring_idx, int v_idx) 1164 { 1165 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1166 1167 q_vector->rx.ring = adapter->rx_ring[ring_idx]; 1168 q_vector->rx.ring->q_vector = q_vector; 1169 q_vector->rx.count++; 1170 q_vector->itr_val = adapter->rx_itr_setting; 1171 if (q_vector->itr_val && q_vector->itr_val <= 3) 1172 q_vector->itr_val = IGB_START_ITR; 1173 } 1174 1175 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter, 1176 int ring_idx, int v_idx) 1177 { 1178 struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; 1179 1180 q_vector->tx.ring = adapter->tx_ring[ring_idx]; 1181 q_vector->tx.ring->q_vector = q_vector; 1182 q_vector->tx.count++; 1183 q_vector->itr_val = adapter->tx_itr_setting; 1184 q_vector->tx.work_limit = adapter->tx_work_limit; 1185 if (q_vector->itr_val && q_vector->itr_val <= 3) 1186 q_vector->itr_val = IGB_START_ITR; 1187 } 1188 1189 /** 1190 * igb_map_ring_to_vector - maps allocated queues to vectors 1191 * 1192 * This function maps the recently allocated queues to vectors. 1193 **/ 1194 static int igb_map_ring_to_vector(struct igb_adapter *adapter) 1195 { 1196 int i; 1197 int v_idx = 0; 1198 1199 if ((adapter->num_q_vectors < adapter->num_rx_queues) || 1200 (adapter->num_q_vectors < adapter->num_tx_queues)) 1201 return -ENOMEM; 1202 1203 if (adapter->num_q_vectors >= 1204 (adapter->num_rx_queues + adapter->num_tx_queues)) { 1205 for (i = 0; i < adapter->num_rx_queues; i++) 1206 igb_map_rx_ring_to_vector(adapter, i, v_idx++); 1207 for (i = 0; i < adapter->num_tx_queues; i++) 1208 igb_map_tx_ring_to_vector(adapter, i, v_idx++); 1209 } else { 1210 for (i = 0; i < adapter->num_rx_queues; i++) { 1211 if (i < adapter->num_tx_queues) 1212 igb_map_tx_ring_to_vector(adapter, i, v_idx); 1213 igb_map_rx_ring_to_vector(adapter, i, v_idx++); 1214 } 1215 for (; i < adapter->num_tx_queues; i++) 1216 igb_map_tx_ring_to_vector(adapter, i, v_idx++); 1217 } 1218 return 0; 1219 } 1220 1221 /** 1222 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 1223 * 1224 * This function initializes the interrupts and allocates all of the queues. 1225 **/ 1226 static int igb_init_interrupt_scheme(struct igb_adapter *adapter) 1227 { 1228 struct pci_dev *pdev = adapter->pdev; 1229 int err; 1230 1231 err = igb_set_interrupt_capability(adapter); 1232 if (err) 1233 return err; 1234 1235 err = igb_alloc_q_vectors(adapter); 1236 if (err) { 1237 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 1238 goto err_alloc_q_vectors; 1239 } 1240 1241 err = igb_alloc_queues(adapter); 1242 if (err) { 1243 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 1244 goto err_alloc_queues; 1245 } 1246 1247 err = igb_map_ring_to_vector(adapter); 1248 if (err) { 1249 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n"); 1250 goto err_map_queues; 1251 } 1252 1253 1254 return 0; 1255 err_map_queues: 1256 igb_free_queues(adapter); 1257 err_alloc_queues: 1258 igb_free_q_vectors(adapter); 1259 err_alloc_q_vectors: 1260 igb_reset_interrupt_capability(adapter); 1261 return err; 1262 } 1263 1264 /** 1265 * igb_request_irq - initialize interrupts 1266 * 1267 * Attempts to configure interrupts using the best available 1268 * capabilities of the hardware and kernel. 1269 **/ 1270 static int igb_request_irq(struct igb_adapter *adapter) 1271 { 1272 struct net_device *netdev = adapter->netdev; 1273 struct pci_dev *pdev = adapter->pdev; 1274 int err = 0; 1275 1276 if (adapter->msix_entries) { 1277 err = igb_request_msix(adapter); 1278 if (!err) 1279 goto request_done; 1280 /* fall back to MSI */ 1281 igb_clear_interrupt_scheme(adapter); 1282 if (!pci_enable_msi(pdev)) 1283 adapter->flags |= IGB_FLAG_HAS_MSI; 1284 igb_free_all_tx_resources(adapter); 1285 igb_free_all_rx_resources(adapter); 1286 adapter->num_tx_queues = 1; 1287 adapter->num_rx_queues = 1; 1288 adapter->num_q_vectors = 1; 1289 err = igb_alloc_q_vectors(adapter); 1290 if (err) { 1291 dev_err(&pdev->dev, 1292 "Unable to allocate memory for vectors\n"); 1293 goto request_done; 1294 } 1295 err = igb_alloc_queues(adapter); 1296 if (err) { 1297 dev_err(&pdev->dev, 1298 "Unable to allocate memory for queues\n"); 1299 igb_free_q_vectors(adapter); 1300 goto request_done; 1301 } 1302 igb_setup_all_tx_resources(adapter); 1303 igb_setup_all_rx_resources(adapter); 1304 } 1305 1306 igb_assign_vector(adapter->q_vector[0], 0); 1307 1308 if (adapter->flags & IGB_FLAG_HAS_MSI) { 1309 err = request_irq(pdev->irq, igb_intr_msi, 0, 1310 netdev->name, adapter); 1311 if (!err) 1312 goto request_done; 1313 1314 /* fall back to legacy interrupts */ 1315 igb_reset_interrupt_capability(adapter); 1316 adapter->flags &= ~IGB_FLAG_HAS_MSI; 1317 } 1318 1319 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED, 1320 netdev->name, adapter); 1321 1322 if (err) 1323 dev_err(&pdev->dev, "Error %d getting interrupt\n", 1324 err); 1325 1326 request_done: 1327 return err; 1328 } 1329 1330 static void igb_free_irq(struct igb_adapter *adapter) 1331 { 1332 if (adapter->msix_entries) { 1333 int vector = 0, i; 1334 1335 free_irq(adapter->msix_entries[vector++].vector, adapter); 1336 1337 for (i = 0; i < adapter->num_q_vectors; i++) 1338 free_irq(adapter->msix_entries[vector++].vector, 1339 adapter->q_vector[i]); 1340 } else { 1341 free_irq(adapter->pdev->irq, adapter); 1342 } 1343 } 1344 1345 /** 1346 * igb_irq_disable - Mask off interrupt generation on the NIC 1347 * @adapter: board private structure 1348 **/ 1349 static void igb_irq_disable(struct igb_adapter *adapter) 1350 { 1351 struct e1000_hw *hw = &adapter->hw; 1352 1353 /* 1354 * we need to be careful when disabling interrupts. The VFs are also 1355 * mapped into these registers and so clearing the bits can cause 1356 * issues on the VF drivers so we only need to clear what we set 1357 */ 1358 if (adapter->msix_entries) { 1359 u32 regval = rd32(E1000_EIAM); 1360 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask); 1361 wr32(E1000_EIMC, adapter->eims_enable_mask); 1362 regval = rd32(E1000_EIAC); 1363 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask); 1364 } 1365 1366 wr32(E1000_IAM, 0); 1367 wr32(E1000_IMC, ~0); 1368 wrfl(); 1369 if (adapter->msix_entries) { 1370 int i; 1371 for (i = 0; i < adapter->num_q_vectors; i++) 1372 synchronize_irq(adapter->msix_entries[i].vector); 1373 } else { 1374 synchronize_irq(adapter->pdev->irq); 1375 } 1376 } 1377 1378 /** 1379 * igb_irq_enable - Enable default interrupt generation settings 1380 * @adapter: board private structure 1381 **/ 1382 static void igb_irq_enable(struct igb_adapter *adapter) 1383 { 1384 struct e1000_hw *hw = &adapter->hw; 1385 1386 if (adapter->msix_entries) { 1387 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA; 1388 u32 regval = rd32(E1000_EIAC); 1389 wr32(E1000_EIAC, regval | adapter->eims_enable_mask); 1390 regval = rd32(E1000_EIAM); 1391 wr32(E1000_EIAM, regval | adapter->eims_enable_mask); 1392 wr32(E1000_EIMS, adapter->eims_enable_mask); 1393 if (adapter->vfs_allocated_count) { 1394 wr32(E1000_MBVFIMR, 0xFF); 1395 ims |= E1000_IMS_VMMB; 1396 } 1397 wr32(E1000_IMS, ims); 1398 } else { 1399 wr32(E1000_IMS, IMS_ENABLE_MASK | 1400 E1000_IMS_DRSTA); 1401 wr32(E1000_IAM, IMS_ENABLE_MASK | 1402 E1000_IMS_DRSTA); 1403 } 1404 } 1405 1406 static void igb_update_mng_vlan(struct igb_adapter *adapter) 1407 { 1408 struct e1000_hw *hw = &adapter->hw; 1409 u16 vid = adapter->hw.mng_cookie.vlan_id; 1410 u16 old_vid = adapter->mng_vlan_id; 1411 1412 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 1413 /* add VID to filter table */ 1414 igb_vfta_set(hw, vid, true); 1415 adapter->mng_vlan_id = vid; 1416 } else { 1417 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; 1418 } 1419 1420 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && 1421 (vid != old_vid) && 1422 !test_bit(old_vid, adapter->active_vlans)) { 1423 /* remove VID from filter table */ 1424 igb_vfta_set(hw, old_vid, false); 1425 } 1426 } 1427 1428 /** 1429 * igb_release_hw_control - release control of the h/w to f/w 1430 * @adapter: address of board private structure 1431 * 1432 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 1433 * For ASF and Pass Through versions of f/w this means that the 1434 * driver is no longer loaded. 1435 * 1436 **/ 1437 static void igb_release_hw_control(struct igb_adapter *adapter) 1438 { 1439 struct e1000_hw *hw = &adapter->hw; 1440 u32 ctrl_ext; 1441 1442 /* Let firmware take over control of h/w */ 1443 ctrl_ext = rd32(E1000_CTRL_EXT); 1444 wr32(E1000_CTRL_EXT, 1445 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 1446 } 1447 1448 /** 1449 * igb_get_hw_control - get control of the h/w from f/w 1450 * @adapter: address of board private structure 1451 * 1452 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 1453 * For ASF and Pass Through versions of f/w this means that 1454 * the driver is loaded. 1455 * 1456 **/ 1457 static void igb_get_hw_control(struct igb_adapter *adapter) 1458 { 1459 struct e1000_hw *hw = &adapter->hw; 1460 u32 ctrl_ext; 1461 1462 /* Let firmware know the driver has taken over */ 1463 ctrl_ext = rd32(E1000_CTRL_EXT); 1464 wr32(E1000_CTRL_EXT, 1465 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 1466 } 1467 1468 /** 1469 * igb_configure - configure the hardware for RX and TX 1470 * @adapter: private board structure 1471 **/ 1472 static void igb_configure(struct igb_adapter *adapter) 1473 { 1474 struct net_device *netdev = adapter->netdev; 1475 int i; 1476 1477 igb_get_hw_control(adapter); 1478 igb_set_rx_mode(netdev); 1479 1480 igb_restore_vlan(adapter); 1481 1482 igb_setup_tctl(adapter); 1483 igb_setup_mrqc(adapter); 1484 igb_setup_rctl(adapter); 1485 1486 igb_configure_tx(adapter); 1487 igb_configure_rx(adapter); 1488 1489 igb_rx_fifo_flush_82575(&adapter->hw); 1490 1491 /* call igb_desc_unused which always leaves 1492 * at least 1 descriptor unused to make sure 1493 * next_to_use != next_to_clean */ 1494 for (i = 0; i < adapter->num_rx_queues; i++) { 1495 struct igb_ring *ring = adapter->rx_ring[i]; 1496 igb_alloc_rx_buffers(ring, igb_desc_unused(ring)); 1497 } 1498 } 1499 1500 /** 1501 * igb_power_up_link - Power up the phy/serdes link 1502 * @adapter: address of board private structure 1503 **/ 1504 void igb_power_up_link(struct igb_adapter *adapter) 1505 { 1506 if (adapter->hw.phy.media_type == e1000_media_type_copper) 1507 igb_power_up_phy_copper(&adapter->hw); 1508 else 1509 igb_power_up_serdes_link_82575(&adapter->hw); 1510 igb_reset_phy(&adapter->hw); 1511 } 1512 1513 /** 1514 * igb_power_down_link - Power down the phy/serdes link 1515 * @adapter: address of board private structure 1516 */ 1517 static void igb_power_down_link(struct igb_adapter *adapter) 1518 { 1519 if (adapter->hw.phy.media_type == e1000_media_type_copper) 1520 igb_power_down_phy_copper_82575(&adapter->hw); 1521 else 1522 igb_shutdown_serdes_link_82575(&adapter->hw); 1523 } 1524 1525 /** 1526 * igb_up - Open the interface and prepare it to handle traffic 1527 * @adapter: board private structure 1528 **/ 1529 int igb_up(struct igb_adapter *adapter) 1530 { 1531 struct e1000_hw *hw = &adapter->hw; 1532 int i; 1533 1534 /* hardware has been reset, we need to reload some things */ 1535 igb_configure(adapter); 1536 1537 clear_bit(__IGB_DOWN, &adapter->state); 1538 1539 for (i = 0; i < adapter->num_q_vectors; i++) 1540 napi_enable(&(adapter->q_vector[i]->napi)); 1541 1542 if (adapter->msix_entries) 1543 igb_configure_msix(adapter); 1544 else 1545 igb_assign_vector(adapter->q_vector[0], 0); 1546 1547 /* Clear any pending interrupts. */ 1548 rd32(E1000_ICR); 1549 igb_irq_enable(adapter); 1550 1551 /* notify VFs that reset has been completed */ 1552 if (adapter->vfs_allocated_count) { 1553 u32 reg_data = rd32(E1000_CTRL_EXT); 1554 reg_data |= E1000_CTRL_EXT_PFRSTD; 1555 wr32(E1000_CTRL_EXT, reg_data); 1556 } 1557 1558 netif_tx_start_all_queues(adapter->netdev); 1559 1560 /* start the watchdog. */ 1561 hw->mac.get_link_status = 1; 1562 schedule_work(&adapter->watchdog_task); 1563 1564 return 0; 1565 } 1566 1567 void igb_down(struct igb_adapter *adapter) 1568 { 1569 struct net_device *netdev = adapter->netdev; 1570 struct e1000_hw *hw = &adapter->hw; 1571 u32 tctl, rctl; 1572 int i; 1573 1574 /* signal that we're down so the interrupt handler does not 1575 * reschedule our watchdog timer */ 1576 set_bit(__IGB_DOWN, &adapter->state); 1577 1578 /* disable receives in the hardware */ 1579 rctl = rd32(E1000_RCTL); 1580 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN); 1581 /* flush and sleep below */ 1582 1583 netif_tx_stop_all_queues(netdev); 1584 1585 /* disable transmits in the hardware */ 1586 tctl = rd32(E1000_TCTL); 1587 tctl &= ~E1000_TCTL_EN; 1588 wr32(E1000_TCTL, tctl); 1589 /* flush both disables and wait for them to finish */ 1590 wrfl(); 1591 msleep(10); 1592 1593 for (i = 0; i < adapter->num_q_vectors; i++) 1594 napi_disable(&(adapter->q_vector[i]->napi)); 1595 1596 igb_irq_disable(adapter); 1597 1598 del_timer_sync(&adapter->watchdog_timer); 1599 del_timer_sync(&adapter->phy_info_timer); 1600 1601 netif_carrier_off(netdev); 1602 1603 /* record the stats before reset*/ 1604 spin_lock(&adapter->stats64_lock); 1605 igb_update_stats(adapter, &adapter->stats64); 1606 spin_unlock(&adapter->stats64_lock); 1607 1608 adapter->link_speed = 0; 1609 adapter->link_duplex = 0; 1610 1611 if (!pci_channel_offline(adapter->pdev)) 1612 igb_reset(adapter); 1613 igb_clean_all_tx_rings(adapter); 1614 igb_clean_all_rx_rings(adapter); 1615 #ifdef CONFIG_IGB_DCA 1616 1617 /* since we reset the hardware DCA settings were cleared */ 1618 igb_setup_dca(adapter); 1619 #endif 1620 } 1621 1622 void igb_reinit_locked(struct igb_adapter *adapter) 1623 { 1624 WARN_ON(in_interrupt()); 1625 while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) 1626 msleep(1); 1627 igb_down(adapter); 1628 igb_up(adapter); 1629 clear_bit(__IGB_RESETTING, &adapter->state); 1630 } 1631 1632 void igb_reset(struct igb_adapter *adapter) 1633 { 1634 struct pci_dev *pdev = adapter->pdev; 1635 struct e1000_hw *hw = &adapter->hw; 1636 struct e1000_mac_info *mac = &hw->mac; 1637 struct e1000_fc_info *fc = &hw->fc; 1638 u32 pba = 0, tx_space, min_tx_space, min_rx_space; 1639 u16 hwm; 1640 1641 /* Repartition Pba for greater than 9k mtu 1642 * To take effect CTRL.RST is required. 1643 */ 1644 switch (mac->type) { 1645 case e1000_i350: 1646 case e1000_82580: 1647 pba = rd32(E1000_RXPBS); 1648 pba = igb_rxpbs_adjust_82580(pba); 1649 break; 1650 case e1000_82576: 1651 pba = rd32(E1000_RXPBS); 1652 pba &= E1000_RXPBS_SIZE_MASK_82576; 1653 break; 1654 case e1000_82575: 1655 default: 1656 pba = E1000_PBA_34K; 1657 break; 1658 } 1659 1660 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) && 1661 (mac->type < e1000_82576)) { 1662 /* adjust PBA for jumbo frames */ 1663 wr32(E1000_PBA, pba); 1664 1665 /* To maintain wire speed transmits, the Tx FIFO should be 1666 * large enough to accommodate two full transmit packets, 1667 * rounded up to the next 1KB and expressed in KB. Likewise, 1668 * the Rx FIFO should be large enough to accommodate at least 1669 * one full receive packet and is similarly rounded up and 1670 * expressed in KB. */ 1671 pba = rd32(E1000_PBA); 1672 /* upper 16 bits has Tx packet buffer allocation size in KB */ 1673 tx_space = pba >> 16; 1674 /* lower 16 bits has Rx packet buffer allocation size in KB */ 1675 pba &= 0xffff; 1676 /* the tx fifo also stores 16 bytes of information about the tx 1677 * but don't include ethernet FCS because hardware appends it */ 1678 min_tx_space = (adapter->max_frame_size + 1679 sizeof(union e1000_adv_tx_desc) - 1680 ETH_FCS_LEN) * 2; 1681 min_tx_space = ALIGN(min_tx_space, 1024); 1682 min_tx_space >>= 10; 1683 /* software strips receive CRC, so leave room for it */ 1684 min_rx_space = adapter->max_frame_size; 1685 min_rx_space = ALIGN(min_rx_space, 1024); 1686 min_rx_space >>= 10; 1687 1688 /* If current Tx allocation is less than the min Tx FIFO size, 1689 * and the min Tx FIFO size is less than the current Rx FIFO 1690 * allocation, take space away from current Rx allocation */ 1691 if (tx_space < min_tx_space && 1692 ((min_tx_space - tx_space) < pba)) { 1693 pba = pba - (min_tx_space - tx_space); 1694 1695 /* if short on rx space, rx wins and must trump tx 1696 * adjustment */ 1697 if (pba < min_rx_space) 1698 pba = min_rx_space; 1699 } 1700 wr32(E1000_PBA, pba); 1701 } 1702 1703 /* flow control settings */ 1704 /* The high water mark must be low enough to fit one full frame 1705 * (or the size used for early receive) above it in the Rx FIFO. 1706 * Set it to the lower of: 1707 * - 90% of the Rx FIFO size, or 1708 * - the full Rx FIFO size minus one full frame */ 1709 hwm = min(((pba << 10) * 9 / 10), 1710 ((pba << 10) - 2 * adapter->max_frame_size)); 1711 1712 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ 1713 fc->low_water = fc->high_water - 16; 1714 fc->pause_time = 0xFFFF; 1715 fc->send_xon = 1; 1716 fc->current_mode = fc->requested_mode; 1717 1718 /* disable receive for all VFs and wait one second */ 1719 if (adapter->vfs_allocated_count) { 1720 int i; 1721 for (i = 0 ; i < adapter->vfs_allocated_count; i++) 1722 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC; 1723 1724 /* ping all the active vfs to let them know we are going down */ 1725 igb_ping_all_vfs(adapter); 1726 1727 /* disable transmits and receives */ 1728 wr32(E1000_VFRE, 0); 1729 wr32(E1000_VFTE, 0); 1730 } 1731 1732 /* Allow time for pending master requests to run */ 1733 hw->mac.ops.reset_hw(hw); 1734 wr32(E1000_WUC, 0); 1735 1736 if (hw->mac.ops.init_hw(hw)) 1737 dev_err(&pdev->dev, "Hardware Error\n"); 1738 1739 igb_init_dmac(adapter, pba); 1740 if (!netif_running(adapter->netdev)) 1741 igb_power_down_link(adapter); 1742 1743 igb_update_mng_vlan(adapter); 1744 1745 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 1746 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE); 1747 1748 igb_get_phy_info(hw); 1749 } 1750 1751 static netdev_features_t igb_fix_features(struct net_device *netdev, 1752 netdev_features_t features) 1753 { 1754 /* 1755 * Since there is no support for separate rx/tx vlan accel 1756 * enable/disable make sure tx flag is always in same state as rx. 1757 */ 1758 if (features & NETIF_F_HW_VLAN_RX) 1759 features |= NETIF_F_HW_VLAN_TX; 1760 else 1761 features &= ~NETIF_F_HW_VLAN_TX; 1762 1763 return features; 1764 } 1765 1766 static int igb_set_features(struct net_device *netdev, 1767 netdev_features_t features) 1768 { 1769 netdev_features_t changed = netdev->features ^ features; 1770 1771 if (changed & NETIF_F_HW_VLAN_RX) 1772 igb_vlan_mode(netdev, features); 1773 1774 return 0; 1775 } 1776 1777 static const struct net_device_ops igb_netdev_ops = { 1778 .ndo_open = igb_open, 1779 .ndo_stop = igb_close, 1780 .ndo_start_xmit = igb_xmit_frame, 1781 .ndo_get_stats64 = igb_get_stats64, 1782 .ndo_set_rx_mode = igb_set_rx_mode, 1783 .ndo_set_mac_address = igb_set_mac, 1784 .ndo_change_mtu = igb_change_mtu, 1785 .ndo_do_ioctl = igb_ioctl, 1786 .ndo_tx_timeout = igb_tx_timeout, 1787 .ndo_validate_addr = eth_validate_addr, 1788 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid, 1789 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid, 1790 .ndo_set_vf_mac = igb_ndo_set_vf_mac, 1791 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan, 1792 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw, 1793 .ndo_get_vf_config = igb_ndo_get_vf_config, 1794 #ifdef CONFIG_NET_POLL_CONTROLLER 1795 .ndo_poll_controller = igb_netpoll, 1796 #endif 1797 .ndo_fix_features = igb_fix_features, 1798 .ndo_set_features = igb_set_features, 1799 }; 1800 1801 /** 1802 * igb_probe - Device Initialization Routine 1803 * @pdev: PCI device information struct 1804 * @ent: entry in igb_pci_tbl 1805 * 1806 * Returns 0 on success, negative on failure 1807 * 1808 * igb_probe initializes an adapter identified by a pci_dev structure. 1809 * The OS initialization, configuring of the adapter private structure, 1810 * and a hardware reset occur. 1811 **/ 1812 static int __devinit igb_probe(struct pci_dev *pdev, 1813 const struct pci_device_id *ent) 1814 { 1815 struct net_device *netdev; 1816 struct igb_adapter *adapter; 1817 struct e1000_hw *hw; 1818 u16 eeprom_data = 0; 1819 s32 ret_val; 1820 static int global_quad_port_a; /* global quad port a indication */ 1821 const struct e1000_info *ei = igb_info_tbl[ent->driver_data]; 1822 unsigned long mmio_start, mmio_len; 1823 int err, pci_using_dac; 1824 u16 eeprom_apme_mask = IGB_EEPROM_APME; 1825 u8 part_str[E1000_PBANUM_LENGTH]; 1826 1827 /* Catch broken hardware that put the wrong VF device ID in 1828 * the PCIe SR-IOV capability. 1829 */ 1830 if (pdev->is_virtfn) { 1831 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", 1832 pci_name(pdev), pdev->vendor, pdev->device); 1833 return -EINVAL; 1834 } 1835 1836 err = pci_enable_device_mem(pdev); 1837 if (err) 1838 return err; 1839 1840 pci_using_dac = 0; 1841 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 1842 if (!err) { 1843 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 1844 if (!err) 1845 pci_using_dac = 1; 1846 } else { 1847 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 1848 if (err) { 1849 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); 1850 if (err) { 1851 dev_err(&pdev->dev, "No usable DMA " 1852 "configuration, aborting\n"); 1853 goto err_dma; 1854 } 1855 } 1856 } 1857 1858 err = pci_request_selected_regions(pdev, pci_select_bars(pdev, 1859 IORESOURCE_MEM), 1860 igb_driver_name); 1861 if (err) 1862 goto err_pci_reg; 1863 1864 pci_enable_pcie_error_reporting(pdev); 1865 1866 pci_set_master(pdev); 1867 pci_save_state(pdev); 1868 1869 err = -ENOMEM; 1870 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter), 1871 IGB_MAX_TX_QUEUES); 1872 if (!netdev) 1873 goto err_alloc_etherdev; 1874 1875 SET_NETDEV_DEV(netdev, &pdev->dev); 1876 1877 pci_set_drvdata(pdev, netdev); 1878 adapter = netdev_priv(netdev); 1879 adapter->netdev = netdev; 1880 adapter->pdev = pdev; 1881 hw = &adapter->hw; 1882 hw->back = adapter; 1883 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE; 1884 1885 mmio_start = pci_resource_start(pdev, 0); 1886 mmio_len = pci_resource_len(pdev, 0); 1887 1888 err = -EIO; 1889 hw->hw_addr = ioremap(mmio_start, mmio_len); 1890 if (!hw->hw_addr) 1891 goto err_ioremap; 1892 1893 netdev->netdev_ops = &igb_netdev_ops; 1894 igb_set_ethtool_ops(netdev); 1895 netdev->watchdog_timeo = 5 * HZ; 1896 1897 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 1898 1899 netdev->mem_start = mmio_start; 1900 netdev->mem_end = mmio_start + mmio_len; 1901 1902 /* PCI config space info */ 1903 hw->vendor_id = pdev->vendor; 1904 hw->device_id = pdev->device; 1905 hw->revision_id = pdev->revision; 1906 hw->subsystem_vendor_id = pdev->subsystem_vendor; 1907 hw->subsystem_device_id = pdev->subsystem_device; 1908 1909 /* Copy the default MAC, PHY and NVM function pointers */ 1910 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 1911 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 1912 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 1913 /* Initialize skew-specific constants */ 1914 err = ei->get_invariants(hw); 1915 if (err) 1916 goto err_sw_init; 1917 1918 /* setup the private structure */ 1919 err = igb_sw_init(adapter); 1920 if (err) 1921 goto err_sw_init; 1922 1923 igb_get_bus_info_pcie(hw); 1924 1925 hw->phy.autoneg_wait_to_complete = false; 1926 1927 /* Copper options */ 1928 if (hw->phy.media_type == e1000_media_type_copper) { 1929 hw->phy.mdix = AUTO_ALL_MODES; 1930 hw->phy.disable_polarity_correction = false; 1931 hw->phy.ms_type = e1000_ms_hw_default; 1932 } 1933 1934 if (igb_check_reset_block(hw)) 1935 dev_info(&pdev->dev, 1936 "PHY reset is blocked due to SOL/IDER session.\n"); 1937 1938 /* 1939 * features is initialized to 0 in allocation, it might have bits 1940 * set by igb_sw_init so we should use an or instead of an 1941 * assignment. 1942 */ 1943 netdev->features |= NETIF_F_SG | 1944 NETIF_F_IP_CSUM | 1945 NETIF_F_IPV6_CSUM | 1946 NETIF_F_TSO | 1947 NETIF_F_TSO6 | 1948 NETIF_F_RXHASH | 1949 NETIF_F_RXCSUM | 1950 NETIF_F_HW_VLAN_RX | 1951 NETIF_F_HW_VLAN_TX; 1952 1953 /* copy netdev features into list of user selectable features */ 1954 netdev->hw_features |= netdev->features; 1955 1956 /* set this bit last since it cannot be part of hw_features */ 1957 netdev->features |= NETIF_F_HW_VLAN_FILTER; 1958 1959 netdev->vlan_features |= NETIF_F_TSO | 1960 NETIF_F_TSO6 | 1961 NETIF_F_IP_CSUM | 1962 NETIF_F_IPV6_CSUM | 1963 NETIF_F_SG; 1964 1965 if (pci_using_dac) { 1966 netdev->features |= NETIF_F_HIGHDMA; 1967 netdev->vlan_features |= NETIF_F_HIGHDMA; 1968 } 1969 1970 if (hw->mac.type >= e1000_82576) { 1971 netdev->hw_features |= NETIF_F_SCTP_CSUM; 1972 netdev->features |= NETIF_F_SCTP_CSUM; 1973 } 1974 1975 netdev->priv_flags |= IFF_UNICAST_FLT; 1976 1977 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw); 1978 1979 /* before reading the NVM, reset the controller to put the device in a 1980 * known good starting state */ 1981 hw->mac.ops.reset_hw(hw); 1982 1983 /* make sure the NVM is good */ 1984 if (hw->nvm.ops.validate(hw) < 0) { 1985 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 1986 err = -EIO; 1987 goto err_eeprom; 1988 } 1989 1990 /* copy the MAC address out of the NVM */ 1991 if (hw->mac.ops.read_mac_addr(hw)) 1992 dev_err(&pdev->dev, "NVM Read Error\n"); 1993 1994 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 1995 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len); 1996 1997 if (!is_valid_ether_addr(netdev->perm_addr)) { 1998 dev_err(&pdev->dev, "Invalid MAC Address\n"); 1999 err = -EIO; 2000 goto err_eeprom; 2001 } 2002 2003 setup_timer(&adapter->watchdog_timer, igb_watchdog, 2004 (unsigned long) adapter); 2005 setup_timer(&adapter->phy_info_timer, igb_update_phy_info, 2006 (unsigned long) adapter); 2007 2008 INIT_WORK(&adapter->reset_task, igb_reset_task); 2009 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task); 2010 2011 /* Initialize link properties that are user-changeable */ 2012 adapter->fc_autoneg = true; 2013 hw->mac.autoneg = true; 2014 hw->phy.autoneg_advertised = 0x2f; 2015 2016 hw->fc.requested_mode = e1000_fc_default; 2017 hw->fc.current_mode = e1000_fc_default; 2018 2019 igb_validate_mdi_setting(hw); 2020 2021 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM, 2022 * enable the ACPI Magic Packet filter 2023 */ 2024 2025 if (hw->bus.func == 0) 2026 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 2027 else if (hw->mac.type >= e1000_82580) 2028 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A + 2029 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1, 2030 &eeprom_data); 2031 else if (hw->bus.func == 1) 2032 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 2033 2034 if (eeprom_data & eeprom_apme_mask) 2035 adapter->eeprom_wol |= E1000_WUFC_MAG; 2036 2037 /* now that we have the eeprom settings, apply the special cases where 2038 * the eeprom may be wrong or the board simply won't support wake on 2039 * lan on a particular port */ 2040 switch (pdev->device) { 2041 case E1000_DEV_ID_82575GB_QUAD_COPPER: 2042 adapter->eeprom_wol = 0; 2043 break; 2044 case E1000_DEV_ID_82575EB_FIBER_SERDES: 2045 case E1000_DEV_ID_82576_FIBER: 2046 case E1000_DEV_ID_82576_SERDES: 2047 /* Wake events only supported on port A for dual fiber 2048 * regardless of eeprom setting */ 2049 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1) 2050 adapter->eeprom_wol = 0; 2051 break; 2052 case E1000_DEV_ID_82576_QUAD_COPPER: 2053 case E1000_DEV_ID_82576_QUAD_COPPER_ET2: 2054 /* if quad port adapter, disable WoL on all but port A */ 2055 if (global_quad_port_a != 0) 2056 adapter->eeprom_wol = 0; 2057 else 2058 adapter->flags |= IGB_FLAG_QUAD_PORT_A; 2059 /* Reset for multiple quad port adapters */ 2060 if (++global_quad_port_a == 4) 2061 global_quad_port_a = 0; 2062 break; 2063 } 2064 2065 /* initialize the wol settings based on the eeprom settings */ 2066 adapter->wol = adapter->eeprom_wol; 2067 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); 2068 2069 /* reset the hardware with the new settings */ 2070 igb_reset(adapter); 2071 2072 /* let the f/w know that the h/w is now under the control of the 2073 * driver. */ 2074 igb_get_hw_control(adapter); 2075 2076 strcpy(netdev->name, "eth%d"); 2077 err = register_netdev(netdev); 2078 if (err) 2079 goto err_register; 2080 2081 /* carrier off reporting is important to ethtool even BEFORE open */ 2082 netif_carrier_off(netdev); 2083 2084 #ifdef CONFIG_IGB_DCA 2085 if (dca_add_requester(&pdev->dev) == 0) { 2086 adapter->flags |= IGB_FLAG_DCA_ENABLED; 2087 dev_info(&pdev->dev, "DCA enabled\n"); 2088 igb_setup_dca(adapter); 2089 } 2090 2091 #endif 2092 /* do hw tstamp init after resetting */ 2093 igb_init_hw_timer(adapter); 2094 2095 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n"); 2096 /* print bus type/speed/width info */ 2097 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n", 2098 netdev->name, 2099 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" : 2100 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" : 2101 "unknown"), 2102 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" : 2103 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" : 2104 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" : 2105 "unknown"), 2106 netdev->dev_addr); 2107 2108 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH); 2109 if (ret_val) 2110 strcpy(part_str, "Unknown"); 2111 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str); 2112 dev_info(&pdev->dev, 2113 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n", 2114 adapter->msix_entries ? "MSI-X" : 2115 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy", 2116 adapter->num_rx_queues, adapter->num_tx_queues); 2117 switch (hw->mac.type) { 2118 case e1000_i350: 2119 igb_set_eee_i350(hw); 2120 break; 2121 default: 2122 break; 2123 } 2124 2125 pm_runtime_put_noidle(&pdev->dev); 2126 return 0; 2127 2128 err_register: 2129 igb_release_hw_control(adapter); 2130 err_eeprom: 2131 if (!igb_check_reset_block(hw)) 2132 igb_reset_phy(hw); 2133 2134 if (hw->flash_address) 2135 iounmap(hw->flash_address); 2136 err_sw_init: 2137 igb_clear_interrupt_scheme(adapter); 2138 iounmap(hw->hw_addr); 2139 err_ioremap: 2140 free_netdev(netdev); 2141 err_alloc_etherdev: 2142 pci_release_selected_regions(pdev, 2143 pci_select_bars(pdev, IORESOURCE_MEM)); 2144 err_pci_reg: 2145 err_dma: 2146 pci_disable_device(pdev); 2147 return err; 2148 } 2149 2150 /** 2151 * igb_remove - Device Removal Routine 2152 * @pdev: PCI device information struct 2153 * 2154 * igb_remove is called by the PCI subsystem to alert the driver 2155 * that it should release a PCI device. The could be caused by a 2156 * Hot-Plug event, or because the driver is going to be removed from 2157 * memory. 2158 **/ 2159 static void __devexit igb_remove(struct pci_dev *pdev) 2160 { 2161 struct net_device *netdev = pci_get_drvdata(pdev); 2162 struct igb_adapter *adapter = netdev_priv(netdev); 2163 struct e1000_hw *hw = &adapter->hw; 2164 2165 pm_runtime_get_noresume(&pdev->dev); 2166 2167 /* 2168 * The watchdog timer may be rescheduled, so explicitly 2169 * disable watchdog from being rescheduled. 2170 */ 2171 set_bit(__IGB_DOWN, &adapter->state); 2172 del_timer_sync(&adapter->watchdog_timer); 2173 del_timer_sync(&adapter->phy_info_timer); 2174 2175 cancel_work_sync(&adapter->reset_task); 2176 cancel_work_sync(&adapter->watchdog_task); 2177 2178 #ifdef CONFIG_IGB_DCA 2179 if (adapter->flags & IGB_FLAG_DCA_ENABLED) { 2180 dev_info(&pdev->dev, "DCA disabled\n"); 2181 dca_remove_requester(&pdev->dev); 2182 adapter->flags &= ~IGB_FLAG_DCA_ENABLED; 2183 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); 2184 } 2185 #endif 2186 2187 /* Release control of h/w to f/w. If f/w is AMT enabled, this 2188 * would have already happened in close and is redundant. */ 2189 igb_release_hw_control(adapter); 2190 2191 unregister_netdev(netdev); 2192 2193 igb_clear_interrupt_scheme(adapter); 2194 2195 #ifdef CONFIG_PCI_IOV 2196 /* reclaim resources allocated to VFs */ 2197 if (adapter->vf_data) { 2198 /* disable iov and allow time for transactions to clear */ 2199 if (!igb_check_vf_assignment(adapter)) { 2200 pci_disable_sriov(pdev); 2201 msleep(500); 2202 } else { 2203 dev_info(&pdev->dev, "VF(s) assigned to guests!\n"); 2204 } 2205 2206 kfree(adapter->vf_data); 2207 adapter->vf_data = NULL; 2208 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); 2209 wrfl(); 2210 msleep(100); 2211 dev_info(&pdev->dev, "IOV Disabled\n"); 2212 } 2213 #endif 2214 2215 iounmap(hw->hw_addr); 2216 if (hw->flash_address) 2217 iounmap(hw->flash_address); 2218 pci_release_selected_regions(pdev, 2219 pci_select_bars(pdev, IORESOURCE_MEM)); 2220 2221 kfree(adapter->shadow_vfta); 2222 free_netdev(netdev); 2223 2224 pci_disable_pcie_error_reporting(pdev); 2225 2226 pci_disable_device(pdev); 2227 } 2228 2229 /** 2230 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space 2231 * @adapter: board private structure to initialize 2232 * 2233 * This function initializes the vf specific data storage and then attempts to 2234 * allocate the VFs. The reason for ordering it this way is because it is much 2235 * mor expensive time wise to disable SR-IOV than it is to allocate and free 2236 * the memory for the VFs. 2237 **/ 2238 static void __devinit igb_probe_vfs(struct igb_adapter * adapter) 2239 { 2240 #ifdef CONFIG_PCI_IOV 2241 struct pci_dev *pdev = adapter->pdev; 2242 int old_vfs = igb_find_enabled_vfs(adapter); 2243 int i; 2244 2245 if (old_vfs) { 2246 dev_info(&pdev->dev, "%d pre-allocated VFs found - override " 2247 "max_vfs setting of %d\n", old_vfs, max_vfs); 2248 adapter->vfs_allocated_count = old_vfs; 2249 } 2250 2251 if (!adapter->vfs_allocated_count) 2252 return; 2253 2254 adapter->vf_data = kcalloc(adapter->vfs_allocated_count, 2255 sizeof(struct vf_data_storage), GFP_KERNEL); 2256 /* if allocation failed then we do not support SR-IOV */ 2257 if (!adapter->vf_data) { 2258 adapter->vfs_allocated_count = 0; 2259 dev_err(&pdev->dev, "Unable to allocate memory for VF " 2260 "Data Storage\n"); 2261 goto out; 2262 } 2263 2264 if (!old_vfs) { 2265 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) 2266 goto err_out; 2267 } 2268 dev_info(&pdev->dev, "%d VFs allocated\n", 2269 adapter->vfs_allocated_count); 2270 for (i = 0; i < adapter->vfs_allocated_count; i++) 2271 igb_vf_configure(adapter, i); 2272 2273 /* DMA Coalescing is not supported in IOV mode. */ 2274 adapter->flags &= ~IGB_FLAG_DMAC; 2275 goto out; 2276 err_out: 2277 kfree(adapter->vf_data); 2278 adapter->vf_data = NULL; 2279 adapter->vfs_allocated_count = 0; 2280 out: 2281 return; 2282 #endif /* CONFIG_PCI_IOV */ 2283 } 2284 2285 /** 2286 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp 2287 * @adapter: board private structure to initialize 2288 * 2289 * igb_init_hw_timer initializes the function pointer and values for the hw 2290 * timer found in hardware. 2291 **/ 2292 static void igb_init_hw_timer(struct igb_adapter *adapter) 2293 { 2294 struct e1000_hw *hw = &adapter->hw; 2295 2296 switch (hw->mac.type) { 2297 case e1000_i350: 2298 case e1000_82580: 2299 memset(&adapter->cycles, 0, sizeof(adapter->cycles)); 2300 adapter->cycles.read = igb_read_clock; 2301 adapter->cycles.mask = CLOCKSOURCE_MASK(64); 2302 adapter->cycles.mult = 1; 2303 /* 2304 * The 82580 timesync updates the system timer every 8ns by 8ns 2305 * and the value cannot be shifted. Instead we need to shift 2306 * the registers to generate a 64bit timer value. As a result 2307 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by 2308 * 24 in order to generate a larger value for synchronization. 2309 */ 2310 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT; 2311 /* disable system timer temporarily by setting bit 31 */ 2312 wr32(E1000_TSAUXC, 0x80000000); 2313 wrfl(); 2314 2315 /* Set registers so that rollover occurs soon to test this. */ 2316 wr32(E1000_SYSTIMR, 0x00000000); 2317 wr32(E1000_SYSTIML, 0x80000000); 2318 wr32(E1000_SYSTIMH, 0x000000FF); 2319 wrfl(); 2320 2321 /* enable system timer by clearing bit 31 */ 2322 wr32(E1000_TSAUXC, 0x0); 2323 wrfl(); 2324 2325 timecounter_init(&adapter->clock, 2326 &adapter->cycles, 2327 ktime_to_ns(ktime_get_real())); 2328 /* 2329 * Synchronize our NIC clock against system wall clock. NIC 2330 * time stamp reading requires ~3us per sample, each sample 2331 * was pretty stable even under load => only require 10 2332 * samples for each offset comparison. 2333 */ 2334 memset(&adapter->compare, 0, sizeof(adapter->compare)); 2335 adapter->compare.source = &adapter->clock; 2336 adapter->compare.target = ktime_get_real; 2337 adapter->compare.num_samples = 10; 2338 timecompare_update(&adapter->compare, 0); 2339 break; 2340 case e1000_82576: 2341 /* 2342 * Initialize hardware timer: we keep it running just in case 2343 * that some program needs it later on. 2344 */ 2345 memset(&adapter->cycles, 0, sizeof(adapter->cycles)); 2346 adapter->cycles.read = igb_read_clock; 2347 adapter->cycles.mask = CLOCKSOURCE_MASK(64); 2348 adapter->cycles.mult = 1; 2349 /** 2350 * Scale the NIC clock cycle by a large factor so that 2351 * relatively small clock corrections can be added or 2352 * subtracted at each clock tick. The drawbacks of a large 2353 * factor are a) that the clock register overflows more quickly 2354 * (not such a big deal) and b) that the increment per tick has 2355 * to fit into 24 bits. As a result we need to use a shift of 2356 * 19 so we can fit a value of 16 into the TIMINCA register. 2357 */ 2358 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT; 2359 wr32(E1000_TIMINCA, 2360 (1 << E1000_TIMINCA_16NS_SHIFT) | 2361 (16 << IGB_82576_TSYNC_SHIFT)); 2362 2363 /* Set registers so that rollover occurs soon to test this. */ 2364 wr32(E1000_SYSTIML, 0x00000000); 2365 wr32(E1000_SYSTIMH, 0xFF800000); 2366 wrfl(); 2367 2368 timecounter_init(&adapter->clock, 2369 &adapter->cycles, 2370 ktime_to_ns(ktime_get_real())); 2371 /* 2372 * Synchronize our NIC clock against system wall clock. NIC 2373 * time stamp reading requires ~3us per sample, each sample 2374 * was pretty stable even under load => only require 10 2375 * samples for each offset comparison. 2376 */ 2377 memset(&adapter->compare, 0, sizeof(adapter->compare)); 2378 adapter->compare.source = &adapter->clock; 2379 adapter->compare.target = ktime_get_real; 2380 adapter->compare.num_samples = 10; 2381 timecompare_update(&adapter->compare, 0); 2382 break; 2383 case e1000_82575: 2384 /* 82575 does not support timesync */ 2385 default: 2386 break; 2387 } 2388 2389 } 2390 2391 /** 2392 * igb_sw_init - Initialize general software structures (struct igb_adapter) 2393 * @adapter: board private structure to initialize 2394 * 2395 * igb_sw_init initializes the Adapter private data structure. 2396 * Fields are initialized based on PCI device information and 2397 * OS network device settings (MTU size). 2398 **/ 2399 static int __devinit igb_sw_init(struct igb_adapter *adapter) 2400 { 2401 struct e1000_hw *hw = &adapter->hw; 2402 struct net_device *netdev = adapter->netdev; 2403 struct pci_dev *pdev = adapter->pdev; 2404 2405 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 2406 2407 /* set default ring sizes */ 2408 adapter->tx_ring_count = IGB_DEFAULT_TXD; 2409 adapter->rx_ring_count = IGB_DEFAULT_RXD; 2410 2411 /* set default ITR values */ 2412 adapter->rx_itr_setting = IGB_DEFAULT_ITR; 2413 adapter->tx_itr_setting = IGB_DEFAULT_ITR; 2414 2415 /* set default work limits */ 2416 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK; 2417 2418 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 2419 VLAN_HLEN; 2420 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 2421 2422 adapter->node = -1; 2423 2424 spin_lock_init(&adapter->stats64_lock); 2425 #ifdef CONFIG_PCI_IOV 2426 switch (hw->mac.type) { 2427 case e1000_82576: 2428 case e1000_i350: 2429 if (max_vfs > 7) { 2430 dev_warn(&pdev->dev, 2431 "Maximum of 7 VFs per PF, using max\n"); 2432 adapter->vfs_allocated_count = 7; 2433 } else 2434 adapter->vfs_allocated_count = max_vfs; 2435 break; 2436 default: 2437 break; 2438 } 2439 #endif /* CONFIG_PCI_IOV */ 2440 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus()); 2441 /* i350 cannot do RSS and SR-IOV at the same time */ 2442 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count) 2443 adapter->rss_queues = 1; 2444 2445 /* 2446 * if rss_queues > 4 or vfs are going to be allocated with rss_queues 2447 * then we should combine the queues into a queue pair in order to 2448 * conserve interrupts due to limited supply 2449 */ 2450 if ((adapter->rss_queues > 4) || 2451 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6))) 2452 adapter->flags |= IGB_FLAG_QUEUE_PAIRS; 2453 2454 /* Setup and initialize a copy of the hw vlan table array */ 2455 adapter->shadow_vfta = kzalloc(sizeof(u32) * 2456 E1000_VLAN_FILTER_TBL_SIZE, 2457 GFP_ATOMIC); 2458 2459 /* This call may decrease the number of queues */ 2460 if (igb_init_interrupt_scheme(adapter)) { 2461 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 2462 return -ENOMEM; 2463 } 2464 2465 igb_probe_vfs(adapter); 2466 2467 /* Explicitly disable IRQ since the NIC can be in any state. */ 2468 igb_irq_disable(adapter); 2469 2470 if (hw->mac.type == e1000_i350) 2471 adapter->flags &= ~IGB_FLAG_DMAC; 2472 2473 set_bit(__IGB_DOWN, &adapter->state); 2474 return 0; 2475 } 2476 2477 /** 2478 * igb_open - Called when a network interface is made active 2479 * @netdev: network interface device structure 2480 * 2481 * Returns 0 on success, negative value on failure 2482 * 2483 * The open entry point is called when a network interface is made 2484 * active by the system (IFF_UP). At this point all resources needed 2485 * for transmit and receive operations are allocated, the interrupt 2486 * handler is registered with the OS, the watchdog timer is started, 2487 * and the stack is notified that the interface is ready. 2488 **/ 2489 static int __igb_open(struct net_device *netdev, bool resuming) 2490 { 2491 struct igb_adapter *adapter = netdev_priv(netdev); 2492 struct e1000_hw *hw = &adapter->hw; 2493 struct pci_dev *pdev = adapter->pdev; 2494 int err; 2495 int i; 2496 2497 /* disallow open during test */ 2498 if (test_bit(__IGB_TESTING, &adapter->state)) { 2499 WARN_ON(resuming); 2500 return -EBUSY; 2501 } 2502 2503 if (!resuming) 2504 pm_runtime_get_sync(&pdev->dev); 2505 2506 netif_carrier_off(netdev); 2507 2508 /* allocate transmit descriptors */ 2509 err = igb_setup_all_tx_resources(adapter); 2510 if (err) 2511 goto err_setup_tx; 2512 2513 /* allocate receive descriptors */ 2514 err = igb_setup_all_rx_resources(adapter); 2515 if (err) 2516 goto err_setup_rx; 2517 2518 igb_power_up_link(adapter); 2519 2520 /* before we allocate an interrupt, we must be ready to handle it. 2521 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 2522 * as soon as we call pci_request_irq, so we have to setup our 2523 * clean_rx handler before we do so. */ 2524 igb_configure(adapter); 2525 2526 err = igb_request_irq(adapter); 2527 if (err) 2528 goto err_req_irq; 2529 2530 /* From here on the code is the same as igb_up() */ 2531 clear_bit(__IGB_DOWN, &adapter->state); 2532 2533 for (i = 0; i < adapter->num_q_vectors; i++) 2534 napi_enable(&(adapter->q_vector[i]->napi)); 2535 2536 /* Clear any pending interrupts. */ 2537 rd32(E1000_ICR); 2538 2539 igb_irq_enable(adapter); 2540 2541 /* notify VFs that reset has been completed */ 2542 if (adapter->vfs_allocated_count) { 2543 u32 reg_data = rd32(E1000_CTRL_EXT); 2544 reg_data |= E1000_CTRL_EXT_PFRSTD; 2545 wr32(E1000_CTRL_EXT, reg_data); 2546 } 2547 2548 netif_tx_start_all_queues(netdev); 2549 2550 if (!resuming) 2551 pm_runtime_put(&pdev->dev); 2552 2553 /* start the watchdog. */ 2554 hw->mac.get_link_status = 1; 2555 schedule_work(&adapter->watchdog_task); 2556 2557 return 0; 2558 2559 err_req_irq: 2560 igb_release_hw_control(adapter); 2561 igb_power_down_link(adapter); 2562 igb_free_all_rx_resources(adapter); 2563 err_setup_rx: 2564 igb_free_all_tx_resources(adapter); 2565 err_setup_tx: 2566 igb_reset(adapter); 2567 if (!resuming) 2568 pm_runtime_put(&pdev->dev); 2569 2570 return err; 2571 } 2572 2573 static int igb_open(struct net_device *netdev) 2574 { 2575 return __igb_open(netdev, false); 2576 } 2577 2578 /** 2579 * igb_close - Disables a network interface 2580 * @netdev: network interface device structure 2581 * 2582 * Returns 0, this is not allowed to fail 2583 * 2584 * The close entry point is called when an interface is de-activated 2585 * by the OS. The hardware is still under the driver's control, but 2586 * needs to be disabled. A global MAC reset is issued to stop the 2587 * hardware, and all transmit and receive resources are freed. 2588 **/ 2589 static int __igb_close(struct net_device *netdev, bool suspending) 2590 { 2591 struct igb_adapter *adapter = netdev_priv(netdev); 2592 struct pci_dev *pdev = adapter->pdev; 2593 2594 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state)); 2595 2596 if (!suspending) 2597 pm_runtime_get_sync(&pdev->dev); 2598 2599 igb_down(adapter); 2600 igb_free_irq(adapter); 2601 2602 igb_free_all_tx_resources(adapter); 2603 igb_free_all_rx_resources(adapter); 2604 2605 if (!suspending) 2606 pm_runtime_put_sync(&pdev->dev); 2607 return 0; 2608 } 2609 2610 static int igb_close(struct net_device *netdev) 2611 { 2612 return __igb_close(netdev, false); 2613 } 2614 2615 /** 2616 * igb_setup_tx_resources - allocate Tx resources (Descriptors) 2617 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2618 * 2619 * Return 0 on success, negative on failure 2620 **/ 2621 int igb_setup_tx_resources(struct igb_ring *tx_ring) 2622 { 2623 struct device *dev = tx_ring->dev; 2624 int orig_node = dev_to_node(dev); 2625 int size; 2626 2627 size = sizeof(struct igb_tx_buffer) * tx_ring->count; 2628 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node); 2629 if (!tx_ring->tx_buffer_info) 2630 tx_ring->tx_buffer_info = vzalloc(size); 2631 if (!tx_ring->tx_buffer_info) 2632 goto err; 2633 2634 /* round up to nearest 4K */ 2635 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc); 2636 tx_ring->size = ALIGN(tx_ring->size, 4096); 2637 2638 set_dev_node(dev, tx_ring->numa_node); 2639 tx_ring->desc = dma_alloc_coherent(dev, 2640 tx_ring->size, 2641 &tx_ring->dma, 2642 GFP_KERNEL); 2643 set_dev_node(dev, orig_node); 2644 if (!tx_ring->desc) 2645 tx_ring->desc = dma_alloc_coherent(dev, 2646 tx_ring->size, 2647 &tx_ring->dma, 2648 GFP_KERNEL); 2649 2650 if (!tx_ring->desc) 2651 goto err; 2652 2653 tx_ring->next_to_use = 0; 2654 tx_ring->next_to_clean = 0; 2655 2656 return 0; 2657 2658 err: 2659 vfree(tx_ring->tx_buffer_info); 2660 dev_err(dev, 2661 "Unable to allocate memory for the transmit descriptor ring\n"); 2662 return -ENOMEM; 2663 } 2664 2665 /** 2666 * igb_setup_all_tx_resources - wrapper to allocate Tx resources 2667 * (Descriptors) for all queues 2668 * @adapter: board private structure 2669 * 2670 * Return 0 on success, negative on failure 2671 **/ 2672 static int igb_setup_all_tx_resources(struct igb_adapter *adapter) 2673 { 2674 struct pci_dev *pdev = adapter->pdev; 2675 int i, err = 0; 2676 2677 for (i = 0; i < adapter->num_tx_queues; i++) { 2678 err = igb_setup_tx_resources(adapter->tx_ring[i]); 2679 if (err) { 2680 dev_err(&pdev->dev, 2681 "Allocation for Tx Queue %u failed\n", i); 2682 for (i--; i >= 0; i--) 2683 igb_free_tx_resources(adapter->tx_ring[i]); 2684 break; 2685 } 2686 } 2687 2688 return err; 2689 } 2690 2691 /** 2692 * igb_setup_tctl - configure the transmit control registers 2693 * @adapter: Board private structure 2694 **/ 2695 void igb_setup_tctl(struct igb_adapter *adapter) 2696 { 2697 struct e1000_hw *hw = &adapter->hw; 2698 u32 tctl; 2699 2700 /* disable queue 0 which is enabled by default on 82575 and 82576 */ 2701 wr32(E1000_TXDCTL(0), 0); 2702 2703 /* Program the Transmit Control Register */ 2704 tctl = rd32(E1000_TCTL); 2705 tctl &= ~E1000_TCTL_CT; 2706 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2707 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2708 2709 igb_config_collision_dist(hw); 2710 2711 /* Enable transmits */ 2712 tctl |= E1000_TCTL_EN; 2713 2714 wr32(E1000_TCTL, tctl); 2715 } 2716 2717 /** 2718 * igb_configure_tx_ring - Configure transmit ring after Reset 2719 * @adapter: board private structure 2720 * @ring: tx ring to configure 2721 * 2722 * Configure a transmit ring after a reset. 2723 **/ 2724 void igb_configure_tx_ring(struct igb_adapter *adapter, 2725 struct igb_ring *ring) 2726 { 2727 struct e1000_hw *hw = &adapter->hw; 2728 u32 txdctl = 0; 2729 u64 tdba = ring->dma; 2730 int reg_idx = ring->reg_idx; 2731 2732 /* disable the queue */ 2733 wr32(E1000_TXDCTL(reg_idx), 0); 2734 wrfl(); 2735 mdelay(10); 2736 2737 wr32(E1000_TDLEN(reg_idx), 2738 ring->count * sizeof(union e1000_adv_tx_desc)); 2739 wr32(E1000_TDBAL(reg_idx), 2740 tdba & 0x00000000ffffffffULL); 2741 wr32(E1000_TDBAH(reg_idx), tdba >> 32); 2742 2743 ring->tail = hw->hw_addr + E1000_TDT(reg_idx); 2744 wr32(E1000_TDH(reg_idx), 0); 2745 writel(0, ring->tail); 2746 2747 txdctl |= IGB_TX_PTHRESH; 2748 txdctl |= IGB_TX_HTHRESH << 8; 2749 txdctl |= IGB_TX_WTHRESH << 16; 2750 2751 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 2752 wr32(E1000_TXDCTL(reg_idx), txdctl); 2753 } 2754 2755 /** 2756 * igb_configure_tx - Configure transmit Unit after Reset 2757 * @adapter: board private structure 2758 * 2759 * Configure the Tx unit of the MAC after a reset. 2760 **/ 2761 static void igb_configure_tx(struct igb_adapter *adapter) 2762 { 2763 int i; 2764 2765 for (i = 0; i < adapter->num_tx_queues; i++) 2766 igb_configure_tx_ring(adapter, adapter->tx_ring[i]); 2767 } 2768 2769 /** 2770 * igb_setup_rx_resources - allocate Rx resources (Descriptors) 2771 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2772 * 2773 * Returns 0 on success, negative on failure 2774 **/ 2775 int igb_setup_rx_resources(struct igb_ring *rx_ring) 2776 { 2777 struct device *dev = rx_ring->dev; 2778 int orig_node = dev_to_node(dev); 2779 int size, desc_len; 2780 2781 size = sizeof(struct igb_rx_buffer) * rx_ring->count; 2782 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node); 2783 if (!rx_ring->rx_buffer_info) 2784 rx_ring->rx_buffer_info = vzalloc(size); 2785 if (!rx_ring->rx_buffer_info) 2786 goto err; 2787 2788 desc_len = sizeof(union e1000_adv_rx_desc); 2789 2790 /* Round up to nearest 4K */ 2791 rx_ring->size = rx_ring->count * desc_len; 2792 rx_ring->size = ALIGN(rx_ring->size, 4096); 2793 2794 set_dev_node(dev, rx_ring->numa_node); 2795 rx_ring->desc = dma_alloc_coherent(dev, 2796 rx_ring->size, 2797 &rx_ring->dma, 2798 GFP_KERNEL); 2799 set_dev_node(dev, orig_node); 2800 if (!rx_ring->desc) 2801 rx_ring->desc = dma_alloc_coherent(dev, 2802 rx_ring->size, 2803 &rx_ring->dma, 2804 GFP_KERNEL); 2805 2806 if (!rx_ring->desc) 2807 goto err; 2808 2809 rx_ring->next_to_clean = 0; 2810 rx_ring->next_to_use = 0; 2811 2812 return 0; 2813 2814 err: 2815 vfree(rx_ring->rx_buffer_info); 2816 rx_ring->rx_buffer_info = NULL; 2817 dev_err(dev, "Unable to allocate memory for the receive descriptor" 2818 " ring\n"); 2819 return -ENOMEM; 2820 } 2821 2822 /** 2823 * igb_setup_all_rx_resources - wrapper to allocate Rx resources 2824 * (Descriptors) for all queues 2825 * @adapter: board private structure 2826 * 2827 * Return 0 on success, negative on failure 2828 **/ 2829 static int igb_setup_all_rx_resources(struct igb_adapter *adapter) 2830 { 2831 struct pci_dev *pdev = adapter->pdev; 2832 int i, err = 0; 2833 2834 for (i = 0; i < adapter->num_rx_queues; i++) { 2835 err = igb_setup_rx_resources(adapter->rx_ring[i]); 2836 if (err) { 2837 dev_err(&pdev->dev, 2838 "Allocation for Rx Queue %u failed\n", i); 2839 for (i--; i >= 0; i--) 2840 igb_free_rx_resources(adapter->rx_ring[i]); 2841 break; 2842 } 2843 } 2844 2845 return err; 2846 } 2847 2848 /** 2849 * igb_setup_mrqc - configure the multiple receive queue control registers 2850 * @adapter: Board private structure 2851 **/ 2852 static void igb_setup_mrqc(struct igb_adapter *adapter) 2853 { 2854 struct e1000_hw *hw = &adapter->hw; 2855 u32 mrqc, rxcsum; 2856 u32 j, num_rx_queues, shift = 0, shift2 = 0; 2857 union e1000_reta { 2858 u32 dword; 2859 u8 bytes[4]; 2860 } reta; 2861 static const u8 rsshash[40] = { 2862 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 2863 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 2864 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 2865 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa }; 2866 2867 /* Fill out hash function seeds */ 2868 for (j = 0; j < 10; j++) { 2869 u32 rsskey = rsshash[(j * 4)]; 2870 rsskey |= rsshash[(j * 4) + 1] << 8; 2871 rsskey |= rsshash[(j * 4) + 2] << 16; 2872 rsskey |= rsshash[(j * 4) + 3] << 24; 2873 array_wr32(E1000_RSSRK(0), j, rsskey); 2874 } 2875 2876 num_rx_queues = adapter->rss_queues; 2877 2878 if (adapter->vfs_allocated_count) { 2879 /* 82575 and 82576 supports 2 RSS queues for VMDq */ 2880 switch (hw->mac.type) { 2881 case e1000_i350: 2882 case e1000_82580: 2883 num_rx_queues = 1; 2884 shift = 0; 2885 break; 2886 case e1000_82576: 2887 shift = 3; 2888 num_rx_queues = 2; 2889 break; 2890 case e1000_82575: 2891 shift = 2; 2892 shift2 = 6; 2893 default: 2894 break; 2895 } 2896 } else { 2897 if (hw->mac.type == e1000_82575) 2898 shift = 6; 2899 } 2900 2901 for (j = 0; j < (32 * 4); j++) { 2902 reta.bytes[j & 3] = (j % num_rx_queues) << shift; 2903 if (shift2) 2904 reta.bytes[j & 3] |= num_rx_queues << shift2; 2905 if ((j & 3) == 3) 2906 wr32(E1000_RETA(j >> 2), reta.dword); 2907 } 2908 2909 /* 2910 * Disable raw packet checksumming so that RSS hash is placed in 2911 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 2912 * offloads as they are enabled by default 2913 */ 2914 rxcsum = rd32(E1000_RXCSUM); 2915 rxcsum |= E1000_RXCSUM_PCSD; 2916 2917 if (adapter->hw.mac.type >= e1000_82576) 2918 /* Enable Receive Checksum Offload for SCTP */ 2919 rxcsum |= E1000_RXCSUM_CRCOFL; 2920 2921 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 2922 wr32(E1000_RXCSUM, rxcsum); 2923 2924 /* If VMDq is enabled then we set the appropriate mode for that, else 2925 * we default to RSS so that an RSS hash is calculated per packet even 2926 * if we are only using one queue */ 2927 if (adapter->vfs_allocated_count) { 2928 if (hw->mac.type > e1000_82575) { 2929 /* Set the default pool for the PF's first queue */ 2930 u32 vtctl = rd32(E1000_VT_CTL); 2931 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK | 2932 E1000_VT_CTL_DISABLE_DEF_POOL); 2933 vtctl |= adapter->vfs_allocated_count << 2934 E1000_VT_CTL_DEFAULT_POOL_SHIFT; 2935 wr32(E1000_VT_CTL, vtctl); 2936 } 2937 if (adapter->rss_queues > 1) 2938 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q; 2939 else 2940 mrqc = E1000_MRQC_ENABLE_VMDQ; 2941 } else { 2942 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 2943 } 2944 igb_vmm_control(adapter); 2945 2946 /* 2947 * Generate RSS hash based on TCP port numbers and/or 2948 * IPv4/v6 src and dst addresses since UDP cannot be 2949 * hashed reliably due to IP fragmentation 2950 */ 2951 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 | 2952 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2953 E1000_MRQC_RSS_FIELD_IPV6 | 2954 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2955 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX; 2956 2957 wr32(E1000_MRQC, mrqc); 2958 } 2959 2960 /** 2961 * igb_setup_rctl - configure the receive control registers 2962 * @adapter: Board private structure 2963 **/ 2964 void igb_setup_rctl(struct igb_adapter *adapter) 2965 { 2966 struct e1000_hw *hw = &adapter->hw; 2967 u32 rctl; 2968 2969 rctl = rd32(E1000_RCTL); 2970 2971 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 2972 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 2973 2974 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF | 2975 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 2976 2977 /* 2978 * enable stripping of CRC. It's unlikely this will break BMC 2979 * redirection as it did with e1000. Newer features require 2980 * that the HW strips the CRC. 2981 */ 2982 rctl |= E1000_RCTL_SECRC; 2983 2984 /* disable store bad packets and clear size bits. */ 2985 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256); 2986 2987 /* enable LPE to prevent packets larger than max_frame_size */ 2988 rctl |= E1000_RCTL_LPE; 2989 2990 /* disable queue 0 to prevent tail write w/o re-config */ 2991 wr32(E1000_RXDCTL(0), 0); 2992 2993 /* Attention!!! For SR-IOV PF driver operations you must enable 2994 * queue drop for all VF and PF queues to prevent head of line blocking 2995 * if an un-trusted VF does not provide descriptors to hardware. 2996 */ 2997 if (adapter->vfs_allocated_count) { 2998 /* set all queue drop enable bits */ 2999 wr32(E1000_QDE, ALL_QUEUES); 3000 } 3001 3002 wr32(E1000_RCTL, rctl); 3003 } 3004 3005 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size, 3006 int vfn) 3007 { 3008 struct e1000_hw *hw = &adapter->hw; 3009 u32 vmolr; 3010 3011 /* if it isn't the PF check to see if VFs are enabled and 3012 * increase the size to support vlan tags */ 3013 if (vfn < adapter->vfs_allocated_count && 3014 adapter->vf_data[vfn].vlans_enabled) 3015 size += VLAN_TAG_SIZE; 3016 3017 vmolr = rd32(E1000_VMOLR(vfn)); 3018 vmolr &= ~E1000_VMOLR_RLPML_MASK; 3019 vmolr |= size | E1000_VMOLR_LPE; 3020 wr32(E1000_VMOLR(vfn), vmolr); 3021 3022 return 0; 3023 } 3024 3025 /** 3026 * igb_rlpml_set - set maximum receive packet size 3027 * @adapter: board private structure 3028 * 3029 * Configure maximum receivable packet size. 3030 **/ 3031 static void igb_rlpml_set(struct igb_adapter *adapter) 3032 { 3033 u32 max_frame_size = adapter->max_frame_size; 3034 struct e1000_hw *hw = &adapter->hw; 3035 u16 pf_id = adapter->vfs_allocated_count; 3036 3037 if (pf_id) { 3038 igb_set_vf_rlpml(adapter, max_frame_size, pf_id); 3039 /* 3040 * If we're in VMDQ or SR-IOV mode, then set global RLPML 3041 * to our max jumbo frame size, in case we need to enable 3042 * jumbo frames on one of the rings later. 3043 * This will not pass over-length frames into the default 3044 * queue because it's gated by the VMOLR.RLPML. 3045 */ 3046 max_frame_size = MAX_JUMBO_FRAME_SIZE; 3047 } 3048 3049 wr32(E1000_RLPML, max_frame_size); 3050 } 3051 3052 static inline void igb_set_vmolr(struct igb_adapter *adapter, 3053 int vfn, bool aupe) 3054 { 3055 struct e1000_hw *hw = &adapter->hw; 3056 u32 vmolr; 3057 3058 /* 3059 * This register exists only on 82576 and newer so if we are older then 3060 * we should exit and do nothing 3061 */ 3062 if (hw->mac.type < e1000_82576) 3063 return; 3064 3065 vmolr = rd32(E1000_VMOLR(vfn)); 3066 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */ 3067 if (aupe) 3068 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */ 3069 else 3070 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */ 3071 3072 /* clear all bits that might not be set */ 3073 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE); 3074 3075 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count) 3076 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */ 3077 /* 3078 * for VMDq only allow the VFs and pool 0 to accept broadcast and 3079 * multicast packets 3080 */ 3081 if (vfn <= adapter->vfs_allocated_count) 3082 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */ 3083 3084 wr32(E1000_VMOLR(vfn), vmolr); 3085 } 3086 3087 /** 3088 * igb_configure_rx_ring - Configure a receive ring after Reset 3089 * @adapter: board private structure 3090 * @ring: receive ring to be configured 3091 * 3092 * Configure the Rx unit of the MAC after a reset. 3093 **/ 3094 void igb_configure_rx_ring(struct igb_adapter *adapter, 3095 struct igb_ring *ring) 3096 { 3097 struct e1000_hw *hw = &adapter->hw; 3098 u64 rdba = ring->dma; 3099 int reg_idx = ring->reg_idx; 3100 u32 srrctl = 0, rxdctl = 0; 3101 3102 /* disable the queue */ 3103 wr32(E1000_RXDCTL(reg_idx), 0); 3104 3105 /* Set DMA base address registers */ 3106 wr32(E1000_RDBAL(reg_idx), 3107 rdba & 0x00000000ffffffffULL); 3108 wr32(E1000_RDBAH(reg_idx), rdba >> 32); 3109 wr32(E1000_RDLEN(reg_idx), 3110 ring->count * sizeof(union e1000_adv_rx_desc)); 3111 3112 /* initialize head and tail */ 3113 ring->tail = hw->hw_addr + E1000_RDT(reg_idx); 3114 wr32(E1000_RDH(reg_idx), 0); 3115 writel(0, ring->tail); 3116 3117 /* set descriptor configuration */ 3118 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 3119 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384 3120 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3121 #else 3122 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3123 #endif 3124 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3125 if (hw->mac.type >= e1000_82580) 3126 srrctl |= E1000_SRRCTL_TIMESTAMP; 3127 /* Only set Drop Enable if we are supporting multiple queues */ 3128 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1) 3129 srrctl |= E1000_SRRCTL_DROP_EN; 3130 3131 wr32(E1000_SRRCTL(reg_idx), srrctl); 3132 3133 /* set filtering for VMDQ pools */ 3134 igb_set_vmolr(adapter, reg_idx & 0x7, true); 3135 3136 rxdctl |= IGB_RX_PTHRESH; 3137 rxdctl |= IGB_RX_HTHRESH << 8; 3138 rxdctl |= IGB_RX_WTHRESH << 16; 3139 3140 /* enable receive descriptor fetching */ 3141 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 3142 wr32(E1000_RXDCTL(reg_idx), rxdctl); 3143 } 3144 3145 /** 3146 * igb_configure_rx - Configure receive Unit after Reset 3147 * @adapter: board private structure 3148 * 3149 * Configure the Rx unit of the MAC after a reset. 3150 **/ 3151 static void igb_configure_rx(struct igb_adapter *adapter) 3152 { 3153 int i; 3154 3155 /* set UTA to appropriate mode */ 3156 igb_set_uta(adapter); 3157 3158 /* set the correct pool for the PF default MAC address in entry 0 */ 3159 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0, 3160 adapter->vfs_allocated_count); 3161 3162 /* Setup the HW Rx Head and Tail Descriptor Pointers and 3163 * the Base and Length of the Rx Descriptor Ring */ 3164 for (i = 0; i < adapter->num_rx_queues; i++) 3165 igb_configure_rx_ring(adapter, adapter->rx_ring[i]); 3166 } 3167 3168 /** 3169 * igb_free_tx_resources - Free Tx Resources per Queue 3170 * @tx_ring: Tx descriptor ring for a specific queue 3171 * 3172 * Free all transmit software resources 3173 **/ 3174 void igb_free_tx_resources(struct igb_ring *tx_ring) 3175 { 3176 igb_clean_tx_ring(tx_ring); 3177 3178 vfree(tx_ring->tx_buffer_info); 3179 tx_ring->tx_buffer_info = NULL; 3180 3181 /* if not set, then don't free */ 3182 if (!tx_ring->desc) 3183 return; 3184 3185 dma_free_coherent(tx_ring->dev, tx_ring->size, 3186 tx_ring->desc, tx_ring->dma); 3187 3188 tx_ring->desc = NULL; 3189 } 3190 3191 /** 3192 * igb_free_all_tx_resources - Free Tx Resources for All Queues 3193 * @adapter: board private structure 3194 * 3195 * Free all transmit software resources 3196 **/ 3197 static void igb_free_all_tx_resources(struct igb_adapter *adapter) 3198 { 3199 int i; 3200 3201 for (i = 0; i < adapter->num_tx_queues; i++) 3202 igb_free_tx_resources(adapter->tx_ring[i]); 3203 } 3204 3205 void igb_unmap_and_free_tx_resource(struct igb_ring *ring, 3206 struct igb_tx_buffer *tx_buffer) 3207 { 3208 if (tx_buffer->skb) { 3209 dev_kfree_skb_any(tx_buffer->skb); 3210 if (tx_buffer->dma) 3211 dma_unmap_single(ring->dev, 3212 tx_buffer->dma, 3213 tx_buffer->length, 3214 DMA_TO_DEVICE); 3215 } else if (tx_buffer->dma) { 3216 dma_unmap_page(ring->dev, 3217 tx_buffer->dma, 3218 tx_buffer->length, 3219 DMA_TO_DEVICE); 3220 } 3221 tx_buffer->next_to_watch = NULL; 3222 tx_buffer->skb = NULL; 3223 tx_buffer->dma = 0; 3224 /* buffer_info must be completely set up in the transmit path */ 3225 } 3226 3227 /** 3228 * igb_clean_tx_ring - Free Tx Buffers 3229 * @tx_ring: ring to be cleaned 3230 **/ 3231 static void igb_clean_tx_ring(struct igb_ring *tx_ring) 3232 { 3233 struct igb_tx_buffer *buffer_info; 3234 unsigned long size; 3235 u16 i; 3236 3237 if (!tx_ring->tx_buffer_info) 3238 return; 3239 /* Free all the Tx ring sk_buffs */ 3240 3241 for (i = 0; i < tx_ring->count; i++) { 3242 buffer_info = &tx_ring->tx_buffer_info[i]; 3243 igb_unmap_and_free_tx_resource(tx_ring, buffer_info); 3244 } 3245 netdev_tx_reset_queue(txring_txq(tx_ring)); 3246 3247 size = sizeof(struct igb_tx_buffer) * tx_ring->count; 3248 memset(tx_ring->tx_buffer_info, 0, size); 3249 3250 /* Zero out the descriptor ring */ 3251 memset(tx_ring->desc, 0, tx_ring->size); 3252 3253 tx_ring->next_to_use = 0; 3254 tx_ring->next_to_clean = 0; 3255 } 3256 3257 /** 3258 * igb_clean_all_tx_rings - Free Tx Buffers for all queues 3259 * @adapter: board private structure 3260 **/ 3261 static void igb_clean_all_tx_rings(struct igb_adapter *adapter) 3262 { 3263 int i; 3264 3265 for (i = 0; i < adapter->num_tx_queues; i++) 3266 igb_clean_tx_ring(adapter->tx_ring[i]); 3267 } 3268 3269 /** 3270 * igb_free_rx_resources - Free Rx Resources 3271 * @rx_ring: ring to clean the resources from 3272 * 3273 * Free all receive software resources 3274 **/ 3275 void igb_free_rx_resources(struct igb_ring *rx_ring) 3276 { 3277 igb_clean_rx_ring(rx_ring); 3278 3279 vfree(rx_ring->rx_buffer_info); 3280 rx_ring->rx_buffer_info = NULL; 3281 3282 /* if not set, then don't free */ 3283 if (!rx_ring->desc) 3284 return; 3285 3286 dma_free_coherent(rx_ring->dev, rx_ring->size, 3287 rx_ring->desc, rx_ring->dma); 3288 3289 rx_ring->desc = NULL; 3290 } 3291 3292 /** 3293 * igb_free_all_rx_resources - Free Rx Resources for All Queues 3294 * @adapter: board private structure 3295 * 3296 * Free all receive software resources 3297 **/ 3298 static void igb_free_all_rx_resources(struct igb_adapter *adapter) 3299 { 3300 int i; 3301 3302 for (i = 0; i < adapter->num_rx_queues; i++) 3303 igb_free_rx_resources(adapter->rx_ring[i]); 3304 } 3305 3306 /** 3307 * igb_clean_rx_ring - Free Rx Buffers per Queue 3308 * @rx_ring: ring to free buffers from 3309 **/ 3310 static void igb_clean_rx_ring(struct igb_ring *rx_ring) 3311 { 3312 unsigned long size; 3313 u16 i; 3314 3315 if (!rx_ring->rx_buffer_info) 3316 return; 3317 3318 /* Free all the Rx ring sk_buffs */ 3319 for (i = 0; i < rx_ring->count; i++) { 3320 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 3321 if (buffer_info->dma) { 3322 dma_unmap_single(rx_ring->dev, 3323 buffer_info->dma, 3324 IGB_RX_HDR_LEN, 3325 DMA_FROM_DEVICE); 3326 buffer_info->dma = 0; 3327 } 3328 3329 if (buffer_info->skb) { 3330 dev_kfree_skb(buffer_info->skb); 3331 buffer_info->skb = NULL; 3332 } 3333 if (buffer_info->page_dma) { 3334 dma_unmap_page(rx_ring->dev, 3335 buffer_info->page_dma, 3336 PAGE_SIZE / 2, 3337 DMA_FROM_DEVICE); 3338 buffer_info->page_dma = 0; 3339 } 3340 if (buffer_info->page) { 3341 put_page(buffer_info->page); 3342 buffer_info->page = NULL; 3343 buffer_info->page_offset = 0; 3344 } 3345 } 3346 3347 size = sizeof(struct igb_rx_buffer) * rx_ring->count; 3348 memset(rx_ring->rx_buffer_info, 0, size); 3349 3350 /* Zero out the descriptor ring */ 3351 memset(rx_ring->desc, 0, rx_ring->size); 3352 3353 rx_ring->next_to_clean = 0; 3354 rx_ring->next_to_use = 0; 3355 } 3356 3357 /** 3358 * igb_clean_all_rx_rings - Free Rx Buffers for all queues 3359 * @adapter: board private structure 3360 **/ 3361 static void igb_clean_all_rx_rings(struct igb_adapter *adapter) 3362 { 3363 int i; 3364 3365 for (i = 0; i < adapter->num_rx_queues; i++) 3366 igb_clean_rx_ring(adapter->rx_ring[i]); 3367 } 3368 3369 /** 3370 * igb_set_mac - Change the Ethernet Address of the NIC 3371 * @netdev: network interface device structure 3372 * @p: pointer to an address structure 3373 * 3374 * Returns 0 on success, negative on failure 3375 **/ 3376 static int igb_set_mac(struct net_device *netdev, void *p) 3377 { 3378 struct igb_adapter *adapter = netdev_priv(netdev); 3379 struct e1000_hw *hw = &adapter->hw; 3380 struct sockaddr *addr = p; 3381 3382 if (!is_valid_ether_addr(addr->sa_data)) 3383 return -EADDRNOTAVAIL; 3384 3385 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3386 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3387 3388 /* set the correct pool for the new PF MAC address in entry 0 */ 3389 igb_rar_set_qsel(adapter, hw->mac.addr, 0, 3390 adapter->vfs_allocated_count); 3391 3392 return 0; 3393 } 3394 3395 /** 3396 * igb_write_mc_addr_list - write multicast addresses to MTA 3397 * @netdev: network interface device structure 3398 * 3399 * Writes multicast address list to the MTA hash table. 3400 * Returns: -ENOMEM on failure 3401 * 0 on no addresses written 3402 * X on writing X addresses to MTA 3403 **/ 3404 static int igb_write_mc_addr_list(struct net_device *netdev) 3405 { 3406 struct igb_adapter *adapter = netdev_priv(netdev); 3407 struct e1000_hw *hw = &adapter->hw; 3408 struct netdev_hw_addr *ha; 3409 u8 *mta_list; 3410 int i; 3411 3412 if (netdev_mc_empty(netdev)) { 3413 /* nothing to program, so clear mc list */ 3414 igb_update_mc_addr_list(hw, NULL, 0); 3415 igb_restore_vf_multicasts(adapter); 3416 return 0; 3417 } 3418 3419 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC); 3420 if (!mta_list) 3421 return -ENOMEM; 3422 3423 /* The shared function expects a packed array of only addresses. */ 3424 i = 0; 3425 netdev_for_each_mc_addr(ha, netdev) 3426 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 3427 3428 igb_update_mc_addr_list(hw, mta_list, i); 3429 kfree(mta_list); 3430 3431 return netdev_mc_count(netdev); 3432 } 3433 3434 /** 3435 * igb_write_uc_addr_list - write unicast addresses to RAR table 3436 * @netdev: network interface device structure 3437 * 3438 * Writes unicast address list to the RAR table. 3439 * Returns: -ENOMEM on failure/insufficient address space 3440 * 0 on no addresses written 3441 * X on writing X addresses to the RAR table 3442 **/ 3443 static int igb_write_uc_addr_list(struct net_device *netdev) 3444 { 3445 struct igb_adapter *adapter = netdev_priv(netdev); 3446 struct e1000_hw *hw = &adapter->hw; 3447 unsigned int vfn = adapter->vfs_allocated_count; 3448 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1); 3449 int count = 0; 3450 3451 /* return ENOMEM indicating insufficient memory for addresses */ 3452 if (netdev_uc_count(netdev) > rar_entries) 3453 return -ENOMEM; 3454 3455 if (!netdev_uc_empty(netdev) && rar_entries) { 3456 struct netdev_hw_addr *ha; 3457 3458 netdev_for_each_uc_addr(ha, netdev) { 3459 if (!rar_entries) 3460 break; 3461 igb_rar_set_qsel(adapter, ha->addr, 3462 rar_entries--, 3463 vfn); 3464 count++; 3465 } 3466 } 3467 /* write the addresses in reverse order to avoid write combining */ 3468 for (; rar_entries > 0 ; rar_entries--) { 3469 wr32(E1000_RAH(rar_entries), 0); 3470 wr32(E1000_RAL(rar_entries), 0); 3471 } 3472 wrfl(); 3473 3474 return count; 3475 } 3476 3477 /** 3478 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 3479 * @netdev: network interface device structure 3480 * 3481 * The set_rx_mode entry point is called whenever the unicast or multicast 3482 * address lists or the network interface flags are updated. This routine is 3483 * responsible for configuring the hardware for proper unicast, multicast, 3484 * promiscuous mode, and all-multi behavior. 3485 **/ 3486 static void igb_set_rx_mode(struct net_device *netdev) 3487 { 3488 struct igb_adapter *adapter = netdev_priv(netdev); 3489 struct e1000_hw *hw = &adapter->hw; 3490 unsigned int vfn = adapter->vfs_allocated_count; 3491 u32 rctl, vmolr = 0; 3492 int count; 3493 3494 /* Check for Promiscuous and All Multicast modes */ 3495 rctl = rd32(E1000_RCTL); 3496 3497 /* clear the effected bits */ 3498 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE); 3499 3500 if (netdev->flags & IFF_PROMISC) { 3501 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3502 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME); 3503 } else { 3504 if (netdev->flags & IFF_ALLMULTI) { 3505 rctl |= E1000_RCTL_MPE; 3506 vmolr |= E1000_VMOLR_MPME; 3507 } else { 3508 /* 3509 * Write addresses to the MTA, if the attempt fails 3510 * then we should just turn on promiscuous mode so 3511 * that we can at least receive multicast traffic 3512 */ 3513 count = igb_write_mc_addr_list(netdev); 3514 if (count < 0) { 3515 rctl |= E1000_RCTL_MPE; 3516 vmolr |= E1000_VMOLR_MPME; 3517 } else if (count) { 3518 vmolr |= E1000_VMOLR_ROMPE; 3519 } 3520 } 3521 /* 3522 * Write addresses to available RAR registers, if there is not 3523 * sufficient space to store all the addresses then enable 3524 * unicast promiscuous mode 3525 */ 3526 count = igb_write_uc_addr_list(netdev); 3527 if (count < 0) { 3528 rctl |= E1000_RCTL_UPE; 3529 vmolr |= E1000_VMOLR_ROPE; 3530 } 3531 rctl |= E1000_RCTL_VFE; 3532 } 3533 wr32(E1000_RCTL, rctl); 3534 3535 /* 3536 * In order to support SR-IOV and eventually VMDq it is necessary to set 3537 * the VMOLR to enable the appropriate modes. Without this workaround 3538 * we will have issues with VLAN tag stripping not being done for frames 3539 * that are only arriving because we are the default pool 3540 */ 3541 if (hw->mac.type < e1000_82576) 3542 return; 3543 3544 vmolr |= rd32(E1000_VMOLR(vfn)) & 3545 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE); 3546 wr32(E1000_VMOLR(vfn), vmolr); 3547 igb_restore_vf_multicasts(adapter); 3548 } 3549 3550 static void igb_check_wvbr(struct igb_adapter *adapter) 3551 { 3552 struct e1000_hw *hw = &adapter->hw; 3553 u32 wvbr = 0; 3554 3555 switch (hw->mac.type) { 3556 case e1000_82576: 3557 case e1000_i350: 3558 if (!(wvbr = rd32(E1000_WVBR))) 3559 return; 3560 break; 3561 default: 3562 break; 3563 } 3564 3565 adapter->wvbr |= wvbr; 3566 } 3567 3568 #define IGB_STAGGERED_QUEUE_OFFSET 8 3569 3570 static void igb_spoof_check(struct igb_adapter *adapter) 3571 { 3572 int j; 3573 3574 if (!adapter->wvbr) 3575 return; 3576 3577 for(j = 0; j < adapter->vfs_allocated_count; j++) { 3578 if (adapter->wvbr & (1 << j) || 3579 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) { 3580 dev_warn(&adapter->pdev->dev, 3581 "Spoof event(s) detected on VF %d\n", j); 3582 adapter->wvbr &= 3583 ~((1 << j) | 3584 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))); 3585 } 3586 } 3587 } 3588 3589 /* Need to wait a few seconds after link up to get diagnostic information from 3590 * the phy */ 3591 static void igb_update_phy_info(unsigned long data) 3592 { 3593 struct igb_adapter *adapter = (struct igb_adapter *) data; 3594 igb_get_phy_info(&adapter->hw); 3595 } 3596 3597 /** 3598 * igb_has_link - check shared code for link and determine up/down 3599 * @adapter: pointer to driver private info 3600 **/ 3601 bool igb_has_link(struct igb_adapter *adapter) 3602 { 3603 struct e1000_hw *hw = &adapter->hw; 3604 bool link_active = false; 3605 s32 ret_val = 0; 3606 3607 /* get_link_status is set on LSC (link status) interrupt or 3608 * rx sequence error interrupt. get_link_status will stay 3609 * false until the e1000_check_for_link establishes link 3610 * for copper adapters ONLY 3611 */ 3612 switch (hw->phy.media_type) { 3613 case e1000_media_type_copper: 3614 if (hw->mac.get_link_status) { 3615 ret_val = hw->mac.ops.check_for_link(hw); 3616 link_active = !hw->mac.get_link_status; 3617 } else { 3618 link_active = true; 3619 } 3620 break; 3621 case e1000_media_type_internal_serdes: 3622 ret_val = hw->mac.ops.check_for_link(hw); 3623 link_active = hw->mac.serdes_has_link; 3624 break; 3625 default: 3626 case e1000_media_type_unknown: 3627 break; 3628 } 3629 3630 return link_active; 3631 } 3632 3633 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event) 3634 { 3635 bool ret = false; 3636 u32 ctrl_ext, thstat; 3637 3638 /* check for thermal sensor event on i350, copper only */ 3639 if (hw->mac.type == e1000_i350) { 3640 thstat = rd32(E1000_THSTAT); 3641 ctrl_ext = rd32(E1000_CTRL_EXT); 3642 3643 if ((hw->phy.media_type == e1000_media_type_copper) && 3644 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) { 3645 ret = !!(thstat & event); 3646 } 3647 } 3648 3649 return ret; 3650 } 3651 3652 /** 3653 * igb_watchdog - Timer Call-back 3654 * @data: pointer to adapter cast into an unsigned long 3655 **/ 3656 static void igb_watchdog(unsigned long data) 3657 { 3658 struct igb_adapter *adapter = (struct igb_adapter *)data; 3659 /* Do the rest outside of interrupt context */ 3660 schedule_work(&adapter->watchdog_task); 3661 } 3662 3663 static void igb_watchdog_task(struct work_struct *work) 3664 { 3665 struct igb_adapter *adapter = container_of(work, 3666 struct igb_adapter, 3667 watchdog_task); 3668 struct e1000_hw *hw = &adapter->hw; 3669 struct net_device *netdev = adapter->netdev; 3670 u32 link; 3671 int i; 3672 3673 link = igb_has_link(adapter); 3674 if (link) { 3675 /* Cancel scheduled suspend requests. */ 3676 pm_runtime_resume(netdev->dev.parent); 3677 3678 if (!netif_carrier_ok(netdev)) { 3679 u32 ctrl; 3680 hw->mac.ops.get_speed_and_duplex(hw, 3681 &adapter->link_speed, 3682 &adapter->link_duplex); 3683 3684 ctrl = rd32(E1000_CTRL); 3685 /* Links status message must follow this format */ 3686 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s " 3687 "Duplex, Flow Control: %s\n", 3688 netdev->name, 3689 adapter->link_speed, 3690 adapter->link_duplex == FULL_DUPLEX ? 3691 "Full" : "Half", 3692 (ctrl & E1000_CTRL_TFCE) && 3693 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" : 3694 (ctrl & E1000_CTRL_RFCE) ? "RX" : 3695 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None"); 3696 3697 /* check for thermal sensor event */ 3698 if (igb_thermal_sensor_event(hw, 3699 E1000_THSTAT_LINK_THROTTLE)) { 3700 netdev_info(netdev, "The network adapter link " 3701 "speed was downshifted because it " 3702 "overheated\n"); 3703 } 3704 3705 /* adjust timeout factor according to speed/duplex */ 3706 adapter->tx_timeout_factor = 1; 3707 switch (adapter->link_speed) { 3708 case SPEED_10: 3709 adapter->tx_timeout_factor = 14; 3710 break; 3711 case SPEED_100: 3712 /* maybe add some timeout factor ? */ 3713 break; 3714 } 3715 3716 netif_carrier_on(netdev); 3717 3718 igb_ping_all_vfs(adapter); 3719 igb_check_vf_rate_limit(adapter); 3720 3721 /* link state has changed, schedule phy info update */ 3722 if (!test_bit(__IGB_DOWN, &adapter->state)) 3723 mod_timer(&adapter->phy_info_timer, 3724 round_jiffies(jiffies + 2 * HZ)); 3725 } 3726 } else { 3727 if (netif_carrier_ok(netdev)) { 3728 adapter->link_speed = 0; 3729 adapter->link_duplex = 0; 3730 3731 /* check for thermal sensor event */ 3732 if (igb_thermal_sensor_event(hw, 3733 E1000_THSTAT_PWR_DOWN)) { 3734 netdev_err(netdev, "The network adapter was " 3735 "stopped because it overheated\n"); 3736 } 3737 3738 /* Links status message must follow this format */ 3739 printk(KERN_INFO "igb: %s NIC Link is Down\n", 3740 netdev->name); 3741 netif_carrier_off(netdev); 3742 3743 igb_ping_all_vfs(adapter); 3744 3745 /* link state has changed, schedule phy info update */ 3746 if (!test_bit(__IGB_DOWN, &adapter->state)) 3747 mod_timer(&adapter->phy_info_timer, 3748 round_jiffies(jiffies + 2 * HZ)); 3749 3750 pm_schedule_suspend(netdev->dev.parent, 3751 MSEC_PER_SEC * 5); 3752 } 3753 } 3754 3755 spin_lock(&adapter->stats64_lock); 3756 igb_update_stats(adapter, &adapter->stats64); 3757 spin_unlock(&adapter->stats64_lock); 3758 3759 for (i = 0; i < adapter->num_tx_queues; i++) { 3760 struct igb_ring *tx_ring = adapter->tx_ring[i]; 3761 if (!netif_carrier_ok(netdev)) { 3762 /* We've lost link, so the controller stops DMA, 3763 * but we've got queued Tx work that's never going 3764 * to get done, so reset controller to flush Tx. 3765 * (Do the reset outside of interrupt context). */ 3766 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) { 3767 adapter->tx_timeout_count++; 3768 schedule_work(&adapter->reset_task); 3769 /* return immediately since reset is imminent */ 3770 return; 3771 } 3772 } 3773 3774 /* Force detection of hung controller every watchdog period */ 3775 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 3776 } 3777 3778 /* Cause software interrupt to ensure rx ring is cleaned */ 3779 if (adapter->msix_entries) { 3780 u32 eics = 0; 3781 for (i = 0; i < adapter->num_q_vectors; i++) 3782 eics |= adapter->q_vector[i]->eims_value; 3783 wr32(E1000_EICS, eics); 3784 } else { 3785 wr32(E1000_ICS, E1000_ICS_RXDMT0); 3786 } 3787 3788 igb_spoof_check(adapter); 3789 3790 /* Reset the timer */ 3791 if (!test_bit(__IGB_DOWN, &adapter->state)) 3792 mod_timer(&adapter->watchdog_timer, 3793 round_jiffies(jiffies + 2 * HZ)); 3794 } 3795 3796 enum latency_range { 3797 lowest_latency = 0, 3798 low_latency = 1, 3799 bulk_latency = 2, 3800 latency_invalid = 255 3801 }; 3802 3803 /** 3804 * igb_update_ring_itr - update the dynamic ITR value based on packet size 3805 * 3806 * Stores a new ITR value based on strictly on packet size. This 3807 * algorithm is less sophisticated than that used in igb_update_itr, 3808 * due to the difficulty of synchronizing statistics across multiple 3809 * receive rings. The divisors and thresholds used by this function 3810 * were determined based on theoretical maximum wire speed and testing 3811 * data, in order to minimize response time while increasing bulk 3812 * throughput. 3813 * This functionality is controlled by the InterruptThrottleRate module 3814 * parameter (see igb_param.c) 3815 * NOTE: This function is called only when operating in a multiqueue 3816 * receive environment. 3817 * @q_vector: pointer to q_vector 3818 **/ 3819 static void igb_update_ring_itr(struct igb_q_vector *q_vector) 3820 { 3821 int new_val = q_vector->itr_val; 3822 int avg_wire_size = 0; 3823 struct igb_adapter *adapter = q_vector->adapter; 3824 unsigned int packets; 3825 3826 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3827 * ints/sec - ITR timer value of 120 ticks. 3828 */ 3829 if (adapter->link_speed != SPEED_1000) { 3830 new_val = IGB_4K_ITR; 3831 goto set_itr_val; 3832 } 3833 3834 packets = q_vector->rx.total_packets; 3835 if (packets) 3836 avg_wire_size = q_vector->rx.total_bytes / packets; 3837 3838 packets = q_vector->tx.total_packets; 3839 if (packets) 3840 avg_wire_size = max_t(u32, avg_wire_size, 3841 q_vector->tx.total_bytes / packets); 3842 3843 /* if avg_wire_size isn't set no work was done */ 3844 if (!avg_wire_size) 3845 goto clear_counts; 3846 3847 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3848 avg_wire_size += 24; 3849 3850 /* Don't starve jumbo frames */ 3851 avg_wire_size = min(avg_wire_size, 3000); 3852 3853 /* Give a little boost to mid-size frames */ 3854 if ((avg_wire_size > 300) && (avg_wire_size < 1200)) 3855 new_val = avg_wire_size / 3; 3856 else 3857 new_val = avg_wire_size / 2; 3858 3859 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3860 if (new_val < IGB_20K_ITR && 3861 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3862 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3863 new_val = IGB_20K_ITR; 3864 3865 set_itr_val: 3866 if (new_val != q_vector->itr_val) { 3867 q_vector->itr_val = new_val; 3868 q_vector->set_itr = 1; 3869 } 3870 clear_counts: 3871 q_vector->rx.total_bytes = 0; 3872 q_vector->rx.total_packets = 0; 3873 q_vector->tx.total_bytes = 0; 3874 q_vector->tx.total_packets = 0; 3875 } 3876 3877 /** 3878 * igb_update_itr - update the dynamic ITR value based on statistics 3879 * Stores a new ITR value based on packets and byte 3880 * counts during the last interrupt. The advantage of per interrupt 3881 * computation is faster updates and more accurate ITR for the current 3882 * traffic pattern. Constants in this function were computed 3883 * based on theoretical maximum wire speed and thresholds were set based 3884 * on testing data as well as attempting to minimize response time 3885 * while increasing bulk throughput. 3886 * this functionality is controlled by the InterruptThrottleRate module 3887 * parameter (see igb_param.c) 3888 * NOTE: These calculations are only valid when operating in a single- 3889 * queue environment. 3890 * @q_vector: pointer to q_vector 3891 * @ring_container: ring info to update the itr for 3892 **/ 3893 static void igb_update_itr(struct igb_q_vector *q_vector, 3894 struct igb_ring_container *ring_container) 3895 { 3896 unsigned int packets = ring_container->total_packets; 3897 unsigned int bytes = ring_container->total_bytes; 3898 u8 itrval = ring_container->itr; 3899 3900 /* no packets, exit with status unchanged */ 3901 if (packets == 0) 3902 return; 3903 3904 switch (itrval) { 3905 case lowest_latency: 3906 /* handle TSO and jumbo frames */ 3907 if (bytes/packets > 8000) 3908 itrval = bulk_latency; 3909 else if ((packets < 5) && (bytes > 512)) 3910 itrval = low_latency; 3911 break; 3912 case low_latency: /* 50 usec aka 20000 ints/s */ 3913 if (bytes > 10000) { 3914 /* this if handles the TSO accounting */ 3915 if (bytes/packets > 8000) { 3916 itrval = bulk_latency; 3917 } else if ((packets < 10) || ((bytes/packets) > 1200)) { 3918 itrval = bulk_latency; 3919 } else if ((packets > 35)) { 3920 itrval = lowest_latency; 3921 } 3922 } else if (bytes/packets > 2000) { 3923 itrval = bulk_latency; 3924 } else if (packets <= 2 && bytes < 512) { 3925 itrval = lowest_latency; 3926 } 3927 break; 3928 case bulk_latency: /* 250 usec aka 4000 ints/s */ 3929 if (bytes > 25000) { 3930 if (packets > 35) 3931 itrval = low_latency; 3932 } else if (bytes < 1500) { 3933 itrval = low_latency; 3934 } 3935 break; 3936 } 3937 3938 /* clear work counters since we have the values we need */ 3939 ring_container->total_bytes = 0; 3940 ring_container->total_packets = 0; 3941 3942 /* write updated itr to ring container */ 3943 ring_container->itr = itrval; 3944 } 3945 3946 static void igb_set_itr(struct igb_q_vector *q_vector) 3947 { 3948 struct igb_adapter *adapter = q_vector->adapter; 3949 u32 new_itr = q_vector->itr_val; 3950 u8 current_itr = 0; 3951 3952 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3953 if (adapter->link_speed != SPEED_1000) { 3954 current_itr = 0; 3955 new_itr = IGB_4K_ITR; 3956 goto set_itr_now; 3957 } 3958 3959 igb_update_itr(q_vector, &q_vector->tx); 3960 igb_update_itr(q_vector, &q_vector->rx); 3961 3962 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3963 3964 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3965 if (current_itr == lowest_latency && 3966 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3967 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3968 current_itr = low_latency; 3969 3970 switch (current_itr) { 3971 /* counts and packets in update_itr are dependent on these numbers */ 3972 case lowest_latency: 3973 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */ 3974 break; 3975 case low_latency: 3976 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */ 3977 break; 3978 case bulk_latency: 3979 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */ 3980 break; 3981 default: 3982 break; 3983 } 3984 3985 set_itr_now: 3986 if (new_itr != q_vector->itr_val) { 3987 /* this attempts to bias the interrupt rate towards Bulk 3988 * by adding intermediate steps when interrupt rate is 3989 * increasing */ 3990 new_itr = new_itr > q_vector->itr_val ? 3991 max((new_itr * q_vector->itr_val) / 3992 (new_itr + (q_vector->itr_val >> 2)), 3993 new_itr) : 3994 new_itr; 3995 /* Don't write the value here; it resets the adapter's 3996 * internal timer, and causes us to delay far longer than 3997 * we should between interrupts. Instead, we write the ITR 3998 * value at the beginning of the next interrupt so the timing 3999 * ends up being correct. 4000 */ 4001 q_vector->itr_val = new_itr; 4002 q_vector->set_itr = 1; 4003 } 4004 } 4005 4006 void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens, 4007 u32 type_tucmd, u32 mss_l4len_idx) 4008 { 4009 struct e1000_adv_tx_context_desc *context_desc; 4010 u16 i = tx_ring->next_to_use; 4011 4012 context_desc = IGB_TX_CTXTDESC(tx_ring, i); 4013 4014 i++; 4015 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 4016 4017 /* set bits to identify this as an advanced context descriptor */ 4018 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 4019 4020 /* For 82575, context index must be unique per ring. */ 4021 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 4022 mss_l4len_idx |= tx_ring->reg_idx << 4; 4023 4024 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 4025 context_desc->seqnum_seed = 0; 4026 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 4027 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 4028 } 4029 4030 static int igb_tso(struct igb_ring *tx_ring, 4031 struct igb_tx_buffer *first, 4032 u8 *hdr_len) 4033 { 4034 struct sk_buff *skb = first->skb; 4035 u32 vlan_macip_lens, type_tucmd; 4036 u32 mss_l4len_idx, l4len; 4037 4038 if (!skb_is_gso(skb)) 4039 return 0; 4040 4041 if (skb_header_cloned(skb)) { 4042 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 4043 if (err) 4044 return err; 4045 } 4046 4047 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 4048 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP; 4049 4050 if (first->protocol == __constant_htons(ETH_P_IP)) { 4051 struct iphdr *iph = ip_hdr(skb); 4052 iph->tot_len = 0; 4053 iph->check = 0; 4054 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 4055 iph->daddr, 0, 4056 IPPROTO_TCP, 4057 0); 4058 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4; 4059 first->tx_flags |= IGB_TX_FLAGS_TSO | 4060 IGB_TX_FLAGS_CSUM | 4061 IGB_TX_FLAGS_IPV4; 4062 } else if (skb_is_gso_v6(skb)) { 4063 ipv6_hdr(skb)->payload_len = 0; 4064 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 4065 &ipv6_hdr(skb)->daddr, 4066 0, IPPROTO_TCP, 0); 4067 first->tx_flags |= IGB_TX_FLAGS_TSO | 4068 IGB_TX_FLAGS_CSUM; 4069 } 4070 4071 /* compute header lengths */ 4072 l4len = tcp_hdrlen(skb); 4073 *hdr_len = skb_transport_offset(skb) + l4len; 4074 4075 /* update gso size and bytecount with header size */ 4076 first->gso_segs = skb_shinfo(skb)->gso_segs; 4077 first->bytecount += (first->gso_segs - 1) * *hdr_len; 4078 4079 /* MSS L4LEN IDX */ 4080 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT; 4081 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT; 4082 4083 /* VLAN MACLEN IPLEN */ 4084 vlan_macip_lens = skb_network_header_len(skb); 4085 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT; 4086 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; 4087 4088 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx); 4089 4090 return 1; 4091 } 4092 4093 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first) 4094 { 4095 struct sk_buff *skb = first->skb; 4096 u32 vlan_macip_lens = 0; 4097 u32 mss_l4len_idx = 0; 4098 u32 type_tucmd = 0; 4099 4100 if (skb->ip_summed != CHECKSUM_PARTIAL) { 4101 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN)) 4102 return; 4103 } else { 4104 u8 l4_hdr = 0; 4105 switch (first->protocol) { 4106 case __constant_htons(ETH_P_IP): 4107 vlan_macip_lens |= skb_network_header_len(skb); 4108 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4; 4109 l4_hdr = ip_hdr(skb)->protocol; 4110 break; 4111 case __constant_htons(ETH_P_IPV6): 4112 vlan_macip_lens |= skb_network_header_len(skb); 4113 l4_hdr = ipv6_hdr(skb)->nexthdr; 4114 break; 4115 default: 4116 if (unlikely(net_ratelimit())) { 4117 dev_warn(tx_ring->dev, 4118 "partial checksum but proto=%x!\n", 4119 first->protocol); 4120 } 4121 break; 4122 } 4123 4124 switch (l4_hdr) { 4125 case IPPROTO_TCP: 4126 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP; 4127 mss_l4len_idx = tcp_hdrlen(skb) << 4128 E1000_ADVTXD_L4LEN_SHIFT; 4129 break; 4130 case IPPROTO_SCTP: 4131 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP; 4132 mss_l4len_idx = sizeof(struct sctphdr) << 4133 E1000_ADVTXD_L4LEN_SHIFT; 4134 break; 4135 case IPPROTO_UDP: 4136 mss_l4len_idx = sizeof(struct udphdr) << 4137 E1000_ADVTXD_L4LEN_SHIFT; 4138 break; 4139 default: 4140 if (unlikely(net_ratelimit())) { 4141 dev_warn(tx_ring->dev, 4142 "partial checksum but l4 proto=%x!\n", 4143 l4_hdr); 4144 } 4145 break; 4146 } 4147 4148 /* update TX checksum flag */ 4149 first->tx_flags |= IGB_TX_FLAGS_CSUM; 4150 } 4151 4152 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT; 4153 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; 4154 4155 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx); 4156 } 4157 4158 static __le32 igb_tx_cmd_type(u32 tx_flags) 4159 { 4160 /* set type for advanced descriptor with frame checksum insertion */ 4161 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA | 4162 E1000_ADVTXD_DCMD_IFCS | 4163 E1000_ADVTXD_DCMD_DEXT); 4164 4165 /* set HW vlan bit if vlan is present */ 4166 if (tx_flags & IGB_TX_FLAGS_VLAN) 4167 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE); 4168 4169 /* set timestamp bit if present */ 4170 if (tx_flags & IGB_TX_FLAGS_TSTAMP) 4171 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP); 4172 4173 /* set segmentation bits for TSO */ 4174 if (tx_flags & IGB_TX_FLAGS_TSO) 4175 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE); 4176 4177 return cmd_type; 4178 } 4179 4180 static void igb_tx_olinfo_status(struct igb_ring *tx_ring, 4181 union e1000_adv_tx_desc *tx_desc, 4182 u32 tx_flags, unsigned int paylen) 4183 { 4184 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT; 4185 4186 /* 82575 requires a unique index per ring if any offload is enabled */ 4187 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) && 4188 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 4189 olinfo_status |= tx_ring->reg_idx << 4; 4190 4191 /* insert L4 checksum */ 4192 if (tx_flags & IGB_TX_FLAGS_CSUM) { 4193 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 4194 4195 /* insert IPv4 checksum */ 4196 if (tx_flags & IGB_TX_FLAGS_IPV4) 4197 olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 4198 } 4199 4200 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 4201 } 4202 4203 /* 4204 * The largest size we can write to the descriptor is 65535. In order to 4205 * maintain a power of two alignment we have to limit ourselves to 32K. 4206 */ 4207 #define IGB_MAX_TXD_PWR 15 4208 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR) 4209 4210 static void igb_tx_map(struct igb_ring *tx_ring, 4211 struct igb_tx_buffer *first, 4212 const u8 hdr_len) 4213 { 4214 struct sk_buff *skb = first->skb; 4215 struct igb_tx_buffer *tx_buffer_info; 4216 union e1000_adv_tx_desc *tx_desc; 4217 dma_addr_t dma; 4218 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 4219 unsigned int data_len = skb->data_len; 4220 unsigned int size = skb_headlen(skb); 4221 unsigned int paylen = skb->len - hdr_len; 4222 __le32 cmd_type; 4223 u32 tx_flags = first->tx_flags; 4224 u16 i = tx_ring->next_to_use; 4225 4226 tx_desc = IGB_TX_DESC(tx_ring, i); 4227 4228 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen); 4229 cmd_type = igb_tx_cmd_type(tx_flags); 4230 4231 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 4232 if (dma_mapping_error(tx_ring->dev, dma)) 4233 goto dma_error; 4234 4235 /* record length, and DMA address */ 4236 first->length = size; 4237 first->dma = dma; 4238 tx_desc->read.buffer_addr = cpu_to_le64(dma); 4239 4240 for (;;) { 4241 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) { 4242 tx_desc->read.cmd_type_len = 4243 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD); 4244 4245 i++; 4246 tx_desc++; 4247 if (i == tx_ring->count) { 4248 tx_desc = IGB_TX_DESC(tx_ring, 0); 4249 i = 0; 4250 } 4251 4252 dma += IGB_MAX_DATA_PER_TXD; 4253 size -= IGB_MAX_DATA_PER_TXD; 4254 4255 tx_desc->read.olinfo_status = 0; 4256 tx_desc->read.buffer_addr = cpu_to_le64(dma); 4257 } 4258 4259 if (likely(!data_len)) 4260 break; 4261 4262 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 4263 4264 i++; 4265 tx_desc++; 4266 if (i == tx_ring->count) { 4267 tx_desc = IGB_TX_DESC(tx_ring, 0); 4268 i = 0; 4269 } 4270 4271 size = skb_frag_size(frag); 4272 data_len -= size; 4273 4274 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 4275 size, DMA_TO_DEVICE); 4276 if (dma_mapping_error(tx_ring->dev, dma)) 4277 goto dma_error; 4278 4279 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 4280 tx_buffer_info->length = size; 4281 tx_buffer_info->dma = dma; 4282 4283 tx_desc->read.olinfo_status = 0; 4284 tx_desc->read.buffer_addr = cpu_to_le64(dma); 4285 4286 frag++; 4287 } 4288 4289 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 4290 4291 /* write last descriptor with RS and EOP bits */ 4292 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD); 4293 tx_desc->read.cmd_type_len = cmd_type; 4294 4295 /* set the timestamp */ 4296 first->time_stamp = jiffies; 4297 4298 /* 4299 * Force memory writes to complete before letting h/w know there 4300 * are new descriptors to fetch. (Only applicable for weak-ordered 4301 * memory model archs, such as IA-64). 4302 * 4303 * We also need this memory barrier to make certain all of the 4304 * status bits have been updated before next_to_watch is written. 4305 */ 4306 wmb(); 4307 4308 /* set next_to_watch value indicating a packet is present */ 4309 first->next_to_watch = tx_desc; 4310 4311 i++; 4312 if (i == tx_ring->count) 4313 i = 0; 4314 4315 tx_ring->next_to_use = i; 4316 4317 writel(i, tx_ring->tail); 4318 4319 /* we need this if more than one processor can write to our tail 4320 * at a time, it syncronizes IO on IA64/Altix systems */ 4321 mmiowb(); 4322 4323 return; 4324 4325 dma_error: 4326 dev_err(tx_ring->dev, "TX DMA map failed\n"); 4327 4328 /* clear dma mappings for failed tx_buffer_info map */ 4329 for (;;) { 4330 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 4331 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 4332 if (tx_buffer_info == first) 4333 break; 4334 if (i == 0) 4335 i = tx_ring->count; 4336 i--; 4337 } 4338 4339 tx_ring->next_to_use = i; 4340 } 4341 4342 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) 4343 { 4344 struct net_device *netdev = tx_ring->netdev; 4345 4346 netif_stop_subqueue(netdev, tx_ring->queue_index); 4347 4348 /* Herbert's original patch had: 4349 * smp_mb__after_netif_stop_queue(); 4350 * but since that doesn't exist yet, just open code it. */ 4351 smp_mb(); 4352 4353 /* We need to check again in a case another CPU has just 4354 * made room available. */ 4355 if (igb_desc_unused(tx_ring) < size) 4356 return -EBUSY; 4357 4358 /* A reprieve! */ 4359 netif_wake_subqueue(netdev, tx_ring->queue_index); 4360 4361 u64_stats_update_begin(&tx_ring->tx_syncp2); 4362 tx_ring->tx_stats.restart_queue2++; 4363 u64_stats_update_end(&tx_ring->tx_syncp2); 4364 4365 return 0; 4366 } 4367 4368 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) 4369 { 4370 if (igb_desc_unused(tx_ring) >= size) 4371 return 0; 4372 return __igb_maybe_stop_tx(tx_ring, size); 4373 } 4374 4375 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb, 4376 struct igb_ring *tx_ring) 4377 { 4378 struct igb_tx_buffer *first; 4379 int tso; 4380 u32 tx_flags = 0; 4381 __be16 protocol = vlan_get_protocol(skb); 4382 u8 hdr_len = 0; 4383 4384 /* need: 1 descriptor per page, 4385 * + 2 desc gap to keep tail from touching head, 4386 * + 1 desc for skb->data, 4387 * + 1 desc for context descriptor, 4388 * otherwise try next time */ 4389 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) { 4390 /* this is a hard error */ 4391 return NETDEV_TX_BUSY; 4392 } 4393 4394 /* record the location of the first descriptor for this packet */ 4395 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 4396 first->skb = skb; 4397 first->bytecount = skb->len; 4398 first->gso_segs = 1; 4399 4400 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 4401 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4402 tx_flags |= IGB_TX_FLAGS_TSTAMP; 4403 } 4404 4405 if (vlan_tx_tag_present(skb)) { 4406 tx_flags |= IGB_TX_FLAGS_VLAN; 4407 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT); 4408 } 4409 4410 /* record initial flags and protocol */ 4411 first->tx_flags = tx_flags; 4412 first->protocol = protocol; 4413 4414 tso = igb_tso(tx_ring, first, &hdr_len); 4415 if (tso < 0) 4416 goto out_drop; 4417 else if (!tso) 4418 igb_tx_csum(tx_ring, first); 4419 4420 igb_tx_map(tx_ring, first, hdr_len); 4421 4422 /* Make sure there is space in the ring for the next send. */ 4423 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4); 4424 4425 return NETDEV_TX_OK; 4426 4427 out_drop: 4428 igb_unmap_and_free_tx_resource(tx_ring, first); 4429 4430 return NETDEV_TX_OK; 4431 } 4432 4433 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter, 4434 struct sk_buff *skb) 4435 { 4436 unsigned int r_idx = skb->queue_mapping; 4437 4438 if (r_idx >= adapter->num_tx_queues) 4439 r_idx = r_idx % adapter->num_tx_queues; 4440 4441 return adapter->tx_ring[r_idx]; 4442 } 4443 4444 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, 4445 struct net_device *netdev) 4446 { 4447 struct igb_adapter *adapter = netdev_priv(netdev); 4448 4449 if (test_bit(__IGB_DOWN, &adapter->state)) { 4450 dev_kfree_skb_any(skb); 4451 return NETDEV_TX_OK; 4452 } 4453 4454 if (skb->len <= 0) { 4455 dev_kfree_skb_any(skb); 4456 return NETDEV_TX_OK; 4457 } 4458 4459 /* 4460 * The minimum packet size with TCTL.PSP set is 17 so pad the skb 4461 * in order to meet this minimum size requirement. 4462 */ 4463 if (skb->len < 17) { 4464 if (skb_padto(skb, 17)) 4465 return NETDEV_TX_OK; 4466 skb->len = 17; 4467 } 4468 4469 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb)); 4470 } 4471 4472 /** 4473 * igb_tx_timeout - Respond to a Tx Hang 4474 * @netdev: network interface device structure 4475 **/ 4476 static void igb_tx_timeout(struct net_device *netdev) 4477 { 4478 struct igb_adapter *adapter = netdev_priv(netdev); 4479 struct e1000_hw *hw = &adapter->hw; 4480 4481 /* Do the reset outside of interrupt context */ 4482 adapter->tx_timeout_count++; 4483 4484 if (hw->mac.type >= e1000_82580) 4485 hw->dev_spec._82575.global_device_reset = true; 4486 4487 schedule_work(&adapter->reset_task); 4488 wr32(E1000_EICS, 4489 (adapter->eims_enable_mask & ~adapter->eims_other)); 4490 } 4491 4492 static void igb_reset_task(struct work_struct *work) 4493 { 4494 struct igb_adapter *adapter; 4495 adapter = container_of(work, struct igb_adapter, reset_task); 4496 4497 igb_dump(adapter); 4498 netdev_err(adapter->netdev, "Reset adapter\n"); 4499 igb_reinit_locked(adapter); 4500 } 4501 4502 /** 4503 * igb_get_stats64 - Get System Network Statistics 4504 * @netdev: network interface device structure 4505 * @stats: rtnl_link_stats64 pointer 4506 * 4507 **/ 4508 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev, 4509 struct rtnl_link_stats64 *stats) 4510 { 4511 struct igb_adapter *adapter = netdev_priv(netdev); 4512 4513 spin_lock(&adapter->stats64_lock); 4514 igb_update_stats(adapter, &adapter->stats64); 4515 memcpy(stats, &adapter->stats64, sizeof(*stats)); 4516 spin_unlock(&adapter->stats64_lock); 4517 4518 return stats; 4519 } 4520 4521 /** 4522 * igb_change_mtu - Change the Maximum Transfer Unit 4523 * @netdev: network interface device structure 4524 * @new_mtu: new value for maximum frame size 4525 * 4526 * Returns 0 on success, negative on failure 4527 **/ 4528 static int igb_change_mtu(struct net_device *netdev, int new_mtu) 4529 { 4530 struct igb_adapter *adapter = netdev_priv(netdev); 4531 struct pci_dev *pdev = adapter->pdev; 4532 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 4533 4534 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) { 4535 dev_err(&pdev->dev, "Invalid MTU setting\n"); 4536 return -EINVAL; 4537 } 4538 4539 #define MAX_STD_JUMBO_FRAME_SIZE 9238 4540 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) { 4541 dev_err(&pdev->dev, "MTU > 9216 not supported.\n"); 4542 return -EINVAL; 4543 } 4544 4545 while (test_and_set_bit(__IGB_RESETTING, &adapter->state)) 4546 msleep(1); 4547 4548 /* igb_down has a dependency on max_frame_size */ 4549 adapter->max_frame_size = max_frame; 4550 4551 if (netif_running(netdev)) 4552 igb_down(adapter); 4553 4554 dev_info(&pdev->dev, "changing MTU from %d to %d\n", 4555 netdev->mtu, new_mtu); 4556 netdev->mtu = new_mtu; 4557 4558 if (netif_running(netdev)) 4559 igb_up(adapter); 4560 else 4561 igb_reset(adapter); 4562 4563 clear_bit(__IGB_RESETTING, &adapter->state); 4564 4565 return 0; 4566 } 4567 4568 /** 4569 * igb_update_stats - Update the board statistics counters 4570 * @adapter: board private structure 4571 **/ 4572 4573 void igb_update_stats(struct igb_adapter *adapter, 4574 struct rtnl_link_stats64 *net_stats) 4575 { 4576 struct e1000_hw *hw = &adapter->hw; 4577 struct pci_dev *pdev = adapter->pdev; 4578 u32 reg, mpc; 4579 u16 phy_tmp; 4580 int i; 4581 u64 bytes, packets; 4582 unsigned int start; 4583 u64 _bytes, _packets; 4584 4585 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF 4586 4587 /* 4588 * Prevent stats update while adapter is being reset, or if the pci 4589 * connection is down. 4590 */ 4591 if (adapter->link_speed == 0) 4592 return; 4593 if (pci_channel_offline(pdev)) 4594 return; 4595 4596 bytes = 0; 4597 packets = 0; 4598 for (i = 0; i < adapter->num_rx_queues; i++) { 4599 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF; 4600 struct igb_ring *ring = adapter->rx_ring[i]; 4601 4602 ring->rx_stats.drops += rqdpc_tmp; 4603 net_stats->rx_fifo_errors += rqdpc_tmp; 4604 4605 do { 4606 start = u64_stats_fetch_begin_bh(&ring->rx_syncp); 4607 _bytes = ring->rx_stats.bytes; 4608 _packets = ring->rx_stats.packets; 4609 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start)); 4610 bytes += _bytes; 4611 packets += _packets; 4612 } 4613 4614 net_stats->rx_bytes = bytes; 4615 net_stats->rx_packets = packets; 4616 4617 bytes = 0; 4618 packets = 0; 4619 for (i = 0; i < adapter->num_tx_queues; i++) { 4620 struct igb_ring *ring = adapter->tx_ring[i]; 4621 do { 4622 start = u64_stats_fetch_begin_bh(&ring->tx_syncp); 4623 _bytes = ring->tx_stats.bytes; 4624 _packets = ring->tx_stats.packets; 4625 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start)); 4626 bytes += _bytes; 4627 packets += _packets; 4628 } 4629 net_stats->tx_bytes = bytes; 4630 net_stats->tx_packets = packets; 4631 4632 /* read stats registers */ 4633 adapter->stats.crcerrs += rd32(E1000_CRCERRS); 4634 adapter->stats.gprc += rd32(E1000_GPRC); 4635 adapter->stats.gorc += rd32(E1000_GORCL); 4636 rd32(E1000_GORCH); /* clear GORCL */ 4637 adapter->stats.bprc += rd32(E1000_BPRC); 4638 adapter->stats.mprc += rd32(E1000_MPRC); 4639 adapter->stats.roc += rd32(E1000_ROC); 4640 4641 adapter->stats.prc64 += rd32(E1000_PRC64); 4642 adapter->stats.prc127 += rd32(E1000_PRC127); 4643 adapter->stats.prc255 += rd32(E1000_PRC255); 4644 adapter->stats.prc511 += rd32(E1000_PRC511); 4645 adapter->stats.prc1023 += rd32(E1000_PRC1023); 4646 adapter->stats.prc1522 += rd32(E1000_PRC1522); 4647 adapter->stats.symerrs += rd32(E1000_SYMERRS); 4648 adapter->stats.sec += rd32(E1000_SEC); 4649 4650 mpc = rd32(E1000_MPC); 4651 adapter->stats.mpc += mpc; 4652 net_stats->rx_fifo_errors += mpc; 4653 adapter->stats.scc += rd32(E1000_SCC); 4654 adapter->stats.ecol += rd32(E1000_ECOL); 4655 adapter->stats.mcc += rd32(E1000_MCC); 4656 adapter->stats.latecol += rd32(E1000_LATECOL); 4657 adapter->stats.dc += rd32(E1000_DC); 4658 adapter->stats.rlec += rd32(E1000_RLEC); 4659 adapter->stats.xonrxc += rd32(E1000_XONRXC); 4660 adapter->stats.xontxc += rd32(E1000_XONTXC); 4661 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC); 4662 adapter->stats.xofftxc += rd32(E1000_XOFFTXC); 4663 adapter->stats.fcruc += rd32(E1000_FCRUC); 4664 adapter->stats.gptc += rd32(E1000_GPTC); 4665 adapter->stats.gotc += rd32(E1000_GOTCL); 4666 rd32(E1000_GOTCH); /* clear GOTCL */ 4667 adapter->stats.rnbc += rd32(E1000_RNBC); 4668 adapter->stats.ruc += rd32(E1000_RUC); 4669 adapter->stats.rfc += rd32(E1000_RFC); 4670 adapter->stats.rjc += rd32(E1000_RJC); 4671 adapter->stats.tor += rd32(E1000_TORH); 4672 adapter->stats.tot += rd32(E1000_TOTH); 4673 adapter->stats.tpr += rd32(E1000_TPR); 4674 4675 adapter->stats.ptc64 += rd32(E1000_PTC64); 4676 adapter->stats.ptc127 += rd32(E1000_PTC127); 4677 adapter->stats.ptc255 += rd32(E1000_PTC255); 4678 adapter->stats.ptc511 += rd32(E1000_PTC511); 4679 adapter->stats.ptc1023 += rd32(E1000_PTC1023); 4680 adapter->stats.ptc1522 += rd32(E1000_PTC1522); 4681 4682 adapter->stats.mptc += rd32(E1000_MPTC); 4683 adapter->stats.bptc += rd32(E1000_BPTC); 4684 4685 adapter->stats.tpt += rd32(E1000_TPT); 4686 adapter->stats.colc += rd32(E1000_COLC); 4687 4688 adapter->stats.algnerrc += rd32(E1000_ALGNERRC); 4689 /* read internal phy specific stats */ 4690 reg = rd32(E1000_CTRL_EXT); 4691 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) { 4692 adapter->stats.rxerrc += rd32(E1000_RXERRC); 4693 adapter->stats.tncrs += rd32(E1000_TNCRS); 4694 } 4695 4696 adapter->stats.tsctc += rd32(E1000_TSCTC); 4697 adapter->stats.tsctfc += rd32(E1000_TSCTFC); 4698 4699 adapter->stats.iac += rd32(E1000_IAC); 4700 adapter->stats.icrxoc += rd32(E1000_ICRXOC); 4701 adapter->stats.icrxptc += rd32(E1000_ICRXPTC); 4702 adapter->stats.icrxatc += rd32(E1000_ICRXATC); 4703 adapter->stats.ictxptc += rd32(E1000_ICTXPTC); 4704 adapter->stats.ictxatc += rd32(E1000_ICTXATC); 4705 adapter->stats.ictxqec += rd32(E1000_ICTXQEC); 4706 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC); 4707 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC); 4708 4709 /* Fill out the OS statistics structure */ 4710 net_stats->multicast = adapter->stats.mprc; 4711 net_stats->collisions = adapter->stats.colc; 4712 4713 /* Rx Errors */ 4714 4715 /* RLEC on some newer hardware can be incorrect so build 4716 * our own version based on RUC and ROC */ 4717 net_stats->rx_errors = adapter->stats.rxerrc + 4718 adapter->stats.crcerrs + adapter->stats.algnerrc + 4719 adapter->stats.ruc + adapter->stats.roc + 4720 adapter->stats.cexterr; 4721 net_stats->rx_length_errors = adapter->stats.ruc + 4722 adapter->stats.roc; 4723 net_stats->rx_crc_errors = adapter->stats.crcerrs; 4724 net_stats->rx_frame_errors = adapter->stats.algnerrc; 4725 net_stats->rx_missed_errors = adapter->stats.mpc; 4726 4727 /* Tx Errors */ 4728 net_stats->tx_errors = adapter->stats.ecol + 4729 adapter->stats.latecol; 4730 net_stats->tx_aborted_errors = adapter->stats.ecol; 4731 net_stats->tx_window_errors = adapter->stats.latecol; 4732 net_stats->tx_carrier_errors = adapter->stats.tncrs; 4733 4734 /* Tx Dropped needs to be maintained elsewhere */ 4735 4736 /* Phy Stats */ 4737 if (hw->phy.media_type == e1000_media_type_copper) { 4738 if ((adapter->link_speed == SPEED_1000) && 4739 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) { 4740 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK; 4741 adapter->phy_stats.idle_errors += phy_tmp; 4742 } 4743 } 4744 4745 /* Management Stats */ 4746 adapter->stats.mgptc += rd32(E1000_MGTPTC); 4747 adapter->stats.mgprc += rd32(E1000_MGTPRC); 4748 adapter->stats.mgpdc += rd32(E1000_MGTPDC); 4749 4750 /* OS2BMC Stats */ 4751 reg = rd32(E1000_MANC); 4752 if (reg & E1000_MANC_EN_BMC2OS) { 4753 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC); 4754 adapter->stats.o2bspc += rd32(E1000_O2BSPC); 4755 adapter->stats.b2ospc += rd32(E1000_B2OSPC); 4756 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC); 4757 } 4758 } 4759 4760 static irqreturn_t igb_msix_other(int irq, void *data) 4761 { 4762 struct igb_adapter *adapter = data; 4763 struct e1000_hw *hw = &adapter->hw; 4764 u32 icr = rd32(E1000_ICR); 4765 /* reading ICR causes bit 31 of EICR to be cleared */ 4766 4767 if (icr & E1000_ICR_DRSTA) 4768 schedule_work(&adapter->reset_task); 4769 4770 if (icr & E1000_ICR_DOUTSYNC) { 4771 /* HW is reporting DMA is out of sync */ 4772 adapter->stats.doosync++; 4773 /* The DMA Out of Sync is also indication of a spoof event 4774 * in IOV mode. Check the Wrong VM Behavior register to 4775 * see if it is really a spoof event. */ 4776 igb_check_wvbr(adapter); 4777 } 4778 4779 /* Check for a mailbox event */ 4780 if (icr & E1000_ICR_VMMB) 4781 igb_msg_task(adapter); 4782 4783 if (icr & E1000_ICR_LSC) { 4784 hw->mac.get_link_status = 1; 4785 /* guard against interrupt when we're going down */ 4786 if (!test_bit(__IGB_DOWN, &adapter->state)) 4787 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4788 } 4789 4790 wr32(E1000_EIMS, adapter->eims_other); 4791 4792 return IRQ_HANDLED; 4793 } 4794 4795 static void igb_write_itr(struct igb_q_vector *q_vector) 4796 { 4797 struct igb_adapter *adapter = q_vector->adapter; 4798 u32 itr_val = q_vector->itr_val & 0x7FFC; 4799 4800 if (!q_vector->set_itr) 4801 return; 4802 4803 if (!itr_val) 4804 itr_val = 0x4; 4805 4806 if (adapter->hw.mac.type == e1000_82575) 4807 itr_val |= itr_val << 16; 4808 else 4809 itr_val |= E1000_EITR_CNT_IGNR; 4810 4811 writel(itr_val, q_vector->itr_register); 4812 q_vector->set_itr = 0; 4813 } 4814 4815 static irqreturn_t igb_msix_ring(int irq, void *data) 4816 { 4817 struct igb_q_vector *q_vector = data; 4818 4819 /* Write the ITR value calculated from the previous interrupt. */ 4820 igb_write_itr(q_vector); 4821 4822 napi_schedule(&q_vector->napi); 4823 4824 return IRQ_HANDLED; 4825 } 4826 4827 #ifdef CONFIG_IGB_DCA 4828 static void igb_update_dca(struct igb_q_vector *q_vector) 4829 { 4830 struct igb_adapter *adapter = q_vector->adapter; 4831 struct e1000_hw *hw = &adapter->hw; 4832 int cpu = get_cpu(); 4833 4834 if (q_vector->cpu == cpu) 4835 goto out_no_update; 4836 4837 if (q_vector->tx.ring) { 4838 int q = q_vector->tx.ring->reg_idx; 4839 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q)); 4840 if (hw->mac.type == e1000_82575) { 4841 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK; 4842 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); 4843 } else { 4844 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576; 4845 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) << 4846 E1000_DCA_TXCTRL_CPUID_SHIFT; 4847 } 4848 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN; 4849 wr32(E1000_DCA_TXCTRL(q), dca_txctrl); 4850 } 4851 if (q_vector->rx.ring) { 4852 int q = q_vector->rx.ring->reg_idx; 4853 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q)); 4854 if (hw->mac.type == e1000_82575) { 4855 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK; 4856 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); 4857 } else { 4858 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576; 4859 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) << 4860 E1000_DCA_RXCTRL_CPUID_SHIFT; 4861 } 4862 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN; 4863 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN; 4864 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN; 4865 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl); 4866 } 4867 q_vector->cpu = cpu; 4868 out_no_update: 4869 put_cpu(); 4870 } 4871 4872 static void igb_setup_dca(struct igb_adapter *adapter) 4873 { 4874 struct e1000_hw *hw = &adapter->hw; 4875 int i; 4876 4877 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED)) 4878 return; 4879 4880 /* Always use CB2 mode, difference is masked in the CB driver. */ 4881 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2); 4882 4883 for (i = 0; i < adapter->num_q_vectors; i++) { 4884 adapter->q_vector[i]->cpu = -1; 4885 igb_update_dca(adapter->q_vector[i]); 4886 } 4887 } 4888 4889 static int __igb_notify_dca(struct device *dev, void *data) 4890 { 4891 struct net_device *netdev = dev_get_drvdata(dev); 4892 struct igb_adapter *adapter = netdev_priv(netdev); 4893 struct pci_dev *pdev = adapter->pdev; 4894 struct e1000_hw *hw = &adapter->hw; 4895 unsigned long event = *(unsigned long *)data; 4896 4897 switch (event) { 4898 case DCA_PROVIDER_ADD: 4899 /* if already enabled, don't do it again */ 4900 if (adapter->flags & IGB_FLAG_DCA_ENABLED) 4901 break; 4902 if (dca_add_requester(dev) == 0) { 4903 adapter->flags |= IGB_FLAG_DCA_ENABLED; 4904 dev_info(&pdev->dev, "DCA enabled\n"); 4905 igb_setup_dca(adapter); 4906 break; 4907 } 4908 /* Fall Through since DCA is disabled. */ 4909 case DCA_PROVIDER_REMOVE: 4910 if (adapter->flags & IGB_FLAG_DCA_ENABLED) { 4911 /* without this a class_device is left 4912 * hanging around in the sysfs model */ 4913 dca_remove_requester(dev); 4914 dev_info(&pdev->dev, "DCA disabled\n"); 4915 adapter->flags &= ~IGB_FLAG_DCA_ENABLED; 4916 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); 4917 } 4918 break; 4919 } 4920 4921 return 0; 4922 } 4923 4924 static int igb_notify_dca(struct notifier_block *nb, unsigned long event, 4925 void *p) 4926 { 4927 int ret_val; 4928 4929 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event, 4930 __igb_notify_dca); 4931 4932 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 4933 } 4934 #endif /* CONFIG_IGB_DCA */ 4935 4936 #ifdef CONFIG_PCI_IOV 4937 static int igb_vf_configure(struct igb_adapter *adapter, int vf) 4938 { 4939 unsigned char mac_addr[ETH_ALEN]; 4940 struct pci_dev *pdev = adapter->pdev; 4941 struct e1000_hw *hw = &adapter->hw; 4942 struct pci_dev *pvfdev; 4943 unsigned int device_id; 4944 u16 thisvf_devfn; 4945 4946 random_ether_addr(mac_addr); 4947 igb_set_vf_mac(adapter, vf, mac_addr); 4948 4949 switch (adapter->hw.mac.type) { 4950 case e1000_82576: 4951 device_id = IGB_82576_VF_DEV_ID; 4952 /* VF Stride for 82576 is 2 */ 4953 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) | 4954 (pdev->devfn & 1); 4955 break; 4956 case e1000_i350: 4957 device_id = IGB_I350_VF_DEV_ID; 4958 /* VF Stride for I350 is 4 */ 4959 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) | 4960 (pdev->devfn & 3); 4961 break; 4962 default: 4963 device_id = 0; 4964 thisvf_devfn = 0; 4965 break; 4966 } 4967 4968 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL); 4969 while (pvfdev) { 4970 if (pvfdev->devfn == thisvf_devfn) 4971 break; 4972 pvfdev = pci_get_device(hw->vendor_id, 4973 device_id, pvfdev); 4974 } 4975 4976 if (pvfdev) 4977 adapter->vf_data[vf].vfdev = pvfdev; 4978 else 4979 dev_err(&pdev->dev, 4980 "Couldn't find pci dev ptr for VF %4.4x\n", 4981 thisvf_devfn); 4982 return pvfdev != NULL; 4983 } 4984 4985 static int igb_find_enabled_vfs(struct igb_adapter *adapter) 4986 { 4987 struct e1000_hw *hw = &adapter->hw; 4988 struct pci_dev *pdev = adapter->pdev; 4989 struct pci_dev *pvfdev; 4990 u16 vf_devfn = 0; 4991 u16 vf_stride; 4992 unsigned int device_id; 4993 int vfs_found = 0; 4994 4995 switch (adapter->hw.mac.type) { 4996 case e1000_82576: 4997 device_id = IGB_82576_VF_DEV_ID; 4998 /* VF Stride for 82576 is 2 */ 4999 vf_stride = 2; 5000 break; 5001 case e1000_i350: 5002 device_id = IGB_I350_VF_DEV_ID; 5003 /* VF Stride for I350 is 4 */ 5004 vf_stride = 4; 5005 break; 5006 default: 5007 device_id = 0; 5008 vf_stride = 0; 5009 break; 5010 } 5011 5012 vf_devfn = pdev->devfn + 0x80; 5013 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL); 5014 while (pvfdev) { 5015 if (pvfdev->devfn == vf_devfn) 5016 vfs_found++; 5017 vf_devfn += vf_stride; 5018 pvfdev = pci_get_device(hw->vendor_id, 5019 device_id, pvfdev); 5020 } 5021 5022 return vfs_found; 5023 } 5024 5025 static int igb_check_vf_assignment(struct igb_adapter *adapter) 5026 { 5027 int i; 5028 for (i = 0; i < adapter->vfs_allocated_count; i++) { 5029 if (adapter->vf_data[i].vfdev) { 5030 if (adapter->vf_data[i].vfdev->dev_flags & 5031 PCI_DEV_FLAGS_ASSIGNED) 5032 return true; 5033 } 5034 } 5035 return false; 5036 } 5037 5038 #endif 5039 static void igb_ping_all_vfs(struct igb_adapter *adapter) 5040 { 5041 struct e1000_hw *hw = &adapter->hw; 5042 u32 ping; 5043 int i; 5044 5045 for (i = 0 ; i < adapter->vfs_allocated_count; i++) { 5046 ping = E1000_PF_CONTROL_MSG; 5047 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS) 5048 ping |= E1000_VT_MSGTYPE_CTS; 5049 igb_write_mbx(hw, &ping, 1, i); 5050 } 5051 } 5052 5053 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) 5054 { 5055 struct e1000_hw *hw = &adapter->hw; 5056 u32 vmolr = rd32(E1000_VMOLR(vf)); 5057 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 5058 5059 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC | 5060 IGB_VF_FLAG_MULTI_PROMISC); 5061 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); 5062 5063 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) { 5064 vmolr |= E1000_VMOLR_MPME; 5065 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC; 5066 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST; 5067 } else { 5068 /* 5069 * if we have hashes and we are clearing a multicast promisc 5070 * flag we need to write the hashes to the MTA as this step 5071 * was previously skipped 5072 */ 5073 if (vf_data->num_vf_mc_hashes > 30) { 5074 vmolr |= E1000_VMOLR_MPME; 5075 } else if (vf_data->num_vf_mc_hashes) { 5076 int j; 5077 vmolr |= E1000_VMOLR_ROMPE; 5078 for (j = 0; j < vf_data->num_vf_mc_hashes; j++) 5079 igb_mta_set(hw, vf_data->vf_mc_hashes[j]); 5080 } 5081 } 5082 5083 wr32(E1000_VMOLR(vf), vmolr); 5084 5085 /* there are flags left unprocessed, likely not supported */ 5086 if (*msgbuf & E1000_VT_MSGINFO_MASK) 5087 return -EINVAL; 5088 5089 return 0; 5090 5091 } 5092 5093 static int igb_set_vf_multicasts(struct igb_adapter *adapter, 5094 u32 *msgbuf, u32 vf) 5095 { 5096 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT; 5097 u16 *hash_list = (u16 *)&msgbuf[1]; 5098 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 5099 int i; 5100 5101 /* salt away the number of multicast addresses assigned 5102 * to this VF for later use to restore when the PF multi cast 5103 * list changes 5104 */ 5105 vf_data->num_vf_mc_hashes = n; 5106 5107 /* only up to 30 hash values supported */ 5108 if (n > 30) 5109 n = 30; 5110 5111 /* store the hashes for later use */ 5112 for (i = 0; i < n; i++) 5113 vf_data->vf_mc_hashes[i] = hash_list[i]; 5114 5115 /* Flush and reset the mta with the new values */ 5116 igb_set_rx_mode(adapter->netdev); 5117 5118 return 0; 5119 } 5120 5121 static void igb_restore_vf_multicasts(struct igb_adapter *adapter) 5122 { 5123 struct e1000_hw *hw = &adapter->hw; 5124 struct vf_data_storage *vf_data; 5125 int i, j; 5126 5127 for (i = 0; i < adapter->vfs_allocated_count; i++) { 5128 u32 vmolr = rd32(E1000_VMOLR(i)); 5129 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); 5130 5131 vf_data = &adapter->vf_data[i]; 5132 5133 if ((vf_data->num_vf_mc_hashes > 30) || 5134 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) { 5135 vmolr |= E1000_VMOLR_MPME; 5136 } else if (vf_data->num_vf_mc_hashes) { 5137 vmolr |= E1000_VMOLR_ROMPE; 5138 for (j = 0; j < vf_data->num_vf_mc_hashes; j++) 5139 igb_mta_set(hw, vf_data->vf_mc_hashes[j]); 5140 } 5141 wr32(E1000_VMOLR(i), vmolr); 5142 } 5143 } 5144 5145 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf) 5146 { 5147 struct e1000_hw *hw = &adapter->hw; 5148 u32 pool_mask, reg, vid; 5149 int i; 5150 5151 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf); 5152 5153 /* Find the vlan filter for this id */ 5154 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) { 5155 reg = rd32(E1000_VLVF(i)); 5156 5157 /* remove the vf from the pool */ 5158 reg &= ~pool_mask; 5159 5160 /* if pool is empty then remove entry from vfta */ 5161 if (!(reg & E1000_VLVF_POOLSEL_MASK) && 5162 (reg & E1000_VLVF_VLANID_ENABLE)) { 5163 reg = 0; 5164 vid = reg & E1000_VLVF_VLANID_MASK; 5165 igb_vfta_set(hw, vid, false); 5166 } 5167 5168 wr32(E1000_VLVF(i), reg); 5169 } 5170 5171 adapter->vf_data[vf].vlans_enabled = 0; 5172 } 5173 5174 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf) 5175 { 5176 struct e1000_hw *hw = &adapter->hw; 5177 u32 reg, i; 5178 5179 /* The vlvf table only exists on 82576 hardware and newer */ 5180 if (hw->mac.type < e1000_82576) 5181 return -1; 5182 5183 /* we only need to do this if VMDq is enabled */ 5184 if (!adapter->vfs_allocated_count) 5185 return -1; 5186 5187 /* Find the vlan filter for this id */ 5188 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) { 5189 reg = rd32(E1000_VLVF(i)); 5190 if ((reg & E1000_VLVF_VLANID_ENABLE) && 5191 vid == (reg & E1000_VLVF_VLANID_MASK)) 5192 break; 5193 } 5194 5195 if (add) { 5196 if (i == E1000_VLVF_ARRAY_SIZE) { 5197 /* Did not find a matching VLAN ID entry that was 5198 * enabled. Search for a free filter entry, i.e. 5199 * one without the enable bit set 5200 */ 5201 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) { 5202 reg = rd32(E1000_VLVF(i)); 5203 if (!(reg & E1000_VLVF_VLANID_ENABLE)) 5204 break; 5205 } 5206 } 5207 if (i < E1000_VLVF_ARRAY_SIZE) { 5208 /* Found an enabled/available entry */ 5209 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf); 5210 5211 /* if !enabled we need to set this up in vfta */ 5212 if (!(reg & E1000_VLVF_VLANID_ENABLE)) { 5213 /* add VID to filter table */ 5214 igb_vfta_set(hw, vid, true); 5215 reg |= E1000_VLVF_VLANID_ENABLE; 5216 } 5217 reg &= ~E1000_VLVF_VLANID_MASK; 5218 reg |= vid; 5219 wr32(E1000_VLVF(i), reg); 5220 5221 /* do not modify RLPML for PF devices */ 5222 if (vf >= adapter->vfs_allocated_count) 5223 return 0; 5224 5225 if (!adapter->vf_data[vf].vlans_enabled) { 5226 u32 size; 5227 reg = rd32(E1000_VMOLR(vf)); 5228 size = reg & E1000_VMOLR_RLPML_MASK; 5229 size += 4; 5230 reg &= ~E1000_VMOLR_RLPML_MASK; 5231 reg |= size; 5232 wr32(E1000_VMOLR(vf), reg); 5233 } 5234 5235 adapter->vf_data[vf].vlans_enabled++; 5236 } 5237 } else { 5238 if (i < E1000_VLVF_ARRAY_SIZE) { 5239 /* remove vf from the pool */ 5240 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf)); 5241 /* if pool is empty then remove entry from vfta */ 5242 if (!(reg & E1000_VLVF_POOLSEL_MASK)) { 5243 reg = 0; 5244 igb_vfta_set(hw, vid, false); 5245 } 5246 wr32(E1000_VLVF(i), reg); 5247 5248 /* do not modify RLPML for PF devices */ 5249 if (vf >= adapter->vfs_allocated_count) 5250 return 0; 5251 5252 adapter->vf_data[vf].vlans_enabled--; 5253 if (!adapter->vf_data[vf].vlans_enabled) { 5254 u32 size; 5255 reg = rd32(E1000_VMOLR(vf)); 5256 size = reg & E1000_VMOLR_RLPML_MASK; 5257 size -= 4; 5258 reg &= ~E1000_VMOLR_RLPML_MASK; 5259 reg |= size; 5260 wr32(E1000_VMOLR(vf), reg); 5261 } 5262 } 5263 } 5264 return 0; 5265 } 5266 5267 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf) 5268 { 5269 struct e1000_hw *hw = &adapter->hw; 5270 5271 if (vid) 5272 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT)); 5273 else 5274 wr32(E1000_VMVIR(vf), 0); 5275 } 5276 5277 static int igb_ndo_set_vf_vlan(struct net_device *netdev, 5278 int vf, u16 vlan, u8 qos) 5279 { 5280 int err = 0; 5281 struct igb_adapter *adapter = netdev_priv(netdev); 5282 5283 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7)) 5284 return -EINVAL; 5285 if (vlan || qos) { 5286 err = igb_vlvf_set(adapter, vlan, !!vlan, vf); 5287 if (err) 5288 goto out; 5289 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf); 5290 igb_set_vmolr(adapter, vf, !vlan); 5291 adapter->vf_data[vf].pf_vlan = vlan; 5292 adapter->vf_data[vf].pf_qos = qos; 5293 dev_info(&adapter->pdev->dev, 5294 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf); 5295 if (test_bit(__IGB_DOWN, &adapter->state)) { 5296 dev_warn(&adapter->pdev->dev, 5297 "The VF VLAN has been set," 5298 " but the PF device is not up.\n"); 5299 dev_warn(&adapter->pdev->dev, 5300 "Bring the PF device up before" 5301 " attempting to use the VF device.\n"); 5302 } 5303 } else { 5304 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan, 5305 false, vf); 5306 igb_set_vmvir(adapter, vlan, vf); 5307 igb_set_vmolr(adapter, vf, true); 5308 adapter->vf_data[vf].pf_vlan = 0; 5309 adapter->vf_data[vf].pf_qos = 0; 5310 } 5311 out: 5312 return err; 5313 } 5314 5315 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) 5316 { 5317 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT; 5318 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK); 5319 5320 return igb_vlvf_set(adapter, vid, add, vf); 5321 } 5322 5323 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf) 5324 { 5325 /* clear flags - except flag that indicates PF has set the MAC */ 5326 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC; 5327 adapter->vf_data[vf].last_nack = jiffies; 5328 5329 /* reset offloads to defaults */ 5330 igb_set_vmolr(adapter, vf, true); 5331 5332 /* reset vlans for device */ 5333 igb_clear_vf_vfta(adapter, vf); 5334 if (adapter->vf_data[vf].pf_vlan) 5335 igb_ndo_set_vf_vlan(adapter->netdev, vf, 5336 adapter->vf_data[vf].pf_vlan, 5337 adapter->vf_data[vf].pf_qos); 5338 else 5339 igb_clear_vf_vfta(adapter, vf); 5340 5341 /* reset multicast table array for vf */ 5342 adapter->vf_data[vf].num_vf_mc_hashes = 0; 5343 5344 /* Flush and reset the mta with the new values */ 5345 igb_set_rx_mode(adapter->netdev); 5346 } 5347 5348 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf) 5349 { 5350 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; 5351 5352 /* generate a new mac address as we were hotplug removed/added */ 5353 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC)) 5354 random_ether_addr(vf_mac); 5355 5356 /* process remaining reset events */ 5357 igb_vf_reset(adapter, vf); 5358 } 5359 5360 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf) 5361 { 5362 struct e1000_hw *hw = &adapter->hw; 5363 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; 5364 int rar_entry = hw->mac.rar_entry_count - (vf + 1); 5365 u32 reg, msgbuf[3]; 5366 u8 *addr = (u8 *)(&msgbuf[1]); 5367 5368 /* process all the same items cleared in a function level reset */ 5369 igb_vf_reset(adapter, vf); 5370 5371 /* set vf mac address */ 5372 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf); 5373 5374 /* enable transmit and receive for vf */ 5375 reg = rd32(E1000_VFTE); 5376 wr32(E1000_VFTE, reg | (1 << vf)); 5377 reg = rd32(E1000_VFRE); 5378 wr32(E1000_VFRE, reg | (1 << vf)); 5379 5380 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS; 5381 5382 /* reply to reset with ack and vf mac address */ 5383 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK; 5384 memcpy(addr, vf_mac, 6); 5385 igb_write_mbx(hw, msgbuf, 3, vf); 5386 } 5387 5388 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf) 5389 { 5390 /* 5391 * The VF MAC Address is stored in a packed array of bytes 5392 * starting at the second 32 bit word of the msg array 5393 */ 5394 unsigned char *addr = (char *)&msg[1]; 5395 int err = -1; 5396 5397 if (is_valid_ether_addr(addr)) 5398 err = igb_set_vf_mac(adapter, vf, addr); 5399 5400 return err; 5401 } 5402 5403 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf) 5404 { 5405 struct e1000_hw *hw = &adapter->hw; 5406 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 5407 u32 msg = E1000_VT_MSGTYPE_NACK; 5408 5409 /* if device isn't clear to send it shouldn't be reading either */ 5410 if (!(vf_data->flags & IGB_VF_FLAG_CTS) && 5411 time_after(jiffies, vf_data->last_nack + (2 * HZ))) { 5412 igb_write_mbx(hw, &msg, 1, vf); 5413 vf_data->last_nack = jiffies; 5414 } 5415 } 5416 5417 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf) 5418 { 5419 struct pci_dev *pdev = adapter->pdev; 5420 u32 msgbuf[E1000_VFMAILBOX_SIZE]; 5421 struct e1000_hw *hw = &adapter->hw; 5422 struct vf_data_storage *vf_data = &adapter->vf_data[vf]; 5423 s32 retval; 5424 5425 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf); 5426 5427 if (retval) { 5428 /* if receive failed revoke VF CTS stats and restart init */ 5429 dev_err(&pdev->dev, "Error receiving message from VF\n"); 5430 vf_data->flags &= ~IGB_VF_FLAG_CTS; 5431 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) 5432 return; 5433 goto out; 5434 } 5435 5436 /* this is a message we already processed, do nothing */ 5437 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) 5438 return; 5439 5440 /* 5441 * until the vf completes a reset it should not be 5442 * allowed to start any configuration. 5443 */ 5444 5445 if (msgbuf[0] == E1000_VF_RESET) { 5446 igb_vf_reset_msg(adapter, vf); 5447 return; 5448 } 5449 5450 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) { 5451 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) 5452 return; 5453 retval = -1; 5454 goto out; 5455 } 5456 5457 switch ((msgbuf[0] & 0xFFFF)) { 5458 case E1000_VF_SET_MAC_ADDR: 5459 retval = -EINVAL; 5460 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC)) 5461 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf); 5462 else 5463 dev_warn(&pdev->dev, 5464 "VF %d attempted to override administratively " 5465 "set MAC address\nReload the VF driver to " 5466 "resume operations\n", vf); 5467 break; 5468 case E1000_VF_SET_PROMISC: 5469 retval = igb_set_vf_promisc(adapter, msgbuf, vf); 5470 break; 5471 case E1000_VF_SET_MULTICAST: 5472 retval = igb_set_vf_multicasts(adapter, msgbuf, vf); 5473 break; 5474 case E1000_VF_SET_LPE: 5475 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf); 5476 break; 5477 case E1000_VF_SET_VLAN: 5478 retval = -1; 5479 if (vf_data->pf_vlan) 5480 dev_warn(&pdev->dev, 5481 "VF %d attempted to override administratively " 5482 "set VLAN tag\nReload the VF driver to " 5483 "resume operations\n", vf); 5484 else 5485 retval = igb_set_vf_vlan(adapter, msgbuf, vf); 5486 break; 5487 default: 5488 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]); 5489 retval = -1; 5490 break; 5491 } 5492 5493 msgbuf[0] |= E1000_VT_MSGTYPE_CTS; 5494 out: 5495 /* notify the VF of the results of what it sent us */ 5496 if (retval) 5497 msgbuf[0] |= E1000_VT_MSGTYPE_NACK; 5498 else 5499 msgbuf[0] |= E1000_VT_MSGTYPE_ACK; 5500 5501 igb_write_mbx(hw, msgbuf, 1, vf); 5502 } 5503 5504 static void igb_msg_task(struct igb_adapter *adapter) 5505 { 5506 struct e1000_hw *hw = &adapter->hw; 5507 u32 vf; 5508 5509 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) { 5510 /* process any reset requests */ 5511 if (!igb_check_for_rst(hw, vf)) 5512 igb_vf_reset_event(adapter, vf); 5513 5514 /* process any messages pending */ 5515 if (!igb_check_for_msg(hw, vf)) 5516 igb_rcv_msg_from_vf(adapter, vf); 5517 5518 /* process any acks */ 5519 if (!igb_check_for_ack(hw, vf)) 5520 igb_rcv_ack_from_vf(adapter, vf); 5521 } 5522 } 5523 5524 /** 5525 * igb_set_uta - Set unicast filter table address 5526 * @adapter: board private structure 5527 * 5528 * The unicast table address is a register array of 32-bit registers. 5529 * The table is meant to be used in a way similar to how the MTA is used 5530 * however due to certain limitations in the hardware it is necessary to 5531 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous 5532 * enable bit to allow vlan tag stripping when promiscuous mode is enabled 5533 **/ 5534 static void igb_set_uta(struct igb_adapter *adapter) 5535 { 5536 struct e1000_hw *hw = &adapter->hw; 5537 int i; 5538 5539 /* The UTA table only exists on 82576 hardware and newer */ 5540 if (hw->mac.type < e1000_82576) 5541 return; 5542 5543 /* we only need to do this if VMDq is enabled */ 5544 if (!adapter->vfs_allocated_count) 5545 return; 5546 5547 for (i = 0; i < hw->mac.uta_reg_count; i++) 5548 array_wr32(E1000_UTA, i, ~0); 5549 } 5550 5551 /** 5552 * igb_intr_msi - Interrupt Handler 5553 * @irq: interrupt number 5554 * @data: pointer to a network interface device structure 5555 **/ 5556 static irqreturn_t igb_intr_msi(int irq, void *data) 5557 { 5558 struct igb_adapter *adapter = data; 5559 struct igb_q_vector *q_vector = adapter->q_vector[0]; 5560 struct e1000_hw *hw = &adapter->hw; 5561 /* read ICR disables interrupts using IAM */ 5562 u32 icr = rd32(E1000_ICR); 5563 5564 igb_write_itr(q_vector); 5565 5566 if (icr & E1000_ICR_DRSTA) 5567 schedule_work(&adapter->reset_task); 5568 5569 if (icr & E1000_ICR_DOUTSYNC) { 5570 /* HW is reporting DMA is out of sync */ 5571 adapter->stats.doosync++; 5572 } 5573 5574 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 5575 hw->mac.get_link_status = 1; 5576 if (!test_bit(__IGB_DOWN, &adapter->state)) 5577 mod_timer(&adapter->watchdog_timer, jiffies + 1); 5578 } 5579 5580 napi_schedule(&q_vector->napi); 5581 5582 return IRQ_HANDLED; 5583 } 5584 5585 /** 5586 * igb_intr - Legacy Interrupt Handler 5587 * @irq: interrupt number 5588 * @data: pointer to a network interface device structure 5589 **/ 5590 static irqreturn_t igb_intr(int irq, void *data) 5591 { 5592 struct igb_adapter *adapter = data; 5593 struct igb_q_vector *q_vector = adapter->q_vector[0]; 5594 struct e1000_hw *hw = &adapter->hw; 5595 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 5596 * need for the IMC write */ 5597 u32 icr = rd32(E1000_ICR); 5598 5599 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 5600 * not set, then the adapter didn't send an interrupt */ 5601 if (!(icr & E1000_ICR_INT_ASSERTED)) 5602 return IRQ_NONE; 5603 5604 igb_write_itr(q_vector); 5605 5606 if (icr & E1000_ICR_DRSTA) 5607 schedule_work(&adapter->reset_task); 5608 5609 if (icr & E1000_ICR_DOUTSYNC) { 5610 /* HW is reporting DMA is out of sync */ 5611 adapter->stats.doosync++; 5612 } 5613 5614 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 5615 hw->mac.get_link_status = 1; 5616 /* guard against interrupt when we're going down */ 5617 if (!test_bit(__IGB_DOWN, &adapter->state)) 5618 mod_timer(&adapter->watchdog_timer, jiffies + 1); 5619 } 5620 5621 napi_schedule(&q_vector->napi); 5622 5623 return IRQ_HANDLED; 5624 } 5625 5626 void igb_ring_irq_enable(struct igb_q_vector *q_vector) 5627 { 5628 struct igb_adapter *adapter = q_vector->adapter; 5629 struct e1000_hw *hw = &adapter->hw; 5630 5631 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 5632 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 5633 if ((adapter->num_q_vectors == 1) && !adapter->vf_data) 5634 igb_set_itr(q_vector); 5635 else 5636 igb_update_ring_itr(q_vector); 5637 } 5638 5639 if (!test_bit(__IGB_DOWN, &adapter->state)) { 5640 if (adapter->msix_entries) 5641 wr32(E1000_EIMS, q_vector->eims_value); 5642 else 5643 igb_irq_enable(adapter); 5644 } 5645 } 5646 5647 /** 5648 * igb_poll - NAPI Rx polling callback 5649 * @napi: napi polling structure 5650 * @budget: count of how many packets we should handle 5651 **/ 5652 static int igb_poll(struct napi_struct *napi, int budget) 5653 { 5654 struct igb_q_vector *q_vector = container_of(napi, 5655 struct igb_q_vector, 5656 napi); 5657 bool clean_complete = true; 5658 5659 #ifdef CONFIG_IGB_DCA 5660 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED) 5661 igb_update_dca(q_vector); 5662 #endif 5663 if (q_vector->tx.ring) 5664 clean_complete = igb_clean_tx_irq(q_vector); 5665 5666 if (q_vector->rx.ring) 5667 clean_complete &= igb_clean_rx_irq(q_vector, budget); 5668 5669 /* If all work not completed, return budget and keep polling */ 5670 if (!clean_complete) 5671 return budget; 5672 5673 /* If not enough Rx work done, exit the polling mode */ 5674 napi_complete(napi); 5675 igb_ring_irq_enable(q_vector); 5676 5677 return 0; 5678 } 5679 5680 /** 5681 * igb_systim_to_hwtstamp - convert system time value to hw timestamp 5682 * @adapter: board private structure 5683 * @shhwtstamps: timestamp structure to update 5684 * @regval: unsigned 64bit system time value. 5685 * 5686 * We need to convert the system time value stored in the RX/TXSTMP registers 5687 * into a hwtstamp which can be used by the upper level timestamping functions 5688 */ 5689 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter, 5690 struct skb_shared_hwtstamps *shhwtstamps, 5691 u64 regval) 5692 { 5693 u64 ns; 5694 5695 /* 5696 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to 5697 * 24 to match clock shift we setup earlier. 5698 */ 5699 if (adapter->hw.mac.type >= e1000_82580) 5700 regval <<= IGB_82580_TSYNC_SHIFT; 5701 5702 ns = timecounter_cyc2time(&adapter->clock, regval); 5703 timecompare_update(&adapter->compare, ns); 5704 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps)); 5705 shhwtstamps->hwtstamp = ns_to_ktime(ns); 5706 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns); 5707 } 5708 5709 /** 5710 * igb_tx_hwtstamp - utility function which checks for TX time stamp 5711 * @q_vector: pointer to q_vector containing needed info 5712 * @buffer: pointer to igb_tx_buffer structure 5713 * 5714 * If we were asked to do hardware stamping and such a time stamp is 5715 * available, then it must have been for this skb here because we only 5716 * allow only one such packet into the queue. 5717 */ 5718 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, 5719 struct igb_tx_buffer *buffer_info) 5720 { 5721 struct igb_adapter *adapter = q_vector->adapter; 5722 struct e1000_hw *hw = &adapter->hw; 5723 struct skb_shared_hwtstamps shhwtstamps; 5724 u64 regval; 5725 5726 /* if skb does not support hw timestamp or TX stamp not valid exit */ 5727 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) || 5728 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID)) 5729 return; 5730 5731 regval = rd32(E1000_TXSTMPL); 5732 regval |= (u64)rd32(E1000_TXSTMPH) << 32; 5733 5734 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval); 5735 skb_tstamp_tx(buffer_info->skb, &shhwtstamps); 5736 } 5737 5738 /** 5739 * igb_clean_tx_irq - Reclaim resources after transmit completes 5740 * @q_vector: pointer to q_vector containing needed info 5741 * returns true if ring is completely cleaned 5742 **/ 5743 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector) 5744 { 5745 struct igb_adapter *adapter = q_vector->adapter; 5746 struct igb_ring *tx_ring = q_vector->tx.ring; 5747 struct igb_tx_buffer *tx_buffer; 5748 union e1000_adv_tx_desc *tx_desc, *eop_desc; 5749 unsigned int total_bytes = 0, total_packets = 0; 5750 unsigned int budget = q_vector->tx.work_limit; 5751 unsigned int i = tx_ring->next_to_clean; 5752 5753 if (test_bit(__IGB_DOWN, &adapter->state)) 5754 return true; 5755 5756 tx_buffer = &tx_ring->tx_buffer_info[i]; 5757 tx_desc = IGB_TX_DESC(tx_ring, i); 5758 i -= tx_ring->count; 5759 5760 for (; budget; budget--) { 5761 eop_desc = tx_buffer->next_to_watch; 5762 5763 /* prevent any other reads prior to eop_desc */ 5764 rmb(); 5765 5766 /* if next_to_watch is not set then there is no work pending */ 5767 if (!eop_desc) 5768 break; 5769 5770 /* if DD is not set pending work has not been completed */ 5771 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD))) 5772 break; 5773 5774 /* clear next_to_watch to prevent false hangs */ 5775 tx_buffer->next_to_watch = NULL; 5776 5777 /* update the statistics for this packet */ 5778 total_bytes += tx_buffer->bytecount; 5779 total_packets += tx_buffer->gso_segs; 5780 5781 /* retrieve hardware timestamp */ 5782 igb_tx_hwtstamp(q_vector, tx_buffer); 5783 5784 /* free the skb */ 5785 dev_kfree_skb_any(tx_buffer->skb); 5786 tx_buffer->skb = NULL; 5787 5788 /* unmap skb header data */ 5789 dma_unmap_single(tx_ring->dev, 5790 tx_buffer->dma, 5791 tx_buffer->length, 5792 DMA_TO_DEVICE); 5793 5794 /* clear last DMA location and unmap remaining buffers */ 5795 while (tx_desc != eop_desc) { 5796 tx_buffer->dma = 0; 5797 5798 tx_buffer++; 5799 tx_desc++; 5800 i++; 5801 if (unlikely(!i)) { 5802 i -= tx_ring->count; 5803 tx_buffer = tx_ring->tx_buffer_info; 5804 tx_desc = IGB_TX_DESC(tx_ring, 0); 5805 } 5806 5807 /* unmap any remaining paged data */ 5808 if (tx_buffer->dma) { 5809 dma_unmap_page(tx_ring->dev, 5810 tx_buffer->dma, 5811 tx_buffer->length, 5812 DMA_TO_DEVICE); 5813 } 5814 } 5815 5816 /* clear last DMA location */ 5817 tx_buffer->dma = 0; 5818 5819 /* move us one more past the eop_desc for start of next pkt */ 5820 tx_buffer++; 5821 tx_desc++; 5822 i++; 5823 if (unlikely(!i)) { 5824 i -= tx_ring->count; 5825 tx_buffer = tx_ring->tx_buffer_info; 5826 tx_desc = IGB_TX_DESC(tx_ring, 0); 5827 } 5828 } 5829 5830 netdev_tx_completed_queue(txring_txq(tx_ring), 5831 total_packets, total_bytes); 5832 i += tx_ring->count; 5833 tx_ring->next_to_clean = i; 5834 u64_stats_update_begin(&tx_ring->tx_syncp); 5835 tx_ring->tx_stats.bytes += total_bytes; 5836 tx_ring->tx_stats.packets += total_packets; 5837 u64_stats_update_end(&tx_ring->tx_syncp); 5838 q_vector->tx.total_bytes += total_bytes; 5839 q_vector->tx.total_packets += total_packets; 5840 5841 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 5842 struct e1000_hw *hw = &adapter->hw; 5843 5844 eop_desc = tx_buffer->next_to_watch; 5845 5846 /* Detect a transmit hang in hardware, this serializes the 5847 * check with the clearing of time_stamp and movement of i */ 5848 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 5849 if (eop_desc && 5850 time_after(jiffies, tx_buffer->time_stamp + 5851 (adapter->tx_timeout_factor * HZ)) && 5852 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) { 5853 5854 /* detected Tx unit hang */ 5855 dev_err(tx_ring->dev, 5856 "Detected Tx Unit Hang\n" 5857 " Tx Queue <%d>\n" 5858 " TDH <%x>\n" 5859 " TDT <%x>\n" 5860 " next_to_use <%x>\n" 5861 " next_to_clean <%x>\n" 5862 "buffer_info[next_to_clean]\n" 5863 " time_stamp <%lx>\n" 5864 " next_to_watch <%p>\n" 5865 " jiffies <%lx>\n" 5866 " desc.status <%x>\n", 5867 tx_ring->queue_index, 5868 rd32(E1000_TDH(tx_ring->reg_idx)), 5869 readl(tx_ring->tail), 5870 tx_ring->next_to_use, 5871 tx_ring->next_to_clean, 5872 tx_buffer->time_stamp, 5873 eop_desc, 5874 jiffies, 5875 eop_desc->wb.status); 5876 netif_stop_subqueue(tx_ring->netdev, 5877 tx_ring->queue_index); 5878 5879 /* we are about to reset, no point in enabling stuff */ 5880 return true; 5881 } 5882 } 5883 5884 if (unlikely(total_packets && 5885 netif_carrier_ok(tx_ring->netdev) && 5886 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) { 5887 /* Make sure that anybody stopping the queue after this 5888 * sees the new next_to_clean. 5889 */ 5890 smp_mb(); 5891 if (__netif_subqueue_stopped(tx_ring->netdev, 5892 tx_ring->queue_index) && 5893 !(test_bit(__IGB_DOWN, &adapter->state))) { 5894 netif_wake_subqueue(tx_ring->netdev, 5895 tx_ring->queue_index); 5896 5897 u64_stats_update_begin(&tx_ring->tx_syncp); 5898 tx_ring->tx_stats.restart_queue++; 5899 u64_stats_update_end(&tx_ring->tx_syncp); 5900 } 5901 } 5902 5903 return !!budget; 5904 } 5905 5906 static inline void igb_rx_checksum(struct igb_ring *ring, 5907 union e1000_adv_rx_desc *rx_desc, 5908 struct sk_buff *skb) 5909 { 5910 skb_checksum_none_assert(skb); 5911 5912 /* Ignore Checksum bit is set */ 5913 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM)) 5914 return; 5915 5916 /* Rx checksum disabled via ethtool */ 5917 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 5918 return; 5919 5920 /* TCP/UDP checksum error bit is set */ 5921 if (igb_test_staterr(rx_desc, 5922 E1000_RXDEXT_STATERR_TCPE | 5923 E1000_RXDEXT_STATERR_IPE)) { 5924 /* 5925 * work around errata with sctp packets where the TCPE aka 5926 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 5927 * packets, (aka let the stack check the crc32c) 5928 */ 5929 if (!((skb->len == 60) && 5930 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 5931 u64_stats_update_begin(&ring->rx_syncp); 5932 ring->rx_stats.csum_err++; 5933 u64_stats_update_end(&ring->rx_syncp); 5934 } 5935 /* let the stack verify checksum errors */ 5936 return; 5937 } 5938 /* It must be a TCP or UDP packet with a valid checksum */ 5939 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS | 5940 E1000_RXD_STAT_UDPCS)) 5941 skb->ip_summed = CHECKSUM_UNNECESSARY; 5942 5943 dev_dbg(ring->dev, "cksum success: bits %08X\n", 5944 le32_to_cpu(rx_desc->wb.upper.status_error)); 5945 } 5946 5947 static inline void igb_rx_hash(struct igb_ring *ring, 5948 union e1000_adv_rx_desc *rx_desc, 5949 struct sk_buff *skb) 5950 { 5951 if (ring->netdev->features & NETIF_F_RXHASH) 5952 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); 5953 } 5954 5955 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, 5956 union e1000_adv_rx_desc *rx_desc, 5957 struct sk_buff *skb) 5958 { 5959 struct igb_adapter *adapter = q_vector->adapter; 5960 struct e1000_hw *hw = &adapter->hw; 5961 u64 regval; 5962 5963 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP | 5964 E1000_RXDADV_STAT_TS)) 5965 return; 5966 5967 /* 5968 * If this bit is set, then the RX registers contain the time stamp. No 5969 * other packet will be time stamped until we read these registers, so 5970 * read the registers to make them available again. Because only one 5971 * packet can be time stamped at a time, we know that the register 5972 * values must belong to this one here and therefore we don't need to 5973 * compare any of the additional attributes stored for it. 5974 * 5975 * If nothing went wrong, then it should have a shared tx_flags that we 5976 * can turn into a skb_shared_hwtstamps. 5977 */ 5978 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { 5979 u32 *stamp = (u32 *)skb->data; 5980 regval = le32_to_cpu(*(stamp + 2)); 5981 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32; 5982 skb_pull(skb, IGB_TS_HDR_LEN); 5983 } else { 5984 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) 5985 return; 5986 5987 regval = rd32(E1000_RXSTMPL); 5988 regval |= (u64)rd32(E1000_RXSTMPH) << 32; 5989 } 5990 5991 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval); 5992 } 5993 5994 static void igb_rx_vlan(struct igb_ring *ring, 5995 union e1000_adv_rx_desc *rx_desc, 5996 struct sk_buff *skb) 5997 { 5998 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) { 5999 u16 vid; 6000 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) && 6001 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags)) 6002 vid = be16_to_cpu(rx_desc->wb.upper.vlan); 6003 else 6004 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 6005 6006 __vlan_hwaccel_put_tag(skb, vid); 6007 } 6008 } 6009 6010 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc) 6011 { 6012 /* HW will not DMA in data larger than the given buffer, even if it 6013 * parses the (NFS, of course) header to be larger. In that case, it 6014 * fills the header buffer and spills the rest into the page. 6015 */ 6016 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) & 6017 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT; 6018 if (hlen > IGB_RX_HDR_LEN) 6019 hlen = IGB_RX_HDR_LEN; 6020 return hlen; 6021 } 6022 6023 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget) 6024 { 6025 struct igb_ring *rx_ring = q_vector->rx.ring; 6026 union e1000_adv_rx_desc *rx_desc; 6027 const int current_node = numa_node_id(); 6028 unsigned int total_bytes = 0, total_packets = 0; 6029 u16 cleaned_count = igb_desc_unused(rx_ring); 6030 u16 i = rx_ring->next_to_clean; 6031 6032 rx_desc = IGB_RX_DESC(rx_ring, i); 6033 6034 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) { 6035 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 6036 struct sk_buff *skb = buffer_info->skb; 6037 union e1000_adv_rx_desc *next_rxd; 6038 6039 buffer_info->skb = NULL; 6040 prefetch(skb->data); 6041 6042 i++; 6043 if (i == rx_ring->count) 6044 i = 0; 6045 6046 next_rxd = IGB_RX_DESC(rx_ring, i); 6047 prefetch(next_rxd); 6048 6049 /* 6050 * This memory barrier is needed to keep us from reading 6051 * any other fields out of the rx_desc until we know the 6052 * RXD_STAT_DD bit is set 6053 */ 6054 rmb(); 6055 6056 if (!skb_is_nonlinear(skb)) { 6057 __skb_put(skb, igb_get_hlen(rx_desc)); 6058 dma_unmap_single(rx_ring->dev, buffer_info->dma, 6059 IGB_RX_HDR_LEN, 6060 DMA_FROM_DEVICE); 6061 buffer_info->dma = 0; 6062 } 6063 6064 if (rx_desc->wb.upper.length) { 6065 u16 length = le16_to_cpu(rx_desc->wb.upper.length); 6066 6067 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 6068 buffer_info->page, 6069 buffer_info->page_offset, 6070 length); 6071 6072 skb->len += length; 6073 skb->data_len += length; 6074 skb->truesize += PAGE_SIZE / 2; 6075 6076 if ((page_count(buffer_info->page) != 1) || 6077 (page_to_nid(buffer_info->page) != current_node)) 6078 buffer_info->page = NULL; 6079 else 6080 get_page(buffer_info->page); 6081 6082 dma_unmap_page(rx_ring->dev, buffer_info->page_dma, 6083 PAGE_SIZE / 2, DMA_FROM_DEVICE); 6084 buffer_info->page_dma = 0; 6085 } 6086 6087 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) { 6088 struct igb_rx_buffer *next_buffer; 6089 next_buffer = &rx_ring->rx_buffer_info[i]; 6090 buffer_info->skb = next_buffer->skb; 6091 buffer_info->dma = next_buffer->dma; 6092 next_buffer->skb = skb; 6093 next_buffer->dma = 0; 6094 goto next_desc; 6095 } 6096 6097 if (igb_test_staterr(rx_desc, 6098 E1000_RXDEXT_ERR_FRAME_ERR_MASK)) { 6099 dev_kfree_skb_any(skb); 6100 goto next_desc; 6101 } 6102 6103 igb_rx_hwtstamp(q_vector, rx_desc, skb); 6104 igb_rx_hash(rx_ring, rx_desc, skb); 6105 igb_rx_checksum(rx_ring, rx_desc, skb); 6106 igb_rx_vlan(rx_ring, rx_desc, skb); 6107 6108 total_bytes += skb->len; 6109 total_packets++; 6110 6111 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 6112 6113 napi_gro_receive(&q_vector->napi, skb); 6114 6115 budget--; 6116 next_desc: 6117 if (!budget) 6118 break; 6119 6120 cleaned_count++; 6121 /* return some buffers to hardware, one at a time is too slow */ 6122 if (cleaned_count >= IGB_RX_BUFFER_WRITE) { 6123 igb_alloc_rx_buffers(rx_ring, cleaned_count); 6124 cleaned_count = 0; 6125 } 6126 6127 /* use prefetched values */ 6128 rx_desc = next_rxd; 6129 } 6130 6131 rx_ring->next_to_clean = i; 6132 u64_stats_update_begin(&rx_ring->rx_syncp); 6133 rx_ring->rx_stats.packets += total_packets; 6134 rx_ring->rx_stats.bytes += total_bytes; 6135 u64_stats_update_end(&rx_ring->rx_syncp); 6136 q_vector->rx.total_packets += total_packets; 6137 q_vector->rx.total_bytes += total_bytes; 6138 6139 if (cleaned_count) 6140 igb_alloc_rx_buffers(rx_ring, cleaned_count); 6141 6142 return !!budget; 6143 } 6144 6145 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring, 6146 struct igb_rx_buffer *bi) 6147 { 6148 struct sk_buff *skb = bi->skb; 6149 dma_addr_t dma = bi->dma; 6150 6151 if (dma) 6152 return true; 6153 6154 if (likely(!skb)) { 6155 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 6156 IGB_RX_HDR_LEN); 6157 bi->skb = skb; 6158 if (!skb) { 6159 rx_ring->rx_stats.alloc_failed++; 6160 return false; 6161 } 6162 6163 /* initialize skb for ring */ 6164 skb_record_rx_queue(skb, rx_ring->queue_index); 6165 } 6166 6167 dma = dma_map_single(rx_ring->dev, skb->data, 6168 IGB_RX_HDR_LEN, DMA_FROM_DEVICE); 6169 6170 if (dma_mapping_error(rx_ring->dev, dma)) { 6171 rx_ring->rx_stats.alloc_failed++; 6172 return false; 6173 } 6174 6175 bi->dma = dma; 6176 return true; 6177 } 6178 6179 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, 6180 struct igb_rx_buffer *bi) 6181 { 6182 struct page *page = bi->page; 6183 dma_addr_t page_dma = bi->page_dma; 6184 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2); 6185 6186 if (page_dma) 6187 return true; 6188 6189 if (!page) { 6190 page = alloc_page(GFP_ATOMIC | __GFP_COLD); 6191 bi->page = page; 6192 if (unlikely(!page)) { 6193 rx_ring->rx_stats.alloc_failed++; 6194 return false; 6195 } 6196 } 6197 6198 page_dma = dma_map_page(rx_ring->dev, page, 6199 page_offset, PAGE_SIZE / 2, 6200 DMA_FROM_DEVICE); 6201 6202 if (dma_mapping_error(rx_ring->dev, page_dma)) { 6203 rx_ring->rx_stats.alloc_failed++; 6204 return false; 6205 } 6206 6207 bi->page_dma = page_dma; 6208 bi->page_offset = page_offset; 6209 return true; 6210 } 6211 6212 /** 6213 * igb_alloc_rx_buffers - Replace used receive buffers; packet split 6214 * @adapter: address of board private structure 6215 **/ 6216 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count) 6217 { 6218 union e1000_adv_rx_desc *rx_desc; 6219 struct igb_rx_buffer *bi; 6220 u16 i = rx_ring->next_to_use; 6221 6222 rx_desc = IGB_RX_DESC(rx_ring, i); 6223 bi = &rx_ring->rx_buffer_info[i]; 6224 i -= rx_ring->count; 6225 6226 while (cleaned_count--) { 6227 if (!igb_alloc_mapped_skb(rx_ring, bi)) 6228 break; 6229 6230 /* Refresh the desc even if buffer_addrs didn't change 6231 * because each write-back erases this info. */ 6232 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); 6233 6234 if (!igb_alloc_mapped_page(rx_ring, bi)) 6235 break; 6236 6237 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); 6238 6239 rx_desc++; 6240 bi++; 6241 i++; 6242 if (unlikely(!i)) { 6243 rx_desc = IGB_RX_DESC(rx_ring, 0); 6244 bi = rx_ring->rx_buffer_info; 6245 i -= rx_ring->count; 6246 } 6247 6248 /* clear the hdr_addr for the next_to_use descriptor */ 6249 rx_desc->read.hdr_addr = 0; 6250 } 6251 6252 i += rx_ring->count; 6253 6254 if (rx_ring->next_to_use != i) { 6255 rx_ring->next_to_use = i; 6256 6257 /* Force memory writes to complete before letting h/w 6258 * know there are new descriptors to fetch. (Only 6259 * applicable for weak-ordered memory model archs, 6260 * such as IA-64). */ 6261 wmb(); 6262 writel(i, rx_ring->tail); 6263 } 6264 } 6265 6266 /** 6267 * igb_mii_ioctl - 6268 * @netdev: 6269 * @ifreq: 6270 * @cmd: 6271 **/ 6272 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 6273 { 6274 struct igb_adapter *adapter = netdev_priv(netdev); 6275 struct mii_ioctl_data *data = if_mii(ifr); 6276 6277 if (adapter->hw.phy.media_type != e1000_media_type_copper) 6278 return -EOPNOTSUPP; 6279 6280 switch (cmd) { 6281 case SIOCGMIIPHY: 6282 data->phy_id = adapter->hw.phy.addr; 6283 break; 6284 case SIOCGMIIREG: 6285 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F, 6286 &data->val_out)) 6287 return -EIO; 6288 break; 6289 case SIOCSMIIREG: 6290 default: 6291 return -EOPNOTSUPP; 6292 } 6293 return 0; 6294 } 6295 6296 /** 6297 * igb_hwtstamp_ioctl - control hardware time stamping 6298 * @netdev: 6299 * @ifreq: 6300 * @cmd: 6301 * 6302 * Outgoing time stamping can be enabled and disabled. Play nice and 6303 * disable it when requested, although it shouldn't case any overhead 6304 * when no packet needs it. At most one packet in the queue may be 6305 * marked for time stamping, otherwise it would be impossible to tell 6306 * for sure to which packet the hardware time stamp belongs. 6307 * 6308 * Incoming time stamping has to be configured via the hardware 6309 * filters. Not all combinations are supported, in particular event 6310 * type has to be specified. Matching the kind of event packet is 6311 * not supported, with the exception of "all V2 events regardless of 6312 * level 2 or 4". 6313 * 6314 **/ 6315 static int igb_hwtstamp_ioctl(struct net_device *netdev, 6316 struct ifreq *ifr, int cmd) 6317 { 6318 struct igb_adapter *adapter = netdev_priv(netdev); 6319 struct e1000_hw *hw = &adapter->hw; 6320 struct hwtstamp_config config; 6321 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED; 6322 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; 6323 u32 tsync_rx_cfg = 0; 6324 bool is_l4 = false; 6325 bool is_l2 = false; 6326 u32 regval; 6327 6328 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 6329 return -EFAULT; 6330 6331 /* reserved for future extensions */ 6332 if (config.flags) 6333 return -EINVAL; 6334 6335 switch (config.tx_type) { 6336 case HWTSTAMP_TX_OFF: 6337 tsync_tx_ctl = 0; 6338 case HWTSTAMP_TX_ON: 6339 break; 6340 default: 6341 return -ERANGE; 6342 } 6343 6344 switch (config.rx_filter) { 6345 case HWTSTAMP_FILTER_NONE: 6346 tsync_rx_ctl = 0; 6347 break; 6348 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 6349 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 6350 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 6351 case HWTSTAMP_FILTER_ALL: 6352 /* 6353 * register TSYNCRXCFG must be set, therefore it is not 6354 * possible to time stamp both Sync and Delay_Req messages 6355 * => fall back to time stamping all packets 6356 */ 6357 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; 6358 config.rx_filter = HWTSTAMP_FILTER_ALL; 6359 break; 6360 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 6361 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 6362 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE; 6363 is_l4 = true; 6364 break; 6365 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 6366 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 6367 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE; 6368 is_l4 = true; 6369 break; 6370 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 6371 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 6372 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 6373 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE; 6374 is_l2 = true; 6375 is_l4 = true; 6376 config.rx_filter = HWTSTAMP_FILTER_SOME; 6377 break; 6378 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 6379 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 6380 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 6381 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE; 6382 is_l2 = true; 6383 is_l4 = true; 6384 config.rx_filter = HWTSTAMP_FILTER_SOME; 6385 break; 6386 case HWTSTAMP_FILTER_PTP_V2_EVENT: 6387 case HWTSTAMP_FILTER_PTP_V2_SYNC: 6388 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 6389 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2; 6390 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 6391 is_l2 = true; 6392 is_l4 = true; 6393 break; 6394 default: 6395 return -ERANGE; 6396 } 6397 6398 if (hw->mac.type == e1000_82575) { 6399 if (tsync_rx_ctl | tsync_tx_ctl) 6400 return -EINVAL; 6401 return 0; 6402 } 6403 6404 /* 6405 * Per-packet timestamping only works if all packets are 6406 * timestamped, so enable timestamping in all packets as 6407 * long as one rx filter was configured. 6408 */ 6409 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) { 6410 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; 6411 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; 6412 } 6413 6414 /* enable/disable TX */ 6415 regval = rd32(E1000_TSYNCTXCTL); 6416 regval &= ~E1000_TSYNCTXCTL_ENABLED; 6417 regval |= tsync_tx_ctl; 6418 wr32(E1000_TSYNCTXCTL, regval); 6419 6420 /* enable/disable RX */ 6421 regval = rd32(E1000_TSYNCRXCTL); 6422 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK); 6423 regval |= tsync_rx_ctl; 6424 wr32(E1000_TSYNCRXCTL, regval); 6425 6426 /* define which PTP packets are time stamped */ 6427 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg); 6428 6429 /* define ethertype filter for timestamped packets */ 6430 if (is_l2) 6431 wr32(E1000_ETQF(3), 6432 (E1000_ETQF_FILTER_ENABLE | /* enable filter */ 6433 E1000_ETQF_1588 | /* enable timestamping */ 6434 ETH_P_1588)); /* 1588 eth protocol type */ 6435 else 6436 wr32(E1000_ETQF(3), 0); 6437 6438 #define PTP_PORT 319 6439 /* L4 Queue Filter[3]: filter by destination port and protocol */ 6440 if (is_l4) { 6441 u32 ftqf = (IPPROTO_UDP /* UDP */ 6442 | E1000_FTQF_VF_BP /* VF not compared */ 6443 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */ 6444 | E1000_FTQF_MASK); /* mask all inputs */ 6445 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */ 6446 6447 wr32(E1000_IMIR(3), htons(PTP_PORT)); 6448 wr32(E1000_IMIREXT(3), 6449 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP)); 6450 if (hw->mac.type == e1000_82576) { 6451 /* enable source port check */ 6452 wr32(E1000_SPQF(3), htons(PTP_PORT)); 6453 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; 6454 } 6455 wr32(E1000_FTQF(3), ftqf); 6456 } else { 6457 wr32(E1000_FTQF(3), E1000_FTQF_MASK); 6458 } 6459 wrfl(); 6460 6461 adapter->hwtstamp_config = config; 6462 6463 /* clear TX/RX time stamp registers, just to be sure */ 6464 regval = rd32(E1000_TXSTMPH); 6465 regval = rd32(E1000_RXSTMPH); 6466 6467 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? 6468 -EFAULT : 0; 6469 } 6470 6471 /** 6472 * igb_ioctl - 6473 * @netdev: 6474 * @ifreq: 6475 * @cmd: 6476 **/ 6477 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 6478 { 6479 switch (cmd) { 6480 case SIOCGMIIPHY: 6481 case SIOCGMIIREG: 6482 case SIOCSMIIREG: 6483 return igb_mii_ioctl(netdev, ifr, cmd); 6484 case SIOCSHWTSTAMP: 6485 return igb_hwtstamp_ioctl(netdev, ifr, cmd); 6486 default: 6487 return -EOPNOTSUPP; 6488 } 6489 } 6490 6491 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) 6492 { 6493 struct igb_adapter *adapter = hw->back; 6494 u16 cap_offset; 6495 6496 cap_offset = adapter->pdev->pcie_cap; 6497 if (!cap_offset) 6498 return -E1000_ERR_CONFIG; 6499 6500 pci_read_config_word(adapter->pdev, cap_offset + reg, value); 6501 6502 return 0; 6503 } 6504 6505 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) 6506 { 6507 struct igb_adapter *adapter = hw->back; 6508 u16 cap_offset; 6509 6510 cap_offset = adapter->pdev->pcie_cap; 6511 if (!cap_offset) 6512 return -E1000_ERR_CONFIG; 6513 6514 pci_write_config_word(adapter->pdev, cap_offset + reg, *value); 6515 6516 return 0; 6517 } 6518 6519 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features) 6520 { 6521 struct igb_adapter *adapter = netdev_priv(netdev); 6522 struct e1000_hw *hw = &adapter->hw; 6523 u32 ctrl, rctl; 6524 bool enable = !!(features & NETIF_F_HW_VLAN_RX); 6525 6526 if (enable) { 6527 /* enable VLAN tag insert/strip */ 6528 ctrl = rd32(E1000_CTRL); 6529 ctrl |= E1000_CTRL_VME; 6530 wr32(E1000_CTRL, ctrl); 6531 6532 /* Disable CFI check */ 6533 rctl = rd32(E1000_RCTL); 6534 rctl &= ~E1000_RCTL_CFIEN; 6535 wr32(E1000_RCTL, rctl); 6536 } else { 6537 /* disable VLAN tag insert/strip */ 6538 ctrl = rd32(E1000_CTRL); 6539 ctrl &= ~E1000_CTRL_VME; 6540 wr32(E1000_CTRL, ctrl); 6541 } 6542 6543 igb_rlpml_set(adapter); 6544 } 6545 6546 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 6547 { 6548 struct igb_adapter *adapter = netdev_priv(netdev); 6549 struct e1000_hw *hw = &adapter->hw; 6550 int pf_id = adapter->vfs_allocated_count; 6551 6552 /* attempt to add filter to vlvf array */ 6553 igb_vlvf_set(adapter, vid, true, pf_id); 6554 6555 /* add the filter since PF can receive vlans w/o entry in vlvf */ 6556 igb_vfta_set(hw, vid, true); 6557 6558 set_bit(vid, adapter->active_vlans); 6559 6560 return 0; 6561 } 6562 6563 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 6564 { 6565 struct igb_adapter *adapter = netdev_priv(netdev); 6566 struct e1000_hw *hw = &adapter->hw; 6567 int pf_id = adapter->vfs_allocated_count; 6568 s32 err; 6569 6570 /* remove vlan from VLVF table array */ 6571 err = igb_vlvf_set(adapter, vid, false, pf_id); 6572 6573 /* if vid was not present in VLVF just remove it from table */ 6574 if (err) 6575 igb_vfta_set(hw, vid, false); 6576 6577 clear_bit(vid, adapter->active_vlans); 6578 6579 return 0; 6580 } 6581 6582 static void igb_restore_vlan(struct igb_adapter *adapter) 6583 { 6584 u16 vid; 6585 6586 igb_vlan_mode(adapter->netdev, adapter->netdev->features); 6587 6588 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 6589 igb_vlan_rx_add_vid(adapter->netdev, vid); 6590 } 6591 6592 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx) 6593 { 6594 struct pci_dev *pdev = adapter->pdev; 6595 struct e1000_mac_info *mac = &adapter->hw.mac; 6596 6597 mac->autoneg = 0; 6598 6599 /* Make sure dplx is at most 1 bit and lsb of speed is not set 6600 * for the switch() below to work */ 6601 if ((spd & 1) || (dplx & ~1)) 6602 goto err_inval; 6603 6604 /* Fiber NIC's only allow 1000 Gbps Full duplex */ 6605 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) && 6606 spd != SPEED_1000 && 6607 dplx != DUPLEX_FULL) 6608 goto err_inval; 6609 6610 switch (spd + dplx) { 6611 case SPEED_10 + DUPLEX_HALF: 6612 mac->forced_speed_duplex = ADVERTISE_10_HALF; 6613 break; 6614 case SPEED_10 + DUPLEX_FULL: 6615 mac->forced_speed_duplex = ADVERTISE_10_FULL; 6616 break; 6617 case SPEED_100 + DUPLEX_HALF: 6618 mac->forced_speed_duplex = ADVERTISE_100_HALF; 6619 break; 6620 case SPEED_100 + DUPLEX_FULL: 6621 mac->forced_speed_duplex = ADVERTISE_100_FULL; 6622 break; 6623 case SPEED_1000 + DUPLEX_FULL: 6624 mac->autoneg = 1; 6625 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 6626 break; 6627 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 6628 default: 6629 goto err_inval; 6630 } 6631 return 0; 6632 6633 err_inval: 6634 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 6635 return -EINVAL; 6636 } 6637 6638 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake, 6639 bool runtime) 6640 { 6641 struct net_device *netdev = pci_get_drvdata(pdev); 6642 struct igb_adapter *adapter = netdev_priv(netdev); 6643 struct e1000_hw *hw = &adapter->hw; 6644 u32 ctrl, rctl, status; 6645 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol; 6646 #ifdef CONFIG_PM 6647 int retval = 0; 6648 #endif 6649 6650 netif_device_detach(netdev); 6651 6652 if (netif_running(netdev)) 6653 __igb_close(netdev, true); 6654 6655 igb_clear_interrupt_scheme(adapter); 6656 6657 #ifdef CONFIG_PM 6658 retval = pci_save_state(pdev); 6659 if (retval) 6660 return retval; 6661 #endif 6662 6663 status = rd32(E1000_STATUS); 6664 if (status & E1000_STATUS_LU) 6665 wufc &= ~E1000_WUFC_LNKC; 6666 6667 if (wufc) { 6668 igb_setup_rctl(adapter); 6669 igb_set_rx_mode(netdev); 6670 6671 /* turn on all-multi mode if wake on multicast is enabled */ 6672 if (wufc & E1000_WUFC_MC) { 6673 rctl = rd32(E1000_RCTL); 6674 rctl |= E1000_RCTL_MPE; 6675 wr32(E1000_RCTL, rctl); 6676 } 6677 6678 ctrl = rd32(E1000_CTRL); 6679 /* advertise wake from D3Cold */ 6680 #define E1000_CTRL_ADVD3WUC 0x00100000 6681 /* phy power management enable */ 6682 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000 6683 ctrl |= E1000_CTRL_ADVD3WUC; 6684 wr32(E1000_CTRL, ctrl); 6685 6686 /* Allow time for pending master requests to run */ 6687 igb_disable_pcie_master(hw); 6688 6689 wr32(E1000_WUC, E1000_WUC_PME_EN); 6690 wr32(E1000_WUFC, wufc); 6691 } else { 6692 wr32(E1000_WUC, 0); 6693 wr32(E1000_WUFC, 0); 6694 } 6695 6696 *enable_wake = wufc || adapter->en_mng_pt; 6697 if (!*enable_wake) 6698 igb_power_down_link(adapter); 6699 else 6700 igb_power_up_link(adapter); 6701 6702 /* Release control of h/w to f/w. If f/w is AMT enabled, this 6703 * would have already happened in close and is redundant. */ 6704 igb_release_hw_control(adapter); 6705 6706 pci_disable_device(pdev); 6707 6708 return 0; 6709 } 6710 6711 #ifdef CONFIG_PM 6712 static int igb_suspend(struct device *dev) 6713 { 6714 int retval; 6715 bool wake; 6716 struct pci_dev *pdev = to_pci_dev(dev); 6717 6718 retval = __igb_shutdown(pdev, &wake, 0); 6719 if (retval) 6720 return retval; 6721 6722 if (wake) { 6723 pci_prepare_to_sleep(pdev); 6724 } else { 6725 pci_wake_from_d3(pdev, false); 6726 pci_set_power_state(pdev, PCI_D3hot); 6727 } 6728 6729 return 0; 6730 } 6731 6732 static int igb_resume(struct device *dev) 6733 { 6734 struct pci_dev *pdev = to_pci_dev(dev); 6735 struct net_device *netdev = pci_get_drvdata(pdev); 6736 struct igb_adapter *adapter = netdev_priv(netdev); 6737 struct e1000_hw *hw = &adapter->hw; 6738 u32 err; 6739 6740 pci_set_power_state(pdev, PCI_D0); 6741 pci_restore_state(pdev); 6742 pci_save_state(pdev); 6743 6744 err = pci_enable_device_mem(pdev); 6745 if (err) { 6746 dev_err(&pdev->dev, 6747 "igb: Cannot enable PCI device from suspend\n"); 6748 return err; 6749 } 6750 pci_set_master(pdev); 6751 6752 pci_enable_wake(pdev, PCI_D3hot, 0); 6753 pci_enable_wake(pdev, PCI_D3cold, 0); 6754 6755 if (!rtnl_is_locked()) { 6756 /* 6757 * shut up ASSERT_RTNL() warning in 6758 * netif_set_real_num_tx/rx_queues. 6759 */ 6760 rtnl_lock(); 6761 err = igb_init_interrupt_scheme(adapter); 6762 rtnl_unlock(); 6763 } else { 6764 err = igb_init_interrupt_scheme(adapter); 6765 } 6766 if (err) { 6767 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 6768 return -ENOMEM; 6769 } 6770 6771 igb_reset(adapter); 6772 6773 /* let the f/w know that the h/w is now under the control of the 6774 * driver. */ 6775 igb_get_hw_control(adapter); 6776 6777 wr32(E1000_WUS, ~0); 6778 6779 if (netdev->flags & IFF_UP) { 6780 err = __igb_open(netdev, true); 6781 if (err) 6782 return err; 6783 } 6784 6785 netif_device_attach(netdev); 6786 return 0; 6787 } 6788 6789 #ifdef CONFIG_PM_RUNTIME 6790 static int igb_runtime_idle(struct device *dev) 6791 { 6792 struct pci_dev *pdev = to_pci_dev(dev); 6793 struct net_device *netdev = pci_get_drvdata(pdev); 6794 struct igb_adapter *adapter = netdev_priv(netdev); 6795 6796 if (!igb_has_link(adapter)) 6797 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 6798 6799 return -EBUSY; 6800 } 6801 6802 static int igb_runtime_suspend(struct device *dev) 6803 { 6804 struct pci_dev *pdev = to_pci_dev(dev); 6805 int retval; 6806 bool wake; 6807 6808 retval = __igb_shutdown(pdev, &wake, 1); 6809 if (retval) 6810 return retval; 6811 6812 if (wake) { 6813 pci_prepare_to_sleep(pdev); 6814 } else { 6815 pci_wake_from_d3(pdev, false); 6816 pci_set_power_state(pdev, PCI_D3hot); 6817 } 6818 6819 return 0; 6820 } 6821 6822 static int igb_runtime_resume(struct device *dev) 6823 { 6824 return igb_resume(dev); 6825 } 6826 #endif /* CONFIG_PM_RUNTIME */ 6827 #endif 6828 6829 static void igb_shutdown(struct pci_dev *pdev) 6830 { 6831 bool wake; 6832 6833 __igb_shutdown(pdev, &wake, 0); 6834 6835 if (system_state == SYSTEM_POWER_OFF) { 6836 pci_wake_from_d3(pdev, wake); 6837 pci_set_power_state(pdev, PCI_D3hot); 6838 } 6839 } 6840 6841 #ifdef CONFIG_NET_POLL_CONTROLLER 6842 /* 6843 * Polling 'interrupt' - used by things like netconsole to send skbs 6844 * without having to re-enable interrupts. It's not called while 6845 * the interrupt routine is executing. 6846 */ 6847 static void igb_netpoll(struct net_device *netdev) 6848 { 6849 struct igb_adapter *adapter = netdev_priv(netdev); 6850 struct e1000_hw *hw = &adapter->hw; 6851 struct igb_q_vector *q_vector; 6852 int i; 6853 6854 for (i = 0; i < adapter->num_q_vectors; i++) { 6855 q_vector = adapter->q_vector[i]; 6856 if (adapter->msix_entries) 6857 wr32(E1000_EIMC, q_vector->eims_value); 6858 else 6859 igb_irq_disable(adapter); 6860 napi_schedule(&q_vector->napi); 6861 } 6862 } 6863 #endif /* CONFIG_NET_POLL_CONTROLLER */ 6864 6865 /** 6866 * igb_io_error_detected - called when PCI error is detected 6867 * @pdev: Pointer to PCI device 6868 * @state: The current pci connection state 6869 * 6870 * This function is called after a PCI bus error affecting 6871 * this device has been detected. 6872 */ 6873 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev, 6874 pci_channel_state_t state) 6875 { 6876 struct net_device *netdev = pci_get_drvdata(pdev); 6877 struct igb_adapter *adapter = netdev_priv(netdev); 6878 6879 netif_device_detach(netdev); 6880 6881 if (state == pci_channel_io_perm_failure) 6882 return PCI_ERS_RESULT_DISCONNECT; 6883 6884 if (netif_running(netdev)) 6885 igb_down(adapter); 6886 pci_disable_device(pdev); 6887 6888 /* Request a slot slot reset. */ 6889 return PCI_ERS_RESULT_NEED_RESET; 6890 } 6891 6892 /** 6893 * igb_io_slot_reset - called after the pci bus has been reset. 6894 * @pdev: Pointer to PCI device 6895 * 6896 * Restart the card from scratch, as if from a cold-boot. Implementation 6897 * resembles the first-half of the igb_resume routine. 6898 */ 6899 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev) 6900 { 6901 struct net_device *netdev = pci_get_drvdata(pdev); 6902 struct igb_adapter *adapter = netdev_priv(netdev); 6903 struct e1000_hw *hw = &adapter->hw; 6904 pci_ers_result_t result; 6905 int err; 6906 6907 if (pci_enable_device_mem(pdev)) { 6908 dev_err(&pdev->dev, 6909 "Cannot re-enable PCI device after reset.\n"); 6910 result = PCI_ERS_RESULT_DISCONNECT; 6911 } else { 6912 pci_set_master(pdev); 6913 pci_restore_state(pdev); 6914 pci_save_state(pdev); 6915 6916 pci_enable_wake(pdev, PCI_D3hot, 0); 6917 pci_enable_wake(pdev, PCI_D3cold, 0); 6918 6919 igb_reset(adapter); 6920 wr32(E1000_WUS, ~0); 6921 result = PCI_ERS_RESULT_RECOVERED; 6922 } 6923 6924 err = pci_cleanup_aer_uncorrect_error_status(pdev); 6925 if (err) { 6926 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status " 6927 "failed 0x%0x\n", err); 6928 /* non-fatal, continue */ 6929 } 6930 6931 return result; 6932 } 6933 6934 /** 6935 * igb_io_resume - called when traffic can start flowing again. 6936 * @pdev: Pointer to PCI device 6937 * 6938 * This callback is called when the error recovery driver tells us that 6939 * its OK to resume normal operation. Implementation resembles the 6940 * second-half of the igb_resume routine. 6941 */ 6942 static void igb_io_resume(struct pci_dev *pdev) 6943 { 6944 struct net_device *netdev = pci_get_drvdata(pdev); 6945 struct igb_adapter *adapter = netdev_priv(netdev); 6946 6947 if (netif_running(netdev)) { 6948 if (igb_up(adapter)) { 6949 dev_err(&pdev->dev, "igb_up failed after reset\n"); 6950 return; 6951 } 6952 } 6953 6954 netif_device_attach(netdev); 6955 6956 /* let the f/w know that the h/w is now under the control of the 6957 * driver. */ 6958 igb_get_hw_control(adapter); 6959 } 6960 6961 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index, 6962 u8 qsel) 6963 { 6964 u32 rar_low, rar_high; 6965 struct e1000_hw *hw = &adapter->hw; 6966 6967 /* HW expects these in little endian so we reverse the byte order 6968 * from network order (big endian) to little endian 6969 */ 6970 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) | 6971 ((u32) addr[2] << 16) | ((u32) addr[3] << 24)); 6972 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8)); 6973 6974 /* Indicate to hardware the Address is Valid. */ 6975 rar_high |= E1000_RAH_AV; 6976 6977 if (hw->mac.type == e1000_82575) 6978 rar_high |= E1000_RAH_POOL_1 * qsel; 6979 else 6980 rar_high |= E1000_RAH_POOL_1 << qsel; 6981 6982 wr32(E1000_RAL(index), rar_low); 6983 wrfl(); 6984 wr32(E1000_RAH(index), rar_high); 6985 wrfl(); 6986 } 6987 6988 static int igb_set_vf_mac(struct igb_adapter *adapter, 6989 int vf, unsigned char *mac_addr) 6990 { 6991 struct e1000_hw *hw = &adapter->hw; 6992 /* VF MAC addresses start at end of receive addresses and moves 6993 * torwards the first, as a result a collision should not be possible */ 6994 int rar_entry = hw->mac.rar_entry_count - (vf + 1); 6995 6996 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN); 6997 6998 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf); 6999 7000 return 0; 7001 } 7002 7003 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 7004 { 7005 struct igb_adapter *adapter = netdev_priv(netdev); 7006 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count)) 7007 return -EINVAL; 7008 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC; 7009 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf); 7010 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this" 7011 " change effective."); 7012 if (test_bit(__IGB_DOWN, &adapter->state)) { 7013 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set," 7014 " but the PF device is not up.\n"); 7015 dev_warn(&adapter->pdev->dev, "Bring the PF device up before" 7016 " attempting to use the VF device.\n"); 7017 } 7018 return igb_set_vf_mac(adapter, vf, mac); 7019 } 7020 7021 static int igb_link_mbps(int internal_link_speed) 7022 { 7023 switch (internal_link_speed) { 7024 case SPEED_100: 7025 return 100; 7026 case SPEED_1000: 7027 return 1000; 7028 default: 7029 return 0; 7030 } 7031 } 7032 7033 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate, 7034 int link_speed) 7035 { 7036 int rf_dec, rf_int; 7037 u32 bcnrc_val; 7038 7039 if (tx_rate != 0) { 7040 /* Calculate the rate factor values to set */ 7041 rf_int = link_speed / tx_rate; 7042 rf_dec = (link_speed - (rf_int * tx_rate)); 7043 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate; 7044 7045 bcnrc_val = E1000_RTTBCNRC_RS_ENA; 7046 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) & 7047 E1000_RTTBCNRC_RF_INT_MASK); 7048 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK); 7049 } else { 7050 bcnrc_val = 0; 7051 } 7052 7053 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */ 7054 wr32(E1000_RTTBCNRC, bcnrc_val); 7055 } 7056 7057 static void igb_check_vf_rate_limit(struct igb_adapter *adapter) 7058 { 7059 int actual_link_speed, i; 7060 bool reset_rate = false; 7061 7062 /* VF TX rate limit was not set or not supported */ 7063 if ((adapter->vf_rate_link_speed == 0) || 7064 (adapter->hw.mac.type != e1000_82576)) 7065 return; 7066 7067 actual_link_speed = igb_link_mbps(adapter->link_speed); 7068 if (actual_link_speed != adapter->vf_rate_link_speed) { 7069 reset_rate = true; 7070 adapter->vf_rate_link_speed = 0; 7071 dev_info(&adapter->pdev->dev, 7072 "Link speed has been changed. VF Transmit " 7073 "rate is disabled\n"); 7074 } 7075 7076 for (i = 0; i < adapter->vfs_allocated_count; i++) { 7077 if (reset_rate) 7078 adapter->vf_data[i].tx_rate = 0; 7079 7080 igb_set_vf_rate_limit(&adapter->hw, i, 7081 adapter->vf_data[i].tx_rate, 7082 actual_link_speed); 7083 } 7084 } 7085 7086 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate) 7087 { 7088 struct igb_adapter *adapter = netdev_priv(netdev); 7089 struct e1000_hw *hw = &adapter->hw; 7090 int actual_link_speed; 7091 7092 if (hw->mac.type != e1000_82576) 7093 return -EOPNOTSUPP; 7094 7095 actual_link_speed = igb_link_mbps(adapter->link_speed); 7096 if ((vf >= adapter->vfs_allocated_count) || 7097 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) || 7098 (tx_rate < 0) || (tx_rate > actual_link_speed)) 7099 return -EINVAL; 7100 7101 adapter->vf_rate_link_speed = actual_link_speed; 7102 adapter->vf_data[vf].tx_rate = (u16)tx_rate; 7103 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed); 7104 7105 return 0; 7106 } 7107 7108 static int igb_ndo_get_vf_config(struct net_device *netdev, 7109 int vf, struct ifla_vf_info *ivi) 7110 { 7111 struct igb_adapter *adapter = netdev_priv(netdev); 7112 if (vf >= adapter->vfs_allocated_count) 7113 return -EINVAL; 7114 ivi->vf = vf; 7115 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN); 7116 ivi->tx_rate = adapter->vf_data[vf].tx_rate; 7117 ivi->vlan = adapter->vf_data[vf].pf_vlan; 7118 ivi->qos = adapter->vf_data[vf].pf_qos; 7119 return 0; 7120 } 7121 7122 static void igb_vmm_control(struct igb_adapter *adapter) 7123 { 7124 struct e1000_hw *hw = &adapter->hw; 7125 u32 reg; 7126 7127 switch (hw->mac.type) { 7128 case e1000_82575: 7129 default: 7130 /* replication is not supported for 82575 */ 7131 return; 7132 case e1000_82576: 7133 /* notify HW that the MAC is adding vlan tags */ 7134 reg = rd32(E1000_DTXCTL); 7135 reg |= E1000_DTXCTL_VLAN_ADDED; 7136 wr32(E1000_DTXCTL, reg); 7137 case e1000_82580: 7138 /* enable replication vlan tag stripping */ 7139 reg = rd32(E1000_RPLOLR); 7140 reg |= E1000_RPLOLR_STRVLAN; 7141 wr32(E1000_RPLOLR, reg); 7142 case e1000_i350: 7143 /* none of the above registers are supported by i350 */ 7144 break; 7145 } 7146 7147 if (adapter->vfs_allocated_count) { 7148 igb_vmdq_set_loopback_pf(hw, true); 7149 igb_vmdq_set_replication_pf(hw, true); 7150 igb_vmdq_set_anti_spoofing_pf(hw, true, 7151 adapter->vfs_allocated_count); 7152 } else { 7153 igb_vmdq_set_loopback_pf(hw, false); 7154 igb_vmdq_set_replication_pf(hw, false); 7155 } 7156 } 7157 7158 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba) 7159 { 7160 struct e1000_hw *hw = &adapter->hw; 7161 u32 dmac_thr; 7162 u16 hwm; 7163 7164 if (hw->mac.type > e1000_82580) { 7165 if (adapter->flags & IGB_FLAG_DMAC) { 7166 u32 reg; 7167 7168 /* force threshold to 0. */ 7169 wr32(E1000_DMCTXTH, 0); 7170 7171 /* 7172 * DMA Coalescing high water mark needs to be greater 7173 * than the Rx threshold. Set hwm to PBA - max frame 7174 * size in 16B units, capping it at PBA - 6KB. 7175 */ 7176 hwm = 64 * pba - adapter->max_frame_size / 16; 7177 if (hwm < 64 * (pba - 6)) 7178 hwm = 64 * (pba - 6); 7179 reg = rd32(E1000_FCRTC); 7180 reg &= ~E1000_FCRTC_RTH_COAL_MASK; 7181 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) 7182 & E1000_FCRTC_RTH_COAL_MASK); 7183 wr32(E1000_FCRTC, reg); 7184 7185 /* 7186 * Set the DMA Coalescing Rx threshold to PBA - 2 * max 7187 * frame size, capping it at PBA - 10KB. 7188 */ 7189 dmac_thr = pba - adapter->max_frame_size / 512; 7190 if (dmac_thr < pba - 10) 7191 dmac_thr = pba - 10; 7192 reg = rd32(E1000_DMACR); 7193 reg &= ~E1000_DMACR_DMACTHR_MASK; 7194 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT) 7195 & E1000_DMACR_DMACTHR_MASK); 7196 7197 /* transition to L0x or L1 if available..*/ 7198 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); 7199 7200 /* watchdog timer= +-1000 usec in 32usec intervals */ 7201 reg |= (1000 >> 5); 7202 wr32(E1000_DMACR, reg); 7203 7204 /* 7205 * no lower threshold to disable 7206 * coalescing(smart fifb)-UTRESH=0 7207 */ 7208 wr32(E1000_DMCRTRH, 0); 7209 7210 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4); 7211 7212 wr32(E1000_DMCTLX, reg); 7213 7214 /* 7215 * free space in tx packet buffer to wake from 7216 * DMA coal 7217 */ 7218 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE - 7219 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6); 7220 7221 /* 7222 * make low power state decision controlled 7223 * by DMA coal 7224 */ 7225 reg = rd32(E1000_PCIEMISC); 7226 reg &= ~E1000_PCIEMISC_LX_DECISION; 7227 wr32(E1000_PCIEMISC, reg); 7228 } /* endif adapter->dmac is not disabled */ 7229 } else if (hw->mac.type == e1000_82580) { 7230 u32 reg = rd32(E1000_PCIEMISC); 7231 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION); 7232 wr32(E1000_DMACR, 0); 7233 } 7234 } 7235 7236 /* igb_main.c */ 7237