1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 1999 - 2018 Intel Corporation. */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/module.h> 7 #include <linux/types.h> 8 #include <linux/init.h> 9 #include <linux/pci.h> 10 #include <linux/vmalloc.h> 11 #include <linux/pagemap.h> 12 #include <linux/delay.h> 13 #include <linux/netdevice.h> 14 #include <linux/interrupt.h> 15 #include <linux/tcp.h> 16 #include <linux/ipv6.h> 17 #include <linux/slab.h> 18 #include <net/checksum.h> 19 #include <net/ip6_checksum.h> 20 #include <linux/ethtool.h> 21 #include <linux/if_vlan.h> 22 #include <linux/cpu.h> 23 #include <linux/smp.h> 24 #include <linux/pm_qos.h> 25 #include <linux/pm_runtime.h> 26 #include <linux/prefetch.h> 27 #include <linux/suspend.h> 28 #include <linux/dmi.h> 29 30 #include "e1000.h" 31 #define CREATE_TRACE_POINTS 32 #include "e1000e_trace.h" 33 34 char e1000e_driver_name[] = "e1000e"; 35 36 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 37 static int debug = -1; 38 module_param(debug, int, 0); 39 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 40 41 static const struct e1000_info *e1000_info_tbl[] = { 42 [board_82571] = &e1000_82571_info, 43 [board_82572] = &e1000_82572_info, 44 [board_82573] = &e1000_82573_info, 45 [board_82574] = &e1000_82574_info, 46 [board_82583] = &e1000_82583_info, 47 [board_80003es2lan] = &e1000_es2_info, 48 [board_ich8lan] = &e1000_ich8_info, 49 [board_ich9lan] = &e1000_ich9_info, 50 [board_ich10lan] = &e1000_ich10_info, 51 [board_pchlan] = &e1000_pch_info, 52 [board_pch2lan] = &e1000_pch2_info, 53 [board_pch_lpt] = &e1000_pch_lpt_info, 54 [board_pch_spt] = &e1000_pch_spt_info, 55 [board_pch_cnp] = &e1000_pch_cnp_info, 56 [board_pch_tgp] = &e1000_pch_tgp_info, 57 [board_pch_adp] = &e1000_pch_adp_info, 58 [board_pch_mtp] = &e1000_pch_mtp_info, 59 [board_pch_ptp] = &e1000_pch_ptp_info, 60 }; 61 62 static const struct dmi_system_id disable_k1_list[] = { 63 { 64 .ident = "Dell Pro 16 Plus PB16250", 65 .matches = { 66 DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), 67 DMI_MATCH(DMI_PRODUCT_NAME, "Dell Pro 16 Plus PB16250"), 68 }, 69 }, 70 { 71 .ident = "Lenovo ThinkPad P14s Gen 5", 72 .matches = { 73 DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), 74 DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad P14s Gen 5"), 75 }, 76 }, 77 {} 78 }; 79 80 struct e1000_reg_info { 81 u32 ofs; 82 char *name; 83 }; 84 85 static const struct e1000_reg_info e1000_reg_info_tbl[] = { 86 /* General Registers */ 87 {E1000_CTRL, "CTRL"}, 88 {E1000_STATUS, "STATUS"}, 89 {E1000_CTRL_EXT, "CTRL_EXT"}, 90 91 /* Interrupt Registers */ 92 {E1000_ICR, "ICR"}, 93 94 /* Rx Registers */ 95 {E1000_RCTL, "RCTL"}, 96 {E1000_RDLEN(0), "RDLEN"}, 97 {E1000_RDH(0), "RDH"}, 98 {E1000_RDT(0), "RDT"}, 99 {E1000_RDTR, "RDTR"}, 100 {E1000_RXDCTL(0), "RXDCTL"}, 101 {E1000_ERT, "ERT"}, 102 {E1000_RDBAL(0), "RDBAL"}, 103 {E1000_RDBAH(0), "RDBAH"}, 104 {E1000_RDFH, "RDFH"}, 105 {E1000_RDFT, "RDFT"}, 106 {E1000_RDFHS, "RDFHS"}, 107 {E1000_RDFTS, "RDFTS"}, 108 {E1000_RDFPC, "RDFPC"}, 109 110 /* Tx Registers */ 111 {E1000_TCTL, "TCTL"}, 112 {E1000_TDBAL(0), "TDBAL"}, 113 {E1000_TDBAH(0), "TDBAH"}, 114 {E1000_TDLEN(0), "TDLEN"}, 115 {E1000_TDH(0), "TDH"}, 116 {E1000_TDT(0), "TDT"}, 117 {E1000_TIDV, "TIDV"}, 118 {E1000_TXDCTL(0), "TXDCTL"}, 119 {E1000_TADV, "TADV"}, 120 {E1000_TARC(0), "TARC"}, 121 {E1000_TDFH, "TDFH"}, 122 {E1000_TDFT, "TDFT"}, 123 {E1000_TDFHS, "TDFHS"}, 124 {E1000_TDFTS, "TDFTS"}, 125 {E1000_TDFPC, "TDFPC"}, 126 127 /* List Terminator */ 128 {0, NULL} 129 }; 130 131 /** 132 * __ew32_prepare - prepare to write to MAC CSR register on certain parts 133 * @hw: pointer to the HW structure 134 * 135 * When updating the MAC CSR registers, the Manageability Engine (ME) could 136 * be accessing the registers at the same time. Normally, this is handled in 137 * h/w by an arbiter but on some parts there is a bug that acknowledges Host 138 * accesses later than it should which could result in the register to have 139 * an incorrect value. Workaround this by checking the FWSM register which 140 * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set 141 * and try again a number of times. 142 **/ 143 static void __ew32_prepare(struct e1000_hw *hw) 144 { 145 s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT; 146 147 while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i) 148 udelay(50); 149 } 150 151 void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val) 152 { 153 if (hw->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 154 __ew32_prepare(hw); 155 156 writel(val, hw->hw_addr + reg); 157 } 158 159 /** 160 * e1000_regdump - register printout routine 161 * @hw: pointer to the HW structure 162 * @reginfo: pointer to the register info table 163 **/ 164 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo) 165 { 166 int n = 0; 167 char rname[16]; 168 u32 regs[8]; 169 170 switch (reginfo->ofs) { 171 case E1000_RXDCTL(0): 172 for (n = 0; n < 2; n++) 173 regs[n] = __er32(hw, E1000_RXDCTL(n)); 174 break; 175 case E1000_TXDCTL(0): 176 for (n = 0; n < 2; n++) 177 regs[n] = __er32(hw, E1000_TXDCTL(n)); 178 break; 179 case E1000_TARC(0): 180 for (n = 0; n < 2; n++) 181 regs[n] = __er32(hw, E1000_TARC(n)); 182 break; 183 default: 184 pr_info("%-15s %08x\n", 185 reginfo->name, __er32(hw, reginfo->ofs)); 186 return; 187 } 188 189 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]"); 190 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]); 191 } 192 193 static void e1000e_dump_ps_pages(struct e1000_adapter *adapter, 194 struct e1000_buffer *bi) 195 { 196 int i; 197 struct e1000_ps_page *ps_page; 198 199 for (i = 0; i < adapter->rx_ps_pages; i++) { 200 ps_page = &bi->ps_pages[i]; 201 202 if (ps_page->page) { 203 pr_info("packet dump for ps_page %d:\n", i); 204 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 205 16, 1, page_address(ps_page->page), 206 PAGE_SIZE, true); 207 } 208 } 209 } 210 211 /** 212 * e1000e_dump - Print registers, Tx-ring and Rx-ring 213 * @adapter: board private structure 214 **/ 215 static void e1000e_dump(struct e1000_adapter *adapter) 216 { 217 struct net_device *netdev = adapter->netdev; 218 struct e1000_hw *hw = &adapter->hw; 219 struct e1000_reg_info *reginfo; 220 struct e1000_ring *tx_ring = adapter->tx_ring; 221 struct e1000_tx_desc *tx_desc; 222 struct my_u0 { 223 __le64 a; 224 __le64 b; 225 } *u0; 226 struct e1000_buffer *buffer_info; 227 struct e1000_ring *rx_ring = adapter->rx_ring; 228 union e1000_rx_desc_packet_split *rx_desc_ps; 229 union e1000_rx_desc_extended *rx_desc; 230 struct my_u1 { 231 __le64 a; 232 __le64 b; 233 __le64 c; 234 __le64 d; 235 } *u1; 236 u32 staterr; 237 int i = 0; 238 239 if (!netif_msg_hw(adapter)) 240 return; 241 242 /* Print netdevice Info */ 243 if (netdev) { 244 dev_info(&adapter->pdev->dev, "Net device Info\n"); 245 pr_info("Device Name state trans_start\n"); 246 pr_info("%-15s %016lX %016lX\n", netdev->name, 247 netdev->state, dev_trans_start(netdev)); 248 } 249 250 /* Print Registers */ 251 dev_info(&adapter->pdev->dev, "Register Dump\n"); 252 pr_info(" Register Name Value\n"); 253 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl; 254 reginfo->name; reginfo++) { 255 e1000_regdump(hw, reginfo); 256 } 257 258 /* Print Tx Ring Summary */ 259 if (!netdev || !netif_running(netdev)) 260 return; 261 262 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n"); 263 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 264 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean]; 265 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n", 266 0, tx_ring->next_to_use, tx_ring->next_to_clean, 267 (unsigned long long)buffer_info->dma, 268 buffer_info->length, 269 buffer_info->next_to_watch, 270 (unsigned long long)buffer_info->time_stamp); 271 272 /* Print Tx Ring */ 273 if (!netif_msg_tx_done(adapter)) 274 goto rx_ring_summary; 275 276 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n"); 277 278 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended) 279 * 280 * Legacy Transmit Descriptor 281 * +--------------------------------------------------------------+ 282 * 0 | Buffer Address [63:0] (Reserved on Write Back) | 283 * +--------------------------------------------------------------+ 284 * 8 | Special | CSS | Status | CMD | CSO | Length | 285 * +--------------------------------------------------------------+ 286 * 63 48 47 36 35 32 31 24 23 16 15 0 287 * 288 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload 289 * 63 48 47 40 39 32 31 16 15 8 7 0 290 * +----------------------------------------------------------------+ 291 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS | 292 * +----------------------------------------------------------------+ 293 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN | 294 * +----------------------------------------------------------------+ 295 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 296 * 297 * Extended Data Descriptor (DTYP=0x1) 298 * +----------------------------------------------------------------+ 299 * 0 | Buffer Address [63:0] | 300 * +----------------------------------------------------------------+ 301 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN | 302 * +----------------------------------------------------------------+ 303 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0 304 */ 305 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n"); 306 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n"); 307 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n"); 308 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 309 const char *next_desc; 310 tx_desc = E1000_TX_DESC(*tx_ring, i); 311 buffer_info = &tx_ring->buffer_info[i]; 312 u0 = (struct my_u0 *)tx_desc; 313 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean) 314 next_desc = " NTC/U"; 315 else if (i == tx_ring->next_to_use) 316 next_desc = " NTU"; 317 else if (i == tx_ring->next_to_clean) 318 next_desc = " NTC"; 319 else 320 next_desc = ""; 321 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n", 322 (!(le64_to_cpu(u0->b) & BIT(29)) ? 'l' : 323 ((le64_to_cpu(u0->b) & BIT(20)) ? 'd' : 'c')), 324 i, 325 (unsigned long long)le64_to_cpu(u0->a), 326 (unsigned long long)le64_to_cpu(u0->b), 327 (unsigned long long)buffer_info->dma, 328 buffer_info->length, buffer_info->next_to_watch, 329 (unsigned long long)buffer_info->time_stamp, 330 buffer_info->skb, next_desc); 331 332 if (netif_msg_pktdata(adapter) && buffer_info->skb) 333 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 334 16, 1, buffer_info->skb->data, 335 buffer_info->skb->len, true); 336 } 337 338 /* Print Rx Ring Summary */ 339 rx_ring_summary: 340 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n"); 341 pr_info("Queue [NTU] [NTC]\n"); 342 pr_info(" %5d %5X %5X\n", 343 0, rx_ring->next_to_use, rx_ring->next_to_clean); 344 345 /* Print Rx Ring */ 346 if (!netif_msg_rx_status(adapter)) 347 return; 348 349 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n"); 350 switch (adapter->rx_ps_pages) { 351 case 1: 352 case 2: 353 case 3: 354 /* [Extended] Packet Split Receive Descriptor Format 355 * 356 * +-----------------------------------------------------+ 357 * 0 | Buffer Address 0 [63:0] | 358 * +-----------------------------------------------------+ 359 * 8 | Buffer Address 1 [63:0] | 360 * +-----------------------------------------------------+ 361 * 16 | Buffer Address 2 [63:0] | 362 * +-----------------------------------------------------+ 363 * 24 | Buffer Address 3 [63:0] | 364 * +-----------------------------------------------------+ 365 */ 366 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n"); 367 /* [Extended] Receive Descriptor (Write-Back) Format 368 * 369 * 63 48 47 32 31 13 12 8 7 4 3 0 370 * +------------------------------------------------------+ 371 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS | 372 * | Checksum | Ident | | Queue | | Type | 373 * +------------------------------------------------------+ 374 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 375 * +------------------------------------------------------+ 376 * 63 48 47 32 31 20 19 0 377 */ 378 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n"); 379 for (i = 0; i < rx_ring->count; i++) { 380 const char *next_desc; 381 buffer_info = &rx_ring->buffer_info[i]; 382 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i); 383 u1 = (struct my_u1 *)rx_desc_ps; 384 staterr = 385 le32_to_cpu(rx_desc_ps->wb.middle.status_error); 386 387 if (i == rx_ring->next_to_use) 388 next_desc = " NTU"; 389 else if (i == rx_ring->next_to_clean) 390 next_desc = " NTC"; 391 else 392 next_desc = ""; 393 394 if (staterr & E1000_RXD_STAT_DD) { 395 /* Descriptor Done */ 396 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n", 397 "RWB", i, 398 (unsigned long long)le64_to_cpu(u1->a), 399 (unsigned long long)le64_to_cpu(u1->b), 400 (unsigned long long)le64_to_cpu(u1->c), 401 (unsigned long long)le64_to_cpu(u1->d), 402 buffer_info->skb, next_desc); 403 } else { 404 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n", 405 "R ", i, 406 (unsigned long long)le64_to_cpu(u1->a), 407 (unsigned long long)le64_to_cpu(u1->b), 408 (unsigned long long)le64_to_cpu(u1->c), 409 (unsigned long long)le64_to_cpu(u1->d), 410 (unsigned long long)buffer_info->dma, 411 buffer_info->skb, next_desc); 412 413 if (netif_msg_pktdata(adapter)) 414 e1000e_dump_ps_pages(adapter, 415 buffer_info); 416 } 417 } 418 break; 419 default: 420 case 0: 421 /* Extended Receive Descriptor (Read) Format 422 * 423 * +-----------------------------------------------------+ 424 * 0 | Buffer Address [63:0] | 425 * +-----------------------------------------------------+ 426 * 8 | Reserved | 427 * +-----------------------------------------------------+ 428 */ 429 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n"); 430 /* Extended Receive Descriptor (Write-Back) Format 431 * 432 * 63 48 47 32 31 24 23 4 3 0 433 * +------------------------------------------------------+ 434 * | RSS Hash | | | | 435 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS | 436 * | Packet | IP | | | Type | 437 * | Checksum | Ident | | | | 438 * +------------------------------------------------------+ 439 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 440 * +------------------------------------------------------+ 441 * 63 48 47 32 31 20 19 0 442 */ 443 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n"); 444 445 for (i = 0; i < rx_ring->count; i++) { 446 const char *next_desc; 447 448 buffer_info = &rx_ring->buffer_info[i]; 449 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 450 u1 = (struct my_u1 *)rx_desc; 451 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 452 453 if (i == rx_ring->next_to_use) 454 next_desc = " NTU"; 455 else if (i == rx_ring->next_to_clean) 456 next_desc = " NTC"; 457 else 458 next_desc = ""; 459 460 if (staterr & E1000_RXD_STAT_DD) { 461 /* Descriptor Done */ 462 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n", 463 "RWB", i, 464 (unsigned long long)le64_to_cpu(u1->a), 465 (unsigned long long)le64_to_cpu(u1->b), 466 buffer_info->skb, next_desc); 467 } else { 468 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n", 469 "R ", i, 470 (unsigned long long)le64_to_cpu(u1->a), 471 (unsigned long long)le64_to_cpu(u1->b), 472 (unsigned long long)buffer_info->dma, 473 buffer_info->skb, next_desc); 474 475 if (netif_msg_pktdata(adapter) && 476 buffer_info->skb) 477 print_hex_dump(KERN_INFO, "", 478 DUMP_PREFIX_ADDRESS, 16, 479 1, 480 buffer_info->skb->data, 481 adapter->rx_buffer_len, 482 true); 483 } 484 } 485 } 486 } 487 488 /** 489 * e1000_desc_unused - calculate if we have unused descriptors 490 * @ring: pointer to ring struct to perform calculation on 491 **/ 492 static int e1000_desc_unused(struct e1000_ring *ring) 493 { 494 if (ring->next_to_clean > ring->next_to_use) 495 return ring->next_to_clean - ring->next_to_use - 1; 496 497 return ring->count + ring->next_to_clean - ring->next_to_use - 1; 498 } 499 500 /** 501 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp 502 * @adapter: board private structure 503 * @hwtstamps: time stamp structure to update 504 * @systim: unsigned 64bit system time value. 505 * 506 * Convert the system time value stored in the RX/TXSTMP registers into a 507 * hwtstamp which can be used by the upper level time stamping functions. 508 * 509 * The 'systim_lock' spinlock is used to protect the consistency of the 510 * system time value. This is needed because reading the 64 bit time 511 * value involves reading two 32 bit registers. The first read latches the 512 * value. 513 **/ 514 static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter, 515 struct skb_shared_hwtstamps *hwtstamps, 516 u64 systim) 517 { 518 u64 ns; 519 unsigned long flags; 520 521 spin_lock_irqsave(&adapter->systim_lock, flags); 522 ns = timecounter_cyc2time(&adapter->tc, systim); 523 spin_unlock_irqrestore(&adapter->systim_lock, flags); 524 525 memset(hwtstamps, 0, sizeof(*hwtstamps)); 526 hwtstamps->hwtstamp = ns_to_ktime(ns); 527 } 528 529 /** 530 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp 531 * @adapter: board private structure 532 * @status: descriptor extended error and status field 533 * @skb: particular skb to include time stamp 534 * 535 * If the time stamp is valid, convert it into the timecounter ns value 536 * and store that result into the shhwtstamps structure which is passed 537 * up the network stack. 538 **/ 539 static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status, 540 struct sk_buff *skb) 541 { 542 struct e1000_hw *hw = &adapter->hw; 543 u64 rxstmp; 544 545 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) || 546 !(status & E1000_RXDEXT_STATERR_TST) || 547 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) 548 return; 549 550 /* The Rx time stamp registers contain the time stamp. No other 551 * received packet will be time stamped until the Rx time stamp 552 * registers are read. Because only one packet can be time stamped 553 * at a time, the register values must belong to this packet and 554 * therefore none of the other additional attributes need to be 555 * compared. 556 */ 557 rxstmp = (u64)er32(RXSTMPL); 558 rxstmp |= (u64)er32(RXSTMPH) << 32; 559 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp); 560 561 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP; 562 } 563 564 /** 565 * e1000_receive_skb - helper function to handle Rx indications 566 * @adapter: board private structure 567 * @netdev: pointer to netdev struct 568 * @staterr: descriptor extended error and status field as written by hardware 569 * @vlan: descriptor vlan field as written by hardware (no le/be conversion) 570 * @skb: pointer to sk_buff to be indicated to stack 571 **/ 572 static void e1000_receive_skb(struct e1000_adapter *adapter, 573 struct net_device *netdev, struct sk_buff *skb, 574 u32 staterr, __le16 vlan) 575 { 576 u16 tag = le16_to_cpu(vlan); 577 578 e1000e_rx_hwtstamp(adapter, staterr, skb); 579 580 skb->protocol = eth_type_trans(skb, netdev); 581 582 if (staterr & E1000_RXD_STAT_VP) 583 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag); 584 585 napi_gro_receive(&adapter->napi, skb); 586 } 587 588 /** 589 * e1000_rx_checksum - Receive Checksum Offload 590 * @adapter: board private structure 591 * @status_err: receive descriptor status and error fields 592 * @skb: socket buffer with received data 593 **/ 594 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err, 595 struct sk_buff *skb) 596 { 597 u16 status = (u16)status_err; 598 u8 errors = (u8)(status_err >> 24); 599 600 skb_checksum_none_assert(skb); 601 602 /* Rx checksum disabled */ 603 if (!(adapter->netdev->features & NETIF_F_RXCSUM)) 604 return; 605 606 /* Ignore Checksum bit is set */ 607 if (status & E1000_RXD_STAT_IXSM) 608 return; 609 610 /* TCP/UDP checksum error bit or IP checksum error bit is set */ 611 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) { 612 /* let the stack verify checksum errors */ 613 adapter->hw_csum_err++; 614 return; 615 } 616 617 /* TCP/UDP Checksum has not been calculated */ 618 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) 619 return; 620 621 /* It must be a TCP or UDP packet with a valid checksum */ 622 skb->ip_summed = CHECKSUM_UNNECESSARY; 623 adapter->hw_csum_good++; 624 } 625 626 static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i) 627 { 628 struct e1000_adapter *adapter = rx_ring->adapter; 629 struct e1000_hw *hw = &adapter->hw; 630 631 __ew32_prepare(hw); 632 writel(i, rx_ring->tail); 633 634 if (unlikely(i != readl(rx_ring->tail))) { 635 u32 rctl = er32(RCTL); 636 637 ew32(RCTL, rctl & ~E1000_RCTL_EN); 638 e_err("ME firmware caused invalid RDT - resetting\n"); 639 schedule_work(&adapter->reset_task); 640 } 641 } 642 643 static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i) 644 { 645 struct e1000_adapter *adapter = tx_ring->adapter; 646 struct e1000_hw *hw = &adapter->hw; 647 648 __ew32_prepare(hw); 649 writel(i, tx_ring->tail); 650 651 if (unlikely(i != readl(tx_ring->tail))) { 652 u32 tctl = er32(TCTL); 653 654 ew32(TCTL, tctl & ~E1000_TCTL_EN); 655 e_err("ME firmware caused invalid TDT - resetting\n"); 656 schedule_work(&adapter->reset_task); 657 } 658 } 659 660 /** 661 * e1000_alloc_rx_buffers - Replace used receive buffers 662 * @rx_ring: Rx descriptor ring 663 * @cleaned_count: number to reallocate 664 * @gfp: flags for allocation 665 **/ 666 static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring, 667 int cleaned_count, gfp_t gfp) 668 { 669 struct e1000_adapter *adapter = rx_ring->adapter; 670 struct net_device *netdev = adapter->netdev; 671 struct pci_dev *pdev = adapter->pdev; 672 union e1000_rx_desc_extended *rx_desc; 673 struct e1000_buffer *buffer_info; 674 struct sk_buff *skb; 675 unsigned int i; 676 unsigned int bufsz = adapter->rx_buffer_len; 677 678 i = rx_ring->next_to_use; 679 buffer_info = &rx_ring->buffer_info[i]; 680 681 while (cleaned_count--) { 682 skb = buffer_info->skb; 683 if (skb) { 684 skb_trim(skb, 0); 685 goto map_skb; 686 } 687 688 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 689 if (!skb) { 690 /* Better luck next round */ 691 adapter->alloc_rx_buff_failed++; 692 break; 693 } 694 695 buffer_info->skb = skb; 696 map_skb: 697 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 698 adapter->rx_buffer_len, 699 DMA_FROM_DEVICE); 700 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 701 dev_err(&pdev->dev, "Rx DMA map failed\n"); 702 adapter->rx_dma_failed++; 703 break; 704 } 705 706 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 707 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 708 709 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 710 /* Force memory writes to complete before letting h/w 711 * know there are new descriptors to fetch. (Only 712 * applicable for weak-ordered memory model archs, 713 * such as IA-64). 714 */ 715 wmb(); 716 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 717 e1000e_update_rdt_wa(rx_ring, i); 718 else 719 writel(i, rx_ring->tail); 720 } 721 i++; 722 if (i == rx_ring->count) 723 i = 0; 724 buffer_info = &rx_ring->buffer_info[i]; 725 } 726 727 rx_ring->next_to_use = i; 728 } 729 730 /** 731 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split 732 * @rx_ring: Rx descriptor ring 733 * @cleaned_count: number to reallocate 734 * @gfp: flags for allocation 735 **/ 736 static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring, 737 int cleaned_count, gfp_t gfp) 738 { 739 struct e1000_adapter *adapter = rx_ring->adapter; 740 struct net_device *netdev = adapter->netdev; 741 struct pci_dev *pdev = adapter->pdev; 742 union e1000_rx_desc_packet_split *rx_desc; 743 struct e1000_buffer *buffer_info; 744 struct e1000_ps_page *ps_page; 745 struct sk_buff *skb; 746 unsigned int i, j; 747 748 i = rx_ring->next_to_use; 749 buffer_info = &rx_ring->buffer_info[i]; 750 751 while (cleaned_count--) { 752 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 753 754 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 755 ps_page = &buffer_info->ps_pages[j]; 756 if (j >= adapter->rx_ps_pages) { 757 /* all unused desc entries get hw null ptr */ 758 rx_desc->read.buffer_addr[j + 1] = 759 ~cpu_to_le64(0); 760 continue; 761 } 762 if (!ps_page->page) { 763 ps_page->page = alloc_page(gfp); 764 if (!ps_page->page) { 765 adapter->alloc_rx_buff_failed++; 766 goto no_buffers; 767 } 768 ps_page->dma = dma_map_page(&pdev->dev, 769 ps_page->page, 770 0, PAGE_SIZE, 771 DMA_FROM_DEVICE); 772 if (dma_mapping_error(&pdev->dev, 773 ps_page->dma)) { 774 dev_err(&adapter->pdev->dev, 775 "Rx DMA page map failed\n"); 776 adapter->rx_dma_failed++; 777 goto no_buffers; 778 } 779 } 780 /* Refresh the desc even if buffer_addrs 781 * didn't change because each write-back 782 * erases this info. 783 */ 784 rx_desc->read.buffer_addr[j + 1] = 785 cpu_to_le64(ps_page->dma); 786 } 787 788 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0, 789 gfp); 790 791 if (!skb) { 792 adapter->alloc_rx_buff_failed++; 793 break; 794 } 795 796 buffer_info->skb = skb; 797 buffer_info->dma = dma_map_single(&pdev->dev, skb->data, 798 adapter->rx_ps_bsize0, 799 DMA_FROM_DEVICE); 800 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 801 dev_err(&pdev->dev, "Rx DMA map failed\n"); 802 adapter->rx_dma_failed++; 803 /* cleanup skb */ 804 dev_kfree_skb_any(skb); 805 buffer_info->skb = NULL; 806 break; 807 } 808 809 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma); 810 811 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) { 812 /* Force memory writes to complete before letting h/w 813 * know there are new descriptors to fetch. (Only 814 * applicable for weak-ordered memory model archs, 815 * such as IA-64). 816 */ 817 wmb(); 818 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 819 e1000e_update_rdt_wa(rx_ring, i << 1); 820 else 821 writel(i << 1, rx_ring->tail); 822 } 823 824 i++; 825 if (i == rx_ring->count) 826 i = 0; 827 buffer_info = &rx_ring->buffer_info[i]; 828 } 829 830 no_buffers: 831 rx_ring->next_to_use = i; 832 } 833 834 /** 835 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers 836 * @rx_ring: Rx descriptor ring 837 * @cleaned_count: number of buffers to allocate this pass 838 * @gfp: flags for allocation 839 **/ 840 841 static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring, 842 int cleaned_count, gfp_t gfp) 843 { 844 struct e1000_adapter *adapter = rx_ring->adapter; 845 struct net_device *netdev = adapter->netdev; 846 struct pci_dev *pdev = adapter->pdev; 847 union e1000_rx_desc_extended *rx_desc; 848 struct e1000_buffer *buffer_info; 849 struct sk_buff *skb; 850 unsigned int i; 851 unsigned int bufsz = 256 - 16; /* for skb_reserve */ 852 853 i = rx_ring->next_to_use; 854 buffer_info = &rx_ring->buffer_info[i]; 855 856 while (cleaned_count--) { 857 skb = buffer_info->skb; 858 if (skb) { 859 skb_trim(skb, 0); 860 goto check_page; 861 } 862 863 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp); 864 if (unlikely(!skb)) { 865 /* Better luck next round */ 866 adapter->alloc_rx_buff_failed++; 867 break; 868 } 869 870 buffer_info->skb = skb; 871 check_page: 872 /* allocate a new page if necessary */ 873 if (!buffer_info->page) { 874 buffer_info->page = alloc_page(gfp); 875 if (unlikely(!buffer_info->page)) { 876 adapter->alloc_rx_buff_failed++; 877 break; 878 } 879 } 880 881 if (!buffer_info->dma) { 882 buffer_info->dma = dma_map_page(&pdev->dev, 883 buffer_info->page, 0, 884 PAGE_SIZE, 885 DMA_FROM_DEVICE); 886 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 887 adapter->alloc_rx_buff_failed++; 888 break; 889 } 890 } 891 892 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 893 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma); 894 895 if (unlikely(++i == rx_ring->count)) 896 i = 0; 897 buffer_info = &rx_ring->buffer_info[i]; 898 } 899 900 if (likely(rx_ring->next_to_use != i)) { 901 rx_ring->next_to_use = i; 902 if (unlikely(i-- == 0)) 903 i = (rx_ring->count - 1); 904 905 /* Force memory writes to complete before letting h/w 906 * know there are new descriptors to fetch. (Only 907 * applicable for weak-ordered memory model archs, 908 * such as IA-64). 909 */ 910 wmb(); 911 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 912 e1000e_update_rdt_wa(rx_ring, i); 913 else 914 writel(i, rx_ring->tail); 915 } 916 } 917 918 static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss, 919 struct sk_buff *skb) 920 { 921 if (netdev->features & NETIF_F_RXHASH) 922 skb_set_hash(skb, le32_to_cpu(rss), PKT_HASH_TYPE_L3); 923 } 924 925 /** 926 * e1000_clean_rx_irq - Send received data up the network stack 927 * @rx_ring: Rx descriptor ring 928 * @work_done: output parameter for indicating completed work 929 * @work_to_do: how many packets we can clean 930 * 931 * the return value indicates whether actual cleaning was done, there 932 * is no guarantee that everything was cleaned 933 **/ 934 static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done, 935 int work_to_do) 936 { 937 struct e1000_adapter *adapter = rx_ring->adapter; 938 struct net_device *netdev = adapter->netdev; 939 struct pci_dev *pdev = adapter->pdev; 940 struct e1000_hw *hw = &adapter->hw; 941 union e1000_rx_desc_extended *rx_desc, *next_rxd; 942 struct e1000_buffer *buffer_info, *next_buffer; 943 u32 length, staterr; 944 unsigned int i; 945 int cleaned_count = 0; 946 bool cleaned = false; 947 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 948 949 i = rx_ring->next_to_clean; 950 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 951 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 952 buffer_info = &rx_ring->buffer_info[i]; 953 954 while (staterr & E1000_RXD_STAT_DD) { 955 struct sk_buff *skb; 956 957 if (*work_done >= work_to_do) 958 break; 959 (*work_done)++; 960 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 961 962 skb = buffer_info->skb; 963 buffer_info->skb = NULL; 964 965 prefetch(skb->data - NET_IP_ALIGN); 966 967 i++; 968 if (i == rx_ring->count) 969 i = 0; 970 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 971 prefetch(next_rxd); 972 973 next_buffer = &rx_ring->buffer_info[i]; 974 975 cleaned = true; 976 cleaned_count++; 977 dma_unmap_single(&pdev->dev, buffer_info->dma, 978 adapter->rx_buffer_len, DMA_FROM_DEVICE); 979 buffer_info->dma = 0; 980 981 length = le16_to_cpu(rx_desc->wb.upper.length); 982 983 /* !EOP means multiple descriptors were used to store a single 984 * packet, if that's the case we need to toss it. In fact, we 985 * need to toss every packet with the EOP bit clear and the 986 * next frame that _does_ have the EOP bit set, as it is by 987 * definition only a frame fragment 988 */ 989 if (unlikely(!(staterr & E1000_RXD_STAT_EOP))) 990 adapter->flags2 |= FLAG2_IS_DISCARDING; 991 992 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 993 /* All receives must fit into a single buffer */ 994 e_dbg("Receive packet consumed multiple buffers\n"); 995 /* recycle */ 996 buffer_info->skb = skb; 997 if (staterr & E1000_RXD_STAT_EOP) 998 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 999 goto next_desc; 1000 } 1001 1002 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1003 !(netdev->features & NETIF_F_RXALL))) { 1004 /* recycle */ 1005 buffer_info->skb = skb; 1006 goto next_desc; 1007 } 1008 1009 /* adjust length to remove Ethernet CRC */ 1010 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1011 /* If configured to store CRC, don't subtract FCS, 1012 * but keep the FCS bytes out of the total_rx_bytes 1013 * counter 1014 */ 1015 if (netdev->features & NETIF_F_RXFCS) 1016 total_rx_bytes -= 4; 1017 else 1018 length -= 4; 1019 } 1020 1021 total_rx_bytes += length; 1022 total_rx_packets++; 1023 1024 /* code added for copybreak, this should improve 1025 * performance for small packets with large amounts 1026 * of reassembly being done in the stack 1027 */ 1028 if (length < copybreak) { 1029 struct sk_buff *new_skb = 1030 napi_alloc_skb(&adapter->napi, length); 1031 if (new_skb) { 1032 skb_copy_to_linear_data_offset(new_skb, 1033 -NET_IP_ALIGN, 1034 (skb->data - 1035 NET_IP_ALIGN), 1036 (length + 1037 NET_IP_ALIGN)); 1038 /* save the skb in buffer_info as good */ 1039 buffer_info->skb = skb; 1040 skb = new_skb; 1041 } 1042 /* else just continue with the old one */ 1043 } 1044 /* end copybreak code */ 1045 skb_put(skb, length); 1046 1047 /* Receive Checksum Offload */ 1048 e1000_rx_checksum(adapter, staterr, skb); 1049 1050 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1051 1052 e1000_receive_skb(adapter, netdev, skb, staterr, 1053 rx_desc->wb.upper.vlan); 1054 1055 next_desc: 1056 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1057 1058 /* return some buffers to hardware, one at a time is too slow */ 1059 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1060 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1061 GFP_ATOMIC); 1062 cleaned_count = 0; 1063 } 1064 1065 /* use prefetched values */ 1066 rx_desc = next_rxd; 1067 buffer_info = next_buffer; 1068 1069 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1070 } 1071 rx_ring->next_to_clean = i; 1072 1073 cleaned_count = e1000_desc_unused(rx_ring); 1074 if (cleaned_count) 1075 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1076 1077 adapter->total_rx_bytes += total_rx_bytes; 1078 adapter->total_rx_packets += total_rx_packets; 1079 return cleaned; 1080 } 1081 1082 static void e1000_put_txbuf(struct e1000_ring *tx_ring, 1083 struct e1000_buffer *buffer_info, 1084 bool drop) 1085 { 1086 struct e1000_adapter *adapter = tx_ring->adapter; 1087 1088 if (buffer_info->dma) { 1089 if (buffer_info->mapped_as_page) 1090 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma, 1091 buffer_info->length, DMA_TO_DEVICE); 1092 else 1093 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1094 buffer_info->length, DMA_TO_DEVICE); 1095 buffer_info->dma = 0; 1096 } 1097 if (buffer_info->skb) { 1098 if (drop) 1099 dev_kfree_skb_any(buffer_info->skb); 1100 else 1101 dev_consume_skb_any(buffer_info->skb); 1102 buffer_info->skb = NULL; 1103 } 1104 buffer_info->time_stamp = 0; 1105 } 1106 1107 static void e1000_print_hw_hang(struct work_struct *work) 1108 { 1109 struct e1000_adapter *adapter = container_of(work, 1110 struct e1000_adapter, 1111 print_hang_task); 1112 struct net_device *netdev = adapter->netdev; 1113 struct e1000_ring *tx_ring = adapter->tx_ring; 1114 unsigned int i = tx_ring->next_to_clean; 1115 unsigned int eop = tx_ring->buffer_info[i].next_to_watch; 1116 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop); 1117 struct e1000_hw *hw = &adapter->hw; 1118 u16 phy_status, phy_1000t_status, phy_ext_status; 1119 u16 pci_status; 1120 1121 if (test_bit(__E1000_DOWN, &adapter->state)) 1122 return; 1123 1124 if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) { 1125 /* May be block on write-back, flush and detect again 1126 * flush pending descriptor writebacks to memory 1127 */ 1128 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1129 /* execute the writes immediately */ 1130 e1e_flush(); 1131 /* Due to rare timing issues, write to TIDV again to ensure 1132 * the write is successful 1133 */ 1134 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 1135 /* execute the writes immediately */ 1136 e1e_flush(); 1137 adapter->tx_hang_recheck = true; 1138 return; 1139 } 1140 adapter->tx_hang_recheck = false; 1141 1142 if (er32(TDH(0)) == er32(TDT(0))) { 1143 e_dbg("false hang detected, ignoring\n"); 1144 return; 1145 } 1146 1147 /* Real hang detected */ 1148 netif_stop_queue(netdev); 1149 1150 e1e_rphy(hw, MII_BMSR, &phy_status); 1151 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status); 1152 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status); 1153 1154 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status); 1155 1156 /* detected Hardware unit hang */ 1157 e_err("Detected Hardware Unit Hang:\n" 1158 " TDH <%x>\n" 1159 " TDT <%x>\n" 1160 " next_to_use <%x>\n" 1161 " next_to_clean <%x>\n" 1162 "buffer_info[next_to_clean]:\n" 1163 " time_stamp <%lx>\n" 1164 " next_to_watch <%x>\n" 1165 " jiffies <%lx>\n" 1166 " next_to_watch.status <%x>\n" 1167 "MAC Status <%x>\n" 1168 "PHY Status <%x>\n" 1169 "PHY 1000BASE-T Status <%x>\n" 1170 "PHY Extended Status <%x>\n" 1171 "PCI Status <%x>\n", 1172 readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use, 1173 tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp, 1174 eop, jiffies, eop_desc->upper.fields.status, er32(STATUS), 1175 phy_status, phy_1000t_status, phy_ext_status, pci_status); 1176 1177 e1000e_dump(adapter); 1178 1179 /* Suggest workaround for known h/w issue */ 1180 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE)) 1181 e_err("Try turning off Tx pause (flow control) via ethtool\n"); 1182 } 1183 1184 /** 1185 * e1000e_tx_hwtstamp_work - check for Tx time stamp 1186 * @work: pointer to work struct 1187 * 1188 * This work function polls the TSYNCTXCTL valid bit to determine when a 1189 * timestamp has been taken for the current stored skb. The timestamp must 1190 * be for this skb because only one such packet is allowed in the queue. 1191 */ 1192 static void e1000e_tx_hwtstamp_work(struct work_struct *work) 1193 { 1194 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter, 1195 tx_hwtstamp_work); 1196 struct e1000_hw *hw = &adapter->hw; 1197 1198 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) { 1199 struct sk_buff *skb = adapter->tx_hwtstamp_skb; 1200 struct skb_shared_hwtstamps shhwtstamps; 1201 u64 txstmp; 1202 1203 txstmp = er32(TXSTMPL); 1204 txstmp |= (u64)er32(TXSTMPH) << 32; 1205 1206 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp); 1207 1208 /* Clear the global tx_hwtstamp_skb pointer and force writes 1209 * prior to notifying the stack of a Tx timestamp. 1210 */ 1211 adapter->tx_hwtstamp_skb = NULL; 1212 wmb(); /* force write prior to skb_tstamp_tx */ 1213 1214 skb_tstamp_tx(skb, &shhwtstamps); 1215 dev_consume_skb_any(skb); 1216 } else if (time_after(jiffies, adapter->tx_hwtstamp_start 1217 + adapter->tx_timeout_factor * HZ)) { 1218 dev_kfree_skb_any(adapter->tx_hwtstamp_skb); 1219 adapter->tx_hwtstamp_skb = NULL; 1220 adapter->tx_hwtstamp_timeouts++; 1221 e_warn("clearing Tx timestamp hang\n"); 1222 } else { 1223 /* reschedule to check later */ 1224 schedule_work(&adapter->tx_hwtstamp_work); 1225 } 1226 } 1227 1228 /** 1229 * e1000_clean_tx_irq - Reclaim resources after transmit completes 1230 * @tx_ring: Tx descriptor ring 1231 * 1232 * the return value indicates whether actual cleaning was done, there 1233 * is no guarantee that everything was cleaned 1234 **/ 1235 static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring) 1236 { 1237 struct e1000_adapter *adapter = tx_ring->adapter; 1238 struct net_device *netdev = adapter->netdev; 1239 struct e1000_hw *hw = &adapter->hw; 1240 struct e1000_tx_desc *tx_desc, *eop_desc; 1241 struct e1000_buffer *buffer_info; 1242 unsigned int i, eop; 1243 unsigned int count = 0; 1244 unsigned int total_tx_bytes = 0, total_tx_packets = 0; 1245 unsigned int bytes_compl = 0, pkts_compl = 0; 1246 1247 i = tx_ring->next_to_clean; 1248 eop = tx_ring->buffer_info[i].next_to_watch; 1249 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1250 1251 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) && 1252 (count < tx_ring->count)) { 1253 bool cleaned = false; 1254 1255 dma_rmb(); /* read buffer_info after eop_desc */ 1256 for (; !cleaned; count++) { 1257 tx_desc = E1000_TX_DESC(*tx_ring, i); 1258 buffer_info = &tx_ring->buffer_info[i]; 1259 cleaned = (i == eop); 1260 1261 if (cleaned) { 1262 total_tx_packets += buffer_info->segs; 1263 total_tx_bytes += buffer_info->bytecount; 1264 if (buffer_info->skb) { 1265 bytes_compl += buffer_info->skb->len; 1266 pkts_compl++; 1267 } 1268 } 1269 1270 e1000_put_txbuf(tx_ring, buffer_info, false); 1271 tx_desc->upper.data = 0; 1272 1273 i++; 1274 if (i == tx_ring->count) 1275 i = 0; 1276 } 1277 1278 if (i == tx_ring->next_to_use) 1279 break; 1280 eop = tx_ring->buffer_info[i].next_to_watch; 1281 eop_desc = E1000_TX_DESC(*tx_ring, eop); 1282 } 1283 1284 tx_ring->next_to_clean = i; 1285 1286 netdev_completed_queue(netdev, pkts_compl, bytes_compl); 1287 1288 #define TX_WAKE_THRESHOLD 32 1289 if (count && netif_carrier_ok(netdev) && 1290 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) { 1291 /* Make sure that anybody stopping the queue after this 1292 * sees the new next_to_clean. 1293 */ 1294 smp_mb(); 1295 1296 if (netif_queue_stopped(netdev) && 1297 !(test_bit(__E1000_DOWN, &adapter->state))) { 1298 netif_wake_queue(netdev); 1299 ++adapter->restart_queue; 1300 } 1301 } 1302 1303 if (adapter->detect_tx_hung) { 1304 /* Detect a transmit hang in hardware, this serializes the 1305 * check with the clearing of time_stamp and movement of i 1306 */ 1307 adapter->detect_tx_hung = false; 1308 if (tx_ring->buffer_info[i].time_stamp && 1309 time_after(jiffies, tx_ring->buffer_info[i].time_stamp 1310 + (adapter->tx_timeout_factor * HZ)) && 1311 !(er32(STATUS) & E1000_STATUS_TXOFF)) 1312 schedule_work(&adapter->print_hang_task); 1313 else 1314 adapter->tx_hang_recheck = false; 1315 } 1316 adapter->total_tx_bytes += total_tx_bytes; 1317 adapter->total_tx_packets += total_tx_packets; 1318 return count < tx_ring->count; 1319 } 1320 1321 /** 1322 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split 1323 * @rx_ring: Rx descriptor ring 1324 * @work_done: output parameter for indicating completed work 1325 * @work_to_do: how many packets we can clean 1326 * 1327 * the return value indicates whether actual cleaning was done, there 1328 * is no guarantee that everything was cleaned 1329 **/ 1330 static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done, 1331 int work_to_do) 1332 { 1333 struct e1000_adapter *adapter = rx_ring->adapter; 1334 struct e1000_hw *hw = &adapter->hw; 1335 union e1000_rx_desc_packet_split *rx_desc, *next_rxd; 1336 struct net_device *netdev = adapter->netdev; 1337 struct pci_dev *pdev = adapter->pdev; 1338 struct e1000_buffer *buffer_info, *next_buffer; 1339 struct e1000_ps_page *ps_page; 1340 struct sk_buff *skb; 1341 unsigned int i, j; 1342 u32 length, staterr; 1343 int cleaned_count = 0; 1344 bool cleaned = false; 1345 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1346 1347 i = rx_ring->next_to_clean; 1348 rx_desc = E1000_RX_DESC_PS(*rx_ring, i); 1349 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1350 buffer_info = &rx_ring->buffer_info[i]; 1351 1352 while (staterr & E1000_RXD_STAT_DD) { 1353 if (*work_done >= work_to_do) 1354 break; 1355 (*work_done)++; 1356 skb = buffer_info->skb; 1357 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1358 1359 /* in the packet split case this is header only */ 1360 prefetch(skb->data - NET_IP_ALIGN); 1361 1362 i++; 1363 if (i == rx_ring->count) 1364 i = 0; 1365 next_rxd = E1000_RX_DESC_PS(*rx_ring, i); 1366 prefetch(next_rxd); 1367 1368 next_buffer = &rx_ring->buffer_info[i]; 1369 1370 cleaned = true; 1371 cleaned_count++; 1372 dma_unmap_single(&pdev->dev, buffer_info->dma, 1373 adapter->rx_ps_bsize0, DMA_FROM_DEVICE); 1374 buffer_info->dma = 0; 1375 1376 /* see !EOP comment in other Rx routine */ 1377 if (!(staterr & E1000_RXD_STAT_EOP)) 1378 adapter->flags2 |= FLAG2_IS_DISCARDING; 1379 1380 if (adapter->flags2 & FLAG2_IS_DISCARDING) { 1381 e_dbg("Packet Split buffers didn't pick up the full packet\n"); 1382 dev_kfree_skb_irq(skb); 1383 if (staterr & E1000_RXD_STAT_EOP) 1384 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1385 goto next_desc; 1386 } 1387 1388 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1389 !(netdev->features & NETIF_F_RXALL))) { 1390 dev_kfree_skb_irq(skb); 1391 goto next_desc; 1392 } 1393 1394 length = le16_to_cpu(rx_desc->wb.middle.length0); 1395 1396 if (!length) { 1397 e_dbg("Last part of the packet spanning multiple descriptors\n"); 1398 dev_kfree_skb_irq(skb); 1399 goto next_desc; 1400 } 1401 1402 /* Good Receive */ 1403 skb_put(skb, length); 1404 1405 { 1406 /* this looks ugly, but it seems compiler issues make 1407 * it more efficient than reusing j 1408 */ 1409 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]); 1410 1411 /* page alloc/put takes too long and effects small 1412 * packet throughput, so unsplit small packets and 1413 * save the alloc/put 1414 */ 1415 if (l1 && (l1 <= copybreak) && 1416 ((length + l1) <= adapter->rx_ps_bsize0)) { 1417 ps_page = &buffer_info->ps_pages[0]; 1418 1419 dma_sync_single_for_cpu(&pdev->dev, 1420 ps_page->dma, 1421 PAGE_SIZE, 1422 DMA_FROM_DEVICE); 1423 memcpy(skb_tail_pointer(skb), 1424 page_address(ps_page->page), l1); 1425 dma_sync_single_for_device(&pdev->dev, 1426 ps_page->dma, 1427 PAGE_SIZE, 1428 DMA_FROM_DEVICE); 1429 1430 /* remove the CRC */ 1431 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1432 if (!(netdev->features & NETIF_F_RXFCS)) 1433 l1 -= 4; 1434 } 1435 1436 skb_put(skb, l1); 1437 goto copydone; 1438 } /* if */ 1439 } 1440 1441 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1442 length = le16_to_cpu(rx_desc->wb.upper.length[j]); 1443 if (!length) 1444 break; 1445 1446 ps_page = &buffer_info->ps_pages[j]; 1447 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1448 DMA_FROM_DEVICE); 1449 ps_page->dma = 0; 1450 skb_fill_page_desc(skb, j, ps_page->page, 0, length); 1451 ps_page->page = NULL; 1452 skb->len += length; 1453 skb->data_len += length; 1454 skb->truesize += PAGE_SIZE; 1455 } 1456 1457 /* strip the ethernet crc, problem is we're using pages now so 1458 * this whole operation can get a little cpu intensive 1459 */ 1460 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) { 1461 if (!(netdev->features & NETIF_F_RXFCS)) 1462 pskb_trim(skb, skb->len - 4); 1463 } 1464 1465 copydone: 1466 total_rx_bytes += skb->len; 1467 total_rx_packets++; 1468 1469 e1000_rx_checksum(adapter, staterr, skb); 1470 1471 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1472 1473 if (rx_desc->wb.upper.header_status & 1474 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP)) 1475 adapter->rx_hdr_split++; 1476 1477 e1000_receive_skb(adapter, netdev, skb, staterr, 1478 rx_desc->wb.middle.vlan); 1479 1480 next_desc: 1481 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF); 1482 buffer_info->skb = NULL; 1483 1484 /* return some buffers to hardware, one at a time is too slow */ 1485 if (cleaned_count >= E1000_RX_BUFFER_WRITE) { 1486 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1487 GFP_ATOMIC); 1488 cleaned_count = 0; 1489 } 1490 1491 /* use prefetched values */ 1492 rx_desc = next_rxd; 1493 buffer_info = next_buffer; 1494 1495 staterr = le32_to_cpu(rx_desc->wb.middle.status_error); 1496 } 1497 rx_ring->next_to_clean = i; 1498 1499 cleaned_count = e1000_desc_unused(rx_ring); 1500 if (cleaned_count) 1501 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1502 1503 adapter->total_rx_bytes += total_rx_bytes; 1504 adapter->total_rx_packets += total_rx_packets; 1505 return cleaned; 1506 } 1507 1508 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb, 1509 u16 length) 1510 { 1511 bi->page = NULL; 1512 skb->len += length; 1513 skb->data_len += length; 1514 skb->truesize += PAGE_SIZE; 1515 } 1516 1517 /** 1518 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy 1519 * @rx_ring: Rx descriptor ring 1520 * @work_done: output parameter for indicating completed work 1521 * @work_to_do: how many packets we can clean 1522 * 1523 * the return value indicates whether actual cleaning was done, there 1524 * is no guarantee that everything was cleaned 1525 **/ 1526 static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done, 1527 int work_to_do) 1528 { 1529 struct e1000_adapter *adapter = rx_ring->adapter; 1530 struct net_device *netdev = adapter->netdev; 1531 struct pci_dev *pdev = adapter->pdev; 1532 union e1000_rx_desc_extended *rx_desc, *next_rxd; 1533 struct e1000_buffer *buffer_info, *next_buffer; 1534 u32 length, staterr; 1535 unsigned int i; 1536 int cleaned_count = 0; 1537 bool cleaned = false; 1538 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1539 struct skb_shared_info *shinfo; 1540 1541 i = rx_ring->next_to_clean; 1542 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i); 1543 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1544 buffer_info = &rx_ring->buffer_info[i]; 1545 1546 while (staterr & E1000_RXD_STAT_DD) { 1547 struct sk_buff *skb; 1548 1549 if (*work_done >= work_to_do) 1550 break; 1551 (*work_done)++; 1552 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */ 1553 1554 skb = buffer_info->skb; 1555 buffer_info->skb = NULL; 1556 1557 ++i; 1558 if (i == rx_ring->count) 1559 i = 0; 1560 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i); 1561 prefetch(next_rxd); 1562 1563 next_buffer = &rx_ring->buffer_info[i]; 1564 1565 cleaned = true; 1566 cleaned_count++; 1567 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE, 1568 DMA_FROM_DEVICE); 1569 buffer_info->dma = 0; 1570 1571 length = le16_to_cpu(rx_desc->wb.upper.length); 1572 1573 /* errors is only valid for DD + EOP descriptors */ 1574 if (unlikely((staterr & E1000_RXD_STAT_EOP) && 1575 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) && 1576 !(netdev->features & NETIF_F_RXALL)))) { 1577 /* recycle both page and skb */ 1578 buffer_info->skb = skb; 1579 /* an error means any chain goes out the window too */ 1580 if (rx_ring->rx_skb_top) 1581 dev_kfree_skb_irq(rx_ring->rx_skb_top); 1582 rx_ring->rx_skb_top = NULL; 1583 goto next_desc; 1584 } 1585 #define rxtop (rx_ring->rx_skb_top) 1586 if (!(staterr & E1000_RXD_STAT_EOP)) { 1587 /* this descriptor is only the beginning (or middle) */ 1588 if (!rxtop) { 1589 /* this is the beginning of a chain */ 1590 rxtop = skb; 1591 skb_fill_page_desc(rxtop, 0, buffer_info->page, 1592 0, length); 1593 } else { 1594 /* this is the middle of a chain */ 1595 shinfo = skb_shinfo(rxtop); 1596 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1597 buffer_info->page, 0, 1598 length); 1599 /* re-use the skb, only consumed the page */ 1600 buffer_info->skb = skb; 1601 } 1602 e1000_consume_page(buffer_info, rxtop, length); 1603 goto next_desc; 1604 } else { 1605 if (rxtop) { 1606 /* end of the chain */ 1607 shinfo = skb_shinfo(rxtop); 1608 skb_fill_page_desc(rxtop, shinfo->nr_frags, 1609 buffer_info->page, 0, 1610 length); 1611 /* re-use the current skb, we only consumed the 1612 * page 1613 */ 1614 buffer_info->skb = skb; 1615 skb = rxtop; 1616 rxtop = NULL; 1617 e1000_consume_page(buffer_info, skb, length); 1618 } else { 1619 /* no chain, got EOP, this buf is the packet 1620 * copybreak to save the put_page/alloc_page 1621 */ 1622 if (length <= copybreak && 1623 skb_tailroom(skb) >= length) { 1624 memcpy(skb_tail_pointer(skb), 1625 page_address(buffer_info->page), 1626 length); 1627 /* re-use the page, so don't erase 1628 * buffer_info->page 1629 */ 1630 skb_put(skb, length); 1631 } else { 1632 skb_fill_page_desc(skb, 0, 1633 buffer_info->page, 0, 1634 length); 1635 e1000_consume_page(buffer_info, skb, 1636 length); 1637 } 1638 } 1639 } 1640 1641 /* Receive Checksum Offload */ 1642 e1000_rx_checksum(adapter, staterr, skb); 1643 1644 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb); 1645 1646 /* probably a little skewed due to removing CRC */ 1647 total_rx_bytes += skb->len; 1648 total_rx_packets++; 1649 1650 /* eth type trans needs skb->data to point to something */ 1651 if (!pskb_may_pull(skb, ETH_HLEN)) { 1652 e_err("pskb_may_pull failed.\n"); 1653 dev_kfree_skb_irq(skb); 1654 goto next_desc; 1655 } 1656 1657 e1000_receive_skb(adapter, netdev, skb, staterr, 1658 rx_desc->wb.upper.vlan); 1659 1660 next_desc: 1661 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF); 1662 1663 /* return some buffers to hardware, one at a time is too slow */ 1664 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) { 1665 adapter->alloc_rx_buf(rx_ring, cleaned_count, 1666 GFP_ATOMIC); 1667 cleaned_count = 0; 1668 } 1669 1670 /* use prefetched values */ 1671 rx_desc = next_rxd; 1672 buffer_info = next_buffer; 1673 1674 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 1675 } 1676 rx_ring->next_to_clean = i; 1677 1678 cleaned_count = e1000_desc_unused(rx_ring); 1679 if (cleaned_count) 1680 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC); 1681 1682 adapter->total_rx_bytes += total_rx_bytes; 1683 adapter->total_rx_packets += total_rx_packets; 1684 return cleaned; 1685 } 1686 1687 /** 1688 * e1000_clean_rx_ring - Free Rx Buffers per Queue 1689 * @rx_ring: Rx descriptor ring 1690 **/ 1691 static void e1000_clean_rx_ring(struct e1000_ring *rx_ring) 1692 { 1693 struct e1000_adapter *adapter = rx_ring->adapter; 1694 struct e1000_buffer *buffer_info; 1695 struct e1000_ps_page *ps_page; 1696 struct pci_dev *pdev = adapter->pdev; 1697 unsigned int i, j; 1698 1699 /* Free all the Rx ring sk_buffs */ 1700 for (i = 0; i < rx_ring->count; i++) { 1701 buffer_info = &rx_ring->buffer_info[i]; 1702 if (buffer_info->dma) { 1703 if (adapter->clean_rx == e1000_clean_rx_irq) 1704 dma_unmap_single(&pdev->dev, buffer_info->dma, 1705 adapter->rx_buffer_len, 1706 DMA_FROM_DEVICE); 1707 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) 1708 dma_unmap_page(&pdev->dev, buffer_info->dma, 1709 PAGE_SIZE, DMA_FROM_DEVICE); 1710 else if (adapter->clean_rx == e1000_clean_rx_irq_ps) 1711 dma_unmap_single(&pdev->dev, buffer_info->dma, 1712 adapter->rx_ps_bsize0, 1713 DMA_FROM_DEVICE); 1714 buffer_info->dma = 0; 1715 } 1716 1717 if (buffer_info->page) { 1718 put_page(buffer_info->page); 1719 buffer_info->page = NULL; 1720 } 1721 1722 if (buffer_info->skb) { 1723 dev_kfree_skb(buffer_info->skb); 1724 buffer_info->skb = NULL; 1725 } 1726 1727 for (j = 0; j < PS_PAGE_BUFFERS; j++) { 1728 ps_page = &buffer_info->ps_pages[j]; 1729 if (!ps_page->page) 1730 break; 1731 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE, 1732 DMA_FROM_DEVICE); 1733 ps_page->dma = 0; 1734 put_page(ps_page->page); 1735 ps_page->page = NULL; 1736 } 1737 } 1738 1739 /* there also may be some cached data from a chained receive */ 1740 if (rx_ring->rx_skb_top) { 1741 dev_kfree_skb(rx_ring->rx_skb_top); 1742 rx_ring->rx_skb_top = NULL; 1743 } 1744 1745 /* Zero out the descriptor ring */ 1746 memset(rx_ring->desc, 0, rx_ring->size); 1747 1748 rx_ring->next_to_clean = 0; 1749 rx_ring->next_to_use = 0; 1750 adapter->flags2 &= ~FLAG2_IS_DISCARDING; 1751 } 1752 1753 static void e1000e_downshift_workaround(struct work_struct *work) 1754 { 1755 struct e1000_adapter *adapter = container_of(work, 1756 struct e1000_adapter, 1757 downshift_task); 1758 1759 if (test_bit(__E1000_DOWN, &adapter->state)) 1760 return; 1761 1762 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw); 1763 } 1764 1765 /** 1766 * e1000_intr_msi - Interrupt Handler 1767 * @irq: interrupt number 1768 * @data: pointer to a network interface device structure 1769 **/ 1770 static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data) 1771 { 1772 struct net_device *netdev = data; 1773 struct e1000_adapter *adapter = netdev_priv(netdev); 1774 struct e1000_hw *hw = &adapter->hw; 1775 u32 icr = er32(ICR); 1776 1777 /* read ICR disables interrupts using IAM */ 1778 if (icr & E1000_ICR_LSC) { 1779 hw->mac.get_link_status = true; 1780 /* ICH8 workaround-- Call gig speed drop workaround on cable 1781 * disconnect (LSC) before accessing any PHY registers 1782 */ 1783 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1784 (!(er32(STATUS) & E1000_STATUS_LU))) 1785 schedule_work(&adapter->downshift_task); 1786 1787 /* 80003ES2LAN workaround-- For packet buffer work-around on 1788 * link down event; disable receives here in the ISR and reset 1789 * adapter in watchdog 1790 */ 1791 if (netif_carrier_ok(netdev) && 1792 adapter->flags & FLAG_RX_NEEDS_RESTART) { 1793 /* disable receives */ 1794 u32 rctl = er32(RCTL); 1795 1796 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1797 adapter->flags |= FLAG_RESTART_NOW; 1798 } 1799 /* guard against interrupt when we're going down */ 1800 if (!test_bit(__E1000_DOWN, &adapter->state)) 1801 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1802 } 1803 1804 /* Reset on uncorrectable ECC error */ 1805 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1806 u32 pbeccsts = er32(PBECCSTS); 1807 1808 adapter->corr_errors += 1809 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1810 adapter->uncorr_errors += 1811 FIELD_GET(E1000_PBECCSTS_UNCORR_ERR_CNT_MASK, pbeccsts); 1812 1813 /* Do the reset outside of interrupt context */ 1814 schedule_work(&adapter->reset_task); 1815 1816 /* return immediately since reset is imminent */ 1817 return IRQ_HANDLED; 1818 } 1819 1820 if (napi_schedule_prep(&adapter->napi)) { 1821 adapter->total_tx_bytes = 0; 1822 adapter->total_tx_packets = 0; 1823 adapter->total_rx_bytes = 0; 1824 adapter->total_rx_packets = 0; 1825 __napi_schedule_irqoff(&adapter->napi); 1826 } 1827 1828 return IRQ_HANDLED; 1829 } 1830 1831 /** 1832 * e1000_intr - Interrupt Handler 1833 * @irq: interrupt number 1834 * @data: pointer to a network interface device structure 1835 **/ 1836 static irqreturn_t e1000_intr(int __always_unused irq, void *data) 1837 { 1838 struct net_device *netdev = data; 1839 struct e1000_adapter *adapter = netdev_priv(netdev); 1840 struct e1000_hw *hw = &adapter->hw; 1841 u32 rctl, icr = er32(ICR); 1842 1843 if (!icr || test_bit(__E1000_DOWN, &adapter->state)) 1844 return IRQ_NONE; /* Not our interrupt */ 1845 1846 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 1847 * not set, then the adapter didn't send an interrupt 1848 */ 1849 if (!(icr & E1000_ICR_INT_ASSERTED)) 1850 return IRQ_NONE; 1851 1852 /* Interrupt Auto-Mask...upon reading ICR, 1853 * interrupts are masked. No need for the 1854 * IMC write 1855 */ 1856 1857 if (icr & E1000_ICR_LSC) { 1858 hw->mac.get_link_status = true; 1859 /* ICH8 workaround-- Call gig speed drop workaround on cable 1860 * disconnect (LSC) before accessing any PHY registers 1861 */ 1862 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) && 1863 (!(er32(STATUS) & E1000_STATUS_LU))) 1864 schedule_work(&adapter->downshift_task); 1865 1866 /* 80003ES2LAN workaround-- 1867 * For packet buffer work-around on link down event; 1868 * disable receives here in the ISR and 1869 * reset adapter in watchdog 1870 */ 1871 if (netif_carrier_ok(netdev) && 1872 (adapter->flags & FLAG_RX_NEEDS_RESTART)) { 1873 /* disable receives */ 1874 rctl = er32(RCTL); 1875 ew32(RCTL, rctl & ~E1000_RCTL_EN); 1876 adapter->flags |= FLAG_RESTART_NOW; 1877 } 1878 /* guard against interrupt when we're going down */ 1879 if (!test_bit(__E1000_DOWN, &adapter->state)) 1880 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1881 } 1882 1883 /* Reset on uncorrectable ECC error */ 1884 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) { 1885 u32 pbeccsts = er32(PBECCSTS); 1886 1887 adapter->corr_errors += 1888 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 1889 adapter->uncorr_errors += 1890 FIELD_GET(E1000_PBECCSTS_UNCORR_ERR_CNT_MASK, pbeccsts); 1891 1892 /* Do the reset outside of interrupt context */ 1893 schedule_work(&adapter->reset_task); 1894 1895 /* return immediately since reset is imminent */ 1896 return IRQ_HANDLED; 1897 } 1898 1899 if (napi_schedule_prep(&adapter->napi)) { 1900 adapter->total_tx_bytes = 0; 1901 adapter->total_tx_packets = 0; 1902 adapter->total_rx_bytes = 0; 1903 adapter->total_rx_packets = 0; 1904 __napi_schedule_irqoff(&adapter->napi); 1905 } 1906 1907 return IRQ_HANDLED; 1908 } 1909 1910 static irqreturn_t e1000_msix_other(int __always_unused irq, void *data) 1911 { 1912 struct net_device *netdev = data; 1913 struct e1000_adapter *adapter = netdev_priv(netdev); 1914 struct e1000_hw *hw = &adapter->hw; 1915 u32 icr = er32(ICR); 1916 1917 if (icr & adapter->eiac_mask) 1918 ew32(ICS, (icr & adapter->eiac_mask)); 1919 1920 if (icr & E1000_ICR_LSC) { 1921 hw->mac.get_link_status = true; 1922 /* guard against interrupt when we're going down */ 1923 if (!test_bit(__E1000_DOWN, &adapter->state)) 1924 mod_timer(&adapter->watchdog_timer, jiffies + 1); 1925 } 1926 1927 if (!test_bit(__E1000_DOWN, &adapter->state)) 1928 ew32(IMS, E1000_IMS_OTHER | IMS_OTHER_MASK); 1929 1930 return IRQ_HANDLED; 1931 } 1932 1933 static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data) 1934 { 1935 struct net_device *netdev = data; 1936 struct e1000_adapter *adapter = netdev_priv(netdev); 1937 struct e1000_hw *hw = &adapter->hw; 1938 struct e1000_ring *tx_ring = adapter->tx_ring; 1939 1940 adapter->total_tx_bytes = 0; 1941 adapter->total_tx_packets = 0; 1942 1943 if (!e1000_clean_tx_irq(tx_ring)) 1944 /* Ring was not completely cleaned, so fire another interrupt */ 1945 ew32(ICS, tx_ring->ims_val); 1946 1947 if (!test_bit(__E1000_DOWN, &adapter->state)) 1948 ew32(IMS, adapter->tx_ring->ims_val); 1949 1950 return IRQ_HANDLED; 1951 } 1952 1953 static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data) 1954 { 1955 struct net_device *netdev = data; 1956 struct e1000_adapter *adapter = netdev_priv(netdev); 1957 struct e1000_ring *rx_ring = adapter->rx_ring; 1958 1959 /* Write the ITR value calculated at the end of the 1960 * previous interrupt. 1961 */ 1962 if (rx_ring->set_itr) { 1963 u32 itr = rx_ring->itr_val ? 1964 1000000000 / (rx_ring->itr_val * 256) : 0; 1965 1966 writel(itr, rx_ring->itr_register); 1967 rx_ring->set_itr = 0; 1968 } 1969 1970 if (napi_schedule_prep(&adapter->napi)) { 1971 adapter->total_rx_bytes = 0; 1972 adapter->total_rx_packets = 0; 1973 __napi_schedule_irqoff(&adapter->napi); 1974 } 1975 return IRQ_HANDLED; 1976 } 1977 1978 /** 1979 * e1000_configure_msix - Configure MSI-X hardware 1980 * @adapter: board private structure 1981 * 1982 * e1000_configure_msix sets up the hardware to properly 1983 * generate MSI-X interrupts. 1984 **/ 1985 static void e1000_configure_msix(struct e1000_adapter *adapter) 1986 { 1987 struct e1000_hw *hw = &adapter->hw; 1988 struct e1000_ring *rx_ring = adapter->rx_ring; 1989 struct e1000_ring *tx_ring = adapter->tx_ring; 1990 int vector = 0; 1991 u32 ctrl_ext, ivar = 0; 1992 1993 adapter->eiac_mask = 0; 1994 1995 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */ 1996 if (hw->mac.type == e1000_82574) { 1997 u32 rfctl = er32(RFCTL); 1998 1999 rfctl |= E1000_RFCTL_ACK_DIS; 2000 ew32(RFCTL, rfctl); 2001 } 2002 2003 /* Configure Rx vector */ 2004 rx_ring->ims_val = E1000_IMS_RXQ0; 2005 adapter->eiac_mask |= rx_ring->ims_val; 2006 if (rx_ring->itr_val) 2007 writel(1000000000 / (rx_ring->itr_val * 256), 2008 rx_ring->itr_register); 2009 else 2010 writel(1, rx_ring->itr_register); 2011 ivar = E1000_IVAR_INT_ALLOC_VALID | vector; 2012 2013 /* Configure Tx vector */ 2014 tx_ring->ims_val = E1000_IMS_TXQ0; 2015 vector++; 2016 if (tx_ring->itr_val) 2017 writel(1000000000 / (tx_ring->itr_val * 256), 2018 tx_ring->itr_register); 2019 else 2020 writel(1, tx_ring->itr_register); 2021 adapter->eiac_mask |= tx_ring->ims_val; 2022 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8); 2023 2024 /* set vector for Other Causes, e.g. link changes */ 2025 vector++; 2026 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16); 2027 if (rx_ring->itr_val) 2028 writel(1000000000 / (rx_ring->itr_val * 256), 2029 hw->hw_addr + E1000_EITR_82574(vector)); 2030 else 2031 writel(1, hw->hw_addr + E1000_EITR_82574(vector)); 2032 2033 /* Cause Tx interrupts on every write back */ 2034 ivar |= BIT(31); 2035 2036 ew32(IVAR, ivar); 2037 2038 /* enable MSI-X PBA support */ 2039 ctrl_ext = er32(CTRL_EXT) & ~E1000_CTRL_EXT_IAME; 2040 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR | E1000_CTRL_EXT_EIAME; 2041 ew32(CTRL_EXT, ctrl_ext); 2042 e1e_flush(); 2043 } 2044 2045 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter) 2046 { 2047 if (adapter->msix_entries) { 2048 pci_disable_msix(adapter->pdev); 2049 kfree(adapter->msix_entries); 2050 adapter->msix_entries = NULL; 2051 } else if (adapter->flags & FLAG_MSI_ENABLED) { 2052 pci_disable_msi(adapter->pdev); 2053 adapter->flags &= ~FLAG_MSI_ENABLED; 2054 } 2055 } 2056 2057 /** 2058 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported 2059 * @adapter: board private structure 2060 * 2061 * Attempt to configure interrupts using the best available 2062 * capabilities of the hardware and kernel. 2063 **/ 2064 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter) 2065 { 2066 int err; 2067 int i; 2068 2069 switch (adapter->int_mode) { 2070 case E1000E_INT_MODE_MSIX: 2071 if (adapter->flags & FLAG_HAS_MSIX) { 2072 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */ 2073 adapter->msix_entries = kzalloc_objs(struct msix_entry, 2074 adapter->num_vectors); 2075 if (adapter->msix_entries) { 2076 struct e1000_adapter *a = adapter; 2077 2078 for (i = 0; i < adapter->num_vectors; i++) 2079 adapter->msix_entries[i].entry = i; 2080 2081 err = pci_enable_msix_range(a->pdev, 2082 a->msix_entries, 2083 a->num_vectors, 2084 a->num_vectors); 2085 if (err > 0) 2086 return; 2087 } 2088 /* MSI-X failed, so fall through and try MSI */ 2089 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n"); 2090 e1000e_reset_interrupt_capability(adapter); 2091 } 2092 adapter->int_mode = E1000E_INT_MODE_MSI; 2093 fallthrough; 2094 case E1000E_INT_MODE_MSI: 2095 if (!pci_enable_msi(adapter->pdev)) { 2096 adapter->flags |= FLAG_MSI_ENABLED; 2097 } else { 2098 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2099 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n"); 2100 } 2101 fallthrough; 2102 case E1000E_INT_MODE_LEGACY: 2103 /* Don't do anything; this is the system default */ 2104 break; 2105 } 2106 2107 /* store the number of vectors being used */ 2108 adapter->num_vectors = 1; 2109 } 2110 2111 /** 2112 * e1000_request_msix - Initialize MSI-X interrupts 2113 * @adapter: board private structure 2114 * 2115 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the 2116 * kernel. 2117 **/ 2118 static int e1000_request_msix(struct e1000_adapter *adapter) 2119 { 2120 struct net_device *netdev = adapter->netdev; 2121 int err = 0, vector = 0, i; 2122 2123 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2124 snprintf(adapter->rx_ring->name, 2125 sizeof(adapter->rx_ring->name) - 1, 2126 "%.14s-rx-0", netdev->name); 2127 else 2128 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ); 2129 err = request_irq(adapter->msix_entries[vector].vector, 2130 e1000_intr_msix_rx, 0, adapter->rx_ring->name, 2131 netdev); 2132 if (err) 2133 goto err_free; 2134 adapter->rx_ring->itr_register = adapter->hw.hw_addr + 2135 E1000_EITR_82574(vector); 2136 adapter->rx_ring->itr_val = adapter->itr; 2137 vector++; 2138 2139 if (strlen(netdev->name) < (IFNAMSIZ - 5)) 2140 snprintf(adapter->tx_ring->name, 2141 sizeof(adapter->tx_ring->name) - 1, 2142 "%.14s-tx-0", netdev->name); 2143 else 2144 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ); 2145 err = request_irq(adapter->msix_entries[vector].vector, 2146 e1000_intr_msix_tx, 0, adapter->tx_ring->name, 2147 netdev); 2148 if (err) 2149 goto err_free; 2150 adapter->tx_ring->itr_register = adapter->hw.hw_addr + 2151 E1000_EITR_82574(vector); 2152 adapter->tx_ring->itr_val = adapter->itr; 2153 vector++; 2154 2155 err = request_irq(adapter->msix_entries[vector].vector, 2156 e1000_msix_other, 0, netdev->name, netdev); 2157 if (err) 2158 goto err_free; 2159 2160 e1000_configure_msix(adapter); 2161 2162 return 0; 2163 2164 err_free: 2165 for (i = vector - 1; i >= 0; i--) 2166 free_irq(adapter->msix_entries[i].vector, netdev); 2167 return err; 2168 } 2169 2170 /** 2171 * e1000_request_irq - initialize interrupts 2172 * @adapter: board private structure 2173 * 2174 * Attempts to configure interrupts using the best available 2175 * capabilities of the hardware and kernel. 2176 **/ 2177 static int e1000_request_irq(struct e1000_adapter *adapter) 2178 { 2179 struct net_device *netdev = adapter->netdev; 2180 int err; 2181 2182 if (adapter->msix_entries) { 2183 err = e1000_request_msix(adapter); 2184 if (!err) 2185 return err; 2186 /* fall back to MSI */ 2187 e1000e_reset_interrupt_capability(adapter); 2188 adapter->int_mode = E1000E_INT_MODE_MSI; 2189 e1000e_set_interrupt_capability(adapter); 2190 } 2191 if (adapter->flags & FLAG_MSI_ENABLED) { 2192 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0, 2193 netdev->name, netdev); 2194 if (!err) 2195 return err; 2196 2197 /* fall back to legacy interrupt */ 2198 e1000e_reset_interrupt_capability(adapter); 2199 adapter->int_mode = E1000E_INT_MODE_LEGACY; 2200 } 2201 2202 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED, 2203 netdev->name, netdev); 2204 if (err) 2205 e_err("Unable to allocate interrupt, Error: %d\n", err); 2206 2207 return err; 2208 } 2209 2210 static void e1000_free_irq(struct e1000_adapter *adapter) 2211 { 2212 struct net_device *netdev = adapter->netdev; 2213 2214 if (adapter->msix_entries) { 2215 int vector = 0; 2216 2217 free_irq(adapter->msix_entries[vector].vector, netdev); 2218 vector++; 2219 2220 free_irq(adapter->msix_entries[vector].vector, netdev); 2221 vector++; 2222 2223 /* Other Causes interrupt vector */ 2224 free_irq(adapter->msix_entries[vector].vector, netdev); 2225 return; 2226 } 2227 2228 free_irq(adapter->pdev->irq, netdev); 2229 } 2230 2231 /** 2232 * e1000_irq_disable - Mask off interrupt generation on the NIC 2233 * @adapter: board private structure 2234 **/ 2235 static void e1000_irq_disable(struct e1000_adapter *adapter) 2236 { 2237 struct e1000_hw *hw = &adapter->hw; 2238 2239 ew32(IMC, ~0); 2240 if (adapter->msix_entries) 2241 ew32(EIAC_82574, 0); 2242 e1e_flush(); 2243 2244 if (adapter->msix_entries) { 2245 int i; 2246 2247 for (i = 0; i < adapter->num_vectors; i++) 2248 synchronize_irq(adapter->msix_entries[i].vector); 2249 } else { 2250 synchronize_irq(adapter->pdev->irq); 2251 } 2252 } 2253 2254 /** 2255 * e1000_irq_enable - Enable default interrupt generation settings 2256 * @adapter: board private structure 2257 **/ 2258 static void e1000_irq_enable(struct e1000_adapter *adapter) 2259 { 2260 struct e1000_hw *hw = &adapter->hw; 2261 2262 if (adapter->msix_entries) { 2263 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574); 2264 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | 2265 IMS_OTHER_MASK); 2266 } else if (hw->mac.type >= e1000_pch_lpt) { 2267 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER); 2268 } else { 2269 ew32(IMS, IMS_ENABLE_MASK); 2270 } 2271 e1e_flush(); 2272 } 2273 2274 /** 2275 * e1000e_get_hw_control - get control of the h/w from f/w 2276 * @adapter: address of board private structure 2277 * 2278 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2279 * For ASF and Pass Through versions of f/w this means that 2280 * the driver is loaded. For AMT version (only with 82573) 2281 * of the f/w this means that the network i/f is open. 2282 **/ 2283 void e1000e_get_hw_control(struct e1000_adapter *adapter) 2284 { 2285 struct e1000_hw *hw = &adapter->hw; 2286 u32 ctrl_ext; 2287 u32 swsm; 2288 2289 /* Let firmware know the driver has taken over */ 2290 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2291 swsm = er32(SWSM); 2292 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD); 2293 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2294 ctrl_ext = er32(CTRL_EXT); 2295 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 2296 } 2297 } 2298 2299 /** 2300 * e1000e_release_hw_control - release control of the h/w to f/w 2301 * @adapter: address of board private structure 2302 * 2303 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit. 2304 * For ASF and Pass Through versions of f/w this means that the 2305 * driver is no longer loaded. For AMT version (only with 82573) i 2306 * of the f/w this means that the network i/f is closed. 2307 * 2308 **/ 2309 void e1000e_release_hw_control(struct e1000_adapter *adapter) 2310 { 2311 struct e1000_hw *hw = &adapter->hw; 2312 u32 ctrl_ext; 2313 u32 swsm; 2314 2315 /* Let firmware taken over control of h/w */ 2316 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) { 2317 swsm = er32(SWSM); 2318 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD); 2319 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) { 2320 ctrl_ext = er32(CTRL_EXT); 2321 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 2322 } 2323 } 2324 2325 /** 2326 * e1000_alloc_ring_dma - allocate memory for a ring structure 2327 * @adapter: board private structure 2328 * @ring: ring struct for which to allocate dma 2329 **/ 2330 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter, 2331 struct e1000_ring *ring) 2332 { 2333 struct pci_dev *pdev = adapter->pdev; 2334 2335 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma, 2336 GFP_KERNEL); 2337 if (!ring->desc) 2338 return -ENOMEM; 2339 2340 return 0; 2341 } 2342 2343 /** 2344 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors) 2345 * @tx_ring: Tx descriptor ring 2346 * 2347 * Return 0 on success, negative on failure 2348 **/ 2349 int e1000e_setup_tx_resources(struct e1000_ring *tx_ring) 2350 { 2351 struct e1000_adapter *adapter = tx_ring->adapter; 2352 int err = -ENOMEM, size; 2353 2354 size = sizeof(struct e1000_buffer) * tx_ring->count; 2355 tx_ring->buffer_info = vzalloc(size); 2356 if (!tx_ring->buffer_info) 2357 goto err; 2358 2359 /* round up to nearest 4K */ 2360 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc); 2361 tx_ring->size = ALIGN(tx_ring->size, 4096); 2362 2363 err = e1000_alloc_ring_dma(adapter, tx_ring); 2364 if (err) 2365 goto err; 2366 2367 tx_ring->next_to_use = 0; 2368 tx_ring->next_to_clean = 0; 2369 2370 return 0; 2371 err: 2372 vfree(tx_ring->buffer_info); 2373 e_err("Unable to allocate memory for the transmit descriptor ring\n"); 2374 return err; 2375 } 2376 2377 /** 2378 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors) 2379 * @rx_ring: Rx descriptor ring 2380 * 2381 * Returns 0 on success, negative on failure 2382 **/ 2383 int e1000e_setup_rx_resources(struct e1000_ring *rx_ring) 2384 { 2385 struct e1000_adapter *adapter = rx_ring->adapter; 2386 struct e1000_buffer *buffer_info; 2387 int i, size, desc_len, err = -ENOMEM; 2388 2389 size = sizeof(struct e1000_buffer) * rx_ring->count; 2390 rx_ring->buffer_info = vzalloc(size); 2391 if (!rx_ring->buffer_info) 2392 goto err; 2393 2394 for (i = 0; i < rx_ring->count; i++) { 2395 buffer_info = &rx_ring->buffer_info[i]; 2396 buffer_info->ps_pages = kzalloc_objs(struct e1000_ps_page, 2397 PS_PAGE_BUFFERS); 2398 if (!buffer_info->ps_pages) 2399 goto err_pages; 2400 } 2401 2402 desc_len = sizeof(union e1000_rx_desc_packet_split); 2403 2404 /* Round up to nearest 4K */ 2405 rx_ring->size = rx_ring->count * desc_len; 2406 rx_ring->size = ALIGN(rx_ring->size, 4096); 2407 2408 err = e1000_alloc_ring_dma(adapter, rx_ring); 2409 if (err) 2410 goto err_pages; 2411 2412 rx_ring->next_to_clean = 0; 2413 rx_ring->next_to_use = 0; 2414 rx_ring->rx_skb_top = NULL; 2415 2416 return 0; 2417 2418 err_pages: 2419 for (i = 0; i < rx_ring->count; i++) { 2420 buffer_info = &rx_ring->buffer_info[i]; 2421 kfree(buffer_info->ps_pages); 2422 } 2423 err: 2424 vfree(rx_ring->buffer_info); 2425 e_err("Unable to allocate memory for the receive descriptor ring\n"); 2426 return err; 2427 } 2428 2429 /** 2430 * e1000_clean_tx_ring - Free Tx Buffers 2431 * @tx_ring: Tx descriptor ring 2432 **/ 2433 static void e1000_clean_tx_ring(struct e1000_ring *tx_ring) 2434 { 2435 struct e1000_adapter *adapter = tx_ring->adapter; 2436 struct e1000_buffer *buffer_info; 2437 unsigned long size; 2438 unsigned int i; 2439 2440 for (i = 0; i < tx_ring->count; i++) { 2441 buffer_info = &tx_ring->buffer_info[i]; 2442 e1000_put_txbuf(tx_ring, buffer_info, false); 2443 } 2444 2445 netdev_reset_queue(adapter->netdev); 2446 size = sizeof(struct e1000_buffer) * tx_ring->count; 2447 memset(tx_ring->buffer_info, 0, size); 2448 2449 memset(tx_ring->desc, 0, tx_ring->size); 2450 2451 tx_ring->next_to_use = 0; 2452 tx_ring->next_to_clean = 0; 2453 } 2454 2455 /** 2456 * e1000e_free_tx_resources - Free Tx Resources per Queue 2457 * @tx_ring: Tx descriptor ring 2458 * 2459 * Free all transmit software resources 2460 **/ 2461 void e1000e_free_tx_resources(struct e1000_ring *tx_ring) 2462 { 2463 struct e1000_adapter *adapter = tx_ring->adapter; 2464 struct pci_dev *pdev = adapter->pdev; 2465 2466 e1000_clean_tx_ring(tx_ring); 2467 2468 vfree(tx_ring->buffer_info); 2469 tx_ring->buffer_info = NULL; 2470 2471 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2472 tx_ring->dma); 2473 tx_ring->desc = NULL; 2474 } 2475 2476 /** 2477 * e1000e_free_rx_resources - Free Rx Resources 2478 * @rx_ring: Rx descriptor ring 2479 * 2480 * Free all receive software resources 2481 **/ 2482 void e1000e_free_rx_resources(struct e1000_ring *rx_ring) 2483 { 2484 struct e1000_adapter *adapter = rx_ring->adapter; 2485 struct pci_dev *pdev = adapter->pdev; 2486 int i; 2487 2488 e1000_clean_rx_ring(rx_ring); 2489 2490 for (i = 0; i < rx_ring->count; i++) 2491 kfree(rx_ring->buffer_info[i].ps_pages); 2492 2493 vfree(rx_ring->buffer_info); 2494 rx_ring->buffer_info = NULL; 2495 2496 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2497 rx_ring->dma); 2498 rx_ring->desc = NULL; 2499 } 2500 2501 /** 2502 * e1000_update_itr - update the dynamic ITR value based on statistics 2503 * @itr_setting: current adapter->itr 2504 * @packets: the number of packets during this measurement interval 2505 * @bytes: the number of bytes during this measurement interval 2506 * 2507 * Stores a new ITR value based on packets and byte 2508 * counts during the last interrupt. The advantage of per interrupt 2509 * computation is faster updates and more accurate ITR for the current 2510 * traffic pattern. Constants in this function were computed 2511 * based on theoretical maximum wire speed and thresholds were set based 2512 * on testing data as well as attempting to minimize response time 2513 * while increasing bulk throughput. This functionality is controlled 2514 * by the InterruptThrottleRate module parameter. 2515 **/ 2516 static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes) 2517 { 2518 unsigned int retval = itr_setting; 2519 2520 if (packets == 0) 2521 return itr_setting; 2522 2523 switch (itr_setting) { 2524 case lowest_latency: 2525 /* handle TSO and jumbo frames */ 2526 if (bytes / packets > 8000) 2527 retval = bulk_latency; 2528 else if ((packets < 5) && (bytes > 512)) 2529 retval = low_latency; 2530 break; 2531 case low_latency: /* 50 usec aka 20000 ints/s */ 2532 if (bytes > 10000) { 2533 /* this if handles the TSO accounting */ 2534 if (bytes / packets > 8000) 2535 retval = bulk_latency; 2536 else if ((packets < 10) || ((bytes / packets) > 1200)) 2537 retval = bulk_latency; 2538 else if ((packets > 35)) 2539 retval = lowest_latency; 2540 } else if (bytes / packets > 2000) { 2541 retval = bulk_latency; 2542 } else if (packets <= 2 && bytes < 512) { 2543 retval = lowest_latency; 2544 } 2545 break; 2546 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2547 if (bytes > 25000) { 2548 if (packets > 35) 2549 retval = low_latency; 2550 } else if (bytes < 6000) { 2551 retval = low_latency; 2552 } 2553 break; 2554 } 2555 2556 return retval; 2557 } 2558 2559 static void e1000_set_itr(struct e1000_adapter *adapter) 2560 { 2561 u16 current_itr; 2562 u32 new_itr = adapter->itr; 2563 2564 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 2565 if (adapter->link_speed != SPEED_1000) { 2566 new_itr = 4000; 2567 goto set_itr_now; 2568 } 2569 2570 if (adapter->flags2 & FLAG2_DISABLE_AIM) { 2571 new_itr = 0; 2572 goto set_itr_now; 2573 } 2574 2575 adapter->tx_itr = e1000_update_itr(adapter->tx_itr, 2576 adapter->total_tx_packets, 2577 adapter->total_tx_bytes); 2578 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2579 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency) 2580 adapter->tx_itr = low_latency; 2581 2582 adapter->rx_itr = e1000_update_itr(adapter->rx_itr, 2583 adapter->total_rx_packets, 2584 adapter->total_rx_bytes); 2585 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2586 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency) 2587 adapter->rx_itr = low_latency; 2588 2589 current_itr = max(adapter->rx_itr, adapter->tx_itr); 2590 2591 /* counts and packets in update_itr are dependent on these numbers */ 2592 switch (current_itr) { 2593 case lowest_latency: 2594 new_itr = 70000; 2595 break; 2596 case low_latency: 2597 new_itr = 20000; /* aka hwitr = ~200 */ 2598 break; 2599 case bulk_latency: 2600 new_itr = 4000; 2601 break; 2602 default: 2603 break; 2604 } 2605 2606 set_itr_now: 2607 if (new_itr != adapter->itr) { 2608 /* this attempts to bias the interrupt rate towards Bulk 2609 * by adding intermediate steps when interrupt rate is 2610 * increasing 2611 */ 2612 new_itr = new_itr > adapter->itr ? 2613 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr; 2614 adapter->itr = new_itr; 2615 adapter->rx_ring->itr_val = new_itr; 2616 if (adapter->msix_entries) 2617 adapter->rx_ring->set_itr = 1; 2618 else 2619 e1000e_write_itr(adapter, new_itr); 2620 } 2621 } 2622 2623 /** 2624 * e1000e_write_itr - write the ITR value to the appropriate registers 2625 * @adapter: address of board private structure 2626 * @itr: new ITR value to program 2627 * 2628 * e1000e_write_itr determines if the adapter is in MSI-X mode 2629 * and, if so, writes the EITR registers with the ITR value. 2630 * Otherwise, it writes the ITR value into the ITR register. 2631 **/ 2632 void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr) 2633 { 2634 struct e1000_hw *hw = &adapter->hw; 2635 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0; 2636 2637 if (adapter->msix_entries) { 2638 int vector; 2639 2640 for (vector = 0; vector < adapter->num_vectors; vector++) 2641 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector)); 2642 } else { 2643 ew32(ITR, new_itr); 2644 } 2645 } 2646 2647 /** 2648 * e1000_alloc_queues - Allocate memory for all rings 2649 * @adapter: board private structure to initialize 2650 **/ 2651 static int e1000_alloc_queues(struct e1000_adapter *adapter) 2652 { 2653 int size = sizeof(struct e1000_ring); 2654 2655 adapter->tx_ring = kzalloc(size, GFP_KERNEL); 2656 if (!adapter->tx_ring) 2657 goto err; 2658 adapter->tx_ring->count = adapter->tx_ring_count; 2659 adapter->tx_ring->adapter = adapter; 2660 2661 adapter->rx_ring = kzalloc(size, GFP_KERNEL); 2662 if (!adapter->rx_ring) 2663 goto err; 2664 adapter->rx_ring->count = adapter->rx_ring_count; 2665 adapter->rx_ring->adapter = adapter; 2666 2667 return 0; 2668 err: 2669 e_err("Unable to allocate memory for queues\n"); 2670 kfree(adapter->rx_ring); 2671 kfree(adapter->tx_ring); 2672 return -ENOMEM; 2673 } 2674 2675 /** 2676 * e1000e_poll - NAPI Rx polling callback 2677 * @napi: struct associated with this polling callback 2678 * @budget: number of packets driver is allowed to process this poll 2679 **/ 2680 static int e1000e_poll(struct napi_struct *napi, int budget) 2681 { 2682 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter, 2683 napi); 2684 struct e1000_hw *hw = &adapter->hw; 2685 struct net_device *poll_dev = adapter->netdev; 2686 int tx_cleaned = 1, work_done = 0; 2687 2688 adapter = netdev_priv(poll_dev); 2689 2690 if (!adapter->msix_entries || 2691 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val)) 2692 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring); 2693 2694 adapter->clean_rx(adapter->rx_ring, &work_done, budget); 2695 2696 if (!tx_cleaned || work_done == budget) 2697 return budget; 2698 2699 /* Exit the polling mode, but don't re-enable interrupts if stack might 2700 * poll us due to busy-polling 2701 */ 2702 if (likely(napi_complete_done(napi, work_done))) { 2703 if (adapter->itr_setting & 3) 2704 e1000_set_itr(adapter); 2705 if (!test_bit(__E1000_DOWN, &adapter->state)) { 2706 if (adapter->msix_entries) 2707 ew32(IMS, adapter->rx_ring->ims_val); 2708 else 2709 e1000_irq_enable(adapter); 2710 } 2711 } 2712 2713 return work_done; 2714 } 2715 2716 static int e1000_vlan_rx_add_vid(struct net_device *netdev, 2717 __always_unused __be16 proto, u16 vid) 2718 { 2719 struct e1000_adapter *adapter = netdev_priv(netdev); 2720 struct e1000_hw *hw = &adapter->hw; 2721 u32 vfta, index; 2722 2723 /* don't update vlan cookie if already programmed */ 2724 if ((adapter->hw.mng_cookie.status & 2725 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2726 (vid == adapter->mng_vlan_id)) 2727 return 0; 2728 2729 /* add VID to filter table */ 2730 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2731 index = (vid >> 5) & 0x7F; 2732 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2733 vfta |= BIT((vid & 0x1F)); 2734 hw->mac.ops.write_vfta(hw, index, vfta); 2735 } 2736 2737 set_bit(vid, adapter->active_vlans); 2738 2739 return 0; 2740 } 2741 2742 static int e1000_vlan_rx_kill_vid(struct net_device *netdev, 2743 __always_unused __be16 proto, u16 vid) 2744 { 2745 struct e1000_adapter *adapter = netdev_priv(netdev); 2746 struct e1000_hw *hw = &adapter->hw; 2747 u32 vfta, index; 2748 2749 if ((adapter->hw.mng_cookie.status & 2750 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) && 2751 (vid == adapter->mng_vlan_id)) { 2752 /* release control to f/w */ 2753 e1000e_release_hw_control(adapter); 2754 return 0; 2755 } 2756 2757 /* remove VID from filter table */ 2758 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2759 index = (vid >> 5) & 0x7F; 2760 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index); 2761 vfta &= ~BIT((vid & 0x1F)); 2762 hw->mac.ops.write_vfta(hw, index, vfta); 2763 } 2764 2765 clear_bit(vid, adapter->active_vlans); 2766 2767 return 0; 2768 } 2769 2770 /** 2771 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering 2772 * @adapter: board private structure to initialize 2773 **/ 2774 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter) 2775 { 2776 struct net_device *netdev = adapter->netdev; 2777 struct e1000_hw *hw = &adapter->hw; 2778 u32 rctl; 2779 2780 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2781 /* disable VLAN receive filtering */ 2782 rctl = er32(RCTL); 2783 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); 2784 ew32(RCTL, rctl); 2785 2786 if (adapter->mng_vlan_id != E1000_MNG_VLAN_NONE) { 2787 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 2788 adapter->mng_vlan_id); 2789 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 2790 } 2791 } 2792 } 2793 2794 /** 2795 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering 2796 * @adapter: board private structure to initialize 2797 **/ 2798 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter) 2799 { 2800 struct e1000_hw *hw = &adapter->hw; 2801 u32 rctl; 2802 2803 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) { 2804 /* enable VLAN receive filtering */ 2805 rctl = er32(RCTL); 2806 rctl |= E1000_RCTL_VFE; 2807 rctl &= ~E1000_RCTL_CFIEN; 2808 ew32(RCTL, rctl); 2809 } 2810 } 2811 2812 /** 2813 * e1000e_vlan_strip_disable - helper to disable HW VLAN stripping 2814 * @adapter: board private structure to initialize 2815 **/ 2816 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter) 2817 { 2818 struct e1000_hw *hw = &adapter->hw; 2819 u32 ctrl; 2820 2821 /* disable VLAN tag insert/strip */ 2822 ctrl = er32(CTRL); 2823 ctrl &= ~E1000_CTRL_VME; 2824 ew32(CTRL, ctrl); 2825 } 2826 2827 /** 2828 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping 2829 * @adapter: board private structure to initialize 2830 **/ 2831 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter) 2832 { 2833 struct e1000_hw *hw = &adapter->hw; 2834 u32 ctrl; 2835 2836 /* enable VLAN tag insert/strip */ 2837 ctrl = er32(CTRL); 2838 ctrl |= E1000_CTRL_VME; 2839 ew32(CTRL, ctrl); 2840 } 2841 2842 static void e1000_update_mng_vlan(struct e1000_adapter *adapter) 2843 { 2844 struct net_device *netdev = adapter->netdev; 2845 u16 vid = adapter->hw.mng_cookie.vlan_id; 2846 u16 old_vid = adapter->mng_vlan_id; 2847 2848 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { 2849 e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid); 2850 adapter->mng_vlan_id = vid; 2851 } 2852 2853 if (old_vid != E1000_MNG_VLAN_NONE && vid != old_vid) 2854 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); 2855 } 2856 2857 static void e1000_restore_vlan(struct e1000_adapter *adapter) 2858 { 2859 u16 vid; 2860 2861 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); 2862 2863 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2864 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 2865 } 2866 2867 static void e1000_init_manageability_pt(struct e1000_adapter *adapter) 2868 { 2869 struct e1000_hw *hw = &adapter->hw; 2870 u32 manc, manc2h, mdef, i, j; 2871 2872 if (!(adapter->flags & FLAG_MNG_PT_ENABLED)) 2873 return; 2874 2875 manc = er32(MANC); 2876 2877 /* enable receiving management packets to the host. this will probably 2878 * generate destination unreachable messages from the host OS, but 2879 * the packets will be handled on SMBUS 2880 */ 2881 manc |= E1000_MANC_EN_MNG2HOST; 2882 manc2h = er32(MANC2H); 2883 2884 switch (hw->mac.type) { 2885 default: 2886 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664); 2887 break; 2888 case e1000_82574: 2889 case e1000_82583: 2890 /* Check if IPMI pass-through decision filter already exists; 2891 * if so, enable it. 2892 */ 2893 for (i = 0, j = 0; i < 8; i++) { 2894 mdef = er32(MDEF(i)); 2895 2896 /* Ignore filters with anything other than IPMI ports */ 2897 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2898 continue; 2899 2900 /* Enable this decision filter in MANC2H */ 2901 if (mdef) 2902 manc2h |= BIT(i); 2903 2904 j |= mdef; 2905 } 2906 2907 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664)) 2908 break; 2909 2910 /* Create new decision filter in an empty filter */ 2911 for (i = 0, j = 0; i < 8; i++) 2912 if (er32(MDEF(i)) == 0) { 2913 ew32(MDEF(i), (E1000_MDEF_PORT_623 | 2914 E1000_MDEF_PORT_664)); 2915 manc2h |= BIT(1); 2916 j++; 2917 break; 2918 } 2919 2920 if (!j) 2921 e_warn("Unable to create IPMI pass-through filter\n"); 2922 break; 2923 } 2924 2925 ew32(MANC2H, manc2h); 2926 ew32(MANC, manc); 2927 } 2928 2929 /** 2930 * e1000_configure_tx - Configure Transmit Unit after Reset 2931 * @adapter: board private structure 2932 * 2933 * Configure the Tx unit of the MAC after a reset. 2934 **/ 2935 static void e1000_configure_tx(struct e1000_adapter *adapter) 2936 { 2937 struct e1000_hw *hw = &adapter->hw; 2938 struct e1000_ring *tx_ring = adapter->tx_ring; 2939 u64 tdba; 2940 u32 tdlen, tctl, tarc; 2941 2942 /* Setup the HW Tx Head and Tail descriptor pointers */ 2943 tdba = tx_ring->dma; 2944 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc); 2945 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32))); 2946 ew32(TDBAH(0), (tdba >> 32)); 2947 ew32(TDLEN(0), tdlen); 2948 ew32(TDH(0), 0); 2949 ew32(TDT(0), 0); 2950 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0); 2951 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0); 2952 2953 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 2954 e1000e_update_tdt_wa(tx_ring, 0); 2955 2956 /* Set the Tx Interrupt Delay register */ 2957 ew32(TIDV, adapter->tx_int_delay); 2958 /* Tx irq moderation */ 2959 ew32(TADV, adapter->tx_abs_int_delay); 2960 2961 if (adapter->flags2 & FLAG2_DMA_BURST) { 2962 u32 txdctl = er32(TXDCTL(0)); 2963 2964 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH | 2965 E1000_TXDCTL_WTHRESH); 2966 /* set up some performance related parameters to encourage the 2967 * hardware to use the bus more efficiently in bursts, depends 2968 * on the tx_int_delay to be enabled, 2969 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls 2970 * hthresh = 1 ==> prefetch when one or more available 2971 * pthresh = 0x1f ==> prefetch if internal cache 31 or less 2972 * BEWARE: this seems to work but should be considered first if 2973 * there are Tx hangs or other Tx related bugs 2974 */ 2975 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE; 2976 ew32(TXDCTL(0), txdctl); 2977 } 2978 /* erratum work around: set txdctl the same for both queues */ 2979 ew32(TXDCTL(1), er32(TXDCTL(0))); 2980 2981 /* Program the Transmit Control Register */ 2982 tctl = er32(TCTL); 2983 tctl &= ~E1000_TCTL_CT; 2984 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2985 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2986 2987 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) { 2988 tarc = er32(TARC(0)); 2989 /* set the speed mode bit, we'll clear it if we're not at 2990 * gigabit link later 2991 */ 2992 #define SPEED_MODE_BIT BIT(21) 2993 tarc |= SPEED_MODE_BIT; 2994 ew32(TARC(0), tarc); 2995 } 2996 2997 /* errata: program both queues to unweighted RR */ 2998 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) { 2999 tarc = er32(TARC(0)); 3000 tarc |= 1; 3001 ew32(TARC(0), tarc); 3002 tarc = er32(TARC(1)); 3003 tarc |= 1; 3004 ew32(TARC(1), tarc); 3005 } 3006 3007 /* Setup Transmit Descriptor Settings for eop descriptor */ 3008 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS; 3009 3010 /* only set IDE if we are delaying interrupts using the timers */ 3011 if (adapter->tx_int_delay) 3012 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3013 3014 /* enable Report Status bit */ 3015 adapter->txd_cmd |= E1000_TXD_CMD_RS; 3016 3017 ew32(TCTL, tctl); 3018 3019 hw->mac.ops.config_collision_dist(hw); 3020 3021 /* SPT and KBL Si errata workaround to avoid data corruption */ 3022 if (hw->mac.type == e1000_pch_spt) { 3023 u32 reg_val; 3024 3025 reg_val = er32(IOSFPC); 3026 reg_val |= E1000_RCTL_RDMTS_HEX; 3027 ew32(IOSFPC, reg_val); 3028 3029 reg_val = er32(TARC(0)); 3030 /* SPT and KBL Si errata workaround to avoid Tx hang. 3031 * Dropping the number of outstanding requests from 3032 * 3 to 2 in order to avoid a buffer overrun. 3033 */ 3034 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ; 3035 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ; 3036 ew32(TARC(0), reg_val); 3037 } 3038 } 3039 3040 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \ 3041 (((S) & (PAGE_SIZE - 1)) ? 1 : 0)) 3042 3043 /** 3044 * e1000_setup_rctl - configure the receive control registers 3045 * @adapter: Board private structure 3046 **/ 3047 static void e1000_setup_rctl(struct e1000_adapter *adapter) 3048 { 3049 struct e1000_hw *hw = &adapter->hw; 3050 u32 rctl, rfctl; 3051 u32 pages = 0; 3052 3053 /* Workaround Si errata on PCHx - configure jumbo frame flow. 3054 * If jumbo frames not set, program related MAC/PHY registers 3055 * to h/w defaults 3056 */ 3057 if (hw->mac.type >= e1000_pch2lan) { 3058 s32 ret_val; 3059 3060 if (adapter->netdev->mtu > ETH_DATA_LEN) 3061 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true); 3062 else 3063 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false); 3064 3065 if (ret_val) 3066 e_dbg("failed to enable|disable jumbo frame workaround mode\n"); 3067 } 3068 3069 /* Program MC offset vector base */ 3070 rctl = er32(RCTL); 3071 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3072 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 3073 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 3074 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3075 3076 /* Do not Store bad packets */ 3077 rctl &= ~E1000_RCTL_SBP; 3078 3079 /* Enable Long Packet receive */ 3080 if (adapter->netdev->mtu <= ETH_DATA_LEN) 3081 rctl &= ~E1000_RCTL_LPE; 3082 else 3083 rctl |= E1000_RCTL_LPE; 3084 3085 /* Some systems expect that the CRC is included in SMBUS traffic. The 3086 * hardware strips the CRC before sending to both SMBUS (BMC) and to 3087 * host memory when this is enabled 3088 */ 3089 if (adapter->flags2 & FLAG2_CRC_STRIPPING) 3090 rctl |= E1000_RCTL_SECRC; 3091 3092 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */ 3093 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) { 3094 u16 phy_data; 3095 3096 e1e_rphy(hw, PHY_REG(770, 26), &phy_data); 3097 phy_data &= 0xfff8; 3098 phy_data |= BIT(2); 3099 e1e_wphy(hw, PHY_REG(770, 26), phy_data); 3100 3101 e1e_rphy(hw, 22, &phy_data); 3102 phy_data &= 0x0fff; 3103 phy_data |= BIT(14); 3104 e1e_wphy(hw, 0x10, 0x2823); 3105 e1e_wphy(hw, 0x11, 0x0003); 3106 e1e_wphy(hw, 22, phy_data); 3107 } 3108 3109 /* Setup buffer sizes */ 3110 rctl &= ~E1000_RCTL_SZ_4096; 3111 rctl |= E1000_RCTL_BSEX; 3112 switch (adapter->rx_buffer_len) { 3113 case 2048: 3114 default: 3115 rctl |= E1000_RCTL_SZ_2048; 3116 rctl &= ~E1000_RCTL_BSEX; 3117 break; 3118 case 4096: 3119 rctl |= E1000_RCTL_SZ_4096; 3120 break; 3121 case 8192: 3122 rctl |= E1000_RCTL_SZ_8192; 3123 break; 3124 case 16384: 3125 rctl |= E1000_RCTL_SZ_16384; 3126 break; 3127 } 3128 3129 /* Enable Extended Status in all Receive Descriptors */ 3130 rfctl = er32(RFCTL); 3131 rfctl |= E1000_RFCTL_EXTEN; 3132 ew32(RFCTL, rfctl); 3133 3134 /* 82571 and greater support packet-split where the protocol 3135 * header is placed in skb->data and the packet data is 3136 * placed in pages hanging off of skb_shinfo(skb)->nr_frags. 3137 * In the case of a non-split, skb->data is linearly filled, 3138 * followed by the page buffers. Therefore, skb->data is 3139 * sized to hold the largest protocol header. 3140 * 3141 * allocations using alloc_page take too long for regular MTU 3142 * so only enable packet split for jumbo frames 3143 * 3144 * Using pages when the page size is greater than 16k wastes 3145 * a lot of memory, since we allocate 3 pages at all times 3146 * per packet. 3147 */ 3148 pages = PAGE_USE_COUNT(adapter->netdev->mtu); 3149 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE)) 3150 adapter->rx_ps_pages = pages; 3151 else 3152 adapter->rx_ps_pages = 0; 3153 3154 if (adapter->rx_ps_pages) { 3155 u32 psrctl = 0; 3156 3157 /* Enable Packet split descriptors */ 3158 rctl |= E1000_RCTL_DTYP_PS; 3159 3160 psrctl |= adapter->rx_ps_bsize0 >> E1000_PSRCTL_BSIZE0_SHIFT; 3161 3162 switch (adapter->rx_ps_pages) { 3163 case 3: 3164 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE3_SHIFT; 3165 fallthrough; 3166 case 2: 3167 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE2_SHIFT; 3168 fallthrough; 3169 case 1: 3170 psrctl |= PAGE_SIZE >> E1000_PSRCTL_BSIZE1_SHIFT; 3171 break; 3172 } 3173 3174 ew32(PSRCTL, psrctl); 3175 } 3176 3177 /* This is useful for sniffing bad packets. */ 3178 if (adapter->netdev->features & NETIF_F_RXALL) { 3179 /* UPE and MPE will be handled by normal PROMISC logic 3180 * in e1000e_set_rx_mode 3181 */ 3182 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ 3183 E1000_RCTL_BAM | /* RX All Bcast Pkts */ 3184 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 3185 3186 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */ 3187 E1000_RCTL_DPF | /* Allow filtered pause */ 3188 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ 3189 /* Do not mess with E1000_CTRL_VME, it affects transmit as well, 3190 * and that breaks VLANs. 3191 */ 3192 } 3193 3194 ew32(RCTL, rctl); 3195 /* just started the receive unit, no need to restart */ 3196 adapter->flags &= ~FLAG_RESTART_NOW; 3197 } 3198 3199 /** 3200 * e1000_configure_rx - Configure Receive Unit after Reset 3201 * @adapter: board private structure 3202 * 3203 * Configure the Rx unit of the MAC after a reset. 3204 **/ 3205 static void e1000_configure_rx(struct e1000_adapter *adapter) 3206 { 3207 struct e1000_hw *hw = &adapter->hw; 3208 struct e1000_ring *rx_ring = adapter->rx_ring; 3209 u64 rdba; 3210 u32 rdlen, rctl, rxcsum, ctrl_ext; 3211 3212 if (adapter->rx_ps_pages) { 3213 /* this is a 32 byte descriptor */ 3214 rdlen = rx_ring->count * 3215 sizeof(union e1000_rx_desc_packet_split); 3216 adapter->clean_rx = e1000_clean_rx_irq_ps; 3217 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps; 3218 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) { 3219 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3220 adapter->clean_rx = e1000_clean_jumbo_rx_irq; 3221 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers; 3222 } else { 3223 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended); 3224 adapter->clean_rx = e1000_clean_rx_irq; 3225 adapter->alloc_rx_buf = e1000_alloc_rx_buffers; 3226 } 3227 3228 /* disable receives while setting up the descriptors */ 3229 rctl = er32(RCTL); 3230 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 3231 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3232 e1e_flush(); 3233 usleep_range(10000, 11000); 3234 3235 if (adapter->flags2 & FLAG2_DMA_BURST) { 3236 /* set the writeback threshold (only takes effect if the RDTR 3237 * is set). set GRAN=1 and write back up to 0x4 worth, and 3238 * enable prefetching of 0x20 Rx descriptors 3239 * granularity = 01 3240 * wthresh = 04, 3241 * hthresh = 04, 3242 * pthresh = 0x20 3243 */ 3244 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE); 3245 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE); 3246 } 3247 3248 /* set the Receive Delay Timer Register */ 3249 ew32(RDTR, adapter->rx_int_delay); 3250 3251 /* irq moderation */ 3252 ew32(RADV, adapter->rx_abs_int_delay); 3253 if ((adapter->itr_setting != 0) && (adapter->itr != 0)) 3254 e1000e_write_itr(adapter, adapter->itr); 3255 3256 ctrl_ext = er32(CTRL_EXT); 3257 /* Auto-Mask interrupts upon ICR access */ 3258 ctrl_ext |= E1000_CTRL_EXT_IAME; 3259 ew32(IAM, 0xffffffff); 3260 ew32(CTRL_EXT, ctrl_ext); 3261 e1e_flush(); 3262 3263 /* Setup the HW Rx Head and Tail Descriptor Pointers and 3264 * the Base and Length of the Rx Descriptor Ring 3265 */ 3266 rdba = rx_ring->dma; 3267 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32))); 3268 ew32(RDBAH(0), (rdba >> 32)); 3269 ew32(RDLEN(0), rdlen); 3270 ew32(RDH(0), 0); 3271 ew32(RDT(0), 0); 3272 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0); 3273 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0); 3274 3275 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 3276 e1000e_update_rdt_wa(rx_ring, 0); 3277 3278 /* Enable Receive Checksum Offload for TCP and UDP */ 3279 rxcsum = er32(RXCSUM); 3280 if (adapter->netdev->features & NETIF_F_RXCSUM) 3281 rxcsum |= E1000_RXCSUM_TUOFL; 3282 else 3283 rxcsum &= ~E1000_RXCSUM_TUOFL; 3284 ew32(RXCSUM, rxcsum); 3285 3286 /* With jumbo frames, excessive C-state transition latencies result 3287 * in dropped transactions. 3288 */ 3289 if (adapter->netdev->mtu > ETH_DATA_LEN) { 3290 u32 lat = 3291 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 - 3292 adapter->max_frame_size) * 8 / 1000; 3293 3294 if (adapter->flags & FLAG_IS_ICH) { 3295 u32 rxdctl = er32(RXDCTL(0)); 3296 3297 ew32(RXDCTL(0), rxdctl | 0x3 | BIT(8)); 3298 } 3299 3300 dev_info(&adapter->pdev->dev, 3301 "Some CPU C-states have been disabled in order to enable jumbo frames\n"); 3302 cpu_latency_qos_update_request(&adapter->pm_qos_req, lat); 3303 } else { 3304 cpu_latency_qos_update_request(&adapter->pm_qos_req, 3305 PM_QOS_DEFAULT_VALUE); 3306 } 3307 3308 /* Enable Receives */ 3309 ew32(RCTL, rctl); 3310 } 3311 3312 /** 3313 * e1000e_write_mc_addr_list - write multicast addresses to MTA 3314 * @netdev: network interface device structure 3315 * 3316 * Writes multicast address list to the MTA hash table. 3317 * Returns: -ENOMEM on failure 3318 * 0 on no addresses written 3319 * X on writing X addresses to MTA 3320 */ 3321 static int e1000e_write_mc_addr_list(struct net_device *netdev) 3322 { 3323 struct e1000_adapter *adapter = netdev_priv(netdev); 3324 struct e1000_hw *hw = &adapter->hw; 3325 struct netdev_hw_addr *ha; 3326 u8 *mta_list; 3327 int i; 3328 3329 if (netdev_mc_empty(netdev)) { 3330 /* nothing to program, so clear mc list */ 3331 hw->mac.ops.update_mc_addr_list(hw, NULL, 0); 3332 return 0; 3333 } 3334 3335 mta_list = kcalloc(netdev_mc_count(netdev), ETH_ALEN, GFP_ATOMIC); 3336 if (!mta_list) 3337 return -ENOMEM; 3338 3339 /* update_mc_addr_list expects a packed array of only addresses. */ 3340 i = 0; 3341 netdev_for_each_mc_addr(ha, netdev) 3342 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 3343 3344 hw->mac.ops.update_mc_addr_list(hw, mta_list, i); 3345 kfree(mta_list); 3346 3347 return netdev_mc_count(netdev); 3348 } 3349 3350 /** 3351 * e1000e_write_uc_addr_list - write unicast addresses to RAR table 3352 * @netdev: network interface device structure 3353 * 3354 * Writes unicast address list to the RAR table. 3355 * Returns: -ENOMEM on failure/insufficient address space 3356 * 0 on no addresses written 3357 * X on writing X addresses to the RAR table 3358 **/ 3359 static int e1000e_write_uc_addr_list(struct net_device *netdev) 3360 { 3361 struct e1000_adapter *adapter = netdev_priv(netdev); 3362 struct e1000_hw *hw = &adapter->hw; 3363 unsigned int rar_entries; 3364 int count = 0; 3365 3366 rar_entries = hw->mac.ops.rar_get_count(hw); 3367 3368 /* save a rar entry for our hardware address */ 3369 rar_entries--; 3370 3371 /* save a rar entry for the LAA workaround */ 3372 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) 3373 rar_entries--; 3374 3375 /* return ENOMEM indicating insufficient memory for addresses */ 3376 if (netdev_uc_count(netdev) > rar_entries) 3377 return -ENOMEM; 3378 3379 if (!netdev_uc_empty(netdev) && rar_entries) { 3380 struct netdev_hw_addr *ha; 3381 3382 /* write the addresses in reverse order to avoid write 3383 * combining 3384 */ 3385 netdev_for_each_uc_addr(ha, netdev) { 3386 int ret_val; 3387 3388 if (!rar_entries) 3389 break; 3390 ret_val = hw->mac.ops.rar_set(hw, ha->addr, rar_entries--); 3391 if (ret_val < 0) 3392 return -ENOMEM; 3393 count++; 3394 } 3395 } 3396 3397 /* zero out the remaining RAR entries not used above */ 3398 for (; rar_entries > 0; rar_entries--) { 3399 ew32(RAH(rar_entries), 0); 3400 ew32(RAL(rar_entries), 0); 3401 } 3402 e1e_flush(); 3403 3404 return count; 3405 } 3406 3407 /** 3408 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set 3409 * @netdev: network interface device structure 3410 * 3411 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast 3412 * address list or the network interface flags are updated. This routine is 3413 * responsible for configuring the hardware for proper unicast, multicast, 3414 * promiscuous mode, and all-multi behavior. 3415 **/ 3416 static void e1000e_set_rx_mode(struct net_device *netdev) 3417 { 3418 struct e1000_adapter *adapter = netdev_priv(netdev); 3419 struct e1000_hw *hw = &adapter->hw; 3420 u32 rctl; 3421 3422 if (pm_runtime_suspended(netdev->dev.parent)) 3423 return; 3424 3425 /* Check for Promiscuous and All Multicast modes */ 3426 rctl = er32(RCTL); 3427 3428 /* clear the affected bits */ 3429 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE); 3430 3431 if (netdev->flags & IFF_PROMISC) { 3432 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 3433 /* Do not hardware filter VLANs in promisc mode */ 3434 e1000e_vlan_filter_disable(adapter); 3435 } else { 3436 int count; 3437 3438 if (netdev->flags & IFF_ALLMULTI) { 3439 rctl |= E1000_RCTL_MPE; 3440 } else { 3441 /* Write addresses to the MTA, if the attempt fails 3442 * then we should just turn on promiscuous mode so 3443 * that we can at least receive multicast traffic 3444 */ 3445 count = e1000e_write_mc_addr_list(netdev); 3446 if (count < 0) 3447 rctl |= E1000_RCTL_MPE; 3448 } 3449 e1000e_vlan_filter_enable(adapter); 3450 /* Write addresses to available RAR registers, if there is not 3451 * sufficient space to store all the addresses then enable 3452 * unicast promiscuous mode 3453 */ 3454 count = e1000e_write_uc_addr_list(netdev); 3455 if (count < 0) 3456 rctl |= E1000_RCTL_UPE; 3457 } 3458 3459 ew32(RCTL, rctl); 3460 3461 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3462 e1000e_vlan_strip_enable(adapter); 3463 else 3464 e1000e_vlan_strip_disable(adapter); 3465 } 3466 3467 static void e1000e_setup_rss_hash(struct e1000_adapter *adapter) 3468 { 3469 struct e1000_hw *hw = &adapter->hw; 3470 u32 mrqc, rxcsum; 3471 u32 rss_key[10]; 3472 int i; 3473 3474 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 3475 for (i = 0; i < 10; i++) 3476 ew32(RSSRK(i), rss_key[i]); 3477 3478 /* Direct all traffic to queue 0 */ 3479 for (i = 0; i < 32; i++) 3480 ew32(RETA(i), 0); 3481 3482 /* Disable raw packet checksumming so that RSS hash is placed in 3483 * descriptor on writeback. 3484 */ 3485 rxcsum = er32(RXCSUM); 3486 rxcsum |= E1000_RXCSUM_PCSD; 3487 3488 ew32(RXCSUM, rxcsum); 3489 3490 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 | 3491 E1000_MRQC_RSS_FIELD_IPV4_TCP | 3492 E1000_MRQC_RSS_FIELD_IPV6 | 3493 E1000_MRQC_RSS_FIELD_IPV6_TCP | 3494 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3495 3496 ew32(MRQC, mrqc); 3497 } 3498 3499 /** 3500 * e1000e_get_base_timinca - get default SYSTIM time increment attributes 3501 * @adapter: board private structure 3502 * @timinca: pointer to returned time increment attributes 3503 * 3504 * Get attributes for incrementing the System Time Register SYSTIML/H at 3505 * the default base frequency, and set the cyclecounter shift value. 3506 **/ 3507 s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca) 3508 { 3509 struct e1000_hw *hw = &adapter->hw; 3510 u32 incvalue, incperiod, shift; 3511 3512 /* Make sure clock is enabled on I217/I218/I219 before checking 3513 * the frequency 3514 */ 3515 if ((hw->mac.type >= e1000_pch_lpt) && 3516 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) && 3517 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) { 3518 u32 fextnvm7 = er32(FEXTNVM7); 3519 3520 if (!(fextnvm7 & BIT(0))) { 3521 ew32(FEXTNVM7, fextnvm7 | BIT(0)); 3522 e1e_flush(); 3523 } 3524 } 3525 3526 switch (hw->mac.type) { 3527 case e1000_pch2lan: 3528 /* Stable 96MHz frequency */ 3529 incperiod = INCPERIOD_96MHZ; 3530 incvalue = INCVALUE_96MHZ; 3531 shift = INCVALUE_SHIFT_96MHZ; 3532 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3533 break; 3534 case e1000_pch_lpt: 3535 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3536 /* Stable 96MHz frequency */ 3537 incperiod = INCPERIOD_96MHZ; 3538 incvalue = INCVALUE_96MHZ; 3539 shift = INCVALUE_SHIFT_96MHZ; 3540 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ; 3541 } else { 3542 /* Stable 25MHz frequency */ 3543 incperiod = INCPERIOD_25MHZ; 3544 incvalue = INCVALUE_25MHZ; 3545 shift = INCVALUE_SHIFT_25MHZ; 3546 adapter->cc.shift = shift; 3547 } 3548 break; 3549 case e1000_pch_spt: 3550 /* Stable 24MHz frequency */ 3551 incperiod = INCPERIOD_24MHZ; 3552 incvalue = INCVALUE_24MHZ; 3553 shift = INCVALUE_SHIFT_24MHZ; 3554 adapter->cc.shift = shift; 3555 break; 3556 case e1000_pch_cnp: 3557 case e1000_pch_tgp: 3558 case e1000_pch_adp: 3559 case e1000_pch_nvp: 3560 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) { 3561 /* Stable 24MHz frequency */ 3562 incperiod = INCPERIOD_24MHZ; 3563 incvalue = INCVALUE_24MHZ; 3564 shift = INCVALUE_SHIFT_24MHZ; 3565 adapter->cc.shift = shift; 3566 } else { 3567 /* Stable 38400KHz frequency */ 3568 incperiod = INCPERIOD_38400KHZ; 3569 incvalue = INCVALUE_38400KHZ; 3570 shift = INCVALUE_SHIFT_38400KHZ; 3571 adapter->cc.shift = shift; 3572 } 3573 break; 3574 case e1000_pch_mtp: 3575 case e1000_pch_lnp: 3576 case e1000_pch_ptp: 3577 /* System firmware can misreport this value, so set it to a 3578 * stable 38400KHz frequency. 3579 */ 3580 incperiod = INCPERIOD_38400KHZ; 3581 incvalue = INCVALUE_38400KHZ; 3582 shift = INCVALUE_SHIFT_38400KHZ; 3583 adapter->cc.shift = shift; 3584 break; 3585 case e1000_82574: 3586 case e1000_82583: 3587 /* Stable 25MHz frequency */ 3588 incperiod = INCPERIOD_25MHZ; 3589 incvalue = INCVALUE_25MHZ; 3590 shift = INCVALUE_SHIFT_25MHZ; 3591 adapter->cc.shift = shift; 3592 break; 3593 default: 3594 return -EINVAL; 3595 } 3596 3597 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) | 3598 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK)); 3599 3600 return 0; 3601 } 3602 3603 /** 3604 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable 3605 * @adapter: board private structure 3606 * @config: timestamp configuration 3607 * @extack: netlink extended ACK for error report 3608 * 3609 * Outgoing time stamping can be enabled and disabled. Play nice and 3610 * disable it when requested, although it shouldn't cause any overhead 3611 * when no packet needs it. At most one packet in the queue may be 3612 * marked for time stamping, otherwise it would be impossible to tell 3613 * for sure to which packet the hardware time stamp belongs. 3614 * 3615 * Incoming time stamping has to be configured via the hardware filters. 3616 * Not all combinations are supported, in particular event type has to be 3617 * specified. Matching the kind of event packet is not supported, with the 3618 * exception of "all V2 events regardless of level 2 or 4". 3619 **/ 3620 static int e1000e_config_hwtstamp(struct e1000_adapter *adapter, 3621 struct kernel_hwtstamp_config *config, 3622 struct netlink_ext_ack *extack) 3623 { 3624 struct e1000_hw *hw = &adapter->hw; 3625 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED; 3626 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED; 3627 u32 rxmtrl = 0; 3628 u16 rxudp = 0; 3629 bool is_l4 = false; 3630 bool is_l2 = false; 3631 u32 regval; 3632 3633 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) { 3634 NL_SET_ERR_MSG(extack, "No HW timestamp support"); 3635 return -EINVAL; 3636 } 3637 3638 switch (config->tx_type) { 3639 case HWTSTAMP_TX_OFF: 3640 tsync_tx_ctl = 0; 3641 break; 3642 case HWTSTAMP_TX_ON: 3643 break; 3644 default: 3645 NL_SET_ERR_MSG(extack, "Unsupported TX HW timestamp type"); 3646 return -ERANGE; 3647 } 3648 3649 switch (config->rx_filter) { 3650 case HWTSTAMP_FILTER_NONE: 3651 tsync_rx_ctl = 0; 3652 break; 3653 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3654 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3655 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE; 3656 is_l4 = true; 3657 break; 3658 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3659 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1; 3660 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE; 3661 is_l4 = true; 3662 break; 3663 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3664 /* Also time stamps V2 L2 Path Delay Request/Response */ 3665 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3666 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3667 is_l2 = true; 3668 break; 3669 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3670 /* Also time stamps V2 L2 Path Delay Request/Response. */ 3671 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2; 3672 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3673 is_l2 = true; 3674 break; 3675 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3676 /* Hardware cannot filter just V2 L4 Sync messages */ 3677 fallthrough; 3678 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3679 /* Also time stamps V2 Path Delay Request/Response. */ 3680 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3681 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE; 3682 is_l2 = true; 3683 is_l4 = true; 3684 break; 3685 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3686 /* Hardware cannot filter just V2 L4 Delay Request messages */ 3687 fallthrough; 3688 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3689 /* Also time stamps V2 Path Delay Request/Response. */ 3690 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2; 3691 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE; 3692 is_l2 = true; 3693 is_l4 = true; 3694 break; 3695 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3696 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3697 /* Hardware cannot filter just V2 L4 or L2 Event messages */ 3698 fallthrough; 3699 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3700 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2; 3701 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 3702 is_l2 = true; 3703 is_l4 = true; 3704 break; 3705 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3706 /* For V1, the hardware can only filter Sync messages or 3707 * Delay Request messages but not both so fall-through to 3708 * time stamp all packets. 3709 */ 3710 fallthrough; 3711 case HWTSTAMP_FILTER_NTP_ALL: 3712 case HWTSTAMP_FILTER_ALL: 3713 is_l2 = true; 3714 is_l4 = true; 3715 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL; 3716 config->rx_filter = HWTSTAMP_FILTER_ALL; 3717 break; 3718 default: 3719 NL_SET_ERR_MSG(extack, "Unsupported RX HW timestamp filter"); 3720 return -ERANGE; 3721 } 3722 3723 adapter->hwtstamp_config = *config; 3724 3725 /* enable/disable Tx h/w time stamping */ 3726 regval = er32(TSYNCTXCTL); 3727 regval &= ~E1000_TSYNCTXCTL_ENABLED; 3728 regval |= tsync_tx_ctl; 3729 ew32(TSYNCTXCTL, regval); 3730 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) != 3731 (regval & E1000_TSYNCTXCTL_ENABLED)) { 3732 NL_SET_ERR_MSG(extack, 3733 "Timesync Tx Control register not set as expected"); 3734 return -EAGAIN; 3735 } 3736 3737 /* enable/disable Rx h/w time stamping */ 3738 regval = er32(TSYNCRXCTL); 3739 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK); 3740 regval |= tsync_rx_ctl; 3741 ew32(TSYNCRXCTL, regval); 3742 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED | 3743 E1000_TSYNCRXCTL_TYPE_MASK)) != 3744 (regval & (E1000_TSYNCRXCTL_ENABLED | 3745 E1000_TSYNCRXCTL_TYPE_MASK))) { 3746 NL_SET_ERR_MSG(extack, 3747 "Timesync Rx Control register not set as expected"); 3748 return -EAGAIN; 3749 } 3750 3751 /* L2: define ethertype filter for time stamped packets */ 3752 if (is_l2) 3753 rxmtrl |= ETH_P_1588; 3754 3755 /* define which PTP packets get time stamped */ 3756 ew32(RXMTRL, rxmtrl); 3757 3758 /* Filter by destination port */ 3759 if (is_l4) { 3760 rxudp = PTP_EV_PORT; 3761 cpu_to_be16s(&rxudp); 3762 } 3763 ew32(RXUDP, rxudp); 3764 3765 e1e_flush(); 3766 3767 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */ 3768 er32(RXSTMPH); 3769 er32(TXSTMPH); 3770 3771 return 0; 3772 } 3773 3774 /** 3775 * e1000_configure - configure the hardware for Rx and Tx 3776 * @adapter: private board structure 3777 **/ 3778 static void e1000_configure(struct e1000_adapter *adapter) 3779 { 3780 struct e1000_ring *rx_ring = adapter->rx_ring; 3781 3782 e1000e_set_rx_mode(adapter->netdev); 3783 3784 e1000_restore_vlan(adapter); 3785 e1000_init_manageability_pt(adapter); 3786 3787 e1000_configure_tx(adapter); 3788 3789 if (adapter->netdev->features & NETIF_F_RXHASH) 3790 e1000e_setup_rss_hash(adapter); 3791 e1000_setup_rctl(adapter); 3792 e1000_configure_rx(adapter); 3793 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL); 3794 } 3795 3796 /** 3797 * e1000e_power_up_phy - restore link in case the phy was powered down 3798 * @adapter: address of board private structure 3799 * 3800 * The phy may be powered down to save power and turn off link when the 3801 * driver is unloaded and wake on lan is not enabled (among others) 3802 * *** this routine MUST be followed by a call to e1000e_reset *** 3803 **/ 3804 void e1000e_power_up_phy(struct e1000_adapter *adapter) 3805 { 3806 if (adapter->hw.phy.ops.power_up) 3807 adapter->hw.phy.ops.power_up(&adapter->hw); 3808 3809 adapter->hw.mac.ops.setup_link(&adapter->hw); 3810 } 3811 3812 /** 3813 * e1000_power_down_phy - Power down the PHY 3814 * @adapter: board private structure 3815 * 3816 * Power down the PHY so no link is implied when interface is down. 3817 * The PHY cannot be powered down if management or WoL is active. 3818 */ 3819 static void e1000_power_down_phy(struct e1000_adapter *adapter) 3820 { 3821 if (adapter->hw.phy.ops.power_down) 3822 adapter->hw.phy.ops.power_down(&adapter->hw); 3823 } 3824 3825 /** 3826 * e1000_flush_tx_ring - remove all descriptors from the tx_ring 3827 * @adapter: board private structure 3828 * 3829 * We want to clear all pending descriptors from the TX ring. 3830 * zeroing happens when the HW reads the regs. We assign the ring itself as 3831 * the data of the next descriptor. We don't care about the data we are about 3832 * to reset the HW. 3833 */ 3834 static void e1000_flush_tx_ring(struct e1000_adapter *adapter) 3835 { 3836 struct e1000_hw *hw = &adapter->hw; 3837 struct e1000_ring *tx_ring = adapter->tx_ring; 3838 struct e1000_tx_desc *tx_desc = NULL; 3839 u32 tdt, tctl, txd_lower = E1000_TXD_CMD_IFCS; 3840 u16 size = 512; 3841 3842 tctl = er32(TCTL); 3843 ew32(TCTL, tctl | E1000_TCTL_EN); 3844 tdt = er32(TDT(0)); 3845 BUG_ON(tdt != tx_ring->next_to_use); 3846 tx_desc = E1000_TX_DESC(*tx_ring, tx_ring->next_to_use); 3847 tx_desc->buffer_addr = cpu_to_le64(tx_ring->dma); 3848 3849 tx_desc->lower.data = cpu_to_le32(txd_lower | size); 3850 tx_desc->upper.data = 0; 3851 /* flush descriptors to memory before notifying the HW */ 3852 wmb(); 3853 tx_ring->next_to_use++; 3854 if (tx_ring->next_to_use == tx_ring->count) 3855 tx_ring->next_to_use = 0; 3856 ew32(TDT(0), tx_ring->next_to_use); 3857 usleep_range(200, 250); 3858 } 3859 3860 /** 3861 * e1000_flush_rx_ring - remove all descriptors from the rx_ring 3862 * @adapter: board private structure 3863 * 3864 * Mark all descriptors in the RX ring as consumed and disable the rx ring 3865 */ 3866 static void e1000_flush_rx_ring(struct e1000_adapter *adapter) 3867 { 3868 u32 rctl, rxdctl; 3869 struct e1000_hw *hw = &adapter->hw; 3870 3871 rctl = er32(RCTL); 3872 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3873 e1e_flush(); 3874 usleep_range(100, 150); 3875 3876 rxdctl = er32(RXDCTL(0)); 3877 /* zero the lower 14 bits (prefetch and host thresholds) */ 3878 rxdctl &= 0xffffc000; 3879 3880 /* update thresholds: prefetch threshold to 31, host threshold to 1 3881 * and make sure the granularity is "descriptors" and not "cache lines" 3882 */ 3883 rxdctl |= (0x1F | BIT(8) | E1000_RXDCTL_THRESH_UNIT_DESC); 3884 3885 ew32(RXDCTL(0), rxdctl); 3886 /* momentarily enable the RX ring for the changes to take effect */ 3887 ew32(RCTL, rctl | E1000_RCTL_EN); 3888 e1e_flush(); 3889 usleep_range(100, 150); 3890 ew32(RCTL, rctl & ~E1000_RCTL_EN); 3891 } 3892 3893 /** 3894 * e1000_flush_desc_rings - remove all descriptors from the descriptor rings 3895 * @adapter: board private structure 3896 * 3897 * In i219, the descriptor rings must be emptied before resetting the HW 3898 * or before changing the device state to D3 during runtime (runtime PM). 3899 * 3900 * Failure to do this will cause the HW to enter a unit hang state which can 3901 * only be released by PCI reset on the device 3902 * 3903 */ 3904 3905 static void e1000_flush_desc_rings(struct e1000_adapter *adapter) 3906 { 3907 u16 hang_state; 3908 u32 fext_nvm11, tdlen; 3909 struct e1000_hw *hw = &adapter->hw; 3910 3911 /* First, disable MULR fix in FEXTNVM11 */ 3912 fext_nvm11 = er32(FEXTNVM11); 3913 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; 3914 ew32(FEXTNVM11, fext_nvm11); 3915 /* do nothing if we're not in faulty state, or if the queue is empty */ 3916 tdlen = er32(TDLEN(0)); 3917 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3918 &hang_state); 3919 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) 3920 return; 3921 e1000_flush_tx_ring(adapter); 3922 /* recheck, maybe the fault is caused by the rx ring */ 3923 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS, 3924 &hang_state); 3925 if (hang_state & FLUSH_DESC_REQUIRED) 3926 e1000_flush_rx_ring(adapter); 3927 } 3928 3929 /** 3930 * e1000e_systim_reset - reset the timesync registers after a hardware reset 3931 * @adapter: board private structure 3932 * 3933 * When the MAC is reset, all hardware bits for timesync will be reset to the 3934 * default values. This function will restore the settings last in place. 3935 * Since the clock SYSTIME registers are reset, we will simply restore the 3936 * cyclecounter to the kernel real clock time. 3937 **/ 3938 static void e1000e_systim_reset(struct e1000_adapter *adapter) 3939 { 3940 struct ptp_clock_info *info = &adapter->ptp_clock_info; 3941 struct e1000_hw *hw = &adapter->hw; 3942 struct netlink_ext_ack extack = {}; 3943 unsigned long flags; 3944 u32 timinca; 3945 s32 ret_val; 3946 3947 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP)) 3948 return; 3949 3950 if (info->adjfine) { 3951 /* restore the previous ptp frequency delta */ 3952 ret_val = info->adjfine(info, adapter->ptp_delta); 3953 } else { 3954 /* set the default base frequency if no adjustment possible */ 3955 ret_val = e1000e_get_base_timinca(adapter, &timinca); 3956 if (!ret_val) 3957 ew32(TIMINCA, timinca); 3958 } 3959 3960 if (ret_val) { 3961 dev_warn(&adapter->pdev->dev, 3962 "Failed to restore TIMINCA clock rate delta: %d\n", 3963 ret_val); 3964 return; 3965 } 3966 3967 /* reset the systim ns time counter */ 3968 spin_lock_irqsave(&adapter->systim_lock, flags); 3969 timecounter_init(&adapter->tc, &adapter->cc, 3970 ktime_get_real_ns()); 3971 spin_unlock_irqrestore(&adapter->systim_lock, flags); 3972 3973 /* restore the previous hwtstamp configuration settings */ 3974 ret_val = e1000e_config_hwtstamp(adapter, &adapter->hwtstamp_config, 3975 &extack); 3976 if (ret_val) { 3977 if (extack._msg) 3978 e_err("%s\n", extack._msg); 3979 } 3980 } 3981 3982 /** 3983 * e1000e_reset - bring the hardware into a known good state 3984 * @adapter: board private structure 3985 * 3986 * This function boots the hardware and enables some settings that 3987 * require a configuration cycle of the hardware - those cannot be 3988 * set/changed during runtime. After reset the device needs to be 3989 * properly configured for Rx, Tx etc. 3990 */ 3991 void e1000e_reset(struct e1000_adapter *adapter) 3992 { 3993 struct e1000_mac_info *mac = &adapter->hw.mac; 3994 struct e1000_fc_info *fc = &adapter->hw.fc; 3995 struct e1000_hw *hw = &adapter->hw; 3996 u32 tx_space, min_tx_space, min_rx_space; 3997 u32 pba = adapter->pba; 3998 u16 hwm; 3999 4000 /* reset Packet Buffer Allocation to default */ 4001 ew32(PBA, pba); 4002 4003 if (adapter->max_frame_size > (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) { 4004 /* To maintain wire speed transmits, the Tx FIFO should be 4005 * large enough to accommodate two full transmit packets, 4006 * rounded up to the next 1KB and expressed in KB. Likewise, 4007 * the Rx FIFO should be large enough to accommodate at least 4008 * one full receive packet and is similarly rounded up and 4009 * expressed in KB. 4010 */ 4011 pba = er32(PBA); 4012 /* upper 16 bits has Tx packet buffer allocation size in KB */ 4013 tx_space = pba >> 16; 4014 /* lower 16 bits has Rx packet buffer allocation size in KB */ 4015 pba &= 0xffff; 4016 /* the Tx fifo also stores 16 bytes of information about the Tx 4017 * but don't include ethernet FCS because hardware appends it 4018 */ 4019 min_tx_space = (adapter->max_frame_size + 4020 sizeof(struct e1000_tx_desc) - ETH_FCS_LEN) * 2; 4021 min_tx_space = ALIGN(min_tx_space, 1024); 4022 min_tx_space >>= 10; 4023 /* software strips receive CRC, so leave room for it */ 4024 min_rx_space = adapter->max_frame_size; 4025 min_rx_space = ALIGN(min_rx_space, 1024); 4026 min_rx_space >>= 10; 4027 4028 /* If current Tx allocation is less than the min Tx FIFO size, 4029 * and the min Tx FIFO size is less than the current Rx FIFO 4030 * allocation, take space away from current Rx allocation 4031 */ 4032 if ((tx_space < min_tx_space) && 4033 ((min_tx_space - tx_space) < pba)) { 4034 pba -= min_tx_space - tx_space; 4035 4036 /* if short on Rx space, Rx wins and must trump Tx 4037 * adjustment 4038 */ 4039 if (pba < min_rx_space) 4040 pba = min_rx_space; 4041 } 4042 4043 ew32(PBA, pba); 4044 } 4045 4046 /* flow control settings 4047 * 4048 * The high water mark must be low enough to fit one full frame 4049 * (or the size used for early receive) above it in the Rx FIFO. 4050 * Set it to the lower of: 4051 * - 90% of the Rx FIFO size, and 4052 * - the full Rx FIFO size minus one full frame 4053 */ 4054 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME) 4055 fc->pause_time = 0xFFFF; 4056 else 4057 fc->pause_time = E1000_FC_PAUSE_TIME; 4058 fc->send_xon = true; 4059 fc->current_mode = fc->requested_mode; 4060 4061 switch (hw->mac.type) { 4062 case e1000_ich9lan: 4063 case e1000_ich10lan: 4064 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4065 pba = 14; 4066 ew32(PBA, pba); 4067 fc->high_water = 0x2800; 4068 fc->low_water = fc->high_water - 8; 4069 break; 4070 } 4071 fallthrough; 4072 default: 4073 hwm = min(((pba << 10) * 9 / 10), 4074 ((pba << 10) - adapter->max_frame_size)); 4075 4076 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */ 4077 fc->low_water = fc->high_water - 8; 4078 break; 4079 case e1000_pchlan: 4080 /* Workaround PCH LOM adapter hangs with certain network 4081 * loads. If hangs persist, try disabling Tx flow control. 4082 */ 4083 if (adapter->netdev->mtu > ETH_DATA_LEN) { 4084 fc->high_water = 0x3500; 4085 fc->low_water = 0x1500; 4086 } else { 4087 fc->high_water = 0x5000; 4088 fc->low_water = 0x3000; 4089 } 4090 fc->refresh_time = 0x1000; 4091 break; 4092 case e1000_pch2lan: 4093 case e1000_pch_lpt: 4094 case e1000_pch_spt: 4095 case e1000_pch_cnp: 4096 case e1000_pch_tgp: 4097 case e1000_pch_adp: 4098 case e1000_pch_mtp: 4099 case e1000_pch_lnp: 4100 case e1000_pch_ptp: 4101 case e1000_pch_nvp: 4102 fc->refresh_time = 0xFFFF; 4103 fc->pause_time = 0xFFFF; 4104 4105 if (adapter->netdev->mtu <= ETH_DATA_LEN) { 4106 fc->high_water = 0x05C20; 4107 fc->low_water = 0x05048; 4108 break; 4109 } 4110 4111 pba = 14; 4112 ew32(PBA, pba); 4113 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH; 4114 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL; 4115 break; 4116 } 4117 4118 /* Alignment of Tx data is on an arbitrary byte boundary with the 4119 * maximum size per Tx descriptor limited only to the transmit 4120 * allocation of the packet buffer minus 96 bytes with an upper 4121 * limit of 24KB due to receive synchronization limitations. 4122 */ 4123 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96, 4124 24 << 10); 4125 4126 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot 4127 * fit in receive buffer. 4128 */ 4129 if (adapter->itr_setting & 0x3) { 4130 if ((adapter->max_frame_size * 2) > (pba << 10)) { 4131 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) { 4132 dev_info(&adapter->pdev->dev, 4133 "Interrupt Throttle Rate off\n"); 4134 adapter->flags2 |= FLAG2_DISABLE_AIM; 4135 e1000e_write_itr(adapter, 0); 4136 } 4137 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) { 4138 dev_info(&adapter->pdev->dev, 4139 "Interrupt Throttle Rate on\n"); 4140 adapter->flags2 &= ~FLAG2_DISABLE_AIM; 4141 adapter->itr = 20000; 4142 e1000e_write_itr(adapter, adapter->itr); 4143 } 4144 } 4145 4146 if (hw->mac.type >= e1000_pch_spt) 4147 e1000_flush_desc_rings(adapter); 4148 /* Allow time for pending master requests to run */ 4149 mac->ops.reset_hw(hw); 4150 4151 /* For parts with AMT enabled, let the firmware know 4152 * that the network interface is in control 4153 */ 4154 if (adapter->flags & FLAG_HAS_AMT) 4155 e1000e_get_hw_control(adapter); 4156 4157 ew32(WUC, 0); 4158 4159 if (mac->ops.init_hw(hw)) 4160 e_err("Hardware Error\n"); 4161 4162 e1000_update_mng_vlan(adapter); 4163 4164 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ 4165 ew32(VET, ETH_P_8021Q); 4166 4167 e1000e_reset_adaptive(hw); 4168 4169 /* restore systim and hwtstamp settings */ 4170 e1000e_systim_reset(adapter); 4171 4172 /* Set EEE advertisement as appropriate */ 4173 if (adapter->flags2 & FLAG2_HAS_EEE) { 4174 s32 ret_val; 4175 u16 adv_addr; 4176 4177 switch (hw->phy.type) { 4178 case e1000_phy_82579: 4179 adv_addr = I82579_EEE_ADVERTISEMENT; 4180 break; 4181 case e1000_phy_i217: 4182 adv_addr = I217_EEE_ADVERTISEMENT; 4183 break; 4184 default: 4185 dev_err(&adapter->pdev->dev, 4186 "Invalid PHY type setting EEE advertisement\n"); 4187 return; 4188 } 4189 4190 ret_val = hw->phy.ops.acquire(hw); 4191 if (ret_val) { 4192 dev_err(&adapter->pdev->dev, 4193 "EEE advertisement - unable to acquire PHY\n"); 4194 return; 4195 } 4196 4197 e1000_write_emi_reg_locked(hw, adv_addr, 4198 hw->dev_spec.ich8lan.eee_disable ? 4199 0 : adapter->eee_advert); 4200 4201 hw->phy.ops.release(hw); 4202 } 4203 4204 if (!netif_running(adapter->netdev) && 4205 !test_bit(__E1000_TESTING, &adapter->state)) 4206 e1000_power_down_phy(adapter); 4207 4208 e1000_get_phy_info(hw); 4209 4210 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) && 4211 !(adapter->flags & FLAG_SMART_POWER_DOWN)) { 4212 u16 phy_data = 0; 4213 /* speed up time to link by disabling smart power down, ignore 4214 * the return value of this function because there is nothing 4215 * different we would do if it failed 4216 */ 4217 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data); 4218 phy_data &= ~IGP02E1000_PM_SPD; 4219 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data); 4220 } 4221 if (hw->mac.type >= e1000_pch_spt && adapter->int_mode == 0) { 4222 u32 reg; 4223 4224 /* Fextnvm7 @ 0xe4[2] = 1 */ 4225 reg = er32(FEXTNVM7); 4226 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE; 4227 ew32(FEXTNVM7, reg); 4228 /* Fextnvm9 @ 0x5bb4[13:12] = 11 */ 4229 reg = er32(FEXTNVM9); 4230 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS | 4231 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS; 4232 ew32(FEXTNVM9, reg); 4233 } 4234 4235 } 4236 4237 /** 4238 * e1000e_trigger_lsc - trigger an LSC interrupt 4239 * @adapter: board private structure 4240 * 4241 * Fire a link status change interrupt to start the watchdog. 4242 **/ 4243 static void e1000e_trigger_lsc(struct e1000_adapter *adapter) 4244 { 4245 struct e1000_hw *hw = &adapter->hw; 4246 4247 if (adapter->msix_entries) 4248 ew32(ICS, E1000_ICS_LSC | E1000_ICS_OTHER); 4249 else 4250 ew32(ICS, E1000_ICS_LSC); 4251 } 4252 4253 void e1000e_up(struct e1000_adapter *adapter) 4254 { 4255 /* hardware has been reset, we need to reload some things */ 4256 e1000_configure(adapter); 4257 4258 clear_bit(__E1000_DOWN, &adapter->state); 4259 4260 if (adapter->msix_entries) 4261 e1000_configure_msix(adapter); 4262 e1000_irq_enable(adapter); 4263 4264 /* Tx queue started by watchdog timer when link is up */ 4265 4266 e1000e_trigger_lsc(adapter); 4267 } 4268 4269 static void e1000e_flush_descriptors(struct e1000_adapter *adapter) 4270 { 4271 struct e1000_hw *hw = &adapter->hw; 4272 4273 if (!(adapter->flags2 & FLAG2_DMA_BURST)) 4274 return; 4275 4276 /* flush pending descriptor writebacks to memory */ 4277 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4278 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4279 4280 /* execute the writes immediately */ 4281 e1e_flush(); 4282 4283 /* due to rare timing issues, write to TIDV/RDTR again to ensure the 4284 * write is successful 4285 */ 4286 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD); 4287 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD); 4288 4289 /* execute the writes immediately */ 4290 e1e_flush(); 4291 } 4292 4293 static void e1000e_update_stats(struct e1000_adapter *adapter); 4294 4295 /** 4296 * e1000e_down - quiesce the device and optionally reset the hardware 4297 * @adapter: board private structure 4298 * @reset: boolean flag to reset the hardware or not 4299 */ 4300 void e1000e_down(struct e1000_adapter *adapter, bool reset) 4301 { 4302 struct net_device *netdev = adapter->netdev; 4303 struct e1000_hw *hw = &adapter->hw; 4304 u32 tctl, rctl; 4305 4306 /* signal that we're down so the interrupt handler does not 4307 * reschedule our watchdog timer 4308 */ 4309 set_bit(__E1000_DOWN, &adapter->state); 4310 4311 netif_carrier_off(netdev); 4312 4313 /* disable receives in the hardware */ 4314 rctl = er32(RCTL); 4315 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX)) 4316 ew32(RCTL, rctl & ~E1000_RCTL_EN); 4317 /* flush and sleep below */ 4318 4319 netif_stop_queue(netdev); 4320 4321 /* disable transmits in the hardware */ 4322 tctl = er32(TCTL); 4323 tctl &= ~E1000_TCTL_EN; 4324 ew32(TCTL, tctl); 4325 4326 /* flush both disables and wait for them to finish */ 4327 e1e_flush(); 4328 usleep_range(10000, 11000); 4329 4330 e1000_irq_disable(adapter); 4331 4332 napi_synchronize(&adapter->napi); 4333 4334 timer_delete_sync(&adapter->watchdog_timer); 4335 timer_delete_sync(&adapter->phy_info_timer); 4336 4337 spin_lock(&adapter->stats64_lock); 4338 e1000e_update_stats(adapter); 4339 spin_unlock(&adapter->stats64_lock); 4340 4341 e1000e_flush_descriptors(adapter); 4342 4343 adapter->link_speed = 0; 4344 adapter->link_duplex = 0; 4345 4346 /* Disable Si errata workaround on PCHx for jumbo frame flow */ 4347 if ((hw->mac.type >= e1000_pch2lan) && 4348 (adapter->netdev->mtu > ETH_DATA_LEN) && 4349 e1000_lv_jumbo_workaround_ich8lan(hw, false)) 4350 e_dbg("failed to disable jumbo frame workaround mode\n"); 4351 4352 if (!pci_channel_offline(adapter->pdev)) { 4353 if (reset) 4354 e1000e_reset(adapter); 4355 else if (hw->mac.type >= e1000_pch_spt) 4356 e1000_flush_desc_rings(adapter); 4357 } 4358 e1000_clean_tx_ring(adapter->tx_ring); 4359 e1000_clean_rx_ring(adapter->rx_ring); 4360 } 4361 4362 void e1000e_reinit_locked(struct e1000_adapter *adapter) 4363 { 4364 might_sleep(); 4365 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 4366 usleep_range(1000, 1100); 4367 e1000e_down(adapter, true); 4368 e1000e_up(adapter); 4369 clear_bit(__E1000_RESETTING, &adapter->state); 4370 } 4371 4372 /** 4373 * e1000e_sanitize_systim - sanitize raw cycle counter reads 4374 * @hw: pointer to the HW structure 4375 * @systim: PHC time value read, sanitized and returned 4376 * @sts: structure to hold system time before and after reading SYSTIML, 4377 * may be NULL 4378 * 4379 * Errata for 82574/82583 possible bad bits read from SYSTIMH/L: 4380 * check to see that the time is incrementing at a reasonable 4381 * rate and is a multiple of incvalue. 4382 **/ 4383 static u64 e1000e_sanitize_systim(struct e1000_hw *hw, u64 systim, 4384 struct ptp_system_timestamp *sts) 4385 { 4386 u64 time_delta, rem, temp; 4387 u64 systim_next; 4388 u32 incvalue; 4389 int i; 4390 4391 incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK; 4392 for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) { 4393 /* latch SYSTIMH on read of SYSTIML */ 4394 ptp_read_system_prets(sts); 4395 systim_next = (u64)er32(SYSTIML); 4396 ptp_read_system_postts(sts); 4397 systim_next |= (u64)er32(SYSTIMH) << 32; 4398 4399 time_delta = systim_next - systim; 4400 temp = time_delta; 4401 /* VMWare users have seen incvalue of zero, don't div / 0 */ 4402 rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0); 4403 4404 systim = systim_next; 4405 4406 if ((time_delta < E1000_82574_SYSTIM_EPSILON) && (rem == 0)) 4407 break; 4408 } 4409 4410 return systim; 4411 } 4412 4413 /** 4414 * e1000e_read_systim - read SYSTIM register 4415 * @adapter: board private structure 4416 * @sts: structure which will contain system time before and after reading 4417 * SYSTIML, may be NULL 4418 **/ 4419 u64 e1000e_read_systim(struct e1000_adapter *adapter, 4420 struct ptp_system_timestamp *sts) 4421 { 4422 struct e1000_hw *hw = &adapter->hw; 4423 u32 systimel, systimel_2, systimeh; 4424 u64 systim; 4425 /* SYSTIMH latching upon SYSTIML read does not work well. 4426 * This means that if SYSTIML overflows after we read it but before 4427 * we read SYSTIMH, the value of SYSTIMH has been incremented and we 4428 * will experience a huge non linear increment in the systime value 4429 * to fix that we test for overflow and if true, we re-read systime. 4430 */ 4431 ptp_read_system_prets(sts); 4432 systimel = er32(SYSTIML); 4433 ptp_read_system_postts(sts); 4434 systimeh = er32(SYSTIMH); 4435 /* Is systimel is so large that overflow is possible? */ 4436 if (systimel >= (u32)0xffffffff - E1000_TIMINCA_INCVALUE_MASK) { 4437 ptp_read_system_prets(sts); 4438 systimel_2 = er32(SYSTIML); 4439 ptp_read_system_postts(sts); 4440 if (systimel > systimel_2) { 4441 /* There was an overflow, read again SYSTIMH, and use 4442 * systimel_2 4443 */ 4444 systimeh = er32(SYSTIMH); 4445 systimel = systimel_2; 4446 } 4447 } 4448 systim = (u64)systimel; 4449 systim |= (u64)systimeh << 32; 4450 4451 if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW) 4452 systim = e1000e_sanitize_systim(hw, systim, sts); 4453 4454 return systim; 4455 } 4456 4457 /** 4458 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter) 4459 * @cc: cyclecounter structure 4460 **/ 4461 static u64 e1000e_cyclecounter_read(struct cyclecounter *cc) 4462 { 4463 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter, 4464 cc); 4465 4466 return e1000e_read_systim(adapter, NULL); 4467 } 4468 4469 /** 4470 * e1000_sw_init - Initialize general software structures (struct e1000_adapter) 4471 * @adapter: board private structure to initialize 4472 * 4473 * e1000_sw_init initializes the Adapter private data structure. 4474 * Fields are initialized based on PCI device information and 4475 * OS network device settings (MTU size). 4476 **/ 4477 static int e1000_sw_init(struct e1000_adapter *adapter) 4478 { 4479 struct net_device *netdev = adapter->netdev; 4480 4481 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 4482 adapter->rx_ps_bsize0 = 128; 4483 adapter->max_frame_size = netdev->mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 4484 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4485 adapter->tx_ring_count = E1000_DEFAULT_TXD; 4486 adapter->rx_ring_count = E1000_DEFAULT_RXD; 4487 4488 spin_lock_init(&adapter->stats64_lock); 4489 4490 e1000e_set_interrupt_capability(adapter); 4491 4492 if (e1000_alloc_queues(adapter)) 4493 return -ENOMEM; 4494 4495 /* Setup hardware time stamping cyclecounter */ 4496 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 4497 adapter->cc.read = e1000e_cyclecounter_read; 4498 adapter->cc.mask = CYCLECOUNTER_MASK(64); 4499 adapter->cc.mult = 1; 4500 /* cc.shift set in e1000e_get_base_tininca() */ 4501 4502 spin_lock_init(&adapter->systim_lock); 4503 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work); 4504 } 4505 4506 /* Explicitly disable IRQ since the NIC can be in any state. */ 4507 e1000_irq_disable(adapter); 4508 4509 set_bit(__E1000_DOWN, &adapter->state); 4510 return 0; 4511 } 4512 4513 /** 4514 * e1000_intr_msi_test - Interrupt Handler 4515 * @irq: interrupt number 4516 * @data: pointer to a network interface device structure 4517 **/ 4518 static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data) 4519 { 4520 struct net_device *netdev = data; 4521 struct e1000_adapter *adapter = netdev_priv(netdev); 4522 struct e1000_hw *hw = &adapter->hw; 4523 u32 icr = er32(ICR); 4524 4525 e_dbg("icr is %08X\n", icr); 4526 if (icr & E1000_ICR_RXSEQ) { 4527 adapter->flags &= ~FLAG_MSI_TEST_FAILED; 4528 /* Force memory writes to complete before acknowledging the 4529 * interrupt is handled. 4530 */ 4531 wmb(); 4532 } 4533 4534 return IRQ_HANDLED; 4535 } 4536 4537 /** 4538 * e1000_test_msi_interrupt - Returns 0 for successful test 4539 * @adapter: board private struct 4540 * 4541 * code flow taken from tg3.c 4542 **/ 4543 static int e1000_test_msi_interrupt(struct e1000_adapter *adapter) 4544 { 4545 struct net_device *netdev = adapter->netdev; 4546 struct e1000_hw *hw = &adapter->hw; 4547 int err; 4548 4549 /* poll_enable hasn't been called yet, so don't need disable */ 4550 /* clear any pending events */ 4551 er32(ICR); 4552 4553 /* free the real vector and request a test handler */ 4554 e1000_free_irq(adapter); 4555 e1000e_reset_interrupt_capability(adapter); 4556 4557 /* Assume that the test fails, if it succeeds then the test 4558 * MSI irq handler will unset this flag 4559 */ 4560 adapter->flags |= FLAG_MSI_TEST_FAILED; 4561 4562 err = pci_enable_msi(adapter->pdev); 4563 if (err) 4564 goto msi_test_failed; 4565 4566 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0, 4567 netdev->name, netdev); 4568 if (err) { 4569 pci_disable_msi(adapter->pdev); 4570 goto msi_test_failed; 4571 } 4572 4573 /* Force memory writes to complete before enabling and firing an 4574 * interrupt. 4575 */ 4576 wmb(); 4577 4578 e1000_irq_enable(adapter); 4579 4580 /* fire an unusual interrupt on the test handler */ 4581 ew32(ICS, E1000_ICS_RXSEQ); 4582 e1e_flush(); 4583 msleep(100); 4584 4585 e1000_irq_disable(adapter); 4586 4587 rmb(); /* read flags after interrupt has been fired */ 4588 4589 if (adapter->flags & FLAG_MSI_TEST_FAILED) { 4590 adapter->int_mode = E1000E_INT_MODE_LEGACY; 4591 e_info("MSI interrupt test failed, using legacy interrupt.\n"); 4592 } else { 4593 e_dbg("MSI interrupt test succeeded!\n"); 4594 } 4595 4596 free_irq(adapter->pdev->irq, netdev); 4597 pci_disable_msi(adapter->pdev); 4598 4599 msi_test_failed: 4600 e1000e_set_interrupt_capability(adapter); 4601 return e1000_request_irq(adapter); 4602 } 4603 4604 /** 4605 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored 4606 * @adapter: board private struct 4607 * 4608 * code flow taken from tg3.c, called with e1000 interrupts disabled. 4609 **/ 4610 static int e1000_test_msi(struct e1000_adapter *adapter) 4611 { 4612 int err; 4613 u16 pci_cmd; 4614 4615 if (!(adapter->flags & FLAG_MSI_ENABLED)) 4616 return 0; 4617 4618 /* disable SERR in case the MSI write causes a master abort */ 4619 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4620 if (pci_cmd & PCI_COMMAND_SERR) 4621 pci_write_config_word(adapter->pdev, PCI_COMMAND, 4622 pci_cmd & ~PCI_COMMAND_SERR); 4623 4624 err = e1000_test_msi_interrupt(adapter); 4625 4626 /* re-enable SERR */ 4627 if (pci_cmd & PCI_COMMAND_SERR) { 4628 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd); 4629 pci_cmd |= PCI_COMMAND_SERR; 4630 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd); 4631 } 4632 4633 return err; 4634 } 4635 4636 /** 4637 * e1000e_open - Called when a network interface is made active 4638 * @netdev: network interface device structure 4639 * 4640 * Returns 0 on success, negative value on failure 4641 * 4642 * The open entry point is called when a network interface is made 4643 * active by the system (IFF_UP). At this point all resources needed 4644 * for transmit and receive operations are allocated, the interrupt 4645 * handler is registered with the OS, the watchdog timer is started, 4646 * and the stack is notified that the interface is ready. 4647 **/ 4648 int e1000e_open(struct net_device *netdev) 4649 { 4650 struct e1000_adapter *adapter = netdev_priv(netdev); 4651 struct e1000_hw *hw = &adapter->hw; 4652 struct pci_dev *pdev = adapter->pdev; 4653 int err; 4654 int irq; 4655 4656 /* disallow open during test */ 4657 if (test_bit(__E1000_TESTING, &adapter->state)) 4658 return -EBUSY; 4659 4660 pm_runtime_get_sync(&pdev->dev); 4661 4662 netif_carrier_off(netdev); 4663 netif_stop_queue(netdev); 4664 4665 /* allocate transmit descriptors */ 4666 err = e1000e_setup_tx_resources(adapter->tx_ring); 4667 if (err) 4668 goto err_setup_tx; 4669 4670 /* allocate receive descriptors */ 4671 err = e1000e_setup_rx_resources(adapter->rx_ring); 4672 if (err) 4673 goto err_setup_rx; 4674 4675 /* If AMT is enabled, let the firmware know that the network 4676 * interface is now open and reset the part to a known state. 4677 */ 4678 if (adapter->flags & FLAG_HAS_AMT) { 4679 e1000e_get_hw_control(adapter); 4680 e1000e_reset(adapter); 4681 } 4682 4683 e1000e_power_up_phy(adapter); 4684 4685 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; 4686 if ((adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN)) 4687 e1000_update_mng_vlan(adapter); 4688 4689 /* DMA latency requirement to workaround jumbo issue */ 4690 cpu_latency_qos_add_request(&adapter->pm_qos_req, PM_QOS_DEFAULT_VALUE); 4691 4692 /* before we allocate an interrupt, we must be ready to handle it. 4693 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt 4694 * as soon as we call pci_request_irq, so we have to setup our 4695 * clean_rx handler before we do so. 4696 */ 4697 e1000_configure(adapter); 4698 4699 err = e1000_request_irq(adapter); 4700 if (err) 4701 goto err_req_irq; 4702 4703 /* Work around PCIe errata with MSI interrupts causing some chipsets to 4704 * ignore e1000e MSI messages, which means we need to test our MSI 4705 * interrupt now 4706 */ 4707 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) { 4708 err = e1000_test_msi(adapter); 4709 if (err) { 4710 e_err("Interrupt allocation failed\n"); 4711 goto err_req_irq; 4712 } 4713 } 4714 4715 /* From here on the code is the same as e1000e_up() */ 4716 clear_bit(__E1000_DOWN, &adapter->state); 4717 4718 if (adapter->int_mode == E1000E_INT_MODE_MSIX) 4719 irq = adapter->msix_entries[0].vector; 4720 else 4721 irq = adapter->pdev->irq; 4722 4723 netif_napi_set_irq(&adapter->napi, irq); 4724 napi_enable(&adapter->napi); 4725 netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_RX, &adapter->napi); 4726 netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_TX, &adapter->napi); 4727 4728 e1000_irq_enable(adapter); 4729 4730 adapter->tx_hang_recheck = false; 4731 4732 hw->mac.get_link_status = true; 4733 pm_runtime_put(&pdev->dev); 4734 4735 e1000e_trigger_lsc(adapter); 4736 4737 return 0; 4738 4739 err_req_irq: 4740 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4741 e1000e_release_hw_control(adapter); 4742 e1000_power_down_phy(adapter); 4743 e1000e_free_rx_resources(adapter->rx_ring); 4744 err_setup_rx: 4745 e1000e_free_tx_resources(adapter->tx_ring); 4746 err_setup_tx: 4747 e1000e_reset(adapter); 4748 pm_runtime_put_sync(&pdev->dev); 4749 4750 return err; 4751 } 4752 4753 /** 4754 * e1000e_close - Disables a network interface 4755 * @netdev: network interface device structure 4756 * 4757 * Returns 0, this is not allowed to fail 4758 * 4759 * The close entry point is called when an interface is de-activated 4760 * by the OS. The hardware is still under the drivers control, but 4761 * needs to be disabled. A global MAC reset is issued to stop the 4762 * hardware, and all transmit and receive resources are freed. 4763 **/ 4764 int e1000e_close(struct net_device *netdev) 4765 { 4766 struct e1000_adapter *adapter = netdev_priv(netdev); 4767 struct pci_dev *pdev = adapter->pdev; 4768 int count = E1000_CHECK_RESET_COUNT; 4769 4770 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 4771 usleep_range(10000, 11000); 4772 4773 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 4774 4775 pm_runtime_get_sync(&pdev->dev); 4776 4777 if (netif_device_present(netdev)) { 4778 e1000e_down(adapter, true); 4779 e1000_free_irq(adapter); 4780 4781 /* Link status message must follow this format */ 4782 netdev_info(netdev, "NIC Link is Down\n"); 4783 } 4784 4785 netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_RX, NULL); 4786 netif_queue_set_napi(netdev, 0, NETDEV_QUEUE_TYPE_TX, NULL); 4787 napi_disable(&adapter->napi); 4788 4789 e1000e_free_tx_resources(adapter->tx_ring); 4790 e1000e_free_rx_resources(adapter->rx_ring); 4791 4792 /* kill manageability vlan ID if supported, but not if a vlan with 4793 * the same ID is registered on the host OS (let 8021q kill it) 4794 */ 4795 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) 4796 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), 4797 adapter->mng_vlan_id); 4798 4799 /* If AMT is enabled, let the firmware know that the network 4800 * interface is now closed 4801 */ 4802 if ((adapter->flags & FLAG_HAS_AMT) && 4803 !test_bit(__E1000_TESTING, &adapter->state)) 4804 e1000e_release_hw_control(adapter); 4805 4806 cpu_latency_qos_remove_request(&adapter->pm_qos_req); 4807 4808 pm_runtime_put_sync(&pdev->dev); 4809 4810 return 0; 4811 } 4812 4813 /** 4814 * e1000_set_mac - Change the Ethernet Address of the NIC 4815 * @netdev: network interface device structure 4816 * @p: pointer to an address structure 4817 * 4818 * Returns 0 on success, negative on failure 4819 **/ 4820 static int e1000_set_mac(struct net_device *netdev, void *p) 4821 { 4822 struct e1000_adapter *adapter = netdev_priv(netdev); 4823 struct e1000_hw *hw = &adapter->hw; 4824 struct sockaddr *addr = p; 4825 4826 if (!is_valid_ether_addr(addr->sa_data)) 4827 return -EADDRNOTAVAIL; 4828 4829 eth_hw_addr_set(netdev, addr->sa_data); 4830 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len); 4831 4832 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 4833 4834 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) { 4835 /* activate the work around */ 4836 e1000e_set_laa_state_82571(&adapter->hw, 1); 4837 4838 /* Hold a copy of the LAA in RAR[14] This is done so that 4839 * between the time RAR[0] gets clobbered and the time it 4840 * gets fixed (in e1000_watchdog), the actual LAA is in one 4841 * of the RARs and no incoming packets directed to this port 4842 * are dropped. Eventually the LAA will be in RAR[0] and 4843 * RAR[14] 4844 */ 4845 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 4846 adapter->hw.mac.rar_entry_count - 1); 4847 } 4848 4849 return 0; 4850 } 4851 4852 /** 4853 * e1000e_update_phy_task - work thread to update phy 4854 * @work: pointer to our work struct 4855 * 4856 * this worker thread exists because we must acquire a 4857 * semaphore to read the phy, which we could msleep while 4858 * waiting for it, and we can't msleep in a timer. 4859 **/ 4860 static void e1000e_update_phy_task(struct work_struct *work) 4861 { 4862 struct e1000_adapter *adapter = container_of(work, 4863 struct e1000_adapter, 4864 update_phy_task); 4865 struct e1000_hw *hw = &adapter->hw; 4866 4867 if (test_bit(__E1000_DOWN, &adapter->state)) 4868 return; 4869 4870 e1000_get_phy_info(hw); 4871 4872 /* Enable EEE on 82579 after link up */ 4873 if (hw->phy.type >= e1000_phy_82579) 4874 e1000_set_eee_pchlan(hw); 4875 } 4876 4877 /** 4878 * e1000_update_phy_info - timre call-back to update PHY info 4879 * @t: pointer to timer_list containing private info adapter 4880 * 4881 * Need to wait a few seconds after link up to get diagnostic information from 4882 * the phy 4883 **/ 4884 static void e1000_update_phy_info(struct timer_list *t) 4885 { 4886 struct e1000_adapter *adapter = timer_container_of(adapter, t, 4887 phy_info_timer); 4888 4889 if (test_bit(__E1000_DOWN, &adapter->state)) 4890 return; 4891 4892 schedule_work(&adapter->update_phy_task); 4893 } 4894 4895 /** 4896 * e1000e_update_phy_stats - Update the PHY statistics counters 4897 * @adapter: board private structure 4898 * 4899 * Read/clear the upper 16-bit PHY registers and read/accumulate lower 4900 **/ 4901 static void e1000e_update_phy_stats(struct e1000_adapter *adapter) 4902 { 4903 struct e1000_hw *hw = &adapter->hw; 4904 s32 ret_val; 4905 u16 phy_data; 4906 4907 ret_val = hw->phy.ops.acquire(hw); 4908 if (ret_val) 4909 return; 4910 4911 /* A page set is expensive so check if already on desired page. 4912 * If not, set to the page with the PHY status registers. 4913 */ 4914 hw->phy.addr = 1; 4915 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4916 &phy_data); 4917 if (ret_val) 4918 goto release; 4919 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) { 4920 ret_val = hw->phy.ops.set_page(hw, 4921 HV_STATS_PAGE << IGP_PAGE_SHIFT); 4922 if (ret_val) 4923 goto release; 4924 } 4925 4926 /* Single Collision Count */ 4927 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data); 4928 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data); 4929 if (!ret_val) 4930 adapter->stats.scc += phy_data; 4931 4932 /* Excessive Collision Count */ 4933 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data); 4934 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data); 4935 if (!ret_val) 4936 adapter->stats.ecol += phy_data; 4937 4938 /* Multiple Collision Count */ 4939 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data); 4940 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data); 4941 if (!ret_val) 4942 adapter->stats.mcc += phy_data; 4943 4944 /* Late Collision Count */ 4945 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data); 4946 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data); 4947 if (!ret_val) 4948 adapter->stats.latecol += phy_data; 4949 4950 /* Collision Count - also used for adaptive IFS */ 4951 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data); 4952 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data); 4953 if (!ret_val) 4954 hw->mac.collision_delta = phy_data; 4955 4956 /* Defer Count */ 4957 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data); 4958 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data); 4959 if (!ret_val) 4960 adapter->stats.dc += phy_data; 4961 4962 /* Transmit with no CRS */ 4963 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data); 4964 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data); 4965 if (!ret_val) 4966 adapter->stats.tncrs += phy_data; 4967 4968 release: 4969 hw->phy.ops.release(hw); 4970 } 4971 4972 /** 4973 * e1000e_update_stats - Update the board statistics counters 4974 * @adapter: board private structure 4975 **/ 4976 static void e1000e_update_stats(struct e1000_adapter *adapter) 4977 { 4978 struct net_device *netdev = adapter->netdev; 4979 struct e1000_hw *hw = &adapter->hw; 4980 struct pci_dev *pdev = adapter->pdev; 4981 4982 /* Prevent stats update while adapter is being reset, or if the pci 4983 * connection is down. 4984 */ 4985 if (adapter->link_speed == 0) 4986 return; 4987 if (pci_channel_offline(pdev)) 4988 return; 4989 4990 adapter->stats.crcerrs += er32(CRCERRS); 4991 adapter->stats.gprc += er32(GPRC); 4992 adapter->stats.gorc += er32(GORCL); 4993 er32(GORCH); /* Clear gorc */ 4994 adapter->stats.bprc += er32(BPRC); 4995 adapter->stats.mprc += er32(MPRC); 4996 adapter->stats.roc += er32(ROC); 4997 4998 adapter->stats.mpc += er32(MPC); 4999 5000 /* Half-duplex statistics */ 5001 if (adapter->link_duplex == HALF_DUPLEX) { 5002 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) { 5003 e1000e_update_phy_stats(adapter); 5004 } else { 5005 adapter->stats.scc += er32(SCC); 5006 adapter->stats.ecol += er32(ECOL); 5007 adapter->stats.mcc += er32(MCC); 5008 adapter->stats.latecol += er32(LATECOL); 5009 adapter->stats.dc += er32(DC); 5010 5011 hw->mac.collision_delta = er32(COLC); 5012 5013 if ((hw->mac.type != e1000_82574) && 5014 (hw->mac.type != e1000_82583)) 5015 adapter->stats.tncrs += er32(TNCRS); 5016 } 5017 adapter->stats.colc += hw->mac.collision_delta; 5018 } 5019 5020 adapter->stats.xonrxc += er32(XONRXC); 5021 adapter->stats.xontxc += er32(XONTXC); 5022 adapter->stats.xoffrxc += er32(XOFFRXC); 5023 adapter->stats.xofftxc += er32(XOFFTXC); 5024 adapter->stats.gptc += er32(GPTC); 5025 adapter->stats.gotc += er32(GOTCL); 5026 er32(GOTCH); /* Clear gotc */ 5027 adapter->stats.rnbc += er32(RNBC); 5028 adapter->stats.ruc += er32(RUC); 5029 5030 adapter->stats.mptc += er32(MPTC); 5031 adapter->stats.bptc += er32(BPTC); 5032 5033 /* used for adaptive IFS */ 5034 5035 hw->mac.tx_packet_delta = er32(TPT); 5036 adapter->stats.tpt += hw->mac.tx_packet_delta; 5037 5038 adapter->stats.algnerrc += er32(ALGNERRC); 5039 adapter->stats.rxerrc += er32(RXERRC); 5040 adapter->stats.cexterr += er32(CEXTERR); 5041 adapter->stats.tsctc += er32(TSCTC); 5042 adapter->stats.tsctfc += er32(TSCTFC); 5043 5044 /* Fill out the OS statistics structure */ 5045 netdev->stats.multicast = adapter->stats.mprc; 5046 netdev->stats.collisions = adapter->stats.colc; 5047 5048 /* Rx Errors */ 5049 5050 /* RLEC on some newer hardware can be incorrect so build 5051 * our own version based on RUC and ROC 5052 */ 5053 netdev->stats.rx_errors = adapter->stats.rxerrc + 5054 adapter->stats.crcerrs + adapter->stats.algnerrc + 5055 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 5056 netdev->stats.rx_length_errors = adapter->stats.ruc + 5057 adapter->stats.roc; 5058 netdev->stats.rx_crc_errors = adapter->stats.crcerrs; 5059 netdev->stats.rx_frame_errors = adapter->stats.algnerrc; 5060 netdev->stats.rx_missed_errors = adapter->stats.mpc; 5061 5062 /* Tx Errors */ 5063 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol; 5064 netdev->stats.tx_aborted_errors = adapter->stats.ecol; 5065 netdev->stats.tx_window_errors = adapter->stats.latecol; 5066 netdev->stats.tx_carrier_errors = adapter->stats.tncrs; 5067 5068 /* Tx Dropped needs to be maintained elsewhere */ 5069 5070 /* Management Stats */ 5071 adapter->stats.mgptc += er32(MGTPTC); 5072 adapter->stats.mgprc += er32(MGTPRC); 5073 adapter->stats.mgpdc += er32(MGTPDC); 5074 5075 /* Correctable ECC Errors */ 5076 if (hw->mac.type >= e1000_pch_lpt) { 5077 u32 pbeccsts = er32(PBECCSTS); 5078 5079 adapter->corr_errors += 5080 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK; 5081 adapter->uncorr_errors += 5082 FIELD_GET(E1000_PBECCSTS_UNCORR_ERR_CNT_MASK, pbeccsts); 5083 } 5084 } 5085 5086 /** 5087 * e1000_phy_read_status - Update the PHY register status snapshot 5088 * @adapter: board private structure 5089 **/ 5090 static void e1000_phy_read_status(struct e1000_adapter *adapter) 5091 { 5092 struct e1000_hw *hw = &adapter->hw; 5093 struct e1000_phy_regs *phy = &adapter->phy_regs; 5094 5095 if (!pm_runtime_suspended((&adapter->pdev->dev)->parent) && 5096 (er32(STATUS) & E1000_STATUS_LU) && 5097 (adapter->hw.phy.media_type == e1000_media_type_copper)) { 5098 int ret_val; 5099 5100 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr); 5101 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr); 5102 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise); 5103 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa); 5104 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion); 5105 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000); 5106 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000); 5107 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus); 5108 if (ret_val) 5109 e_warn("Error reading PHY register\n"); 5110 } else { 5111 /* Do not read PHY registers if link is not up 5112 * Set values to typical power-on defaults 5113 */ 5114 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX); 5115 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL | 5116 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE | 5117 BMSR_ERCAP); 5118 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP | 5119 ADVERTISE_ALL | ADVERTISE_CSMA); 5120 phy->lpa = 0; 5121 phy->expansion = EXPANSION_ENABLENPAGE; 5122 phy->ctrl1000 = ADVERTISE_1000FULL; 5123 phy->stat1000 = 0; 5124 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF); 5125 } 5126 } 5127 5128 static void e1000_print_link_info(struct e1000_adapter *adapter) 5129 { 5130 struct e1000_hw *hw = &adapter->hw; 5131 u32 ctrl = er32(CTRL); 5132 5133 /* Link status message must follow this format for user tools */ 5134 netdev_info(adapter->netdev, 5135 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5136 adapter->link_speed, 5137 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half", 5138 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" : 5139 (ctrl & E1000_CTRL_RFCE) ? "Rx" : 5140 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None"); 5141 } 5142 5143 static bool e1000e_has_link(struct e1000_adapter *adapter) 5144 { 5145 struct e1000_hw *hw = &adapter->hw; 5146 bool link_active = false; 5147 s32 ret_val = 0; 5148 5149 /* get_link_status is set on LSC (link status) interrupt or 5150 * Rx sequence error interrupt. get_link_status will stay 5151 * true until the check_for_link establishes link 5152 * for copper adapters ONLY 5153 */ 5154 switch (hw->phy.media_type) { 5155 case e1000_media_type_copper: 5156 if (hw->mac.get_link_status) { 5157 ret_val = hw->mac.ops.check_for_link(hw); 5158 link_active = !hw->mac.get_link_status; 5159 } else { 5160 link_active = true; 5161 } 5162 break; 5163 case e1000_media_type_fiber: 5164 ret_val = hw->mac.ops.check_for_link(hw); 5165 link_active = !!(er32(STATUS) & E1000_STATUS_LU); 5166 break; 5167 case e1000_media_type_internal_serdes: 5168 ret_val = hw->mac.ops.check_for_link(hw); 5169 link_active = hw->mac.serdes_has_link; 5170 break; 5171 default: 5172 case e1000_media_type_unknown: 5173 break; 5174 } 5175 5176 if ((ret_val == -E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) && 5177 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) { 5178 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */ 5179 e_info("Gigabit has been disabled, downgrading speed\n"); 5180 } 5181 5182 return link_active; 5183 } 5184 5185 static void e1000e_enable_receives(struct e1000_adapter *adapter) 5186 { 5187 /* make sure the receive unit is started */ 5188 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) && 5189 (adapter->flags & FLAG_RESTART_NOW)) { 5190 struct e1000_hw *hw = &adapter->hw; 5191 u32 rctl = er32(RCTL); 5192 5193 ew32(RCTL, rctl | E1000_RCTL_EN); 5194 adapter->flags &= ~FLAG_RESTART_NOW; 5195 } 5196 } 5197 5198 static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter) 5199 { 5200 struct e1000_hw *hw = &adapter->hw; 5201 5202 /* With 82574 controllers, PHY needs to be checked periodically 5203 * for hung state and reset, if two calls return true 5204 */ 5205 if (e1000_check_phy_82574(hw)) 5206 adapter->phy_hang_count++; 5207 else 5208 adapter->phy_hang_count = 0; 5209 5210 if (adapter->phy_hang_count > 1) { 5211 adapter->phy_hang_count = 0; 5212 e_dbg("PHY appears hung - resetting\n"); 5213 schedule_work(&adapter->reset_task); 5214 } 5215 } 5216 5217 /** 5218 * e1000_watchdog - Timer Call-back 5219 * @t: pointer to timer_list containing private info adapter 5220 **/ 5221 static void e1000_watchdog(struct timer_list *t) 5222 { 5223 struct e1000_adapter *adapter = timer_container_of(adapter, t, 5224 watchdog_timer); 5225 5226 /* Do the rest outside of interrupt context */ 5227 schedule_work(&adapter->watchdog_task); 5228 5229 /* TODO: make this use queue_delayed_work() */ 5230 } 5231 5232 static void e1000_watchdog_task(struct work_struct *work) 5233 { 5234 struct e1000_adapter *adapter = container_of(work, 5235 struct e1000_adapter, 5236 watchdog_task); 5237 struct net_device *netdev = adapter->netdev; 5238 struct e1000_mac_info *mac = &adapter->hw.mac; 5239 struct e1000_phy_info *phy = &adapter->hw.phy; 5240 struct e1000_ring *tx_ring = adapter->tx_ring; 5241 u32 dmoff_exit_timeout = 100, tries = 0; 5242 struct e1000_hw *hw = &adapter->hw; 5243 u32 link, tctl, pcim_state; 5244 5245 if (test_bit(__E1000_DOWN, &adapter->state)) 5246 return; 5247 5248 link = e1000e_has_link(adapter); 5249 if ((netif_carrier_ok(netdev)) && link) { 5250 /* Cancel scheduled suspend requests. */ 5251 pm_runtime_resume(netdev->dev.parent); 5252 5253 e1000e_enable_receives(adapter); 5254 goto link_up; 5255 } 5256 5257 if ((e1000e_enable_tx_pkt_filtering(hw)) && 5258 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id)) 5259 e1000_update_mng_vlan(adapter); 5260 5261 if (link) { 5262 if (!netif_carrier_ok(netdev)) { 5263 bool txb2b = true; 5264 5265 /* Cancel scheduled suspend requests. */ 5266 pm_runtime_resume(netdev->dev.parent); 5267 5268 /* Checking if MAC is in DMoff state*/ 5269 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) { 5270 pcim_state = er32(STATUS); 5271 while (pcim_state & E1000_STATUS_PCIM_STATE) { 5272 if (tries++ == dmoff_exit_timeout) { 5273 e_dbg("Error in exiting dmoff\n"); 5274 break; 5275 } 5276 usleep_range(10000, 20000); 5277 pcim_state = er32(STATUS); 5278 5279 /* Checking if MAC exited DMoff state */ 5280 if (!(pcim_state & E1000_STATUS_PCIM_STATE)) 5281 e1000_phy_hw_reset(&adapter->hw); 5282 } 5283 } 5284 5285 /* update snapshot of PHY registers on LSC */ 5286 e1000_phy_read_status(adapter); 5287 mac->ops.get_link_up_info(&adapter->hw, 5288 &adapter->link_speed, 5289 &adapter->link_duplex); 5290 e1000_print_link_info(adapter); 5291 5292 /* check if SmartSpeed worked */ 5293 e1000e_check_downshift(hw); 5294 if (phy->speed_downgraded) 5295 netdev_warn(netdev, 5296 "Link Speed was downgraded by SmartSpeed\n"); 5297 5298 /* On supported PHYs, check for duplex mismatch only 5299 * if link has autonegotiated at 10/100 half 5300 */ 5301 if ((hw->phy.type == e1000_phy_igp_3 || 5302 hw->phy.type == e1000_phy_bm) && 5303 hw->mac.autoneg && 5304 (adapter->link_speed == SPEED_10 || 5305 adapter->link_speed == SPEED_100) && 5306 (adapter->link_duplex == HALF_DUPLEX)) { 5307 u16 autoneg_exp; 5308 5309 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp); 5310 5311 if (!(autoneg_exp & EXPANSION_NWAY)) 5312 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n"); 5313 } 5314 5315 /* adjust timeout factor according to speed/duplex */ 5316 adapter->tx_timeout_factor = 1; 5317 switch (adapter->link_speed) { 5318 case SPEED_10: 5319 txb2b = false; 5320 adapter->tx_timeout_factor = 16; 5321 break; 5322 case SPEED_100: 5323 txb2b = false; 5324 adapter->tx_timeout_factor = 10; 5325 break; 5326 } 5327 5328 /* workaround: re-program speed mode bit after 5329 * link-up event 5330 */ 5331 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) && 5332 !txb2b) { 5333 u32 tarc0; 5334 5335 tarc0 = er32(TARC(0)); 5336 tarc0 &= ~SPEED_MODE_BIT; 5337 ew32(TARC(0), tarc0); 5338 } 5339 5340 /* enable transmits in the hardware, need to do this 5341 * after setting TARC(0) 5342 */ 5343 tctl = er32(TCTL); 5344 tctl |= E1000_TCTL_EN; 5345 ew32(TCTL, tctl); 5346 5347 /* Perform any post-link-up configuration before 5348 * reporting link up. 5349 */ 5350 if (phy->ops.cfg_on_link_up) 5351 phy->ops.cfg_on_link_up(hw); 5352 5353 netif_wake_queue(netdev); 5354 netif_carrier_on(netdev); 5355 5356 if (!test_bit(__E1000_DOWN, &adapter->state)) 5357 mod_timer(&adapter->phy_info_timer, 5358 round_jiffies(jiffies + 2 * HZ)); 5359 } 5360 } else { 5361 if (netif_carrier_ok(netdev)) { 5362 adapter->link_speed = 0; 5363 adapter->link_duplex = 0; 5364 /* Link status message must follow this format */ 5365 netdev_info(netdev, "NIC Link is Down\n"); 5366 netif_carrier_off(netdev); 5367 netif_stop_queue(netdev); 5368 if (!test_bit(__E1000_DOWN, &adapter->state)) 5369 mod_timer(&adapter->phy_info_timer, 5370 round_jiffies(jiffies + 2 * HZ)); 5371 5372 /* 8000ES2LAN requires a Rx packet buffer work-around 5373 * on link down event; reset the controller to flush 5374 * the Rx packet buffer. 5375 */ 5376 if (adapter->flags & FLAG_RX_NEEDS_RESTART) 5377 adapter->flags |= FLAG_RESTART_NOW; 5378 else 5379 pm_schedule_suspend(netdev->dev.parent, 5380 LINK_TIMEOUT); 5381 } 5382 } 5383 5384 link_up: 5385 spin_lock(&adapter->stats64_lock); 5386 e1000e_update_stats(adapter); 5387 5388 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old; 5389 adapter->tpt_old = adapter->stats.tpt; 5390 mac->collision_delta = adapter->stats.colc - adapter->colc_old; 5391 adapter->colc_old = adapter->stats.colc; 5392 5393 adapter->gorc = adapter->stats.gorc - adapter->gorc_old; 5394 adapter->gorc_old = adapter->stats.gorc; 5395 adapter->gotc = adapter->stats.gotc - adapter->gotc_old; 5396 adapter->gotc_old = adapter->stats.gotc; 5397 spin_unlock(&adapter->stats64_lock); 5398 5399 /* If the link is lost the controller stops DMA, but 5400 * if there is queued Tx work it cannot be done. So 5401 * reset the controller to flush the Tx packet buffers. 5402 */ 5403 if (!netif_carrier_ok(netdev) && 5404 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count)) 5405 adapter->flags |= FLAG_RESTART_NOW; 5406 5407 /* If reset is necessary, do it outside of interrupt context. */ 5408 if (adapter->flags & FLAG_RESTART_NOW) { 5409 schedule_work(&adapter->reset_task); 5410 /* return immediately since reset is imminent */ 5411 return; 5412 } 5413 5414 e1000e_update_adaptive(&adapter->hw); 5415 5416 /* Simple mode for Interrupt Throttle Rate (ITR) */ 5417 if (adapter->itr_setting == 4) { 5418 /* Symmetric Tx/Rx gets a reduced ITR=2000; 5419 * Total asymmetrical Tx or Rx gets ITR=8000; 5420 * everyone else is between 2000-8000. 5421 */ 5422 u32 goc = (adapter->gotc + adapter->gorc) / 10000; 5423 u32 dif = (adapter->gotc > adapter->gorc ? 5424 adapter->gotc - adapter->gorc : 5425 adapter->gorc - adapter->gotc) / 10000; 5426 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000; 5427 5428 e1000e_write_itr(adapter, itr); 5429 } 5430 5431 /* Cause software interrupt to ensure Rx ring is cleaned */ 5432 if (adapter->msix_entries) 5433 ew32(ICS, adapter->rx_ring->ims_val); 5434 else 5435 ew32(ICS, E1000_ICS_RXDMT0); 5436 5437 /* flush pending descriptors to memory before detecting Tx hang */ 5438 e1000e_flush_descriptors(adapter); 5439 5440 /* Force detection of hung controller every watchdog period */ 5441 adapter->detect_tx_hung = true; 5442 5443 /* With 82571 controllers, LAA may be overwritten due to controller 5444 * reset from the other port. Set the appropriate LAA in RAR[0] 5445 */ 5446 if (e1000e_get_laa_state_82571(hw)) 5447 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0); 5448 5449 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG) 5450 e1000e_check_82574_phy_workaround(adapter); 5451 5452 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */ 5453 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) { 5454 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) && 5455 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) { 5456 er32(RXSTMPH); 5457 adapter->rx_hwtstamp_cleared++; 5458 } else { 5459 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP; 5460 } 5461 } 5462 5463 /* Reset the timer */ 5464 if (!test_bit(__E1000_DOWN, &adapter->state)) 5465 mod_timer(&adapter->watchdog_timer, 5466 round_jiffies(jiffies + 2 * HZ)); 5467 } 5468 5469 #define E1000_TX_FLAGS_CSUM 0x00000001 5470 #define E1000_TX_FLAGS_VLAN 0x00000002 5471 #define E1000_TX_FLAGS_TSO 0x00000004 5472 #define E1000_TX_FLAGS_IPV4 0x00000008 5473 #define E1000_TX_FLAGS_NO_FCS 0x00000010 5474 #define E1000_TX_FLAGS_HWTSTAMP 0x00000020 5475 #define E1000_TX_FLAGS_VLAN_MASK 0xffff0000 5476 #define E1000_TX_FLAGS_VLAN_SHIFT 16 5477 5478 static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb, 5479 __be16 protocol) 5480 { 5481 struct e1000_context_desc *context_desc; 5482 struct e1000_buffer *buffer_info; 5483 unsigned int i; 5484 u32 cmd_length = 0; 5485 u16 ipcse = 0, mss; 5486 u8 ipcss, ipcso, tucss, tucso, hdr_len; 5487 int err; 5488 5489 if (!skb_is_gso(skb)) 5490 return 0; 5491 5492 err = skb_cow_head(skb, 0); 5493 if (err < 0) 5494 return err; 5495 5496 hdr_len = skb_tcp_all_headers(skb); 5497 mss = skb_shinfo(skb)->gso_size; 5498 if (protocol == htons(ETH_P_IP)) { 5499 struct iphdr *iph = ip_hdr(skb); 5500 iph->tot_len = 0; 5501 iph->check = 0; 5502 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 5503 0, IPPROTO_TCP, 0); 5504 cmd_length = E1000_TXD_CMD_IP; 5505 ipcse = skb_transport_offset(skb) - 1; 5506 } else if (skb_is_gso_v6(skb)) { 5507 tcp_v6_gso_csum_prep(skb); 5508 ipcse = 0; 5509 } 5510 ipcss = skb_network_offset(skb); 5511 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data; 5512 tucss = skb_transport_offset(skb); 5513 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data; 5514 5515 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE | 5516 E1000_TXD_CMD_TCP | (skb->len - (hdr_len))); 5517 5518 i = tx_ring->next_to_use; 5519 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5520 buffer_info = &tx_ring->buffer_info[i]; 5521 5522 context_desc->lower_setup.ip_fields.ipcss = ipcss; 5523 context_desc->lower_setup.ip_fields.ipcso = ipcso; 5524 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse); 5525 context_desc->upper_setup.tcp_fields.tucss = tucss; 5526 context_desc->upper_setup.tcp_fields.tucso = tucso; 5527 context_desc->upper_setup.tcp_fields.tucse = 0; 5528 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss); 5529 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len; 5530 context_desc->cmd_and_length = cpu_to_le32(cmd_length); 5531 5532 buffer_info->time_stamp = jiffies; 5533 buffer_info->next_to_watch = i; 5534 5535 i++; 5536 if (i == tx_ring->count) 5537 i = 0; 5538 tx_ring->next_to_use = i; 5539 5540 return 1; 5541 } 5542 5543 static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb, 5544 __be16 protocol) 5545 { 5546 struct e1000_adapter *adapter = tx_ring->adapter; 5547 struct e1000_context_desc *context_desc; 5548 struct e1000_buffer *buffer_info; 5549 unsigned int i; 5550 u8 css; 5551 u32 cmd_len = E1000_TXD_CMD_DEXT; 5552 5553 if (skb->ip_summed != CHECKSUM_PARTIAL) 5554 return false; 5555 5556 switch (protocol) { 5557 case cpu_to_be16(ETH_P_IP): 5558 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 5559 cmd_len |= E1000_TXD_CMD_TCP; 5560 break; 5561 case cpu_to_be16(ETH_P_IPV6): 5562 /* XXX not handling all IPV6 headers */ 5563 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 5564 cmd_len |= E1000_TXD_CMD_TCP; 5565 break; 5566 default: 5567 if (unlikely(net_ratelimit())) 5568 e_warn("checksum_partial proto=%x!\n", 5569 be16_to_cpu(protocol)); 5570 break; 5571 } 5572 5573 css = skb_checksum_start_offset(skb); 5574 5575 i = tx_ring->next_to_use; 5576 buffer_info = &tx_ring->buffer_info[i]; 5577 context_desc = E1000_CONTEXT_DESC(*tx_ring, i); 5578 5579 context_desc->lower_setup.ip_config = 0; 5580 context_desc->upper_setup.tcp_fields.tucss = css; 5581 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset; 5582 context_desc->upper_setup.tcp_fields.tucse = 0; 5583 context_desc->tcp_seg_setup.data = 0; 5584 context_desc->cmd_and_length = cpu_to_le32(cmd_len); 5585 5586 buffer_info->time_stamp = jiffies; 5587 buffer_info->next_to_watch = i; 5588 5589 i++; 5590 if (i == tx_ring->count) 5591 i = 0; 5592 tx_ring->next_to_use = i; 5593 5594 return true; 5595 } 5596 5597 static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb, 5598 unsigned int first, unsigned int max_per_txd, 5599 unsigned int nr_frags) 5600 { 5601 struct e1000_adapter *adapter = tx_ring->adapter; 5602 struct pci_dev *pdev = adapter->pdev; 5603 struct e1000_buffer *buffer_info; 5604 unsigned int len = skb_headlen(skb); 5605 unsigned int offset = 0, size, count = 0, i; 5606 unsigned int f, bytecount, segs; 5607 5608 i = tx_ring->next_to_use; 5609 5610 while (len) { 5611 buffer_info = &tx_ring->buffer_info[i]; 5612 size = min(len, max_per_txd); 5613 5614 buffer_info->length = size; 5615 buffer_info->time_stamp = jiffies; 5616 buffer_info->next_to_watch = i; 5617 buffer_info->dma = dma_map_single(&pdev->dev, 5618 skb->data + offset, 5619 size, DMA_TO_DEVICE); 5620 buffer_info->mapped_as_page = false; 5621 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5622 goto dma_error; 5623 5624 len -= size; 5625 offset += size; 5626 count++; 5627 5628 if (len) { 5629 i++; 5630 if (i == tx_ring->count) 5631 i = 0; 5632 } 5633 } 5634 5635 for (f = 0; f < nr_frags; f++) { 5636 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 5637 5638 len = skb_frag_size(frag); 5639 offset = 0; 5640 5641 while (len) { 5642 i++; 5643 if (i == tx_ring->count) 5644 i = 0; 5645 5646 buffer_info = &tx_ring->buffer_info[i]; 5647 size = min(len, max_per_txd); 5648 5649 buffer_info->length = size; 5650 buffer_info->time_stamp = jiffies; 5651 buffer_info->next_to_watch = i; 5652 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 5653 offset, size, 5654 DMA_TO_DEVICE); 5655 buffer_info->mapped_as_page = true; 5656 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 5657 goto dma_error; 5658 5659 len -= size; 5660 offset += size; 5661 count++; 5662 } 5663 } 5664 5665 segs = skb_shinfo(skb)->gso_segs ? : 1; 5666 /* multiply data chunks by size of headers */ 5667 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len; 5668 5669 tx_ring->buffer_info[i].skb = skb; 5670 tx_ring->buffer_info[i].segs = segs; 5671 tx_ring->buffer_info[i].bytecount = bytecount; 5672 tx_ring->buffer_info[first].next_to_watch = i; 5673 5674 return count; 5675 5676 dma_error: 5677 dev_err(&pdev->dev, "Tx DMA map failed\n"); 5678 buffer_info->dma = 0; 5679 5680 while (count--) { 5681 if (i == 0) 5682 i += tx_ring->count; 5683 i--; 5684 buffer_info = &tx_ring->buffer_info[i]; 5685 e1000_put_txbuf(tx_ring, buffer_info, true); 5686 } 5687 5688 return 0; 5689 } 5690 5691 static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count) 5692 { 5693 struct e1000_adapter *adapter = tx_ring->adapter; 5694 struct e1000_tx_desc *tx_desc = NULL; 5695 struct e1000_buffer *buffer_info; 5696 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS; 5697 unsigned int i; 5698 5699 if (tx_flags & E1000_TX_FLAGS_TSO) { 5700 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D | 5701 E1000_TXD_CMD_TSE; 5702 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5703 5704 if (tx_flags & E1000_TX_FLAGS_IPV4) 5705 txd_upper |= E1000_TXD_POPTS_IXSM << 8; 5706 } 5707 5708 if (tx_flags & E1000_TX_FLAGS_CSUM) { 5709 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5710 txd_upper |= E1000_TXD_POPTS_TXSM << 8; 5711 } 5712 5713 if (tx_flags & E1000_TX_FLAGS_VLAN) { 5714 txd_lower |= E1000_TXD_CMD_VLE; 5715 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK); 5716 } 5717 5718 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5719 txd_lower &= ~(E1000_TXD_CMD_IFCS); 5720 5721 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) { 5722 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 5723 txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 5724 } 5725 5726 i = tx_ring->next_to_use; 5727 5728 do { 5729 buffer_info = &tx_ring->buffer_info[i]; 5730 tx_desc = E1000_TX_DESC(*tx_ring, i); 5731 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma); 5732 tx_desc->lower.data = cpu_to_le32(txd_lower | 5733 buffer_info->length); 5734 tx_desc->upper.data = cpu_to_le32(txd_upper); 5735 5736 i++; 5737 if (i == tx_ring->count) 5738 i = 0; 5739 } while (--count > 0); 5740 5741 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd); 5742 5743 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */ 5744 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS)) 5745 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS)); 5746 5747 /* Force memory writes to complete before letting h/w 5748 * know there are new descriptors to fetch. (Only 5749 * applicable for weak-ordered memory model archs, 5750 * such as IA-64). 5751 */ 5752 wmb(); 5753 5754 tx_ring->next_to_use = i; 5755 } 5756 5757 #define MINIMUM_DHCP_PACKET_SIZE 282 5758 static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter, 5759 struct sk_buff *skb) 5760 { 5761 struct e1000_hw *hw = &adapter->hw; 5762 u16 length, offset; 5763 5764 if (skb_vlan_tag_present(skb) && 5765 !((skb_vlan_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) && 5766 (adapter->hw.mng_cookie.status & 5767 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))) 5768 return 0; 5769 5770 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE) 5771 return 0; 5772 5773 if (((struct ethhdr *)skb->data)->h_proto != htons(ETH_P_IP)) 5774 return 0; 5775 5776 { 5777 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14); 5778 struct udphdr *udp; 5779 5780 if (ip->protocol != IPPROTO_UDP) 5781 return 0; 5782 5783 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2)); 5784 if (ntohs(udp->dest) != 67) 5785 return 0; 5786 5787 offset = (u8 *)udp + 8 - skb->data; 5788 length = skb->len - offset; 5789 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length); 5790 } 5791 5792 return 0; 5793 } 5794 5795 static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5796 { 5797 struct e1000_adapter *adapter = tx_ring->adapter; 5798 5799 netif_stop_queue(adapter->netdev); 5800 /* Herbert's original patch had: 5801 * smp_mb__after_netif_stop_queue(); 5802 * but since that doesn't exist yet, just open code it. 5803 */ 5804 smp_mb(); 5805 5806 /* We need to check again in a case another CPU has just 5807 * made room available. 5808 */ 5809 if (e1000_desc_unused(tx_ring) < size) 5810 return -EBUSY; 5811 5812 /* A reprieve! */ 5813 netif_start_queue(adapter->netdev); 5814 ++adapter->restart_queue; 5815 return 0; 5816 } 5817 5818 static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size) 5819 { 5820 BUG_ON(size > tx_ring->count); 5821 5822 if (e1000_desc_unused(tx_ring) >= size) 5823 return 0; 5824 return __e1000_maybe_stop_tx(tx_ring, size); 5825 } 5826 5827 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb, 5828 struct net_device *netdev) 5829 { 5830 struct e1000_adapter *adapter = netdev_priv(netdev); 5831 struct e1000_ring *tx_ring = adapter->tx_ring; 5832 unsigned int first; 5833 unsigned int tx_flags = 0; 5834 unsigned int len = skb_headlen(skb); 5835 unsigned int nr_frags; 5836 unsigned int mss; 5837 int count = 0; 5838 int tso; 5839 unsigned int f; 5840 __be16 protocol = vlan_get_protocol(skb); 5841 5842 if (test_bit(__E1000_DOWN, &adapter->state)) { 5843 dev_kfree_skb_any(skb); 5844 return NETDEV_TX_OK; 5845 } 5846 5847 if (skb->len <= 0) { 5848 dev_kfree_skb_any(skb); 5849 return NETDEV_TX_OK; 5850 } 5851 5852 /* The minimum packet size with TCTL.PSP set is 17 bytes so 5853 * pad skb in order to meet this minimum size requirement 5854 */ 5855 if (skb_put_padto(skb, 17)) 5856 return NETDEV_TX_OK; 5857 5858 mss = skb_shinfo(skb)->gso_size; 5859 if (mss) { 5860 u8 hdr_len; 5861 5862 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data 5863 * points to just header, pull a few bytes of payload from 5864 * frags into skb->data 5865 */ 5866 hdr_len = skb_tcp_all_headers(skb); 5867 /* we do this workaround for ES2LAN, but it is un-necessary, 5868 * avoiding it could save a lot of cycles 5869 */ 5870 if (skb->data_len && (hdr_len == len)) { 5871 unsigned int pull_size; 5872 5873 pull_size = min_t(unsigned int, 4, skb->data_len); 5874 if (!__pskb_pull_tail(skb, pull_size)) { 5875 e_err("__pskb_pull_tail failed.\n"); 5876 dev_kfree_skb_any(skb); 5877 return NETDEV_TX_OK; 5878 } 5879 len = skb_headlen(skb); 5880 } 5881 } 5882 5883 /* reserve a descriptor for the offload context */ 5884 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL)) 5885 count++; 5886 count++; 5887 5888 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit); 5889 5890 nr_frags = skb_shinfo(skb)->nr_frags; 5891 for (f = 0; f < nr_frags; f++) 5892 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]), 5893 adapter->tx_fifo_limit); 5894 5895 if (adapter->hw.mac.tx_pkt_filtering) 5896 e1000_transfer_dhcp_info(adapter, skb); 5897 5898 /* need: count + 2 desc gap to keep tail from touching 5899 * head, otherwise try next time 5900 */ 5901 if (e1000_maybe_stop_tx(tx_ring, count + 2)) 5902 return NETDEV_TX_BUSY; 5903 5904 if (skb_vlan_tag_present(skb)) { 5905 tx_flags |= E1000_TX_FLAGS_VLAN; 5906 tx_flags |= (skb_vlan_tag_get(skb) << 5907 E1000_TX_FLAGS_VLAN_SHIFT); 5908 } 5909 5910 first = tx_ring->next_to_use; 5911 5912 tso = e1000_tso(tx_ring, skb, protocol); 5913 if (tso < 0) { 5914 dev_kfree_skb_any(skb); 5915 return NETDEV_TX_OK; 5916 } 5917 5918 if (tso) 5919 tx_flags |= E1000_TX_FLAGS_TSO; 5920 else if (e1000_tx_csum(tx_ring, skb, protocol)) 5921 tx_flags |= E1000_TX_FLAGS_CSUM; 5922 5923 /* Old method was to assume IPv4 packet by default if TSO was enabled. 5924 * 82571 hardware supports TSO capabilities for IPv6 as well... 5925 * no longer assume, we must. 5926 */ 5927 if (protocol == htons(ETH_P_IP)) 5928 tx_flags |= E1000_TX_FLAGS_IPV4; 5929 5930 if (unlikely(skb->no_fcs)) 5931 tx_flags |= E1000_TX_FLAGS_NO_FCS; 5932 5933 /* if count is 0 then mapping error has occurred */ 5934 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit, 5935 nr_frags); 5936 if (count) { 5937 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 5938 (adapter->flags & FLAG_HAS_HW_TIMESTAMP)) { 5939 if (!adapter->tx_hwtstamp_skb) { 5940 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 5941 tx_flags |= E1000_TX_FLAGS_HWTSTAMP; 5942 adapter->tx_hwtstamp_skb = skb_get(skb); 5943 adapter->tx_hwtstamp_start = jiffies; 5944 schedule_work(&adapter->tx_hwtstamp_work); 5945 } else { 5946 adapter->tx_hwtstamp_skipped++; 5947 } 5948 } 5949 5950 skb_tx_timestamp(skb); 5951 5952 netdev_sent_queue(netdev, skb->len); 5953 e1000_tx_queue(tx_ring, tx_flags, count); 5954 /* Make sure there is space in the ring for the next send. */ 5955 e1000_maybe_stop_tx(tx_ring, 5956 ((MAX_SKB_FRAGS + 1) * 5957 DIV_ROUND_UP(PAGE_SIZE, 5958 adapter->tx_fifo_limit) + 4)); 5959 5960 if (!netdev_xmit_more() || 5961 netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) { 5962 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA) 5963 e1000e_update_tdt_wa(tx_ring, 5964 tx_ring->next_to_use); 5965 else 5966 writel(tx_ring->next_to_use, tx_ring->tail); 5967 } 5968 } else { 5969 dev_kfree_skb_any(skb); 5970 tx_ring->buffer_info[first].time_stamp = 0; 5971 tx_ring->next_to_use = first; 5972 } 5973 5974 return NETDEV_TX_OK; 5975 } 5976 5977 /** 5978 * e1000_tx_timeout - Respond to a Tx Hang 5979 * @netdev: network interface device structure 5980 * @txqueue: index of the hung queue (unused) 5981 **/ 5982 static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) 5983 { 5984 struct e1000_adapter *adapter = netdev_priv(netdev); 5985 5986 /* Do the reset outside of interrupt context */ 5987 adapter->tx_timeout_count++; 5988 schedule_work(&adapter->reset_task); 5989 } 5990 5991 static void e1000_reset_task(struct work_struct *work) 5992 { 5993 struct e1000_adapter *adapter; 5994 adapter = container_of(work, struct e1000_adapter, reset_task); 5995 5996 rtnl_lock(); 5997 /* don't run the task if already down */ 5998 if (test_bit(__E1000_DOWN, &adapter->state)) { 5999 rtnl_unlock(); 6000 return; 6001 } 6002 6003 if (!(adapter->flags & FLAG_RESTART_NOW)) { 6004 e1000e_dump(adapter); 6005 e_err("Reset adapter unexpectedly\n"); 6006 } 6007 e1000e_reinit_locked(adapter); 6008 rtnl_unlock(); 6009 } 6010 6011 /** 6012 * e1000e_get_stats64 - Get System Network Statistics 6013 * @netdev: network interface device structure 6014 * @stats: rtnl_link_stats64 pointer 6015 * 6016 * Returns the address of the device statistics structure. 6017 **/ 6018 void e1000e_get_stats64(struct net_device *netdev, 6019 struct rtnl_link_stats64 *stats) 6020 { 6021 struct e1000_adapter *adapter = netdev_priv(netdev); 6022 6023 spin_lock(&adapter->stats64_lock); 6024 e1000e_update_stats(adapter); 6025 /* Fill out the OS statistics structure */ 6026 stats->rx_bytes = adapter->stats.gorc; 6027 stats->rx_packets = adapter->stats.gprc; 6028 stats->tx_bytes = adapter->stats.gotc; 6029 stats->tx_packets = adapter->stats.gptc; 6030 stats->multicast = adapter->stats.mprc; 6031 stats->collisions = adapter->stats.colc; 6032 6033 /* Rx Errors */ 6034 6035 /* RLEC on some newer hardware can be incorrect so build 6036 * our own version based on RUC and ROC 6037 */ 6038 stats->rx_errors = adapter->stats.rxerrc + 6039 adapter->stats.crcerrs + adapter->stats.algnerrc + 6040 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr; 6041 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc; 6042 stats->rx_crc_errors = adapter->stats.crcerrs; 6043 stats->rx_frame_errors = adapter->stats.algnerrc; 6044 stats->rx_missed_errors = adapter->stats.mpc; 6045 6046 /* Tx Errors */ 6047 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol; 6048 stats->tx_aborted_errors = adapter->stats.ecol; 6049 stats->tx_window_errors = adapter->stats.latecol; 6050 stats->tx_carrier_errors = adapter->stats.tncrs; 6051 6052 /* Tx Dropped needs to be maintained elsewhere */ 6053 6054 spin_unlock(&adapter->stats64_lock); 6055 } 6056 6057 /** 6058 * e1000_change_mtu - Change the Maximum Transfer Unit 6059 * @netdev: network interface device structure 6060 * @new_mtu: new value for maximum frame size 6061 * 6062 * Returns 0 on success, negative on failure 6063 **/ 6064 static int e1000_change_mtu(struct net_device *netdev, int new_mtu) 6065 { 6066 struct e1000_adapter *adapter = netdev_priv(netdev); 6067 int max_frame = new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN; 6068 6069 /* Jumbo frame support */ 6070 if ((new_mtu > ETH_DATA_LEN) && 6071 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) { 6072 e_err("Jumbo Frames not supported.\n"); 6073 return -EINVAL; 6074 } 6075 6076 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 6077 if ((adapter->hw.mac.type >= e1000_pch2lan) && 6078 !(adapter->flags2 & FLAG2_CRC_STRIPPING) && 6079 (new_mtu > ETH_DATA_LEN)) { 6080 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n"); 6081 return -EINVAL; 6082 } 6083 6084 while (test_and_set_bit(__E1000_RESETTING, &adapter->state)) 6085 usleep_range(1000, 1100); 6086 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */ 6087 adapter->max_frame_size = max_frame; 6088 netdev_dbg(netdev, "changing MTU from %d to %d\n", 6089 netdev->mtu, new_mtu); 6090 WRITE_ONCE(netdev->mtu, new_mtu); 6091 6092 pm_runtime_get_sync(netdev->dev.parent); 6093 6094 if (netif_running(netdev)) 6095 e1000e_down(adapter, true); 6096 6097 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN 6098 * means we reserve 2 more, this pushes us to allocate from the next 6099 * larger slab size. 6100 * i.e. RXBUFFER_2048 --> size-4096 slab 6101 * However with the new *_jumbo_rx* routines, jumbo receives will use 6102 * fragmented skbs 6103 */ 6104 6105 if (max_frame <= 2048) 6106 adapter->rx_buffer_len = 2048; 6107 else 6108 adapter->rx_buffer_len = 4096; 6109 6110 /* adjust allocation if LPE protects us, and we aren't using SBP */ 6111 if (max_frame <= (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) 6112 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN; 6113 6114 if (netif_running(netdev)) 6115 e1000e_up(adapter); 6116 else 6117 e1000e_reset(adapter); 6118 6119 pm_runtime_put_sync(netdev->dev.parent); 6120 6121 clear_bit(__E1000_RESETTING, &adapter->state); 6122 6123 return 0; 6124 } 6125 6126 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 6127 { 6128 struct e1000_adapter *adapter = netdev_priv(netdev); 6129 struct mii_ioctl_data *data = if_mii(ifr); 6130 6131 if (adapter->hw.phy.media_type != e1000_media_type_copper) 6132 return -EOPNOTSUPP; 6133 6134 switch (cmd) { 6135 case SIOCGMIIPHY: 6136 data->phy_id = adapter->hw.phy.addr; 6137 break; 6138 case SIOCGMIIREG: 6139 e1000_phy_read_status(adapter); 6140 6141 switch (data->reg_num & 0x1F) { 6142 case MII_BMCR: 6143 data->val_out = adapter->phy_regs.bmcr; 6144 break; 6145 case MII_BMSR: 6146 data->val_out = adapter->phy_regs.bmsr; 6147 break; 6148 case MII_PHYSID1: 6149 data->val_out = (adapter->hw.phy.id >> 16); 6150 break; 6151 case MII_PHYSID2: 6152 data->val_out = (adapter->hw.phy.id & 0xFFFF); 6153 break; 6154 case MII_ADVERTISE: 6155 data->val_out = adapter->phy_regs.advertise; 6156 break; 6157 case MII_LPA: 6158 data->val_out = adapter->phy_regs.lpa; 6159 break; 6160 case MII_EXPANSION: 6161 data->val_out = adapter->phy_regs.expansion; 6162 break; 6163 case MII_CTRL1000: 6164 data->val_out = adapter->phy_regs.ctrl1000; 6165 break; 6166 case MII_STAT1000: 6167 data->val_out = adapter->phy_regs.stat1000; 6168 break; 6169 case MII_ESTATUS: 6170 data->val_out = adapter->phy_regs.estatus; 6171 break; 6172 default: 6173 return -EIO; 6174 } 6175 break; 6176 case SIOCSMIIREG: 6177 default: 6178 return -EOPNOTSUPP; 6179 } 6180 return 0; 6181 } 6182 6183 /** 6184 * e1000e_hwtstamp_set - control hardware time stamping 6185 * @netdev: network interface device structure 6186 * @config: timestamp configuration 6187 * @extack: netlink extended ACK report 6188 * 6189 * Outgoing time stamping can be enabled and disabled. Play nice and 6190 * disable it when requested, although it shouldn't cause any overhead 6191 * when no packet needs it. At most one packet in the queue may be 6192 * marked for time stamping, otherwise it would be impossible to tell 6193 * for sure to which packet the hardware time stamp belongs. 6194 * 6195 * Incoming time stamping has to be configured via the hardware filters. 6196 * Not all combinations are supported, in particular event type has to be 6197 * specified. Matching the kind of event packet is not supported, with the 6198 * exception of "all V2 events regardless of level 2 or 4". 6199 **/ 6200 static int e1000e_hwtstamp_set(struct net_device *netdev, 6201 struct kernel_hwtstamp_config *config, 6202 struct netlink_ext_ack *extack) 6203 { 6204 struct e1000_adapter *adapter = netdev_priv(netdev); 6205 int ret_val; 6206 6207 ret_val = e1000e_config_hwtstamp(adapter, config, extack); 6208 if (ret_val) 6209 return ret_val; 6210 6211 switch (config->rx_filter) { 6212 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 6213 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 6214 case HWTSTAMP_FILTER_PTP_V2_SYNC: 6215 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 6216 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 6217 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 6218 /* With V2 type filters which specify a Sync or Delay Request, 6219 * Path Delay Request/Response messages are also time stamped 6220 * by hardware so notify the caller the requested packets plus 6221 * some others are time stamped. 6222 */ 6223 config->rx_filter = HWTSTAMP_FILTER_SOME; 6224 break; 6225 default: 6226 break; 6227 } 6228 6229 return 0; 6230 } 6231 6232 static int e1000e_hwtstamp_get(struct net_device *netdev, 6233 struct kernel_hwtstamp_config *kernel_config) 6234 { 6235 struct e1000_adapter *adapter = netdev_priv(netdev); 6236 6237 *kernel_config = adapter->hwtstamp_config; 6238 6239 return 0; 6240 } 6241 6242 static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc) 6243 { 6244 struct e1000_hw *hw = &adapter->hw; 6245 u32 i, mac_reg, wuc; 6246 u16 phy_reg, wuc_enable; 6247 int retval; 6248 6249 /* copy MAC RARs to PHY RARs */ 6250 e1000_copy_rx_addrs_to_phy_ich8lan(hw); 6251 6252 retval = hw->phy.ops.acquire(hw); 6253 if (retval) { 6254 e_err("Could not acquire PHY\n"); 6255 return retval; 6256 } 6257 6258 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */ 6259 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6260 if (retval) 6261 goto release; 6262 6263 /* copy MAC MTA to PHY MTA - only needed for pchlan */ 6264 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 6265 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 6266 hw->phy.ops.write_reg_page(hw, BM_MTA(i), 6267 (u16)(mac_reg & 0xFFFF)); 6268 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1, 6269 (u16)((mac_reg >> 16) & 0xFFFF)); 6270 } 6271 6272 /* configure PHY Rx Control register */ 6273 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg); 6274 mac_reg = er32(RCTL); 6275 if (mac_reg & E1000_RCTL_UPE) 6276 phy_reg |= BM_RCTL_UPE; 6277 if (mac_reg & E1000_RCTL_MPE) 6278 phy_reg |= BM_RCTL_MPE; 6279 phy_reg &= ~(BM_RCTL_MO_MASK); 6280 if (mac_reg & E1000_RCTL_MO_3) 6281 phy_reg |= (FIELD_GET(E1000_RCTL_MO_3, mac_reg) 6282 << BM_RCTL_MO_SHIFT); 6283 if (mac_reg & E1000_RCTL_BAM) 6284 phy_reg |= BM_RCTL_BAM; 6285 if (mac_reg & E1000_RCTL_PMCF) 6286 phy_reg |= BM_RCTL_PMCF; 6287 mac_reg = er32(CTRL); 6288 if (mac_reg & E1000_CTRL_RFCE) 6289 phy_reg |= BM_RCTL_RFCE; 6290 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg); 6291 6292 wuc = E1000_WUC_PME_EN; 6293 if (wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC)) 6294 wuc |= E1000_WUC_APME; 6295 6296 /* enable PHY wakeup in MAC register */ 6297 ew32(WUFC, wufc); 6298 ew32(WUC, (E1000_WUC_PHY_WAKE | E1000_WUC_APMPME | 6299 E1000_WUC_PME_STATUS | wuc)); 6300 6301 /* configure and enable PHY wakeup in PHY registers */ 6302 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc); 6303 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, wuc); 6304 6305 /* activate PHY wakeup */ 6306 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 6307 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable); 6308 if (retval) 6309 e_err("Could not set PHY Host Wakeup bit\n"); 6310 release: 6311 hw->phy.ops.release(hw); 6312 6313 return retval; 6314 } 6315 6316 static void e1000e_flush_lpic(struct pci_dev *pdev) 6317 { 6318 struct net_device *netdev = pci_get_drvdata(pdev); 6319 struct e1000_adapter *adapter = netdev_priv(netdev); 6320 struct e1000_hw *hw = &adapter->hw; 6321 u32 ret_val; 6322 6323 pm_runtime_get_sync(netdev->dev.parent); 6324 6325 ret_val = hw->phy.ops.acquire(hw); 6326 if (ret_val) 6327 goto fl_out; 6328 6329 pr_info("EEE TX LPI TIMER: %08X\n", 6330 er32(LPIC) >> E1000_LPIC_LPIET_SHIFT); 6331 6332 hw->phy.ops.release(hw); 6333 6334 fl_out: 6335 pm_runtime_put_sync(netdev->dev.parent); 6336 } 6337 6338 /* S0ix implementation */ 6339 static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter) 6340 { 6341 struct e1000_hw *hw = &adapter->hw; 6342 u32 mac_data; 6343 u16 phy_data; 6344 6345 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID && 6346 hw->mac.type >= e1000_pch_adp) { 6347 /* Request ME configure the device for S0ix */ 6348 mac_data = er32(H2ME); 6349 mac_data |= E1000_H2ME_START_DPG; 6350 mac_data &= ~E1000_H2ME_EXIT_DPG; 6351 trace_e1000e_trace_mac_register(mac_data); 6352 ew32(H2ME, mac_data); 6353 } else { 6354 /* Request driver configure the device to S0ix */ 6355 /* Disable the periodic inband message, 6356 * don't request PCIe clock in K1 page770_17[10:9] = 10b 6357 */ 6358 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6359 phy_data &= ~HV_PM_CTRL_K1_CLK_REQ; 6360 phy_data |= BIT(10); 6361 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6362 6363 /* Make sure we don't exit K1 every time a new packet arrives 6364 * 772_29[5] = 1 CS_Mode_Stay_In_K1 6365 */ 6366 e1e_rphy(hw, I217_CGFREG, &phy_data); 6367 phy_data |= BIT(5); 6368 e1e_wphy(hw, I217_CGFREG, phy_data); 6369 6370 /* Change the MAC/PHY interface to SMBus 6371 * Force the SMBus in PHY page769_23[0] = 1 6372 * Force the SMBus in MAC CTRL_EXT[11] = 1 6373 */ 6374 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6375 phy_data |= CV_SMB_CTRL_FORCE_SMBUS; 6376 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6377 mac_data = er32(CTRL_EXT); 6378 mac_data |= E1000_CTRL_EXT_FORCE_SMBUS; 6379 ew32(CTRL_EXT, mac_data); 6380 6381 /* DFT control: PHY bit: page769_20[0] = 1 6382 * page769_20[7] - PHY PLL stop 6383 * page769_20[8] - PHY go to the electrical idle 6384 * page769_20[9] - PHY serdes disable 6385 * Gate PPW via EXTCNF_CTRL - set 0x0F00[7] = 1 6386 */ 6387 e1e_rphy(hw, I82579_DFT_CTRL, &phy_data); 6388 phy_data |= BIT(0); 6389 phy_data |= BIT(7); 6390 phy_data |= BIT(8); 6391 phy_data |= BIT(9); 6392 e1e_wphy(hw, I82579_DFT_CTRL, phy_data); 6393 6394 mac_data = er32(EXTCNF_CTRL); 6395 mac_data |= E1000_EXTCNF_CTRL_GATE_PHY_CFG; 6396 ew32(EXTCNF_CTRL, mac_data); 6397 6398 /* Disable disconnected cable conditioning for Power Gating */ 6399 mac_data = er32(DPGFR); 6400 mac_data |= BIT(2); 6401 ew32(DPGFR, mac_data); 6402 6403 /* Enable the Dynamic Clock Gating in the DMA and MAC */ 6404 mac_data = er32(CTRL_EXT); 6405 mac_data |= E1000_CTRL_EXT_DMA_DYN_CLK_EN; 6406 ew32(CTRL_EXT, mac_data); 6407 } 6408 6409 /* Enable the Dynamic Power Gating in the MAC */ 6410 mac_data = er32(FEXTNVM7); 6411 mac_data |= BIT(22); 6412 ew32(FEXTNVM7, mac_data); 6413 6414 /* Don't wake from dynamic Power Gating with clock request */ 6415 mac_data = er32(FEXTNVM12); 6416 mac_data |= BIT(12); 6417 ew32(FEXTNVM12, mac_data); 6418 6419 /* Ungate PGCB clock */ 6420 mac_data = er32(FEXTNVM9); 6421 mac_data &= ~BIT(28); 6422 ew32(FEXTNVM9, mac_data); 6423 6424 /* Enable K1 off to enable mPHY Power Gating */ 6425 mac_data = er32(FEXTNVM6); 6426 mac_data |= BIT(31); 6427 ew32(FEXTNVM6, mac_data); 6428 6429 /* Enable mPHY power gating for any link and speed */ 6430 mac_data = er32(FEXTNVM8); 6431 mac_data |= BIT(9); 6432 ew32(FEXTNVM8, mac_data); 6433 6434 /* No MAC DPG gating SLP_S0 in modern standby 6435 * Switch the logic of the lanphypc to use PMC counter 6436 */ 6437 mac_data = er32(FEXTNVM5); 6438 mac_data |= BIT(7); 6439 ew32(FEXTNVM5, mac_data); 6440 6441 /* Disable the time synchronization clock */ 6442 mac_data = er32(FEXTNVM7); 6443 mac_data |= BIT(31); 6444 mac_data &= ~BIT(0); 6445 ew32(FEXTNVM7, mac_data); 6446 6447 /* Dynamic Power Gating Enable */ 6448 mac_data = er32(CTRL_EXT); 6449 mac_data |= BIT(3); 6450 ew32(CTRL_EXT, mac_data); 6451 6452 /* Check MAC Tx/Rx packet buffer pointers. 6453 * Reset MAC Tx/Rx packet buffer pointers to suppress any 6454 * pending traffic indication that would prevent power gating. 6455 */ 6456 mac_data = er32(TDFH); 6457 if (mac_data) 6458 ew32(TDFH, 0); 6459 mac_data = er32(TDFT); 6460 if (mac_data) 6461 ew32(TDFT, 0); 6462 mac_data = er32(TDFHS); 6463 if (mac_data) 6464 ew32(TDFHS, 0); 6465 mac_data = er32(TDFTS); 6466 if (mac_data) 6467 ew32(TDFTS, 0); 6468 mac_data = er32(TDFPC); 6469 if (mac_data) 6470 ew32(TDFPC, 0); 6471 mac_data = er32(RDFH); 6472 if (mac_data) 6473 ew32(RDFH, 0); 6474 mac_data = er32(RDFT); 6475 if (mac_data) 6476 ew32(RDFT, 0); 6477 mac_data = er32(RDFHS); 6478 if (mac_data) 6479 ew32(RDFHS, 0); 6480 mac_data = er32(RDFTS); 6481 if (mac_data) 6482 ew32(RDFTS, 0); 6483 mac_data = er32(RDFPC); 6484 if (mac_data) 6485 ew32(RDFPC, 0); 6486 } 6487 6488 static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter) 6489 { 6490 struct e1000_hw *hw = &adapter->hw; 6491 bool firmware_bug = false; 6492 u32 mac_data; 6493 u16 phy_data; 6494 u32 i = 0; 6495 6496 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID && 6497 hw->mac.type >= e1000_pch_adp) { 6498 /* Keep the GPT clock enabled for CSME */ 6499 mac_data = er32(FEXTNVM); 6500 mac_data |= BIT(3); 6501 ew32(FEXTNVM, mac_data); 6502 /* Request ME unconfigure the device from S0ix */ 6503 mac_data = er32(H2ME); 6504 mac_data &= ~E1000_H2ME_START_DPG; 6505 mac_data |= E1000_H2ME_EXIT_DPG; 6506 trace_e1000e_trace_mac_register(mac_data); 6507 ew32(H2ME, mac_data); 6508 6509 /* Poll up to 2.5 seconds for ME to unconfigure DPG. 6510 * If this takes more than 1 second, show a warning indicating a 6511 * firmware bug 6512 */ 6513 while (!(er32(EXFWSM) & E1000_EXFWSM_DPG_EXIT_DONE)) { 6514 if (i > 100 && !firmware_bug) 6515 firmware_bug = true; 6516 6517 if (i++ == 250) { 6518 e_dbg("Timeout (firmware bug): %d msec\n", 6519 i * 10); 6520 break; 6521 } 6522 6523 usleep_range(10000, 11000); 6524 } 6525 if (firmware_bug) 6526 e_warn("DPG_EXIT_DONE took %d msec. This is a firmware bug\n", 6527 i * 10); 6528 else 6529 e_dbg("DPG_EXIT_DONE cleared after %d msec\n", i * 10); 6530 } else { 6531 /* Request driver unconfigure the device from S0ix */ 6532 6533 /* Cancel disable disconnected cable conditioning 6534 * for Power Gating 6535 */ 6536 mac_data = er32(DPGFR); 6537 mac_data &= ~BIT(2); 6538 ew32(DPGFR, mac_data); 6539 6540 /* Disable the Dynamic Clock Gating in the DMA and MAC */ 6541 mac_data = er32(CTRL_EXT); 6542 mac_data &= 0xFFF7FFFF; 6543 ew32(CTRL_EXT, mac_data); 6544 6545 /* Enable the periodic inband message, 6546 * Request PCIe clock in K1 page770_17[10:9] =01b 6547 */ 6548 e1e_rphy(hw, HV_PM_CTRL, &phy_data); 6549 phy_data &= 0xFBFF; 6550 phy_data |= HV_PM_CTRL_K1_CLK_REQ; 6551 e1e_wphy(hw, HV_PM_CTRL, phy_data); 6552 6553 /* Return back configuration 6554 * 772_29[5] = 0 CS_Mode_Stay_In_K1 6555 */ 6556 e1e_rphy(hw, I217_CGFREG, &phy_data); 6557 phy_data &= 0xFFDF; 6558 e1e_wphy(hw, I217_CGFREG, phy_data); 6559 6560 /* Change the MAC/PHY interface to Kumeran 6561 * Unforce the SMBus in PHY page769_23[0] = 0 6562 * Unforce the SMBus in MAC CTRL_EXT[11] = 0 6563 */ 6564 e1e_rphy(hw, CV_SMB_CTRL, &phy_data); 6565 phy_data &= ~CV_SMB_CTRL_FORCE_SMBUS; 6566 e1e_wphy(hw, CV_SMB_CTRL, phy_data); 6567 mac_data = er32(CTRL_EXT); 6568 mac_data &= ~E1000_CTRL_EXT_FORCE_SMBUS; 6569 ew32(CTRL_EXT, mac_data); 6570 } 6571 6572 /* Disable Dynamic Power Gating */ 6573 mac_data = er32(CTRL_EXT); 6574 mac_data &= 0xFFFFFFF7; 6575 ew32(CTRL_EXT, mac_data); 6576 6577 /* Enable the time synchronization clock */ 6578 mac_data = er32(FEXTNVM7); 6579 mac_data &= ~BIT(31); 6580 mac_data |= BIT(0); 6581 ew32(FEXTNVM7, mac_data); 6582 6583 /* Disable the Dynamic Power Gating in the MAC */ 6584 mac_data = er32(FEXTNVM7); 6585 mac_data &= 0xFFBFFFFF; 6586 ew32(FEXTNVM7, mac_data); 6587 6588 /* Disable mPHY power gating for any link and speed */ 6589 mac_data = er32(FEXTNVM8); 6590 mac_data &= ~BIT(9); 6591 ew32(FEXTNVM8, mac_data); 6592 6593 /* Disable K1 off */ 6594 mac_data = er32(FEXTNVM6); 6595 mac_data &= ~BIT(31); 6596 ew32(FEXTNVM6, mac_data); 6597 6598 /* Disable Ungate PGCB clock */ 6599 mac_data = er32(FEXTNVM9); 6600 mac_data |= BIT(28); 6601 ew32(FEXTNVM9, mac_data); 6602 6603 /* Cancel not waking from dynamic 6604 * Power Gating with clock request 6605 */ 6606 mac_data = er32(FEXTNVM12); 6607 mac_data &= ~BIT(12); 6608 ew32(FEXTNVM12, mac_data); 6609 6610 /* Revert the lanphypc logic to use the internal Gbe counter 6611 * and not the PMC counter 6612 */ 6613 mac_data = er32(FEXTNVM5); 6614 mac_data &= 0xFFFFFF7F; 6615 ew32(FEXTNVM5, mac_data); 6616 } 6617 6618 static int e1000e_pm_freeze(struct device *dev) 6619 { 6620 struct net_device *netdev = dev_get_drvdata(dev); 6621 struct e1000_adapter *adapter = netdev_priv(netdev); 6622 bool present; 6623 6624 rtnl_lock(); 6625 6626 present = netif_device_present(netdev); 6627 netif_device_detach(netdev); 6628 6629 if (present && netif_running(netdev)) { 6630 int count = E1000_CHECK_RESET_COUNT; 6631 6632 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 6633 usleep_range(10000, 11000); 6634 6635 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 6636 6637 /* Quiesce the device without resetting the hardware */ 6638 e1000e_down(adapter, false); 6639 e1000_free_irq(adapter); 6640 } 6641 rtnl_unlock(); 6642 6643 e1000e_reset_interrupt_capability(adapter); 6644 6645 /* Allow time for pending master requests to run */ 6646 e1000e_disable_pcie_master(&adapter->hw); 6647 6648 return 0; 6649 } 6650 6651 static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) 6652 { 6653 struct net_device *netdev = pci_get_drvdata(pdev); 6654 struct e1000_adapter *adapter = netdev_priv(netdev); 6655 struct e1000_hw *hw = &adapter->hw; 6656 u32 ctrl, ctrl_ext, rctl, status, wufc; 6657 int retval = 0; 6658 6659 /* Runtime suspend should only enable wakeup for link changes */ 6660 if (runtime) 6661 wufc = E1000_WUFC_LNKC; 6662 else if (device_may_wakeup(&pdev->dev)) 6663 wufc = adapter->wol; 6664 else 6665 wufc = 0; 6666 6667 status = er32(STATUS); 6668 if (status & E1000_STATUS_LU) 6669 wufc &= ~E1000_WUFC_LNKC; 6670 6671 if (wufc) { 6672 e1000_setup_rctl(adapter); 6673 e1000e_set_rx_mode(netdev); 6674 6675 /* turn on all-multi mode if wake on multicast is enabled */ 6676 if (wufc & E1000_WUFC_MC) { 6677 rctl = er32(RCTL); 6678 rctl |= E1000_RCTL_MPE; 6679 ew32(RCTL, rctl); 6680 } 6681 6682 ctrl = er32(CTRL); 6683 ctrl |= E1000_CTRL_ADVD3WUC; 6684 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP)) 6685 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT; 6686 ew32(CTRL, ctrl); 6687 6688 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 6689 adapter->hw.phy.media_type == 6690 e1000_media_type_internal_serdes) { 6691 /* keep the laser running in D3 */ 6692 ctrl_ext = er32(CTRL_EXT); 6693 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 6694 ew32(CTRL_EXT, ctrl_ext); 6695 } 6696 6697 if (!runtime) 6698 e1000e_power_up_phy(adapter); 6699 6700 if (adapter->flags & FLAG_IS_ICH) 6701 e1000_suspend_workarounds_ich8lan(&adapter->hw); 6702 6703 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6704 /* enable wakeup by the PHY */ 6705 retval = e1000_init_phy_wakeup(adapter, wufc); 6706 if (retval) { 6707 e_err("Failed to enable wakeup\n"); 6708 goto skip_phy_configurations; 6709 } 6710 } else { 6711 /* enable wakeup by the MAC */ 6712 ew32(WUFC, wufc); 6713 ew32(WUC, E1000_WUC_PME_EN); 6714 } 6715 } else { 6716 ew32(WUC, 0); 6717 ew32(WUFC, 0); 6718 6719 e1000_power_down_phy(adapter); 6720 } 6721 6722 if (adapter->hw.phy.type == e1000_phy_igp_3) { 6723 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 6724 } else if (hw->mac.type >= e1000_pch_lpt) { 6725 if (wufc && !(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC))) { 6726 /* ULP does not support wake from unicast, multicast 6727 * or broadcast. 6728 */ 6729 retval = e1000_enable_ulp_lpt_lp(hw, !runtime); 6730 if (retval) { 6731 e_err("Failed to enable ULP\n"); 6732 goto skip_phy_configurations; 6733 } 6734 } 6735 } 6736 6737 /* Ensure that the appropriate bits are set in LPI_CTRL 6738 * for EEE in Sx 6739 */ 6740 if ((hw->phy.type >= e1000_phy_i217) && 6741 adapter->eee_advert && hw->dev_spec.ich8lan.eee_lp_ability) { 6742 u16 lpi_ctrl = 0; 6743 6744 retval = hw->phy.ops.acquire(hw); 6745 if (!retval) { 6746 retval = e1e_rphy_locked(hw, I82579_LPI_CTRL, 6747 &lpi_ctrl); 6748 if (!retval) { 6749 if (adapter->eee_advert & 6750 hw->dev_spec.ich8lan.eee_lp_ability & 6751 I82579_EEE_100_SUPPORTED) 6752 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE; 6753 if (adapter->eee_advert & 6754 hw->dev_spec.ich8lan.eee_lp_ability & 6755 I82579_EEE_1000_SUPPORTED) 6756 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE; 6757 6758 retval = e1e_wphy_locked(hw, I82579_LPI_CTRL, 6759 lpi_ctrl); 6760 } 6761 } 6762 hw->phy.ops.release(hw); 6763 } 6764 6765 skip_phy_configurations: 6766 /* Release control of h/w to f/w. If f/w is AMT enabled, this 6767 * would have already happened in close and is redundant. 6768 */ 6769 e1000e_release_hw_control(adapter); 6770 6771 pci_clear_master(pdev); 6772 6773 /* The pci-e switch on some quad port adapters will report a 6774 * correctable error when the MAC transitions from D0 to D3. To 6775 * prevent this we need to mask off the correctable errors on the 6776 * downstream port of the pci-e switch. 6777 * 6778 * We don't have the associated upstream bridge while assigning 6779 * the PCI device into guest. For example, the KVM on power is 6780 * one of the cases. 6781 */ 6782 if (adapter->flags & FLAG_IS_QUAD_PORT) { 6783 struct pci_dev *us_dev = pdev->bus->self; 6784 u16 devctl; 6785 6786 if (!us_dev) 6787 return 0; 6788 6789 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl); 6790 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, 6791 (devctl & ~PCI_EXP_DEVCTL_CERE)); 6792 6793 pci_save_state(pdev); 6794 pci_prepare_to_sleep(pdev); 6795 6796 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl); 6797 } 6798 6799 return 0; 6800 } 6801 6802 /** 6803 * __e1000e_disable_aspm - Disable ASPM states 6804 * @pdev: pointer to PCI device struct 6805 * @state: bit-mask of ASPM states to disable 6806 * @locked: indication if this context holds pci_bus_sem locked. 6807 * 6808 * Some devices *must* have certain ASPM states disabled per hardware errata. 6809 **/ 6810 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state, int locked) 6811 { 6812 struct pci_dev *parent = pdev->bus->self; 6813 u16 aspm_dis_mask = 0; 6814 u16 pdev_aspmc, parent_aspmc; 6815 6816 switch (state) { 6817 case PCIE_LINK_STATE_L0S: 6818 case PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1: 6819 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L0S; 6820 fallthrough; /* can't have L1 without L0s */ 6821 case PCIE_LINK_STATE_L1: 6822 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L1; 6823 break; 6824 default: 6825 return; 6826 } 6827 6828 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6829 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6830 6831 if (parent) { 6832 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, 6833 &parent_aspmc); 6834 parent_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6835 } 6836 6837 /* Nothing to do if the ASPM states to be disabled already are */ 6838 if (!(pdev_aspmc & aspm_dis_mask) && 6839 (!parent || !(parent_aspmc & aspm_dis_mask))) 6840 return; 6841 6842 dev_info(&pdev->dev, "Disabling ASPM %s %s\n", 6843 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L0S) ? 6844 "L0s" : "", 6845 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L1) ? 6846 "L1" : ""); 6847 6848 #ifdef CONFIG_PCIEASPM 6849 if (locked) 6850 pci_disable_link_state_locked(pdev, state); 6851 else 6852 pci_disable_link_state(pdev, state); 6853 6854 /* Double-check ASPM control. If not disabled by the above, the 6855 * BIOS is preventing that from happening (or CONFIG_PCIEASPM is 6856 * not enabled); override by writing PCI config space directly. 6857 */ 6858 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc); 6859 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC; 6860 6861 if (!(aspm_dis_mask & pdev_aspmc)) 6862 return; 6863 #endif 6864 6865 /* Both device and parent should have the same ASPM setting. 6866 * Disable ASPM in downstream component first and then upstream. 6867 */ 6868 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_dis_mask); 6869 6870 if (parent) 6871 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL, 6872 aspm_dis_mask); 6873 } 6874 6875 /** 6876 * e1000e_disable_aspm - Disable ASPM states. 6877 * @pdev: pointer to PCI device struct 6878 * @state: bit-mask of ASPM states to disable 6879 * 6880 * This function acquires the pci_bus_sem! 6881 * Some devices *must* have certain ASPM states disabled per hardware errata. 6882 **/ 6883 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state) 6884 { 6885 __e1000e_disable_aspm(pdev, state, 0); 6886 } 6887 6888 /** 6889 * e1000e_disable_aspm_locked - Disable ASPM states. 6890 * @pdev: pointer to PCI device struct 6891 * @state: bit-mask of ASPM states to disable 6892 * 6893 * This function must be called with pci_bus_sem acquired! 6894 * Some devices *must* have certain ASPM states disabled per hardware errata. 6895 **/ 6896 static void e1000e_disable_aspm_locked(struct pci_dev *pdev, u16 state) 6897 { 6898 __e1000e_disable_aspm(pdev, state, 1); 6899 } 6900 6901 static int e1000e_pm_thaw(struct device *dev) 6902 { 6903 struct net_device *netdev = dev_get_drvdata(dev); 6904 struct e1000_adapter *adapter = netdev_priv(netdev); 6905 int rc = 0; 6906 6907 e1000e_set_interrupt_capability(adapter); 6908 6909 rtnl_lock(); 6910 if (netif_running(netdev)) { 6911 rc = e1000_request_irq(adapter); 6912 if (rc) 6913 goto err_irq; 6914 6915 e1000e_up(adapter); 6916 } 6917 6918 netif_device_attach(netdev); 6919 err_irq: 6920 rtnl_unlock(); 6921 6922 return rc; 6923 } 6924 6925 static int __e1000_resume(struct pci_dev *pdev) 6926 { 6927 struct net_device *netdev = pci_get_drvdata(pdev); 6928 struct e1000_adapter *adapter = netdev_priv(netdev); 6929 struct e1000_hw *hw = &adapter->hw; 6930 u16 aspm_disable_flag = 0; 6931 6932 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 6933 aspm_disable_flag = PCIE_LINK_STATE_L0S; 6934 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 6935 aspm_disable_flag |= PCIE_LINK_STATE_L1; 6936 if (aspm_disable_flag) 6937 e1000e_disable_aspm(pdev, aspm_disable_flag); 6938 6939 pci_set_master(pdev); 6940 6941 if (hw->mac.type >= e1000_pch2lan) 6942 e1000_resume_workarounds_pchlan(&adapter->hw); 6943 6944 e1000e_power_up_phy(adapter); 6945 6946 /* report the system wakeup cause from S3/S4 */ 6947 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { 6948 u16 phy_data; 6949 6950 e1e_rphy(&adapter->hw, BM_WUS, &phy_data); 6951 if (phy_data) { 6952 e_info("PHY Wakeup cause - %s\n", 6953 phy_data & E1000_WUS_EX ? "Unicast Packet" : 6954 phy_data & E1000_WUS_MC ? "Multicast Packet" : 6955 phy_data & E1000_WUS_BC ? "Broadcast Packet" : 6956 phy_data & E1000_WUS_MAG ? "Magic Packet" : 6957 phy_data & E1000_WUS_LNKC ? 6958 "Link Status Change" : "other"); 6959 } 6960 e1e_wphy(&adapter->hw, BM_WUS, ~0); 6961 } else { 6962 u32 wus = er32(WUS); 6963 6964 if (wus) { 6965 e_info("MAC Wakeup cause - %s\n", 6966 wus & E1000_WUS_EX ? "Unicast Packet" : 6967 wus & E1000_WUS_MC ? "Multicast Packet" : 6968 wus & E1000_WUS_BC ? "Broadcast Packet" : 6969 wus & E1000_WUS_MAG ? "Magic Packet" : 6970 wus & E1000_WUS_LNKC ? "Link Status Change" : 6971 "other"); 6972 } 6973 ew32(WUS, ~0); 6974 } 6975 6976 e1000e_reset(adapter); 6977 6978 e1000_init_manageability_pt(adapter); 6979 6980 /* If the controller has AMT, do not set DRV_LOAD until the interface 6981 * is up. For all other cases, let the f/w know that the h/w is now 6982 * under the control of the driver. 6983 */ 6984 if (!(adapter->flags & FLAG_HAS_AMT)) 6985 e1000e_get_hw_control(adapter); 6986 6987 return 0; 6988 } 6989 6990 static int e1000e_pm_prepare(struct device *dev) 6991 { 6992 return pm_runtime_suspended(dev) && 6993 pm_suspend_via_firmware(); 6994 } 6995 6996 static int e1000e_pm_suspend(struct device *dev) 6997 { 6998 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 6999 struct e1000_adapter *adapter = netdev_priv(netdev); 7000 struct pci_dev *pdev = to_pci_dev(dev); 7001 int rc; 7002 7003 e1000e_flush_lpic(pdev); 7004 7005 e1000e_pm_freeze(dev); 7006 7007 rc = __e1000_shutdown(pdev, false); 7008 if (!rc) { 7009 /* Introduce S0ix implementation */ 7010 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 7011 e1000e_s0ix_entry_flow(adapter); 7012 } 7013 7014 return 0; 7015 } 7016 7017 static int e1000e_pm_resume(struct device *dev) 7018 { 7019 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev)); 7020 struct e1000_adapter *adapter = netdev_priv(netdev); 7021 struct pci_dev *pdev = to_pci_dev(dev); 7022 int rc; 7023 7024 /* Introduce S0ix implementation */ 7025 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) 7026 e1000e_s0ix_exit_flow(adapter); 7027 7028 rc = __e1000_resume(pdev); 7029 if (rc) 7030 return rc; 7031 7032 return e1000e_pm_thaw(dev); 7033 } 7034 7035 static __maybe_unused int e1000e_pm_runtime_idle(struct device *dev) 7036 { 7037 struct net_device *netdev = dev_get_drvdata(dev); 7038 struct e1000_adapter *adapter = netdev_priv(netdev); 7039 u16 eee_lp; 7040 7041 eee_lp = adapter->hw.dev_spec.ich8lan.eee_lp_ability; 7042 7043 if (!e1000e_has_link(adapter)) { 7044 adapter->hw.dev_spec.ich8lan.eee_lp_ability = eee_lp; 7045 pm_schedule_suspend(dev, 5 * MSEC_PER_SEC); 7046 } 7047 7048 return -EBUSY; 7049 } 7050 7051 static int e1000e_pm_runtime_resume(struct device *dev) 7052 { 7053 struct pci_dev *pdev = to_pci_dev(dev); 7054 struct net_device *netdev = pci_get_drvdata(pdev); 7055 struct e1000_adapter *adapter = netdev_priv(netdev); 7056 int rc; 7057 7058 pdev->pme_poll = true; 7059 7060 rc = __e1000_resume(pdev); 7061 if (rc) 7062 return rc; 7063 7064 if (netdev->flags & IFF_UP) 7065 e1000e_up(adapter); 7066 7067 return rc; 7068 } 7069 7070 static int e1000e_pm_runtime_suspend(struct device *dev) 7071 { 7072 struct pci_dev *pdev = to_pci_dev(dev); 7073 struct net_device *netdev = pci_get_drvdata(pdev); 7074 struct e1000_adapter *adapter = netdev_priv(netdev); 7075 7076 if (netdev->flags & IFF_UP) { 7077 int count = E1000_CHECK_RESET_COUNT; 7078 7079 while (test_bit(__E1000_RESETTING, &adapter->state) && count--) 7080 usleep_range(10000, 11000); 7081 7082 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state)); 7083 7084 /* Down the device without resetting the hardware */ 7085 e1000e_down(adapter, false); 7086 } 7087 7088 if (__e1000_shutdown(pdev, true)) { 7089 e1000e_pm_runtime_resume(dev); 7090 return -EBUSY; 7091 } 7092 7093 return 0; 7094 } 7095 7096 static void e1000_shutdown(struct pci_dev *pdev) 7097 { 7098 e1000e_flush_lpic(pdev); 7099 7100 e1000e_pm_freeze(&pdev->dev); 7101 7102 __e1000_shutdown(pdev, false); 7103 } 7104 7105 #ifdef CONFIG_NET_POLL_CONTROLLER 7106 7107 static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data) 7108 { 7109 struct net_device *netdev = data; 7110 struct e1000_adapter *adapter = netdev_priv(netdev); 7111 7112 if (adapter->msix_entries) { 7113 int vector, msix_irq; 7114 7115 vector = 0; 7116 msix_irq = adapter->msix_entries[vector].vector; 7117 if (disable_hardirq(msix_irq)) 7118 e1000_intr_msix_rx(msix_irq, netdev); 7119 enable_irq(msix_irq); 7120 7121 vector++; 7122 msix_irq = adapter->msix_entries[vector].vector; 7123 if (disable_hardirq(msix_irq)) 7124 e1000_intr_msix_tx(msix_irq, netdev); 7125 enable_irq(msix_irq); 7126 7127 vector++; 7128 msix_irq = adapter->msix_entries[vector].vector; 7129 if (disable_hardirq(msix_irq)) 7130 e1000_msix_other(msix_irq, netdev); 7131 enable_irq(msix_irq); 7132 } 7133 7134 return IRQ_HANDLED; 7135 } 7136 7137 /** 7138 * e1000_netpoll 7139 * @netdev: network interface device structure 7140 * 7141 * Polling 'interrupt' - used by things like netconsole to send skbs 7142 * without having to re-enable interrupts. It's not called while 7143 * the interrupt routine is executing. 7144 */ 7145 static void e1000_netpoll(struct net_device *netdev) 7146 { 7147 struct e1000_adapter *adapter = netdev_priv(netdev); 7148 7149 switch (adapter->int_mode) { 7150 case E1000E_INT_MODE_MSIX: 7151 e1000_intr_msix(adapter->pdev->irq, netdev); 7152 break; 7153 case E1000E_INT_MODE_MSI: 7154 if (disable_hardirq(adapter->pdev->irq)) 7155 e1000_intr_msi(adapter->pdev->irq, netdev); 7156 enable_irq(adapter->pdev->irq); 7157 break; 7158 default: /* E1000E_INT_MODE_LEGACY */ 7159 if (disable_hardirq(adapter->pdev->irq)) 7160 e1000_intr(adapter->pdev->irq, netdev); 7161 enable_irq(adapter->pdev->irq); 7162 break; 7163 } 7164 } 7165 #endif 7166 7167 /** 7168 * e1000_io_error_detected - called when PCI error is detected 7169 * @pdev: Pointer to PCI device 7170 * @state: The current pci connection state 7171 * 7172 * This function is called after a PCI bus error affecting 7173 * this device has been detected. 7174 */ 7175 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, 7176 pci_channel_state_t state) 7177 { 7178 e1000e_pm_freeze(&pdev->dev); 7179 7180 if (state == pci_channel_io_perm_failure) 7181 return PCI_ERS_RESULT_DISCONNECT; 7182 7183 pci_disable_device(pdev); 7184 7185 /* Request a slot reset. */ 7186 return PCI_ERS_RESULT_NEED_RESET; 7187 } 7188 7189 /** 7190 * e1000_io_slot_reset - called after the pci bus has been reset. 7191 * @pdev: Pointer to PCI device 7192 * 7193 * Restart the card from scratch, as if from a cold-boot. Implementation 7194 * resembles the first-half of the e1000e_pm_resume routine. 7195 */ 7196 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev) 7197 { 7198 struct net_device *netdev = pci_get_drvdata(pdev); 7199 struct e1000_adapter *adapter = netdev_priv(netdev); 7200 struct e1000_hw *hw = &adapter->hw; 7201 u16 aspm_disable_flag = 0; 7202 int err; 7203 pci_ers_result_t result; 7204 7205 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S) 7206 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7207 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1) 7208 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7209 if (aspm_disable_flag) 7210 e1000e_disable_aspm_locked(pdev, aspm_disable_flag); 7211 7212 err = pci_enable_device_mem(pdev); 7213 if (err) { 7214 dev_err(&pdev->dev, 7215 "Cannot re-enable PCI device after reset.\n"); 7216 result = PCI_ERS_RESULT_DISCONNECT; 7217 } else { 7218 pci_restore_state(pdev); 7219 pci_set_master(pdev); 7220 7221 pci_enable_wake(pdev, PCI_D3hot, 0); 7222 pci_enable_wake(pdev, PCI_D3cold, 0); 7223 7224 e1000e_reset(adapter); 7225 ew32(WUS, ~0); 7226 result = PCI_ERS_RESULT_RECOVERED; 7227 } 7228 7229 return result; 7230 } 7231 7232 /** 7233 * e1000_io_resume - called when traffic can start flowing again. 7234 * @pdev: Pointer to PCI device 7235 * 7236 * This callback is called when the error recovery driver tells us that 7237 * its OK to resume normal operation. Implementation resembles the 7238 * second-half of the e1000e_pm_resume routine. 7239 */ 7240 static void e1000_io_resume(struct pci_dev *pdev) 7241 { 7242 struct net_device *netdev = pci_get_drvdata(pdev); 7243 struct e1000_adapter *adapter = netdev_priv(netdev); 7244 7245 e1000_init_manageability_pt(adapter); 7246 7247 e1000e_pm_thaw(&pdev->dev); 7248 7249 /* If the controller has AMT, do not set DRV_LOAD until the interface 7250 * is up. For all other cases, let the f/w know that the h/w is now 7251 * under the control of the driver. 7252 */ 7253 if (!(adapter->flags & FLAG_HAS_AMT)) 7254 e1000e_get_hw_control(adapter); 7255 } 7256 7257 static void e1000_print_device_info(struct e1000_adapter *adapter) 7258 { 7259 struct e1000_hw *hw = &adapter->hw; 7260 struct net_device *netdev = adapter->netdev; 7261 u32 ret_val; 7262 u8 pba_str[E1000_PBANUM_LENGTH]; 7263 7264 /* print bus type/speed/width info */ 7265 e_info("(PCI Express:2.5GT/s:%s) %pM\n", 7266 /* bus width */ 7267 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" : 7268 "Width x1"), 7269 /* MAC address */ 7270 netdev->dev_addr); 7271 e_info("Intel(R) PRO/%s Network Connection\n", 7272 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000"); 7273 ret_val = e1000_read_pba_string_generic(hw, pba_str, 7274 E1000_PBANUM_LENGTH); 7275 if (ret_val) 7276 strscpy((char *)pba_str, "Unknown", sizeof(pba_str)); 7277 e_info("MAC: %d, PHY: %d, PBA No: %s\n", 7278 hw->mac.type, hw->phy.type, pba_str); 7279 } 7280 7281 static void e1000_eeprom_checks(struct e1000_adapter *adapter) 7282 { 7283 struct e1000_hw *hw = &adapter->hw; 7284 int ret_val; 7285 u16 buf = 0; 7286 7287 if (hw->mac.type != e1000_82573) 7288 return; 7289 7290 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf); 7291 le16_to_cpus(&buf); 7292 if (!ret_val && (!(buf & BIT(0)))) { 7293 /* Deep Smart Power Down (DSPD) */ 7294 dev_warn(&adapter->pdev->dev, 7295 "Warning: detected DSPD enabled in EEPROM\n"); 7296 } 7297 } 7298 7299 static netdev_features_t e1000_fix_features(struct net_device *netdev, 7300 netdev_features_t features) 7301 { 7302 struct e1000_adapter *adapter = netdev_priv(netdev); 7303 struct e1000_hw *hw = &adapter->hw; 7304 7305 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */ 7306 if ((hw->mac.type >= e1000_pch2lan) && (netdev->mtu > ETH_DATA_LEN)) 7307 features &= ~NETIF_F_RXFCS; 7308 7309 /* Since there is no support for separate Rx/Tx vlan accel 7310 * enable/disable make sure Tx flag is always in same state as Rx. 7311 */ 7312 if (features & NETIF_F_HW_VLAN_CTAG_RX) 7313 features |= NETIF_F_HW_VLAN_CTAG_TX; 7314 else 7315 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 7316 7317 return features; 7318 } 7319 7320 static int e1000_set_features(struct net_device *netdev, 7321 netdev_features_t features) 7322 { 7323 struct e1000_adapter *adapter = netdev_priv(netdev); 7324 netdev_features_t changed = features ^ netdev->features; 7325 7326 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) 7327 adapter->flags |= FLAG_TSO_FORCE; 7328 7329 if (!(changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX | 7330 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS | 7331 NETIF_F_RXALL))) 7332 return 0; 7333 7334 if (changed & NETIF_F_RXFCS) { 7335 if (features & NETIF_F_RXFCS) { 7336 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7337 } else { 7338 /* We need to take it back to defaults, which might mean 7339 * stripping is still disabled at the adapter level. 7340 */ 7341 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING) 7342 adapter->flags2 |= FLAG2_CRC_STRIPPING; 7343 else 7344 adapter->flags2 &= ~FLAG2_CRC_STRIPPING; 7345 } 7346 } 7347 7348 netdev->features = features; 7349 7350 if (netif_running(netdev)) 7351 e1000e_reinit_locked(adapter); 7352 else 7353 e1000e_reset(adapter); 7354 7355 return 1; 7356 } 7357 7358 static const struct net_device_ops e1000e_netdev_ops = { 7359 .ndo_open = e1000e_open, 7360 .ndo_stop = e1000e_close, 7361 .ndo_start_xmit = e1000_xmit_frame, 7362 .ndo_get_stats64 = e1000e_get_stats64, 7363 .ndo_set_rx_mode = e1000e_set_rx_mode, 7364 .ndo_set_mac_address = e1000_set_mac, 7365 .ndo_change_mtu = e1000_change_mtu, 7366 .ndo_eth_ioctl = e1000_ioctl, 7367 .ndo_tx_timeout = e1000_tx_timeout, 7368 .ndo_validate_addr = eth_validate_addr, 7369 7370 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid, 7371 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid, 7372 #ifdef CONFIG_NET_POLL_CONTROLLER 7373 .ndo_poll_controller = e1000_netpoll, 7374 #endif 7375 .ndo_set_features = e1000_set_features, 7376 .ndo_fix_features = e1000_fix_features, 7377 .ndo_features_check = passthru_features_check, 7378 .ndo_hwtstamp_get = e1000e_hwtstamp_get, 7379 .ndo_hwtstamp_set = e1000e_hwtstamp_set, 7380 }; 7381 7382 /** 7383 * e1000_probe - Device Initialization Routine 7384 * @pdev: PCI device information struct 7385 * @ent: entry in e1000_pci_tbl 7386 * 7387 * Returns 0 on success, negative on failure 7388 * 7389 * e1000_probe initializes an adapter identified by a pci_dev structure. 7390 * The OS initialization, configuring of the adapter private structure, 7391 * and a hardware reset occur. 7392 **/ 7393 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 7394 { 7395 struct net_device *netdev; 7396 struct e1000_adapter *adapter; 7397 struct e1000_hw *hw; 7398 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data]; 7399 resource_size_t mmio_start, mmio_len; 7400 resource_size_t flash_start, flash_len; 7401 static int cards_found; 7402 u16 aspm_disable_flag = 0; 7403 u16 eeprom_data = 0; 7404 u16 eeprom_apme_mask = E1000_EEPROM_APME; 7405 int bars, i, err; 7406 s32 ret_val = 0; 7407 7408 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S) 7409 aspm_disable_flag = PCIE_LINK_STATE_L0S; 7410 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1) 7411 aspm_disable_flag |= PCIE_LINK_STATE_L1; 7412 if (aspm_disable_flag) 7413 e1000e_disable_aspm(pdev, aspm_disable_flag); 7414 7415 err = pci_enable_device_mem(pdev); 7416 if (err) 7417 return err; 7418 7419 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 7420 if (err) { 7421 dev_err(&pdev->dev, 7422 "No usable DMA configuration, aborting\n"); 7423 goto err_dma; 7424 } 7425 7426 bars = pci_select_bars(pdev, IORESOURCE_MEM); 7427 err = pci_request_selected_regions_exclusive(pdev, bars, 7428 e1000e_driver_name); 7429 if (err) 7430 goto err_pci_reg; 7431 7432 pci_set_master(pdev); 7433 /* PCI config space info */ 7434 err = pci_save_state(pdev); 7435 if (err) 7436 goto err_alloc_etherdev; 7437 7438 err = -ENOMEM; 7439 netdev = alloc_etherdev(sizeof(struct e1000_adapter)); 7440 if (!netdev) 7441 goto err_alloc_etherdev; 7442 7443 SET_NETDEV_DEV(netdev, &pdev->dev); 7444 7445 netdev->irq = pdev->irq; 7446 7447 pci_set_drvdata(pdev, netdev); 7448 adapter = netdev_priv(netdev); 7449 hw = &adapter->hw; 7450 adapter->netdev = netdev; 7451 adapter->pdev = pdev; 7452 adapter->ei = ei; 7453 adapter->pba = ei->pba; 7454 adapter->flags = ei->flags; 7455 adapter->flags2 = ei->flags2; 7456 adapter->hw.adapter = adapter; 7457 adapter->hw.mac.type = ei->mac; 7458 adapter->max_hw_frame_size = ei->max_hw_frame_size; 7459 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7460 7461 mmio_start = pci_resource_start(pdev, 0); 7462 mmio_len = pci_resource_len(pdev, 0); 7463 7464 err = -EIO; 7465 /* Smallest BAR0 that covers every register the driver accesses */ 7466 if (mmio_len < SZ_64K) { 7467 dev_err(&pdev->dev, "MMIO len is too small\n"); 7468 goto err_ioremap; 7469 } 7470 7471 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len); 7472 if (!adapter->hw.hw_addr) 7473 goto err_ioremap; 7474 7475 if ((adapter->flags & FLAG_HAS_FLASH) && 7476 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM) && 7477 (hw->mac.type < e1000_pch_spt)) { 7478 flash_start = pci_resource_start(pdev, 1); 7479 flash_len = pci_resource_len(pdev, 1); 7480 adapter->hw.flash_address = ioremap(flash_start, flash_len); 7481 if (!adapter->hw.flash_address) 7482 goto err_flashmap; 7483 } 7484 7485 /* Set default EEE advertisement */ 7486 if (adapter->flags2 & FLAG2_HAS_EEE) 7487 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; 7488 7489 /* construct the net_device struct */ 7490 netdev->netdev_ops = &e1000e_netdev_ops; 7491 e1000e_set_ethtool_ops(netdev); 7492 netdev->watchdog_timeo = 5 * HZ; 7493 netif_napi_add(netdev, &adapter->napi, e1000e_poll); 7494 strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); 7495 7496 netdev->mem_start = mmio_start; 7497 netdev->mem_end = mmio_start + mmio_len; 7498 7499 adapter->bd_number = cards_found++; 7500 7501 e1000e_check_options(adapter); 7502 7503 /* setup adapter struct */ 7504 err = e1000_sw_init(adapter); 7505 if (err) 7506 goto err_sw_init; 7507 7508 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 7509 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); 7510 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 7511 7512 err = ei->get_variants(adapter); 7513 if (err) 7514 goto err_hw_init; 7515 7516 if ((adapter->flags & FLAG_IS_ICH) && 7517 (adapter->flags & FLAG_READ_ONLY_NVM) && 7518 (hw->mac.type < e1000_pch_spt)) 7519 e1000e_write_protect_nvm_ich8lan(&adapter->hw); 7520 7521 hw->mac.ops.get_bus_info(&adapter->hw); 7522 7523 adapter->hw.phy.autoneg_wait_to_complete = 0; 7524 7525 /* Copper options */ 7526 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 7527 adapter->hw.phy.mdix = AUTO_ALL_MODES; 7528 adapter->hw.phy.disable_polarity_correction = 0; 7529 adapter->hw.phy.ms_type = e1000_ms_hw_default; 7530 } 7531 7532 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw)) 7533 dev_info(&pdev->dev, 7534 "PHY reset is blocked due to SOL/IDER session.\n"); 7535 7536 /* Set initial default active device features */ 7537 netdev->features = (NETIF_F_SG | 7538 NETIF_F_HW_VLAN_CTAG_RX | 7539 NETIF_F_HW_VLAN_CTAG_TX | 7540 NETIF_F_TSO | 7541 NETIF_F_TSO6 | 7542 NETIF_F_RXHASH | 7543 NETIF_F_RXCSUM | 7544 NETIF_F_HW_CSUM); 7545 7546 /* disable TSO for pcie and 10/100 speeds to avoid 7547 * some hardware issues and for i219 to fix transfer 7548 * speed being capped at 60% 7549 */ 7550 if (!(adapter->flags & FLAG_TSO_FORCE)) { 7551 switch (adapter->link_speed) { 7552 case SPEED_10: 7553 case SPEED_100: 7554 e_info("10/100 speed: disabling TSO\n"); 7555 netdev->features &= ~NETIF_F_TSO; 7556 netdev->features &= ~NETIF_F_TSO6; 7557 break; 7558 case SPEED_1000: 7559 netdev->features |= NETIF_F_TSO; 7560 netdev->features |= NETIF_F_TSO6; 7561 break; 7562 default: 7563 /* oops */ 7564 break; 7565 } 7566 if (hw->mac.type == e1000_pch_spt) { 7567 netdev->features &= ~NETIF_F_TSO; 7568 netdev->features &= ~NETIF_F_TSO6; 7569 } 7570 } 7571 7572 /* Set user-changeable features (subset of all device features) */ 7573 netdev->hw_features = netdev->features; 7574 netdev->hw_features |= NETIF_F_RXFCS; 7575 netdev->priv_flags |= IFF_SUPP_NOFCS; 7576 netdev->hw_features |= NETIF_F_RXALL; 7577 7578 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) 7579 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7580 7581 netdev->vlan_features |= (NETIF_F_SG | 7582 NETIF_F_TSO | 7583 NETIF_F_TSO6 | 7584 NETIF_F_HW_CSUM); 7585 7586 netdev->priv_flags |= IFF_UNICAST_FLT; 7587 7588 netdev->features |= NETIF_F_HIGHDMA; 7589 netdev->vlan_features |= NETIF_F_HIGHDMA; 7590 7591 /* MTU range: 68 - max_hw_frame_size */ 7592 netdev->min_mtu = ETH_MIN_MTU; 7593 netdev->max_mtu = adapter->max_hw_frame_size - 7594 (VLAN_ETH_HLEN + ETH_FCS_LEN); 7595 7596 if (e1000e_enable_mng_pass_thru(&adapter->hw)) 7597 adapter->flags |= FLAG_MNG_PT_ENABLED; 7598 7599 /* before reading the NVM, reset the controller to 7600 * put the device in a known good starting state 7601 */ 7602 adapter->hw.mac.ops.reset_hw(&adapter->hw); 7603 7604 /* systems with ASPM and others may see the checksum fail on the first 7605 * attempt. Let's give it a few tries 7606 */ 7607 for (i = 0;; i++) { 7608 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0) 7609 break; 7610 if (i == 2) { 7611 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 7612 err = -EIO; 7613 goto err_eeprom; 7614 } 7615 } 7616 7617 e1000_eeprom_checks(adapter); 7618 7619 /* copy the MAC address */ 7620 if (e1000e_read_mac_addr(&adapter->hw)) 7621 dev_err(&pdev->dev, 7622 "NVM Read Error while reading MAC address\n"); 7623 7624 eth_hw_addr_set(netdev, adapter->hw.mac.addr); 7625 7626 if (!is_valid_ether_addr(netdev->dev_addr)) { 7627 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n", 7628 netdev->dev_addr); 7629 err = -EIO; 7630 goto err_eeprom; 7631 } 7632 7633 timer_setup(&adapter->watchdog_timer, e1000_watchdog, 0); 7634 timer_setup(&adapter->phy_info_timer, e1000_update_phy_info, 0); 7635 7636 INIT_WORK(&adapter->reset_task, e1000_reset_task); 7637 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task); 7638 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround); 7639 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task); 7640 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang); 7641 7642 /* Initialize link parameters. User can change them with ethtool */ 7643 adapter->hw.mac.autoneg = 1; 7644 adapter->fc_autoneg = true; 7645 adapter->hw.fc.requested_mode = e1000_fc_default; 7646 adapter->hw.fc.current_mode = e1000_fc_default; 7647 adapter->hw.phy.autoneg_advertised = 0x2f; 7648 7649 /* Initial Wake on LAN setting - If APM wake is enabled in 7650 * the EEPROM, enable the ACPI Magic Packet filter 7651 */ 7652 if (adapter->flags & FLAG_APME_IN_WUC) { 7653 /* APME bit in EEPROM is mapped to WUC.APME */ 7654 eeprom_data = er32(WUC); 7655 eeprom_apme_mask = E1000_WUC_APME; 7656 if ((hw->mac.type > e1000_ich10lan) && 7657 (eeprom_data & E1000_WUC_PHY_WAKE)) 7658 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP; 7659 } else if (adapter->flags & FLAG_APME_IN_CTRL3) { 7660 if (adapter->flags & FLAG_APME_CHECK_PORT_B && 7661 (adapter->hw.bus.func == 1)) 7662 ret_val = e1000_read_nvm(&adapter->hw, 7663 NVM_INIT_CONTROL3_PORT_B, 7664 1, &eeprom_data); 7665 else 7666 ret_val = e1000_read_nvm(&adapter->hw, 7667 NVM_INIT_CONTROL3_PORT_A, 7668 1, &eeprom_data); 7669 } 7670 7671 /* fetch WoL from EEPROM */ 7672 if (ret_val) 7673 e_dbg("NVM read error getting WoL initial values: %d\n", ret_val); 7674 else if (eeprom_data & eeprom_apme_mask) 7675 adapter->eeprom_wol |= E1000_WUFC_MAG; 7676 7677 /* now that we have the eeprom settings, apply the special cases 7678 * where the eeprom may be wrong or the board simply won't support 7679 * wake on lan on a particular port 7680 */ 7681 if (!(adapter->flags & FLAG_HAS_WOL)) 7682 adapter->eeprom_wol = 0; 7683 7684 /* initialize the wol settings based on the eeprom settings */ 7685 adapter->wol = adapter->eeprom_wol; 7686 7687 /* make sure adapter isn't asleep if manageability is enabled */ 7688 if (adapter->wol || (adapter->flags & FLAG_MNG_PT_ENABLED) || 7689 (hw->mac.ops.check_mng_mode(hw))) 7690 device_wakeup_enable(&pdev->dev); 7691 7692 /* save off EEPROM version number */ 7693 ret_val = e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers); 7694 7695 if (ret_val) { 7696 e_dbg("NVM read error getting EEPROM version: %d\n", ret_val); 7697 adapter->eeprom_vers = 0; 7698 } 7699 7700 /* init PTP hardware clock */ 7701 e1000e_ptp_init(adapter); 7702 7703 /* disable K1 by default on known problematic systems */ 7704 if (hw->mac.type >= e1000_pch_mtp && dmi_check_system(disable_k1_list)) 7705 adapter->flags2 |= FLAG2_DISABLE_K1; 7706 7707 /* reset the hardware with the new settings */ 7708 e1000e_reset(adapter); 7709 7710 /* If the controller has AMT, do not set DRV_LOAD until the interface 7711 * is up. For all other cases, let the f/w know that the h/w is now 7712 * under the control of the driver. 7713 */ 7714 if (!(adapter->flags & FLAG_HAS_AMT)) 7715 e1000e_get_hw_control(adapter); 7716 7717 if (hw->mac.type >= e1000_pch_cnp) 7718 adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS; 7719 7720 strscpy(netdev->name, "eth%d", sizeof(netdev->name)); 7721 err = register_netdev(netdev); 7722 if (err) 7723 goto err_register; 7724 7725 /* carrier off reporting is important to ethtool even BEFORE open */ 7726 netif_carrier_off(netdev); 7727 7728 e1000_print_device_info(adapter); 7729 7730 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_SMART_PREPARE); 7731 7732 if (pci_dev_run_wake(pdev)) 7733 pm_runtime_put_noidle(&pdev->dev); 7734 7735 return 0; 7736 7737 err_register: 7738 if (!(adapter->flags & FLAG_HAS_AMT)) 7739 e1000e_release_hw_control(adapter); 7740 e1000e_ptp_remove(adapter); 7741 err_eeprom: 7742 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw)) 7743 e1000_phy_hw_reset(&adapter->hw); 7744 err_hw_init: 7745 kfree(adapter->tx_ring); 7746 kfree(adapter->rx_ring); 7747 err_sw_init: 7748 if ((adapter->hw.flash_address) && (hw->mac.type < e1000_pch_spt)) 7749 iounmap(adapter->hw.flash_address); 7750 e1000e_reset_interrupt_capability(adapter); 7751 err_flashmap: 7752 iounmap(adapter->hw.hw_addr); 7753 err_ioremap: 7754 free_netdev(netdev); 7755 err_alloc_etherdev: 7756 pci_release_mem_regions(pdev); 7757 err_pci_reg: 7758 err_dma: 7759 pci_disable_device(pdev); 7760 return err; 7761 } 7762 7763 /** 7764 * e1000_remove - Device Removal Routine 7765 * @pdev: PCI device information struct 7766 * 7767 * e1000_remove is called by the PCI subsystem to alert the driver 7768 * that it should release a PCI device. This could be caused by a 7769 * Hot-Plug event, or because the driver is going to be removed from 7770 * memory. 7771 **/ 7772 static void e1000_remove(struct pci_dev *pdev) 7773 { 7774 struct net_device *netdev = pci_get_drvdata(pdev); 7775 struct e1000_adapter *adapter = netdev_priv(netdev); 7776 7777 e1000e_ptp_remove(adapter); 7778 7779 /* The timers may be rescheduled, so explicitly disable them 7780 * from being rescheduled. 7781 */ 7782 set_bit(__E1000_DOWN, &adapter->state); 7783 timer_delete_sync(&adapter->watchdog_timer); 7784 timer_delete_sync(&adapter->phy_info_timer); 7785 7786 cancel_work_sync(&adapter->reset_task); 7787 cancel_work_sync(&adapter->watchdog_task); 7788 cancel_work_sync(&adapter->downshift_task); 7789 cancel_work_sync(&adapter->update_phy_task); 7790 cancel_work_sync(&adapter->print_hang_task); 7791 7792 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) { 7793 cancel_work_sync(&adapter->tx_hwtstamp_work); 7794 if (adapter->tx_hwtstamp_skb) { 7795 dev_consume_skb_any(adapter->tx_hwtstamp_skb); 7796 adapter->tx_hwtstamp_skb = NULL; 7797 } 7798 } 7799 7800 unregister_netdev(netdev); 7801 7802 if (pci_dev_run_wake(pdev)) 7803 pm_runtime_get_noresume(&pdev->dev); 7804 7805 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7806 * would have already happened in close and is redundant. 7807 */ 7808 e1000e_release_hw_control(adapter); 7809 7810 e1000e_reset_interrupt_capability(adapter); 7811 kfree(adapter->tx_ring); 7812 kfree(adapter->rx_ring); 7813 7814 iounmap(adapter->hw.hw_addr); 7815 if ((adapter->hw.flash_address) && 7816 (adapter->hw.mac.type < e1000_pch_spt)) 7817 iounmap(adapter->hw.flash_address); 7818 pci_release_mem_regions(pdev); 7819 7820 free_netdev(netdev); 7821 7822 pci_disable_device(pdev); 7823 } 7824 7825 /* PCI Error Recovery (ERS) */ 7826 static const struct pci_error_handlers e1000_err_handler = { 7827 .error_detected = e1000_io_error_detected, 7828 .slot_reset = e1000_io_slot_reset, 7829 .resume = e1000_io_resume, 7830 }; 7831 7832 static const struct pci_device_id e1000_pci_tbl[] = { 7833 { 7834 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), 7835 .driver_data = board_82571, 7836 }, { 7837 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), 7838 .driver_data = board_82571, 7839 }, { 7840 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), 7841 .driver_data = board_82571, 7842 }, { 7843 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), 7844 .driver_data = board_82571, 7845 }, { 7846 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), 7847 .driver_data = board_82571, 7848 }, { 7849 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), 7850 .driver_data = board_82571, 7851 }, { 7852 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), 7853 .driver_data = board_82571, 7854 }, { 7855 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), 7856 .driver_data = board_82571, 7857 }, { 7858 PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), 7859 .driver_data = board_82571, 7860 }, 7861 7862 { 7863 PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), 7864 .driver_data = board_82572, 7865 }, { 7866 PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), 7867 .driver_data = board_82572, 7868 }, { 7869 PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), 7870 .driver_data = board_82572, 7871 }, { 7872 PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), 7873 .driver_data = board_82572, 7874 }, 7875 7876 { 7877 PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), 7878 .driver_data = board_82573, 7879 }, { 7880 PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), 7881 .driver_data = board_82573, 7882 }, { 7883 PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), 7884 .driver_data = board_82573, 7885 }, 7886 7887 { 7888 PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), 7889 .driver_data = board_82574, 7890 }, { 7891 PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), 7892 .driver_data = board_82574, 7893 }, { 7894 PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), 7895 .driver_data = board_82583, 7896 }, 7897 7898 { 7899 PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT), 7900 .driver_data = board_80003es2lan, 7901 }, { 7902 PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT), 7903 .driver_data = board_80003es2lan, 7904 }, { 7905 PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT), 7906 .driver_data = board_80003es2lan, 7907 }, { 7908 PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT), 7909 .driver_data = board_80003es2lan, 7910 }, 7911 7912 { 7913 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), 7914 .driver_data = board_ich8lan, 7915 }, { 7916 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), 7917 .driver_data = board_ich8lan, 7918 }, { 7919 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), 7920 .driver_data = board_ich8lan, 7921 }, { 7922 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), 7923 .driver_data = board_ich8lan, 7924 }, { 7925 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), 7926 .driver_data = board_ich8lan, 7927 }, { 7928 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), 7929 .driver_data = board_ich8lan, 7930 }, { 7931 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), 7932 .driver_data = board_ich8lan, 7933 }, { 7934 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), 7935 .driver_data = board_ich8lan, 7936 }, 7937 7938 { 7939 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), 7940 .driver_data = board_ich9lan, 7941 }, { 7942 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), 7943 .driver_data = board_ich9lan, 7944 }, { 7945 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), 7946 .driver_data = board_ich9lan, 7947 }, { 7948 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), 7949 .driver_data = board_ich9lan, 7950 }, { 7951 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), 7952 .driver_data = board_ich9lan, 7953 }, { 7954 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), 7955 .driver_data = board_ich9lan, 7956 }, { 7957 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), 7958 .driver_data = board_ich9lan, 7959 }, { 7960 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), 7961 .driver_data = board_ich9lan, 7962 }, { 7963 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), 7964 .driver_data = board_ich9lan 7965 }, 7966 7967 { 7968 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), 7969 .driver_data = board_ich9lan, 7970 }, { 7971 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), 7972 .driver_data = board_ich9lan, 7973 }, { 7974 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), 7975 .driver_data = board_ich9lan, 7976 }, 7977 7978 { 7979 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), 7980 .driver_data = board_ich10lan, 7981 }, { 7982 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), 7983 .driver_data = board_ich10lan, 7984 }, { 7985 PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), 7986 .driver_data = board_ich10lan, 7987 }, 7988 7989 { 7990 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), 7991 .driver_data = board_pchlan, 7992 }, { 7993 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), 7994 .driver_data = board_pchlan, 7995 }, { 7996 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), 7997 .driver_data = board_pchlan, 7998 }, { 7999 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), 8000 .driver_data = board_pchlan 8001 }, 8002 8003 { 8004 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), 8005 .driver_data = board_pch2lan, 8006 }, { 8007 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), 8008 .driver_data = board_pch2lan 8009 }, 8010 8011 { 8012 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), 8013 .driver_data = board_pch_lpt, 8014 }, { 8015 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), 8016 .driver_data = board_pch_lpt, 8017 }, { 8018 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), 8019 .driver_data = board_pch_lpt, 8020 }, { 8021 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), 8022 .driver_data = board_pch_lpt, 8023 }, { 8024 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2), 8025 .driver_data = board_pch_lpt, 8026 }, { 8027 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2), 8028 .driver_data = board_pch_lpt, 8029 }, { 8030 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3), 8031 .driver_data = board_pch_lpt, 8032 }, { 8033 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3), 8034 .driver_data = board_pch_lpt, 8035 }, { 8036 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM), 8037 .driver_data = board_pch_spt, 8038 }, { 8039 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V), 8040 .driver_data = board_pch_spt, 8041 }, { 8042 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2), 8043 .driver_data = board_pch_spt, 8044 }, { 8045 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2), 8046 .driver_data = board_pch_spt, 8047 }, { 8048 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3), 8049 .driver_data = board_pch_spt, 8050 }, { 8051 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4), 8052 .driver_data = board_pch_spt, 8053 }, { 8054 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4), 8055 .driver_data = board_pch_spt, 8056 }, { 8057 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5), 8058 .driver_data = board_pch_spt, 8059 }, { 8060 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5), 8061 .driver_data = board_pch_spt, 8062 }, { 8063 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6), 8064 .driver_data = board_pch_cnp, 8065 }, { 8066 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), 8067 .driver_data = board_pch_cnp, 8068 }, { 8069 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), 8070 .driver_data = board_pch_cnp, 8071 }, { 8072 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), 8073 .driver_data = board_pch_cnp, 8074 }, { 8075 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), 8076 .driver_data = board_pch_cnp, 8077 }, { 8078 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), 8079 .driver_data = board_pch_cnp, 8080 }, { 8081 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), 8082 .driver_data = board_pch_cnp, 8083 }, { 8084 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), 8085 .driver_data = board_pch_cnp, 8086 }, { 8087 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10), 8088 .driver_data = board_pch_cnp, 8089 }, { 8090 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10), 8091 .driver_data = board_pch_cnp, 8092 }, { 8093 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11), 8094 .driver_data = board_pch_cnp, 8095 }, { 8096 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11), 8097 .driver_data = board_pch_cnp, 8098 }, { 8099 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12), 8100 .driver_data = board_pch_spt, 8101 }, { 8102 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12), 8103 .driver_data = board_pch_spt, 8104 }, { 8105 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13), 8106 .driver_data = board_pch_tgp, 8107 }, { 8108 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13), 8109 .driver_data = board_pch_tgp, 8110 }, { 8111 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14), 8112 .driver_data = board_pch_tgp, 8113 }, { 8114 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), 8115 .driver_data = board_pch_tgp, 8116 }, { 8117 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), 8118 .driver_data = board_pch_tgp, 8119 }, { 8120 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), 8121 .driver_data = board_pch_tgp, 8122 }, { 8123 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM23), 8124 .driver_data = board_pch_adp, 8125 }, { 8126 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V23), 8127 .driver_data = board_pch_adp, 8128 }, { 8129 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), 8130 .driver_data = board_pch_adp, 8131 }, { 8132 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), 8133 .driver_data = board_pch_adp, 8134 }, { 8135 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), 8136 .driver_data = board_pch_adp, 8137 }, { 8138 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), 8139 .driver_data = board_pch_adp, 8140 }, { 8141 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM22), 8142 .driver_data = board_pch_adp, 8143 }, { 8144 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V22), 8145 .driver_data = board_pch_adp, 8146 }, { 8147 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM19), 8148 .driver_data = board_pch_adp, 8149 }, { 8150 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V19), 8151 .driver_data = board_pch_adp, 8152 }, { 8153 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), 8154 .driver_data = board_pch_mtp, 8155 }, { 8156 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), 8157 .driver_data = board_pch_mtp, 8158 }, { 8159 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM20), 8160 .driver_data = board_pch_mtp, 8161 }, { 8162 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V20), 8163 .driver_data = board_pch_mtp, 8164 }, { 8165 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM21), 8166 .driver_data = board_pch_mtp, 8167 }, { 8168 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V21), 8169 .driver_data = board_pch_mtp, 8170 }, { 8171 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_LM24), 8172 .driver_data = board_pch_mtp, 8173 }, { 8174 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_V24), 8175 .driver_data = board_pch_mtp, 8176 }, { 8177 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM25), 8178 .driver_data = board_pch_ptp, 8179 }, { 8180 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V25), 8181 .driver_data = board_pch_ptp, 8182 }, { 8183 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM27), 8184 .driver_data = board_pch_ptp, 8185 }, { 8186 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V27), 8187 .driver_data = board_pch_ptp, 8188 }, { 8189 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_LM29), 8190 .driver_data = board_pch_ptp, 8191 }, { 8192 PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_V29), 8193 .driver_data = board_pch_ptp 8194 }, 8195 8196 { } /* terminate list */ 8197 }; 8198 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl); 8199 8200 static const struct dev_pm_ops e1000e_pm_ops = { 8201 .prepare = e1000e_pm_prepare, 8202 .suspend = e1000e_pm_suspend, 8203 .resume = e1000e_pm_resume, 8204 .freeze = e1000e_pm_freeze, 8205 .thaw = e1000e_pm_thaw, 8206 .poweroff = e1000e_pm_suspend, 8207 .restore = e1000e_pm_resume, 8208 RUNTIME_PM_OPS(e1000e_pm_runtime_suspend, e1000e_pm_runtime_resume, 8209 e1000e_pm_runtime_idle) 8210 }; 8211 8212 /* PCI Device API Driver */ 8213 static struct pci_driver e1000_driver = { 8214 .name = e1000e_driver_name, 8215 .id_table = e1000_pci_tbl, 8216 .probe = e1000_probe, 8217 .remove = e1000_remove, 8218 .driver.pm = pm_ptr(&e1000e_pm_ops), 8219 .shutdown = e1000_shutdown, 8220 .err_handler = &e1000_err_handler 8221 }; 8222 8223 /** 8224 * e1000_init_module - Driver Registration Routine 8225 * 8226 * e1000_init_module is the first routine called when the driver is 8227 * loaded. All it does is register with the PCI subsystem. 8228 **/ 8229 static int __init e1000_init_module(void) 8230 { 8231 pr_info("Intel(R) PRO/1000 Network Driver\n"); 8232 pr_info("Copyright(c) 1999 - 2015 Intel Corporation.\n"); 8233 8234 return pci_register_driver(&e1000_driver); 8235 } 8236 module_init(e1000_init_module); 8237 8238 /** 8239 * e1000_exit_module - Driver Exit Cleanup Routine 8240 * 8241 * e1000_exit_module is called just before the driver is removed 8242 * from memory. 8243 **/ 8244 static void __exit e1000_exit_module(void) 8245 { 8246 pci_unregister_driver(&e1000_driver); 8247 } 8248 module_exit(e1000_exit_module); 8249 8250 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver"); 8251 MODULE_LICENSE("GPL v2"); 8252 8253 /* netdev.c */ 8254