1 /******************************************************************************* 2 3 Intel 10 Gigabit PCI Express Linux driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 #include <linux/types.h> 29 #include <linux/module.h> 30 #include <linux/pci.h> 31 #include <linux/netdevice.h> 32 #include <linux/vmalloc.h> 33 #include <linux/string.h> 34 #include <linux/in.h> 35 #include <linux/interrupt.h> 36 #include <linux/ip.h> 37 #include <linux/tcp.h> 38 #include <linux/sctp.h> 39 #include <linux/pkt_sched.h> 40 #include <linux/ipv6.h> 41 #include <linux/slab.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <linux/ethtool.h> 45 #include <linux/if.h> 46 #include <linux/if_vlan.h> 47 #include <linux/prefetch.h> 48 #include <scsi/fc/fc_fcoe.h> 49 50 #include "ixgbe.h" 51 #include "ixgbe_common.h" 52 #include "ixgbe_dcb_82599.h" 53 #include "ixgbe_sriov.h" 54 55 char ixgbe_driver_name[] = "ixgbe"; 56 static const char ixgbe_driver_string[] = 57 "Intel(R) 10 Gigabit PCI Express Network Driver"; 58 #ifdef IXGBE_FCOE 59 char ixgbe_default_device_descr[] = 60 "Intel(R) 10 Gigabit Network Connection"; 61 #else 62 static char ixgbe_default_device_descr[] = 63 "Intel(R) 10 Gigabit Network Connection"; 64 #endif 65 #define MAJ 3 66 #define MIN 9 67 #define BUILD 15 68 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ 69 __stringify(BUILD) "-k" 70 const char ixgbe_driver_version[] = DRV_VERSION; 71 static const char ixgbe_copyright[] = 72 "Copyright (c) 1999-2012 Intel Corporation."; 73 74 static const struct ixgbe_info *ixgbe_info_tbl[] = { 75 [board_82598] = &ixgbe_82598_info, 76 [board_82599] = &ixgbe_82599_info, 77 [board_X540] = &ixgbe_X540_info, 78 }; 79 80 /* ixgbe_pci_tbl - PCI Device ID Table 81 * 82 * Wildcard entries (PCI_ANY_ID) should come last 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 86 * Class, Class Mask, private data (not used) } 87 */ 88 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = { 89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 }, 90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 }, 91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 }, 92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 }, 93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 }, 94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 }, 95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 }, 96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 }, 97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 }, 98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 }, 99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 }, 100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 }, 101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 }, 102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 }, 103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 }, 104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 }, 105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 }, 106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 }, 107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 }, 108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 }, 109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 }, 110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 }, 111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 }, 112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 }, 113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 }, 114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 }, 115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 }, 116 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 }, 117 /* required last entry */ 118 {0, } 119 }; 120 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); 121 122 #ifdef CONFIG_IXGBE_DCA 123 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, 124 void *p); 125 static struct notifier_block dca_notifier = { 126 .notifier_call = ixgbe_notify_dca, 127 .next = NULL, 128 .priority = 0 129 }; 130 #endif 131 132 #ifdef CONFIG_PCI_IOV 133 static unsigned int max_vfs; 134 module_param(max_vfs, uint, 0); 135 MODULE_PARM_DESC(max_vfs, 136 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63"); 137 #endif /* CONFIG_PCI_IOV */ 138 139 static unsigned int allow_unsupported_sfp; 140 module_param(allow_unsupported_sfp, uint, 0); 141 MODULE_PARM_DESC(allow_unsupported_sfp, 142 "Allow unsupported and untested SFP+ modules on 82599-based adapters"); 143 144 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 145 static int debug = -1; 146 module_param(debug, int, 0); 147 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 148 149 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 150 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); 151 MODULE_LICENSE("GPL"); 152 MODULE_VERSION(DRV_VERSION); 153 154 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) 155 { 156 if (!test_bit(__IXGBE_DOWN, &adapter->state) && 157 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) 158 schedule_work(&adapter->service_task); 159 } 160 161 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter) 162 { 163 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state)); 164 165 /* flush memory to make sure state is correct before next watchdog */ 166 smp_mb__before_clear_bit(); 167 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 168 } 169 170 struct ixgbe_reg_info { 171 u32 ofs; 172 char *name; 173 }; 174 175 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = { 176 177 /* General Registers */ 178 {IXGBE_CTRL, "CTRL"}, 179 {IXGBE_STATUS, "STATUS"}, 180 {IXGBE_CTRL_EXT, "CTRL_EXT"}, 181 182 /* Interrupt Registers */ 183 {IXGBE_EICR, "EICR"}, 184 185 /* RX Registers */ 186 {IXGBE_SRRCTL(0), "SRRCTL"}, 187 {IXGBE_DCA_RXCTRL(0), "DRXCTL"}, 188 {IXGBE_RDLEN(0), "RDLEN"}, 189 {IXGBE_RDH(0), "RDH"}, 190 {IXGBE_RDT(0), "RDT"}, 191 {IXGBE_RXDCTL(0), "RXDCTL"}, 192 {IXGBE_RDBAL(0), "RDBAL"}, 193 {IXGBE_RDBAH(0), "RDBAH"}, 194 195 /* TX Registers */ 196 {IXGBE_TDBAL(0), "TDBAL"}, 197 {IXGBE_TDBAH(0), "TDBAH"}, 198 {IXGBE_TDLEN(0), "TDLEN"}, 199 {IXGBE_TDH(0), "TDH"}, 200 {IXGBE_TDT(0), "TDT"}, 201 {IXGBE_TXDCTL(0), "TXDCTL"}, 202 203 /* List Terminator */ 204 {} 205 }; 206 207 208 /* 209 * ixgbe_regdump - register printout routine 210 */ 211 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo) 212 { 213 int i = 0, j = 0; 214 char rname[16]; 215 u32 regs[64]; 216 217 switch (reginfo->ofs) { 218 case IXGBE_SRRCTL(0): 219 for (i = 0; i < 64; i++) 220 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 221 break; 222 case IXGBE_DCA_RXCTRL(0): 223 for (i = 0; i < 64; i++) 224 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i)); 225 break; 226 case IXGBE_RDLEN(0): 227 for (i = 0; i < 64; i++) 228 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i)); 229 break; 230 case IXGBE_RDH(0): 231 for (i = 0; i < 64; i++) 232 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i)); 233 break; 234 case IXGBE_RDT(0): 235 for (i = 0; i < 64; i++) 236 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i)); 237 break; 238 case IXGBE_RXDCTL(0): 239 for (i = 0; i < 64; i++) 240 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 241 break; 242 case IXGBE_RDBAL(0): 243 for (i = 0; i < 64; i++) 244 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i)); 245 break; 246 case IXGBE_RDBAH(0): 247 for (i = 0; i < 64; i++) 248 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i)); 249 break; 250 case IXGBE_TDBAL(0): 251 for (i = 0; i < 64; i++) 252 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i)); 253 break; 254 case IXGBE_TDBAH(0): 255 for (i = 0; i < 64; i++) 256 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i)); 257 break; 258 case IXGBE_TDLEN(0): 259 for (i = 0; i < 64; i++) 260 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i)); 261 break; 262 case IXGBE_TDH(0): 263 for (i = 0; i < 64; i++) 264 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i)); 265 break; 266 case IXGBE_TDT(0): 267 for (i = 0; i < 64; i++) 268 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i)); 269 break; 270 case IXGBE_TXDCTL(0): 271 for (i = 0; i < 64; i++) 272 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 273 break; 274 default: 275 pr_info("%-15s %08x\n", reginfo->name, 276 IXGBE_READ_REG(hw, reginfo->ofs)); 277 return; 278 } 279 280 for (i = 0; i < 8; i++) { 281 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7); 282 pr_err("%-15s", rname); 283 for (j = 0; j < 8; j++) 284 pr_cont(" %08x", regs[i*8+j]); 285 pr_cont("\n"); 286 } 287 288 } 289 290 /* 291 * ixgbe_dump - Print registers, tx-rings and rx-rings 292 */ 293 static void ixgbe_dump(struct ixgbe_adapter *adapter) 294 { 295 struct net_device *netdev = adapter->netdev; 296 struct ixgbe_hw *hw = &adapter->hw; 297 struct ixgbe_reg_info *reginfo; 298 int n = 0; 299 struct ixgbe_ring *tx_ring; 300 struct ixgbe_tx_buffer *tx_buffer; 301 union ixgbe_adv_tx_desc *tx_desc; 302 struct my_u0 { u64 a; u64 b; } *u0; 303 struct ixgbe_ring *rx_ring; 304 union ixgbe_adv_rx_desc *rx_desc; 305 struct ixgbe_rx_buffer *rx_buffer_info; 306 u32 staterr; 307 int i = 0; 308 309 if (!netif_msg_hw(adapter)) 310 return; 311 312 /* Print netdevice Info */ 313 if (netdev) { 314 dev_info(&adapter->pdev->dev, "Net device Info\n"); 315 pr_info("Device Name state " 316 "trans_start last_rx\n"); 317 pr_info("%-15s %016lX %016lX %016lX\n", 318 netdev->name, 319 netdev->state, 320 netdev->trans_start, 321 netdev->last_rx); 322 } 323 324 /* Print Registers */ 325 dev_info(&adapter->pdev->dev, "Register Dump\n"); 326 pr_info(" Register Name Value\n"); 327 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; 328 reginfo->name; reginfo++) { 329 ixgbe_regdump(hw, reginfo); 330 } 331 332 /* Print TX Ring Summary */ 333 if (!netdev || !netif_running(netdev)) 334 goto exit; 335 336 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 337 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 338 for (n = 0; n < adapter->num_tx_queues; n++) { 339 tx_ring = adapter->tx_ring[n]; 340 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 341 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", 342 n, tx_ring->next_to_use, tx_ring->next_to_clean, 343 (u64)dma_unmap_addr(tx_buffer, dma), 344 dma_unmap_len(tx_buffer, len), 345 tx_buffer->next_to_watch, 346 (u64)tx_buffer->time_stamp); 347 } 348 349 /* Print TX Rings */ 350 if (!netif_msg_tx_done(adapter)) 351 goto rx_ring_summary; 352 353 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 354 355 /* Transmit Descriptor Formats 356 * 357 * Advanced Transmit Descriptor 358 * +--------------------------------------------------------------+ 359 * 0 | Buffer Address [63:0] | 360 * +--------------------------------------------------------------+ 361 * 8 | PAYLEN | PORTS | IDX | STA | DCMD |DTYP | RSV | DTALEN | 362 * +--------------------------------------------------------------+ 363 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0 364 */ 365 366 for (n = 0; n < adapter->num_tx_queues; n++) { 367 tx_ring = adapter->tx_ring[n]; 368 pr_info("------------------------------------\n"); 369 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 370 pr_info("------------------------------------\n"); 371 pr_info("T [desc] [address 63:0 ] " 372 "[PlPOIdStDDt Ln] [bi->dma ] " 373 "leng ntw timestamp bi->skb\n"); 374 375 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 376 tx_desc = IXGBE_TX_DESC(tx_ring, i); 377 tx_buffer = &tx_ring->tx_buffer_info[i]; 378 u0 = (struct my_u0 *)tx_desc; 379 pr_info("T [0x%03X] %016llX %016llX %016llX" 380 " %04X %p %016llX %p", i, 381 le64_to_cpu(u0->a), 382 le64_to_cpu(u0->b), 383 (u64)dma_unmap_addr(tx_buffer, dma), 384 dma_unmap_len(tx_buffer, len), 385 tx_buffer->next_to_watch, 386 (u64)tx_buffer->time_stamp, 387 tx_buffer->skb); 388 if (i == tx_ring->next_to_use && 389 i == tx_ring->next_to_clean) 390 pr_cont(" NTC/U\n"); 391 else if (i == tx_ring->next_to_use) 392 pr_cont(" NTU\n"); 393 else if (i == tx_ring->next_to_clean) 394 pr_cont(" NTC\n"); 395 else 396 pr_cont("\n"); 397 398 if (netif_msg_pktdata(adapter) && 399 dma_unmap_len(tx_buffer, len) != 0) 400 print_hex_dump(KERN_INFO, "", 401 DUMP_PREFIX_ADDRESS, 16, 1, 402 phys_to_virt(dma_unmap_addr(tx_buffer, 403 dma)), 404 dma_unmap_len(tx_buffer, len), 405 true); 406 } 407 } 408 409 /* Print RX Rings Summary */ 410 rx_ring_summary: 411 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 412 pr_info("Queue [NTU] [NTC]\n"); 413 for (n = 0; n < adapter->num_rx_queues; n++) { 414 rx_ring = adapter->rx_ring[n]; 415 pr_info("%5d %5X %5X\n", 416 n, rx_ring->next_to_use, rx_ring->next_to_clean); 417 } 418 419 /* Print RX Rings */ 420 if (!netif_msg_rx_status(adapter)) 421 goto exit; 422 423 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 424 425 /* Advanced Receive Descriptor (Read) Format 426 * 63 1 0 427 * +-----------------------------------------------------+ 428 * 0 | Packet Buffer Address [63:1] |A0/NSE| 429 * +----------------------------------------------+------+ 430 * 8 | Header Buffer Address [63:1] | DD | 431 * +-----------------------------------------------------+ 432 * 433 * 434 * Advanced Receive Descriptor (Write-Back) Format 435 * 436 * 63 48 47 32 31 30 21 20 16 15 4 3 0 437 * +------------------------------------------------------+ 438 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | 439 * | Checksum Ident | | | | Type | Type | 440 * +------------------------------------------------------+ 441 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 442 * +------------------------------------------------------+ 443 * 63 48 47 32 31 20 19 0 444 */ 445 for (n = 0; n < adapter->num_rx_queues; n++) { 446 rx_ring = adapter->rx_ring[n]; 447 pr_info("------------------------------------\n"); 448 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 449 pr_info("------------------------------------\n"); 450 pr_info("R [desc] [ PktBuf A0] " 451 "[ HeadBuf DD] [bi->dma ] [bi->skb] " 452 "<-- Adv Rx Read format\n"); 453 pr_info("RWB[desc] [PcsmIpSHl PtRs] " 454 "[vl er S cks ln] ---------------- [bi->skb] " 455 "<-- Adv Rx Write-Back format\n"); 456 457 for (i = 0; i < rx_ring->count; i++) { 458 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 459 rx_desc = IXGBE_RX_DESC(rx_ring, i); 460 u0 = (struct my_u0 *)rx_desc; 461 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 462 if (staterr & IXGBE_RXD_STAT_DD) { 463 /* Descriptor Done */ 464 pr_info("RWB[0x%03X] %016llX " 465 "%016llX ---------------- %p", i, 466 le64_to_cpu(u0->a), 467 le64_to_cpu(u0->b), 468 rx_buffer_info->skb); 469 } else { 470 pr_info("R [0x%03X] %016llX " 471 "%016llX %016llX %p", i, 472 le64_to_cpu(u0->a), 473 le64_to_cpu(u0->b), 474 (u64)rx_buffer_info->dma, 475 rx_buffer_info->skb); 476 477 if (netif_msg_pktdata(adapter)) { 478 print_hex_dump(KERN_INFO, "", 479 DUMP_PREFIX_ADDRESS, 16, 1, 480 phys_to_virt(rx_buffer_info->dma), 481 ixgbe_rx_bufsz(rx_ring), true); 482 } 483 } 484 485 if (i == rx_ring->next_to_use) 486 pr_cont(" NTU\n"); 487 else if (i == rx_ring->next_to_clean) 488 pr_cont(" NTC\n"); 489 else 490 pr_cont("\n"); 491 492 } 493 } 494 495 exit: 496 return; 497 } 498 499 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter) 500 { 501 u32 ctrl_ext; 502 503 /* Let firmware take over control of h/w */ 504 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 505 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 506 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD); 507 } 508 509 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter) 510 { 511 u32 ctrl_ext; 512 513 /* Let firmware know the driver has taken over */ 514 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 515 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 516 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD); 517 } 518 519 /* 520 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors 521 * @adapter: pointer to adapter struct 522 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 523 * @queue: queue to map the corresponding interrupt to 524 * @msix_vector: the vector to map to the corresponding queue 525 * 526 */ 527 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, 528 u8 queue, u8 msix_vector) 529 { 530 u32 ivar, index; 531 struct ixgbe_hw *hw = &adapter->hw; 532 switch (hw->mac.type) { 533 case ixgbe_mac_82598EB: 534 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 535 if (direction == -1) 536 direction = 0; 537 index = (((direction * 64) + queue) >> 2) & 0x1F; 538 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 539 ivar &= ~(0xFF << (8 * (queue & 0x3))); 540 ivar |= (msix_vector << (8 * (queue & 0x3))); 541 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 542 break; 543 case ixgbe_mac_82599EB: 544 case ixgbe_mac_X540: 545 if (direction == -1) { 546 /* other causes */ 547 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 548 index = ((queue & 1) * 8); 549 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC); 550 ivar &= ~(0xFF << index); 551 ivar |= (msix_vector << index); 552 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar); 553 break; 554 } else { 555 /* tx or rx causes */ 556 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 557 index = ((16 * (queue & 1)) + (8 * direction)); 558 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 559 ivar &= ~(0xFF << index); 560 ivar |= (msix_vector << index); 561 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar); 562 break; 563 } 564 default: 565 break; 566 } 567 } 568 569 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, 570 u64 qmask) 571 { 572 u32 mask; 573 574 switch (adapter->hw.mac.type) { 575 case ixgbe_mac_82598EB: 576 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 577 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 578 break; 579 case ixgbe_mac_82599EB: 580 case ixgbe_mac_X540: 581 mask = (qmask & 0xFFFFFFFF); 582 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 583 mask = (qmask >> 32); 584 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 585 break; 586 default: 587 break; 588 } 589 } 590 591 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring, 592 struct ixgbe_tx_buffer *tx_buffer) 593 { 594 if (tx_buffer->skb) { 595 dev_kfree_skb_any(tx_buffer->skb); 596 if (dma_unmap_len(tx_buffer, len)) 597 dma_unmap_single(ring->dev, 598 dma_unmap_addr(tx_buffer, dma), 599 dma_unmap_len(tx_buffer, len), 600 DMA_TO_DEVICE); 601 } else if (dma_unmap_len(tx_buffer, len)) { 602 dma_unmap_page(ring->dev, 603 dma_unmap_addr(tx_buffer, dma), 604 dma_unmap_len(tx_buffer, len), 605 DMA_TO_DEVICE); 606 } 607 tx_buffer->next_to_watch = NULL; 608 tx_buffer->skb = NULL; 609 dma_unmap_len_set(tx_buffer, len, 0); 610 /* tx_buffer must be completely set up in the transmit path */ 611 } 612 613 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) 614 { 615 struct ixgbe_hw *hw = &adapter->hw; 616 struct ixgbe_hw_stats *hwstats = &adapter->stats; 617 int i; 618 u32 data; 619 620 if ((hw->fc.current_mode != ixgbe_fc_full) && 621 (hw->fc.current_mode != ixgbe_fc_rx_pause)) 622 return; 623 624 switch (hw->mac.type) { 625 case ixgbe_mac_82598EB: 626 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 627 break; 628 default: 629 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 630 } 631 hwstats->lxoffrxc += data; 632 633 /* refill credits (no tx hang) if we received xoff */ 634 if (!data) 635 return; 636 637 for (i = 0; i < adapter->num_tx_queues; i++) 638 clear_bit(__IXGBE_HANG_CHECK_ARMED, 639 &adapter->tx_ring[i]->state); 640 } 641 642 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) 643 { 644 struct ixgbe_hw *hw = &adapter->hw; 645 struct ixgbe_hw_stats *hwstats = &adapter->stats; 646 u32 xoff[8] = {0}; 647 int i; 648 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 649 650 if (adapter->ixgbe_ieee_pfc) 651 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 652 653 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) { 654 ixgbe_update_xoff_rx_lfc(adapter); 655 return; 656 } 657 658 /* update stats for each tc, only valid with PFC enabled */ 659 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) { 660 switch (hw->mac.type) { 661 case ixgbe_mac_82598EB: 662 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 663 break; 664 default: 665 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 666 } 667 hwstats->pxoffrxc[i] += xoff[i]; 668 } 669 670 /* disarm tx queues that have received xoff frames */ 671 for (i = 0; i < adapter->num_tx_queues; i++) { 672 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 673 u8 tc = tx_ring->dcb_tc; 674 675 if (xoff[tc]) 676 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 677 } 678 } 679 680 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring) 681 { 682 return ring->stats.packets; 683 } 684 685 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring) 686 { 687 struct ixgbe_adapter *adapter = netdev_priv(ring->netdev); 688 struct ixgbe_hw *hw = &adapter->hw; 689 690 u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx)); 691 u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx)); 692 693 if (head != tail) 694 return (head < tail) ? 695 tail - head : (tail + ring->count - head); 696 697 return 0; 698 } 699 700 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring) 701 { 702 u32 tx_done = ixgbe_get_tx_completed(tx_ring); 703 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 704 u32 tx_pending = ixgbe_get_tx_pending(tx_ring); 705 bool ret = false; 706 707 clear_check_for_tx_hang(tx_ring); 708 709 /* 710 * Check for a hung queue, but be thorough. This verifies 711 * that a transmit has been completed since the previous 712 * check AND there is at least one packet pending. The 713 * ARMED bit is set to indicate a potential hang. The 714 * bit is cleared if a pause frame is received to remove 715 * false hang detection due to PFC or 802.3x frames. By 716 * requiring this to fail twice we avoid races with 717 * pfc clearing the ARMED bit and conditions where we 718 * run the check_tx_hang logic with a transmit completion 719 * pending but without time to complete it yet. 720 */ 721 if ((tx_done_old == tx_done) && tx_pending) { 722 /* make sure it is true for two checks in a row */ 723 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED, 724 &tx_ring->state); 725 } else { 726 /* update completed stats and continue */ 727 tx_ring->tx_stats.tx_done_old = tx_done; 728 /* reset the countdown */ 729 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 730 } 731 732 return ret; 733 } 734 735 /** 736 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout 737 * @adapter: driver private struct 738 **/ 739 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) 740 { 741 742 /* Do the reset outside of interrupt context */ 743 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 744 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; 745 ixgbe_service_event_schedule(adapter); 746 } 747 } 748 749 /** 750 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes 751 * @q_vector: structure containing interrupt and ring information 752 * @tx_ring: tx ring to clean 753 **/ 754 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, 755 struct ixgbe_ring *tx_ring) 756 { 757 struct ixgbe_adapter *adapter = q_vector->adapter; 758 struct ixgbe_tx_buffer *tx_buffer; 759 union ixgbe_adv_tx_desc *tx_desc; 760 unsigned int total_bytes = 0, total_packets = 0; 761 unsigned int budget = q_vector->tx.work_limit; 762 unsigned int i = tx_ring->next_to_clean; 763 764 if (test_bit(__IXGBE_DOWN, &adapter->state)) 765 return true; 766 767 tx_buffer = &tx_ring->tx_buffer_info[i]; 768 tx_desc = IXGBE_TX_DESC(tx_ring, i); 769 i -= tx_ring->count; 770 771 do { 772 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 773 774 /* if next_to_watch is not set then there is no work pending */ 775 if (!eop_desc) 776 break; 777 778 /* prevent any other reads prior to eop_desc */ 779 rmb(); 780 781 /* if DD is not set pending work has not been completed */ 782 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 783 break; 784 785 /* clear next_to_watch to prevent false hangs */ 786 tx_buffer->next_to_watch = NULL; 787 788 /* update the statistics for this packet */ 789 total_bytes += tx_buffer->bytecount; 790 total_packets += tx_buffer->gso_segs; 791 792 #ifdef CONFIG_IXGBE_PTP 793 if (unlikely(tx_buffer->tx_flags & 794 IXGBE_TX_FLAGS_TSTAMP)) 795 ixgbe_ptp_tx_hwtstamp(q_vector, 796 tx_buffer->skb); 797 798 #endif 799 /* free the skb */ 800 dev_kfree_skb_any(tx_buffer->skb); 801 802 /* unmap skb header data */ 803 dma_unmap_single(tx_ring->dev, 804 dma_unmap_addr(tx_buffer, dma), 805 dma_unmap_len(tx_buffer, len), 806 DMA_TO_DEVICE); 807 808 /* clear tx_buffer data */ 809 tx_buffer->skb = NULL; 810 dma_unmap_len_set(tx_buffer, len, 0); 811 812 /* unmap remaining buffers */ 813 while (tx_desc != eop_desc) { 814 tx_buffer++; 815 tx_desc++; 816 i++; 817 if (unlikely(!i)) { 818 i -= tx_ring->count; 819 tx_buffer = tx_ring->tx_buffer_info; 820 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 821 } 822 823 /* unmap any remaining paged data */ 824 if (dma_unmap_len(tx_buffer, len)) { 825 dma_unmap_page(tx_ring->dev, 826 dma_unmap_addr(tx_buffer, dma), 827 dma_unmap_len(tx_buffer, len), 828 DMA_TO_DEVICE); 829 dma_unmap_len_set(tx_buffer, len, 0); 830 } 831 } 832 833 /* move us one more past the eop_desc for start of next pkt */ 834 tx_buffer++; 835 tx_desc++; 836 i++; 837 if (unlikely(!i)) { 838 i -= tx_ring->count; 839 tx_buffer = tx_ring->tx_buffer_info; 840 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 841 } 842 843 /* issue prefetch for next Tx descriptor */ 844 prefetch(tx_desc); 845 846 /* update budget accounting */ 847 budget--; 848 } while (likely(budget)); 849 850 i += tx_ring->count; 851 tx_ring->next_to_clean = i; 852 u64_stats_update_begin(&tx_ring->syncp); 853 tx_ring->stats.bytes += total_bytes; 854 tx_ring->stats.packets += total_packets; 855 u64_stats_update_end(&tx_ring->syncp); 856 q_vector->tx.total_bytes += total_bytes; 857 q_vector->tx.total_packets += total_packets; 858 859 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { 860 /* schedule immediate reset if we believe we hung */ 861 struct ixgbe_hw *hw = &adapter->hw; 862 e_err(drv, "Detected Tx Unit Hang\n" 863 " Tx Queue <%d>\n" 864 " TDH, TDT <%x>, <%x>\n" 865 " next_to_use <%x>\n" 866 " next_to_clean <%x>\n" 867 "tx_buffer_info[next_to_clean]\n" 868 " time_stamp <%lx>\n" 869 " jiffies <%lx>\n", 870 tx_ring->queue_index, 871 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), 872 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), 873 tx_ring->next_to_use, i, 874 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 875 876 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 877 878 e_info(probe, 879 "tx hang %d detected on queue %d, resetting adapter\n", 880 adapter->tx_timeout_count + 1, tx_ring->queue_index); 881 882 /* schedule immediate reset if we believe we hung */ 883 ixgbe_tx_timeout_reset(adapter); 884 885 /* the adapter is about to reset, no point in enabling stuff */ 886 return true; 887 } 888 889 netdev_tx_completed_queue(txring_txq(tx_ring), 890 total_packets, total_bytes); 891 892 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 893 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 894 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 895 /* Make sure that anybody stopping the queue after this 896 * sees the new next_to_clean. 897 */ 898 smp_mb(); 899 if (__netif_subqueue_stopped(tx_ring->netdev, 900 tx_ring->queue_index) 901 && !test_bit(__IXGBE_DOWN, &adapter->state)) { 902 netif_wake_subqueue(tx_ring->netdev, 903 tx_ring->queue_index); 904 ++tx_ring->tx_stats.restart_queue; 905 } 906 } 907 908 return !!budget; 909 } 910 911 #ifdef CONFIG_IXGBE_DCA 912 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, 913 struct ixgbe_ring *tx_ring, 914 int cpu) 915 { 916 struct ixgbe_hw *hw = &adapter->hw; 917 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu); 918 u16 reg_offset; 919 920 switch (hw->mac.type) { 921 case ixgbe_mac_82598EB: 922 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx); 923 break; 924 case ixgbe_mac_82599EB: 925 case ixgbe_mac_X540: 926 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx); 927 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599; 928 break; 929 default: 930 /* for unknown hardware do not write register */ 931 return; 932 } 933 934 /* 935 * We can enable relaxed ordering for reads, but not writes when 936 * DCA is enabled. This is due to a known issue in some chipsets 937 * which will cause the DCA tag to be cleared. 938 */ 939 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN | 940 IXGBE_DCA_TXCTRL_DATA_RRO_EN | 941 IXGBE_DCA_TXCTRL_DESC_DCA_EN; 942 943 IXGBE_WRITE_REG(hw, reg_offset, txctrl); 944 } 945 946 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, 947 struct ixgbe_ring *rx_ring, 948 int cpu) 949 { 950 struct ixgbe_hw *hw = &adapter->hw; 951 u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu); 952 u8 reg_idx = rx_ring->reg_idx; 953 954 955 switch (hw->mac.type) { 956 case ixgbe_mac_82599EB: 957 case ixgbe_mac_X540: 958 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599; 959 break; 960 default: 961 break; 962 } 963 964 /* 965 * We can enable relaxed ordering for reads, but not writes when 966 * DCA is enabled. This is due to a known issue in some chipsets 967 * which will cause the DCA tag to be cleared. 968 */ 969 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN | 970 IXGBE_DCA_RXCTRL_DATA_DCA_EN | 971 IXGBE_DCA_RXCTRL_DESC_DCA_EN; 972 973 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl); 974 } 975 976 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector) 977 { 978 struct ixgbe_adapter *adapter = q_vector->adapter; 979 struct ixgbe_ring *ring; 980 int cpu = get_cpu(); 981 982 if (q_vector->cpu == cpu) 983 goto out_no_update; 984 985 ixgbe_for_each_ring(ring, q_vector->tx) 986 ixgbe_update_tx_dca(adapter, ring, cpu); 987 988 ixgbe_for_each_ring(ring, q_vector->rx) 989 ixgbe_update_rx_dca(adapter, ring, cpu); 990 991 q_vector->cpu = cpu; 992 out_no_update: 993 put_cpu(); 994 } 995 996 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) 997 { 998 int num_q_vectors; 999 int i; 1000 1001 if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) 1002 return; 1003 1004 /* always use CB2 mode, difference is masked in the CB driver */ 1005 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2); 1006 1007 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 1008 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1009 else 1010 num_q_vectors = 1; 1011 1012 for (i = 0; i < num_q_vectors; i++) { 1013 adapter->q_vector[i]->cpu = -1; 1014 ixgbe_update_dca(adapter->q_vector[i]); 1015 } 1016 } 1017 1018 static int __ixgbe_notify_dca(struct device *dev, void *data) 1019 { 1020 struct ixgbe_adapter *adapter = dev_get_drvdata(dev); 1021 unsigned long event = *(unsigned long *)data; 1022 1023 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE)) 1024 return 0; 1025 1026 switch (event) { 1027 case DCA_PROVIDER_ADD: 1028 /* if we're already enabled, don't do it again */ 1029 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1030 break; 1031 if (dca_add_requester(dev) == 0) { 1032 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 1033 ixgbe_setup_dca(adapter); 1034 break; 1035 } 1036 /* Fall Through since DCA is disabled. */ 1037 case DCA_PROVIDER_REMOVE: 1038 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 1039 dca_remove_requester(dev); 1040 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 1041 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); 1042 } 1043 break; 1044 } 1045 1046 return 0; 1047 } 1048 1049 #endif /* CONFIG_IXGBE_DCA */ 1050 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, 1051 union ixgbe_adv_rx_desc *rx_desc, 1052 struct sk_buff *skb) 1053 { 1054 if (ring->netdev->features & NETIF_F_RXHASH) 1055 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); 1056 } 1057 1058 #ifdef IXGBE_FCOE 1059 /** 1060 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type 1061 * @adapter: address of board private structure 1062 * @rx_desc: advanced rx descriptor 1063 * 1064 * Returns : true if it is FCoE pkt 1065 */ 1066 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter, 1067 union ixgbe_adv_rx_desc *rx_desc) 1068 { 1069 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1070 1071 return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && 1072 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == 1073 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << 1074 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); 1075 } 1076 1077 #endif /* IXGBE_FCOE */ 1078 /** 1079 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum 1080 * @ring: structure containing ring specific data 1081 * @rx_desc: current Rx descriptor being processed 1082 * @skb: skb currently being received and modified 1083 **/ 1084 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, 1085 union ixgbe_adv_rx_desc *rx_desc, 1086 struct sk_buff *skb) 1087 { 1088 skb_checksum_none_assert(skb); 1089 1090 /* Rx csum disabled */ 1091 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1092 return; 1093 1094 /* if IP and error */ 1095 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 1096 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 1097 ring->rx_stats.csum_err++; 1098 return; 1099 } 1100 1101 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 1102 return; 1103 1104 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 1105 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1106 1107 /* 1108 * 82599 errata, UDP frames with a 0 checksum can be marked as 1109 * checksum errors. 1110 */ 1111 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && 1112 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) 1113 return; 1114 1115 ring->rx_stats.csum_err++; 1116 return; 1117 } 1118 1119 /* It must be a TCP or UDP packet with a valid checksum */ 1120 skb->ip_summed = CHECKSUM_UNNECESSARY; 1121 } 1122 1123 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val) 1124 { 1125 rx_ring->next_to_use = val; 1126 1127 /* update next to alloc since we have filled the ring */ 1128 rx_ring->next_to_alloc = val; 1129 /* 1130 * Force memory writes to complete before letting h/w 1131 * know there are new descriptors to fetch. (Only 1132 * applicable for weak-ordered memory model archs, 1133 * such as IA-64). 1134 */ 1135 wmb(); 1136 writel(val, rx_ring->tail); 1137 } 1138 1139 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, 1140 struct ixgbe_rx_buffer *bi) 1141 { 1142 struct page *page = bi->page; 1143 dma_addr_t dma = bi->dma; 1144 1145 /* since we are recycling buffers we should seldom need to alloc */ 1146 if (likely(dma)) 1147 return true; 1148 1149 /* alloc new page for storage */ 1150 if (likely(!page)) { 1151 page = alloc_pages(GFP_ATOMIC | __GFP_COLD, 1152 ixgbe_rx_pg_order(rx_ring)); 1153 if (unlikely(!page)) { 1154 rx_ring->rx_stats.alloc_rx_page_failed++; 1155 return false; 1156 } 1157 bi->page = page; 1158 } 1159 1160 /* map page for use */ 1161 dma = dma_map_page(rx_ring->dev, page, 0, 1162 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1163 1164 /* 1165 * if mapping failed free memory back to system since 1166 * there isn't much point in holding memory we can't use 1167 */ 1168 if (dma_mapping_error(rx_ring->dev, dma)) { 1169 __free_pages(page, ixgbe_rx_pg_order(rx_ring)); 1170 bi->page = NULL; 1171 1172 rx_ring->rx_stats.alloc_rx_page_failed++; 1173 return false; 1174 } 1175 1176 bi->dma = dma; 1177 bi->page_offset ^= ixgbe_rx_bufsz(rx_ring); 1178 1179 return true; 1180 } 1181 1182 /** 1183 * ixgbe_alloc_rx_buffers - Replace used receive buffers 1184 * @rx_ring: ring to place buffers on 1185 * @cleaned_count: number of buffers to replace 1186 **/ 1187 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) 1188 { 1189 union ixgbe_adv_rx_desc *rx_desc; 1190 struct ixgbe_rx_buffer *bi; 1191 u16 i = rx_ring->next_to_use; 1192 1193 /* nothing to do */ 1194 if (!cleaned_count) 1195 return; 1196 1197 rx_desc = IXGBE_RX_DESC(rx_ring, i); 1198 bi = &rx_ring->rx_buffer_info[i]; 1199 i -= rx_ring->count; 1200 1201 do { 1202 if (!ixgbe_alloc_mapped_page(rx_ring, bi)) 1203 break; 1204 1205 /* 1206 * Refresh the desc even if buffer_addrs didn't change 1207 * because each write-back erases this info. 1208 */ 1209 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1210 1211 rx_desc++; 1212 bi++; 1213 i++; 1214 if (unlikely(!i)) { 1215 rx_desc = IXGBE_RX_DESC(rx_ring, 0); 1216 bi = rx_ring->rx_buffer_info; 1217 i -= rx_ring->count; 1218 } 1219 1220 /* clear the hdr_addr for the next_to_use descriptor */ 1221 rx_desc->read.hdr_addr = 0; 1222 1223 cleaned_count--; 1224 } while (cleaned_count); 1225 1226 i += rx_ring->count; 1227 1228 if (rx_ring->next_to_use != i) 1229 ixgbe_release_rx_desc(rx_ring, i); 1230 } 1231 1232 /** 1233 * ixgbe_get_headlen - determine size of header for RSC/LRO/GRO/FCOE 1234 * @data: pointer to the start of the headers 1235 * @max_len: total length of section to find headers in 1236 * 1237 * This function is meant to determine the length of headers that will 1238 * be recognized by hardware for LRO, GRO, and RSC offloads. The main 1239 * motivation of doing this is to only perform one pull for IPv4 TCP 1240 * packets so that we can do basic things like calculating the gso_size 1241 * based on the average data per packet. 1242 **/ 1243 static unsigned int ixgbe_get_headlen(unsigned char *data, 1244 unsigned int max_len) 1245 { 1246 union { 1247 unsigned char *network; 1248 /* l2 headers */ 1249 struct ethhdr *eth; 1250 struct vlan_hdr *vlan; 1251 /* l3 headers */ 1252 struct iphdr *ipv4; 1253 } hdr; 1254 __be16 protocol; 1255 u8 nexthdr = 0; /* default to not TCP */ 1256 u8 hlen; 1257 1258 /* this should never happen, but better safe than sorry */ 1259 if (max_len < ETH_HLEN) 1260 return max_len; 1261 1262 /* initialize network frame pointer */ 1263 hdr.network = data; 1264 1265 /* set first protocol and move network header forward */ 1266 protocol = hdr.eth->h_proto; 1267 hdr.network += ETH_HLEN; 1268 1269 /* handle any vlan tag if present */ 1270 if (protocol == __constant_htons(ETH_P_8021Q)) { 1271 if ((hdr.network - data) > (max_len - VLAN_HLEN)) 1272 return max_len; 1273 1274 protocol = hdr.vlan->h_vlan_encapsulated_proto; 1275 hdr.network += VLAN_HLEN; 1276 } 1277 1278 /* handle L3 protocols */ 1279 if (protocol == __constant_htons(ETH_P_IP)) { 1280 if ((hdr.network - data) > (max_len - sizeof(struct iphdr))) 1281 return max_len; 1282 1283 /* access ihl as a u8 to avoid unaligned access on ia64 */ 1284 hlen = (hdr.network[0] & 0x0F) << 2; 1285 1286 /* verify hlen meets minimum size requirements */ 1287 if (hlen < sizeof(struct iphdr)) 1288 return hdr.network - data; 1289 1290 /* record next protocol */ 1291 nexthdr = hdr.ipv4->protocol; 1292 hdr.network += hlen; 1293 #ifdef IXGBE_FCOE 1294 } else if (protocol == __constant_htons(ETH_P_FCOE)) { 1295 if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN)) 1296 return max_len; 1297 hdr.network += FCOE_HEADER_LEN; 1298 #endif 1299 } else { 1300 return hdr.network - data; 1301 } 1302 1303 /* finally sort out TCP */ 1304 if (nexthdr == IPPROTO_TCP) { 1305 if ((hdr.network - data) > (max_len - sizeof(struct tcphdr))) 1306 return max_len; 1307 1308 /* access doff as a u8 to avoid unaligned access on ia64 */ 1309 hlen = (hdr.network[12] & 0xF0) >> 2; 1310 1311 /* verify hlen meets minimum size requirements */ 1312 if (hlen < sizeof(struct tcphdr)) 1313 return hdr.network - data; 1314 1315 hdr.network += hlen; 1316 } 1317 1318 /* 1319 * If everything has gone correctly hdr.network should be the 1320 * data section of the packet and will be the end of the header. 1321 * If not then it probably represents the end of the last recognized 1322 * header. 1323 */ 1324 if ((hdr.network - data) < max_len) 1325 return hdr.network - data; 1326 else 1327 return max_len; 1328 } 1329 1330 static void ixgbe_get_rsc_cnt(struct ixgbe_ring *rx_ring, 1331 union ixgbe_adv_rx_desc *rx_desc, 1332 struct sk_buff *skb) 1333 { 1334 __le32 rsc_enabled; 1335 u32 rsc_cnt; 1336 1337 if (!ring_is_rsc_enabled(rx_ring)) 1338 return; 1339 1340 rsc_enabled = rx_desc->wb.lower.lo_dword.data & 1341 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); 1342 1343 /* If this is an RSC frame rsc_cnt should be non-zero */ 1344 if (!rsc_enabled) 1345 return; 1346 1347 rsc_cnt = le32_to_cpu(rsc_enabled); 1348 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 1349 1350 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; 1351 } 1352 1353 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring, 1354 struct sk_buff *skb) 1355 { 1356 u16 hdr_len = skb_headlen(skb); 1357 1358 /* set gso_size to avoid messing up TCP MSS */ 1359 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len), 1360 IXGBE_CB(skb)->append_cnt); 1361 } 1362 1363 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, 1364 struct sk_buff *skb) 1365 { 1366 /* if append_cnt is 0 then frame is not RSC */ 1367 if (!IXGBE_CB(skb)->append_cnt) 1368 return; 1369 1370 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt; 1371 rx_ring->rx_stats.rsc_flush++; 1372 1373 ixgbe_set_rsc_gso_size(rx_ring, skb); 1374 1375 /* gso_size is computed using append_cnt so always clear it last */ 1376 IXGBE_CB(skb)->append_cnt = 0; 1377 } 1378 1379 /** 1380 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor 1381 * @rx_ring: rx descriptor ring packet is being transacted on 1382 * @rx_desc: pointer to the EOP Rx descriptor 1383 * @skb: pointer to current skb being populated 1384 * 1385 * This function checks the ring, descriptor, and packet information in 1386 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1387 * other fields within the skb. 1388 **/ 1389 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, 1390 union ixgbe_adv_rx_desc *rx_desc, 1391 struct sk_buff *skb) 1392 { 1393 ixgbe_update_rsc_stats(rx_ring, skb); 1394 1395 ixgbe_rx_hash(rx_ring, rx_desc, skb); 1396 1397 ixgbe_rx_checksum(rx_ring, rx_desc, skb); 1398 1399 #ifdef CONFIG_IXGBE_PTP 1400 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_TS)) 1401 ixgbe_ptp_rx_hwtstamp(rx_ring->q_vector, skb); 1402 #endif 1403 1404 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 1405 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 1406 __vlan_hwaccel_put_tag(skb, vid); 1407 } 1408 1409 skb_record_rx_queue(skb, rx_ring->queue_index); 1410 1411 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1412 } 1413 1414 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, 1415 struct sk_buff *skb) 1416 { 1417 struct ixgbe_adapter *adapter = q_vector->adapter; 1418 1419 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 1420 napi_gro_receive(&q_vector->napi, skb); 1421 else 1422 netif_rx(skb); 1423 } 1424 1425 /** 1426 * ixgbe_is_non_eop - process handling of non-EOP buffers 1427 * @rx_ring: Rx ring being processed 1428 * @rx_desc: Rx descriptor for current buffer 1429 * @skb: Current socket buffer containing buffer in progress 1430 * 1431 * This function updates next to clean. If the buffer is an EOP buffer 1432 * this function exits returning false, otherwise it will place the 1433 * sk_buff in the next buffer to be chained and return true indicating 1434 * that this is in fact a non-EOP buffer. 1435 **/ 1436 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, 1437 union ixgbe_adv_rx_desc *rx_desc, 1438 struct sk_buff *skb) 1439 { 1440 u32 ntc = rx_ring->next_to_clean + 1; 1441 1442 /* fetch, update, and store next to clean */ 1443 ntc = (ntc < rx_ring->count) ? ntc : 0; 1444 rx_ring->next_to_clean = ntc; 1445 1446 prefetch(IXGBE_RX_DESC(rx_ring, ntc)); 1447 1448 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 1449 return false; 1450 1451 /* append_cnt indicates packet is RSC, if so fetch nextp */ 1452 if (IXGBE_CB(skb)->append_cnt) { 1453 ntc = le32_to_cpu(rx_desc->wb.upper.status_error); 1454 ntc &= IXGBE_RXDADV_NEXTP_MASK; 1455 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT; 1456 } 1457 1458 /* place skb in next buffer to be received */ 1459 rx_ring->rx_buffer_info[ntc].skb = skb; 1460 rx_ring->rx_stats.non_eop_descs++; 1461 1462 return true; 1463 } 1464 1465 /** 1466 * ixgbe_cleanup_headers - Correct corrupted or empty headers 1467 * @rx_ring: rx descriptor ring packet is being transacted on 1468 * @rx_desc: pointer to the EOP Rx descriptor 1469 * @skb: pointer to current skb being fixed 1470 * 1471 * Check for corrupted packet headers caused by senders on the local L2 1472 * embedded NIC switch not setting up their Tx Descriptors right. These 1473 * should be very rare. 1474 * 1475 * Also address the case where we are pulling data in on pages only 1476 * and as such no data is present in the skb header. 1477 * 1478 * In addition if skb is not at least 60 bytes we need to pad it so that 1479 * it is large enough to qualify as a valid Ethernet frame. 1480 * 1481 * Returns true if an error was encountered and skb was freed. 1482 **/ 1483 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, 1484 union ixgbe_adv_rx_desc *rx_desc, 1485 struct sk_buff *skb) 1486 { 1487 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1488 struct net_device *netdev = rx_ring->netdev; 1489 unsigned char *va; 1490 unsigned int pull_len; 1491 1492 /* if the page was released unmap it, else just sync our portion */ 1493 if (unlikely(IXGBE_CB(skb)->page_released)) { 1494 dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma, 1495 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1496 IXGBE_CB(skb)->page_released = false; 1497 } else { 1498 dma_sync_single_range_for_cpu(rx_ring->dev, 1499 IXGBE_CB(skb)->dma, 1500 frag->page_offset, 1501 ixgbe_rx_bufsz(rx_ring), 1502 DMA_FROM_DEVICE); 1503 } 1504 IXGBE_CB(skb)->dma = 0; 1505 1506 /* verify that the packet does not have any known errors */ 1507 if (unlikely(ixgbe_test_staterr(rx_desc, 1508 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && 1509 !(netdev->features & NETIF_F_RXALL))) { 1510 dev_kfree_skb_any(skb); 1511 return true; 1512 } 1513 1514 /* 1515 * it is valid to use page_address instead of kmap since we are 1516 * working with pages allocated out of the lomem pool per 1517 * alloc_page(GFP_ATOMIC) 1518 */ 1519 va = skb_frag_address(frag); 1520 1521 /* 1522 * we need the header to contain the greater of either ETH_HLEN or 1523 * 60 bytes if the skb->len is less than 60 for skb_pad. 1524 */ 1525 pull_len = skb_frag_size(frag); 1526 if (pull_len > 256) 1527 pull_len = ixgbe_get_headlen(va, pull_len); 1528 1529 /* align pull length to size of long to optimize memcpy performance */ 1530 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 1531 1532 /* update all of the pointers */ 1533 skb_frag_size_sub(frag, pull_len); 1534 frag->page_offset += pull_len; 1535 skb->data_len -= pull_len; 1536 skb->tail += pull_len; 1537 1538 /* 1539 * if we sucked the frag empty then we should free it, 1540 * if there are other frags here something is screwed up in hardware 1541 */ 1542 if (skb_frag_size(frag) == 0) { 1543 BUG_ON(skb_shinfo(skb)->nr_frags != 1); 1544 skb_shinfo(skb)->nr_frags = 0; 1545 __skb_frag_unref(frag); 1546 skb->truesize -= ixgbe_rx_bufsz(rx_ring); 1547 } 1548 1549 /* if skb_pad returns an error the skb was freed */ 1550 if (unlikely(skb->len < 60)) { 1551 int pad_len = 60 - skb->len; 1552 1553 if (skb_pad(skb, pad_len)) 1554 return true; 1555 __skb_put(skb, pad_len); 1556 } 1557 1558 return false; 1559 } 1560 1561 /** 1562 * ixgbe_can_reuse_page - determine if we can reuse a page 1563 * @rx_buffer: pointer to rx_buffer containing the page we want to reuse 1564 * 1565 * Returns true if page can be reused in another Rx buffer 1566 **/ 1567 static inline bool ixgbe_can_reuse_page(struct ixgbe_rx_buffer *rx_buffer) 1568 { 1569 struct page *page = rx_buffer->page; 1570 1571 /* if we are only owner of page and it is local we can reuse it */ 1572 return likely(page_count(page) == 1) && 1573 likely(page_to_nid(page) == numa_node_id()); 1574 } 1575 1576 /** 1577 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring 1578 * @rx_ring: rx descriptor ring to store buffers on 1579 * @old_buff: donor buffer to have page reused 1580 * 1581 * Syncronizes page for reuse by the adapter 1582 **/ 1583 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, 1584 struct ixgbe_rx_buffer *old_buff) 1585 { 1586 struct ixgbe_rx_buffer *new_buff; 1587 u16 nta = rx_ring->next_to_alloc; 1588 u16 bufsz = ixgbe_rx_bufsz(rx_ring); 1589 1590 new_buff = &rx_ring->rx_buffer_info[nta]; 1591 1592 /* update, and store next to alloc */ 1593 nta++; 1594 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1595 1596 /* transfer page from old buffer to new buffer */ 1597 new_buff->page = old_buff->page; 1598 new_buff->dma = old_buff->dma; 1599 1600 /* flip page offset to other buffer and store to new_buff */ 1601 new_buff->page_offset = old_buff->page_offset ^ bufsz; 1602 1603 /* sync the buffer for use by the device */ 1604 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 1605 new_buff->page_offset, bufsz, 1606 DMA_FROM_DEVICE); 1607 1608 /* bump ref count on page before it is given to the stack */ 1609 get_page(new_buff->page); 1610 } 1611 1612 /** 1613 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff 1614 * @rx_ring: rx descriptor ring to transact packets on 1615 * @rx_buffer: buffer containing page to add 1616 * @rx_desc: descriptor containing length of buffer written by hardware 1617 * @skb: sk_buff to place the data into 1618 * 1619 * This function is based on skb_add_rx_frag. I would have used that 1620 * function however it doesn't handle the truesize case correctly since we 1621 * are allocating more memory than might be used for a single receive. 1622 **/ 1623 static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, 1624 struct ixgbe_rx_buffer *rx_buffer, 1625 struct sk_buff *skb, int size) 1626 { 1627 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 1628 rx_buffer->page, rx_buffer->page_offset, 1629 size); 1630 skb->len += size; 1631 skb->data_len += size; 1632 skb->truesize += ixgbe_rx_bufsz(rx_ring); 1633 } 1634 1635 /** 1636 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 1637 * @q_vector: structure containing interrupt and ring information 1638 * @rx_ring: rx descriptor ring to transact packets on 1639 * @budget: Total limit on number of packets to process 1640 * 1641 * This function provides a "bounce buffer" approach to Rx interrupt 1642 * processing. The advantage to this is that on systems that have 1643 * expensive overhead for IOMMU access this provides a means of avoiding 1644 * it by maintaining the mapping of the page to the syste. 1645 * 1646 * Returns true if all work is completed without reaching budget 1647 **/ 1648 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, 1649 struct ixgbe_ring *rx_ring, 1650 int budget) 1651 { 1652 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1653 #ifdef IXGBE_FCOE 1654 struct ixgbe_adapter *adapter = q_vector->adapter; 1655 int ddp_bytes = 0; 1656 #endif /* IXGBE_FCOE */ 1657 u16 cleaned_count = ixgbe_desc_unused(rx_ring); 1658 1659 do { 1660 struct ixgbe_rx_buffer *rx_buffer; 1661 union ixgbe_adv_rx_desc *rx_desc; 1662 struct sk_buff *skb; 1663 struct page *page; 1664 u16 ntc; 1665 1666 /* return some buffers to hardware, one at a time is too slow */ 1667 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { 1668 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 1669 cleaned_count = 0; 1670 } 1671 1672 ntc = rx_ring->next_to_clean; 1673 rx_desc = IXGBE_RX_DESC(rx_ring, ntc); 1674 rx_buffer = &rx_ring->rx_buffer_info[ntc]; 1675 1676 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 1677 break; 1678 1679 /* 1680 * This memory barrier is needed to keep us from reading 1681 * any other fields out of the rx_desc until we know the 1682 * RXD_STAT_DD bit is set 1683 */ 1684 rmb(); 1685 1686 page = rx_buffer->page; 1687 prefetchw(page); 1688 1689 skb = rx_buffer->skb; 1690 1691 if (likely(!skb)) { 1692 void *page_addr = page_address(page) + 1693 rx_buffer->page_offset; 1694 1695 /* prefetch first cache line of first page */ 1696 prefetch(page_addr); 1697 #if L1_CACHE_BYTES < 128 1698 prefetch(page_addr + L1_CACHE_BYTES); 1699 #endif 1700 1701 /* allocate a skb to store the frags */ 1702 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 1703 IXGBE_RX_HDR_SIZE); 1704 if (unlikely(!skb)) { 1705 rx_ring->rx_stats.alloc_rx_buff_failed++; 1706 break; 1707 } 1708 1709 /* 1710 * we will be copying header into skb->data in 1711 * pskb_may_pull so it is in our interest to prefetch 1712 * it now to avoid a possible cache miss 1713 */ 1714 prefetchw(skb->data); 1715 1716 /* 1717 * Delay unmapping of the first packet. It carries the 1718 * header information, HW may still access the header 1719 * after the writeback. Only unmap it when EOP is 1720 * reached 1721 */ 1722 IXGBE_CB(skb)->dma = rx_buffer->dma; 1723 } else { 1724 /* we are reusing so sync this buffer for CPU use */ 1725 dma_sync_single_range_for_cpu(rx_ring->dev, 1726 rx_buffer->dma, 1727 rx_buffer->page_offset, 1728 ixgbe_rx_bufsz(rx_ring), 1729 DMA_FROM_DEVICE); 1730 } 1731 1732 /* pull page into skb */ 1733 ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, 1734 le16_to_cpu(rx_desc->wb.upper.length)); 1735 1736 if (ixgbe_can_reuse_page(rx_buffer)) { 1737 /* hand second half of page back to the ring */ 1738 ixgbe_reuse_rx_page(rx_ring, rx_buffer); 1739 } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) { 1740 /* the page has been released from the ring */ 1741 IXGBE_CB(skb)->page_released = true; 1742 } else { 1743 /* we are not reusing the buffer so unmap it */ 1744 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 1745 ixgbe_rx_pg_size(rx_ring), 1746 DMA_FROM_DEVICE); 1747 } 1748 1749 /* clear contents of buffer_info */ 1750 rx_buffer->skb = NULL; 1751 rx_buffer->dma = 0; 1752 rx_buffer->page = NULL; 1753 1754 ixgbe_get_rsc_cnt(rx_ring, rx_desc, skb); 1755 1756 cleaned_count++; 1757 1758 /* place incomplete frames back on ring for completion */ 1759 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) 1760 continue; 1761 1762 /* verify the packet layout is correct */ 1763 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) 1764 continue; 1765 1766 /* probably a little skewed due to removing CRC */ 1767 total_rx_bytes += skb->len; 1768 total_rx_packets++; 1769 1770 /* populate checksum, timestamp, VLAN, and protocol */ 1771 ixgbe_process_skb_fields(rx_ring, rx_desc, skb); 1772 1773 #ifdef IXGBE_FCOE 1774 /* if ddp, not passing to ULD unless for FCP_RSP or error */ 1775 if (ixgbe_rx_is_fcoe(adapter, rx_desc)) { 1776 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb); 1777 if (!ddp_bytes) { 1778 dev_kfree_skb_any(skb); 1779 continue; 1780 } 1781 } 1782 1783 #endif /* IXGBE_FCOE */ 1784 ixgbe_rx_skb(q_vector, skb); 1785 1786 /* update budget accounting */ 1787 budget--; 1788 } while (likely(budget)); 1789 1790 #ifdef IXGBE_FCOE 1791 /* include DDPed FCoE data */ 1792 if (ddp_bytes > 0) { 1793 unsigned int mss; 1794 1795 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) - 1796 sizeof(struct fc_frame_header) - 1797 sizeof(struct fcoe_crc_eof); 1798 if (mss > 512) 1799 mss &= ~511; 1800 total_rx_bytes += ddp_bytes; 1801 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss); 1802 } 1803 1804 #endif /* IXGBE_FCOE */ 1805 u64_stats_update_begin(&rx_ring->syncp); 1806 rx_ring->stats.packets += total_rx_packets; 1807 rx_ring->stats.bytes += total_rx_bytes; 1808 u64_stats_update_end(&rx_ring->syncp); 1809 q_vector->rx.total_packets += total_rx_packets; 1810 q_vector->rx.total_bytes += total_rx_bytes; 1811 1812 if (cleaned_count) 1813 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 1814 1815 return !!budget; 1816 } 1817 1818 /** 1819 * ixgbe_configure_msix - Configure MSI-X hardware 1820 * @adapter: board private structure 1821 * 1822 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X 1823 * interrupts. 1824 **/ 1825 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) 1826 { 1827 struct ixgbe_q_vector *q_vector; 1828 int q_vectors, v_idx; 1829 u32 mask; 1830 1831 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1832 1833 /* Populate MSIX to EITR Select */ 1834 if (adapter->num_vfs > 32) { 1835 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1; 1836 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel); 1837 } 1838 1839 /* 1840 * Populate the IVAR table and set the ITR values to the 1841 * corresponding register. 1842 */ 1843 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1844 struct ixgbe_ring *ring; 1845 q_vector = adapter->q_vector[v_idx]; 1846 1847 ixgbe_for_each_ring(ring, q_vector->rx) 1848 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1849 1850 ixgbe_for_each_ring(ring, q_vector->tx) 1851 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1852 1853 if (q_vector->tx.ring && !q_vector->rx.ring) { 1854 /* tx only vector */ 1855 if (adapter->tx_itr_setting == 1) 1856 q_vector->itr = IXGBE_10K_ITR; 1857 else 1858 q_vector->itr = adapter->tx_itr_setting; 1859 } else { 1860 /* rx or rx/tx vector */ 1861 if (adapter->rx_itr_setting == 1) 1862 q_vector->itr = IXGBE_20K_ITR; 1863 else 1864 q_vector->itr = adapter->rx_itr_setting; 1865 } 1866 1867 ixgbe_write_eitr(q_vector); 1868 } 1869 1870 switch (adapter->hw.mac.type) { 1871 case ixgbe_mac_82598EB: 1872 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, 1873 v_idx); 1874 break; 1875 case ixgbe_mac_82599EB: 1876 case ixgbe_mac_X540: 1877 ixgbe_set_ivar(adapter, -1, 1, v_idx); 1878 break; 1879 default: 1880 break; 1881 } 1882 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950); 1883 1884 /* set up to autoclear timer, and the vectors */ 1885 mask = IXGBE_EIMS_ENABLE_MASK; 1886 mask &= ~(IXGBE_EIMS_OTHER | 1887 IXGBE_EIMS_MAILBOX | 1888 IXGBE_EIMS_LSC); 1889 1890 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); 1891 } 1892 1893 enum latency_range { 1894 lowest_latency = 0, 1895 low_latency = 1, 1896 bulk_latency = 2, 1897 latency_invalid = 255 1898 }; 1899 1900 /** 1901 * ixgbe_update_itr - update the dynamic ITR value based on statistics 1902 * @q_vector: structure containing interrupt and ring information 1903 * @ring_container: structure containing ring performance data 1904 * 1905 * Stores a new ITR value based on packets and byte 1906 * counts during the last interrupt. The advantage of per interrupt 1907 * computation is faster updates and more accurate ITR for the current 1908 * traffic pattern. Constants in this function were computed 1909 * based on theoretical maximum wire speed and thresholds were set based 1910 * on testing data as well as attempting to minimize response time 1911 * while increasing bulk throughput. 1912 * this functionality is controlled by the InterruptThrottleRate module 1913 * parameter (see ixgbe_param.c) 1914 **/ 1915 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, 1916 struct ixgbe_ring_container *ring_container) 1917 { 1918 int bytes = ring_container->total_bytes; 1919 int packets = ring_container->total_packets; 1920 u32 timepassed_us; 1921 u64 bytes_perint; 1922 u8 itr_setting = ring_container->itr; 1923 1924 if (packets == 0) 1925 return; 1926 1927 /* simple throttlerate management 1928 * 0-10MB/s lowest (100000 ints/s) 1929 * 10-20MB/s low (20000 ints/s) 1930 * 20-1249MB/s bulk (8000 ints/s) 1931 */ 1932 /* what was last interrupt timeslice? */ 1933 timepassed_us = q_vector->itr >> 2; 1934 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1935 1936 switch (itr_setting) { 1937 case lowest_latency: 1938 if (bytes_perint > 10) 1939 itr_setting = low_latency; 1940 break; 1941 case low_latency: 1942 if (bytes_perint > 20) 1943 itr_setting = bulk_latency; 1944 else if (bytes_perint <= 10) 1945 itr_setting = lowest_latency; 1946 break; 1947 case bulk_latency: 1948 if (bytes_perint <= 20) 1949 itr_setting = low_latency; 1950 break; 1951 } 1952 1953 /* clear work counters since we have the values we need */ 1954 ring_container->total_bytes = 0; 1955 ring_container->total_packets = 0; 1956 1957 /* write updated itr to ring container */ 1958 ring_container->itr = itr_setting; 1959 } 1960 1961 /** 1962 * ixgbe_write_eitr - write EITR register in hardware specific way 1963 * @q_vector: structure containing interrupt and ring information 1964 * 1965 * This function is made to be called by ethtool and by the driver 1966 * when it needs to update EITR registers at runtime. Hardware 1967 * specific quirks/differences are taken care of here. 1968 */ 1969 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector) 1970 { 1971 struct ixgbe_adapter *adapter = q_vector->adapter; 1972 struct ixgbe_hw *hw = &adapter->hw; 1973 int v_idx = q_vector->v_idx; 1974 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1975 1976 switch (adapter->hw.mac.type) { 1977 case ixgbe_mac_82598EB: 1978 /* must write high and low 16 bits to reset counter */ 1979 itr_reg |= (itr_reg << 16); 1980 break; 1981 case ixgbe_mac_82599EB: 1982 case ixgbe_mac_X540: 1983 /* 1984 * set the WDIS bit to not clear the timer bits and cause an 1985 * immediate assertion of the interrupt 1986 */ 1987 itr_reg |= IXGBE_EITR_CNT_WDIS; 1988 break; 1989 default: 1990 break; 1991 } 1992 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg); 1993 } 1994 1995 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector) 1996 { 1997 u32 new_itr = q_vector->itr; 1998 u8 current_itr; 1999 2000 ixgbe_update_itr(q_vector, &q_vector->tx); 2001 ixgbe_update_itr(q_vector, &q_vector->rx); 2002 2003 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 2004 2005 switch (current_itr) { 2006 /* counts and packets in update_itr are dependent on these numbers */ 2007 case lowest_latency: 2008 new_itr = IXGBE_100K_ITR; 2009 break; 2010 case low_latency: 2011 new_itr = IXGBE_20K_ITR; 2012 break; 2013 case bulk_latency: 2014 new_itr = IXGBE_8K_ITR; 2015 break; 2016 default: 2017 break; 2018 } 2019 2020 if (new_itr != q_vector->itr) { 2021 /* do an exponential smoothing */ 2022 new_itr = (10 * new_itr * q_vector->itr) / 2023 ((9 * new_itr) + q_vector->itr); 2024 2025 /* save the algorithm value here */ 2026 q_vector->itr = new_itr; 2027 2028 ixgbe_write_eitr(q_vector); 2029 } 2030 } 2031 2032 /** 2033 * ixgbe_check_overtemp_subtask - check for over temperature 2034 * @adapter: pointer to adapter 2035 **/ 2036 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) 2037 { 2038 struct ixgbe_hw *hw = &adapter->hw; 2039 u32 eicr = adapter->interrupt_event; 2040 2041 if (test_bit(__IXGBE_DOWN, &adapter->state)) 2042 return; 2043 2044 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) && 2045 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT)) 2046 return; 2047 2048 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2049 2050 switch (hw->device_id) { 2051 case IXGBE_DEV_ID_82599_T3_LOM: 2052 /* 2053 * Since the warning interrupt is for both ports 2054 * we don't have to check if: 2055 * - This interrupt wasn't for our port. 2056 * - We may have missed the interrupt so always have to 2057 * check if we got a LSC 2058 */ 2059 if (!(eicr & IXGBE_EICR_GPI_SDP0) && 2060 !(eicr & IXGBE_EICR_LSC)) 2061 return; 2062 2063 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { 2064 u32 autoneg; 2065 bool link_up = false; 2066 2067 hw->mac.ops.check_link(hw, &autoneg, &link_up, false); 2068 2069 if (link_up) 2070 return; 2071 } 2072 2073 /* Check if this is not due to overtemp */ 2074 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP) 2075 return; 2076 2077 break; 2078 default: 2079 if (!(eicr & IXGBE_EICR_GPI_SDP0)) 2080 return; 2081 break; 2082 } 2083 e_crit(drv, 2084 "Network adapter has been stopped because it has over heated. " 2085 "Restart the computer. If the problem persists, " 2086 "power off the system and replace the adapter\n"); 2087 2088 adapter->interrupt_event = 0; 2089 } 2090 2091 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) 2092 { 2093 struct ixgbe_hw *hw = &adapter->hw; 2094 2095 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 2096 (eicr & IXGBE_EICR_GPI_SDP1)) { 2097 e_crit(probe, "Fan has stopped, replace the adapter\n"); 2098 /* write to clear the interrupt */ 2099 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2100 } 2101 } 2102 2103 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) 2104 { 2105 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)) 2106 return; 2107 2108 switch (adapter->hw.mac.type) { 2109 case ixgbe_mac_82599EB: 2110 /* 2111 * Need to check link state so complete overtemp check 2112 * on service task 2113 */ 2114 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) && 2115 (!test_bit(__IXGBE_DOWN, &adapter->state))) { 2116 adapter->interrupt_event = eicr; 2117 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2118 ixgbe_service_event_schedule(adapter); 2119 return; 2120 } 2121 return; 2122 case ixgbe_mac_X540: 2123 if (!(eicr & IXGBE_EICR_TS)) 2124 return; 2125 break; 2126 default: 2127 return; 2128 } 2129 2130 e_crit(drv, 2131 "Network adapter has been stopped because it has over heated. " 2132 "Restart the computer. If the problem persists, " 2133 "power off the system and replace the adapter\n"); 2134 } 2135 2136 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) 2137 { 2138 struct ixgbe_hw *hw = &adapter->hw; 2139 2140 if (eicr & IXGBE_EICR_GPI_SDP2) { 2141 /* Clear the interrupt */ 2142 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 2143 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2144 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 2145 ixgbe_service_event_schedule(adapter); 2146 } 2147 } 2148 2149 if (eicr & IXGBE_EICR_GPI_SDP1) { 2150 /* Clear the interrupt */ 2151 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2152 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2153 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 2154 ixgbe_service_event_schedule(adapter); 2155 } 2156 } 2157 } 2158 2159 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) 2160 { 2161 struct ixgbe_hw *hw = &adapter->hw; 2162 2163 adapter->lsc_int++; 2164 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2165 adapter->link_check_timeout = jiffies; 2166 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2167 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 2168 IXGBE_WRITE_FLUSH(hw); 2169 ixgbe_service_event_schedule(adapter); 2170 } 2171 } 2172 2173 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, 2174 u64 qmask) 2175 { 2176 u32 mask; 2177 struct ixgbe_hw *hw = &adapter->hw; 2178 2179 switch (hw->mac.type) { 2180 case ixgbe_mac_82598EB: 2181 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2182 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2183 break; 2184 case ixgbe_mac_82599EB: 2185 case ixgbe_mac_X540: 2186 mask = (qmask & 0xFFFFFFFF); 2187 if (mask) 2188 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2189 mask = (qmask >> 32); 2190 if (mask) 2191 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2192 break; 2193 default: 2194 break; 2195 } 2196 /* skip the flush */ 2197 } 2198 2199 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter, 2200 u64 qmask) 2201 { 2202 u32 mask; 2203 struct ixgbe_hw *hw = &adapter->hw; 2204 2205 switch (hw->mac.type) { 2206 case ixgbe_mac_82598EB: 2207 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2208 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2209 break; 2210 case ixgbe_mac_82599EB: 2211 case ixgbe_mac_X540: 2212 mask = (qmask & 0xFFFFFFFF); 2213 if (mask) 2214 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2215 mask = (qmask >> 32); 2216 if (mask) 2217 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2218 break; 2219 default: 2220 break; 2221 } 2222 /* skip the flush */ 2223 } 2224 2225 /** 2226 * ixgbe_irq_enable - Enable default interrupt generation settings 2227 * @adapter: board private structure 2228 **/ 2229 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, 2230 bool flush) 2231 { 2232 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2233 2234 /* don't reenable LSC while waiting for link */ 2235 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 2236 mask &= ~IXGBE_EIMS_LSC; 2237 2238 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) 2239 switch (adapter->hw.mac.type) { 2240 case ixgbe_mac_82599EB: 2241 mask |= IXGBE_EIMS_GPI_SDP0; 2242 break; 2243 case ixgbe_mac_X540: 2244 mask |= IXGBE_EIMS_TS; 2245 break; 2246 default: 2247 break; 2248 } 2249 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 2250 mask |= IXGBE_EIMS_GPI_SDP1; 2251 switch (adapter->hw.mac.type) { 2252 case ixgbe_mac_82599EB: 2253 mask |= IXGBE_EIMS_GPI_SDP1; 2254 mask |= IXGBE_EIMS_GPI_SDP2; 2255 case ixgbe_mac_X540: 2256 mask |= IXGBE_EIMS_ECC; 2257 mask |= IXGBE_EIMS_MAILBOX; 2258 break; 2259 default: 2260 break; 2261 } 2262 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && 2263 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 2264 mask |= IXGBE_EIMS_FLOW_DIR; 2265 2266 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); 2267 if (queues) 2268 ixgbe_irq_enable_queues(adapter, ~0); 2269 if (flush) 2270 IXGBE_WRITE_FLUSH(&adapter->hw); 2271 } 2272 2273 static irqreturn_t ixgbe_msix_other(int irq, void *data) 2274 { 2275 struct ixgbe_adapter *adapter = data; 2276 struct ixgbe_hw *hw = &adapter->hw; 2277 u32 eicr; 2278 2279 /* 2280 * Workaround for Silicon errata. Use clear-by-write instead 2281 * of clear-by-read. Reading with EICS will return the 2282 * interrupt causes without clearing, which later be done 2283 * with the write to EICR. 2284 */ 2285 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 2286 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 2287 2288 if (eicr & IXGBE_EICR_LSC) 2289 ixgbe_check_lsc(adapter); 2290 2291 if (eicr & IXGBE_EICR_MAILBOX) 2292 ixgbe_msg_task(adapter); 2293 2294 switch (hw->mac.type) { 2295 case ixgbe_mac_82599EB: 2296 case ixgbe_mac_X540: 2297 if (eicr & IXGBE_EICR_ECC) 2298 e_info(link, "Received unrecoverable ECC Err, please " 2299 "reboot\n"); 2300 /* Handle Flow Director Full threshold interrupt */ 2301 if (eicr & IXGBE_EICR_FLOW_DIR) { 2302 int reinit_count = 0; 2303 int i; 2304 for (i = 0; i < adapter->num_tx_queues; i++) { 2305 struct ixgbe_ring *ring = adapter->tx_ring[i]; 2306 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, 2307 &ring->state)) 2308 reinit_count++; 2309 } 2310 if (reinit_count) { 2311 /* no more flow director interrupts until after init */ 2312 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR); 2313 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 2314 ixgbe_service_event_schedule(adapter); 2315 } 2316 } 2317 ixgbe_check_sfp_event(adapter, eicr); 2318 ixgbe_check_overtemp_event(adapter, eicr); 2319 break; 2320 default: 2321 break; 2322 } 2323 2324 ixgbe_check_fan_failure(adapter, eicr); 2325 #ifdef CONFIG_IXGBE_PTP 2326 ixgbe_ptp_check_pps_event(adapter, eicr); 2327 #endif 2328 2329 /* re-enable the original interrupt state, no lsc, no queues */ 2330 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2331 ixgbe_irq_enable(adapter, false, false); 2332 2333 return IRQ_HANDLED; 2334 } 2335 2336 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data) 2337 { 2338 struct ixgbe_q_vector *q_vector = data; 2339 2340 /* EIAM disabled interrupts (on this vector) for us */ 2341 2342 if (q_vector->rx.ring || q_vector->tx.ring) 2343 napi_schedule(&q_vector->napi); 2344 2345 return IRQ_HANDLED; 2346 } 2347 2348 /** 2349 * ixgbe_poll - NAPI Rx polling callback 2350 * @napi: structure for representing this polling device 2351 * @budget: how many packets driver is allowed to clean 2352 * 2353 * This function is used for legacy and MSI, NAPI mode 2354 **/ 2355 int ixgbe_poll(struct napi_struct *napi, int budget) 2356 { 2357 struct ixgbe_q_vector *q_vector = 2358 container_of(napi, struct ixgbe_q_vector, napi); 2359 struct ixgbe_adapter *adapter = q_vector->adapter; 2360 struct ixgbe_ring *ring; 2361 int per_ring_budget; 2362 bool clean_complete = true; 2363 2364 #ifdef CONFIG_IXGBE_DCA 2365 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 2366 ixgbe_update_dca(q_vector); 2367 #endif 2368 2369 ixgbe_for_each_ring(ring, q_vector->tx) 2370 clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring); 2371 2372 /* attempt to distribute budget to each queue fairly, but don't allow 2373 * the budget to go below 1 because we'll exit polling */ 2374 if (q_vector->rx.count > 1) 2375 per_ring_budget = max(budget/q_vector->rx.count, 1); 2376 else 2377 per_ring_budget = budget; 2378 2379 ixgbe_for_each_ring(ring, q_vector->rx) 2380 clean_complete &= ixgbe_clean_rx_irq(q_vector, ring, 2381 per_ring_budget); 2382 2383 /* If all work not completed, return budget and keep polling */ 2384 if (!clean_complete) 2385 return budget; 2386 2387 /* all work done, exit the polling mode */ 2388 napi_complete(napi); 2389 if (adapter->rx_itr_setting & 1) 2390 ixgbe_set_itr(q_vector); 2391 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2392 ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx)); 2393 2394 return 0; 2395 } 2396 2397 /** 2398 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts 2399 * @adapter: board private structure 2400 * 2401 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests 2402 * interrupts from the kernel. 2403 **/ 2404 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) 2405 { 2406 struct net_device *netdev = adapter->netdev; 2407 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2408 int vector, err; 2409 int ri = 0, ti = 0; 2410 2411 for (vector = 0; vector < q_vectors; vector++) { 2412 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 2413 struct msix_entry *entry = &adapter->msix_entries[vector]; 2414 2415 if (q_vector->tx.ring && q_vector->rx.ring) { 2416 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2417 "%s-%s-%d", netdev->name, "TxRx", ri++); 2418 ti++; 2419 } else if (q_vector->rx.ring) { 2420 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2421 "%s-%s-%d", netdev->name, "rx", ri++); 2422 } else if (q_vector->tx.ring) { 2423 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2424 "%s-%s-%d", netdev->name, "tx", ti++); 2425 } else { 2426 /* skip this unused q_vector */ 2427 continue; 2428 } 2429 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0, 2430 q_vector->name, q_vector); 2431 if (err) { 2432 e_err(probe, "request_irq failed for MSIX interrupt " 2433 "Error: %d\n", err); 2434 goto free_queue_irqs; 2435 } 2436 /* If Flow Director is enabled, set interrupt affinity */ 2437 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 2438 /* assign the mask for this irq */ 2439 irq_set_affinity_hint(entry->vector, 2440 &q_vector->affinity_mask); 2441 } 2442 } 2443 2444 err = request_irq(adapter->msix_entries[vector].vector, 2445 ixgbe_msix_other, 0, netdev->name, adapter); 2446 if (err) { 2447 e_err(probe, "request_irq for msix_other failed: %d\n", err); 2448 goto free_queue_irqs; 2449 } 2450 2451 return 0; 2452 2453 free_queue_irqs: 2454 while (vector) { 2455 vector--; 2456 irq_set_affinity_hint(adapter->msix_entries[vector].vector, 2457 NULL); 2458 free_irq(adapter->msix_entries[vector].vector, 2459 adapter->q_vector[vector]); 2460 } 2461 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; 2462 pci_disable_msix(adapter->pdev); 2463 kfree(adapter->msix_entries); 2464 adapter->msix_entries = NULL; 2465 return err; 2466 } 2467 2468 /** 2469 * ixgbe_intr - legacy mode Interrupt Handler 2470 * @irq: interrupt number 2471 * @data: pointer to a network interface device structure 2472 **/ 2473 static irqreturn_t ixgbe_intr(int irq, void *data) 2474 { 2475 struct ixgbe_adapter *adapter = data; 2476 struct ixgbe_hw *hw = &adapter->hw; 2477 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2478 u32 eicr; 2479 2480 /* 2481 * Workaround for silicon errata #26 on 82598. Mask the interrupt 2482 * before the read of EICR. 2483 */ 2484 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 2485 2486 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read 2487 * therefore no explicit interrupt disable is necessary */ 2488 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 2489 if (!eicr) { 2490 /* 2491 * shared interrupt alert! 2492 * make sure interrupts are enabled because the read will 2493 * have disabled interrupts due to EIAM 2494 * finish the workaround of silicon errata on 82598. Unmask 2495 * the interrupt that we masked before the EICR read. 2496 */ 2497 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2498 ixgbe_irq_enable(adapter, true, true); 2499 return IRQ_NONE; /* Not our interrupt */ 2500 } 2501 2502 if (eicr & IXGBE_EICR_LSC) 2503 ixgbe_check_lsc(adapter); 2504 2505 switch (hw->mac.type) { 2506 case ixgbe_mac_82599EB: 2507 ixgbe_check_sfp_event(adapter, eicr); 2508 /* Fall through */ 2509 case ixgbe_mac_X540: 2510 if (eicr & IXGBE_EICR_ECC) 2511 e_info(link, "Received unrecoverable ECC err, please " 2512 "reboot\n"); 2513 ixgbe_check_overtemp_event(adapter, eicr); 2514 break; 2515 default: 2516 break; 2517 } 2518 2519 ixgbe_check_fan_failure(adapter, eicr); 2520 #ifdef CONFIG_IXGBE_PTP 2521 ixgbe_ptp_check_pps_event(adapter, eicr); 2522 #endif 2523 2524 /* would disable interrupts here but EIAM disabled it */ 2525 napi_schedule(&q_vector->napi); 2526 2527 /* 2528 * re-enable link(maybe) and non-queue interrupts, no flush. 2529 * ixgbe_poll will re-enable the queue interrupts 2530 */ 2531 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2532 ixgbe_irq_enable(adapter, false, false); 2533 2534 return IRQ_HANDLED; 2535 } 2536 2537 /** 2538 * ixgbe_request_irq - initialize interrupts 2539 * @adapter: board private structure 2540 * 2541 * Attempts to configure interrupts using the best available 2542 * capabilities of the hardware and kernel. 2543 **/ 2544 static int ixgbe_request_irq(struct ixgbe_adapter *adapter) 2545 { 2546 struct net_device *netdev = adapter->netdev; 2547 int err; 2548 2549 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 2550 err = ixgbe_request_msix_irqs(adapter); 2551 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) 2552 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0, 2553 netdev->name, adapter); 2554 else 2555 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED, 2556 netdev->name, adapter); 2557 2558 if (err) 2559 e_err(probe, "request_irq failed, Error %d\n", err); 2560 2561 return err; 2562 } 2563 2564 static void ixgbe_free_irq(struct ixgbe_adapter *adapter) 2565 { 2566 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 2567 int i, q_vectors; 2568 2569 q_vectors = adapter->num_msix_vectors; 2570 i = q_vectors - 1; 2571 free_irq(adapter->msix_entries[i].vector, adapter); 2572 i--; 2573 2574 for (; i >= 0; i--) { 2575 /* free only the irqs that were actually requested */ 2576 if (!adapter->q_vector[i]->rx.ring && 2577 !adapter->q_vector[i]->tx.ring) 2578 continue; 2579 2580 /* clear the affinity_mask in the IRQ descriptor */ 2581 irq_set_affinity_hint(adapter->msix_entries[i].vector, 2582 NULL); 2583 2584 free_irq(adapter->msix_entries[i].vector, 2585 adapter->q_vector[i]); 2586 } 2587 } else { 2588 free_irq(adapter->pdev->irq, adapter); 2589 } 2590 } 2591 2592 /** 2593 * ixgbe_irq_disable - Mask off interrupt generation on the NIC 2594 * @adapter: board private structure 2595 **/ 2596 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter) 2597 { 2598 switch (adapter->hw.mac.type) { 2599 case ixgbe_mac_82598EB: 2600 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 2601 break; 2602 case ixgbe_mac_82599EB: 2603 case ixgbe_mac_X540: 2604 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 2605 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 2606 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 2607 break; 2608 default: 2609 break; 2610 } 2611 IXGBE_WRITE_FLUSH(&adapter->hw); 2612 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 2613 int i; 2614 for (i = 0; i < adapter->num_msix_vectors; i++) 2615 synchronize_irq(adapter->msix_entries[i].vector); 2616 } else { 2617 synchronize_irq(adapter->pdev->irq); 2618 } 2619 } 2620 2621 /** 2622 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts 2623 * 2624 **/ 2625 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) 2626 { 2627 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2628 2629 /* rx/tx vector */ 2630 if (adapter->rx_itr_setting == 1) 2631 q_vector->itr = IXGBE_20K_ITR; 2632 else 2633 q_vector->itr = adapter->rx_itr_setting; 2634 2635 ixgbe_write_eitr(q_vector); 2636 2637 ixgbe_set_ivar(adapter, 0, 0, 0); 2638 ixgbe_set_ivar(adapter, 1, 0, 0); 2639 2640 e_info(hw, "Legacy interrupt IVAR setup done\n"); 2641 } 2642 2643 /** 2644 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset 2645 * @adapter: board private structure 2646 * @ring: structure containing ring specific data 2647 * 2648 * Configure the Tx descriptor ring after a reset. 2649 **/ 2650 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, 2651 struct ixgbe_ring *ring) 2652 { 2653 struct ixgbe_hw *hw = &adapter->hw; 2654 u64 tdba = ring->dma; 2655 int wait_loop = 10; 2656 u32 txdctl = IXGBE_TXDCTL_ENABLE; 2657 u8 reg_idx = ring->reg_idx; 2658 2659 /* disable queue to avoid issues while updating state */ 2660 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); 2661 IXGBE_WRITE_FLUSH(hw); 2662 2663 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx), 2664 (tdba & DMA_BIT_MASK(32))); 2665 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32)); 2666 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx), 2667 ring->count * sizeof(union ixgbe_adv_tx_desc)); 2668 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0); 2669 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0); 2670 ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx); 2671 2672 /* 2673 * set WTHRESH to encourage burst writeback, it should not be set 2674 * higher than 1 when ITR is 0 as it could cause false TX hangs 2675 * 2676 * In order to avoid issues WTHRESH + PTHRESH should always be equal 2677 * to or less than the number of on chip descriptors, which is 2678 * currently 40. 2679 */ 2680 if (!ring->q_vector || (ring->q_vector->itr < 8)) 2681 txdctl |= (1 << 16); /* WTHRESH = 1 */ 2682 else 2683 txdctl |= (8 << 16); /* WTHRESH = 8 */ 2684 2685 /* 2686 * Setting PTHRESH to 32 both improves performance 2687 * and avoids a TX hang with DFP enabled 2688 */ 2689 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 2690 32; /* PTHRESH = 32 */ 2691 2692 /* reinitialize flowdirector state */ 2693 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && 2694 adapter->atr_sample_rate) { 2695 ring->atr_sample_rate = adapter->atr_sample_rate; 2696 ring->atr_count = 0; 2697 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state); 2698 } else { 2699 ring->atr_sample_rate = 0; 2700 } 2701 2702 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); 2703 2704 /* enable queue */ 2705 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); 2706 2707 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 2708 if (hw->mac.type == ixgbe_mac_82598EB && 2709 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 2710 return; 2711 2712 /* poll to verify queue is enabled */ 2713 do { 2714 usleep_range(1000, 2000); 2715 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx)); 2716 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 2717 if (!wait_loop) 2718 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx); 2719 } 2720 2721 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter) 2722 { 2723 struct ixgbe_hw *hw = &adapter->hw; 2724 u32 rttdcs; 2725 u32 reg; 2726 u8 tcs = netdev_get_num_tc(adapter->netdev); 2727 2728 if (hw->mac.type == ixgbe_mac_82598EB) 2729 return; 2730 2731 /* disable the arbiter while setting MTQC */ 2732 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2733 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2734 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2735 2736 /* set transmit pool layout */ 2737 switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 2738 case (IXGBE_FLAG_SRIOV_ENABLED): 2739 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 2740 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF)); 2741 break; 2742 default: 2743 if (!tcs) 2744 reg = IXGBE_MTQC_64Q_1PB; 2745 else if (tcs <= 4) 2746 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 2747 else 2748 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 2749 2750 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); 2751 2752 /* Enable Security TX Buffer IFG for multiple pb */ 2753 if (tcs) { 2754 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 2755 reg |= IXGBE_SECTX_DCB; 2756 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); 2757 } 2758 break; 2759 } 2760 2761 /* re-enable the arbiter */ 2762 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2763 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2764 } 2765 2766 /** 2767 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset 2768 * @adapter: board private structure 2769 * 2770 * Configure the Tx unit of the MAC after a reset. 2771 **/ 2772 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter) 2773 { 2774 struct ixgbe_hw *hw = &adapter->hw; 2775 u32 dmatxctl; 2776 u32 i; 2777 2778 ixgbe_setup_mtqc(adapter); 2779 2780 if (hw->mac.type != ixgbe_mac_82598EB) { 2781 /* DMATXCTL.EN must be before Tx queues are enabled */ 2782 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2783 dmatxctl |= IXGBE_DMATXCTL_TE; 2784 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2785 } 2786 2787 /* Setup the HW Tx Head and Tail descriptor pointers */ 2788 for (i = 0; i < adapter->num_tx_queues; i++) 2789 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]); 2790 } 2791 2792 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter, 2793 struct ixgbe_ring *ring) 2794 { 2795 struct ixgbe_hw *hw = &adapter->hw; 2796 u8 reg_idx = ring->reg_idx; 2797 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 2798 2799 srrctl |= IXGBE_SRRCTL_DROP_EN; 2800 2801 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 2802 } 2803 2804 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter, 2805 struct ixgbe_ring *ring) 2806 { 2807 struct ixgbe_hw *hw = &adapter->hw; 2808 u8 reg_idx = ring->reg_idx; 2809 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 2810 2811 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 2812 2813 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 2814 } 2815 2816 #ifdef CONFIG_IXGBE_DCB 2817 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 2818 #else 2819 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 2820 #endif 2821 { 2822 int i; 2823 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 2824 2825 if (adapter->ixgbe_ieee_pfc) 2826 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 2827 2828 /* 2829 * We should set the drop enable bit if: 2830 * SR-IOV is enabled 2831 * or 2832 * Number of Rx queues > 1 and flow control is disabled 2833 * 2834 * This allows us to avoid head of line blocking for security 2835 * and performance reasons. 2836 */ 2837 if (adapter->num_vfs || (adapter->num_rx_queues > 1 && 2838 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { 2839 for (i = 0; i < adapter->num_rx_queues; i++) 2840 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); 2841 } else { 2842 for (i = 0; i < adapter->num_rx_queues; i++) 2843 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]); 2844 } 2845 } 2846 2847 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2848 2849 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, 2850 struct ixgbe_ring *rx_ring) 2851 { 2852 u32 srrctl; 2853 u8 reg_idx = rx_ring->reg_idx; 2854 2855 switch (adapter->hw.mac.type) { 2856 case ixgbe_mac_82598EB: { 2857 struct ixgbe_ring_feature *feature = adapter->ring_feature; 2858 const int mask = feature[RING_F_RSS].mask; 2859 reg_idx = reg_idx & mask; 2860 } 2861 break; 2862 case ixgbe_mac_82599EB: 2863 case ixgbe_mac_X540: 2864 default: 2865 break; 2866 } 2867 2868 srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx)); 2869 2870 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2871 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2872 if (adapter->num_vfs) 2873 srrctl |= IXGBE_SRRCTL_DROP_EN; 2874 2875 srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & 2876 IXGBE_SRRCTL_BSIZEHDR_MASK; 2877 2878 #if PAGE_SIZE > IXGBE_MAX_RXBUFFER 2879 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2880 #else 2881 srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2882 #endif 2883 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2884 2885 IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl); 2886 } 2887 2888 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) 2889 { 2890 struct ixgbe_hw *hw = &adapter->hw; 2891 static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D, 2892 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE, 2893 0x6A3E67EA, 0x14364D17, 0x3BED200D}; 2894 u32 mrqc = 0, reta = 0; 2895 u32 rxcsum; 2896 int i, j; 2897 u8 tcs = netdev_get_num_tc(adapter->netdev); 2898 int maxq = adapter->ring_feature[RING_F_RSS].indices; 2899 2900 if (tcs) 2901 maxq = min(maxq, adapter->num_tx_queues / tcs); 2902 2903 /* Fill out hash function seeds */ 2904 for (i = 0; i < 10; i++) 2905 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]); 2906 2907 /* Fill out redirection table */ 2908 for (i = 0, j = 0; i < 128; i++, j++) { 2909 if (j == maxq) 2910 j = 0; 2911 /* reta = 4-byte sliding window of 2912 * 0x00..(indices-1)(indices-1)00..etc. */ 2913 reta = (reta << 8) | (j * 0x11); 2914 if ((i & 3) == 3) 2915 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2916 } 2917 2918 /* Disable indicating checksum in descriptor, enables RSS hash */ 2919 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2920 rxcsum |= IXGBE_RXCSUM_PCSD; 2921 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2922 2923 if (adapter->hw.mac.type == ixgbe_mac_82598EB && 2924 (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) { 2925 mrqc = IXGBE_MRQC_RSSEN; 2926 } else { 2927 int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED 2928 | IXGBE_FLAG_SRIOV_ENABLED); 2929 2930 switch (mask) { 2931 case (IXGBE_FLAG_RSS_ENABLED): 2932 if (!tcs) 2933 mrqc = IXGBE_MRQC_RSSEN; 2934 else if (tcs <= 4) 2935 mrqc = IXGBE_MRQC_RTRSS4TCEN; 2936 else 2937 mrqc = IXGBE_MRQC_RTRSS8TCEN; 2938 break; 2939 case (IXGBE_FLAG_SRIOV_ENABLED): 2940 mrqc = IXGBE_MRQC_VMDQEN; 2941 break; 2942 default: 2943 break; 2944 } 2945 } 2946 2947 /* Perform hash on these packet types */ 2948 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 2949 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 2950 | IXGBE_MRQC_RSS_FIELD_IPV6 2951 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 2952 2953 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) 2954 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 2955 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP) 2956 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 2957 2958 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2959 } 2960 2961 /** 2962 * ixgbe_configure_rscctl - enable RSC for the indicated ring 2963 * @adapter: address of board private structure 2964 * @index: index of ring to set 2965 **/ 2966 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, 2967 struct ixgbe_ring *ring) 2968 { 2969 struct ixgbe_hw *hw = &adapter->hw; 2970 u32 rscctrl; 2971 u8 reg_idx = ring->reg_idx; 2972 2973 if (!ring_is_rsc_enabled(ring)) 2974 return; 2975 2976 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx)); 2977 rscctrl |= IXGBE_RSCCTL_RSCEN; 2978 /* 2979 * we must limit the number of descriptors so that the 2980 * total size of max desc * buf_len is not greater 2981 * than 65536 2982 */ 2983 #if (PAGE_SIZE <= 8192) 2984 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 2985 #elif (PAGE_SIZE <= 16384) 2986 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 2987 #else 2988 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 2989 #endif 2990 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); 2991 } 2992 2993 #define IXGBE_MAX_RX_DESC_POLL 10 2994 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, 2995 struct ixgbe_ring *ring) 2996 { 2997 struct ixgbe_hw *hw = &adapter->hw; 2998 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 2999 u32 rxdctl; 3000 u8 reg_idx = ring->reg_idx; 3001 3002 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 3003 if (hw->mac.type == ixgbe_mac_82598EB && 3004 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3005 return; 3006 3007 do { 3008 usleep_range(1000, 2000); 3009 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3010 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 3011 3012 if (!wait_loop) { 3013 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " 3014 "the polling period\n", reg_idx); 3015 } 3016 } 3017 3018 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter, 3019 struct ixgbe_ring *ring) 3020 { 3021 struct ixgbe_hw *hw = &adapter->hw; 3022 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3023 u32 rxdctl; 3024 u8 reg_idx = ring->reg_idx; 3025 3026 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3027 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 3028 3029 /* write value back with RXDCTL.ENABLE bit cleared */ 3030 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3031 3032 if (hw->mac.type == ixgbe_mac_82598EB && 3033 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3034 return; 3035 3036 /* the hardware may take up to 100us to really disable the rx queue */ 3037 do { 3038 udelay(10); 3039 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3040 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 3041 3042 if (!wait_loop) { 3043 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within " 3044 "the polling period\n", reg_idx); 3045 } 3046 } 3047 3048 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 3049 struct ixgbe_ring *ring) 3050 { 3051 struct ixgbe_hw *hw = &adapter->hw; 3052 u64 rdba = ring->dma; 3053 u32 rxdctl; 3054 u8 reg_idx = ring->reg_idx; 3055 3056 /* disable queue to avoid issues while updating state */ 3057 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3058 ixgbe_disable_rx_queue(adapter, ring); 3059 3060 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32))); 3061 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32)); 3062 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx), 3063 ring->count * sizeof(union ixgbe_adv_rx_desc)); 3064 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0); 3065 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); 3066 ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx); 3067 3068 ixgbe_configure_srrctl(adapter, ring); 3069 ixgbe_configure_rscctl(adapter, ring); 3070 3071 /* If operating in IOV mode set RLPML for X540 */ 3072 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && 3073 hw->mac.type == ixgbe_mac_X540) { 3074 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 3075 rxdctl |= ((ring->netdev->mtu + ETH_HLEN + 3076 ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN); 3077 } 3078 3079 if (hw->mac.type == ixgbe_mac_82598EB) { 3080 /* 3081 * enable cache line friendly hardware writes: 3082 * PTHRESH=32 descriptors (half the internal cache), 3083 * this also removes ugly rx_no_buffer_count increment 3084 * HTHRESH=4 descriptors (to minimize latency on fetch) 3085 * WTHRESH=8 burst writeback up to two cache lines 3086 */ 3087 rxdctl &= ~0x3FFFFF; 3088 rxdctl |= 0x080420; 3089 } 3090 3091 /* enable receive descriptor ring */ 3092 rxdctl |= IXGBE_RXDCTL_ENABLE; 3093 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3094 3095 ixgbe_rx_desc_queue_enable(adapter, ring); 3096 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); 3097 } 3098 3099 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) 3100 { 3101 struct ixgbe_hw *hw = &adapter->hw; 3102 int p; 3103 3104 /* PSRTYPE must be initialized in non 82598 adapters */ 3105 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 3106 IXGBE_PSRTYPE_UDPHDR | 3107 IXGBE_PSRTYPE_IPV4HDR | 3108 IXGBE_PSRTYPE_L2HDR | 3109 IXGBE_PSRTYPE_IPV6HDR; 3110 3111 if (hw->mac.type == ixgbe_mac_82598EB) 3112 return; 3113 3114 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) 3115 psrtype |= (adapter->num_rx_queues_per_pool << 29); 3116 3117 for (p = 0; p < adapter->num_rx_pools; p++) 3118 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p), 3119 psrtype); 3120 } 3121 3122 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) 3123 { 3124 struct ixgbe_hw *hw = &adapter->hw; 3125 u32 gcr_ext; 3126 u32 vt_reg_bits; 3127 u32 reg_offset, vf_shift; 3128 u32 vmdctl; 3129 int i; 3130 3131 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 3132 return; 3133 3134 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 3135 vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN; 3136 vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT); 3137 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits); 3138 3139 vf_shift = adapter->num_vfs % 32; 3140 reg_offset = (adapter->num_vfs >= 32) ? 1 : 0; 3141 3142 /* Enable only the PF's pool for Tx/Rx */ 3143 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift)); 3144 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0); 3145 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift)); 3146 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0); 3147 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 3148 3149 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */ 3150 hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs); 3151 3152 /* 3153 * Set up VF register offsets for selected VT Mode, 3154 * i.e. 32 or 64 VFs for SR-IOV 3155 */ 3156 gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT); 3157 gcr_ext |= IXGBE_GCR_EXT_MSIX_EN; 3158 gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64; 3159 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext); 3160 3161 /* enable Tx loopback for VF/PF communication */ 3162 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 3163 /* Enable MAC Anti-Spoofing */ 3164 hw->mac.ops.set_mac_anti_spoofing(hw, 3165 (adapter->num_vfs != 0), 3166 adapter->num_vfs); 3167 /* For VFs that have spoof checking turned off */ 3168 for (i = 0; i < adapter->num_vfs; i++) { 3169 if (!adapter->vfinfo[i].spoofchk_enabled) 3170 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false); 3171 } 3172 } 3173 3174 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) 3175 { 3176 struct ixgbe_hw *hw = &adapter->hw; 3177 struct net_device *netdev = adapter->netdev; 3178 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3179 struct ixgbe_ring *rx_ring; 3180 int i; 3181 u32 mhadd, hlreg0; 3182 3183 #ifdef IXGBE_FCOE 3184 /* adjust max frame to be able to do baby jumbo for FCoE */ 3185 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && 3186 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) 3187 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3188 3189 #endif /* IXGBE_FCOE */ 3190 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 3191 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { 3192 mhadd &= ~IXGBE_MHADD_MFS_MASK; 3193 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; 3194 3195 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 3196 } 3197 3198 /* MHADD will allow an extra 4 bytes past for vlan tagged frames */ 3199 max_frame += VLAN_HLEN; 3200 3201 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3202 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ 3203 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 3204 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 3205 3206 /* 3207 * Setup the HW Rx Head and Tail Descriptor Pointers and 3208 * the Base and Length of the Rx Descriptor Ring 3209 */ 3210 for (i = 0; i < adapter->num_rx_queues; i++) { 3211 rx_ring = adapter->rx_ring[i]; 3212 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 3213 set_ring_rsc_enabled(rx_ring); 3214 else 3215 clear_ring_rsc_enabled(rx_ring); 3216 } 3217 } 3218 3219 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) 3220 { 3221 struct ixgbe_hw *hw = &adapter->hw; 3222 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3223 3224 switch (hw->mac.type) { 3225 case ixgbe_mac_82598EB: 3226 /* 3227 * For VMDq support of different descriptor types or 3228 * buffer sizes through the use of multiple SRRCTL 3229 * registers, RDRXCTL.MVMEN must be set to 1 3230 * 3231 * also, the manual doesn't mention it clearly but DCA hints 3232 * will only use queue 0's tags unless this bit is set. Side 3233 * effects of setting this bit are only that SRRCTL must be 3234 * fully programmed [0..15] 3235 */ 3236 rdrxctl |= IXGBE_RDRXCTL_MVMEN; 3237 break; 3238 case ixgbe_mac_82599EB: 3239 case ixgbe_mac_X540: 3240 /* Disable RSC for ACK packets */ 3241 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3242 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3243 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3244 /* hardware requires some bits to be set by default */ 3245 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX); 3246 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3247 break; 3248 default: 3249 /* We should do nothing since we don't know this hardware */ 3250 return; 3251 } 3252 3253 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3254 } 3255 3256 /** 3257 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset 3258 * @adapter: board private structure 3259 * 3260 * Configure the Rx unit of the MAC after a reset. 3261 **/ 3262 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) 3263 { 3264 struct ixgbe_hw *hw = &adapter->hw; 3265 int i; 3266 u32 rxctrl; 3267 3268 /* disable receives while setting up the descriptors */ 3269 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3270 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 3271 3272 ixgbe_setup_psrtype(adapter); 3273 ixgbe_setup_rdrxctl(adapter); 3274 3275 /* Program registers for the distribution of queues */ 3276 ixgbe_setup_mrqc(adapter); 3277 3278 /* set_rx_buffer_len must be called before ring initialization */ 3279 ixgbe_set_rx_buffer_len(adapter); 3280 3281 /* 3282 * Setup the HW Rx Head and Tail Descriptor Pointers and 3283 * the Base and Length of the Rx Descriptor Ring 3284 */ 3285 for (i = 0; i < adapter->num_rx_queues; i++) 3286 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); 3287 3288 /* disable drop enable for 82598 parts */ 3289 if (hw->mac.type == ixgbe_mac_82598EB) 3290 rxctrl |= IXGBE_RXCTRL_DMBYPS; 3291 3292 /* enable all receives */ 3293 rxctrl |= IXGBE_RXCTRL_RXEN; 3294 hw->mac.ops.enable_rx_dma(hw, rxctrl); 3295 } 3296 3297 static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 3298 { 3299 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3300 struct ixgbe_hw *hw = &adapter->hw; 3301 int pool_ndx = adapter->num_vfs; 3302 3303 /* add VID to filter table */ 3304 hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, true); 3305 set_bit(vid, adapter->active_vlans); 3306 3307 return 0; 3308 } 3309 3310 static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 3311 { 3312 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3313 struct ixgbe_hw *hw = &adapter->hw; 3314 int pool_ndx = adapter->num_vfs; 3315 3316 /* remove VID from filter table */ 3317 hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, false); 3318 clear_bit(vid, adapter->active_vlans); 3319 3320 return 0; 3321 } 3322 3323 /** 3324 * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering 3325 * @adapter: driver data 3326 */ 3327 static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter) 3328 { 3329 struct ixgbe_hw *hw = &adapter->hw; 3330 u32 vlnctrl; 3331 3332 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3333 vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN); 3334 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3335 } 3336 3337 /** 3338 * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering 3339 * @adapter: driver data 3340 */ 3341 static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter) 3342 { 3343 struct ixgbe_hw *hw = &adapter->hw; 3344 u32 vlnctrl; 3345 3346 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3347 vlnctrl |= IXGBE_VLNCTRL_VFE; 3348 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 3349 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3350 } 3351 3352 /** 3353 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping 3354 * @adapter: driver data 3355 */ 3356 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter) 3357 { 3358 struct ixgbe_hw *hw = &adapter->hw; 3359 u32 vlnctrl; 3360 int i, j; 3361 3362 switch (hw->mac.type) { 3363 case ixgbe_mac_82598EB: 3364 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3365 vlnctrl &= ~IXGBE_VLNCTRL_VME; 3366 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3367 break; 3368 case ixgbe_mac_82599EB: 3369 case ixgbe_mac_X540: 3370 for (i = 0; i < adapter->num_rx_queues; i++) { 3371 j = adapter->rx_ring[i]->reg_idx; 3372 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 3373 vlnctrl &= ~IXGBE_RXDCTL_VME; 3374 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 3375 } 3376 break; 3377 default: 3378 break; 3379 } 3380 } 3381 3382 /** 3383 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping 3384 * @adapter: driver data 3385 */ 3386 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter) 3387 { 3388 struct ixgbe_hw *hw = &adapter->hw; 3389 u32 vlnctrl; 3390 int i, j; 3391 3392 switch (hw->mac.type) { 3393 case ixgbe_mac_82598EB: 3394 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3395 vlnctrl |= IXGBE_VLNCTRL_VME; 3396 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3397 break; 3398 case ixgbe_mac_82599EB: 3399 case ixgbe_mac_X540: 3400 for (i = 0; i < adapter->num_rx_queues; i++) { 3401 j = adapter->rx_ring[i]->reg_idx; 3402 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 3403 vlnctrl |= IXGBE_RXDCTL_VME; 3404 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 3405 } 3406 break; 3407 default: 3408 break; 3409 } 3410 } 3411 3412 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter) 3413 { 3414 u16 vid; 3415 3416 ixgbe_vlan_rx_add_vid(adapter->netdev, 0); 3417 3418 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 3419 ixgbe_vlan_rx_add_vid(adapter->netdev, vid); 3420 } 3421 3422 /** 3423 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table 3424 * @netdev: network interface device structure 3425 * 3426 * Writes unicast address list to the RAR table. 3427 * Returns: -ENOMEM on failure/insufficient address space 3428 * 0 on no addresses written 3429 * X on writing X addresses to the RAR table 3430 **/ 3431 static int ixgbe_write_uc_addr_list(struct net_device *netdev) 3432 { 3433 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3434 struct ixgbe_hw *hw = &adapter->hw; 3435 unsigned int vfn = adapter->num_vfs; 3436 unsigned int rar_entries = IXGBE_MAX_PF_MACVLANS; 3437 int count = 0; 3438 3439 /* return ENOMEM indicating insufficient memory for addresses */ 3440 if (netdev_uc_count(netdev) > rar_entries) 3441 return -ENOMEM; 3442 3443 if (!netdev_uc_empty(netdev) && rar_entries) { 3444 struct netdev_hw_addr *ha; 3445 /* return error if we do not support writing to RAR table */ 3446 if (!hw->mac.ops.set_rar) 3447 return -ENOMEM; 3448 3449 netdev_for_each_uc_addr(ha, netdev) { 3450 if (!rar_entries) 3451 break; 3452 hw->mac.ops.set_rar(hw, rar_entries--, ha->addr, 3453 vfn, IXGBE_RAH_AV); 3454 count++; 3455 } 3456 } 3457 /* write the addresses in reverse order to avoid write combining */ 3458 for (; rar_entries > 0 ; rar_entries--) 3459 hw->mac.ops.clear_rar(hw, rar_entries); 3460 3461 return count; 3462 } 3463 3464 /** 3465 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set 3466 * @netdev: network interface device structure 3467 * 3468 * The set_rx_method entry point is called whenever the unicast/multicast 3469 * address list or the network interface flags are updated. This routine is 3470 * responsible for configuring the hardware for proper unicast, multicast and 3471 * promiscuous mode. 3472 **/ 3473 void ixgbe_set_rx_mode(struct net_device *netdev) 3474 { 3475 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3476 struct ixgbe_hw *hw = &adapter->hw; 3477 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE; 3478 int count; 3479 3480 /* Check for Promiscuous and All Multicast modes */ 3481 3482 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3483 3484 /* set all bits that we expect to always be set */ 3485 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */ 3486 fctrl |= IXGBE_FCTRL_BAM; 3487 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */ 3488 fctrl |= IXGBE_FCTRL_PMCF; 3489 3490 /* clear the bits we are changing the status of */ 3491 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 3492 3493 if (netdev->flags & IFF_PROMISC) { 3494 hw->addr_ctrl.user_set_promisc = true; 3495 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 3496 vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE); 3497 /* don't hardware filter vlans in promisc mode */ 3498 ixgbe_vlan_filter_disable(adapter); 3499 } else { 3500 if (netdev->flags & IFF_ALLMULTI) { 3501 fctrl |= IXGBE_FCTRL_MPE; 3502 vmolr |= IXGBE_VMOLR_MPE; 3503 } else { 3504 /* 3505 * Write addresses to the MTA, if the attempt fails 3506 * then we should just turn on promiscuous mode so 3507 * that we can at least receive multicast traffic 3508 */ 3509 hw->mac.ops.update_mc_addr_list(hw, netdev); 3510 vmolr |= IXGBE_VMOLR_ROMPE; 3511 } 3512 ixgbe_vlan_filter_enable(adapter); 3513 hw->addr_ctrl.user_set_promisc = false; 3514 } 3515 3516 /* 3517 * Write addresses to available RAR registers, if there is not 3518 * sufficient space to store all the addresses then enable 3519 * unicast promiscuous mode 3520 */ 3521 count = ixgbe_write_uc_addr_list(netdev); 3522 if (count < 0) { 3523 fctrl |= IXGBE_FCTRL_UPE; 3524 vmolr |= IXGBE_VMOLR_ROPE; 3525 } 3526 3527 if (adapter->num_vfs) { 3528 ixgbe_restore_vf_multicasts(adapter); 3529 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) & 3530 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE | 3531 IXGBE_VMOLR_ROPE); 3532 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr); 3533 } 3534 3535 /* This is useful for sniffing bad packets. */ 3536 if (adapter->netdev->features & NETIF_F_RXALL) { 3537 /* UPE and MPE will be handled by normal PROMISC logic 3538 * in e1000e_set_rx_mode */ 3539 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */ 3540 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */ 3541 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */ 3542 3543 fctrl &= ~(IXGBE_FCTRL_DPF); 3544 /* NOTE: VLAN filtering is disabled by setting PROMISC */ 3545 } 3546 3547 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3548 3549 if (netdev->features & NETIF_F_HW_VLAN_RX) 3550 ixgbe_vlan_strip_enable(adapter); 3551 else 3552 ixgbe_vlan_strip_disable(adapter); 3553 } 3554 3555 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter) 3556 { 3557 int q_idx; 3558 struct ixgbe_q_vector *q_vector; 3559 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 3560 3561 /* legacy and MSI only use one vector */ 3562 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) 3563 q_vectors = 1; 3564 3565 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 3566 q_vector = adapter->q_vector[q_idx]; 3567 napi_enable(&q_vector->napi); 3568 } 3569 } 3570 3571 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) 3572 { 3573 int q_idx; 3574 struct ixgbe_q_vector *q_vector; 3575 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 3576 3577 /* legacy and MSI only use one vector */ 3578 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) 3579 q_vectors = 1; 3580 3581 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 3582 q_vector = adapter->q_vector[q_idx]; 3583 napi_disable(&q_vector->napi); 3584 } 3585 } 3586 3587 #ifdef CONFIG_IXGBE_DCB 3588 /* 3589 * ixgbe_configure_dcb - Configure DCB hardware 3590 * @adapter: ixgbe adapter struct 3591 * 3592 * This is called by the driver on open to configure the DCB hardware. 3593 * This is also called by the gennetlink interface when reconfiguring 3594 * the DCB state. 3595 */ 3596 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) 3597 { 3598 struct ixgbe_hw *hw = &adapter->hw; 3599 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3600 3601 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) { 3602 if (hw->mac.type == ixgbe_mac_82598EB) 3603 netif_set_gso_max_size(adapter->netdev, 65536); 3604 return; 3605 } 3606 3607 if (hw->mac.type == ixgbe_mac_82598EB) 3608 netif_set_gso_max_size(adapter->netdev, 32768); 3609 3610 3611 /* Enable VLAN tag insert/strip */ 3612 adapter->netdev->features |= NETIF_F_HW_VLAN_RX; 3613 3614 hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true); 3615 3616 #ifdef IXGBE_FCOE 3617 if (adapter->netdev->features & NETIF_F_FCOE_MTU) 3618 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); 3619 #endif 3620 3621 /* reconfigure the hardware */ 3622 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) { 3623 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 3624 DCB_TX_CONFIG); 3625 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 3626 DCB_RX_CONFIG); 3627 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg); 3628 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) { 3629 ixgbe_dcb_hw_ets(&adapter->hw, 3630 adapter->ixgbe_ieee_ets, 3631 max_frame); 3632 ixgbe_dcb_hw_pfc_config(&adapter->hw, 3633 adapter->ixgbe_ieee_pfc->pfc_en, 3634 adapter->ixgbe_ieee_ets->prio_tc); 3635 } 3636 3637 /* Enable RSS Hash per TC */ 3638 if (hw->mac.type != ixgbe_mac_82598EB) { 3639 int i; 3640 u32 reg = 0; 3641 3642 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) { 3643 u8 msb = 0; 3644 u8 cnt = adapter->netdev->tc_to_txq[i].count; 3645 3646 while (cnt >>= 1) 3647 msb++; 3648 3649 reg |= msb << IXGBE_RQTC_SHIFT_TC(i); 3650 } 3651 IXGBE_WRITE_REG(hw, IXGBE_RQTC, reg); 3652 } 3653 } 3654 #endif 3655 3656 /* Additional bittime to account for IXGBE framing */ 3657 #define IXGBE_ETH_FRAMING 20 3658 3659 /* 3660 * ixgbe_hpbthresh - calculate high water mark for flow control 3661 * 3662 * @adapter: board private structure to calculate for 3663 * @pb - packet buffer to calculate 3664 */ 3665 static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) 3666 { 3667 struct ixgbe_hw *hw = &adapter->hw; 3668 struct net_device *dev = adapter->netdev; 3669 int link, tc, kb, marker; 3670 u32 dv_id, rx_pba; 3671 3672 /* Calculate max LAN frame size */ 3673 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING; 3674 3675 #ifdef IXGBE_FCOE 3676 /* FCoE traffic class uses FCOE jumbo frames */ 3677 if (dev->features & NETIF_F_FCOE_MTU) { 3678 int fcoe_pb = 0; 3679 3680 #ifdef CONFIG_IXGBE_DCB 3681 fcoe_pb = netdev_get_prio_tc_map(dev, adapter->fcoe.up); 3682 3683 #endif 3684 if (fcoe_pb == pb && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) 3685 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3686 } 3687 #endif 3688 3689 /* Calculate delay value for device */ 3690 switch (hw->mac.type) { 3691 case ixgbe_mac_X540: 3692 dv_id = IXGBE_DV_X540(link, tc); 3693 break; 3694 default: 3695 dv_id = IXGBE_DV(link, tc); 3696 break; 3697 } 3698 3699 /* Loopback switch introduces additional latency */ 3700 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 3701 dv_id += IXGBE_B2BT(tc); 3702 3703 /* Delay value is calculated in bit times convert to KB */ 3704 kb = IXGBE_BT2KB(dv_id); 3705 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10; 3706 3707 marker = rx_pba - kb; 3708 3709 /* It is possible that the packet buffer is not large enough 3710 * to provide required headroom. In this case throw an error 3711 * to user and a do the best we can. 3712 */ 3713 if (marker < 0) { 3714 e_warn(drv, "Packet Buffer(%i) can not provide enough" 3715 "headroom to support flow control." 3716 "Decrease MTU or number of traffic classes\n", pb); 3717 marker = tc + 1; 3718 } 3719 3720 return marker; 3721 } 3722 3723 /* 3724 * ixgbe_lpbthresh - calculate low water mark for for flow control 3725 * 3726 * @adapter: board private structure to calculate for 3727 * @pb - packet buffer to calculate 3728 */ 3729 static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter) 3730 { 3731 struct ixgbe_hw *hw = &adapter->hw; 3732 struct net_device *dev = adapter->netdev; 3733 int tc; 3734 u32 dv_id; 3735 3736 /* Calculate max LAN frame size */ 3737 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN; 3738 3739 /* Calculate delay value for device */ 3740 switch (hw->mac.type) { 3741 case ixgbe_mac_X540: 3742 dv_id = IXGBE_LOW_DV_X540(tc); 3743 break; 3744 default: 3745 dv_id = IXGBE_LOW_DV(tc); 3746 break; 3747 } 3748 3749 /* Delay value is calculated in bit times convert to KB */ 3750 return IXGBE_BT2KB(dv_id); 3751 } 3752 3753 /* 3754 * ixgbe_pbthresh_setup - calculate and setup high low water marks 3755 */ 3756 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter) 3757 { 3758 struct ixgbe_hw *hw = &adapter->hw; 3759 int num_tc = netdev_get_num_tc(adapter->netdev); 3760 int i; 3761 3762 if (!num_tc) 3763 num_tc = 1; 3764 3765 hw->fc.low_water = ixgbe_lpbthresh(adapter); 3766 3767 for (i = 0; i < num_tc; i++) { 3768 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i); 3769 3770 /* Low water marks must not be larger than high water marks */ 3771 if (hw->fc.low_water > hw->fc.high_water[i]) 3772 hw->fc.low_water = 0; 3773 } 3774 } 3775 3776 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) 3777 { 3778 struct ixgbe_hw *hw = &adapter->hw; 3779 int hdrm; 3780 u8 tc = netdev_get_num_tc(adapter->netdev); 3781 3782 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || 3783 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 3784 hdrm = 32 << adapter->fdir_pballoc; 3785 else 3786 hdrm = 0; 3787 3788 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); 3789 ixgbe_pbthresh_setup(adapter); 3790 } 3791 3792 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) 3793 { 3794 struct ixgbe_hw *hw = &adapter->hw; 3795 struct hlist_node *node, *node2; 3796 struct ixgbe_fdir_filter *filter; 3797 3798 spin_lock(&adapter->fdir_perfect_lock); 3799 3800 if (!hlist_empty(&adapter->fdir_filter_list)) 3801 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask); 3802 3803 hlist_for_each_entry_safe(filter, node, node2, 3804 &adapter->fdir_filter_list, fdir_node) { 3805 ixgbe_fdir_write_perfect_filter_82599(hw, 3806 &filter->filter, 3807 filter->sw_idx, 3808 (filter->action == IXGBE_FDIR_DROP_QUEUE) ? 3809 IXGBE_FDIR_DROP_QUEUE : 3810 adapter->rx_ring[filter->action]->reg_idx); 3811 } 3812 3813 spin_unlock(&adapter->fdir_perfect_lock); 3814 } 3815 3816 static void ixgbe_configure(struct ixgbe_adapter *adapter) 3817 { 3818 struct ixgbe_hw *hw = &adapter->hw; 3819 3820 ixgbe_configure_pb(adapter); 3821 #ifdef CONFIG_IXGBE_DCB 3822 ixgbe_configure_dcb(adapter); 3823 #endif 3824 3825 ixgbe_set_rx_mode(adapter->netdev); 3826 ixgbe_restore_vlan(adapter); 3827 3828 #ifdef IXGBE_FCOE 3829 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) 3830 ixgbe_configure_fcoe(adapter); 3831 3832 #endif /* IXGBE_FCOE */ 3833 3834 switch (hw->mac.type) { 3835 case ixgbe_mac_82599EB: 3836 case ixgbe_mac_X540: 3837 hw->mac.ops.disable_rx_buff(hw); 3838 break; 3839 default: 3840 break; 3841 } 3842 3843 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 3844 ixgbe_init_fdir_signature_82599(&adapter->hw, 3845 adapter->fdir_pballoc); 3846 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { 3847 ixgbe_init_fdir_perfect_82599(&adapter->hw, 3848 adapter->fdir_pballoc); 3849 ixgbe_fdir_filter_restore(adapter); 3850 } 3851 3852 switch (hw->mac.type) { 3853 case ixgbe_mac_82599EB: 3854 case ixgbe_mac_X540: 3855 hw->mac.ops.enable_rx_buff(hw); 3856 break; 3857 default: 3858 break; 3859 } 3860 3861 ixgbe_configure_virtualization(adapter); 3862 3863 ixgbe_configure_tx(adapter); 3864 ixgbe_configure_rx(adapter); 3865 } 3866 3867 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw) 3868 { 3869 switch (hw->phy.type) { 3870 case ixgbe_phy_sfp_avago: 3871 case ixgbe_phy_sfp_ftl: 3872 case ixgbe_phy_sfp_intel: 3873 case ixgbe_phy_sfp_unknown: 3874 case ixgbe_phy_sfp_passive_tyco: 3875 case ixgbe_phy_sfp_passive_unknown: 3876 case ixgbe_phy_sfp_active_unknown: 3877 case ixgbe_phy_sfp_ftl_active: 3878 return true; 3879 case ixgbe_phy_nl: 3880 if (hw->mac.type == ixgbe_mac_82598EB) 3881 return true; 3882 default: 3883 return false; 3884 } 3885 } 3886 3887 /** 3888 * ixgbe_sfp_link_config - set up SFP+ link 3889 * @adapter: pointer to private adapter struct 3890 **/ 3891 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) 3892 { 3893 /* 3894 * We are assuming the worst case scenario here, and that 3895 * is that an SFP was inserted/removed after the reset 3896 * but before SFP detection was enabled. As such the best 3897 * solution is to just start searching as soon as we start 3898 */ 3899 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3900 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 3901 3902 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 3903 } 3904 3905 /** 3906 * ixgbe_non_sfp_link_config - set up non-SFP+ link 3907 * @hw: pointer to private hardware struct 3908 * 3909 * Returns 0 on success, negative on failure 3910 **/ 3911 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) 3912 { 3913 u32 autoneg; 3914 bool negotiation, link_up = false; 3915 u32 ret = IXGBE_ERR_LINK_SETUP; 3916 3917 if (hw->mac.ops.check_link) 3918 ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false); 3919 3920 if (ret) 3921 goto link_cfg_out; 3922 3923 autoneg = hw->phy.autoneg_advertised; 3924 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 3925 ret = hw->mac.ops.get_link_capabilities(hw, &autoneg, 3926 &negotiation); 3927 if (ret) 3928 goto link_cfg_out; 3929 3930 if (hw->mac.ops.setup_link) 3931 ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up); 3932 link_cfg_out: 3933 return ret; 3934 } 3935 3936 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) 3937 { 3938 struct ixgbe_hw *hw = &adapter->hw; 3939 u32 gpie = 0; 3940 3941 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 3942 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 3943 IXGBE_GPIE_OCD; 3944 gpie |= IXGBE_GPIE_EIAME; 3945 /* 3946 * use EIAM to auto-mask when MSI-X interrupt is asserted 3947 * this saves a register write for every interrupt 3948 */ 3949 switch (hw->mac.type) { 3950 case ixgbe_mac_82598EB: 3951 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 3952 break; 3953 case ixgbe_mac_82599EB: 3954 case ixgbe_mac_X540: 3955 default: 3956 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 3957 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 3958 break; 3959 } 3960 } else { 3961 /* legacy interrupts, use EIAM to auto-mask when reading EICR, 3962 * specifically only auto mask tx and rx interrupts */ 3963 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 3964 } 3965 3966 /* XXX: to interrupt immediately for EICS writes, enable this */ 3967 /* gpie |= IXGBE_GPIE_EIMEN; */ 3968 3969 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 3970 gpie &= ~IXGBE_GPIE_VTMODE_MASK; 3971 gpie |= IXGBE_GPIE_VTMODE_64; 3972 } 3973 3974 /* Enable Thermal over heat sensor interrupt */ 3975 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) { 3976 switch (adapter->hw.mac.type) { 3977 case ixgbe_mac_82599EB: 3978 gpie |= IXGBE_SDP0_GPIEN; 3979 break; 3980 case ixgbe_mac_X540: 3981 gpie |= IXGBE_EIMS_TS; 3982 break; 3983 default: 3984 break; 3985 } 3986 } 3987 3988 /* Enable fan failure interrupt */ 3989 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 3990 gpie |= IXGBE_SDP1_GPIEN; 3991 3992 if (hw->mac.type == ixgbe_mac_82599EB) { 3993 gpie |= IXGBE_SDP1_GPIEN; 3994 gpie |= IXGBE_SDP2_GPIEN; 3995 } 3996 3997 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3998 } 3999 4000 static void ixgbe_up_complete(struct ixgbe_adapter *adapter) 4001 { 4002 struct ixgbe_hw *hw = &adapter->hw; 4003 int err; 4004 u32 ctrl_ext; 4005 4006 ixgbe_get_hw_control(adapter); 4007 ixgbe_setup_gpie(adapter); 4008 4009 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 4010 ixgbe_configure_msix(adapter); 4011 else 4012 ixgbe_configure_msi_and_legacy(adapter); 4013 4014 /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */ 4015 if (hw->mac.ops.enable_tx_laser && 4016 ((hw->phy.multispeed_fiber) || 4017 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && 4018 (hw->mac.type == ixgbe_mac_82599EB)))) 4019 hw->mac.ops.enable_tx_laser(hw); 4020 4021 clear_bit(__IXGBE_DOWN, &adapter->state); 4022 ixgbe_napi_enable_all(adapter); 4023 4024 if (ixgbe_is_sfp(hw)) { 4025 ixgbe_sfp_link_config(adapter); 4026 } else { 4027 err = ixgbe_non_sfp_link_config(hw); 4028 if (err) 4029 e_err(probe, "link_config FAILED %d\n", err); 4030 } 4031 4032 /* clear any pending interrupts, may auto mask */ 4033 IXGBE_READ_REG(hw, IXGBE_EICR); 4034 ixgbe_irq_enable(adapter, true, true); 4035 4036 /* 4037 * If this adapter has a fan, check to see if we had a failure 4038 * before we enabled the interrupt. 4039 */ 4040 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 4041 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 4042 if (esdp & IXGBE_ESDP_SDP1) 4043 e_crit(drv, "Fan has stopped, replace the adapter\n"); 4044 } 4045 4046 /* enable transmits */ 4047 netif_tx_start_all_queues(adapter->netdev); 4048 4049 /* bring the link up in the watchdog, this could race with our first 4050 * link up interrupt but shouldn't be a problem */ 4051 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4052 adapter->link_check_timeout = jiffies; 4053 mod_timer(&adapter->service_timer, jiffies); 4054 4055 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 4056 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4057 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4058 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4059 } 4060 4061 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) 4062 { 4063 WARN_ON(in_interrupt()); 4064 /* put off any impending NetWatchDogTimeout */ 4065 adapter->netdev->trans_start = jiffies; 4066 4067 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state)) 4068 usleep_range(1000, 2000); 4069 ixgbe_down(adapter); 4070 /* 4071 * If SR-IOV enabled then wait a bit before bringing the adapter 4072 * back up to give the VFs time to respond to the reset. The 4073 * two second wait is based upon the watchdog timer cycle in 4074 * the VF driver. 4075 */ 4076 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 4077 msleep(2000); 4078 ixgbe_up(adapter); 4079 clear_bit(__IXGBE_RESETTING, &adapter->state); 4080 } 4081 4082 void ixgbe_up(struct ixgbe_adapter *adapter) 4083 { 4084 /* hardware has been reset, we need to reload some things */ 4085 ixgbe_configure(adapter); 4086 4087 ixgbe_up_complete(adapter); 4088 } 4089 4090 void ixgbe_reset(struct ixgbe_adapter *adapter) 4091 { 4092 struct ixgbe_hw *hw = &adapter->hw; 4093 int err; 4094 4095 /* lock SFP init bit to prevent race conditions with the watchdog */ 4096 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 4097 usleep_range(1000, 2000); 4098 4099 /* clear all SFP and link config related flags while holding SFP_INIT */ 4100 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | 4101 IXGBE_FLAG2_SFP_NEEDS_RESET); 4102 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4103 4104 err = hw->mac.ops.init_hw(hw); 4105 switch (err) { 4106 case 0: 4107 case IXGBE_ERR_SFP_NOT_PRESENT: 4108 case IXGBE_ERR_SFP_NOT_SUPPORTED: 4109 break; 4110 case IXGBE_ERR_MASTER_REQUESTS_PENDING: 4111 e_dev_err("master disable timed out\n"); 4112 break; 4113 case IXGBE_ERR_EEPROM_VERSION: 4114 /* We are running on a pre-production device, log a warning */ 4115 e_dev_warn("This device is a pre-production adapter/LOM. " 4116 "Please be aware there may be issues associated with " 4117 "your hardware. If you are experiencing problems " 4118 "please contact your Intel or hardware " 4119 "representative who provided you with this " 4120 "hardware.\n"); 4121 break; 4122 default: 4123 e_dev_err("Hardware Error: %d\n", err); 4124 } 4125 4126 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 4127 4128 /* reprogram the RAR[0] in case user changed it. */ 4129 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs, 4130 IXGBE_RAH_AV); 4131 } 4132 4133 /** 4134 * ixgbe_init_rx_page_offset - initialize page offset values for Rx buffers 4135 * @rx_ring: ring to setup 4136 * 4137 * On many IA platforms the L1 cache has a critical stride of 4K, this 4138 * results in each receive buffer starting in the same cache set. To help 4139 * reduce the pressure on this cache set we can interleave the offsets so 4140 * that only every other buffer will be in the same cache set. 4141 **/ 4142 static void ixgbe_init_rx_page_offset(struct ixgbe_ring *rx_ring) 4143 { 4144 struct ixgbe_rx_buffer *rx_buffer = rx_ring->rx_buffer_info; 4145 u16 i; 4146 4147 for (i = 0; i < rx_ring->count; i += 2) { 4148 rx_buffer[0].page_offset = 0; 4149 rx_buffer[1].page_offset = ixgbe_rx_bufsz(rx_ring); 4150 rx_buffer = &rx_buffer[2]; 4151 } 4152 } 4153 4154 /** 4155 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue 4156 * @rx_ring: ring to free buffers from 4157 **/ 4158 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) 4159 { 4160 struct device *dev = rx_ring->dev; 4161 unsigned long size; 4162 u16 i; 4163 4164 /* ring already cleared, nothing to do */ 4165 if (!rx_ring->rx_buffer_info) 4166 return; 4167 4168 /* Free all the Rx ring sk_buffs */ 4169 for (i = 0; i < rx_ring->count; i++) { 4170 struct ixgbe_rx_buffer *rx_buffer; 4171 4172 rx_buffer = &rx_ring->rx_buffer_info[i]; 4173 if (rx_buffer->skb) { 4174 struct sk_buff *skb = rx_buffer->skb; 4175 if (IXGBE_CB(skb)->page_released) { 4176 dma_unmap_page(dev, 4177 IXGBE_CB(skb)->dma, 4178 ixgbe_rx_bufsz(rx_ring), 4179 DMA_FROM_DEVICE); 4180 IXGBE_CB(skb)->page_released = false; 4181 } 4182 dev_kfree_skb(skb); 4183 } 4184 rx_buffer->skb = NULL; 4185 if (rx_buffer->dma) 4186 dma_unmap_page(dev, rx_buffer->dma, 4187 ixgbe_rx_pg_size(rx_ring), 4188 DMA_FROM_DEVICE); 4189 rx_buffer->dma = 0; 4190 if (rx_buffer->page) 4191 __free_pages(rx_buffer->page, 4192 ixgbe_rx_pg_order(rx_ring)); 4193 rx_buffer->page = NULL; 4194 } 4195 4196 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4197 memset(rx_ring->rx_buffer_info, 0, size); 4198 4199 ixgbe_init_rx_page_offset(rx_ring); 4200 4201 /* Zero out the descriptor ring */ 4202 memset(rx_ring->desc, 0, rx_ring->size); 4203 4204 rx_ring->next_to_alloc = 0; 4205 rx_ring->next_to_clean = 0; 4206 rx_ring->next_to_use = 0; 4207 } 4208 4209 /** 4210 * ixgbe_clean_tx_ring - Free Tx Buffers 4211 * @tx_ring: ring to be cleaned 4212 **/ 4213 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) 4214 { 4215 struct ixgbe_tx_buffer *tx_buffer_info; 4216 unsigned long size; 4217 u16 i; 4218 4219 /* ring already cleared, nothing to do */ 4220 if (!tx_ring->tx_buffer_info) 4221 return; 4222 4223 /* Free all the Tx ring sk_buffs */ 4224 for (i = 0; i < tx_ring->count; i++) { 4225 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 4226 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 4227 } 4228 4229 netdev_tx_reset_queue(txring_txq(tx_ring)); 4230 4231 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 4232 memset(tx_ring->tx_buffer_info, 0, size); 4233 4234 /* Zero out the descriptor ring */ 4235 memset(tx_ring->desc, 0, tx_ring->size); 4236 4237 tx_ring->next_to_use = 0; 4238 tx_ring->next_to_clean = 0; 4239 } 4240 4241 /** 4242 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues 4243 * @adapter: board private structure 4244 **/ 4245 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) 4246 { 4247 int i; 4248 4249 for (i = 0; i < adapter->num_rx_queues; i++) 4250 ixgbe_clean_rx_ring(adapter->rx_ring[i]); 4251 } 4252 4253 /** 4254 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues 4255 * @adapter: board private structure 4256 **/ 4257 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter) 4258 { 4259 int i; 4260 4261 for (i = 0; i < adapter->num_tx_queues; i++) 4262 ixgbe_clean_tx_ring(adapter->tx_ring[i]); 4263 } 4264 4265 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter) 4266 { 4267 struct hlist_node *node, *node2; 4268 struct ixgbe_fdir_filter *filter; 4269 4270 spin_lock(&adapter->fdir_perfect_lock); 4271 4272 hlist_for_each_entry_safe(filter, node, node2, 4273 &adapter->fdir_filter_list, fdir_node) { 4274 hlist_del(&filter->fdir_node); 4275 kfree(filter); 4276 } 4277 adapter->fdir_filter_count = 0; 4278 4279 spin_unlock(&adapter->fdir_perfect_lock); 4280 } 4281 4282 void ixgbe_down(struct ixgbe_adapter *adapter) 4283 { 4284 struct net_device *netdev = adapter->netdev; 4285 struct ixgbe_hw *hw = &adapter->hw; 4286 u32 rxctrl; 4287 int i; 4288 4289 /* signal that we are down to the interrupt handler */ 4290 set_bit(__IXGBE_DOWN, &adapter->state); 4291 4292 /* disable receives */ 4293 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4294 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 4295 4296 /* disable all enabled rx queues */ 4297 for (i = 0; i < adapter->num_rx_queues; i++) 4298 /* this call also flushes the previous write */ 4299 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]); 4300 4301 usleep_range(10000, 20000); 4302 4303 netif_tx_stop_all_queues(netdev); 4304 4305 /* call carrier off first to avoid false dev_watchdog timeouts */ 4306 netif_carrier_off(netdev); 4307 netif_tx_disable(netdev); 4308 4309 ixgbe_irq_disable(adapter); 4310 4311 ixgbe_napi_disable_all(adapter); 4312 4313 adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT | 4314 IXGBE_FLAG2_RESET_REQUESTED); 4315 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4316 4317 del_timer_sync(&adapter->service_timer); 4318 4319 if (adapter->num_vfs) { 4320 /* Clear EITR Select mapping */ 4321 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0); 4322 4323 /* Mark all the VFs as inactive */ 4324 for (i = 0 ; i < adapter->num_vfs; i++) 4325 adapter->vfinfo[i].clear_to_send = false; 4326 4327 /* ping all the active vfs to let them know we are going down */ 4328 ixgbe_ping_all_vfs(adapter); 4329 4330 /* Disable all VFTE/VFRE TX/RX */ 4331 ixgbe_disable_tx_rx(adapter); 4332 } 4333 4334 /* disable transmits in the hardware now that interrupts are off */ 4335 for (i = 0; i < adapter->num_tx_queues; i++) { 4336 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 4337 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 4338 } 4339 4340 /* Disable the Tx DMA engine on 82599 and X540 */ 4341 switch (hw->mac.type) { 4342 case ixgbe_mac_82599EB: 4343 case ixgbe_mac_X540: 4344 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, 4345 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) & 4346 ~IXGBE_DMATXCTL_TE)); 4347 break; 4348 default: 4349 break; 4350 } 4351 4352 if (!pci_channel_offline(adapter->pdev)) 4353 ixgbe_reset(adapter); 4354 4355 /* power down the optics for multispeed fiber and 82599 SFP+ fiber */ 4356 if (hw->mac.ops.disable_tx_laser && 4357 ((hw->phy.multispeed_fiber) || 4358 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && 4359 (hw->mac.type == ixgbe_mac_82599EB)))) 4360 hw->mac.ops.disable_tx_laser(hw); 4361 4362 ixgbe_clean_all_tx_rings(adapter); 4363 ixgbe_clean_all_rx_rings(adapter); 4364 4365 #ifdef CONFIG_IXGBE_DCA 4366 /* since we reset the hardware DCA settings were cleared */ 4367 ixgbe_setup_dca(adapter); 4368 #endif 4369 } 4370 4371 /** 4372 * ixgbe_tx_timeout - Respond to a Tx Hang 4373 * @netdev: network interface device structure 4374 **/ 4375 static void ixgbe_tx_timeout(struct net_device *netdev) 4376 { 4377 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4378 4379 /* Do the reset outside of interrupt context */ 4380 ixgbe_tx_timeout_reset(adapter); 4381 } 4382 4383 /** 4384 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter) 4385 * @adapter: board private structure to initialize 4386 * 4387 * ixgbe_sw_init initializes the Adapter private data structure. 4388 * Fields are initialized based on PCI device information and 4389 * OS network device settings (MTU size). 4390 **/ 4391 static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) 4392 { 4393 struct ixgbe_hw *hw = &adapter->hw; 4394 struct pci_dev *pdev = adapter->pdev; 4395 unsigned int rss; 4396 #ifdef CONFIG_IXGBE_DCB 4397 int j; 4398 struct tc_configuration *tc; 4399 #endif 4400 4401 /* PCI config space info */ 4402 4403 hw->vendor_id = pdev->vendor; 4404 hw->device_id = pdev->device; 4405 hw->revision_id = pdev->revision; 4406 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4407 hw->subsystem_device_id = pdev->subsystem_device; 4408 4409 /* Set capability flags */ 4410 rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus()); 4411 adapter->ring_feature[RING_F_RSS].indices = rss; 4412 adapter->flags |= IXGBE_FLAG_RSS_ENABLED; 4413 switch (hw->mac.type) { 4414 case ixgbe_mac_82598EB: 4415 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4416 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 4417 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598; 4418 break; 4419 case ixgbe_mac_X540: 4420 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 4421 case ixgbe_mac_82599EB: 4422 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599; 4423 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE; 4424 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 4425 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) 4426 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 4427 /* Flow Director hash filters enabled */ 4428 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 4429 adapter->atr_sample_rate = 20; 4430 adapter->ring_feature[RING_F_FDIR].indices = 4431 IXGBE_MAX_FDIR_INDICES; 4432 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K; 4433 #ifdef IXGBE_FCOE 4434 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE; 4435 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 4436 adapter->ring_feature[RING_F_FCOE].indices = 0; 4437 #ifdef CONFIG_IXGBE_DCB 4438 /* Default traffic class to use for FCoE */ 4439 adapter->fcoe.up = IXGBE_FCOE_DEFTC; 4440 #endif 4441 #endif /* IXGBE_FCOE */ 4442 break; 4443 default: 4444 break; 4445 } 4446 4447 /* n-tuple support exists, always init our spinlock */ 4448 spin_lock_init(&adapter->fdir_perfect_lock); 4449 4450 #ifdef CONFIG_IXGBE_DCB 4451 switch (hw->mac.type) { 4452 case ixgbe_mac_X540: 4453 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS; 4454 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS; 4455 break; 4456 default: 4457 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS; 4458 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS; 4459 break; 4460 } 4461 4462 /* Configure DCB traffic classes */ 4463 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) { 4464 tc = &adapter->dcb_cfg.tc_config[j]; 4465 tc->path[DCB_TX_CONFIG].bwg_id = 0; 4466 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1); 4467 tc->path[DCB_RX_CONFIG].bwg_id = 0; 4468 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1); 4469 tc->dcb_pfc = pfc_disabled; 4470 } 4471 4472 /* Initialize default user to priority mapping, UPx->TC0 */ 4473 tc = &adapter->dcb_cfg.tc_config[0]; 4474 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 4475 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 4476 4477 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100; 4478 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100; 4479 adapter->dcb_cfg.pfc_mode_enable = false; 4480 adapter->dcb_set_bitmap = 0x00; 4481 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; 4482 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, 4483 sizeof(adapter->temp_dcb_cfg)); 4484 4485 #endif 4486 4487 /* default flow control settings */ 4488 hw->fc.requested_mode = ixgbe_fc_full; 4489 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */ 4490 ixgbe_pbthresh_setup(adapter); 4491 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE; 4492 hw->fc.send_xon = true; 4493 hw->fc.disable_fc_autoneg = false; 4494 4495 /* enable itr by default in dynamic mode */ 4496 adapter->rx_itr_setting = 1; 4497 adapter->tx_itr_setting = 1; 4498 4499 /* set default ring sizes */ 4500 adapter->tx_ring_count = IXGBE_DEFAULT_TXD; 4501 adapter->rx_ring_count = IXGBE_DEFAULT_RXD; 4502 4503 /* set default work limits */ 4504 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK; 4505 4506 /* initialize eeprom parameters */ 4507 if (ixgbe_init_eeprom_params_generic(hw)) { 4508 e_dev_err("EEPROM initialization failed\n"); 4509 return -EIO; 4510 } 4511 4512 set_bit(__IXGBE_DOWN, &adapter->state); 4513 4514 return 0; 4515 } 4516 4517 /** 4518 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors) 4519 * @tx_ring: tx descriptor ring (for a specific queue) to setup 4520 * 4521 * Return 0 on success, negative on failure 4522 **/ 4523 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) 4524 { 4525 struct device *dev = tx_ring->dev; 4526 int orig_node = dev_to_node(dev); 4527 int numa_node = -1; 4528 int size; 4529 4530 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 4531 4532 if (tx_ring->q_vector) 4533 numa_node = tx_ring->q_vector->numa_node; 4534 4535 tx_ring->tx_buffer_info = vzalloc_node(size, numa_node); 4536 if (!tx_ring->tx_buffer_info) 4537 tx_ring->tx_buffer_info = vzalloc(size); 4538 if (!tx_ring->tx_buffer_info) 4539 goto err; 4540 4541 /* round up to nearest 4K */ 4542 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 4543 tx_ring->size = ALIGN(tx_ring->size, 4096); 4544 4545 set_dev_node(dev, numa_node); 4546 tx_ring->desc = dma_alloc_coherent(dev, 4547 tx_ring->size, 4548 &tx_ring->dma, 4549 GFP_KERNEL); 4550 set_dev_node(dev, orig_node); 4551 if (!tx_ring->desc) 4552 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 4553 &tx_ring->dma, GFP_KERNEL); 4554 if (!tx_ring->desc) 4555 goto err; 4556 4557 tx_ring->next_to_use = 0; 4558 tx_ring->next_to_clean = 0; 4559 return 0; 4560 4561 err: 4562 vfree(tx_ring->tx_buffer_info); 4563 tx_ring->tx_buffer_info = NULL; 4564 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); 4565 return -ENOMEM; 4566 } 4567 4568 /** 4569 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources 4570 * @adapter: board private structure 4571 * 4572 * If this function returns with an error, then it's possible one or 4573 * more of the rings is populated (while the rest are not). It is the 4574 * callers duty to clean those orphaned rings. 4575 * 4576 * Return 0 on success, negative on failure 4577 **/ 4578 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) 4579 { 4580 int i, err = 0; 4581 4582 for (i = 0; i < adapter->num_tx_queues; i++) { 4583 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]); 4584 if (!err) 4585 continue; 4586 e_err(probe, "Allocation for Tx Queue %u failed\n", i); 4587 break; 4588 } 4589 4590 return err; 4591 } 4592 4593 /** 4594 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) 4595 * @rx_ring: rx descriptor ring (for a specific queue) to setup 4596 * 4597 * Returns 0 on success, negative on failure 4598 **/ 4599 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) 4600 { 4601 struct device *dev = rx_ring->dev; 4602 int orig_node = dev_to_node(dev); 4603 int numa_node = -1; 4604 int size; 4605 4606 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4607 4608 if (rx_ring->q_vector) 4609 numa_node = rx_ring->q_vector->numa_node; 4610 4611 rx_ring->rx_buffer_info = vzalloc_node(size, numa_node); 4612 if (!rx_ring->rx_buffer_info) 4613 rx_ring->rx_buffer_info = vzalloc(size); 4614 if (!rx_ring->rx_buffer_info) 4615 goto err; 4616 4617 /* Round up to nearest 4K */ 4618 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 4619 rx_ring->size = ALIGN(rx_ring->size, 4096); 4620 4621 set_dev_node(dev, numa_node); 4622 rx_ring->desc = dma_alloc_coherent(dev, 4623 rx_ring->size, 4624 &rx_ring->dma, 4625 GFP_KERNEL); 4626 set_dev_node(dev, orig_node); 4627 if (!rx_ring->desc) 4628 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 4629 &rx_ring->dma, GFP_KERNEL); 4630 if (!rx_ring->desc) 4631 goto err; 4632 4633 rx_ring->next_to_clean = 0; 4634 rx_ring->next_to_use = 0; 4635 4636 ixgbe_init_rx_page_offset(rx_ring); 4637 4638 return 0; 4639 err: 4640 vfree(rx_ring->rx_buffer_info); 4641 rx_ring->rx_buffer_info = NULL; 4642 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); 4643 return -ENOMEM; 4644 } 4645 4646 /** 4647 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources 4648 * @adapter: board private structure 4649 * 4650 * If this function returns with an error, then it's possible one or 4651 * more of the rings is populated (while the rest are not). It is the 4652 * callers duty to clean those orphaned rings. 4653 * 4654 * Return 0 on success, negative on failure 4655 **/ 4656 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) 4657 { 4658 int i, err = 0; 4659 4660 for (i = 0; i < adapter->num_rx_queues; i++) { 4661 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]); 4662 if (!err) 4663 continue; 4664 e_err(probe, "Allocation for Rx Queue %u failed\n", i); 4665 break; 4666 } 4667 4668 return err; 4669 } 4670 4671 /** 4672 * ixgbe_free_tx_resources - Free Tx Resources per Queue 4673 * @tx_ring: Tx descriptor ring for a specific queue 4674 * 4675 * Free all transmit software resources 4676 **/ 4677 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring) 4678 { 4679 ixgbe_clean_tx_ring(tx_ring); 4680 4681 vfree(tx_ring->tx_buffer_info); 4682 tx_ring->tx_buffer_info = NULL; 4683 4684 /* if not set, then don't free */ 4685 if (!tx_ring->desc) 4686 return; 4687 4688 dma_free_coherent(tx_ring->dev, tx_ring->size, 4689 tx_ring->desc, tx_ring->dma); 4690 4691 tx_ring->desc = NULL; 4692 } 4693 4694 /** 4695 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues 4696 * @adapter: board private structure 4697 * 4698 * Free all transmit software resources 4699 **/ 4700 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) 4701 { 4702 int i; 4703 4704 for (i = 0; i < adapter->num_tx_queues; i++) 4705 if (adapter->tx_ring[i]->desc) 4706 ixgbe_free_tx_resources(adapter->tx_ring[i]); 4707 } 4708 4709 /** 4710 * ixgbe_free_rx_resources - Free Rx Resources 4711 * @rx_ring: ring to clean the resources from 4712 * 4713 * Free all receive software resources 4714 **/ 4715 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) 4716 { 4717 ixgbe_clean_rx_ring(rx_ring); 4718 4719 vfree(rx_ring->rx_buffer_info); 4720 rx_ring->rx_buffer_info = NULL; 4721 4722 /* if not set, then don't free */ 4723 if (!rx_ring->desc) 4724 return; 4725 4726 dma_free_coherent(rx_ring->dev, rx_ring->size, 4727 rx_ring->desc, rx_ring->dma); 4728 4729 rx_ring->desc = NULL; 4730 } 4731 4732 /** 4733 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues 4734 * @adapter: board private structure 4735 * 4736 * Free all receive software resources 4737 **/ 4738 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) 4739 { 4740 int i; 4741 4742 for (i = 0; i < adapter->num_rx_queues; i++) 4743 if (adapter->rx_ring[i]->desc) 4744 ixgbe_free_rx_resources(adapter->rx_ring[i]); 4745 } 4746 4747 /** 4748 * ixgbe_change_mtu - Change the Maximum Transfer Unit 4749 * @netdev: network interface device structure 4750 * @new_mtu: new value for maximum frame size 4751 * 4752 * Returns 0 on success, negative on failure 4753 **/ 4754 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) 4755 { 4756 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4757 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 4758 4759 /* MTU < 68 is an error and causes problems on some kernels */ 4760 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE)) 4761 return -EINVAL; 4762 4763 /* 4764 * For 82599EB we cannot allow PF to change MTU greater than 1500 4765 * in SR-IOV mode as it may cause buffer overruns in guest VFs that 4766 * don't allocate and chain buffers correctly. 4767 */ 4768 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && 4769 (adapter->hw.mac.type == ixgbe_mac_82599EB) && 4770 (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE)) 4771 return -EINVAL; 4772 4773 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 4774 4775 /* must set new MTU before calling down or up */ 4776 netdev->mtu = new_mtu; 4777 4778 if (netif_running(netdev)) 4779 ixgbe_reinit_locked(adapter); 4780 4781 return 0; 4782 } 4783 4784 /** 4785 * ixgbe_open - Called when a network interface is made active 4786 * @netdev: network interface device structure 4787 * 4788 * Returns 0 on success, negative value on failure 4789 * 4790 * The open entry point is called when a network interface is made 4791 * active by the system (IFF_UP). At this point all resources needed 4792 * for transmit and receive operations are allocated, the interrupt 4793 * handler is registered with the OS, the watchdog timer is started, 4794 * and the stack is notified that the interface is ready. 4795 **/ 4796 static int ixgbe_open(struct net_device *netdev) 4797 { 4798 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4799 int err; 4800 4801 /* disallow open during test */ 4802 if (test_bit(__IXGBE_TESTING, &adapter->state)) 4803 return -EBUSY; 4804 4805 netif_carrier_off(netdev); 4806 4807 /* allocate transmit descriptors */ 4808 err = ixgbe_setup_all_tx_resources(adapter); 4809 if (err) 4810 goto err_setup_tx; 4811 4812 /* allocate receive descriptors */ 4813 err = ixgbe_setup_all_rx_resources(adapter); 4814 if (err) 4815 goto err_setup_rx; 4816 4817 ixgbe_configure(adapter); 4818 4819 err = ixgbe_request_irq(adapter); 4820 if (err) 4821 goto err_req_irq; 4822 4823 ixgbe_up_complete(adapter); 4824 4825 return 0; 4826 4827 err_req_irq: 4828 err_setup_rx: 4829 ixgbe_free_all_rx_resources(adapter); 4830 err_setup_tx: 4831 ixgbe_free_all_tx_resources(adapter); 4832 ixgbe_reset(adapter); 4833 4834 return err; 4835 } 4836 4837 /** 4838 * ixgbe_close - Disables a network interface 4839 * @netdev: network interface device structure 4840 * 4841 * Returns 0, this is not allowed to fail 4842 * 4843 * The close entry point is called when an interface is de-activated 4844 * by the OS. The hardware is still under the drivers control, but 4845 * needs to be disabled. A global MAC reset is issued to stop the 4846 * hardware, and all transmit and receive resources are freed. 4847 **/ 4848 static int ixgbe_close(struct net_device *netdev) 4849 { 4850 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4851 4852 ixgbe_down(adapter); 4853 ixgbe_free_irq(adapter); 4854 4855 ixgbe_fdir_filter_exit(adapter); 4856 4857 ixgbe_free_all_tx_resources(adapter); 4858 ixgbe_free_all_rx_resources(adapter); 4859 4860 ixgbe_release_hw_control(adapter); 4861 4862 return 0; 4863 } 4864 4865 #ifdef CONFIG_PM 4866 static int ixgbe_resume(struct pci_dev *pdev) 4867 { 4868 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 4869 struct net_device *netdev = adapter->netdev; 4870 u32 err; 4871 4872 pci_set_power_state(pdev, PCI_D0); 4873 pci_restore_state(pdev); 4874 /* 4875 * pci_restore_state clears dev->state_saved so call 4876 * pci_save_state to restore it. 4877 */ 4878 pci_save_state(pdev); 4879 4880 err = pci_enable_device_mem(pdev); 4881 if (err) { 4882 e_dev_err("Cannot enable PCI device from suspend\n"); 4883 return err; 4884 } 4885 pci_set_master(pdev); 4886 4887 pci_wake_from_d3(pdev, false); 4888 4889 rtnl_lock(); 4890 err = ixgbe_init_interrupt_scheme(adapter); 4891 rtnl_unlock(); 4892 if (err) { 4893 e_dev_err("Cannot initialize interrupts for device\n"); 4894 return err; 4895 } 4896 4897 ixgbe_reset(adapter); 4898 4899 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 4900 4901 if (netif_running(netdev)) { 4902 err = ixgbe_open(netdev); 4903 if (err) 4904 return err; 4905 } 4906 4907 netif_device_attach(netdev); 4908 4909 return 0; 4910 } 4911 #endif /* CONFIG_PM */ 4912 4913 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) 4914 { 4915 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 4916 struct net_device *netdev = adapter->netdev; 4917 struct ixgbe_hw *hw = &adapter->hw; 4918 u32 ctrl, fctrl; 4919 u32 wufc = adapter->wol; 4920 #ifdef CONFIG_PM 4921 int retval = 0; 4922 #endif 4923 4924 netif_device_detach(netdev); 4925 4926 if (netif_running(netdev)) { 4927 rtnl_lock(); 4928 ixgbe_down(adapter); 4929 ixgbe_free_irq(adapter); 4930 ixgbe_free_all_tx_resources(adapter); 4931 ixgbe_free_all_rx_resources(adapter); 4932 rtnl_unlock(); 4933 } 4934 4935 ixgbe_clear_interrupt_scheme(adapter); 4936 4937 #ifdef CONFIG_PM 4938 retval = pci_save_state(pdev); 4939 if (retval) 4940 return retval; 4941 4942 #endif 4943 if (wufc) { 4944 ixgbe_set_rx_mode(netdev); 4945 4946 /* 4947 * enable the optics for both mult-speed fiber and 4948 * 82599 SFP+ fiber as we can WoL. 4949 */ 4950 if (hw->mac.ops.enable_tx_laser && 4951 (hw->phy.multispeed_fiber || 4952 (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber && 4953 hw->mac.type == ixgbe_mac_82599EB))) 4954 hw->mac.ops.enable_tx_laser(hw); 4955 4956 /* turn on all-multi mode if wake on multicast is enabled */ 4957 if (wufc & IXGBE_WUFC_MC) { 4958 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4959 fctrl |= IXGBE_FCTRL_MPE; 4960 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4961 } 4962 4963 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL); 4964 ctrl |= IXGBE_CTRL_GIO_DIS; 4965 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); 4966 4967 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc); 4968 } else { 4969 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 4970 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 4971 } 4972 4973 switch (hw->mac.type) { 4974 case ixgbe_mac_82598EB: 4975 pci_wake_from_d3(pdev, false); 4976 break; 4977 case ixgbe_mac_82599EB: 4978 case ixgbe_mac_X540: 4979 pci_wake_from_d3(pdev, !!wufc); 4980 break; 4981 default: 4982 break; 4983 } 4984 4985 *enable_wake = !!wufc; 4986 4987 ixgbe_release_hw_control(adapter); 4988 4989 pci_disable_device(pdev); 4990 4991 return 0; 4992 } 4993 4994 #ifdef CONFIG_PM 4995 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state) 4996 { 4997 int retval; 4998 bool wake; 4999 5000 retval = __ixgbe_shutdown(pdev, &wake); 5001 if (retval) 5002 return retval; 5003 5004 if (wake) { 5005 pci_prepare_to_sleep(pdev); 5006 } else { 5007 pci_wake_from_d3(pdev, false); 5008 pci_set_power_state(pdev, PCI_D3hot); 5009 } 5010 5011 return 0; 5012 } 5013 #endif /* CONFIG_PM */ 5014 5015 static void ixgbe_shutdown(struct pci_dev *pdev) 5016 { 5017 bool wake; 5018 5019 __ixgbe_shutdown(pdev, &wake); 5020 5021 if (system_state == SYSTEM_POWER_OFF) { 5022 pci_wake_from_d3(pdev, wake); 5023 pci_set_power_state(pdev, PCI_D3hot); 5024 } 5025 } 5026 5027 /** 5028 * ixgbe_update_stats - Update the board statistics counters. 5029 * @adapter: board private structure 5030 **/ 5031 void ixgbe_update_stats(struct ixgbe_adapter *adapter) 5032 { 5033 struct net_device *netdev = adapter->netdev; 5034 struct ixgbe_hw *hw = &adapter->hw; 5035 struct ixgbe_hw_stats *hwstats = &adapter->stats; 5036 u64 total_mpc = 0; 5037 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; 5038 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0; 5039 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 5040 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; 5041 #ifdef IXGBE_FCOE 5042 struct ixgbe_fcoe *fcoe = &adapter->fcoe; 5043 unsigned int cpu; 5044 u64 fcoe_noddp_counts_sum = 0, fcoe_noddp_ext_buff_counts_sum = 0; 5045 #endif /* IXGBE_FCOE */ 5046 5047 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5048 test_bit(__IXGBE_RESETTING, &adapter->state)) 5049 return; 5050 5051 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { 5052 u64 rsc_count = 0; 5053 u64 rsc_flush = 0; 5054 for (i = 0; i < adapter->num_rx_queues; i++) { 5055 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count; 5056 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush; 5057 } 5058 adapter->rsc_total_count = rsc_count; 5059 adapter->rsc_total_flush = rsc_flush; 5060 } 5061 5062 for (i = 0; i < adapter->num_rx_queues; i++) { 5063 struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; 5064 non_eop_descs += rx_ring->rx_stats.non_eop_descs; 5065 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 5066 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 5067 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 5068 bytes += rx_ring->stats.bytes; 5069 packets += rx_ring->stats.packets; 5070 } 5071 adapter->non_eop_descs = non_eop_descs; 5072 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 5073 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 5074 adapter->hw_csum_rx_error = hw_csum_rx_error; 5075 netdev->stats.rx_bytes = bytes; 5076 netdev->stats.rx_packets = packets; 5077 5078 bytes = 0; 5079 packets = 0; 5080 /* gather some stats to the adapter struct that are per queue */ 5081 for (i = 0; i < adapter->num_tx_queues; i++) { 5082 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 5083 restart_queue += tx_ring->tx_stats.restart_queue; 5084 tx_busy += tx_ring->tx_stats.tx_busy; 5085 bytes += tx_ring->stats.bytes; 5086 packets += tx_ring->stats.packets; 5087 } 5088 adapter->restart_queue = restart_queue; 5089 adapter->tx_busy = tx_busy; 5090 netdev->stats.tx_bytes = bytes; 5091 netdev->stats.tx_packets = packets; 5092 5093 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5094 5095 /* 8 register reads */ 5096 for (i = 0; i < 8; i++) { 5097 /* for packet buffers not used, the register should read 0 */ 5098 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5099 missed_rx += mpc; 5100 hwstats->mpc[i] += mpc; 5101 total_mpc += hwstats->mpc[i]; 5102 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5103 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5104 switch (hw->mac.type) { 5105 case ixgbe_mac_82598EB: 5106 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5107 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5108 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5109 hwstats->pxonrxc[i] += 5110 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5111 break; 5112 case ixgbe_mac_82599EB: 5113 case ixgbe_mac_X540: 5114 hwstats->pxonrxc[i] += 5115 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 5116 break; 5117 default: 5118 break; 5119 } 5120 } 5121 5122 /*16 register reads */ 5123 for (i = 0; i < 16; i++) { 5124 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5125 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5126 if ((hw->mac.type == ixgbe_mac_82599EB) || 5127 (hw->mac.type == ixgbe_mac_X540)) { 5128 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 5129 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ 5130 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 5131 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */ 5132 } 5133 } 5134 5135 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 5136 /* work around hardware counting issue */ 5137 hwstats->gprc -= missed_rx; 5138 5139 ixgbe_update_xoff_received(adapter); 5140 5141 /* 82598 hardware only has a 32 bit counter in the high register */ 5142 switch (hw->mac.type) { 5143 case ixgbe_mac_82598EB: 5144 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5145 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 5146 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 5147 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 5148 break; 5149 case ixgbe_mac_X540: 5150 /* OS2BMC stats are X540 only*/ 5151 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC); 5152 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC); 5153 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC); 5154 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC); 5155 case ixgbe_mac_82599EB: 5156 for (i = 0; i < 16; i++) 5157 adapter->hw_rx_no_dma_resources += 5158 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5159 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 5160 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ 5161 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 5162 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */ 5163 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 5164 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */ 5165 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5166 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 5167 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 5168 #ifdef IXGBE_FCOE 5169 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5170 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5171 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5172 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5173 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5174 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5175 /* Add up per cpu counters for total ddp aloc fail */ 5176 if (fcoe->pcpu_noddp && fcoe->pcpu_noddp_ext_buff) { 5177 for_each_possible_cpu(cpu) { 5178 fcoe_noddp_counts_sum += 5179 *per_cpu_ptr(fcoe->pcpu_noddp, cpu); 5180 fcoe_noddp_ext_buff_counts_sum += 5181 *per_cpu_ptr(fcoe-> 5182 pcpu_noddp_ext_buff, cpu); 5183 } 5184 } 5185 hwstats->fcoe_noddp = fcoe_noddp_counts_sum; 5186 hwstats->fcoe_noddp_ext_buff = fcoe_noddp_ext_buff_counts_sum; 5187 #endif /* IXGBE_FCOE */ 5188 break; 5189 default: 5190 break; 5191 } 5192 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5193 hwstats->bprc += bprc; 5194 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 5195 if (hw->mac.type == ixgbe_mac_82598EB) 5196 hwstats->mprc -= bprc; 5197 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 5198 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 5199 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 5200 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 5201 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 5202 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5203 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5204 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 5205 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5206 hwstats->lxontxc += lxon; 5207 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5208 hwstats->lxofftxc += lxoff; 5209 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 5210 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 5211 /* 5212 * 82598 errata - tx of flow control packets is included in tx counters 5213 */ 5214 xon_off_tot = lxon + lxoff; 5215 hwstats->gptc -= xon_off_tot; 5216 hwstats->mptc -= xon_off_tot; 5217 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN)); 5218 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 5219 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 5220 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 5221 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 5222 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 5223 hwstats->ptc64 -= xon_off_tot; 5224 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 5225 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 5226 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 5227 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5228 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5229 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 5230 5231 /* Fill out the OS statistics structure */ 5232 netdev->stats.multicast = hwstats->mprc; 5233 5234 /* Rx Errors */ 5235 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec; 5236 netdev->stats.rx_dropped = 0; 5237 netdev->stats.rx_length_errors = hwstats->rlec; 5238 netdev->stats.rx_crc_errors = hwstats->crcerrs; 5239 netdev->stats.rx_missed_errors = total_mpc; 5240 } 5241 5242 /** 5243 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table 5244 * @adapter - pointer to the device adapter structure 5245 **/ 5246 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) 5247 { 5248 struct ixgbe_hw *hw = &adapter->hw; 5249 int i; 5250 5251 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 5252 return; 5253 5254 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 5255 5256 /* if interface is down do nothing */ 5257 if (test_bit(__IXGBE_DOWN, &adapter->state)) 5258 return; 5259 5260 /* do nothing if we are not using signature filters */ 5261 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) 5262 return; 5263 5264 adapter->fdir_overflow++; 5265 5266 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) { 5267 for (i = 0; i < adapter->num_tx_queues; i++) 5268 set_bit(__IXGBE_TX_FDIR_INIT_DONE, 5269 &(adapter->tx_ring[i]->state)); 5270 /* re-enable flow director interrupts */ 5271 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); 5272 } else { 5273 e_err(probe, "failed to finish FDIR re-initialization, " 5274 "ignored adding FDIR ATR filters\n"); 5275 } 5276 } 5277 5278 /** 5279 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts 5280 * @adapter - pointer to the device adapter structure 5281 * 5282 * This function serves two purposes. First it strobes the interrupt lines 5283 * in order to make certain interrupts are occurring. Secondly it sets the 5284 * bits needed to check for TX hangs. As a result we should immediately 5285 * determine if a hang has occurred. 5286 */ 5287 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) 5288 { 5289 struct ixgbe_hw *hw = &adapter->hw; 5290 u64 eics = 0; 5291 int i; 5292 5293 /* If we're down or resetting, just bail */ 5294 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5295 test_bit(__IXGBE_RESETTING, &adapter->state)) 5296 return; 5297 5298 /* Force detection of hung controller */ 5299 if (netif_carrier_ok(adapter->netdev)) { 5300 for (i = 0; i < adapter->num_tx_queues; i++) 5301 set_check_for_tx_hang(adapter->tx_ring[i]); 5302 } 5303 5304 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 5305 /* 5306 * for legacy and MSI interrupts don't set any bits 5307 * that are enabled for EIAM, because this operation 5308 * would set *both* EIMS and EICS for any bit in EIAM 5309 */ 5310 IXGBE_WRITE_REG(hw, IXGBE_EICS, 5311 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER)); 5312 } else { 5313 /* get one bit for every active tx/rx interrupt vector */ 5314 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 5315 struct ixgbe_q_vector *qv = adapter->q_vector[i]; 5316 if (qv->rx.ring || qv->tx.ring) 5317 eics |= ((u64)1 << i); 5318 } 5319 } 5320 5321 /* Cause software interrupt to ensure rings are cleaned */ 5322 ixgbe_irq_rearm_queues(adapter, eics); 5323 5324 } 5325 5326 /** 5327 * ixgbe_watchdog_update_link - update the link status 5328 * @adapter - pointer to the device adapter structure 5329 * @link_speed - pointer to a u32 to store the link_speed 5330 **/ 5331 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) 5332 { 5333 struct ixgbe_hw *hw = &adapter->hw; 5334 u32 link_speed = adapter->link_speed; 5335 bool link_up = adapter->link_up; 5336 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 5337 5338 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 5339 return; 5340 5341 if (hw->mac.ops.check_link) { 5342 hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 5343 } else { 5344 /* always assume link is up, if no check link function */ 5345 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 5346 link_up = true; 5347 } 5348 5349 if (adapter->ixgbe_ieee_pfc) 5350 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 5351 5352 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) { 5353 hw->mac.ops.fc_enable(hw); 5354 ixgbe_set_rx_drop_en(adapter); 5355 } 5356 5357 if (link_up || 5358 time_after(jiffies, (adapter->link_check_timeout + 5359 IXGBE_TRY_LINK_TIMEOUT))) { 5360 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 5361 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); 5362 IXGBE_WRITE_FLUSH(hw); 5363 } 5364 5365 adapter->link_up = link_up; 5366 adapter->link_speed = link_speed; 5367 } 5368 5369 /** 5370 * ixgbe_watchdog_link_is_up - update netif_carrier status and 5371 * print link up message 5372 * @adapter - pointer to the device adapter structure 5373 **/ 5374 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) 5375 { 5376 struct net_device *netdev = adapter->netdev; 5377 struct ixgbe_hw *hw = &adapter->hw; 5378 u32 link_speed = adapter->link_speed; 5379 bool flow_rx, flow_tx; 5380 5381 /* only continue if link was previously down */ 5382 if (netif_carrier_ok(netdev)) 5383 return; 5384 5385 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 5386 5387 switch (hw->mac.type) { 5388 case ixgbe_mac_82598EB: { 5389 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5390 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS); 5391 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE); 5392 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X); 5393 } 5394 break; 5395 case ixgbe_mac_X540: 5396 case ixgbe_mac_82599EB: { 5397 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 5398 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 5399 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE); 5400 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X); 5401 } 5402 break; 5403 default: 5404 flow_tx = false; 5405 flow_rx = false; 5406 break; 5407 } 5408 5409 #ifdef CONFIG_IXGBE_PTP 5410 ixgbe_ptp_start_cyclecounter(adapter); 5411 #endif 5412 5413 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", 5414 (link_speed == IXGBE_LINK_SPEED_10GB_FULL ? 5415 "10 Gbps" : 5416 (link_speed == IXGBE_LINK_SPEED_1GB_FULL ? 5417 "1 Gbps" : 5418 (link_speed == IXGBE_LINK_SPEED_100_FULL ? 5419 "100 Mbps" : 5420 "unknown speed"))), 5421 ((flow_rx && flow_tx) ? "RX/TX" : 5422 (flow_rx ? "RX" : 5423 (flow_tx ? "TX" : "None")))); 5424 5425 netif_carrier_on(netdev); 5426 ixgbe_check_vf_rate_limit(adapter); 5427 } 5428 5429 /** 5430 * ixgbe_watchdog_link_is_down - update netif_carrier status and 5431 * print link down message 5432 * @adapter - pointer to the adapter structure 5433 **/ 5434 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter) 5435 { 5436 struct net_device *netdev = adapter->netdev; 5437 struct ixgbe_hw *hw = &adapter->hw; 5438 5439 adapter->link_up = false; 5440 adapter->link_speed = 0; 5441 5442 /* only continue if link was up previously */ 5443 if (!netif_carrier_ok(netdev)) 5444 return; 5445 5446 /* poll for SFP+ cable when link is down */ 5447 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB) 5448 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 5449 5450 #ifdef CONFIG_IXGBE_PTP 5451 ixgbe_ptp_start_cyclecounter(adapter); 5452 #endif 5453 5454 e_info(drv, "NIC Link is Down\n"); 5455 netif_carrier_off(netdev); 5456 } 5457 5458 /** 5459 * ixgbe_watchdog_flush_tx - flush queues on link down 5460 * @adapter - pointer to the device adapter structure 5461 **/ 5462 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) 5463 { 5464 int i; 5465 int some_tx_pending = 0; 5466 5467 if (!netif_carrier_ok(adapter->netdev)) { 5468 for (i = 0; i < adapter->num_tx_queues; i++) { 5469 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 5470 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 5471 some_tx_pending = 1; 5472 break; 5473 } 5474 } 5475 5476 if (some_tx_pending) { 5477 /* We've lost link, so the controller stops DMA, 5478 * but we've got queued Tx work that's never going 5479 * to get done, so reset controller to flush Tx. 5480 * (Do the reset outside of interrupt context). 5481 */ 5482 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; 5483 } 5484 } 5485 } 5486 5487 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) 5488 { 5489 u32 ssvpc; 5490 5491 /* Do not perform spoof check for 82598 */ 5492 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 5493 return; 5494 5495 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); 5496 5497 /* 5498 * ssvpc register is cleared on read, if zero then no 5499 * spoofed packets in the last interval. 5500 */ 5501 if (!ssvpc) 5502 return; 5503 5504 e_warn(drv, "%d Spoofed packets detected\n", ssvpc); 5505 } 5506 5507 /** 5508 * ixgbe_watchdog_subtask - check and bring link up 5509 * @adapter - pointer to the device adapter structure 5510 **/ 5511 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) 5512 { 5513 /* if interface is down do nothing */ 5514 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5515 test_bit(__IXGBE_RESETTING, &adapter->state)) 5516 return; 5517 5518 ixgbe_watchdog_update_link(adapter); 5519 5520 if (adapter->link_up) 5521 ixgbe_watchdog_link_is_up(adapter); 5522 else 5523 ixgbe_watchdog_link_is_down(adapter); 5524 5525 ixgbe_spoof_check(adapter); 5526 ixgbe_update_stats(adapter); 5527 5528 ixgbe_watchdog_flush_tx(adapter); 5529 } 5530 5531 /** 5532 * ixgbe_sfp_detection_subtask - poll for SFP+ cable 5533 * @adapter - the ixgbe adapter structure 5534 **/ 5535 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) 5536 { 5537 struct ixgbe_hw *hw = &adapter->hw; 5538 s32 err; 5539 5540 /* not searching for SFP so there is nothing to do here */ 5541 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && 5542 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 5543 return; 5544 5545 /* someone else is in init, wait until next service event */ 5546 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5547 return; 5548 5549 err = hw->phy.ops.identify_sfp(hw); 5550 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 5551 goto sfp_out; 5552 5553 if (err == IXGBE_ERR_SFP_NOT_PRESENT) { 5554 /* If no cable is present, then we need to reset 5555 * the next time we find a good cable. */ 5556 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 5557 } 5558 5559 /* exit on error */ 5560 if (err) 5561 goto sfp_out; 5562 5563 /* exit if reset not needed */ 5564 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 5565 goto sfp_out; 5566 5567 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET; 5568 5569 /* 5570 * A module may be identified correctly, but the EEPROM may not have 5571 * support for that module. setup_sfp() will fail in that case, so 5572 * we should not allow that module to load. 5573 */ 5574 if (hw->mac.type == ixgbe_mac_82598EB) 5575 err = hw->phy.ops.reset(hw); 5576 else 5577 err = hw->mac.ops.setup_sfp(hw); 5578 5579 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 5580 goto sfp_out; 5581 5582 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 5583 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type); 5584 5585 sfp_out: 5586 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 5587 5588 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) && 5589 (adapter->netdev->reg_state == NETREG_REGISTERED)) { 5590 e_dev_err("failed to initialize because an unsupported " 5591 "SFP+ module type was detected.\n"); 5592 e_dev_err("Reload the driver after installing a " 5593 "supported module.\n"); 5594 unregister_netdev(adapter->netdev); 5595 } 5596 } 5597 5598 /** 5599 * ixgbe_sfp_link_config_subtask - set up link SFP after module install 5600 * @adapter - the ixgbe adapter structure 5601 **/ 5602 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) 5603 { 5604 struct ixgbe_hw *hw = &adapter->hw; 5605 u32 autoneg; 5606 bool negotiation; 5607 5608 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG)) 5609 return; 5610 5611 /* someone else is in init, wait until next service event */ 5612 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5613 return; 5614 5615 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 5616 5617 autoneg = hw->phy.autoneg_advertised; 5618 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5619 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation); 5620 if (hw->mac.ops.setup_link) 5621 hw->mac.ops.setup_link(hw, autoneg, negotiation, true); 5622 5623 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 5624 adapter->link_check_timeout = jiffies; 5625 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 5626 } 5627 5628 #ifdef CONFIG_PCI_IOV 5629 static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) 5630 { 5631 int vf; 5632 struct ixgbe_hw *hw = &adapter->hw; 5633 struct net_device *netdev = adapter->netdev; 5634 u32 gpc; 5635 u32 ciaa, ciad; 5636 5637 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC); 5638 if (gpc) /* If incrementing then no need for the check below */ 5639 return; 5640 /* 5641 * Check to see if a bad DMA write target from an errant or 5642 * malicious VF has caused a PCIe error. If so then we can 5643 * issue a VFLR to the offending VF(s) and then resume without 5644 * requesting a full slot reset. 5645 */ 5646 5647 for (vf = 0; vf < adapter->num_vfs; vf++) { 5648 ciaa = (vf << 16) | 0x80000000; 5649 /* 32 bit read so align, we really want status at offset 6 */ 5650 ciaa |= PCI_COMMAND; 5651 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5652 ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599); 5653 ciaa &= 0x7FFFFFFF; 5654 /* disable debug mode asap after reading data */ 5655 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5656 /* Get the upper 16 bits which will be the PCI status reg */ 5657 ciad >>= 16; 5658 if (ciad & PCI_STATUS_REC_MASTER_ABORT) { 5659 netdev_err(netdev, "VF %d Hung DMA\n", vf); 5660 /* Issue VFLR */ 5661 ciaa = (vf << 16) | 0x80000000; 5662 ciaa |= 0xA8; 5663 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5664 ciad = 0x00008000; /* VFLR */ 5665 IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad); 5666 ciaa &= 0x7FFFFFFF; 5667 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5668 } 5669 } 5670 } 5671 5672 #endif 5673 /** 5674 * ixgbe_service_timer - Timer Call-back 5675 * @data: pointer to adapter cast into an unsigned long 5676 **/ 5677 static void ixgbe_service_timer(unsigned long data) 5678 { 5679 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data; 5680 unsigned long next_event_offset; 5681 bool ready = true; 5682 5683 /* poll faster when waiting for link */ 5684 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 5685 next_event_offset = HZ / 10; 5686 else 5687 next_event_offset = HZ * 2; 5688 5689 #ifdef CONFIG_PCI_IOV 5690 /* 5691 * don't bother with SR-IOV VF DMA hang check if there are 5692 * no VFs or the link is down 5693 */ 5694 if (!adapter->num_vfs || 5695 (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 5696 goto normal_timer_service; 5697 5698 /* If we have VFs allocated then we must check for DMA hangs */ 5699 ixgbe_check_for_bad_vf(adapter); 5700 next_event_offset = HZ / 50; 5701 adapter->timer_event_accumulator++; 5702 5703 if (adapter->timer_event_accumulator >= 100) 5704 adapter->timer_event_accumulator = 0; 5705 else 5706 ready = false; 5707 5708 normal_timer_service: 5709 #endif 5710 /* Reset the timer */ 5711 mod_timer(&adapter->service_timer, next_event_offset + jiffies); 5712 5713 if (ready) 5714 ixgbe_service_event_schedule(adapter); 5715 } 5716 5717 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) 5718 { 5719 if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED)) 5720 return; 5721 5722 adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED; 5723 5724 /* If we're already down or resetting, just bail */ 5725 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5726 test_bit(__IXGBE_RESETTING, &adapter->state)) 5727 return; 5728 5729 ixgbe_dump(adapter); 5730 netdev_err(adapter->netdev, "Reset adapter\n"); 5731 adapter->tx_timeout_count++; 5732 5733 ixgbe_reinit_locked(adapter); 5734 } 5735 5736 /** 5737 * ixgbe_service_task - manages and runs subtasks 5738 * @work: pointer to work_struct containing our data 5739 **/ 5740 static void ixgbe_service_task(struct work_struct *work) 5741 { 5742 struct ixgbe_adapter *adapter = container_of(work, 5743 struct ixgbe_adapter, 5744 service_task); 5745 5746 ixgbe_reset_subtask(adapter); 5747 ixgbe_sfp_detection_subtask(adapter); 5748 ixgbe_sfp_link_config_subtask(adapter); 5749 ixgbe_check_overtemp_subtask(adapter); 5750 ixgbe_watchdog_subtask(adapter); 5751 ixgbe_fdir_reinit_subtask(adapter); 5752 ixgbe_check_hang_subtask(adapter); 5753 #ifdef CONFIG_IXGBE_PTP 5754 ixgbe_ptp_overflow_check(adapter); 5755 #endif 5756 5757 ixgbe_service_event_complete(adapter); 5758 } 5759 5760 static int ixgbe_tso(struct ixgbe_ring *tx_ring, 5761 struct ixgbe_tx_buffer *first, 5762 u8 *hdr_len) 5763 { 5764 struct sk_buff *skb = first->skb; 5765 u32 vlan_macip_lens, type_tucmd; 5766 u32 mss_l4len_idx, l4len; 5767 5768 if (!skb_is_gso(skb)) 5769 return 0; 5770 5771 if (skb_header_cloned(skb)) { 5772 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5773 if (err) 5774 return err; 5775 } 5776 5777 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 5778 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 5779 5780 if (first->protocol == __constant_htons(ETH_P_IP)) { 5781 struct iphdr *iph = ip_hdr(skb); 5782 iph->tot_len = 0; 5783 iph->check = 0; 5784 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 5785 iph->daddr, 0, 5786 IPPROTO_TCP, 5787 0); 5788 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 5789 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 5790 IXGBE_TX_FLAGS_CSUM | 5791 IXGBE_TX_FLAGS_IPV4; 5792 } else if (skb_is_gso_v6(skb)) { 5793 ipv6_hdr(skb)->payload_len = 0; 5794 tcp_hdr(skb)->check = 5795 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5796 &ipv6_hdr(skb)->daddr, 5797 0, IPPROTO_TCP, 0); 5798 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 5799 IXGBE_TX_FLAGS_CSUM; 5800 } 5801 5802 /* compute header lengths */ 5803 l4len = tcp_hdrlen(skb); 5804 *hdr_len = skb_transport_offset(skb) + l4len; 5805 5806 /* update gso size and bytecount with header size */ 5807 first->gso_segs = skb_shinfo(skb)->gso_segs; 5808 first->bytecount += (first->gso_segs - 1) * *hdr_len; 5809 5810 /* mss_l4len_id: use 1 as index for TSO */ 5811 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 5812 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 5813 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 5814 5815 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 5816 vlan_macip_lens = skb_network_header_len(skb); 5817 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 5818 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 5819 5820 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 5821 mss_l4len_idx); 5822 5823 return 1; 5824 } 5825 5826 static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, 5827 struct ixgbe_tx_buffer *first) 5828 { 5829 struct sk_buff *skb = first->skb; 5830 u32 vlan_macip_lens = 0; 5831 u32 mss_l4len_idx = 0; 5832 u32 type_tucmd = 0; 5833 5834 if (skb->ip_summed != CHECKSUM_PARTIAL) { 5835 if (!(first->tx_flags & IXGBE_TX_FLAGS_HW_VLAN) && 5836 !(first->tx_flags & IXGBE_TX_FLAGS_TXSW)) 5837 return; 5838 } else { 5839 u8 l4_hdr = 0; 5840 switch (first->protocol) { 5841 case __constant_htons(ETH_P_IP): 5842 vlan_macip_lens |= skb_network_header_len(skb); 5843 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 5844 l4_hdr = ip_hdr(skb)->protocol; 5845 break; 5846 case __constant_htons(ETH_P_IPV6): 5847 vlan_macip_lens |= skb_network_header_len(skb); 5848 l4_hdr = ipv6_hdr(skb)->nexthdr; 5849 break; 5850 default: 5851 if (unlikely(net_ratelimit())) { 5852 dev_warn(tx_ring->dev, 5853 "partial checksum but proto=%x!\n", 5854 first->protocol); 5855 } 5856 break; 5857 } 5858 5859 switch (l4_hdr) { 5860 case IPPROTO_TCP: 5861 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 5862 mss_l4len_idx = tcp_hdrlen(skb) << 5863 IXGBE_ADVTXD_L4LEN_SHIFT; 5864 break; 5865 case IPPROTO_SCTP: 5866 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 5867 mss_l4len_idx = sizeof(struct sctphdr) << 5868 IXGBE_ADVTXD_L4LEN_SHIFT; 5869 break; 5870 case IPPROTO_UDP: 5871 mss_l4len_idx = sizeof(struct udphdr) << 5872 IXGBE_ADVTXD_L4LEN_SHIFT; 5873 break; 5874 default: 5875 if (unlikely(net_ratelimit())) { 5876 dev_warn(tx_ring->dev, 5877 "partial checksum but l4 proto=%x!\n", 5878 l4_hdr); 5879 } 5880 break; 5881 } 5882 5883 /* update TX checksum flag */ 5884 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 5885 } 5886 5887 /* vlan_macip_lens: MACLEN, VLAN tag */ 5888 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 5889 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 5890 5891 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, 5892 type_tucmd, mss_l4len_idx); 5893 } 5894 5895 static __le32 ixgbe_tx_cmd_type(u32 tx_flags) 5896 { 5897 /* set type for advanced descriptor with frame checksum insertion */ 5898 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 5899 IXGBE_ADVTXD_DCMD_IFCS | 5900 IXGBE_ADVTXD_DCMD_DEXT); 5901 5902 /* set HW vlan bit if vlan is present */ 5903 if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN) 5904 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 5905 5906 #ifdef CONFIG_IXGBE_PTP 5907 if (tx_flags & IXGBE_TX_FLAGS_TSTAMP) 5908 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_MAC_TSTAMP); 5909 #endif 5910 5911 /* set segmentation enable bits for TSO/FSO */ 5912 #ifdef IXGBE_FCOE 5913 if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_FSO)) 5914 #else 5915 if (tx_flags & IXGBE_TX_FLAGS_TSO) 5916 #endif 5917 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 5918 5919 return cmd_type; 5920 } 5921 5922 static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 5923 u32 tx_flags, unsigned int paylen) 5924 { 5925 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 5926 5927 /* enable L4 checksum for TSO and TX checksum offload */ 5928 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 5929 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 5930 5931 /* enble IPv4 checksum for TSO */ 5932 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 5933 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 5934 5935 /* use index 1 context for TSO/FSO/FCOE */ 5936 #ifdef IXGBE_FCOE 5937 if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_FCOE)) 5938 #else 5939 if (tx_flags & IXGBE_TX_FLAGS_TSO) 5940 #endif 5941 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 5942 5943 /* 5944 * Check Context must be set if Tx switch is enabled, which it 5945 * always is for case where virtual functions are running 5946 */ 5947 #ifdef IXGBE_FCOE 5948 if (tx_flags & (IXGBE_TX_FLAGS_TXSW | IXGBE_TX_FLAGS_FCOE)) 5949 #else 5950 if (tx_flags & IXGBE_TX_FLAGS_TXSW) 5951 #endif 5952 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 5953 5954 tx_desc->read.olinfo_status = olinfo_status; 5955 } 5956 5957 #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ 5958 IXGBE_TXD_CMD_RS) 5959 5960 static void ixgbe_tx_map(struct ixgbe_ring *tx_ring, 5961 struct ixgbe_tx_buffer *first, 5962 const u8 hdr_len) 5963 { 5964 dma_addr_t dma; 5965 struct sk_buff *skb = first->skb; 5966 struct ixgbe_tx_buffer *tx_buffer; 5967 union ixgbe_adv_tx_desc *tx_desc; 5968 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 5969 unsigned int data_len = skb->data_len; 5970 unsigned int size = skb_headlen(skb); 5971 unsigned int paylen = skb->len - hdr_len; 5972 u32 tx_flags = first->tx_flags; 5973 __le32 cmd_type; 5974 u16 i = tx_ring->next_to_use; 5975 5976 tx_desc = IXGBE_TX_DESC(tx_ring, i); 5977 5978 ixgbe_tx_olinfo_status(tx_desc, tx_flags, paylen); 5979 cmd_type = ixgbe_tx_cmd_type(tx_flags); 5980 5981 #ifdef IXGBE_FCOE 5982 if (tx_flags & IXGBE_TX_FLAGS_FCOE) { 5983 if (data_len < sizeof(struct fcoe_crc_eof)) { 5984 size -= sizeof(struct fcoe_crc_eof) - data_len; 5985 data_len = 0; 5986 } else { 5987 data_len -= sizeof(struct fcoe_crc_eof); 5988 } 5989 } 5990 5991 #endif 5992 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 5993 if (dma_mapping_error(tx_ring->dev, dma)) 5994 goto dma_error; 5995 5996 /* record length, and DMA address */ 5997 dma_unmap_len_set(first, len, size); 5998 dma_unmap_addr_set(first, dma, dma); 5999 6000 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6001 6002 for (;;) { 6003 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 6004 tx_desc->read.cmd_type_len = 6005 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 6006 6007 i++; 6008 tx_desc++; 6009 if (i == tx_ring->count) { 6010 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 6011 i = 0; 6012 } 6013 6014 dma += IXGBE_MAX_DATA_PER_TXD; 6015 size -= IXGBE_MAX_DATA_PER_TXD; 6016 6017 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6018 tx_desc->read.olinfo_status = 0; 6019 } 6020 6021 if (likely(!data_len)) 6022 break; 6023 6024 if (unlikely(skb->no_fcs)) 6025 cmd_type &= ~(cpu_to_le32(IXGBE_ADVTXD_DCMD_IFCS)); 6026 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 6027 6028 i++; 6029 tx_desc++; 6030 if (i == tx_ring->count) { 6031 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 6032 i = 0; 6033 } 6034 6035 #ifdef IXGBE_FCOE 6036 size = min_t(unsigned int, data_len, skb_frag_size(frag)); 6037 #else 6038 size = skb_frag_size(frag); 6039 #endif 6040 data_len -= size; 6041 6042 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 6043 DMA_TO_DEVICE); 6044 if (dma_mapping_error(tx_ring->dev, dma)) 6045 goto dma_error; 6046 6047 tx_buffer = &tx_ring->tx_buffer_info[i]; 6048 dma_unmap_len_set(tx_buffer, len, size); 6049 dma_unmap_addr_set(tx_buffer, dma, dma); 6050 6051 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6052 tx_desc->read.olinfo_status = 0; 6053 6054 frag++; 6055 } 6056 6057 /* write last descriptor with RS and EOP bits */ 6058 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 6059 tx_desc->read.cmd_type_len = cmd_type; 6060 6061 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 6062 6063 /* set the timestamp */ 6064 first->time_stamp = jiffies; 6065 6066 /* 6067 * Force memory writes to complete before letting h/w know there 6068 * are new descriptors to fetch. (Only applicable for weak-ordered 6069 * memory model archs, such as IA-64). 6070 * 6071 * We also need this memory barrier to make certain all of the 6072 * status bits have been updated before next_to_watch is written. 6073 */ 6074 wmb(); 6075 6076 /* set next_to_watch value indicating a packet is present */ 6077 first->next_to_watch = tx_desc; 6078 6079 i++; 6080 if (i == tx_ring->count) 6081 i = 0; 6082 6083 tx_ring->next_to_use = i; 6084 6085 /* notify HW of packet */ 6086 writel(i, tx_ring->tail); 6087 6088 return; 6089 dma_error: 6090 dev_err(tx_ring->dev, "TX DMA map failed\n"); 6091 6092 /* clear dma mappings for failed tx_buffer_info map */ 6093 for (;;) { 6094 tx_buffer = &tx_ring->tx_buffer_info[i]; 6095 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer); 6096 if (tx_buffer == first) 6097 break; 6098 if (i == 0) 6099 i = tx_ring->count; 6100 i--; 6101 } 6102 6103 tx_ring->next_to_use = i; 6104 } 6105 6106 static void ixgbe_atr(struct ixgbe_ring *ring, 6107 struct ixgbe_tx_buffer *first) 6108 { 6109 struct ixgbe_q_vector *q_vector = ring->q_vector; 6110 union ixgbe_atr_hash_dword input = { .dword = 0 }; 6111 union ixgbe_atr_hash_dword common = { .dword = 0 }; 6112 union { 6113 unsigned char *network; 6114 struct iphdr *ipv4; 6115 struct ipv6hdr *ipv6; 6116 } hdr; 6117 struct tcphdr *th; 6118 __be16 vlan_id; 6119 6120 /* if ring doesn't have a interrupt vector, cannot perform ATR */ 6121 if (!q_vector) 6122 return; 6123 6124 /* do nothing if sampling is disabled */ 6125 if (!ring->atr_sample_rate) 6126 return; 6127 6128 ring->atr_count++; 6129 6130 /* snag network header to get L4 type and address */ 6131 hdr.network = skb_network_header(first->skb); 6132 6133 /* Currently only IPv4/IPv6 with TCP is supported */ 6134 if ((first->protocol != __constant_htons(ETH_P_IPV6) || 6135 hdr.ipv6->nexthdr != IPPROTO_TCP) && 6136 (first->protocol != __constant_htons(ETH_P_IP) || 6137 hdr.ipv4->protocol != IPPROTO_TCP)) 6138 return; 6139 6140 th = tcp_hdr(first->skb); 6141 6142 /* skip this packet since it is invalid or the socket is closing */ 6143 if (!th || th->fin) 6144 return; 6145 6146 /* sample on all syn packets or once every atr sample count */ 6147 if (!th->syn && (ring->atr_count < ring->atr_sample_rate)) 6148 return; 6149 6150 /* reset sample count */ 6151 ring->atr_count = 0; 6152 6153 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT); 6154 6155 /* 6156 * src and dst are inverted, think how the receiver sees them 6157 * 6158 * The input is broken into two sections, a non-compressed section 6159 * containing vm_pool, vlan_id, and flow_type. The rest of the data 6160 * is XORed together and stored in the compressed dword. 6161 */ 6162 input.formatted.vlan_id = vlan_id; 6163 6164 /* 6165 * since src port and flex bytes occupy the same word XOR them together 6166 * and write the value to source port portion of compressed dword 6167 */ 6168 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN)) 6169 common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q); 6170 else 6171 common.port.src ^= th->dest ^ first->protocol; 6172 common.port.dst ^= th->source; 6173 6174 if (first->protocol == __constant_htons(ETH_P_IP)) { 6175 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4; 6176 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr; 6177 } else { 6178 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; 6179 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ 6180 hdr.ipv6->saddr.s6_addr32[1] ^ 6181 hdr.ipv6->saddr.s6_addr32[2] ^ 6182 hdr.ipv6->saddr.s6_addr32[3] ^ 6183 hdr.ipv6->daddr.s6_addr32[0] ^ 6184 hdr.ipv6->daddr.s6_addr32[1] ^ 6185 hdr.ipv6->daddr.s6_addr32[2] ^ 6186 hdr.ipv6->daddr.s6_addr32[3]; 6187 } 6188 6189 /* This assumes the Rx queue and Tx queue are bound to the same CPU */ 6190 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, 6191 input, common, ring->queue_index); 6192 } 6193 6194 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 6195 { 6196 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 6197 /* Herbert's original patch had: 6198 * smp_mb__after_netif_stop_queue(); 6199 * but since that doesn't exist yet, just open code it. */ 6200 smp_mb(); 6201 6202 /* We need to check again in a case another CPU has just 6203 * made room available. */ 6204 if (likely(ixgbe_desc_unused(tx_ring) < size)) 6205 return -EBUSY; 6206 6207 /* A reprieve! - use start_queue because it doesn't call schedule */ 6208 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 6209 ++tx_ring->tx_stats.restart_queue; 6210 return 0; 6211 } 6212 6213 static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 6214 { 6215 if (likely(ixgbe_desc_unused(tx_ring) >= size)) 6216 return 0; 6217 return __ixgbe_maybe_stop_tx(tx_ring, size); 6218 } 6219 6220 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb) 6221 { 6222 struct ixgbe_adapter *adapter = netdev_priv(dev); 6223 int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 6224 smp_processor_id(); 6225 #ifdef IXGBE_FCOE 6226 __be16 protocol = vlan_get_protocol(skb); 6227 6228 if (((protocol == htons(ETH_P_FCOE)) || 6229 (protocol == htons(ETH_P_FIP))) && 6230 (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) { 6231 txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1); 6232 txq += adapter->ring_feature[RING_F_FCOE].mask; 6233 return txq; 6234 } 6235 #endif 6236 6237 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 6238 while (unlikely(txq >= dev->real_num_tx_queues)) 6239 txq -= dev->real_num_tx_queues; 6240 return txq; 6241 } 6242 6243 return skb_tx_hash(dev, skb); 6244 } 6245 6246 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, 6247 struct ixgbe_adapter *adapter, 6248 struct ixgbe_ring *tx_ring) 6249 { 6250 struct ixgbe_tx_buffer *first; 6251 int tso; 6252 u32 tx_flags = 0; 6253 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 6254 unsigned short f; 6255 #endif 6256 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 6257 __be16 protocol = skb->protocol; 6258 u8 hdr_len = 0; 6259 6260 /* 6261 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 6262 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 6263 * + 2 desc gap to keep tail from touching head, 6264 * + 1 desc for context descriptor, 6265 * otherwise try next time 6266 */ 6267 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 6268 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 6269 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 6270 #else 6271 count += skb_shinfo(skb)->nr_frags; 6272 #endif 6273 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) { 6274 tx_ring->tx_stats.tx_busy++; 6275 return NETDEV_TX_BUSY; 6276 } 6277 6278 /* record the location of the first descriptor for this packet */ 6279 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 6280 first->skb = skb; 6281 first->bytecount = skb->len; 6282 first->gso_segs = 1; 6283 6284 /* if we have a HW VLAN tag being added default to the HW one */ 6285 if (vlan_tx_tag_present(skb)) { 6286 tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT; 6287 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 6288 /* else if it is a SW VLAN check the next protocol and store the tag */ 6289 } else if (protocol == __constant_htons(ETH_P_8021Q)) { 6290 struct vlan_hdr *vhdr, _vhdr; 6291 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 6292 if (!vhdr) 6293 goto out_drop; 6294 6295 protocol = vhdr->h_vlan_encapsulated_proto; 6296 tx_flags |= ntohs(vhdr->h_vlan_TCI) << 6297 IXGBE_TX_FLAGS_VLAN_SHIFT; 6298 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN; 6299 } 6300 6301 skb_tx_timestamp(skb); 6302 6303 #ifdef CONFIG_IXGBE_PTP 6304 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 6305 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 6306 tx_flags |= IXGBE_TX_FLAGS_TSTAMP; 6307 } 6308 #endif 6309 6310 #ifdef CONFIG_PCI_IOV 6311 /* 6312 * Use the l2switch_enable flag - would be false if the DMA 6313 * Tx switch had been disabled. 6314 */ 6315 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 6316 tx_flags |= IXGBE_TX_FLAGS_TXSW; 6317 6318 #endif 6319 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ 6320 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && 6321 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || 6322 (skb->priority != TC_PRIO_CONTROL))) { 6323 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; 6324 tx_flags |= (skb->priority & 0x7) << 6325 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; 6326 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) { 6327 struct vlan_ethhdr *vhdr; 6328 if (skb_header_cloned(skb) && 6329 pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) 6330 goto out_drop; 6331 vhdr = (struct vlan_ethhdr *)skb->data; 6332 vhdr->h_vlan_TCI = htons(tx_flags >> 6333 IXGBE_TX_FLAGS_VLAN_SHIFT); 6334 } else { 6335 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 6336 } 6337 } 6338 6339 /* record initial flags and protocol */ 6340 first->tx_flags = tx_flags; 6341 first->protocol = protocol; 6342 6343 #ifdef IXGBE_FCOE 6344 /* setup tx offload for FCoE */ 6345 if ((protocol == __constant_htons(ETH_P_FCOE)) && 6346 (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) { 6347 tso = ixgbe_fso(tx_ring, first, &hdr_len); 6348 if (tso < 0) 6349 goto out_drop; 6350 6351 goto xmit_fcoe; 6352 } 6353 6354 #endif /* IXGBE_FCOE */ 6355 tso = ixgbe_tso(tx_ring, first, &hdr_len); 6356 if (tso < 0) 6357 goto out_drop; 6358 else if (!tso) 6359 ixgbe_tx_csum(tx_ring, first); 6360 6361 /* add the ATR filter if ATR is on */ 6362 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state)) 6363 ixgbe_atr(tx_ring, first); 6364 6365 #ifdef IXGBE_FCOE 6366 xmit_fcoe: 6367 #endif /* IXGBE_FCOE */ 6368 ixgbe_tx_map(tx_ring, first, hdr_len); 6369 6370 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED); 6371 6372 return NETDEV_TX_OK; 6373 6374 out_drop: 6375 dev_kfree_skb_any(first->skb); 6376 first->skb = NULL; 6377 6378 return NETDEV_TX_OK; 6379 } 6380 6381 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, 6382 struct net_device *netdev) 6383 { 6384 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6385 struct ixgbe_ring *tx_ring; 6386 6387 if (skb->len <= 0) { 6388 dev_kfree_skb_any(skb); 6389 return NETDEV_TX_OK; 6390 } 6391 6392 /* 6393 * The minimum packet size for olinfo paylen is 17 so pad the skb 6394 * in order to meet this minimum size requirement. 6395 */ 6396 if (skb->len < 17) { 6397 if (skb_padto(skb, 17)) 6398 return NETDEV_TX_OK; 6399 skb->len = 17; 6400 } 6401 6402 tx_ring = adapter->tx_ring[skb->queue_mapping]; 6403 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); 6404 } 6405 6406 /** 6407 * ixgbe_set_mac - Change the Ethernet Address of the NIC 6408 * @netdev: network interface device structure 6409 * @p: pointer to an address structure 6410 * 6411 * Returns 0 on success, negative on failure 6412 **/ 6413 static int ixgbe_set_mac(struct net_device *netdev, void *p) 6414 { 6415 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6416 struct ixgbe_hw *hw = &adapter->hw; 6417 struct sockaddr *addr = p; 6418 6419 if (!is_valid_ether_addr(addr->sa_data)) 6420 return -EADDRNOTAVAIL; 6421 6422 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 6423 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 6424 6425 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs, 6426 IXGBE_RAH_AV); 6427 6428 return 0; 6429 } 6430 6431 static int 6432 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) 6433 { 6434 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6435 struct ixgbe_hw *hw = &adapter->hw; 6436 u16 value; 6437 int rc; 6438 6439 if (prtad != hw->phy.mdio.prtad) 6440 return -EINVAL; 6441 rc = hw->phy.ops.read_reg(hw, addr, devad, &value); 6442 if (!rc) 6443 rc = value; 6444 return rc; 6445 } 6446 6447 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, 6448 u16 addr, u16 value) 6449 { 6450 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6451 struct ixgbe_hw *hw = &adapter->hw; 6452 6453 if (prtad != hw->phy.mdio.prtad) 6454 return -EINVAL; 6455 return hw->phy.ops.write_reg(hw, addr, devad, value); 6456 } 6457 6458 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) 6459 { 6460 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6461 6462 switch (cmd) { 6463 #ifdef CONFIG_IXGBE_PTP 6464 case SIOCSHWTSTAMP: 6465 return ixgbe_ptp_hwtstamp_ioctl(adapter, req, cmd); 6466 #endif 6467 default: 6468 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd); 6469 } 6470 } 6471 6472 /** 6473 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding 6474 * netdev->dev_addrs 6475 * @netdev: network interface device structure 6476 * 6477 * Returns non-zero on failure 6478 **/ 6479 static int ixgbe_add_sanmac_netdev(struct net_device *dev) 6480 { 6481 int err = 0; 6482 struct ixgbe_adapter *adapter = netdev_priv(dev); 6483 struct ixgbe_mac_info *mac = &adapter->hw.mac; 6484 6485 if (is_valid_ether_addr(mac->san_addr)) { 6486 rtnl_lock(); 6487 err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); 6488 rtnl_unlock(); 6489 } 6490 return err; 6491 } 6492 6493 /** 6494 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding 6495 * netdev->dev_addrs 6496 * @netdev: network interface device structure 6497 * 6498 * Returns non-zero on failure 6499 **/ 6500 static int ixgbe_del_sanmac_netdev(struct net_device *dev) 6501 { 6502 int err = 0; 6503 struct ixgbe_adapter *adapter = netdev_priv(dev); 6504 struct ixgbe_mac_info *mac = &adapter->hw.mac; 6505 6506 if (is_valid_ether_addr(mac->san_addr)) { 6507 rtnl_lock(); 6508 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); 6509 rtnl_unlock(); 6510 } 6511 return err; 6512 } 6513 6514 #ifdef CONFIG_NET_POLL_CONTROLLER 6515 /* 6516 * Polling 'interrupt' - used by things like netconsole to send skbs 6517 * without having to re-enable interrupts. It's not called while 6518 * the interrupt routine is executing. 6519 */ 6520 static void ixgbe_netpoll(struct net_device *netdev) 6521 { 6522 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6523 int i; 6524 6525 /* if interface is down do nothing */ 6526 if (test_bit(__IXGBE_DOWN, &adapter->state)) 6527 return; 6528 6529 adapter->flags |= IXGBE_FLAG_IN_NETPOLL; 6530 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 6531 int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 6532 for (i = 0; i < num_q_vectors; i++) { 6533 struct ixgbe_q_vector *q_vector = adapter->q_vector[i]; 6534 ixgbe_msix_clean_rings(0, q_vector); 6535 } 6536 } else { 6537 ixgbe_intr(adapter->pdev->irq, netdev); 6538 } 6539 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL; 6540 } 6541 6542 #endif 6543 static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev, 6544 struct rtnl_link_stats64 *stats) 6545 { 6546 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6547 int i; 6548 6549 rcu_read_lock(); 6550 for (i = 0; i < adapter->num_rx_queues; i++) { 6551 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]); 6552 u64 bytes, packets; 6553 unsigned int start; 6554 6555 if (ring) { 6556 do { 6557 start = u64_stats_fetch_begin_bh(&ring->syncp); 6558 packets = ring->stats.packets; 6559 bytes = ring->stats.bytes; 6560 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 6561 stats->rx_packets += packets; 6562 stats->rx_bytes += bytes; 6563 } 6564 } 6565 6566 for (i = 0; i < adapter->num_tx_queues; i++) { 6567 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]); 6568 u64 bytes, packets; 6569 unsigned int start; 6570 6571 if (ring) { 6572 do { 6573 start = u64_stats_fetch_begin_bh(&ring->syncp); 6574 packets = ring->stats.packets; 6575 bytes = ring->stats.bytes; 6576 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 6577 stats->tx_packets += packets; 6578 stats->tx_bytes += bytes; 6579 } 6580 } 6581 rcu_read_unlock(); 6582 /* following stats updated by ixgbe_watchdog_task() */ 6583 stats->multicast = netdev->stats.multicast; 6584 stats->rx_errors = netdev->stats.rx_errors; 6585 stats->rx_length_errors = netdev->stats.rx_length_errors; 6586 stats->rx_crc_errors = netdev->stats.rx_crc_errors; 6587 stats->rx_missed_errors = netdev->stats.rx_missed_errors; 6588 return stats; 6589 } 6590 6591 #ifdef CONFIG_IXGBE_DCB 6592 /* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid. 6593 * #adapter: pointer to ixgbe_adapter 6594 * @tc: number of traffic classes currently enabled 6595 * 6596 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm 6597 * 802.1Q priority maps to a packet buffer that exists. 6598 */ 6599 static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc) 6600 { 6601 struct ixgbe_hw *hw = &adapter->hw; 6602 u32 reg, rsave; 6603 int i; 6604 6605 /* 82598 have a static priority to TC mapping that can not 6606 * be changed so no validation is needed. 6607 */ 6608 if (hw->mac.type == ixgbe_mac_82598EB) 6609 return; 6610 6611 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC); 6612 rsave = reg; 6613 6614 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) { 6615 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT); 6616 6617 /* If up2tc is out of bounds default to zero */ 6618 if (up2tc > tc) 6619 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT); 6620 } 6621 6622 if (reg != rsave) 6623 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg); 6624 6625 return; 6626 } 6627 6628 /* ixgbe_setup_tc - routine to configure net_device for multiple traffic 6629 * classes. 6630 * 6631 * @netdev: net device to configure 6632 * @tc: number of traffic classes to enable 6633 */ 6634 int ixgbe_setup_tc(struct net_device *dev, u8 tc) 6635 { 6636 struct ixgbe_adapter *adapter = netdev_priv(dev); 6637 struct ixgbe_hw *hw = &adapter->hw; 6638 6639 /* Multiple traffic classes requires multiple queues */ 6640 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 6641 e_err(drv, "Enable failed, needs MSI-X\n"); 6642 return -EINVAL; 6643 } 6644 6645 /* Hardware supports up to 8 traffic classes */ 6646 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs || 6647 (hw->mac.type == ixgbe_mac_82598EB && 6648 tc < MAX_TRAFFIC_CLASS)) 6649 return -EINVAL; 6650 6651 /* Hardware has to reinitialize queues and interrupts to 6652 * match packet buffer alignment. Unfortunately, the 6653 * hardware is not flexible enough to do this dynamically. 6654 */ 6655 if (netif_running(dev)) 6656 ixgbe_close(dev); 6657 ixgbe_clear_interrupt_scheme(adapter); 6658 6659 if (tc) { 6660 netdev_set_num_tc(dev, tc); 6661 adapter->flags |= IXGBE_FLAG_DCB_ENABLED; 6662 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 6663 6664 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 6665 adapter->last_lfc_mode = adapter->hw.fc.requested_mode; 6666 adapter->hw.fc.requested_mode = ixgbe_fc_none; 6667 } 6668 } else { 6669 netdev_reset_tc(dev); 6670 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 6671 adapter->hw.fc.requested_mode = adapter->last_lfc_mode; 6672 6673 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; 6674 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 6675 6676 adapter->temp_dcb_cfg.pfc_mode_enable = false; 6677 adapter->dcb_cfg.pfc_mode_enable = false; 6678 } 6679 6680 ixgbe_init_interrupt_scheme(adapter); 6681 ixgbe_validate_rtr(adapter, tc); 6682 if (netif_running(dev)) 6683 ixgbe_open(dev); 6684 6685 return 0; 6686 } 6687 6688 #endif /* CONFIG_IXGBE_DCB */ 6689 void ixgbe_do_reset(struct net_device *netdev) 6690 { 6691 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6692 6693 if (netif_running(netdev)) 6694 ixgbe_reinit_locked(adapter); 6695 else 6696 ixgbe_reset(adapter); 6697 } 6698 6699 static netdev_features_t ixgbe_fix_features(struct net_device *netdev, 6700 netdev_features_t features) 6701 { 6702 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6703 6704 #ifdef CONFIG_DCB 6705 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) 6706 features &= ~NETIF_F_HW_VLAN_RX; 6707 #endif 6708 6709 /* return error if RXHASH is being enabled when RSS is not supported */ 6710 if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) 6711 features &= ~NETIF_F_RXHASH; 6712 6713 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ 6714 if (!(features & NETIF_F_RXCSUM)) 6715 features &= ~NETIF_F_LRO; 6716 6717 /* Turn off LRO if not RSC capable */ 6718 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) 6719 features &= ~NETIF_F_LRO; 6720 6721 6722 return features; 6723 } 6724 6725 static int ixgbe_set_features(struct net_device *netdev, 6726 netdev_features_t features) 6727 { 6728 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6729 netdev_features_t changed = netdev->features ^ features; 6730 bool need_reset = false; 6731 6732 /* Make sure RSC matches LRO, reset if change */ 6733 if (!(features & NETIF_F_LRO)) { 6734 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 6735 need_reset = true; 6736 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED; 6737 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && 6738 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { 6739 if (adapter->rx_itr_setting == 1 || 6740 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { 6741 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 6742 need_reset = true; 6743 } else if ((changed ^ features) & NETIF_F_LRO) { 6744 e_info(probe, "rx-usecs set too low, " 6745 "disabling RSC\n"); 6746 } 6747 } 6748 6749 /* 6750 * Check if Flow Director n-tuple support was enabled or disabled. If 6751 * the state changed, we need to reset. 6752 */ 6753 if (!(features & NETIF_F_NTUPLE)) { 6754 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { 6755 /* turn off Flow Director, set ATR and reset */ 6756 if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) && 6757 !(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) 6758 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 6759 need_reset = true; 6760 } 6761 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 6762 } else if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) { 6763 /* turn off ATR, enable perfect filters and reset */ 6764 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 6765 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 6766 need_reset = true; 6767 } 6768 6769 if (changed & NETIF_F_RXALL) 6770 need_reset = true; 6771 6772 netdev->features = features; 6773 if (need_reset) 6774 ixgbe_do_reset(netdev); 6775 6776 return 0; 6777 } 6778 6779 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, 6780 struct net_device *dev, 6781 unsigned char *addr, 6782 u16 flags) 6783 { 6784 struct ixgbe_adapter *adapter = netdev_priv(dev); 6785 int err = -EOPNOTSUPP; 6786 6787 if (ndm->ndm_state & NUD_PERMANENT) { 6788 pr_info("%s: FDB only supports static addresses\n", 6789 ixgbe_driver_name); 6790 return -EINVAL; 6791 } 6792 6793 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 6794 if (is_unicast_ether_addr(addr)) 6795 err = dev_uc_add_excl(dev, addr); 6796 else if (is_multicast_ether_addr(addr)) 6797 err = dev_mc_add_excl(dev, addr); 6798 else 6799 err = -EINVAL; 6800 } 6801 6802 /* Only return duplicate errors if NLM_F_EXCL is set */ 6803 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 6804 err = 0; 6805 6806 return err; 6807 } 6808 6809 static int ixgbe_ndo_fdb_del(struct ndmsg *ndm, 6810 struct net_device *dev, 6811 unsigned char *addr) 6812 { 6813 struct ixgbe_adapter *adapter = netdev_priv(dev); 6814 int err = -EOPNOTSUPP; 6815 6816 if (ndm->ndm_state & NUD_PERMANENT) { 6817 pr_info("%s: FDB only supports static addresses\n", 6818 ixgbe_driver_name); 6819 return -EINVAL; 6820 } 6821 6822 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 6823 if (is_unicast_ether_addr(addr)) 6824 err = dev_uc_del(dev, addr); 6825 else if (is_multicast_ether_addr(addr)) 6826 err = dev_mc_del(dev, addr); 6827 else 6828 err = -EINVAL; 6829 } 6830 6831 return err; 6832 } 6833 6834 static int ixgbe_ndo_fdb_dump(struct sk_buff *skb, 6835 struct netlink_callback *cb, 6836 struct net_device *dev, 6837 int idx) 6838 { 6839 struct ixgbe_adapter *adapter = netdev_priv(dev); 6840 6841 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 6842 idx = ndo_dflt_fdb_dump(skb, cb, dev, idx); 6843 6844 return idx; 6845 } 6846 6847 static const struct net_device_ops ixgbe_netdev_ops = { 6848 .ndo_open = ixgbe_open, 6849 .ndo_stop = ixgbe_close, 6850 .ndo_start_xmit = ixgbe_xmit_frame, 6851 .ndo_select_queue = ixgbe_select_queue, 6852 .ndo_set_rx_mode = ixgbe_set_rx_mode, 6853 .ndo_validate_addr = eth_validate_addr, 6854 .ndo_set_mac_address = ixgbe_set_mac, 6855 .ndo_change_mtu = ixgbe_change_mtu, 6856 .ndo_tx_timeout = ixgbe_tx_timeout, 6857 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid, 6858 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid, 6859 .ndo_do_ioctl = ixgbe_ioctl, 6860 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac, 6861 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan, 6862 .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw, 6863 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk, 6864 .ndo_get_vf_config = ixgbe_ndo_get_vf_config, 6865 .ndo_get_stats64 = ixgbe_get_stats64, 6866 #ifdef CONFIG_IXGBE_DCB 6867 .ndo_setup_tc = ixgbe_setup_tc, 6868 #endif 6869 #ifdef CONFIG_NET_POLL_CONTROLLER 6870 .ndo_poll_controller = ixgbe_netpoll, 6871 #endif 6872 #ifdef IXGBE_FCOE 6873 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get, 6874 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target, 6875 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put, 6876 .ndo_fcoe_enable = ixgbe_fcoe_enable, 6877 .ndo_fcoe_disable = ixgbe_fcoe_disable, 6878 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn, 6879 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo, 6880 #endif /* IXGBE_FCOE */ 6881 .ndo_set_features = ixgbe_set_features, 6882 .ndo_fix_features = ixgbe_fix_features, 6883 .ndo_fdb_add = ixgbe_ndo_fdb_add, 6884 .ndo_fdb_del = ixgbe_ndo_fdb_del, 6885 .ndo_fdb_dump = ixgbe_ndo_fdb_dump, 6886 }; 6887 6888 static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter, 6889 const struct ixgbe_info *ii) 6890 { 6891 #ifdef CONFIG_PCI_IOV 6892 struct ixgbe_hw *hw = &adapter->hw; 6893 6894 if (hw->mac.type == ixgbe_mac_82598EB) 6895 return; 6896 6897 /* The 82599 supports up to 64 VFs per physical function 6898 * but this implementation limits allocation to 63 so that 6899 * basic networking resources are still available to the 6900 * physical function. If the user requests greater thn 6901 * 63 VFs then it is an error - reset to default of zero. 6902 */ 6903 adapter->num_vfs = (max_vfs > 63) ? 0 : max_vfs; 6904 ixgbe_enable_sriov(adapter, ii); 6905 #endif /* CONFIG_PCI_IOV */ 6906 } 6907 6908 /** 6909 * ixgbe_wol_supported - Check whether device supports WoL 6910 * @hw: hw specific details 6911 * @device_id: the device ID 6912 * @subdev_id: the subsystem device ID 6913 * 6914 * This function is used by probe and ethtool to determine 6915 * which devices have WoL support 6916 * 6917 **/ 6918 int ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, 6919 u16 subdevice_id) 6920 { 6921 struct ixgbe_hw *hw = &adapter->hw; 6922 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK; 6923 int is_wol_supported = 0; 6924 6925 switch (device_id) { 6926 case IXGBE_DEV_ID_82599_SFP: 6927 /* Only these subdevices could supports WOL */ 6928 switch (subdevice_id) { 6929 case IXGBE_SUBDEV_ID_82599_560FLR: 6930 /* only support first port */ 6931 if (hw->bus.func != 0) 6932 break; 6933 case IXGBE_SUBDEV_ID_82599_SFP: 6934 is_wol_supported = 1; 6935 break; 6936 } 6937 break; 6938 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: 6939 /* All except this subdevice support WOL */ 6940 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) 6941 is_wol_supported = 1; 6942 break; 6943 case IXGBE_DEV_ID_82599_KX4: 6944 is_wol_supported = 1; 6945 break; 6946 case IXGBE_DEV_ID_X540T: 6947 /* check eeprom to see if enabled wol */ 6948 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 6949 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && 6950 (hw->bus.func == 0))) { 6951 is_wol_supported = 1; 6952 } 6953 break; 6954 } 6955 6956 return is_wol_supported; 6957 } 6958 6959 /** 6960 * ixgbe_probe - Device Initialization Routine 6961 * @pdev: PCI device information struct 6962 * @ent: entry in ixgbe_pci_tbl 6963 * 6964 * Returns 0 on success, negative on failure 6965 * 6966 * ixgbe_probe initializes an adapter identified by a pci_dev structure. 6967 * The OS initialization, configuring of the adapter private structure, 6968 * and a hardware reset occur. 6969 **/ 6970 static int __devinit ixgbe_probe(struct pci_dev *pdev, 6971 const struct pci_device_id *ent) 6972 { 6973 struct net_device *netdev; 6974 struct ixgbe_adapter *adapter = NULL; 6975 struct ixgbe_hw *hw; 6976 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data]; 6977 static int cards_found; 6978 int i, err, pci_using_dac; 6979 u8 part_str[IXGBE_PBANUM_LENGTH]; 6980 unsigned int indices = num_possible_cpus(); 6981 #ifdef IXGBE_FCOE 6982 u16 device_caps; 6983 #endif 6984 u32 eec; 6985 6986 /* Catch broken hardware that put the wrong VF device ID in 6987 * the PCIe SR-IOV capability. 6988 */ 6989 if (pdev->is_virtfn) { 6990 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", 6991 pci_name(pdev), pdev->vendor, pdev->device); 6992 return -EINVAL; 6993 } 6994 6995 err = pci_enable_device_mem(pdev); 6996 if (err) 6997 return err; 6998 6999 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 7000 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 7001 pci_using_dac = 1; 7002 } else { 7003 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 7004 if (err) { 7005 err = dma_set_coherent_mask(&pdev->dev, 7006 DMA_BIT_MASK(32)); 7007 if (err) { 7008 dev_err(&pdev->dev, 7009 "No usable DMA configuration, aborting\n"); 7010 goto err_dma; 7011 } 7012 } 7013 pci_using_dac = 0; 7014 } 7015 7016 err = pci_request_selected_regions(pdev, pci_select_bars(pdev, 7017 IORESOURCE_MEM), ixgbe_driver_name); 7018 if (err) { 7019 dev_err(&pdev->dev, 7020 "pci_request_selected_regions failed 0x%x\n", err); 7021 goto err_pci_reg; 7022 } 7023 7024 pci_enable_pcie_error_reporting(pdev); 7025 7026 pci_set_master(pdev); 7027 pci_save_state(pdev); 7028 7029 #ifdef CONFIG_IXGBE_DCB 7030 indices *= MAX_TRAFFIC_CLASS; 7031 #endif 7032 7033 if (ii->mac == ixgbe_mac_82598EB) 7034 indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES); 7035 else 7036 indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES); 7037 7038 #ifdef IXGBE_FCOE 7039 indices += min_t(unsigned int, num_possible_cpus(), 7040 IXGBE_MAX_FCOE_INDICES); 7041 #endif 7042 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices); 7043 if (!netdev) { 7044 err = -ENOMEM; 7045 goto err_alloc_etherdev; 7046 } 7047 7048 SET_NETDEV_DEV(netdev, &pdev->dev); 7049 7050 adapter = netdev_priv(netdev); 7051 pci_set_drvdata(pdev, adapter); 7052 7053 adapter->netdev = netdev; 7054 adapter->pdev = pdev; 7055 hw = &adapter->hw; 7056 hw->back = adapter; 7057 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7058 7059 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 7060 pci_resource_len(pdev, 0)); 7061 if (!hw->hw_addr) { 7062 err = -EIO; 7063 goto err_ioremap; 7064 } 7065 7066 for (i = 1; i <= 5; i++) { 7067 if (pci_resource_len(pdev, i) == 0) 7068 continue; 7069 } 7070 7071 netdev->netdev_ops = &ixgbe_netdev_ops; 7072 ixgbe_set_ethtool_ops(netdev); 7073 netdev->watchdog_timeo = 5 * HZ; 7074 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 7075 7076 adapter->bd_number = cards_found; 7077 7078 /* Setup hw api */ 7079 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 7080 hw->mac.type = ii->mac; 7081 7082 /* EEPROM */ 7083 memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops)); 7084 eec = IXGBE_READ_REG(hw, IXGBE_EEC); 7085 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */ 7086 if (!(eec & (1 << 8))) 7087 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic; 7088 7089 /* PHY */ 7090 memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops)); 7091 hw->phy.sfp_type = ixgbe_sfp_type_unknown; 7092 /* ixgbe_identify_phy_generic will set prtad and mmds properly */ 7093 hw->phy.mdio.prtad = MDIO_PRTAD_NONE; 7094 hw->phy.mdio.mmds = 0; 7095 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22; 7096 hw->phy.mdio.dev = netdev; 7097 hw->phy.mdio.mdio_read = ixgbe_mdio_read; 7098 hw->phy.mdio.mdio_write = ixgbe_mdio_write; 7099 7100 ii->get_invariants(hw); 7101 7102 /* setup the private structure */ 7103 err = ixgbe_sw_init(adapter); 7104 if (err) 7105 goto err_sw_init; 7106 7107 /* Make it possible the adapter to be woken up via WOL */ 7108 switch (adapter->hw.mac.type) { 7109 case ixgbe_mac_82599EB: 7110 case ixgbe_mac_X540: 7111 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 7112 break; 7113 default: 7114 break; 7115 } 7116 7117 /* 7118 * If there is a fan on this device and it has failed log the 7119 * failure. 7120 */ 7121 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 7122 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 7123 if (esdp & IXGBE_ESDP_SDP1) 7124 e_crit(probe, "Fan has stopped, replace the adapter\n"); 7125 } 7126 7127 if (allow_unsupported_sfp) 7128 hw->allow_unsupported_sfp = allow_unsupported_sfp; 7129 7130 /* reset_hw fills in the perm_addr as well */ 7131 hw->phy.reset_if_overtemp = true; 7132 err = hw->mac.ops.reset_hw(hw); 7133 hw->phy.reset_if_overtemp = false; 7134 if (err == IXGBE_ERR_SFP_NOT_PRESENT && 7135 hw->mac.type == ixgbe_mac_82598EB) { 7136 err = 0; 7137 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 7138 e_dev_err("failed to load because an unsupported SFP+ " 7139 "module type was detected.\n"); 7140 e_dev_err("Reload the driver after installing a supported " 7141 "module.\n"); 7142 goto err_sw_init; 7143 } else if (err) { 7144 e_dev_err("HW Init failed: %d\n", err); 7145 goto err_sw_init; 7146 } 7147 7148 ixgbe_probe_vf(adapter, ii); 7149 7150 netdev->features = NETIF_F_SG | 7151 NETIF_F_IP_CSUM | 7152 NETIF_F_IPV6_CSUM | 7153 NETIF_F_HW_VLAN_TX | 7154 NETIF_F_HW_VLAN_RX | 7155 NETIF_F_HW_VLAN_FILTER | 7156 NETIF_F_TSO | 7157 NETIF_F_TSO6 | 7158 NETIF_F_RXHASH | 7159 NETIF_F_RXCSUM; 7160 7161 netdev->hw_features = netdev->features; 7162 7163 switch (adapter->hw.mac.type) { 7164 case ixgbe_mac_82599EB: 7165 case ixgbe_mac_X540: 7166 netdev->features |= NETIF_F_SCTP_CSUM; 7167 netdev->hw_features |= NETIF_F_SCTP_CSUM | 7168 NETIF_F_NTUPLE; 7169 break; 7170 default: 7171 break; 7172 } 7173 7174 netdev->hw_features |= NETIF_F_RXALL; 7175 7176 netdev->vlan_features |= NETIF_F_TSO; 7177 netdev->vlan_features |= NETIF_F_TSO6; 7178 netdev->vlan_features |= NETIF_F_IP_CSUM; 7179 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 7180 netdev->vlan_features |= NETIF_F_SG; 7181 7182 netdev->priv_flags |= IFF_UNICAST_FLT; 7183 netdev->priv_flags |= IFF_SUPP_NOFCS; 7184 7185 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 7186 adapter->flags &= ~(IXGBE_FLAG_RSS_ENABLED | 7187 IXGBE_FLAG_DCB_ENABLED); 7188 7189 #ifdef CONFIG_IXGBE_DCB 7190 netdev->dcbnl_ops = &dcbnl_ops; 7191 #endif 7192 7193 #ifdef IXGBE_FCOE 7194 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) { 7195 if (hw->mac.ops.get_device_caps) { 7196 hw->mac.ops.get_device_caps(hw, &device_caps); 7197 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS) 7198 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 7199 } 7200 } 7201 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) { 7202 netdev->vlan_features |= NETIF_F_FCOE_CRC; 7203 netdev->vlan_features |= NETIF_F_FSO; 7204 netdev->vlan_features |= NETIF_F_FCOE_MTU; 7205 } 7206 #endif /* IXGBE_FCOE */ 7207 if (pci_using_dac) { 7208 netdev->features |= NETIF_F_HIGHDMA; 7209 netdev->vlan_features |= NETIF_F_HIGHDMA; 7210 } 7211 7212 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) 7213 netdev->hw_features |= NETIF_F_LRO; 7214 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 7215 netdev->features |= NETIF_F_LRO; 7216 7217 /* make sure the EEPROM is good */ 7218 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) { 7219 e_dev_err("The EEPROM Checksum Is Not Valid\n"); 7220 err = -EIO; 7221 goto err_sw_init; 7222 } 7223 7224 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len); 7225 memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len); 7226 7227 if (ixgbe_validate_mac_addr(netdev->perm_addr)) { 7228 e_dev_err("invalid MAC address\n"); 7229 err = -EIO; 7230 goto err_sw_init; 7231 } 7232 7233 setup_timer(&adapter->service_timer, &ixgbe_service_timer, 7234 (unsigned long) adapter); 7235 7236 INIT_WORK(&adapter->service_task, ixgbe_service_task); 7237 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 7238 7239 err = ixgbe_init_interrupt_scheme(adapter); 7240 if (err) 7241 goto err_sw_init; 7242 7243 if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) { 7244 netdev->hw_features &= ~NETIF_F_RXHASH; 7245 netdev->features &= ~NETIF_F_RXHASH; 7246 } 7247 7248 /* WOL not supported for all devices */ 7249 adapter->wol = 0; 7250 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap); 7251 if (ixgbe_wol_supported(adapter, pdev->device, pdev->subsystem_device)) 7252 adapter->wol = IXGBE_WUFC_MAG; 7253 7254 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); 7255 7256 #ifdef CONFIG_IXGBE_PTP 7257 ixgbe_ptp_init(adapter); 7258 #endif /* CONFIG_IXGBE_PTP*/ 7259 7260 /* save off EEPROM version number */ 7261 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh); 7262 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl); 7263 7264 /* pick up the PCI bus settings for reporting later */ 7265 hw->mac.ops.get_bus_info(hw); 7266 7267 /* print bus type/speed/width info */ 7268 e_dev_info("(PCI Express:%s:%s) %pM\n", 7269 (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : 7270 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : 7271 "Unknown"), 7272 (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : 7273 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : 7274 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : 7275 "Unknown"), 7276 netdev->dev_addr); 7277 7278 err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH); 7279 if (err) 7280 strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH); 7281 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 7282 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n", 7283 hw->mac.type, hw->phy.type, hw->phy.sfp_type, 7284 part_str); 7285 else 7286 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n", 7287 hw->mac.type, hw->phy.type, part_str); 7288 7289 if (hw->bus.width <= ixgbe_bus_width_pcie_x4) { 7290 e_dev_warn("PCI-Express bandwidth available for this card is " 7291 "not sufficient for optimal performance.\n"); 7292 e_dev_warn("For optimal performance a x8 PCI-Express slot " 7293 "is required.\n"); 7294 } 7295 7296 /* reset the hardware with the new settings */ 7297 err = hw->mac.ops.start_hw(hw); 7298 if (err == IXGBE_ERR_EEPROM_VERSION) { 7299 /* We are running on a pre-production device, log a warning */ 7300 e_dev_warn("This device is a pre-production adapter/LOM. " 7301 "Please be aware there may be issues associated " 7302 "with your hardware. If you are experiencing " 7303 "problems please contact your Intel or hardware " 7304 "representative who provided you with this " 7305 "hardware.\n"); 7306 } 7307 strcpy(netdev->name, "eth%d"); 7308 err = register_netdev(netdev); 7309 if (err) 7310 goto err_register; 7311 7312 /* power down the optics for multispeed fiber and 82599 SFP+ fiber */ 7313 if (hw->mac.ops.disable_tx_laser && 7314 ((hw->phy.multispeed_fiber) || 7315 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && 7316 (hw->mac.type == ixgbe_mac_82599EB)))) 7317 hw->mac.ops.disable_tx_laser(hw); 7318 7319 /* carrier off reporting is important to ethtool even BEFORE open */ 7320 netif_carrier_off(netdev); 7321 7322 #ifdef CONFIG_IXGBE_DCA 7323 if (dca_add_requester(&pdev->dev) == 0) { 7324 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 7325 ixgbe_setup_dca(adapter); 7326 } 7327 #endif 7328 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 7329 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs); 7330 for (i = 0; i < adapter->num_vfs; i++) 7331 ixgbe_vf_configuration(pdev, (i | 0x10000000)); 7332 } 7333 7334 /* firmware requires driver version to be 0xFFFFFFFF 7335 * since os does not support feature 7336 */ 7337 if (hw->mac.ops.set_fw_drv_ver) 7338 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 7339 0xFF); 7340 7341 /* add san mac addr to netdev */ 7342 ixgbe_add_sanmac_netdev(netdev); 7343 7344 e_dev_info("%s\n", ixgbe_default_device_descr); 7345 cards_found++; 7346 7347 #ifdef CONFIG_IXGBE_HWMON 7348 if (ixgbe_sysfs_init(adapter)) 7349 e_err(probe, "failed to allocate sysfs resources\n"); 7350 #endif /* CONFIG_IXGBE_HWMON */ 7351 7352 return 0; 7353 7354 err_register: 7355 ixgbe_release_hw_control(adapter); 7356 ixgbe_clear_interrupt_scheme(adapter); 7357 err_sw_init: 7358 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 7359 ixgbe_disable_sriov(adapter); 7360 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 7361 iounmap(hw->hw_addr); 7362 err_ioremap: 7363 free_netdev(netdev); 7364 err_alloc_etherdev: 7365 pci_release_selected_regions(pdev, 7366 pci_select_bars(pdev, IORESOURCE_MEM)); 7367 err_pci_reg: 7368 err_dma: 7369 pci_disable_device(pdev); 7370 return err; 7371 } 7372 7373 /** 7374 * ixgbe_remove - Device Removal Routine 7375 * @pdev: PCI device information struct 7376 * 7377 * ixgbe_remove is called by the PCI subsystem to alert the driver 7378 * that it should release a PCI device. The could be caused by a 7379 * Hot-Plug event, or because the driver is going to be removed from 7380 * memory. 7381 **/ 7382 static void __devexit ixgbe_remove(struct pci_dev *pdev) 7383 { 7384 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7385 struct net_device *netdev = adapter->netdev; 7386 7387 set_bit(__IXGBE_DOWN, &adapter->state); 7388 cancel_work_sync(&adapter->service_task); 7389 7390 #ifdef CONFIG_IXGBE_PTP 7391 ixgbe_ptp_stop(adapter); 7392 #endif 7393 7394 #ifdef CONFIG_IXGBE_DCA 7395 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 7396 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 7397 dca_remove_requester(&pdev->dev); 7398 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); 7399 } 7400 7401 #endif 7402 #ifdef CONFIG_IXGBE_HWMON 7403 ixgbe_sysfs_exit(adapter); 7404 #endif /* CONFIG_IXGBE_HWMON */ 7405 7406 #ifdef IXGBE_FCOE 7407 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) 7408 ixgbe_cleanup_fcoe(adapter); 7409 7410 #endif /* IXGBE_FCOE */ 7411 7412 /* remove the added san mac */ 7413 ixgbe_del_sanmac_netdev(netdev); 7414 7415 if (netdev->reg_state == NETREG_REGISTERED) 7416 unregister_netdev(netdev); 7417 7418 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 7419 if (!(ixgbe_check_vf_assignment(adapter))) 7420 ixgbe_disable_sriov(adapter); 7421 else 7422 e_dev_warn("Unloading driver while VFs are assigned " 7423 "- VFs will not be deallocated\n"); 7424 } 7425 7426 ixgbe_clear_interrupt_scheme(adapter); 7427 7428 ixgbe_release_hw_control(adapter); 7429 7430 #ifdef CONFIG_DCB 7431 kfree(adapter->ixgbe_ieee_pfc); 7432 kfree(adapter->ixgbe_ieee_ets); 7433 7434 #endif 7435 iounmap(adapter->hw.hw_addr); 7436 pci_release_selected_regions(pdev, pci_select_bars(pdev, 7437 IORESOURCE_MEM)); 7438 7439 e_dev_info("complete\n"); 7440 7441 free_netdev(netdev); 7442 7443 pci_disable_pcie_error_reporting(pdev); 7444 7445 pci_disable_device(pdev); 7446 } 7447 7448 /** 7449 * ixgbe_io_error_detected - called when PCI error is detected 7450 * @pdev: Pointer to PCI device 7451 * @state: The current pci connection state 7452 * 7453 * This function is called after a PCI bus error affecting 7454 * this device has been detected. 7455 */ 7456 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, 7457 pci_channel_state_t state) 7458 { 7459 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7460 struct net_device *netdev = adapter->netdev; 7461 7462 #ifdef CONFIG_PCI_IOV 7463 struct pci_dev *bdev, *vfdev; 7464 u32 dw0, dw1, dw2, dw3; 7465 int vf, pos; 7466 u16 req_id, pf_func; 7467 7468 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 7469 adapter->num_vfs == 0) 7470 goto skip_bad_vf_detection; 7471 7472 bdev = pdev->bus->self; 7473 while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT)) 7474 bdev = bdev->bus->self; 7475 7476 if (!bdev) 7477 goto skip_bad_vf_detection; 7478 7479 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR); 7480 if (!pos) 7481 goto skip_bad_vf_detection; 7482 7483 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0); 7484 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1); 7485 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2); 7486 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3); 7487 7488 req_id = dw1 >> 16; 7489 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */ 7490 if (!(req_id & 0x0080)) 7491 goto skip_bad_vf_detection; 7492 7493 pf_func = req_id & 0x01; 7494 if ((pf_func & 1) == (pdev->devfn & 1)) { 7495 unsigned int device_id; 7496 7497 vf = (req_id & 0x7F) >> 1; 7498 e_dev_err("VF %d has caused a PCIe error\n", vf); 7499 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: " 7500 "%8.8x\tdw3: %8.8x\n", 7501 dw0, dw1, dw2, dw3); 7502 switch (adapter->hw.mac.type) { 7503 case ixgbe_mac_82599EB: 7504 device_id = IXGBE_82599_VF_DEVICE_ID; 7505 break; 7506 case ixgbe_mac_X540: 7507 device_id = IXGBE_X540_VF_DEVICE_ID; 7508 break; 7509 default: 7510 device_id = 0; 7511 break; 7512 } 7513 7514 /* Find the pci device of the offending VF */ 7515 vfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID, device_id, NULL); 7516 while (vfdev) { 7517 if (vfdev->devfn == (req_id & 0xFF)) 7518 break; 7519 vfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID, 7520 device_id, vfdev); 7521 } 7522 /* 7523 * There's a slim chance the VF could have been hot plugged, 7524 * so if it is no longer present we don't need to issue the 7525 * VFLR. Just clean up the AER in that case. 7526 */ 7527 if (vfdev) { 7528 e_dev_err("Issuing VFLR to VF %d\n", vf); 7529 pci_write_config_dword(vfdev, 0xA8, 0x00008000); 7530 } 7531 7532 pci_cleanup_aer_uncorrect_error_status(pdev); 7533 } 7534 7535 /* 7536 * Even though the error may have occurred on the other port 7537 * we still need to increment the vf error reference count for 7538 * both ports because the I/O resume function will be called 7539 * for both of them. 7540 */ 7541 adapter->vferr_refcount++; 7542 7543 return PCI_ERS_RESULT_RECOVERED; 7544 7545 skip_bad_vf_detection: 7546 #endif /* CONFIG_PCI_IOV */ 7547 netif_device_detach(netdev); 7548 7549 if (state == pci_channel_io_perm_failure) 7550 return PCI_ERS_RESULT_DISCONNECT; 7551 7552 if (netif_running(netdev)) 7553 ixgbe_down(adapter); 7554 pci_disable_device(pdev); 7555 7556 /* Request a slot reset. */ 7557 return PCI_ERS_RESULT_NEED_RESET; 7558 } 7559 7560 /** 7561 * ixgbe_io_slot_reset - called after the pci bus has been reset. 7562 * @pdev: Pointer to PCI device 7563 * 7564 * Restart the card from scratch, as if from a cold-boot. 7565 */ 7566 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) 7567 { 7568 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7569 pci_ers_result_t result; 7570 int err; 7571 7572 if (pci_enable_device_mem(pdev)) { 7573 e_err(probe, "Cannot re-enable PCI device after reset.\n"); 7574 result = PCI_ERS_RESULT_DISCONNECT; 7575 } else { 7576 pci_set_master(pdev); 7577 pci_restore_state(pdev); 7578 pci_save_state(pdev); 7579 7580 pci_wake_from_d3(pdev, false); 7581 7582 ixgbe_reset(adapter); 7583 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 7584 result = PCI_ERS_RESULT_RECOVERED; 7585 } 7586 7587 err = pci_cleanup_aer_uncorrect_error_status(pdev); 7588 if (err) { 7589 e_dev_err("pci_cleanup_aer_uncorrect_error_status " 7590 "failed 0x%0x\n", err); 7591 /* non-fatal, continue */ 7592 } 7593 7594 return result; 7595 } 7596 7597 /** 7598 * ixgbe_io_resume - called when traffic can start flowing again. 7599 * @pdev: Pointer to PCI device 7600 * 7601 * This callback is called when the error recovery driver tells us that 7602 * its OK to resume normal operation. 7603 */ 7604 static void ixgbe_io_resume(struct pci_dev *pdev) 7605 { 7606 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7607 struct net_device *netdev = adapter->netdev; 7608 7609 #ifdef CONFIG_PCI_IOV 7610 if (adapter->vferr_refcount) { 7611 e_info(drv, "Resuming after VF err\n"); 7612 adapter->vferr_refcount--; 7613 return; 7614 } 7615 7616 #endif 7617 if (netif_running(netdev)) 7618 ixgbe_up(adapter); 7619 7620 netif_device_attach(netdev); 7621 } 7622 7623 static struct pci_error_handlers ixgbe_err_handler = { 7624 .error_detected = ixgbe_io_error_detected, 7625 .slot_reset = ixgbe_io_slot_reset, 7626 .resume = ixgbe_io_resume, 7627 }; 7628 7629 static struct pci_driver ixgbe_driver = { 7630 .name = ixgbe_driver_name, 7631 .id_table = ixgbe_pci_tbl, 7632 .probe = ixgbe_probe, 7633 .remove = __devexit_p(ixgbe_remove), 7634 #ifdef CONFIG_PM 7635 .suspend = ixgbe_suspend, 7636 .resume = ixgbe_resume, 7637 #endif 7638 .shutdown = ixgbe_shutdown, 7639 .err_handler = &ixgbe_err_handler 7640 }; 7641 7642 /** 7643 * ixgbe_init_module - Driver Registration Routine 7644 * 7645 * ixgbe_init_module is the first routine called when the driver is 7646 * loaded. All it does is register with the PCI subsystem. 7647 **/ 7648 static int __init ixgbe_init_module(void) 7649 { 7650 int ret; 7651 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version); 7652 pr_info("%s\n", ixgbe_copyright); 7653 7654 #ifdef CONFIG_IXGBE_DCA 7655 dca_register_notify(&dca_notifier); 7656 #endif 7657 7658 ret = pci_register_driver(&ixgbe_driver); 7659 return ret; 7660 } 7661 7662 module_init(ixgbe_init_module); 7663 7664 /** 7665 * ixgbe_exit_module - Driver Exit Cleanup Routine 7666 * 7667 * ixgbe_exit_module is called just before the driver is removed 7668 * from memory. 7669 **/ 7670 static void __exit ixgbe_exit_module(void) 7671 { 7672 #ifdef CONFIG_IXGBE_DCA 7673 dca_unregister_notify(&dca_notifier); 7674 #endif 7675 pci_unregister_driver(&ixgbe_driver); 7676 rcu_barrier(); /* Wait for completion of call_rcu()'s */ 7677 } 7678 7679 #ifdef CONFIG_IXGBE_DCA 7680 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, 7681 void *p) 7682 { 7683 int ret_val; 7684 7685 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event, 7686 __ixgbe_notify_dca); 7687 7688 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 7689 } 7690 7691 #endif /* CONFIG_IXGBE_DCA */ 7692 7693 module_exit(ixgbe_exit_module); 7694 7695 /* ixgbe_main.c */ 7696