1 /******************************************************************************* 2 3 Intel 10 Gigabit PCI Express Linux driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26 *******************************************************************************/ 27 28 #include <linux/types.h> 29 #include <linux/module.h> 30 #include <linux/pci.h> 31 #include <linux/netdevice.h> 32 #include <linux/vmalloc.h> 33 #include <linux/string.h> 34 #include <linux/in.h> 35 #include <linux/interrupt.h> 36 #include <linux/ip.h> 37 #include <linux/tcp.h> 38 #include <linux/sctp.h> 39 #include <linux/pkt_sched.h> 40 #include <linux/ipv6.h> 41 #include <linux/slab.h> 42 #include <net/checksum.h> 43 #include <net/ip6_checksum.h> 44 #include <linux/ethtool.h> 45 #include <linux/if.h> 46 #include <linux/if_vlan.h> 47 #include <linux/prefetch.h> 48 #include <scsi/fc/fc_fcoe.h> 49 50 #include "ixgbe.h" 51 #include "ixgbe_common.h" 52 #include "ixgbe_dcb_82599.h" 53 #include "ixgbe_sriov.h" 54 55 char ixgbe_driver_name[] = "ixgbe"; 56 static const char ixgbe_driver_string[] = 57 "Intel(R) 10 Gigabit PCI Express Network Driver"; 58 #ifdef IXGBE_FCOE 59 char ixgbe_default_device_descr[] = 60 "Intel(R) 10 Gigabit Network Connection"; 61 #else 62 static char ixgbe_default_device_descr[] = 63 "Intel(R) 10 Gigabit Network Connection"; 64 #endif 65 #define MAJ 3 66 #define MIN 9 67 #define BUILD 15 68 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \ 69 __stringify(BUILD) "-k" 70 const char ixgbe_driver_version[] = DRV_VERSION; 71 static const char ixgbe_copyright[] = 72 "Copyright (c) 1999-2012 Intel Corporation."; 73 74 static const struct ixgbe_info *ixgbe_info_tbl[] = { 75 [board_82598] = &ixgbe_82598_info, 76 [board_82599] = &ixgbe_82599_info, 77 [board_X540] = &ixgbe_X540_info, 78 }; 79 80 /* ixgbe_pci_tbl - PCI Device ID Table 81 * 82 * Wildcard entries (PCI_ANY_ID) should come last 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 86 * Class, Class Mask, private data (not used) } 87 */ 88 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = { 89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 }, 90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 }, 91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 }, 92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 }, 93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 }, 94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 }, 95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 }, 96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 }, 97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 }, 98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 }, 99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 }, 100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 }, 101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 }, 102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 }, 103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 }, 104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 }, 105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 }, 106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 }, 107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 }, 108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 }, 109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 }, 110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 }, 111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 }, 112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 }, 113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 }, 114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 }, 115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 }, 116 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 }, 117 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 }, 118 /* required last entry */ 119 {0, } 120 }; 121 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); 122 123 #ifdef CONFIG_IXGBE_DCA 124 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, 125 void *p); 126 static struct notifier_block dca_notifier = { 127 .notifier_call = ixgbe_notify_dca, 128 .next = NULL, 129 .priority = 0 130 }; 131 #endif 132 133 #ifdef CONFIG_PCI_IOV 134 static unsigned int max_vfs; 135 module_param(max_vfs, uint, 0); 136 MODULE_PARM_DESC(max_vfs, 137 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63"); 138 #endif /* CONFIG_PCI_IOV */ 139 140 static unsigned int allow_unsupported_sfp; 141 module_param(allow_unsupported_sfp, uint, 0); 142 MODULE_PARM_DESC(allow_unsupported_sfp, 143 "Allow unsupported and untested SFP+ modules on 82599-based adapters"); 144 145 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 146 static int debug = -1; 147 module_param(debug, int, 0); 148 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 149 150 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 151 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); 152 MODULE_LICENSE("GPL"); 153 MODULE_VERSION(DRV_VERSION); 154 155 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) 156 { 157 if (!test_bit(__IXGBE_DOWN, &adapter->state) && 158 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) 159 schedule_work(&adapter->service_task); 160 } 161 162 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter) 163 { 164 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state)); 165 166 /* flush memory to make sure state is correct before next watchdog */ 167 smp_mb__before_clear_bit(); 168 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 169 } 170 171 struct ixgbe_reg_info { 172 u32 ofs; 173 char *name; 174 }; 175 176 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = { 177 178 /* General Registers */ 179 {IXGBE_CTRL, "CTRL"}, 180 {IXGBE_STATUS, "STATUS"}, 181 {IXGBE_CTRL_EXT, "CTRL_EXT"}, 182 183 /* Interrupt Registers */ 184 {IXGBE_EICR, "EICR"}, 185 186 /* RX Registers */ 187 {IXGBE_SRRCTL(0), "SRRCTL"}, 188 {IXGBE_DCA_RXCTRL(0), "DRXCTL"}, 189 {IXGBE_RDLEN(0), "RDLEN"}, 190 {IXGBE_RDH(0), "RDH"}, 191 {IXGBE_RDT(0), "RDT"}, 192 {IXGBE_RXDCTL(0), "RXDCTL"}, 193 {IXGBE_RDBAL(0), "RDBAL"}, 194 {IXGBE_RDBAH(0), "RDBAH"}, 195 196 /* TX Registers */ 197 {IXGBE_TDBAL(0), "TDBAL"}, 198 {IXGBE_TDBAH(0), "TDBAH"}, 199 {IXGBE_TDLEN(0), "TDLEN"}, 200 {IXGBE_TDH(0), "TDH"}, 201 {IXGBE_TDT(0), "TDT"}, 202 {IXGBE_TXDCTL(0), "TXDCTL"}, 203 204 /* List Terminator */ 205 {} 206 }; 207 208 209 /* 210 * ixgbe_regdump - register printout routine 211 */ 212 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo) 213 { 214 int i = 0, j = 0; 215 char rname[16]; 216 u32 regs[64]; 217 218 switch (reginfo->ofs) { 219 case IXGBE_SRRCTL(0): 220 for (i = 0; i < 64; i++) 221 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 222 break; 223 case IXGBE_DCA_RXCTRL(0): 224 for (i = 0; i < 64; i++) 225 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i)); 226 break; 227 case IXGBE_RDLEN(0): 228 for (i = 0; i < 64; i++) 229 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i)); 230 break; 231 case IXGBE_RDH(0): 232 for (i = 0; i < 64; i++) 233 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i)); 234 break; 235 case IXGBE_RDT(0): 236 for (i = 0; i < 64; i++) 237 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i)); 238 break; 239 case IXGBE_RXDCTL(0): 240 for (i = 0; i < 64; i++) 241 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 242 break; 243 case IXGBE_RDBAL(0): 244 for (i = 0; i < 64; i++) 245 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i)); 246 break; 247 case IXGBE_RDBAH(0): 248 for (i = 0; i < 64; i++) 249 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i)); 250 break; 251 case IXGBE_TDBAL(0): 252 for (i = 0; i < 64; i++) 253 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i)); 254 break; 255 case IXGBE_TDBAH(0): 256 for (i = 0; i < 64; i++) 257 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i)); 258 break; 259 case IXGBE_TDLEN(0): 260 for (i = 0; i < 64; i++) 261 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i)); 262 break; 263 case IXGBE_TDH(0): 264 for (i = 0; i < 64; i++) 265 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i)); 266 break; 267 case IXGBE_TDT(0): 268 for (i = 0; i < 64; i++) 269 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i)); 270 break; 271 case IXGBE_TXDCTL(0): 272 for (i = 0; i < 64; i++) 273 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 274 break; 275 default: 276 pr_info("%-15s %08x\n", reginfo->name, 277 IXGBE_READ_REG(hw, reginfo->ofs)); 278 return; 279 } 280 281 for (i = 0; i < 8; i++) { 282 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7); 283 pr_err("%-15s", rname); 284 for (j = 0; j < 8; j++) 285 pr_cont(" %08x", regs[i*8+j]); 286 pr_cont("\n"); 287 } 288 289 } 290 291 /* 292 * ixgbe_dump - Print registers, tx-rings and rx-rings 293 */ 294 static void ixgbe_dump(struct ixgbe_adapter *adapter) 295 { 296 struct net_device *netdev = adapter->netdev; 297 struct ixgbe_hw *hw = &adapter->hw; 298 struct ixgbe_reg_info *reginfo; 299 int n = 0; 300 struct ixgbe_ring *tx_ring; 301 struct ixgbe_tx_buffer *tx_buffer; 302 union ixgbe_adv_tx_desc *tx_desc; 303 struct my_u0 { u64 a; u64 b; } *u0; 304 struct ixgbe_ring *rx_ring; 305 union ixgbe_adv_rx_desc *rx_desc; 306 struct ixgbe_rx_buffer *rx_buffer_info; 307 u32 staterr; 308 int i = 0; 309 310 if (!netif_msg_hw(adapter)) 311 return; 312 313 /* Print netdevice Info */ 314 if (netdev) { 315 dev_info(&adapter->pdev->dev, "Net device Info\n"); 316 pr_info("Device Name state " 317 "trans_start last_rx\n"); 318 pr_info("%-15s %016lX %016lX %016lX\n", 319 netdev->name, 320 netdev->state, 321 netdev->trans_start, 322 netdev->last_rx); 323 } 324 325 /* Print Registers */ 326 dev_info(&adapter->pdev->dev, "Register Dump\n"); 327 pr_info(" Register Name Value\n"); 328 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; 329 reginfo->name; reginfo++) { 330 ixgbe_regdump(hw, reginfo); 331 } 332 333 /* Print TX Ring Summary */ 334 if (!netdev || !netif_running(netdev)) 335 goto exit; 336 337 dev_info(&adapter->pdev->dev, "TX Rings Summary\n"); 338 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n"); 339 for (n = 0; n < adapter->num_tx_queues; n++) { 340 tx_ring = adapter->tx_ring[n]; 341 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; 342 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n", 343 n, tx_ring->next_to_use, tx_ring->next_to_clean, 344 (u64)dma_unmap_addr(tx_buffer, dma), 345 dma_unmap_len(tx_buffer, len), 346 tx_buffer->next_to_watch, 347 (u64)tx_buffer->time_stamp); 348 } 349 350 /* Print TX Rings */ 351 if (!netif_msg_tx_done(adapter)) 352 goto rx_ring_summary; 353 354 dev_info(&adapter->pdev->dev, "TX Rings Dump\n"); 355 356 /* Transmit Descriptor Formats 357 * 358 * Advanced Transmit Descriptor 359 * +--------------------------------------------------------------+ 360 * 0 | Buffer Address [63:0] | 361 * +--------------------------------------------------------------+ 362 * 8 | PAYLEN | PORTS | IDX | STA | DCMD |DTYP | RSV | DTALEN | 363 * +--------------------------------------------------------------+ 364 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0 365 */ 366 367 for (n = 0; n < adapter->num_tx_queues; n++) { 368 tx_ring = adapter->tx_ring[n]; 369 pr_info("------------------------------------\n"); 370 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index); 371 pr_info("------------------------------------\n"); 372 pr_info("T [desc] [address 63:0 ] " 373 "[PlPOIdStDDt Ln] [bi->dma ] " 374 "leng ntw timestamp bi->skb\n"); 375 376 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { 377 tx_desc = IXGBE_TX_DESC(tx_ring, i); 378 tx_buffer = &tx_ring->tx_buffer_info[i]; 379 u0 = (struct my_u0 *)tx_desc; 380 pr_info("T [0x%03X] %016llX %016llX %016llX" 381 " %04X %p %016llX %p", i, 382 le64_to_cpu(u0->a), 383 le64_to_cpu(u0->b), 384 (u64)dma_unmap_addr(tx_buffer, dma), 385 dma_unmap_len(tx_buffer, len), 386 tx_buffer->next_to_watch, 387 (u64)tx_buffer->time_stamp, 388 tx_buffer->skb); 389 if (i == tx_ring->next_to_use && 390 i == tx_ring->next_to_clean) 391 pr_cont(" NTC/U\n"); 392 else if (i == tx_ring->next_to_use) 393 pr_cont(" NTU\n"); 394 else if (i == tx_ring->next_to_clean) 395 pr_cont(" NTC\n"); 396 else 397 pr_cont("\n"); 398 399 if (netif_msg_pktdata(adapter) && 400 tx_buffer->skb) 401 print_hex_dump(KERN_INFO, "", 402 DUMP_PREFIX_ADDRESS, 16, 1, 403 tx_buffer->skb->data, 404 dma_unmap_len(tx_buffer, len), 405 true); 406 } 407 } 408 409 /* Print RX Rings Summary */ 410 rx_ring_summary: 411 dev_info(&adapter->pdev->dev, "RX Rings Summary\n"); 412 pr_info("Queue [NTU] [NTC]\n"); 413 for (n = 0; n < adapter->num_rx_queues; n++) { 414 rx_ring = adapter->rx_ring[n]; 415 pr_info("%5d %5X %5X\n", 416 n, rx_ring->next_to_use, rx_ring->next_to_clean); 417 } 418 419 /* Print RX Rings */ 420 if (!netif_msg_rx_status(adapter)) 421 goto exit; 422 423 dev_info(&adapter->pdev->dev, "RX Rings Dump\n"); 424 425 /* Advanced Receive Descriptor (Read) Format 426 * 63 1 0 427 * +-----------------------------------------------------+ 428 * 0 | Packet Buffer Address [63:1] |A0/NSE| 429 * +----------------------------------------------+------+ 430 * 8 | Header Buffer Address [63:1] | DD | 431 * +-----------------------------------------------------+ 432 * 433 * 434 * Advanced Receive Descriptor (Write-Back) Format 435 * 436 * 63 48 47 32 31 30 21 20 16 15 4 3 0 437 * +------------------------------------------------------+ 438 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | 439 * | Checksum Ident | | | | Type | Type | 440 * +------------------------------------------------------+ 441 * 8 | VLAN Tag | Length | Extended Error | Extended Status | 442 * +------------------------------------------------------+ 443 * 63 48 47 32 31 20 19 0 444 */ 445 for (n = 0; n < adapter->num_rx_queues; n++) { 446 rx_ring = adapter->rx_ring[n]; 447 pr_info("------------------------------------\n"); 448 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); 449 pr_info("------------------------------------\n"); 450 pr_info("R [desc] [ PktBuf A0] " 451 "[ HeadBuf DD] [bi->dma ] [bi->skb] " 452 "<-- Adv Rx Read format\n"); 453 pr_info("RWB[desc] [PcsmIpSHl PtRs] " 454 "[vl er S cks ln] ---------------- [bi->skb] " 455 "<-- Adv Rx Write-Back format\n"); 456 457 for (i = 0; i < rx_ring->count; i++) { 458 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 459 rx_desc = IXGBE_RX_DESC(rx_ring, i); 460 u0 = (struct my_u0 *)rx_desc; 461 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 462 if (staterr & IXGBE_RXD_STAT_DD) { 463 /* Descriptor Done */ 464 pr_info("RWB[0x%03X] %016llX " 465 "%016llX ---------------- %p", i, 466 le64_to_cpu(u0->a), 467 le64_to_cpu(u0->b), 468 rx_buffer_info->skb); 469 } else { 470 pr_info("R [0x%03X] %016llX " 471 "%016llX %016llX %p", i, 472 le64_to_cpu(u0->a), 473 le64_to_cpu(u0->b), 474 (u64)rx_buffer_info->dma, 475 rx_buffer_info->skb); 476 477 if (netif_msg_pktdata(adapter) && 478 rx_buffer_info->dma) { 479 print_hex_dump(KERN_INFO, "", 480 DUMP_PREFIX_ADDRESS, 16, 1, 481 page_address(rx_buffer_info->page) + 482 rx_buffer_info->page_offset, 483 ixgbe_rx_bufsz(rx_ring), true); 484 } 485 } 486 487 if (i == rx_ring->next_to_use) 488 pr_cont(" NTU\n"); 489 else if (i == rx_ring->next_to_clean) 490 pr_cont(" NTC\n"); 491 else 492 pr_cont("\n"); 493 494 } 495 } 496 497 exit: 498 return; 499 } 500 501 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter) 502 { 503 u32 ctrl_ext; 504 505 /* Let firmware take over control of h/w */ 506 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 507 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 508 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD); 509 } 510 511 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter) 512 { 513 u32 ctrl_ext; 514 515 /* Let firmware know the driver has taken over */ 516 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 517 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, 518 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD); 519 } 520 521 /** 522 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors 523 * @adapter: pointer to adapter struct 524 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 525 * @queue: queue to map the corresponding interrupt to 526 * @msix_vector: the vector to map to the corresponding queue 527 * 528 */ 529 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, 530 u8 queue, u8 msix_vector) 531 { 532 u32 ivar, index; 533 struct ixgbe_hw *hw = &adapter->hw; 534 switch (hw->mac.type) { 535 case ixgbe_mac_82598EB: 536 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 537 if (direction == -1) 538 direction = 0; 539 index = (((direction * 64) + queue) >> 2) & 0x1F; 540 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 541 ivar &= ~(0xFF << (8 * (queue & 0x3))); 542 ivar |= (msix_vector << (8 * (queue & 0x3))); 543 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 544 break; 545 case ixgbe_mac_82599EB: 546 case ixgbe_mac_X540: 547 if (direction == -1) { 548 /* other causes */ 549 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 550 index = ((queue & 1) * 8); 551 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC); 552 ivar &= ~(0xFF << index); 553 ivar |= (msix_vector << index); 554 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar); 555 break; 556 } else { 557 /* tx or rx causes */ 558 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 559 index = ((16 * (queue & 1)) + (8 * direction)); 560 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1)); 561 ivar &= ~(0xFF << index); 562 ivar |= (msix_vector << index); 563 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar); 564 break; 565 } 566 default: 567 break; 568 } 569 } 570 571 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, 572 u64 qmask) 573 { 574 u32 mask; 575 576 switch (adapter->hw.mac.type) { 577 case ixgbe_mac_82598EB: 578 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 579 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 580 break; 581 case ixgbe_mac_82599EB: 582 case ixgbe_mac_X540: 583 mask = (qmask & 0xFFFFFFFF); 584 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 585 mask = (qmask >> 32); 586 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 587 break; 588 default: 589 break; 590 } 591 } 592 593 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring, 594 struct ixgbe_tx_buffer *tx_buffer) 595 { 596 if (tx_buffer->skb) { 597 dev_kfree_skb_any(tx_buffer->skb); 598 if (dma_unmap_len(tx_buffer, len)) 599 dma_unmap_single(ring->dev, 600 dma_unmap_addr(tx_buffer, dma), 601 dma_unmap_len(tx_buffer, len), 602 DMA_TO_DEVICE); 603 } else if (dma_unmap_len(tx_buffer, len)) { 604 dma_unmap_page(ring->dev, 605 dma_unmap_addr(tx_buffer, dma), 606 dma_unmap_len(tx_buffer, len), 607 DMA_TO_DEVICE); 608 } 609 tx_buffer->next_to_watch = NULL; 610 tx_buffer->skb = NULL; 611 dma_unmap_len_set(tx_buffer, len, 0); 612 /* tx_buffer must be completely set up in the transmit path */ 613 } 614 615 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) 616 { 617 struct ixgbe_hw *hw = &adapter->hw; 618 struct ixgbe_hw_stats *hwstats = &adapter->stats; 619 int i; 620 u32 data; 621 622 if ((hw->fc.current_mode != ixgbe_fc_full) && 623 (hw->fc.current_mode != ixgbe_fc_rx_pause)) 624 return; 625 626 switch (hw->mac.type) { 627 case ixgbe_mac_82598EB: 628 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 629 break; 630 default: 631 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 632 } 633 hwstats->lxoffrxc += data; 634 635 /* refill credits (no tx hang) if we received xoff */ 636 if (!data) 637 return; 638 639 for (i = 0; i < adapter->num_tx_queues; i++) 640 clear_bit(__IXGBE_HANG_CHECK_ARMED, 641 &adapter->tx_ring[i]->state); 642 } 643 644 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) 645 { 646 struct ixgbe_hw *hw = &adapter->hw; 647 struct ixgbe_hw_stats *hwstats = &adapter->stats; 648 u32 xoff[8] = {0}; 649 int i; 650 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 651 652 if (adapter->ixgbe_ieee_pfc) 653 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 654 655 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) { 656 ixgbe_update_xoff_rx_lfc(adapter); 657 return; 658 } 659 660 /* update stats for each tc, only valid with PFC enabled */ 661 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) { 662 switch (hw->mac.type) { 663 case ixgbe_mac_82598EB: 664 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 665 break; 666 default: 667 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i)); 668 } 669 hwstats->pxoffrxc[i] += xoff[i]; 670 } 671 672 /* disarm tx queues that have received xoff frames */ 673 for (i = 0; i < adapter->num_tx_queues; i++) { 674 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 675 u8 tc = tx_ring->dcb_tc; 676 677 if (xoff[tc]) 678 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 679 } 680 } 681 682 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring) 683 { 684 return ring->stats.packets; 685 } 686 687 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring) 688 { 689 struct ixgbe_adapter *adapter = netdev_priv(ring->netdev); 690 struct ixgbe_hw *hw = &adapter->hw; 691 692 u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx)); 693 u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx)); 694 695 if (head != tail) 696 return (head < tail) ? 697 tail - head : (tail + ring->count - head); 698 699 return 0; 700 } 701 702 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring) 703 { 704 u32 tx_done = ixgbe_get_tx_completed(tx_ring); 705 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 706 u32 tx_pending = ixgbe_get_tx_pending(tx_ring); 707 bool ret = false; 708 709 clear_check_for_tx_hang(tx_ring); 710 711 /* 712 * Check for a hung queue, but be thorough. This verifies 713 * that a transmit has been completed since the previous 714 * check AND there is at least one packet pending. The 715 * ARMED bit is set to indicate a potential hang. The 716 * bit is cleared if a pause frame is received to remove 717 * false hang detection due to PFC or 802.3x frames. By 718 * requiring this to fail twice we avoid races with 719 * pfc clearing the ARMED bit and conditions where we 720 * run the check_tx_hang logic with a transmit completion 721 * pending but without time to complete it yet. 722 */ 723 if ((tx_done_old == tx_done) && tx_pending) { 724 /* make sure it is true for two checks in a row */ 725 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED, 726 &tx_ring->state); 727 } else { 728 /* update completed stats and continue */ 729 tx_ring->tx_stats.tx_done_old = tx_done; 730 /* reset the countdown */ 731 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); 732 } 733 734 return ret; 735 } 736 737 /** 738 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout 739 * @adapter: driver private struct 740 **/ 741 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) 742 { 743 744 /* Do the reset outside of interrupt context */ 745 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 746 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; 747 ixgbe_service_event_schedule(adapter); 748 } 749 } 750 751 /** 752 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes 753 * @q_vector: structure containing interrupt and ring information 754 * @tx_ring: tx ring to clean 755 **/ 756 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, 757 struct ixgbe_ring *tx_ring) 758 { 759 struct ixgbe_adapter *adapter = q_vector->adapter; 760 struct ixgbe_tx_buffer *tx_buffer; 761 union ixgbe_adv_tx_desc *tx_desc; 762 unsigned int total_bytes = 0, total_packets = 0; 763 unsigned int budget = q_vector->tx.work_limit; 764 unsigned int i = tx_ring->next_to_clean; 765 766 if (test_bit(__IXGBE_DOWN, &adapter->state)) 767 return true; 768 769 tx_buffer = &tx_ring->tx_buffer_info[i]; 770 tx_desc = IXGBE_TX_DESC(tx_ring, i); 771 i -= tx_ring->count; 772 773 do { 774 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 775 776 /* if next_to_watch is not set then there is no work pending */ 777 if (!eop_desc) 778 break; 779 780 /* prevent any other reads prior to eop_desc */ 781 rmb(); 782 783 /* if DD is not set pending work has not been completed */ 784 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 785 break; 786 787 /* clear next_to_watch to prevent false hangs */ 788 tx_buffer->next_to_watch = NULL; 789 790 /* update the statistics for this packet */ 791 total_bytes += tx_buffer->bytecount; 792 total_packets += tx_buffer->gso_segs; 793 794 #ifdef CONFIG_IXGBE_PTP 795 if (unlikely(tx_buffer->tx_flags & IXGBE_TX_FLAGS_TSTAMP)) 796 ixgbe_ptp_tx_hwtstamp(q_vector, tx_buffer->skb); 797 #endif 798 799 /* free the skb */ 800 dev_kfree_skb_any(tx_buffer->skb); 801 802 /* unmap skb header data */ 803 dma_unmap_single(tx_ring->dev, 804 dma_unmap_addr(tx_buffer, dma), 805 dma_unmap_len(tx_buffer, len), 806 DMA_TO_DEVICE); 807 808 /* clear tx_buffer data */ 809 tx_buffer->skb = NULL; 810 dma_unmap_len_set(tx_buffer, len, 0); 811 812 /* unmap remaining buffers */ 813 while (tx_desc != eop_desc) { 814 tx_buffer++; 815 tx_desc++; 816 i++; 817 if (unlikely(!i)) { 818 i -= tx_ring->count; 819 tx_buffer = tx_ring->tx_buffer_info; 820 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 821 } 822 823 /* unmap any remaining paged data */ 824 if (dma_unmap_len(tx_buffer, len)) { 825 dma_unmap_page(tx_ring->dev, 826 dma_unmap_addr(tx_buffer, dma), 827 dma_unmap_len(tx_buffer, len), 828 DMA_TO_DEVICE); 829 dma_unmap_len_set(tx_buffer, len, 0); 830 } 831 } 832 833 /* move us one more past the eop_desc for start of next pkt */ 834 tx_buffer++; 835 tx_desc++; 836 i++; 837 if (unlikely(!i)) { 838 i -= tx_ring->count; 839 tx_buffer = tx_ring->tx_buffer_info; 840 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 841 } 842 843 /* issue prefetch for next Tx descriptor */ 844 prefetch(tx_desc); 845 846 /* update budget accounting */ 847 budget--; 848 } while (likely(budget)); 849 850 i += tx_ring->count; 851 tx_ring->next_to_clean = i; 852 u64_stats_update_begin(&tx_ring->syncp); 853 tx_ring->stats.bytes += total_bytes; 854 tx_ring->stats.packets += total_packets; 855 u64_stats_update_end(&tx_ring->syncp); 856 q_vector->tx.total_bytes += total_bytes; 857 q_vector->tx.total_packets += total_packets; 858 859 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { 860 /* schedule immediate reset if we believe we hung */ 861 struct ixgbe_hw *hw = &adapter->hw; 862 e_err(drv, "Detected Tx Unit Hang\n" 863 " Tx Queue <%d>\n" 864 " TDH, TDT <%x>, <%x>\n" 865 " next_to_use <%x>\n" 866 " next_to_clean <%x>\n" 867 "tx_buffer_info[next_to_clean]\n" 868 " time_stamp <%lx>\n" 869 " jiffies <%lx>\n", 870 tx_ring->queue_index, 871 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), 872 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), 873 tx_ring->next_to_use, i, 874 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 875 876 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 877 878 e_info(probe, 879 "tx hang %d detected on queue %d, resetting adapter\n", 880 adapter->tx_timeout_count + 1, tx_ring->queue_index); 881 882 /* schedule immediate reset if we believe we hung */ 883 ixgbe_tx_timeout_reset(adapter); 884 885 /* the adapter is about to reset, no point in enabling stuff */ 886 return true; 887 } 888 889 netdev_tx_completed_queue(txring_txq(tx_ring), 890 total_packets, total_bytes); 891 892 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 893 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 894 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 895 /* Make sure that anybody stopping the queue after this 896 * sees the new next_to_clean. 897 */ 898 smp_mb(); 899 if (__netif_subqueue_stopped(tx_ring->netdev, 900 tx_ring->queue_index) 901 && !test_bit(__IXGBE_DOWN, &adapter->state)) { 902 netif_wake_subqueue(tx_ring->netdev, 903 tx_ring->queue_index); 904 ++tx_ring->tx_stats.restart_queue; 905 } 906 } 907 908 return !!budget; 909 } 910 911 #ifdef CONFIG_IXGBE_DCA 912 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, 913 struct ixgbe_ring *tx_ring, 914 int cpu) 915 { 916 struct ixgbe_hw *hw = &adapter->hw; 917 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu); 918 u16 reg_offset; 919 920 switch (hw->mac.type) { 921 case ixgbe_mac_82598EB: 922 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx); 923 break; 924 case ixgbe_mac_82599EB: 925 case ixgbe_mac_X540: 926 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx); 927 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599; 928 break; 929 default: 930 /* for unknown hardware do not write register */ 931 return; 932 } 933 934 /* 935 * We can enable relaxed ordering for reads, but not writes when 936 * DCA is enabled. This is due to a known issue in some chipsets 937 * which will cause the DCA tag to be cleared. 938 */ 939 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN | 940 IXGBE_DCA_TXCTRL_DATA_RRO_EN | 941 IXGBE_DCA_TXCTRL_DESC_DCA_EN; 942 943 IXGBE_WRITE_REG(hw, reg_offset, txctrl); 944 } 945 946 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, 947 struct ixgbe_ring *rx_ring, 948 int cpu) 949 { 950 struct ixgbe_hw *hw = &adapter->hw; 951 u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu); 952 u8 reg_idx = rx_ring->reg_idx; 953 954 955 switch (hw->mac.type) { 956 case ixgbe_mac_82599EB: 957 case ixgbe_mac_X540: 958 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599; 959 break; 960 default: 961 break; 962 } 963 964 /* 965 * We can enable relaxed ordering for reads, but not writes when 966 * DCA is enabled. This is due to a known issue in some chipsets 967 * which will cause the DCA tag to be cleared. 968 */ 969 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN | 970 IXGBE_DCA_RXCTRL_DATA_DCA_EN | 971 IXGBE_DCA_RXCTRL_DESC_DCA_EN; 972 973 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl); 974 } 975 976 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector) 977 { 978 struct ixgbe_adapter *adapter = q_vector->adapter; 979 struct ixgbe_ring *ring; 980 int cpu = get_cpu(); 981 982 if (q_vector->cpu == cpu) 983 goto out_no_update; 984 985 ixgbe_for_each_ring(ring, q_vector->tx) 986 ixgbe_update_tx_dca(adapter, ring, cpu); 987 988 ixgbe_for_each_ring(ring, q_vector->rx) 989 ixgbe_update_rx_dca(adapter, ring, cpu); 990 991 q_vector->cpu = cpu; 992 out_no_update: 993 put_cpu(); 994 } 995 996 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) 997 { 998 int i; 999 1000 if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) 1001 return; 1002 1003 /* always use CB2 mode, difference is masked in the CB driver */ 1004 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2); 1005 1006 for (i = 0; i < adapter->num_q_vectors; i++) { 1007 adapter->q_vector[i]->cpu = -1; 1008 ixgbe_update_dca(adapter->q_vector[i]); 1009 } 1010 } 1011 1012 static int __ixgbe_notify_dca(struct device *dev, void *data) 1013 { 1014 struct ixgbe_adapter *adapter = dev_get_drvdata(dev); 1015 unsigned long event = *(unsigned long *)data; 1016 1017 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE)) 1018 return 0; 1019 1020 switch (event) { 1021 case DCA_PROVIDER_ADD: 1022 /* if we're already enabled, don't do it again */ 1023 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 1024 break; 1025 if (dca_add_requester(dev) == 0) { 1026 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 1027 ixgbe_setup_dca(adapter); 1028 break; 1029 } 1030 /* Fall Through since DCA is disabled. */ 1031 case DCA_PROVIDER_REMOVE: 1032 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 1033 dca_remove_requester(dev); 1034 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 1035 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); 1036 } 1037 break; 1038 } 1039 1040 return 0; 1041 } 1042 1043 #endif /* CONFIG_IXGBE_DCA */ 1044 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, 1045 union ixgbe_adv_rx_desc *rx_desc, 1046 struct sk_buff *skb) 1047 { 1048 if (ring->netdev->features & NETIF_F_RXHASH) 1049 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); 1050 } 1051 1052 #ifdef IXGBE_FCOE 1053 /** 1054 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type 1055 * @ring: structure containing ring specific data 1056 * @rx_desc: advanced rx descriptor 1057 * 1058 * Returns : true if it is FCoE pkt 1059 */ 1060 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, 1061 union ixgbe_adv_rx_desc *rx_desc) 1062 { 1063 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1064 1065 return test_bit(__IXGBE_RX_FCOE, &ring->state) && 1066 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == 1067 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << 1068 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); 1069 } 1070 1071 #endif /* IXGBE_FCOE */ 1072 /** 1073 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum 1074 * @ring: structure containing ring specific data 1075 * @rx_desc: current Rx descriptor being processed 1076 * @skb: skb currently being received and modified 1077 **/ 1078 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, 1079 union ixgbe_adv_rx_desc *rx_desc, 1080 struct sk_buff *skb) 1081 { 1082 skb_checksum_none_assert(skb); 1083 1084 /* Rx csum disabled */ 1085 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1086 return; 1087 1088 /* if IP and error */ 1089 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 1090 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 1091 ring->rx_stats.csum_err++; 1092 return; 1093 } 1094 1095 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 1096 return; 1097 1098 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 1099 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 1100 1101 /* 1102 * 82599 errata, UDP frames with a 0 checksum can be marked as 1103 * checksum errors. 1104 */ 1105 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && 1106 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) 1107 return; 1108 1109 ring->rx_stats.csum_err++; 1110 return; 1111 } 1112 1113 /* It must be a TCP or UDP packet with a valid checksum */ 1114 skb->ip_summed = CHECKSUM_UNNECESSARY; 1115 } 1116 1117 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val) 1118 { 1119 rx_ring->next_to_use = val; 1120 1121 /* update next to alloc since we have filled the ring */ 1122 rx_ring->next_to_alloc = val; 1123 /* 1124 * Force memory writes to complete before letting h/w 1125 * know there are new descriptors to fetch. (Only 1126 * applicable for weak-ordered memory model archs, 1127 * such as IA-64). 1128 */ 1129 wmb(); 1130 writel(val, rx_ring->tail); 1131 } 1132 1133 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, 1134 struct ixgbe_rx_buffer *bi) 1135 { 1136 struct page *page = bi->page; 1137 dma_addr_t dma = bi->dma; 1138 1139 /* since we are recycling buffers we should seldom need to alloc */ 1140 if (likely(dma)) 1141 return true; 1142 1143 /* alloc new page for storage */ 1144 if (likely(!page)) { 1145 page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP, 1146 bi->skb, ixgbe_rx_pg_order(rx_ring)); 1147 if (unlikely(!page)) { 1148 rx_ring->rx_stats.alloc_rx_page_failed++; 1149 return false; 1150 } 1151 bi->page = page; 1152 } 1153 1154 /* map page for use */ 1155 dma = dma_map_page(rx_ring->dev, page, 0, 1156 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1157 1158 /* 1159 * if mapping failed free memory back to system since 1160 * there isn't much point in holding memory we can't use 1161 */ 1162 if (dma_mapping_error(rx_ring->dev, dma)) { 1163 __free_pages(page, ixgbe_rx_pg_order(rx_ring)); 1164 bi->page = NULL; 1165 1166 rx_ring->rx_stats.alloc_rx_page_failed++; 1167 return false; 1168 } 1169 1170 bi->dma = dma; 1171 bi->page_offset = 0; 1172 1173 return true; 1174 } 1175 1176 /** 1177 * ixgbe_alloc_rx_buffers - Replace used receive buffers 1178 * @rx_ring: ring to place buffers on 1179 * @cleaned_count: number of buffers to replace 1180 **/ 1181 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) 1182 { 1183 union ixgbe_adv_rx_desc *rx_desc; 1184 struct ixgbe_rx_buffer *bi; 1185 u16 i = rx_ring->next_to_use; 1186 1187 /* nothing to do */ 1188 if (!cleaned_count) 1189 return; 1190 1191 rx_desc = IXGBE_RX_DESC(rx_ring, i); 1192 bi = &rx_ring->rx_buffer_info[i]; 1193 i -= rx_ring->count; 1194 1195 do { 1196 if (!ixgbe_alloc_mapped_page(rx_ring, bi)) 1197 break; 1198 1199 /* 1200 * Refresh the desc even if buffer_addrs didn't change 1201 * because each write-back erases this info. 1202 */ 1203 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1204 1205 rx_desc++; 1206 bi++; 1207 i++; 1208 if (unlikely(!i)) { 1209 rx_desc = IXGBE_RX_DESC(rx_ring, 0); 1210 bi = rx_ring->rx_buffer_info; 1211 i -= rx_ring->count; 1212 } 1213 1214 /* clear the hdr_addr for the next_to_use descriptor */ 1215 rx_desc->read.hdr_addr = 0; 1216 1217 cleaned_count--; 1218 } while (cleaned_count); 1219 1220 i += rx_ring->count; 1221 1222 if (rx_ring->next_to_use != i) 1223 ixgbe_release_rx_desc(rx_ring, i); 1224 } 1225 1226 /** 1227 * ixgbe_get_headlen - determine size of header for RSC/LRO/GRO/FCOE 1228 * @data: pointer to the start of the headers 1229 * @max_len: total length of section to find headers in 1230 * 1231 * This function is meant to determine the length of headers that will 1232 * be recognized by hardware for LRO, GRO, and RSC offloads. The main 1233 * motivation of doing this is to only perform one pull for IPv4 TCP 1234 * packets so that we can do basic things like calculating the gso_size 1235 * based on the average data per packet. 1236 **/ 1237 static unsigned int ixgbe_get_headlen(unsigned char *data, 1238 unsigned int max_len) 1239 { 1240 union { 1241 unsigned char *network; 1242 /* l2 headers */ 1243 struct ethhdr *eth; 1244 struct vlan_hdr *vlan; 1245 /* l3 headers */ 1246 struct iphdr *ipv4; 1247 } hdr; 1248 __be16 protocol; 1249 u8 nexthdr = 0; /* default to not TCP */ 1250 u8 hlen; 1251 1252 /* this should never happen, but better safe than sorry */ 1253 if (max_len < ETH_HLEN) 1254 return max_len; 1255 1256 /* initialize network frame pointer */ 1257 hdr.network = data; 1258 1259 /* set first protocol and move network header forward */ 1260 protocol = hdr.eth->h_proto; 1261 hdr.network += ETH_HLEN; 1262 1263 /* handle any vlan tag if present */ 1264 if (protocol == __constant_htons(ETH_P_8021Q)) { 1265 if ((hdr.network - data) > (max_len - VLAN_HLEN)) 1266 return max_len; 1267 1268 protocol = hdr.vlan->h_vlan_encapsulated_proto; 1269 hdr.network += VLAN_HLEN; 1270 } 1271 1272 /* handle L3 protocols */ 1273 if (protocol == __constant_htons(ETH_P_IP)) { 1274 if ((hdr.network - data) > (max_len - sizeof(struct iphdr))) 1275 return max_len; 1276 1277 /* access ihl as a u8 to avoid unaligned access on ia64 */ 1278 hlen = (hdr.network[0] & 0x0F) << 2; 1279 1280 /* verify hlen meets minimum size requirements */ 1281 if (hlen < sizeof(struct iphdr)) 1282 return hdr.network - data; 1283 1284 /* record next protocol */ 1285 nexthdr = hdr.ipv4->protocol; 1286 hdr.network += hlen; 1287 #ifdef IXGBE_FCOE 1288 } else if (protocol == __constant_htons(ETH_P_FCOE)) { 1289 if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN)) 1290 return max_len; 1291 hdr.network += FCOE_HEADER_LEN; 1292 #endif 1293 } else { 1294 return hdr.network - data; 1295 } 1296 1297 /* finally sort out TCP */ 1298 if (nexthdr == IPPROTO_TCP) { 1299 if ((hdr.network - data) > (max_len - sizeof(struct tcphdr))) 1300 return max_len; 1301 1302 /* access doff as a u8 to avoid unaligned access on ia64 */ 1303 hlen = (hdr.network[12] & 0xF0) >> 2; 1304 1305 /* verify hlen meets minimum size requirements */ 1306 if (hlen < sizeof(struct tcphdr)) 1307 return hdr.network - data; 1308 1309 hdr.network += hlen; 1310 } 1311 1312 /* 1313 * If everything has gone correctly hdr.network should be the 1314 * data section of the packet and will be the end of the header. 1315 * If not then it probably represents the end of the last recognized 1316 * header. 1317 */ 1318 if ((hdr.network - data) < max_len) 1319 return hdr.network - data; 1320 else 1321 return max_len; 1322 } 1323 1324 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring, 1325 struct sk_buff *skb) 1326 { 1327 u16 hdr_len = skb_headlen(skb); 1328 1329 /* set gso_size to avoid messing up TCP MSS */ 1330 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len), 1331 IXGBE_CB(skb)->append_cnt); 1332 } 1333 1334 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, 1335 struct sk_buff *skb) 1336 { 1337 /* if append_cnt is 0 then frame is not RSC */ 1338 if (!IXGBE_CB(skb)->append_cnt) 1339 return; 1340 1341 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt; 1342 rx_ring->rx_stats.rsc_flush++; 1343 1344 ixgbe_set_rsc_gso_size(rx_ring, skb); 1345 1346 /* gso_size is computed using append_cnt so always clear it last */ 1347 IXGBE_CB(skb)->append_cnt = 0; 1348 } 1349 1350 /** 1351 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor 1352 * @rx_ring: rx descriptor ring packet is being transacted on 1353 * @rx_desc: pointer to the EOP Rx descriptor 1354 * @skb: pointer to current skb being populated 1355 * 1356 * This function checks the ring, descriptor, and packet information in 1357 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1358 * other fields within the skb. 1359 **/ 1360 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, 1361 union ixgbe_adv_rx_desc *rx_desc, 1362 struct sk_buff *skb) 1363 { 1364 struct net_device *dev = rx_ring->netdev; 1365 1366 ixgbe_update_rsc_stats(rx_ring, skb); 1367 1368 ixgbe_rx_hash(rx_ring, rx_desc, skb); 1369 1370 ixgbe_rx_checksum(rx_ring, rx_desc, skb); 1371 1372 #ifdef CONFIG_IXGBE_PTP 1373 ixgbe_ptp_rx_hwtstamp(rx_ring->q_vector, rx_desc, skb); 1374 #endif 1375 1376 if ((dev->features & NETIF_F_HW_VLAN_RX) && 1377 ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 1378 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 1379 __vlan_hwaccel_put_tag(skb, vid); 1380 } 1381 1382 skb_record_rx_queue(skb, rx_ring->queue_index); 1383 1384 skb->protocol = eth_type_trans(skb, dev); 1385 } 1386 1387 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, 1388 struct sk_buff *skb) 1389 { 1390 struct ixgbe_adapter *adapter = q_vector->adapter; 1391 1392 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 1393 napi_gro_receive(&q_vector->napi, skb); 1394 else 1395 netif_rx(skb); 1396 } 1397 1398 /** 1399 * ixgbe_is_non_eop - process handling of non-EOP buffers 1400 * @rx_ring: Rx ring being processed 1401 * @rx_desc: Rx descriptor for current buffer 1402 * @skb: Current socket buffer containing buffer in progress 1403 * 1404 * This function updates next to clean. If the buffer is an EOP buffer 1405 * this function exits returning false, otherwise it will place the 1406 * sk_buff in the next buffer to be chained and return true indicating 1407 * that this is in fact a non-EOP buffer. 1408 **/ 1409 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, 1410 union ixgbe_adv_rx_desc *rx_desc, 1411 struct sk_buff *skb) 1412 { 1413 u32 ntc = rx_ring->next_to_clean + 1; 1414 1415 /* fetch, update, and store next to clean */ 1416 ntc = (ntc < rx_ring->count) ? ntc : 0; 1417 rx_ring->next_to_clean = ntc; 1418 1419 prefetch(IXGBE_RX_DESC(rx_ring, ntc)); 1420 1421 /* update RSC append count if present */ 1422 if (ring_is_rsc_enabled(rx_ring)) { 1423 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & 1424 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); 1425 1426 if (unlikely(rsc_enabled)) { 1427 u32 rsc_cnt = le32_to_cpu(rsc_enabled); 1428 1429 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 1430 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; 1431 1432 /* update ntc based on RSC value */ 1433 ntc = le32_to_cpu(rx_desc->wb.upper.status_error); 1434 ntc &= IXGBE_RXDADV_NEXTP_MASK; 1435 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT; 1436 } 1437 } 1438 1439 /* if we are the last buffer then there is nothing else to do */ 1440 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 1441 return false; 1442 1443 /* place skb in next buffer to be received */ 1444 rx_ring->rx_buffer_info[ntc].skb = skb; 1445 rx_ring->rx_stats.non_eop_descs++; 1446 1447 return true; 1448 } 1449 1450 /** 1451 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail 1452 * @rx_ring: rx descriptor ring packet is being transacted on 1453 * @skb: pointer to current skb being adjusted 1454 * 1455 * This function is an ixgbe specific version of __pskb_pull_tail. The 1456 * main difference between this version and the original function is that 1457 * this function can make several assumptions about the state of things 1458 * that allow for significant optimizations versus the standard function. 1459 * As a result we can do things like drop a frag and maintain an accurate 1460 * truesize for the skb. 1461 */ 1462 static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, 1463 struct sk_buff *skb) 1464 { 1465 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1466 unsigned char *va; 1467 unsigned int pull_len; 1468 1469 /* 1470 * it is valid to use page_address instead of kmap since we are 1471 * working with pages allocated out of the lomem pool per 1472 * alloc_page(GFP_ATOMIC) 1473 */ 1474 va = skb_frag_address(frag); 1475 1476 /* 1477 * we need the header to contain the greater of either ETH_HLEN or 1478 * 60 bytes if the skb->len is less than 60 for skb_pad. 1479 */ 1480 pull_len = ixgbe_get_headlen(va, IXGBE_RX_HDR_SIZE); 1481 1482 /* align pull length to size of long to optimize memcpy performance */ 1483 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); 1484 1485 /* update all of the pointers */ 1486 skb_frag_size_sub(frag, pull_len); 1487 frag->page_offset += pull_len; 1488 skb->data_len -= pull_len; 1489 skb->tail += pull_len; 1490 } 1491 1492 /** 1493 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB 1494 * @rx_ring: rx descriptor ring packet is being transacted on 1495 * @skb: pointer to current skb being updated 1496 * 1497 * This function provides a basic DMA sync up for the first fragment of an 1498 * skb. The reason for doing this is that the first fragment cannot be 1499 * unmapped until we have reached the end of packet descriptor for a buffer 1500 * chain. 1501 */ 1502 static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, 1503 struct sk_buff *skb) 1504 { 1505 /* if the page was released unmap it, else just sync our portion */ 1506 if (unlikely(IXGBE_CB(skb)->page_released)) { 1507 dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma, 1508 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE); 1509 IXGBE_CB(skb)->page_released = false; 1510 } else { 1511 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 1512 1513 dma_sync_single_range_for_cpu(rx_ring->dev, 1514 IXGBE_CB(skb)->dma, 1515 frag->page_offset, 1516 ixgbe_rx_bufsz(rx_ring), 1517 DMA_FROM_DEVICE); 1518 } 1519 IXGBE_CB(skb)->dma = 0; 1520 } 1521 1522 /** 1523 * ixgbe_cleanup_headers - Correct corrupted or empty headers 1524 * @rx_ring: rx descriptor ring packet is being transacted on 1525 * @rx_desc: pointer to the EOP Rx descriptor 1526 * @skb: pointer to current skb being fixed 1527 * 1528 * Check for corrupted packet headers caused by senders on the local L2 1529 * embedded NIC switch not setting up their Tx Descriptors right. These 1530 * should be very rare. 1531 * 1532 * Also address the case where we are pulling data in on pages only 1533 * and as such no data is present in the skb header. 1534 * 1535 * In addition if skb is not at least 60 bytes we need to pad it so that 1536 * it is large enough to qualify as a valid Ethernet frame. 1537 * 1538 * Returns true if an error was encountered and skb was freed. 1539 **/ 1540 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, 1541 union ixgbe_adv_rx_desc *rx_desc, 1542 struct sk_buff *skb) 1543 { 1544 struct net_device *netdev = rx_ring->netdev; 1545 1546 /* verify that the packet does not have any known errors */ 1547 if (unlikely(ixgbe_test_staterr(rx_desc, 1548 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && 1549 !(netdev->features & NETIF_F_RXALL))) { 1550 dev_kfree_skb_any(skb); 1551 return true; 1552 } 1553 1554 /* place header in linear portion of buffer */ 1555 if (skb_is_nonlinear(skb)) 1556 ixgbe_pull_tail(rx_ring, skb); 1557 1558 #ifdef IXGBE_FCOE 1559 /* do not attempt to pad FCoE Frames as this will disrupt DDP */ 1560 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) 1561 return false; 1562 1563 #endif 1564 /* if skb_pad returns an error the skb was freed */ 1565 if (unlikely(skb->len < 60)) { 1566 int pad_len = 60 - skb->len; 1567 1568 if (skb_pad(skb, pad_len)) 1569 return true; 1570 __skb_put(skb, pad_len); 1571 } 1572 1573 return false; 1574 } 1575 1576 /** 1577 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring 1578 * @rx_ring: rx descriptor ring to store buffers on 1579 * @old_buff: donor buffer to have page reused 1580 * 1581 * Synchronizes page for reuse by the adapter 1582 **/ 1583 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, 1584 struct ixgbe_rx_buffer *old_buff) 1585 { 1586 struct ixgbe_rx_buffer *new_buff; 1587 u16 nta = rx_ring->next_to_alloc; 1588 1589 new_buff = &rx_ring->rx_buffer_info[nta]; 1590 1591 /* update, and store next to alloc */ 1592 nta++; 1593 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1594 1595 /* transfer page from old buffer to new buffer */ 1596 new_buff->page = old_buff->page; 1597 new_buff->dma = old_buff->dma; 1598 new_buff->page_offset = old_buff->page_offset; 1599 1600 /* sync the buffer for use by the device */ 1601 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 1602 new_buff->page_offset, 1603 ixgbe_rx_bufsz(rx_ring), 1604 DMA_FROM_DEVICE); 1605 } 1606 1607 /** 1608 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff 1609 * @rx_ring: rx descriptor ring to transact packets on 1610 * @rx_buffer: buffer containing page to add 1611 * @rx_desc: descriptor containing length of buffer written by hardware 1612 * @skb: sk_buff to place the data into 1613 * 1614 * This function will add the data contained in rx_buffer->page to the skb. 1615 * This is done either through a direct copy if the data in the buffer is 1616 * less than the skb header size, otherwise it will just attach the page as 1617 * a frag to the skb. 1618 * 1619 * The function will then update the page offset if necessary and return 1620 * true if the buffer can be reused by the adapter. 1621 **/ 1622 static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, 1623 struct ixgbe_rx_buffer *rx_buffer, 1624 union ixgbe_adv_rx_desc *rx_desc, 1625 struct sk_buff *skb) 1626 { 1627 struct page *page = rx_buffer->page; 1628 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 1629 #if (PAGE_SIZE < 8192) 1630 unsigned int truesize = ixgbe_rx_bufsz(rx_ring); 1631 #else 1632 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 1633 unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) - 1634 ixgbe_rx_bufsz(rx_ring); 1635 #endif 1636 1637 if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) { 1638 unsigned char *va = page_address(page) + rx_buffer->page_offset; 1639 1640 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 1641 1642 /* we can reuse buffer as-is, just make sure it is local */ 1643 if (likely(page_to_nid(page) == numa_node_id())) 1644 return true; 1645 1646 /* this page cannot be reused so discard it */ 1647 put_page(page); 1648 return false; 1649 } 1650 1651 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 1652 rx_buffer->page_offset, size, truesize); 1653 1654 /* avoid re-using remote pages */ 1655 if (unlikely(page_to_nid(page) != numa_node_id())) 1656 return false; 1657 1658 #if (PAGE_SIZE < 8192) 1659 /* if we are only owner of page we can reuse it */ 1660 if (unlikely(page_count(page) != 1)) 1661 return false; 1662 1663 /* flip page offset to other buffer */ 1664 rx_buffer->page_offset ^= truesize; 1665 1666 /* 1667 * since we are the only owner of the page and we need to 1668 * increment it, just set the value to 2 in order to avoid 1669 * an unecessary locked operation 1670 */ 1671 atomic_set(&page->_count, 2); 1672 #else 1673 /* move offset up to the next cache line */ 1674 rx_buffer->page_offset += truesize; 1675 1676 if (rx_buffer->page_offset > last_offset) 1677 return false; 1678 1679 /* bump ref count on page before it is given to the stack */ 1680 get_page(page); 1681 #endif 1682 1683 return true; 1684 } 1685 1686 static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring, 1687 union ixgbe_adv_rx_desc *rx_desc) 1688 { 1689 struct ixgbe_rx_buffer *rx_buffer; 1690 struct sk_buff *skb; 1691 struct page *page; 1692 1693 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1694 page = rx_buffer->page; 1695 prefetchw(page); 1696 1697 skb = rx_buffer->skb; 1698 1699 if (likely(!skb)) { 1700 void *page_addr = page_address(page) + 1701 rx_buffer->page_offset; 1702 1703 /* prefetch first cache line of first page */ 1704 prefetch(page_addr); 1705 #if L1_CACHE_BYTES < 128 1706 prefetch(page_addr + L1_CACHE_BYTES); 1707 #endif 1708 1709 /* allocate a skb to store the frags */ 1710 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 1711 IXGBE_RX_HDR_SIZE); 1712 if (unlikely(!skb)) { 1713 rx_ring->rx_stats.alloc_rx_buff_failed++; 1714 return NULL; 1715 } 1716 1717 /* 1718 * we will be copying header into skb->data in 1719 * pskb_may_pull so it is in our interest to prefetch 1720 * it now to avoid a possible cache miss 1721 */ 1722 prefetchw(skb->data); 1723 1724 /* 1725 * Delay unmapping of the first packet. It carries the 1726 * header information, HW may still access the header 1727 * after the writeback. Only unmap it when EOP is 1728 * reached 1729 */ 1730 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 1731 goto dma_sync; 1732 1733 IXGBE_CB(skb)->dma = rx_buffer->dma; 1734 } else { 1735 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) 1736 ixgbe_dma_sync_frag(rx_ring, skb); 1737 1738 dma_sync: 1739 /* we are reusing so sync this buffer for CPU use */ 1740 dma_sync_single_range_for_cpu(rx_ring->dev, 1741 rx_buffer->dma, 1742 rx_buffer->page_offset, 1743 ixgbe_rx_bufsz(rx_ring), 1744 DMA_FROM_DEVICE); 1745 } 1746 1747 /* pull page into skb */ 1748 if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 1749 /* hand second half of page back to the ring */ 1750 ixgbe_reuse_rx_page(rx_ring, rx_buffer); 1751 } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) { 1752 /* the page has been released from the ring */ 1753 IXGBE_CB(skb)->page_released = true; 1754 } else { 1755 /* we are not reusing the buffer so unmap it */ 1756 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 1757 ixgbe_rx_pg_size(rx_ring), 1758 DMA_FROM_DEVICE); 1759 } 1760 1761 /* clear contents of buffer_info */ 1762 rx_buffer->skb = NULL; 1763 rx_buffer->dma = 0; 1764 rx_buffer->page = NULL; 1765 1766 return skb; 1767 } 1768 1769 /** 1770 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 1771 * @q_vector: structure containing interrupt and ring information 1772 * @rx_ring: rx descriptor ring to transact packets on 1773 * @budget: Total limit on number of packets to process 1774 * 1775 * This function provides a "bounce buffer" approach to Rx interrupt 1776 * processing. The advantage to this is that on systems that have 1777 * expensive overhead for IOMMU access this provides a means of avoiding 1778 * it by maintaining the mapping of the page to the syste. 1779 * 1780 * Returns true if all work is completed without reaching budget 1781 **/ 1782 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, 1783 struct ixgbe_ring *rx_ring, 1784 int budget) 1785 { 1786 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 1787 #ifdef IXGBE_FCOE 1788 struct ixgbe_adapter *adapter = q_vector->adapter; 1789 int ddp_bytes; 1790 unsigned int mss = 0; 1791 #endif /* IXGBE_FCOE */ 1792 u16 cleaned_count = ixgbe_desc_unused(rx_ring); 1793 1794 do { 1795 union ixgbe_adv_rx_desc *rx_desc; 1796 struct sk_buff *skb; 1797 1798 /* return some buffers to hardware, one at a time is too slow */ 1799 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { 1800 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 1801 cleaned_count = 0; 1802 } 1803 1804 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); 1805 1806 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 1807 break; 1808 1809 /* 1810 * This memory barrier is needed to keep us from reading 1811 * any other fields out of the rx_desc until we know the 1812 * RXD_STAT_DD bit is set 1813 */ 1814 rmb(); 1815 1816 /* retrieve a buffer from the ring */ 1817 skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc); 1818 1819 /* exit if we failed to retrieve a buffer */ 1820 if (!skb) 1821 break; 1822 1823 cleaned_count++; 1824 1825 /* place incomplete frames back on ring for completion */ 1826 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) 1827 continue; 1828 1829 /* verify the packet layout is correct */ 1830 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) 1831 continue; 1832 1833 /* probably a little skewed due to removing CRC */ 1834 total_rx_bytes += skb->len; 1835 total_rx_packets++; 1836 1837 /* populate checksum, timestamp, VLAN, and protocol */ 1838 ixgbe_process_skb_fields(rx_ring, rx_desc, skb); 1839 1840 #ifdef IXGBE_FCOE 1841 /* if ddp, not passing to ULD unless for FCP_RSP or error */ 1842 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) { 1843 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb); 1844 /* include DDPed FCoE data */ 1845 if (ddp_bytes > 0) { 1846 if (!mss) { 1847 mss = rx_ring->netdev->mtu - 1848 sizeof(struct fcoe_hdr) - 1849 sizeof(struct fc_frame_header) - 1850 sizeof(struct fcoe_crc_eof); 1851 if (mss > 512) 1852 mss &= ~511; 1853 } 1854 total_rx_bytes += ddp_bytes; 1855 total_rx_packets += DIV_ROUND_UP(ddp_bytes, 1856 mss); 1857 } 1858 if (!ddp_bytes) { 1859 dev_kfree_skb_any(skb); 1860 continue; 1861 } 1862 } 1863 1864 #endif /* IXGBE_FCOE */ 1865 ixgbe_rx_skb(q_vector, skb); 1866 1867 /* update budget accounting */ 1868 budget--; 1869 } while (likely(budget)); 1870 1871 u64_stats_update_begin(&rx_ring->syncp); 1872 rx_ring->stats.packets += total_rx_packets; 1873 rx_ring->stats.bytes += total_rx_bytes; 1874 u64_stats_update_end(&rx_ring->syncp); 1875 q_vector->rx.total_packets += total_rx_packets; 1876 q_vector->rx.total_bytes += total_rx_bytes; 1877 1878 if (cleaned_count) 1879 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count); 1880 1881 return !!budget; 1882 } 1883 1884 /** 1885 * ixgbe_configure_msix - Configure MSI-X hardware 1886 * @adapter: board private structure 1887 * 1888 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X 1889 * interrupts. 1890 **/ 1891 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) 1892 { 1893 struct ixgbe_q_vector *q_vector; 1894 int v_idx; 1895 u32 mask; 1896 1897 /* Populate MSIX to EITR Select */ 1898 if (adapter->num_vfs > 32) { 1899 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1; 1900 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel); 1901 } 1902 1903 /* 1904 * Populate the IVAR table and set the ITR values to the 1905 * corresponding register. 1906 */ 1907 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { 1908 struct ixgbe_ring *ring; 1909 q_vector = adapter->q_vector[v_idx]; 1910 1911 ixgbe_for_each_ring(ring, q_vector->rx) 1912 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1913 1914 ixgbe_for_each_ring(ring, q_vector->tx) 1915 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1916 1917 if (q_vector->tx.ring && !q_vector->rx.ring) { 1918 /* tx only vector */ 1919 if (adapter->tx_itr_setting == 1) 1920 q_vector->itr = IXGBE_10K_ITR; 1921 else 1922 q_vector->itr = adapter->tx_itr_setting; 1923 } else { 1924 /* rx or rx/tx vector */ 1925 if (adapter->rx_itr_setting == 1) 1926 q_vector->itr = IXGBE_20K_ITR; 1927 else 1928 q_vector->itr = adapter->rx_itr_setting; 1929 } 1930 1931 ixgbe_write_eitr(q_vector); 1932 } 1933 1934 switch (adapter->hw.mac.type) { 1935 case ixgbe_mac_82598EB: 1936 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, 1937 v_idx); 1938 break; 1939 case ixgbe_mac_82599EB: 1940 case ixgbe_mac_X540: 1941 ixgbe_set_ivar(adapter, -1, 1, v_idx); 1942 break; 1943 default: 1944 break; 1945 } 1946 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950); 1947 1948 /* set up to autoclear timer, and the vectors */ 1949 mask = IXGBE_EIMS_ENABLE_MASK; 1950 mask &= ~(IXGBE_EIMS_OTHER | 1951 IXGBE_EIMS_MAILBOX | 1952 IXGBE_EIMS_LSC); 1953 1954 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); 1955 } 1956 1957 enum latency_range { 1958 lowest_latency = 0, 1959 low_latency = 1, 1960 bulk_latency = 2, 1961 latency_invalid = 255 1962 }; 1963 1964 /** 1965 * ixgbe_update_itr - update the dynamic ITR value based on statistics 1966 * @q_vector: structure containing interrupt and ring information 1967 * @ring_container: structure containing ring performance data 1968 * 1969 * Stores a new ITR value based on packets and byte 1970 * counts during the last interrupt. The advantage of per interrupt 1971 * computation is faster updates and more accurate ITR for the current 1972 * traffic pattern. Constants in this function were computed 1973 * based on theoretical maximum wire speed and thresholds were set based 1974 * on testing data as well as attempting to minimize response time 1975 * while increasing bulk throughput. 1976 * this functionality is controlled by the InterruptThrottleRate module 1977 * parameter (see ixgbe_param.c) 1978 **/ 1979 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, 1980 struct ixgbe_ring_container *ring_container) 1981 { 1982 int bytes = ring_container->total_bytes; 1983 int packets = ring_container->total_packets; 1984 u32 timepassed_us; 1985 u64 bytes_perint; 1986 u8 itr_setting = ring_container->itr; 1987 1988 if (packets == 0) 1989 return; 1990 1991 /* simple throttlerate management 1992 * 0-10MB/s lowest (100000 ints/s) 1993 * 10-20MB/s low (20000 ints/s) 1994 * 20-1249MB/s bulk (8000 ints/s) 1995 */ 1996 /* what was last interrupt timeslice? */ 1997 timepassed_us = q_vector->itr >> 2; 1998 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1999 2000 switch (itr_setting) { 2001 case lowest_latency: 2002 if (bytes_perint > 10) 2003 itr_setting = low_latency; 2004 break; 2005 case low_latency: 2006 if (bytes_perint > 20) 2007 itr_setting = bulk_latency; 2008 else if (bytes_perint <= 10) 2009 itr_setting = lowest_latency; 2010 break; 2011 case bulk_latency: 2012 if (bytes_perint <= 20) 2013 itr_setting = low_latency; 2014 break; 2015 } 2016 2017 /* clear work counters since we have the values we need */ 2018 ring_container->total_bytes = 0; 2019 ring_container->total_packets = 0; 2020 2021 /* write updated itr to ring container */ 2022 ring_container->itr = itr_setting; 2023 } 2024 2025 /** 2026 * ixgbe_write_eitr - write EITR register in hardware specific way 2027 * @q_vector: structure containing interrupt and ring information 2028 * 2029 * This function is made to be called by ethtool and by the driver 2030 * when it needs to update EITR registers at runtime. Hardware 2031 * specific quirks/differences are taken care of here. 2032 */ 2033 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector) 2034 { 2035 struct ixgbe_adapter *adapter = q_vector->adapter; 2036 struct ixgbe_hw *hw = &adapter->hw; 2037 int v_idx = q_vector->v_idx; 2038 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 2039 2040 switch (adapter->hw.mac.type) { 2041 case ixgbe_mac_82598EB: 2042 /* must write high and low 16 bits to reset counter */ 2043 itr_reg |= (itr_reg << 16); 2044 break; 2045 case ixgbe_mac_82599EB: 2046 case ixgbe_mac_X540: 2047 /* 2048 * set the WDIS bit to not clear the timer bits and cause an 2049 * immediate assertion of the interrupt 2050 */ 2051 itr_reg |= IXGBE_EITR_CNT_WDIS; 2052 break; 2053 default: 2054 break; 2055 } 2056 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg); 2057 } 2058 2059 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector) 2060 { 2061 u32 new_itr = q_vector->itr; 2062 u8 current_itr; 2063 2064 ixgbe_update_itr(q_vector, &q_vector->tx); 2065 ixgbe_update_itr(q_vector, &q_vector->rx); 2066 2067 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 2068 2069 switch (current_itr) { 2070 /* counts and packets in update_itr are dependent on these numbers */ 2071 case lowest_latency: 2072 new_itr = IXGBE_100K_ITR; 2073 break; 2074 case low_latency: 2075 new_itr = IXGBE_20K_ITR; 2076 break; 2077 case bulk_latency: 2078 new_itr = IXGBE_8K_ITR; 2079 break; 2080 default: 2081 break; 2082 } 2083 2084 if (new_itr != q_vector->itr) { 2085 /* do an exponential smoothing */ 2086 new_itr = (10 * new_itr * q_vector->itr) / 2087 ((9 * new_itr) + q_vector->itr); 2088 2089 /* save the algorithm value here */ 2090 q_vector->itr = new_itr; 2091 2092 ixgbe_write_eitr(q_vector); 2093 } 2094 } 2095 2096 /** 2097 * ixgbe_check_overtemp_subtask - check for over temperature 2098 * @adapter: pointer to adapter 2099 **/ 2100 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) 2101 { 2102 struct ixgbe_hw *hw = &adapter->hw; 2103 u32 eicr = adapter->interrupt_event; 2104 2105 if (test_bit(__IXGBE_DOWN, &adapter->state)) 2106 return; 2107 2108 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) && 2109 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT)) 2110 return; 2111 2112 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2113 2114 switch (hw->device_id) { 2115 case IXGBE_DEV_ID_82599_T3_LOM: 2116 /* 2117 * Since the warning interrupt is for both ports 2118 * we don't have to check if: 2119 * - This interrupt wasn't for our port. 2120 * - We may have missed the interrupt so always have to 2121 * check if we got a LSC 2122 */ 2123 if (!(eicr & IXGBE_EICR_GPI_SDP0) && 2124 !(eicr & IXGBE_EICR_LSC)) 2125 return; 2126 2127 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { 2128 u32 autoneg; 2129 bool link_up = false; 2130 2131 hw->mac.ops.check_link(hw, &autoneg, &link_up, false); 2132 2133 if (link_up) 2134 return; 2135 } 2136 2137 /* Check if this is not due to overtemp */ 2138 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP) 2139 return; 2140 2141 break; 2142 default: 2143 if (!(eicr & IXGBE_EICR_GPI_SDP0)) 2144 return; 2145 break; 2146 } 2147 e_crit(drv, 2148 "Network adapter has been stopped because it has over heated. " 2149 "Restart the computer. If the problem persists, " 2150 "power off the system and replace the adapter\n"); 2151 2152 adapter->interrupt_event = 0; 2153 } 2154 2155 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) 2156 { 2157 struct ixgbe_hw *hw = &adapter->hw; 2158 2159 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 2160 (eicr & IXGBE_EICR_GPI_SDP1)) { 2161 e_crit(probe, "Fan has stopped, replace the adapter\n"); 2162 /* write to clear the interrupt */ 2163 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2164 } 2165 } 2166 2167 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) 2168 { 2169 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)) 2170 return; 2171 2172 switch (adapter->hw.mac.type) { 2173 case ixgbe_mac_82599EB: 2174 /* 2175 * Need to check link state so complete overtemp check 2176 * on service task 2177 */ 2178 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) && 2179 (!test_bit(__IXGBE_DOWN, &adapter->state))) { 2180 adapter->interrupt_event = eicr; 2181 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; 2182 ixgbe_service_event_schedule(adapter); 2183 return; 2184 } 2185 return; 2186 case ixgbe_mac_X540: 2187 if (!(eicr & IXGBE_EICR_TS)) 2188 return; 2189 break; 2190 default: 2191 return; 2192 } 2193 2194 e_crit(drv, 2195 "Network adapter has been stopped because it has over heated. " 2196 "Restart the computer. If the problem persists, " 2197 "power off the system and replace the adapter\n"); 2198 } 2199 2200 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) 2201 { 2202 struct ixgbe_hw *hw = &adapter->hw; 2203 2204 if (eicr & IXGBE_EICR_GPI_SDP2) { 2205 /* Clear the interrupt */ 2206 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 2207 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2208 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 2209 ixgbe_service_event_schedule(adapter); 2210 } 2211 } 2212 2213 if (eicr & IXGBE_EICR_GPI_SDP1) { 2214 /* Clear the interrupt */ 2215 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2216 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2217 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 2218 ixgbe_service_event_schedule(adapter); 2219 } 2220 } 2221 } 2222 2223 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) 2224 { 2225 struct ixgbe_hw *hw = &adapter->hw; 2226 2227 adapter->lsc_int++; 2228 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 2229 adapter->link_check_timeout = jiffies; 2230 if (!test_bit(__IXGBE_DOWN, &adapter->state)) { 2231 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 2232 IXGBE_WRITE_FLUSH(hw); 2233 ixgbe_service_event_schedule(adapter); 2234 } 2235 } 2236 2237 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, 2238 u64 qmask) 2239 { 2240 u32 mask; 2241 struct ixgbe_hw *hw = &adapter->hw; 2242 2243 switch (hw->mac.type) { 2244 case ixgbe_mac_82598EB: 2245 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2246 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2247 break; 2248 case ixgbe_mac_82599EB: 2249 case ixgbe_mac_X540: 2250 mask = (qmask & 0xFFFFFFFF); 2251 if (mask) 2252 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2253 mask = (qmask >> 32); 2254 if (mask) 2255 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2256 break; 2257 default: 2258 break; 2259 } 2260 /* skip the flush */ 2261 } 2262 2263 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter, 2264 u64 qmask) 2265 { 2266 u32 mask; 2267 struct ixgbe_hw *hw = &adapter->hw; 2268 2269 switch (hw->mac.type) { 2270 case ixgbe_mac_82598EB: 2271 mask = (IXGBE_EIMS_RTX_QUEUE & qmask); 2272 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2273 break; 2274 case ixgbe_mac_82599EB: 2275 case ixgbe_mac_X540: 2276 mask = (qmask & 0xFFFFFFFF); 2277 if (mask) 2278 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2279 mask = (qmask >> 32); 2280 if (mask) 2281 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2282 break; 2283 default: 2284 break; 2285 } 2286 /* skip the flush */ 2287 } 2288 2289 /** 2290 * ixgbe_irq_enable - Enable default interrupt generation settings 2291 * @adapter: board private structure 2292 **/ 2293 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, 2294 bool flush) 2295 { 2296 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2297 2298 /* don't reenable LSC while waiting for link */ 2299 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 2300 mask &= ~IXGBE_EIMS_LSC; 2301 2302 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) 2303 switch (adapter->hw.mac.type) { 2304 case ixgbe_mac_82599EB: 2305 mask |= IXGBE_EIMS_GPI_SDP0; 2306 break; 2307 case ixgbe_mac_X540: 2308 mask |= IXGBE_EIMS_TS; 2309 break; 2310 default: 2311 break; 2312 } 2313 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 2314 mask |= IXGBE_EIMS_GPI_SDP1; 2315 switch (adapter->hw.mac.type) { 2316 case ixgbe_mac_82599EB: 2317 mask |= IXGBE_EIMS_GPI_SDP1; 2318 mask |= IXGBE_EIMS_GPI_SDP2; 2319 case ixgbe_mac_X540: 2320 mask |= IXGBE_EIMS_ECC; 2321 mask |= IXGBE_EIMS_MAILBOX; 2322 break; 2323 default: 2324 break; 2325 } 2326 2327 #ifdef CONFIG_IXGBE_PTP 2328 if (adapter->hw.mac.type == ixgbe_mac_X540) 2329 mask |= IXGBE_EIMS_TIMESYNC; 2330 #endif 2331 2332 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && 2333 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 2334 mask |= IXGBE_EIMS_FLOW_DIR; 2335 2336 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); 2337 if (queues) 2338 ixgbe_irq_enable_queues(adapter, ~0); 2339 if (flush) 2340 IXGBE_WRITE_FLUSH(&adapter->hw); 2341 } 2342 2343 static irqreturn_t ixgbe_msix_other(int irq, void *data) 2344 { 2345 struct ixgbe_adapter *adapter = data; 2346 struct ixgbe_hw *hw = &adapter->hw; 2347 u32 eicr; 2348 2349 /* 2350 * Workaround for Silicon errata. Use clear-by-write instead 2351 * of clear-by-read. Reading with EICS will return the 2352 * interrupt causes without clearing, which later be done 2353 * with the write to EICR. 2354 */ 2355 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 2356 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 2357 2358 if (eicr & IXGBE_EICR_LSC) 2359 ixgbe_check_lsc(adapter); 2360 2361 if (eicr & IXGBE_EICR_MAILBOX) 2362 ixgbe_msg_task(adapter); 2363 2364 switch (hw->mac.type) { 2365 case ixgbe_mac_82599EB: 2366 case ixgbe_mac_X540: 2367 if (eicr & IXGBE_EICR_ECC) 2368 e_info(link, "Received unrecoverable ECC Err, please " 2369 "reboot\n"); 2370 /* Handle Flow Director Full threshold interrupt */ 2371 if (eicr & IXGBE_EICR_FLOW_DIR) { 2372 int reinit_count = 0; 2373 int i; 2374 for (i = 0; i < adapter->num_tx_queues; i++) { 2375 struct ixgbe_ring *ring = adapter->tx_ring[i]; 2376 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, 2377 &ring->state)) 2378 reinit_count++; 2379 } 2380 if (reinit_count) { 2381 /* no more flow director interrupts until after init */ 2382 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR); 2383 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 2384 ixgbe_service_event_schedule(adapter); 2385 } 2386 } 2387 ixgbe_check_sfp_event(adapter, eicr); 2388 ixgbe_check_overtemp_event(adapter, eicr); 2389 break; 2390 default: 2391 break; 2392 } 2393 2394 ixgbe_check_fan_failure(adapter, eicr); 2395 2396 #ifdef CONFIG_IXGBE_PTP 2397 if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) 2398 ixgbe_ptp_check_pps_event(adapter, eicr); 2399 #endif 2400 2401 /* re-enable the original interrupt state, no lsc, no queues */ 2402 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2403 ixgbe_irq_enable(adapter, false, false); 2404 2405 return IRQ_HANDLED; 2406 } 2407 2408 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data) 2409 { 2410 struct ixgbe_q_vector *q_vector = data; 2411 2412 /* EIAM disabled interrupts (on this vector) for us */ 2413 2414 if (q_vector->rx.ring || q_vector->tx.ring) 2415 napi_schedule(&q_vector->napi); 2416 2417 return IRQ_HANDLED; 2418 } 2419 2420 /** 2421 * ixgbe_poll - NAPI Rx polling callback 2422 * @napi: structure for representing this polling device 2423 * @budget: how many packets driver is allowed to clean 2424 * 2425 * This function is used for legacy and MSI, NAPI mode 2426 **/ 2427 int ixgbe_poll(struct napi_struct *napi, int budget) 2428 { 2429 struct ixgbe_q_vector *q_vector = 2430 container_of(napi, struct ixgbe_q_vector, napi); 2431 struct ixgbe_adapter *adapter = q_vector->adapter; 2432 struct ixgbe_ring *ring; 2433 int per_ring_budget; 2434 bool clean_complete = true; 2435 2436 #ifdef CONFIG_IXGBE_DCA 2437 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) 2438 ixgbe_update_dca(q_vector); 2439 #endif 2440 2441 ixgbe_for_each_ring(ring, q_vector->tx) 2442 clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring); 2443 2444 /* attempt to distribute budget to each queue fairly, but don't allow 2445 * the budget to go below 1 because we'll exit polling */ 2446 if (q_vector->rx.count > 1) 2447 per_ring_budget = max(budget/q_vector->rx.count, 1); 2448 else 2449 per_ring_budget = budget; 2450 2451 ixgbe_for_each_ring(ring, q_vector->rx) 2452 clean_complete &= ixgbe_clean_rx_irq(q_vector, ring, 2453 per_ring_budget); 2454 2455 /* If all work not completed, return budget and keep polling */ 2456 if (!clean_complete) 2457 return budget; 2458 2459 /* all work done, exit the polling mode */ 2460 napi_complete(napi); 2461 if (adapter->rx_itr_setting & 1) 2462 ixgbe_set_itr(q_vector); 2463 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2464 ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx)); 2465 2466 return 0; 2467 } 2468 2469 /** 2470 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts 2471 * @adapter: board private structure 2472 * 2473 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests 2474 * interrupts from the kernel. 2475 **/ 2476 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) 2477 { 2478 struct net_device *netdev = adapter->netdev; 2479 int vector, err; 2480 int ri = 0, ti = 0; 2481 2482 for (vector = 0; vector < adapter->num_q_vectors; vector++) { 2483 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 2484 struct msix_entry *entry = &adapter->msix_entries[vector]; 2485 2486 if (q_vector->tx.ring && q_vector->rx.ring) { 2487 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2488 "%s-%s-%d", netdev->name, "TxRx", ri++); 2489 ti++; 2490 } else if (q_vector->rx.ring) { 2491 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2492 "%s-%s-%d", netdev->name, "rx", ri++); 2493 } else if (q_vector->tx.ring) { 2494 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2495 "%s-%s-%d", netdev->name, "tx", ti++); 2496 } else { 2497 /* skip this unused q_vector */ 2498 continue; 2499 } 2500 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0, 2501 q_vector->name, q_vector); 2502 if (err) { 2503 e_err(probe, "request_irq failed for MSIX interrupt " 2504 "Error: %d\n", err); 2505 goto free_queue_irqs; 2506 } 2507 /* If Flow Director is enabled, set interrupt affinity */ 2508 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 2509 /* assign the mask for this irq */ 2510 irq_set_affinity_hint(entry->vector, 2511 &q_vector->affinity_mask); 2512 } 2513 } 2514 2515 err = request_irq(adapter->msix_entries[vector].vector, 2516 ixgbe_msix_other, 0, netdev->name, adapter); 2517 if (err) { 2518 e_err(probe, "request_irq for msix_other failed: %d\n", err); 2519 goto free_queue_irqs; 2520 } 2521 2522 return 0; 2523 2524 free_queue_irqs: 2525 while (vector) { 2526 vector--; 2527 irq_set_affinity_hint(adapter->msix_entries[vector].vector, 2528 NULL); 2529 free_irq(adapter->msix_entries[vector].vector, 2530 adapter->q_vector[vector]); 2531 } 2532 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; 2533 pci_disable_msix(adapter->pdev); 2534 kfree(adapter->msix_entries); 2535 adapter->msix_entries = NULL; 2536 return err; 2537 } 2538 2539 /** 2540 * ixgbe_intr - legacy mode Interrupt Handler 2541 * @irq: interrupt number 2542 * @data: pointer to a network interface device structure 2543 **/ 2544 static irqreturn_t ixgbe_intr(int irq, void *data) 2545 { 2546 struct ixgbe_adapter *adapter = data; 2547 struct ixgbe_hw *hw = &adapter->hw; 2548 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2549 u32 eicr; 2550 2551 /* 2552 * Workaround for silicon errata #26 on 82598. Mask the interrupt 2553 * before the read of EICR. 2554 */ 2555 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 2556 2557 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read 2558 * therefore no explicit interrupt disable is necessary */ 2559 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 2560 if (!eicr) { 2561 /* 2562 * shared interrupt alert! 2563 * make sure interrupts are enabled because the read will 2564 * have disabled interrupts due to EIAM 2565 * finish the workaround of silicon errata on 82598. Unmask 2566 * the interrupt that we masked before the EICR read. 2567 */ 2568 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2569 ixgbe_irq_enable(adapter, true, true); 2570 return IRQ_NONE; /* Not our interrupt */ 2571 } 2572 2573 if (eicr & IXGBE_EICR_LSC) 2574 ixgbe_check_lsc(adapter); 2575 2576 switch (hw->mac.type) { 2577 case ixgbe_mac_82599EB: 2578 ixgbe_check_sfp_event(adapter, eicr); 2579 /* Fall through */ 2580 case ixgbe_mac_X540: 2581 if (eicr & IXGBE_EICR_ECC) 2582 e_info(link, "Received unrecoverable ECC err, please " 2583 "reboot\n"); 2584 ixgbe_check_overtemp_event(adapter, eicr); 2585 break; 2586 default: 2587 break; 2588 } 2589 2590 ixgbe_check_fan_failure(adapter, eicr); 2591 #ifdef CONFIG_IXGBE_PTP 2592 if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) 2593 ixgbe_ptp_check_pps_event(adapter, eicr); 2594 #endif 2595 2596 /* would disable interrupts here but EIAM disabled it */ 2597 napi_schedule(&q_vector->napi); 2598 2599 /* 2600 * re-enable link(maybe) and non-queue interrupts, no flush. 2601 * ixgbe_poll will re-enable the queue interrupts 2602 */ 2603 if (!test_bit(__IXGBE_DOWN, &adapter->state)) 2604 ixgbe_irq_enable(adapter, false, false); 2605 2606 return IRQ_HANDLED; 2607 } 2608 2609 /** 2610 * ixgbe_request_irq - initialize interrupts 2611 * @adapter: board private structure 2612 * 2613 * Attempts to configure interrupts using the best available 2614 * capabilities of the hardware and kernel. 2615 **/ 2616 static int ixgbe_request_irq(struct ixgbe_adapter *adapter) 2617 { 2618 struct net_device *netdev = adapter->netdev; 2619 int err; 2620 2621 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 2622 err = ixgbe_request_msix_irqs(adapter); 2623 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) 2624 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0, 2625 netdev->name, adapter); 2626 else 2627 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED, 2628 netdev->name, adapter); 2629 2630 if (err) 2631 e_err(probe, "request_irq failed, Error %d\n", err); 2632 2633 return err; 2634 } 2635 2636 static void ixgbe_free_irq(struct ixgbe_adapter *adapter) 2637 { 2638 int vector; 2639 2640 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 2641 free_irq(adapter->pdev->irq, adapter); 2642 return; 2643 } 2644 2645 for (vector = 0; vector < adapter->num_q_vectors; vector++) { 2646 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector]; 2647 struct msix_entry *entry = &adapter->msix_entries[vector]; 2648 2649 /* free only the irqs that were actually requested */ 2650 if (!q_vector->rx.ring && !q_vector->tx.ring) 2651 continue; 2652 2653 /* clear the affinity_mask in the IRQ descriptor */ 2654 irq_set_affinity_hint(entry->vector, NULL); 2655 2656 free_irq(entry->vector, q_vector); 2657 } 2658 2659 free_irq(adapter->msix_entries[vector++].vector, adapter); 2660 } 2661 2662 /** 2663 * ixgbe_irq_disable - Mask off interrupt generation on the NIC 2664 * @adapter: board private structure 2665 **/ 2666 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter) 2667 { 2668 switch (adapter->hw.mac.type) { 2669 case ixgbe_mac_82598EB: 2670 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 2671 break; 2672 case ixgbe_mac_82599EB: 2673 case ixgbe_mac_X540: 2674 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 2675 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 2676 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 2677 break; 2678 default: 2679 break; 2680 } 2681 IXGBE_WRITE_FLUSH(&adapter->hw); 2682 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 2683 int vector; 2684 2685 for (vector = 0; vector < adapter->num_q_vectors; vector++) 2686 synchronize_irq(adapter->msix_entries[vector].vector); 2687 2688 synchronize_irq(adapter->msix_entries[vector++].vector); 2689 } else { 2690 synchronize_irq(adapter->pdev->irq); 2691 } 2692 } 2693 2694 /** 2695 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts 2696 * 2697 **/ 2698 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) 2699 { 2700 struct ixgbe_q_vector *q_vector = adapter->q_vector[0]; 2701 2702 /* rx/tx vector */ 2703 if (adapter->rx_itr_setting == 1) 2704 q_vector->itr = IXGBE_20K_ITR; 2705 else 2706 q_vector->itr = adapter->rx_itr_setting; 2707 2708 ixgbe_write_eitr(q_vector); 2709 2710 ixgbe_set_ivar(adapter, 0, 0, 0); 2711 ixgbe_set_ivar(adapter, 1, 0, 0); 2712 2713 e_info(hw, "Legacy interrupt IVAR setup done\n"); 2714 } 2715 2716 /** 2717 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset 2718 * @adapter: board private structure 2719 * @ring: structure containing ring specific data 2720 * 2721 * Configure the Tx descriptor ring after a reset. 2722 **/ 2723 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, 2724 struct ixgbe_ring *ring) 2725 { 2726 struct ixgbe_hw *hw = &adapter->hw; 2727 u64 tdba = ring->dma; 2728 int wait_loop = 10; 2729 u32 txdctl = IXGBE_TXDCTL_ENABLE; 2730 u8 reg_idx = ring->reg_idx; 2731 2732 /* disable queue to avoid issues while updating state */ 2733 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); 2734 IXGBE_WRITE_FLUSH(hw); 2735 2736 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx), 2737 (tdba & DMA_BIT_MASK(32))); 2738 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32)); 2739 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx), 2740 ring->count * sizeof(union ixgbe_adv_tx_desc)); 2741 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0); 2742 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0); 2743 ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx); 2744 2745 /* 2746 * set WTHRESH to encourage burst writeback, it should not be set 2747 * higher than 1 when ITR is 0 as it could cause false TX hangs 2748 * 2749 * In order to avoid issues WTHRESH + PTHRESH should always be equal 2750 * to or less than the number of on chip descriptors, which is 2751 * currently 40. 2752 */ 2753 if (!ring->q_vector || (ring->q_vector->itr < 8)) 2754 txdctl |= (1 << 16); /* WTHRESH = 1 */ 2755 else 2756 txdctl |= (8 << 16); /* WTHRESH = 8 */ 2757 2758 /* 2759 * Setting PTHRESH to 32 both improves performance 2760 * and avoids a TX hang with DFP enabled 2761 */ 2762 txdctl |= (1 << 8) | /* HTHRESH = 1 */ 2763 32; /* PTHRESH = 32 */ 2764 2765 /* reinitialize flowdirector state */ 2766 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 2767 ring->atr_sample_rate = adapter->atr_sample_rate; 2768 ring->atr_count = 0; 2769 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state); 2770 } else { 2771 ring->atr_sample_rate = 0; 2772 } 2773 2774 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); 2775 2776 /* enable queue */ 2777 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); 2778 2779 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 2780 if (hw->mac.type == ixgbe_mac_82598EB && 2781 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 2782 return; 2783 2784 /* poll to verify queue is enabled */ 2785 do { 2786 usleep_range(1000, 2000); 2787 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx)); 2788 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 2789 if (!wait_loop) 2790 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx); 2791 } 2792 2793 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter) 2794 { 2795 struct ixgbe_hw *hw = &adapter->hw; 2796 u32 rttdcs, mtqc; 2797 u8 tcs = netdev_get_num_tc(adapter->netdev); 2798 2799 if (hw->mac.type == ixgbe_mac_82598EB) 2800 return; 2801 2802 /* disable the arbiter while setting MTQC */ 2803 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2804 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2805 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2806 2807 /* set transmit pool layout */ 2808 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 2809 mtqc = IXGBE_MTQC_VT_ENA; 2810 if (tcs > 4) 2811 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 2812 else if (tcs > 1) 2813 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 2814 else if (adapter->ring_feature[RING_F_RSS].indices == 4) 2815 mtqc |= IXGBE_MTQC_32VF; 2816 else 2817 mtqc |= IXGBE_MTQC_64VF; 2818 } else { 2819 if (tcs > 4) 2820 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 2821 else if (tcs > 1) 2822 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 2823 else 2824 mtqc = IXGBE_MTQC_64Q_1PB; 2825 } 2826 2827 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 2828 2829 /* Enable Security TX Buffer IFG for multiple pb */ 2830 if (tcs) { 2831 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 2832 sectx |= IXGBE_SECTX_DCB; 2833 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx); 2834 } 2835 2836 /* re-enable the arbiter */ 2837 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2838 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2839 } 2840 2841 /** 2842 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset 2843 * @adapter: board private structure 2844 * 2845 * Configure the Tx unit of the MAC after a reset. 2846 **/ 2847 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter) 2848 { 2849 struct ixgbe_hw *hw = &adapter->hw; 2850 u32 dmatxctl; 2851 u32 i; 2852 2853 ixgbe_setup_mtqc(adapter); 2854 2855 if (hw->mac.type != ixgbe_mac_82598EB) { 2856 /* DMATXCTL.EN must be before Tx queues are enabled */ 2857 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2858 dmatxctl |= IXGBE_DMATXCTL_TE; 2859 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2860 } 2861 2862 /* Setup the HW Tx Head and Tail descriptor pointers */ 2863 for (i = 0; i < adapter->num_tx_queues; i++) 2864 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]); 2865 } 2866 2867 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter, 2868 struct ixgbe_ring *ring) 2869 { 2870 struct ixgbe_hw *hw = &adapter->hw; 2871 u8 reg_idx = ring->reg_idx; 2872 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 2873 2874 srrctl |= IXGBE_SRRCTL_DROP_EN; 2875 2876 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 2877 } 2878 2879 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter, 2880 struct ixgbe_ring *ring) 2881 { 2882 struct ixgbe_hw *hw = &adapter->hw; 2883 u8 reg_idx = ring->reg_idx; 2884 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx)); 2885 2886 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 2887 2888 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 2889 } 2890 2891 #ifdef CONFIG_IXGBE_DCB 2892 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 2893 #else 2894 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) 2895 #endif 2896 { 2897 int i; 2898 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 2899 2900 if (adapter->ixgbe_ieee_pfc) 2901 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 2902 2903 /* 2904 * We should set the drop enable bit if: 2905 * SR-IOV is enabled 2906 * or 2907 * Number of Rx queues > 1 and flow control is disabled 2908 * 2909 * This allows us to avoid head of line blocking for security 2910 * and performance reasons. 2911 */ 2912 if (adapter->num_vfs || (adapter->num_rx_queues > 1 && 2913 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { 2914 for (i = 0; i < adapter->num_rx_queues; i++) 2915 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); 2916 } else { 2917 for (i = 0; i < adapter->num_rx_queues; i++) 2918 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]); 2919 } 2920 } 2921 2922 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2923 2924 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, 2925 struct ixgbe_ring *rx_ring) 2926 { 2927 struct ixgbe_hw *hw = &adapter->hw; 2928 u32 srrctl; 2929 u8 reg_idx = rx_ring->reg_idx; 2930 2931 if (hw->mac.type == ixgbe_mac_82598EB) { 2932 u16 mask = adapter->ring_feature[RING_F_RSS].mask; 2933 2934 /* 2935 * if VMDq is not active we must program one srrctl register 2936 * per RSS queue since we have enabled RDRXCTL.MVMEN 2937 */ 2938 reg_idx &= mask; 2939 } 2940 2941 /* configure header buffer length, needed for RSC */ 2942 srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 2943 2944 /* configure the packet buffer length */ 2945 srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2946 2947 /* configure descriptor type */ 2948 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2949 2950 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl); 2951 } 2952 2953 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) 2954 { 2955 struct ixgbe_hw *hw = &adapter->hw; 2956 static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D, 2957 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE, 2958 0x6A3E67EA, 0x14364D17, 0x3BED200D}; 2959 u32 mrqc = 0, reta = 0; 2960 u32 rxcsum; 2961 int i, j; 2962 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; 2963 2964 /* 2965 * Program table for at least 2 queues w/ SR-IOV so that VFs can 2966 * make full use of any rings they may have. We will use the 2967 * PSRTYPE register to control how many rings we use within the PF. 2968 */ 2969 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2)) 2970 rss_i = 2; 2971 2972 /* Fill out hash function seeds */ 2973 for (i = 0; i < 10; i++) 2974 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]); 2975 2976 /* Fill out redirection table */ 2977 for (i = 0, j = 0; i < 128; i++, j++) { 2978 if (j == rss_i) 2979 j = 0; 2980 /* reta = 4-byte sliding window of 2981 * 0x00..(indices-1)(indices-1)00..etc. */ 2982 reta = (reta << 8) | (j * 0x11); 2983 if ((i & 3) == 3) 2984 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2985 } 2986 2987 /* Disable indicating checksum in descriptor, enables RSS hash */ 2988 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2989 rxcsum |= IXGBE_RXCSUM_PCSD; 2990 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2991 2992 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 2993 if (adapter->ring_feature[RING_F_RSS].mask) 2994 mrqc = IXGBE_MRQC_RSSEN; 2995 } else { 2996 u8 tcs = netdev_get_num_tc(adapter->netdev); 2997 2998 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 2999 if (tcs > 4) 3000 mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */ 3001 else if (tcs > 1) 3002 mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */ 3003 else if (adapter->ring_feature[RING_F_RSS].indices == 4) 3004 mrqc = IXGBE_MRQC_VMDQRSS32EN; 3005 else 3006 mrqc = IXGBE_MRQC_VMDQRSS64EN; 3007 } else { 3008 if (tcs > 4) 3009 mrqc = IXGBE_MRQC_RTRSS8TCEN; 3010 else if (tcs > 1) 3011 mrqc = IXGBE_MRQC_RTRSS4TCEN; 3012 else 3013 mrqc = IXGBE_MRQC_RSSEN; 3014 } 3015 } 3016 3017 /* Perform hash on these packet types */ 3018 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4 | 3019 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 3020 IXGBE_MRQC_RSS_FIELD_IPV6 | 3021 IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 3022 3023 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) 3024 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 3025 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP) 3026 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 3027 3028 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3029 } 3030 3031 /** 3032 * ixgbe_configure_rscctl - enable RSC for the indicated ring 3033 * @adapter: address of board private structure 3034 * @index: index of ring to set 3035 **/ 3036 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, 3037 struct ixgbe_ring *ring) 3038 { 3039 struct ixgbe_hw *hw = &adapter->hw; 3040 u32 rscctrl; 3041 u8 reg_idx = ring->reg_idx; 3042 3043 if (!ring_is_rsc_enabled(ring)) 3044 return; 3045 3046 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx)); 3047 rscctrl |= IXGBE_RSCCTL_RSCEN; 3048 /* 3049 * we must limit the number of descriptors so that the 3050 * total size of max desc * buf_len is not greater 3051 * than 65536 3052 */ 3053 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3054 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); 3055 } 3056 3057 #define IXGBE_MAX_RX_DESC_POLL 10 3058 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, 3059 struct ixgbe_ring *ring) 3060 { 3061 struct ixgbe_hw *hw = &adapter->hw; 3062 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3063 u32 rxdctl; 3064 u8 reg_idx = ring->reg_idx; 3065 3066 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ 3067 if (hw->mac.type == ixgbe_mac_82598EB && 3068 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3069 return; 3070 3071 do { 3072 usleep_range(1000, 2000); 3073 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3074 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 3075 3076 if (!wait_loop) { 3077 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " 3078 "the polling period\n", reg_idx); 3079 } 3080 } 3081 3082 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter, 3083 struct ixgbe_ring *ring) 3084 { 3085 struct ixgbe_hw *hw = &adapter->hw; 3086 int wait_loop = IXGBE_MAX_RX_DESC_POLL; 3087 u32 rxdctl; 3088 u8 reg_idx = ring->reg_idx; 3089 3090 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3091 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 3092 3093 /* write value back with RXDCTL.ENABLE bit cleared */ 3094 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3095 3096 if (hw->mac.type == ixgbe_mac_82598EB && 3097 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) 3098 return; 3099 3100 /* the hardware may take up to 100us to really disable the rx queue */ 3101 do { 3102 udelay(10); 3103 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3104 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 3105 3106 if (!wait_loop) { 3107 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within " 3108 "the polling period\n", reg_idx); 3109 } 3110 } 3111 3112 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 3113 struct ixgbe_ring *ring) 3114 { 3115 struct ixgbe_hw *hw = &adapter->hw; 3116 u64 rdba = ring->dma; 3117 u32 rxdctl; 3118 u8 reg_idx = ring->reg_idx; 3119 3120 /* disable queue to avoid issues while updating state */ 3121 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx)); 3122 ixgbe_disable_rx_queue(adapter, ring); 3123 3124 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32))); 3125 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32)); 3126 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx), 3127 ring->count * sizeof(union ixgbe_adv_rx_desc)); 3128 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0); 3129 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); 3130 ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx); 3131 3132 ixgbe_configure_srrctl(adapter, ring); 3133 ixgbe_configure_rscctl(adapter, ring); 3134 3135 /* If operating in IOV mode set RLPML for X540 */ 3136 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && 3137 hw->mac.type == ixgbe_mac_X540) { 3138 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 3139 rxdctl |= ((ring->netdev->mtu + ETH_HLEN + 3140 ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN); 3141 } 3142 3143 if (hw->mac.type == ixgbe_mac_82598EB) { 3144 /* 3145 * enable cache line friendly hardware writes: 3146 * PTHRESH=32 descriptors (half the internal cache), 3147 * this also removes ugly rx_no_buffer_count increment 3148 * HTHRESH=4 descriptors (to minimize latency on fetch) 3149 * WTHRESH=8 burst writeback up to two cache lines 3150 */ 3151 rxdctl &= ~0x3FFFFF; 3152 rxdctl |= 0x080420; 3153 } 3154 3155 /* enable receive descriptor ring */ 3156 rxdctl |= IXGBE_RXDCTL_ENABLE; 3157 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); 3158 3159 ixgbe_rx_desc_queue_enable(adapter, ring); 3160 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); 3161 } 3162 3163 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) 3164 { 3165 struct ixgbe_hw *hw = &adapter->hw; 3166 int rss_i = adapter->ring_feature[RING_F_RSS].indices; 3167 int p; 3168 3169 /* PSRTYPE must be initialized in non 82598 adapters */ 3170 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 3171 IXGBE_PSRTYPE_UDPHDR | 3172 IXGBE_PSRTYPE_IPV4HDR | 3173 IXGBE_PSRTYPE_L2HDR | 3174 IXGBE_PSRTYPE_IPV6HDR; 3175 3176 if (hw->mac.type == ixgbe_mac_82598EB) 3177 return; 3178 3179 if (rss_i > 3) 3180 psrtype |= 2 << 29; 3181 else if (rss_i > 1) 3182 psrtype |= 1 << 29; 3183 3184 for (p = 0; p < adapter->num_rx_pools; p++) 3185 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(p)), 3186 psrtype); 3187 } 3188 3189 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) 3190 { 3191 struct ixgbe_hw *hw = &adapter->hw; 3192 u32 reg_offset, vf_shift; 3193 u32 gcr_ext, vmdctl; 3194 int i; 3195 3196 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 3197 return; 3198 3199 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL); 3200 vmdctl |= IXGBE_VMD_CTL_VMDQ_EN; 3201 vmdctl &= ~IXGBE_VT_CTL_POOL_MASK; 3202 vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT; 3203 vmdctl |= IXGBE_VT_CTL_REPLEN; 3204 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl); 3205 3206 vf_shift = VMDQ_P(0) % 32; 3207 reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0; 3208 3209 /* Enable only the PF's pool for Tx/Rx */ 3210 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (~0) << vf_shift); 3211 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1); 3212 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (~0) << vf_shift); 3213 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1); 3214 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 3215 3216 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */ 3217 hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0)); 3218 3219 /* 3220 * Set up VF register offsets for selected VT Mode, 3221 * i.e. 32 or 64 VFs for SR-IOV 3222 */ 3223 switch (adapter->ring_feature[RING_F_VMDQ].mask) { 3224 case IXGBE_82599_VMDQ_8Q_MASK: 3225 gcr_ext = IXGBE_GCR_EXT_VT_MODE_16; 3226 break; 3227 case IXGBE_82599_VMDQ_4Q_MASK: 3228 gcr_ext = IXGBE_GCR_EXT_VT_MODE_32; 3229 break; 3230 default: 3231 gcr_ext = IXGBE_GCR_EXT_VT_MODE_64; 3232 break; 3233 } 3234 3235 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext); 3236 3237 /* enable Tx loopback for VF/PF communication */ 3238 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 3239 3240 /* Enable MAC Anti-Spoofing */ 3241 hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0), 3242 adapter->num_vfs); 3243 /* For VFs that have spoof checking turned off */ 3244 for (i = 0; i < adapter->num_vfs; i++) { 3245 if (!adapter->vfinfo[i].spoofchk_enabled) 3246 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false); 3247 } 3248 } 3249 3250 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) 3251 { 3252 struct ixgbe_hw *hw = &adapter->hw; 3253 struct net_device *netdev = adapter->netdev; 3254 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3255 struct ixgbe_ring *rx_ring; 3256 int i; 3257 u32 mhadd, hlreg0; 3258 3259 #ifdef IXGBE_FCOE 3260 /* adjust max frame to be able to do baby jumbo for FCoE */ 3261 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && 3262 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) 3263 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3264 3265 #endif /* IXGBE_FCOE */ 3266 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 3267 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { 3268 mhadd &= ~IXGBE_MHADD_MFS_MASK; 3269 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; 3270 3271 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 3272 } 3273 3274 /* MHADD will allow an extra 4 bytes past for vlan tagged frames */ 3275 max_frame += VLAN_HLEN; 3276 3277 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3278 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ 3279 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 3280 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 3281 3282 /* 3283 * Setup the HW Rx Head and Tail Descriptor Pointers and 3284 * the Base and Length of the Rx Descriptor Ring 3285 */ 3286 for (i = 0; i < adapter->num_rx_queues; i++) { 3287 rx_ring = adapter->rx_ring[i]; 3288 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 3289 set_ring_rsc_enabled(rx_ring); 3290 else 3291 clear_ring_rsc_enabled(rx_ring); 3292 } 3293 } 3294 3295 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) 3296 { 3297 struct ixgbe_hw *hw = &adapter->hw; 3298 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3299 3300 switch (hw->mac.type) { 3301 case ixgbe_mac_82598EB: 3302 /* 3303 * For VMDq support of different descriptor types or 3304 * buffer sizes through the use of multiple SRRCTL 3305 * registers, RDRXCTL.MVMEN must be set to 1 3306 * 3307 * also, the manual doesn't mention it clearly but DCA hints 3308 * will only use queue 0's tags unless this bit is set. Side 3309 * effects of setting this bit are only that SRRCTL must be 3310 * fully programmed [0..15] 3311 */ 3312 rdrxctl |= IXGBE_RDRXCTL_MVMEN; 3313 break; 3314 case ixgbe_mac_82599EB: 3315 case ixgbe_mac_X540: 3316 /* Disable RSC for ACK packets */ 3317 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3318 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3319 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3320 /* hardware requires some bits to be set by default */ 3321 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX); 3322 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3323 break; 3324 default: 3325 /* We should do nothing since we don't know this hardware */ 3326 return; 3327 } 3328 3329 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3330 } 3331 3332 /** 3333 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset 3334 * @adapter: board private structure 3335 * 3336 * Configure the Rx unit of the MAC after a reset. 3337 **/ 3338 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) 3339 { 3340 struct ixgbe_hw *hw = &adapter->hw; 3341 int i; 3342 u32 rxctrl; 3343 3344 /* disable receives while setting up the descriptors */ 3345 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3346 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 3347 3348 ixgbe_setup_psrtype(adapter); 3349 ixgbe_setup_rdrxctl(adapter); 3350 3351 /* Program registers for the distribution of queues */ 3352 ixgbe_setup_mrqc(adapter); 3353 3354 /* set_rx_buffer_len must be called before ring initialization */ 3355 ixgbe_set_rx_buffer_len(adapter); 3356 3357 /* 3358 * Setup the HW Rx Head and Tail Descriptor Pointers and 3359 * the Base and Length of the Rx Descriptor Ring 3360 */ 3361 for (i = 0; i < adapter->num_rx_queues; i++) 3362 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); 3363 3364 /* disable drop enable for 82598 parts */ 3365 if (hw->mac.type == ixgbe_mac_82598EB) 3366 rxctrl |= IXGBE_RXCTRL_DMBYPS; 3367 3368 /* enable all receives */ 3369 rxctrl |= IXGBE_RXCTRL_RXEN; 3370 hw->mac.ops.enable_rx_dma(hw, rxctrl); 3371 } 3372 3373 static int ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 3374 { 3375 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3376 struct ixgbe_hw *hw = &adapter->hw; 3377 3378 /* add VID to filter table */ 3379 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true); 3380 set_bit(vid, adapter->active_vlans); 3381 3382 return 0; 3383 } 3384 3385 static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 3386 { 3387 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3388 struct ixgbe_hw *hw = &adapter->hw; 3389 3390 /* remove VID from filter table */ 3391 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), false); 3392 clear_bit(vid, adapter->active_vlans); 3393 3394 return 0; 3395 } 3396 3397 /** 3398 * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering 3399 * @adapter: driver data 3400 */ 3401 static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter) 3402 { 3403 struct ixgbe_hw *hw = &adapter->hw; 3404 u32 vlnctrl; 3405 3406 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3407 vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN); 3408 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3409 } 3410 3411 /** 3412 * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering 3413 * @adapter: driver data 3414 */ 3415 static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter) 3416 { 3417 struct ixgbe_hw *hw = &adapter->hw; 3418 u32 vlnctrl; 3419 3420 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3421 vlnctrl |= IXGBE_VLNCTRL_VFE; 3422 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN; 3423 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3424 } 3425 3426 /** 3427 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping 3428 * @adapter: driver data 3429 */ 3430 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter) 3431 { 3432 struct ixgbe_hw *hw = &adapter->hw; 3433 u32 vlnctrl; 3434 int i, j; 3435 3436 switch (hw->mac.type) { 3437 case ixgbe_mac_82598EB: 3438 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3439 vlnctrl &= ~IXGBE_VLNCTRL_VME; 3440 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3441 break; 3442 case ixgbe_mac_82599EB: 3443 case ixgbe_mac_X540: 3444 for (i = 0; i < adapter->num_rx_queues; i++) { 3445 j = adapter->rx_ring[i]->reg_idx; 3446 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 3447 vlnctrl &= ~IXGBE_RXDCTL_VME; 3448 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 3449 } 3450 break; 3451 default: 3452 break; 3453 } 3454 } 3455 3456 /** 3457 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping 3458 * @adapter: driver data 3459 */ 3460 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter) 3461 { 3462 struct ixgbe_hw *hw = &adapter->hw; 3463 u32 vlnctrl; 3464 int i, j; 3465 3466 switch (hw->mac.type) { 3467 case ixgbe_mac_82598EB: 3468 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3469 vlnctrl |= IXGBE_VLNCTRL_VME; 3470 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); 3471 break; 3472 case ixgbe_mac_82599EB: 3473 case ixgbe_mac_X540: 3474 for (i = 0; i < adapter->num_rx_queues; i++) { 3475 j = adapter->rx_ring[i]->reg_idx; 3476 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j)); 3477 vlnctrl |= IXGBE_RXDCTL_VME; 3478 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl); 3479 } 3480 break; 3481 default: 3482 break; 3483 } 3484 } 3485 3486 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter) 3487 { 3488 u16 vid; 3489 3490 ixgbe_vlan_rx_add_vid(adapter->netdev, 0); 3491 3492 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 3493 ixgbe_vlan_rx_add_vid(adapter->netdev, vid); 3494 } 3495 3496 /** 3497 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table 3498 * @netdev: network interface device structure 3499 * 3500 * Writes unicast address list to the RAR table. 3501 * Returns: -ENOMEM on failure/insufficient address space 3502 * 0 on no addresses written 3503 * X on writing X addresses to the RAR table 3504 **/ 3505 static int ixgbe_write_uc_addr_list(struct net_device *netdev) 3506 { 3507 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3508 struct ixgbe_hw *hw = &adapter->hw; 3509 unsigned int rar_entries = hw->mac.num_rar_entries - 1; 3510 int count = 0; 3511 3512 /* In SR-IOV mode significantly less RAR entries are available */ 3513 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 3514 rar_entries = IXGBE_MAX_PF_MACVLANS - 1; 3515 3516 /* return ENOMEM indicating insufficient memory for addresses */ 3517 if (netdev_uc_count(netdev) > rar_entries) 3518 return -ENOMEM; 3519 3520 if (!netdev_uc_empty(netdev)) { 3521 struct netdev_hw_addr *ha; 3522 /* return error if we do not support writing to RAR table */ 3523 if (!hw->mac.ops.set_rar) 3524 return -ENOMEM; 3525 3526 netdev_for_each_uc_addr(ha, netdev) { 3527 if (!rar_entries) 3528 break; 3529 hw->mac.ops.set_rar(hw, rar_entries--, ha->addr, 3530 VMDQ_P(0), IXGBE_RAH_AV); 3531 count++; 3532 } 3533 } 3534 /* write the addresses in reverse order to avoid write combining */ 3535 for (; rar_entries > 0 ; rar_entries--) 3536 hw->mac.ops.clear_rar(hw, rar_entries); 3537 3538 return count; 3539 } 3540 3541 /** 3542 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set 3543 * @netdev: network interface device structure 3544 * 3545 * The set_rx_method entry point is called whenever the unicast/multicast 3546 * address list or the network interface flags are updated. This routine is 3547 * responsible for configuring the hardware for proper unicast, multicast and 3548 * promiscuous mode. 3549 **/ 3550 void ixgbe_set_rx_mode(struct net_device *netdev) 3551 { 3552 struct ixgbe_adapter *adapter = netdev_priv(netdev); 3553 struct ixgbe_hw *hw = &adapter->hw; 3554 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE; 3555 int count; 3556 3557 /* Check for Promiscuous and All Multicast modes */ 3558 3559 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3560 3561 /* set all bits that we expect to always be set */ 3562 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */ 3563 fctrl |= IXGBE_FCTRL_BAM; 3564 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */ 3565 fctrl |= IXGBE_FCTRL_PMCF; 3566 3567 /* clear the bits we are changing the status of */ 3568 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 3569 3570 if (netdev->flags & IFF_PROMISC) { 3571 hw->addr_ctrl.user_set_promisc = true; 3572 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 3573 vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE); 3574 /* don't hardware filter vlans in promisc mode */ 3575 ixgbe_vlan_filter_disable(adapter); 3576 } else { 3577 if (netdev->flags & IFF_ALLMULTI) { 3578 fctrl |= IXGBE_FCTRL_MPE; 3579 vmolr |= IXGBE_VMOLR_MPE; 3580 } else { 3581 /* 3582 * Write addresses to the MTA, if the attempt fails 3583 * then we should just turn on promiscuous mode so 3584 * that we can at least receive multicast traffic 3585 */ 3586 hw->mac.ops.update_mc_addr_list(hw, netdev); 3587 vmolr |= IXGBE_VMOLR_ROMPE; 3588 } 3589 ixgbe_vlan_filter_enable(adapter); 3590 hw->addr_ctrl.user_set_promisc = false; 3591 } 3592 3593 /* 3594 * Write addresses to available RAR registers, if there is not 3595 * sufficient space to store all the addresses then enable 3596 * unicast promiscuous mode 3597 */ 3598 count = ixgbe_write_uc_addr_list(netdev); 3599 if (count < 0) { 3600 fctrl |= IXGBE_FCTRL_UPE; 3601 vmolr |= IXGBE_VMOLR_ROPE; 3602 } 3603 3604 if (adapter->num_vfs) 3605 ixgbe_restore_vf_multicasts(adapter); 3606 3607 if (hw->mac.type != ixgbe_mac_82598EB) { 3608 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) & 3609 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE | 3610 IXGBE_VMOLR_ROPE); 3611 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr); 3612 } 3613 3614 /* This is useful for sniffing bad packets. */ 3615 if (adapter->netdev->features & NETIF_F_RXALL) { 3616 /* UPE and MPE will be handled by normal PROMISC logic 3617 * in e1000e_set_rx_mode */ 3618 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */ 3619 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */ 3620 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */ 3621 3622 fctrl &= ~(IXGBE_FCTRL_DPF); 3623 /* NOTE: VLAN filtering is disabled by setting PROMISC */ 3624 } 3625 3626 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3627 3628 if (netdev->features & NETIF_F_HW_VLAN_RX) 3629 ixgbe_vlan_strip_enable(adapter); 3630 else 3631 ixgbe_vlan_strip_disable(adapter); 3632 } 3633 3634 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter) 3635 { 3636 int q_idx; 3637 3638 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) 3639 napi_enable(&adapter->q_vector[q_idx]->napi); 3640 } 3641 3642 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) 3643 { 3644 int q_idx; 3645 3646 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) 3647 napi_disable(&adapter->q_vector[q_idx]->napi); 3648 } 3649 3650 #ifdef CONFIG_IXGBE_DCB 3651 /** 3652 * ixgbe_configure_dcb - Configure DCB hardware 3653 * @adapter: ixgbe adapter struct 3654 * 3655 * This is called by the driver on open to configure the DCB hardware. 3656 * This is also called by the gennetlink interface when reconfiguring 3657 * the DCB state. 3658 */ 3659 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) 3660 { 3661 struct ixgbe_hw *hw = &adapter->hw; 3662 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 3663 3664 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) { 3665 if (hw->mac.type == ixgbe_mac_82598EB) 3666 netif_set_gso_max_size(adapter->netdev, 65536); 3667 return; 3668 } 3669 3670 if (hw->mac.type == ixgbe_mac_82598EB) 3671 netif_set_gso_max_size(adapter->netdev, 32768); 3672 3673 #ifdef IXGBE_FCOE 3674 if (adapter->netdev->features & NETIF_F_FCOE_MTU) 3675 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); 3676 #endif 3677 3678 /* reconfigure the hardware */ 3679 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) { 3680 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 3681 DCB_TX_CONFIG); 3682 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame, 3683 DCB_RX_CONFIG); 3684 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg); 3685 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) { 3686 ixgbe_dcb_hw_ets(&adapter->hw, 3687 adapter->ixgbe_ieee_ets, 3688 max_frame); 3689 ixgbe_dcb_hw_pfc_config(&adapter->hw, 3690 adapter->ixgbe_ieee_pfc->pfc_en, 3691 adapter->ixgbe_ieee_ets->prio_tc); 3692 } 3693 3694 /* Enable RSS Hash per TC */ 3695 if (hw->mac.type != ixgbe_mac_82598EB) { 3696 u32 msb = 0; 3697 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1; 3698 3699 while (rss_i) { 3700 msb++; 3701 rss_i >>= 1; 3702 } 3703 3704 /* write msb to all 8 TCs in one write */ 3705 IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111); 3706 } 3707 } 3708 #endif 3709 3710 /* Additional bittime to account for IXGBE framing */ 3711 #define IXGBE_ETH_FRAMING 20 3712 3713 /** 3714 * ixgbe_hpbthresh - calculate high water mark for flow control 3715 * 3716 * @adapter: board private structure to calculate for 3717 * @pb: packet buffer to calculate 3718 */ 3719 static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) 3720 { 3721 struct ixgbe_hw *hw = &adapter->hw; 3722 struct net_device *dev = adapter->netdev; 3723 int link, tc, kb, marker; 3724 u32 dv_id, rx_pba; 3725 3726 /* Calculate max LAN frame size */ 3727 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING; 3728 3729 #ifdef IXGBE_FCOE 3730 /* FCoE traffic class uses FCOE jumbo frames */ 3731 if ((dev->features & NETIF_F_FCOE_MTU) && 3732 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && 3733 (pb == ixgbe_fcoe_get_tc(adapter))) 3734 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; 3735 3736 #endif 3737 /* Calculate delay value for device */ 3738 switch (hw->mac.type) { 3739 case ixgbe_mac_X540: 3740 dv_id = IXGBE_DV_X540(link, tc); 3741 break; 3742 default: 3743 dv_id = IXGBE_DV(link, tc); 3744 break; 3745 } 3746 3747 /* Loopback switch introduces additional latency */ 3748 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 3749 dv_id += IXGBE_B2BT(tc); 3750 3751 /* Delay value is calculated in bit times convert to KB */ 3752 kb = IXGBE_BT2KB(dv_id); 3753 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10; 3754 3755 marker = rx_pba - kb; 3756 3757 /* It is possible that the packet buffer is not large enough 3758 * to provide required headroom. In this case throw an error 3759 * to user and a do the best we can. 3760 */ 3761 if (marker < 0) { 3762 e_warn(drv, "Packet Buffer(%i) can not provide enough" 3763 "headroom to support flow control." 3764 "Decrease MTU or number of traffic classes\n", pb); 3765 marker = tc + 1; 3766 } 3767 3768 return marker; 3769 } 3770 3771 /** 3772 * ixgbe_lpbthresh - calculate low water mark for for flow control 3773 * 3774 * @adapter: board private structure to calculate for 3775 * @pb: packet buffer to calculate 3776 */ 3777 static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter) 3778 { 3779 struct ixgbe_hw *hw = &adapter->hw; 3780 struct net_device *dev = adapter->netdev; 3781 int tc; 3782 u32 dv_id; 3783 3784 /* Calculate max LAN frame size */ 3785 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN; 3786 3787 /* Calculate delay value for device */ 3788 switch (hw->mac.type) { 3789 case ixgbe_mac_X540: 3790 dv_id = IXGBE_LOW_DV_X540(tc); 3791 break; 3792 default: 3793 dv_id = IXGBE_LOW_DV(tc); 3794 break; 3795 } 3796 3797 /* Delay value is calculated in bit times convert to KB */ 3798 return IXGBE_BT2KB(dv_id); 3799 } 3800 3801 /* 3802 * ixgbe_pbthresh_setup - calculate and setup high low water marks 3803 */ 3804 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter) 3805 { 3806 struct ixgbe_hw *hw = &adapter->hw; 3807 int num_tc = netdev_get_num_tc(adapter->netdev); 3808 int i; 3809 3810 if (!num_tc) 3811 num_tc = 1; 3812 3813 hw->fc.low_water = ixgbe_lpbthresh(adapter); 3814 3815 for (i = 0; i < num_tc; i++) { 3816 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i); 3817 3818 /* Low water marks must not be larger than high water marks */ 3819 if (hw->fc.low_water > hw->fc.high_water[i]) 3820 hw->fc.low_water = 0; 3821 } 3822 } 3823 3824 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) 3825 { 3826 struct ixgbe_hw *hw = &adapter->hw; 3827 int hdrm; 3828 u8 tc = netdev_get_num_tc(adapter->netdev); 3829 3830 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || 3831 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 3832 hdrm = 32 << adapter->fdir_pballoc; 3833 else 3834 hdrm = 0; 3835 3836 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); 3837 ixgbe_pbthresh_setup(adapter); 3838 } 3839 3840 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) 3841 { 3842 struct ixgbe_hw *hw = &adapter->hw; 3843 struct hlist_node *node, *node2; 3844 struct ixgbe_fdir_filter *filter; 3845 3846 spin_lock(&adapter->fdir_perfect_lock); 3847 3848 if (!hlist_empty(&adapter->fdir_filter_list)) 3849 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask); 3850 3851 hlist_for_each_entry_safe(filter, node, node2, 3852 &adapter->fdir_filter_list, fdir_node) { 3853 ixgbe_fdir_write_perfect_filter_82599(hw, 3854 &filter->filter, 3855 filter->sw_idx, 3856 (filter->action == IXGBE_FDIR_DROP_QUEUE) ? 3857 IXGBE_FDIR_DROP_QUEUE : 3858 adapter->rx_ring[filter->action]->reg_idx); 3859 } 3860 3861 spin_unlock(&adapter->fdir_perfect_lock); 3862 } 3863 3864 static void ixgbe_configure(struct ixgbe_adapter *adapter) 3865 { 3866 struct ixgbe_hw *hw = &adapter->hw; 3867 3868 ixgbe_configure_pb(adapter); 3869 #ifdef CONFIG_IXGBE_DCB 3870 ixgbe_configure_dcb(adapter); 3871 #endif 3872 /* 3873 * We must restore virtualization before VLANs or else 3874 * the VLVF registers will not be populated 3875 */ 3876 ixgbe_configure_virtualization(adapter); 3877 3878 ixgbe_set_rx_mode(adapter->netdev); 3879 ixgbe_restore_vlan(adapter); 3880 3881 switch (hw->mac.type) { 3882 case ixgbe_mac_82599EB: 3883 case ixgbe_mac_X540: 3884 hw->mac.ops.disable_rx_buff(hw); 3885 break; 3886 default: 3887 break; 3888 } 3889 3890 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 3891 ixgbe_init_fdir_signature_82599(&adapter->hw, 3892 adapter->fdir_pballoc); 3893 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { 3894 ixgbe_init_fdir_perfect_82599(&adapter->hw, 3895 adapter->fdir_pballoc); 3896 ixgbe_fdir_filter_restore(adapter); 3897 } 3898 3899 switch (hw->mac.type) { 3900 case ixgbe_mac_82599EB: 3901 case ixgbe_mac_X540: 3902 hw->mac.ops.enable_rx_buff(hw); 3903 break; 3904 default: 3905 break; 3906 } 3907 3908 #ifdef IXGBE_FCOE 3909 /* configure FCoE L2 filters, redirection table, and Rx control */ 3910 ixgbe_configure_fcoe(adapter); 3911 3912 #endif /* IXGBE_FCOE */ 3913 ixgbe_configure_tx(adapter); 3914 ixgbe_configure_rx(adapter); 3915 } 3916 3917 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw) 3918 { 3919 switch (hw->phy.type) { 3920 case ixgbe_phy_sfp_avago: 3921 case ixgbe_phy_sfp_ftl: 3922 case ixgbe_phy_sfp_intel: 3923 case ixgbe_phy_sfp_unknown: 3924 case ixgbe_phy_sfp_passive_tyco: 3925 case ixgbe_phy_sfp_passive_unknown: 3926 case ixgbe_phy_sfp_active_unknown: 3927 case ixgbe_phy_sfp_ftl_active: 3928 return true; 3929 case ixgbe_phy_nl: 3930 if (hw->mac.type == ixgbe_mac_82598EB) 3931 return true; 3932 default: 3933 return false; 3934 } 3935 } 3936 3937 /** 3938 * ixgbe_sfp_link_config - set up SFP+ link 3939 * @adapter: pointer to private adapter struct 3940 **/ 3941 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) 3942 { 3943 /* 3944 * We are assuming the worst case scenario here, and that 3945 * is that an SFP was inserted/removed after the reset 3946 * but before SFP detection was enabled. As such the best 3947 * solution is to just start searching as soon as we start 3948 */ 3949 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3950 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 3951 3952 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 3953 } 3954 3955 /** 3956 * ixgbe_non_sfp_link_config - set up non-SFP+ link 3957 * @hw: pointer to private hardware struct 3958 * 3959 * Returns 0 on success, negative on failure 3960 **/ 3961 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) 3962 { 3963 u32 autoneg; 3964 bool negotiation, link_up = false; 3965 u32 ret = IXGBE_ERR_LINK_SETUP; 3966 3967 if (hw->mac.ops.check_link) 3968 ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false); 3969 3970 if (ret) 3971 goto link_cfg_out; 3972 3973 autoneg = hw->phy.autoneg_advertised; 3974 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 3975 ret = hw->mac.ops.get_link_capabilities(hw, &autoneg, 3976 &negotiation); 3977 if (ret) 3978 goto link_cfg_out; 3979 3980 if (hw->mac.ops.setup_link) 3981 ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up); 3982 link_cfg_out: 3983 return ret; 3984 } 3985 3986 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) 3987 { 3988 struct ixgbe_hw *hw = &adapter->hw; 3989 u32 gpie = 0; 3990 3991 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 3992 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | 3993 IXGBE_GPIE_OCD; 3994 gpie |= IXGBE_GPIE_EIAME; 3995 /* 3996 * use EIAM to auto-mask when MSI-X interrupt is asserted 3997 * this saves a register write for every interrupt 3998 */ 3999 switch (hw->mac.type) { 4000 case ixgbe_mac_82598EB: 4001 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4002 break; 4003 case ixgbe_mac_82599EB: 4004 case ixgbe_mac_X540: 4005 default: 4006 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4007 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4008 break; 4009 } 4010 } else { 4011 /* legacy interrupts, use EIAM to auto-mask when reading EICR, 4012 * specifically only auto mask tx and rx interrupts */ 4013 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4014 } 4015 4016 /* XXX: to interrupt immediately for EICS writes, enable this */ 4017 /* gpie |= IXGBE_GPIE_EIMEN; */ 4018 4019 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 4020 gpie &= ~IXGBE_GPIE_VTMODE_MASK; 4021 4022 switch (adapter->ring_feature[RING_F_VMDQ].mask) { 4023 case IXGBE_82599_VMDQ_8Q_MASK: 4024 gpie |= IXGBE_GPIE_VTMODE_16; 4025 break; 4026 case IXGBE_82599_VMDQ_4Q_MASK: 4027 gpie |= IXGBE_GPIE_VTMODE_32; 4028 break; 4029 default: 4030 gpie |= IXGBE_GPIE_VTMODE_64; 4031 break; 4032 } 4033 } 4034 4035 /* Enable Thermal over heat sensor interrupt */ 4036 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) { 4037 switch (adapter->hw.mac.type) { 4038 case ixgbe_mac_82599EB: 4039 gpie |= IXGBE_SDP0_GPIEN; 4040 break; 4041 case ixgbe_mac_X540: 4042 gpie |= IXGBE_EIMS_TS; 4043 break; 4044 default: 4045 break; 4046 } 4047 } 4048 4049 /* Enable fan failure interrupt */ 4050 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) 4051 gpie |= IXGBE_SDP1_GPIEN; 4052 4053 if (hw->mac.type == ixgbe_mac_82599EB) { 4054 gpie |= IXGBE_SDP1_GPIEN; 4055 gpie |= IXGBE_SDP2_GPIEN; 4056 } 4057 4058 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4059 } 4060 4061 static void ixgbe_up_complete(struct ixgbe_adapter *adapter) 4062 { 4063 struct ixgbe_hw *hw = &adapter->hw; 4064 int err; 4065 u32 ctrl_ext; 4066 4067 ixgbe_get_hw_control(adapter); 4068 ixgbe_setup_gpie(adapter); 4069 4070 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) 4071 ixgbe_configure_msix(adapter); 4072 else 4073 ixgbe_configure_msi_and_legacy(adapter); 4074 4075 /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */ 4076 if (hw->mac.ops.enable_tx_laser && 4077 ((hw->phy.multispeed_fiber) || 4078 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && 4079 (hw->mac.type == ixgbe_mac_82599EB)))) 4080 hw->mac.ops.enable_tx_laser(hw); 4081 4082 clear_bit(__IXGBE_DOWN, &adapter->state); 4083 ixgbe_napi_enable_all(adapter); 4084 4085 if (ixgbe_is_sfp(hw)) { 4086 ixgbe_sfp_link_config(adapter); 4087 } else { 4088 err = ixgbe_non_sfp_link_config(hw); 4089 if (err) 4090 e_err(probe, "link_config FAILED %d\n", err); 4091 } 4092 4093 /* clear any pending interrupts, may auto mask */ 4094 IXGBE_READ_REG(hw, IXGBE_EICR); 4095 ixgbe_irq_enable(adapter, true, true); 4096 4097 /* 4098 * If this adapter has a fan, check to see if we had a failure 4099 * before we enabled the interrupt. 4100 */ 4101 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 4102 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 4103 if (esdp & IXGBE_ESDP_SDP1) 4104 e_crit(drv, "Fan has stopped, replace the adapter\n"); 4105 } 4106 4107 /* enable transmits */ 4108 netif_tx_start_all_queues(adapter->netdev); 4109 4110 /* bring the link up in the watchdog, this could race with our first 4111 * link up interrupt but shouldn't be a problem */ 4112 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 4113 adapter->link_check_timeout = jiffies; 4114 mod_timer(&adapter->service_timer, jiffies); 4115 4116 /* Set PF Reset Done bit so PF/VF Mail Ops can work */ 4117 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4118 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4119 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4120 } 4121 4122 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter) 4123 { 4124 WARN_ON(in_interrupt()); 4125 /* put off any impending NetWatchDogTimeout */ 4126 adapter->netdev->trans_start = jiffies; 4127 4128 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state)) 4129 usleep_range(1000, 2000); 4130 ixgbe_down(adapter); 4131 /* 4132 * If SR-IOV enabled then wait a bit before bringing the adapter 4133 * back up to give the VFs time to respond to the reset. The 4134 * two second wait is based upon the watchdog timer cycle in 4135 * the VF driver. 4136 */ 4137 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 4138 msleep(2000); 4139 ixgbe_up(adapter); 4140 clear_bit(__IXGBE_RESETTING, &adapter->state); 4141 } 4142 4143 void ixgbe_up(struct ixgbe_adapter *adapter) 4144 { 4145 /* hardware has been reset, we need to reload some things */ 4146 ixgbe_configure(adapter); 4147 4148 ixgbe_up_complete(adapter); 4149 } 4150 4151 void ixgbe_reset(struct ixgbe_adapter *adapter) 4152 { 4153 struct ixgbe_hw *hw = &adapter->hw; 4154 int err; 4155 4156 /* lock SFP init bit to prevent race conditions with the watchdog */ 4157 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 4158 usleep_range(1000, 2000); 4159 4160 /* clear all SFP and link config related flags while holding SFP_INIT */ 4161 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | 4162 IXGBE_FLAG2_SFP_NEEDS_RESET); 4163 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 4164 4165 err = hw->mac.ops.init_hw(hw); 4166 switch (err) { 4167 case 0: 4168 case IXGBE_ERR_SFP_NOT_PRESENT: 4169 case IXGBE_ERR_SFP_NOT_SUPPORTED: 4170 break; 4171 case IXGBE_ERR_MASTER_REQUESTS_PENDING: 4172 e_dev_err("master disable timed out\n"); 4173 break; 4174 case IXGBE_ERR_EEPROM_VERSION: 4175 /* We are running on a pre-production device, log a warning */ 4176 e_dev_warn("This device is a pre-production adapter/LOM. " 4177 "Please be aware there may be issues associated with " 4178 "your hardware. If you are experiencing problems " 4179 "please contact your Intel or hardware " 4180 "representative who provided you with this " 4181 "hardware.\n"); 4182 break; 4183 default: 4184 e_dev_err("Hardware Error: %d\n", err); 4185 } 4186 4187 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 4188 4189 /* reprogram the RAR[0] in case user changed it. */ 4190 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV); 4191 4192 /* update SAN MAC vmdq pool selection */ 4193 if (hw->mac.san_mac_rar_index) 4194 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0)); 4195 } 4196 4197 /** 4198 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue 4199 * @rx_ring: ring to free buffers from 4200 **/ 4201 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) 4202 { 4203 struct device *dev = rx_ring->dev; 4204 unsigned long size; 4205 u16 i; 4206 4207 /* ring already cleared, nothing to do */ 4208 if (!rx_ring->rx_buffer_info) 4209 return; 4210 4211 /* Free all the Rx ring sk_buffs */ 4212 for (i = 0; i < rx_ring->count; i++) { 4213 struct ixgbe_rx_buffer *rx_buffer; 4214 4215 rx_buffer = &rx_ring->rx_buffer_info[i]; 4216 if (rx_buffer->skb) { 4217 struct sk_buff *skb = rx_buffer->skb; 4218 if (IXGBE_CB(skb)->page_released) { 4219 dma_unmap_page(dev, 4220 IXGBE_CB(skb)->dma, 4221 ixgbe_rx_bufsz(rx_ring), 4222 DMA_FROM_DEVICE); 4223 IXGBE_CB(skb)->page_released = false; 4224 } 4225 dev_kfree_skb(skb); 4226 } 4227 rx_buffer->skb = NULL; 4228 if (rx_buffer->dma) 4229 dma_unmap_page(dev, rx_buffer->dma, 4230 ixgbe_rx_pg_size(rx_ring), 4231 DMA_FROM_DEVICE); 4232 rx_buffer->dma = 0; 4233 if (rx_buffer->page) 4234 __free_pages(rx_buffer->page, 4235 ixgbe_rx_pg_order(rx_ring)); 4236 rx_buffer->page = NULL; 4237 } 4238 4239 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4240 memset(rx_ring->rx_buffer_info, 0, size); 4241 4242 /* Zero out the descriptor ring */ 4243 memset(rx_ring->desc, 0, rx_ring->size); 4244 4245 rx_ring->next_to_alloc = 0; 4246 rx_ring->next_to_clean = 0; 4247 rx_ring->next_to_use = 0; 4248 } 4249 4250 /** 4251 * ixgbe_clean_tx_ring - Free Tx Buffers 4252 * @tx_ring: ring to be cleaned 4253 **/ 4254 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) 4255 { 4256 struct ixgbe_tx_buffer *tx_buffer_info; 4257 unsigned long size; 4258 u16 i; 4259 4260 /* ring already cleared, nothing to do */ 4261 if (!tx_ring->tx_buffer_info) 4262 return; 4263 4264 /* Free all the Tx ring sk_buffs */ 4265 for (i = 0; i < tx_ring->count; i++) { 4266 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 4267 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 4268 } 4269 4270 netdev_tx_reset_queue(txring_txq(tx_ring)); 4271 4272 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 4273 memset(tx_ring->tx_buffer_info, 0, size); 4274 4275 /* Zero out the descriptor ring */ 4276 memset(tx_ring->desc, 0, tx_ring->size); 4277 4278 tx_ring->next_to_use = 0; 4279 tx_ring->next_to_clean = 0; 4280 } 4281 4282 /** 4283 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues 4284 * @adapter: board private structure 4285 **/ 4286 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) 4287 { 4288 int i; 4289 4290 for (i = 0; i < adapter->num_rx_queues; i++) 4291 ixgbe_clean_rx_ring(adapter->rx_ring[i]); 4292 } 4293 4294 /** 4295 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues 4296 * @adapter: board private structure 4297 **/ 4298 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter) 4299 { 4300 int i; 4301 4302 for (i = 0; i < adapter->num_tx_queues; i++) 4303 ixgbe_clean_tx_ring(adapter->tx_ring[i]); 4304 } 4305 4306 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter) 4307 { 4308 struct hlist_node *node, *node2; 4309 struct ixgbe_fdir_filter *filter; 4310 4311 spin_lock(&adapter->fdir_perfect_lock); 4312 4313 hlist_for_each_entry_safe(filter, node, node2, 4314 &adapter->fdir_filter_list, fdir_node) { 4315 hlist_del(&filter->fdir_node); 4316 kfree(filter); 4317 } 4318 adapter->fdir_filter_count = 0; 4319 4320 spin_unlock(&adapter->fdir_perfect_lock); 4321 } 4322 4323 void ixgbe_down(struct ixgbe_adapter *adapter) 4324 { 4325 struct net_device *netdev = adapter->netdev; 4326 struct ixgbe_hw *hw = &adapter->hw; 4327 u32 rxctrl; 4328 int i; 4329 4330 /* signal that we are down to the interrupt handler */ 4331 set_bit(__IXGBE_DOWN, &adapter->state); 4332 4333 /* disable receives */ 4334 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4335 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 4336 4337 /* disable all enabled rx queues */ 4338 for (i = 0; i < adapter->num_rx_queues; i++) 4339 /* this call also flushes the previous write */ 4340 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]); 4341 4342 usleep_range(10000, 20000); 4343 4344 netif_tx_stop_all_queues(netdev); 4345 4346 /* call carrier off first to avoid false dev_watchdog timeouts */ 4347 netif_carrier_off(netdev); 4348 netif_tx_disable(netdev); 4349 4350 ixgbe_irq_disable(adapter); 4351 4352 ixgbe_napi_disable_all(adapter); 4353 4354 adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT | 4355 IXGBE_FLAG2_RESET_REQUESTED); 4356 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 4357 4358 del_timer_sync(&adapter->service_timer); 4359 4360 if (adapter->num_vfs) { 4361 /* Clear EITR Select mapping */ 4362 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0); 4363 4364 /* Mark all the VFs as inactive */ 4365 for (i = 0 ; i < adapter->num_vfs; i++) 4366 adapter->vfinfo[i].clear_to_send = false; 4367 4368 /* ping all the active vfs to let them know we are going down */ 4369 ixgbe_ping_all_vfs(adapter); 4370 4371 /* Disable all VFTE/VFRE TX/RX */ 4372 ixgbe_disable_tx_rx(adapter); 4373 } 4374 4375 /* disable transmits in the hardware now that interrupts are off */ 4376 for (i = 0; i < adapter->num_tx_queues; i++) { 4377 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 4378 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 4379 } 4380 4381 /* Disable the Tx DMA engine on 82599 and X540 */ 4382 switch (hw->mac.type) { 4383 case ixgbe_mac_82599EB: 4384 case ixgbe_mac_X540: 4385 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, 4386 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) & 4387 ~IXGBE_DMATXCTL_TE)); 4388 break; 4389 default: 4390 break; 4391 } 4392 4393 if (!pci_channel_offline(adapter->pdev)) 4394 ixgbe_reset(adapter); 4395 4396 /* power down the optics for multispeed fiber and 82599 SFP+ fiber */ 4397 if (hw->mac.ops.disable_tx_laser && 4398 ((hw->phy.multispeed_fiber) || 4399 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && 4400 (hw->mac.type == ixgbe_mac_82599EB)))) 4401 hw->mac.ops.disable_tx_laser(hw); 4402 4403 ixgbe_clean_all_tx_rings(adapter); 4404 ixgbe_clean_all_rx_rings(adapter); 4405 4406 #ifdef CONFIG_IXGBE_DCA 4407 /* since we reset the hardware DCA settings were cleared */ 4408 ixgbe_setup_dca(adapter); 4409 #endif 4410 } 4411 4412 /** 4413 * ixgbe_tx_timeout - Respond to a Tx Hang 4414 * @netdev: network interface device structure 4415 **/ 4416 static void ixgbe_tx_timeout(struct net_device *netdev) 4417 { 4418 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4419 4420 /* Do the reset outside of interrupt context */ 4421 ixgbe_tx_timeout_reset(adapter); 4422 } 4423 4424 /** 4425 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter) 4426 * @adapter: board private structure to initialize 4427 * 4428 * ixgbe_sw_init initializes the Adapter private data structure. 4429 * Fields are initialized based on PCI device information and 4430 * OS network device settings (MTU size). 4431 **/ 4432 static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) 4433 { 4434 struct ixgbe_hw *hw = &adapter->hw; 4435 struct pci_dev *pdev = adapter->pdev; 4436 unsigned int rss; 4437 #ifdef CONFIG_IXGBE_DCB 4438 int j; 4439 struct tc_configuration *tc; 4440 #endif 4441 4442 /* PCI config space info */ 4443 4444 hw->vendor_id = pdev->vendor; 4445 hw->device_id = pdev->device; 4446 hw->revision_id = pdev->revision; 4447 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4448 hw->subsystem_device_id = pdev->subsystem_device; 4449 4450 /* Set capability flags */ 4451 rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus()); 4452 adapter->ring_feature[RING_F_RSS].limit = rss; 4453 switch (hw->mac.type) { 4454 case ixgbe_mac_82598EB: 4455 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4456 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 4457 adapter->max_q_vectors = MAX_Q_VECTORS_82598; 4458 break; 4459 case ixgbe_mac_X540: 4460 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 4461 case ixgbe_mac_82599EB: 4462 adapter->max_q_vectors = MAX_Q_VECTORS_82599; 4463 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE; 4464 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 4465 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) 4466 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; 4467 /* Flow Director hash filters enabled */ 4468 adapter->atr_sample_rate = 20; 4469 adapter->ring_feature[RING_F_FDIR].limit = 4470 IXGBE_MAX_FDIR_INDICES; 4471 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K; 4472 #ifdef IXGBE_FCOE 4473 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE; 4474 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; 4475 #ifdef CONFIG_IXGBE_DCB 4476 /* Default traffic class to use for FCoE */ 4477 adapter->fcoe.up = IXGBE_FCOE_DEFTC; 4478 #endif 4479 #endif /* IXGBE_FCOE */ 4480 break; 4481 default: 4482 break; 4483 } 4484 4485 #ifdef IXGBE_FCOE 4486 /* FCoE support exists, always init the FCoE lock */ 4487 spin_lock_init(&adapter->fcoe.lock); 4488 4489 #endif 4490 /* n-tuple support exists, always init our spinlock */ 4491 spin_lock_init(&adapter->fdir_perfect_lock); 4492 4493 #ifdef CONFIG_IXGBE_DCB 4494 switch (hw->mac.type) { 4495 case ixgbe_mac_X540: 4496 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS; 4497 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS; 4498 break; 4499 default: 4500 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS; 4501 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS; 4502 break; 4503 } 4504 4505 /* Configure DCB traffic classes */ 4506 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) { 4507 tc = &adapter->dcb_cfg.tc_config[j]; 4508 tc->path[DCB_TX_CONFIG].bwg_id = 0; 4509 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1); 4510 tc->path[DCB_RX_CONFIG].bwg_id = 0; 4511 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1); 4512 tc->dcb_pfc = pfc_disabled; 4513 } 4514 4515 /* Initialize default user to priority mapping, UPx->TC0 */ 4516 tc = &adapter->dcb_cfg.tc_config[0]; 4517 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF; 4518 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF; 4519 4520 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100; 4521 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100; 4522 adapter->dcb_cfg.pfc_mode_enable = false; 4523 adapter->dcb_set_bitmap = 0x00; 4524 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; 4525 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, 4526 sizeof(adapter->temp_dcb_cfg)); 4527 4528 #endif 4529 4530 /* default flow control settings */ 4531 hw->fc.requested_mode = ixgbe_fc_full; 4532 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */ 4533 ixgbe_pbthresh_setup(adapter); 4534 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE; 4535 hw->fc.send_xon = true; 4536 hw->fc.disable_fc_autoneg = false; 4537 4538 #ifdef CONFIG_PCI_IOV 4539 /* assign number of SR-IOV VFs */ 4540 if (hw->mac.type != ixgbe_mac_82598EB) 4541 adapter->num_vfs = (max_vfs > 63) ? 0 : max_vfs; 4542 4543 #endif 4544 /* enable itr by default in dynamic mode */ 4545 adapter->rx_itr_setting = 1; 4546 adapter->tx_itr_setting = 1; 4547 4548 /* set default ring sizes */ 4549 adapter->tx_ring_count = IXGBE_DEFAULT_TXD; 4550 adapter->rx_ring_count = IXGBE_DEFAULT_RXD; 4551 4552 /* set default work limits */ 4553 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK; 4554 4555 /* initialize eeprom parameters */ 4556 if (ixgbe_init_eeprom_params_generic(hw)) { 4557 e_dev_err("EEPROM initialization failed\n"); 4558 return -EIO; 4559 } 4560 4561 set_bit(__IXGBE_DOWN, &adapter->state); 4562 4563 return 0; 4564 } 4565 4566 /** 4567 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors) 4568 * @tx_ring: tx descriptor ring (for a specific queue) to setup 4569 * 4570 * Return 0 on success, negative on failure 4571 **/ 4572 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) 4573 { 4574 struct device *dev = tx_ring->dev; 4575 int orig_node = dev_to_node(dev); 4576 int numa_node = -1; 4577 int size; 4578 4579 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; 4580 4581 if (tx_ring->q_vector) 4582 numa_node = tx_ring->q_vector->numa_node; 4583 4584 tx_ring->tx_buffer_info = vzalloc_node(size, numa_node); 4585 if (!tx_ring->tx_buffer_info) 4586 tx_ring->tx_buffer_info = vzalloc(size); 4587 if (!tx_ring->tx_buffer_info) 4588 goto err; 4589 4590 /* round up to nearest 4K */ 4591 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 4592 tx_ring->size = ALIGN(tx_ring->size, 4096); 4593 4594 set_dev_node(dev, numa_node); 4595 tx_ring->desc = dma_alloc_coherent(dev, 4596 tx_ring->size, 4597 &tx_ring->dma, 4598 GFP_KERNEL); 4599 set_dev_node(dev, orig_node); 4600 if (!tx_ring->desc) 4601 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 4602 &tx_ring->dma, GFP_KERNEL); 4603 if (!tx_ring->desc) 4604 goto err; 4605 4606 tx_ring->next_to_use = 0; 4607 tx_ring->next_to_clean = 0; 4608 return 0; 4609 4610 err: 4611 vfree(tx_ring->tx_buffer_info); 4612 tx_ring->tx_buffer_info = NULL; 4613 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n"); 4614 return -ENOMEM; 4615 } 4616 4617 /** 4618 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources 4619 * @adapter: board private structure 4620 * 4621 * If this function returns with an error, then it's possible one or 4622 * more of the rings is populated (while the rest are not). It is the 4623 * callers duty to clean those orphaned rings. 4624 * 4625 * Return 0 on success, negative on failure 4626 **/ 4627 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) 4628 { 4629 int i, err = 0; 4630 4631 for (i = 0; i < adapter->num_tx_queues; i++) { 4632 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]); 4633 if (!err) 4634 continue; 4635 4636 e_err(probe, "Allocation for Tx Queue %u failed\n", i); 4637 goto err_setup_tx; 4638 } 4639 4640 return 0; 4641 err_setup_tx: 4642 /* rewind the index freeing the rings as we go */ 4643 while (i--) 4644 ixgbe_free_tx_resources(adapter->tx_ring[i]); 4645 return err; 4646 } 4647 4648 /** 4649 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) 4650 * @rx_ring: rx descriptor ring (for a specific queue) to setup 4651 * 4652 * Returns 0 on success, negative on failure 4653 **/ 4654 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) 4655 { 4656 struct device *dev = rx_ring->dev; 4657 int orig_node = dev_to_node(dev); 4658 int numa_node = -1; 4659 int size; 4660 4661 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; 4662 4663 if (rx_ring->q_vector) 4664 numa_node = rx_ring->q_vector->numa_node; 4665 4666 rx_ring->rx_buffer_info = vzalloc_node(size, numa_node); 4667 if (!rx_ring->rx_buffer_info) 4668 rx_ring->rx_buffer_info = vzalloc(size); 4669 if (!rx_ring->rx_buffer_info) 4670 goto err; 4671 4672 /* Round up to nearest 4K */ 4673 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 4674 rx_ring->size = ALIGN(rx_ring->size, 4096); 4675 4676 set_dev_node(dev, numa_node); 4677 rx_ring->desc = dma_alloc_coherent(dev, 4678 rx_ring->size, 4679 &rx_ring->dma, 4680 GFP_KERNEL); 4681 set_dev_node(dev, orig_node); 4682 if (!rx_ring->desc) 4683 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 4684 &rx_ring->dma, GFP_KERNEL); 4685 if (!rx_ring->desc) 4686 goto err; 4687 4688 rx_ring->next_to_clean = 0; 4689 rx_ring->next_to_use = 0; 4690 4691 return 0; 4692 err: 4693 vfree(rx_ring->rx_buffer_info); 4694 rx_ring->rx_buffer_info = NULL; 4695 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); 4696 return -ENOMEM; 4697 } 4698 4699 /** 4700 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources 4701 * @adapter: board private structure 4702 * 4703 * If this function returns with an error, then it's possible one or 4704 * more of the rings is populated (while the rest are not). It is the 4705 * callers duty to clean those orphaned rings. 4706 * 4707 * Return 0 on success, negative on failure 4708 **/ 4709 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter) 4710 { 4711 int i, err = 0; 4712 4713 for (i = 0; i < adapter->num_rx_queues; i++) { 4714 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]); 4715 if (!err) 4716 continue; 4717 4718 e_err(probe, "Allocation for Rx Queue %u failed\n", i); 4719 goto err_setup_rx; 4720 } 4721 4722 #ifdef IXGBE_FCOE 4723 err = ixgbe_setup_fcoe_ddp_resources(adapter); 4724 if (!err) 4725 #endif 4726 return 0; 4727 err_setup_rx: 4728 /* rewind the index freeing the rings as we go */ 4729 while (i--) 4730 ixgbe_free_rx_resources(adapter->rx_ring[i]); 4731 return err; 4732 } 4733 4734 /** 4735 * ixgbe_free_tx_resources - Free Tx Resources per Queue 4736 * @tx_ring: Tx descriptor ring for a specific queue 4737 * 4738 * Free all transmit software resources 4739 **/ 4740 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring) 4741 { 4742 ixgbe_clean_tx_ring(tx_ring); 4743 4744 vfree(tx_ring->tx_buffer_info); 4745 tx_ring->tx_buffer_info = NULL; 4746 4747 /* if not set, then don't free */ 4748 if (!tx_ring->desc) 4749 return; 4750 4751 dma_free_coherent(tx_ring->dev, tx_ring->size, 4752 tx_ring->desc, tx_ring->dma); 4753 4754 tx_ring->desc = NULL; 4755 } 4756 4757 /** 4758 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues 4759 * @adapter: board private structure 4760 * 4761 * Free all transmit software resources 4762 **/ 4763 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) 4764 { 4765 int i; 4766 4767 for (i = 0; i < adapter->num_tx_queues; i++) 4768 if (adapter->tx_ring[i]->desc) 4769 ixgbe_free_tx_resources(adapter->tx_ring[i]); 4770 } 4771 4772 /** 4773 * ixgbe_free_rx_resources - Free Rx Resources 4774 * @rx_ring: ring to clean the resources from 4775 * 4776 * Free all receive software resources 4777 **/ 4778 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) 4779 { 4780 ixgbe_clean_rx_ring(rx_ring); 4781 4782 vfree(rx_ring->rx_buffer_info); 4783 rx_ring->rx_buffer_info = NULL; 4784 4785 /* if not set, then don't free */ 4786 if (!rx_ring->desc) 4787 return; 4788 4789 dma_free_coherent(rx_ring->dev, rx_ring->size, 4790 rx_ring->desc, rx_ring->dma); 4791 4792 rx_ring->desc = NULL; 4793 } 4794 4795 /** 4796 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues 4797 * @adapter: board private structure 4798 * 4799 * Free all receive software resources 4800 **/ 4801 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) 4802 { 4803 int i; 4804 4805 #ifdef IXGBE_FCOE 4806 ixgbe_free_fcoe_ddp_resources(adapter); 4807 4808 #endif 4809 for (i = 0; i < adapter->num_rx_queues; i++) 4810 if (adapter->rx_ring[i]->desc) 4811 ixgbe_free_rx_resources(adapter->rx_ring[i]); 4812 } 4813 4814 /** 4815 * ixgbe_change_mtu - Change the Maximum Transfer Unit 4816 * @netdev: network interface device structure 4817 * @new_mtu: new value for maximum frame size 4818 * 4819 * Returns 0 on success, negative on failure 4820 **/ 4821 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) 4822 { 4823 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4824 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 4825 4826 /* MTU < 68 is an error and causes problems on some kernels */ 4827 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE)) 4828 return -EINVAL; 4829 4830 /* 4831 * For 82599EB we cannot allow PF to change MTU greater than 1500 4832 * in SR-IOV mode as it may cause buffer overruns in guest VFs that 4833 * don't allocate and chain buffers correctly. 4834 */ 4835 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && 4836 (adapter->hw.mac.type == ixgbe_mac_82599EB) && 4837 (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE)) 4838 return -EINVAL; 4839 4840 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 4841 4842 /* must set new MTU before calling down or up */ 4843 netdev->mtu = new_mtu; 4844 4845 if (netif_running(netdev)) 4846 ixgbe_reinit_locked(adapter); 4847 4848 return 0; 4849 } 4850 4851 /** 4852 * ixgbe_open - Called when a network interface is made active 4853 * @netdev: network interface device structure 4854 * 4855 * Returns 0 on success, negative value on failure 4856 * 4857 * The open entry point is called when a network interface is made 4858 * active by the system (IFF_UP). At this point all resources needed 4859 * for transmit and receive operations are allocated, the interrupt 4860 * handler is registered with the OS, the watchdog timer is started, 4861 * and the stack is notified that the interface is ready. 4862 **/ 4863 static int ixgbe_open(struct net_device *netdev) 4864 { 4865 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4866 int err; 4867 4868 /* disallow open during test */ 4869 if (test_bit(__IXGBE_TESTING, &adapter->state)) 4870 return -EBUSY; 4871 4872 netif_carrier_off(netdev); 4873 4874 /* allocate transmit descriptors */ 4875 err = ixgbe_setup_all_tx_resources(adapter); 4876 if (err) 4877 goto err_setup_tx; 4878 4879 /* allocate receive descriptors */ 4880 err = ixgbe_setup_all_rx_resources(adapter); 4881 if (err) 4882 goto err_setup_rx; 4883 4884 ixgbe_configure(adapter); 4885 4886 err = ixgbe_request_irq(adapter); 4887 if (err) 4888 goto err_req_irq; 4889 4890 /* Notify the stack of the actual queue counts. */ 4891 err = netif_set_real_num_tx_queues(netdev, 4892 adapter->num_rx_pools > 1 ? 1 : 4893 adapter->num_tx_queues); 4894 if (err) 4895 goto err_set_queues; 4896 4897 4898 err = netif_set_real_num_rx_queues(netdev, 4899 adapter->num_rx_pools > 1 ? 1 : 4900 adapter->num_rx_queues); 4901 if (err) 4902 goto err_set_queues; 4903 4904 ixgbe_up_complete(adapter); 4905 4906 return 0; 4907 4908 err_set_queues: 4909 ixgbe_free_irq(adapter); 4910 err_req_irq: 4911 ixgbe_free_all_rx_resources(adapter); 4912 err_setup_rx: 4913 ixgbe_free_all_tx_resources(adapter); 4914 err_setup_tx: 4915 ixgbe_reset(adapter); 4916 4917 return err; 4918 } 4919 4920 /** 4921 * ixgbe_close - Disables a network interface 4922 * @netdev: network interface device structure 4923 * 4924 * Returns 0, this is not allowed to fail 4925 * 4926 * The close entry point is called when an interface is de-activated 4927 * by the OS. The hardware is still under the drivers control, but 4928 * needs to be disabled. A global MAC reset is issued to stop the 4929 * hardware, and all transmit and receive resources are freed. 4930 **/ 4931 static int ixgbe_close(struct net_device *netdev) 4932 { 4933 struct ixgbe_adapter *adapter = netdev_priv(netdev); 4934 4935 ixgbe_down(adapter); 4936 ixgbe_free_irq(adapter); 4937 4938 ixgbe_fdir_filter_exit(adapter); 4939 4940 ixgbe_free_all_tx_resources(adapter); 4941 ixgbe_free_all_rx_resources(adapter); 4942 4943 ixgbe_release_hw_control(adapter); 4944 4945 return 0; 4946 } 4947 4948 #ifdef CONFIG_PM 4949 static int ixgbe_resume(struct pci_dev *pdev) 4950 { 4951 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 4952 struct net_device *netdev = adapter->netdev; 4953 u32 err; 4954 4955 pci_set_power_state(pdev, PCI_D0); 4956 pci_restore_state(pdev); 4957 /* 4958 * pci_restore_state clears dev->state_saved so call 4959 * pci_save_state to restore it. 4960 */ 4961 pci_save_state(pdev); 4962 4963 err = pci_enable_device_mem(pdev); 4964 if (err) { 4965 e_dev_err("Cannot enable PCI device from suspend\n"); 4966 return err; 4967 } 4968 pci_set_master(pdev); 4969 4970 pci_wake_from_d3(pdev, false); 4971 4972 ixgbe_reset(adapter); 4973 4974 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 4975 4976 rtnl_lock(); 4977 err = ixgbe_init_interrupt_scheme(adapter); 4978 if (!err && netif_running(netdev)) 4979 err = ixgbe_open(netdev); 4980 4981 rtnl_unlock(); 4982 4983 if (err) 4984 return err; 4985 4986 netif_device_attach(netdev); 4987 4988 return 0; 4989 } 4990 #endif /* CONFIG_PM */ 4991 4992 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake) 4993 { 4994 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 4995 struct net_device *netdev = adapter->netdev; 4996 struct ixgbe_hw *hw = &adapter->hw; 4997 u32 ctrl, fctrl; 4998 u32 wufc = adapter->wol; 4999 #ifdef CONFIG_PM 5000 int retval = 0; 5001 #endif 5002 5003 netif_device_detach(netdev); 5004 5005 if (netif_running(netdev)) { 5006 rtnl_lock(); 5007 ixgbe_down(adapter); 5008 ixgbe_free_irq(adapter); 5009 ixgbe_free_all_tx_resources(adapter); 5010 ixgbe_free_all_rx_resources(adapter); 5011 rtnl_unlock(); 5012 } 5013 5014 ixgbe_clear_interrupt_scheme(adapter); 5015 5016 #ifdef CONFIG_PM 5017 retval = pci_save_state(pdev); 5018 if (retval) 5019 return retval; 5020 5021 #endif 5022 if (wufc) { 5023 ixgbe_set_rx_mode(netdev); 5024 5025 /* 5026 * enable the optics for both mult-speed fiber and 5027 * 82599 SFP+ fiber as we can WoL. 5028 */ 5029 if (hw->mac.ops.enable_tx_laser && 5030 (hw->phy.multispeed_fiber || 5031 (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber && 5032 hw->mac.type == ixgbe_mac_82599EB))) 5033 hw->mac.ops.enable_tx_laser(hw); 5034 5035 /* turn on all-multi mode if wake on multicast is enabled */ 5036 if (wufc & IXGBE_WUFC_MC) { 5037 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5038 fctrl |= IXGBE_FCTRL_MPE; 5039 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 5040 } 5041 5042 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL); 5043 ctrl |= IXGBE_CTRL_GIO_DIS; 5044 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl); 5045 5046 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc); 5047 } else { 5048 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 5049 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 5050 } 5051 5052 switch (hw->mac.type) { 5053 case ixgbe_mac_82598EB: 5054 pci_wake_from_d3(pdev, false); 5055 break; 5056 case ixgbe_mac_82599EB: 5057 case ixgbe_mac_X540: 5058 pci_wake_from_d3(pdev, !!wufc); 5059 break; 5060 default: 5061 break; 5062 } 5063 5064 *enable_wake = !!wufc; 5065 5066 ixgbe_release_hw_control(adapter); 5067 5068 pci_disable_device(pdev); 5069 5070 return 0; 5071 } 5072 5073 #ifdef CONFIG_PM 5074 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state) 5075 { 5076 int retval; 5077 bool wake; 5078 5079 retval = __ixgbe_shutdown(pdev, &wake); 5080 if (retval) 5081 return retval; 5082 5083 if (wake) { 5084 pci_prepare_to_sleep(pdev); 5085 } else { 5086 pci_wake_from_d3(pdev, false); 5087 pci_set_power_state(pdev, PCI_D3hot); 5088 } 5089 5090 return 0; 5091 } 5092 #endif /* CONFIG_PM */ 5093 5094 static void ixgbe_shutdown(struct pci_dev *pdev) 5095 { 5096 bool wake; 5097 5098 __ixgbe_shutdown(pdev, &wake); 5099 5100 if (system_state == SYSTEM_POWER_OFF) { 5101 pci_wake_from_d3(pdev, wake); 5102 pci_set_power_state(pdev, PCI_D3hot); 5103 } 5104 } 5105 5106 /** 5107 * ixgbe_update_stats - Update the board statistics counters. 5108 * @adapter: board private structure 5109 **/ 5110 void ixgbe_update_stats(struct ixgbe_adapter *adapter) 5111 { 5112 struct net_device *netdev = adapter->netdev; 5113 struct ixgbe_hw *hw = &adapter->hw; 5114 struct ixgbe_hw_stats *hwstats = &adapter->stats; 5115 u64 total_mpc = 0; 5116 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot; 5117 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0; 5118 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0; 5119 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; 5120 5121 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5122 test_bit(__IXGBE_RESETTING, &adapter->state)) 5123 return; 5124 5125 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { 5126 u64 rsc_count = 0; 5127 u64 rsc_flush = 0; 5128 for (i = 0; i < adapter->num_rx_queues; i++) { 5129 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count; 5130 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush; 5131 } 5132 adapter->rsc_total_count = rsc_count; 5133 adapter->rsc_total_flush = rsc_flush; 5134 } 5135 5136 for (i = 0; i < adapter->num_rx_queues; i++) { 5137 struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; 5138 non_eop_descs += rx_ring->rx_stats.non_eop_descs; 5139 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed; 5140 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed; 5141 hw_csum_rx_error += rx_ring->rx_stats.csum_err; 5142 bytes += rx_ring->stats.bytes; 5143 packets += rx_ring->stats.packets; 5144 } 5145 adapter->non_eop_descs = non_eop_descs; 5146 adapter->alloc_rx_page_failed = alloc_rx_page_failed; 5147 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed; 5148 adapter->hw_csum_rx_error = hw_csum_rx_error; 5149 netdev->stats.rx_bytes = bytes; 5150 netdev->stats.rx_packets = packets; 5151 5152 bytes = 0; 5153 packets = 0; 5154 /* gather some stats to the adapter struct that are per queue */ 5155 for (i = 0; i < adapter->num_tx_queues; i++) { 5156 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 5157 restart_queue += tx_ring->tx_stats.restart_queue; 5158 tx_busy += tx_ring->tx_stats.tx_busy; 5159 bytes += tx_ring->stats.bytes; 5160 packets += tx_ring->stats.packets; 5161 } 5162 adapter->restart_queue = restart_queue; 5163 adapter->tx_busy = tx_busy; 5164 netdev->stats.tx_bytes = bytes; 5165 netdev->stats.tx_packets = packets; 5166 5167 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5168 5169 /* 8 register reads */ 5170 for (i = 0; i < 8; i++) { 5171 /* for packet buffers not used, the register should read 0 */ 5172 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5173 missed_rx += mpc; 5174 hwstats->mpc[i] += mpc; 5175 total_mpc += hwstats->mpc[i]; 5176 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5177 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5178 switch (hw->mac.type) { 5179 case ixgbe_mac_82598EB: 5180 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5181 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5182 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5183 hwstats->pxonrxc[i] += 5184 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5185 break; 5186 case ixgbe_mac_82599EB: 5187 case ixgbe_mac_X540: 5188 hwstats->pxonrxc[i] += 5189 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i)); 5190 break; 5191 default: 5192 break; 5193 } 5194 } 5195 5196 /*16 register reads */ 5197 for (i = 0; i < 16; i++) { 5198 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5199 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5200 if ((hw->mac.type == ixgbe_mac_82599EB) || 5201 (hw->mac.type == ixgbe_mac_X540)) { 5202 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); 5203 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ 5204 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); 5205 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */ 5206 } 5207 } 5208 5209 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); 5210 /* work around hardware counting issue */ 5211 hwstats->gprc -= missed_rx; 5212 5213 ixgbe_update_xoff_received(adapter); 5214 5215 /* 82598 hardware only has a 32 bit counter in the high register */ 5216 switch (hw->mac.type) { 5217 case ixgbe_mac_82598EB: 5218 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5219 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); 5220 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH); 5221 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH); 5222 break; 5223 case ixgbe_mac_X540: 5224 /* OS2BMC stats are X540 only*/ 5225 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC); 5226 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC); 5227 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC); 5228 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC); 5229 case ixgbe_mac_82599EB: 5230 for (i = 0; i < 16; i++) 5231 adapter->hw_rx_no_dma_resources += 5232 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5233 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); 5234 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ 5235 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); 5236 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */ 5237 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL); 5238 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */ 5239 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5240 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH); 5241 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS); 5242 #ifdef IXGBE_FCOE 5243 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5244 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5245 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5246 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5247 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5248 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5249 /* Add up per cpu counters for total ddp aloc fail */ 5250 if (adapter->fcoe.ddp_pool) { 5251 struct ixgbe_fcoe *fcoe = &adapter->fcoe; 5252 struct ixgbe_fcoe_ddp_pool *ddp_pool; 5253 unsigned int cpu; 5254 u64 noddp = 0, noddp_ext_buff = 0; 5255 for_each_possible_cpu(cpu) { 5256 ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu); 5257 noddp += ddp_pool->noddp; 5258 noddp_ext_buff += ddp_pool->noddp_ext_buff; 5259 } 5260 hwstats->fcoe_noddp = noddp; 5261 hwstats->fcoe_noddp_ext_buff = noddp_ext_buff; 5262 } 5263 #endif /* IXGBE_FCOE */ 5264 break; 5265 default: 5266 break; 5267 } 5268 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5269 hwstats->bprc += bprc; 5270 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); 5271 if (hw->mac.type == ixgbe_mac_82598EB) 5272 hwstats->mprc -= bprc; 5273 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC); 5274 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 5275 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 5276 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 5277 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 5278 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5279 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5280 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 5281 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5282 hwstats->lxontxc += lxon; 5283 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5284 hwstats->lxofftxc += lxoff; 5285 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 5286 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); 5287 /* 5288 * 82598 errata - tx of flow control packets is included in tx counters 5289 */ 5290 xon_off_tot = lxon + lxoff; 5291 hwstats->gptc -= xon_off_tot; 5292 hwstats->mptc -= xon_off_tot; 5293 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN)); 5294 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC); 5295 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC); 5296 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC); 5297 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR); 5298 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); 5299 hwstats->ptc64 -= xon_off_tot; 5300 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); 5301 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); 5302 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); 5303 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5304 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5305 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); 5306 5307 /* Fill out the OS statistics structure */ 5308 netdev->stats.multicast = hwstats->mprc; 5309 5310 /* Rx Errors */ 5311 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec; 5312 netdev->stats.rx_dropped = 0; 5313 netdev->stats.rx_length_errors = hwstats->rlec; 5314 netdev->stats.rx_crc_errors = hwstats->crcerrs; 5315 netdev->stats.rx_missed_errors = total_mpc; 5316 } 5317 5318 /** 5319 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table 5320 * @adapter: pointer to the device adapter structure 5321 **/ 5322 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) 5323 { 5324 struct ixgbe_hw *hw = &adapter->hw; 5325 int i; 5326 5327 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) 5328 return; 5329 5330 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT; 5331 5332 /* if interface is down do nothing */ 5333 if (test_bit(__IXGBE_DOWN, &adapter->state)) 5334 return; 5335 5336 /* do nothing if we are not using signature filters */ 5337 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) 5338 return; 5339 5340 adapter->fdir_overflow++; 5341 5342 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) { 5343 for (i = 0; i < adapter->num_tx_queues; i++) 5344 set_bit(__IXGBE_TX_FDIR_INIT_DONE, 5345 &(adapter->tx_ring[i]->state)); 5346 /* re-enable flow director interrupts */ 5347 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); 5348 } else { 5349 e_err(probe, "failed to finish FDIR re-initialization, " 5350 "ignored adding FDIR ATR filters\n"); 5351 } 5352 } 5353 5354 /** 5355 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts 5356 * @adapter: pointer to the device adapter structure 5357 * 5358 * This function serves two purposes. First it strobes the interrupt lines 5359 * in order to make certain interrupts are occurring. Secondly it sets the 5360 * bits needed to check for TX hangs. As a result we should immediately 5361 * determine if a hang has occurred. 5362 */ 5363 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) 5364 { 5365 struct ixgbe_hw *hw = &adapter->hw; 5366 u64 eics = 0; 5367 int i; 5368 5369 /* If we're down or resetting, just bail */ 5370 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5371 test_bit(__IXGBE_RESETTING, &adapter->state)) 5372 return; 5373 5374 /* Force detection of hung controller */ 5375 if (netif_carrier_ok(adapter->netdev)) { 5376 for (i = 0; i < adapter->num_tx_queues; i++) 5377 set_check_for_tx_hang(adapter->tx_ring[i]); 5378 } 5379 5380 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) { 5381 /* 5382 * for legacy and MSI interrupts don't set any bits 5383 * that are enabled for EIAM, because this operation 5384 * would set *both* EIMS and EICS for any bit in EIAM 5385 */ 5386 IXGBE_WRITE_REG(hw, IXGBE_EICS, 5387 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER)); 5388 } else { 5389 /* get one bit for every active tx/rx interrupt vector */ 5390 for (i = 0; i < adapter->num_q_vectors; i++) { 5391 struct ixgbe_q_vector *qv = adapter->q_vector[i]; 5392 if (qv->rx.ring || qv->tx.ring) 5393 eics |= ((u64)1 << i); 5394 } 5395 } 5396 5397 /* Cause software interrupt to ensure rings are cleaned */ 5398 ixgbe_irq_rearm_queues(adapter, eics); 5399 5400 } 5401 5402 /** 5403 * ixgbe_watchdog_update_link - update the link status 5404 * @adapter: pointer to the device adapter structure 5405 * @link_speed: pointer to a u32 to store the link_speed 5406 **/ 5407 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) 5408 { 5409 struct ixgbe_hw *hw = &adapter->hw; 5410 u32 link_speed = adapter->link_speed; 5411 bool link_up = adapter->link_up; 5412 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable; 5413 5414 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 5415 return; 5416 5417 if (hw->mac.ops.check_link) { 5418 hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 5419 } else { 5420 /* always assume link is up, if no check link function */ 5421 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 5422 link_up = true; 5423 } 5424 5425 if (adapter->ixgbe_ieee_pfc) 5426 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en); 5427 5428 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) { 5429 hw->mac.ops.fc_enable(hw); 5430 ixgbe_set_rx_drop_en(adapter); 5431 } 5432 5433 if (link_up || 5434 time_after(jiffies, (adapter->link_check_timeout + 5435 IXGBE_TRY_LINK_TIMEOUT))) { 5436 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; 5437 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); 5438 IXGBE_WRITE_FLUSH(hw); 5439 } 5440 5441 adapter->link_up = link_up; 5442 adapter->link_speed = link_speed; 5443 } 5444 5445 /** 5446 * ixgbe_watchdog_link_is_up - update netif_carrier status and 5447 * print link up message 5448 * @adapter: pointer to the device adapter structure 5449 **/ 5450 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) 5451 { 5452 struct net_device *netdev = adapter->netdev; 5453 struct ixgbe_hw *hw = &adapter->hw; 5454 u32 link_speed = adapter->link_speed; 5455 bool flow_rx, flow_tx; 5456 5457 /* only continue if link was previously down */ 5458 if (netif_carrier_ok(netdev)) 5459 return; 5460 5461 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 5462 5463 switch (hw->mac.type) { 5464 case ixgbe_mac_82598EB: { 5465 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5466 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS); 5467 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE); 5468 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X); 5469 } 5470 break; 5471 case ixgbe_mac_X540: 5472 case ixgbe_mac_82599EB: { 5473 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN); 5474 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG); 5475 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE); 5476 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X); 5477 } 5478 break; 5479 default: 5480 flow_tx = false; 5481 flow_rx = false; 5482 break; 5483 } 5484 5485 #ifdef CONFIG_IXGBE_PTP 5486 ixgbe_ptp_start_cyclecounter(adapter); 5487 #endif 5488 5489 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", 5490 (link_speed == IXGBE_LINK_SPEED_10GB_FULL ? 5491 "10 Gbps" : 5492 (link_speed == IXGBE_LINK_SPEED_1GB_FULL ? 5493 "1 Gbps" : 5494 (link_speed == IXGBE_LINK_SPEED_100_FULL ? 5495 "100 Mbps" : 5496 "unknown speed"))), 5497 ((flow_rx && flow_tx) ? "RX/TX" : 5498 (flow_rx ? "RX" : 5499 (flow_tx ? "TX" : "None")))); 5500 5501 netif_carrier_on(netdev); 5502 ixgbe_check_vf_rate_limit(adapter); 5503 5504 /* ping all the active vfs to let them know link has changed */ 5505 ixgbe_ping_all_vfs(adapter); 5506 } 5507 5508 /** 5509 * ixgbe_watchdog_link_is_down - update netif_carrier status and 5510 * print link down message 5511 * @adapter: pointer to the adapter structure 5512 **/ 5513 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter) 5514 { 5515 struct net_device *netdev = adapter->netdev; 5516 struct ixgbe_hw *hw = &adapter->hw; 5517 5518 adapter->link_up = false; 5519 adapter->link_speed = 0; 5520 5521 /* only continue if link was up previously */ 5522 if (!netif_carrier_ok(netdev)) 5523 return; 5524 5525 /* poll for SFP+ cable when link is down */ 5526 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB) 5527 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP; 5528 5529 #ifdef CONFIG_IXGBE_PTP 5530 ixgbe_ptp_start_cyclecounter(adapter); 5531 #endif 5532 5533 e_info(drv, "NIC Link is Down\n"); 5534 netif_carrier_off(netdev); 5535 5536 /* ping all the active vfs to let them know link has changed */ 5537 ixgbe_ping_all_vfs(adapter); 5538 } 5539 5540 /** 5541 * ixgbe_watchdog_flush_tx - flush queues on link down 5542 * @adapter: pointer to the device adapter structure 5543 **/ 5544 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) 5545 { 5546 int i; 5547 int some_tx_pending = 0; 5548 5549 if (!netif_carrier_ok(adapter->netdev)) { 5550 for (i = 0; i < adapter->num_tx_queues; i++) { 5551 struct ixgbe_ring *tx_ring = adapter->tx_ring[i]; 5552 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 5553 some_tx_pending = 1; 5554 break; 5555 } 5556 } 5557 5558 if (some_tx_pending) { 5559 /* We've lost link, so the controller stops DMA, 5560 * but we've got queued Tx work that's never going 5561 * to get done, so reset controller to flush Tx. 5562 * (Do the reset outside of interrupt context). 5563 */ 5564 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED; 5565 } 5566 } 5567 } 5568 5569 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) 5570 { 5571 u32 ssvpc; 5572 5573 /* Do not perform spoof check for 82598 or if not in IOV mode */ 5574 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 5575 adapter->num_vfs == 0) 5576 return; 5577 5578 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); 5579 5580 /* 5581 * ssvpc register is cleared on read, if zero then no 5582 * spoofed packets in the last interval. 5583 */ 5584 if (!ssvpc) 5585 return; 5586 5587 e_warn(drv, "%u Spoofed packets detected\n", ssvpc); 5588 } 5589 5590 /** 5591 * ixgbe_watchdog_subtask - check and bring link up 5592 * @adapter: pointer to the device adapter structure 5593 **/ 5594 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) 5595 { 5596 /* if interface is down do nothing */ 5597 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5598 test_bit(__IXGBE_RESETTING, &adapter->state)) 5599 return; 5600 5601 ixgbe_watchdog_update_link(adapter); 5602 5603 if (adapter->link_up) 5604 ixgbe_watchdog_link_is_up(adapter); 5605 else 5606 ixgbe_watchdog_link_is_down(adapter); 5607 5608 ixgbe_spoof_check(adapter); 5609 ixgbe_update_stats(adapter); 5610 5611 ixgbe_watchdog_flush_tx(adapter); 5612 } 5613 5614 /** 5615 * ixgbe_sfp_detection_subtask - poll for SFP+ cable 5616 * @adapter: the ixgbe adapter structure 5617 **/ 5618 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) 5619 { 5620 struct ixgbe_hw *hw = &adapter->hw; 5621 s32 err; 5622 5623 /* not searching for SFP so there is nothing to do here */ 5624 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && 5625 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 5626 return; 5627 5628 /* someone else is in init, wait until next service event */ 5629 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5630 return; 5631 5632 err = hw->phy.ops.identify_sfp(hw); 5633 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 5634 goto sfp_out; 5635 5636 if (err == IXGBE_ERR_SFP_NOT_PRESENT) { 5637 /* If no cable is present, then we need to reset 5638 * the next time we find a good cable. */ 5639 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET; 5640 } 5641 5642 /* exit on error */ 5643 if (err) 5644 goto sfp_out; 5645 5646 /* exit if reset not needed */ 5647 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) 5648 goto sfp_out; 5649 5650 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET; 5651 5652 /* 5653 * A module may be identified correctly, but the EEPROM may not have 5654 * support for that module. setup_sfp() will fail in that case, so 5655 * we should not allow that module to load. 5656 */ 5657 if (hw->mac.type == ixgbe_mac_82598EB) 5658 err = hw->phy.ops.reset(hw); 5659 else 5660 err = hw->mac.ops.setup_sfp(hw); 5661 5662 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) 5663 goto sfp_out; 5664 5665 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG; 5666 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type); 5667 5668 sfp_out: 5669 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 5670 5671 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) && 5672 (adapter->netdev->reg_state == NETREG_REGISTERED)) { 5673 e_dev_err("failed to initialize because an unsupported " 5674 "SFP+ module type was detected.\n"); 5675 e_dev_err("Reload the driver after installing a " 5676 "supported module.\n"); 5677 unregister_netdev(adapter->netdev); 5678 } 5679 } 5680 5681 /** 5682 * ixgbe_sfp_link_config_subtask - set up link SFP after module install 5683 * @adapter: the ixgbe adapter structure 5684 **/ 5685 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) 5686 { 5687 struct ixgbe_hw *hw = &adapter->hw; 5688 u32 autoneg; 5689 bool negotiation; 5690 5691 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG)) 5692 return; 5693 5694 /* someone else is in init, wait until next service event */ 5695 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state)) 5696 return; 5697 5698 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; 5699 5700 autoneg = hw->phy.autoneg_advertised; 5701 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5702 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation); 5703 if (hw->mac.ops.setup_link) 5704 hw->mac.ops.setup_link(hw, autoneg, negotiation, true); 5705 5706 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 5707 adapter->link_check_timeout = jiffies; 5708 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); 5709 } 5710 5711 #ifdef CONFIG_PCI_IOV 5712 static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) 5713 { 5714 int vf; 5715 struct ixgbe_hw *hw = &adapter->hw; 5716 struct net_device *netdev = adapter->netdev; 5717 u32 gpc; 5718 u32 ciaa, ciad; 5719 5720 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC); 5721 if (gpc) /* If incrementing then no need for the check below */ 5722 return; 5723 /* 5724 * Check to see if a bad DMA write target from an errant or 5725 * malicious VF has caused a PCIe error. If so then we can 5726 * issue a VFLR to the offending VF(s) and then resume without 5727 * requesting a full slot reset. 5728 */ 5729 5730 for (vf = 0; vf < adapter->num_vfs; vf++) { 5731 ciaa = (vf << 16) | 0x80000000; 5732 /* 32 bit read so align, we really want status at offset 6 */ 5733 ciaa |= PCI_COMMAND; 5734 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5735 ciad = IXGBE_READ_REG(hw, IXGBE_CIAD_82599); 5736 ciaa &= 0x7FFFFFFF; 5737 /* disable debug mode asap after reading data */ 5738 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5739 /* Get the upper 16 bits which will be the PCI status reg */ 5740 ciad >>= 16; 5741 if (ciad & PCI_STATUS_REC_MASTER_ABORT) { 5742 netdev_err(netdev, "VF %d Hung DMA\n", vf); 5743 /* Issue VFLR */ 5744 ciaa = (vf << 16) | 0x80000000; 5745 ciaa |= 0xA8; 5746 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5747 ciad = 0x00008000; /* VFLR */ 5748 IXGBE_WRITE_REG(hw, IXGBE_CIAD_82599, ciad); 5749 ciaa &= 0x7FFFFFFF; 5750 IXGBE_WRITE_REG(hw, IXGBE_CIAA_82599, ciaa); 5751 } 5752 } 5753 } 5754 5755 #endif 5756 /** 5757 * ixgbe_service_timer - Timer Call-back 5758 * @data: pointer to adapter cast into an unsigned long 5759 **/ 5760 static void ixgbe_service_timer(unsigned long data) 5761 { 5762 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data; 5763 unsigned long next_event_offset; 5764 bool ready = true; 5765 5766 /* poll faster when waiting for link */ 5767 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) 5768 next_event_offset = HZ / 10; 5769 else 5770 next_event_offset = HZ * 2; 5771 5772 #ifdef CONFIG_PCI_IOV 5773 /* 5774 * don't bother with SR-IOV VF DMA hang check if there are 5775 * no VFs or the link is down 5776 */ 5777 if (!adapter->num_vfs || 5778 (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) 5779 goto normal_timer_service; 5780 5781 /* If we have VFs allocated then we must check for DMA hangs */ 5782 ixgbe_check_for_bad_vf(adapter); 5783 next_event_offset = HZ / 50; 5784 adapter->timer_event_accumulator++; 5785 5786 if (adapter->timer_event_accumulator >= 100) 5787 adapter->timer_event_accumulator = 0; 5788 else 5789 ready = false; 5790 5791 normal_timer_service: 5792 #endif 5793 /* Reset the timer */ 5794 mod_timer(&adapter->service_timer, next_event_offset + jiffies); 5795 5796 if (ready) 5797 ixgbe_service_event_schedule(adapter); 5798 } 5799 5800 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) 5801 { 5802 if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED)) 5803 return; 5804 5805 adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED; 5806 5807 /* If we're already down or resetting, just bail */ 5808 if (test_bit(__IXGBE_DOWN, &adapter->state) || 5809 test_bit(__IXGBE_RESETTING, &adapter->state)) 5810 return; 5811 5812 ixgbe_dump(adapter); 5813 netdev_err(adapter->netdev, "Reset adapter\n"); 5814 adapter->tx_timeout_count++; 5815 5816 ixgbe_reinit_locked(adapter); 5817 } 5818 5819 /** 5820 * ixgbe_service_task - manages and runs subtasks 5821 * @work: pointer to work_struct containing our data 5822 **/ 5823 static void ixgbe_service_task(struct work_struct *work) 5824 { 5825 struct ixgbe_adapter *adapter = container_of(work, 5826 struct ixgbe_adapter, 5827 service_task); 5828 5829 ixgbe_reset_subtask(adapter); 5830 ixgbe_sfp_detection_subtask(adapter); 5831 ixgbe_sfp_link_config_subtask(adapter); 5832 ixgbe_check_overtemp_subtask(adapter); 5833 ixgbe_watchdog_subtask(adapter); 5834 ixgbe_fdir_reinit_subtask(adapter); 5835 ixgbe_check_hang_subtask(adapter); 5836 #ifdef CONFIG_IXGBE_PTP 5837 ixgbe_ptp_overflow_check(adapter); 5838 #endif 5839 5840 ixgbe_service_event_complete(adapter); 5841 } 5842 5843 static int ixgbe_tso(struct ixgbe_ring *tx_ring, 5844 struct ixgbe_tx_buffer *first, 5845 u8 *hdr_len) 5846 { 5847 struct sk_buff *skb = first->skb; 5848 u32 vlan_macip_lens, type_tucmd; 5849 u32 mss_l4len_idx, l4len; 5850 5851 if (!skb_is_gso(skb)) 5852 return 0; 5853 5854 if (skb_header_cloned(skb)) { 5855 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 5856 if (err) 5857 return err; 5858 } 5859 5860 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 5861 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 5862 5863 if (first->protocol == __constant_htons(ETH_P_IP)) { 5864 struct iphdr *iph = ip_hdr(skb); 5865 iph->tot_len = 0; 5866 iph->check = 0; 5867 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 5868 iph->daddr, 0, 5869 IPPROTO_TCP, 5870 0); 5871 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 5872 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 5873 IXGBE_TX_FLAGS_CSUM | 5874 IXGBE_TX_FLAGS_IPV4; 5875 } else if (skb_is_gso_v6(skb)) { 5876 ipv6_hdr(skb)->payload_len = 0; 5877 tcp_hdr(skb)->check = 5878 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 5879 &ipv6_hdr(skb)->daddr, 5880 0, IPPROTO_TCP, 0); 5881 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 5882 IXGBE_TX_FLAGS_CSUM; 5883 } 5884 5885 /* compute header lengths */ 5886 l4len = tcp_hdrlen(skb); 5887 *hdr_len = skb_transport_offset(skb) + l4len; 5888 5889 /* update gso size and bytecount with header size */ 5890 first->gso_segs = skb_shinfo(skb)->gso_segs; 5891 first->bytecount += (first->gso_segs - 1) * *hdr_len; 5892 5893 /* mss_l4len_id: use 1 as index for TSO */ 5894 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 5895 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 5896 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 5897 5898 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 5899 vlan_macip_lens = skb_network_header_len(skb); 5900 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 5901 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 5902 5903 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 5904 mss_l4len_idx); 5905 5906 return 1; 5907 } 5908 5909 static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, 5910 struct ixgbe_tx_buffer *first) 5911 { 5912 struct sk_buff *skb = first->skb; 5913 u32 vlan_macip_lens = 0; 5914 u32 mss_l4len_idx = 0; 5915 u32 type_tucmd = 0; 5916 5917 if (skb->ip_summed != CHECKSUM_PARTIAL) { 5918 if (!(first->tx_flags & IXGBE_TX_FLAGS_HW_VLAN)) { 5919 if (unlikely(skb->no_fcs)) 5920 first->tx_flags |= IXGBE_TX_FLAGS_NO_IFCS; 5921 if (!(first->tx_flags & IXGBE_TX_FLAGS_TXSW)) 5922 return; 5923 } 5924 } else { 5925 u8 l4_hdr = 0; 5926 switch (first->protocol) { 5927 case __constant_htons(ETH_P_IP): 5928 vlan_macip_lens |= skb_network_header_len(skb); 5929 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 5930 l4_hdr = ip_hdr(skb)->protocol; 5931 break; 5932 case __constant_htons(ETH_P_IPV6): 5933 vlan_macip_lens |= skb_network_header_len(skb); 5934 l4_hdr = ipv6_hdr(skb)->nexthdr; 5935 break; 5936 default: 5937 if (unlikely(net_ratelimit())) { 5938 dev_warn(tx_ring->dev, 5939 "partial checksum but proto=%x!\n", 5940 first->protocol); 5941 } 5942 break; 5943 } 5944 5945 switch (l4_hdr) { 5946 case IPPROTO_TCP: 5947 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 5948 mss_l4len_idx = tcp_hdrlen(skb) << 5949 IXGBE_ADVTXD_L4LEN_SHIFT; 5950 break; 5951 case IPPROTO_SCTP: 5952 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 5953 mss_l4len_idx = sizeof(struct sctphdr) << 5954 IXGBE_ADVTXD_L4LEN_SHIFT; 5955 break; 5956 case IPPROTO_UDP: 5957 mss_l4len_idx = sizeof(struct udphdr) << 5958 IXGBE_ADVTXD_L4LEN_SHIFT; 5959 break; 5960 default: 5961 if (unlikely(net_ratelimit())) { 5962 dev_warn(tx_ring->dev, 5963 "partial checksum but l4 proto=%x!\n", 5964 l4_hdr); 5965 } 5966 break; 5967 } 5968 5969 /* update TX checksum flag */ 5970 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 5971 } 5972 5973 /* vlan_macip_lens: MACLEN, VLAN tag */ 5974 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 5975 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 5976 5977 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, 5978 type_tucmd, mss_l4len_idx); 5979 } 5980 5981 static __le32 ixgbe_tx_cmd_type(u32 tx_flags) 5982 { 5983 /* set type for advanced descriptor with frame checksum insertion */ 5984 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 5985 IXGBE_ADVTXD_DCMD_DEXT); 5986 5987 /* set HW vlan bit if vlan is present */ 5988 if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN) 5989 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 5990 5991 #ifdef CONFIG_IXGBE_PTP 5992 if (tx_flags & IXGBE_TX_FLAGS_TSTAMP) 5993 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_MAC_TSTAMP); 5994 #endif 5995 5996 /* set segmentation enable bits for TSO/FSO */ 5997 #ifdef IXGBE_FCOE 5998 if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_FSO)) 5999 #else 6000 if (tx_flags & IXGBE_TX_FLAGS_TSO) 6001 #endif 6002 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 6003 6004 /* insert frame checksum */ 6005 if (!(tx_flags & IXGBE_TX_FLAGS_NO_IFCS)) 6006 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_IFCS); 6007 6008 return cmd_type; 6009 } 6010 6011 static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 6012 u32 tx_flags, unsigned int paylen) 6013 { 6014 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 6015 6016 /* enable L4 checksum for TSO and TX checksum offload */ 6017 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 6018 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 6019 6020 /* enble IPv4 checksum for TSO */ 6021 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 6022 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 6023 6024 /* use index 1 context for TSO/FSO/FCOE */ 6025 #ifdef IXGBE_FCOE 6026 if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_FCOE)) 6027 #else 6028 if (tx_flags & IXGBE_TX_FLAGS_TSO) 6029 #endif 6030 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); 6031 6032 /* 6033 * Check Context must be set if Tx switch is enabled, which it 6034 * always is for case where virtual functions are running 6035 */ 6036 #ifdef IXGBE_FCOE 6037 if (tx_flags & (IXGBE_TX_FLAGS_TXSW | IXGBE_TX_FLAGS_FCOE)) 6038 #else 6039 if (tx_flags & IXGBE_TX_FLAGS_TXSW) 6040 #endif 6041 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 6042 6043 tx_desc->read.olinfo_status = olinfo_status; 6044 } 6045 6046 #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ 6047 IXGBE_TXD_CMD_RS) 6048 6049 static void ixgbe_tx_map(struct ixgbe_ring *tx_ring, 6050 struct ixgbe_tx_buffer *first, 6051 const u8 hdr_len) 6052 { 6053 dma_addr_t dma; 6054 struct sk_buff *skb = first->skb; 6055 struct ixgbe_tx_buffer *tx_buffer; 6056 union ixgbe_adv_tx_desc *tx_desc; 6057 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 6058 unsigned int data_len = skb->data_len; 6059 unsigned int size = skb_headlen(skb); 6060 unsigned int paylen = skb->len - hdr_len; 6061 u32 tx_flags = first->tx_flags; 6062 __le32 cmd_type; 6063 u16 i = tx_ring->next_to_use; 6064 6065 tx_desc = IXGBE_TX_DESC(tx_ring, i); 6066 6067 ixgbe_tx_olinfo_status(tx_desc, tx_flags, paylen); 6068 cmd_type = ixgbe_tx_cmd_type(tx_flags); 6069 6070 #ifdef IXGBE_FCOE 6071 if (tx_flags & IXGBE_TX_FLAGS_FCOE) { 6072 if (data_len < sizeof(struct fcoe_crc_eof)) { 6073 size -= sizeof(struct fcoe_crc_eof) - data_len; 6074 data_len = 0; 6075 } else { 6076 data_len -= sizeof(struct fcoe_crc_eof); 6077 } 6078 } 6079 6080 #endif 6081 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 6082 if (dma_mapping_error(tx_ring->dev, dma)) 6083 goto dma_error; 6084 6085 /* record length, and DMA address */ 6086 dma_unmap_len_set(first, len, size); 6087 dma_unmap_addr_set(first, dma, dma); 6088 6089 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6090 6091 for (;;) { 6092 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 6093 tx_desc->read.cmd_type_len = 6094 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 6095 6096 i++; 6097 tx_desc++; 6098 if (i == tx_ring->count) { 6099 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 6100 i = 0; 6101 } 6102 6103 dma += IXGBE_MAX_DATA_PER_TXD; 6104 size -= IXGBE_MAX_DATA_PER_TXD; 6105 6106 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6107 tx_desc->read.olinfo_status = 0; 6108 } 6109 6110 if (likely(!data_len)) 6111 break; 6112 6113 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 6114 6115 i++; 6116 tx_desc++; 6117 if (i == tx_ring->count) { 6118 tx_desc = IXGBE_TX_DESC(tx_ring, 0); 6119 i = 0; 6120 } 6121 6122 #ifdef IXGBE_FCOE 6123 size = min_t(unsigned int, data_len, skb_frag_size(frag)); 6124 #else 6125 size = skb_frag_size(frag); 6126 #endif 6127 data_len -= size; 6128 6129 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 6130 DMA_TO_DEVICE); 6131 if (dma_mapping_error(tx_ring->dev, dma)) 6132 goto dma_error; 6133 6134 tx_buffer = &tx_ring->tx_buffer_info[i]; 6135 dma_unmap_len_set(tx_buffer, len, size); 6136 dma_unmap_addr_set(tx_buffer, dma, dma); 6137 6138 tx_desc->read.buffer_addr = cpu_to_le64(dma); 6139 tx_desc->read.olinfo_status = 0; 6140 6141 frag++; 6142 } 6143 6144 /* write last descriptor with RS and EOP bits */ 6145 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 6146 tx_desc->read.cmd_type_len = cmd_type; 6147 6148 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 6149 6150 /* set the timestamp */ 6151 first->time_stamp = jiffies; 6152 6153 /* 6154 * Force memory writes to complete before letting h/w know there 6155 * are new descriptors to fetch. (Only applicable for weak-ordered 6156 * memory model archs, such as IA-64). 6157 * 6158 * We also need this memory barrier to make certain all of the 6159 * status bits have been updated before next_to_watch is written. 6160 */ 6161 wmb(); 6162 6163 /* set next_to_watch value indicating a packet is present */ 6164 first->next_to_watch = tx_desc; 6165 6166 i++; 6167 if (i == tx_ring->count) 6168 i = 0; 6169 6170 tx_ring->next_to_use = i; 6171 6172 /* notify HW of packet */ 6173 writel(i, tx_ring->tail); 6174 6175 return; 6176 dma_error: 6177 dev_err(tx_ring->dev, "TX DMA map failed\n"); 6178 6179 /* clear dma mappings for failed tx_buffer_info map */ 6180 for (;;) { 6181 tx_buffer = &tx_ring->tx_buffer_info[i]; 6182 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer); 6183 if (tx_buffer == first) 6184 break; 6185 if (i == 0) 6186 i = tx_ring->count; 6187 i--; 6188 } 6189 6190 tx_ring->next_to_use = i; 6191 } 6192 6193 static void ixgbe_atr(struct ixgbe_ring *ring, 6194 struct ixgbe_tx_buffer *first) 6195 { 6196 struct ixgbe_q_vector *q_vector = ring->q_vector; 6197 union ixgbe_atr_hash_dword input = { .dword = 0 }; 6198 union ixgbe_atr_hash_dword common = { .dword = 0 }; 6199 union { 6200 unsigned char *network; 6201 struct iphdr *ipv4; 6202 struct ipv6hdr *ipv6; 6203 } hdr; 6204 struct tcphdr *th; 6205 __be16 vlan_id; 6206 6207 /* if ring doesn't have a interrupt vector, cannot perform ATR */ 6208 if (!q_vector) 6209 return; 6210 6211 /* do nothing if sampling is disabled */ 6212 if (!ring->atr_sample_rate) 6213 return; 6214 6215 ring->atr_count++; 6216 6217 /* snag network header to get L4 type and address */ 6218 hdr.network = skb_network_header(first->skb); 6219 6220 /* Currently only IPv4/IPv6 with TCP is supported */ 6221 if ((first->protocol != __constant_htons(ETH_P_IPV6) || 6222 hdr.ipv6->nexthdr != IPPROTO_TCP) && 6223 (first->protocol != __constant_htons(ETH_P_IP) || 6224 hdr.ipv4->protocol != IPPROTO_TCP)) 6225 return; 6226 6227 th = tcp_hdr(first->skb); 6228 6229 /* skip this packet since it is invalid or the socket is closing */ 6230 if (!th || th->fin) 6231 return; 6232 6233 /* sample on all syn packets or once every atr sample count */ 6234 if (!th->syn && (ring->atr_count < ring->atr_sample_rate)) 6235 return; 6236 6237 /* reset sample count */ 6238 ring->atr_count = 0; 6239 6240 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT); 6241 6242 /* 6243 * src and dst are inverted, think how the receiver sees them 6244 * 6245 * The input is broken into two sections, a non-compressed section 6246 * containing vm_pool, vlan_id, and flow_type. The rest of the data 6247 * is XORed together and stored in the compressed dword. 6248 */ 6249 input.formatted.vlan_id = vlan_id; 6250 6251 /* 6252 * since src port and flex bytes occupy the same word XOR them together 6253 * and write the value to source port portion of compressed dword 6254 */ 6255 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN)) 6256 common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q); 6257 else 6258 common.port.src ^= th->dest ^ first->protocol; 6259 common.port.dst ^= th->source; 6260 6261 if (first->protocol == __constant_htons(ETH_P_IP)) { 6262 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4; 6263 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr; 6264 } else { 6265 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; 6266 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ 6267 hdr.ipv6->saddr.s6_addr32[1] ^ 6268 hdr.ipv6->saddr.s6_addr32[2] ^ 6269 hdr.ipv6->saddr.s6_addr32[3] ^ 6270 hdr.ipv6->daddr.s6_addr32[0] ^ 6271 hdr.ipv6->daddr.s6_addr32[1] ^ 6272 hdr.ipv6->daddr.s6_addr32[2] ^ 6273 hdr.ipv6->daddr.s6_addr32[3]; 6274 } 6275 6276 /* This assumes the Rx queue and Tx queue are bound to the same CPU */ 6277 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, 6278 input, common, ring->queue_index); 6279 } 6280 6281 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 6282 { 6283 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 6284 /* Herbert's original patch had: 6285 * smp_mb__after_netif_stop_queue(); 6286 * but since that doesn't exist yet, just open code it. */ 6287 smp_mb(); 6288 6289 /* We need to check again in a case another CPU has just 6290 * made room available. */ 6291 if (likely(ixgbe_desc_unused(tx_ring) < size)) 6292 return -EBUSY; 6293 6294 /* A reprieve! - use start_queue because it doesn't call schedule */ 6295 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 6296 ++tx_ring->tx_stats.restart_queue; 6297 return 0; 6298 } 6299 6300 static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) 6301 { 6302 if (likely(ixgbe_desc_unused(tx_ring) >= size)) 6303 return 0; 6304 return __ixgbe_maybe_stop_tx(tx_ring, size); 6305 } 6306 6307 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb) 6308 { 6309 struct ixgbe_adapter *adapter = netdev_priv(dev); 6310 int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 6311 smp_processor_id(); 6312 #ifdef IXGBE_FCOE 6313 __be16 protocol = vlan_get_protocol(skb); 6314 6315 if (((protocol == htons(ETH_P_FCOE)) || 6316 (protocol == htons(ETH_P_FIP))) && 6317 (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) { 6318 struct ixgbe_ring_feature *f; 6319 6320 f = &adapter->ring_feature[RING_F_FCOE]; 6321 6322 while (txq >= f->indices) 6323 txq -= f->indices; 6324 txq += adapter->ring_feature[RING_F_FCOE].offset; 6325 6326 return txq; 6327 } 6328 #endif 6329 6330 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { 6331 while (unlikely(txq >= dev->real_num_tx_queues)) 6332 txq -= dev->real_num_tx_queues; 6333 return txq; 6334 } 6335 6336 return skb_tx_hash(dev, skb); 6337 } 6338 6339 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, 6340 struct ixgbe_adapter *adapter, 6341 struct ixgbe_ring *tx_ring) 6342 { 6343 struct ixgbe_tx_buffer *first; 6344 int tso; 6345 u32 tx_flags = 0; 6346 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 6347 unsigned short f; 6348 #endif 6349 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 6350 __be16 protocol = skb->protocol; 6351 u8 hdr_len = 0; 6352 6353 /* 6354 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 6355 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 6356 * + 2 desc gap to keep tail from touching head, 6357 * + 1 desc for context descriptor, 6358 * otherwise try next time 6359 */ 6360 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 6361 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 6362 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 6363 #else 6364 count += skb_shinfo(skb)->nr_frags; 6365 #endif 6366 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) { 6367 tx_ring->tx_stats.tx_busy++; 6368 return NETDEV_TX_BUSY; 6369 } 6370 6371 /* record the location of the first descriptor for this packet */ 6372 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 6373 first->skb = skb; 6374 first->bytecount = skb->len; 6375 first->gso_segs = 1; 6376 6377 /* if we have a HW VLAN tag being added default to the HW one */ 6378 if (vlan_tx_tag_present(skb)) { 6379 tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT; 6380 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 6381 /* else if it is a SW VLAN check the next protocol and store the tag */ 6382 } else if (protocol == __constant_htons(ETH_P_8021Q)) { 6383 struct vlan_hdr *vhdr, _vhdr; 6384 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 6385 if (!vhdr) 6386 goto out_drop; 6387 6388 protocol = vhdr->h_vlan_encapsulated_proto; 6389 tx_flags |= ntohs(vhdr->h_vlan_TCI) << 6390 IXGBE_TX_FLAGS_VLAN_SHIFT; 6391 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN; 6392 } 6393 6394 skb_tx_timestamp(skb); 6395 6396 #ifdef CONFIG_IXGBE_PTP 6397 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 6398 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 6399 tx_flags |= IXGBE_TX_FLAGS_TSTAMP; 6400 } 6401 #endif 6402 6403 #ifdef CONFIG_PCI_IOV 6404 /* 6405 * Use the l2switch_enable flag - would be false if the DMA 6406 * Tx switch had been disabled. 6407 */ 6408 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 6409 tx_flags |= IXGBE_TX_FLAGS_TXSW; 6410 6411 #endif 6412 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ 6413 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && 6414 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || 6415 (skb->priority != TC_PRIO_CONTROL))) { 6416 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; 6417 tx_flags |= (skb->priority & 0x7) << 6418 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; 6419 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) { 6420 struct vlan_ethhdr *vhdr; 6421 if (skb_header_cloned(skb) && 6422 pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) 6423 goto out_drop; 6424 vhdr = (struct vlan_ethhdr *)skb->data; 6425 vhdr->h_vlan_TCI = htons(tx_flags >> 6426 IXGBE_TX_FLAGS_VLAN_SHIFT); 6427 } else { 6428 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN; 6429 } 6430 } 6431 6432 /* record initial flags and protocol */ 6433 first->tx_flags = tx_flags; 6434 first->protocol = protocol; 6435 6436 #ifdef IXGBE_FCOE 6437 /* setup tx offload for FCoE */ 6438 if ((protocol == __constant_htons(ETH_P_FCOE)) && 6439 (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { 6440 tso = ixgbe_fso(tx_ring, first, &hdr_len); 6441 if (tso < 0) 6442 goto out_drop; 6443 6444 goto xmit_fcoe; 6445 } 6446 6447 #endif /* IXGBE_FCOE */ 6448 tso = ixgbe_tso(tx_ring, first, &hdr_len); 6449 if (tso < 0) 6450 goto out_drop; 6451 else if (!tso) 6452 ixgbe_tx_csum(tx_ring, first); 6453 6454 /* add the ATR filter if ATR is on */ 6455 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state)) 6456 ixgbe_atr(tx_ring, first); 6457 6458 #ifdef IXGBE_FCOE 6459 xmit_fcoe: 6460 #endif /* IXGBE_FCOE */ 6461 ixgbe_tx_map(tx_ring, first, hdr_len); 6462 6463 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED); 6464 6465 return NETDEV_TX_OK; 6466 6467 out_drop: 6468 dev_kfree_skb_any(first->skb); 6469 first->skb = NULL; 6470 6471 return NETDEV_TX_OK; 6472 } 6473 6474 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, 6475 struct net_device *netdev) 6476 { 6477 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6478 struct ixgbe_ring *tx_ring; 6479 6480 /* 6481 * The minimum packet size for olinfo paylen is 17 so pad the skb 6482 * in order to meet this minimum size requirement. 6483 */ 6484 if (unlikely(skb->len < 17)) { 6485 if (skb_pad(skb, 17 - skb->len)) 6486 return NETDEV_TX_OK; 6487 skb->len = 17; 6488 } 6489 6490 tx_ring = adapter->tx_ring[skb->queue_mapping]; 6491 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); 6492 } 6493 6494 /** 6495 * ixgbe_set_mac - Change the Ethernet Address of the NIC 6496 * @netdev: network interface device structure 6497 * @p: pointer to an address structure 6498 * 6499 * Returns 0 on success, negative on failure 6500 **/ 6501 static int ixgbe_set_mac(struct net_device *netdev, void *p) 6502 { 6503 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6504 struct ixgbe_hw *hw = &adapter->hw; 6505 struct sockaddr *addr = p; 6506 6507 if (!is_valid_ether_addr(addr->sa_data)) 6508 return -EADDRNOTAVAIL; 6509 6510 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 6511 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 6512 6513 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, VMDQ_P(0), IXGBE_RAH_AV); 6514 6515 return 0; 6516 } 6517 6518 static int 6519 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) 6520 { 6521 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6522 struct ixgbe_hw *hw = &adapter->hw; 6523 u16 value; 6524 int rc; 6525 6526 if (prtad != hw->phy.mdio.prtad) 6527 return -EINVAL; 6528 rc = hw->phy.ops.read_reg(hw, addr, devad, &value); 6529 if (!rc) 6530 rc = value; 6531 return rc; 6532 } 6533 6534 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, 6535 u16 addr, u16 value) 6536 { 6537 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6538 struct ixgbe_hw *hw = &adapter->hw; 6539 6540 if (prtad != hw->phy.mdio.prtad) 6541 return -EINVAL; 6542 return hw->phy.ops.write_reg(hw, addr, devad, value); 6543 } 6544 6545 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) 6546 { 6547 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6548 6549 switch (cmd) { 6550 #ifdef CONFIG_IXGBE_PTP 6551 case SIOCSHWTSTAMP: 6552 return ixgbe_ptp_hwtstamp_ioctl(adapter, req, cmd); 6553 #endif 6554 default: 6555 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd); 6556 } 6557 } 6558 6559 /** 6560 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding 6561 * netdev->dev_addrs 6562 * @netdev: network interface device structure 6563 * 6564 * Returns non-zero on failure 6565 **/ 6566 static int ixgbe_add_sanmac_netdev(struct net_device *dev) 6567 { 6568 int err = 0; 6569 struct ixgbe_adapter *adapter = netdev_priv(dev); 6570 struct ixgbe_hw *hw = &adapter->hw; 6571 6572 if (is_valid_ether_addr(hw->mac.san_addr)) { 6573 rtnl_lock(); 6574 err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN); 6575 rtnl_unlock(); 6576 6577 /* update SAN MAC vmdq pool selection */ 6578 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0)); 6579 } 6580 return err; 6581 } 6582 6583 /** 6584 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding 6585 * netdev->dev_addrs 6586 * @netdev: network interface device structure 6587 * 6588 * Returns non-zero on failure 6589 **/ 6590 static int ixgbe_del_sanmac_netdev(struct net_device *dev) 6591 { 6592 int err = 0; 6593 struct ixgbe_adapter *adapter = netdev_priv(dev); 6594 struct ixgbe_mac_info *mac = &adapter->hw.mac; 6595 6596 if (is_valid_ether_addr(mac->san_addr)) { 6597 rtnl_lock(); 6598 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN); 6599 rtnl_unlock(); 6600 } 6601 return err; 6602 } 6603 6604 #ifdef CONFIG_NET_POLL_CONTROLLER 6605 /* 6606 * Polling 'interrupt' - used by things like netconsole to send skbs 6607 * without having to re-enable interrupts. It's not called while 6608 * the interrupt routine is executing. 6609 */ 6610 static void ixgbe_netpoll(struct net_device *netdev) 6611 { 6612 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6613 int i; 6614 6615 /* if interface is down do nothing */ 6616 if (test_bit(__IXGBE_DOWN, &adapter->state)) 6617 return; 6618 6619 adapter->flags |= IXGBE_FLAG_IN_NETPOLL; 6620 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { 6621 for (i = 0; i < adapter->num_q_vectors; i++) 6622 ixgbe_msix_clean_rings(0, adapter->q_vector[i]); 6623 } else { 6624 ixgbe_intr(adapter->pdev->irq, netdev); 6625 } 6626 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL; 6627 } 6628 6629 #endif 6630 static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev, 6631 struct rtnl_link_stats64 *stats) 6632 { 6633 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6634 int i; 6635 6636 rcu_read_lock(); 6637 for (i = 0; i < adapter->num_rx_queues; i++) { 6638 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]); 6639 u64 bytes, packets; 6640 unsigned int start; 6641 6642 if (ring) { 6643 do { 6644 start = u64_stats_fetch_begin_bh(&ring->syncp); 6645 packets = ring->stats.packets; 6646 bytes = ring->stats.bytes; 6647 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 6648 stats->rx_packets += packets; 6649 stats->rx_bytes += bytes; 6650 } 6651 } 6652 6653 for (i = 0; i < adapter->num_tx_queues; i++) { 6654 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]); 6655 u64 bytes, packets; 6656 unsigned int start; 6657 6658 if (ring) { 6659 do { 6660 start = u64_stats_fetch_begin_bh(&ring->syncp); 6661 packets = ring->stats.packets; 6662 bytes = ring->stats.bytes; 6663 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 6664 stats->tx_packets += packets; 6665 stats->tx_bytes += bytes; 6666 } 6667 } 6668 rcu_read_unlock(); 6669 /* following stats updated by ixgbe_watchdog_task() */ 6670 stats->multicast = netdev->stats.multicast; 6671 stats->rx_errors = netdev->stats.rx_errors; 6672 stats->rx_length_errors = netdev->stats.rx_length_errors; 6673 stats->rx_crc_errors = netdev->stats.rx_crc_errors; 6674 stats->rx_missed_errors = netdev->stats.rx_missed_errors; 6675 return stats; 6676 } 6677 6678 #ifdef CONFIG_IXGBE_DCB 6679 /** 6680 * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid. 6681 * @adapter: pointer to ixgbe_adapter 6682 * @tc: number of traffic classes currently enabled 6683 * 6684 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm 6685 * 802.1Q priority maps to a packet buffer that exists. 6686 */ 6687 static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc) 6688 { 6689 struct ixgbe_hw *hw = &adapter->hw; 6690 u32 reg, rsave; 6691 int i; 6692 6693 /* 82598 have a static priority to TC mapping that can not 6694 * be changed so no validation is needed. 6695 */ 6696 if (hw->mac.type == ixgbe_mac_82598EB) 6697 return; 6698 6699 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC); 6700 rsave = reg; 6701 6702 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) { 6703 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT); 6704 6705 /* If up2tc is out of bounds default to zero */ 6706 if (up2tc > tc) 6707 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT); 6708 } 6709 6710 if (reg != rsave) 6711 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg); 6712 6713 return; 6714 } 6715 6716 /** 6717 * ixgbe_set_prio_tc_map - Configure netdev prio tc map 6718 * @adapter: Pointer to adapter struct 6719 * 6720 * Populate the netdev user priority to tc map 6721 */ 6722 static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) 6723 { 6724 struct net_device *dev = adapter->netdev; 6725 struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg; 6726 struct ieee_ets *ets = adapter->ixgbe_ieee_ets; 6727 u8 prio; 6728 6729 for (prio = 0; prio < MAX_USER_PRIORITY; prio++) { 6730 u8 tc = 0; 6731 6732 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) 6733 tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio); 6734 else if (ets) 6735 tc = ets->prio_tc[prio]; 6736 6737 netdev_set_prio_tc_map(dev, prio, tc); 6738 } 6739 } 6740 6741 /** 6742 * ixgbe_setup_tc - configure net_device for multiple traffic classes 6743 * 6744 * @netdev: net device to configure 6745 * @tc: number of traffic classes to enable 6746 */ 6747 int ixgbe_setup_tc(struct net_device *dev, u8 tc) 6748 { 6749 struct ixgbe_adapter *adapter = netdev_priv(dev); 6750 struct ixgbe_hw *hw = &adapter->hw; 6751 6752 /* Hardware supports up to 8 traffic classes */ 6753 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs || 6754 (hw->mac.type == ixgbe_mac_82598EB && 6755 tc < MAX_TRAFFIC_CLASS)) 6756 return -EINVAL; 6757 6758 /* Hardware has to reinitialize queues and interrupts to 6759 * match packet buffer alignment. Unfortunately, the 6760 * hardware is not flexible enough to do this dynamically. 6761 */ 6762 if (netif_running(dev)) 6763 ixgbe_close(dev); 6764 ixgbe_clear_interrupt_scheme(adapter); 6765 6766 if (tc) { 6767 netdev_set_num_tc(dev, tc); 6768 ixgbe_set_prio_tc_map(adapter); 6769 6770 adapter->flags |= IXGBE_FLAG_DCB_ENABLED; 6771 6772 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 6773 adapter->last_lfc_mode = adapter->hw.fc.requested_mode; 6774 adapter->hw.fc.requested_mode = ixgbe_fc_none; 6775 } 6776 } else { 6777 netdev_reset_tc(dev); 6778 6779 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 6780 adapter->hw.fc.requested_mode = adapter->last_lfc_mode; 6781 6782 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; 6783 6784 adapter->temp_dcb_cfg.pfc_mode_enable = false; 6785 adapter->dcb_cfg.pfc_mode_enable = false; 6786 } 6787 6788 ixgbe_init_interrupt_scheme(adapter); 6789 ixgbe_validate_rtr(adapter, tc); 6790 if (netif_running(dev)) 6791 ixgbe_open(dev); 6792 6793 return 0; 6794 } 6795 6796 #endif /* CONFIG_IXGBE_DCB */ 6797 void ixgbe_do_reset(struct net_device *netdev) 6798 { 6799 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6800 6801 if (netif_running(netdev)) 6802 ixgbe_reinit_locked(adapter); 6803 else 6804 ixgbe_reset(adapter); 6805 } 6806 6807 static netdev_features_t ixgbe_fix_features(struct net_device *netdev, 6808 netdev_features_t features) 6809 { 6810 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6811 6812 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ 6813 if (!(features & NETIF_F_RXCSUM)) 6814 features &= ~NETIF_F_LRO; 6815 6816 /* Turn off LRO if not RSC capable */ 6817 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) 6818 features &= ~NETIF_F_LRO; 6819 6820 return features; 6821 } 6822 6823 static int ixgbe_set_features(struct net_device *netdev, 6824 netdev_features_t features) 6825 { 6826 struct ixgbe_adapter *adapter = netdev_priv(netdev); 6827 netdev_features_t changed = netdev->features ^ features; 6828 bool need_reset = false; 6829 6830 /* Make sure RSC matches LRO, reset if change */ 6831 if (!(features & NETIF_F_LRO)) { 6832 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 6833 need_reset = true; 6834 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED; 6835 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && 6836 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { 6837 if (adapter->rx_itr_setting == 1 || 6838 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { 6839 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; 6840 need_reset = true; 6841 } else if ((changed ^ features) & NETIF_F_LRO) { 6842 e_info(probe, "rx-usecs set too low, " 6843 "disabling RSC\n"); 6844 } 6845 } 6846 6847 /* 6848 * Check if Flow Director n-tuple support was enabled or disabled. If 6849 * the state changed, we need to reset. 6850 */ 6851 switch (features & NETIF_F_NTUPLE) { 6852 case NETIF_F_NTUPLE: 6853 /* turn off ATR, enable perfect filters and reset */ 6854 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) 6855 need_reset = true; 6856 6857 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; 6858 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 6859 break; 6860 default: 6861 /* turn off perfect filters, enable ATR and reset */ 6862 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) 6863 need_reset = true; 6864 6865 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE; 6866 6867 /* We cannot enable ATR if SR-IOV is enabled */ 6868 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 6869 break; 6870 6871 /* We cannot enable ATR if we have 2 or more traffic classes */ 6872 if (netdev_get_num_tc(netdev) > 1) 6873 break; 6874 6875 /* We cannot enable ATR if RSS is disabled */ 6876 if (adapter->ring_feature[RING_F_RSS].limit <= 1) 6877 break; 6878 6879 /* A sample rate of 0 indicates ATR disabled */ 6880 if (!adapter->atr_sample_rate) 6881 break; 6882 6883 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; 6884 break; 6885 } 6886 6887 if (features & NETIF_F_HW_VLAN_RX) 6888 ixgbe_vlan_strip_enable(adapter); 6889 else 6890 ixgbe_vlan_strip_disable(adapter); 6891 6892 if (changed & NETIF_F_RXALL) 6893 need_reset = true; 6894 6895 netdev->features = features; 6896 if (need_reset) 6897 ixgbe_do_reset(netdev); 6898 6899 return 0; 6900 } 6901 6902 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 6903 struct net_device *dev, 6904 const unsigned char *addr, 6905 u16 flags) 6906 { 6907 struct ixgbe_adapter *adapter = netdev_priv(dev); 6908 int err; 6909 6910 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) 6911 return -EOPNOTSUPP; 6912 6913 if (ndm->ndm_state & NUD_PERMANENT) { 6914 pr_info("%s: FDB only supports static addresses\n", 6915 ixgbe_driver_name); 6916 return -EINVAL; 6917 } 6918 6919 if (is_unicast_ether_addr(addr)) { 6920 u32 rar_uc_entries = IXGBE_MAX_PF_MACVLANS; 6921 6922 if (netdev_uc_count(dev) < rar_uc_entries) 6923 err = dev_uc_add_excl(dev, addr); 6924 else 6925 err = -ENOMEM; 6926 } else if (is_multicast_ether_addr(addr)) { 6927 err = dev_mc_add_excl(dev, addr); 6928 } else { 6929 err = -EINVAL; 6930 } 6931 6932 /* Only return duplicate errors if NLM_F_EXCL is set */ 6933 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 6934 err = 0; 6935 6936 return err; 6937 } 6938 6939 static int ixgbe_ndo_fdb_del(struct ndmsg *ndm, 6940 struct net_device *dev, 6941 const unsigned char *addr) 6942 { 6943 struct ixgbe_adapter *adapter = netdev_priv(dev); 6944 int err = -EOPNOTSUPP; 6945 6946 if (ndm->ndm_state & NUD_PERMANENT) { 6947 pr_info("%s: FDB only supports static addresses\n", 6948 ixgbe_driver_name); 6949 return -EINVAL; 6950 } 6951 6952 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 6953 if (is_unicast_ether_addr(addr)) 6954 err = dev_uc_del(dev, addr); 6955 else if (is_multicast_ether_addr(addr)) 6956 err = dev_mc_del(dev, addr); 6957 else 6958 err = -EINVAL; 6959 } 6960 6961 return err; 6962 } 6963 6964 static int ixgbe_ndo_fdb_dump(struct sk_buff *skb, 6965 struct netlink_callback *cb, 6966 struct net_device *dev, 6967 int idx) 6968 { 6969 struct ixgbe_adapter *adapter = netdev_priv(dev); 6970 6971 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) 6972 idx = ndo_dflt_fdb_dump(skb, cb, dev, idx); 6973 6974 return idx; 6975 } 6976 6977 static const struct net_device_ops ixgbe_netdev_ops = { 6978 .ndo_open = ixgbe_open, 6979 .ndo_stop = ixgbe_close, 6980 .ndo_start_xmit = ixgbe_xmit_frame, 6981 .ndo_select_queue = ixgbe_select_queue, 6982 .ndo_set_rx_mode = ixgbe_set_rx_mode, 6983 .ndo_validate_addr = eth_validate_addr, 6984 .ndo_set_mac_address = ixgbe_set_mac, 6985 .ndo_change_mtu = ixgbe_change_mtu, 6986 .ndo_tx_timeout = ixgbe_tx_timeout, 6987 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid, 6988 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid, 6989 .ndo_do_ioctl = ixgbe_ioctl, 6990 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac, 6991 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan, 6992 .ndo_set_vf_tx_rate = ixgbe_ndo_set_vf_bw, 6993 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk, 6994 .ndo_get_vf_config = ixgbe_ndo_get_vf_config, 6995 .ndo_get_stats64 = ixgbe_get_stats64, 6996 #ifdef CONFIG_IXGBE_DCB 6997 .ndo_setup_tc = ixgbe_setup_tc, 6998 #endif 6999 #ifdef CONFIG_NET_POLL_CONTROLLER 7000 .ndo_poll_controller = ixgbe_netpoll, 7001 #endif 7002 #ifdef IXGBE_FCOE 7003 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get, 7004 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target, 7005 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put, 7006 .ndo_fcoe_enable = ixgbe_fcoe_enable, 7007 .ndo_fcoe_disable = ixgbe_fcoe_disable, 7008 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn, 7009 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo, 7010 #endif /* IXGBE_FCOE */ 7011 .ndo_set_features = ixgbe_set_features, 7012 .ndo_fix_features = ixgbe_fix_features, 7013 .ndo_fdb_add = ixgbe_ndo_fdb_add, 7014 .ndo_fdb_del = ixgbe_ndo_fdb_del, 7015 .ndo_fdb_dump = ixgbe_ndo_fdb_dump, 7016 }; 7017 7018 /** 7019 * ixgbe_wol_supported - Check whether device supports WoL 7020 * @hw: hw specific details 7021 * @device_id: the device ID 7022 * @subdev_id: the subsystem device ID 7023 * 7024 * This function is used by probe and ethtool to determine 7025 * which devices have WoL support 7026 * 7027 **/ 7028 int ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, 7029 u16 subdevice_id) 7030 { 7031 struct ixgbe_hw *hw = &adapter->hw; 7032 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK; 7033 int is_wol_supported = 0; 7034 7035 switch (device_id) { 7036 case IXGBE_DEV_ID_82599_SFP: 7037 /* Only these subdevices could supports WOL */ 7038 switch (subdevice_id) { 7039 case IXGBE_SUBDEV_ID_82599_560FLR: 7040 /* only support first port */ 7041 if (hw->bus.func != 0) 7042 break; 7043 case IXGBE_SUBDEV_ID_82599_SFP: 7044 case IXGBE_SUBDEV_ID_82599_RNDC: 7045 is_wol_supported = 1; 7046 break; 7047 } 7048 break; 7049 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE: 7050 /* All except this subdevice support WOL */ 7051 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) 7052 is_wol_supported = 1; 7053 break; 7054 case IXGBE_DEV_ID_82599_KX4: 7055 is_wol_supported = 1; 7056 break; 7057 case IXGBE_DEV_ID_X540T: 7058 case IXGBE_DEV_ID_X540T1: 7059 /* check eeprom to see if enabled wol */ 7060 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 7061 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && 7062 (hw->bus.func == 0))) { 7063 is_wol_supported = 1; 7064 } 7065 break; 7066 } 7067 7068 return is_wol_supported; 7069 } 7070 7071 /** 7072 * ixgbe_probe - Device Initialization Routine 7073 * @pdev: PCI device information struct 7074 * @ent: entry in ixgbe_pci_tbl 7075 * 7076 * Returns 0 on success, negative on failure 7077 * 7078 * ixgbe_probe initializes an adapter identified by a pci_dev structure. 7079 * The OS initialization, configuring of the adapter private structure, 7080 * and a hardware reset occur. 7081 **/ 7082 static int __devinit ixgbe_probe(struct pci_dev *pdev, 7083 const struct pci_device_id *ent) 7084 { 7085 struct net_device *netdev; 7086 struct ixgbe_adapter *adapter = NULL; 7087 struct ixgbe_hw *hw; 7088 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data]; 7089 static int cards_found; 7090 int i, err, pci_using_dac; 7091 u8 part_str[IXGBE_PBANUM_LENGTH]; 7092 unsigned int indices = num_possible_cpus(); 7093 unsigned int dcb_max = 0; 7094 #ifdef IXGBE_FCOE 7095 u16 device_caps; 7096 #endif 7097 u32 eec; 7098 7099 /* Catch broken hardware that put the wrong VF device ID in 7100 * the PCIe SR-IOV capability. 7101 */ 7102 if (pdev->is_virtfn) { 7103 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", 7104 pci_name(pdev), pdev->vendor, pdev->device); 7105 return -EINVAL; 7106 } 7107 7108 err = pci_enable_device_mem(pdev); 7109 if (err) 7110 return err; 7111 7112 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 7113 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 7114 pci_using_dac = 1; 7115 } else { 7116 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 7117 if (err) { 7118 err = dma_set_coherent_mask(&pdev->dev, 7119 DMA_BIT_MASK(32)); 7120 if (err) { 7121 dev_err(&pdev->dev, 7122 "No usable DMA configuration, aborting\n"); 7123 goto err_dma; 7124 } 7125 } 7126 pci_using_dac = 0; 7127 } 7128 7129 err = pci_request_selected_regions(pdev, pci_select_bars(pdev, 7130 IORESOURCE_MEM), ixgbe_driver_name); 7131 if (err) { 7132 dev_err(&pdev->dev, 7133 "pci_request_selected_regions failed 0x%x\n", err); 7134 goto err_pci_reg; 7135 } 7136 7137 pci_enable_pcie_error_reporting(pdev); 7138 7139 pci_set_master(pdev); 7140 pci_save_state(pdev); 7141 7142 #ifdef CONFIG_IXGBE_DCB 7143 if (ii->mac == ixgbe_mac_82598EB) 7144 dcb_max = min_t(unsigned int, indices * MAX_TRAFFIC_CLASS, 7145 IXGBE_MAX_RSS_INDICES); 7146 else 7147 dcb_max = min_t(unsigned int, indices * MAX_TRAFFIC_CLASS, 7148 IXGBE_MAX_FDIR_INDICES); 7149 #endif 7150 7151 if (ii->mac == ixgbe_mac_82598EB) 7152 indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES); 7153 else 7154 indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES); 7155 7156 #ifdef IXGBE_FCOE 7157 indices += min_t(unsigned int, num_possible_cpus(), 7158 IXGBE_MAX_FCOE_INDICES); 7159 #endif 7160 indices = max_t(unsigned int, dcb_max, indices); 7161 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices); 7162 if (!netdev) { 7163 err = -ENOMEM; 7164 goto err_alloc_etherdev; 7165 } 7166 7167 SET_NETDEV_DEV(netdev, &pdev->dev); 7168 7169 adapter = netdev_priv(netdev); 7170 pci_set_drvdata(pdev, adapter); 7171 7172 adapter->netdev = netdev; 7173 adapter->pdev = pdev; 7174 hw = &adapter->hw; 7175 hw->back = adapter; 7176 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7177 7178 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 7179 pci_resource_len(pdev, 0)); 7180 if (!hw->hw_addr) { 7181 err = -EIO; 7182 goto err_ioremap; 7183 } 7184 7185 netdev->netdev_ops = &ixgbe_netdev_ops; 7186 ixgbe_set_ethtool_ops(netdev); 7187 netdev->watchdog_timeo = 5 * HZ; 7188 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 7189 7190 adapter->bd_number = cards_found; 7191 7192 /* Setup hw api */ 7193 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 7194 hw->mac.type = ii->mac; 7195 7196 /* EEPROM */ 7197 memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops)); 7198 eec = IXGBE_READ_REG(hw, IXGBE_EEC); 7199 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */ 7200 if (!(eec & (1 << 8))) 7201 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic; 7202 7203 /* PHY */ 7204 memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops)); 7205 hw->phy.sfp_type = ixgbe_sfp_type_unknown; 7206 /* ixgbe_identify_phy_generic will set prtad and mmds properly */ 7207 hw->phy.mdio.prtad = MDIO_PRTAD_NONE; 7208 hw->phy.mdio.mmds = 0; 7209 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22; 7210 hw->phy.mdio.dev = netdev; 7211 hw->phy.mdio.mdio_read = ixgbe_mdio_read; 7212 hw->phy.mdio.mdio_write = ixgbe_mdio_write; 7213 7214 ii->get_invariants(hw); 7215 7216 /* setup the private structure */ 7217 err = ixgbe_sw_init(adapter); 7218 if (err) 7219 goto err_sw_init; 7220 7221 /* Make it possible the adapter to be woken up via WOL */ 7222 switch (adapter->hw.mac.type) { 7223 case ixgbe_mac_82599EB: 7224 case ixgbe_mac_X540: 7225 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 7226 break; 7227 default: 7228 break; 7229 } 7230 7231 /* 7232 * If there is a fan on this device and it has failed log the 7233 * failure. 7234 */ 7235 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) { 7236 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 7237 if (esdp & IXGBE_ESDP_SDP1) 7238 e_crit(probe, "Fan has stopped, replace the adapter\n"); 7239 } 7240 7241 if (allow_unsupported_sfp) 7242 hw->allow_unsupported_sfp = allow_unsupported_sfp; 7243 7244 /* reset_hw fills in the perm_addr as well */ 7245 hw->phy.reset_if_overtemp = true; 7246 err = hw->mac.ops.reset_hw(hw); 7247 hw->phy.reset_if_overtemp = false; 7248 if (err == IXGBE_ERR_SFP_NOT_PRESENT && 7249 hw->mac.type == ixgbe_mac_82598EB) { 7250 err = 0; 7251 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 7252 e_dev_err("failed to load because an unsupported SFP+ " 7253 "module type was detected.\n"); 7254 e_dev_err("Reload the driver after installing a supported " 7255 "module.\n"); 7256 goto err_sw_init; 7257 } else if (err) { 7258 e_dev_err("HW Init failed: %d\n", err); 7259 goto err_sw_init; 7260 } 7261 7262 #ifdef CONFIG_PCI_IOV 7263 ixgbe_enable_sriov(adapter, ii); 7264 7265 #endif 7266 netdev->features = NETIF_F_SG | 7267 NETIF_F_IP_CSUM | 7268 NETIF_F_IPV6_CSUM | 7269 NETIF_F_HW_VLAN_TX | 7270 NETIF_F_HW_VLAN_RX | 7271 NETIF_F_HW_VLAN_FILTER | 7272 NETIF_F_TSO | 7273 NETIF_F_TSO6 | 7274 NETIF_F_RXHASH | 7275 NETIF_F_RXCSUM; 7276 7277 netdev->hw_features = netdev->features; 7278 7279 switch (adapter->hw.mac.type) { 7280 case ixgbe_mac_82599EB: 7281 case ixgbe_mac_X540: 7282 netdev->features |= NETIF_F_SCTP_CSUM; 7283 netdev->hw_features |= NETIF_F_SCTP_CSUM | 7284 NETIF_F_NTUPLE; 7285 break; 7286 default: 7287 break; 7288 } 7289 7290 netdev->hw_features |= NETIF_F_RXALL; 7291 7292 netdev->vlan_features |= NETIF_F_TSO; 7293 netdev->vlan_features |= NETIF_F_TSO6; 7294 netdev->vlan_features |= NETIF_F_IP_CSUM; 7295 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 7296 netdev->vlan_features |= NETIF_F_SG; 7297 7298 netdev->priv_flags |= IFF_UNICAST_FLT; 7299 netdev->priv_flags |= IFF_SUPP_NOFCS; 7300 7301 #ifdef CONFIG_IXGBE_DCB 7302 netdev->dcbnl_ops = &dcbnl_ops; 7303 #endif 7304 7305 #ifdef IXGBE_FCOE 7306 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) { 7307 if (hw->mac.ops.get_device_caps) { 7308 hw->mac.ops.get_device_caps(hw, &device_caps); 7309 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS) 7310 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE; 7311 } 7312 7313 adapter->ring_feature[RING_F_FCOE].limit = IXGBE_FCRETA_SIZE; 7314 7315 netdev->features |= NETIF_F_FSO | 7316 NETIF_F_FCOE_CRC; 7317 7318 netdev->vlan_features |= NETIF_F_FSO | 7319 NETIF_F_FCOE_CRC | 7320 NETIF_F_FCOE_MTU; 7321 } 7322 #endif /* IXGBE_FCOE */ 7323 if (pci_using_dac) { 7324 netdev->features |= NETIF_F_HIGHDMA; 7325 netdev->vlan_features |= NETIF_F_HIGHDMA; 7326 } 7327 7328 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) 7329 netdev->hw_features |= NETIF_F_LRO; 7330 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) 7331 netdev->features |= NETIF_F_LRO; 7332 7333 /* make sure the EEPROM is good */ 7334 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) { 7335 e_dev_err("The EEPROM Checksum Is Not Valid\n"); 7336 err = -EIO; 7337 goto err_sw_init; 7338 } 7339 7340 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len); 7341 memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len); 7342 7343 if (ixgbe_validate_mac_addr(netdev->perm_addr)) { 7344 e_dev_err("invalid MAC address\n"); 7345 err = -EIO; 7346 goto err_sw_init; 7347 } 7348 7349 setup_timer(&adapter->service_timer, &ixgbe_service_timer, 7350 (unsigned long) adapter); 7351 7352 INIT_WORK(&adapter->service_task, ixgbe_service_task); 7353 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state); 7354 7355 err = ixgbe_init_interrupt_scheme(adapter); 7356 if (err) 7357 goto err_sw_init; 7358 7359 /* WOL not supported for all devices */ 7360 adapter->wol = 0; 7361 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap); 7362 if (ixgbe_wol_supported(adapter, pdev->device, pdev->subsystem_device)) 7363 adapter->wol = IXGBE_WUFC_MAG; 7364 7365 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol); 7366 7367 #ifdef CONFIG_IXGBE_PTP 7368 ixgbe_ptp_init(adapter); 7369 #endif /* CONFIG_IXGBE_PTP*/ 7370 7371 /* save off EEPROM version number */ 7372 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh); 7373 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl); 7374 7375 /* pick up the PCI bus settings for reporting later */ 7376 hw->mac.ops.get_bus_info(hw); 7377 7378 /* print bus type/speed/width info */ 7379 e_dev_info("(PCI Express:%s:%s) %pM\n", 7380 (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" : 7381 hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" : 7382 "Unknown"), 7383 (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" : 7384 hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" : 7385 hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" : 7386 "Unknown"), 7387 netdev->dev_addr); 7388 7389 err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH); 7390 if (err) 7391 strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH); 7392 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) 7393 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n", 7394 hw->mac.type, hw->phy.type, hw->phy.sfp_type, 7395 part_str); 7396 else 7397 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n", 7398 hw->mac.type, hw->phy.type, part_str); 7399 7400 if (hw->bus.width <= ixgbe_bus_width_pcie_x4) { 7401 e_dev_warn("PCI-Express bandwidth available for this card is " 7402 "not sufficient for optimal performance.\n"); 7403 e_dev_warn("For optimal performance a x8 PCI-Express slot " 7404 "is required.\n"); 7405 } 7406 7407 /* reset the hardware with the new settings */ 7408 err = hw->mac.ops.start_hw(hw); 7409 if (err == IXGBE_ERR_EEPROM_VERSION) { 7410 /* We are running on a pre-production device, log a warning */ 7411 e_dev_warn("This device is a pre-production adapter/LOM. " 7412 "Please be aware there may be issues associated " 7413 "with your hardware. If you are experiencing " 7414 "problems please contact your Intel or hardware " 7415 "representative who provided you with this " 7416 "hardware.\n"); 7417 } 7418 strcpy(netdev->name, "eth%d"); 7419 err = register_netdev(netdev); 7420 if (err) 7421 goto err_register; 7422 7423 /* power down the optics for multispeed fiber and 82599 SFP+ fiber */ 7424 if (hw->mac.ops.disable_tx_laser && 7425 ((hw->phy.multispeed_fiber) || 7426 ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) && 7427 (hw->mac.type == ixgbe_mac_82599EB)))) 7428 hw->mac.ops.disable_tx_laser(hw); 7429 7430 /* carrier off reporting is important to ethtool even BEFORE open */ 7431 netif_carrier_off(netdev); 7432 7433 #ifdef CONFIG_IXGBE_DCA 7434 if (dca_add_requester(&pdev->dev) == 0) { 7435 adapter->flags |= IXGBE_FLAG_DCA_ENABLED; 7436 ixgbe_setup_dca(adapter); 7437 } 7438 #endif 7439 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) { 7440 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs); 7441 for (i = 0; i < adapter->num_vfs; i++) 7442 ixgbe_vf_configuration(pdev, (i | 0x10000000)); 7443 } 7444 7445 /* firmware requires driver version to be 0xFFFFFFFF 7446 * since os does not support feature 7447 */ 7448 if (hw->mac.ops.set_fw_drv_ver) 7449 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 7450 0xFF); 7451 7452 /* add san mac addr to netdev */ 7453 ixgbe_add_sanmac_netdev(netdev); 7454 7455 e_dev_info("%s\n", ixgbe_default_device_descr); 7456 cards_found++; 7457 7458 #ifdef CONFIG_IXGBE_HWMON 7459 if (ixgbe_sysfs_init(adapter)) 7460 e_err(probe, "failed to allocate sysfs resources\n"); 7461 #endif /* CONFIG_IXGBE_HWMON */ 7462 7463 #ifdef CONFIG_DEBUG_FS 7464 ixgbe_dbg_adapter_init(adapter); 7465 #endif /* CONFIG_DEBUG_FS */ 7466 7467 return 0; 7468 7469 err_register: 7470 ixgbe_release_hw_control(adapter); 7471 ixgbe_clear_interrupt_scheme(adapter); 7472 err_sw_init: 7473 ixgbe_disable_sriov(adapter); 7474 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP; 7475 iounmap(hw->hw_addr); 7476 err_ioremap: 7477 free_netdev(netdev); 7478 err_alloc_etherdev: 7479 pci_release_selected_regions(pdev, 7480 pci_select_bars(pdev, IORESOURCE_MEM)); 7481 err_pci_reg: 7482 err_dma: 7483 pci_disable_device(pdev); 7484 return err; 7485 } 7486 7487 /** 7488 * ixgbe_remove - Device Removal Routine 7489 * @pdev: PCI device information struct 7490 * 7491 * ixgbe_remove is called by the PCI subsystem to alert the driver 7492 * that it should release a PCI device. The could be caused by a 7493 * Hot-Plug event, or because the driver is going to be removed from 7494 * memory. 7495 **/ 7496 static void __devexit ixgbe_remove(struct pci_dev *pdev) 7497 { 7498 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7499 struct net_device *netdev = adapter->netdev; 7500 7501 #ifdef CONFIG_DEBUG_FS 7502 ixgbe_dbg_adapter_exit(adapter); 7503 #endif /*CONFIG_DEBUG_FS */ 7504 7505 set_bit(__IXGBE_DOWN, &adapter->state); 7506 cancel_work_sync(&adapter->service_task); 7507 7508 #ifdef CONFIG_IXGBE_PTP 7509 ixgbe_ptp_stop(adapter); 7510 #endif 7511 7512 #ifdef CONFIG_IXGBE_DCA 7513 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { 7514 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; 7515 dca_remove_requester(&pdev->dev); 7516 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); 7517 } 7518 7519 #endif 7520 #ifdef CONFIG_IXGBE_HWMON 7521 ixgbe_sysfs_exit(adapter); 7522 #endif /* CONFIG_IXGBE_HWMON */ 7523 7524 /* remove the added san mac */ 7525 ixgbe_del_sanmac_netdev(netdev); 7526 7527 if (netdev->reg_state == NETREG_REGISTERED) 7528 unregister_netdev(netdev); 7529 7530 ixgbe_disable_sriov(adapter); 7531 7532 ixgbe_clear_interrupt_scheme(adapter); 7533 7534 ixgbe_release_hw_control(adapter); 7535 7536 #ifdef CONFIG_DCB 7537 kfree(adapter->ixgbe_ieee_pfc); 7538 kfree(adapter->ixgbe_ieee_ets); 7539 7540 #endif 7541 iounmap(adapter->hw.hw_addr); 7542 pci_release_selected_regions(pdev, pci_select_bars(pdev, 7543 IORESOURCE_MEM)); 7544 7545 e_dev_info("complete\n"); 7546 7547 free_netdev(netdev); 7548 7549 pci_disable_pcie_error_reporting(pdev); 7550 7551 pci_disable_device(pdev); 7552 } 7553 7554 /** 7555 * ixgbe_io_error_detected - called when PCI error is detected 7556 * @pdev: Pointer to PCI device 7557 * @state: The current pci connection state 7558 * 7559 * This function is called after a PCI bus error affecting 7560 * this device has been detected. 7561 */ 7562 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, 7563 pci_channel_state_t state) 7564 { 7565 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7566 struct net_device *netdev = adapter->netdev; 7567 7568 #ifdef CONFIG_PCI_IOV 7569 struct pci_dev *bdev, *vfdev; 7570 u32 dw0, dw1, dw2, dw3; 7571 int vf, pos; 7572 u16 req_id, pf_func; 7573 7574 if (adapter->hw.mac.type == ixgbe_mac_82598EB || 7575 adapter->num_vfs == 0) 7576 goto skip_bad_vf_detection; 7577 7578 bdev = pdev->bus->self; 7579 while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT)) 7580 bdev = bdev->bus->self; 7581 7582 if (!bdev) 7583 goto skip_bad_vf_detection; 7584 7585 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR); 7586 if (!pos) 7587 goto skip_bad_vf_detection; 7588 7589 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0); 7590 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1); 7591 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2); 7592 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3); 7593 7594 req_id = dw1 >> 16; 7595 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */ 7596 if (!(req_id & 0x0080)) 7597 goto skip_bad_vf_detection; 7598 7599 pf_func = req_id & 0x01; 7600 if ((pf_func & 1) == (pdev->devfn & 1)) { 7601 unsigned int device_id; 7602 7603 vf = (req_id & 0x7F) >> 1; 7604 e_dev_err("VF %d has caused a PCIe error\n", vf); 7605 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: " 7606 "%8.8x\tdw3: %8.8x\n", 7607 dw0, dw1, dw2, dw3); 7608 switch (adapter->hw.mac.type) { 7609 case ixgbe_mac_82599EB: 7610 device_id = IXGBE_82599_VF_DEVICE_ID; 7611 break; 7612 case ixgbe_mac_X540: 7613 device_id = IXGBE_X540_VF_DEVICE_ID; 7614 break; 7615 default: 7616 device_id = 0; 7617 break; 7618 } 7619 7620 /* Find the pci device of the offending VF */ 7621 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL); 7622 while (vfdev) { 7623 if (vfdev->devfn == (req_id & 0xFF)) 7624 break; 7625 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, 7626 device_id, vfdev); 7627 } 7628 /* 7629 * There's a slim chance the VF could have been hot plugged, 7630 * so if it is no longer present we don't need to issue the 7631 * VFLR. Just clean up the AER in that case. 7632 */ 7633 if (vfdev) { 7634 e_dev_err("Issuing VFLR to VF %d\n", vf); 7635 pci_write_config_dword(vfdev, 0xA8, 0x00008000); 7636 } 7637 7638 pci_cleanup_aer_uncorrect_error_status(pdev); 7639 } 7640 7641 /* 7642 * Even though the error may have occurred on the other port 7643 * we still need to increment the vf error reference count for 7644 * both ports because the I/O resume function will be called 7645 * for both of them. 7646 */ 7647 adapter->vferr_refcount++; 7648 7649 return PCI_ERS_RESULT_RECOVERED; 7650 7651 skip_bad_vf_detection: 7652 #endif /* CONFIG_PCI_IOV */ 7653 netif_device_detach(netdev); 7654 7655 if (state == pci_channel_io_perm_failure) 7656 return PCI_ERS_RESULT_DISCONNECT; 7657 7658 if (netif_running(netdev)) 7659 ixgbe_down(adapter); 7660 pci_disable_device(pdev); 7661 7662 /* Request a slot reset. */ 7663 return PCI_ERS_RESULT_NEED_RESET; 7664 } 7665 7666 /** 7667 * ixgbe_io_slot_reset - called after the pci bus has been reset. 7668 * @pdev: Pointer to PCI device 7669 * 7670 * Restart the card from scratch, as if from a cold-boot. 7671 */ 7672 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev) 7673 { 7674 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7675 pci_ers_result_t result; 7676 int err; 7677 7678 if (pci_enable_device_mem(pdev)) { 7679 e_err(probe, "Cannot re-enable PCI device after reset.\n"); 7680 result = PCI_ERS_RESULT_DISCONNECT; 7681 } else { 7682 pci_set_master(pdev); 7683 pci_restore_state(pdev); 7684 pci_save_state(pdev); 7685 7686 pci_wake_from_d3(pdev, false); 7687 7688 ixgbe_reset(adapter); 7689 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0); 7690 result = PCI_ERS_RESULT_RECOVERED; 7691 } 7692 7693 err = pci_cleanup_aer_uncorrect_error_status(pdev); 7694 if (err) { 7695 e_dev_err("pci_cleanup_aer_uncorrect_error_status " 7696 "failed 0x%0x\n", err); 7697 /* non-fatal, continue */ 7698 } 7699 7700 return result; 7701 } 7702 7703 /** 7704 * ixgbe_io_resume - called when traffic can start flowing again. 7705 * @pdev: Pointer to PCI device 7706 * 7707 * This callback is called when the error recovery driver tells us that 7708 * its OK to resume normal operation. 7709 */ 7710 static void ixgbe_io_resume(struct pci_dev *pdev) 7711 { 7712 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev); 7713 struct net_device *netdev = adapter->netdev; 7714 7715 #ifdef CONFIG_PCI_IOV 7716 if (adapter->vferr_refcount) { 7717 e_info(drv, "Resuming after VF err\n"); 7718 adapter->vferr_refcount--; 7719 return; 7720 } 7721 7722 #endif 7723 if (netif_running(netdev)) 7724 ixgbe_up(adapter); 7725 7726 netif_device_attach(netdev); 7727 } 7728 7729 static const struct pci_error_handlers ixgbe_err_handler = { 7730 .error_detected = ixgbe_io_error_detected, 7731 .slot_reset = ixgbe_io_slot_reset, 7732 .resume = ixgbe_io_resume, 7733 }; 7734 7735 static struct pci_driver ixgbe_driver = { 7736 .name = ixgbe_driver_name, 7737 .id_table = ixgbe_pci_tbl, 7738 .probe = ixgbe_probe, 7739 .remove = __devexit_p(ixgbe_remove), 7740 #ifdef CONFIG_PM 7741 .suspend = ixgbe_suspend, 7742 .resume = ixgbe_resume, 7743 #endif 7744 .shutdown = ixgbe_shutdown, 7745 .err_handler = &ixgbe_err_handler 7746 }; 7747 7748 /** 7749 * ixgbe_init_module - Driver Registration Routine 7750 * 7751 * ixgbe_init_module is the first routine called when the driver is 7752 * loaded. All it does is register with the PCI subsystem. 7753 **/ 7754 static int __init ixgbe_init_module(void) 7755 { 7756 int ret; 7757 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version); 7758 pr_info("%s\n", ixgbe_copyright); 7759 7760 #ifdef CONFIG_DEBUG_FS 7761 ixgbe_dbg_init(); 7762 #endif /* CONFIG_DEBUG_FS */ 7763 7764 #ifdef CONFIG_IXGBE_DCA 7765 dca_register_notify(&dca_notifier); 7766 #endif 7767 7768 ret = pci_register_driver(&ixgbe_driver); 7769 return ret; 7770 } 7771 7772 module_init(ixgbe_init_module); 7773 7774 /** 7775 * ixgbe_exit_module - Driver Exit Cleanup Routine 7776 * 7777 * ixgbe_exit_module is called just before the driver is removed 7778 * from memory. 7779 **/ 7780 static void __exit ixgbe_exit_module(void) 7781 { 7782 #ifdef CONFIG_IXGBE_DCA 7783 dca_unregister_notify(&dca_notifier); 7784 #endif 7785 pci_unregister_driver(&ixgbe_driver); 7786 7787 #ifdef CONFIG_DEBUG_FS 7788 ixgbe_dbg_exit(); 7789 #endif /* CONFIG_DEBUG_FS */ 7790 7791 rcu_barrier(); /* Wait for completion of call_rcu()'s */ 7792 } 7793 7794 #ifdef CONFIG_IXGBE_DCA 7795 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, 7796 void *p) 7797 { 7798 int ret_val; 7799 7800 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event, 7801 __ixgbe_notify_dca); 7802 7803 return ret_val ? NOTIFY_BAD : NOTIFY_DONE; 7804 } 7805 7806 #endif /* CONFIG_IXGBE_DCA */ 7807 7808 module_exit(ixgbe_exit_module); 7809 7810 /* ixgbe_main.c */ 7811