1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 21 22 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 23 24 static int debug = -1; 25 26 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 27 MODULE_DESCRIPTION(DRV_SUMMARY); 28 MODULE_LICENSE("GPL v2"); 29 module_param(debug, int, 0); 30 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 31 32 char igc_driver_name[] = "igc"; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 51 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base }, 52 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base }, 53 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base }, 54 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base }, 55 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base }, 56 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 57 /* required last entry */ 58 {0, } 59 }; 60 61 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 62 63 enum latency_range { 64 lowest_latency = 0, 65 low_latency = 1, 66 bulk_latency = 2, 67 latency_invalid = 255 68 }; 69 70 void igc_reset(struct igc_adapter *adapter) 71 { 72 struct net_device *dev = adapter->netdev; 73 struct igc_hw *hw = &adapter->hw; 74 struct igc_fc_info *fc = &hw->fc; 75 u32 pba, hwm; 76 77 /* Repartition PBA for greater than 9k MTU if required */ 78 pba = IGC_PBA_34K; 79 80 /* flow control settings 81 * The high water mark must be low enough to fit one full frame 82 * after transmitting the pause frame. As such we must have enough 83 * space to allow for us to complete our current transmit and then 84 * receive the frame that is in progress from the link partner. 85 * Set it to: 86 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 87 */ 88 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 89 90 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 91 fc->low_water = fc->high_water - 16; 92 fc->pause_time = 0xFFFF; 93 fc->send_xon = 1; 94 fc->current_mode = fc->requested_mode; 95 96 hw->mac.ops.reset_hw(hw); 97 98 if (hw->mac.ops.init_hw(hw)) 99 netdev_err(dev, "Error on hardware initialization\n"); 100 101 /* Re-establish EEE setting */ 102 igc_set_eee_i225(hw, true, true, true); 103 104 if (!netif_running(adapter->netdev)) 105 igc_power_down_phy_copper_base(&adapter->hw); 106 107 /* Re-enable PTP, where applicable. */ 108 igc_ptp_reset(adapter); 109 110 /* Re-enable TSN offloading, where applicable. */ 111 igc_tsn_offload_apply(adapter); 112 113 igc_get_phy_info(hw); 114 } 115 116 /** 117 * igc_power_up_link - Power up the phy link 118 * @adapter: address of board private structure 119 */ 120 static void igc_power_up_link(struct igc_adapter *adapter) 121 { 122 igc_reset_phy(&adapter->hw); 123 124 igc_power_up_phy_copper(&adapter->hw); 125 126 igc_setup_link(&adapter->hw); 127 } 128 129 /** 130 * igc_release_hw_control - release control of the h/w to f/w 131 * @adapter: address of board private structure 132 * 133 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 134 * For ASF and Pass Through versions of f/w this means that the 135 * driver is no longer loaded. 136 */ 137 static void igc_release_hw_control(struct igc_adapter *adapter) 138 { 139 struct igc_hw *hw = &adapter->hw; 140 u32 ctrl_ext; 141 142 /* Let firmware take over control of h/w */ 143 ctrl_ext = rd32(IGC_CTRL_EXT); 144 wr32(IGC_CTRL_EXT, 145 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 146 } 147 148 /** 149 * igc_get_hw_control - get control of the h/w from f/w 150 * @adapter: address of board private structure 151 * 152 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 153 * For ASF and Pass Through versions of f/w this means that 154 * the driver is loaded. 155 */ 156 static void igc_get_hw_control(struct igc_adapter *adapter) 157 { 158 struct igc_hw *hw = &adapter->hw; 159 u32 ctrl_ext; 160 161 /* Let firmware know the driver has taken over */ 162 ctrl_ext = rd32(IGC_CTRL_EXT); 163 wr32(IGC_CTRL_EXT, 164 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 165 } 166 167 /** 168 * igc_clean_tx_ring - Free Tx Buffers 169 * @tx_ring: ring to be cleaned 170 */ 171 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 172 { 173 u16 i = tx_ring->next_to_clean; 174 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 175 176 while (i != tx_ring->next_to_use) { 177 union igc_adv_tx_desc *eop_desc, *tx_desc; 178 179 /* Free all the Tx ring sk_buffs */ 180 dev_kfree_skb_any(tx_buffer->skb); 181 182 /* unmap skb header data */ 183 dma_unmap_single(tx_ring->dev, 184 dma_unmap_addr(tx_buffer, dma), 185 dma_unmap_len(tx_buffer, len), 186 DMA_TO_DEVICE); 187 188 /* check for eop_desc to determine the end of the packet */ 189 eop_desc = tx_buffer->next_to_watch; 190 tx_desc = IGC_TX_DESC(tx_ring, i); 191 192 /* unmap remaining buffers */ 193 while (tx_desc != eop_desc) { 194 tx_buffer++; 195 tx_desc++; 196 i++; 197 if (unlikely(i == tx_ring->count)) { 198 i = 0; 199 tx_buffer = tx_ring->tx_buffer_info; 200 tx_desc = IGC_TX_DESC(tx_ring, 0); 201 } 202 203 /* unmap any remaining paged data */ 204 if (dma_unmap_len(tx_buffer, len)) 205 dma_unmap_page(tx_ring->dev, 206 dma_unmap_addr(tx_buffer, dma), 207 dma_unmap_len(tx_buffer, len), 208 DMA_TO_DEVICE); 209 } 210 211 /* move us one more past the eop_desc for start of next pkt */ 212 tx_buffer++; 213 i++; 214 if (unlikely(i == tx_ring->count)) { 215 i = 0; 216 tx_buffer = tx_ring->tx_buffer_info; 217 } 218 } 219 220 /* reset BQL for queue */ 221 netdev_tx_reset_queue(txring_txq(tx_ring)); 222 223 /* reset next_to_use and next_to_clean */ 224 tx_ring->next_to_use = 0; 225 tx_ring->next_to_clean = 0; 226 } 227 228 /** 229 * igc_free_tx_resources - Free Tx Resources per Queue 230 * @tx_ring: Tx descriptor ring for a specific queue 231 * 232 * Free all transmit software resources 233 */ 234 void igc_free_tx_resources(struct igc_ring *tx_ring) 235 { 236 igc_clean_tx_ring(tx_ring); 237 238 vfree(tx_ring->tx_buffer_info); 239 tx_ring->tx_buffer_info = NULL; 240 241 /* if not set, then don't free */ 242 if (!tx_ring->desc) 243 return; 244 245 dma_free_coherent(tx_ring->dev, tx_ring->size, 246 tx_ring->desc, tx_ring->dma); 247 248 tx_ring->desc = NULL; 249 } 250 251 /** 252 * igc_free_all_tx_resources - Free Tx Resources for All Queues 253 * @adapter: board private structure 254 * 255 * Free all transmit software resources 256 */ 257 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 258 { 259 int i; 260 261 for (i = 0; i < adapter->num_tx_queues; i++) 262 igc_free_tx_resources(adapter->tx_ring[i]); 263 } 264 265 /** 266 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 267 * @adapter: board private structure 268 */ 269 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 270 { 271 int i; 272 273 for (i = 0; i < adapter->num_tx_queues; i++) 274 if (adapter->tx_ring[i]) 275 igc_clean_tx_ring(adapter->tx_ring[i]); 276 } 277 278 /** 279 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 280 * @tx_ring: tx descriptor ring (for a specific queue) to setup 281 * 282 * Return 0 on success, negative on failure 283 */ 284 int igc_setup_tx_resources(struct igc_ring *tx_ring) 285 { 286 struct net_device *ndev = tx_ring->netdev; 287 struct device *dev = tx_ring->dev; 288 int size = 0; 289 290 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 291 tx_ring->tx_buffer_info = vzalloc(size); 292 if (!tx_ring->tx_buffer_info) 293 goto err; 294 295 /* round up to nearest 4K */ 296 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 297 tx_ring->size = ALIGN(tx_ring->size, 4096); 298 299 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 300 &tx_ring->dma, GFP_KERNEL); 301 302 if (!tx_ring->desc) 303 goto err; 304 305 tx_ring->next_to_use = 0; 306 tx_ring->next_to_clean = 0; 307 308 return 0; 309 310 err: 311 vfree(tx_ring->tx_buffer_info); 312 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 313 return -ENOMEM; 314 } 315 316 /** 317 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 318 * @adapter: board private structure 319 * 320 * Return 0 on success, negative on failure 321 */ 322 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 323 { 324 struct net_device *dev = adapter->netdev; 325 int i, err = 0; 326 327 for (i = 0; i < adapter->num_tx_queues; i++) { 328 err = igc_setup_tx_resources(adapter->tx_ring[i]); 329 if (err) { 330 netdev_err(dev, "Error on Tx queue %u setup\n", i); 331 for (i--; i >= 0; i--) 332 igc_free_tx_resources(adapter->tx_ring[i]); 333 break; 334 } 335 } 336 337 return err; 338 } 339 340 /** 341 * igc_clean_rx_ring - Free Rx Buffers per Queue 342 * @rx_ring: ring to free buffers from 343 */ 344 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 345 { 346 u16 i = rx_ring->next_to_clean; 347 348 dev_kfree_skb(rx_ring->skb); 349 rx_ring->skb = NULL; 350 351 /* Free all the Rx ring sk_buffs */ 352 while (i != rx_ring->next_to_alloc) { 353 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 354 355 /* Invalidate cache lines that may have been written to by 356 * device so that we avoid corrupting memory. 357 */ 358 dma_sync_single_range_for_cpu(rx_ring->dev, 359 buffer_info->dma, 360 buffer_info->page_offset, 361 igc_rx_bufsz(rx_ring), 362 DMA_FROM_DEVICE); 363 364 /* free resources associated with mapping */ 365 dma_unmap_page_attrs(rx_ring->dev, 366 buffer_info->dma, 367 igc_rx_pg_size(rx_ring), 368 DMA_FROM_DEVICE, 369 IGC_RX_DMA_ATTR); 370 __page_frag_cache_drain(buffer_info->page, 371 buffer_info->pagecnt_bias); 372 373 i++; 374 if (i == rx_ring->count) 375 i = 0; 376 } 377 378 rx_ring->next_to_alloc = 0; 379 rx_ring->next_to_clean = 0; 380 rx_ring->next_to_use = 0; 381 } 382 383 /** 384 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 385 * @adapter: board private structure 386 */ 387 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 388 { 389 int i; 390 391 for (i = 0; i < adapter->num_rx_queues; i++) 392 if (adapter->rx_ring[i]) 393 igc_clean_rx_ring(adapter->rx_ring[i]); 394 } 395 396 /** 397 * igc_free_rx_resources - Free Rx Resources 398 * @rx_ring: ring to clean the resources from 399 * 400 * Free all receive software resources 401 */ 402 void igc_free_rx_resources(struct igc_ring *rx_ring) 403 { 404 igc_clean_rx_ring(rx_ring); 405 406 vfree(rx_ring->rx_buffer_info); 407 rx_ring->rx_buffer_info = NULL; 408 409 /* if not set, then don't free */ 410 if (!rx_ring->desc) 411 return; 412 413 dma_free_coherent(rx_ring->dev, rx_ring->size, 414 rx_ring->desc, rx_ring->dma); 415 416 rx_ring->desc = NULL; 417 } 418 419 /** 420 * igc_free_all_rx_resources - Free Rx Resources for All Queues 421 * @adapter: board private structure 422 * 423 * Free all receive software resources 424 */ 425 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 426 { 427 int i; 428 429 for (i = 0; i < adapter->num_rx_queues; i++) 430 igc_free_rx_resources(adapter->rx_ring[i]); 431 } 432 433 /** 434 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 435 * @rx_ring: rx descriptor ring (for a specific queue) to setup 436 * 437 * Returns 0 on success, negative on failure 438 */ 439 int igc_setup_rx_resources(struct igc_ring *rx_ring) 440 { 441 struct net_device *ndev = rx_ring->netdev; 442 struct device *dev = rx_ring->dev; 443 int size, desc_len; 444 445 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 446 rx_ring->rx_buffer_info = vzalloc(size); 447 if (!rx_ring->rx_buffer_info) 448 goto err; 449 450 desc_len = sizeof(union igc_adv_rx_desc); 451 452 /* Round up to nearest 4K */ 453 rx_ring->size = rx_ring->count * desc_len; 454 rx_ring->size = ALIGN(rx_ring->size, 4096); 455 456 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 457 &rx_ring->dma, GFP_KERNEL); 458 459 if (!rx_ring->desc) 460 goto err; 461 462 rx_ring->next_to_alloc = 0; 463 rx_ring->next_to_clean = 0; 464 rx_ring->next_to_use = 0; 465 466 return 0; 467 468 err: 469 vfree(rx_ring->rx_buffer_info); 470 rx_ring->rx_buffer_info = NULL; 471 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 472 return -ENOMEM; 473 } 474 475 /** 476 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 477 * (Descriptors) for all queues 478 * @adapter: board private structure 479 * 480 * Return 0 on success, negative on failure 481 */ 482 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 483 { 484 struct net_device *dev = adapter->netdev; 485 int i, err = 0; 486 487 for (i = 0; i < adapter->num_rx_queues; i++) { 488 err = igc_setup_rx_resources(adapter->rx_ring[i]); 489 if (err) { 490 netdev_err(dev, "Error on Rx queue %u setup\n", i); 491 for (i--; i >= 0; i--) 492 igc_free_rx_resources(adapter->rx_ring[i]); 493 break; 494 } 495 } 496 497 return err; 498 } 499 500 /** 501 * igc_configure_rx_ring - Configure a receive ring after Reset 502 * @adapter: board private structure 503 * @ring: receive ring to be configured 504 * 505 * Configure the Rx unit of the MAC after a reset. 506 */ 507 static void igc_configure_rx_ring(struct igc_adapter *adapter, 508 struct igc_ring *ring) 509 { 510 struct igc_hw *hw = &adapter->hw; 511 union igc_adv_rx_desc *rx_desc; 512 int reg_idx = ring->reg_idx; 513 u32 srrctl = 0, rxdctl = 0; 514 u64 rdba = ring->dma; 515 516 /* disable the queue */ 517 wr32(IGC_RXDCTL(reg_idx), 0); 518 519 /* Set DMA base address registers */ 520 wr32(IGC_RDBAL(reg_idx), 521 rdba & 0x00000000ffffffffULL); 522 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 523 wr32(IGC_RDLEN(reg_idx), 524 ring->count * sizeof(union igc_adv_rx_desc)); 525 526 /* initialize head and tail */ 527 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 528 wr32(IGC_RDH(reg_idx), 0); 529 writel(0, ring->tail); 530 531 /* reset next-to- use/clean to place SW in sync with hardware */ 532 ring->next_to_clean = 0; 533 ring->next_to_use = 0; 534 535 /* set descriptor configuration */ 536 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 537 if (ring_uses_large_buffer(ring)) 538 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 539 else 540 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 541 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 542 543 wr32(IGC_SRRCTL(reg_idx), srrctl); 544 545 rxdctl |= IGC_RX_PTHRESH; 546 rxdctl |= IGC_RX_HTHRESH << 8; 547 rxdctl |= IGC_RX_WTHRESH << 16; 548 549 /* initialize rx_buffer_info */ 550 memset(ring->rx_buffer_info, 0, 551 sizeof(struct igc_rx_buffer) * ring->count); 552 553 /* initialize Rx descriptor 0 */ 554 rx_desc = IGC_RX_DESC(ring, 0); 555 rx_desc->wb.upper.length = 0; 556 557 /* enable receive descriptor fetching */ 558 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 559 560 wr32(IGC_RXDCTL(reg_idx), rxdctl); 561 } 562 563 /** 564 * igc_configure_rx - Configure receive Unit after Reset 565 * @adapter: board private structure 566 * 567 * Configure the Rx unit of the MAC after a reset. 568 */ 569 static void igc_configure_rx(struct igc_adapter *adapter) 570 { 571 int i; 572 573 /* Setup the HW Rx Head and Tail Descriptor Pointers and 574 * the Base and Length of the Rx Descriptor Ring 575 */ 576 for (i = 0; i < adapter->num_rx_queues; i++) 577 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 578 } 579 580 /** 581 * igc_configure_tx_ring - Configure transmit ring after Reset 582 * @adapter: board private structure 583 * @ring: tx ring to configure 584 * 585 * Configure a transmit ring after a reset. 586 */ 587 static void igc_configure_tx_ring(struct igc_adapter *adapter, 588 struct igc_ring *ring) 589 { 590 struct igc_hw *hw = &adapter->hw; 591 int reg_idx = ring->reg_idx; 592 u64 tdba = ring->dma; 593 u32 txdctl = 0; 594 595 /* disable the queue */ 596 wr32(IGC_TXDCTL(reg_idx), 0); 597 wrfl(); 598 mdelay(10); 599 600 wr32(IGC_TDLEN(reg_idx), 601 ring->count * sizeof(union igc_adv_tx_desc)); 602 wr32(IGC_TDBAL(reg_idx), 603 tdba & 0x00000000ffffffffULL); 604 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 605 606 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 607 wr32(IGC_TDH(reg_idx), 0); 608 writel(0, ring->tail); 609 610 txdctl |= IGC_TX_PTHRESH; 611 txdctl |= IGC_TX_HTHRESH << 8; 612 txdctl |= IGC_TX_WTHRESH << 16; 613 614 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 615 wr32(IGC_TXDCTL(reg_idx), txdctl); 616 } 617 618 /** 619 * igc_configure_tx - Configure transmit Unit after Reset 620 * @adapter: board private structure 621 * 622 * Configure the Tx unit of the MAC after a reset. 623 */ 624 static void igc_configure_tx(struct igc_adapter *adapter) 625 { 626 int i; 627 628 for (i = 0; i < adapter->num_tx_queues; i++) 629 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 630 } 631 632 /** 633 * igc_setup_mrqc - configure the multiple receive queue control registers 634 * @adapter: Board private structure 635 */ 636 static void igc_setup_mrqc(struct igc_adapter *adapter) 637 { 638 struct igc_hw *hw = &adapter->hw; 639 u32 j, num_rx_queues; 640 u32 mrqc, rxcsum; 641 u32 rss_key[10]; 642 643 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 644 for (j = 0; j < 10; j++) 645 wr32(IGC_RSSRK(j), rss_key[j]); 646 647 num_rx_queues = adapter->rss_queues; 648 649 if (adapter->rss_indir_tbl_init != num_rx_queues) { 650 for (j = 0; j < IGC_RETA_SIZE; j++) 651 adapter->rss_indir_tbl[j] = 652 (j * num_rx_queues) / IGC_RETA_SIZE; 653 adapter->rss_indir_tbl_init = num_rx_queues; 654 } 655 igc_write_rss_indir_tbl(adapter); 656 657 /* Disable raw packet checksumming so that RSS hash is placed in 658 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 659 * offloads as they are enabled by default 660 */ 661 rxcsum = rd32(IGC_RXCSUM); 662 rxcsum |= IGC_RXCSUM_PCSD; 663 664 /* Enable Receive Checksum Offload for SCTP */ 665 rxcsum |= IGC_RXCSUM_CRCOFL; 666 667 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 668 wr32(IGC_RXCSUM, rxcsum); 669 670 /* Generate RSS hash based on packet types, TCP/UDP 671 * port numbers and/or IPv4/v6 src and dst addresses 672 */ 673 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 674 IGC_MRQC_RSS_FIELD_IPV4_TCP | 675 IGC_MRQC_RSS_FIELD_IPV6 | 676 IGC_MRQC_RSS_FIELD_IPV6_TCP | 677 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 678 679 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 680 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 681 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 682 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 683 684 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 685 686 wr32(IGC_MRQC, mrqc); 687 } 688 689 /** 690 * igc_setup_rctl - configure the receive control registers 691 * @adapter: Board private structure 692 */ 693 static void igc_setup_rctl(struct igc_adapter *adapter) 694 { 695 struct igc_hw *hw = &adapter->hw; 696 u32 rctl; 697 698 rctl = rd32(IGC_RCTL); 699 700 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 701 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 702 703 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 704 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 705 706 /* enable stripping of CRC. Newer features require 707 * that the HW strips the CRC. 708 */ 709 rctl |= IGC_RCTL_SECRC; 710 711 /* disable store bad packets and clear size bits. */ 712 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 713 714 /* enable LPE to allow for reception of jumbo frames */ 715 rctl |= IGC_RCTL_LPE; 716 717 /* disable queue 0 to prevent tail write w/o re-config */ 718 wr32(IGC_RXDCTL(0), 0); 719 720 /* This is useful for sniffing bad packets. */ 721 if (adapter->netdev->features & NETIF_F_RXALL) { 722 /* UPE and MPE will be handled by normal PROMISC logic 723 * in set_rx_mode 724 */ 725 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 726 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 727 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 728 729 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 730 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 731 } 732 733 wr32(IGC_RCTL, rctl); 734 } 735 736 /** 737 * igc_setup_tctl - configure the transmit control registers 738 * @adapter: Board private structure 739 */ 740 static void igc_setup_tctl(struct igc_adapter *adapter) 741 { 742 struct igc_hw *hw = &adapter->hw; 743 u32 tctl; 744 745 /* disable queue 0 which icould be enabled by default */ 746 wr32(IGC_TXDCTL(0), 0); 747 748 /* Program the Transmit Control Register */ 749 tctl = rd32(IGC_TCTL); 750 tctl &= ~IGC_TCTL_CT; 751 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 752 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 753 754 /* Enable transmits */ 755 tctl |= IGC_TCTL_EN; 756 757 wr32(IGC_TCTL, tctl); 758 } 759 760 /** 761 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 762 * @adapter: Pointer to adapter where the filter should be set 763 * @index: Filter index 764 * @type: MAC address filter type (source or destination) 765 * @addr: MAC address 766 * @queue: If non-negative, queue assignment feature is enabled and frames 767 * matching the filter are enqueued onto 'queue'. Otherwise, queue 768 * assignment is disabled. 769 */ 770 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 771 enum igc_mac_filter_type type, 772 const u8 *addr, int queue) 773 { 774 struct net_device *dev = adapter->netdev; 775 struct igc_hw *hw = &adapter->hw; 776 u32 ral, rah; 777 778 if (WARN_ON(index >= hw->mac.rar_entry_count)) 779 return; 780 781 ral = le32_to_cpup((__le32 *)(addr)); 782 rah = le16_to_cpup((__le16 *)(addr + 4)); 783 784 if (type == IGC_MAC_FILTER_TYPE_SRC) { 785 rah &= ~IGC_RAH_ASEL_MASK; 786 rah |= IGC_RAH_ASEL_SRC_ADDR; 787 } 788 789 if (queue >= 0) { 790 rah &= ~IGC_RAH_QSEL_MASK; 791 rah |= (queue << IGC_RAH_QSEL_SHIFT); 792 rah |= IGC_RAH_QSEL_ENABLE; 793 } 794 795 rah |= IGC_RAH_AV; 796 797 wr32(IGC_RAL(index), ral); 798 wr32(IGC_RAH(index), rah); 799 800 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 801 } 802 803 /** 804 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 805 * @adapter: Pointer to adapter where the filter should be cleared 806 * @index: Filter index 807 */ 808 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 809 { 810 struct net_device *dev = adapter->netdev; 811 struct igc_hw *hw = &adapter->hw; 812 813 if (WARN_ON(index >= hw->mac.rar_entry_count)) 814 return; 815 816 wr32(IGC_RAL(index), 0); 817 wr32(IGC_RAH(index), 0); 818 819 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 820 } 821 822 /* Set default MAC address for the PF in the first RAR entry */ 823 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 824 { 825 struct net_device *dev = adapter->netdev; 826 u8 *addr = adapter->hw.mac.addr; 827 828 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 829 830 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 831 } 832 833 /** 834 * igc_set_mac - Change the Ethernet Address of the NIC 835 * @netdev: network interface device structure 836 * @p: pointer to an address structure 837 * 838 * Returns 0 on success, negative on failure 839 */ 840 static int igc_set_mac(struct net_device *netdev, void *p) 841 { 842 struct igc_adapter *adapter = netdev_priv(netdev); 843 struct igc_hw *hw = &adapter->hw; 844 struct sockaddr *addr = p; 845 846 if (!is_valid_ether_addr(addr->sa_data)) 847 return -EADDRNOTAVAIL; 848 849 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 850 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 851 852 /* set the correct pool for the new PF MAC address in entry 0 */ 853 igc_set_default_mac_filter(adapter); 854 855 return 0; 856 } 857 858 /** 859 * igc_write_mc_addr_list - write multicast addresses to MTA 860 * @netdev: network interface device structure 861 * 862 * Writes multicast address list to the MTA hash table. 863 * Returns: -ENOMEM on failure 864 * 0 on no addresses written 865 * X on writing X addresses to MTA 866 **/ 867 static int igc_write_mc_addr_list(struct net_device *netdev) 868 { 869 struct igc_adapter *adapter = netdev_priv(netdev); 870 struct igc_hw *hw = &adapter->hw; 871 struct netdev_hw_addr *ha; 872 u8 *mta_list; 873 int i; 874 875 if (netdev_mc_empty(netdev)) { 876 /* nothing to program, so clear mc list */ 877 igc_update_mc_addr_list(hw, NULL, 0); 878 return 0; 879 } 880 881 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 882 if (!mta_list) 883 return -ENOMEM; 884 885 /* The shared function expects a packed array of only addresses. */ 886 i = 0; 887 netdev_for_each_mc_addr(ha, netdev) 888 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 889 890 igc_update_mc_addr_list(hw, mta_list, i); 891 kfree(mta_list); 892 893 return netdev_mc_count(netdev); 894 } 895 896 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 897 { 898 ktime_t cycle_time = adapter->cycle_time; 899 ktime_t base_time = adapter->base_time; 900 u32 launchtime; 901 902 /* FIXME: when using ETF together with taprio, we may have a 903 * case where 'delta' is larger than the cycle_time, this may 904 * cause problems if we don't read the current value of 905 * IGC_BASET, as the value writen into the launchtime 906 * descriptor field may be misinterpreted. 907 */ 908 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 909 910 return cpu_to_le32(launchtime); 911 } 912 913 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 914 struct igc_tx_buffer *first, 915 u32 vlan_macip_lens, u32 type_tucmd, 916 u32 mss_l4len_idx) 917 { 918 struct igc_adv_tx_context_desc *context_desc; 919 u16 i = tx_ring->next_to_use; 920 921 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 922 923 i++; 924 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 925 926 /* set bits to identify this as an advanced context descriptor */ 927 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 928 929 /* For i225, context index must be unique per ring. */ 930 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 931 mss_l4len_idx |= tx_ring->reg_idx << 4; 932 933 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 934 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 935 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 936 937 /* We assume there is always a valid Tx time available. Invalid times 938 * should have been handled by the upper layers. 939 */ 940 if (tx_ring->launchtime_enable) { 941 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 942 ktime_t txtime = first->skb->tstamp; 943 944 first->skb->tstamp = ktime_set(0, 0); 945 context_desc->launch_time = igc_tx_launchtime(adapter, 946 txtime); 947 } else { 948 context_desc->launch_time = 0; 949 } 950 } 951 952 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 953 { 954 unsigned int offset = 0; 955 956 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 957 958 return offset == skb_checksum_start_offset(skb); 959 } 960 961 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 962 { 963 struct sk_buff *skb = first->skb; 964 u32 vlan_macip_lens = 0; 965 u32 type_tucmd = 0; 966 967 if (skb->ip_summed != CHECKSUM_PARTIAL) { 968 csum_failed: 969 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 970 !tx_ring->launchtime_enable) 971 return; 972 goto no_csum; 973 } 974 975 switch (skb->csum_offset) { 976 case offsetof(struct tcphdr, check): 977 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 978 fallthrough; 979 case offsetof(struct udphdr, check): 980 break; 981 case offsetof(struct sctphdr, checksum): 982 /* validate that this is actually an SCTP request */ 983 if ((first->protocol == htons(ETH_P_IP) && 984 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 985 (first->protocol == htons(ETH_P_IPV6) && 986 igc_ipv6_csum_is_sctp(skb))) { 987 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 988 break; 989 } 990 fallthrough; 991 default: 992 skb_checksum_help(skb); 993 goto csum_failed; 994 } 995 996 /* update TX checksum flag */ 997 first->tx_flags |= IGC_TX_FLAGS_CSUM; 998 vlan_macip_lens = skb_checksum_start_offset(skb) - 999 skb_network_offset(skb); 1000 no_csum: 1001 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 1002 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1003 1004 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 1005 } 1006 1007 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1008 { 1009 struct net_device *netdev = tx_ring->netdev; 1010 1011 netif_stop_subqueue(netdev, tx_ring->queue_index); 1012 1013 /* memory barriier comment */ 1014 smp_mb(); 1015 1016 /* We need to check again in a case another CPU has just 1017 * made room available. 1018 */ 1019 if (igc_desc_unused(tx_ring) < size) 1020 return -EBUSY; 1021 1022 /* A reprieve! */ 1023 netif_wake_subqueue(netdev, tx_ring->queue_index); 1024 1025 u64_stats_update_begin(&tx_ring->tx_syncp2); 1026 tx_ring->tx_stats.restart_queue2++; 1027 u64_stats_update_end(&tx_ring->tx_syncp2); 1028 1029 return 0; 1030 } 1031 1032 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1033 { 1034 if (igc_desc_unused(tx_ring) >= size) 1035 return 0; 1036 return __igc_maybe_stop_tx(tx_ring, size); 1037 } 1038 1039 #define IGC_SET_FLAG(_input, _flag, _result) \ 1040 (((_flag) <= (_result)) ? \ 1041 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1042 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1043 1044 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1045 { 1046 /* set type for advanced descriptor with frame checksum insertion */ 1047 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1048 IGC_ADVTXD_DCMD_DEXT | 1049 IGC_ADVTXD_DCMD_IFCS; 1050 1051 /* set segmentation bits for TSO */ 1052 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1053 (IGC_ADVTXD_DCMD_TSE)); 1054 1055 /* set timestamp bit if present */ 1056 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1057 (IGC_ADVTXD_MAC_TSTAMP)); 1058 1059 return cmd_type; 1060 } 1061 1062 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1063 union igc_adv_tx_desc *tx_desc, 1064 u32 tx_flags, unsigned int paylen) 1065 { 1066 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1067 1068 /* insert L4 checksum */ 1069 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1070 ((IGC_TXD_POPTS_TXSM << 8) / 1071 IGC_TX_FLAGS_CSUM); 1072 1073 /* insert IPv4 checksum */ 1074 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1075 (((IGC_TXD_POPTS_IXSM << 8)) / 1076 IGC_TX_FLAGS_IPV4); 1077 1078 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1079 } 1080 1081 static int igc_tx_map(struct igc_ring *tx_ring, 1082 struct igc_tx_buffer *first, 1083 const u8 hdr_len) 1084 { 1085 struct sk_buff *skb = first->skb; 1086 struct igc_tx_buffer *tx_buffer; 1087 union igc_adv_tx_desc *tx_desc; 1088 u32 tx_flags = first->tx_flags; 1089 skb_frag_t *frag; 1090 u16 i = tx_ring->next_to_use; 1091 unsigned int data_len, size; 1092 dma_addr_t dma; 1093 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1094 1095 tx_desc = IGC_TX_DESC(tx_ring, i); 1096 1097 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1098 1099 size = skb_headlen(skb); 1100 data_len = skb->data_len; 1101 1102 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1103 1104 tx_buffer = first; 1105 1106 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1107 if (dma_mapping_error(tx_ring->dev, dma)) 1108 goto dma_error; 1109 1110 /* record length, and DMA address */ 1111 dma_unmap_len_set(tx_buffer, len, size); 1112 dma_unmap_addr_set(tx_buffer, dma, dma); 1113 1114 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1115 1116 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1117 tx_desc->read.cmd_type_len = 1118 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1119 1120 i++; 1121 tx_desc++; 1122 if (i == tx_ring->count) { 1123 tx_desc = IGC_TX_DESC(tx_ring, 0); 1124 i = 0; 1125 } 1126 tx_desc->read.olinfo_status = 0; 1127 1128 dma += IGC_MAX_DATA_PER_TXD; 1129 size -= IGC_MAX_DATA_PER_TXD; 1130 1131 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1132 } 1133 1134 if (likely(!data_len)) 1135 break; 1136 1137 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1138 1139 i++; 1140 tx_desc++; 1141 if (i == tx_ring->count) { 1142 tx_desc = IGC_TX_DESC(tx_ring, 0); 1143 i = 0; 1144 } 1145 tx_desc->read.olinfo_status = 0; 1146 1147 size = skb_frag_size(frag); 1148 data_len -= size; 1149 1150 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1151 size, DMA_TO_DEVICE); 1152 1153 tx_buffer = &tx_ring->tx_buffer_info[i]; 1154 } 1155 1156 /* write last descriptor with RS and EOP bits */ 1157 cmd_type |= size | IGC_TXD_DCMD; 1158 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1159 1160 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1161 1162 /* set the timestamp */ 1163 first->time_stamp = jiffies; 1164 1165 skb_tx_timestamp(skb); 1166 1167 /* Force memory writes to complete before letting h/w know there 1168 * are new descriptors to fetch. (Only applicable for weak-ordered 1169 * memory model archs, such as IA-64). 1170 * 1171 * We also need this memory barrier to make certain all of the 1172 * status bits have been updated before next_to_watch is written. 1173 */ 1174 wmb(); 1175 1176 /* set next_to_watch value indicating a packet is present */ 1177 first->next_to_watch = tx_desc; 1178 1179 i++; 1180 if (i == tx_ring->count) 1181 i = 0; 1182 1183 tx_ring->next_to_use = i; 1184 1185 /* Make sure there is space in the ring for the next send. */ 1186 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1187 1188 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1189 writel(i, tx_ring->tail); 1190 } 1191 1192 return 0; 1193 dma_error: 1194 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1195 tx_buffer = &tx_ring->tx_buffer_info[i]; 1196 1197 /* clear dma mappings for failed tx_buffer_info map */ 1198 while (tx_buffer != first) { 1199 if (dma_unmap_len(tx_buffer, len)) 1200 dma_unmap_page(tx_ring->dev, 1201 dma_unmap_addr(tx_buffer, dma), 1202 dma_unmap_len(tx_buffer, len), 1203 DMA_TO_DEVICE); 1204 dma_unmap_len_set(tx_buffer, len, 0); 1205 1206 if (i-- == 0) 1207 i += tx_ring->count; 1208 tx_buffer = &tx_ring->tx_buffer_info[i]; 1209 } 1210 1211 if (dma_unmap_len(tx_buffer, len)) 1212 dma_unmap_single(tx_ring->dev, 1213 dma_unmap_addr(tx_buffer, dma), 1214 dma_unmap_len(tx_buffer, len), 1215 DMA_TO_DEVICE); 1216 dma_unmap_len_set(tx_buffer, len, 0); 1217 1218 dev_kfree_skb_any(tx_buffer->skb); 1219 tx_buffer->skb = NULL; 1220 1221 tx_ring->next_to_use = i; 1222 1223 return -1; 1224 } 1225 1226 static int igc_tso(struct igc_ring *tx_ring, 1227 struct igc_tx_buffer *first, 1228 u8 *hdr_len) 1229 { 1230 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1231 struct sk_buff *skb = first->skb; 1232 union { 1233 struct iphdr *v4; 1234 struct ipv6hdr *v6; 1235 unsigned char *hdr; 1236 } ip; 1237 union { 1238 struct tcphdr *tcp; 1239 struct udphdr *udp; 1240 unsigned char *hdr; 1241 } l4; 1242 u32 paylen, l4_offset; 1243 int err; 1244 1245 if (skb->ip_summed != CHECKSUM_PARTIAL) 1246 return 0; 1247 1248 if (!skb_is_gso(skb)) 1249 return 0; 1250 1251 err = skb_cow_head(skb, 0); 1252 if (err < 0) 1253 return err; 1254 1255 ip.hdr = skb_network_header(skb); 1256 l4.hdr = skb_checksum_start(skb); 1257 1258 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1259 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1260 1261 /* initialize outer IP header fields */ 1262 if (ip.v4->version == 4) { 1263 unsigned char *csum_start = skb_checksum_start(skb); 1264 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1265 1266 /* IP header will have to cancel out any data that 1267 * is not a part of the outer IP header 1268 */ 1269 ip.v4->check = csum_fold(csum_partial(trans_start, 1270 csum_start - trans_start, 1271 0)); 1272 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1273 1274 ip.v4->tot_len = 0; 1275 first->tx_flags |= IGC_TX_FLAGS_TSO | 1276 IGC_TX_FLAGS_CSUM | 1277 IGC_TX_FLAGS_IPV4; 1278 } else { 1279 ip.v6->payload_len = 0; 1280 first->tx_flags |= IGC_TX_FLAGS_TSO | 1281 IGC_TX_FLAGS_CSUM; 1282 } 1283 1284 /* determine offset of inner transport header */ 1285 l4_offset = l4.hdr - skb->data; 1286 1287 /* remove payload length from inner checksum */ 1288 paylen = skb->len - l4_offset; 1289 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1290 /* compute length of segmentation header */ 1291 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1292 csum_replace_by_diff(&l4.tcp->check, 1293 (__force __wsum)htonl(paylen)); 1294 } else { 1295 /* compute length of segmentation header */ 1296 *hdr_len = sizeof(*l4.udp) + l4_offset; 1297 csum_replace_by_diff(&l4.udp->check, 1298 (__force __wsum)htonl(paylen)); 1299 } 1300 1301 /* update gso size and bytecount with header size */ 1302 first->gso_segs = skb_shinfo(skb)->gso_segs; 1303 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1304 1305 /* MSS L4LEN IDX */ 1306 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1307 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1308 1309 /* VLAN MACLEN IPLEN */ 1310 vlan_macip_lens = l4.hdr - ip.hdr; 1311 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1312 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1313 1314 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1315 type_tucmd, mss_l4len_idx); 1316 1317 return 1; 1318 } 1319 1320 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1321 struct igc_ring *tx_ring) 1322 { 1323 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1324 __be16 protocol = vlan_get_protocol(skb); 1325 struct igc_tx_buffer *first; 1326 u32 tx_flags = 0; 1327 unsigned short f; 1328 u8 hdr_len = 0; 1329 int tso = 0; 1330 1331 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1332 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1333 * + 2 desc gap to keep tail from touching head, 1334 * + 1 desc for context descriptor, 1335 * otherwise try next time 1336 */ 1337 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1338 count += TXD_USE_COUNT(skb_frag_size( 1339 &skb_shinfo(skb)->frags[f])); 1340 1341 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1342 /* this is a hard error */ 1343 return NETDEV_TX_BUSY; 1344 } 1345 1346 /* record the location of the first descriptor for this packet */ 1347 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1348 first->skb = skb; 1349 first->bytecount = skb->len; 1350 first->gso_segs = 1; 1351 1352 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1353 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1354 1355 /* FIXME: add support for retrieving timestamps from 1356 * the other timer registers before skipping the 1357 * timestamping request. 1358 */ 1359 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1360 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1361 &adapter->state)) { 1362 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1363 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1364 1365 adapter->ptp_tx_skb = skb_get(skb); 1366 adapter->ptp_tx_start = jiffies; 1367 } else { 1368 adapter->tx_hwtstamp_skipped++; 1369 } 1370 } 1371 1372 /* record initial flags and protocol */ 1373 first->tx_flags = tx_flags; 1374 first->protocol = protocol; 1375 1376 tso = igc_tso(tx_ring, first, &hdr_len); 1377 if (tso < 0) 1378 goto out_drop; 1379 else if (!tso) 1380 igc_tx_csum(tx_ring, first); 1381 1382 igc_tx_map(tx_ring, first, hdr_len); 1383 1384 return NETDEV_TX_OK; 1385 1386 out_drop: 1387 dev_kfree_skb_any(first->skb); 1388 first->skb = NULL; 1389 1390 return NETDEV_TX_OK; 1391 } 1392 1393 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1394 struct sk_buff *skb) 1395 { 1396 unsigned int r_idx = skb->queue_mapping; 1397 1398 if (r_idx >= adapter->num_tx_queues) 1399 r_idx = r_idx % adapter->num_tx_queues; 1400 1401 return adapter->tx_ring[r_idx]; 1402 } 1403 1404 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1405 struct net_device *netdev) 1406 { 1407 struct igc_adapter *adapter = netdev_priv(netdev); 1408 1409 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1410 * in order to meet this minimum size requirement. 1411 */ 1412 if (skb->len < 17) { 1413 if (skb_padto(skb, 17)) 1414 return NETDEV_TX_OK; 1415 skb->len = 17; 1416 } 1417 1418 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1419 } 1420 1421 static void igc_rx_checksum(struct igc_ring *ring, 1422 union igc_adv_rx_desc *rx_desc, 1423 struct sk_buff *skb) 1424 { 1425 skb_checksum_none_assert(skb); 1426 1427 /* Ignore Checksum bit is set */ 1428 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1429 return; 1430 1431 /* Rx checksum disabled via ethtool */ 1432 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1433 return; 1434 1435 /* TCP/UDP checksum error bit is set */ 1436 if (igc_test_staterr(rx_desc, 1437 IGC_RXDEXT_STATERR_L4E | 1438 IGC_RXDEXT_STATERR_IPE)) { 1439 /* work around errata with sctp packets where the TCPE aka 1440 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1441 * packets (aka let the stack check the crc32c) 1442 */ 1443 if (!(skb->len == 60 && 1444 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1445 u64_stats_update_begin(&ring->rx_syncp); 1446 ring->rx_stats.csum_err++; 1447 u64_stats_update_end(&ring->rx_syncp); 1448 } 1449 /* let the stack verify checksum errors */ 1450 return; 1451 } 1452 /* It must be a TCP or UDP packet with a valid checksum */ 1453 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1454 IGC_RXD_STAT_UDPCS)) 1455 skb->ip_summed = CHECKSUM_UNNECESSARY; 1456 1457 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1458 le32_to_cpu(rx_desc->wb.upper.status_error)); 1459 } 1460 1461 static inline void igc_rx_hash(struct igc_ring *ring, 1462 union igc_adv_rx_desc *rx_desc, 1463 struct sk_buff *skb) 1464 { 1465 if (ring->netdev->features & NETIF_F_RXHASH) 1466 skb_set_hash(skb, 1467 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1468 PKT_HASH_TYPE_L3); 1469 } 1470 1471 /** 1472 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1473 * @rx_ring: rx descriptor ring packet is being transacted on 1474 * @rx_desc: pointer to the EOP Rx descriptor 1475 * @skb: pointer to current skb being populated 1476 * 1477 * This function checks the ring, descriptor, and packet information in order 1478 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1479 * skb. 1480 */ 1481 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1482 union igc_adv_rx_desc *rx_desc, 1483 struct sk_buff *skb) 1484 { 1485 igc_rx_hash(rx_ring, rx_desc, skb); 1486 1487 igc_rx_checksum(rx_ring, rx_desc, skb); 1488 1489 skb_record_rx_queue(skb, rx_ring->queue_index); 1490 1491 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1492 } 1493 1494 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1495 const unsigned int size) 1496 { 1497 struct igc_rx_buffer *rx_buffer; 1498 1499 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1500 prefetchw(rx_buffer->page); 1501 1502 /* we are reusing so sync this buffer for CPU use */ 1503 dma_sync_single_range_for_cpu(rx_ring->dev, 1504 rx_buffer->dma, 1505 rx_buffer->page_offset, 1506 size, 1507 DMA_FROM_DEVICE); 1508 1509 rx_buffer->pagecnt_bias--; 1510 1511 return rx_buffer; 1512 } 1513 1514 /** 1515 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1516 * @rx_ring: rx descriptor ring to transact packets on 1517 * @rx_buffer: buffer containing page to add 1518 * @skb: sk_buff to place the data into 1519 * @size: size of buffer to be added 1520 * 1521 * This function will add the data contained in rx_buffer->page to the skb. 1522 */ 1523 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1524 struct igc_rx_buffer *rx_buffer, 1525 struct sk_buff *skb, 1526 unsigned int size) 1527 { 1528 #if (PAGE_SIZE < 8192) 1529 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1530 1531 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1532 rx_buffer->page_offset, size, truesize); 1533 rx_buffer->page_offset ^= truesize; 1534 #else 1535 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1536 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1537 SKB_DATA_ALIGN(size); 1538 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1539 rx_buffer->page_offset, size, truesize); 1540 rx_buffer->page_offset += truesize; 1541 #endif 1542 } 1543 1544 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1545 struct igc_rx_buffer *rx_buffer, 1546 union igc_adv_rx_desc *rx_desc, 1547 unsigned int size) 1548 { 1549 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1550 #if (PAGE_SIZE < 8192) 1551 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1552 #else 1553 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1554 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1555 #endif 1556 struct sk_buff *skb; 1557 1558 /* prefetch first cache line of first page */ 1559 net_prefetch(va); 1560 1561 /* build an skb around the page buffer */ 1562 skb = build_skb(va - IGC_SKB_PAD, truesize); 1563 if (unlikely(!skb)) 1564 return NULL; 1565 1566 /* update pointers within the skb to store the data */ 1567 skb_reserve(skb, IGC_SKB_PAD); 1568 __skb_put(skb, size); 1569 1570 /* update buffer offset */ 1571 #if (PAGE_SIZE < 8192) 1572 rx_buffer->page_offset ^= truesize; 1573 #else 1574 rx_buffer->page_offset += truesize; 1575 #endif 1576 1577 return skb; 1578 } 1579 1580 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1581 struct igc_rx_buffer *rx_buffer, 1582 union igc_adv_rx_desc *rx_desc, 1583 unsigned int size) 1584 { 1585 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1586 #if (PAGE_SIZE < 8192) 1587 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1588 #else 1589 unsigned int truesize = SKB_DATA_ALIGN(size); 1590 #endif 1591 unsigned int headlen; 1592 struct sk_buff *skb; 1593 1594 /* prefetch first cache line of first page */ 1595 net_prefetch(va); 1596 1597 /* allocate a skb to store the frags */ 1598 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1599 if (unlikely(!skb)) 1600 return NULL; 1601 1602 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1603 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1604 va += IGC_TS_HDR_LEN; 1605 size -= IGC_TS_HDR_LEN; 1606 } 1607 1608 /* Determine available headroom for copy */ 1609 headlen = size; 1610 if (headlen > IGC_RX_HDR_LEN) 1611 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1612 1613 /* align pull length to size of long to optimize memcpy performance */ 1614 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1615 1616 /* update all of the pointers */ 1617 size -= headlen; 1618 if (size) { 1619 skb_add_rx_frag(skb, 0, rx_buffer->page, 1620 (va + headlen) - page_address(rx_buffer->page), 1621 size, truesize); 1622 #if (PAGE_SIZE < 8192) 1623 rx_buffer->page_offset ^= truesize; 1624 #else 1625 rx_buffer->page_offset += truesize; 1626 #endif 1627 } else { 1628 rx_buffer->pagecnt_bias++; 1629 } 1630 1631 return skb; 1632 } 1633 1634 /** 1635 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1636 * @rx_ring: rx descriptor ring to store buffers on 1637 * @old_buff: donor buffer to have page reused 1638 * 1639 * Synchronizes page for reuse by the adapter 1640 */ 1641 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1642 struct igc_rx_buffer *old_buff) 1643 { 1644 u16 nta = rx_ring->next_to_alloc; 1645 struct igc_rx_buffer *new_buff; 1646 1647 new_buff = &rx_ring->rx_buffer_info[nta]; 1648 1649 /* update, and store next to alloc */ 1650 nta++; 1651 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1652 1653 /* Transfer page from old buffer to new buffer. 1654 * Move each member individually to avoid possible store 1655 * forwarding stalls. 1656 */ 1657 new_buff->dma = old_buff->dma; 1658 new_buff->page = old_buff->page; 1659 new_buff->page_offset = old_buff->page_offset; 1660 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1661 } 1662 1663 static inline bool igc_page_is_reserved(struct page *page) 1664 { 1665 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1666 } 1667 1668 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1669 { 1670 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1671 struct page *page = rx_buffer->page; 1672 1673 /* avoid re-using remote pages */ 1674 if (unlikely(igc_page_is_reserved(page))) 1675 return false; 1676 1677 #if (PAGE_SIZE < 8192) 1678 /* if we are only owner of page we can reuse it */ 1679 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1680 return false; 1681 #else 1682 #define IGC_LAST_OFFSET \ 1683 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1684 1685 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1686 return false; 1687 #endif 1688 1689 /* If we have drained the page fragment pool we need to update 1690 * the pagecnt_bias and page count so that we fully restock the 1691 * number of references the driver holds. 1692 */ 1693 if (unlikely(!pagecnt_bias)) { 1694 page_ref_add(page, USHRT_MAX); 1695 rx_buffer->pagecnt_bias = USHRT_MAX; 1696 } 1697 1698 return true; 1699 } 1700 1701 /** 1702 * igc_is_non_eop - process handling of non-EOP buffers 1703 * @rx_ring: Rx ring being processed 1704 * @rx_desc: Rx descriptor for current buffer 1705 * 1706 * This function updates next to clean. If the buffer is an EOP buffer 1707 * this function exits returning false, otherwise it will place the 1708 * sk_buff in the next buffer to be chained and return true indicating 1709 * that this is in fact a non-EOP buffer. 1710 */ 1711 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1712 union igc_adv_rx_desc *rx_desc) 1713 { 1714 u32 ntc = rx_ring->next_to_clean + 1; 1715 1716 /* fetch, update, and store next to clean */ 1717 ntc = (ntc < rx_ring->count) ? ntc : 0; 1718 rx_ring->next_to_clean = ntc; 1719 1720 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1721 1722 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1723 return false; 1724 1725 return true; 1726 } 1727 1728 /** 1729 * igc_cleanup_headers - Correct corrupted or empty headers 1730 * @rx_ring: rx descriptor ring packet is being transacted on 1731 * @rx_desc: pointer to the EOP Rx descriptor 1732 * @skb: pointer to current skb being fixed 1733 * 1734 * Address the case where we are pulling data in on pages only 1735 * and as such no data is present in the skb header. 1736 * 1737 * In addition if skb is not at least 60 bytes we need to pad it so that 1738 * it is large enough to qualify as a valid Ethernet frame. 1739 * 1740 * Returns true if an error was encountered and skb was freed. 1741 */ 1742 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1743 union igc_adv_rx_desc *rx_desc, 1744 struct sk_buff *skb) 1745 { 1746 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { 1747 struct net_device *netdev = rx_ring->netdev; 1748 1749 if (!(netdev->features & NETIF_F_RXALL)) { 1750 dev_kfree_skb_any(skb); 1751 return true; 1752 } 1753 } 1754 1755 /* if eth_skb_pad returns an error the skb was freed */ 1756 if (eth_skb_pad(skb)) 1757 return true; 1758 1759 return false; 1760 } 1761 1762 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1763 struct igc_rx_buffer *rx_buffer) 1764 { 1765 if (igc_can_reuse_rx_page(rx_buffer)) { 1766 /* hand second half of page back to the ring */ 1767 igc_reuse_rx_page(rx_ring, rx_buffer); 1768 } else { 1769 /* We are not reusing the buffer so unmap it and free 1770 * any references we are holding to it 1771 */ 1772 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1773 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1774 IGC_RX_DMA_ATTR); 1775 __page_frag_cache_drain(rx_buffer->page, 1776 rx_buffer->pagecnt_bias); 1777 } 1778 1779 /* clear contents of rx_buffer */ 1780 rx_buffer->page = NULL; 1781 } 1782 1783 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1784 { 1785 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1786 } 1787 1788 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1789 struct igc_rx_buffer *bi) 1790 { 1791 struct page *page = bi->page; 1792 dma_addr_t dma; 1793 1794 /* since we are recycling buffers we should seldom need to alloc */ 1795 if (likely(page)) 1796 return true; 1797 1798 /* alloc new page for storage */ 1799 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1800 if (unlikely(!page)) { 1801 rx_ring->rx_stats.alloc_failed++; 1802 return false; 1803 } 1804 1805 /* map page for use */ 1806 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1807 igc_rx_pg_size(rx_ring), 1808 DMA_FROM_DEVICE, 1809 IGC_RX_DMA_ATTR); 1810 1811 /* if mapping failed free memory back to system since 1812 * there isn't much point in holding memory we can't use 1813 */ 1814 if (dma_mapping_error(rx_ring->dev, dma)) { 1815 __free_page(page); 1816 1817 rx_ring->rx_stats.alloc_failed++; 1818 return false; 1819 } 1820 1821 bi->dma = dma; 1822 bi->page = page; 1823 bi->page_offset = igc_rx_offset(rx_ring); 1824 bi->pagecnt_bias = 1; 1825 1826 return true; 1827 } 1828 1829 /** 1830 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1831 * @rx_ring: rx descriptor ring 1832 * @cleaned_count: number of buffers to clean 1833 */ 1834 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1835 { 1836 union igc_adv_rx_desc *rx_desc; 1837 u16 i = rx_ring->next_to_use; 1838 struct igc_rx_buffer *bi; 1839 u16 bufsz; 1840 1841 /* nothing to do */ 1842 if (!cleaned_count) 1843 return; 1844 1845 rx_desc = IGC_RX_DESC(rx_ring, i); 1846 bi = &rx_ring->rx_buffer_info[i]; 1847 i -= rx_ring->count; 1848 1849 bufsz = igc_rx_bufsz(rx_ring); 1850 1851 do { 1852 if (!igc_alloc_mapped_page(rx_ring, bi)) 1853 break; 1854 1855 /* sync the buffer for use by the device */ 1856 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1857 bi->page_offset, bufsz, 1858 DMA_FROM_DEVICE); 1859 1860 /* Refresh the desc even if buffer_addrs didn't change 1861 * because each write-back erases this info. 1862 */ 1863 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1864 1865 rx_desc++; 1866 bi++; 1867 i++; 1868 if (unlikely(!i)) { 1869 rx_desc = IGC_RX_DESC(rx_ring, 0); 1870 bi = rx_ring->rx_buffer_info; 1871 i -= rx_ring->count; 1872 } 1873 1874 /* clear the length for the next_to_use descriptor */ 1875 rx_desc->wb.upper.length = 0; 1876 1877 cleaned_count--; 1878 } while (cleaned_count); 1879 1880 i += rx_ring->count; 1881 1882 if (rx_ring->next_to_use != i) { 1883 /* record the next descriptor to use */ 1884 rx_ring->next_to_use = i; 1885 1886 /* update next to alloc since we have filled the ring */ 1887 rx_ring->next_to_alloc = i; 1888 1889 /* Force memory writes to complete before letting h/w 1890 * know there are new descriptors to fetch. (Only 1891 * applicable for weak-ordered memory model archs, 1892 * such as IA-64). 1893 */ 1894 wmb(); 1895 writel(i, rx_ring->tail); 1896 } 1897 } 1898 1899 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1900 { 1901 unsigned int total_bytes = 0, total_packets = 0; 1902 struct igc_ring *rx_ring = q_vector->rx.ring; 1903 struct sk_buff *skb = rx_ring->skb; 1904 u16 cleaned_count = igc_desc_unused(rx_ring); 1905 1906 while (likely(total_packets < budget)) { 1907 union igc_adv_rx_desc *rx_desc; 1908 struct igc_rx_buffer *rx_buffer; 1909 unsigned int size; 1910 1911 /* return some buffers to hardware, one at a time is too slow */ 1912 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1913 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1914 cleaned_count = 0; 1915 } 1916 1917 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1918 size = le16_to_cpu(rx_desc->wb.upper.length); 1919 if (!size) 1920 break; 1921 1922 /* This memory barrier is needed to keep us from reading 1923 * any other fields out of the rx_desc until we know the 1924 * descriptor has been written back 1925 */ 1926 dma_rmb(); 1927 1928 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1929 1930 /* retrieve a buffer from the ring */ 1931 if (skb) 1932 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1933 else if (ring_uses_build_skb(rx_ring)) 1934 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1935 else 1936 skb = igc_construct_skb(rx_ring, rx_buffer, 1937 rx_desc, size); 1938 1939 /* exit if we failed to retrieve a buffer */ 1940 if (!skb) { 1941 rx_ring->rx_stats.alloc_failed++; 1942 rx_buffer->pagecnt_bias++; 1943 break; 1944 } 1945 1946 igc_put_rx_buffer(rx_ring, rx_buffer); 1947 cleaned_count++; 1948 1949 /* fetch next buffer in frame if non-eop */ 1950 if (igc_is_non_eop(rx_ring, rx_desc)) 1951 continue; 1952 1953 /* verify the packet layout is correct */ 1954 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1955 skb = NULL; 1956 continue; 1957 } 1958 1959 /* probably a little skewed due to removing CRC */ 1960 total_bytes += skb->len; 1961 1962 /* populate checksum, VLAN, and protocol */ 1963 igc_process_skb_fields(rx_ring, rx_desc, skb); 1964 1965 napi_gro_receive(&q_vector->napi, skb); 1966 1967 /* reset skb pointer */ 1968 skb = NULL; 1969 1970 /* update budget accounting */ 1971 total_packets++; 1972 } 1973 1974 /* place incomplete frames back on ring for completion */ 1975 rx_ring->skb = skb; 1976 1977 u64_stats_update_begin(&rx_ring->rx_syncp); 1978 rx_ring->rx_stats.packets += total_packets; 1979 rx_ring->rx_stats.bytes += total_bytes; 1980 u64_stats_update_end(&rx_ring->rx_syncp); 1981 q_vector->rx.total_packets += total_packets; 1982 q_vector->rx.total_bytes += total_bytes; 1983 1984 if (cleaned_count) 1985 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1986 1987 return total_packets; 1988 } 1989 1990 /** 1991 * igc_clean_tx_irq - Reclaim resources after transmit completes 1992 * @q_vector: pointer to q_vector containing needed info 1993 * @napi_budget: Used to determine if we are in netpoll 1994 * 1995 * returns true if ring is completely cleaned 1996 */ 1997 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1998 { 1999 struct igc_adapter *adapter = q_vector->adapter; 2000 unsigned int total_bytes = 0, total_packets = 0; 2001 unsigned int budget = q_vector->tx.work_limit; 2002 struct igc_ring *tx_ring = q_vector->tx.ring; 2003 unsigned int i = tx_ring->next_to_clean; 2004 struct igc_tx_buffer *tx_buffer; 2005 union igc_adv_tx_desc *tx_desc; 2006 2007 if (test_bit(__IGC_DOWN, &adapter->state)) 2008 return true; 2009 2010 tx_buffer = &tx_ring->tx_buffer_info[i]; 2011 tx_desc = IGC_TX_DESC(tx_ring, i); 2012 i -= tx_ring->count; 2013 2014 do { 2015 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2016 2017 /* if next_to_watch is not set then there is no work pending */ 2018 if (!eop_desc) 2019 break; 2020 2021 /* prevent any other reads prior to eop_desc */ 2022 smp_rmb(); 2023 2024 /* if DD is not set pending work has not been completed */ 2025 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2026 break; 2027 2028 /* clear next_to_watch to prevent false hangs */ 2029 tx_buffer->next_to_watch = NULL; 2030 2031 /* update the statistics for this packet */ 2032 total_bytes += tx_buffer->bytecount; 2033 total_packets += tx_buffer->gso_segs; 2034 2035 /* free the skb */ 2036 napi_consume_skb(tx_buffer->skb, napi_budget); 2037 2038 /* unmap skb header data */ 2039 dma_unmap_single(tx_ring->dev, 2040 dma_unmap_addr(tx_buffer, dma), 2041 dma_unmap_len(tx_buffer, len), 2042 DMA_TO_DEVICE); 2043 2044 /* clear tx_buffer data */ 2045 dma_unmap_len_set(tx_buffer, len, 0); 2046 2047 /* clear last DMA location and unmap remaining buffers */ 2048 while (tx_desc != eop_desc) { 2049 tx_buffer++; 2050 tx_desc++; 2051 i++; 2052 if (unlikely(!i)) { 2053 i -= tx_ring->count; 2054 tx_buffer = tx_ring->tx_buffer_info; 2055 tx_desc = IGC_TX_DESC(tx_ring, 0); 2056 } 2057 2058 /* unmap any remaining paged data */ 2059 if (dma_unmap_len(tx_buffer, len)) { 2060 dma_unmap_page(tx_ring->dev, 2061 dma_unmap_addr(tx_buffer, dma), 2062 dma_unmap_len(tx_buffer, len), 2063 DMA_TO_DEVICE); 2064 dma_unmap_len_set(tx_buffer, len, 0); 2065 } 2066 } 2067 2068 /* move us one more past the eop_desc for start of next pkt */ 2069 tx_buffer++; 2070 tx_desc++; 2071 i++; 2072 if (unlikely(!i)) { 2073 i -= tx_ring->count; 2074 tx_buffer = tx_ring->tx_buffer_info; 2075 tx_desc = IGC_TX_DESC(tx_ring, 0); 2076 } 2077 2078 /* issue prefetch for next Tx descriptor */ 2079 prefetch(tx_desc); 2080 2081 /* update budget accounting */ 2082 budget--; 2083 } while (likely(budget)); 2084 2085 netdev_tx_completed_queue(txring_txq(tx_ring), 2086 total_packets, total_bytes); 2087 2088 i += tx_ring->count; 2089 tx_ring->next_to_clean = i; 2090 u64_stats_update_begin(&tx_ring->tx_syncp); 2091 tx_ring->tx_stats.bytes += total_bytes; 2092 tx_ring->tx_stats.packets += total_packets; 2093 u64_stats_update_end(&tx_ring->tx_syncp); 2094 q_vector->tx.total_bytes += total_bytes; 2095 q_vector->tx.total_packets += total_packets; 2096 2097 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2098 struct igc_hw *hw = &adapter->hw; 2099 2100 /* Detect a transmit hang in hardware, this serializes the 2101 * check with the clearing of time_stamp and movement of i 2102 */ 2103 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2104 if (tx_buffer->next_to_watch && 2105 time_after(jiffies, tx_buffer->time_stamp + 2106 (adapter->tx_timeout_factor * HZ)) && 2107 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2108 /* detected Tx unit hang */ 2109 netdev_err(tx_ring->netdev, 2110 "Detected Tx Unit Hang\n" 2111 " Tx Queue <%d>\n" 2112 " TDH <%x>\n" 2113 " TDT <%x>\n" 2114 " next_to_use <%x>\n" 2115 " next_to_clean <%x>\n" 2116 "buffer_info[next_to_clean]\n" 2117 " time_stamp <%lx>\n" 2118 " next_to_watch <%p>\n" 2119 " jiffies <%lx>\n" 2120 " desc.status <%x>\n", 2121 tx_ring->queue_index, 2122 rd32(IGC_TDH(tx_ring->reg_idx)), 2123 readl(tx_ring->tail), 2124 tx_ring->next_to_use, 2125 tx_ring->next_to_clean, 2126 tx_buffer->time_stamp, 2127 tx_buffer->next_to_watch, 2128 jiffies, 2129 tx_buffer->next_to_watch->wb.status); 2130 netif_stop_subqueue(tx_ring->netdev, 2131 tx_ring->queue_index); 2132 2133 /* we are about to reset, no point in enabling stuff */ 2134 return true; 2135 } 2136 } 2137 2138 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2139 if (unlikely(total_packets && 2140 netif_carrier_ok(tx_ring->netdev) && 2141 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2142 /* Make sure that anybody stopping the queue after this 2143 * sees the new next_to_clean. 2144 */ 2145 smp_mb(); 2146 if (__netif_subqueue_stopped(tx_ring->netdev, 2147 tx_ring->queue_index) && 2148 !(test_bit(__IGC_DOWN, &adapter->state))) { 2149 netif_wake_subqueue(tx_ring->netdev, 2150 tx_ring->queue_index); 2151 2152 u64_stats_update_begin(&tx_ring->tx_syncp); 2153 tx_ring->tx_stats.restart_queue++; 2154 u64_stats_update_end(&tx_ring->tx_syncp); 2155 } 2156 } 2157 2158 return !!budget; 2159 } 2160 2161 static int igc_find_mac_filter(struct igc_adapter *adapter, 2162 enum igc_mac_filter_type type, const u8 *addr) 2163 { 2164 struct igc_hw *hw = &adapter->hw; 2165 int max_entries = hw->mac.rar_entry_count; 2166 u32 ral, rah; 2167 int i; 2168 2169 for (i = 0; i < max_entries; i++) { 2170 ral = rd32(IGC_RAL(i)); 2171 rah = rd32(IGC_RAH(i)); 2172 2173 if (!(rah & IGC_RAH_AV)) 2174 continue; 2175 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2176 continue; 2177 if ((rah & IGC_RAH_RAH_MASK) != 2178 le16_to_cpup((__le16 *)(addr + 4))) 2179 continue; 2180 if (ral != le32_to_cpup((__le32 *)(addr))) 2181 continue; 2182 2183 return i; 2184 } 2185 2186 return -1; 2187 } 2188 2189 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2190 { 2191 struct igc_hw *hw = &adapter->hw; 2192 int max_entries = hw->mac.rar_entry_count; 2193 u32 rah; 2194 int i; 2195 2196 for (i = 0; i < max_entries; i++) { 2197 rah = rd32(IGC_RAH(i)); 2198 2199 if (!(rah & IGC_RAH_AV)) 2200 return i; 2201 } 2202 2203 return -1; 2204 } 2205 2206 /** 2207 * igc_add_mac_filter() - Add MAC address filter 2208 * @adapter: Pointer to adapter where the filter should be added 2209 * @type: MAC address filter type (source or destination) 2210 * @addr: MAC address 2211 * @queue: If non-negative, queue assignment feature is enabled and frames 2212 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2213 * assignment is disabled. 2214 * 2215 * Return: 0 in case of success, negative errno code otherwise. 2216 */ 2217 static int igc_add_mac_filter(struct igc_adapter *adapter, 2218 enum igc_mac_filter_type type, const u8 *addr, 2219 int queue) 2220 { 2221 struct net_device *dev = adapter->netdev; 2222 int index; 2223 2224 index = igc_find_mac_filter(adapter, type, addr); 2225 if (index >= 0) 2226 goto update_filter; 2227 2228 index = igc_get_avail_mac_filter_slot(adapter); 2229 if (index < 0) 2230 return -ENOSPC; 2231 2232 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2233 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2234 addr, queue); 2235 2236 update_filter: 2237 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2238 return 0; 2239 } 2240 2241 /** 2242 * igc_del_mac_filter() - Delete MAC address filter 2243 * @adapter: Pointer to adapter where the filter should be deleted from 2244 * @type: MAC address filter type (source or destination) 2245 * @addr: MAC address 2246 */ 2247 static void igc_del_mac_filter(struct igc_adapter *adapter, 2248 enum igc_mac_filter_type type, const u8 *addr) 2249 { 2250 struct net_device *dev = adapter->netdev; 2251 int index; 2252 2253 index = igc_find_mac_filter(adapter, type, addr); 2254 if (index < 0) 2255 return; 2256 2257 if (index == 0) { 2258 /* If this is the default filter, we don't actually delete it. 2259 * We just reset to its default value i.e. disable queue 2260 * assignment. 2261 */ 2262 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2263 2264 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2265 } else { 2266 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2267 index, 2268 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2269 addr); 2270 2271 igc_clear_mac_filter_hw(adapter, index); 2272 } 2273 } 2274 2275 /** 2276 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2277 * @adapter: Pointer to adapter where the filter should be added 2278 * @prio: VLAN priority value 2279 * @queue: Queue number which matching frames are assigned to 2280 * 2281 * Return: 0 in case of success, negative errno code otherwise. 2282 */ 2283 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2284 int queue) 2285 { 2286 struct net_device *dev = adapter->netdev; 2287 struct igc_hw *hw = &adapter->hw; 2288 u32 vlanpqf; 2289 2290 vlanpqf = rd32(IGC_VLANPQF); 2291 2292 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2293 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2294 return -EEXIST; 2295 } 2296 2297 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2298 vlanpqf |= IGC_VLANPQF_VALID(prio); 2299 2300 wr32(IGC_VLANPQF, vlanpqf); 2301 2302 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2303 prio, queue); 2304 return 0; 2305 } 2306 2307 /** 2308 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2309 * @adapter: Pointer to adapter where the filter should be deleted from 2310 * @prio: VLAN priority value 2311 */ 2312 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2313 { 2314 struct igc_hw *hw = &adapter->hw; 2315 u32 vlanpqf; 2316 2317 vlanpqf = rd32(IGC_VLANPQF); 2318 2319 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2320 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2321 2322 wr32(IGC_VLANPQF, vlanpqf); 2323 2324 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2325 prio); 2326 } 2327 2328 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2329 { 2330 struct igc_hw *hw = &adapter->hw; 2331 int i; 2332 2333 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2334 u32 etqf = rd32(IGC_ETQF(i)); 2335 2336 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2337 return i; 2338 } 2339 2340 return -1; 2341 } 2342 2343 /** 2344 * igc_add_etype_filter() - Add ethertype filter 2345 * @adapter: Pointer to adapter where the filter should be added 2346 * @etype: Ethertype value 2347 * @queue: If non-negative, queue assignment feature is enabled and frames 2348 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2349 * assignment is disabled. 2350 * 2351 * Return: 0 in case of success, negative errno code otherwise. 2352 */ 2353 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2354 int queue) 2355 { 2356 struct igc_hw *hw = &adapter->hw; 2357 int index; 2358 u32 etqf; 2359 2360 index = igc_get_avail_etype_filter_slot(adapter); 2361 if (index < 0) 2362 return -ENOSPC; 2363 2364 etqf = rd32(IGC_ETQF(index)); 2365 2366 etqf &= ~IGC_ETQF_ETYPE_MASK; 2367 etqf |= etype; 2368 2369 if (queue >= 0) { 2370 etqf &= ~IGC_ETQF_QUEUE_MASK; 2371 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2372 etqf |= IGC_ETQF_QUEUE_ENABLE; 2373 } 2374 2375 etqf |= IGC_ETQF_FILTER_ENABLE; 2376 2377 wr32(IGC_ETQF(index), etqf); 2378 2379 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2380 etype, queue); 2381 return 0; 2382 } 2383 2384 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2385 { 2386 struct igc_hw *hw = &adapter->hw; 2387 int i; 2388 2389 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2390 u32 etqf = rd32(IGC_ETQF(i)); 2391 2392 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2393 return i; 2394 } 2395 2396 return -1; 2397 } 2398 2399 /** 2400 * igc_del_etype_filter() - Delete ethertype filter 2401 * @adapter: Pointer to adapter where the filter should be deleted from 2402 * @etype: Ethertype value 2403 */ 2404 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2405 { 2406 struct igc_hw *hw = &adapter->hw; 2407 int index; 2408 2409 index = igc_find_etype_filter(adapter, etype); 2410 if (index < 0) 2411 return; 2412 2413 wr32(IGC_ETQF(index), 0); 2414 2415 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2416 etype); 2417 } 2418 2419 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2420 const struct igc_nfc_rule *rule) 2421 { 2422 int err; 2423 2424 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2425 err = igc_add_etype_filter(adapter, rule->filter.etype, 2426 rule->action); 2427 if (err) 2428 return err; 2429 } 2430 2431 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2432 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2433 rule->filter.src_addr, rule->action); 2434 if (err) 2435 return err; 2436 } 2437 2438 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2439 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2440 rule->filter.dst_addr, rule->action); 2441 if (err) 2442 return err; 2443 } 2444 2445 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2446 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2447 VLAN_PRIO_SHIFT; 2448 2449 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2450 if (err) 2451 return err; 2452 } 2453 2454 return 0; 2455 } 2456 2457 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2458 const struct igc_nfc_rule *rule) 2459 { 2460 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2461 igc_del_etype_filter(adapter, rule->filter.etype); 2462 2463 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2464 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2465 VLAN_PRIO_SHIFT; 2466 2467 igc_del_vlan_prio_filter(adapter, prio); 2468 } 2469 2470 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2471 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2472 rule->filter.src_addr); 2473 2474 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2475 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2476 rule->filter.dst_addr); 2477 } 2478 2479 /** 2480 * igc_get_nfc_rule() - Get NFC rule 2481 * @adapter: Pointer to adapter 2482 * @location: Rule location 2483 * 2484 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2485 * 2486 * Return: Pointer to NFC rule at @location. If not found, NULL. 2487 */ 2488 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2489 u32 location) 2490 { 2491 struct igc_nfc_rule *rule; 2492 2493 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2494 if (rule->location == location) 2495 return rule; 2496 if (rule->location > location) 2497 break; 2498 } 2499 2500 return NULL; 2501 } 2502 2503 /** 2504 * igc_del_nfc_rule() - Delete NFC rule 2505 * @adapter: Pointer to adapter 2506 * @rule: Pointer to rule to be deleted 2507 * 2508 * Disable NFC rule in hardware and delete it from adapter. 2509 * 2510 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2511 */ 2512 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2513 { 2514 igc_disable_nfc_rule(adapter, rule); 2515 2516 list_del(&rule->list); 2517 adapter->nfc_rule_count--; 2518 2519 kfree(rule); 2520 } 2521 2522 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2523 { 2524 struct igc_nfc_rule *rule, *tmp; 2525 2526 mutex_lock(&adapter->nfc_rule_lock); 2527 2528 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2529 igc_del_nfc_rule(adapter, rule); 2530 2531 mutex_unlock(&adapter->nfc_rule_lock); 2532 } 2533 2534 /** 2535 * igc_add_nfc_rule() - Add NFC rule 2536 * @adapter: Pointer to adapter 2537 * @rule: Pointer to rule to be added 2538 * 2539 * Enable NFC rule in hardware and add it to adapter. 2540 * 2541 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2542 * 2543 * Return: 0 on success, negative errno on failure. 2544 */ 2545 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2546 { 2547 struct igc_nfc_rule *pred, *cur; 2548 int err; 2549 2550 err = igc_enable_nfc_rule(adapter, rule); 2551 if (err) 2552 return err; 2553 2554 pred = NULL; 2555 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2556 if (cur->location >= rule->location) 2557 break; 2558 pred = cur; 2559 } 2560 2561 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2562 adapter->nfc_rule_count++; 2563 return 0; 2564 } 2565 2566 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2567 { 2568 struct igc_nfc_rule *rule; 2569 2570 mutex_lock(&adapter->nfc_rule_lock); 2571 2572 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2573 igc_enable_nfc_rule(adapter, rule); 2574 2575 mutex_unlock(&adapter->nfc_rule_lock); 2576 } 2577 2578 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2579 { 2580 struct igc_adapter *adapter = netdev_priv(netdev); 2581 2582 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2583 } 2584 2585 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2586 { 2587 struct igc_adapter *adapter = netdev_priv(netdev); 2588 2589 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2590 return 0; 2591 } 2592 2593 /** 2594 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2595 * @netdev: network interface device structure 2596 * 2597 * The set_rx_mode entry point is called whenever the unicast or multicast 2598 * address lists or the network interface flags are updated. This routine is 2599 * responsible for configuring the hardware for proper unicast, multicast, 2600 * promiscuous mode, and all-multi behavior. 2601 */ 2602 static void igc_set_rx_mode(struct net_device *netdev) 2603 { 2604 struct igc_adapter *adapter = netdev_priv(netdev); 2605 struct igc_hw *hw = &adapter->hw; 2606 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2607 int count; 2608 2609 /* Check for Promiscuous and All Multicast modes */ 2610 if (netdev->flags & IFF_PROMISC) { 2611 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2612 } else { 2613 if (netdev->flags & IFF_ALLMULTI) { 2614 rctl |= IGC_RCTL_MPE; 2615 } else { 2616 /* Write addresses to the MTA, if the attempt fails 2617 * then we should just turn on promiscuous mode so 2618 * that we can at least receive multicast traffic 2619 */ 2620 count = igc_write_mc_addr_list(netdev); 2621 if (count < 0) 2622 rctl |= IGC_RCTL_MPE; 2623 } 2624 } 2625 2626 /* Write addresses to available RAR registers, if there is not 2627 * sufficient space to store all the addresses then enable 2628 * unicast promiscuous mode 2629 */ 2630 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2631 rctl |= IGC_RCTL_UPE; 2632 2633 /* update state of unicast and multicast */ 2634 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2635 wr32(IGC_RCTL, rctl); 2636 2637 #if (PAGE_SIZE < 8192) 2638 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2639 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2640 #endif 2641 wr32(IGC_RLPML, rlpml); 2642 } 2643 2644 /** 2645 * igc_configure - configure the hardware for RX and TX 2646 * @adapter: private board structure 2647 */ 2648 static void igc_configure(struct igc_adapter *adapter) 2649 { 2650 struct net_device *netdev = adapter->netdev; 2651 int i = 0; 2652 2653 igc_get_hw_control(adapter); 2654 igc_set_rx_mode(netdev); 2655 2656 igc_setup_tctl(adapter); 2657 igc_setup_mrqc(adapter); 2658 igc_setup_rctl(adapter); 2659 2660 igc_set_default_mac_filter(adapter); 2661 igc_restore_nfc_rules(adapter); 2662 2663 igc_configure_tx(adapter); 2664 igc_configure_rx(adapter); 2665 2666 igc_rx_fifo_flush_base(&adapter->hw); 2667 2668 /* call igc_desc_unused which always leaves 2669 * at least 1 descriptor unused to make sure 2670 * next_to_use != next_to_clean 2671 */ 2672 for (i = 0; i < adapter->num_rx_queues; i++) { 2673 struct igc_ring *ring = adapter->rx_ring[i]; 2674 2675 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2676 } 2677 } 2678 2679 /** 2680 * igc_write_ivar - configure ivar for given MSI-X vector 2681 * @hw: pointer to the HW structure 2682 * @msix_vector: vector number we are allocating to a given ring 2683 * @index: row index of IVAR register to write within IVAR table 2684 * @offset: column offset of in IVAR, should be multiple of 8 2685 * 2686 * The IVAR table consists of 2 columns, 2687 * each containing an cause allocation for an Rx and Tx ring, and a 2688 * variable number of rows depending on the number of queues supported. 2689 */ 2690 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2691 int index, int offset) 2692 { 2693 u32 ivar = array_rd32(IGC_IVAR0, index); 2694 2695 /* clear any bits that are currently set */ 2696 ivar &= ~((u32)0xFF << offset); 2697 2698 /* write vector and valid bit */ 2699 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2700 2701 array_wr32(IGC_IVAR0, index, ivar); 2702 } 2703 2704 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2705 { 2706 struct igc_adapter *adapter = q_vector->adapter; 2707 struct igc_hw *hw = &adapter->hw; 2708 int rx_queue = IGC_N0_QUEUE; 2709 int tx_queue = IGC_N0_QUEUE; 2710 2711 if (q_vector->rx.ring) 2712 rx_queue = q_vector->rx.ring->reg_idx; 2713 if (q_vector->tx.ring) 2714 tx_queue = q_vector->tx.ring->reg_idx; 2715 2716 switch (hw->mac.type) { 2717 case igc_i225: 2718 if (rx_queue > IGC_N0_QUEUE) 2719 igc_write_ivar(hw, msix_vector, 2720 rx_queue >> 1, 2721 (rx_queue & 0x1) << 4); 2722 if (tx_queue > IGC_N0_QUEUE) 2723 igc_write_ivar(hw, msix_vector, 2724 tx_queue >> 1, 2725 ((tx_queue & 0x1) << 4) + 8); 2726 q_vector->eims_value = BIT(msix_vector); 2727 break; 2728 default: 2729 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2730 break; 2731 } 2732 2733 /* add q_vector eims value to global eims_enable_mask */ 2734 adapter->eims_enable_mask |= q_vector->eims_value; 2735 2736 /* configure q_vector to set itr on first interrupt */ 2737 q_vector->set_itr = 1; 2738 } 2739 2740 /** 2741 * igc_configure_msix - Configure MSI-X hardware 2742 * @adapter: Pointer to adapter structure 2743 * 2744 * igc_configure_msix sets up the hardware to properly 2745 * generate MSI-X interrupts. 2746 */ 2747 static void igc_configure_msix(struct igc_adapter *adapter) 2748 { 2749 struct igc_hw *hw = &adapter->hw; 2750 int i, vector = 0; 2751 u32 tmp; 2752 2753 adapter->eims_enable_mask = 0; 2754 2755 /* set vector for other causes, i.e. link changes */ 2756 switch (hw->mac.type) { 2757 case igc_i225: 2758 /* Turn on MSI-X capability first, or our settings 2759 * won't stick. And it will take days to debug. 2760 */ 2761 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2762 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2763 IGC_GPIE_NSICR); 2764 2765 /* enable msix_other interrupt */ 2766 adapter->eims_other = BIT(vector); 2767 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2768 2769 wr32(IGC_IVAR_MISC, tmp); 2770 break; 2771 default: 2772 /* do nothing, since nothing else supports MSI-X */ 2773 break; 2774 } /* switch (hw->mac.type) */ 2775 2776 adapter->eims_enable_mask |= adapter->eims_other; 2777 2778 for (i = 0; i < adapter->num_q_vectors; i++) 2779 igc_assign_vector(adapter->q_vector[i], vector++); 2780 2781 wrfl(); 2782 } 2783 2784 /** 2785 * igc_irq_enable - Enable default interrupt generation settings 2786 * @adapter: board private structure 2787 */ 2788 static void igc_irq_enable(struct igc_adapter *adapter) 2789 { 2790 struct igc_hw *hw = &adapter->hw; 2791 2792 if (adapter->msix_entries) { 2793 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2794 u32 regval = rd32(IGC_EIAC); 2795 2796 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2797 regval = rd32(IGC_EIAM); 2798 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2799 wr32(IGC_EIMS, adapter->eims_enable_mask); 2800 wr32(IGC_IMS, ims); 2801 } else { 2802 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2803 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2804 } 2805 } 2806 2807 /** 2808 * igc_irq_disable - Mask off interrupt generation on the NIC 2809 * @adapter: board private structure 2810 */ 2811 static void igc_irq_disable(struct igc_adapter *adapter) 2812 { 2813 struct igc_hw *hw = &adapter->hw; 2814 2815 if (adapter->msix_entries) { 2816 u32 regval = rd32(IGC_EIAM); 2817 2818 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2819 wr32(IGC_EIMC, adapter->eims_enable_mask); 2820 regval = rd32(IGC_EIAC); 2821 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2822 } 2823 2824 wr32(IGC_IAM, 0); 2825 wr32(IGC_IMC, ~0); 2826 wrfl(); 2827 2828 if (adapter->msix_entries) { 2829 int vector = 0, i; 2830 2831 synchronize_irq(adapter->msix_entries[vector++].vector); 2832 2833 for (i = 0; i < adapter->num_q_vectors; i++) 2834 synchronize_irq(adapter->msix_entries[vector++].vector); 2835 } else { 2836 synchronize_irq(adapter->pdev->irq); 2837 } 2838 } 2839 2840 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2841 const u32 max_rss_queues) 2842 { 2843 /* Determine if we need to pair queues. */ 2844 /* If rss_queues > half of max_rss_queues, pair the queues in 2845 * order to conserve interrupts due to limited supply. 2846 */ 2847 if (adapter->rss_queues > (max_rss_queues / 2)) 2848 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2849 else 2850 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2851 } 2852 2853 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2854 { 2855 return IGC_MAX_RX_QUEUES; 2856 } 2857 2858 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2859 { 2860 u32 max_rss_queues; 2861 2862 max_rss_queues = igc_get_max_rss_queues(adapter); 2863 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2864 2865 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2866 } 2867 2868 /** 2869 * igc_reset_q_vector - Reset config for interrupt vector 2870 * @adapter: board private structure to initialize 2871 * @v_idx: Index of vector to be reset 2872 * 2873 * If NAPI is enabled it will delete any references to the 2874 * NAPI struct. This is preparation for igc_free_q_vector. 2875 */ 2876 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2877 { 2878 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2879 2880 /* if we're coming from igc_set_interrupt_capability, the vectors are 2881 * not yet allocated 2882 */ 2883 if (!q_vector) 2884 return; 2885 2886 if (q_vector->tx.ring) 2887 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2888 2889 if (q_vector->rx.ring) 2890 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2891 2892 netif_napi_del(&q_vector->napi); 2893 } 2894 2895 /** 2896 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2897 * @adapter: board private structure to initialize 2898 * @v_idx: Index of vector to be freed 2899 * 2900 * This function frees the memory allocated to the q_vector. 2901 */ 2902 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2903 { 2904 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2905 2906 adapter->q_vector[v_idx] = NULL; 2907 2908 /* igc_get_stats64() might access the rings on this vector, 2909 * we must wait a grace period before freeing it. 2910 */ 2911 if (q_vector) 2912 kfree_rcu(q_vector, rcu); 2913 } 2914 2915 /** 2916 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2917 * @adapter: board private structure to initialize 2918 * 2919 * This function frees the memory allocated to the q_vectors. In addition if 2920 * NAPI is enabled it will delete any references to the NAPI struct prior 2921 * to freeing the q_vector. 2922 */ 2923 static void igc_free_q_vectors(struct igc_adapter *adapter) 2924 { 2925 int v_idx = adapter->num_q_vectors; 2926 2927 adapter->num_tx_queues = 0; 2928 adapter->num_rx_queues = 0; 2929 adapter->num_q_vectors = 0; 2930 2931 while (v_idx--) { 2932 igc_reset_q_vector(adapter, v_idx); 2933 igc_free_q_vector(adapter, v_idx); 2934 } 2935 } 2936 2937 /** 2938 * igc_update_itr - update the dynamic ITR value based on statistics 2939 * @q_vector: pointer to q_vector 2940 * @ring_container: ring info to update the itr for 2941 * 2942 * Stores a new ITR value based on packets and byte 2943 * counts during the last interrupt. The advantage of per interrupt 2944 * computation is faster updates and more accurate ITR for the current 2945 * traffic pattern. Constants in this function were computed 2946 * based on theoretical maximum wire speed and thresholds were set based 2947 * on testing data as well as attempting to minimize response time 2948 * while increasing bulk throughput. 2949 * NOTE: These calculations are only valid when operating in a single- 2950 * queue environment. 2951 */ 2952 static void igc_update_itr(struct igc_q_vector *q_vector, 2953 struct igc_ring_container *ring_container) 2954 { 2955 unsigned int packets = ring_container->total_packets; 2956 unsigned int bytes = ring_container->total_bytes; 2957 u8 itrval = ring_container->itr; 2958 2959 /* no packets, exit with status unchanged */ 2960 if (packets == 0) 2961 return; 2962 2963 switch (itrval) { 2964 case lowest_latency: 2965 /* handle TSO and jumbo frames */ 2966 if (bytes / packets > 8000) 2967 itrval = bulk_latency; 2968 else if ((packets < 5) && (bytes > 512)) 2969 itrval = low_latency; 2970 break; 2971 case low_latency: /* 50 usec aka 20000 ints/s */ 2972 if (bytes > 10000) { 2973 /* this if handles the TSO accounting */ 2974 if (bytes / packets > 8000) 2975 itrval = bulk_latency; 2976 else if ((packets < 10) || ((bytes / packets) > 1200)) 2977 itrval = bulk_latency; 2978 else if ((packets > 35)) 2979 itrval = lowest_latency; 2980 } else if (bytes / packets > 2000) { 2981 itrval = bulk_latency; 2982 } else if (packets <= 2 && bytes < 512) { 2983 itrval = lowest_latency; 2984 } 2985 break; 2986 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2987 if (bytes > 25000) { 2988 if (packets > 35) 2989 itrval = low_latency; 2990 } else if (bytes < 1500) { 2991 itrval = low_latency; 2992 } 2993 break; 2994 } 2995 2996 /* clear work counters since we have the values we need */ 2997 ring_container->total_bytes = 0; 2998 ring_container->total_packets = 0; 2999 3000 /* write updated itr to ring container */ 3001 ring_container->itr = itrval; 3002 } 3003 3004 static void igc_set_itr(struct igc_q_vector *q_vector) 3005 { 3006 struct igc_adapter *adapter = q_vector->adapter; 3007 u32 new_itr = q_vector->itr_val; 3008 u8 current_itr = 0; 3009 3010 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3011 switch (adapter->link_speed) { 3012 case SPEED_10: 3013 case SPEED_100: 3014 current_itr = 0; 3015 new_itr = IGC_4K_ITR; 3016 goto set_itr_now; 3017 default: 3018 break; 3019 } 3020 3021 igc_update_itr(q_vector, &q_vector->tx); 3022 igc_update_itr(q_vector, &q_vector->rx); 3023 3024 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3025 3026 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3027 if (current_itr == lowest_latency && 3028 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3029 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3030 current_itr = low_latency; 3031 3032 switch (current_itr) { 3033 /* counts and packets in update_itr are dependent on these numbers */ 3034 case lowest_latency: 3035 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3036 break; 3037 case low_latency: 3038 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3039 break; 3040 case bulk_latency: 3041 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3042 break; 3043 default: 3044 break; 3045 } 3046 3047 set_itr_now: 3048 if (new_itr != q_vector->itr_val) { 3049 /* this attempts to bias the interrupt rate towards Bulk 3050 * by adding intermediate steps when interrupt rate is 3051 * increasing 3052 */ 3053 new_itr = new_itr > q_vector->itr_val ? 3054 max((new_itr * q_vector->itr_val) / 3055 (new_itr + (q_vector->itr_val >> 2)), 3056 new_itr) : new_itr; 3057 /* Don't write the value here; it resets the adapter's 3058 * internal timer, and causes us to delay far longer than 3059 * we should between interrupts. Instead, we write the ITR 3060 * value at the beginning of the next interrupt so the timing 3061 * ends up being correct. 3062 */ 3063 q_vector->itr_val = new_itr; 3064 q_vector->set_itr = 1; 3065 } 3066 } 3067 3068 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3069 { 3070 int v_idx = adapter->num_q_vectors; 3071 3072 if (adapter->msix_entries) { 3073 pci_disable_msix(adapter->pdev); 3074 kfree(adapter->msix_entries); 3075 adapter->msix_entries = NULL; 3076 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3077 pci_disable_msi(adapter->pdev); 3078 } 3079 3080 while (v_idx--) 3081 igc_reset_q_vector(adapter, v_idx); 3082 } 3083 3084 /** 3085 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3086 * @adapter: Pointer to adapter structure 3087 * @msix: boolean value for MSI-X capability 3088 * 3089 * Attempt to configure interrupts using the best available 3090 * capabilities of the hardware and kernel. 3091 */ 3092 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3093 bool msix) 3094 { 3095 int numvecs, i; 3096 int err; 3097 3098 if (!msix) 3099 goto msi_only; 3100 adapter->flags |= IGC_FLAG_HAS_MSIX; 3101 3102 /* Number of supported queues. */ 3103 adapter->num_rx_queues = adapter->rss_queues; 3104 3105 adapter->num_tx_queues = adapter->rss_queues; 3106 3107 /* start with one vector for every Rx queue */ 3108 numvecs = adapter->num_rx_queues; 3109 3110 /* if Tx handler is separate add 1 for every Tx queue */ 3111 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3112 numvecs += adapter->num_tx_queues; 3113 3114 /* store the number of vectors reserved for queues */ 3115 adapter->num_q_vectors = numvecs; 3116 3117 /* add 1 vector for link status interrupts */ 3118 numvecs++; 3119 3120 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3121 GFP_KERNEL); 3122 3123 if (!adapter->msix_entries) 3124 return; 3125 3126 /* populate entry values */ 3127 for (i = 0; i < numvecs; i++) 3128 adapter->msix_entries[i].entry = i; 3129 3130 err = pci_enable_msix_range(adapter->pdev, 3131 adapter->msix_entries, 3132 numvecs, 3133 numvecs); 3134 if (err > 0) 3135 return; 3136 3137 kfree(adapter->msix_entries); 3138 adapter->msix_entries = NULL; 3139 3140 igc_reset_interrupt_capability(adapter); 3141 3142 msi_only: 3143 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3144 3145 adapter->rss_queues = 1; 3146 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3147 adapter->num_rx_queues = 1; 3148 adapter->num_tx_queues = 1; 3149 adapter->num_q_vectors = 1; 3150 if (!pci_enable_msi(adapter->pdev)) 3151 adapter->flags |= IGC_FLAG_HAS_MSI; 3152 } 3153 3154 /** 3155 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3156 * @q_vector: pointer to q_vector 3157 * 3158 * Stores a new ITR value based on strictly on packet size. This 3159 * algorithm is less sophisticated than that used in igc_update_itr, 3160 * due to the difficulty of synchronizing statistics across multiple 3161 * receive rings. The divisors and thresholds used by this function 3162 * were determined based on theoretical maximum wire speed and testing 3163 * data, in order to minimize response time while increasing bulk 3164 * throughput. 3165 * NOTE: This function is called only when operating in a multiqueue 3166 * receive environment. 3167 */ 3168 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3169 { 3170 struct igc_adapter *adapter = q_vector->adapter; 3171 int new_val = q_vector->itr_val; 3172 int avg_wire_size = 0; 3173 unsigned int packets; 3174 3175 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3176 * ints/sec - ITR timer value of 120 ticks. 3177 */ 3178 switch (adapter->link_speed) { 3179 case SPEED_10: 3180 case SPEED_100: 3181 new_val = IGC_4K_ITR; 3182 goto set_itr_val; 3183 default: 3184 break; 3185 } 3186 3187 packets = q_vector->rx.total_packets; 3188 if (packets) 3189 avg_wire_size = q_vector->rx.total_bytes / packets; 3190 3191 packets = q_vector->tx.total_packets; 3192 if (packets) 3193 avg_wire_size = max_t(u32, avg_wire_size, 3194 q_vector->tx.total_bytes / packets); 3195 3196 /* if avg_wire_size isn't set no work was done */ 3197 if (!avg_wire_size) 3198 goto clear_counts; 3199 3200 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3201 avg_wire_size += 24; 3202 3203 /* Don't starve jumbo frames */ 3204 avg_wire_size = min(avg_wire_size, 3000); 3205 3206 /* Give a little boost to mid-size frames */ 3207 if (avg_wire_size > 300 && avg_wire_size < 1200) 3208 new_val = avg_wire_size / 3; 3209 else 3210 new_val = avg_wire_size / 2; 3211 3212 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3213 if (new_val < IGC_20K_ITR && 3214 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3215 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3216 new_val = IGC_20K_ITR; 3217 3218 set_itr_val: 3219 if (new_val != q_vector->itr_val) { 3220 q_vector->itr_val = new_val; 3221 q_vector->set_itr = 1; 3222 } 3223 clear_counts: 3224 q_vector->rx.total_bytes = 0; 3225 q_vector->rx.total_packets = 0; 3226 q_vector->tx.total_bytes = 0; 3227 q_vector->tx.total_packets = 0; 3228 } 3229 3230 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3231 { 3232 struct igc_adapter *adapter = q_vector->adapter; 3233 struct igc_hw *hw = &adapter->hw; 3234 3235 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3236 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3237 if (adapter->num_q_vectors == 1) 3238 igc_set_itr(q_vector); 3239 else 3240 igc_update_ring_itr(q_vector); 3241 } 3242 3243 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3244 if (adapter->msix_entries) 3245 wr32(IGC_EIMS, q_vector->eims_value); 3246 else 3247 igc_irq_enable(adapter); 3248 } 3249 } 3250 3251 static void igc_add_ring(struct igc_ring *ring, 3252 struct igc_ring_container *head) 3253 { 3254 head->ring = ring; 3255 head->count++; 3256 } 3257 3258 /** 3259 * igc_cache_ring_register - Descriptor ring to register mapping 3260 * @adapter: board private structure to initialize 3261 * 3262 * Once we know the feature-set enabled for the device, we'll cache 3263 * the register offset the descriptor ring is assigned to. 3264 */ 3265 static void igc_cache_ring_register(struct igc_adapter *adapter) 3266 { 3267 int i = 0, j = 0; 3268 3269 switch (adapter->hw.mac.type) { 3270 case igc_i225: 3271 default: 3272 for (; i < adapter->num_rx_queues; i++) 3273 adapter->rx_ring[i]->reg_idx = i; 3274 for (; j < adapter->num_tx_queues; j++) 3275 adapter->tx_ring[j]->reg_idx = j; 3276 break; 3277 } 3278 } 3279 3280 /** 3281 * igc_poll - NAPI Rx polling callback 3282 * @napi: napi polling structure 3283 * @budget: count of how many packets we should handle 3284 */ 3285 static int igc_poll(struct napi_struct *napi, int budget) 3286 { 3287 struct igc_q_vector *q_vector = container_of(napi, 3288 struct igc_q_vector, 3289 napi); 3290 bool clean_complete = true; 3291 int work_done = 0; 3292 3293 if (q_vector->tx.ring) 3294 clean_complete = igc_clean_tx_irq(q_vector, budget); 3295 3296 if (q_vector->rx.ring) { 3297 int cleaned = igc_clean_rx_irq(q_vector, budget); 3298 3299 work_done += cleaned; 3300 if (cleaned >= budget) 3301 clean_complete = false; 3302 } 3303 3304 /* If all work not completed, return budget and keep polling */ 3305 if (!clean_complete) 3306 return budget; 3307 3308 /* Exit the polling mode, but don't re-enable interrupts if stack might 3309 * poll us due to busy-polling 3310 */ 3311 if (likely(napi_complete_done(napi, work_done))) 3312 igc_ring_irq_enable(q_vector); 3313 3314 return min(work_done, budget - 1); 3315 } 3316 3317 /** 3318 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3319 * @adapter: board private structure to initialize 3320 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3321 * @v_idx: index of vector in adapter struct 3322 * @txr_count: total number of Tx rings to allocate 3323 * @txr_idx: index of first Tx ring to allocate 3324 * @rxr_count: total number of Rx rings to allocate 3325 * @rxr_idx: index of first Rx ring to allocate 3326 * 3327 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3328 */ 3329 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3330 unsigned int v_count, unsigned int v_idx, 3331 unsigned int txr_count, unsigned int txr_idx, 3332 unsigned int rxr_count, unsigned int rxr_idx) 3333 { 3334 struct igc_q_vector *q_vector; 3335 struct igc_ring *ring; 3336 int ring_count; 3337 3338 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3339 if (txr_count > 1 || rxr_count > 1) 3340 return -ENOMEM; 3341 3342 ring_count = txr_count + rxr_count; 3343 3344 /* allocate q_vector and rings */ 3345 q_vector = adapter->q_vector[v_idx]; 3346 if (!q_vector) 3347 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3348 GFP_KERNEL); 3349 else 3350 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3351 if (!q_vector) 3352 return -ENOMEM; 3353 3354 /* initialize NAPI */ 3355 netif_napi_add(adapter->netdev, &q_vector->napi, 3356 igc_poll, 64); 3357 3358 /* tie q_vector and adapter together */ 3359 adapter->q_vector[v_idx] = q_vector; 3360 q_vector->adapter = adapter; 3361 3362 /* initialize work limits */ 3363 q_vector->tx.work_limit = adapter->tx_work_limit; 3364 3365 /* initialize ITR configuration */ 3366 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3367 q_vector->itr_val = IGC_START_ITR; 3368 3369 /* initialize pointer to rings */ 3370 ring = q_vector->ring; 3371 3372 /* initialize ITR */ 3373 if (rxr_count) { 3374 /* rx or rx/tx vector */ 3375 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3376 q_vector->itr_val = adapter->rx_itr_setting; 3377 } else { 3378 /* tx only vector */ 3379 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3380 q_vector->itr_val = adapter->tx_itr_setting; 3381 } 3382 3383 if (txr_count) { 3384 /* assign generic ring traits */ 3385 ring->dev = &adapter->pdev->dev; 3386 ring->netdev = adapter->netdev; 3387 3388 /* configure backlink on ring */ 3389 ring->q_vector = q_vector; 3390 3391 /* update q_vector Tx values */ 3392 igc_add_ring(ring, &q_vector->tx); 3393 3394 /* apply Tx specific ring traits */ 3395 ring->count = adapter->tx_ring_count; 3396 ring->queue_index = txr_idx; 3397 3398 /* assign ring to adapter */ 3399 adapter->tx_ring[txr_idx] = ring; 3400 3401 /* push pointer to next ring */ 3402 ring++; 3403 } 3404 3405 if (rxr_count) { 3406 /* assign generic ring traits */ 3407 ring->dev = &adapter->pdev->dev; 3408 ring->netdev = adapter->netdev; 3409 3410 /* configure backlink on ring */ 3411 ring->q_vector = q_vector; 3412 3413 /* update q_vector Rx values */ 3414 igc_add_ring(ring, &q_vector->rx); 3415 3416 /* apply Rx specific ring traits */ 3417 ring->count = adapter->rx_ring_count; 3418 ring->queue_index = rxr_idx; 3419 3420 /* assign ring to adapter */ 3421 adapter->rx_ring[rxr_idx] = ring; 3422 } 3423 3424 return 0; 3425 } 3426 3427 /** 3428 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3429 * @adapter: board private structure to initialize 3430 * 3431 * We allocate one q_vector per queue interrupt. If allocation fails we 3432 * return -ENOMEM. 3433 */ 3434 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3435 { 3436 int rxr_remaining = adapter->num_rx_queues; 3437 int txr_remaining = adapter->num_tx_queues; 3438 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3439 int q_vectors = adapter->num_q_vectors; 3440 int err; 3441 3442 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3443 for (; rxr_remaining; v_idx++) { 3444 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3445 0, 0, 1, rxr_idx); 3446 3447 if (err) 3448 goto err_out; 3449 3450 /* update counts and index */ 3451 rxr_remaining--; 3452 rxr_idx++; 3453 } 3454 } 3455 3456 for (; v_idx < q_vectors; v_idx++) { 3457 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3458 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3459 3460 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3461 tqpv, txr_idx, rqpv, rxr_idx); 3462 3463 if (err) 3464 goto err_out; 3465 3466 /* update counts and index */ 3467 rxr_remaining -= rqpv; 3468 txr_remaining -= tqpv; 3469 rxr_idx++; 3470 txr_idx++; 3471 } 3472 3473 return 0; 3474 3475 err_out: 3476 adapter->num_tx_queues = 0; 3477 adapter->num_rx_queues = 0; 3478 adapter->num_q_vectors = 0; 3479 3480 while (v_idx--) 3481 igc_free_q_vector(adapter, v_idx); 3482 3483 return -ENOMEM; 3484 } 3485 3486 /** 3487 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3488 * @adapter: Pointer to adapter structure 3489 * @msix: boolean for MSI-X capability 3490 * 3491 * This function initializes the interrupts and allocates all of the queues. 3492 */ 3493 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3494 { 3495 struct net_device *dev = adapter->netdev; 3496 int err = 0; 3497 3498 igc_set_interrupt_capability(adapter, msix); 3499 3500 err = igc_alloc_q_vectors(adapter); 3501 if (err) { 3502 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3503 goto err_alloc_q_vectors; 3504 } 3505 3506 igc_cache_ring_register(adapter); 3507 3508 return 0; 3509 3510 err_alloc_q_vectors: 3511 igc_reset_interrupt_capability(adapter); 3512 return err; 3513 } 3514 3515 /** 3516 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3517 * @adapter: board private structure to initialize 3518 * 3519 * igc_sw_init initializes the Adapter private data structure. 3520 * Fields are initialized based on PCI device information and 3521 * OS network device settings (MTU size). 3522 */ 3523 static int igc_sw_init(struct igc_adapter *adapter) 3524 { 3525 struct net_device *netdev = adapter->netdev; 3526 struct pci_dev *pdev = adapter->pdev; 3527 struct igc_hw *hw = &adapter->hw; 3528 3529 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3530 3531 /* set default ring sizes */ 3532 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3533 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3534 3535 /* set default ITR values */ 3536 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3537 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3538 3539 /* set default work limits */ 3540 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3541 3542 /* adjust max frame to be at least the size of a standard frame */ 3543 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3544 VLAN_HLEN; 3545 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3546 3547 mutex_init(&adapter->nfc_rule_lock); 3548 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3549 adapter->nfc_rule_count = 0; 3550 3551 spin_lock_init(&adapter->stats64_lock); 3552 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3553 adapter->flags |= IGC_FLAG_HAS_MSIX; 3554 3555 igc_init_queue_configuration(adapter); 3556 3557 /* This call may decrease the number of queues */ 3558 if (igc_init_interrupt_scheme(adapter, true)) { 3559 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3560 return -ENOMEM; 3561 } 3562 3563 /* Explicitly disable IRQ since the NIC can be in any state. */ 3564 igc_irq_disable(adapter); 3565 3566 set_bit(__IGC_DOWN, &adapter->state); 3567 3568 return 0; 3569 } 3570 3571 /** 3572 * igc_up - Open the interface and prepare it to handle traffic 3573 * @adapter: board private structure 3574 */ 3575 void igc_up(struct igc_adapter *adapter) 3576 { 3577 struct igc_hw *hw = &adapter->hw; 3578 int i = 0; 3579 3580 /* hardware has been reset, we need to reload some things */ 3581 igc_configure(adapter); 3582 3583 clear_bit(__IGC_DOWN, &adapter->state); 3584 3585 for (i = 0; i < adapter->num_q_vectors; i++) 3586 napi_enable(&adapter->q_vector[i]->napi); 3587 3588 if (adapter->msix_entries) 3589 igc_configure_msix(adapter); 3590 else 3591 igc_assign_vector(adapter->q_vector[0], 0); 3592 3593 /* Clear any pending interrupts. */ 3594 rd32(IGC_ICR); 3595 igc_irq_enable(adapter); 3596 3597 netif_tx_start_all_queues(adapter->netdev); 3598 3599 /* start the watchdog. */ 3600 hw->mac.get_link_status = 1; 3601 schedule_work(&adapter->watchdog_task); 3602 } 3603 3604 /** 3605 * igc_update_stats - Update the board statistics counters 3606 * @adapter: board private structure 3607 */ 3608 void igc_update_stats(struct igc_adapter *adapter) 3609 { 3610 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3611 struct pci_dev *pdev = adapter->pdev; 3612 struct igc_hw *hw = &adapter->hw; 3613 u64 _bytes, _packets; 3614 u64 bytes, packets; 3615 unsigned int start; 3616 u32 mpc; 3617 int i; 3618 3619 /* Prevent stats update while adapter is being reset, or if the pci 3620 * connection is down. 3621 */ 3622 if (adapter->link_speed == 0) 3623 return; 3624 if (pci_channel_offline(pdev)) 3625 return; 3626 3627 packets = 0; 3628 bytes = 0; 3629 3630 rcu_read_lock(); 3631 for (i = 0; i < adapter->num_rx_queues; i++) { 3632 struct igc_ring *ring = adapter->rx_ring[i]; 3633 u32 rqdpc = rd32(IGC_RQDPC(i)); 3634 3635 if (hw->mac.type >= igc_i225) 3636 wr32(IGC_RQDPC(i), 0); 3637 3638 if (rqdpc) { 3639 ring->rx_stats.drops += rqdpc; 3640 net_stats->rx_fifo_errors += rqdpc; 3641 } 3642 3643 do { 3644 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3645 _bytes = ring->rx_stats.bytes; 3646 _packets = ring->rx_stats.packets; 3647 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3648 bytes += _bytes; 3649 packets += _packets; 3650 } 3651 3652 net_stats->rx_bytes = bytes; 3653 net_stats->rx_packets = packets; 3654 3655 packets = 0; 3656 bytes = 0; 3657 for (i = 0; i < adapter->num_tx_queues; i++) { 3658 struct igc_ring *ring = adapter->tx_ring[i]; 3659 3660 do { 3661 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3662 _bytes = ring->tx_stats.bytes; 3663 _packets = ring->tx_stats.packets; 3664 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3665 bytes += _bytes; 3666 packets += _packets; 3667 } 3668 net_stats->tx_bytes = bytes; 3669 net_stats->tx_packets = packets; 3670 rcu_read_unlock(); 3671 3672 /* read stats registers */ 3673 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3674 adapter->stats.gprc += rd32(IGC_GPRC); 3675 adapter->stats.gorc += rd32(IGC_GORCL); 3676 rd32(IGC_GORCH); /* clear GORCL */ 3677 adapter->stats.bprc += rd32(IGC_BPRC); 3678 adapter->stats.mprc += rd32(IGC_MPRC); 3679 adapter->stats.roc += rd32(IGC_ROC); 3680 3681 adapter->stats.prc64 += rd32(IGC_PRC64); 3682 adapter->stats.prc127 += rd32(IGC_PRC127); 3683 adapter->stats.prc255 += rd32(IGC_PRC255); 3684 adapter->stats.prc511 += rd32(IGC_PRC511); 3685 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3686 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3687 adapter->stats.tlpic += rd32(IGC_TLPIC); 3688 adapter->stats.rlpic += rd32(IGC_RLPIC); 3689 3690 mpc = rd32(IGC_MPC); 3691 adapter->stats.mpc += mpc; 3692 net_stats->rx_fifo_errors += mpc; 3693 adapter->stats.scc += rd32(IGC_SCC); 3694 adapter->stats.ecol += rd32(IGC_ECOL); 3695 adapter->stats.mcc += rd32(IGC_MCC); 3696 adapter->stats.latecol += rd32(IGC_LATECOL); 3697 adapter->stats.dc += rd32(IGC_DC); 3698 adapter->stats.rlec += rd32(IGC_RLEC); 3699 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3700 adapter->stats.xontxc += rd32(IGC_XONTXC); 3701 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3702 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3703 adapter->stats.fcruc += rd32(IGC_FCRUC); 3704 adapter->stats.gptc += rd32(IGC_GPTC); 3705 adapter->stats.gotc += rd32(IGC_GOTCL); 3706 rd32(IGC_GOTCH); /* clear GOTCL */ 3707 adapter->stats.rnbc += rd32(IGC_RNBC); 3708 adapter->stats.ruc += rd32(IGC_RUC); 3709 adapter->stats.rfc += rd32(IGC_RFC); 3710 adapter->stats.rjc += rd32(IGC_RJC); 3711 adapter->stats.tor += rd32(IGC_TORH); 3712 adapter->stats.tot += rd32(IGC_TOTH); 3713 adapter->stats.tpr += rd32(IGC_TPR); 3714 3715 adapter->stats.ptc64 += rd32(IGC_PTC64); 3716 adapter->stats.ptc127 += rd32(IGC_PTC127); 3717 adapter->stats.ptc255 += rd32(IGC_PTC255); 3718 adapter->stats.ptc511 += rd32(IGC_PTC511); 3719 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3720 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3721 3722 adapter->stats.mptc += rd32(IGC_MPTC); 3723 adapter->stats.bptc += rd32(IGC_BPTC); 3724 3725 adapter->stats.tpt += rd32(IGC_TPT); 3726 adapter->stats.colc += rd32(IGC_COLC); 3727 adapter->stats.colc += rd32(IGC_RERC); 3728 3729 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3730 3731 adapter->stats.tsctc += rd32(IGC_TSCTC); 3732 3733 adapter->stats.iac += rd32(IGC_IAC); 3734 3735 /* Fill out the OS statistics structure */ 3736 net_stats->multicast = adapter->stats.mprc; 3737 net_stats->collisions = adapter->stats.colc; 3738 3739 /* Rx Errors */ 3740 3741 /* RLEC on some newer hardware can be incorrect so build 3742 * our own version based on RUC and ROC 3743 */ 3744 net_stats->rx_errors = adapter->stats.rxerrc + 3745 adapter->stats.crcerrs + adapter->stats.algnerrc + 3746 adapter->stats.ruc + adapter->stats.roc + 3747 adapter->stats.cexterr; 3748 net_stats->rx_length_errors = adapter->stats.ruc + 3749 adapter->stats.roc; 3750 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3751 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3752 net_stats->rx_missed_errors = adapter->stats.mpc; 3753 3754 /* Tx Errors */ 3755 net_stats->tx_errors = adapter->stats.ecol + 3756 adapter->stats.latecol; 3757 net_stats->tx_aborted_errors = adapter->stats.ecol; 3758 net_stats->tx_window_errors = adapter->stats.latecol; 3759 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3760 3761 /* Tx Dropped needs to be maintained elsewhere */ 3762 3763 /* Management Stats */ 3764 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3765 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3766 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3767 } 3768 3769 /** 3770 * igc_down - Close the interface 3771 * @adapter: board private structure 3772 */ 3773 void igc_down(struct igc_adapter *adapter) 3774 { 3775 struct net_device *netdev = adapter->netdev; 3776 struct igc_hw *hw = &adapter->hw; 3777 u32 tctl, rctl; 3778 int i = 0; 3779 3780 set_bit(__IGC_DOWN, &adapter->state); 3781 3782 igc_ptp_suspend(adapter); 3783 3784 /* disable receives in the hardware */ 3785 rctl = rd32(IGC_RCTL); 3786 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3787 /* flush and sleep below */ 3788 3789 /* set trans_start so we don't get spurious watchdogs during reset */ 3790 netif_trans_update(netdev); 3791 3792 netif_carrier_off(netdev); 3793 netif_tx_stop_all_queues(netdev); 3794 3795 /* disable transmits in the hardware */ 3796 tctl = rd32(IGC_TCTL); 3797 tctl &= ~IGC_TCTL_EN; 3798 wr32(IGC_TCTL, tctl); 3799 /* flush both disables and wait for them to finish */ 3800 wrfl(); 3801 usleep_range(10000, 20000); 3802 3803 igc_irq_disable(adapter); 3804 3805 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3806 3807 for (i = 0; i < adapter->num_q_vectors; i++) { 3808 if (adapter->q_vector[i]) { 3809 napi_synchronize(&adapter->q_vector[i]->napi); 3810 napi_disable(&adapter->q_vector[i]->napi); 3811 } 3812 } 3813 3814 del_timer_sync(&adapter->watchdog_timer); 3815 del_timer_sync(&adapter->phy_info_timer); 3816 3817 /* record the stats before reset*/ 3818 spin_lock(&adapter->stats64_lock); 3819 igc_update_stats(adapter); 3820 spin_unlock(&adapter->stats64_lock); 3821 3822 adapter->link_speed = 0; 3823 adapter->link_duplex = 0; 3824 3825 if (!pci_channel_offline(adapter->pdev)) 3826 igc_reset(adapter); 3827 3828 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3829 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3830 3831 igc_clean_all_tx_rings(adapter); 3832 igc_clean_all_rx_rings(adapter); 3833 } 3834 3835 void igc_reinit_locked(struct igc_adapter *adapter) 3836 { 3837 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3838 usleep_range(1000, 2000); 3839 igc_down(adapter); 3840 igc_up(adapter); 3841 clear_bit(__IGC_RESETTING, &adapter->state); 3842 } 3843 3844 static void igc_reset_task(struct work_struct *work) 3845 { 3846 struct igc_adapter *adapter; 3847 3848 adapter = container_of(work, struct igc_adapter, reset_task); 3849 3850 igc_rings_dump(adapter); 3851 igc_regs_dump(adapter); 3852 netdev_err(adapter->netdev, "Reset adapter\n"); 3853 igc_reinit_locked(adapter); 3854 } 3855 3856 /** 3857 * igc_change_mtu - Change the Maximum Transfer Unit 3858 * @netdev: network interface device structure 3859 * @new_mtu: new value for maximum frame size 3860 * 3861 * Returns 0 on success, negative on failure 3862 */ 3863 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3864 { 3865 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3866 struct igc_adapter *adapter = netdev_priv(netdev); 3867 3868 /* adjust max frame to be at least the size of a standard frame */ 3869 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3870 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3871 3872 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3873 usleep_range(1000, 2000); 3874 3875 /* igc_down has a dependency on max_frame_size */ 3876 adapter->max_frame_size = max_frame; 3877 3878 if (netif_running(netdev)) 3879 igc_down(adapter); 3880 3881 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3882 netdev->mtu = new_mtu; 3883 3884 if (netif_running(netdev)) 3885 igc_up(adapter); 3886 else 3887 igc_reset(adapter); 3888 3889 clear_bit(__IGC_RESETTING, &adapter->state); 3890 3891 return 0; 3892 } 3893 3894 /** 3895 * igc_get_stats64 - Get System Network Statistics 3896 * @netdev: network interface device structure 3897 * @stats: rtnl_link_stats64 pointer 3898 * 3899 * Returns the address of the device statistics structure. 3900 * The statistics are updated here and also from the timer callback. 3901 */ 3902 static void igc_get_stats64(struct net_device *netdev, 3903 struct rtnl_link_stats64 *stats) 3904 { 3905 struct igc_adapter *adapter = netdev_priv(netdev); 3906 3907 spin_lock(&adapter->stats64_lock); 3908 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3909 igc_update_stats(adapter); 3910 memcpy(stats, &adapter->stats64, sizeof(*stats)); 3911 spin_unlock(&adapter->stats64_lock); 3912 } 3913 3914 static netdev_features_t igc_fix_features(struct net_device *netdev, 3915 netdev_features_t features) 3916 { 3917 /* Since there is no support for separate Rx/Tx vlan accel 3918 * enable/disable make sure Tx flag is always in same state as Rx. 3919 */ 3920 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3921 features |= NETIF_F_HW_VLAN_CTAG_TX; 3922 else 3923 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3924 3925 return features; 3926 } 3927 3928 static int igc_set_features(struct net_device *netdev, 3929 netdev_features_t features) 3930 { 3931 netdev_features_t changed = netdev->features ^ features; 3932 struct igc_adapter *adapter = netdev_priv(netdev); 3933 3934 /* Add VLAN support */ 3935 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3936 return 0; 3937 3938 if (!(features & NETIF_F_NTUPLE)) 3939 igc_flush_nfc_rules(adapter); 3940 3941 netdev->features = features; 3942 3943 if (netif_running(netdev)) 3944 igc_reinit_locked(adapter); 3945 else 3946 igc_reset(adapter); 3947 3948 return 1; 3949 } 3950 3951 static netdev_features_t 3952 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3953 netdev_features_t features) 3954 { 3955 unsigned int network_hdr_len, mac_hdr_len; 3956 3957 /* Make certain the headers can be described by a context descriptor */ 3958 mac_hdr_len = skb_network_header(skb) - skb->data; 3959 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3960 return features & ~(NETIF_F_HW_CSUM | 3961 NETIF_F_SCTP_CRC | 3962 NETIF_F_HW_VLAN_CTAG_TX | 3963 NETIF_F_TSO | 3964 NETIF_F_TSO6); 3965 3966 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3967 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3968 return features & ~(NETIF_F_HW_CSUM | 3969 NETIF_F_SCTP_CRC | 3970 NETIF_F_TSO | 3971 NETIF_F_TSO6); 3972 3973 /* We can only support IPv4 TSO in tunnels if we can mangle the 3974 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3975 */ 3976 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3977 features &= ~NETIF_F_TSO; 3978 3979 return features; 3980 } 3981 3982 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3983 { 3984 struct igc_hw *hw = &adapter->hw; 3985 u32 tsicr = rd32(IGC_TSICR); 3986 u32 ack = 0; 3987 3988 if (tsicr & IGC_TSICR_TXTS) { 3989 /* retrieve hardware timestamp */ 3990 schedule_work(&adapter->ptp_tx_work); 3991 ack |= IGC_TSICR_TXTS; 3992 } 3993 3994 /* acknowledge the interrupts */ 3995 wr32(IGC_TSICR, ack); 3996 } 3997 3998 /** 3999 * igc_msix_other - msix other interrupt handler 4000 * @irq: interrupt number 4001 * @data: pointer to a q_vector 4002 */ 4003 static irqreturn_t igc_msix_other(int irq, void *data) 4004 { 4005 struct igc_adapter *adapter = data; 4006 struct igc_hw *hw = &adapter->hw; 4007 u32 icr = rd32(IGC_ICR); 4008 4009 /* reading ICR causes bit 31 of EICR to be cleared */ 4010 if (icr & IGC_ICR_DRSTA) 4011 schedule_work(&adapter->reset_task); 4012 4013 if (icr & IGC_ICR_DOUTSYNC) { 4014 /* HW is reporting DMA is out of sync */ 4015 adapter->stats.doosync++; 4016 } 4017 4018 if (icr & IGC_ICR_LSC) { 4019 hw->mac.get_link_status = 1; 4020 /* guard against interrupt when we're going down */ 4021 if (!test_bit(__IGC_DOWN, &adapter->state)) 4022 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4023 } 4024 4025 if (icr & IGC_ICR_TS) 4026 igc_tsync_interrupt(adapter); 4027 4028 wr32(IGC_EIMS, adapter->eims_other); 4029 4030 return IRQ_HANDLED; 4031 } 4032 4033 static void igc_write_itr(struct igc_q_vector *q_vector) 4034 { 4035 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4036 4037 if (!q_vector->set_itr) 4038 return; 4039 4040 if (!itr_val) 4041 itr_val = IGC_ITR_VAL_MASK; 4042 4043 itr_val |= IGC_EITR_CNT_IGNR; 4044 4045 writel(itr_val, q_vector->itr_register); 4046 q_vector->set_itr = 0; 4047 } 4048 4049 static irqreturn_t igc_msix_ring(int irq, void *data) 4050 { 4051 struct igc_q_vector *q_vector = data; 4052 4053 /* Write the ITR value calculated from the previous interrupt. */ 4054 igc_write_itr(q_vector); 4055 4056 napi_schedule(&q_vector->napi); 4057 4058 return IRQ_HANDLED; 4059 } 4060 4061 /** 4062 * igc_request_msix - Initialize MSI-X interrupts 4063 * @adapter: Pointer to adapter structure 4064 * 4065 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4066 * kernel. 4067 */ 4068 static int igc_request_msix(struct igc_adapter *adapter) 4069 { 4070 int i = 0, err = 0, vector = 0, free_vector = 0; 4071 struct net_device *netdev = adapter->netdev; 4072 4073 err = request_irq(adapter->msix_entries[vector].vector, 4074 &igc_msix_other, 0, netdev->name, adapter); 4075 if (err) 4076 goto err_out; 4077 4078 for (i = 0; i < adapter->num_q_vectors; i++) { 4079 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4080 4081 vector++; 4082 4083 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4084 4085 if (q_vector->rx.ring && q_vector->tx.ring) 4086 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4087 q_vector->rx.ring->queue_index); 4088 else if (q_vector->tx.ring) 4089 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4090 q_vector->tx.ring->queue_index); 4091 else if (q_vector->rx.ring) 4092 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4093 q_vector->rx.ring->queue_index); 4094 else 4095 sprintf(q_vector->name, "%s-unused", netdev->name); 4096 4097 err = request_irq(adapter->msix_entries[vector].vector, 4098 igc_msix_ring, 0, q_vector->name, 4099 q_vector); 4100 if (err) 4101 goto err_free; 4102 } 4103 4104 igc_configure_msix(adapter); 4105 return 0; 4106 4107 err_free: 4108 /* free already assigned IRQs */ 4109 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4110 4111 vector--; 4112 for (i = 0; i < vector; i++) { 4113 free_irq(adapter->msix_entries[free_vector++].vector, 4114 adapter->q_vector[i]); 4115 } 4116 err_out: 4117 return err; 4118 } 4119 4120 /** 4121 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4122 * @adapter: Pointer to adapter structure 4123 * 4124 * This function resets the device so that it has 0 rx queues, tx queues, and 4125 * MSI-X interrupts allocated. 4126 */ 4127 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4128 { 4129 igc_free_q_vectors(adapter); 4130 igc_reset_interrupt_capability(adapter); 4131 } 4132 4133 /* Need to wait a few seconds after link up to get diagnostic information from 4134 * the phy 4135 */ 4136 static void igc_update_phy_info(struct timer_list *t) 4137 { 4138 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4139 4140 igc_get_phy_info(&adapter->hw); 4141 } 4142 4143 /** 4144 * igc_has_link - check shared code for link and determine up/down 4145 * @adapter: pointer to driver private info 4146 */ 4147 bool igc_has_link(struct igc_adapter *adapter) 4148 { 4149 struct igc_hw *hw = &adapter->hw; 4150 bool link_active = false; 4151 4152 /* get_link_status is set on LSC (link status) interrupt or 4153 * rx sequence error interrupt. get_link_status will stay 4154 * false until the igc_check_for_link establishes link 4155 * for copper adapters ONLY 4156 */ 4157 switch (hw->phy.media_type) { 4158 case igc_media_type_copper: 4159 if (!hw->mac.get_link_status) 4160 return true; 4161 hw->mac.ops.check_for_link(hw); 4162 link_active = !hw->mac.get_link_status; 4163 break; 4164 default: 4165 case igc_media_type_unknown: 4166 break; 4167 } 4168 4169 if (hw->mac.type == igc_i225 && 4170 hw->phy.id == I225_I_PHY_ID) { 4171 if (!netif_carrier_ok(adapter->netdev)) { 4172 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4173 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4174 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4175 adapter->link_check_timeout = jiffies; 4176 } 4177 } 4178 4179 return link_active; 4180 } 4181 4182 /** 4183 * igc_watchdog - Timer Call-back 4184 * @t: timer for the watchdog 4185 */ 4186 static void igc_watchdog(struct timer_list *t) 4187 { 4188 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4189 /* Do the rest outside of interrupt context */ 4190 schedule_work(&adapter->watchdog_task); 4191 } 4192 4193 static void igc_watchdog_task(struct work_struct *work) 4194 { 4195 struct igc_adapter *adapter = container_of(work, 4196 struct igc_adapter, 4197 watchdog_task); 4198 struct net_device *netdev = adapter->netdev; 4199 struct igc_hw *hw = &adapter->hw; 4200 struct igc_phy_info *phy = &hw->phy; 4201 u16 phy_data, retry_count = 20; 4202 u32 link; 4203 int i; 4204 4205 link = igc_has_link(adapter); 4206 4207 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4208 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4209 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4210 else 4211 link = false; 4212 } 4213 4214 if (link) { 4215 /* Cancel scheduled suspend requests. */ 4216 pm_runtime_resume(netdev->dev.parent); 4217 4218 if (!netif_carrier_ok(netdev)) { 4219 u32 ctrl; 4220 4221 hw->mac.ops.get_speed_and_duplex(hw, 4222 &adapter->link_speed, 4223 &adapter->link_duplex); 4224 4225 ctrl = rd32(IGC_CTRL); 4226 /* Link status message must follow this format */ 4227 netdev_info(netdev, 4228 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4229 adapter->link_speed, 4230 adapter->link_duplex == FULL_DUPLEX ? 4231 "Full" : "Half", 4232 (ctrl & IGC_CTRL_TFCE) && 4233 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4234 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4235 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4236 4237 /* disable EEE if enabled */ 4238 if ((adapter->flags & IGC_FLAG_EEE) && 4239 adapter->link_duplex == HALF_DUPLEX) { 4240 netdev_info(netdev, 4241 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 4242 adapter->hw.dev_spec._base.eee_enable = false; 4243 adapter->flags &= ~IGC_FLAG_EEE; 4244 } 4245 4246 /* check if SmartSpeed worked */ 4247 igc_check_downshift(hw); 4248 if (phy->speed_downgraded) 4249 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4250 4251 /* adjust timeout factor according to speed/duplex */ 4252 adapter->tx_timeout_factor = 1; 4253 switch (adapter->link_speed) { 4254 case SPEED_10: 4255 adapter->tx_timeout_factor = 14; 4256 break; 4257 case SPEED_100: 4258 /* maybe add some timeout factor ? */ 4259 break; 4260 } 4261 4262 if (adapter->link_speed != SPEED_1000) 4263 goto no_wait; 4264 4265 /* wait for Remote receiver status OK */ 4266 retry_read_status: 4267 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4268 &phy_data)) { 4269 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4270 retry_count) { 4271 msleep(100); 4272 retry_count--; 4273 goto retry_read_status; 4274 } else if (!retry_count) { 4275 netdev_err(netdev, "exceed max 2 second\n"); 4276 } 4277 } else { 4278 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4279 } 4280 no_wait: 4281 netif_carrier_on(netdev); 4282 4283 /* link state has changed, schedule phy info update */ 4284 if (!test_bit(__IGC_DOWN, &adapter->state)) 4285 mod_timer(&adapter->phy_info_timer, 4286 round_jiffies(jiffies + 2 * HZ)); 4287 } 4288 } else { 4289 if (netif_carrier_ok(netdev)) { 4290 adapter->link_speed = 0; 4291 adapter->link_duplex = 0; 4292 4293 /* Links status message must follow this format */ 4294 netdev_info(netdev, "NIC Link is Down\n"); 4295 netif_carrier_off(netdev); 4296 4297 /* link state has changed, schedule phy info update */ 4298 if (!test_bit(__IGC_DOWN, &adapter->state)) 4299 mod_timer(&adapter->phy_info_timer, 4300 round_jiffies(jiffies + 2 * HZ)); 4301 4302 /* link is down, time to check for alternate media */ 4303 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4304 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4305 schedule_work(&adapter->reset_task); 4306 /* return immediately */ 4307 return; 4308 } 4309 } 4310 pm_schedule_suspend(netdev->dev.parent, 4311 MSEC_PER_SEC * 5); 4312 4313 /* also check for alternate media here */ 4314 } else if (!netif_carrier_ok(netdev) && 4315 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4316 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4317 schedule_work(&adapter->reset_task); 4318 /* return immediately */ 4319 return; 4320 } 4321 } 4322 } 4323 4324 spin_lock(&adapter->stats64_lock); 4325 igc_update_stats(adapter); 4326 spin_unlock(&adapter->stats64_lock); 4327 4328 for (i = 0; i < adapter->num_tx_queues; i++) { 4329 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4330 4331 if (!netif_carrier_ok(netdev)) { 4332 /* We've lost link, so the controller stops DMA, 4333 * but we've got queued Tx work that's never going 4334 * to get done, so reset controller to flush Tx. 4335 * (Do the reset outside of interrupt context). 4336 */ 4337 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4338 adapter->tx_timeout_count++; 4339 schedule_work(&adapter->reset_task); 4340 /* return immediately since reset is imminent */ 4341 return; 4342 } 4343 } 4344 4345 /* Force detection of hung controller every watchdog period */ 4346 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4347 } 4348 4349 /* Cause software interrupt to ensure Rx ring is cleaned */ 4350 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4351 u32 eics = 0; 4352 4353 for (i = 0; i < adapter->num_q_vectors; i++) 4354 eics |= adapter->q_vector[i]->eims_value; 4355 wr32(IGC_EICS, eics); 4356 } else { 4357 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4358 } 4359 4360 igc_ptp_tx_hang(adapter); 4361 4362 /* Reset the timer */ 4363 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4364 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4365 mod_timer(&adapter->watchdog_timer, 4366 round_jiffies(jiffies + HZ)); 4367 else 4368 mod_timer(&adapter->watchdog_timer, 4369 round_jiffies(jiffies + 2 * HZ)); 4370 } 4371 } 4372 4373 /** 4374 * igc_intr_msi - Interrupt Handler 4375 * @irq: interrupt number 4376 * @data: pointer to a network interface device structure 4377 */ 4378 static irqreturn_t igc_intr_msi(int irq, void *data) 4379 { 4380 struct igc_adapter *adapter = data; 4381 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4382 struct igc_hw *hw = &adapter->hw; 4383 /* read ICR disables interrupts using IAM */ 4384 u32 icr = rd32(IGC_ICR); 4385 4386 igc_write_itr(q_vector); 4387 4388 if (icr & IGC_ICR_DRSTA) 4389 schedule_work(&adapter->reset_task); 4390 4391 if (icr & IGC_ICR_DOUTSYNC) { 4392 /* HW is reporting DMA is out of sync */ 4393 adapter->stats.doosync++; 4394 } 4395 4396 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4397 hw->mac.get_link_status = 1; 4398 if (!test_bit(__IGC_DOWN, &adapter->state)) 4399 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4400 } 4401 4402 napi_schedule(&q_vector->napi); 4403 4404 return IRQ_HANDLED; 4405 } 4406 4407 /** 4408 * igc_intr - Legacy Interrupt Handler 4409 * @irq: interrupt number 4410 * @data: pointer to a network interface device structure 4411 */ 4412 static irqreturn_t igc_intr(int irq, void *data) 4413 { 4414 struct igc_adapter *adapter = data; 4415 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4416 struct igc_hw *hw = &adapter->hw; 4417 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4418 * need for the IMC write 4419 */ 4420 u32 icr = rd32(IGC_ICR); 4421 4422 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4423 * not set, then the adapter didn't send an interrupt 4424 */ 4425 if (!(icr & IGC_ICR_INT_ASSERTED)) 4426 return IRQ_NONE; 4427 4428 igc_write_itr(q_vector); 4429 4430 if (icr & IGC_ICR_DRSTA) 4431 schedule_work(&adapter->reset_task); 4432 4433 if (icr & IGC_ICR_DOUTSYNC) { 4434 /* HW is reporting DMA is out of sync */ 4435 adapter->stats.doosync++; 4436 } 4437 4438 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4439 hw->mac.get_link_status = 1; 4440 /* guard against interrupt when we're going down */ 4441 if (!test_bit(__IGC_DOWN, &adapter->state)) 4442 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4443 } 4444 4445 napi_schedule(&q_vector->napi); 4446 4447 return IRQ_HANDLED; 4448 } 4449 4450 static void igc_free_irq(struct igc_adapter *adapter) 4451 { 4452 if (adapter->msix_entries) { 4453 int vector = 0, i; 4454 4455 free_irq(adapter->msix_entries[vector++].vector, adapter); 4456 4457 for (i = 0; i < adapter->num_q_vectors; i++) 4458 free_irq(adapter->msix_entries[vector++].vector, 4459 adapter->q_vector[i]); 4460 } else { 4461 free_irq(adapter->pdev->irq, adapter); 4462 } 4463 } 4464 4465 /** 4466 * igc_request_irq - initialize interrupts 4467 * @adapter: Pointer to adapter structure 4468 * 4469 * Attempts to configure interrupts using the best available 4470 * capabilities of the hardware and kernel. 4471 */ 4472 static int igc_request_irq(struct igc_adapter *adapter) 4473 { 4474 struct net_device *netdev = adapter->netdev; 4475 struct pci_dev *pdev = adapter->pdev; 4476 int err = 0; 4477 4478 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4479 err = igc_request_msix(adapter); 4480 if (!err) 4481 goto request_done; 4482 /* fall back to MSI */ 4483 igc_free_all_tx_resources(adapter); 4484 igc_free_all_rx_resources(adapter); 4485 4486 igc_clear_interrupt_scheme(adapter); 4487 err = igc_init_interrupt_scheme(adapter, false); 4488 if (err) 4489 goto request_done; 4490 igc_setup_all_tx_resources(adapter); 4491 igc_setup_all_rx_resources(adapter); 4492 igc_configure(adapter); 4493 } 4494 4495 igc_assign_vector(adapter->q_vector[0], 0); 4496 4497 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4498 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4499 netdev->name, adapter); 4500 if (!err) 4501 goto request_done; 4502 4503 /* fall back to legacy interrupts */ 4504 igc_reset_interrupt_capability(adapter); 4505 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4506 } 4507 4508 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4509 netdev->name, adapter); 4510 4511 if (err) 4512 netdev_err(netdev, "Error %d getting interrupt\n", err); 4513 4514 request_done: 4515 return err; 4516 } 4517 4518 /** 4519 * __igc_open - Called when a network interface is made active 4520 * @netdev: network interface device structure 4521 * @resuming: boolean indicating if the device is resuming 4522 * 4523 * Returns 0 on success, negative value on failure 4524 * 4525 * The open entry point is called when a network interface is made 4526 * active by the system (IFF_UP). At this point all resources needed 4527 * for transmit and receive operations are allocated, the interrupt 4528 * handler is registered with the OS, the watchdog timer is started, 4529 * and the stack is notified that the interface is ready. 4530 */ 4531 static int __igc_open(struct net_device *netdev, bool resuming) 4532 { 4533 struct igc_adapter *adapter = netdev_priv(netdev); 4534 struct pci_dev *pdev = adapter->pdev; 4535 struct igc_hw *hw = &adapter->hw; 4536 int err = 0; 4537 int i = 0; 4538 4539 /* disallow open during test */ 4540 4541 if (test_bit(__IGC_TESTING, &adapter->state)) { 4542 WARN_ON(resuming); 4543 return -EBUSY; 4544 } 4545 4546 if (!resuming) 4547 pm_runtime_get_sync(&pdev->dev); 4548 4549 netif_carrier_off(netdev); 4550 4551 /* allocate transmit descriptors */ 4552 err = igc_setup_all_tx_resources(adapter); 4553 if (err) 4554 goto err_setup_tx; 4555 4556 /* allocate receive descriptors */ 4557 err = igc_setup_all_rx_resources(adapter); 4558 if (err) 4559 goto err_setup_rx; 4560 4561 igc_power_up_link(adapter); 4562 4563 igc_configure(adapter); 4564 4565 err = igc_request_irq(adapter); 4566 if (err) 4567 goto err_req_irq; 4568 4569 /* Notify the stack of the actual queue counts. */ 4570 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4571 if (err) 4572 goto err_set_queues; 4573 4574 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4575 if (err) 4576 goto err_set_queues; 4577 4578 clear_bit(__IGC_DOWN, &adapter->state); 4579 4580 for (i = 0; i < adapter->num_q_vectors; i++) 4581 napi_enable(&adapter->q_vector[i]->napi); 4582 4583 /* Clear any pending interrupts. */ 4584 rd32(IGC_ICR); 4585 igc_irq_enable(adapter); 4586 4587 if (!resuming) 4588 pm_runtime_put(&pdev->dev); 4589 4590 netif_tx_start_all_queues(netdev); 4591 4592 /* start the watchdog. */ 4593 hw->mac.get_link_status = 1; 4594 schedule_work(&adapter->watchdog_task); 4595 4596 return IGC_SUCCESS; 4597 4598 err_set_queues: 4599 igc_free_irq(adapter); 4600 err_req_irq: 4601 igc_release_hw_control(adapter); 4602 igc_power_down_phy_copper_base(&adapter->hw); 4603 igc_free_all_rx_resources(adapter); 4604 err_setup_rx: 4605 igc_free_all_tx_resources(adapter); 4606 err_setup_tx: 4607 igc_reset(adapter); 4608 if (!resuming) 4609 pm_runtime_put(&pdev->dev); 4610 4611 return err; 4612 } 4613 4614 int igc_open(struct net_device *netdev) 4615 { 4616 return __igc_open(netdev, false); 4617 } 4618 4619 /** 4620 * __igc_close - Disables a network interface 4621 * @netdev: network interface device structure 4622 * @suspending: boolean indicating the device is suspending 4623 * 4624 * Returns 0, this is not allowed to fail 4625 * 4626 * The close entry point is called when an interface is de-activated 4627 * by the OS. The hardware is still under the driver's control, but 4628 * needs to be disabled. A global MAC reset is issued to stop the 4629 * hardware, and all transmit and receive resources are freed. 4630 */ 4631 static int __igc_close(struct net_device *netdev, bool suspending) 4632 { 4633 struct igc_adapter *adapter = netdev_priv(netdev); 4634 struct pci_dev *pdev = adapter->pdev; 4635 4636 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4637 4638 if (!suspending) 4639 pm_runtime_get_sync(&pdev->dev); 4640 4641 igc_down(adapter); 4642 4643 igc_release_hw_control(adapter); 4644 4645 igc_free_irq(adapter); 4646 4647 igc_free_all_tx_resources(adapter); 4648 igc_free_all_rx_resources(adapter); 4649 4650 if (!suspending) 4651 pm_runtime_put_sync(&pdev->dev); 4652 4653 return 0; 4654 } 4655 4656 int igc_close(struct net_device *netdev) 4657 { 4658 if (netif_device_present(netdev) || netdev->dismantle) 4659 return __igc_close(netdev, false); 4660 return 0; 4661 } 4662 4663 /** 4664 * igc_ioctl - Access the hwtstamp interface 4665 * @netdev: network interface device structure 4666 * @ifr: interface request data 4667 * @cmd: ioctl command 4668 **/ 4669 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4670 { 4671 switch (cmd) { 4672 case SIOCGHWTSTAMP: 4673 return igc_ptp_get_ts_config(netdev, ifr); 4674 case SIOCSHWTSTAMP: 4675 return igc_ptp_set_ts_config(netdev, ifr); 4676 default: 4677 return -EOPNOTSUPP; 4678 } 4679 } 4680 4681 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4682 bool enable) 4683 { 4684 struct igc_ring *ring; 4685 int i; 4686 4687 if (queue < 0 || queue >= adapter->num_tx_queues) 4688 return -EINVAL; 4689 4690 ring = adapter->tx_ring[queue]; 4691 ring->launchtime_enable = enable; 4692 4693 if (adapter->base_time) 4694 return 0; 4695 4696 adapter->cycle_time = NSEC_PER_SEC; 4697 4698 for (i = 0; i < adapter->num_tx_queues; i++) { 4699 ring = adapter->tx_ring[i]; 4700 ring->start_time = 0; 4701 ring->end_time = NSEC_PER_SEC; 4702 } 4703 4704 return 0; 4705 } 4706 4707 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) 4708 { 4709 struct timespec64 b; 4710 4711 b = ktime_to_timespec64(base_time); 4712 4713 return timespec64_compare(now, &b) > 0; 4714 } 4715 4716 static bool validate_schedule(struct igc_adapter *adapter, 4717 const struct tc_taprio_qopt_offload *qopt) 4718 { 4719 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4720 struct timespec64 now; 4721 size_t n; 4722 4723 if (qopt->cycle_time_extension) 4724 return false; 4725 4726 igc_ptp_read(adapter, &now); 4727 4728 /* If we program the controller's BASET registers with a time 4729 * in the future, it will hold all the packets until that 4730 * time, causing a lot of TX Hangs, so to avoid that, we 4731 * reject schedules that would start in the future. 4732 */ 4733 if (!is_base_time_past(qopt->base_time, &now)) 4734 return false; 4735 4736 for (n = 0; n < qopt->num_entries; n++) { 4737 const struct tc_taprio_sched_entry *e; 4738 int i; 4739 4740 e = &qopt->entries[n]; 4741 4742 /* i225 only supports "global" frame preemption 4743 * settings. 4744 */ 4745 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4746 return false; 4747 4748 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4749 if (e->gate_mask & BIT(i)) 4750 queue_uses[i]++; 4751 4752 if (queue_uses[i] > 1) 4753 return false; 4754 } 4755 } 4756 4757 return true; 4758 } 4759 4760 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4761 struct tc_etf_qopt_offload *qopt) 4762 { 4763 struct igc_hw *hw = &adapter->hw; 4764 int err; 4765 4766 if (hw->mac.type != igc_i225) 4767 return -EOPNOTSUPP; 4768 4769 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4770 if (err) 4771 return err; 4772 4773 return igc_tsn_offload_apply(adapter); 4774 } 4775 4776 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4777 struct tc_taprio_qopt_offload *qopt) 4778 { 4779 u32 start_time = 0, end_time = 0; 4780 size_t n; 4781 4782 if (!qopt->enable) { 4783 adapter->base_time = 0; 4784 return 0; 4785 } 4786 4787 if (adapter->base_time) 4788 return -EALREADY; 4789 4790 if (!validate_schedule(adapter, qopt)) 4791 return -EINVAL; 4792 4793 adapter->cycle_time = qopt->cycle_time; 4794 adapter->base_time = qopt->base_time; 4795 4796 /* FIXME: be a little smarter about cases when the gate for a 4797 * queue stays open for more than one entry. 4798 */ 4799 for (n = 0; n < qopt->num_entries; n++) { 4800 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4801 int i; 4802 4803 end_time += e->interval; 4804 4805 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4806 struct igc_ring *ring = adapter->tx_ring[i]; 4807 4808 if (!(e->gate_mask & BIT(i))) 4809 continue; 4810 4811 ring->start_time = start_time; 4812 ring->end_time = end_time; 4813 } 4814 4815 start_time += e->interval; 4816 } 4817 4818 return 0; 4819 } 4820 4821 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4822 struct tc_taprio_qopt_offload *qopt) 4823 { 4824 struct igc_hw *hw = &adapter->hw; 4825 int err; 4826 4827 if (hw->mac.type != igc_i225) 4828 return -EOPNOTSUPP; 4829 4830 err = igc_save_qbv_schedule(adapter, qopt); 4831 if (err) 4832 return err; 4833 4834 return igc_tsn_offload_apply(adapter); 4835 } 4836 4837 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4838 void *type_data) 4839 { 4840 struct igc_adapter *adapter = netdev_priv(dev); 4841 4842 switch (type) { 4843 case TC_SETUP_QDISC_TAPRIO: 4844 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4845 4846 case TC_SETUP_QDISC_ETF: 4847 return igc_tsn_enable_launchtime(adapter, type_data); 4848 4849 default: 4850 return -EOPNOTSUPP; 4851 } 4852 } 4853 4854 static const struct net_device_ops igc_netdev_ops = { 4855 .ndo_open = igc_open, 4856 .ndo_stop = igc_close, 4857 .ndo_start_xmit = igc_xmit_frame, 4858 .ndo_set_rx_mode = igc_set_rx_mode, 4859 .ndo_set_mac_address = igc_set_mac, 4860 .ndo_change_mtu = igc_change_mtu, 4861 .ndo_get_stats64 = igc_get_stats64, 4862 .ndo_fix_features = igc_fix_features, 4863 .ndo_set_features = igc_set_features, 4864 .ndo_features_check = igc_features_check, 4865 .ndo_do_ioctl = igc_ioctl, 4866 .ndo_setup_tc = igc_setup_tc, 4867 }; 4868 4869 /* PCIe configuration access */ 4870 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4871 { 4872 struct igc_adapter *adapter = hw->back; 4873 4874 pci_read_config_word(adapter->pdev, reg, value); 4875 } 4876 4877 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4878 { 4879 struct igc_adapter *adapter = hw->back; 4880 4881 pci_write_config_word(adapter->pdev, reg, *value); 4882 } 4883 4884 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4885 { 4886 struct igc_adapter *adapter = hw->back; 4887 4888 if (!pci_is_pcie(adapter->pdev)) 4889 return -IGC_ERR_CONFIG; 4890 4891 pcie_capability_read_word(adapter->pdev, reg, value); 4892 4893 return IGC_SUCCESS; 4894 } 4895 4896 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4897 { 4898 struct igc_adapter *adapter = hw->back; 4899 4900 if (!pci_is_pcie(adapter->pdev)) 4901 return -IGC_ERR_CONFIG; 4902 4903 pcie_capability_write_word(adapter->pdev, reg, *value); 4904 4905 return IGC_SUCCESS; 4906 } 4907 4908 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4909 { 4910 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4911 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4912 u32 value = 0; 4913 4914 value = readl(&hw_addr[reg]); 4915 4916 /* reads should not return all F's */ 4917 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4918 struct net_device *netdev = igc->netdev; 4919 4920 hw->hw_addr = NULL; 4921 netif_device_detach(netdev); 4922 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4923 WARN(pci_device_is_present(igc->pdev), 4924 "igc: Failed to read reg 0x%x!\n", reg); 4925 } 4926 4927 return value; 4928 } 4929 4930 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4931 { 4932 struct igc_mac_info *mac = &adapter->hw.mac; 4933 4934 mac->autoneg = 0; 4935 4936 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4937 * for the switch() below to work 4938 */ 4939 if ((spd & 1) || (dplx & ~1)) 4940 goto err_inval; 4941 4942 switch (spd + dplx) { 4943 case SPEED_10 + DUPLEX_HALF: 4944 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4945 break; 4946 case SPEED_10 + DUPLEX_FULL: 4947 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4948 break; 4949 case SPEED_100 + DUPLEX_HALF: 4950 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4951 break; 4952 case SPEED_100 + DUPLEX_FULL: 4953 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4954 break; 4955 case SPEED_1000 + DUPLEX_FULL: 4956 mac->autoneg = 1; 4957 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4958 break; 4959 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4960 goto err_inval; 4961 case SPEED_2500 + DUPLEX_FULL: 4962 mac->autoneg = 1; 4963 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4964 break; 4965 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4966 default: 4967 goto err_inval; 4968 } 4969 4970 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4971 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4972 4973 return 0; 4974 4975 err_inval: 4976 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4977 return -EINVAL; 4978 } 4979 4980 /** 4981 * igc_probe - Device Initialization Routine 4982 * @pdev: PCI device information struct 4983 * @ent: entry in igc_pci_tbl 4984 * 4985 * Returns 0 on success, negative on failure 4986 * 4987 * igc_probe initializes an adapter identified by a pci_dev structure. 4988 * The OS initialization, configuring the adapter private structure, 4989 * and a hardware reset occur. 4990 */ 4991 static int igc_probe(struct pci_dev *pdev, 4992 const struct pci_device_id *ent) 4993 { 4994 struct igc_adapter *adapter; 4995 struct net_device *netdev; 4996 struct igc_hw *hw; 4997 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4998 int err, pci_using_dac; 4999 5000 err = pci_enable_device_mem(pdev); 5001 if (err) 5002 return err; 5003 5004 pci_using_dac = 0; 5005 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5006 if (!err) { 5007 pci_using_dac = 1; 5008 } else { 5009 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5010 if (err) { 5011 dev_err(&pdev->dev, 5012 "No usable DMA configuration, aborting\n"); 5013 goto err_dma; 5014 } 5015 } 5016 5017 err = pci_request_mem_regions(pdev, igc_driver_name); 5018 if (err) 5019 goto err_pci_reg; 5020 5021 pci_enable_pcie_error_reporting(pdev); 5022 5023 pci_set_master(pdev); 5024 5025 err = -ENOMEM; 5026 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 5027 IGC_MAX_TX_QUEUES); 5028 5029 if (!netdev) 5030 goto err_alloc_etherdev; 5031 5032 SET_NETDEV_DEV(netdev, &pdev->dev); 5033 5034 pci_set_drvdata(pdev, netdev); 5035 adapter = netdev_priv(netdev); 5036 adapter->netdev = netdev; 5037 adapter->pdev = pdev; 5038 hw = &adapter->hw; 5039 hw->back = adapter; 5040 adapter->port_num = hw->bus.func; 5041 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5042 5043 err = pci_save_state(pdev); 5044 if (err) 5045 goto err_ioremap; 5046 5047 err = -EIO; 5048 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5049 pci_resource_len(pdev, 0)); 5050 if (!adapter->io_addr) 5051 goto err_ioremap; 5052 5053 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5054 hw->hw_addr = adapter->io_addr; 5055 5056 netdev->netdev_ops = &igc_netdev_ops; 5057 igc_ethtool_set_ops(netdev); 5058 netdev->watchdog_timeo = 5 * HZ; 5059 5060 netdev->mem_start = pci_resource_start(pdev, 0); 5061 netdev->mem_end = pci_resource_end(pdev, 0); 5062 5063 /* PCI config space info */ 5064 hw->vendor_id = pdev->vendor; 5065 hw->device_id = pdev->device; 5066 hw->revision_id = pdev->revision; 5067 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5068 hw->subsystem_device_id = pdev->subsystem_device; 5069 5070 /* Copy the default MAC and PHY function pointers */ 5071 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5072 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5073 5074 /* Initialize skew-specific constants */ 5075 err = ei->get_invariants(hw); 5076 if (err) 5077 goto err_sw_init; 5078 5079 /* Add supported features to the features list*/ 5080 netdev->features |= NETIF_F_SG; 5081 netdev->features |= NETIF_F_TSO; 5082 netdev->features |= NETIF_F_TSO6; 5083 netdev->features |= NETIF_F_TSO_ECN; 5084 netdev->features |= NETIF_F_RXCSUM; 5085 netdev->features |= NETIF_F_HW_CSUM; 5086 netdev->features |= NETIF_F_SCTP_CRC; 5087 netdev->features |= NETIF_F_HW_TC; 5088 5089 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5090 NETIF_F_GSO_GRE_CSUM | \ 5091 NETIF_F_GSO_IPXIP4 | \ 5092 NETIF_F_GSO_IPXIP6 | \ 5093 NETIF_F_GSO_UDP_TUNNEL | \ 5094 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5095 5096 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5097 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5098 5099 /* setup the private structure */ 5100 err = igc_sw_init(adapter); 5101 if (err) 5102 goto err_sw_init; 5103 5104 /* copy netdev features into list of user selectable features */ 5105 netdev->hw_features |= NETIF_F_NTUPLE; 5106 netdev->hw_features |= netdev->features; 5107 5108 if (pci_using_dac) 5109 netdev->features |= NETIF_F_HIGHDMA; 5110 5111 /* MTU range: 68 - 9216 */ 5112 netdev->min_mtu = ETH_MIN_MTU; 5113 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5114 5115 /* before reading the NVM, reset the controller to put the device in a 5116 * known good starting state 5117 */ 5118 hw->mac.ops.reset_hw(hw); 5119 5120 if (igc_get_flash_presence_i225(hw)) { 5121 if (hw->nvm.ops.validate(hw) < 0) { 5122 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5123 err = -EIO; 5124 goto err_eeprom; 5125 } 5126 } 5127 5128 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5129 /* copy the MAC address out of the NVM */ 5130 if (hw->mac.ops.read_mac_addr(hw)) 5131 dev_err(&pdev->dev, "NVM Read Error\n"); 5132 } 5133 5134 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5135 5136 if (!is_valid_ether_addr(netdev->dev_addr)) { 5137 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5138 err = -EIO; 5139 goto err_eeprom; 5140 } 5141 5142 /* configure RXPBSIZE and TXPBSIZE */ 5143 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5144 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5145 5146 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5147 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5148 5149 INIT_WORK(&adapter->reset_task, igc_reset_task); 5150 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5151 5152 /* Initialize link properties that are user-changeable */ 5153 adapter->fc_autoneg = true; 5154 hw->mac.autoneg = true; 5155 hw->phy.autoneg_advertised = 0xaf; 5156 5157 hw->fc.requested_mode = igc_fc_default; 5158 hw->fc.current_mode = igc_fc_default; 5159 5160 /* By default, support wake on port A */ 5161 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5162 5163 /* initialize the wol settings based on the eeprom settings */ 5164 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5165 adapter->wol |= IGC_WUFC_MAG; 5166 5167 device_set_wakeup_enable(&adapter->pdev->dev, 5168 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5169 5170 igc_ptp_init(adapter); 5171 5172 /* reset the hardware with the new settings */ 5173 igc_reset(adapter); 5174 5175 /* let the f/w know that the h/w is now under the control of the 5176 * driver. 5177 */ 5178 igc_get_hw_control(adapter); 5179 5180 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5181 err = register_netdev(netdev); 5182 if (err) 5183 goto err_register; 5184 5185 /* carrier off reporting is important to ethtool even BEFORE open */ 5186 netif_carrier_off(netdev); 5187 5188 /* Check if Media Autosense is enabled */ 5189 adapter->ei = *ei; 5190 5191 /* print pcie link status and MAC address */ 5192 pcie_print_link_status(pdev); 5193 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5194 5195 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5196 /* Disable EEE for internal PHY devices */ 5197 hw->dev_spec._base.eee_enable = false; 5198 adapter->flags &= ~IGC_FLAG_EEE; 5199 igc_set_eee_i225(hw, false, false, false); 5200 5201 pm_runtime_put_noidle(&pdev->dev); 5202 5203 return 0; 5204 5205 err_register: 5206 igc_release_hw_control(adapter); 5207 err_eeprom: 5208 if (!igc_check_reset_block(hw)) 5209 igc_reset_phy(hw); 5210 err_sw_init: 5211 igc_clear_interrupt_scheme(adapter); 5212 iounmap(adapter->io_addr); 5213 err_ioremap: 5214 free_netdev(netdev); 5215 err_alloc_etherdev: 5216 pci_release_mem_regions(pdev); 5217 err_pci_reg: 5218 err_dma: 5219 pci_disable_device(pdev); 5220 return err; 5221 } 5222 5223 /** 5224 * igc_remove - Device Removal Routine 5225 * @pdev: PCI device information struct 5226 * 5227 * igc_remove is called by the PCI subsystem to alert the driver 5228 * that it should release a PCI device. This could be caused by a 5229 * Hot-Plug event, or because the driver is going to be removed from 5230 * memory. 5231 */ 5232 static void igc_remove(struct pci_dev *pdev) 5233 { 5234 struct net_device *netdev = pci_get_drvdata(pdev); 5235 struct igc_adapter *adapter = netdev_priv(netdev); 5236 5237 pm_runtime_get_noresume(&pdev->dev); 5238 5239 igc_flush_nfc_rules(adapter); 5240 5241 igc_ptp_stop(adapter); 5242 5243 set_bit(__IGC_DOWN, &adapter->state); 5244 5245 del_timer_sync(&adapter->watchdog_timer); 5246 del_timer_sync(&adapter->phy_info_timer); 5247 5248 cancel_work_sync(&adapter->reset_task); 5249 cancel_work_sync(&adapter->watchdog_task); 5250 5251 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5252 * would have already happened in close and is redundant. 5253 */ 5254 igc_release_hw_control(adapter); 5255 unregister_netdev(netdev); 5256 5257 igc_clear_interrupt_scheme(adapter); 5258 pci_iounmap(pdev, adapter->io_addr); 5259 pci_release_mem_regions(pdev); 5260 5261 free_netdev(netdev); 5262 5263 pci_disable_pcie_error_reporting(pdev); 5264 5265 pci_disable_device(pdev); 5266 } 5267 5268 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5269 bool runtime) 5270 { 5271 struct net_device *netdev = pci_get_drvdata(pdev); 5272 struct igc_adapter *adapter = netdev_priv(netdev); 5273 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5274 struct igc_hw *hw = &adapter->hw; 5275 u32 ctrl, rctl, status; 5276 bool wake; 5277 5278 rtnl_lock(); 5279 netif_device_detach(netdev); 5280 5281 if (netif_running(netdev)) 5282 __igc_close(netdev, true); 5283 5284 igc_ptp_suspend(adapter); 5285 5286 igc_clear_interrupt_scheme(adapter); 5287 rtnl_unlock(); 5288 5289 status = rd32(IGC_STATUS); 5290 if (status & IGC_STATUS_LU) 5291 wufc &= ~IGC_WUFC_LNKC; 5292 5293 if (wufc) { 5294 igc_setup_rctl(adapter); 5295 igc_set_rx_mode(netdev); 5296 5297 /* turn on all-multi mode if wake on multicast is enabled */ 5298 if (wufc & IGC_WUFC_MC) { 5299 rctl = rd32(IGC_RCTL); 5300 rctl |= IGC_RCTL_MPE; 5301 wr32(IGC_RCTL, rctl); 5302 } 5303 5304 ctrl = rd32(IGC_CTRL); 5305 ctrl |= IGC_CTRL_ADVD3WUC; 5306 wr32(IGC_CTRL, ctrl); 5307 5308 /* Allow time for pending master requests to run */ 5309 igc_disable_pcie_master(hw); 5310 5311 wr32(IGC_WUC, IGC_WUC_PME_EN); 5312 wr32(IGC_WUFC, wufc); 5313 } else { 5314 wr32(IGC_WUC, 0); 5315 wr32(IGC_WUFC, 0); 5316 } 5317 5318 wake = wufc || adapter->en_mng_pt; 5319 if (!wake) 5320 igc_power_down_phy_copper_base(&adapter->hw); 5321 else 5322 igc_power_up_link(adapter); 5323 5324 if (enable_wake) 5325 *enable_wake = wake; 5326 5327 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5328 * would have already happened in close and is redundant. 5329 */ 5330 igc_release_hw_control(adapter); 5331 5332 pci_disable_device(pdev); 5333 5334 return 0; 5335 } 5336 5337 #ifdef CONFIG_PM 5338 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5339 { 5340 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5341 } 5342 5343 static void igc_deliver_wake_packet(struct net_device *netdev) 5344 { 5345 struct igc_adapter *adapter = netdev_priv(netdev); 5346 struct igc_hw *hw = &adapter->hw; 5347 struct sk_buff *skb; 5348 u32 wupl; 5349 5350 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5351 5352 /* WUPM stores only the first 128 bytes of the wake packet. 5353 * Read the packet only if we have the whole thing. 5354 */ 5355 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5356 return; 5357 5358 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5359 if (!skb) 5360 return; 5361 5362 skb_put(skb, wupl); 5363 5364 /* Ensure reads are 32-bit aligned */ 5365 wupl = roundup(wupl, 4); 5366 5367 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5368 5369 skb->protocol = eth_type_trans(skb, netdev); 5370 netif_rx(skb); 5371 } 5372 5373 static int __maybe_unused igc_resume(struct device *dev) 5374 { 5375 struct pci_dev *pdev = to_pci_dev(dev); 5376 struct net_device *netdev = pci_get_drvdata(pdev); 5377 struct igc_adapter *adapter = netdev_priv(netdev); 5378 struct igc_hw *hw = &adapter->hw; 5379 u32 err, val; 5380 5381 pci_set_power_state(pdev, PCI_D0); 5382 pci_restore_state(pdev); 5383 pci_save_state(pdev); 5384 5385 if (!pci_device_is_present(pdev)) 5386 return -ENODEV; 5387 err = pci_enable_device_mem(pdev); 5388 if (err) { 5389 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5390 return err; 5391 } 5392 pci_set_master(pdev); 5393 5394 pci_enable_wake(pdev, PCI_D3hot, 0); 5395 pci_enable_wake(pdev, PCI_D3cold, 0); 5396 5397 if (igc_init_interrupt_scheme(adapter, true)) { 5398 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5399 return -ENOMEM; 5400 } 5401 5402 igc_reset(adapter); 5403 5404 /* let the f/w know that the h/w is now under the control of the 5405 * driver. 5406 */ 5407 igc_get_hw_control(adapter); 5408 5409 val = rd32(IGC_WUS); 5410 if (val & WAKE_PKT_WUS) 5411 igc_deliver_wake_packet(netdev); 5412 5413 wr32(IGC_WUS, ~0); 5414 5415 rtnl_lock(); 5416 if (!err && netif_running(netdev)) 5417 err = __igc_open(netdev, true); 5418 5419 if (!err) 5420 netif_device_attach(netdev); 5421 rtnl_unlock(); 5422 5423 return err; 5424 } 5425 5426 static int __maybe_unused igc_runtime_resume(struct device *dev) 5427 { 5428 return igc_resume(dev); 5429 } 5430 5431 static int __maybe_unused igc_suspend(struct device *dev) 5432 { 5433 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5434 } 5435 5436 static int __maybe_unused igc_runtime_idle(struct device *dev) 5437 { 5438 struct net_device *netdev = dev_get_drvdata(dev); 5439 struct igc_adapter *adapter = netdev_priv(netdev); 5440 5441 if (!igc_has_link(adapter)) 5442 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5443 5444 return -EBUSY; 5445 } 5446 #endif /* CONFIG_PM */ 5447 5448 static void igc_shutdown(struct pci_dev *pdev) 5449 { 5450 bool wake; 5451 5452 __igc_shutdown(pdev, &wake, 0); 5453 5454 if (system_state == SYSTEM_POWER_OFF) { 5455 pci_wake_from_d3(pdev, wake); 5456 pci_set_power_state(pdev, PCI_D3hot); 5457 } 5458 } 5459 5460 /** 5461 * igc_io_error_detected - called when PCI error is detected 5462 * @pdev: Pointer to PCI device 5463 * @state: The current PCI connection state 5464 * 5465 * This function is called after a PCI bus error affecting 5466 * this device has been detected. 5467 **/ 5468 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5469 pci_channel_state_t state) 5470 { 5471 struct net_device *netdev = pci_get_drvdata(pdev); 5472 struct igc_adapter *adapter = netdev_priv(netdev); 5473 5474 netif_device_detach(netdev); 5475 5476 if (state == pci_channel_io_perm_failure) 5477 return PCI_ERS_RESULT_DISCONNECT; 5478 5479 if (netif_running(netdev)) 5480 igc_down(adapter); 5481 pci_disable_device(pdev); 5482 5483 /* Request a slot reset. */ 5484 return PCI_ERS_RESULT_NEED_RESET; 5485 } 5486 5487 /** 5488 * igc_io_slot_reset - called after the PCI bus has been reset. 5489 * @pdev: Pointer to PCI device 5490 * 5491 * Restart the card from scratch, as if from a cold-boot. Implementation 5492 * resembles the first-half of the igc_resume routine. 5493 **/ 5494 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5495 { 5496 struct net_device *netdev = pci_get_drvdata(pdev); 5497 struct igc_adapter *adapter = netdev_priv(netdev); 5498 struct igc_hw *hw = &adapter->hw; 5499 pci_ers_result_t result; 5500 5501 if (pci_enable_device_mem(pdev)) { 5502 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5503 result = PCI_ERS_RESULT_DISCONNECT; 5504 } else { 5505 pci_set_master(pdev); 5506 pci_restore_state(pdev); 5507 pci_save_state(pdev); 5508 5509 pci_enable_wake(pdev, PCI_D3hot, 0); 5510 pci_enable_wake(pdev, PCI_D3cold, 0); 5511 5512 /* In case of PCI error, adapter loses its HW address 5513 * so we should re-assign it here. 5514 */ 5515 hw->hw_addr = adapter->io_addr; 5516 5517 igc_reset(adapter); 5518 wr32(IGC_WUS, ~0); 5519 result = PCI_ERS_RESULT_RECOVERED; 5520 } 5521 5522 return result; 5523 } 5524 5525 /** 5526 * igc_io_resume - called when traffic can start to flow again. 5527 * @pdev: Pointer to PCI device 5528 * 5529 * This callback is called when the error recovery driver tells us that 5530 * its OK to resume normal operation. Implementation resembles the 5531 * second-half of the igc_resume routine. 5532 */ 5533 static void igc_io_resume(struct pci_dev *pdev) 5534 { 5535 struct net_device *netdev = pci_get_drvdata(pdev); 5536 struct igc_adapter *adapter = netdev_priv(netdev); 5537 5538 rtnl_lock(); 5539 if (netif_running(netdev)) { 5540 if (igc_open(netdev)) { 5541 netdev_err(netdev, "igc_open failed after reset\n"); 5542 return; 5543 } 5544 } 5545 5546 netif_device_attach(netdev); 5547 5548 /* let the f/w know that the h/w is now under the control of the 5549 * driver. 5550 */ 5551 igc_get_hw_control(adapter); 5552 rtnl_unlock(); 5553 } 5554 5555 static const struct pci_error_handlers igc_err_handler = { 5556 .error_detected = igc_io_error_detected, 5557 .slot_reset = igc_io_slot_reset, 5558 .resume = igc_io_resume, 5559 }; 5560 5561 #ifdef CONFIG_PM 5562 static const struct dev_pm_ops igc_pm_ops = { 5563 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5564 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5565 igc_runtime_idle) 5566 }; 5567 #endif 5568 5569 static struct pci_driver igc_driver = { 5570 .name = igc_driver_name, 5571 .id_table = igc_pci_tbl, 5572 .probe = igc_probe, 5573 .remove = igc_remove, 5574 #ifdef CONFIG_PM 5575 .driver.pm = &igc_pm_ops, 5576 #endif 5577 .shutdown = igc_shutdown, 5578 .err_handler = &igc_err_handler, 5579 }; 5580 5581 /** 5582 * igc_reinit_queues - return error 5583 * @adapter: pointer to adapter structure 5584 */ 5585 int igc_reinit_queues(struct igc_adapter *adapter) 5586 { 5587 struct net_device *netdev = adapter->netdev; 5588 int err = 0; 5589 5590 if (netif_running(netdev)) 5591 igc_close(netdev); 5592 5593 igc_reset_interrupt_capability(adapter); 5594 5595 if (igc_init_interrupt_scheme(adapter, true)) { 5596 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5597 return -ENOMEM; 5598 } 5599 5600 if (netif_running(netdev)) 5601 err = igc_open(netdev); 5602 5603 return err; 5604 } 5605 5606 /** 5607 * igc_get_hw_dev - return device 5608 * @hw: pointer to hardware structure 5609 * 5610 * used by hardware layer to print debugging information 5611 */ 5612 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5613 { 5614 struct igc_adapter *adapter = hw->back; 5615 5616 return adapter->netdev; 5617 } 5618 5619 /** 5620 * igc_init_module - Driver Registration Routine 5621 * 5622 * igc_init_module is the first routine called when the driver is 5623 * loaded. All it does is register with the PCI subsystem. 5624 */ 5625 static int __init igc_init_module(void) 5626 { 5627 int ret; 5628 5629 pr_info("%s\n", igc_driver_string); 5630 pr_info("%s\n", igc_copyright); 5631 5632 ret = pci_register_driver(&igc_driver); 5633 return ret; 5634 } 5635 5636 module_init(igc_init_module); 5637 5638 /** 5639 * igc_exit_module - Driver Exit Cleanup Routine 5640 * 5641 * igc_exit_module is called just before the driver is removed 5642 * from memory. 5643 */ 5644 static void __exit igc_exit_module(void) 5645 { 5646 pci_unregister_driver(&igc_driver); 5647 } 5648 5649 module_exit(igc_exit_module); 5650 /* igc_main.c */ 5651