1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_VERSION "0.0.1-k" 21 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 22 23 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 24 25 static int debug = -1; 26 27 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 28 MODULE_DESCRIPTION(DRV_SUMMARY); 29 MODULE_LICENSE("GPL v2"); 30 MODULE_VERSION(DRV_VERSION); 31 module_param(debug, int, 0); 32 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 33 34 char igc_driver_name[] = "igc"; 35 char igc_driver_version[] = DRV_VERSION; 36 static const char igc_driver_string[] = DRV_SUMMARY; 37 static const char igc_copyright[] = 38 "Copyright(c) 2018 Intel Corporation."; 39 40 static const struct igc_info *igc_info_tbl[] = { 41 [board_base] = &igc_base_info, 42 }; 43 44 static const struct pci_device_id igc_pci_tbl[] = { 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 51 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 52 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 53 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 54 /* required last entry */ 55 {0, } 56 }; 57 58 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 59 60 enum latency_range { 61 lowest_latency = 0, 62 low_latency = 1, 63 bulk_latency = 2, 64 latency_invalid = 255 65 }; 66 67 /** 68 * igc_power_down_link - Power down the phy/serdes link 69 * @adapter: address of board private structure 70 */ 71 static void igc_power_down_link(struct igc_adapter *adapter) 72 { 73 if (adapter->hw.phy.media_type == igc_media_type_copper) 74 igc_power_down_phy_copper_base(&adapter->hw); 75 } 76 77 void igc_reset(struct igc_adapter *adapter) 78 { 79 struct pci_dev *pdev = adapter->pdev; 80 struct igc_hw *hw = &adapter->hw; 81 struct igc_fc_info *fc = &hw->fc; 82 u32 pba, hwm; 83 84 /* Repartition PBA for greater than 9k MTU if required */ 85 pba = IGC_PBA_34K; 86 87 /* flow control settings 88 * The high water mark must be low enough to fit one full frame 89 * after transmitting the pause frame. As such we must have enough 90 * space to allow for us to complete our current transmit and then 91 * receive the frame that is in progress from the link partner. 92 * Set it to: 93 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 94 */ 95 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 96 97 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 98 fc->low_water = fc->high_water - 16; 99 fc->pause_time = 0xFFFF; 100 fc->send_xon = 1; 101 fc->current_mode = fc->requested_mode; 102 103 hw->mac.ops.reset_hw(hw); 104 105 if (hw->mac.ops.init_hw(hw)) 106 dev_err(&pdev->dev, "Hardware Error\n"); 107 108 if (!netif_running(adapter->netdev)) 109 igc_power_down_link(adapter); 110 111 /* Re-enable PTP, where applicable. */ 112 igc_ptp_reset(adapter); 113 114 /* Re-enable TSN offloading, where applicable. */ 115 igc_tsn_offload_apply(adapter); 116 117 igc_get_phy_info(hw); 118 } 119 120 /** 121 * igc_power_up_link - Power up the phy link 122 * @adapter: address of board private structure 123 */ 124 static void igc_power_up_link(struct igc_adapter *adapter) 125 { 126 igc_reset_phy(&adapter->hw); 127 128 if (adapter->hw.phy.media_type == igc_media_type_copper) 129 igc_power_up_phy_copper(&adapter->hw); 130 131 igc_setup_link(&adapter->hw); 132 } 133 134 /** 135 * igc_release_hw_control - release control of the h/w to f/w 136 * @adapter: address of board private structure 137 * 138 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 139 * For ASF and Pass Through versions of f/w this means that the 140 * driver is no longer loaded. 141 */ 142 static void igc_release_hw_control(struct igc_adapter *adapter) 143 { 144 struct igc_hw *hw = &adapter->hw; 145 u32 ctrl_ext; 146 147 /* Let firmware take over control of h/w */ 148 ctrl_ext = rd32(IGC_CTRL_EXT); 149 wr32(IGC_CTRL_EXT, 150 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 151 } 152 153 /** 154 * igc_get_hw_control - get control of the h/w from f/w 155 * @adapter: address of board private structure 156 * 157 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 158 * For ASF and Pass Through versions of f/w this means that 159 * the driver is loaded. 160 */ 161 static void igc_get_hw_control(struct igc_adapter *adapter) 162 { 163 struct igc_hw *hw = &adapter->hw; 164 u32 ctrl_ext; 165 166 /* Let firmware know the driver has taken over */ 167 ctrl_ext = rd32(IGC_CTRL_EXT); 168 wr32(IGC_CTRL_EXT, 169 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 170 } 171 172 /** 173 * igc_clean_tx_ring - Free Tx Buffers 174 * @tx_ring: ring to be cleaned 175 */ 176 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 177 { 178 u16 i = tx_ring->next_to_clean; 179 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 180 181 while (i != tx_ring->next_to_use) { 182 union igc_adv_tx_desc *eop_desc, *tx_desc; 183 184 /* Free all the Tx ring sk_buffs */ 185 dev_kfree_skb_any(tx_buffer->skb); 186 187 /* unmap skb header data */ 188 dma_unmap_single(tx_ring->dev, 189 dma_unmap_addr(tx_buffer, dma), 190 dma_unmap_len(tx_buffer, len), 191 DMA_TO_DEVICE); 192 193 /* check for eop_desc to determine the end of the packet */ 194 eop_desc = tx_buffer->next_to_watch; 195 tx_desc = IGC_TX_DESC(tx_ring, i); 196 197 /* unmap remaining buffers */ 198 while (tx_desc != eop_desc) { 199 tx_buffer++; 200 tx_desc++; 201 i++; 202 if (unlikely(i == tx_ring->count)) { 203 i = 0; 204 tx_buffer = tx_ring->tx_buffer_info; 205 tx_desc = IGC_TX_DESC(tx_ring, 0); 206 } 207 208 /* unmap any remaining paged data */ 209 if (dma_unmap_len(tx_buffer, len)) 210 dma_unmap_page(tx_ring->dev, 211 dma_unmap_addr(tx_buffer, dma), 212 dma_unmap_len(tx_buffer, len), 213 DMA_TO_DEVICE); 214 } 215 216 /* move us one more past the eop_desc for start of next pkt */ 217 tx_buffer++; 218 i++; 219 if (unlikely(i == tx_ring->count)) { 220 i = 0; 221 tx_buffer = tx_ring->tx_buffer_info; 222 } 223 } 224 225 /* reset BQL for queue */ 226 netdev_tx_reset_queue(txring_txq(tx_ring)); 227 228 /* reset next_to_use and next_to_clean */ 229 tx_ring->next_to_use = 0; 230 tx_ring->next_to_clean = 0; 231 } 232 233 /** 234 * igc_free_tx_resources - Free Tx Resources per Queue 235 * @tx_ring: Tx descriptor ring for a specific queue 236 * 237 * Free all transmit software resources 238 */ 239 void igc_free_tx_resources(struct igc_ring *tx_ring) 240 { 241 igc_clean_tx_ring(tx_ring); 242 243 vfree(tx_ring->tx_buffer_info); 244 tx_ring->tx_buffer_info = NULL; 245 246 /* if not set, then don't free */ 247 if (!tx_ring->desc) 248 return; 249 250 dma_free_coherent(tx_ring->dev, tx_ring->size, 251 tx_ring->desc, tx_ring->dma); 252 253 tx_ring->desc = NULL; 254 } 255 256 /** 257 * igc_free_all_tx_resources - Free Tx Resources for All Queues 258 * @adapter: board private structure 259 * 260 * Free all transmit software resources 261 */ 262 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 263 { 264 int i; 265 266 for (i = 0; i < adapter->num_tx_queues; i++) 267 igc_free_tx_resources(adapter->tx_ring[i]); 268 } 269 270 /** 271 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 272 * @adapter: board private structure 273 */ 274 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 275 { 276 int i; 277 278 for (i = 0; i < adapter->num_tx_queues; i++) 279 if (adapter->tx_ring[i]) 280 igc_clean_tx_ring(adapter->tx_ring[i]); 281 } 282 283 /** 284 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 285 * @tx_ring: tx descriptor ring (for a specific queue) to setup 286 * 287 * Return 0 on success, negative on failure 288 */ 289 int igc_setup_tx_resources(struct igc_ring *tx_ring) 290 { 291 struct device *dev = tx_ring->dev; 292 int size = 0; 293 294 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 295 tx_ring->tx_buffer_info = vzalloc(size); 296 if (!tx_ring->tx_buffer_info) 297 goto err; 298 299 /* round up to nearest 4K */ 300 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 301 tx_ring->size = ALIGN(tx_ring->size, 4096); 302 303 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 304 &tx_ring->dma, GFP_KERNEL); 305 306 if (!tx_ring->desc) 307 goto err; 308 309 tx_ring->next_to_use = 0; 310 tx_ring->next_to_clean = 0; 311 312 return 0; 313 314 err: 315 vfree(tx_ring->tx_buffer_info); 316 dev_err(dev, 317 "Unable to allocate memory for the transmit descriptor ring\n"); 318 return -ENOMEM; 319 } 320 321 /** 322 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 323 * @adapter: board private structure 324 * 325 * Return 0 on success, negative on failure 326 */ 327 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 328 { 329 struct pci_dev *pdev = adapter->pdev; 330 int i, err = 0; 331 332 for (i = 0; i < adapter->num_tx_queues; i++) { 333 err = igc_setup_tx_resources(adapter->tx_ring[i]); 334 if (err) { 335 dev_err(&pdev->dev, 336 "Allocation for Tx Queue %u failed\n", i); 337 for (i--; i >= 0; i--) 338 igc_free_tx_resources(adapter->tx_ring[i]); 339 break; 340 } 341 } 342 343 return err; 344 } 345 346 /** 347 * igc_clean_rx_ring - Free Rx Buffers per Queue 348 * @rx_ring: ring to free buffers from 349 */ 350 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 351 { 352 u16 i = rx_ring->next_to_clean; 353 354 dev_kfree_skb(rx_ring->skb); 355 rx_ring->skb = NULL; 356 357 /* Free all the Rx ring sk_buffs */ 358 while (i != rx_ring->next_to_alloc) { 359 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 360 361 /* Invalidate cache lines that may have been written to by 362 * device so that we avoid corrupting memory. 363 */ 364 dma_sync_single_range_for_cpu(rx_ring->dev, 365 buffer_info->dma, 366 buffer_info->page_offset, 367 igc_rx_bufsz(rx_ring), 368 DMA_FROM_DEVICE); 369 370 /* free resources associated with mapping */ 371 dma_unmap_page_attrs(rx_ring->dev, 372 buffer_info->dma, 373 igc_rx_pg_size(rx_ring), 374 DMA_FROM_DEVICE, 375 IGC_RX_DMA_ATTR); 376 __page_frag_cache_drain(buffer_info->page, 377 buffer_info->pagecnt_bias); 378 379 i++; 380 if (i == rx_ring->count) 381 i = 0; 382 } 383 384 rx_ring->next_to_alloc = 0; 385 rx_ring->next_to_clean = 0; 386 rx_ring->next_to_use = 0; 387 } 388 389 /** 390 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 391 * @adapter: board private structure 392 */ 393 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 394 { 395 int i; 396 397 for (i = 0; i < adapter->num_rx_queues; i++) 398 if (adapter->rx_ring[i]) 399 igc_clean_rx_ring(adapter->rx_ring[i]); 400 } 401 402 /** 403 * igc_free_rx_resources - Free Rx Resources 404 * @rx_ring: ring to clean the resources from 405 * 406 * Free all receive software resources 407 */ 408 void igc_free_rx_resources(struct igc_ring *rx_ring) 409 { 410 igc_clean_rx_ring(rx_ring); 411 412 vfree(rx_ring->rx_buffer_info); 413 rx_ring->rx_buffer_info = NULL; 414 415 /* if not set, then don't free */ 416 if (!rx_ring->desc) 417 return; 418 419 dma_free_coherent(rx_ring->dev, rx_ring->size, 420 rx_ring->desc, rx_ring->dma); 421 422 rx_ring->desc = NULL; 423 } 424 425 /** 426 * igc_free_all_rx_resources - Free Rx Resources for All Queues 427 * @adapter: board private structure 428 * 429 * Free all receive software resources 430 */ 431 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 432 { 433 int i; 434 435 for (i = 0; i < adapter->num_rx_queues; i++) 436 igc_free_rx_resources(adapter->rx_ring[i]); 437 } 438 439 /** 440 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 441 * @rx_ring: rx descriptor ring (for a specific queue) to setup 442 * 443 * Returns 0 on success, negative on failure 444 */ 445 int igc_setup_rx_resources(struct igc_ring *rx_ring) 446 { 447 struct device *dev = rx_ring->dev; 448 int size, desc_len; 449 450 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 451 rx_ring->rx_buffer_info = vzalloc(size); 452 if (!rx_ring->rx_buffer_info) 453 goto err; 454 455 desc_len = sizeof(union igc_adv_rx_desc); 456 457 /* Round up to nearest 4K */ 458 rx_ring->size = rx_ring->count * desc_len; 459 rx_ring->size = ALIGN(rx_ring->size, 4096); 460 461 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 462 &rx_ring->dma, GFP_KERNEL); 463 464 if (!rx_ring->desc) 465 goto err; 466 467 rx_ring->next_to_alloc = 0; 468 rx_ring->next_to_clean = 0; 469 rx_ring->next_to_use = 0; 470 471 return 0; 472 473 err: 474 vfree(rx_ring->rx_buffer_info); 475 rx_ring->rx_buffer_info = NULL; 476 dev_err(dev, 477 "Unable to allocate memory for the receive descriptor ring\n"); 478 return -ENOMEM; 479 } 480 481 /** 482 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 483 * (Descriptors) for all queues 484 * @adapter: board private structure 485 * 486 * Return 0 on success, negative on failure 487 */ 488 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 489 { 490 struct pci_dev *pdev = adapter->pdev; 491 int i, err = 0; 492 493 for (i = 0; i < adapter->num_rx_queues; i++) { 494 err = igc_setup_rx_resources(adapter->rx_ring[i]); 495 if (err) { 496 dev_err(&pdev->dev, 497 "Allocation for Rx Queue %u failed\n", i); 498 for (i--; i >= 0; i--) 499 igc_free_rx_resources(adapter->rx_ring[i]); 500 break; 501 } 502 } 503 504 return err; 505 } 506 507 /** 508 * igc_configure_rx_ring - Configure a receive ring after Reset 509 * @adapter: board private structure 510 * @ring: receive ring to be configured 511 * 512 * Configure the Rx unit of the MAC after a reset. 513 */ 514 static void igc_configure_rx_ring(struct igc_adapter *adapter, 515 struct igc_ring *ring) 516 { 517 struct igc_hw *hw = &adapter->hw; 518 union igc_adv_rx_desc *rx_desc; 519 int reg_idx = ring->reg_idx; 520 u32 srrctl = 0, rxdctl = 0; 521 u64 rdba = ring->dma; 522 523 /* disable the queue */ 524 wr32(IGC_RXDCTL(reg_idx), 0); 525 526 /* Set DMA base address registers */ 527 wr32(IGC_RDBAL(reg_idx), 528 rdba & 0x00000000ffffffffULL); 529 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 530 wr32(IGC_RDLEN(reg_idx), 531 ring->count * sizeof(union igc_adv_rx_desc)); 532 533 /* initialize head and tail */ 534 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 535 wr32(IGC_RDH(reg_idx), 0); 536 writel(0, ring->tail); 537 538 /* reset next-to- use/clean to place SW in sync with hardware */ 539 ring->next_to_clean = 0; 540 ring->next_to_use = 0; 541 542 /* set descriptor configuration */ 543 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 544 if (ring_uses_large_buffer(ring)) 545 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 546 else 547 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 548 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 549 550 wr32(IGC_SRRCTL(reg_idx), srrctl); 551 552 rxdctl |= IGC_RX_PTHRESH; 553 rxdctl |= IGC_RX_HTHRESH << 8; 554 rxdctl |= IGC_RX_WTHRESH << 16; 555 556 /* initialize rx_buffer_info */ 557 memset(ring->rx_buffer_info, 0, 558 sizeof(struct igc_rx_buffer) * ring->count); 559 560 /* initialize Rx descriptor 0 */ 561 rx_desc = IGC_RX_DESC(ring, 0); 562 rx_desc->wb.upper.length = 0; 563 564 /* enable receive descriptor fetching */ 565 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 566 567 wr32(IGC_RXDCTL(reg_idx), rxdctl); 568 } 569 570 /** 571 * igc_configure_rx - Configure receive Unit after Reset 572 * @adapter: board private structure 573 * 574 * Configure the Rx unit of the MAC after a reset. 575 */ 576 static void igc_configure_rx(struct igc_adapter *adapter) 577 { 578 int i; 579 580 /* Setup the HW Rx Head and Tail Descriptor Pointers and 581 * the Base and Length of the Rx Descriptor Ring 582 */ 583 for (i = 0; i < adapter->num_rx_queues; i++) 584 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 585 } 586 587 /** 588 * igc_configure_tx_ring - Configure transmit ring after Reset 589 * @adapter: board private structure 590 * @ring: tx ring to configure 591 * 592 * Configure a transmit ring after a reset. 593 */ 594 static void igc_configure_tx_ring(struct igc_adapter *adapter, 595 struct igc_ring *ring) 596 { 597 struct igc_hw *hw = &adapter->hw; 598 int reg_idx = ring->reg_idx; 599 u64 tdba = ring->dma; 600 u32 txdctl = 0; 601 602 /* disable the queue */ 603 wr32(IGC_TXDCTL(reg_idx), 0); 604 wrfl(); 605 mdelay(10); 606 607 wr32(IGC_TDLEN(reg_idx), 608 ring->count * sizeof(union igc_adv_tx_desc)); 609 wr32(IGC_TDBAL(reg_idx), 610 tdba & 0x00000000ffffffffULL); 611 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 612 613 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 614 wr32(IGC_TDH(reg_idx), 0); 615 writel(0, ring->tail); 616 617 txdctl |= IGC_TX_PTHRESH; 618 txdctl |= IGC_TX_HTHRESH << 8; 619 txdctl |= IGC_TX_WTHRESH << 16; 620 621 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 622 wr32(IGC_TXDCTL(reg_idx), txdctl); 623 } 624 625 /** 626 * igc_configure_tx - Configure transmit Unit after Reset 627 * @adapter: board private structure 628 * 629 * Configure the Tx unit of the MAC after a reset. 630 */ 631 static void igc_configure_tx(struct igc_adapter *adapter) 632 { 633 int i; 634 635 for (i = 0; i < adapter->num_tx_queues; i++) 636 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 637 } 638 639 /** 640 * igc_setup_mrqc - configure the multiple receive queue control registers 641 * @adapter: Board private structure 642 */ 643 static void igc_setup_mrqc(struct igc_adapter *adapter) 644 { 645 struct igc_hw *hw = &adapter->hw; 646 u32 j, num_rx_queues; 647 u32 mrqc, rxcsum; 648 u32 rss_key[10]; 649 650 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 651 for (j = 0; j < 10; j++) 652 wr32(IGC_RSSRK(j), rss_key[j]); 653 654 num_rx_queues = adapter->rss_queues; 655 656 if (adapter->rss_indir_tbl_init != num_rx_queues) { 657 for (j = 0; j < IGC_RETA_SIZE; j++) 658 adapter->rss_indir_tbl[j] = 659 (j * num_rx_queues) / IGC_RETA_SIZE; 660 adapter->rss_indir_tbl_init = num_rx_queues; 661 } 662 igc_write_rss_indir_tbl(adapter); 663 664 /* Disable raw packet checksumming so that RSS hash is placed in 665 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 666 * offloads as they are enabled by default 667 */ 668 rxcsum = rd32(IGC_RXCSUM); 669 rxcsum |= IGC_RXCSUM_PCSD; 670 671 /* Enable Receive Checksum Offload for SCTP */ 672 rxcsum |= IGC_RXCSUM_CRCOFL; 673 674 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 675 wr32(IGC_RXCSUM, rxcsum); 676 677 /* Generate RSS hash based on packet types, TCP/UDP 678 * port numbers and/or IPv4/v6 src and dst addresses 679 */ 680 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 681 IGC_MRQC_RSS_FIELD_IPV4_TCP | 682 IGC_MRQC_RSS_FIELD_IPV6 | 683 IGC_MRQC_RSS_FIELD_IPV6_TCP | 684 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 685 686 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 687 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 688 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 689 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 690 691 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 692 693 wr32(IGC_MRQC, mrqc); 694 } 695 696 /** 697 * igc_setup_rctl - configure the receive control registers 698 * @adapter: Board private structure 699 */ 700 static void igc_setup_rctl(struct igc_adapter *adapter) 701 { 702 struct igc_hw *hw = &adapter->hw; 703 u32 rctl; 704 705 rctl = rd32(IGC_RCTL); 706 707 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 708 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 709 710 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 711 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 712 713 /* enable stripping of CRC. Newer features require 714 * that the HW strips the CRC. 715 */ 716 rctl |= IGC_RCTL_SECRC; 717 718 /* disable store bad packets and clear size bits. */ 719 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 720 721 /* enable LPE to allow for reception of jumbo frames */ 722 rctl |= IGC_RCTL_LPE; 723 724 /* disable queue 0 to prevent tail write w/o re-config */ 725 wr32(IGC_RXDCTL(0), 0); 726 727 /* This is useful for sniffing bad packets. */ 728 if (adapter->netdev->features & NETIF_F_RXALL) { 729 /* UPE and MPE will be handled by normal PROMISC logic 730 * in set_rx_mode 731 */ 732 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 733 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 734 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 735 736 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 737 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 738 } 739 740 wr32(IGC_RCTL, rctl); 741 } 742 743 /** 744 * igc_setup_tctl - configure the transmit control registers 745 * @adapter: Board private structure 746 */ 747 static void igc_setup_tctl(struct igc_adapter *adapter) 748 { 749 struct igc_hw *hw = &adapter->hw; 750 u32 tctl; 751 752 /* disable queue 0 which icould be enabled by default */ 753 wr32(IGC_TXDCTL(0), 0); 754 755 /* Program the Transmit Control Register */ 756 tctl = rd32(IGC_TCTL); 757 tctl &= ~IGC_TCTL_CT; 758 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 759 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 760 761 /* Enable transmits */ 762 tctl |= IGC_TCTL_EN; 763 764 wr32(IGC_TCTL, tctl); 765 } 766 767 /** 768 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 769 * @adapter: Pointer to adapter where the filter should be set 770 * @index: Filter index 771 * @addr: Destination MAC address 772 * @queue: If non-negative, queue assignment feature is enabled and frames 773 * matching the filter are enqueued onto 'queue'. Otherwise, queue 774 * assignment is disabled. 775 */ 776 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 777 const u8 *addr, int queue) 778 { 779 struct net_device *dev = adapter->netdev; 780 struct igc_hw *hw = &adapter->hw; 781 u32 ral, rah; 782 783 if (WARN_ON(index >= hw->mac.rar_entry_count)) 784 return; 785 786 ral = le32_to_cpup((__le32 *)(addr)); 787 rah = le16_to_cpup((__le16 *)(addr + 4)); 788 789 if (queue >= 0) { 790 rah &= ~IGC_RAH_QSEL_MASK; 791 rah |= (queue << IGC_RAH_QSEL_SHIFT); 792 rah |= IGC_RAH_QSEL_ENABLE; 793 } 794 795 rah |= IGC_RAH_AV; 796 797 wr32(IGC_RAL(index), ral); 798 wr32(IGC_RAH(index), rah); 799 800 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 801 } 802 803 /** 804 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 805 * @adapter: Pointer to adapter where the filter should be cleared 806 * @index: Filter index 807 */ 808 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 809 { 810 struct net_device *dev = adapter->netdev; 811 struct igc_hw *hw = &adapter->hw; 812 813 if (WARN_ON(index >= hw->mac.rar_entry_count)) 814 return; 815 816 wr32(IGC_RAL(index), 0); 817 wr32(IGC_RAH(index), 0); 818 819 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 820 } 821 822 /* Set default MAC address for the PF in the first RAR entry */ 823 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 824 { 825 struct igc_mac_addr *mac_table = &adapter->mac_table[0]; 826 struct net_device *dev = adapter->netdev; 827 u8 *addr = adapter->hw.mac.addr; 828 829 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 830 831 ether_addr_copy(mac_table->addr, addr); 832 mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 833 mac_table->queue = -1; 834 835 igc_set_mac_filter_hw(adapter, 0, addr, mac_table->queue); 836 } 837 838 /** 839 * igc_set_mac - Change the Ethernet Address of the NIC 840 * @netdev: network interface device structure 841 * @p: pointer to an address structure 842 * 843 * Returns 0 on success, negative on failure 844 */ 845 static int igc_set_mac(struct net_device *netdev, void *p) 846 { 847 struct igc_adapter *adapter = netdev_priv(netdev); 848 struct igc_hw *hw = &adapter->hw; 849 struct sockaddr *addr = p; 850 851 if (!is_valid_ether_addr(addr->sa_data)) 852 return -EADDRNOTAVAIL; 853 854 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 855 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 856 857 /* set the correct pool for the new PF MAC address in entry 0 */ 858 igc_set_default_mac_filter(adapter); 859 860 return 0; 861 } 862 863 /** 864 * igc_write_mc_addr_list - write multicast addresses to MTA 865 * @netdev: network interface device structure 866 * 867 * Writes multicast address list to the MTA hash table. 868 * Returns: -ENOMEM on failure 869 * 0 on no addresses written 870 * X on writing X addresses to MTA 871 **/ 872 static int igc_write_mc_addr_list(struct net_device *netdev) 873 { 874 struct igc_adapter *adapter = netdev_priv(netdev); 875 struct igc_hw *hw = &adapter->hw; 876 struct netdev_hw_addr *ha; 877 u8 *mta_list; 878 int i; 879 880 if (netdev_mc_empty(netdev)) { 881 /* nothing to program, so clear mc list */ 882 igc_update_mc_addr_list(hw, NULL, 0); 883 return 0; 884 } 885 886 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 887 if (!mta_list) 888 return -ENOMEM; 889 890 /* The shared function expects a packed array of only addresses. */ 891 i = 0; 892 netdev_for_each_mc_addr(ha, netdev) 893 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 894 895 igc_update_mc_addr_list(hw, mta_list, i); 896 kfree(mta_list); 897 898 return netdev_mc_count(netdev); 899 } 900 901 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 902 { 903 ktime_t cycle_time = adapter->cycle_time; 904 ktime_t base_time = adapter->base_time; 905 u32 launchtime; 906 907 /* FIXME: when using ETF together with taprio, we may have a 908 * case where 'delta' is larger than the cycle_time, this may 909 * cause problems if we don't read the current value of 910 * IGC_BASET, as the value writen into the launchtime 911 * descriptor field may be misinterpreted. 912 */ 913 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 914 915 return cpu_to_le32(launchtime); 916 } 917 918 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 919 struct igc_tx_buffer *first, 920 u32 vlan_macip_lens, u32 type_tucmd, 921 u32 mss_l4len_idx) 922 { 923 struct igc_adv_tx_context_desc *context_desc; 924 u16 i = tx_ring->next_to_use; 925 926 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 927 928 i++; 929 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 930 931 /* set bits to identify this as an advanced context descriptor */ 932 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 933 934 /* For i225, context index must be unique per ring. */ 935 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 936 mss_l4len_idx |= tx_ring->reg_idx << 4; 937 938 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 939 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 940 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 941 942 /* We assume there is always a valid Tx time available. Invalid times 943 * should have been handled by the upper layers. 944 */ 945 if (tx_ring->launchtime_enable) { 946 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 947 ktime_t txtime = first->skb->tstamp; 948 949 first->skb->tstamp = ktime_set(0, 0); 950 context_desc->launch_time = igc_tx_launchtime(adapter, 951 txtime); 952 } else { 953 context_desc->launch_time = 0; 954 } 955 } 956 957 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 958 { 959 unsigned int offset = 0; 960 961 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 962 963 return offset == skb_checksum_start_offset(skb); 964 } 965 966 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 967 { 968 struct sk_buff *skb = first->skb; 969 u32 vlan_macip_lens = 0; 970 u32 type_tucmd = 0; 971 972 if (skb->ip_summed != CHECKSUM_PARTIAL) { 973 csum_failed: 974 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 975 !tx_ring->launchtime_enable) 976 return; 977 goto no_csum; 978 } 979 980 switch (skb->csum_offset) { 981 case offsetof(struct tcphdr, check): 982 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 983 /* fall through */ 984 case offsetof(struct udphdr, check): 985 break; 986 case offsetof(struct sctphdr, checksum): 987 /* validate that this is actually an SCTP request */ 988 if ((first->protocol == htons(ETH_P_IP) && 989 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 990 (first->protocol == htons(ETH_P_IPV6) && 991 igc_ipv6_csum_is_sctp(skb))) { 992 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 993 break; 994 } 995 /* fall through */ 996 default: 997 skb_checksum_help(skb); 998 goto csum_failed; 999 } 1000 1001 /* update TX checksum flag */ 1002 first->tx_flags |= IGC_TX_FLAGS_CSUM; 1003 vlan_macip_lens = skb_checksum_start_offset(skb) - 1004 skb_network_offset(skb); 1005 no_csum: 1006 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 1007 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1008 1009 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 1010 } 1011 1012 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1013 { 1014 struct net_device *netdev = tx_ring->netdev; 1015 1016 netif_stop_subqueue(netdev, tx_ring->queue_index); 1017 1018 /* memory barriier comment */ 1019 smp_mb(); 1020 1021 /* We need to check again in a case another CPU has just 1022 * made room available. 1023 */ 1024 if (igc_desc_unused(tx_ring) < size) 1025 return -EBUSY; 1026 1027 /* A reprieve! */ 1028 netif_wake_subqueue(netdev, tx_ring->queue_index); 1029 1030 u64_stats_update_begin(&tx_ring->tx_syncp2); 1031 tx_ring->tx_stats.restart_queue2++; 1032 u64_stats_update_end(&tx_ring->tx_syncp2); 1033 1034 return 0; 1035 } 1036 1037 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1038 { 1039 if (igc_desc_unused(tx_ring) >= size) 1040 return 0; 1041 return __igc_maybe_stop_tx(tx_ring, size); 1042 } 1043 1044 #define IGC_SET_FLAG(_input, _flag, _result) \ 1045 (((_flag) <= (_result)) ? \ 1046 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1047 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1048 1049 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1050 { 1051 /* set type for advanced descriptor with frame checksum insertion */ 1052 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1053 IGC_ADVTXD_DCMD_DEXT | 1054 IGC_ADVTXD_DCMD_IFCS; 1055 1056 /* set segmentation bits for TSO */ 1057 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1058 (IGC_ADVTXD_DCMD_TSE)); 1059 1060 /* set timestamp bit if present */ 1061 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1062 (IGC_ADVTXD_MAC_TSTAMP)); 1063 1064 return cmd_type; 1065 } 1066 1067 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1068 union igc_adv_tx_desc *tx_desc, 1069 u32 tx_flags, unsigned int paylen) 1070 { 1071 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1072 1073 /* insert L4 checksum */ 1074 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1075 ((IGC_TXD_POPTS_TXSM << 8) / 1076 IGC_TX_FLAGS_CSUM); 1077 1078 /* insert IPv4 checksum */ 1079 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1080 (((IGC_TXD_POPTS_IXSM << 8)) / 1081 IGC_TX_FLAGS_IPV4); 1082 1083 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1084 } 1085 1086 static int igc_tx_map(struct igc_ring *tx_ring, 1087 struct igc_tx_buffer *first, 1088 const u8 hdr_len) 1089 { 1090 struct sk_buff *skb = first->skb; 1091 struct igc_tx_buffer *tx_buffer; 1092 union igc_adv_tx_desc *tx_desc; 1093 u32 tx_flags = first->tx_flags; 1094 skb_frag_t *frag; 1095 u16 i = tx_ring->next_to_use; 1096 unsigned int data_len, size; 1097 dma_addr_t dma; 1098 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1099 1100 tx_desc = IGC_TX_DESC(tx_ring, i); 1101 1102 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1103 1104 size = skb_headlen(skb); 1105 data_len = skb->data_len; 1106 1107 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1108 1109 tx_buffer = first; 1110 1111 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1112 if (dma_mapping_error(tx_ring->dev, dma)) 1113 goto dma_error; 1114 1115 /* record length, and DMA address */ 1116 dma_unmap_len_set(tx_buffer, len, size); 1117 dma_unmap_addr_set(tx_buffer, dma, dma); 1118 1119 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1120 1121 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1122 tx_desc->read.cmd_type_len = 1123 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1124 1125 i++; 1126 tx_desc++; 1127 if (i == tx_ring->count) { 1128 tx_desc = IGC_TX_DESC(tx_ring, 0); 1129 i = 0; 1130 } 1131 tx_desc->read.olinfo_status = 0; 1132 1133 dma += IGC_MAX_DATA_PER_TXD; 1134 size -= IGC_MAX_DATA_PER_TXD; 1135 1136 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1137 } 1138 1139 if (likely(!data_len)) 1140 break; 1141 1142 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1143 1144 i++; 1145 tx_desc++; 1146 if (i == tx_ring->count) { 1147 tx_desc = IGC_TX_DESC(tx_ring, 0); 1148 i = 0; 1149 } 1150 tx_desc->read.olinfo_status = 0; 1151 1152 size = skb_frag_size(frag); 1153 data_len -= size; 1154 1155 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1156 size, DMA_TO_DEVICE); 1157 1158 tx_buffer = &tx_ring->tx_buffer_info[i]; 1159 } 1160 1161 /* write last descriptor with RS and EOP bits */ 1162 cmd_type |= size | IGC_TXD_DCMD; 1163 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1164 1165 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1166 1167 /* set the timestamp */ 1168 first->time_stamp = jiffies; 1169 1170 skb_tx_timestamp(skb); 1171 1172 /* Force memory writes to complete before letting h/w know there 1173 * are new descriptors to fetch. (Only applicable for weak-ordered 1174 * memory model archs, such as IA-64). 1175 * 1176 * We also need this memory barrier to make certain all of the 1177 * status bits have been updated before next_to_watch is written. 1178 */ 1179 wmb(); 1180 1181 /* set next_to_watch value indicating a packet is present */ 1182 first->next_to_watch = tx_desc; 1183 1184 i++; 1185 if (i == tx_ring->count) 1186 i = 0; 1187 1188 tx_ring->next_to_use = i; 1189 1190 /* Make sure there is space in the ring for the next send. */ 1191 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1192 1193 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1194 writel(i, tx_ring->tail); 1195 } 1196 1197 return 0; 1198 dma_error: 1199 dev_err(tx_ring->dev, "TX DMA map failed\n"); 1200 tx_buffer = &tx_ring->tx_buffer_info[i]; 1201 1202 /* clear dma mappings for failed tx_buffer_info map */ 1203 while (tx_buffer != first) { 1204 if (dma_unmap_len(tx_buffer, len)) 1205 dma_unmap_page(tx_ring->dev, 1206 dma_unmap_addr(tx_buffer, dma), 1207 dma_unmap_len(tx_buffer, len), 1208 DMA_TO_DEVICE); 1209 dma_unmap_len_set(tx_buffer, len, 0); 1210 1211 if (i-- == 0) 1212 i += tx_ring->count; 1213 tx_buffer = &tx_ring->tx_buffer_info[i]; 1214 } 1215 1216 if (dma_unmap_len(tx_buffer, len)) 1217 dma_unmap_single(tx_ring->dev, 1218 dma_unmap_addr(tx_buffer, dma), 1219 dma_unmap_len(tx_buffer, len), 1220 DMA_TO_DEVICE); 1221 dma_unmap_len_set(tx_buffer, len, 0); 1222 1223 dev_kfree_skb_any(tx_buffer->skb); 1224 tx_buffer->skb = NULL; 1225 1226 tx_ring->next_to_use = i; 1227 1228 return -1; 1229 } 1230 1231 static int igc_tso(struct igc_ring *tx_ring, 1232 struct igc_tx_buffer *first, 1233 u8 *hdr_len) 1234 { 1235 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1236 struct sk_buff *skb = first->skb; 1237 union { 1238 struct iphdr *v4; 1239 struct ipv6hdr *v6; 1240 unsigned char *hdr; 1241 } ip; 1242 union { 1243 struct tcphdr *tcp; 1244 struct udphdr *udp; 1245 unsigned char *hdr; 1246 } l4; 1247 u32 paylen, l4_offset; 1248 int err; 1249 1250 if (skb->ip_summed != CHECKSUM_PARTIAL) 1251 return 0; 1252 1253 if (!skb_is_gso(skb)) 1254 return 0; 1255 1256 err = skb_cow_head(skb, 0); 1257 if (err < 0) 1258 return err; 1259 1260 ip.hdr = skb_network_header(skb); 1261 l4.hdr = skb_checksum_start(skb); 1262 1263 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1264 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1265 1266 /* initialize outer IP header fields */ 1267 if (ip.v4->version == 4) { 1268 unsigned char *csum_start = skb_checksum_start(skb); 1269 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1270 1271 /* IP header will have to cancel out any data that 1272 * is not a part of the outer IP header 1273 */ 1274 ip.v4->check = csum_fold(csum_partial(trans_start, 1275 csum_start - trans_start, 1276 0)); 1277 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1278 1279 ip.v4->tot_len = 0; 1280 first->tx_flags |= IGC_TX_FLAGS_TSO | 1281 IGC_TX_FLAGS_CSUM | 1282 IGC_TX_FLAGS_IPV4; 1283 } else { 1284 ip.v6->payload_len = 0; 1285 first->tx_flags |= IGC_TX_FLAGS_TSO | 1286 IGC_TX_FLAGS_CSUM; 1287 } 1288 1289 /* determine offset of inner transport header */ 1290 l4_offset = l4.hdr - skb->data; 1291 1292 /* remove payload length from inner checksum */ 1293 paylen = skb->len - l4_offset; 1294 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1295 /* compute length of segmentation header */ 1296 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1297 csum_replace_by_diff(&l4.tcp->check, 1298 (__force __wsum)htonl(paylen)); 1299 } else { 1300 /* compute length of segmentation header */ 1301 *hdr_len = sizeof(*l4.udp) + l4_offset; 1302 csum_replace_by_diff(&l4.udp->check, 1303 (__force __wsum)htonl(paylen)); 1304 } 1305 1306 /* update gso size and bytecount with header size */ 1307 first->gso_segs = skb_shinfo(skb)->gso_segs; 1308 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1309 1310 /* MSS L4LEN IDX */ 1311 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1312 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1313 1314 /* VLAN MACLEN IPLEN */ 1315 vlan_macip_lens = l4.hdr - ip.hdr; 1316 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1317 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1318 1319 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1320 type_tucmd, mss_l4len_idx); 1321 1322 return 1; 1323 } 1324 1325 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1326 struct igc_ring *tx_ring) 1327 { 1328 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1329 __be16 protocol = vlan_get_protocol(skb); 1330 struct igc_tx_buffer *first; 1331 u32 tx_flags = 0; 1332 unsigned short f; 1333 u8 hdr_len = 0; 1334 int tso = 0; 1335 1336 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1337 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1338 * + 2 desc gap to keep tail from touching head, 1339 * + 1 desc for context descriptor, 1340 * otherwise try next time 1341 */ 1342 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1343 count += TXD_USE_COUNT(skb_frag_size( 1344 &skb_shinfo(skb)->frags[f])); 1345 1346 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1347 /* this is a hard error */ 1348 return NETDEV_TX_BUSY; 1349 } 1350 1351 /* record the location of the first descriptor for this packet */ 1352 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1353 first->skb = skb; 1354 first->bytecount = skb->len; 1355 first->gso_segs = 1; 1356 1357 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1358 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1359 1360 /* FIXME: add support for retrieving timestamps from 1361 * the other timer registers before skipping the 1362 * timestamping request. 1363 */ 1364 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1365 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1366 &adapter->state)) { 1367 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1368 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1369 1370 adapter->ptp_tx_skb = skb_get(skb); 1371 adapter->ptp_tx_start = jiffies; 1372 } else { 1373 adapter->tx_hwtstamp_skipped++; 1374 } 1375 } 1376 1377 /* record initial flags and protocol */ 1378 first->tx_flags = tx_flags; 1379 first->protocol = protocol; 1380 1381 tso = igc_tso(tx_ring, first, &hdr_len); 1382 if (tso < 0) 1383 goto out_drop; 1384 else if (!tso) 1385 igc_tx_csum(tx_ring, first); 1386 1387 igc_tx_map(tx_ring, first, hdr_len); 1388 1389 return NETDEV_TX_OK; 1390 1391 out_drop: 1392 dev_kfree_skb_any(first->skb); 1393 first->skb = NULL; 1394 1395 return NETDEV_TX_OK; 1396 } 1397 1398 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1399 struct sk_buff *skb) 1400 { 1401 unsigned int r_idx = skb->queue_mapping; 1402 1403 if (r_idx >= adapter->num_tx_queues) 1404 r_idx = r_idx % adapter->num_tx_queues; 1405 1406 return adapter->tx_ring[r_idx]; 1407 } 1408 1409 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1410 struct net_device *netdev) 1411 { 1412 struct igc_adapter *adapter = netdev_priv(netdev); 1413 1414 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1415 * in order to meet this minimum size requirement. 1416 */ 1417 if (skb->len < 17) { 1418 if (skb_padto(skb, 17)) 1419 return NETDEV_TX_OK; 1420 skb->len = 17; 1421 } 1422 1423 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1424 } 1425 1426 static void igc_rx_checksum(struct igc_ring *ring, 1427 union igc_adv_rx_desc *rx_desc, 1428 struct sk_buff *skb) 1429 { 1430 skb_checksum_none_assert(skb); 1431 1432 /* Ignore Checksum bit is set */ 1433 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1434 return; 1435 1436 /* Rx checksum disabled via ethtool */ 1437 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1438 return; 1439 1440 /* TCP/UDP checksum error bit is set */ 1441 if (igc_test_staterr(rx_desc, 1442 IGC_RXDEXT_STATERR_TCPE | 1443 IGC_RXDEXT_STATERR_IPE)) { 1444 /* work around errata with sctp packets where the TCPE aka 1445 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1446 * packets (aka let the stack check the crc32c) 1447 */ 1448 if (!(skb->len == 60 && 1449 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1450 u64_stats_update_begin(&ring->rx_syncp); 1451 ring->rx_stats.csum_err++; 1452 u64_stats_update_end(&ring->rx_syncp); 1453 } 1454 /* let the stack verify checksum errors */ 1455 return; 1456 } 1457 /* It must be a TCP or UDP packet with a valid checksum */ 1458 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1459 IGC_RXD_STAT_UDPCS)) 1460 skb->ip_summed = CHECKSUM_UNNECESSARY; 1461 1462 dev_dbg(ring->dev, "cksum success: bits %08X\n", 1463 le32_to_cpu(rx_desc->wb.upper.status_error)); 1464 } 1465 1466 static inline void igc_rx_hash(struct igc_ring *ring, 1467 union igc_adv_rx_desc *rx_desc, 1468 struct sk_buff *skb) 1469 { 1470 if (ring->netdev->features & NETIF_F_RXHASH) 1471 skb_set_hash(skb, 1472 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1473 PKT_HASH_TYPE_L3); 1474 } 1475 1476 /** 1477 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1478 * @rx_ring: rx descriptor ring packet is being transacted on 1479 * @rx_desc: pointer to the EOP Rx descriptor 1480 * @skb: pointer to current skb being populated 1481 * 1482 * This function checks the ring, descriptor, and packet information in 1483 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1484 * other fields within the skb. 1485 */ 1486 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1487 union igc_adv_rx_desc *rx_desc, 1488 struct sk_buff *skb) 1489 { 1490 igc_rx_hash(rx_ring, rx_desc, skb); 1491 1492 igc_rx_checksum(rx_ring, rx_desc, skb); 1493 1494 if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TS) && 1495 !igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) 1496 igc_ptp_rx_rgtstamp(rx_ring->q_vector, skb); 1497 1498 skb_record_rx_queue(skb, rx_ring->queue_index); 1499 1500 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1501 } 1502 1503 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1504 const unsigned int size) 1505 { 1506 struct igc_rx_buffer *rx_buffer; 1507 1508 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1509 prefetchw(rx_buffer->page); 1510 1511 /* we are reusing so sync this buffer for CPU use */ 1512 dma_sync_single_range_for_cpu(rx_ring->dev, 1513 rx_buffer->dma, 1514 rx_buffer->page_offset, 1515 size, 1516 DMA_FROM_DEVICE); 1517 1518 rx_buffer->pagecnt_bias--; 1519 1520 return rx_buffer; 1521 } 1522 1523 /** 1524 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1525 * @rx_ring: rx descriptor ring to transact packets on 1526 * @rx_buffer: buffer containing page to add 1527 * @skb: sk_buff to place the data into 1528 * @size: size of buffer to be added 1529 * 1530 * This function will add the data contained in rx_buffer->page to the skb. 1531 */ 1532 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1533 struct igc_rx_buffer *rx_buffer, 1534 struct sk_buff *skb, 1535 unsigned int size) 1536 { 1537 #if (PAGE_SIZE < 8192) 1538 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1539 1540 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1541 rx_buffer->page_offset, size, truesize); 1542 rx_buffer->page_offset ^= truesize; 1543 #else 1544 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1545 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1546 SKB_DATA_ALIGN(size); 1547 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1548 rx_buffer->page_offset, size, truesize); 1549 rx_buffer->page_offset += truesize; 1550 #endif 1551 } 1552 1553 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1554 struct igc_rx_buffer *rx_buffer, 1555 union igc_adv_rx_desc *rx_desc, 1556 unsigned int size) 1557 { 1558 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1559 #if (PAGE_SIZE < 8192) 1560 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1561 #else 1562 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1563 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1564 #endif 1565 struct sk_buff *skb; 1566 1567 /* prefetch first cache line of first page */ 1568 prefetch(va); 1569 #if L1_CACHE_BYTES < 128 1570 prefetch(va + L1_CACHE_BYTES); 1571 #endif 1572 1573 /* build an skb around the page buffer */ 1574 skb = build_skb(va - IGC_SKB_PAD, truesize); 1575 if (unlikely(!skb)) 1576 return NULL; 1577 1578 /* update pointers within the skb to store the data */ 1579 skb_reserve(skb, IGC_SKB_PAD); 1580 __skb_put(skb, size); 1581 1582 /* update buffer offset */ 1583 #if (PAGE_SIZE < 8192) 1584 rx_buffer->page_offset ^= truesize; 1585 #else 1586 rx_buffer->page_offset += truesize; 1587 #endif 1588 1589 return skb; 1590 } 1591 1592 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1593 struct igc_rx_buffer *rx_buffer, 1594 union igc_adv_rx_desc *rx_desc, 1595 unsigned int size) 1596 { 1597 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1598 #if (PAGE_SIZE < 8192) 1599 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1600 #else 1601 unsigned int truesize = SKB_DATA_ALIGN(size); 1602 #endif 1603 unsigned int headlen; 1604 struct sk_buff *skb; 1605 1606 /* prefetch first cache line of first page */ 1607 prefetch(va); 1608 #if L1_CACHE_BYTES < 128 1609 prefetch(va + L1_CACHE_BYTES); 1610 #endif 1611 1612 /* allocate a skb to store the frags */ 1613 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1614 if (unlikely(!skb)) 1615 return NULL; 1616 1617 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1618 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1619 va += IGC_TS_HDR_LEN; 1620 size -= IGC_TS_HDR_LEN; 1621 } 1622 1623 /* Determine available headroom for copy */ 1624 headlen = size; 1625 if (headlen > IGC_RX_HDR_LEN) 1626 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1627 1628 /* align pull length to size of long to optimize memcpy performance */ 1629 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1630 1631 /* update all of the pointers */ 1632 size -= headlen; 1633 if (size) { 1634 skb_add_rx_frag(skb, 0, rx_buffer->page, 1635 (va + headlen) - page_address(rx_buffer->page), 1636 size, truesize); 1637 #if (PAGE_SIZE < 8192) 1638 rx_buffer->page_offset ^= truesize; 1639 #else 1640 rx_buffer->page_offset += truesize; 1641 #endif 1642 } else { 1643 rx_buffer->pagecnt_bias++; 1644 } 1645 1646 return skb; 1647 } 1648 1649 /** 1650 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1651 * @rx_ring: rx descriptor ring to store buffers on 1652 * @old_buff: donor buffer to have page reused 1653 * 1654 * Synchronizes page for reuse by the adapter 1655 */ 1656 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1657 struct igc_rx_buffer *old_buff) 1658 { 1659 u16 nta = rx_ring->next_to_alloc; 1660 struct igc_rx_buffer *new_buff; 1661 1662 new_buff = &rx_ring->rx_buffer_info[nta]; 1663 1664 /* update, and store next to alloc */ 1665 nta++; 1666 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1667 1668 /* Transfer page from old buffer to new buffer. 1669 * Move each member individually to avoid possible store 1670 * forwarding stalls. 1671 */ 1672 new_buff->dma = old_buff->dma; 1673 new_buff->page = old_buff->page; 1674 new_buff->page_offset = old_buff->page_offset; 1675 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1676 } 1677 1678 static inline bool igc_page_is_reserved(struct page *page) 1679 { 1680 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1681 } 1682 1683 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1684 { 1685 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1686 struct page *page = rx_buffer->page; 1687 1688 /* avoid re-using remote pages */ 1689 if (unlikely(igc_page_is_reserved(page))) 1690 return false; 1691 1692 #if (PAGE_SIZE < 8192) 1693 /* if we are only owner of page we can reuse it */ 1694 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1695 return false; 1696 #else 1697 #define IGC_LAST_OFFSET \ 1698 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1699 1700 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1701 return false; 1702 #endif 1703 1704 /* If we have drained the page fragment pool we need to update 1705 * the pagecnt_bias and page count so that we fully restock the 1706 * number of references the driver holds. 1707 */ 1708 if (unlikely(!pagecnt_bias)) { 1709 page_ref_add(page, USHRT_MAX); 1710 rx_buffer->pagecnt_bias = USHRT_MAX; 1711 } 1712 1713 return true; 1714 } 1715 1716 /** 1717 * igc_is_non_eop - process handling of non-EOP buffers 1718 * @rx_ring: Rx ring being processed 1719 * @rx_desc: Rx descriptor for current buffer 1720 * 1721 * This function updates next to clean. If the buffer is an EOP buffer 1722 * this function exits returning false, otherwise it will place the 1723 * sk_buff in the next buffer to be chained and return true indicating 1724 * that this is in fact a non-EOP buffer. 1725 */ 1726 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1727 union igc_adv_rx_desc *rx_desc) 1728 { 1729 u32 ntc = rx_ring->next_to_clean + 1; 1730 1731 /* fetch, update, and store next to clean */ 1732 ntc = (ntc < rx_ring->count) ? ntc : 0; 1733 rx_ring->next_to_clean = ntc; 1734 1735 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1736 1737 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1738 return false; 1739 1740 return true; 1741 } 1742 1743 /** 1744 * igc_cleanup_headers - Correct corrupted or empty headers 1745 * @rx_ring: rx descriptor ring packet is being transacted on 1746 * @rx_desc: pointer to the EOP Rx descriptor 1747 * @skb: pointer to current skb being fixed 1748 * 1749 * Address the case where we are pulling data in on pages only 1750 * and as such no data is present in the skb header. 1751 * 1752 * In addition if skb is not at least 60 bytes we need to pad it so that 1753 * it is large enough to qualify as a valid Ethernet frame. 1754 * 1755 * Returns true if an error was encountered and skb was freed. 1756 */ 1757 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1758 union igc_adv_rx_desc *rx_desc, 1759 struct sk_buff *skb) 1760 { 1761 if (unlikely((igc_test_staterr(rx_desc, 1762 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1763 struct net_device *netdev = rx_ring->netdev; 1764 1765 if (!(netdev->features & NETIF_F_RXALL)) { 1766 dev_kfree_skb_any(skb); 1767 return true; 1768 } 1769 } 1770 1771 /* if eth_skb_pad returns an error the skb was freed */ 1772 if (eth_skb_pad(skb)) 1773 return true; 1774 1775 return false; 1776 } 1777 1778 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1779 struct igc_rx_buffer *rx_buffer) 1780 { 1781 if (igc_can_reuse_rx_page(rx_buffer)) { 1782 /* hand second half of page back to the ring */ 1783 igc_reuse_rx_page(rx_ring, rx_buffer); 1784 } else { 1785 /* We are not reusing the buffer so unmap it and free 1786 * any references we are holding to it 1787 */ 1788 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1789 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1790 IGC_RX_DMA_ATTR); 1791 __page_frag_cache_drain(rx_buffer->page, 1792 rx_buffer->pagecnt_bias); 1793 } 1794 1795 /* clear contents of rx_buffer */ 1796 rx_buffer->page = NULL; 1797 } 1798 1799 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1800 { 1801 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1802 } 1803 1804 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1805 struct igc_rx_buffer *bi) 1806 { 1807 struct page *page = bi->page; 1808 dma_addr_t dma; 1809 1810 /* since we are recycling buffers we should seldom need to alloc */ 1811 if (likely(page)) 1812 return true; 1813 1814 /* alloc new page for storage */ 1815 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1816 if (unlikely(!page)) { 1817 rx_ring->rx_stats.alloc_failed++; 1818 return false; 1819 } 1820 1821 /* map page for use */ 1822 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1823 igc_rx_pg_size(rx_ring), 1824 DMA_FROM_DEVICE, 1825 IGC_RX_DMA_ATTR); 1826 1827 /* if mapping failed free memory back to system since 1828 * there isn't much point in holding memory we can't use 1829 */ 1830 if (dma_mapping_error(rx_ring->dev, dma)) { 1831 __free_page(page); 1832 1833 rx_ring->rx_stats.alloc_failed++; 1834 return false; 1835 } 1836 1837 bi->dma = dma; 1838 bi->page = page; 1839 bi->page_offset = igc_rx_offset(rx_ring); 1840 bi->pagecnt_bias = 1; 1841 1842 return true; 1843 } 1844 1845 /** 1846 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1847 * @rx_ring: rx descriptor ring 1848 * @cleaned_count: number of buffers to clean 1849 */ 1850 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1851 { 1852 union igc_adv_rx_desc *rx_desc; 1853 u16 i = rx_ring->next_to_use; 1854 struct igc_rx_buffer *bi; 1855 u16 bufsz; 1856 1857 /* nothing to do */ 1858 if (!cleaned_count) 1859 return; 1860 1861 rx_desc = IGC_RX_DESC(rx_ring, i); 1862 bi = &rx_ring->rx_buffer_info[i]; 1863 i -= rx_ring->count; 1864 1865 bufsz = igc_rx_bufsz(rx_ring); 1866 1867 do { 1868 if (!igc_alloc_mapped_page(rx_ring, bi)) 1869 break; 1870 1871 /* sync the buffer for use by the device */ 1872 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1873 bi->page_offset, bufsz, 1874 DMA_FROM_DEVICE); 1875 1876 /* Refresh the desc even if buffer_addrs didn't change 1877 * because each write-back erases this info. 1878 */ 1879 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1880 1881 rx_desc++; 1882 bi++; 1883 i++; 1884 if (unlikely(!i)) { 1885 rx_desc = IGC_RX_DESC(rx_ring, 0); 1886 bi = rx_ring->rx_buffer_info; 1887 i -= rx_ring->count; 1888 } 1889 1890 /* clear the length for the next_to_use descriptor */ 1891 rx_desc->wb.upper.length = 0; 1892 1893 cleaned_count--; 1894 } while (cleaned_count); 1895 1896 i += rx_ring->count; 1897 1898 if (rx_ring->next_to_use != i) { 1899 /* record the next descriptor to use */ 1900 rx_ring->next_to_use = i; 1901 1902 /* update next to alloc since we have filled the ring */ 1903 rx_ring->next_to_alloc = i; 1904 1905 /* Force memory writes to complete before letting h/w 1906 * know there are new descriptors to fetch. (Only 1907 * applicable for weak-ordered memory model archs, 1908 * such as IA-64). 1909 */ 1910 wmb(); 1911 writel(i, rx_ring->tail); 1912 } 1913 } 1914 1915 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1916 { 1917 unsigned int total_bytes = 0, total_packets = 0; 1918 struct igc_ring *rx_ring = q_vector->rx.ring; 1919 struct sk_buff *skb = rx_ring->skb; 1920 u16 cleaned_count = igc_desc_unused(rx_ring); 1921 1922 while (likely(total_packets < budget)) { 1923 union igc_adv_rx_desc *rx_desc; 1924 struct igc_rx_buffer *rx_buffer; 1925 unsigned int size; 1926 1927 /* return some buffers to hardware, one at a time is too slow */ 1928 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1929 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1930 cleaned_count = 0; 1931 } 1932 1933 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1934 size = le16_to_cpu(rx_desc->wb.upper.length); 1935 if (!size) 1936 break; 1937 1938 /* This memory barrier is needed to keep us from reading 1939 * any other fields out of the rx_desc until we know the 1940 * descriptor has been written back 1941 */ 1942 dma_rmb(); 1943 1944 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1945 1946 /* retrieve a buffer from the ring */ 1947 if (skb) 1948 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1949 else if (ring_uses_build_skb(rx_ring)) 1950 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1951 else 1952 skb = igc_construct_skb(rx_ring, rx_buffer, 1953 rx_desc, size); 1954 1955 /* exit if we failed to retrieve a buffer */ 1956 if (!skb) { 1957 rx_ring->rx_stats.alloc_failed++; 1958 rx_buffer->pagecnt_bias++; 1959 break; 1960 } 1961 1962 igc_put_rx_buffer(rx_ring, rx_buffer); 1963 cleaned_count++; 1964 1965 /* fetch next buffer in frame if non-eop */ 1966 if (igc_is_non_eop(rx_ring, rx_desc)) 1967 continue; 1968 1969 /* verify the packet layout is correct */ 1970 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1971 skb = NULL; 1972 continue; 1973 } 1974 1975 /* probably a little skewed due to removing CRC */ 1976 total_bytes += skb->len; 1977 1978 /* populate checksum, timestamp, VLAN, and protocol */ 1979 igc_process_skb_fields(rx_ring, rx_desc, skb); 1980 1981 napi_gro_receive(&q_vector->napi, skb); 1982 1983 /* reset skb pointer */ 1984 skb = NULL; 1985 1986 /* update budget accounting */ 1987 total_packets++; 1988 } 1989 1990 /* place incomplete frames back on ring for completion */ 1991 rx_ring->skb = skb; 1992 1993 u64_stats_update_begin(&rx_ring->rx_syncp); 1994 rx_ring->rx_stats.packets += total_packets; 1995 rx_ring->rx_stats.bytes += total_bytes; 1996 u64_stats_update_end(&rx_ring->rx_syncp); 1997 q_vector->rx.total_packets += total_packets; 1998 q_vector->rx.total_bytes += total_bytes; 1999 2000 if (cleaned_count) 2001 igc_alloc_rx_buffers(rx_ring, cleaned_count); 2002 2003 return total_packets; 2004 } 2005 2006 /** 2007 * igc_clean_tx_irq - Reclaim resources after transmit completes 2008 * @q_vector: pointer to q_vector containing needed info 2009 * @napi_budget: Used to determine if we are in netpoll 2010 * 2011 * returns true if ring is completely cleaned 2012 */ 2013 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 2014 { 2015 struct igc_adapter *adapter = q_vector->adapter; 2016 unsigned int total_bytes = 0, total_packets = 0; 2017 unsigned int budget = q_vector->tx.work_limit; 2018 struct igc_ring *tx_ring = q_vector->tx.ring; 2019 unsigned int i = tx_ring->next_to_clean; 2020 struct igc_tx_buffer *tx_buffer; 2021 union igc_adv_tx_desc *tx_desc; 2022 2023 if (test_bit(__IGC_DOWN, &adapter->state)) 2024 return true; 2025 2026 tx_buffer = &tx_ring->tx_buffer_info[i]; 2027 tx_desc = IGC_TX_DESC(tx_ring, i); 2028 i -= tx_ring->count; 2029 2030 do { 2031 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2032 2033 /* if next_to_watch is not set then there is no work pending */ 2034 if (!eop_desc) 2035 break; 2036 2037 /* prevent any other reads prior to eop_desc */ 2038 smp_rmb(); 2039 2040 /* if DD is not set pending work has not been completed */ 2041 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2042 break; 2043 2044 /* clear next_to_watch to prevent false hangs */ 2045 tx_buffer->next_to_watch = NULL; 2046 2047 /* update the statistics for this packet */ 2048 total_bytes += tx_buffer->bytecount; 2049 total_packets += tx_buffer->gso_segs; 2050 2051 /* free the skb */ 2052 napi_consume_skb(tx_buffer->skb, napi_budget); 2053 2054 /* unmap skb header data */ 2055 dma_unmap_single(tx_ring->dev, 2056 dma_unmap_addr(tx_buffer, dma), 2057 dma_unmap_len(tx_buffer, len), 2058 DMA_TO_DEVICE); 2059 2060 /* clear tx_buffer data */ 2061 dma_unmap_len_set(tx_buffer, len, 0); 2062 2063 /* clear last DMA location and unmap remaining buffers */ 2064 while (tx_desc != eop_desc) { 2065 tx_buffer++; 2066 tx_desc++; 2067 i++; 2068 if (unlikely(!i)) { 2069 i -= tx_ring->count; 2070 tx_buffer = tx_ring->tx_buffer_info; 2071 tx_desc = IGC_TX_DESC(tx_ring, 0); 2072 } 2073 2074 /* unmap any remaining paged data */ 2075 if (dma_unmap_len(tx_buffer, len)) { 2076 dma_unmap_page(tx_ring->dev, 2077 dma_unmap_addr(tx_buffer, dma), 2078 dma_unmap_len(tx_buffer, len), 2079 DMA_TO_DEVICE); 2080 dma_unmap_len_set(tx_buffer, len, 0); 2081 } 2082 } 2083 2084 /* move us one more past the eop_desc for start of next pkt */ 2085 tx_buffer++; 2086 tx_desc++; 2087 i++; 2088 if (unlikely(!i)) { 2089 i -= tx_ring->count; 2090 tx_buffer = tx_ring->tx_buffer_info; 2091 tx_desc = IGC_TX_DESC(tx_ring, 0); 2092 } 2093 2094 /* issue prefetch for next Tx descriptor */ 2095 prefetch(tx_desc); 2096 2097 /* update budget accounting */ 2098 budget--; 2099 } while (likely(budget)); 2100 2101 netdev_tx_completed_queue(txring_txq(tx_ring), 2102 total_packets, total_bytes); 2103 2104 i += tx_ring->count; 2105 tx_ring->next_to_clean = i; 2106 u64_stats_update_begin(&tx_ring->tx_syncp); 2107 tx_ring->tx_stats.bytes += total_bytes; 2108 tx_ring->tx_stats.packets += total_packets; 2109 u64_stats_update_end(&tx_ring->tx_syncp); 2110 q_vector->tx.total_bytes += total_bytes; 2111 q_vector->tx.total_packets += total_packets; 2112 2113 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2114 struct igc_hw *hw = &adapter->hw; 2115 2116 /* Detect a transmit hang in hardware, this serializes the 2117 * check with the clearing of time_stamp and movement of i 2118 */ 2119 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2120 if (tx_buffer->next_to_watch && 2121 time_after(jiffies, tx_buffer->time_stamp + 2122 (adapter->tx_timeout_factor * HZ)) && 2123 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2124 /* detected Tx unit hang */ 2125 dev_err(tx_ring->dev, 2126 "Detected Tx Unit Hang\n" 2127 " Tx Queue <%d>\n" 2128 " TDH <%x>\n" 2129 " TDT <%x>\n" 2130 " next_to_use <%x>\n" 2131 " next_to_clean <%x>\n" 2132 "buffer_info[next_to_clean]\n" 2133 " time_stamp <%lx>\n" 2134 " next_to_watch <%p>\n" 2135 " jiffies <%lx>\n" 2136 " desc.status <%x>\n", 2137 tx_ring->queue_index, 2138 rd32(IGC_TDH(tx_ring->reg_idx)), 2139 readl(tx_ring->tail), 2140 tx_ring->next_to_use, 2141 tx_ring->next_to_clean, 2142 tx_buffer->time_stamp, 2143 tx_buffer->next_to_watch, 2144 jiffies, 2145 tx_buffer->next_to_watch->wb.status); 2146 netif_stop_subqueue(tx_ring->netdev, 2147 tx_ring->queue_index); 2148 2149 /* we are about to reset, no point in enabling stuff */ 2150 return true; 2151 } 2152 } 2153 2154 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2155 if (unlikely(total_packets && 2156 netif_carrier_ok(tx_ring->netdev) && 2157 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2158 /* Make sure that anybody stopping the queue after this 2159 * sees the new next_to_clean. 2160 */ 2161 smp_mb(); 2162 if (__netif_subqueue_stopped(tx_ring->netdev, 2163 tx_ring->queue_index) && 2164 !(test_bit(__IGC_DOWN, &adapter->state))) { 2165 netif_wake_subqueue(tx_ring->netdev, 2166 tx_ring->queue_index); 2167 2168 u64_stats_update_begin(&tx_ring->tx_syncp); 2169 tx_ring->tx_stats.restart_queue++; 2170 u64_stats_update_end(&tx_ring->tx_syncp); 2171 } 2172 } 2173 2174 return !!budget; 2175 } 2176 2177 static void igc_nfc_filter_restore(struct igc_adapter *adapter) 2178 { 2179 struct igc_nfc_filter *rule; 2180 2181 spin_lock(&adapter->nfc_lock); 2182 2183 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 2184 igc_add_filter(adapter, rule); 2185 2186 spin_unlock(&adapter->nfc_lock); 2187 } 2188 2189 static int igc_find_mac_filter(struct igc_adapter *adapter, const u8 *addr, 2190 u8 flags) 2191 { 2192 int max_entries = adapter->hw.mac.rar_entry_count; 2193 struct igc_mac_addr *entry; 2194 int i; 2195 2196 for (i = 0; i < max_entries; i++) { 2197 entry = &adapter->mac_table[i]; 2198 2199 if (!(entry->state & IGC_MAC_STATE_IN_USE)) 2200 continue; 2201 if (!ether_addr_equal(addr, entry->addr)) 2202 continue; 2203 if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != 2204 (flags & IGC_MAC_STATE_SRC_ADDR)) 2205 continue; 2206 2207 return i; 2208 } 2209 2210 return -1; 2211 } 2212 2213 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2214 { 2215 int max_entries = adapter->hw.mac.rar_entry_count; 2216 struct igc_mac_addr *entry; 2217 int i; 2218 2219 for (i = 0; i < max_entries; i++) { 2220 entry = &adapter->mac_table[i]; 2221 2222 if (!(entry->state & IGC_MAC_STATE_IN_USE)) 2223 return i; 2224 } 2225 2226 return -1; 2227 } 2228 2229 /** 2230 * igc_add_mac_filter() - Add MAC address filter 2231 * @adapter: Pointer to adapter where the filter should be added 2232 * @addr: MAC address 2233 * @queue: If non-negative, queue assignment feature is enabled and frames 2234 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2235 * assignment is disabled. 2236 * @flags: Set IGC_MAC_STATE_SRC_ADDR bit to indicate @address is a source 2237 * address 2238 * 2239 * Return: 0 in case of success, negative errno code otherwise. 2240 */ 2241 int igc_add_mac_filter(struct igc_adapter *adapter, const u8 *addr, 2242 const s8 queue, const u8 flags) 2243 { 2244 struct net_device *dev = adapter->netdev; 2245 int index; 2246 2247 if (!is_valid_ether_addr(addr)) 2248 return -EINVAL; 2249 if (flags & IGC_MAC_STATE_SRC_ADDR) 2250 return -ENOTSUPP; 2251 2252 index = igc_find_mac_filter(adapter, addr, flags); 2253 if (index >= 0) 2254 goto update_queue_assignment; 2255 2256 index = igc_get_avail_mac_filter_slot(adapter); 2257 if (index < 0) 2258 return -ENOSPC; 2259 2260 netdev_dbg(dev, "Add MAC address filter: index %d address %pM queue %d", 2261 index, addr, queue); 2262 2263 ether_addr_copy(adapter->mac_table[index].addr, addr); 2264 adapter->mac_table[index].state |= IGC_MAC_STATE_IN_USE | flags; 2265 update_queue_assignment: 2266 adapter->mac_table[index].queue = queue; 2267 2268 igc_set_mac_filter_hw(adapter, index, addr, queue); 2269 return 0; 2270 } 2271 2272 /** 2273 * igc_del_mac_filter() - Delete MAC address filter 2274 * @adapter: Pointer to adapter where the filter should be deleted from 2275 * @addr: MAC address 2276 * @flags: Set IGC_MAC_STATE_SRC_ADDR bit to indicate @address is a source 2277 * address 2278 * 2279 * Return: 0 in case of success, negative errno code otherwise. 2280 */ 2281 int igc_del_mac_filter(struct igc_adapter *adapter, const u8 *addr, 2282 const u8 flags) 2283 { 2284 struct net_device *dev = adapter->netdev; 2285 struct igc_mac_addr *entry; 2286 int index; 2287 2288 if (!is_valid_ether_addr(addr)) 2289 return -EINVAL; 2290 2291 index = igc_find_mac_filter(adapter, addr, flags); 2292 if (index < 0) 2293 return -ENOENT; 2294 2295 entry = &adapter->mac_table[index]; 2296 2297 if (entry->state & IGC_MAC_STATE_DEFAULT) { 2298 /* If this is the default filter, we don't actually delete it. 2299 * We just reset to its default value i.e. disable queue 2300 * assignment. 2301 */ 2302 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2303 2304 entry->queue = -1; 2305 igc_set_mac_filter_hw(adapter, 0, addr, entry->queue); 2306 } else { 2307 netdev_dbg(dev, "Delete MAC address filter: index %d address %pM", 2308 index, addr); 2309 2310 entry->state = 0; 2311 entry->queue = -1; 2312 memset(entry->addr, 0, ETH_ALEN); 2313 igc_clear_mac_filter_hw(adapter, index); 2314 } 2315 2316 return 0; 2317 } 2318 2319 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2320 { 2321 struct igc_adapter *adapter = netdev_priv(netdev); 2322 2323 return igc_add_mac_filter(adapter, addr, -1, 0); 2324 } 2325 2326 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2327 { 2328 struct igc_adapter *adapter = netdev_priv(netdev); 2329 2330 return igc_del_mac_filter(adapter, addr, 0); 2331 } 2332 2333 /** 2334 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2335 * @netdev: network interface device structure 2336 * 2337 * The set_rx_mode entry point is called whenever the unicast or multicast 2338 * address lists or the network interface flags are updated. This routine is 2339 * responsible for configuring the hardware for proper unicast, multicast, 2340 * promiscuous mode, and all-multi behavior. 2341 */ 2342 static void igc_set_rx_mode(struct net_device *netdev) 2343 { 2344 struct igc_adapter *adapter = netdev_priv(netdev); 2345 struct igc_hw *hw = &adapter->hw; 2346 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2347 int count; 2348 2349 /* Check for Promiscuous and All Multicast modes */ 2350 if (netdev->flags & IFF_PROMISC) { 2351 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2352 } else { 2353 if (netdev->flags & IFF_ALLMULTI) { 2354 rctl |= IGC_RCTL_MPE; 2355 } else { 2356 /* Write addresses to the MTA, if the attempt fails 2357 * then we should just turn on promiscuous mode so 2358 * that we can at least receive multicast traffic 2359 */ 2360 count = igc_write_mc_addr_list(netdev); 2361 if (count < 0) 2362 rctl |= IGC_RCTL_MPE; 2363 } 2364 } 2365 2366 /* Write addresses to available RAR registers, if there is not 2367 * sufficient space to store all the addresses then enable 2368 * unicast promiscuous mode 2369 */ 2370 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2371 rctl |= IGC_RCTL_UPE; 2372 2373 /* update state of unicast and multicast */ 2374 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2375 wr32(IGC_RCTL, rctl); 2376 2377 #if (PAGE_SIZE < 8192) 2378 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2379 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2380 #endif 2381 wr32(IGC_RLPML, rlpml); 2382 } 2383 2384 /** 2385 * igc_configure - configure the hardware for RX and TX 2386 * @adapter: private board structure 2387 */ 2388 static void igc_configure(struct igc_adapter *adapter) 2389 { 2390 struct net_device *netdev = adapter->netdev; 2391 int i = 0; 2392 2393 igc_get_hw_control(adapter); 2394 igc_set_rx_mode(netdev); 2395 2396 igc_setup_tctl(adapter); 2397 igc_setup_mrqc(adapter); 2398 igc_setup_rctl(adapter); 2399 2400 igc_set_default_mac_filter(adapter); 2401 igc_nfc_filter_restore(adapter); 2402 2403 igc_configure_tx(adapter); 2404 igc_configure_rx(adapter); 2405 2406 igc_rx_fifo_flush_base(&adapter->hw); 2407 2408 /* call igc_desc_unused which always leaves 2409 * at least 1 descriptor unused to make sure 2410 * next_to_use != next_to_clean 2411 */ 2412 for (i = 0; i < adapter->num_rx_queues; i++) { 2413 struct igc_ring *ring = adapter->rx_ring[i]; 2414 2415 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2416 } 2417 } 2418 2419 /** 2420 * igc_write_ivar - configure ivar for given MSI-X vector 2421 * @hw: pointer to the HW structure 2422 * @msix_vector: vector number we are allocating to a given ring 2423 * @index: row index of IVAR register to write within IVAR table 2424 * @offset: column offset of in IVAR, should be multiple of 8 2425 * 2426 * The IVAR table consists of 2 columns, 2427 * each containing an cause allocation for an Rx and Tx ring, and a 2428 * variable number of rows depending on the number of queues supported. 2429 */ 2430 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2431 int index, int offset) 2432 { 2433 u32 ivar = array_rd32(IGC_IVAR0, index); 2434 2435 /* clear any bits that are currently set */ 2436 ivar &= ~((u32)0xFF << offset); 2437 2438 /* write vector and valid bit */ 2439 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2440 2441 array_wr32(IGC_IVAR0, index, ivar); 2442 } 2443 2444 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2445 { 2446 struct igc_adapter *adapter = q_vector->adapter; 2447 struct igc_hw *hw = &adapter->hw; 2448 int rx_queue = IGC_N0_QUEUE; 2449 int tx_queue = IGC_N0_QUEUE; 2450 2451 if (q_vector->rx.ring) 2452 rx_queue = q_vector->rx.ring->reg_idx; 2453 if (q_vector->tx.ring) 2454 tx_queue = q_vector->tx.ring->reg_idx; 2455 2456 switch (hw->mac.type) { 2457 case igc_i225: 2458 if (rx_queue > IGC_N0_QUEUE) 2459 igc_write_ivar(hw, msix_vector, 2460 rx_queue >> 1, 2461 (rx_queue & 0x1) << 4); 2462 if (tx_queue > IGC_N0_QUEUE) 2463 igc_write_ivar(hw, msix_vector, 2464 tx_queue >> 1, 2465 ((tx_queue & 0x1) << 4) + 8); 2466 q_vector->eims_value = BIT(msix_vector); 2467 break; 2468 default: 2469 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2470 break; 2471 } 2472 2473 /* add q_vector eims value to global eims_enable_mask */ 2474 adapter->eims_enable_mask |= q_vector->eims_value; 2475 2476 /* configure q_vector to set itr on first interrupt */ 2477 q_vector->set_itr = 1; 2478 } 2479 2480 /** 2481 * igc_configure_msix - Configure MSI-X hardware 2482 * @adapter: Pointer to adapter structure 2483 * 2484 * igc_configure_msix sets up the hardware to properly 2485 * generate MSI-X interrupts. 2486 */ 2487 static void igc_configure_msix(struct igc_adapter *adapter) 2488 { 2489 struct igc_hw *hw = &adapter->hw; 2490 int i, vector = 0; 2491 u32 tmp; 2492 2493 adapter->eims_enable_mask = 0; 2494 2495 /* set vector for other causes, i.e. link changes */ 2496 switch (hw->mac.type) { 2497 case igc_i225: 2498 /* Turn on MSI-X capability first, or our settings 2499 * won't stick. And it will take days to debug. 2500 */ 2501 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2502 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2503 IGC_GPIE_NSICR); 2504 2505 /* enable msix_other interrupt */ 2506 adapter->eims_other = BIT(vector); 2507 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2508 2509 wr32(IGC_IVAR_MISC, tmp); 2510 break; 2511 default: 2512 /* do nothing, since nothing else supports MSI-X */ 2513 break; 2514 } /* switch (hw->mac.type) */ 2515 2516 adapter->eims_enable_mask |= adapter->eims_other; 2517 2518 for (i = 0; i < adapter->num_q_vectors; i++) 2519 igc_assign_vector(adapter->q_vector[i], vector++); 2520 2521 wrfl(); 2522 } 2523 2524 /** 2525 * igc_irq_enable - Enable default interrupt generation settings 2526 * @adapter: board private structure 2527 */ 2528 static void igc_irq_enable(struct igc_adapter *adapter) 2529 { 2530 struct igc_hw *hw = &adapter->hw; 2531 2532 if (adapter->msix_entries) { 2533 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2534 u32 regval = rd32(IGC_EIAC); 2535 2536 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2537 regval = rd32(IGC_EIAM); 2538 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2539 wr32(IGC_EIMS, adapter->eims_enable_mask); 2540 wr32(IGC_IMS, ims); 2541 } else { 2542 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2543 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2544 } 2545 } 2546 2547 /** 2548 * igc_irq_disable - Mask off interrupt generation on the NIC 2549 * @adapter: board private structure 2550 */ 2551 static void igc_irq_disable(struct igc_adapter *adapter) 2552 { 2553 struct igc_hw *hw = &adapter->hw; 2554 2555 if (adapter->msix_entries) { 2556 u32 regval = rd32(IGC_EIAM); 2557 2558 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2559 wr32(IGC_EIMC, adapter->eims_enable_mask); 2560 regval = rd32(IGC_EIAC); 2561 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2562 } 2563 2564 wr32(IGC_IAM, 0); 2565 wr32(IGC_IMC, ~0); 2566 wrfl(); 2567 2568 if (adapter->msix_entries) { 2569 int vector = 0, i; 2570 2571 synchronize_irq(adapter->msix_entries[vector++].vector); 2572 2573 for (i = 0; i < adapter->num_q_vectors; i++) 2574 synchronize_irq(adapter->msix_entries[vector++].vector); 2575 } else { 2576 synchronize_irq(adapter->pdev->irq); 2577 } 2578 } 2579 2580 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2581 const u32 max_rss_queues) 2582 { 2583 /* Determine if we need to pair queues. */ 2584 /* If rss_queues > half of max_rss_queues, pair the queues in 2585 * order to conserve interrupts due to limited supply. 2586 */ 2587 if (adapter->rss_queues > (max_rss_queues / 2)) 2588 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2589 else 2590 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2591 } 2592 2593 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2594 { 2595 unsigned int max_rss_queues; 2596 2597 /* Determine the maximum number of RSS queues supported. */ 2598 max_rss_queues = IGC_MAX_RX_QUEUES; 2599 2600 return max_rss_queues; 2601 } 2602 2603 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2604 { 2605 u32 max_rss_queues; 2606 2607 max_rss_queues = igc_get_max_rss_queues(adapter); 2608 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2609 2610 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2611 } 2612 2613 /** 2614 * igc_reset_q_vector - Reset config for interrupt vector 2615 * @adapter: board private structure to initialize 2616 * @v_idx: Index of vector to be reset 2617 * 2618 * If NAPI is enabled it will delete any references to the 2619 * NAPI struct. This is preparation for igc_free_q_vector. 2620 */ 2621 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2622 { 2623 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2624 2625 /* if we're coming from igc_set_interrupt_capability, the vectors are 2626 * not yet allocated 2627 */ 2628 if (!q_vector) 2629 return; 2630 2631 if (q_vector->tx.ring) 2632 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2633 2634 if (q_vector->rx.ring) 2635 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2636 2637 netif_napi_del(&q_vector->napi); 2638 } 2639 2640 /** 2641 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2642 * @adapter: board private structure to initialize 2643 * @v_idx: Index of vector to be freed 2644 * 2645 * This function frees the memory allocated to the q_vector. 2646 */ 2647 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2648 { 2649 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2650 2651 adapter->q_vector[v_idx] = NULL; 2652 2653 /* igc_get_stats64() might access the rings on this vector, 2654 * we must wait a grace period before freeing it. 2655 */ 2656 if (q_vector) 2657 kfree_rcu(q_vector, rcu); 2658 } 2659 2660 /** 2661 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2662 * @adapter: board private structure to initialize 2663 * 2664 * This function frees the memory allocated to the q_vectors. In addition if 2665 * NAPI is enabled it will delete any references to the NAPI struct prior 2666 * to freeing the q_vector. 2667 */ 2668 static void igc_free_q_vectors(struct igc_adapter *adapter) 2669 { 2670 int v_idx = adapter->num_q_vectors; 2671 2672 adapter->num_tx_queues = 0; 2673 adapter->num_rx_queues = 0; 2674 adapter->num_q_vectors = 0; 2675 2676 while (v_idx--) { 2677 igc_reset_q_vector(adapter, v_idx); 2678 igc_free_q_vector(adapter, v_idx); 2679 } 2680 } 2681 2682 /** 2683 * igc_update_itr - update the dynamic ITR value based on statistics 2684 * @q_vector: pointer to q_vector 2685 * @ring_container: ring info to update the itr for 2686 * 2687 * Stores a new ITR value based on packets and byte 2688 * counts during the last interrupt. The advantage of per interrupt 2689 * computation is faster updates and more accurate ITR for the current 2690 * traffic pattern. Constants in this function were computed 2691 * based on theoretical maximum wire speed and thresholds were set based 2692 * on testing data as well as attempting to minimize response time 2693 * while increasing bulk throughput. 2694 * NOTE: These calculations are only valid when operating in a single- 2695 * queue environment. 2696 */ 2697 static void igc_update_itr(struct igc_q_vector *q_vector, 2698 struct igc_ring_container *ring_container) 2699 { 2700 unsigned int packets = ring_container->total_packets; 2701 unsigned int bytes = ring_container->total_bytes; 2702 u8 itrval = ring_container->itr; 2703 2704 /* no packets, exit with status unchanged */ 2705 if (packets == 0) 2706 return; 2707 2708 switch (itrval) { 2709 case lowest_latency: 2710 /* handle TSO and jumbo frames */ 2711 if (bytes / packets > 8000) 2712 itrval = bulk_latency; 2713 else if ((packets < 5) && (bytes > 512)) 2714 itrval = low_latency; 2715 break; 2716 case low_latency: /* 50 usec aka 20000 ints/s */ 2717 if (bytes > 10000) { 2718 /* this if handles the TSO accounting */ 2719 if (bytes / packets > 8000) 2720 itrval = bulk_latency; 2721 else if ((packets < 10) || ((bytes / packets) > 1200)) 2722 itrval = bulk_latency; 2723 else if ((packets > 35)) 2724 itrval = lowest_latency; 2725 } else if (bytes / packets > 2000) { 2726 itrval = bulk_latency; 2727 } else if (packets <= 2 && bytes < 512) { 2728 itrval = lowest_latency; 2729 } 2730 break; 2731 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2732 if (bytes > 25000) { 2733 if (packets > 35) 2734 itrval = low_latency; 2735 } else if (bytes < 1500) { 2736 itrval = low_latency; 2737 } 2738 break; 2739 } 2740 2741 /* clear work counters since we have the values we need */ 2742 ring_container->total_bytes = 0; 2743 ring_container->total_packets = 0; 2744 2745 /* write updated itr to ring container */ 2746 ring_container->itr = itrval; 2747 } 2748 2749 static void igc_set_itr(struct igc_q_vector *q_vector) 2750 { 2751 struct igc_adapter *adapter = q_vector->adapter; 2752 u32 new_itr = q_vector->itr_val; 2753 u8 current_itr = 0; 2754 2755 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 2756 switch (adapter->link_speed) { 2757 case SPEED_10: 2758 case SPEED_100: 2759 current_itr = 0; 2760 new_itr = IGC_4K_ITR; 2761 goto set_itr_now; 2762 default: 2763 break; 2764 } 2765 2766 igc_update_itr(q_vector, &q_vector->tx); 2767 igc_update_itr(q_vector, &q_vector->rx); 2768 2769 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 2770 2771 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2772 if (current_itr == lowest_latency && 2773 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 2774 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 2775 current_itr = low_latency; 2776 2777 switch (current_itr) { 2778 /* counts and packets in update_itr are dependent on these numbers */ 2779 case lowest_latency: 2780 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 2781 break; 2782 case low_latency: 2783 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 2784 break; 2785 case bulk_latency: 2786 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 2787 break; 2788 default: 2789 break; 2790 } 2791 2792 set_itr_now: 2793 if (new_itr != q_vector->itr_val) { 2794 /* this attempts to bias the interrupt rate towards Bulk 2795 * by adding intermediate steps when interrupt rate is 2796 * increasing 2797 */ 2798 new_itr = new_itr > q_vector->itr_val ? 2799 max((new_itr * q_vector->itr_val) / 2800 (new_itr + (q_vector->itr_val >> 2)), 2801 new_itr) : new_itr; 2802 /* Don't write the value here; it resets the adapter's 2803 * internal timer, and causes us to delay far longer than 2804 * we should between interrupts. Instead, we write the ITR 2805 * value at the beginning of the next interrupt so the timing 2806 * ends up being correct. 2807 */ 2808 q_vector->itr_val = new_itr; 2809 q_vector->set_itr = 1; 2810 } 2811 } 2812 2813 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 2814 { 2815 int v_idx = adapter->num_q_vectors; 2816 2817 if (adapter->msix_entries) { 2818 pci_disable_msix(adapter->pdev); 2819 kfree(adapter->msix_entries); 2820 adapter->msix_entries = NULL; 2821 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 2822 pci_disable_msi(adapter->pdev); 2823 } 2824 2825 while (v_idx--) 2826 igc_reset_q_vector(adapter, v_idx); 2827 } 2828 2829 /** 2830 * igc_set_interrupt_capability - set MSI or MSI-X if supported 2831 * @adapter: Pointer to adapter structure 2832 * @msix: boolean value for MSI-X capability 2833 * 2834 * Attempt to configure interrupts using the best available 2835 * capabilities of the hardware and kernel. 2836 */ 2837 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 2838 bool msix) 2839 { 2840 int numvecs, i; 2841 int err; 2842 2843 if (!msix) 2844 goto msi_only; 2845 adapter->flags |= IGC_FLAG_HAS_MSIX; 2846 2847 /* Number of supported queues. */ 2848 adapter->num_rx_queues = adapter->rss_queues; 2849 2850 adapter->num_tx_queues = adapter->rss_queues; 2851 2852 /* start with one vector for every Rx queue */ 2853 numvecs = adapter->num_rx_queues; 2854 2855 /* if Tx handler is separate add 1 for every Tx queue */ 2856 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 2857 numvecs += adapter->num_tx_queues; 2858 2859 /* store the number of vectors reserved for queues */ 2860 adapter->num_q_vectors = numvecs; 2861 2862 /* add 1 vector for link status interrupts */ 2863 numvecs++; 2864 2865 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 2866 GFP_KERNEL); 2867 2868 if (!adapter->msix_entries) 2869 return; 2870 2871 /* populate entry values */ 2872 for (i = 0; i < numvecs; i++) 2873 adapter->msix_entries[i].entry = i; 2874 2875 err = pci_enable_msix_range(adapter->pdev, 2876 adapter->msix_entries, 2877 numvecs, 2878 numvecs); 2879 if (err > 0) 2880 return; 2881 2882 kfree(adapter->msix_entries); 2883 adapter->msix_entries = NULL; 2884 2885 igc_reset_interrupt_capability(adapter); 2886 2887 msi_only: 2888 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 2889 2890 adapter->rss_queues = 1; 2891 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2892 adapter->num_rx_queues = 1; 2893 adapter->num_tx_queues = 1; 2894 adapter->num_q_vectors = 1; 2895 if (!pci_enable_msi(adapter->pdev)) 2896 adapter->flags |= IGC_FLAG_HAS_MSI; 2897 } 2898 2899 /** 2900 * igc_update_ring_itr - update the dynamic ITR value based on packet size 2901 * @q_vector: pointer to q_vector 2902 * 2903 * Stores a new ITR value based on strictly on packet size. This 2904 * algorithm is less sophisticated than that used in igc_update_itr, 2905 * due to the difficulty of synchronizing statistics across multiple 2906 * receive rings. The divisors and thresholds used by this function 2907 * were determined based on theoretical maximum wire speed and testing 2908 * data, in order to minimize response time while increasing bulk 2909 * throughput. 2910 * NOTE: This function is called only when operating in a multiqueue 2911 * receive environment. 2912 */ 2913 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 2914 { 2915 struct igc_adapter *adapter = q_vector->adapter; 2916 int new_val = q_vector->itr_val; 2917 int avg_wire_size = 0; 2918 unsigned int packets; 2919 2920 /* For non-gigabit speeds, just fix the interrupt rate at 4000 2921 * ints/sec - ITR timer value of 120 ticks. 2922 */ 2923 switch (adapter->link_speed) { 2924 case SPEED_10: 2925 case SPEED_100: 2926 new_val = IGC_4K_ITR; 2927 goto set_itr_val; 2928 default: 2929 break; 2930 } 2931 2932 packets = q_vector->rx.total_packets; 2933 if (packets) 2934 avg_wire_size = q_vector->rx.total_bytes / packets; 2935 2936 packets = q_vector->tx.total_packets; 2937 if (packets) 2938 avg_wire_size = max_t(u32, avg_wire_size, 2939 q_vector->tx.total_bytes / packets); 2940 2941 /* if avg_wire_size isn't set no work was done */ 2942 if (!avg_wire_size) 2943 goto clear_counts; 2944 2945 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 2946 avg_wire_size += 24; 2947 2948 /* Don't starve jumbo frames */ 2949 avg_wire_size = min(avg_wire_size, 3000); 2950 2951 /* Give a little boost to mid-size frames */ 2952 if (avg_wire_size > 300 && avg_wire_size < 1200) 2953 new_val = avg_wire_size / 3; 2954 else 2955 new_val = avg_wire_size / 2; 2956 2957 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 2958 if (new_val < IGC_20K_ITR && 2959 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 2960 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 2961 new_val = IGC_20K_ITR; 2962 2963 set_itr_val: 2964 if (new_val != q_vector->itr_val) { 2965 q_vector->itr_val = new_val; 2966 q_vector->set_itr = 1; 2967 } 2968 clear_counts: 2969 q_vector->rx.total_bytes = 0; 2970 q_vector->rx.total_packets = 0; 2971 q_vector->tx.total_bytes = 0; 2972 q_vector->tx.total_packets = 0; 2973 } 2974 2975 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 2976 { 2977 struct igc_adapter *adapter = q_vector->adapter; 2978 struct igc_hw *hw = &adapter->hw; 2979 2980 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 2981 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 2982 if (adapter->num_q_vectors == 1) 2983 igc_set_itr(q_vector); 2984 else 2985 igc_update_ring_itr(q_vector); 2986 } 2987 2988 if (!test_bit(__IGC_DOWN, &adapter->state)) { 2989 if (adapter->msix_entries) 2990 wr32(IGC_EIMS, q_vector->eims_value); 2991 else 2992 igc_irq_enable(adapter); 2993 } 2994 } 2995 2996 static void igc_add_ring(struct igc_ring *ring, 2997 struct igc_ring_container *head) 2998 { 2999 head->ring = ring; 3000 head->count++; 3001 } 3002 3003 /** 3004 * igc_cache_ring_register - Descriptor ring to register mapping 3005 * @adapter: board private structure to initialize 3006 * 3007 * Once we know the feature-set enabled for the device, we'll cache 3008 * the register offset the descriptor ring is assigned to. 3009 */ 3010 static void igc_cache_ring_register(struct igc_adapter *adapter) 3011 { 3012 int i = 0, j = 0; 3013 3014 switch (adapter->hw.mac.type) { 3015 case igc_i225: 3016 /* Fall through */ 3017 default: 3018 for (; i < adapter->num_rx_queues; i++) 3019 adapter->rx_ring[i]->reg_idx = i; 3020 for (; j < adapter->num_tx_queues; j++) 3021 adapter->tx_ring[j]->reg_idx = j; 3022 break; 3023 } 3024 } 3025 3026 /** 3027 * igc_poll - NAPI Rx polling callback 3028 * @napi: napi polling structure 3029 * @budget: count of how many packets we should handle 3030 */ 3031 static int igc_poll(struct napi_struct *napi, int budget) 3032 { 3033 struct igc_q_vector *q_vector = container_of(napi, 3034 struct igc_q_vector, 3035 napi); 3036 bool clean_complete = true; 3037 int work_done = 0; 3038 3039 if (q_vector->tx.ring) 3040 clean_complete = igc_clean_tx_irq(q_vector, budget); 3041 3042 if (q_vector->rx.ring) { 3043 int cleaned = igc_clean_rx_irq(q_vector, budget); 3044 3045 work_done += cleaned; 3046 if (cleaned >= budget) 3047 clean_complete = false; 3048 } 3049 3050 /* If all work not completed, return budget and keep polling */ 3051 if (!clean_complete) 3052 return budget; 3053 3054 /* Exit the polling mode, but don't re-enable interrupts if stack might 3055 * poll us due to busy-polling 3056 */ 3057 if (likely(napi_complete_done(napi, work_done))) 3058 igc_ring_irq_enable(q_vector); 3059 3060 return min(work_done, budget - 1); 3061 } 3062 3063 /** 3064 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3065 * @adapter: board private structure to initialize 3066 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3067 * @v_idx: index of vector in adapter struct 3068 * @txr_count: total number of Tx rings to allocate 3069 * @txr_idx: index of first Tx ring to allocate 3070 * @rxr_count: total number of Rx rings to allocate 3071 * @rxr_idx: index of first Rx ring to allocate 3072 * 3073 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3074 */ 3075 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3076 unsigned int v_count, unsigned int v_idx, 3077 unsigned int txr_count, unsigned int txr_idx, 3078 unsigned int rxr_count, unsigned int rxr_idx) 3079 { 3080 struct igc_q_vector *q_vector; 3081 struct igc_ring *ring; 3082 int ring_count; 3083 3084 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3085 if (txr_count > 1 || rxr_count > 1) 3086 return -ENOMEM; 3087 3088 ring_count = txr_count + rxr_count; 3089 3090 /* allocate q_vector and rings */ 3091 q_vector = adapter->q_vector[v_idx]; 3092 if (!q_vector) 3093 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3094 GFP_KERNEL); 3095 else 3096 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3097 if (!q_vector) 3098 return -ENOMEM; 3099 3100 /* initialize NAPI */ 3101 netif_napi_add(adapter->netdev, &q_vector->napi, 3102 igc_poll, 64); 3103 3104 /* tie q_vector and adapter together */ 3105 adapter->q_vector[v_idx] = q_vector; 3106 q_vector->adapter = adapter; 3107 3108 /* initialize work limits */ 3109 q_vector->tx.work_limit = adapter->tx_work_limit; 3110 3111 /* initialize ITR configuration */ 3112 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3113 q_vector->itr_val = IGC_START_ITR; 3114 3115 /* initialize pointer to rings */ 3116 ring = q_vector->ring; 3117 3118 /* initialize ITR */ 3119 if (rxr_count) { 3120 /* rx or rx/tx vector */ 3121 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3122 q_vector->itr_val = adapter->rx_itr_setting; 3123 } else { 3124 /* tx only vector */ 3125 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3126 q_vector->itr_val = adapter->tx_itr_setting; 3127 } 3128 3129 if (txr_count) { 3130 /* assign generic ring traits */ 3131 ring->dev = &adapter->pdev->dev; 3132 ring->netdev = adapter->netdev; 3133 3134 /* configure backlink on ring */ 3135 ring->q_vector = q_vector; 3136 3137 /* update q_vector Tx values */ 3138 igc_add_ring(ring, &q_vector->tx); 3139 3140 /* apply Tx specific ring traits */ 3141 ring->count = adapter->tx_ring_count; 3142 ring->queue_index = txr_idx; 3143 3144 /* assign ring to adapter */ 3145 adapter->tx_ring[txr_idx] = ring; 3146 3147 /* push pointer to next ring */ 3148 ring++; 3149 } 3150 3151 if (rxr_count) { 3152 /* assign generic ring traits */ 3153 ring->dev = &adapter->pdev->dev; 3154 ring->netdev = adapter->netdev; 3155 3156 /* configure backlink on ring */ 3157 ring->q_vector = q_vector; 3158 3159 /* update q_vector Rx values */ 3160 igc_add_ring(ring, &q_vector->rx); 3161 3162 /* apply Rx specific ring traits */ 3163 ring->count = adapter->rx_ring_count; 3164 ring->queue_index = rxr_idx; 3165 3166 /* assign ring to adapter */ 3167 adapter->rx_ring[rxr_idx] = ring; 3168 } 3169 3170 return 0; 3171 } 3172 3173 /** 3174 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3175 * @adapter: board private structure to initialize 3176 * 3177 * We allocate one q_vector per queue interrupt. If allocation fails we 3178 * return -ENOMEM. 3179 */ 3180 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3181 { 3182 int rxr_remaining = adapter->num_rx_queues; 3183 int txr_remaining = adapter->num_tx_queues; 3184 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3185 int q_vectors = adapter->num_q_vectors; 3186 int err; 3187 3188 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3189 for (; rxr_remaining; v_idx++) { 3190 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3191 0, 0, 1, rxr_idx); 3192 3193 if (err) 3194 goto err_out; 3195 3196 /* update counts and index */ 3197 rxr_remaining--; 3198 rxr_idx++; 3199 } 3200 } 3201 3202 for (; v_idx < q_vectors; v_idx++) { 3203 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3204 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3205 3206 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3207 tqpv, txr_idx, rqpv, rxr_idx); 3208 3209 if (err) 3210 goto err_out; 3211 3212 /* update counts and index */ 3213 rxr_remaining -= rqpv; 3214 txr_remaining -= tqpv; 3215 rxr_idx++; 3216 txr_idx++; 3217 } 3218 3219 return 0; 3220 3221 err_out: 3222 adapter->num_tx_queues = 0; 3223 adapter->num_rx_queues = 0; 3224 adapter->num_q_vectors = 0; 3225 3226 while (v_idx--) 3227 igc_free_q_vector(adapter, v_idx); 3228 3229 return -ENOMEM; 3230 } 3231 3232 /** 3233 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3234 * @adapter: Pointer to adapter structure 3235 * @msix: boolean for MSI-X capability 3236 * 3237 * This function initializes the interrupts and allocates all of the queues. 3238 */ 3239 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3240 { 3241 struct pci_dev *pdev = adapter->pdev; 3242 int err = 0; 3243 3244 igc_set_interrupt_capability(adapter, msix); 3245 3246 err = igc_alloc_q_vectors(adapter); 3247 if (err) { 3248 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 3249 goto err_alloc_q_vectors; 3250 } 3251 3252 igc_cache_ring_register(adapter); 3253 3254 return 0; 3255 3256 err_alloc_q_vectors: 3257 igc_reset_interrupt_capability(adapter); 3258 return err; 3259 } 3260 3261 /** 3262 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3263 * @adapter: board private structure to initialize 3264 * 3265 * igc_sw_init initializes the Adapter private data structure. 3266 * Fields are initialized based on PCI device information and 3267 * OS network device settings (MTU size). 3268 */ 3269 static int igc_sw_init(struct igc_adapter *adapter) 3270 { 3271 struct net_device *netdev = adapter->netdev; 3272 struct pci_dev *pdev = adapter->pdev; 3273 struct igc_hw *hw = &adapter->hw; 3274 3275 int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; 3276 3277 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3278 3279 /* set default ring sizes */ 3280 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3281 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3282 3283 /* set default ITR values */ 3284 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3285 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3286 3287 /* set default work limits */ 3288 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3289 3290 /* adjust max frame to be at least the size of a standard frame */ 3291 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3292 VLAN_HLEN; 3293 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3294 3295 spin_lock_init(&adapter->nfc_lock); 3296 spin_lock_init(&adapter->stats64_lock); 3297 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3298 adapter->flags |= IGC_FLAG_HAS_MSIX; 3299 3300 adapter->mac_table = kzalloc(size, GFP_ATOMIC); 3301 if (!adapter->mac_table) 3302 return -ENOMEM; 3303 3304 igc_init_queue_configuration(adapter); 3305 3306 /* This call may decrease the number of queues */ 3307 if (igc_init_interrupt_scheme(adapter, true)) { 3308 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 3309 return -ENOMEM; 3310 } 3311 3312 /* Explicitly disable IRQ since the NIC can be in any state. */ 3313 igc_irq_disable(adapter); 3314 3315 set_bit(__IGC_DOWN, &adapter->state); 3316 3317 return 0; 3318 } 3319 3320 /** 3321 * igc_up - Open the interface and prepare it to handle traffic 3322 * @adapter: board private structure 3323 */ 3324 void igc_up(struct igc_adapter *adapter) 3325 { 3326 struct igc_hw *hw = &adapter->hw; 3327 int i = 0; 3328 3329 /* hardware has been reset, we need to reload some things */ 3330 igc_configure(adapter); 3331 3332 clear_bit(__IGC_DOWN, &adapter->state); 3333 3334 for (i = 0; i < adapter->num_q_vectors; i++) 3335 napi_enable(&adapter->q_vector[i]->napi); 3336 3337 if (adapter->msix_entries) 3338 igc_configure_msix(adapter); 3339 else 3340 igc_assign_vector(adapter->q_vector[0], 0); 3341 3342 /* Clear any pending interrupts. */ 3343 rd32(IGC_ICR); 3344 igc_irq_enable(adapter); 3345 3346 netif_tx_start_all_queues(adapter->netdev); 3347 3348 /* start the watchdog. */ 3349 hw->mac.get_link_status = 1; 3350 schedule_work(&adapter->watchdog_task); 3351 } 3352 3353 /** 3354 * igc_update_stats - Update the board statistics counters 3355 * @adapter: board private structure 3356 */ 3357 void igc_update_stats(struct igc_adapter *adapter) 3358 { 3359 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3360 struct pci_dev *pdev = adapter->pdev; 3361 struct igc_hw *hw = &adapter->hw; 3362 u64 _bytes, _packets; 3363 u64 bytes, packets; 3364 unsigned int start; 3365 u32 mpc; 3366 int i; 3367 3368 /* Prevent stats update while adapter is being reset, or if the pci 3369 * connection is down. 3370 */ 3371 if (adapter->link_speed == 0) 3372 return; 3373 if (pci_channel_offline(pdev)) 3374 return; 3375 3376 packets = 0; 3377 bytes = 0; 3378 3379 rcu_read_lock(); 3380 for (i = 0; i < adapter->num_rx_queues; i++) { 3381 struct igc_ring *ring = adapter->rx_ring[i]; 3382 u32 rqdpc = rd32(IGC_RQDPC(i)); 3383 3384 if (hw->mac.type >= igc_i225) 3385 wr32(IGC_RQDPC(i), 0); 3386 3387 if (rqdpc) { 3388 ring->rx_stats.drops += rqdpc; 3389 net_stats->rx_fifo_errors += rqdpc; 3390 } 3391 3392 do { 3393 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3394 _bytes = ring->rx_stats.bytes; 3395 _packets = ring->rx_stats.packets; 3396 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3397 bytes += _bytes; 3398 packets += _packets; 3399 } 3400 3401 net_stats->rx_bytes = bytes; 3402 net_stats->rx_packets = packets; 3403 3404 packets = 0; 3405 bytes = 0; 3406 for (i = 0; i < adapter->num_tx_queues; i++) { 3407 struct igc_ring *ring = adapter->tx_ring[i]; 3408 3409 do { 3410 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3411 _bytes = ring->tx_stats.bytes; 3412 _packets = ring->tx_stats.packets; 3413 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3414 bytes += _bytes; 3415 packets += _packets; 3416 } 3417 net_stats->tx_bytes = bytes; 3418 net_stats->tx_packets = packets; 3419 rcu_read_unlock(); 3420 3421 /* read stats registers */ 3422 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3423 adapter->stats.gprc += rd32(IGC_GPRC); 3424 adapter->stats.gorc += rd32(IGC_GORCL); 3425 rd32(IGC_GORCH); /* clear GORCL */ 3426 adapter->stats.bprc += rd32(IGC_BPRC); 3427 adapter->stats.mprc += rd32(IGC_MPRC); 3428 adapter->stats.roc += rd32(IGC_ROC); 3429 3430 adapter->stats.prc64 += rd32(IGC_PRC64); 3431 adapter->stats.prc127 += rd32(IGC_PRC127); 3432 adapter->stats.prc255 += rd32(IGC_PRC255); 3433 adapter->stats.prc511 += rd32(IGC_PRC511); 3434 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3435 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3436 adapter->stats.symerrs += rd32(IGC_SYMERRS); 3437 adapter->stats.sec += rd32(IGC_SEC); 3438 3439 mpc = rd32(IGC_MPC); 3440 adapter->stats.mpc += mpc; 3441 net_stats->rx_fifo_errors += mpc; 3442 adapter->stats.scc += rd32(IGC_SCC); 3443 adapter->stats.ecol += rd32(IGC_ECOL); 3444 adapter->stats.mcc += rd32(IGC_MCC); 3445 adapter->stats.latecol += rd32(IGC_LATECOL); 3446 adapter->stats.dc += rd32(IGC_DC); 3447 adapter->stats.rlec += rd32(IGC_RLEC); 3448 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3449 adapter->stats.xontxc += rd32(IGC_XONTXC); 3450 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3451 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3452 adapter->stats.fcruc += rd32(IGC_FCRUC); 3453 adapter->stats.gptc += rd32(IGC_GPTC); 3454 adapter->stats.gotc += rd32(IGC_GOTCL); 3455 rd32(IGC_GOTCH); /* clear GOTCL */ 3456 adapter->stats.rnbc += rd32(IGC_RNBC); 3457 adapter->stats.ruc += rd32(IGC_RUC); 3458 adapter->stats.rfc += rd32(IGC_RFC); 3459 adapter->stats.rjc += rd32(IGC_RJC); 3460 adapter->stats.tor += rd32(IGC_TORH); 3461 adapter->stats.tot += rd32(IGC_TOTH); 3462 adapter->stats.tpr += rd32(IGC_TPR); 3463 3464 adapter->stats.ptc64 += rd32(IGC_PTC64); 3465 adapter->stats.ptc127 += rd32(IGC_PTC127); 3466 adapter->stats.ptc255 += rd32(IGC_PTC255); 3467 adapter->stats.ptc511 += rd32(IGC_PTC511); 3468 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3469 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3470 3471 adapter->stats.mptc += rd32(IGC_MPTC); 3472 adapter->stats.bptc += rd32(IGC_BPTC); 3473 3474 adapter->stats.tpt += rd32(IGC_TPT); 3475 adapter->stats.colc += rd32(IGC_COLC); 3476 3477 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3478 3479 adapter->stats.tsctc += rd32(IGC_TSCTC); 3480 adapter->stats.tsctfc += rd32(IGC_TSCTFC); 3481 3482 adapter->stats.iac += rd32(IGC_IAC); 3483 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 3484 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 3485 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 3486 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 3487 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 3488 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 3489 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 3490 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 3491 3492 /* Fill out the OS statistics structure */ 3493 net_stats->multicast = adapter->stats.mprc; 3494 net_stats->collisions = adapter->stats.colc; 3495 3496 /* Rx Errors */ 3497 3498 /* RLEC on some newer hardware can be incorrect so build 3499 * our own version based on RUC and ROC 3500 */ 3501 net_stats->rx_errors = adapter->stats.rxerrc + 3502 adapter->stats.crcerrs + adapter->stats.algnerrc + 3503 adapter->stats.ruc + adapter->stats.roc + 3504 adapter->stats.cexterr; 3505 net_stats->rx_length_errors = adapter->stats.ruc + 3506 adapter->stats.roc; 3507 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3508 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3509 net_stats->rx_missed_errors = adapter->stats.mpc; 3510 3511 /* Tx Errors */ 3512 net_stats->tx_errors = adapter->stats.ecol + 3513 adapter->stats.latecol; 3514 net_stats->tx_aborted_errors = adapter->stats.ecol; 3515 net_stats->tx_window_errors = adapter->stats.latecol; 3516 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3517 3518 /* Tx Dropped needs to be maintained elsewhere */ 3519 3520 /* Management Stats */ 3521 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3522 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3523 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3524 } 3525 3526 static void igc_nfc_filter_exit(struct igc_adapter *adapter) 3527 { 3528 struct igc_nfc_filter *rule; 3529 3530 spin_lock(&adapter->nfc_lock); 3531 3532 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 3533 igc_erase_filter(adapter, rule); 3534 3535 spin_unlock(&adapter->nfc_lock); 3536 } 3537 3538 /** 3539 * igc_down - Close the interface 3540 * @adapter: board private structure 3541 */ 3542 void igc_down(struct igc_adapter *adapter) 3543 { 3544 struct net_device *netdev = adapter->netdev; 3545 struct igc_hw *hw = &adapter->hw; 3546 u32 tctl, rctl; 3547 int i = 0; 3548 3549 set_bit(__IGC_DOWN, &adapter->state); 3550 3551 /* disable receives in the hardware */ 3552 rctl = rd32(IGC_RCTL); 3553 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3554 /* flush and sleep below */ 3555 3556 igc_nfc_filter_exit(adapter); 3557 3558 /* set trans_start so we don't get spurious watchdogs during reset */ 3559 netif_trans_update(netdev); 3560 3561 netif_carrier_off(netdev); 3562 netif_tx_stop_all_queues(netdev); 3563 3564 /* disable transmits in the hardware */ 3565 tctl = rd32(IGC_TCTL); 3566 tctl &= ~IGC_TCTL_EN; 3567 wr32(IGC_TCTL, tctl); 3568 /* flush both disables and wait for them to finish */ 3569 wrfl(); 3570 usleep_range(10000, 20000); 3571 3572 igc_irq_disable(adapter); 3573 3574 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3575 3576 for (i = 0; i < adapter->num_q_vectors; i++) { 3577 if (adapter->q_vector[i]) { 3578 napi_synchronize(&adapter->q_vector[i]->napi); 3579 napi_disable(&adapter->q_vector[i]->napi); 3580 } 3581 } 3582 3583 del_timer_sync(&adapter->watchdog_timer); 3584 del_timer_sync(&adapter->phy_info_timer); 3585 3586 /* record the stats before reset*/ 3587 spin_lock(&adapter->stats64_lock); 3588 igc_update_stats(adapter); 3589 spin_unlock(&adapter->stats64_lock); 3590 3591 adapter->link_speed = 0; 3592 adapter->link_duplex = 0; 3593 3594 if (!pci_channel_offline(adapter->pdev)) 3595 igc_reset(adapter); 3596 3597 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3598 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3599 3600 igc_clean_all_tx_rings(adapter); 3601 igc_clean_all_rx_rings(adapter); 3602 } 3603 3604 void igc_reinit_locked(struct igc_adapter *adapter) 3605 { 3606 WARN_ON(in_interrupt()); 3607 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3608 usleep_range(1000, 2000); 3609 igc_down(adapter); 3610 igc_up(adapter); 3611 clear_bit(__IGC_RESETTING, &adapter->state); 3612 } 3613 3614 static void igc_reset_task(struct work_struct *work) 3615 { 3616 struct igc_adapter *adapter; 3617 3618 adapter = container_of(work, struct igc_adapter, reset_task); 3619 3620 igc_rings_dump(adapter); 3621 igc_regs_dump(adapter); 3622 netdev_err(adapter->netdev, "Reset adapter\n"); 3623 igc_reinit_locked(adapter); 3624 } 3625 3626 /** 3627 * igc_change_mtu - Change the Maximum Transfer Unit 3628 * @netdev: network interface device structure 3629 * @new_mtu: new value for maximum frame size 3630 * 3631 * Returns 0 on success, negative on failure 3632 */ 3633 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3634 { 3635 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3636 struct igc_adapter *adapter = netdev_priv(netdev); 3637 3638 /* adjust max frame to be at least the size of a standard frame */ 3639 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3640 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3641 3642 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3643 usleep_range(1000, 2000); 3644 3645 /* igc_down has a dependency on max_frame_size */ 3646 adapter->max_frame_size = max_frame; 3647 3648 if (netif_running(netdev)) 3649 igc_down(adapter); 3650 3651 netdev_dbg(netdev, "changing MTU from %d to %d\n", 3652 netdev->mtu, new_mtu); 3653 netdev->mtu = new_mtu; 3654 3655 if (netif_running(netdev)) 3656 igc_up(adapter); 3657 else 3658 igc_reset(adapter); 3659 3660 clear_bit(__IGC_RESETTING, &adapter->state); 3661 3662 return 0; 3663 } 3664 3665 /** 3666 * igc_get_stats - Get System Network Statistics 3667 * @netdev: network interface device structure 3668 * 3669 * Returns the address of the device statistics structure. 3670 * The statistics are updated here and also from the timer callback. 3671 */ 3672 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 3673 { 3674 struct igc_adapter *adapter = netdev_priv(netdev); 3675 3676 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3677 igc_update_stats(adapter); 3678 3679 /* only return the current stats */ 3680 return &netdev->stats; 3681 } 3682 3683 static netdev_features_t igc_fix_features(struct net_device *netdev, 3684 netdev_features_t features) 3685 { 3686 /* Since there is no support for separate Rx/Tx vlan accel 3687 * enable/disable make sure Tx flag is always in same state as Rx. 3688 */ 3689 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3690 features |= NETIF_F_HW_VLAN_CTAG_TX; 3691 else 3692 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3693 3694 return features; 3695 } 3696 3697 static int igc_set_features(struct net_device *netdev, 3698 netdev_features_t features) 3699 { 3700 netdev_features_t changed = netdev->features ^ features; 3701 struct igc_adapter *adapter = netdev_priv(netdev); 3702 3703 /* Add VLAN support */ 3704 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3705 return 0; 3706 3707 if (!(features & NETIF_F_NTUPLE)) { 3708 struct hlist_node *node2; 3709 struct igc_nfc_filter *rule; 3710 3711 spin_lock(&adapter->nfc_lock); 3712 hlist_for_each_entry_safe(rule, node2, 3713 &adapter->nfc_filter_list, nfc_node) { 3714 igc_erase_filter(adapter, rule); 3715 hlist_del(&rule->nfc_node); 3716 kfree(rule); 3717 } 3718 spin_unlock(&adapter->nfc_lock); 3719 adapter->nfc_filter_count = 0; 3720 } 3721 3722 netdev->features = features; 3723 3724 if (netif_running(netdev)) 3725 igc_reinit_locked(adapter); 3726 else 3727 igc_reset(adapter); 3728 3729 return 1; 3730 } 3731 3732 static netdev_features_t 3733 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3734 netdev_features_t features) 3735 { 3736 unsigned int network_hdr_len, mac_hdr_len; 3737 3738 /* Make certain the headers can be described by a context descriptor */ 3739 mac_hdr_len = skb_network_header(skb) - skb->data; 3740 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3741 return features & ~(NETIF_F_HW_CSUM | 3742 NETIF_F_SCTP_CRC | 3743 NETIF_F_HW_VLAN_CTAG_TX | 3744 NETIF_F_TSO | 3745 NETIF_F_TSO6); 3746 3747 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3748 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3749 return features & ~(NETIF_F_HW_CSUM | 3750 NETIF_F_SCTP_CRC | 3751 NETIF_F_TSO | 3752 NETIF_F_TSO6); 3753 3754 /* We can only support IPv4 TSO in tunnels if we can mangle the 3755 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3756 */ 3757 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3758 features &= ~NETIF_F_TSO; 3759 3760 return features; 3761 } 3762 3763 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3764 { 3765 struct igc_hw *hw = &adapter->hw; 3766 u32 tsicr = rd32(IGC_TSICR); 3767 u32 ack = 0; 3768 3769 if (tsicr & IGC_TSICR_TXTS) { 3770 /* retrieve hardware timestamp */ 3771 schedule_work(&adapter->ptp_tx_work); 3772 ack |= IGC_TSICR_TXTS; 3773 } 3774 3775 /* acknowledge the interrupts */ 3776 wr32(IGC_TSICR, ack); 3777 } 3778 3779 /** 3780 * igc_msix_other - msix other interrupt handler 3781 * @irq: interrupt number 3782 * @data: pointer to a q_vector 3783 */ 3784 static irqreturn_t igc_msix_other(int irq, void *data) 3785 { 3786 struct igc_adapter *adapter = data; 3787 struct igc_hw *hw = &adapter->hw; 3788 u32 icr = rd32(IGC_ICR); 3789 3790 /* reading ICR causes bit 31 of EICR to be cleared */ 3791 if (icr & IGC_ICR_DRSTA) 3792 schedule_work(&adapter->reset_task); 3793 3794 if (icr & IGC_ICR_DOUTSYNC) { 3795 /* HW is reporting DMA is out of sync */ 3796 adapter->stats.doosync++; 3797 } 3798 3799 if (icr & IGC_ICR_LSC) { 3800 hw->mac.get_link_status = 1; 3801 /* guard against interrupt when we're going down */ 3802 if (!test_bit(__IGC_DOWN, &adapter->state)) 3803 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3804 } 3805 3806 if (icr & IGC_ICR_TS) 3807 igc_tsync_interrupt(adapter); 3808 3809 wr32(IGC_EIMS, adapter->eims_other); 3810 3811 return IRQ_HANDLED; 3812 } 3813 3814 static void igc_write_itr(struct igc_q_vector *q_vector) 3815 { 3816 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 3817 3818 if (!q_vector->set_itr) 3819 return; 3820 3821 if (!itr_val) 3822 itr_val = IGC_ITR_VAL_MASK; 3823 3824 itr_val |= IGC_EITR_CNT_IGNR; 3825 3826 writel(itr_val, q_vector->itr_register); 3827 q_vector->set_itr = 0; 3828 } 3829 3830 static irqreturn_t igc_msix_ring(int irq, void *data) 3831 { 3832 struct igc_q_vector *q_vector = data; 3833 3834 /* Write the ITR value calculated from the previous interrupt. */ 3835 igc_write_itr(q_vector); 3836 3837 napi_schedule(&q_vector->napi); 3838 3839 return IRQ_HANDLED; 3840 } 3841 3842 /** 3843 * igc_request_msix - Initialize MSI-X interrupts 3844 * @adapter: Pointer to adapter structure 3845 * 3846 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 3847 * kernel. 3848 */ 3849 static int igc_request_msix(struct igc_adapter *adapter) 3850 { 3851 int i = 0, err = 0, vector = 0, free_vector = 0; 3852 struct net_device *netdev = adapter->netdev; 3853 3854 err = request_irq(adapter->msix_entries[vector].vector, 3855 &igc_msix_other, 0, netdev->name, adapter); 3856 if (err) 3857 goto err_out; 3858 3859 for (i = 0; i < adapter->num_q_vectors; i++) { 3860 struct igc_q_vector *q_vector = adapter->q_vector[i]; 3861 3862 vector++; 3863 3864 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 3865 3866 if (q_vector->rx.ring && q_vector->tx.ring) 3867 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 3868 q_vector->rx.ring->queue_index); 3869 else if (q_vector->tx.ring) 3870 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 3871 q_vector->tx.ring->queue_index); 3872 else if (q_vector->rx.ring) 3873 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 3874 q_vector->rx.ring->queue_index); 3875 else 3876 sprintf(q_vector->name, "%s-unused", netdev->name); 3877 3878 err = request_irq(adapter->msix_entries[vector].vector, 3879 igc_msix_ring, 0, q_vector->name, 3880 q_vector); 3881 if (err) 3882 goto err_free; 3883 } 3884 3885 igc_configure_msix(adapter); 3886 return 0; 3887 3888 err_free: 3889 /* free already assigned IRQs */ 3890 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 3891 3892 vector--; 3893 for (i = 0; i < vector; i++) { 3894 free_irq(adapter->msix_entries[free_vector++].vector, 3895 adapter->q_vector[i]); 3896 } 3897 err_out: 3898 return err; 3899 } 3900 3901 /** 3902 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 3903 * @adapter: Pointer to adapter structure 3904 * 3905 * This function resets the device so that it has 0 rx queues, tx queues, and 3906 * MSI-X interrupts allocated. 3907 */ 3908 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 3909 { 3910 igc_free_q_vectors(adapter); 3911 igc_reset_interrupt_capability(adapter); 3912 } 3913 3914 /* Need to wait a few seconds after link up to get diagnostic information from 3915 * the phy 3916 */ 3917 static void igc_update_phy_info(struct timer_list *t) 3918 { 3919 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 3920 3921 igc_get_phy_info(&adapter->hw); 3922 } 3923 3924 /** 3925 * igc_has_link - check shared code for link and determine up/down 3926 * @adapter: pointer to driver private info 3927 */ 3928 bool igc_has_link(struct igc_adapter *adapter) 3929 { 3930 struct igc_hw *hw = &adapter->hw; 3931 bool link_active = false; 3932 3933 /* get_link_status is set on LSC (link status) interrupt or 3934 * rx sequence error interrupt. get_link_status will stay 3935 * false until the igc_check_for_link establishes link 3936 * for copper adapters ONLY 3937 */ 3938 switch (hw->phy.media_type) { 3939 case igc_media_type_copper: 3940 if (!hw->mac.get_link_status) 3941 return true; 3942 hw->mac.ops.check_for_link(hw); 3943 link_active = !hw->mac.get_link_status; 3944 break; 3945 default: 3946 case igc_media_type_unknown: 3947 break; 3948 } 3949 3950 if (hw->mac.type == igc_i225 && 3951 hw->phy.id == I225_I_PHY_ID) { 3952 if (!netif_carrier_ok(adapter->netdev)) { 3953 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3954 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 3955 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 3956 adapter->link_check_timeout = jiffies; 3957 } 3958 } 3959 3960 return link_active; 3961 } 3962 3963 /** 3964 * igc_watchdog - Timer Call-back 3965 * @t: timer for the watchdog 3966 */ 3967 static void igc_watchdog(struct timer_list *t) 3968 { 3969 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 3970 /* Do the rest outside of interrupt context */ 3971 schedule_work(&adapter->watchdog_task); 3972 } 3973 3974 static void igc_watchdog_task(struct work_struct *work) 3975 { 3976 struct igc_adapter *adapter = container_of(work, 3977 struct igc_adapter, 3978 watchdog_task); 3979 struct net_device *netdev = adapter->netdev; 3980 struct igc_hw *hw = &adapter->hw; 3981 struct igc_phy_info *phy = &hw->phy; 3982 u16 phy_data, retry_count = 20; 3983 u32 link; 3984 int i; 3985 3986 link = igc_has_link(adapter); 3987 3988 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 3989 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 3990 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3991 else 3992 link = false; 3993 } 3994 3995 if (link) { 3996 /* Cancel scheduled suspend requests. */ 3997 pm_runtime_resume(netdev->dev.parent); 3998 3999 if (!netif_carrier_ok(netdev)) { 4000 u32 ctrl; 4001 4002 hw->mac.ops.get_speed_and_duplex(hw, 4003 &adapter->link_speed, 4004 &adapter->link_duplex); 4005 4006 ctrl = rd32(IGC_CTRL); 4007 /* Link status message must follow this format */ 4008 netdev_info(netdev, 4009 "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4010 netdev->name, 4011 adapter->link_speed, 4012 adapter->link_duplex == FULL_DUPLEX ? 4013 "Full" : "Half", 4014 (ctrl & IGC_CTRL_TFCE) && 4015 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4016 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4017 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4018 4019 /* check if SmartSpeed worked */ 4020 igc_check_downshift(hw); 4021 if (phy->speed_downgraded) 4022 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4023 4024 /* adjust timeout factor according to speed/duplex */ 4025 adapter->tx_timeout_factor = 1; 4026 switch (adapter->link_speed) { 4027 case SPEED_10: 4028 adapter->tx_timeout_factor = 14; 4029 break; 4030 case SPEED_100: 4031 /* maybe add some timeout factor ? */ 4032 break; 4033 } 4034 4035 if (adapter->link_speed != SPEED_1000) 4036 goto no_wait; 4037 4038 /* wait for Remote receiver status OK */ 4039 retry_read_status: 4040 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4041 &phy_data)) { 4042 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4043 retry_count) { 4044 msleep(100); 4045 retry_count--; 4046 goto retry_read_status; 4047 } else if (!retry_count) { 4048 dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); 4049 } 4050 } else { 4051 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); 4052 } 4053 no_wait: 4054 netif_carrier_on(netdev); 4055 4056 /* link state has changed, schedule phy info update */ 4057 if (!test_bit(__IGC_DOWN, &adapter->state)) 4058 mod_timer(&adapter->phy_info_timer, 4059 round_jiffies(jiffies + 2 * HZ)); 4060 } 4061 } else { 4062 if (netif_carrier_ok(netdev)) { 4063 adapter->link_speed = 0; 4064 adapter->link_duplex = 0; 4065 4066 /* Links status message must follow this format */ 4067 netdev_info(netdev, "igc: %s NIC Link is Down\n", 4068 netdev->name); 4069 netif_carrier_off(netdev); 4070 4071 /* link state has changed, schedule phy info update */ 4072 if (!test_bit(__IGC_DOWN, &adapter->state)) 4073 mod_timer(&adapter->phy_info_timer, 4074 round_jiffies(jiffies + 2 * HZ)); 4075 4076 /* link is down, time to check for alternate media */ 4077 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4078 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4079 schedule_work(&adapter->reset_task); 4080 /* return immediately */ 4081 return; 4082 } 4083 } 4084 pm_schedule_suspend(netdev->dev.parent, 4085 MSEC_PER_SEC * 5); 4086 4087 /* also check for alternate media here */ 4088 } else if (!netif_carrier_ok(netdev) && 4089 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4090 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4091 schedule_work(&adapter->reset_task); 4092 /* return immediately */ 4093 return; 4094 } 4095 } 4096 } 4097 4098 spin_lock(&adapter->stats64_lock); 4099 igc_update_stats(adapter); 4100 spin_unlock(&adapter->stats64_lock); 4101 4102 for (i = 0; i < adapter->num_tx_queues; i++) { 4103 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4104 4105 if (!netif_carrier_ok(netdev)) { 4106 /* We've lost link, so the controller stops DMA, 4107 * but we've got queued Tx work that's never going 4108 * to get done, so reset controller to flush Tx. 4109 * (Do the reset outside of interrupt context). 4110 */ 4111 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4112 adapter->tx_timeout_count++; 4113 schedule_work(&adapter->reset_task); 4114 /* return immediately since reset is imminent */ 4115 return; 4116 } 4117 } 4118 4119 /* Force detection of hung controller every watchdog period */ 4120 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4121 } 4122 4123 /* Cause software interrupt to ensure Rx ring is cleaned */ 4124 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4125 u32 eics = 0; 4126 4127 for (i = 0; i < adapter->num_q_vectors; i++) 4128 eics |= adapter->q_vector[i]->eims_value; 4129 wr32(IGC_EICS, eics); 4130 } else { 4131 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4132 } 4133 4134 igc_ptp_tx_hang(adapter); 4135 4136 /* Reset the timer */ 4137 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4138 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4139 mod_timer(&adapter->watchdog_timer, 4140 round_jiffies(jiffies + HZ)); 4141 else 4142 mod_timer(&adapter->watchdog_timer, 4143 round_jiffies(jiffies + 2 * HZ)); 4144 } 4145 } 4146 4147 /** 4148 * igc_intr_msi - Interrupt Handler 4149 * @irq: interrupt number 4150 * @data: pointer to a network interface device structure 4151 */ 4152 static irqreturn_t igc_intr_msi(int irq, void *data) 4153 { 4154 struct igc_adapter *adapter = data; 4155 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4156 struct igc_hw *hw = &adapter->hw; 4157 /* read ICR disables interrupts using IAM */ 4158 u32 icr = rd32(IGC_ICR); 4159 4160 igc_write_itr(q_vector); 4161 4162 if (icr & IGC_ICR_DRSTA) 4163 schedule_work(&adapter->reset_task); 4164 4165 if (icr & IGC_ICR_DOUTSYNC) { 4166 /* HW is reporting DMA is out of sync */ 4167 adapter->stats.doosync++; 4168 } 4169 4170 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4171 hw->mac.get_link_status = 1; 4172 if (!test_bit(__IGC_DOWN, &adapter->state)) 4173 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4174 } 4175 4176 napi_schedule(&q_vector->napi); 4177 4178 return IRQ_HANDLED; 4179 } 4180 4181 /** 4182 * igc_intr - Legacy Interrupt Handler 4183 * @irq: interrupt number 4184 * @data: pointer to a network interface device structure 4185 */ 4186 static irqreturn_t igc_intr(int irq, void *data) 4187 { 4188 struct igc_adapter *adapter = data; 4189 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4190 struct igc_hw *hw = &adapter->hw; 4191 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4192 * need for the IMC write 4193 */ 4194 u32 icr = rd32(IGC_ICR); 4195 4196 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4197 * not set, then the adapter didn't send an interrupt 4198 */ 4199 if (!(icr & IGC_ICR_INT_ASSERTED)) 4200 return IRQ_NONE; 4201 4202 igc_write_itr(q_vector); 4203 4204 if (icr & IGC_ICR_DRSTA) 4205 schedule_work(&adapter->reset_task); 4206 4207 if (icr & IGC_ICR_DOUTSYNC) { 4208 /* HW is reporting DMA is out of sync */ 4209 adapter->stats.doosync++; 4210 } 4211 4212 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4213 hw->mac.get_link_status = 1; 4214 /* guard against interrupt when we're going down */ 4215 if (!test_bit(__IGC_DOWN, &adapter->state)) 4216 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4217 } 4218 4219 napi_schedule(&q_vector->napi); 4220 4221 return IRQ_HANDLED; 4222 } 4223 4224 static void igc_free_irq(struct igc_adapter *adapter) 4225 { 4226 if (adapter->msix_entries) { 4227 int vector = 0, i; 4228 4229 free_irq(adapter->msix_entries[vector++].vector, adapter); 4230 4231 for (i = 0; i < adapter->num_q_vectors; i++) 4232 free_irq(adapter->msix_entries[vector++].vector, 4233 adapter->q_vector[i]); 4234 } else { 4235 free_irq(adapter->pdev->irq, adapter); 4236 } 4237 } 4238 4239 /** 4240 * igc_request_irq - initialize interrupts 4241 * @adapter: Pointer to adapter structure 4242 * 4243 * Attempts to configure interrupts using the best available 4244 * capabilities of the hardware and kernel. 4245 */ 4246 static int igc_request_irq(struct igc_adapter *adapter) 4247 { 4248 struct net_device *netdev = adapter->netdev; 4249 struct pci_dev *pdev = adapter->pdev; 4250 int err = 0; 4251 4252 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4253 err = igc_request_msix(adapter); 4254 if (!err) 4255 goto request_done; 4256 /* fall back to MSI */ 4257 igc_free_all_tx_resources(adapter); 4258 igc_free_all_rx_resources(adapter); 4259 4260 igc_clear_interrupt_scheme(adapter); 4261 err = igc_init_interrupt_scheme(adapter, false); 4262 if (err) 4263 goto request_done; 4264 igc_setup_all_tx_resources(adapter); 4265 igc_setup_all_rx_resources(adapter); 4266 igc_configure(adapter); 4267 } 4268 4269 igc_assign_vector(adapter->q_vector[0], 0); 4270 4271 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4272 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4273 netdev->name, adapter); 4274 if (!err) 4275 goto request_done; 4276 4277 /* fall back to legacy interrupts */ 4278 igc_reset_interrupt_capability(adapter); 4279 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4280 } 4281 4282 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4283 netdev->name, adapter); 4284 4285 if (err) 4286 dev_err(&pdev->dev, "Error %d getting interrupt\n", 4287 err); 4288 4289 request_done: 4290 return err; 4291 } 4292 4293 /** 4294 * __igc_open - Called when a network interface is made active 4295 * @netdev: network interface device structure 4296 * @resuming: boolean indicating if the device is resuming 4297 * 4298 * Returns 0 on success, negative value on failure 4299 * 4300 * The open entry point is called when a network interface is made 4301 * active by the system (IFF_UP). At this point all resources needed 4302 * for transmit and receive operations are allocated, the interrupt 4303 * handler is registered with the OS, the watchdog timer is started, 4304 * and the stack is notified that the interface is ready. 4305 */ 4306 static int __igc_open(struct net_device *netdev, bool resuming) 4307 { 4308 struct igc_adapter *adapter = netdev_priv(netdev); 4309 struct pci_dev *pdev = adapter->pdev; 4310 struct igc_hw *hw = &adapter->hw; 4311 int err = 0; 4312 int i = 0; 4313 4314 /* disallow open during test */ 4315 4316 if (test_bit(__IGC_TESTING, &adapter->state)) { 4317 WARN_ON(resuming); 4318 return -EBUSY; 4319 } 4320 4321 if (!resuming) 4322 pm_runtime_get_sync(&pdev->dev); 4323 4324 netif_carrier_off(netdev); 4325 4326 /* allocate transmit descriptors */ 4327 err = igc_setup_all_tx_resources(adapter); 4328 if (err) 4329 goto err_setup_tx; 4330 4331 /* allocate receive descriptors */ 4332 err = igc_setup_all_rx_resources(adapter); 4333 if (err) 4334 goto err_setup_rx; 4335 4336 igc_power_up_link(adapter); 4337 4338 igc_configure(adapter); 4339 4340 err = igc_request_irq(adapter); 4341 if (err) 4342 goto err_req_irq; 4343 4344 /* Notify the stack of the actual queue counts. */ 4345 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4346 if (err) 4347 goto err_set_queues; 4348 4349 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4350 if (err) 4351 goto err_set_queues; 4352 4353 clear_bit(__IGC_DOWN, &adapter->state); 4354 4355 for (i = 0; i < adapter->num_q_vectors; i++) 4356 napi_enable(&adapter->q_vector[i]->napi); 4357 4358 /* Clear any pending interrupts. */ 4359 rd32(IGC_ICR); 4360 igc_irq_enable(adapter); 4361 4362 if (!resuming) 4363 pm_runtime_put(&pdev->dev); 4364 4365 netif_tx_start_all_queues(netdev); 4366 4367 /* start the watchdog. */ 4368 hw->mac.get_link_status = 1; 4369 schedule_work(&adapter->watchdog_task); 4370 4371 return IGC_SUCCESS; 4372 4373 err_set_queues: 4374 igc_free_irq(adapter); 4375 err_req_irq: 4376 igc_release_hw_control(adapter); 4377 igc_power_down_link(adapter); 4378 igc_free_all_rx_resources(adapter); 4379 err_setup_rx: 4380 igc_free_all_tx_resources(adapter); 4381 err_setup_tx: 4382 igc_reset(adapter); 4383 if (!resuming) 4384 pm_runtime_put(&pdev->dev); 4385 4386 return err; 4387 } 4388 4389 static int igc_open(struct net_device *netdev) 4390 { 4391 return __igc_open(netdev, false); 4392 } 4393 4394 /** 4395 * __igc_close - Disables a network interface 4396 * @netdev: network interface device structure 4397 * @suspending: boolean indicating the device is suspending 4398 * 4399 * Returns 0, this is not allowed to fail 4400 * 4401 * The close entry point is called when an interface is de-activated 4402 * by the OS. The hardware is still under the driver's control, but 4403 * needs to be disabled. A global MAC reset is issued to stop the 4404 * hardware, and all transmit and receive resources are freed. 4405 */ 4406 static int __igc_close(struct net_device *netdev, bool suspending) 4407 { 4408 struct igc_adapter *adapter = netdev_priv(netdev); 4409 struct pci_dev *pdev = adapter->pdev; 4410 4411 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4412 4413 if (!suspending) 4414 pm_runtime_get_sync(&pdev->dev); 4415 4416 igc_down(adapter); 4417 4418 igc_release_hw_control(adapter); 4419 4420 igc_free_irq(adapter); 4421 4422 igc_free_all_tx_resources(adapter); 4423 igc_free_all_rx_resources(adapter); 4424 4425 if (!suspending) 4426 pm_runtime_put_sync(&pdev->dev); 4427 4428 return 0; 4429 } 4430 4431 static int igc_close(struct net_device *netdev) 4432 { 4433 if (netif_device_present(netdev) || netdev->dismantle) 4434 return __igc_close(netdev, false); 4435 return 0; 4436 } 4437 4438 /** 4439 * igc_ioctl - Access the hwtstamp interface 4440 * @netdev: network interface device structure 4441 * @ifreq: interface request data 4442 * @cmd: ioctl command 4443 **/ 4444 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4445 { 4446 switch (cmd) { 4447 case SIOCGHWTSTAMP: 4448 return igc_ptp_get_ts_config(netdev, ifr); 4449 case SIOCSHWTSTAMP: 4450 return igc_ptp_set_ts_config(netdev, ifr); 4451 default: 4452 return -EOPNOTSUPP; 4453 } 4454 } 4455 4456 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4457 bool enable) 4458 { 4459 struct igc_ring *ring; 4460 int i; 4461 4462 if (queue < 0 || queue >= adapter->num_tx_queues) 4463 return -EINVAL; 4464 4465 ring = adapter->tx_ring[queue]; 4466 ring->launchtime_enable = enable; 4467 4468 if (adapter->base_time) 4469 return 0; 4470 4471 adapter->cycle_time = NSEC_PER_SEC; 4472 4473 for (i = 0; i < adapter->num_tx_queues; i++) { 4474 ring = adapter->tx_ring[i]; 4475 ring->start_time = 0; 4476 ring->end_time = NSEC_PER_SEC; 4477 } 4478 4479 return 0; 4480 } 4481 4482 static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) 4483 { 4484 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4485 size_t n; 4486 4487 if (qopt->cycle_time_extension) 4488 return false; 4489 4490 for (n = 0; n < qopt->num_entries; n++) { 4491 const struct tc_taprio_sched_entry *e; 4492 int i; 4493 4494 e = &qopt->entries[n]; 4495 4496 /* i225 only supports "global" frame preemption 4497 * settings. 4498 */ 4499 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4500 return false; 4501 4502 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4503 if (e->gate_mask & BIT(i)) 4504 queue_uses[i]++; 4505 4506 if (queue_uses[i] > 1) 4507 return false; 4508 } 4509 } 4510 4511 return true; 4512 } 4513 4514 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4515 struct tc_etf_qopt_offload *qopt) 4516 { 4517 struct igc_hw *hw = &adapter->hw; 4518 int err; 4519 4520 if (hw->mac.type != igc_i225) 4521 return -EOPNOTSUPP; 4522 4523 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4524 if (err) 4525 return err; 4526 4527 return igc_tsn_offload_apply(adapter); 4528 } 4529 4530 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4531 struct tc_taprio_qopt_offload *qopt) 4532 { 4533 u32 start_time = 0, end_time = 0; 4534 size_t n; 4535 4536 if (!qopt->enable) { 4537 adapter->base_time = 0; 4538 return 0; 4539 } 4540 4541 if (adapter->base_time) 4542 return -EALREADY; 4543 4544 if (!validate_schedule(qopt)) 4545 return -EINVAL; 4546 4547 adapter->cycle_time = qopt->cycle_time; 4548 adapter->base_time = qopt->base_time; 4549 4550 /* FIXME: be a little smarter about cases when the gate for a 4551 * queue stays open for more than one entry. 4552 */ 4553 for (n = 0; n < qopt->num_entries; n++) { 4554 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4555 int i; 4556 4557 end_time += e->interval; 4558 4559 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4560 struct igc_ring *ring = adapter->tx_ring[i]; 4561 4562 if (!(e->gate_mask & BIT(i))) 4563 continue; 4564 4565 ring->start_time = start_time; 4566 ring->end_time = end_time; 4567 } 4568 4569 start_time += e->interval; 4570 } 4571 4572 return 0; 4573 } 4574 4575 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4576 struct tc_taprio_qopt_offload *qopt) 4577 { 4578 struct igc_hw *hw = &adapter->hw; 4579 int err; 4580 4581 if (hw->mac.type != igc_i225) 4582 return -EOPNOTSUPP; 4583 4584 err = igc_save_qbv_schedule(adapter, qopt); 4585 if (err) 4586 return err; 4587 4588 return igc_tsn_offload_apply(adapter); 4589 } 4590 4591 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4592 void *type_data) 4593 { 4594 struct igc_adapter *adapter = netdev_priv(dev); 4595 4596 switch (type) { 4597 case TC_SETUP_QDISC_TAPRIO: 4598 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4599 4600 case TC_SETUP_QDISC_ETF: 4601 return igc_tsn_enable_launchtime(adapter, type_data); 4602 4603 default: 4604 return -EOPNOTSUPP; 4605 } 4606 } 4607 4608 static const struct net_device_ops igc_netdev_ops = { 4609 .ndo_open = igc_open, 4610 .ndo_stop = igc_close, 4611 .ndo_start_xmit = igc_xmit_frame, 4612 .ndo_set_rx_mode = igc_set_rx_mode, 4613 .ndo_set_mac_address = igc_set_mac, 4614 .ndo_change_mtu = igc_change_mtu, 4615 .ndo_get_stats = igc_get_stats, 4616 .ndo_fix_features = igc_fix_features, 4617 .ndo_set_features = igc_set_features, 4618 .ndo_features_check = igc_features_check, 4619 .ndo_do_ioctl = igc_ioctl, 4620 .ndo_setup_tc = igc_setup_tc, 4621 }; 4622 4623 /* PCIe configuration access */ 4624 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4625 { 4626 struct igc_adapter *adapter = hw->back; 4627 4628 pci_read_config_word(adapter->pdev, reg, value); 4629 } 4630 4631 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4632 { 4633 struct igc_adapter *adapter = hw->back; 4634 4635 pci_write_config_word(adapter->pdev, reg, *value); 4636 } 4637 4638 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4639 { 4640 struct igc_adapter *adapter = hw->back; 4641 4642 if (!pci_is_pcie(adapter->pdev)) 4643 return -IGC_ERR_CONFIG; 4644 4645 pcie_capability_read_word(adapter->pdev, reg, value); 4646 4647 return IGC_SUCCESS; 4648 } 4649 4650 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4651 { 4652 struct igc_adapter *adapter = hw->back; 4653 4654 if (!pci_is_pcie(adapter->pdev)) 4655 return -IGC_ERR_CONFIG; 4656 4657 pcie_capability_write_word(adapter->pdev, reg, *value); 4658 4659 return IGC_SUCCESS; 4660 } 4661 4662 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4663 { 4664 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4665 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4666 u32 value = 0; 4667 4668 if (IGC_REMOVED(hw_addr)) 4669 return ~value; 4670 4671 value = readl(&hw_addr[reg]); 4672 4673 /* reads should not return all F's */ 4674 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4675 struct net_device *netdev = igc->netdev; 4676 4677 hw->hw_addr = NULL; 4678 netif_device_detach(netdev); 4679 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4680 WARN(pci_device_is_present(igc->pdev), 4681 "igc: Failed to read reg 0x%x!\n", reg); 4682 } 4683 4684 return value; 4685 } 4686 4687 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4688 { 4689 struct pci_dev *pdev = adapter->pdev; 4690 struct igc_mac_info *mac = &adapter->hw.mac; 4691 4692 mac->autoneg = 0; 4693 4694 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4695 * for the switch() below to work 4696 */ 4697 if ((spd & 1) || (dplx & ~1)) 4698 goto err_inval; 4699 4700 switch (spd + dplx) { 4701 case SPEED_10 + DUPLEX_HALF: 4702 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4703 break; 4704 case SPEED_10 + DUPLEX_FULL: 4705 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4706 break; 4707 case SPEED_100 + DUPLEX_HALF: 4708 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4709 break; 4710 case SPEED_100 + DUPLEX_FULL: 4711 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4712 break; 4713 case SPEED_1000 + DUPLEX_FULL: 4714 mac->autoneg = 1; 4715 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4716 break; 4717 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4718 goto err_inval; 4719 case SPEED_2500 + DUPLEX_FULL: 4720 mac->autoneg = 1; 4721 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4722 break; 4723 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4724 default: 4725 goto err_inval; 4726 } 4727 4728 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4729 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4730 4731 return 0; 4732 4733 err_inval: 4734 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 4735 return -EINVAL; 4736 } 4737 4738 /** 4739 * igc_probe - Device Initialization Routine 4740 * @pdev: PCI device information struct 4741 * @ent: entry in igc_pci_tbl 4742 * 4743 * Returns 0 on success, negative on failure 4744 * 4745 * igc_probe initializes an adapter identified by a pci_dev structure. 4746 * The OS initialization, configuring the adapter private structure, 4747 * and a hardware reset occur. 4748 */ 4749 static int igc_probe(struct pci_dev *pdev, 4750 const struct pci_device_id *ent) 4751 { 4752 struct igc_adapter *adapter; 4753 struct net_device *netdev; 4754 struct igc_hw *hw; 4755 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4756 int err, pci_using_dac; 4757 4758 err = pci_enable_device_mem(pdev); 4759 if (err) 4760 return err; 4761 4762 pci_using_dac = 0; 4763 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4764 if (!err) { 4765 pci_using_dac = 1; 4766 } else { 4767 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4768 if (err) { 4769 dev_err(&pdev->dev, 4770 "No usable DMA configuration, aborting\n"); 4771 goto err_dma; 4772 } 4773 } 4774 4775 err = pci_request_mem_regions(pdev, igc_driver_name); 4776 if (err) 4777 goto err_pci_reg; 4778 4779 pci_enable_pcie_error_reporting(pdev); 4780 4781 pci_set_master(pdev); 4782 4783 err = -ENOMEM; 4784 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 4785 IGC_MAX_TX_QUEUES); 4786 4787 if (!netdev) 4788 goto err_alloc_etherdev; 4789 4790 SET_NETDEV_DEV(netdev, &pdev->dev); 4791 4792 pci_set_drvdata(pdev, netdev); 4793 adapter = netdev_priv(netdev); 4794 adapter->netdev = netdev; 4795 adapter->pdev = pdev; 4796 hw = &adapter->hw; 4797 hw->back = adapter; 4798 adapter->port_num = hw->bus.func; 4799 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4800 4801 err = pci_save_state(pdev); 4802 if (err) 4803 goto err_ioremap; 4804 4805 err = -EIO; 4806 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 4807 pci_resource_len(pdev, 0)); 4808 if (!adapter->io_addr) 4809 goto err_ioremap; 4810 4811 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 4812 hw->hw_addr = adapter->io_addr; 4813 4814 netdev->netdev_ops = &igc_netdev_ops; 4815 igc_set_ethtool_ops(netdev); 4816 netdev->watchdog_timeo = 5 * HZ; 4817 4818 netdev->mem_start = pci_resource_start(pdev, 0); 4819 netdev->mem_end = pci_resource_end(pdev, 0); 4820 4821 /* PCI config space info */ 4822 hw->vendor_id = pdev->vendor; 4823 hw->device_id = pdev->device; 4824 hw->revision_id = pdev->revision; 4825 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4826 hw->subsystem_device_id = pdev->subsystem_device; 4827 4828 /* Copy the default MAC and PHY function pointers */ 4829 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 4830 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 4831 4832 /* Initialize skew-specific constants */ 4833 err = ei->get_invariants(hw); 4834 if (err) 4835 goto err_sw_init; 4836 4837 /* Add supported features to the features list*/ 4838 netdev->features |= NETIF_F_SG; 4839 netdev->features |= NETIF_F_TSO; 4840 netdev->features |= NETIF_F_TSO6; 4841 netdev->features |= NETIF_F_RXCSUM; 4842 netdev->features |= NETIF_F_HW_CSUM; 4843 netdev->features |= NETIF_F_SCTP_CRC; 4844 netdev->features |= NETIF_F_HW_TC; 4845 4846 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 4847 NETIF_F_GSO_GRE_CSUM | \ 4848 NETIF_F_GSO_IPXIP4 | \ 4849 NETIF_F_GSO_IPXIP6 | \ 4850 NETIF_F_GSO_UDP_TUNNEL | \ 4851 NETIF_F_GSO_UDP_TUNNEL_CSUM) 4852 4853 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 4854 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 4855 4856 /* setup the private structure */ 4857 err = igc_sw_init(adapter); 4858 if (err) 4859 goto err_sw_init; 4860 4861 /* copy netdev features into list of user selectable features */ 4862 netdev->hw_features |= NETIF_F_NTUPLE; 4863 netdev->hw_features |= netdev->features; 4864 4865 if (pci_using_dac) 4866 netdev->features |= NETIF_F_HIGHDMA; 4867 4868 /* MTU range: 68 - 9216 */ 4869 netdev->min_mtu = ETH_MIN_MTU; 4870 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 4871 4872 /* before reading the NVM, reset the controller to put the device in a 4873 * known good starting state 4874 */ 4875 hw->mac.ops.reset_hw(hw); 4876 4877 if (igc_get_flash_presence_i225(hw)) { 4878 if (hw->nvm.ops.validate(hw) < 0) { 4879 dev_err(&pdev->dev, 4880 "The NVM Checksum Is Not Valid\n"); 4881 err = -EIO; 4882 goto err_eeprom; 4883 } 4884 } 4885 4886 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 4887 /* copy the MAC address out of the NVM */ 4888 if (hw->mac.ops.read_mac_addr(hw)) 4889 dev_err(&pdev->dev, "NVM Read Error\n"); 4890 } 4891 4892 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 4893 4894 if (!is_valid_ether_addr(netdev->dev_addr)) { 4895 dev_err(&pdev->dev, "Invalid MAC Address\n"); 4896 err = -EIO; 4897 goto err_eeprom; 4898 } 4899 4900 /* configure RXPBSIZE and TXPBSIZE */ 4901 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 4902 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 4903 4904 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 4905 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 4906 4907 INIT_WORK(&adapter->reset_task, igc_reset_task); 4908 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 4909 4910 /* Initialize link properties that are user-changeable */ 4911 adapter->fc_autoneg = true; 4912 hw->mac.autoneg = true; 4913 hw->phy.autoneg_advertised = 0xaf; 4914 4915 hw->fc.requested_mode = igc_fc_default; 4916 hw->fc.current_mode = igc_fc_default; 4917 4918 /* By default, support wake on port A */ 4919 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 4920 4921 /* initialize the wol settings based on the eeprom settings */ 4922 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 4923 adapter->wol |= IGC_WUFC_MAG; 4924 4925 device_set_wakeup_enable(&adapter->pdev->dev, 4926 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 4927 4928 /* reset the hardware with the new settings */ 4929 igc_reset(adapter); 4930 4931 /* let the f/w know that the h/w is now under the control of the 4932 * driver. 4933 */ 4934 igc_get_hw_control(adapter); 4935 4936 strncpy(netdev->name, "eth%d", IFNAMSIZ); 4937 err = register_netdev(netdev); 4938 if (err) 4939 goto err_register; 4940 4941 /* carrier off reporting is important to ethtool even BEFORE open */ 4942 netif_carrier_off(netdev); 4943 4944 /* do hw tstamp init after resetting */ 4945 igc_ptp_init(adapter); 4946 4947 /* Check if Media Autosense is enabled */ 4948 adapter->ei = *ei; 4949 4950 /* print pcie link status and MAC address */ 4951 pcie_print_link_status(pdev); 4952 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 4953 4954 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NEVER_SKIP); 4955 4956 pm_runtime_put_noidle(&pdev->dev); 4957 4958 return 0; 4959 4960 err_register: 4961 igc_release_hw_control(adapter); 4962 err_eeprom: 4963 if (!igc_check_reset_block(hw)) 4964 igc_reset_phy(hw); 4965 err_sw_init: 4966 igc_clear_interrupt_scheme(adapter); 4967 iounmap(adapter->io_addr); 4968 err_ioremap: 4969 free_netdev(netdev); 4970 err_alloc_etherdev: 4971 pci_release_mem_regions(pdev); 4972 err_pci_reg: 4973 err_dma: 4974 pci_disable_device(pdev); 4975 return err; 4976 } 4977 4978 /** 4979 * igc_remove - Device Removal Routine 4980 * @pdev: PCI device information struct 4981 * 4982 * igc_remove is called by the PCI subsystem to alert the driver 4983 * that it should release a PCI device. This could be caused by a 4984 * Hot-Plug event, or because the driver is going to be removed from 4985 * memory. 4986 */ 4987 static void igc_remove(struct pci_dev *pdev) 4988 { 4989 struct net_device *netdev = pci_get_drvdata(pdev); 4990 struct igc_adapter *adapter = netdev_priv(netdev); 4991 4992 pm_runtime_get_noresume(&pdev->dev); 4993 4994 igc_ptp_stop(adapter); 4995 4996 set_bit(__IGC_DOWN, &adapter->state); 4997 4998 del_timer_sync(&adapter->watchdog_timer); 4999 del_timer_sync(&adapter->phy_info_timer); 5000 5001 cancel_work_sync(&adapter->reset_task); 5002 cancel_work_sync(&adapter->watchdog_task); 5003 5004 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5005 * would have already happened in close and is redundant. 5006 */ 5007 igc_release_hw_control(adapter); 5008 unregister_netdev(netdev); 5009 5010 igc_clear_interrupt_scheme(adapter); 5011 pci_iounmap(pdev, adapter->io_addr); 5012 pci_release_mem_regions(pdev); 5013 5014 kfree(adapter->mac_table); 5015 free_netdev(netdev); 5016 5017 pci_disable_pcie_error_reporting(pdev); 5018 5019 pci_disable_device(pdev); 5020 } 5021 5022 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5023 bool runtime) 5024 { 5025 struct net_device *netdev = pci_get_drvdata(pdev); 5026 struct igc_adapter *adapter = netdev_priv(netdev); 5027 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5028 struct igc_hw *hw = &adapter->hw; 5029 u32 ctrl, rctl, status; 5030 bool wake; 5031 5032 rtnl_lock(); 5033 netif_device_detach(netdev); 5034 5035 if (netif_running(netdev)) 5036 __igc_close(netdev, true); 5037 5038 igc_ptp_suspend(adapter); 5039 5040 igc_clear_interrupt_scheme(adapter); 5041 rtnl_unlock(); 5042 5043 status = rd32(IGC_STATUS); 5044 if (status & IGC_STATUS_LU) 5045 wufc &= ~IGC_WUFC_LNKC; 5046 5047 if (wufc) { 5048 igc_setup_rctl(adapter); 5049 igc_set_rx_mode(netdev); 5050 5051 /* turn on all-multi mode if wake on multicast is enabled */ 5052 if (wufc & IGC_WUFC_MC) { 5053 rctl = rd32(IGC_RCTL); 5054 rctl |= IGC_RCTL_MPE; 5055 wr32(IGC_RCTL, rctl); 5056 } 5057 5058 ctrl = rd32(IGC_CTRL); 5059 ctrl |= IGC_CTRL_ADVD3WUC; 5060 wr32(IGC_CTRL, ctrl); 5061 5062 /* Allow time for pending master requests to run */ 5063 igc_disable_pcie_master(hw); 5064 5065 wr32(IGC_WUC, IGC_WUC_PME_EN); 5066 wr32(IGC_WUFC, wufc); 5067 } else { 5068 wr32(IGC_WUC, 0); 5069 wr32(IGC_WUFC, 0); 5070 } 5071 5072 wake = wufc || adapter->en_mng_pt; 5073 if (!wake) 5074 igc_power_down_link(adapter); 5075 else 5076 igc_power_up_link(adapter); 5077 5078 if (enable_wake) 5079 *enable_wake = wake; 5080 5081 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5082 * would have already happened in close and is redundant. 5083 */ 5084 igc_release_hw_control(adapter); 5085 5086 pci_disable_device(pdev); 5087 5088 return 0; 5089 } 5090 5091 #ifdef CONFIG_PM 5092 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5093 { 5094 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5095 } 5096 5097 static void igc_deliver_wake_packet(struct net_device *netdev) 5098 { 5099 struct igc_adapter *adapter = netdev_priv(netdev); 5100 struct igc_hw *hw = &adapter->hw; 5101 struct sk_buff *skb; 5102 u32 wupl; 5103 5104 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5105 5106 /* WUPM stores only the first 128 bytes of the wake packet. 5107 * Read the packet only if we have the whole thing. 5108 */ 5109 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5110 return; 5111 5112 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5113 if (!skb) 5114 return; 5115 5116 skb_put(skb, wupl); 5117 5118 /* Ensure reads are 32-bit aligned */ 5119 wupl = roundup(wupl, 4); 5120 5121 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5122 5123 skb->protocol = eth_type_trans(skb, netdev); 5124 netif_rx(skb); 5125 } 5126 5127 static int __maybe_unused igc_resume(struct device *dev) 5128 { 5129 struct pci_dev *pdev = to_pci_dev(dev); 5130 struct net_device *netdev = pci_get_drvdata(pdev); 5131 struct igc_adapter *adapter = netdev_priv(netdev); 5132 struct igc_hw *hw = &adapter->hw; 5133 u32 err, val; 5134 5135 pci_set_power_state(pdev, PCI_D0); 5136 pci_restore_state(pdev); 5137 pci_save_state(pdev); 5138 5139 if (!pci_device_is_present(pdev)) 5140 return -ENODEV; 5141 err = pci_enable_device_mem(pdev); 5142 if (err) { 5143 dev_err(&pdev->dev, 5144 "igc: Cannot enable PCI device from suspend\n"); 5145 return err; 5146 } 5147 pci_set_master(pdev); 5148 5149 pci_enable_wake(pdev, PCI_D3hot, 0); 5150 pci_enable_wake(pdev, PCI_D3cold, 0); 5151 5152 if (igc_init_interrupt_scheme(adapter, true)) { 5153 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 5154 return -ENOMEM; 5155 } 5156 5157 igc_reset(adapter); 5158 5159 /* let the f/w know that the h/w is now under the control of the 5160 * driver. 5161 */ 5162 igc_get_hw_control(adapter); 5163 5164 val = rd32(IGC_WUS); 5165 if (val & WAKE_PKT_WUS) 5166 igc_deliver_wake_packet(netdev); 5167 5168 wr32(IGC_WUS, ~0); 5169 5170 rtnl_lock(); 5171 if (!err && netif_running(netdev)) 5172 err = __igc_open(netdev, true); 5173 5174 if (!err) 5175 netif_device_attach(netdev); 5176 rtnl_unlock(); 5177 5178 return err; 5179 } 5180 5181 static int __maybe_unused igc_runtime_resume(struct device *dev) 5182 { 5183 return igc_resume(dev); 5184 } 5185 5186 static int __maybe_unused igc_suspend(struct device *dev) 5187 { 5188 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5189 } 5190 5191 static int __maybe_unused igc_runtime_idle(struct device *dev) 5192 { 5193 struct net_device *netdev = dev_get_drvdata(dev); 5194 struct igc_adapter *adapter = netdev_priv(netdev); 5195 5196 if (!igc_has_link(adapter)) 5197 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5198 5199 return -EBUSY; 5200 } 5201 #endif /* CONFIG_PM */ 5202 5203 static void igc_shutdown(struct pci_dev *pdev) 5204 { 5205 bool wake; 5206 5207 __igc_shutdown(pdev, &wake, 0); 5208 5209 if (system_state == SYSTEM_POWER_OFF) { 5210 pci_wake_from_d3(pdev, wake); 5211 pci_set_power_state(pdev, PCI_D3hot); 5212 } 5213 } 5214 5215 /** 5216 * igc_io_error_detected - called when PCI error is detected 5217 * @pdev: Pointer to PCI device 5218 * @state: The current PCI connection state 5219 * 5220 * This function is called after a PCI bus error affecting 5221 * this device has been detected. 5222 **/ 5223 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5224 pci_channel_state_t state) 5225 { 5226 struct net_device *netdev = pci_get_drvdata(pdev); 5227 struct igc_adapter *adapter = netdev_priv(netdev); 5228 5229 netif_device_detach(netdev); 5230 5231 if (state == pci_channel_io_perm_failure) 5232 return PCI_ERS_RESULT_DISCONNECT; 5233 5234 if (netif_running(netdev)) 5235 igc_down(adapter); 5236 pci_disable_device(pdev); 5237 5238 /* Request a slot reset. */ 5239 return PCI_ERS_RESULT_NEED_RESET; 5240 } 5241 5242 /** 5243 * igc_io_slot_reset - called after the PCI bus has been reset. 5244 * @pdev: Pointer to PCI device 5245 * 5246 * Restart the card from scratch, as if from a cold-boot. Implementation 5247 * resembles the first-half of the igc_resume routine. 5248 **/ 5249 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5250 { 5251 struct net_device *netdev = pci_get_drvdata(pdev); 5252 struct igc_adapter *adapter = netdev_priv(netdev); 5253 struct igc_hw *hw = &adapter->hw; 5254 pci_ers_result_t result; 5255 5256 if (pci_enable_device_mem(pdev)) { 5257 dev_err(&pdev->dev, 5258 "Could not re-enable PCI device after reset.\n"); 5259 result = PCI_ERS_RESULT_DISCONNECT; 5260 } else { 5261 pci_set_master(pdev); 5262 pci_restore_state(pdev); 5263 pci_save_state(pdev); 5264 5265 pci_enable_wake(pdev, PCI_D3hot, 0); 5266 pci_enable_wake(pdev, PCI_D3cold, 0); 5267 5268 /* In case of PCI error, adapter loses its HW address 5269 * so we should re-assign it here. 5270 */ 5271 hw->hw_addr = adapter->io_addr; 5272 5273 igc_reset(adapter); 5274 wr32(IGC_WUS, ~0); 5275 result = PCI_ERS_RESULT_RECOVERED; 5276 } 5277 5278 return result; 5279 } 5280 5281 /** 5282 * igc_io_resume - called when traffic can start to flow again. 5283 * @pdev: Pointer to PCI device 5284 * 5285 * This callback is called when the error recovery driver tells us that 5286 * its OK to resume normal operation. Implementation resembles the 5287 * second-half of the igc_resume routine. 5288 */ 5289 static void igc_io_resume(struct pci_dev *pdev) 5290 { 5291 struct net_device *netdev = pci_get_drvdata(pdev); 5292 struct igc_adapter *adapter = netdev_priv(netdev); 5293 5294 rtnl_lock(); 5295 if (netif_running(netdev)) { 5296 if (igc_open(netdev)) { 5297 dev_err(&pdev->dev, "igc_open failed after reset\n"); 5298 return; 5299 } 5300 } 5301 5302 netif_device_attach(netdev); 5303 5304 /* let the f/w know that the h/w is now under the control of the 5305 * driver. 5306 */ 5307 igc_get_hw_control(adapter); 5308 rtnl_unlock(); 5309 } 5310 5311 static const struct pci_error_handlers igc_err_handler = { 5312 .error_detected = igc_io_error_detected, 5313 .slot_reset = igc_io_slot_reset, 5314 .resume = igc_io_resume, 5315 }; 5316 5317 #ifdef CONFIG_PM 5318 static const struct dev_pm_ops igc_pm_ops = { 5319 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5320 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5321 igc_runtime_idle) 5322 }; 5323 #endif 5324 5325 static struct pci_driver igc_driver = { 5326 .name = igc_driver_name, 5327 .id_table = igc_pci_tbl, 5328 .probe = igc_probe, 5329 .remove = igc_remove, 5330 #ifdef CONFIG_PM 5331 .driver.pm = &igc_pm_ops, 5332 #endif 5333 .shutdown = igc_shutdown, 5334 .err_handler = &igc_err_handler, 5335 }; 5336 5337 /** 5338 * igc_reinit_queues - return error 5339 * @adapter: pointer to adapter structure 5340 */ 5341 int igc_reinit_queues(struct igc_adapter *adapter) 5342 { 5343 struct net_device *netdev = adapter->netdev; 5344 struct pci_dev *pdev = adapter->pdev; 5345 int err = 0; 5346 5347 if (netif_running(netdev)) 5348 igc_close(netdev); 5349 5350 igc_reset_interrupt_capability(adapter); 5351 5352 if (igc_init_interrupt_scheme(adapter, true)) { 5353 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 5354 return -ENOMEM; 5355 } 5356 5357 if (netif_running(netdev)) 5358 err = igc_open(netdev); 5359 5360 return err; 5361 } 5362 5363 /** 5364 * igc_get_hw_dev - return device 5365 * @hw: pointer to hardware structure 5366 * 5367 * used by hardware layer to print debugging information 5368 */ 5369 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5370 { 5371 struct igc_adapter *adapter = hw->back; 5372 5373 return adapter->netdev; 5374 } 5375 5376 /** 5377 * igc_init_module - Driver Registration Routine 5378 * 5379 * igc_init_module is the first routine called when the driver is 5380 * loaded. All it does is register with the PCI subsystem. 5381 */ 5382 static int __init igc_init_module(void) 5383 { 5384 int ret; 5385 5386 pr_info("%s - version %s\n", 5387 igc_driver_string, igc_driver_version); 5388 5389 pr_info("%s\n", igc_copyright); 5390 5391 ret = pci_register_driver(&igc_driver); 5392 return ret; 5393 } 5394 5395 module_init(igc_init_module); 5396 5397 /** 5398 * igc_exit_module - Driver Exit Cleanup Routine 5399 * 5400 * igc_exit_module is called just before the driver is removed 5401 * from memory. 5402 */ 5403 static void __exit igc_exit_module(void) 5404 { 5405 pci_unregister_driver(&igc_driver); 5406 } 5407 5408 module_exit(igc_exit_module); 5409 /* igc_main.c */ 5410