1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 12 #include <net/ipv6.h> 13 14 #include "igc.h" 15 #include "igc_hw.h" 16 17 #define DRV_VERSION "0.0.1-k" 18 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 19 20 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 21 22 static int debug = -1; 23 24 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 25 MODULE_DESCRIPTION(DRV_SUMMARY); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_VERSION(DRV_VERSION); 28 module_param(debug, int, 0); 29 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 30 31 char igc_driver_name[] = "igc"; 32 char igc_driver_version[] = DRV_VERSION; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 /* required last entry */ 48 {0, } 49 }; 50 51 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 52 53 /* forward declaration */ 54 static void igc_clean_tx_ring(struct igc_ring *tx_ring); 55 static int igc_sw_init(struct igc_adapter *); 56 static void igc_configure(struct igc_adapter *adapter); 57 static void igc_power_down_link(struct igc_adapter *adapter); 58 static void igc_set_default_mac_filter(struct igc_adapter *adapter); 59 static void igc_set_rx_mode(struct net_device *netdev); 60 static void igc_write_itr(struct igc_q_vector *q_vector); 61 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); 62 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); 63 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 64 bool msix); 65 static void igc_free_q_vectors(struct igc_adapter *adapter); 66 static void igc_irq_disable(struct igc_adapter *adapter); 67 static void igc_irq_enable(struct igc_adapter *adapter); 68 static void igc_configure_msix(struct igc_adapter *adapter); 69 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 70 struct igc_rx_buffer *bi); 71 72 enum latency_range { 73 lowest_latency = 0, 74 low_latency = 1, 75 bulk_latency = 2, 76 latency_invalid = 255 77 }; 78 79 void igc_reset(struct igc_adapter *adapter) 80 { 81 struct pci_dev *pdev = adapter->pdev; 82 struct igc_hw *hw = &adapter->hw; 83 struct igc_fc_info *fc = &hw->fc; 84 u32 pba, hwm; 85 86 /* Repartition PBA for greater than 9k MTU if required */ 87 pba = IGC_PBA_34K; 88 89 /* flow control settings 90 * The high water mark must be low enough to fit one full frame 91 * after transmitting the pause frame. As such we must have enough 92 * space to allow for us to complete our current transmit and then 93 * receive the frame that is in progress from the link partner. 94 * Set it to: 95 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 96 */ 97 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 98 99 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 100 fc->low_water = fc->high_water - 16; 101 fc->pause_time = 0xFFFF; 102 fc->send_xon = 1; 103 fc->current_mode = fc->requested_mode; 104 105 hw->mac.ops.reset_hw(hw); 106 107 if (hw->mac.ops.init_hw(hw)) 108 dev_err(&pdev->dev, "Hardware Error\n"); 109 110 if (!netif_running(adapter->netdev)) 111 igc_power_down_link(adapter); 112 113 igc_get_phy_info(hw); 114 } 115 116 /** 117 * igc_power_up_link - Power up the phy/serdes link 118 * @adapter: address of board private structure 119 */ 120 static void igc_power_up_link(struct igc_adapter *adapter) 121 { 122 igc_reset_phy(&adapter->hw); 123 124 if (adapter->hw.phy.media_type == igc_media_type_copper) 125 igc_power_up_phy_copper(&adapter->hw); 126 127 igc_setup_link(&adapter->hw); 128 } 129 130 /** 131 * igc_power_down_link - Power down the phy/serdes link 132 * @adapter: address of board private structure 133 */ 134 static void igc_power_down_link(struct igc_adapter *adapter) 135 { 136 if (adapter->hw.phy.media_type == igc_media_type_copper) 137 igc_power_down_phy_copper_base(&adapter->hw); 138 } 139 140 /** 141 * igc_release_hw_control - release control of the h/w to f/w 142 * @adapter: address of board private structure 143 * 144 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 145 * For ASF and Pass Through versions of f/w this means that the 146 * driver is no longer loaded. 147 */ 148 static void igc_release_hw_control(struct igc_adapter *adapter) 149 { 150 struct igc_hw *hw = &adapter->hw; 151 u32 ctrl_ext; 152 153 /* Let firmware take over control of h/w */ 154 ctrl_ext = rd32(IGC_CTRL_EXT); 155 wr32(IGC_CTRL_EXT, 156 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 157 } 158 159 /** 160 * igc_get_hw_control - get control of the h/w from f/w 161 * @adapter: address of board private structure 162 * 163 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 164 * For ASF and Pass Through versions of f/w this means that 165 * the driver is loaded. 166 */ 167 static void igc_get_hw_control(struct igc_adapter *adapter) 168 { 169 struct igc_hw *hw = &adapter->hw; 170 u32 ctrl_ext; 171 172 /* Let firmware know the driver has taken over */ 173 ctrl_ext = rd32(IGC_CTRL_EXT); 174 wr32(IGC_CTRL_EXT, 175 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 176 } 177 178 /** 179 * igc_free_tx_resources - Free Tx Resources per Queue 180 * @tx_ring: Tx descriptor ring for a specific queue 181 * 182 * Free all transmit software resources 183 */ 184 void igc_free_tx_resources(struct igc_ring *tx_ring) 185 { 186 igc_clean_tx_ring(tx_ring); 187 188 vfree(tx_ring->tx_buffer_info); 189 tx_ring->tx_buffer_info = NULL; 190 191 /* if not set, then don't free */ 192 if (!tx_ring->desc) 193 return; 194 195 dma_free_coherent(tx_ring->dev, tx_ring->size, 196 tx_ring->desc, tx_ring->dma); 197 198 tx_ring->desc = NULL; 199 } 200 201 /** 202 * igc_free_all_tx_resources - Free Tx Resources for All Queues 203 * @adapter: board private structure 204 * 205 * Free all transmit software resources 206 */ 207 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 208 { 209 int i; 210 211 for (i = 0; i < adapter->num_tx_queues; i++) 212 igc_free_tx_resources(adapter->tx_ring[i]); 213 } 214 215 /** 216 * igc_clean_tx_ring - Free Tx Buffers 217 * @tx_ring: ring to be cleaned 218 */ 219 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 220 { 221 u16 i = tx_ring->next_to_clean; 222 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 223 224 while (i != tx_ring->next_to_use) { 225 union igc_adv_tx_desc *eop_desc, *tx_desc; 226 227 /* Free all the Tx ring sk_buffs */ 228 dev_kfree_skb_any(tx_buffer->skb); 229 230 /* unmap skb header data */ 231 dma_unmap_single(tx_ring->dev, 232 dma_unmap_addr(tx_buffer, dma), 233 dma_unmap_len(tx_buffer, len), 234 DMA_TO_DEVICE); 235 236 /* check for eop_desc to determine the end of the packet */ 237 eop_desc = tx_buffer->next_to_watch; 238 tx_desc = IGC_TX_DESC(tx_ring, i); 239 240 /* unmap remaining buffers */ 241 while (tx_desc != eop_desc) { 242 tx_buffer++; 243 tx_desc++; 244 i++; 245 if (unlikely(i == tx_ring->count)) { 246 i = 0; 247 tx_buffer = tx_ring->tx_buffer_info; 248 tx_desc = IGC_TX_DESC(tx_ring, 0); 249 } 250 251 /* unmap any remaining paged data */ 252 if (dma_unmap_len(tx_buffer, len)) 253 dma_unmap_page(tx_ring->dev, 254 dma_unmap_addr(tx_buffer, dma), 255 dma_unmap_len(tx_buffer, len), 256 DMA_TO_DEVICE); 257 } 258 259 /* move us one more past the eop_desc for start of next pkt */ 260 tx_buffer++; 261 i++; 262 if (unlikely(i == tx_ring->count)) { 263 i = 0; 264 tx_buffer = tx_ring->tx_buffer_info; 265 } 266 } 267 268 /* reset BQL for queue */ 269 netdev_tx_reset_queue(txring_txq(tx_ring)); 270 271 /* reset next_to_use and next_to_clean */ 272 tx_ring->next_to_use = 0; 273 tx_ring->next_to_clean = 0; 274 } 275 276 /** 277 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 278 * @adapter: board private structure 279 */ 280 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 281 { 282 int i; 283 284 for (i = 0; i < adapter->num_tx_queues; i++) 285 if (adapter->tx_ring[i]) 286 igc_clean_tx_ring(adapter->tx_ring[i]); 287 } 288 289 /** 290 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 291 * @tx_ring: tx descriptor ring (for a specific queue) to setup 292 * 293 * Return 0 on success, negative on failure 294 */ 295 int igc_setup_tx_resources(struct igc_ring *tx_ring) 296 { 297 struct device *dev = tx_ring->dev; 298 int size = 0; 299 300 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 301 tx_ring->tx_buffer_info = vzalloc(size); 302 if (!tx_ring->tx_buffer_info) 303 goto err; 304 305 /* round up to nearest 4K */ 306 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 307 tx_ring->size = ALIGN(tx_ring->size, 4096); 308 309 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 310 &tx_ring->dma, GFP_KERNEL); 311 312 if (!tx_ring->desc) 313 goto err; 314 315 tx_ring->next_to_use = 0; 316 tx_ring->next_to_clean = 0; 317 318 return 0; 319 320 err: 321 vfree(tx_ring->tx_buffer_info); 322 dev_err(dev, 323 "Unable to allocate memory for the transmit descriptor ring\n"); 324 return -ENOMEM; 325 } 326 327 /** 328 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 329 * @adapter: board private structure 330 * 331 * Return 0 on success, negative on failure 332 */ 333 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 334 { 335 struct pci_dev *pdev = adapter->pdev; 336 int i, err = 0; 337 338 for (i = 0; i < adapter->num_tx_queues; i++) { 339 err = igc_setup_tx_resources(adapter->tx_ring[i]); 340 if (err) { 341 dev_err(&pdev->dev, 342 "Allocation for Tx Queue %u failed\n", i); 343 for (i--; i >= 0; i--) 344 igc_free_tx_resources(adapter->tx_ring[i]); 345 break; 346 } 347 } 348 349 return err; 350 } 351 352 /** 353 * igc_clean_rx_ring - Free Rx Buffers per Queue 354 * @rx_ring: ring to free buffers from 355 */ 356 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 357 { 358 u16 i = rx_ring->next_to_clean; 359 360 dev_kfree_skb(rx_ring->skb); 361 rx_ring->skb = NULL; 362 363 /* Free all the Rx ring sk_buffs */ 364 while (i != rx_ring->next_to_alloc) { 365 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 366 367 /* Invalidate cache lines that may have been written to by 368 * device so that we avoid corrupting memory. 369 */ 370 dma_sync_single_range_for_cpu(rx_ring->dev, 371 buffer_info->dma, 372 buffer_info->page_offset, 373 igc_rx_bufsz(rx_ring), 374 DMA_FROM_DEVICE); 375 376 /* free resources associated with mapping */ 377 dma_unmap_page_attrs(rx_ring->dev, 378 buffer_info->dma, 379 igc_rx_pg_size(rx_ring), 380 DMA_FROM_DEVICE, 381 IGC_RX_DMA_ATTR); 382 __page_frag_cache_drain(buffer_info->page, 383 buffer_info->pagecnt_bias); 384 385 i++; 386 if (i == rx_ring->count) 387 i = 0; 388 } 389 390 rx_ring->next_to_alloc = 0; 391 rx_ring->next_to_clean = 0; 392 rx_ring->next_to_use = 0; 393 } 394 395 /** 396 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 397 * @adapter: board private structure 398 */ 399 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 400 { 401 int i; 402 403 for (i = 0; i < adapter->num_rx_queues; i++) 404 if (adapter->rx_ring[i]) 405 igc_clean_rx_ring(adapter->rx_ring[i]); 406 } 407 408 /** 409 * igc_free_rx_resources - Free Rx Resources 410 * @rx_ring: ring to clean the resources from 411 * 412 * Free all receive software resources 413 */ 414 void igc_free_rx_resources(struct igc_ring *rx_ring) 415 { 416 igc_clean_rx_ring(rx_ring); 417 418 vfree(rx_ring->rx_buffer_info); 419 rx_ring->rx_buffer_info = NULL; 420 421 /* if not set, then don't free */ 422 if (!rx_ring->desc) 423 return; 424 425 dma_free_coherent(rx_ring->dev, rx_ring->size, 426 rx_ring->desc, rx_ring->dma); 427 428 rx_ring->desc = NULL; 429 } 430 431 /** 432 * igc_free_all_rx_resources - Free Rx Resources for All Queues 433 * @adapter: board private structure 434 * 435 * Free all receive software resources 436 */ 437 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 438 { 439 int i; 440 441 for (i = 0; i < adapter->num_rx_queues; i++) 442 igc_free_rx_resources(adapter->rx_ring[i]); 443 } 444 445 /** 446 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 447 * @rx_ring: rx descriptor ring (for a specific queue) to setup 448 * 449 * Returns 0 on success, negative on failure 450 */ 451 int igc_setup_rx_resources(struct igc_ring *rx_ring) 452 { 453 struct device *dev = rx_ring->dev; 454 int size, desc_len; 455 456 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 457 rx_ring->rx_buffer_info = vzalloc(size); 458 if (!rx_ring->rx_buffer_info) 459 goto err; 460 461 desc_len = sizeof(union igc_adv_rx_desc); 462 463 /* Round up to nearest 4K */ 464 rx_ring->size = rx_ring->count * desc_len; 465 rx_ring->size = ALIGN(rx_ring->size, 4096); 466 467 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 468 &rx_ring->dma, GFP_KERNEL); 469 470 if (!rx_ring->desc) 471 goto err; 472 473 rx_ring->next_to_alloc = 0; 474 rx_ring->next_to_clean = 0; 475 rx_ring->next_to_use = 0; 476 477 return 0; 478 479 err: 480 vfree(rx_ring->rx_buffer_info); 481 rx_ring->rx_buffer_info = NULL; 482 dev_err(dev, 483 "Unable to allocate memory for the receive descriptor ring\n"); 484 return -ENOMEM; 485 } 486 487 /** 488 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 489 * (Descriptors) for all queues 490 * @adapter: board private structure 491 * 492 * Return 0 on success, negative on failure 493 */ 494 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 495 { 496 struct pci_dev *pdev = adapter->pdev; 497 int i, err = 0; 498 499 for (i = 0; i < adapter->num_rx_queues; i++) { 500 err = igc_setup_rx_resources(adapter->rx_ring[i]); 501 if (err) { 502 dev_err(&pdev->dev, 503 "Allocation for Rx Queue %u failed\n", i); 504 for (i--; i >= 0; i--) 505 igc_free_rx_resources(adapter->rx_ring[i]); 506 break; 507 } 508 } 509 510 return err; 511 } 512 513 /** 514 * igc_configure_rx_ring - Configure a receive ring after Reset 515 * @adapter: board private structure 516 * @ring: receive ring to be configured 517 * 518 * Configure the Rx unit of the MAC after a reset. 519 */ 520 static void igc_configure_rx_ring(struct igc_adapter *adapter, 521 struct igc_ring *ring) 522 { 523 struct igc_hw *hw = &adapter->hw; 524 union igc_adv_rx_desc *rx_desc; 525 int reg_idx = ring->reg_idx; 526 u32 srrctl = 0, rxdctl = 0; 527 u64 rdba = ring->dma; 528 529 /* disable the queue */ 530 wr32(IGC_RXDCTL(reg_idx), 0); 531 532 /* Set DMA base address registers */ 533 wr32(IGC_RDBAL(reg_idx), 534 rdba & 0x00000000ffffffffULL); 535 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 536 wr32(IGC_RDLEN(reg_idx), 537 ring->count * sizeof(union igc_adv_rx_desc)); 538 539 /* initialize head and tail */ 540 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 541 wr32(IGC_RDH(reg_idx), 0); 542 writel(0, ring->tail); 543 544 /* reset next-to- use/clean to place SW in sync with hardware */ 545 ring->next_to_clean = 0; 546 ring->next_to_use = 0; 547 548 /* set descriptor configuration */ 549 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 550 if (ring_uses_large_buffer(ring)) 551 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 552 else 553 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 554 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 555 556 wr32(IGC_SRRCTL(reg_idx), srrctl); 557 558 rxdctl |= IGC_RX_PTHRESH; 559 rxdctl |= IGC_RX_HTHRESH << 8; 560 rxdctl |= IGC_RX_WTHRESH << 16; 561 562 /* initialize rx_buffer_info */ 563 memset(ring->rx_buffer_info, 0, 564 sizeof(struct igc_rx_buffer) * ring->count); 565 566 /* initialize Rx descriptor 0 */ 567 rx_desc = IGC_RX_DESC(ring, 0); 568 rx_desc->wb.upper.length = 0; 569 570 /* enable receive descriptor fetching */ 571 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 572 573 wr32(IGC_RXDCTL(reg_idx), rxdctl); 574 } 575 576 /** 577 * igc_configure_rx - Configure receive Unit after Reset 578 * @adapter: board private structure 579 * 580 * Configure the Rx unit of the MAC after a reset. 581 */ 582 static void igc_configure_rx(struct igc_adapter *adapter) 583 { 584 int i; 585 586 /* Setup the HW Rx Head and Tail Descriptor Pointers and 587 * the Base and Length of the Rx Descriptor Ring 588 */ 589 for (i = 0; i < adapter->num_rx_queues; i++) 590 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 591 } 592 593 /** 594 * igc_configure_tx_ring - Configure transmit ring after Reset 595 * @adapter: board private structure 596 * @ring: tx ring to configure 597 * 598 * Configure a transmit ring after a reset. 599 */ 600 static void igc_configure_tx_ring(struct igc_adapter *adapter, 601 struct igc_ring *ring) 602 { 603 struct igc_hw *hw = &adapter->hw; 604 int reg_idx = ring->reg_idx; 605 u64 tdba = ring->dma; 606 u32 txdctl = 0; 607 608 /* disable the queue */ 609 wr32(IGC_TXDCTL(reg_idx), 0); 610 wrfl(); 611 mdelay(10); 612 613 wr32(IGC_TDLEN(reg_idx), 614 ring->count * sizeof(union igc_adv_tx_desc)); 615 wr32(IGC_TDBAL(reg_idx), 616 tdba & 0x00000000ffffffffULL); 617 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 618 619 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 620 wr32(IGC_TDH(reg_idx), 0); 621 writel(0, ring->tail); 622 623 txdctl |= IGC_TX_PTHRESH; 624 txdctl |= IGC_TX_HTHRESH << 8; 625 txdctl |= IGC_TX_WTHRESH << 16; 626 627 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 628 wr32(IGC_TXDCTL(reg_idx), txdctl); 629 } 630 631 /** 632 * igc_configure_tx - Configure transmit Unit after Reset 633 * @adapter: board private structure 634 * 635 * Configure the Tx unit of the MAC after a reset. 636 */ 637 static void igc_configure_tx(struct igc_adapter *adapter) 638 { 639 int i; 640 641 for (i = 0; i < adapter->num_tx_queues; i++) 642 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 643 } 644 645 /** 646 * igc_setup_mrqc - configure the multiple receive queue control registers 647 * @adapter: Board private structure 648 */ 649 static void igc_setup_mrqc(struct igc_adapter *adapter) 650 { 651 struct igc_hw *hw = &adapter->hw; 652 u32 j, num_rx_queues; 653 u32 mrqc, rxcsum; 654 u32 rss_key[10]; 655 656 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 657 for (j = 0; j < 10; j++) 658 wr32(IGC_RSSRK(j), rss_key[j]); 659 660 num_rx_queues = adapter->rss_queues; 661 662 if (adapter->rss_indir_tbl_init != num_rx_queues) { 663 for (j = 0; j < IGC_RETA_SIZE; j++) 664 adapter->rss_indir_tbl[j] = 665 (j * num_rx_queues) / IGC_RETA_SIZE; 666 adapter->rss_indir_tbl_init = num_rx_queues; 667 } 668 igc_write_rss_indir_tbl(adapter); 669 670 /* Disable raw packet checksumming so that RSS hash is placed in 671 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 672 * offloads as they are enabled by default 673 */ 674 rxcsum = rd32(IGC_RXCSUM); 675 rxcsum |= IGC_RXCSUM_PCSD; 676 677 /* Enable Receive Checksum Offload for SCTP */ 678 rxcsum |= IGC_RXCSUM_CRCOFL; 679 680 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 681 wr32(IGC_RXCSUM, rxcsum); 682 683 /* Generate RSS hash based on packet types, TCP/UDP 684 * port numbers and/or IPv4/v6 src and dst addresses 685 */ 686 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 687 IGC_MRQC_RSS_FIELD_IPV4_TCP | 688 IGC_MRQC_RSS_FIELD_IPV6 | 689 IGC_MRQC_RSS_FIELD_IPV6_TCP | 690 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 691 692 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 693 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 694 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 695 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 696 697 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 698 699 wr32(IGC_MRQC, mrqc); 700 } 701 702 /** 703 * igc_setup_rctl - configure the receive control registers 704 * @adapter: Board private structure 705 */ 706 static void igc_setup_rctl(struct igc_adapter *adapter) 707 { 708 struct igc_hw *hw = &adapter->hw; 709 u32 rctl; 710 711 rctl = rd32(IGC_RCTL); 712 713 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 714 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 715 716 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 717 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 718 719 /* enable stripping of CRC. Newer features require 720 * that the HW strips the CRC. 721 */ 722 rctl |= IGC_RCTL_SECRC; 723 724 /* disable store bad packets and clear size bits. */ 725 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 726 727 /* enable LPE to allow for reception of jumbo frames */ 728 rctl |= IGC_RCTL_LPE; 729 730 /* disable queue 0 to prevent tail write w/o re-config */ 731 wr32(IGC_RXDCTL(0), 0); 732 733 /* This is useful for sniffing bad packets. */ 734 if (adapter->netdev->features & NETIF_F_RXALL) { 735 /* UPE and MPE will be handled by normal PROMISC logic 736 * in set_rx_mode 737 */ 738 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 739 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 740 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 741 742 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 743 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 744 } 745 746 wr32(IGC_RCTL, rctl); 747 } 748 749 /** 750 * igc_setup_tctl - configure the transmit control registers 751 * @adapter: Board private structure 752 */ 753 static void igc_setup_tctl(struct igc_adapter *adapter) 754 { 755 struct igc_hw *hw = &adapter->hw; 756 u32 tctl; 757 758 /* disable queue 0 which icould be enabled by default */ 759 wr32(IGC_TXDCTL(0), 0); 760 761 /* Program the Transmit Control Register */ 762 tctl = rd32(IGC_TCTL); 763 tctl &= ~IGC_TCTL_CT; 764 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 765 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 766 767 /* Enable transmits */ 768 tctl |= IGC_TCTL_EN; 769 770 wr32(IGC_TCTL, tctl); 771 } 772 773 /** 774 * igc_set_mac - Change the Ethernet Address of the NIC 775 * @netdev: network interface device structure 776 * @p: pointer to an address structure 777 * 778 * Returns 0 on success, negative on failure 779 */ 780 static int igc_set_mac(struct net_device *netdev, void *p) 781 { 782 struct igc_adapter *adapter = netdev_priv(netdev); 783 struct igc_hw *hw = &adapter->hw; 784 struct sockaddr *addr = p; 785 786 if (!is_valid_ether_addr(addr->sa_data)) 787 return -EADDRNOTAVAIL; 788 789 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 790 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 791 792 /* set the correct pool for the new PF MAC address in entry 0 */ 793 igc_set_default_mac_filter(adapter); 794 795 return 0; 796 } 797 798 /** 799 * igc_write_mc_addr_list - write multicast addresses to MTA 800 * @netdev: network interface device structure 801 * 802 * Writes multicast address list to the MTA hash table. 803 * Returns: -ENOMEM on failure 804 * 0 on no addresses written 805 * X on writing X addresses to MTA 806 **/ 807 static int igc_write_mc_addr_list(struct net_device *netdev) 808 { 809 struct igc_adapter *adapter = netdev_priv(netdev); 810 struct igc_hw *hw = &adapter->hw; 811 struct netdev_hw_addr *ha; 812 u8 *mta_list; 813 int i; 814 815 if (netdev_mc_empty(netdev)) { 816 /* nothing to program, so clear mc list */ 817 igc_update_mc_addr_list(hw, NULL, 0); 818 return 0; 819 } 820 821 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 822 if (!mta_list) 823 return -ENOMEM; 824 825 /* The shared function expects a packed array of only addresses. */ 826 i = 0; 827 netdev_for_each_mc_addr(ha, netdev) 828 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 829 830 igc_update_mc_addr_list(hw, mta_list, i); 831 kfree(mta_list); 832 833 return netdev_mc_count(netdev); 834 } 835 836 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 837 struct igc_tx_buffer *first, 838 u32 vlan_macip_lens, u32 type_tucmd, 839 u32 mss_l4len_idx) 840 { 841 struct igc_adv_tx_context_desc *context_desc; 842 u16 i = tx_ring->next_to_use; 843 struct timespec64 ts; 844 845 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 846 847 i++; 848 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 849 850 /* set bits to identify this as an advanced context descriptor */ 851 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 852 853 /* For 82575, context index must be unique per ring. */ 854 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 855 mss_l4len_idx |= tx_ring->reg_idx << 4; 856 857 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 858 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 859 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 860 861 /* We assume there is always a valid Tx time available. Invalid times 862 * should have been handled by the upper layers. 863 */ 864 if (tx_ring->launchtime_enable) { 865 ts = ns_to_timespec64(first->skb->tstamp); 866 first->skb->tstamp = 0; 867 context_desc->launch_time = cpu_to_le32(ts.tv_nsec / 32); 868 } else { 869 context_desc->launch_time = 0; 870 } 871 } 872 873 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 874 { 875 unsigned int offset = 0; 876 877 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 878 879 return offset == skb_checksum_start_offset(skb); 880 } 881 882 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 883 { 884 struct sk_buff *skb = first->skb; 885 u32 vlan_macip_lens = 0; 886 u32 type_tucmd = 0; 887 888 if (skb->ip_summed != CHECKSUM_PARTIAL) { 889 csum_failed: 890 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 891 !tx_ring->launchtime_enable) 892 return; 893 goto no_csum; 894 } 895 896 switch (skb->csum_offset) { 897 case offsetof(struct tcphdr, check): 898 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 899 /* fall through */ 900 case offsetof(struct udphdr, check): 901 break; 902 case offsetof(struct sctphdr, checksum): 903 /* validate that this is actually an SCTP request */ 904 if ((first->protocol == htons(ETH_P_IP) && 905 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 906 (first->protocol == htons(ETH_P_IPV6) && 907 igc_ipv6_csum_is_sctp(skb))) { 908 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 909 break; 910 } 911 /* fall through */ 912 default: 913 skb_checksum_help(skb); 914 goto csum_failed; 915 } 916 917 /* update TX checksum flag */ 918 first->tx_flags |= IGC_TX_FLAGS_CSUM; 919 vlan_macip_lens = skb_checksum_start_offset(skb) - 920 skb_network_offset(skb); 921 no_csum: 922 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 923 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 924 925 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 926 } 927 928 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 929 { 930 struct net_device *netdev = tx_ring->netdev; 931 932 netif_stop_subqueue(netdev, tx_ring->queue_index); 933 934 /* memory barriier comment */ 935 smp_mb(); 936 937 /* We need to check again in a case another CPU has just 938 * made room available. 939 */ 940 if (igc_desc_unused(tx_ring) < size) 941 return -EBUSY; 942 943 /* A reprieve! */ 944 netif_wake_subqueue(netdev, tx_ring->queue_index); 945 946 u64_stats_update_begin(&tx_ring->tx_syncp2); 947 tx_ring->tx_stats.restart_queue2++; 948 u64_stats_update_end(&tx_ring->tx_syncp2); 949 950 return 0; 951 } 952 953 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 954 { 955 if (igc_desc_unused(tx_ring) >= size) 956 return 0; 957 return __igc_maybe_stop_tx(tx_ring, size); 958 } 959 960 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 961 { 962 /* set type for advanced descriptor with frame checksum insertion */ 963 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 964 IGC_ADVTXD_DCMD_DEXT | 965 IGC_ADVTXD_DCMD_IFCS; 966 967 return cmd_type; 968 } 969 970 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 971 union igc_adv_tx_desc *tx_desc, 972 u32 tx_flags, unsigned int paylen) 973 { 974 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 975 976 /* insert L4 checksum */ 977 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 978 ((IGC_TXD_POPTS_TXSM << 8) / 979 IGC_TX_FLAGS_CSUM); 980 981 /* insert IPv4 checksum */ 982 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 983 (((IGC_TXD_POPTS_IXSM << 8)) / 984 IGC_TX_FLAGS_IPV4); 985 986 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 987 } 988 989 static int igc_tx_map(struct igc_ring *tx_ring, 990 struct igc_tx_buffer *first, 991 const u8 hdr_len) 992 { 993 struct sk_buff *skb = first->skb; 994 struct igc_tx_buffer *tx_buffer; 995 union igc_adv_tx_desc *tx_desc; 996 u32 tx_flags = first->tx_flags; 997 skb_frag_t *frag; 998 u16 i = tx_ring->next_to_use; 999 unsigned int data_len, size; 1000 dma_addr_t dma; 1001 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1002 1003 tx_desc = IGC_TX_DESC(tx_ring, i); 1004 1005 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1006 1007 size = skb_headlen(skb); 1008 data_len = skb->data_len; 1009 1010 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1011 1012 tx_buffer = first; 1013 1014 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1015 if (dma_mapping_error(tx_ring->dev, dma)) 1016 goto dma_error; 1017 1018 /* record length, and DMA address */ 1019 dma_unmap_len_set(tx_buffer, len, size); 1020 dma_unmap_addr_set(tx_buffer, dma, dma); 1021 1022 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1023 1024 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1025 tx_desc->read.cmd_type_len = 1026 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1027 1028 i++; 1029 tx_desc++; 1030 if (i == tx_ring->count) { 1031 tx_desc = IGC_TX_DESC(tx_ring, 0); 1032 i = 0; 1033 } 1034 tx_desc->read.olinfo_status = 0; 1035 1036 dma += IGC_MAX_DATA_PER_TXD; 1037 size -= IGC_MAX_DATA_PER_TXD; 1038 1039 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1040 } 1041 1042 if (likely(!data_len)) 1043 break; 1044 1045 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1046 1047 i++; 1048 tx_desc++; 1049 if (i == tx_ring->count) { 1050 tx_desc = IGC_TX_DESC(tx_ring, 0); 1051 i = 0; 1052 } 1053 tx_desc->read.olinfo_status = 0; 1054 1055 size = skb_frag_size(frag); 1056 data_len -= size; 1057 1058 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1059 size, DMA_TO_DEVICE); 1060 1061 tx_buffer = &tx_ring->tx_buffer_info[i]; 1062 } 1063 1064 /* write last descriptor with RS and EOP bits */ 1065 cmd_type |= size | IGC_TXD_DCMD; 1066 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1067 1068 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1069 1070 /* set the timestamp */ 1071 first->time_stamp = jiffies; 1072 1073 skb_tx_timestamp(skb); 1074 1075 /* Force memory writes to complete before letting h/w know there 1076 * are new descriptors to fetch. (Only applicable for weak-ordered 1077 * memory model archs, such as IA-64). 1078 * 1079 * We also need this memory barrier to make certain all of the 1080 * status bits have been updated before next_to_watch is written. 1081 */ 1082 wmb(); 1083 1084 /* set next_to_watch value indicating a packet is present */ 1085 first->next_to_watch = tx_desc; 1086 1087 i++; 1088 if (i == tx_ring->count) 1089 i = 0; 1090 1091 tx_ring->next_to_use = i; 1092 1093 /* Make sure there is space in the ring for the next send. */ 1094 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1095 1096 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1097 writel(i, tx_ring->tail); 1098 } 1099 1100 return 0; 1101 dma_error: 1102 dev_err(tx_ring->dev, "TX DMA map failed\n"); 1103 tx_buffer = &tx_ring->tx_buffer_info[i]; 1104 1105 /* clear dma mappings for failed tx_buffer_info map */ 1106 while (tx_buffer != first) { 1107 if (dma_unmap_len(tx_buffer, len)) 1108 dma_unmap_page(tx_ring->dev, 1109 dma_unmap_addr(tx_buffer, dma), 1110 dma_unmap_len(tx_buffer, len), 1111 DMA_TO_DEVICE); 1112 dma_unmap_len_set(tx_buffer, len, 0); 1113 1114 if (i-- == 0) 1115 i += tx_ring->count; 1116 tx_buffer = &tx_ring->tx_buffer_info[i]; 1117 } 1118 1119 if (dma_unmap_len(tx_buffer, len)) 1120 dma_unmap_single(tx_ring->dev, 1121 dma_unmap_addr(tx_buffer, dma), 1122 dma_unmap_len(tx_buffer, len), 1123 DMA_TO_DEVICE); 1124 dma_unmap_len_set(tx_buffer, len, 0); 1125 1126 dev_kfree_skb_any(tx_buffer->skb); 1127 tx_buffer->skb = NULL; 1128 1129 tx_ring->next_to_use = i; 1130 1131 return -1; 1132 } 1133 1134 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1135 struct igc_ring *tx_ring) 1136 { 1137 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1138 __be16 protocol = vlan_get_protocol(skb); 1139 struct igc_tx_buffer *first; 1140 u32 tx_flags = 0; 1141 unsigned short f; 1142 u8 hdr_len = 0; 1143 1144 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1145 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1146 * + 2 desc gap to keep tail from touching head, 1147 * + 1 desc for context descriptor, 1148 * otherwise try next time 1149 */ 1150 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1151 count += TXD_USE_COUNT(skb_frag_size( 1152 &skb_shinfo(skb)->frags[f])); 1153 1154 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1155 /* this is a hard error */ 1156 return NETDEV_TX_BUSY; 1157 } 1158 1159 /* record the location of the first descriptor for this packet */ 1160 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1161 first->skb = skb; 1162 first->bytecount = skb->len; 1163 first->gso_segs = 1; 1164 1165 /* record initial flags and protocol */ 1166 first->tx_flags = tx_flags; 1167 first->protocol = protocol; 1168 1169 igc_tx_csum(tx_ring, first); 1170 1171 igc_tx_map(tx_ring, first, hdr_len); 1172 1173 return NETDEV_TX_OK; 1174 } 1175 1176 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1177 struct sk_buff *skb) 1178 { 1179 unsigned int r_idx = skb->queue_mapping; 1180 1181 if (r_idx >= adapter->num_tx_queues) 1182 r_idx = r_idx % adapter->num_tx_queues; 1183 1184 return adapter->tx_ring[r_idx]; 1185 } 1186 1187 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1188 struct net_device *netdev) 1189 { 1190 struct igc_adapter *adapter = netdev_priv(netdev); 1191 1192 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1193 * in order to meet this minimum size requirement. 1194 */ 1195 if (skb->len < 17) { 1196 if (skb_padto(skb, 17)) 1197 return NETDEV_TX_OK; 1198 skb->len = 17; 1199 } 1200 1201 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1202 } 1203 1204 static void igc_rx_checksum(struct igc_ring *ring, 1205 union igc_adv_rx_desc *rx_desc, 1206 struct sk_buff *skb) 1207 { 1208 skb_checksum_none_assert(skb); 1209 1210 /* Ignore Checksum bit is set */ 1211 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1212 return; 1213 1214 /* Rx checksum disabled via ethtool */ 1215 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1216 return; 1217 1218 /* TCP/UDP checksum error bit is set */ 1219 if (igc_test_staterr(rx_desc, 1220 IGC_RXDEXT_STATERR_TCPE | 1221 IGC_RXDEXT_STATERR_IPE)) { 1222 /* work around errata with sctp packets where the TCPE aka 1223 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1224 * packets (aka let the stack check the crc32c) 1225 */ 1226 if (!(skb->len == 60 && 1227 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1228 u64_stats_update_begin(&ring->rx_syncp); 1229 ring->rx_stats.csum_err++; 1230 u64_stats_update_end(&ring->rx_syncp); 1231 } 1232 /* let the stack verify checksum errors */ 1233 return; 1234 } 1235 /* It must be a TCP or UDP packet with a valid checksum */ 1236 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1237 IGC_RXD_STAT_UDPCS)) 1238 skb->ip_summed = CHECKSUM_UNNECESSARY; 1239 1240 dev_dbg(ring->dev, "cksum success: bits %08X\n", 1241 le32_to_cpu(rx_desc->wb.upper.status_error)); 1242 } 1243 1244 static inline void igc_rx_hash(struct igc_ring *ring, 1245 union igc_adv_rx_desc *rx_desc, 1246 struct sk_buff *skb) 1247 { 1248 if (ring->netdev->features & NETIF_F_RXHASH) 1249 skb_set_hash(skb, 1250 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1251 PKT_HASH_TYPE_L3); 1252 } 1253 1254 /** 1255 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1256 * @rx_ring: rx descriptor ring packet is being transacted on 1257 * @rx_desc: pointer to the EOP Rx descriptor 1258 * @skb: pointer to current skb being populated 1259 * 1260 * This function checks the ring, descriptor, and packet information in 1261 * order to populate the hash, checksum, VLAN, timestamp, protocol, and 1262 * other fields within the skb. 1263 */ 1264 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1265 union igc_adv_rx_desc *rx_desc, 1266 struct sk_buff *skb) 1267 { 1268 igc_rx_hash(rx_ring, rx_desc, skb); 1269 1270 igc_rx_checksum(rx_ring, rx_desc, skb); 1271 1272 skb_record_rx_queue(skb, rx_ring->queue_index); 1273 1274 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1275 } 1276 1277 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1278 const unsigned int size) 1279 { 1280 struct igc_rx_buffer *rx_buffer; 1281 1282 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1283 prefetchw(rx_buffer->page); 1284 1285 /* we are reusing so sync this buffer for CPU use */ 1286 dma_sync_single_range_for_cpu(rx_ring->dev, 1287 rx_buffer->dma, 1288 rx_buffer->page_offset, 1289 size, 1290 DMA_FROM_DEVICE); 1291 1292 rx_buffer->pagecnt_bias--; 1293 1294 return rx_buffer; 1295 } 1296 1297 /** 1298 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1299 * @rx_ring: rx descriptor ring to transact packets on 1300 * @rx_buffer: buffer containing page to add 1301 * @skb: sk_buff to place the data into 1302 * @size: size of buffer to be added 1303 * 1304 * This function will add the data contained in rx_buffer->page to the skb. 1305 */ 1306 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1307 struct igc_rx_buffer *rx_buffer, 1308 struct sk_buff *skb, 1309 unsigned int size) 1310 { 1311 #if (PAGE_SIZE < 8192) 1312 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1313 1314 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1315 rx_buffer->page_offset, size, truesize); 1316 rx_buffer->page_offset ^= truesize; 1317 #else 1318 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1319 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1320 SKB_DATA_ALIGN(size); 1321 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1322 rx_buffer->page_offset, size, truesize); 1323 rx_buffer->page_offset += truesize; 1324 #endif 1325 } 1326 1327 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1328 struct igc_rx_buffer *rx_buffer, 1329 union igc_adv_rx_desc *rx_desc, 1330 unsigned int size) 1331 { 1332 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1333 #if (PAGE_SIZE < 8192) 1334 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1335 #else 1336 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1337 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1338 #endif 1339 struct sk_buff *skb; 1340 1341 /* prefetch first cache line of first page */ 1342 prefetch(va); 1343 #if L1_CACHE_BYTES < 128 1344 prefetch(va + L1_CACHE_BYTES); 1345 #endif 1346 1347 /* build an skb around the page buffer */ 1348 skb = build_skb(va - IGC_SKB_PAD, truesize); 1349 if (unlikely(!skb)) 1350 return NULL; 1351 1352 /* update pointers within the skb to store the data */ 1353 skb_reserve(skb, IGC_SKB_PAD); 1354 __skb_put(skb, size); 1355 1356 /* update buffer offset */ 1357 #if (PAGE_SIZE < 8192) 1358 rx_buffer->page_offset ^= truesize; 1359 #else 1360 rx_buffer->page_offset += truesize; 1361 #endif 1362 1363 return skb; 1364 } 1365 1366 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1367 struct igc_rx_buffer *rx_buffer, 1368 union igc_adv_rx_desc *rx_desc, 1369 unsigned int size) 1370 { 1371 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1372 #if (PAGE_SIZE < 8192) 1373 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1374 #else 1375 unsigned int truesize = SKB_DATA_ALIGN(size); 1376 #endif 1377 unsigned int headlen; 1378 struct sk_buff *skb; 1379 1380 /* prefetch first cache line of first page */ 1381 prefetch(va); 1382 #if L1_CACHE_BYTES < 128 1383 prefetch(va + L1_CACHE_BYTES); 1384 #endif 1385 1386 /* allocate a skb to store the frags */ 1387 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1388 if (unlikely(!skb)) 1389 return NULL; 1390 1391 /* Determine available headroom for copy */ 1392 headlen = size; 1393 if (headlen > IGC_RX_HDR_LEN) 1394 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1395 1396 /* align pull length to size of long to optimize memcpy performance */ 1397 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1398 1399 /* update all of the pointers */ 1400 size -= headlen; 1401 if (size) { 1402 skb_add_rx_frag(skb, 0, rx_buffer->page, 1403 (va + headlen) - page_address(rx_buffer->page), 1404 size, truesize); 1405 #if (PAGE_SIZE < 8192) 1406 rx_buffer->page_offset ^= truesize; 1407 #else 1408 rx_buffer->page_offset += truesize; 1409 #endif 1410 } else { 1411 rx_buffer->pagecnt_bias++; 1412 } 1413 1414 return skb; 1415 } 1416 1417 /** 1418 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1419 * @rx_ring: rx descriptor ring to store buffers on 1420 * @old_buff: donor buffer to have page reused 1421 * 1422 * Synchronizes page for reuse by the adapter 1423 */ 1424 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1425 struct igc_rx_buffer *old_buff) 1426 { 1427 u16 nta = rx_ring->next_to_alloc; 1428 struct igc_rx_buffer *new_buff; 1429 1430 new_buff = &rx_ring->rx_buffer_info[nta]; 1431 1432 /* update, and store next to alloc */ 1433 nta++; 1434 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1435 1436 /* Transfer page from old buffer to new buffer. 1437 * Move each member individually to avoid possible store 1438 * forwarding stalls. 1439 */ 1440 new_buff->dma = old_buff->dma; 1441 new_buff->page = old_buff->page; 1442 new_buff->page_offset = old_buff->page_offset; 1443 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1444 } 1445 1446 static inline bool igc_page_is_reserved(struct page *page) 1447 { 1448 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1449 } 1450 1451 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1452 { 1453 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1454 struct page *page = rx_buffer->page; 1455 1456 /* avoid re-using remote pages */ 1457 if (unlikely(igc_page_is_reserved(page))) 1458 return false; 1459 1460 #if (PAGE_SIZE < 8192) 1461 /* if we are only owner of page we can reuse it */ 1462 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1463 return false; 1464 #else 1465 #define IGC_LAST_OFFSET \ 1466 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1467 1468 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1469 return false; 1470 #endif 1471 1472 /* If we have drained the page fragment pool we need to update 1473 * the pagecnt_bias and page count so that we fully restock the 1474 * number of references the driver holds. 1475 */ 1476 if (unlikely(!pagecnt_bias)) { 1477 page_ref_add(page, USHRT_MAX); 1478 rx_buffer->pagecnt_bias = USHRT_MAX; 1479 } 1480 1481 return true; 1482 } 1483 1484 /** 1485 * igc_is_non_eop - process handling of non-EOP buffers 1486 * @rx_ring: Rx ring being processed 1487 * @rx_desc: Rx descriptor for current buffer 1488 * @skb: current socket buffer containing buffer in progress 1489 * 1490 * This function updates next to clean. If the buffer is an EOP buffer 1491 * this function exits returning false, otherwise it will place the 1492 * sk_buff in the next buffer to be chained and return true indicating 1493 * that this is in fact a non-EOP buffer. 1494 */ 1495 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1496 union igc_adv_rx_desc *rx_desc) 1497 { 1498 u32 ntc = rx_ring->next_to_clean + 1; 1499 1500 /* fetch, update, and store next to clean */ 1501 ntc = (ntc < rx_ring->count) ? ntc : 0; 1502 rx_ring->next_to_clean = ntc; 1503 1504 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1505 1506 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1507 return false; 1508 1509 return true; 1510 } 1511 1512 /** 1513 * igc_cleanup_headers - Correct corrupted or empty headers 1514 * @rx_ring: rx descriptor ring packet is being transacted on 1515 * @rx_desc: pointer to the EOP Rx descriptor 1516 * @skb: pointer to current skb being fixed 1517 * 1518 * Address the case where we are pulling data in on pages only 1519 * and as such no data is present in the skb header. 1520 * 1521 * In addition if skb is not at least 60 bytes we need to pad it so that 1522 * it is large enough to qualify as a valid Ethernet frame. 1523 * 1524 * Returns true if an error was encountered and skb was freed. 1525 */ 1526 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1527 union igc_adv_rx_desc *rx_desc, 1528 struct sk_buff *skb) 1529 { 1530 if (unlikely((igc_test_staterr(rx_desc, 1531 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1532 struct net_device *netdev = rx_ring->netdev; 1533 1534 if (!(netdev->features & NETIF_F_RXALL)) { 1535 dev_kfree_skb_any(skb); 1536 return true; 1537 } 1538 } 1539 1540 /* if eth_skb_pad returns an error the skb was freed */ 1541 if (eth_skb_pad(skb)) 1542 return true; 1543 1544 return false; 1545 } 1546 1547 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1548 struct igc_rx_buffer *rx_buffer) 1549 { 1550 if (igc_can_reuse_rx_page(rx_buffer)) { 1551 /* hand second half of page back to the ring */ 1552 igc_reuse_rx_page(rx_ring, rx_buffer); 1553 } else { 1554 /* We are not reusing the buffer so unmap it and free 1555 * any references we are holding to it 1556 */ 1557 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1558 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1559 IGC_RX_DMA_ATTR); 1560 __page_frag_cache_drain(rx_buffer->page, 1561 rx_buffer->pagecnt_bias); 1562 } 1563 1564 /* clear contents of rx_buffer */ 1565 rx_buffer->page = NULL; 1566 } 1567 1568 /** 1569 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1570 * @adapter: address of board private structure 1571 */ 1572 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1573 { 1574 union igc_adv_rx_desc *rx_desc; 1575 u16 i = rx_ring->next_to_use; 1576 struct igc_rx_buffer *bi; 1577 u16 bufsz; 1578 1579 /* nothing to do */ 1580 if (!cleaned_count) 1581 return; 1582 1583 rx_desc = IGC_RX_DESC(rx_ring, i); 1584 bi = &rx_ring->rx_buffer_info[i]; 1585 i -= rx_ring->count; 1586 1587 bufsz = igc_rx_bufsz(rx_ring); 1588 1589 do { 1590 if (!igc_alloc_mapped_page(rx_ring, bi)) 1591 break; 1592 1593 /* sync the buffer for use by the device */ 1594 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1595 bi->page_offset, bufsz, 1596 DMA_FROM_DEVICE); 1597 1598 /* Refresh the desc even if buffer_addrs didn't change 1599 * because each write-back erases this info. 1600 */ 1601 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1602 1603 rx_desc++; 1604 bi++; 1605 i++; 1606 if (unlikely(!i)) { 1607 rx_desc = IGC_RX_DESC(rx_ring, 0); 1608 bi = rx_ring->rx_buffer_info; 1609 i -= rx_ring->count; 1610 } 1611 1612 /* clear the length for the next_to_use descriptor */ 1613 rx_desc->wb.upper.length = 0; 1614 1615 cleaned_count--; 1616 } while (cleaned_count); 1617 1618 i += rx_ring->count; 1619 1620 if (rx_ring->next_to_use != i) { 1621 /* record the next descriptor to use */ 1622 rx_ring->next_to_use = i; 1623 1624 /* update next to alloc since we have filled the ring */ 1625 rx_ring->next_to_alloc = i; 1626 1627 /* Force memory writes to complete before letting h/w 1628 * know there are new descriptors to fetch. (Only 1629 * applicable for weak-ordered memory model archs, 1630 * such as IA-64). 1631 */ 1632 wmb(); 1633 writel(i, rx_ring->tail); 1634 } 1635 } 1636 1637 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1638 { 1639 unsigned int total_bytes = 0, total_packets = 0; 1640 struct igc_ring *rx_ring = q_vector->rx.ring; 1641 struct sk_buff *skb = rx_ring->skb; 1642 u16 cleaned_count = igc_desc_unused(rx_ring); 1643 1644 while (likely(total_packets < budget)) { 1645 union igc_adv_rx_desc *rx_desc; 1646 struct igc_rx_buffer *rx_buffer; 1647 unsigned int size; 1648 1649 /* return some buffers to hardware, one at a time is too slow */ 1650 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1651 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1652 cleaned_count = 0; 1653 } 1654 1655 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1656 size = le16_to_cpu(rx_desc->wb.upper.length); 1657 if (!size) 1658 break; 1659 1660 /* This memory barrier is needed to keep us from reading 1661 * any other fields out of the rx_desc until we know the 1662 * descriptor has been written back 1663 */ 1664 dma_rmb(); 1665 1666 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1667 1668 /* retrieve a buffer from the ring */ 1669 if (skb) 1670 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1671 else if (ring_uses_build_skb(rx_ring)) 1672 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1673 else 1674 skb = igc_construct_skb(rx_ring, rx_buffer, 1675 rx_desc, size); 1676 1677 /* exit if we failed to retrieve a buffer */ 1678 if (!skb) { 1679 rx_ring->rx_stats.alloc_failed++; 1680 rx_buffer->pagecnt_bias++; 1681 break; 1682 } 1683 1684 igc_put_rx_buffer(rx_ring, rx_buffer); 1685 cleaned_count++; 1686 1687 /* fetch next buffer in frame if non-eop */ 1688 if (igc_is_non_eop(rx_ring, rx_desc)) 1689 continue; 1690 1691 /* verify the packet layout is correct */ 1692 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1693 skb = NULL; 1694 continue; 1695 } 1696 1697 /* probably a little skewed due to removing CRC */ 1698 total_bytes += skb->len; 1699 1700 /* populate checksum, timestamp, VLAN, and protocol */ 1701 igc_process_skb_fields(rx_ring, rx_desc, skb); 1702 1703 napi_gro_receive(&q_vector->napi, skb); 1704 1705 /* reset skb pointer */ 1706 skb = NULL; 1707 1708 /* update budget accounting */ 1709 total_packets++; 1710 } 1711 1712 /* place incomplete frames back on ring for completion */ 1713 rx_ring->skb = skb; 1714 1715 u64_stats_update_begin(&rx_ring->rx_syncp); 1716 rx_ring->rx_stats.packets += total_packets; 1717 rx_ring->rx_stats.bytes += total_bytes; 1718 u64_stats_update_end(&rx_ring->rx_syncp); 1719 q_vector->rx.total_packets += total_packets; 1720 q_vector->rx.total_bytes += total_bytes; 1721 1722 if (cleaned_count) 1723 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1724 1725 return total_packets; 1726 } 1727 1728 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1729 { 1730 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1731 } 1732 1733 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1734 struct igc_rx_buffer *bi) 1735 { 1736 struct page *page = bi->page; 1737 dma_addr_t dma; 1738 1739 /* since we are recycling buffers we should seldom need to alloc */ 1740 if (likely(page)) 1741 return true; 1742 1743 /* alloc new page for storage */ 1744 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1745 if (unlikely(!page)) { 1746 rx_ring->rx_stats.alloc_failed++; 1747 return false; 1748 } 1749 1750 /* map page for use */ 1751 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1752 igc_rx_pg_size(rx_ring), 1753 DMA_FROM_DEVICE, 1754 IGC_RX_DMA_ATTR); 1755 1756 /* if mapping failed free memory back to system since 1757 * there isn't much point in holding memory we can't use 1758 */ 1759 if (dma_mapping_error(rx_ring->dev, dma)) { 1760 __free_page(page); 1761 1762 rx_ring->rx_stats.alloc_failed++; 1763 return false; 1764 } 1765 1766 bi->dma = dma; 1767 bi->page = page; 1768 bi->page_offset = igc_rx_offset(rx_ring); 1769 bi->pagecnt_bias = 1; 1770 1771 return true; 1772 } 1773 1774 /** 1775 * igc_clean_tx_irq - Reclaim resources after transmit completes 1776 * @q_vector: pointer to q_vector containing needed info 1777 * @napi_budget: Used to determine if we are in netpoll 1778 * 1779 * returns true if ring is completely cleaned 1780 */ 1781 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1782 { 1783 struct igc_adapter *adapter = q_vector->adapter; 1784 unsigned int total_bytes = 0, total_packets = 0; 1785 unsigned int budget = q_vector->tx.work_limit; 1786 struct igc_ring *tx_ring = q_vector->tx.ring; 1787 unsigned int i = tx_ring->next_to_clean; 1788 struct igc_tx_buffer *tx_buffer; 1789 union igc_adv_tx_desc *tx_desc; 1790 1791 if (test_bit(__IGC_DOWN, &adapter->state)) 1792 return true; 1793 1794 tx_buffer = &tx_ring->tx_buffer_info[i]; 1795 tx_desc = IGC_TX_DESC(tx_ring, i); 1796 i -= tx_ring->count; 1797 1798 do { 1799 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 1800 1801 /* if next_to_watch is not set then there is no work pending */ 1802 if (!eop_desc) 1803 break; 1804 1805 /* prevent any other reads prior to eop_desc */ 1806 smp_rmb(); 1807 1808 /* if DD is not set pending work has not been completed */ 1809 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 1810 break; 1811 1812 /* clear next_to_watch to prevent false hangs */ 1813 tx_buffer->next_to_watch = NULL; 1814 1815 /* update the statistics for this packet */ 1816 total_bytes += tx_buffer->bytecount; 1817 total_packets += tx_buffer->gso_segs; 1818 1819 /* free the skb */ 1820 napi_consume_skb(tx_buffer->skb, napi_budget); 1821 1822 /* unmap skb header data */ 1823 dma_unmap_single(tx_ring->dev, 1824 dma_unmap_addr(tx_buffer, dma), 1825 dma_unmap_len(tx_buffer, len), 1826 DMA_TO_DEVICE); 1827 1828 /* clear tx_buffer data */ 1829 dma_unmap_len_set(tx_buffer, len, 0); 1830 1831 /* clear last DMA location and unmap remaining buffers */ 1832 while (tx_desc != eop_desc) { 1833 tx_buffer++; 1834 tx_desc++; 1835 i++; 1836 if (unlikely(!i)) { 1837 i -= tx_ring->count; 1838 tx_buffer = tx_ring->tx_buffer_info; 1839 tx_desc = IGC_TX_DESC(tx_ring, 0); 1840 } 1841 1842 /* unmap any remaining paged data */ 1843 if (dma_unmap_len(tx_buffer, len)) { 1844 dma_unmap_page(tx_ring->dev, 1845 dma_unmap_addr(tx_buffer, dma), 1846 dma_unmap_len(tx_buffer, len), 1847 DMA_TO_DEVICE); 1848 dma_unmap_len_set(tx_buffer, len, 0); 1849 } 1850 } 1851 1852 /* move us one more past the eop_desc for start of next pkt */ 1853 tx_buffer++; 1854 tx_desc++; 1855 i++; 1856 if (unlikely(!i)) { 1857 i -= tx_ring->count; 1858 tx_buffer = tx_ring->tx_buffer_info; 1859 tx_desc = IGC_TX_DESC(tx_ring, 0); 1860 } 1861 1862 /* issue prefetch for next Tx descriptor */ 1863 prefetch(tx_desc); 1864 1865 /* update budget accounting */ 1866 budget--; 1867 } while (likely(budget)); 1868 1869 netdev_tx_completed_queue(txring_txq(tx_ring), 1870 total_packets, total_bytes); 1871 1872 i += tx_ring->count; 1873 tx_ring->next_to_clean = i; 1874 u64_stats_update_begin(&tx_ring->tx_syncp); 1875 tx_ring->tx_stats.bytes += total_bytes; 1876 tx_ring->tx_stats.packets += total_packets; 1877 u64_stats_update_end(&tx_ring->tx_syncp); 1878 q_vector->tx.total_bytes += total_bytes; 1879 q_vector->tx.total_packets += total_packets; 1880 1881 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 1882 struct igc_hw *hw = &adapter->hw; 1883 1884 /* Detect a transmit hang in hardware, this serializes the 1885 * check with the clearing of time_stamp and movement of i 1886 */ 1887 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 1888 if (tx_buffer->next_to_watch && 1889 time_after(jiffies, tx_buffer->time_stamp + 1890 (adapter->tx_timeout_factor * HZ)) && 1891 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 1892 /* detected Tx unit hang */ 1893 dev_err(tx_ring->dev, 1894 "Detected Tx Unit Hang\n" 1895 " Tx Queue <%d>\n" 1896 " TDH <%x>\n" 1897 " TDT <%x>\n" 1898 " next_to_use <%x>\n" 1899 " next_to_clean <%x>\n" 1900 "buffer_info[next_to_clean]\n" 1901 " time_stamp <%lx>\n" 1902 " next_to_watch <%p>\n" 1903 " jiffies <%lx>\n" 1904 " desc.status <%x>\n", 1905 tx_ring->queue_index, 1906 rd32(IGC_TDH(tx_ring->reg_idx)), 1907 readl(tx_ring->tail), 1908 tx_ring->next_to_use, 1909 tx_ring->next_to_clean, 1910 tx_buffer->time_stamp, 1911 tx_buffer->next_to_watch, 1912 jiffies, 1913 tx_buffer->next_to_watch->wb.status); 1914 netif_stop_subqueue(tx_ring->netdev, 1915 tx_ring->queue_index); 1916 1917 /* we are about to reset, no point in enabling stuff */ 1918 return true; 1919 } 1920 } 1921 1922 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 1923 if (unlikely(total_packets && 1924 netif_carrier_ok(tx_ring->netdev) && 1925 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 1926 /* Make sure that anybody stopping the queue after this 1927 * sees the new next_to_clean. 1928 */ 1929 smp_mb(); 1930 if (__netif_subqueue_stopped(tx_ring->netdev, 1931 tx_ring->queue_index) && 1932 !(test_bit(__IGC_DOWN, &adapter->state))) { 1933 netif_wake_subqueue(tx_ring->netdev, 1934 tx_ring->queue_index); 1935 1936 u64_stats_update_begin(&tx_ring->tx_syncp); 1937 tx_ring->tx_stats.restart_queue++; 1938 u64_stats_update_end(&tx_ring->tx_syncp); 1939 } 1940 } 1941 1942 return !!budget; 1943 } 1944 1945 /** 1946 * igc_up - Open the interface and prepare it to handle traffic 1947 * @adapter: board private structure 1948 */ 1949 void igc_up(struct igc_adapter *adapter) 1950 { 1951 struct igc_hw *hw = &adapter->hw; 1952 int i = 0; 1953 1954 /* hardware has been reset, we need to reload some things */ 1955 igc_configure(adapter); 1956 1957 clear_bit(__IGC_DOWN, &adapter->state); 1958 1959 for (i = 0; i < adapter->num_q_vectors; i++) 1960 napi_enable(&adapter->q_vector[i]->napi); 1961 1962 if (adapter->msix_entries) 1963 igc_configure_msix(adapter); 1964 else 1965 igc_assign_vector(adapter->q_vector[0], 0); 1966 1967 /* Clear any pending interrupts. */ 1968 rd32(IGC_ICR); 1969 igc_irq_enable(adapter); 1970 1971 netif_tx_start_all_queues(adapter->netdev); 1972 1973 /* start the watchdog. */ 1974 hw->mac.get_link_status = 1; 1975 schedule_work(&adapter->watchdog_task); 1976 } 1977 1978 /** 1979 * igc_update_stats - Update the board statistics counters 1980 * @adapter: board private structure 1981 */ 1982 void igc_update_stats(struct igc_adapter *adapter) 1983 { 1984 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 1985 struct pci_dev *pdev = adapter->pdev; 1986 struct igc_hw *hw = &adapter->hw; 1987 u64 _bytes, _packets; 1988 u64 bytes, packets; 1989 unsigned int start; 1990 u32 mpc; 1991 int i; 1992 1993 /* Prevent stats update while adapter is being reset, or if the pci 1994 * connection is down. 1995 */ 1996 if (adapter->link_speed == 0) 1997 return; 1998 if (pci_channel_offline(pdev)) 1999 return; 2000 2001 packets = 0; 2002 bytes = 0; 2003 2004 rcu_read_lock(); 2005 for (i = 0; i < adapter->num_rx_queues; i++) { 2006 struct igc_ring *ring = adapter->rx_ring[i]; 2007 u32 rqdpc = rd32(IGC_RQDPC(i)); 2008 2009 if (hw->mac.type >= igc_i225) 2010 wr32(IGC_RQDPC(i), 0); 2011 2012 if (rqdpc) { 2013 ring->rx_stats.drops += rqdpc; 2014 net_stats->rx_fifo_errors += rqdpc; 2015 } 2016 2017 do { 2018 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 2019 _bytes = ring->rx_stats.bytes; 2020 _packets = ring->rx_stats.packets; 2021 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 2022 bytes += _bytes; 2023 packets += _packets; 2024 } 2025 2026 net_stats->rx_bytes = bytes; 2027 net_stats->rx_packets = packets; 2028 2029 packets = 0; 2030 bytes = 0; 2031 for (i = 0; i < adapter->num_tx_queues; i++) { 2032 struct igc_ring *ring = adapter->tx_ring[i]; 2033 2034 do { 2035 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 2036 _bytes = ring->tx_stats.bytes; 2037 _packets = ring->tx_stats.packets; 2038 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 2039 bytes += _bytes; 2040 packets += _packets; 2041 } 2042 net_stats->tx_bytes = bytes; 2043 net_stats->tx_packets = packets; 2044 rcu_read_unlock(); 2045 2046 /* read stats registers */ 2047 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 2048 adapter->stats.gprc += rd32(IGC_GPRC); 2049 adapter->stats.gorc += rd32(IGC_GORCL); 2050 rd32(IGC_GORCH); /* clear GORCL */ 2051 adapter->stats.bprc += rd32(IGC_BPRC); 2052 adapter->stats.mprc += rd32(IGC_MPRC); 2053 adapter->stats.roc += rd32(IGC_ROC); 2054 2055 adapter->stats.prc64 += rd32(IGC_PRC64); 2056 adapter->stats.prc127 += rd32(IGC_PRC127); 2057 adapter->stats.prc255 += rd32(IGC_PRC255); 2058 adapter->stats.prc511 += rd32(IGC_PRC511); 2059 adapter->stats.prc1023 += rd32(IGC_PRC1023); 2060 adapter->stats.prc1522 += rd32(IGC_PRC1522); 2061 adapter->stats.symerrs += rd32(IGC_SYMERRS); 2062 adapter->stats.sec += rd32(IGC_SEC); 2063 2064 mpc = rd32(IGC_MPC); 2065 adapter->stats.mpc += mpc; 2066 net_stats->rx_fifo_errors += mpc; 2067 adapter->stats.scc += rd32(IGC_SCC); 2068 adapter->stats.ecol += rd32(IGC_ECOL); 2069 adapter->stats.mcc += rd32(IGC_MCC); 2070 adapter->stats.latecol += rd32(IGC_LATECOL); 2071 adapter->stats.dc += rd32(IGC_DC); 2072 adapter->stats.rlec += rd32(IGC_RLEC); 2073 adapter->stats.xonrxc += rd32(IGC_XONRXC); 2074 adapter->stats.xontxc += rd32(IGC_XONTXC); 2075 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 2076 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 2077 adapter->stats.fcruc += rd32(IGC_FCRUC); 2078 adapter->stats.gptc += rd32(IGC_GPTC); 2079 adapter->stats.gotc += rd32(IGC_GOTCL); 2080 rd32(IGC_GOTCH); /* clear GOTCL */ 2081 adapter->stats.rnbc += rd32(IGC_RNBC); 2082 adapter->stats.ruc += rd32(IGC_RUC); 2083 adapter->stats.rfc += rd32(IGC_RFC); 2084 adapter->stats.rjc += rd32(IGC_RJC); 2085 adapter->stats.tor += rd32(IGC_TORH); 2086 adapter->stats.tot += rd32(IGC_TOTH); 2087 adapter->stats.tpr += rd32(IGC_TPR); 2088 2089 adapter->stats.ptc64 += rd32(IGC_PTC64); 2090 adapter->stats.ptc127 += rd32(IGC_PTC127); 2091 adapter->stats.ptc255 += rd32(IGC_PTC255); 2092 adapter->stats.ptc511 += rd32(IGC_PTC511); 2093 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 2094 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 2095 2096 adapter->stats.mptc += rd32(IGC_MPTC); 2097 adapter->stats.bptc += rd32(IGC_BPTC); 2098 2099 adapter->stats.tpt += rd32(IGC_TPT); 2100 adapter->stats.colc += rd32(IGC_COLC); 2101 2102 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 2103 2104 adapter->stats.tsctc += rd32(IGC_TSCTC); 2105 adapter->stats.tsctfc += rd32(IGC_TSCTFC); 2106 2107 adapter->stats.iac += rd32(IGC_IAC); 2108 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 2109 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 2110 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 2111 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 2112 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 2113 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 2114 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 2115 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 2116 2117 /* Fill out the OS statistics structure */ 2118 net_stats->multicast = adapter->stats.mprc; 2119 net_stats->collisions = adapter->stats.colc; 2120 2121 /* Rx Errors */ 2122 2123 /* RLEC on some newer hardware can be incorrect so build 2124 * our own version based on RUC and ROC 2125 */ 2126 net_stats->rx_errors = adapter->stats.rxerrc + 2127 adapter->stats.crcerrs + adapter->stats.algnerrc + 2128 adapter->stats.ruc + adapter->stats.roc + 2129 adapter->stats.cexterr; 2130 net_stats->rx_length_errors = adapter->stats.ruc + 2131 adapter->stats.roc; 2132 net_stats->rx_crc_errors = adapter->stats.crcerrs; 2133 net_stats->rx_frame_errors = adapter->stats.algnerrc; 2134 net_stats->rx_missed_errors = adapter->stats.mpc; 2135 2136 /* Tx Errors */ 2137 net_stats->tx_errors = adapter->stats.ecol + 2138 adapter->stats.latecol; 2139 net_stats->tx_aborted_errors = adapter->stats.ecol; 2140 net_stats->tx_window_errors = adapter->stats.latecol; 2141 net_stats->tx_carrier_errors = adapter->stats.tncrs; 2142 2143 /* Tx Dropped needs to be maintained elsewhere */ 2144 2145 /* Management Stats */ 2146 adapter->stats.mgptc += rd32(IGC_MGTPTC); 2147 adapter->stats.mgprc += rd32(IGC_MGTPRC); 2148 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 2149 } 2150 2151 static void igc_nfc_filter_exit(struct igc_adapter *adapter) 2152 { 2153 struct igc_nfc_filter *rule; 2154 2155 spin_lock(&adapter->nfc_lock); 2156 2157 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 2158 igc_erase_filter(adapter, rule); 2159 2160 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) 2161 igc_erase_filter(adapter, rule); 2162 2163 spin_unlock(&adapter->nfc_lock); 2164 } 2165 2166 static void igc_nfc_filter_restore(struct igc_adapter *adapter) 2167 { 2168 struct igc_nfc_filter *rule; 2169 2170 spin_lock(&adapter->nfc_lock); 2171 2172 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) 2173 igc_add_filter(adapter, rule); 2174 2175 spin_unlock(&adapter->nfc_lock); 2176 } 2177 2178 /** 2179 * igc_down - Close the interface 2180 * @adapter: board private structure 2181 */ 2182 void igc_down(struct igc_adapter *adapter) 2183 { 2184 struct net_device *netdev = adapter->netdev; 2185 struct igc_hw *hw = &adapter->hw; 2186 u32 tctl, rctl; 2187 int i = 0; 2188 2189 set_bit(__IGC_DOWN, &adapter->state); 2190 2191 /* disable receives in the hardware */ 2192 rctl = rd32(IGC_RCTL); 2193 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 2194 /* flush and sleep below */ 2195 2196 igc_nfc_filter_exit(adapter); 2197 2198 /* set trans_start so we don't get spurious watchdogs during reset */ 2199 netif_trans_update(netdev); 2200 2201 netif_carrier_off(netdev); 2202 netif_tx_stop_all_queues(netdev); 2203 2204 /* disable transmits in the hardware */ 2205 tctl = rd32(IGC_TCTL); 2206 tctl &= ~IGC_TCTL_EN; 2207 wr32(IGC_TCTL, tctl); 2208 /* flush both disables and wait for them to finish */ 2209 wrfl(); 2210 usleep_range(10000, 20000); 2211 2212 igc_irq_disable(adapter); 2213 2214 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 2215 2216 for (i = 0; i < adapter->num_q_vectors; i++) { 2217 if (adapter->q_vector[i]) { 2218 napi_synchronize(&adapter->q_vector[i]->napi); 2219 napi_disable(&adapter->q_vector[i]->napi); 2220 } 2221 } 2222 2223 del_timer_sync(&adapter->watchdog_timer); 2224 del_timer_sync(&adapter->phy_info_timer); 2225 2226 /* record the stats before reset*/ 2227 spin_lock(&adapter->stats64_lock); 2228 igc_update_stats(adapter); 2229 spin_unlock(&adapter->stats64_lock); 2230 2231 adapter->link_speed = 0; 2232 adapter->link_duplex = 0; 2233 2234 if (!pci_channel_offline(adapter->pdev)) 2235 igc_reset(adapter); 2236 2237 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 2238 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 2239 2240 igc_clean_all_tx_rings(adapter); 2241 igc_clean_all_rx_rings(adapter); 2242 } 2243 2244 void igc_reinit_locked(struct igc_adapter *adapter) 2245 { 2246 WARN_ON(in_interrupt()); 2247 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 2248 usleep_range(1000, 2000); 2249 igc_down(adapter); 2250 igc_up(adapter); 2251 clear_bit(__IGC_RESETTING, &adapter->state); 2252 } 2253 2254 static void igc_reset_task(struct work_struct *work) 2255 { 2256 struct igc_adapter *adapter; 2257 2258 adapter = container_of(work, struct igc_adapter, reset_task); 2259 2260 netdev_err(adapter->netdev, "Reset adapter\n"); 2261 igc_reinit_locked(adapter); 2262 } 2263 2264 /** 2265 * igc_change_mtu - Change the Maximum Transfer Unit 2266 * @netdev: network interface device structure 2267 * @new_mtu: new value for maximum frame size 2268 * 2269 * Returns 0 on success, negative on failure 2270 */ 2271 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 2272 { 2273 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2274 struct igc_adapter *adapter = netdev_priv(netdev); 2275 struct pci_dev *pdev = adapter->pdev; 2276 2277 /* adjust max frame to be at least the size of a standard frame */ 2278 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 2279 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 2280 2281 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 2282 usleep_range(1000, 2000); 2283 2284 /* igc_down has a dependency on max_frame_size */ 2285 adapter->max_frame_size = max_frame; 2286 2287 if (netif_running(netdev)) 2288 igc_down(adapter); 2289 2290 dev_info(&pdev->dev, "changing MTU from %d to %d\n", 2291 netdev->mtu, new_mtu); 2292 netdev->mtu = new_mtu; 2293 2294 if (netif_running(netdev)) 2295 igc_up(adapter); 2296 else 2297 igc_reset(adapter); 2298 2299 clear_bit(__IGC_RESETTING, &adapter->state); 2300 2301 return 0; 2302 } 2303 2304 /** 2305 * igc_get_stats - Get System Network Statistics 2306 * @netdev: network interface device structure 2307 * 2308 * Returns the address of the device statistics structure. 2309 * The statistics are updated here and also from the timer callback. 2310 */ 2311 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 2312 { 2313 struct igc_adapter *adapter = netdev_priv(netdev); 2314 2315 if (!test_bit(__IGC_RESETTING, &adapter->state)) 2316 igc_update_stats(adapter); 2317 2318 /* only return the current stats */ 2319 return &netdev->stats; 2320 } 2321 2322 static netdev_features_t igc_fix_features(struct net_device *netdev, 2323 netdev_features_t features) 2324 { 2325 /* Since there is no support for separate Rx/Tx vlan accel 2326 * enable/disable make sure Tx flag is always in same state as Rx. 2327 */ 2328 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2329 features |= NETIF_F_HW_VLAN_CTAG_TX; 2330 else 2331 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 2332 2333 return features; 2334 } 2335 2336 static int igc_set_features(struct net_device *netdev, 2337 netdev_features_t features) 2338 { 2339 netdev_features_t changed = netdev->features ^ features; 2340 struct igc_adapter *adapter = netdev_priv(netdev); 2341 2342 /* Add VLAN support */ 2343 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 2344 return 0; 2345 2346 if (!(features & NETIF_F_NTUPLE)) { 2347 struct hlist_node *node2; 2348 struct igc_nfc_filter *rule; 2349 2350 spin_lock(&adapter->nfc_lock); 2351 hlist_for_each_entry_safe(rule, node2, 2352 &adapter->nfc_filter_list, nfc_node) { 2353 igc_erase_filter(adapter, rule); 2354 hlist_del(&rule->nfc_node); 2355 kfree(rule); 2356 } 2357 spin_unlock(&adapter->nfc_lock); 2358 adapter->nfc_filter_count = 0; 2359 } 2360 2361 netdev->features = features; 2362 2363 if (netif_running(netdev)) 2364 igc_reinit_locked(adapter); 2365 else 2366 igc_reset(adapter); 2367 2368 return 1; 2369 } 2370 2371 static netdev_features_t 2372 igc_features_check(struct sk_buff *skb, struct net_device *dev, 2373 netdev_features_t features) 2374 { 2375 unsigned int network_hdr_len, mac_hdr_len; 2376 2377 /* Make certain the headers can be described by a context descriptor */ 2378 mac_hdr_len = skb_network_header(skb) - skb->data; 2379 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 2380 return features & ~(NETIF_F_HW_CSUM | 2381 NETIF_F_SCTP_CRC | 2382 NETIF_F_HW_VLAN_CTAG_TX | 2383 NETIF_F_TSO | 2384 NETIF_F_TSO6); 2385 2386 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 2387 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 2388 return features & ~(NETIF_F_HW_CSUM | 2389 NETIF_F_SCTP_CRC | 2390 NETIF_F_TSO | 2391 NETIF_F_TSO6); 2392 2393 /* We can only support IPv4 TSO in tunnels if we can mangle the 2394 * inner IP ID field, so strip TSO if MANGLEID is not supported. 2395 */ 2396 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 2397 features &= ~NETIF_F_TSO; 2398 2399 return features; 2400 } 2401 2402 /** 2403 * igc_configure - configure the hardware for RX and TX 2404 * @adapter: private board structure 2405 */ 2406 static void igc_configure(struct igc_adapter *adapter) 2407 { 2408 struct net_device *netdev = adapter->netdev; 2409 int i = 0; 2410 2411 igc_get_hw_control(adapter); 2412 igc_set_rx_mode(netdev); 2413 2414 igc_setup_tctl(adapter); 2415 igc_setup_mrqc(adapter); 2416 igc_setup_rctl(adapter); 2417 2418 igc_nfc_filter_restore(adapter); 2419 igc_configure_tx(adapter); 2420 igc_configure_rx(adapter); 2421 2422 igc_rx_fifo_flush_base(&adapter->hw); 2423 2424 /* call igc_desc_unused which always leaves 2425 * at least 1 descriptor unused to make sure 2426 * next_to_use != next_to_clean 2427 */ 2428 for (i = 0; i < adapter->num_rx_queues; i++) { 2429 struct igc_ring *ring = adapter->rx_ring[i]; 2430 2431 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2432 } 2433 } 2434 2435 /** 2436 * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table 2437 * @adapter: address of board private structure 2438 * @index: Index of the RAR entry which need to be synced with MAC table 2439 */ 2440 static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) 2441 { 2442 u8 *addr = adapter->mac_table[index].addr; 2443 struct igc_hw *hw = &adapter->hw; 2444 u32 rar_low, rar_high; 2445 2446 /* HW expects these to be in network order when they are plugged 2447 * into the registers which are little endian. In order to guarantee 2448 * that ordering we need to do an leXX_to_cpup here in order to be 2449 * ready for the byteswap that occurs with writel 2450 */ 2451 rar_low = le32_to_cpup((__le32 *)(addr)); 2452 rar_high = le16_to_cpup((__le16 *)(addr + 4)); 2453 2454 /* Indicate to hardware the Address is Valid. */ 2455 if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { 2456 if (is_valid_ether_addr(addr)) 2457 rar_high |= IGC_RAH_AV; 2458 2459 rar_high |= IGC_RAH_POOL_1 << 2460 adapter->mac_table[index].queue; 2461 } 2462 2463 wr32(IGC_RAL(index), rar_low); 2464 wrfl(); 2465 wr32(IGC_RAH(index), rar_high); 2466 wrfl(); 2467 } 2468 2469 /* Set default MAC address for the PF in the first RAR entry */ 2470 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 2471 { 2472 struct igc_mac_addr *mac_table = &adapter->mac_table[0]; 2473 2474 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); 2475 mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2476 2477 igc_rar_set_index(adapter, 0); 2478 } 2479 2480 /* If the filter to be added and an already existing filter express 2481 * the same address and address type, it should be possible to only 2482 * override the other configurations, for example the queue to steer 2483 * traffic. 2484 */ 2485 static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, 2486 const u8 *addr, const u8 flags) 2487 { 2488 if (!(entry->state & IGC_MAC_STATE_IN_USE)) 2489 return true; 2490 2491 if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != 2492 (flags & IGC_MAC_STATE_SRC_ADDR)) 2493 return false; 2494 2495 if (!ether_addr_equal(addr, entry->addr)) 2496 return false; 2497 2498 return true; 2499 } 2500 2501 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 2502 * 'flags' is used to indicate what kind of match is made, match is by 2503 * default for the destination address, if matching by source address 2504 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. 2505 */ 2506 static int igc_add_mac_filter_flags(struct igc_adapter *adapter, 2507 const u8 *addr, const u8 queue, 2508 const u8 flags) 2509 { 2510 struct igc_hw *hw = &adapter->hw; 2511 int rar_entries = hw->mac.rar_entry_count; 2512 int i; 2513 2514 if (is_zero_ether_addr(addr)) 2515 return -EINVAL; 2516 2517 /* Search for the first empty entry in the MAC table. 2518 * Do not touch entries at the end of the table reserved for the VF MAC 2519 * addresses. 2520 */ 2521 for (i = 0; i < rar_entries; i++) { 2522 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], 2523 addr, flags)) 2524 continue; 2525 2526 ether_addr_copy(adapter->mac_table[i].addr, addr); 2527 adapter->mac_table[i].queue = queue; 2528 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; 2529 2530 igc_rar_set_index(adapter, i); 2531 return i; 2532 } 2533 2534 return -ENOSPC; 2535 } 2536 2537 int igc_add_mac_steering_filter(struct igc_adapter *adapter, 2538 const u8 *addr, u8 queue, u8 flags) 2539 { 2540 return igc_add_mac_filter_flags(adapter, addr, queue, 2541 IGC_MAC_STATE_QUEUE_STEERING | flags); 2542 } 2543 2544 /* Remove a MAC filter for 'addr' directing matching traffic to 2545 * 'queue', 'flags' is used to indicate what kind of match need to be 2546 * removed, match is by default for the destination address, if 2547 * matching by source address is to be removed the flag 2548 * IGC_MAC_STATE_SRC_ADDR can be used. 2549 */ 2550 static int igc_del_mac_filter_flags(struct igc_adapter *adapter, 2551 const u8 *addr, const u8 queue, 2552 const u8 flags) 2553 { 2554 struct igc_hw *hw = &adapter->hw; 2555 int rar_entries = hw->mac.rar_entry_count; 2556 int i; 2557 2558 if (is_zero_ether_addr(addr)) 2559 return -EINVAL; 2560 2561 /* Search for matching entry in the MAC table based on given address 2562 * and queue. Do not touch entries at the end of the table reserved 2563 * for the VF MAC addresses. 2564 */ 2565 for (i = 0; i < rar_entries; i++) { 2566 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) 2567 continue; 2568 if ((adapter->mac_table[i].state & flags) != flags) 2569 continue; 2570 if (adapter->mac_table[i].queue != queue) 2571 continue; 2572 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 2573 continue; 2574 2575 /* When a filter for the default address is "deleted", 2576 * we return it to its initial configuration 2577 */ 2578 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { 2579 adapter->mac_table[i].state = 2580 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2581 } else { 2582 adapter->mac_table[i].state = 0; 2583 adapter->mac_table[i].queue = 0; 2584 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 2585 } 2586 2587 igc_rar_set_index(adapter, i); 2588 return 0; 2589 } 2590 2591 return -ENOENT; 2592 } 2593 2594 int igc_del_mac_steering_filter(struct igc_adapter *adapter, 2595 const u8 *addr, u8 queue, u8 flags) 2596 { 2597 return igc_del_mac_filter_flags(adapter, addr, queue, 2598 IGC_MAC_STATE_QUEUE_STEERING | flags); 2599 } 2600 2601 /* Add a MAC filter for 'addr' directing matching traffic to 'queue', 2602 * 'flags' is used to indicate what kind of match is made, match is by 2603 * default for the destination address, if matching by source address 2604 * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. 2605 */ 2606 static int igc_add_mac_filter(struct igc_adapter *adapter, 2607 const u8 *addr, const u8 queue) 2608 { 2609 struct igc_hw *hw = &adapter->hw; 2610 int rar_entries = hw->mac.rar_entry_count; 2611 int i; 2612 2613 if (is_zero_ether_addr(addr)) 2614 return -EINVAL; 2615 2616 /* Search for the first empty entry in the MAC table. 2617 * Do not touch entries at the end of the table reserved for the VF MAC 2618 * addresses. 2619 */ 2620 for (i = 0; i < rar_entries; i++) { 2621 if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], 2622 addr, 0)) 2623 continue; 2624 2625 ether_addr_copy(adapter->mac_table[i].addr, addr); 2626 adapter->mac_table[i].queue = queue; 2627 adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE; 2628 2629 igc_rar_set_index(adapter, i); 2630 return i; 2631 } 2632 2633 return -ENOSPC; 2634 } 2635 2636 /* Remove a MAC filter for 'addr' directing matching traffic to 2637 * 'queue', 'flags' is used to indicate what kind of match need to be 2638 * removed, match is by default for the destination address, if 2639 * matching by source address is to be removed the flag 2640 * IGC_MAC_STATE_SRC_ADDR can be used. 2641 */ 2642 static int igc_del_mac_filter(struct igc_adapter *adapter, 2643 const u8 *addr, const u8 queue) 2644 { 2645 struct igc_hw *hw = &adapter->hw; 2646 int rar_entries = hw->mac.rar_entry_count; 2647 int i; 2648 2649 if (is_zero_ether_addr(addr)) 2650 return -EINVAL; 2651 2652 /* Search for matching entry in the MAC table based on given address 2653 * and queue. Do not touch entries at the end of the table reserved 2654 * for the VF MAC addresses. 2655 */ 2656 for (i = 0; i < rar_entries; i++) { 2657 if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) 2658 continue; 2659 if (adapter->mac_table[i].state != 0) 2660 continue; 2661 if (adapter->mac_table[i].queue != queue) 2662 continue; 2663 if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) 2664 continue; 2665 2666 /* When a filter for the default address is "deleted", 2667 * we return it to its initial configuration 2668 */ 2669 if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { 2670 adapter->mac_table[i].state = 2671 IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; 2672 adapter->mac_table[i].queue = 0; 2673 } else { 2674 adapter->mac_table[i].state = 0; 2675 adapter->mac_table[i].queue = 0; 2676 memset(adapter->mac_table[i].addr, 0, ETH_ALEN); 2677 } 2678 2679 igc_rar_set_index(adapter, i); 2680 return 0; 2681 } 2682 2683 return -ENOENT; 2684 } 2685 2686 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2687 { 2688 struct igc_adapter *adapter = netdev_priv(netdev); 2689 int ret; 2690 2691 ret = igc_add_mac_filter(adapter, addr, adapter->num_rx_queues); 2692 2693 return min_t(int, ret, 0); 2694 } 2695 2696 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2697 { 2698 struct igc_adapter *adapter = netdev_priv(netdev); 2699 2700 igc_del_mac_filter(adapter, addr, adapter->num_rx_queues); 2701 2702 return 0; 2703 } 2704 2705 /** 2706 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2707 * @netdev: network interface device structure 2708 * 2709 * The set_rx_mode entry point is called whenever the unicast or multicast 2710 * address lists or the network interface flags are updated. This routine is 2711 * responsible for configuring the hardware for proper unicast, multicast, 2712 * promiscuous mode, and all-multi behavior. 2713 */ 2714 static void igc_set_rx_mode(struct net_device *netdev) 2715 { 2716 struct igc_adapter *adapter = netdev_priv(netdev); 2717 struct igc_hw *hw = &adapter->hw; 2718 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2719 int count; 2720 2721 /* Check for Promiscuous and All Multicast modes */ 2722 if (netdev->flags & IFF_PROMISC) { 2723 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2724 } else { 2725 if (netdev->flags & IFF_ALLMULTI) { 2726 rctl |= IGC_RCTL_MPE; 2727 } else { 2728 /* Write addresses to the MTA, if the attempt fails 2729 * then we should just turn on promiscuous mode so 2730 * that we can at least receive multicast traffic 2731 */ 2732 count = igc_write_mc_addr_list(netdev); 2733 if (count < 0) 2734 rctl |= IGC_RCTL_MPE; 2735 } 2736 } 2737 2738 /* Write addresses to available RAR registers, if there is not 2739 * sufficient space to store all the addresses then enable 2740 * unicast promiscuous mode 2741 */ 2742 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2743 rctl |= IGC_RCTL_UPE; 2744 2745 /* update state of unicast and multicast */ 2746 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2747 wr32(IGC_RCTL, rctl); 2748 2749 #if (PAGE_SIZE < 8192) 2750 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2751 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2752 #endif 2753 wr32(IGC_RLPML, rlpml); 2754 } 2755 2756 /** 2757 * igc_msix_other - msix other interrupt handler 2758 * @irq: interrupt number 2759 * @data: pointer to a q_vector 2760 */ 2761 static irqreturn_t igc_msix_other(int irq, void *data) 2762 { 2763 struct igc_adapter *adapter = data; 2764 struct igc_hw *hw = &adapter->hw; 2765 u32 icr = rd32(IGC_ICR); 2766 2767 /* reading ICR causes bit 31 of EICR to be cleared */ 2768 if (icr & IGC_ICR_DRSTA) 2769 schedule_work(&adapter->reset_task); 2770 2771 if (icr & IGC_ICR_DOUTSYNC) { 2772 /* HW is reporting DMA is out of sync */ 2773 adapter->stats.doosync++; 2774 } 2775 2776 if (icr & IGC_ICR_LSC) { 2777 hw->mac.get_link_status = 1; 2778 /* guard against interrupt when we're going down */ 2779 if (!test_bit(__IGC_DOWN, &adapter->state)) 2780 mod_timer(&adapter->watchdog_timer, jiffies + 1); 2781 } 2782 2783 wr32(IGC_EIMS, adapter->eims_other); 2784 2785 return IRQ_HANDLED; 2786 } 2787 2788 /** 2789 * igc_write_ivar - configure ivar for given MSI-X vector 2790 * @hw: pointer to the HW structure 2791 * @msix_vector: vector number we are allocating to a given ring 2792 * @index: row index of IVAR register to write within IVAR table 2793 * @offset: column offset of in IVAR, should be multiple of 8 2794 * 2795 * The IVAR table consists of 2 columns, 2796 * each containing an cause allocation for an Rx and Tx ring, and a 2797 * variable number of rows depending on the number of queues supported. 2798 */ 2799 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2800 int index, int offset) 2801 { 2802 u32 ivar = array_rd32(IGC_IVAR0, index); 2803 2804 /* clear any bits that are currently set */ 2805 ivar &= ~((u32)0xFF << offset); 2806 2807 /* write vector and valid bit */ 2808 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2809 2810 array_wr32(IGC_IVAR0, index, ivar); 2811 } 2812 2813 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2814 { 2815 struct igc_adapter *adapter = q_vector->adapter; 2816 struct igc_hw *hw = &adapter->hw; 2817 int rx_queue = IGC_N0_QUEUE; 2818 int tx_queue = IGC_N0_QUEUE; 2819 2820 if (q_vector->rx.ring) 2821 rx_queue = q_vector->rx.ring->reg_idx; 2822 if (q_vector->tx.ring) 2823 tx_queue = q_vector->tx.ring->reg_idx; 2824 2825 switch (hw->mac.type) { 2826 case igc_i225: 2827 if (rx_queue > IGC_N0_QUEUE) 2828 igc_write_ivar(hw, msix_vector, 2829 rx_queue >> 1, 2830 (rx_queue & 0x1) << 4); 2831 if (tx_queue > IGC_N0_QUEUE) 2832 igc_write_ivar(hw, msix_vector, 2833 tx_queue >> 1, 2834 ((tx_queue & 0x1) << 4) + 8); 2835 q_vector->eims_value = BIT(msix_vector); 2836 break; 2837 default: 2838 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2839 break; 2840 } 2841 2842 /* add q_vector eims value to global eims_enable_mask */ 2843 adapter->eims_enable_mask |= q_vector->eims_value; 2844 2845 /* configure q_vector to set itr on first interrupt */ 2846 q_vector->set_itr = 1; 2847 } 2848 2849 /** 2850 * igc_configure_msix - Configure MSI-X hardware 2851 * @adapter: Pointer to adapter structure 2852 * 2853 * igc_configure_msix sets up the hardware to properly 2854 * generate MSI-X interrupts. 2855 */ 2856 static void igc_configure_msix(struct igc_adapter *adapter) 2857 { 2858 struct igc_hw *hw = &adapter->hw; 2859 int i, vector = 0; 2860 u32 tmp; 2861 2862 adapter->eims_enable_mask = 0; 2863 2864 /* set vector for other causes, i.e. link changes */ 2865 switch (hw->mac.type) { 2866 case igc_i225: 2867 /* Turn on MSI-X capability first, or our settings 2868 * won't stick. And it will take days to debug. 2869 */ 2870 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2871 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2872 IGC_GPIE_NSICR); 2873 2874 /* enable msix_other interrupt */ 2875 adapter->eims_other = BIT(vector); 2876 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2877 2878 wr32(IGC_IVAR_MISC, tmp); 2879 break; 2880 default: 2881 /* do nothing, since nothing else supports MSI-X */ 2882 break; 2883 } /* switch (hw->mac.type) */ 2884 2885 adapter->eims_enable_mask |= adapter->eims_other; 2886 2887 for (i = 0; i < adapter->num_q_vectors; i++) 2888 igc_assign_vector(adapter->q_vector[i], vector++); 2889 2890 wrfl(); 2891 } 2892 2893 static irqreturn_t igc_msix_ring(int irq, void *data) 2894 { 2895 struct igc_q_vector *q_vector = data; 2896 2897 /* Write the ITR value calculated from the previous interrupt. */ 2898 igc_write_itr(q_vector); 2899 2900 napi_schedule(&q_vector->napi); 2901 2902 return IRQ_HANDLED; 2903 } 2904 2905 /** 2906 * igc_request_msix - Initialize MSI-X interrupts 2907 * @adapter: Pointer to adapter structure 2908 * 2909 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 2910 * kernel. 2911 */ 2912 static int igc_request_msix(struct igc_adapter *adapter) 2913 { 2914 int i = 0, err = 0, vector = 0, free_vector = 0; 2915 struct net_device *netdev = adapter->netdev; 2916 2917 err = request_irq(adapter->msix_entries[vector].vector, 2918 &igc_msix_other, 0, netdev->name, adapter); 2919 if (err) 2920 goto err_out; 2921 2922 for (i = 0; i < adapter->num_q_vectors; i++) { 2923 struct igc_q_vector *q_vector = adapter->q_vector[i]; 2924 2925 vector++; 2926 2927 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 2928 2929 if (q_vector->rx.ring && q_vector->tx.ring) 2930 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 2931 q_vector->rx.ring->queue_index); 2932 else if (q_vector->tx.ring) 2933 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 2934 q_vector->tx.ring->queue_index); 2935 else if (q_vector->rx.ring) 2936 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 2937 q_vector->rx.ring->queue_index); 2938 else 2939 sprintf(q_vector->name, "%s-unused", netdev->name); 2940 2941 err = request_irq(adapter->msix_entries[vector].vector, 2942 igc_msix_ring, 0, q_vector->name, 2943 q_vector); 2944 if (err) 2945 goto err_free; 2946 } 2947 2948 igc_configure_msix(adapter); 2949 return 0; 2950 2951 err_free: 2952 /* free already assigned IRQs */ 2953 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 2954 2955 vector--; 2956 for (i = 0; i < vector; i++) { 2957 free_irq(adapter->msix_entries[free_vector++].vector, 2958 adapter->q_vector[i]); 2959 } 2960 err_out: 2961 return err; 2962 } 2963 2964 /** 2965 * igc_reset_q_vector - Reset config for interrupt vector 2966 * @adapter: board private structure to initialize 2967 * @v_idx: Index of vector to be reset 2968 * 2969 * If NAPI is enabled it will delete any references to the 2970 * NAPI struct. This is preparation for igc_free_q_vector. 2971 */ 2972 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2973 { 2974 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2975 2976 /* if we're coming from igc_set_interrupt_capability, the vectors are 2977 * not yet allocated 2978 */ 2979 if (!q_vector) 2980 return; 2981 2982 if (q_vector->tx.ring) 2983 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2984 2985 if (q_vector->rx.ring) 2986 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2987 2988 netif_napi_del(&q_vector->napi); 2989 } 2990 2991 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 2992 { 2993 int v_idx = adapter->num_q_vectors; 2994 2995 if (adapter->msix_entries) { 2996 pci_disable_msix(adapter->pdev); 2997 kfree(adapter->msix_entries); 2998 adapter->msix_entries = NULL; 2999 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3000 pci_disable_msi(adapter->pdev); 3001 } 3002 3003 while (v_idx--) 3004 igc_reset_q_vector(adapter, v_idx); 3005 } 3006 3007 /** 3008 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 3009 * @adapter: Pointer to adapter structure 3010 * 3011 * This function resets the device so that it has 0 rx queues, tx queues, and 3012 * MSI-X interrupts allocated. 3013 */ 3014 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 3015 { 3016 igc_free_q_vectors(adapter); 3017 igc_reset_interrupt_capability(adapter); 3018 } 3019 3020 /** 3021 * igc_free_q_vectors - Free memory allocated for interrupt vectors 3022 * @adapter: board private structure to initialize 3023 * 3024 * This function frees the memory allocated to the q_vectors. In addition if 3025 * NAPI is enabled it will delete any references to the NAPI struct prior 3026 * to freeing the q_vector. 3027 */ 3028 static void igc_free_q_vectors(struct igc_adapter *adapter) 3029 { 3030 int v_idx = adapter->num_q_vectors; 3031 3032 adapter->num_tx_queues = 0; 3033 adapter->num_rx_queues = 0; 3034 adapter->num_q_vectors = 0; 3035 3036 while (v_idx--) { 3037 igc_reset_q_vector(adapter, v_idx); 3038 igc_free_q_vector(adapter, v_idx); 3039 } 3040 } 3041 3042 /** 3043 * igc_free_q_vector - Free memory allocated for specific interrupt vector 3044 * @adapter: board private structure to initialize 3045 * @v_idx: Index of vector to be freed 3046 * 3047 * This function frees the memory allocated to the q_vector. 3048 */ 3049 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 3050 { 3051 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 3052 3053 adapter->q_vector[v_idx] = NULL; 3054 3055 /* igc_get_stats64() might access the rings on this vector, 3056 * we must wait a grace period before freeing it. 3057 */ 3058 if (q_vector) 3059 kfree_rcu(q_vector, rcu); 3060 } 3061 3062 /* Need to wait a few seconds after link up to get diagnostic information from 3063 * the phy 3064 */ 3065 static void igc_update_phy_info(struct timer_list *t) 3066 { 3067 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 3068 3069 igc_get_phy_info(&adapter->hw); 3070 } 3071 3072 /** 3073 * igc_has_link - check shared code for link and determine up/down 3074 * @adapter: pointer to driver private info 3075 */ 3076 bool igc_has_link(struct igc_adapter *adapter) 3077 { 3078 struct igc_hw *hw = &adapter->hw; 3079 bool link_active = false; 3080 3081 /* get_link_status is set on LSC (link status) interrupt or 3082 * rx sequence error interrupt. get_link_status will stay 3083 * false until the igc_check_for_link establishes link 3084 * for copper adapters ONLY 3085 */ 3086 switch (hw->phy.media_type) { 3087 case igc_media_type_copper: 3088 if (!hw->mac.get_link_status) 3089 return true; 3090 hw->mac.ops.check_for_link(hw); 3091 link_active = !hw->mac.get_link_status; 3092 break; 3093 default: 3094 case igc_media_type_unknown: 3095 break; 3096 } 3097 3098 if (hw->mac.type == igc_i225 && 3099 hw->phy.id == I225_I_PHY_ID) { 3100 if (!netif_carrier_ok(adapter->netdev)) { 3101 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3102 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 3103 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 3104 adapter->link_check_timeout = jiffies; 3105 } 3106 } 3107 3108 return link_active; 3109 } 3110 3111 /** 3112 * igc_watchdog - Timer Call-back 3113 * @data: pointer to adapter cast into an unsigned long 3114 */ 3115 static void igc_watchdog(struct timer_list *t) 3116 { 3117 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 3118 /* Do the rest outside of interrupt context */ 3119 schedule_work(&adapter->watchdog_task); 3120 } 3121 3122 static void igc_watchdog_task(struct work_struct *work) 3123 { 3124 struct igc_adapter *adapter = container_of(work, 3125 struct igc_adapter, 3126 watchdog_task); 3127 struct net_device *netdev = adapter->netdev; 3128 struct igc_hw *hw = &adapter->hw; 3129 struct igc_phy_info *phy = &hw->phy; 3130 u16 phy_data, retry_count = 20; 3131 u32 connsw; 3132 u32 link; 3133 int i; 3134 3135 link = igc_has_link(adapter); 3136 3137 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 3138 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 3139 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3140 else 3141 link = false; 3142 } 3143 3144 /* Force link down if we have fiber to swap to */ 3145 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 3146 if (hw->phy.media_type == igc_media_type_copper) { 3147 connsw = rd32(IGC_CONNSW); 3148 if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) 3149 link = 0; 3150 } 3151 } 3152 if (link) { 3153 if (!netif_carrier_ok(netdev)) { 3154 u32 ctrl; 3155 3156 hw->mac.ops.get_speed_and_duplex(hw, 3157 &adapter->link_speed, 3158 &adapter->link_duplex); 3159 3160 ctrl = rd32(IGC_CTRL); 3161 /* Link status message must follow this format */ 3162 netdev_info(netdev, 3163 "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 3164 netdev->name, 3165 adapter->link_speed, 3166 adapter->link_duplex == FULL_DUPLEX ? 3167 "Full" : "Half", 3168 (ctrl & IGC_CTRL_TFCE) && 3169 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 3170 (ctrl & IGC_CTRL_RFCE) ? "RX" : 3171 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 3172 3173 /* check if SmartSpeed worked */ 3174 igc_check_downshift(hw); 3175 if (phy->speed_downgraded) 3176 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 3177 3178 /* adjust timeout factor according to speed/duplex */ 3179 adapter->tx_timeout_factor = 1; 3180 switch (adapter->link_speed) { 3181 case SPEED_10: 3182 adapter->tx_timeout_factor = 14; 3183 break; 3184 case SPEED_100: 3185 /* maybe add some timeout factor ? */ 3186 break; 3187 } 3188 3189 if (adapter->link_speed != SPEED_1000) 3190 goto no_wait; 3191 3192 /* wait for Remote receiver status OK */ 3193 retry_read_status: 3194 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 3195 &phy_data)) { 3196 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 3197 retry_count) { 3198 msleep(100); 3199 retry_count--; 3200 goto retry_read_status; 3201 } else if (!retry_count) { 3202 dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); 3203 } 3204 } else { 3205 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); 3206 } 3207 no_wait: 3208 netif_carrier_on(netdev); 3209 3210 /* link state has changed, schedule phy info update */ 3211 if (!test_bit(__IGC_DOWN, &adapter->state)) 3212 mod_timer(&adapter->phy_info_timer, 3213 round_jiffies(jiffies + 2 * HZ)); 3214 } 3215 } else { 3216 if (netif_carrier_ok(netdev)) { 3217 adapter->link_speed = 0; 3218 adapter->link_duplex = 0; 3219 3220 /* Links status message must follow this format */ 3221 netdev_info(netdev, "igc: %s NIC Link is Down\n", 3222 netdev->name); 3223 netif_carrier_off(netdev); 3224 3225 /* link state has changed, schedule phy info update */ 3226 if (!test_bit(__IGC_DOWN, &adapter->state)) 3227 mod_timer(&adapter->phy_info_timer, 3228 round_jiffies(jiffies + 2 * HZ)); 3229 3230 /* link is down, time to check for alternate media */ 3231 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 3232 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 3233 schedule_work(&adapter->reset_task); 3234 /* return immediately */ 3235 return; 3236 } 3237 } 3238 3239 /* also check for alternate media here */ 3240 } else if (!netif_carrier_ok(netdev) && 3241 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 3242 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 3243 schedule_work(&adapter->reset_task); 3244 /* return immediately */ 3245 return; 3246 } 3247 } 3248 } 3249 3250 spin_lock(&adapter->stats64_lock); 3251 igc_update_stats(adapter); 3252 spin_unlock(&adapter->stats64_lock); 3253 3254 for (i = 0; i < adapter->num_tx_queues; i++) { 3255 struct igc_ring *tx_ring = adapter->tx_ring[i]; 3256 3257 if (!netif_carrier_ok(netdev)) { 3258 /* We've lost link, so the controller stops DMA, 3259 * but we've got queued Tx work that's never going 3260 * to get done, so reset controller to flush Tx. 3261 * (Do the reset outside of interrupt context). 3262 */ 3263 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 3264 adapter->tx_timeout_count++; 3265 schedule_work(&adapter->reset_task); 3266 /* return immediately since reset is imminent */ 3267 return; 3268 } 3269 } 3270 3271 /* Force detection of hung controller every watchdog period */ 3272 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 3273 } 3274 3275 /* Cause software interrupt to ensure Rx ring is cleaned */ 3276 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 3277 u32 eics = 0; 3278 3279 for (i = 0; i < adapter->num_q_vectors; i++) 3280 eics |= adapter->q_vector[i]->eims_value; 3281 wr32(IGC_EICS, eics); 3282 } else { 3283 wr32(IGC_ICS, IGC_ICS_RXDMT0); 3284 } 3285 3286 /* Reset the timer */ 3287 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3288 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 3289 mod_timer(&adapter->watchdog_timer, 3290 round_jiffies(jiffies + HZ)); 3291 else 3292 mod_timer(&adapter->watchdog_timer, 3293 round_jiffies(jiffies + 2 * HZ)); 3294 } 3295 } 3296 3297 /** 3298 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3299 * @q_vector: pointer to q_vector 3300 * 3301 * Stores a new ITR value based on strictly on packet size. This 3302 * algorithm is less sophisticated than that used in igc_update_itr, 3303 * due to the difficulty of synchronizing statistics across multiple 3304 * receive rings. The divisors and thresholds used by this function 3305 * were determined based on theoretical maximum wire speed and testing 3306 * data, in order to minimize response time while increasing bulk 3307 * throughput. 3308 * NOTE: This function is called only when operating in a multiqueue 3309 * receive environment. 3310 */ 3311 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3312 { 3313 struct igc_adapter *adapter = q_vector->adapter; 3314 int new_val = q_vector->itr_val; 3315 int avg_wire_size = 0; 3316 unsigned int packets; 3317 3318 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3319 * ints/sec - ITR timer value of 120 ticks. 3320 */ 3321 switch (adapter->link_speed) { 3322 case SPEED_10: 3323 case SPEED_100: 3324 new_val = IGC_4K_ITR; 3325 goto set_itr_val; 3326 default: 3327 break; 3328 } 3329 3330 packets = q_vector->rx.total_packets; 3331 if (packets) 3332 avg_wire_size = q_vector->rx.total_bytes / packets; 3333 3334 packets = q_vector->tx.total_packets; 3335 if (packets) 3336 avg_wire_size = max_t(u32, avg_wire_size, 3337 q_vector->tx.total_bytes / packets); 3338 3339 /* if avg_wire_size isn't set no work was done */ 3340 if (!avg_wire_size) 3341 goto clear_counts; 3342 3343 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3344 avg_wire_size += 24; 3345 3346 /* Don't starve jumbo frames */ 3347 avg_wire_size = min(avg_wire_size, 3000); 3348 3349 /* Give a little boost to mid-size frames */ 3350 if (avg_wire_size > 300 && avg_wire_size < 1200) 3351 new_val = avg_wire_size / 3; 3352 else 3353 new_val = avg_wire_size / 2; 3354 3355 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3356 if (new_val < IGC_20K_ITR && 3357 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3358 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3359 new_val = IGC_20K_ITR; 3360 3361 set_itr_val: 3362 if (new_val != q_vector->itr_val) { 3363 q_vector->itr_val = new_val; 3364 q_vector->set_itr = 1; 3365 } 3366 clear_counts: 3367 q_vector->rx.total_bytes = 0; 3368 q_vector->rx.total_packets = 0; 3369 q_vector->tx.total_bytes = 0; 3370 q_vector->tx.total_packets = 0; 3371 } 3372 3373 /** 3374 * igc_update_itr - update the dynamic ITR value based on statistics 3375 * @q_vector: pointer to q_vector 3376 * @ring_container: ring info to update the itr for 3377 * 3378 * Stores a new ITR value based on packets and byte 3379 * counts during the last interrupt. The advantage of per interrupt 3380 * computation is faster updates and more accurate ITR for the current 3381 * traffic pattern. Constants in this function were computed 3382 * based on theoretical maximum wire speed and thresholds were set based 3383 * on testing data as well as attempting to minimize response time 3384 * while increasing bulk throughput. 3385 * NOTE: These calculations are only valid when operating in a single- 3386 * queue environment. 3387 */ 3388 static void igc_update_itr(struct igc_q_vector *q_vector, 3389 struct igc_ring_container *ring_container) 3390 { 3391 unsigned int packets = ring_container->total_packets; 3392 unsigned int bytes = ring_container->total_bytes; 3393 u8 itrval = ring_container->itr; 3394 3395 /* no packets, exit with status unchanged */ 3396 if (packets == 0) 3397 return; 3398 3399 switch (itrval) { 3400 case lowest_latency: 3401 /* handle TSO and jumbo frames */ 3402 if (bytes / packets > 8000) 3403 itrval = bulk_latency; 3404 else if ((packets < 5) && (bytes > 512)) 3405 itrval = low_latency; 3406 break; 3407 case low_latency: /* 50 usec aka 20000 ints/s */ 3408 if (bytes > 10000) { 3409 /* this if handles the TSO accounting */ 3410 if (bytes / packets > 8000) 3411 itrval = bulk_latency; 3412 else if ((packets < 10) || ((bytes / packets) > 1200)) 3413 itrval = bulk_latency; 3414 else if ((packets > 35)) 3415 itrval = lowest_latency; 3416 } else if (bytes / packets > 2000) { 3417 itrval = bulk_latency; 3418 } else if (packets <= 2 && bytes < 512) { 3419 itrval = lowest_latency; 3420 } 3421 break; 3422 case bulk_latency: /* 250 usec aka 4000 ints/s */ 3423 if (bytes > 25000) { 3424 if (packets > 35) 3425 itrval = low_latency; 3426 } else if (bytes < 1500) { 3427 itrval = low_latency; 3428 } 3429 break; 3430 } 3431 3432 /* clear work counters since we have the values we need */ 3433 ring_container->total_bytes = 0; 3434 ring_container->total_packets = 0; 3435 3436 /* write updated itr to ring container */ 3437 ring_container->itr = itrval; 3438 } 3439 3440 /** 3441 * igc_intr_msi - Interrupt Handler 3442 * @irq: interrupt number 3443 * @data: pointer to a network interface device structure 3444 */ 3445 static irqreturn_t igc_intr_msi(int irq, void *data) 3446 { 3447 struct igc_adapter *adapter = data; 3448 struct igc_q_vector *q_vector = adapter->q_vector[0]; 3449 struct igc_hw *hw = &adapter->hw; 3450 /* read ICR disables interrupts using IAM */ 3451 u32 icr = rd32(IGC_ICR); 3452 3453 igc_write_itr(q_vector); 3454 3455 if (icr & IGC_ICR_DRSTA) 3456 schedule_work(&adapter->reset_task); 3457 3458 if (icr & IGC_ICR_DOUTSYNC) { 3459 /* HW is reporting DMA is out of sync */ 3460 adapter->stats.doosync++; 3461 } 3462 3463 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 3464 hw->mac.get_link_status = 1; 3465 if (!test_bit(__IGC_DOWN, &adapter->state)) 3466 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3467 } 3468 3469 napi_schedule(&q_vector->napi); 3470 3471 return IRQ_HANDLED; 3472 } 3473 3474 /** 3475 * igc_intr - Legacy Interrupt Handler 3476 * @irq: interrupt number 3477 * @data: pointer to a network interface device structure 3478 */ 3479 static irqreturn_t igc_intr(int irq, void *data) 3480 { 3481 struct igc_adapter *adapter = data; 3482 struct igc_q_vector *q_vector = adapter->q_vector[0]; 3483 struct igc_hw *hw = &adapter->hw; 3484 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 3485 * need for the IMC write 3486 */ 3487 u32 icr = rd32(IGC_ICR); 3488 3489 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 3490 * not set, then the adapter didn't send an interrupt 3491 */ 3492 if (!(icr & IGC_ICR_INT_ASSERTED)) 3493 return IRQ_NONE; 3494 3495 igc_write_itr(q_vector); 3496 3497 if (icr & IGC_ICR_DRSTA) 3498 schedule_work(&adapter->reset_task); 3499 3500 if (icr & IGC_ICR_DOUTSYNC) { 3501 /* HW is reporting DMA is out of sync */ 3502 adapter->stats.doosync++; 3503 } 3504 3505 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 3506 hw->mac.get_link_status = 1; 3507 /* guard against interrupt when we're going down */ 3508 if (!test_bit(__IGC_DOWN, &adapter->state)) 3509 mod_timer(&adapter->watchdog_timer, jiffies + 1); 3510 } 3511 3512 napi_schedule(&q_vector->napi); 3513 3514 return IRQ_HANDLED; 3515 } 3516 3517 static void igc_set_itr(struct igc_q_vector *q_vector) 3518 { 3519 struct igc_adapter *adapter = q_vector->adapter; 3520 u32 new_itr = q_vector->itr_val; 3521 u8 current_itr = 0; 3522 3523 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3524 switch (adapter->link_speed) { 3525 case SPEED_10: 3526 case SPEED_100: 3527 current_itr = 0; 3528 new_itr = IGC_4K_ITR; 3529 goto set_itr_now; 3530 default: 3531 break; 3532 } 3533 3534 igc_update_itr(q_vector, &q_vector->tx); 3535 igc_update_itr(q_vector, &q_vector->rx); 3536 3537 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3538 3539 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3540 if (current_itr == lowest_latency && 3541 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3542 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3543 current_itr = low_latency; 3544 3545 switch (current_itr) { 3546 /* counts and packets in update_itr are dependent on these numbers */ 3547 case lowest_latency: 3548 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3549 break; 3550 case low_latency: 3551 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3552 break; 3553 case bulk_latency: 3554 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3555 break; 3556 default: 3557 break; 3558 } 3559 3560 set_itr_now: 3561 if (new_itr != q_vector->itr_val) { 3562 /* this attempts to bias the interrupt rate towards Bulk 3563 * by adding intermediate steps when interrupt rate is 3564 * increasing 3565 */ 3566 new_itr = new_itr > q_vector->itr_val ? 3567 max((new_itr * q_vector->itr_val) / 3568 (new_itr + (q_vector->itr_val >> 2)), 3569 new_itr) : new_itr; 3570 /* Don't write the value here; it resets the adapter's 3571 * internal timer, and causes us to delay far longer than 3572 * we should between interrupts. Instead, we write the ITR 3573 * value at the beginning of the next interrupt so the timing 3574 * ends up being correct. 3575 */ 3576 q_vector->itr_val = new_itr; 3577 q_vector->set_itr = 1; 3578 } 3579 } 3580 3581 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3582 { 3583 struct igc_adapter *adapter = q_vector->adapter; 3584 struct igc_hw *hw = &adapter->hw; 3585 3586 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3587 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3588 if (adapter->num_q_vectors == 1) 3589 igc_set_itr(q_vector); 3590 else 3591 igc_update_ring_itr(q_vector); 3592 } 3593 3594 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3595 if (adapter->msix_entries) 3596 wr32(IGC_EIMS, q_vector->eims_value); 3597 else 3598 igc_irq_enable(adapter); 3599 } 3600 } 3601 3602 /** 3603 * igc_poll - NAPI Rx polling callback 3604 * @napi: napi polling structure 3605 * @budget: count of how many packets we should handle 3606 */ 3607 static int igc_poll(struct napi_struct *napi, int budget) 3608 { 3609 struct igc_q_vector *q_vector = container_of(napi, 3610 struct igc_q_vector, 3611 napi); 3612 bool clean_complete = true; 3613 int work_done = 0; 3614 3615 if (q_vector->tx.ring) 3616 clean_complete = igc_clean_tx_irq(q_vector, budget); 3617 3618 if (q_vector->rx.ring) { 3619 int cleaned = igc_clean_rx_irq(q_vector, budget); 3620 3621 work_done += cleaned; 3622 if (cleaned >= budget) 3623 clean_complete = false; 3624 } 3625 3626 /* If all work not completed, return budget and keep polling */ 3627 if (!clean_complete) 3628 return budget; 3629 3630 /* Exit the polling mode, but don't re-enable interrupts if stack might 3631 * poll us due to busy-polling 3632 */ 3633 if (likely(napi_complete_done(napi, work_done))) 3634 igc_ring_irq_enable(q_vector); 3635 3636 return min(work_done, budget - 1); 3637 } 3638 3639 /** 3640 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3641 * @adapter: Pointer to adapter structure 3642 * 3643 * Attempt to configure interrupts using the best available 3644 * capabilities of the hardware and kernel. 3645 */ 3646 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3647 bool msix) 3648 { 3649 int numvecs, i; 3650 int err; 3651 3652 if (!msix) 3653 goto msi_only; 3654 adapter->flags |= IGC_FLAG_HAS_MSIX; 3655 3656 /* Number of supported queues. */ 3657 adapter->num_rx_queues = adapter->rss_queues; 3658 3659 adapter->num_tx_queues = adapter->rss_queues; 3660 3661 /* start with one vector for every Rx queue */ 3662 numvecs = adapter->num_rx_queues; 3663 3664 /* if Tx handler is separate add 1 for every Tx queue */ 3665 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3666 numvecs += adapter->num_tx_queues; 3667 3668 /* store the number of vectors reserved for queues */ 3669 adapter->num_q_vectors = numvecs; 3670 3671 /* add 1 vector for link status interrupts */ 3672 numvecs++; 3673 3674 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3675 GFP_KERNEL); 3676 3677 if (!adapter->msix_entries) 3678 return; 3679 3680 /* populate entry values */ 3681 for (i = 0; i < numvecs; i++) 3682 adapter->msix_entries[i].entry = i; 3683 3684 err = pci_enable_msix_range(adapter->pdev, 3685 adapter->msix_entries, 3686 numvecs, 3687 numvecs); 3688 if (err > 0) 3689 return; 3690 3691 kfree(adapter->msix_entries); 3692 adapter->msix_entries = NULL; 3693 3694 igc_reset_interrupt_capability(adapter); 3695 3696 msi_only: 3697 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3698 3699 adapter->rss_queues = 1; 3700 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3701 adapter->num_rx_queues = 1; 3702 adapter->num_tx_queues = 1; 3703 adapter->num_q_vectors = 1; 3704 if (!pci_enable_msi(adapter->pdev)) 3705 adapter->flags |= IGC_FLAG_HAS_MSI; 3706 } 3707 3708 static void igc_add_ring(struct igc_ring *ring, 3709 struct igc_ring_container *head) 3710 { 3711 head->ring = ring; 3712 head->count++; 3713 } 3714 3715 /** 3716 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3717 * @adapter: board private structure to initialize 3718 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3719 * @v_idx: index of vector in adapter struct 3720 * @txr_count: total number of Tx rings to allocate 3721 * @txr_idx: index of first Tx ring to allocate 3722 * @rxr_count: total number of Rx rings to allocate 3723 * @rxr_idx: index of first Rx ring to allocate 3724 * 3725 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3726 */ 3727 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3728 unsigned int v_count, unsigned int v_idx, 3729 unsigned int txr_count, unsigned int txr_idx, 3730 unsigned int rxr_count, unsigned int rxr_idx) 3731 { 3732 struct igc_q_vector *q_vector; 3733 struct igc_ring *ring; 3734 int ring_count; 3735 3736 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3737 if (txr_count > 1 || rxr_count > 1) 3738 return -ENOMEM; 3739 3740 ring_count = txr_count + rxr_count; 3741 3742 /* allocate q_vector and rings */ 3743 q_vector = adapter->q_vector[v_idx]; 3744 if (!q_vector) 3745 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3746 GFP_KERNEL); 3747 else 3748 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3749 if (!q_vector) 3750 return -ENOMEM; 3751 3752 /* initialize NAPI */ 3753 netif_napi_add(adapter->netdev, &q_vector->napi, 3754 igc_poll, 64); 3755 3756 /* tie q_vector and adapter together */ 3757 adapter->q_vector[v_idx] = q_vector; 3758 q_vector->adapter = adapter; 3759 3760 /* initialize work limits */ 3761 q_vector->tx.work_limit = adapter->tx_work_limit; 3762 3763 /* initialize ITR configuration */ 3764 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3765 q_vector->itr_val = IGC_START_ITR; 3766 3767 /* initialize pointer to rings */ 3768 ring = q_vector->ring; 3769 3770 /* initialize ITR */ 3771 if (rxr_count) { 3772 /* rx or rx/tx vector */ 3773 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3774 q_vector->itr_val = adapter->rx_itr_setting; 3775 } else { 3776 /* tx only vector */ 3777 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3778 q_vector->itr_val = adapter->tx_itr_setting; 3779 } 3780 3781 if (txr_count) { 3782 /* assign generic ring traits */ 3783 ring->dev = &adapter->pdev->dev; 3784 ring->netdev = adapter->netdev; 3785 3786 /* configure backlink on ring */ 3787 ring->q_vector = q_vector; 3788 3789 /* update q_vector Tx values */ 3790 igc_add_ring(ring, &q_vector->tx); 3791 3792 /* apply Tx specific ring traits */ 3793 ring->count = adapter->tx_ring_count; 3794 ring->queue_index = txr_idx; 3795 3796 /* assign ring to adapter */ 3797 adapter->tx_ring[txr_idx] = ring; 3798 3799 /* push pointer to next ring */ 3800 ring++; 3801 } 3802 3803 if (rxr_count) { 3804 /* assign generic ring traits */ 3805 ring->dev = &adapter->pdev->dev; 3806 ring->netdev = adapter->netdev; 3807 3808 /* configure backlink on ring */ 3809 ring->q_vector = q_vector; 3810 3811 /* update q_vector Rx values */ 3812 igc_add_ring(ring, &q_vector->rx); 3813 3814 /* apply Rx specific ring traits */ 3815 ring->count = adapter->rx_ring_count; 3816 ring->queue_index = rxr_idx; 3817 3818 /* assign ring to adapter */ 3819 adapter->rx_ring[rxr_idx] = ring; 3820 } 3821 3822 return 0; 3823 } 3824 3825 /** 3826 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3827 * @adapter: board private structure to initialize 3828 * 3829 * We allocate one q_vector per queue interrupt. If allocation fails we 3830 * return -ENOMEM. 3831 */ 3832 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3833 { 3834 int rxr_remaining = adapter->num_rx_queues; 3835 int txr_remaining = adapter->num_tx_queues; 3836 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3837 int q_vectors = adapter->num_q_vectors; 3838 int err; 3839 3840 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3841 for (; rxr_remaining; v_idx++) { 3842 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3843 0, 0, 1, rxr_idx); 3844 3845 if (err) 3846 goto err_out; 3847 3848 /* update counts and index */ 3849 rxr_remaining--; 3850 rxr_idx++; 3851 } 3852 } 3853 3854 for (; v_idx < q_vectors; v_idx++) { 3855 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3856 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3857 3858 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3859 tqpv, txr_idx, rqpv, rxr_idx); 3860 3861 if (err) 3862 goto err_out; 3863 3864 /* update counts and index */ 3865 rxr_remaining -= rqpv; 3866 txr_remaining -= tqpv; 3867 rxr_idx++; 3868 txr_idx++; 3869 } 3870 3871 return 0; 3872 3873 err_out: 3874 adapter->num_tx_queues = 0; 3875 adapter->num_rx_queues = 0; 3876 adapter->num_q_vectors = 0; 3877 3878 while (v_idx--) 3879 igc_free_q_vector(adapter, v_idx); 3880 3881 return -ENOMEM; 3882 } 3883 3884 /** 3885 * igc_cache_ring_register - Descriptor ring to register mapping 3886 * @adapter: board private structure to initialize 3887 * 3888 * Once we know the feature-set enabled for the device, we'll cache 3889 * the register offset the descriptor ring is assigned to. 3890 */ 3891 static void igc_cache_ring_register(struct igc_adapter *adapter) 3892 { 3893 int i = 0, j = 0; 3894 3895 switch (adapter->hw.mac.type) { 3896 case igc_i225: 3897 /* Fall through */ 3898 default: 3899 for (; i < adapter->num_rx_queues; i++) 3900 adapter->rx_ring[i]->reg_idx = i; 3901 for (; j < adapter->num_tx_queues; j++) 3902 adapter->tx_ring[j]->reg_idx = j; 3903 break; 3904 } 3905 } 3906 3907 /** 3908 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3909 * @adapter: Pointer to adapter structure 3910 * 3911 * This function initializes the interrupts and allocates all of the queues. 3912 */ 3913 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3914 { 3915 struct pci_dev *pdev = adapter->pdev; 3916 int err = 0; 3917 3918 igc_set_interrupt_capability(adapter, msix); 3919 3920 err = igc_alloc_q_vectors(adapter); 3921 if (err) { 3922 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); 3923 goto err_alloc_q_vectors; 3924 } 3925 3926 igc_cache_ring_register(adapter); 3927 3928 return 0; 3929 3930 err_alloc_q_vectors: 3931 igc_reset_interrupt_capability(adapter); 3932 return err; 3933 } 3934 3935 static void igc_free_irq(struct igc_adapter *adapter) 3936 { 3937 if (adapter->msix_entries) { 3938 int vector = 0, i; 3939 3940 free_irq(adapter->msix_entries[vector++].vector, adapter); 3941 3942 for (i = 0; i < adapter->num_q_vectors; i++) 3943 free_irq(adapter->msix_entries[vector++].vector, 3944 adapter->q_vector[i]); 3945 } else { 3946 free_irq(adapter->pdev->irq, adapter); 3947 } 3948 } 3949 3950 /** 3951 * igc_irq_disable - Mask off interrupt generation on the NIC 3952 * @adapter: board private structure 3953 */ 3954 static void igc_irq_disable(struct igc_adapter *adapter) 3955 { 3956 struct igc_hw *hw = &adapter->hw; 3957 3958 if (adapter->msix_entries) { 3959 u32 regval = rd32(IGC_EIAM); 3960 3961 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 3962 wr32(IGC_EIMC, adapter->eims_enable_mask); 3963 regval = rd32(IGC_EIAC); 3964 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 3965 } 3966 3967 wr32(IGC_IAM, 0); 3968 wr32(IGC_IMC, ~0); 3969 wrfl(); 3970 3971 if (adapter->msix_entries) { 3972 int vector = 0, i; 3973 3974 synchronize_irq(adapter->msix_entries[vector++].vector); 3975 3976 for (i = 0; i < adapter->num_q_vectors; i++) 3977 synchronize_irq(adapter->msix_entries[vector++].vector); 3978 } else { 3979 synchronize_irq(adapter->pdev->irq); 3980 } 3981 } 3982 3983 /** 3984 * igc_irq_enable - Enable default interrupt generation settings 3985 * @adapter: board private structure 3986 */ 3987 static void igc_irq_enable(struct igc_adapter *adapter) 3988 { 3989 struct igc_hw *hw = &adapter->hw; 3990 3991 if (adapter->msix_entries) { 3992 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 3993 u32 regval = rd32(IGC_EIAC); 3994 3995 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 3996 regval = rd32(IGC_EIAM); 3997 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 3998 wr32(IGC_EIMS, adapter->eims_enable_mask); 3999 wr32(IGC_IMS, ims); 4000 } else { 4001 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4002 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4003 } 4004 } 4005 4006 /** 4007 * igc_request_irq - initialize interrupts 4008 * @adapter: Pointer to adapter structure 4009 * 4010 * Attempts to configure interrupts using the best available 4011 * capabilities of the hardware and kernel. 4012 */ 4013 static int igc_request_irq(struct igc_adapter *adapter) 4014 { 4015 struct net_device *netdev = adapter->netdev; 4016 struct pci_dev *pdev = adapter->pdev; 4017 int err = 0; 4018 4019 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4020 err = igc_request_msix(adapter); 4021 if (!err) 4022 goto request_done; 4023 /* fall back to MSI */ 4024 igc_free_all_tx_resources(adapter); 4025 igc_free_all_rx_resources(adapter); 4026 4027 igc_clear_interrupt_scheme(adapter); 4028 err = igc_init_interrupt_scheme(adapter, false); 4029 if (err) 4030 goto request_done; 4031 igc_setup_all_tx_resources(adapter); 4032 igc_setup_all_rx_resources(adapter); 4033 igc_configure(adapter); 4034 } 4035 4036 igc_assign_vector(adapter->q_vector[0], 0); 4037 4038 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4039 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4040 netdev->name, adapter); 4041 if (!err) 4042 goto request_done; 4043 4044 /* fall back to legacy interrupts */ 4045 igc_reset_interrupt_capability(adapter); 4046 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4047 } 4048 4049 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4050 netdev->name, adapter); 4051 4052 if (err) 4053 dev_err(&pdev->dev, "Error %d getting interrupt\n", 4054 err); 4055 4056 request_done: 4057 return err; 4058 } 4059 4060 static void igc_write_itr(struct igc_q_vector *q_vector) 4061 { 4062 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4063 4064 if (!q_vector->set_itr) 4065 return; 4066 4067 if (!itr_val) 4068 itr_val = IGC_ITR_VAL_MASK; 4069 4070 itr_val |= IGC_EITR_CNT_IGNR; 4071 4072 writel(itr_val, q_vector->itr_register); 4073 q_vector->set_itr = 0; 4074 } 4075 4076 /** 4077 * igc_open - Called when a network interface is made active 4078 * @netdev: network interface device structure 4079 * 4080 * Returns 0 on success, negative value on failure 4081 * 4082 * The open entry point is called when a network interface is made 4083 * active by the system (IFF_UP). At this point all resources needed 4084 * for transmit and receive operations are allocated, the interrupt 4085 * handler is registered with the OS, the watchdog timer is started, 4086 * and the stack is notified that the interface is ready. 4087 */ 4088 static int __igc_open(struct net_device *netdev, bool resuming) 4089 { 4090 struct igc_adapter *adapter = netdev_priv(netdev); 4091 struct igc_hw *hw = &adapter->hw; 4092 int err = 0; 4093 int i = 0; 4094 4095 /* disallow open during test */ 4096 4097 if (test_bit(__IGC_TESTING, &adapter->state)) { 4098 WARN_ON(resuming); 4099 return -EBUSY; 4100 } 4101 4102 netif_carrier_off(netdev); 4103 4104 /* allocate transmit descriptors */ 4105 err = igc_setup_all_tx_resources(adapter); 4106 if (err) 4107 goto err_setup_tx; 4108 4109 /* allocate receive descriptors */ 4110 err = igc_setup_all_rx_resources(adapter); 4111 if (err) 4112 goto err_setup_rx; 4113 4114 igc_power_up_link(adapter); 4115 4116 igc_configure(adapter); 4117 4118 err = igc_request_irq(adapter); 4119 if (err) 4120 goto err_req_irq; 4121 4122 /* Notify the stack of the actual queue counts. */ 4123 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4124 if (err) 4125 goto err_set_queues; 4126 4127 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4128 if (err) 4129 goto err_set_queues; 4130 4131 clear_bit(__IGC_DOWN, &adapter->state); 4132 4133 for (i = 0; i < adapter->num_q_vectors; i++) 4134 napi_enable(&adapter->q_vector[i]->napi); 4135 4136 /* Clear any pending interrupts. */ 4137 rd32(IGC_ICR); 4138 igc_irq_enable(adapter); 4139 4140 netif_tx_start_all_queues(netdev); 4141 4142 /* start the watchdog. */ 4143 hw->mac.get_link_status = 1; 4144 schedule_work(&adapter->watchdog_task); 4145 4146 return IGC_SUCCESS; 4147 4148 err_set_queues: 4149 igc_free_irq(adapter); 4150 err_req_irq: 4151 igc_release_hw_control(adapter); 4152 igc_power_down_link(adapter); 4153 igc_free_all_rx_resources(adapter); 4154 err_setup_rx: 4155 igc_free_all_tx_resources(adapter); 4156 err_setup_tx: 4157 igc_reset(adapter); 4158 4159 return err; 4160 } 4161 4162 static int igc_open(struct net_device *netdev) 4163 { 4164 return __igc_open(netdev, false); 4165 } 4166 4167 /** 4168 * igc_close - Disables a network interface 4169 * @netdev: network interface device structure 4170 * 4171 * Returns 0, this is not allowed to fail 4172 * 4173 * The close entry point is called when an interface is de-activated 4174 * by the OS. The hardware is still under the driver's control, but 4175 * needs to be disabled. A global MAC reset is issued to stop the 4176 * hardware, and all transmit and receive resources are freed. 4177 */ 4178 static int __igc_close(struct net_device *netdev, bool suspending) 4179 { 4180 struct igc_adapter *adapter = netdev_priv(netdev); 4181 4182 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4183 4184 igc_down(adapter); 4185 4186 igc_release_hw_control(adapter); 4187 4188 igc_free_irq(adapter); 4189 4190 igc_free_all_tx_resources(adapter); 4191 igc_free_all_rx_resources(adapter); 4192 4193 return 0; 4194 } 4195 4196 static int igc_close(struct net_device *netdev) 4197 { 4198 if (netif_device_present(netdev) || netdev->dismantle) 4199 return __igc_close(netdev, false); 4200 return 0; 4201 } 4202 4203 static const struct net_device_ops igc_netdev_ops = { 4204 .ndo_open = igc_open, 4205 .ndo_stop = igc_close, 4206 .ndo_start_xmit = igc_xmit_frame, 4207 .ndo_set_rx_mode = igc_set_rx_mode, 4208 .ndo_set_mac_address = igc_set_mac, 4209 .ndo_change_mtu = igc_change_mtu, 4210 .ndo_get_stats = igc_get_stats, 4211 .ndo_fix_features = igc_fix_features, 4212 .ndo_set_features = igc_set_features, 4213 .ndo_features_check = igc_features_check, 4214 }; 4215 4216 /* PCIe configuration access */ 4217 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4218 { 4219 struct igc_adapter *adapter = hw->back; 4220 4221 pci_read_config_word(adapter->pdev, reg, value); 4222 } 4223 4224 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4225 { 4226 struct igc_adapter *adapter = hw->back; 4227 4228 pci_write_config_word(adapter->pdev, reg, *value); 4229 } 4230 4231 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4232 { 4233 struct igc_adapter *adapter = hw->back; 4234 4235 if (!pci_is_pcie(adapter->pdev)) 4236 return -IGC_ERR_CONFIG; 4237 4238 pcie_capability_read_word(adapter->pdev, reg, value); 4239 4240 return IGC_SUCCESS; 4241 } 4242 4243 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4244 { 4245 struct igc_adapter *adapter = hw->back; 4246 4247 if (!pci_is_pcie(adapter->pdev)) 4248 return -IGC_ERR_CONFIG; 4249 4250 pcie_capability_write_word(adapter->pdev, reg, *value); 4251 4252 return IGC_SUCCESS; 4253 } 4254 4255 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4256 { 4257 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4258 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4259 u32 value = 0; 4260 4261 if (IGC_REMOVED(hw_addr)) 4262 return ~value; 4263 4264 value = readl(&hw_addr[reg]); 4265 4266 /* reads should not return all F's */ 4267 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4268 struct net_device *netdev = igc->netdev; 4269 4270 hw->hw_addr = NULL; 4271 netif_device_detach(netdev); 4272 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4273 WARN(pci_device_is_present(igc->pdev), 4274 "igc: Failed to read reg 0x%x!\n", reg); 4275 } 4276 4277 return value; 4278 } 4279 4280 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4281 { 4282 struct pci_dev *pdev = adapter->pdev; 4283 struct igc_mac_info *mac = &adapter->hw.mac; 4284 4285 mac->autoneg = 0; 4286 4287 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4288 * for the switch() below to work 4289 */ 4290 if ((spd & 1) || (dplx & ~1)) 4291 goto err_inval; 4292 4293 switch (spd + dplx) { 4294 case SPEED_10 + DUPLEX_HALF: 4295 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4296 break; 4297 case SPEED_10 + DUPLEX_FULL: 4298 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4299 break; 4300 case SPEED_100 + DUPLEX_HALF: 4301 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4302 break; 4303 case SPEED_100 + DUPLEX_FULL: 4304 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4305 break; 4306 case SPEED_1000 + DUPLEX_FULL: 4307 mac->autoneg = 1; 4308 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4309 break; 4310 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4311 goto err_inval; 4312 case SPEED_2500 + DUPLEX_FULL: 4313 mac->autoneg = 1; 4314 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4315 break; 4316 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4317 default: 4318 goto err_inval; 4319 } 4320 4321 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4322 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4323 4324 return 0; 4325 4326 err_inval: 4327 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n"); 4328 return -EINVAL; 4329 } 4330 4331 /** 4332 * igc_probe - Device Initialization Routine 4333 * @pdev: PCI device information struct 4334 * @ent: entry in igc_pci_tbl 4335 * 4336 * Returns 0 on success, negative on failure 4337 * 4338 * igc_probe initializes an adapter identified by a pci_dev structure. 4339 * The OS initialization, configuring the adapter private structure, 4340 * and a hardware reset occur. 4341 */ 4342 static int igc_probe(struct pci_dev *pdev, 4343 const struct pci_device_id *ent) 4344 { 4345 struct igc_adapter *adapter; 4346 struct net_device *netdev; 4347 struct igc_hw *hw; 4348 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4349 int err; 4350 4351 err = pci_enable_device_mem(pdev); 4352 if (err) 4353 return err; 4354 4355 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 4356 if (!err) { 4357 err = dma_set_coherent_mask(&pdev->dev, 4358 DMA_BIT_MASK(64)); 4359 } else { 4360 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 4361 if (err) { 4362 err = dma_set_coherent_mask(&pdev->dev, 4363 DMA_BIT_MASK(32)); 4364 if (err) { 4365 dev_err(&pdev->dev, "igc: Wrong DMA config\n"); 4366 goto err_dma; 4367 } 4368 } 4369 } 4370 4371 err = pci_request_selected_regions(pdev, 4372 pci_select_bars(pdev, 4373 IORESOURCE_MEM), 4374 igc_driver_name); 4375 if (err) 4376 goto err_pci_reg; 4377 4378 pci_enable_pcie_error_reporting(pdev); 4379 4380 pci_set_master(pdev); 4381 4382 err = -ENOMEM; 4383 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 4384 IGC_MAX_TX_QUEUES); 4385 4386 if (!netdev) 4387 goto err_alloc_etherdev; 4388 4389 SET_NETDEV_DEV(netdev, &pdev->dev); 4390 4391 pci_set_drvdata(pdev, netdev); 4392 adapter = netdev_priv(netdev); 4393 adapter->netdev = netdev; 4394 adapter->pdev = pdev; 4395 hw = &adapter->hw; 4396 hw->back = adapter; 4397 adapter->port_num = hw->bus.func; 4398 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4399 4400 err = pci_save_state(pdev); 4401 if (err) 4402 goto err_ioremap; 4403 4404 err = -EIO; 4405 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 4406 pci_resource_len(pdev, 0)); 4407 if (!adapter->io_addr) 4408 goto err_ioremap; 4409 4410 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 4411 hw->hw_addr = adapter->io_addr; 4412 4413 netdev->netdev_ops = &igc_netdev_ops; 4414 igc_set_ethtool_ops(netdev); 4415 netdev->watchdog_timeo = 5 * HZ; 4416 4417 netdev->mem_start = pci_resource_start(pdev, 0); 4418 netdev->mem_end = pci_resource_end(pdev, 0); 4419 4420 /* PCI config space info */ 4421 hw->vendor_id = pdev->vendor; 4422 hw->device_id = pdev->device; 4423 hw->revision_id = pdev->revision; 4424 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4425 hw->subsystem_device_id = pdev->subsystem_device; 4426 4427 /* Copy the default MAC and PHY function pointers */ 4428 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 4429 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 4430 4431 /* Initialize skew-specific constants */ 4432 err = ei->get_invariants(hw); 4433 if (err) 4434 goto err_sw_init; 4435 4436 /* Add supported features to the features list*/ 4437 netdev->features |= NETIF_F_RXCSUM; 4438 netdev->features |= NETIF_F_HW_CSUM; 4439 netdev->features |= NETIF_F_SCTP_CRC; 4440 4441 /* setup the private structure */ 4442 err = igc_sw_init(adapter); 4443 if (err) 4444 goto err_sw_init; 4445 4446 /* copy netdev features into list of user selectable features */ 4447 netdev->hw_features |= NETIF_F_NTUPLE; 4448 netdev->hw_features |= netdev->features; 4449 4450 /* MTU range: 68 - 9216 */ 4451 netdev->min_mtu = ETH_MIN_MTU; 4452 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 4453 4454 /* before reading the NVM, reset the controller to put the device in a 4455 * known good starting state 4456 */ 4457 hw->mac.ops.reset_hw(hw); 4458 4459 if (igc_get_flash_presence_i225(hw)) { 4460 if (hw->nvm.ops.validate(hw) < 0) { 4461 dev_err(&pdev->dev, 4462 "The NVM Checksum Is Not Valid\n"); 4463 err = -EIO; 4464 goto err_eeprom; 4465 } 4466 } 4467 4468 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 4469 /* copy the MAC address out of the NVM */ 4470 if (hw->mac.ops.read_mac_addr(hw)) 4471 dev_err(&pdev->dev, "NVM Read Error\n"); 4472 } 4473 4474 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 4475 4476 if (!is_valid_ether_addr(netdev->dev_addr)) { 4477 dev_err(&pdev->dev, "Invalid MAC Address\n"); 4478 err = -EIO; 4479 goto err_eeprom; 4480 } 4481 4482 /* configure RXPBSIZE and TXPBSIZE */ 4483 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 4484 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 4485 4486 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 4487 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 4488 4489 INIT_WORK(&adapter->reset_task, igc_reset_task); 4490 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 4491 4492 /* Initialize link properties that are user-changeable */ 4493 adapter->fc_autoneg = true; 4494 hw->mac.autoneg = true; 4495 hw->phy.autoneg_advertised = 0xaf; 4496 4497 hw->fc.requested_mode = igc_fc_default; 4498 hw->fc.current_mode = igc_fc_default; 4499 4500 /* reset the hardware with the new settings */ 4501 igc_reset(adapter); 4502 4503 /* let the f/w know that the h/w is now under the control of the 4504 * driver. 4505 */ 4506 igc_get_hw_control(adapter); 4507 4508 strncpy(netdev->name, "eth%d", IFNAMSIZ); 4509 err = register_netdev(netdev); 4510 if (err) 4511 goto err_register; 4512 4513 /* carrier off reporting is important to ethtool even BEFORE open */ 4514 netif_carrier_off(netdev); 4515 4516 /* Check if Media Autosense is enabled */ 4517 adapter->ei = *ei; 4518 4519 /* print pcie link status and MAC address */ 4520 pcie_print_link_status(pdev); 4521 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 4522 4523 return 0; 4524 4525 err_register: 4526 igc_release_hw_control(adapter); 4527 err_eeprom: 4528 if (!igc_check_reset_block(hw)) 4529 igc_reset_phy(hw); 4530 err_sw_init: 4531 igc_clear_interrupt_scheme(adapter); 4532 iounmap(adapter->io_addr); 4533 err_ioremap: 4534 free_netdev(netdev); 4535 err_alloc_etherdev: 4536 pci_release_selected_regions(pdev, 4537 pci_select_bars(pdev, IORESOURCE_MEM)); 4538 err_pci_reg: 4539 err_dma: 4540 pci_disable_device(pdev); 4541 return err; 4542 } 4543 4544 /** 4545 * igc_remove - Device Removal Routine 4546 * @pdev: PCI device information struct 4547 * 4548 * igc_remove is called by the PCI subsystem to alert the driver 4549 * that it should release a PCI device. This could be caused by a 4550 * Hot-Plug event, or because the driver is going to be removed from 4551 * memory. 4552 */ 4553 static void igc_remove(struct pci_dev *pdev) 4554 { 4555 struct net_device *netdev = pci_get_drvdata(pdev); 4556 struct igc_adapter *adapter = netdev_priv(netdev); 4557 4558 set_bit(__IGC_DOWN, &adapter->state); 4559 4560 del_timer_sync(&adapter->watchdog_timer); 4561 del_timer_sync(&adapter->phy_info_timer); 4562 4563 cancel_work_sync(&adapter->reset_task); 4564 cancel_work_sync(&adapter->watchdog_task); 4565 4566 /* Release control of h/w to f/w. If f/w is AMT enabled, this 4567 * would have already happened in close and is redundant. 4568 */ 4569 igc_release_hw_control(adapter); 4570 unregister_netdev(netdev); 4571 4572 igc_clear_interrupt_scheme(adapter); 4573 pci_iounmap(pdev, adapter->io_addr); 4574 pci_release_mem_regions(pdev); 4575 4576 kfree(adapter->mac_table); 4577 free_netdev(netdev); 4578 4579 pci_disable_pcie_error_reporting(pdev); 4580 4581 pci_disable_device(pdev); 4582 } 4583 4584 static struct pci_driver igc_driver = { 4585 .name = igc_driver_name, 4586 .id_table = igc_pci_tbl, 4587 .probe = igc_probe, 4588 .remove = igc_remove, 4589 }; 4590 4591 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 4592 const u32 max_rss_queues) 4593 { 4594 /* Determine if we need to pair queues. */ 4595 /* If rss_queues > half of max_rss_queues, pair the queues in 4596 * order to conserve interrupts due to limited supply. 4597 */ 4598 if (adapter->rss_queues > (max_rss_queues / 2)) 4599 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4600 else 4601 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 4602 } 4603 4604 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 4605 { 4606 unsigned int max_rss_queues; 4607 4608 /* Determine the maximum number of RSS queues supported. */ 4609 max_rss_queues = IGC_MAX_RX_QUEUES; 4610 4611 return max_rss_queues; 4612 } 4613 4614 static void igc_init_queue_configuration(struct igc_adapter *adapter) 4615 { 4616 u32 max_rss_queues; 4617 4618 max_rss_queues = igc_get_max_rss_queues(adapter); 4619 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 4620 4621 igc_set_flag_queue_pairs(adapter, max_rss_queues); 4622 } 4623 4624 /** 4625 * igc_sw_init - Initialize general software structures (struct igc_adapter) 4626 * @adapter: board private structure to initialize 4627 * 4628 * igc_sw_init initializes the Adapter private data structure. 4629 * Fields are initialized based on PCI device information and 4630 * OS network device settings (MTU size). 4631 */ 4632 static int igc_sw_init(struct igc_adapter *adapter) 4633 { 4634 struct net_device *netdev = adapter->netdev; 4635 struct pci_dev *pdev = adapter->pdev; 4636 struct igc_hw *hw = &adapter->hw; 4637 4638 int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; 4639 4640 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 4641 4642 /* set default ring sizes */ 4643 adapter->tx_ring_count = IGC_DEFAULT_TXD; 4644 adapter->rx_ring_count = IGC_DEFAULT_RXD; 4645 4646 /* set default ITR values */ 4647 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 4648 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 4649 4650 /* set default work limits */ 4651 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 4652 4653 /* adjust max frame to be at least the size of a standard frame */ 4654 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 4655 VLAN_HLEN; 4656 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 4657 4658 spin_lock_init(&adapter->nfc_lock); 4659 spin_lock_init(&adapter->stats64_lock); 4660 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 4661 adapter->flags |= IGC_FLAG_HAS_MSIX; 4662 4663 adapter->mac_table = kzalloc(size, GFP_ATOMIC); 4664 if (!adapter->mac_table) 4665 return -ENOMEM; 4666 4667 igc_init_queue_configuration(adapter); 4668 4669 /* This call may decrease the number of queues */ 4670 if (igc_init_interrupt_scheme(adapter, true)) { 4671 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4672 return -ENOMEM; 4673 } 4674 4675 /* Explicitly disable IRQ since the NIC can be in any state. */ 4676 igc_irq_disable(adapter); 4677 4678 set_bit(__IGC_DOWN, &adapter->state); 4679 4680 return 0; 4681 } 4682 4683 /** 4684 * igc_reinit_queues - return error 4685 * @adapter: pointer to adapter structure 4686 */ 4687 int igc_reinit_queues(struct igc_adapter *adapter) 4688 { 4689 struct net_device *netdev = adapter->netdev; 4690 struct pci_dev *pdev = adapter->pdev; 4691 int err = 0; 4692 4693 if (netif_running(netdev)) 4694 igc_close(netdev); 4695 4696 igc_reset_interrupt_capability(adapter); 4697 4698 if (igc_init_interrupt_scheme(adapter, true)) { 4699 dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); 4700 return -ENOMEM; 4701 } 4702 4703 if (netif_running(netdev)) 4704 err = igc_open(netdev); 4705 4706 return err; 4707 } 4708 4709 /** 4710 * igc_get_hw_dev - return device 4711 * @hw: pointer to hardware structure 4712 * 4713 * used by hardware layer to print debugging information 4714 */ 4715 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 4716 { 4717 struct igc_adapter *adapter = hw->back; 4718 4719 return adapter->netdev; 4720 } 4721 4722 /** 4723 * igc_init_module - Driver Registration Routine 4724 * 4725 * igc_init_module is the first routine called when the driver is 4726 * loaded. All it does is register with the PCI subsystem. 4727 */ 4728 static int __init igc_init_module(void) 4729 { 4730 int ret; 4731 4732 pr_info("%s - version %s\n", 4733 igc_driver_string, igc_driver_version); 4734 4735 pr_info("%s\n", igc_copyright); 4736 4737 ret = pci_register_driver(&igc_driver); 4738 return ret; 4739 } 4740 4741 module_init(igc_init_module); 4742 4743 /** 4744 * igc_exit_module - Driver Exit Cleanup Routine 4745 * 4746 * igc_exit_module is called just before the driver is removed 4747 * from memory. 4748 */ 4749 static void __exit igc_exit_module(void) 4750 { 4751 pci_unregister_driver(&igc_driver); 4752 } 4753 4754 module_exit(igc_exit_module); 4755 /* igc_main.c */ 4756