1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/aer.h> 8 #include <linux/tcp.h> 9 #include <linux/udp.h> 10 #include <linux/ip.h> 11 #include <linux/pm_runtime.h> 12 #include <net/pkt_sched.h> 13 14 #include <net/ipv6.h> 15 16 #include "igc.h" 17 #include "igc_hw.h" 18 #include "igc_tsn.h" 19 20 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 21 22 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 23 24 static int debug = -1; 25 26 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 27 MODULE_DESCRIPTION(DRV_SUMMARY); 28 MODULE_LICENSE("GPL v2"); 29 module_param(debug, int, 0); 30 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 31 32 char igc_driver_name[] = "igc"; 33 static const char igc_driver_string[] = DRV_SUMMARY; 34 static const char igc_copyright[] = 35 "Copyright(c) 2018 Intel Corporation."; 36 37 static const struct igc_info *igc_info_tbl[] = { 38 [board_base] = &igc_base_info, 39 }; 40 41 static const struct pci_device_id igc_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, 43 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, 44 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, 45 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, 46 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, 47 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, 48 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, 49 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, 51 /* required last entry */ 52 {0, } 53 }; 54 55 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 56 57 enum latency_range { 58 lowest_latency = 0, 59 low_latency = 1, 60 bulk_latency = 2, 61 latency_invalid = 255 62 }; 63 64 void igc_reset(struct igc_adapter *adapter) 65 { 66 struct net_device *dev = adapter->netdev; 67 struct igc_hw *hw = &adapter->hw; 68 struct igc_fc_info *fc = &hw->fc; 69 u32 pba, hwm; 70 71 /* Repartition PBA for greater than 9k MTU if required */ 72 pba = IGC_PBA_34K; 73 74 /* flow control settings 75 * The high water mark must be low enough to fit one full frame 76 * after transmitting the pause frame. As such we must have enough 77 * space to allow for us to complete our current transmit and then 78 * receive the frame that is in progress from the link partner. 79 * Set it to: 80 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 81 */ 82 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 83 84 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 85 fc->low_water = fc->high_water - 16; 86 fc->pause_time = 0xFFFF; 87 fc->send_xon = 1; 88 fc->current_mode = fc->requested_mode; 89 90 hw->mac.ops.reset_hw(hw); 91 92 if (hw->mac.ops.init_hw(hw)) 93 netdev_err(dev, "Error on hardware initialization\n"); 94 95 /* Re-establish EEE setting */ 96 igc_set_eee_i225(hw, true, true, true); 97 98 if (!netif_running(adapter->netdev)) 99 igc_power_down_phy_copper_base(&adapter->hw); 100 101 /* Re-enable PTP, where applicable. */ 102 igc_ptp_reset(adapter); 103 104 /* Re-enable TSN offloading, where applicable. */ 105 igc_tsn_offload_apply(adapter); 106 107 igc_get_phy_info(hw); 108 } 109 110 /** 111 * igc_power_up_link - Power up the phy link 112 * @adapter: address of board private structure 113 */ 114 static void igc_power_up_link(struct igc_adapter *adapter) 115 { 116 igc_reset_phy(&adapter->hw); 117 118 igc_power_up_phy_copper(&adapter->hw); 119 120 igc_setup_link(&adapter->hw); 121 } 122 123 /** 124 * igc_release_hw_control - release control of the h/w to f/w 125 * @adapter: address of board private structure 126 * 127 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 128 * For ASF and Pass Through versions of f/w this means that the 129 * driver is no longer loaded. 130 */ 131 static void igc_release_hw_control(struct igc_adapter *adapter) 132 { 133 struct igc_hw *hw = &adapter->hw; 134 u32 ctrl_ext; 135 136 /* Let firmware take over control of h/w */ 137 ctrl_ext = rd32(IGC_CTRL_EXT); 138 wr32(IGC_CTRL_EXT, 139 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 140 } 141 142 /** 143 * igc_get_hw_control - get control of the h/w from f/w 144 * @adapter: address of board private structure 145 * 146 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 147 * For ASF and Pass Through versions of f/w this means that 148 * the driver is loaded. 149 */ 150 static void igc_get_hw_control(struct igc_adapter *adapter) 151 { 152 struct igc_hw *hw = &adapter->hw; 153 u32 ctrl_ext; 154 155 /* Let firmware know the driver has taken over */ 156 ctrl_ext = rd32(IGC_CTRL_EXT); 157 wr32(IGC_CTRL_EXT, 158 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 159 } 160 161 /** 162 * igc_clean_tx_ring - Free Tx Buffers 163 * @tx_ring: ring to be cleaned 164 */ 165 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 166 { 167 u16 i = tx_ring->next_to_clean; 168 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 169 170 while (i != tx_ring->next_to_use) { 171 union igc_adv_tx_desc *eop_desc, *tx_desc; 172 173 /* Free all the Tx ring sk_buffs */ 174 dev_kfree_skb_any(tx_buffer->skb); 175 176 /* unmap skb header data */ 177 dma_unmap_single(tx_ring->dev, 178 dma_unmap_addr(tx_buffer, dma), 179 dma_unmap_len(tx_buffer, len), 180 DMA_TO_DEVICE); 181 182 /* check for eop_desc to determine the end of the packet */ 183 eop_desc = tx_buffer->next_to_watch; 184 tx_desc = IGC_TX_DESC(tx_ring, i); 185 186 /* unmap remaining buffers */ 187 while (tx_desc != eop_desc) { 188 tx_buffer++; 189 tx_desc++; 190 i++; 191 if (unlikely(i == tx_ring->count)) { 192 i = 0; 193 tx_buffer = tx_ring->tx_buffer_info; 194 tx_desc = IGC_TX_DESC(tx_ring, 0); 195 } 196 197 /* unmap any remaining paged data */ 198 if (dma_unmap_len(tx_buffer, len)) 199 dma_unmap_page(tx_ring->dev, 200 dma_unmap_addr(tx_buffer, dma), 201 dma_unmap_len(tx_buffer, len), 202 DMA_TO_DEVICE); 203 } 204 205 /* move us one more past the eop_desc for start of next pkt */ 206 tx_buffer++; 207 i++; 208 if (unlikely(i == tx_ring->count)) { 209 i = 0; 210 tx_buffer = tx_ring->tx_buffer_info; 211 } 212 } 213 214 /* reset BQL for queue */ 215 netdev_tx_reset_queue(txring_txq(tx_ring)); 216 217 /* reset next_to_use and next_to_clean */ 218 tx_ring->next_to_use = 0; 219 tx_ring->next_to_clean = 0; 220 } 221 222 /** 223 * igc_free_tx_resources - Free Tx Resources per Queue 224 * @tx_ring: Tx descriptor ring for a specific queue 225 * 226 * Free all transmit software resources 227 */ 228 void igc_free_tx_resources(struct igc_ring *tx_ring) 229 { 230 igc_clean_tx_ring(tx_ring); 231 232 vfree(tx_ring->tx_buffer_info); 233 tx_ring->tx_buffer_info = NULL; 234 235 /* if not set, then don't free */ 236 if (!tx_ring->desc) 237 return; 238 239 dma_free_coherent(tx_ring->dev, tx_ring->size, 240 tx_ring->desc, tx_ring->dma); 241 242 tx_ring->desc = NULL; 243 } 244 245 /** 246 * igc_free_all_tx_resources - Free Tx Resources for All Queues 247 * @adapter: board private structure 248 * 249 * Free all transmit software resources 250 */ 251 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 252 { 253 int i; 254 255 for (i = 0; i < adapter->num_tx_queues; i++) 256 igc_free_tx_resources(adapter->tx_ring[i]); 257 } 258 259 /** 260 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 261 * @adapter: board private structure 262 */ 263 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 264 { 265 int i; 266 267 for (i = 0; i < adapter->num_tx_queues; i++) 268 if (adapter->tx_ring[i]) 269 igc_clean_tx_ring(adapter->tx_ring[i]); 270 } 271 272 /** 273 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 274 * @tx_ring: tx descriptor ring (for a specific queue) to setup 275 * 276 * Return 0 on success, negative on failure 277 */ 278 int igc_setup_tx_resources(struct igc_ring *tx_ring) 279 { 280 struct net_device *ndev = tx_ring->netdev; 281 struct device *dev = tx_ring->dev; 282 int size = 0; 283 284 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 285 tx_ring->tx_buffer_info = vzalloc(size); 286 if (!tx_ring->tx_buffer_info) 287 goto err; 288 289 /* round up to nearest 4K */ 290 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 291 tx_ring->size = ALIGN(tx_ring->size, 4096); 292 293 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 294 &tx_ring->dma, GFP_KERNEL); 295 296 if (!tx_ring->desc) 297 goto err; 298 299 tx_ring->next_to_use = 0; 300 tx_ring->next_to_clean = 0; 301 302 return 0; 303 304 err: 305 vfree(tx_ring->tx_buffer_info); 306 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 307 return -ENOMEM; 308 } 309 310 /** 311 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 312 * @adapter: board private structure 313 * 314 * Return 0 on success, negative on failure 315 */ 316 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 317 { 318 struct net_device *dev = adapter->netdev; 319 int i, err = 0; 320 321 for (i = 0; i < adapter->num_tx_queues; i++) { 322 err = igc_setup_tx_resources(adapter->tx_ring[i]); 323 if (err) { 324 netdev_err(dev, "Error on Tx queue %u setup\n", i); 325 for (i--; i >= 0; i--) 326 igc_free_tx_resources(adapter->tx_ring[i]); 327 break; 328 } 329 } 330 331 return err; 332 } 333 334 /** 335 * igc_clean_rx_ring - Free Rx Buffers per Queue 336 * @rx_ring: ring to free buffers from 337 */ 338 static void igc_clean_rx_ring(struct igc_ring *rx_ring) 339 { 340 u16 i = rx_ring->next_to_clean; 341 342 dev_kfree_skb(rx_ring->skb); 343 rx_ring->skb = NULL; 344 345 /* Free all the Rx ring sk_buffs */ 346 while (i != rx_ring->next_to_alloc) { 347 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 348 349 /* Invalidate cache lines that may have been written to by 350 * device so that we avoid corrupting memory. 351 */ 352 dma_sync_single_range_for_cpu(rx_ring->dev, 353 buffer_info->dma, 354 buffer_info->page_offset, 355 igc_rx_bufsz(rx_ring), 356 DMA_FROM_DEVICE); 357 358 /* free resources associated with mapping */ 359 dma_unmap_page_attrs(rx_ring->dev, 360 buffer_info->dma, 361 igc_rx_pg_size(rx_ring), 362 DMA_FROM_DEVICE, 363 IGC_RX_DMA_ATTR); 364 __page_frag_cache_drain(buffer_info->page, 365 buffer_info->pagecnt_bias); 366 367 i++; 368 if (i == rx_ring->count) 369 i = 0; 370 } 371 372 rx_ring->next_to_alloc = 0; 373 rx_ring->next_to_clean = 0; 374 rx_ring->next_to_use = 0; 375 } 376 377 /** 378 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 379 * @adapter: board private structure 380 */ 381 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 382 { 383 int i; 384 385 for (i = 0; i < adapter->num_rx_queues; i++) 386 if (adapter->rx_ring[i]) 387 igc_clean_rx_ring(adapter->rx_ring[i]); 388 } 389 390 /** 391 * igc_free_rx_resources - Free Rx Resources 392 * @rx_ring: ring to clean the resources from 393 * 394 * Free all receive software resources 395 */ 396 void igc_free_rx_resources(struct igc_ring *rx_ring) 397 { 398 igc_clean_rx_ring(rx_ring); 399 400 vfree(rx_ring->rx_buffer_info); 401 rx_ring->rx_buffer_info = NULL; 402 403 /* if not set, then don't free */ 404 if (!rx_ring->desc) 405 return; 406 407 dma_free_coherent(rx_ring->dev, rx_ring->size, 408 rx_ring->desc, rx_ring->dma); 409 410 rx_ring->desc = NULL; 411 } 412 413 /** 414 * igc_free_all_rx_resources - Free Rx Resources for All Queues 415 * @adapter: board private structure 416 * 417 * Free all receive software resources 418 */ 419 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 420 { 421 int i; 422 423 for (i = 0; i < adapter->num_rx_queues; i++) 424 igc_free_rx_resources(adapter->rx_ring[i]); 425 } 426 427 /** 428 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 429 * @rx_ring: rx descriptor ring (for a specific queue) to setup 430 * 431 * Returns 0 on success, negative on failure 432 */ 433 int igc_setup_rx_resources(struct igc_ring *rx_ring) 434 { 435 struct net_device *ndev = rx_ring->netdev; 436 struct device *dev = rx_ring->dev; 437 int size, desc_len; 438 439 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 440 rx_ring->rx_buffer_info = vzalloc(size); 441 if (!rx_ring->rx_buffer_info) 442 goto err; 443 444 desc_len = sizeof(union igc_adv_rx_desc); 445 446 /* Round up to nearest 4K */ 447 rx_ring->size = rx_ring->count * desc_len; 448 rx_ring->size = ALIGN(rx_ring->size, 4096); 449 450 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 451 &rx_ring->dma, GFP_KERNEL); 452 453 if (!rx_ring->desc) 454 goto err; 455 456 rx_ring->next_to_alloc = 0; 457 rx_ring->next_to_clean = 0; 458 rx_ring->next_to_use = 0; 459 460 return 0; 461 462 err: 463 vfree(rx_ring->rx_buffer_info); 464 rx_ring->rx_buffer_info = NULL; 465 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 466 return -ENOMEM; 467 } 468 469 /** 470 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 471 * (Descriptors) for all queues 472 * @adapter: board private structure 473 * 474 * Return 0 on success, negative on failure 475 */ 476 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 477 { 478 struct net_device *dev = adapter->netdev; 479 int i, err = 0; 480 481 for (i = 0; i < adapter->num_rx_queues; i++) { 482 err = igc_setup_rx_resources(adapter->rx_ring[i]); 483 if (err) { 484 netdev_err(dev, "Error on Rx queue %u setup\n", i); 485 for (i--; i >= 0; i--) 486 igc_free_rx_resources(adapter->rx_ring[i]); 487 break; 488 } 489 } 490 491 return err; 492 } 493 494 /** 495 * igc_configure_rx_ring - Configure a receive ring after Reset 496 * @adapter: board private structure 497 * @ring: receive ring to be configured 498 * 499 * Configure the Rx unit of the MAC after a reset. 500 */ 501 static void igc_configure_rx_ring(struct igc_adapter *adapter, 502 struct igc_ring *ring) 503 { 504 struct igc_hw *hw = &adapter->hw; 505 union igc_adv_rx_desc *rx_desc; 506 int reg_idx = ring->reg_idx; 507 u32 srrctl = 0, rxdctl = 0; 508 u64 rdba = ring->dma; 509 510 /* disable the queue */ 511 wr32(IGC_RXDCTL(reg_idx), 0); 512 513 /* Set DMA base address registers */ 514 wr32(IGC_RDBAL(reg_idx), 515 rdba & 0x00000000ffffffffULL); 516 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 517 wr32(IGC_RDLEN(reg_idx), 518 ring->count * sizeof(union igc_adv_rx_desc)); 519 520 /* initialize head and tail */ 521 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 522 wr32(IGC_RDH(reg_idx), 0); 523 writel(0, ring->tail); 524 525 /* reset next-to- use/clean to place SW in sync with hardware */ 526 ring->next_to_clean = 0; 527 ring->next_to_use = 0; 528 529 /* set descriptor configuration */ 530 srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; 531 if (ring_uses_large_buffer(ring)) 532 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 533 else 534 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 535 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 536 537 wr32(IGC_SRRCTL(reg_idx), srrctl); 538 539 rxdctl |= IGC_RX_PTHRESH; 540 rxdctl |= IGC_RX_HTHRESH << 8; 541 rxdctl |= IGC_RX_WTHRESH << 16; 542 543 /* initialize rx_buffer_info */ 544 memset(ring->rx_buffer_info, 0, 545 sizeof(struct igc_rx_buffer) * ring->count); 546 547 /* initialize Rx descriptor 0 */ 548 rx_desc = IGC_RX_DESC(ring, 0); 549 rx_desc->wb.upper.length = 0; 550 551 /* enable receive descriptor fetching */ 552 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 553 554 wr32(IGC_RXDCTL(reg_idx), rxdctl); 555 } 556 557 /** 558 * igc_configure_rx - Configure receive Unit after Reset 559 * @adapter: board private structure 560 * 561 * Configure the Rx unit of the MAC after a reset. 562 */ 563 static void igc_configure_rx(struct igc_adapter *adapter) 564 { 565 int i; 566 567 /* Setup the HW Rx Head and Tail Descriptor Pointers and 568 * the Base and Length of the Rx Descriptor Ring 569 */ 570 for (i = 0; i < adapter->num_rx_queues; i++) 571 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 572 } 573 574 /** 575 * igc_configure_tx_ring - Configure transmit ring after Reset 576 * @adapter: board private structure 577 * @ring: tx ring to configure 578 * 579 * Configure a transmit ring after a reset. 580 */ 581 static void igc_configure_tx_ring(struct igc_adapter *adapter, 582 struct igc_ring *ring) 583 { 584 struct igc_hw *hw = &adapter->hw; 585 int reg_idx = ring->reg_idx; 586 u64 tdba = ring->dma; 587 u32 txdctl = 0; 588 589 /* disable the queue */ 590 wr32(IGC_TXDCTL(reg_idx), 0); 591 wrfl(); 592 mdelay(10); 593 594 wr32(IGC_TDLEN(reg_idx), 595 ring->count * sizeof(union igc_adv_tx_desc)); 596 wr32(IGC_TDBAL(reg_idx), 597 tdba & 0x00000000ffffffffULL); 598 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 599 600 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 601 wr32(IGC_TDH(reg_idx), 0); 602 writel(0, ring->tail); 603 604 txdctl |= IGC_TX_PTHRESH; 605 txdctl |= IGC_TX_HTHRESH << 8; 606 txdctl |= IGC_TX_WTHRESH << 16; 607 608 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 609 wr32(IGC_TXDCTL(reg_idx), txdctl); 610 } 611 612 /** 613 * igc_configure_tx - Configure transmit Unit after Reset 614 * @adapter: board private structure 615 * 616 * Configure the Tx unit of the MAC after a reset. 617 */ 618 static void igc_configure_tx(struct igc_adapter *adapter) 619 { 620 int i; 621 622 for (i = 0; i < adapter->num_tx_queues; i++) 623 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 624 } 625 626 /** 627 * igc_setup_mrqc - configure the multiple receive queue control registers 628 * @adapter: Board private structure 629 */ 630 static void igc_setup_mrqc(struct igc_adapter *adapter) 631 { 632 struct igc_hw *hw = &adapter->hw; 633 u32 j, num_rx_queues; 634 u32 mrqc, rxcsum; 635 u32 rss_key[10]; 636 637 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 638 for (j = 0; j < 10; j++) 639 wr32(IGC_RSSRK(j), rss_key[j]); 640 641 num_rx_queues = adapter->rss_queues; 642 643 if (adapter->rss_indir_tbl_init != num_rx_queues) { 644 for (j = 0; j < IGC_RETA_SIZE; j++) 645 adapter->rss_indir_tbl[j] = 646 (j * num_rx_queues) / IGC_RETA_SIZE; 647 adapter->rss_indir_tbl_init = num_rx_queues; 648 } 649 igc_write_rss_indir_tbl(adapter); 650 651 /* Disable raw packet checksumming so that RSS hash is placed in 652 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 653 * offloads as they are enabled by default 654 */ 655 rxcsum = rd32(IGC_RXCSUM); 656 rxcsum |= IGC_RXCSUM_PCSD; 657 658 /* Enable Receive Checksum Offload for SCTP */ 659 rxcsum |= IGC_RXCSUM_CRCOFL; 660 661 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 662 wr32(IGC_RXCSUM, rxcsum); 663 664 /* Generate RSS hash based on packet types, TCP/UDP 665 * port numbers and/or IPv4/v6 src and dst addresses 666 */ 667 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 668 IGC_MRQC_RSS_FIELD_IPV4_TCP | 669 IGC_MRQC_RSS_FIELD_IPV6 | 670 IGC_MRQC_RSS_FIELD_IPV6_TCP | 671 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 672 673 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 674 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 675 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 676 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 677 678 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 679 680 wr32(IGC_MRQC, mrqc); 681 } 682 683 /** 684 * igc_setup_rctl - configure the receive control registers 685 * @adapter: Board private structure 686 */ 687 static void igc_setup_rctl(struct igc_adapter *adapter) 688 { 689 struct igc_hw *hw = &adapter->hw; 690 u32 rctl; 691 692 rctl = rd32(IGC_RCTL); 693 694 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 695 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 696 697 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 698 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 699 700 /* enable stripping of CRC. Newer features require 701 * that the HW strips the CRC. 702 */ 703 rctl |= IGC_RCTL_SECRC; 704 705 /* disable store bad packets and clear size bits. */ 706 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 707 708 /* enable LPE to allow for reception of jumbo frames */ 709 rctl |= IGC_RCTL_LPE; 710 711 /* disable queue 0 to prevent tail write w/o re-config */ 712 wr32(IGC_RXDCTL(0), 0); 713 714 /* This is useful for sniffing bad packets. */ 715 if (adapter->netdev->features & NETIF_F_RXALL) { 716 /* UPE and MPE will be handled by normal PROMISC logic 717 * in set_rx_mode 718 */ 719 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 720 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 721 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 722 723 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 724 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 725 } 726 727 wr32(IGC_RCTL, rctl); 728 } 729 730 /** 731 * igc_setup_tctl - configure the transmit control registers 732 * @adapter: Board private structure 733 */ 734 static void igc_setup_tctl(struct igc_adapter *adapter) 735 { 736 struct igc_hw *hw = &adapter->hw; 737 u32 tctl; 738 739 /* disable queue 0 which icould be enabled by default */ 740 wr32(IGC_TXDCTL(0), 0); 741 742 /* Program the Transmit Control Register */ 743 tctl = rd32(IGC_TCTL); 744 tctl &= ~IGC_TCTL_CT; 745 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 746 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 747 748 /* Enable transmits */ 749 tctl |= IGC_TCTL_EN; 750 751 wr32(IGC_TCTL, tctl); 752 } 753 754 /** 755 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 756 * @adapter: Pointer to adapter where the filter should be set 757 * @index: Filter index 758 * @type: MAC address filter type (source or destination) 759 * @addr: MAC address 760 * @queue: If non-negative, queue assignment feature is enabled and frames 761 * matching the filter are enqueued onto 'queue'. Otherwise, queue 762 * assignment is disabled. 763 */ 764 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 765 enum igc_mac_filter_type type, 766 const u8 *addr, int queue) 767 { 768 struct net_device *dev = adapter->netdev; 769 struct igc_hw *hw = &adapter->hw; 770 u32 ral, rah; 771 772 if (WARN_ON(index >= hw->mac.rar_entry_count)) 773 return; 774 775 ral = le32_to_cpup((__le32 *)(addr)); 776 rah = le16_to_cpup((__le16 *)(addr + 4)); 777 778 if (type == IGC_MAC_FILTER_TYPE_SRC) { 779 rah &= ~IGC_RAH_ASEL_MASK; 780 rah |= IGC_RAH_ASEL_SRC_ADDR; 781 } 782 783 if (queue >= 0) { 784 rah &= ~IGC_RAH_QSEL_MASK; 785 rah |= (queue << IGC_RAH_QSEL_SHIFT); 786 rah |= IGC_RAH_QSEL_ENABLE; 787 } 788 789 rah |= IGC_RAH_AV; 790 791 wr32(IGC_RAL(index), ral); 792 wr32(IGC_RAH(index), rah); 793 794 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 795 } 796 797 /** 798 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 799 * @adapter: Pointer to adapter where the filter should be cleared 800 * @index: Filter index 801 */ 802 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 803 { 804 struct net_device *dev = adapter->netdev; 805 struct igc_hw *hw = &adapter->hw; 806 807 if (WARN_ON(index >= hw->mac.rar_entry_count)) 808 return; 809 810 wr32(IGC_RAL(index), 0); 811 wr32(IGC_RAH(index), 0); 812 813 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 814 } 815 816 /* Set default MAC address for the PF in the first RAR entry */ 817 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 818 { 819 struct net_device *dev = adapter->netdev; 820 u8 *addr = adapter->hw.mac.addr; 821 822 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 823 824 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 825 } 826 827 /** 828 * igc_set_mac - Change the Ethernet Address of the NIC 829 * @netdev: network interface device structure 830 * @p: pointer to an address structure 831 * 832 * Returns 0 on success, negative on failure 833 */ 834 static int igc_set_mac(struct net_device *netdev, void *p) 835 { 836 struct igc_adapter *adapter = netdev_priv(netdev); 837 struct igc_hw *hw = &adapter->hw; 838 struct sockaddr *addr = p; 839 840 if (!is_valid_ether_addr(addr->sa_data)) 841 return -EADDRNOTAVAIL; 842 843 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 844 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 845 846 /* set the correct pool for the new PF MAC address in entry 0 */ 847 igc_set_default_mac_filter(adapter); 848 849 return 0; 850 } 851 852 /** 853 * igc_write_mc_addr_list - write multicast addresses to MTA 854 * @netdev: network interface device structure 855 * 856 * Writes multicast address list to the MTA hash table. 857 * Returns: -ENOMEM on failure 858 * 0 on no addresses written 859 * X on writing X addresses to MTA 860 **/ 861 static int igc_write_mc_addr_list(struct net_device *netdev) 862 { 863 struct igc_adapter *adapter = netdev_priv(netdev); 864 struct igc_hw *hw = &adapter->hw; 865 struct netdev_hw_addr *ha; 866 u8 *mta_list; 867 int i; 868 869 if (netdev_mc_empty(netdev)) { 870 /* nothing to program, so clear mc list */ 871 igc_update_mc_addr_list(hw, NULL, 0); 872 return 0; 873 } 874 875 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 876 if (!mta_list) 877 return -ENOMEM; 878 879 /* The shared function expects a packed array of only addresses. */ 880 i = 0; 881 netdev_for_each_mc_addr(ha, netdev) 882 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 883 884 igc_update_mc_addr_list(hw, mta_list, i); 885 kfree(mta_list); 886 887 return netdev_mc_count(netdev); 888 } 889 890 static __le32 igc_tx_launchtime(struct igc_adapter *adapter, ktime_t txtime) 891 { 892 ktime_t cycle_time = adapter->cycle_time; 893 ktime_t base_time = adapter->base_time; 894 u32 launchtime; 895 896 /* FIXME: when using ETF together with taprio, we may have a 897 * case where 'delta' is larger than the cycle_time, this may 898 * cause problems if we don't read the current value of 899 * IGC_BASET, as the value writen into the launchtime 900 * descriptor field may be misinterpreted. 901 */ 902 div_s64_rem(ktime_sub_ns(txtime, base_time), cycle_time, &launchtime); 903 904 return cpu_to_le32(launchtime); 905 } 906 907 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 908 struct igc_tx_buffer *first, 909 u32 vlan_macip_lens, u32 type_tucmd, 910 u32 mss_l4len_idx) 911 { 912 struct igc_adv_tx_context_desc *context_desc; 913 u16 i = tx_ring->next_to_use; 914 915 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 916 917 i++; 918 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 919 920 /* set bits to identify this as an advanced context descriptor */ 921 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 922 923 /* For i225, context index must be unique per ring. */ 924 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 925 mss_l4len_idx |= tx_ring->reg_idx << 4; 926 927 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 928 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 929 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 930 931 /* We assume there is always a valid Tx time available. Invalid times 932 * should have been handled by the upper layers. 933 */ 934 if (tx_ring->launchtime_enable) { 935 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 936 ktime_t txtime = first->skb->tstamp; 937 938 first->skb->tstamp = ktime_set(0, 0); 939 context_desc->launch_time = igc_tx_launchtime(adapter, 940 txtime); 941 } else { 942 context_desc->launch_time = 0; 943 } 944 } 945 946 static inline bool igc_ipv6_csum_is_sctp(struct sk_buff *skb) 947 { 948 unsigned int offset = 0; 949 950 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 951 952 return offset == skb_checksum_start_offset(skb); 953 } 954 955 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first) 956 { 957 struct sk_buff *skb = first->skb; 958 u32 vlan_macip_lens = 0; 959 u32 type_tucmd = 0; 960 961 if (skb->ip_summed != CHECKSUM_PARTIAL) { 962 csum_failed: 963 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 964 !tx_ring->launchtime_enable) 965 return; 966 goto no_csum; 967 } 968 969 switch (skb->csum_offset) { 970 case offsetof(struct tcphdr, check): 971 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 972 /* fall through */ 973 case offsetof(struct udphdr, check): 974 break; 975 case offsetof(struct sctphdr, checksum): 976 /* validate that this is actually an SCTP request */ 977 if ((first->protocol == htons(ETH_P_IP) && 978 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 979 (first->protocol == htons(ETH_P_IPV6) && 980 igc_ipv6_csum_is_sctp(skb))) { 981 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 982 break; 983 } 984 /* fall through */ 985 default: 986 skb_checksum_help(skb); 987 goto csum_failed; 988 } 989 990 /* update TX checksum flag */ 991 first->tx_flags |= IGC_TX_FLAGS_CSUM; 992 vlan_macip_lens = skb_checksum_start_offset(skb) - 993 skb_network_offset(skb); 994 no_csum: 995 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 996 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 997 998 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0); 999 } 1000 1001 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1002 { 1003 struct net_device *netdev = tx_ring->netdev; 1004 1005 netif_stop_subqueue(netdev, tx_ring->queue_index); 1006 1007 /* memory barriier comment */ 1008 smp_mb(); 1009 1010 /* We need to check again in a case another CPU has just 1011 * made room available. 1012 */ 1013 if (igc_desc_unused(tx_ring) < size) 1014 return -EBUSY; 1015 1016 /* A reprieve! */ 1017 netif_wake_subqueue(netdev, tx_ring->queue_index); 1018 1019 u64_stats_update_begin(&tx_ring->tx_syncp2); 1020 tx_ring->tx_stats.restart_queue2++; 1021 u64_stats_update_end(&tx_ring->tx_syncp2); 1022 1023 return 0; 1024 } 1025 1026 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1027 { 1028 if (igc_desc_unused(tx_ring) >= size) 1029 return 0; 1030 return __igc_maybe_stop_tx(tx_ring, size); 1031 } 1032 1033 #define IGC_SET_FLAG(_input, _flag, _result) \ 1034 (((_flag) <= (_result)) ? \ 1035 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1036 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1037 1038 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1039 { 1040 /* set type for advanced descriptor with frame checksum insertion */ 1041 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1042 IGC_ADVTXD_DCMD_DEXT | 1043 IGC_ADVTXD_DCMD_IFCS; 1044 1045 /* set segmentation bits for TSO */ 1046 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1047 (IGC_ADVTXD_DCMD_TSE)); 1048 1049 /* set timestamp bit if present */ 1050 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1051 (IGC_ADVTXD_MAC_TSTAMP)); 1052 1053 return cmd_type; 1054 } 1055 1056 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1057 union igc_adv_tx_desc *tx_desc, 1058 u32 tx_flags, unsigned int paylen) 1059 { 1060 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1061 1062 /* insert L4 checksum */ 1063 olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * 1064 ((IGC_TXD_POPTS_TXSM << 8) / 1065 IGC_TX_FLAGS_CSUM); 1066 1067 /* insert IPv4 checksum */ 1068 olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * 1069 (((IGC_TXD_POPTS_IXSM << 8)) / 1070 IGC_TX_FLAGS_IPV4); 1071 1072 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1073 } 1074 1075 static int igc_tx_map(struct igc_ring *tx_ring, 1076 struct igc_tx_buffer *first, 1077 const u8 hdr_len) 1078 { 1079 struct sk_buff *skb = first->skb; 1080 struct igc_tx_buffer *tx_buffer; 1081 union igc_adv_tx_desc *tx_desc; 1082 u32 tx_flags = first->tx_flags; 1083 skb_frag_t *frag; 1084 u16 i = tx_ring->next_to_use; 1085 unsigned int data_len, size; 1086 dma_addr_t dma; 1087 u32 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1088 1089 tx_desc = IGC_TX_DESC(tx_ring, i); 1090 1091 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1092 1093 size = skb_headlen(skb); 1094 data_len = skb->data_len; 1095 1096 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1097 1098 tx_buffer = first; 1099 1100 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1101 if (dma_mapping_error(tx_ring->dev, dma)) 1102 goto dma_error; 1103 1104 /* record length, and DMA address */ 1105 dma_unmap_len_set(tx_buffer, len, size); 1106 dma_unmap_addr_set(tx_buffer, dma, dma); 1107 1108 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1109 1110 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1111 tx_desc->read.cmd_type_len = 1112 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1113 1114 i++; 1115 tx_desc++; 1116 if (i == tx_ring->count) { 1117 tx_desc = IGC_TX_DESC(tx_ring, 0); 1118 i = 0; 1119 } 1120 tx_desc->read.olinfo_status = 0; 1121 1122 dma += IGC_MAX_DATA_PER_TXD; 1123 size -= IGC_MAX_DATA_PER_TXD; 1124 1125 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1126 } 1127 1128 if (likely(!data_len)) 1129 break; 1130 1131 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1132 1133 i++; 1134 tx_desc++; 1135 if (i == tx_ring->count) { 1136 tx_desc = IGC_TX_DESC(tx_ring, 0); 1137 i = 0; 1138 } 1139 tx_desc->read.olinfo_status = 0; 1140 1141 size = skb_frag_size(frag); 1142 data_len -= size; 1143 1144 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1145 size, DMA_TO_DEVICE); 1146 1147 tx_buffer = &tx_ring->tx_buffer_info[i]; 1148 } 1149 1150 /* write last descriptor with RS and EOP bits */ 1151 cmd_type |= size | IGC_TXD_DCMD; 1152 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1153 1154 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1155 1156 /* set the timestamp */ 1157 first->time_stamp = jiffies; 1158 1159 skb_tx_timestamp(skb); 1160 1161 /* Force memory writes to complete before letting h/w know there 1162 * are new descriptors to fetch. (Only applicable for weak-ordered 1163 * memory model archs, such as IA-64). 1164 * 1165 * We also need this memory barrier to make certain all of the 1166 * status bits have been updated before next_to_watch is written. 1167 */ 1168 wmb(); 1169 1170 /* set next_to_watch value indicating a packet is present */ 1171 first->next_to_watch = tx_desc; 1172 1173 i++; 1174 if (i == tx_ring->count) 1175 i = 0; 1176 1177 tx_ring->next_to_use = i; 1178 1179 /* Make sure there is space in the ring for the next send. */ 1180 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1181 1182 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1183 writel(i, tx_ring->tail); 1184 } 1185 1186 return 0; 1187 dma_error: 1188 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1189 tx_buffer = &tx_ring->tx_buffer_info[i]; 1190 1191 /* clear dma mappings for failed tx_buffer_info map */ 1192 while (tx_buffer != first) { 1193 if (dma_unmap_len(tx_buffer, len)) 1194 dma_unmap_page(tx_ring->dev, 1195 dma_unmap_addr(tx_buffer, dma), 1196 dma_unmap_len(tx_buffer, len), 1197 DMA_TO_DEVICE); 1198 dma_unmap_len_set(tx_buffer, len, 0); 1199 1200 if (i-- == 0) 1201 i += tx_ring->count; 1202 tx_buffer = &tx_ring->tx_buffer_info[i]; 1203 } 1204 1205 if (dma_unmap_len(tx_buffer, len)) 1206 dma_unmap_single(tx_ring->dev, 1207 dma_unmap_addr(tx_buffer, dma), 1208 dma_unmap_len(tx_buffer, len), 1209 DMA_TO_DEVICE); 1210 dma_unmap_len_set(tx_buffer, len, 0); 1211 1212 dev_kfree_skb_any(tx_buffer->skb); 1213 tx_buffer->skb = NULL; 1214 1215 tx_ring->next_to_use = i; 1216 1217 return -1; 1218 } 1219 1220 static int igc_tso(struct igc_ring *tx_ring, 1221 struct igc_tx_buffer *first, 1222 u8 *hdr_len) 1223 { 1224 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1225 struct sk_buff *skb = first->skb; 1226 union { 1227 struct iphdr *v4; 1228 struct ipv6hdr *v6; 1229 unsigned char *hdr; 1230 } ip; 1231 union { 1232 struct tcphdr *tcp; 1233 struct udphdr *udp; 1234 unsigned char *hdr; 1235 } l4; 1236 u32 paylen, l4_offset; 1237 int err; 1238 1239 if (skb->ip_summed != CHECKSUM_PARTIAL) 1240 return 0; 1241 1242 if (!skb_is_gso(skb)) 1243 return 0; 1244 1245 err = skb_cow_head(skb, 0); 1246 if (err < 0) 1247 return err; 1248 1249 ip.hdr = skb_network_header(skb); 1250 l4.hdr = skb_checksum_start(skb); 1251 1252 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1253 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1254 1255 /* initialize outer IP header fields */ 1256 if (ip.v4->version == 4) { 1257 unsigned char *csum_start = skb_checksum_start(skb); 1258 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1259 1260 /* IP header will have to cancel out any data that 1261 * is not a part of the outer IP header 1262 */ 1263 ip.v4->check = csum_fold(csum_partial(trans_start, 1264 csum_start - trans_start, 1265 0)); 1266 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1267 1268 ip.v4->tot_len = 0; 1269 first->tx_flags |= IGC_TX_FLAGS_TSO | 1270 IGC_TX_FLAGS_CSUM | 1271 IGC_TX_FLAGS_IPV4; 1272 } else { 1273 ip.v6->payload_len = 0; 1274 first->tx_flags |= IGC_TX_FLAGS_TSO | 1275 IGC_TX_FLAGS_CSUM; 1276 } 1277 1278 /* determine offset of inner transport header */ 1279 l4_offset = l4.hdr - skb->data; 1280 1281 /* remove payload length from inner checksum */ 1282 paylen = skb->len - l4_offset; 1283 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1284 /* compute length of segmentation header */ 1285 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1286 csum_replace_by_diff(&l4.tcp->check, 1287 (__force __wsum)htonl(paylen)); 1288 } else { 1289 /* compute length of segmentation header */ 1290 *hdr_len = sizeof(*l4.udp) + l4_offset; 1291 csum_replace_by_diff(&l4.udp->check, 1292 (__force __wsum)htonl(paylen)); 1293 } 1294 1295 /* update gso size and bytecount with header size */ 1296 first->gso_segs = skb_shinfo(skb)->gso_segs; 1297 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1298 1299 /* MSS L4LEN IDX */ 1300 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1301 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1302 1303 /* VLAN MACLEN IPLEN */ 1304 vlan_macip_lens = l4.hdr - ip.hdr; 1305 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1306 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1307 1308 igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, 1309 type_tucmd, mss_l4len_idx); 1310 1311 return 1; 1312 } 1313 1314 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1315 struct igc_ring *tx_ring) 1316 { 1317 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1318 __be16 protocol = vlan_get_protocol(skb); 1319 struct igc_tx_buffer *first; 1320 u32 tx_flags = 0; 1321 unsigned short f; 1322 u8 hdr_len = 0; 1323 int tso = 0; 1324 1325 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1326 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1327 * + 2 desc gap to keep tail from touching head, 1328 * + 1 desc for context descriptor, 1329 * otherwise try next time 1330 */ 1331 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1332 count += TXD_USE_COUNT(skb_frag_size( 1333 &skb_shinfo(skb)->frags[f])); 1334 1335 if (igc_maybe_stop_tx(tx_ring, count + 3)) { 1336 /* this is a hard error */ 1337 return NETDEV_TX_BUSY; 1338 } 1339 1340 /* record the location of the first descriptor for this packet */ 1341 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1342 first->skb = skb; 1343 first->bytecount = skb->len; 1344 first->gso_segs = 1; 1345 1346 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1347 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1348 1349 /* FIXME: add support for retrieving timestamps from 1350 * the other timer registers before skipping the 1351 * timestamping request. 1352 */ 1353 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && 1354 !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, 1355 &adapter->state)) { 1356 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1357 tx_flags |= IGC_TX_FLAGS_TSTAMP; 1358 1359 adapter->ptp_tx_skb = skb_get(skb); 1360 adapter->ptp_tx_start = jiffies; 1361 } else { 1362 adapter->tx_hwtstamp_skipped++; 1363 } 1364 } 1365 1366 /* record initial flags and protocol */ 1367 first->tx_flags = tx_flags; 1368 first->protocol = protocol; 1369 1370 tso = igc_tso(tx_ring, first, &hdr_len); 1371 if (tso < 0) 1372 goto out_drop; 1373 else if (!tso) 1374 igc_tx_csum(tx_ring, first); 1375 1376 igc_tx_map(tx_ring, first, hdr_len); 1377 1378 return NETDEV_TX_OK; 1379 1380 out_drop: 1381 dev_kfree_skb_any(first->skb); 1382 first->skb = NULL; 1383 1384 return NETDEV_TX_OK; 1385 } 1386 1387 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1388 struct sk_buff *skb) 1389 { 1390 unsigned int r_idx = skb->queue_mapping; 1391 1392 if (r_idx >= adapter->num_tx_queues) 1393 r_idx = r_idx % adapter->num_tx_queues; 1394 1395 return adapter->tx_ring[r_idx]; 1396 } 1397 1398 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1399 struct net_device *netdev) 1400 { 1401 struct igc_adapter *adapter = netdev_priv(netdev); 1402 1403 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1404 * in order to meet this minimum size requirement. 1405 */ 1406 if (skb->len < 17) { 1407 if (skb_padto(skb, 17)) 1408 return NETDEV_TX_OK; 1409 skb->len = 17; 1410 } 1411 1412 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1413 } 1414 1415 static void igc_rx_checksum(struct igc_ring *ring, 1416 union igc_adv_rx_desc *rx_desc, 1417 struct sk_buff *skb) 1418 { 1419 skb_checksum_none_assert(skb); 1420 1421 /* Ignore Checksum bit is set */ 1422 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1423 return; 1424 1425 /* Rx checksum disabled via ethtool */ 1426 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1427 return; 1428 1429 /* TCP/UDP checksum error bit is set */ 1430 if (igc_test_staterr(rx_desc, 1431 IGC_RXDEXT_STATERR_TCPE | 1432 IGC_RXDEXT_STATERR_IPE)) { 1433 /* work around errata with sctp packets where the TCPE aka 1434 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1435 * packets (aka let the stack check the crc32c) 1436 */ 1437 if (!(skb->len == 60 && 1438 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1439 u64_stats_update_begin(&ring->rx_syncp); 1440 ring->rx_stats.csum_err++; 1441 u64_stats_update_end(&ring->rx_syncp); 1442 } 1443 /* let the stack verify checksum errors */ 1444 return; 1445 } 1446 /* It must be a TCP or UDP packet with a valid checksum */ 1447 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1448 IGC_RXD_STAT_UDPCS)) 1449 skb->ip_summed = CHECKSUM_UNNECESSARY; 1450 1451 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1452 le32_to_cpu(rx_desc->wb.upper.status_error)); 1453 } 1454 1455 static inline void igc_rx_hash(struct igc_ring *ring, 1456 union igc_adv_rx_desc *rx_desc, 1457 struct sk_buff *skb) 1458 { 1459 if (ring->netdev->features & NETIF_F_RXHASH) 1460 skb_set_hash(skb, 1461 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 1462 PKT_HASH_TYPE_L3); 1463 } 1464 1465 /** 1466 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1467 * @rx_ring: rx descriptor ring packet is being transacted on 1468 * @rx_desc: pointer to the EOP Rx descriptor 1469 * @skb: pointer to current skb being populated 1470 * 1471 * This function checks the ring, descriptor, and packet information in order 1472 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1473 * skb. 1474 */ 1475 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1476 union igc_adv_rx_desc *rx_desc, 1477 struct sk_buff *skb) 1478 { 1479 igc_rx_hash(rx_ring, rx_desc, skb); 1480 1481 igc_rx_checksum(rx_ring, rx_desc, skb); 1482 1483 skb_record_rx_queue(skb, rx_ring->queue_index); 1484 1485 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1486 } 1487 1488 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1489 const unsigned int size) 1490 { 1491 struct igc_rx_buffer *rx_buffer; 1492 1493 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1494 prefetchw(rx_buffer->page); 1495 1496 /* we are reusing so sync this buffer for CPU use */ 1497 dma_sync_single_range_for_cpu(rx_ring->dev, 1498 rx_buffer->dma, 1499 rx_buffer->page_offset, 1500 size, 1501 DMA_FROM_DEVICE); 1502 1503 rx_buffer->pagecnt_bias--; 1504 1505 return rx_buffer; 1506 } 1507 1508 /** 1509 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1510 * @rx_ring: rx descriptor ring to transact packets on 1511 * @rx_buffer: buffer containing page to add 1512 * @skb: sk_buff to place the data into 1513 * @size: size of buffer to be added 1514 * 1515 * This function will add the data contained in rx_buffer->page to the skb. 1516 */ 1517 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1518 struct igc_rx_buffer *rx_buffer, 1519 struct sk_buff *skb, 1520 unsigned int size) 1521 { 1522 #if (PAGE_SIZE < 8192) 1523 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1524 1525 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1526 rx_buffer->page_offset, size, truesize); 1527 rx_buffer->page_offset ^= truesize; 1528 #else 1529 unsigned int truesize = ring_uses_build_skb(rx_ring) ? 1530 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1531 SKB_DATA_ALIGN(size); 1532 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1533 rx_buffer->page_offset, size, truesize); 1534 rx_buffer->page_offset += truesize; 1535 #endif 1536 } 1537 1538 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1539 struct igc_rx_buffer *rx_buffer, 1540 union igc_adv_rx_desc *rx_desc, 1541 unsigned int size) 1542 { 1543 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1544 #if (PAGE_SIZE < 8192) 1545 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1546 #else 1547 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1548 SKB_DATA_ALIGN(IGC_SKB_PAD + size); 1549 #endif 1550 struct sk_buff *skb; 1551 1552 /* prefetch first cache line of first page */ 1553 prefetch(va); 1554 #if L1_CACHE_BYTES < 128 1555 prefetch(va + L1_CACHE_BYTES); 1556 #endif 1557 1558 /* build an skb around the page buffer */ 1559 skb = build_skb(va - IGC_SKB_PAD, truesize); 1560 if (unlikely(!skb)) 1561 return NULL; 1562 1563 /* update pointers within the skb to store the data */ 1564 skb_reserve(skb, IGC_SKB_PAD); 1565 __skb_put(skb, size); 1566 1567 /* update buffer offset */ 1568 #if (PAGE_SIZE < 8192) 1569 rx_buffer->page_offset ^= truesize; 1570 #else 1571 rx_buffer->page_offset += truesize; 1572 #endif 1573 1574 return skb; 1575 } 1576 1577 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1578 struct igc_rx_buffer *rx_buffer, 1579 union igc_adv_rx_desc *rx_desc, 1580 unsigned int size) 1581 { 1582 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; 1583 #if (PAGE_SIZE < 8192) 1584 unsigned int truesize = igc_rx_pg_size(rx_ring) / 2; 1585 #else 1586 unsigned int truesize = SKB_DATA_ALIGN(size); 1587 #endif 1588 unsigned int headlen; 1589 struct sk_buff *skb; 1590 1591 /* prefetch first cache line of first page */ 1592 prefetch(va); 1593 #if L1_CACHE_BYTES < 128 1594 prefetch(va + L1_CACHE_BYTES); 1595 #endif 1596 1597 /* allocate a skb to store the frags */ 1598 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); 1599 if (unlikely(!skb)) 1600 return NULL; 1601 1602 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { 1603 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); 1604 va += IGC_TS_HDR_LEN; 1605 size -= IGC_TS_HDR_LEN; 1606 } 1607 1608 /* Determine available headroom for copy */ 1609 headlen = size; 1610 if (headlen > IGC_RX_HDR_LEN) 1611 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 1612 1613 /* align pull length to size of long to optimize memcpy performance */ 1614 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); 1615 1616 /* update all of the pointers */ 1617 size -= headlen; 1618 if (size) { 1619 skb_add_rx_frag(skb, 0, rx_buffer->page, 1620 (va + headlen) - page_address(rx_buffer->page), 1621 size, truesize); 1622 #if (PAGE_SIZE < 8192) 1623 rx_buffer->page_offset ^= truesize; 1624 #else 1625 rx_buffer->page_offset += truesize; 1626 #endif 1627 } else { 1628 rx_buffer->pagecnt_bias++; 1629 } 1630 1631 return skb; 1632 } 1633 1634 /** 1635 * igc_reuse_rx_page - page flip buffer and store it back on the ring 1636 * @rx_ring: rx descriptor ring to store buffers on 1637 * @old_buff: donor buffer to have page reused 1638 * 1639 * Synchronizes page for reuse by the adapter 1640 */ 1641 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 1642 struct igc_rx_buffer *old_buff) 1643 { 1644 u16 nta = rx_ring->next_to_alloc; 1645 struct igc_rx_buffer *new_buff; 1646 1647 new_buff = &rx_ring->rx_buffer_info[nta]; 1648 1649 /* update, and store next to alloc */ 1650 nta++; 1651 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1652 1653 /* Transfer page from old buffer to new buffer. 1654 * Move each member individually to avoid possible store 1655 * forwarding stalls. 1656 */ 1657 new_buff->dma = old_buff->dma; 1658 new_buff->page = old_buff->page; 1659 new_buff->page_offset = old_buff->page_offset; 1660 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1661 } 1662 1663 static inline bool igc_page_is_reserved(struct page *page) 1664 { 1665 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 1666 } 1667 1668 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) 1669 { 1670 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1671 struct page *page = rx_buffer->page; 1672 1673 /* avoid re-using remote pages */ 1674 if (unlikely(igc_page_is_reserved(page))) 1675 return false; 1676 1677 #if (PAGE_SIZE < 8192) 1678 /* if we are only owner of page we can reuse it */ 1679 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) 1680 return false; 1681 #else 1682 #define IGC_LAST_OFFSET \ 1683 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 1684 1685 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 1686 return false; 1687 #endif 1688 1689 /* If we have drained the page fragment pool we need to update 1690 * the pagecnt_bias and page count so that we fully restock the 1691 * number of references the driver holds. 1692 */ 1693 if (unlikely(!pagecnt_bias)) { 1694 page_ref_add(page, USHRT_MAX); 1695 rx_buffer->pagecnt_bias = USHRT_MAX; 1696 } 1697 1698 return true; 1699 } 1700 1701 /** 1702 * igc_is_non_eop - process handling of non-EOP buffers 1703 * @rx_ring: Rx ring being processed 1704 * @rx_desc: Rx descriptor for current buffer 1705 * 1706 * This function updates next to clean. If the buffer is an EOP buffer 1707 * this function exits returning false, otherwise it will place the 1708 * sk_buff in the next buffer to be chained and return true indicating 1709 * that this is in fact a non-EOP buffer. 1710 */ 1711 static bool igc_is_non_eop(struct igc_ring *rx_ring, 1712 union igc_adv_rx_desc *rx_desc) 1713 { 1714 u32 ntc = rx_ring->next_to_clean + 1; 1715 1716 /* fetch, update, and store next to clean */ 1717 ntc = (ntc < rx_ring->count) ? ntc : 0; 1718 rx_ring->next_to_clean = ntc; 1719 1720 prefetch(IGC_RX_DESC(rx_ring, ntc)); 1721 1722 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 1723 return false; 1724 1725 return true; 1726 } 1727 1728 /** 1729 * igc_cleanup_headers - Correct corrupted or empty headers 1730 * @rx_ring: rx descriptor ring packet is being transacted on 1731 * @rx_desc: pointer to the EOP Rx descriptor 1732 * @skb: pointer to current skb being fixed 1733 * 1734 * Address the case where we are pulling data in on pages only 1735 * and as such no data is present in the skb header. 1736 * 1737 * In addition if skb is not at least 60 bytes we need to pad it so that 1738 * it is large enough to qualify as a valid Ethernet frame. 1739 * 1740 * Returns true if an error was encountered and skb was freed. 1741 */ 1742 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 1743 union igc_adv_rx_desc *rx_desc, 1744 struct sk_buff *skb) 1745 { 1746 if (unlikely((igc_test_staterr(rx_desc, 1747 IGC_RXDEXT_ERR_FRAME_ERR_MASK)))) { 1748 struct net_device *netdev = rx_ring->netdev; 1749 1750 if (!(netdev->features & NETIF_F_RXALL)) { 1751 dev_kfree_skb_any(skb); 1752 return true; 1753 } 1754 } 1755 1756 /* if eth_skb_pad returns an error the skb was freed */ 1757 if (eth_skb_pad(skb)) 1758 return true; 1759 1760 return false; 1761 } 1762 1763 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 1764 struct igc_rx_buffer *rx_buffer) 1765 { 1766 if (igc_can_reuse_rx_page(rx_buffer)) { 1767 /* hand second half of page back to the ring */ 1768 igc_reuse_rx_page(rx_ring, rx_buffer); 1769 } else { 1770 /* We are not reusing the buffer so unmap it and free 1771 * any references we are holding to it 1772 */ 1773 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 1774 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 1775 IGC_RX_DMA_ATTR); 1776 __page_frag_cache_drain(rx_buffer->page, 1777 rx_buffer->pagecnt_bias); 1778 } 1779 1780 /* clear contents of rx_buffer */ 1781 rx_buffer->page = NULL; 1782 } 1783 1784 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 1785 { 1786 return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; 1787 } 1788 1789 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 1790 struct igc_rx_buffer *bi) 1791 { 1792 struct page *page = bi->page; 1793 dma_addr_t dma; 1794 1795 /* since we are recycling buffers we should seldom need to alloc */ 1796 if (likely(page)) 1797 return true; 1798 1799 /* alloc new page for storage */ 1800 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 1801 if (unlikely(!page)) { 1802 rx_ring->rx_stats.alloc_failed++; 1803 return false; 1804 } 1805 1806 /* map page for use */ 1807 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1808 igc_rx_pg_size(rx_ring), 1809 DMA_FROM_DEVICE, 1810 IGC_RX_DMA_ATTR); 1811 1812 /* if mapping failed free memory back to system since 1813 * there isn't much point in holding memory we can't use 1814 */ 1815 if (dma_mapping_error(rx_ring->dev, dma)) { 1816 __free_page(page); 1817 1818 rx_ring->rx_stats.alloc_failed++; 1819 return false; 1820 } 1821 1822 bi->dma = dma; 1823 bi->page = page; 1824 bi->page_offset = igc_rx_offset(rx_ring); 1825 bi->pagecnt_bias = 1; 1826 1827 return true; 1828 } 1829 1830 /** 1831 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 1832 * @rx_ring: rx descriptor ring 1833 * @cleaned_count: number of buffers to clean 1834 */ 1835 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 1836 { 1837 union igc_adv_rx_desc *rx_desc; 1838 u16 i = rx_ring->next_to_use; 1839 struct igc_rx_buffer *bi; 1840 u16 bufsz; 1841 1842 /* nothing to do */ 1843 if (!cleaned_count) 1844 return; 1845 1846 rx_desc = IGC_RX_DESC(rx_ring, i); 1847 bi = &rx_ring->rx_buffer_info[i]; 1848 i -= rx_ring->count; 1849 1850 bufsz = igc_rx_bufsz(rx_ring); 1851 1852 do { 1853 if (!igc_alloc_mapped_page(rx_ring, bi)) 1854 break; 1855 1856 /* sync the buffer for use by the device */ 1857 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1858 bi->page_offset, bufsz, 1859 DMA_FROM_DEVICE); 1860 1861 /* Refresh the desc even if buffer_addrs didn't change 1862 * because each write-back erases this info. 1863 */ 1864 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1865 1866 rx_desc++; 1867 bi++; 1868 i++; 1869 if (unlikely(!i)) { 1870 rx_desc = IGC_RX_DESC(rx_ring, 0); 1871 bi = rx_ring->rx_buffer_info; 1872 i -= rx_ring->count; 1873 } 1874 1875 /* clear the length for the next_to_use descriptor */ 1876 rx_desc->wb.upper.length = 0; 1877 1878 cleaned_count--; 1879 } while (cleaned_count); 1880 1881 i += rx_ring->count; 1882 1883 if (rx_ring->next_to_use != i) { 1884 /* record the next descriptor to use */ 1885 rx_ring->next_to_use = i; 1886 1887 /* update next to alloc since we have filled the ring */ 1888 rx_ring->next_to_alloc = i; 1889 1890 /* Force memory writes to complete before letting h/w 1891 * know there are new descriptors to fetch. (Only 1892 * applicable for weak-ordered memory model archs, 1893 * such as IA-64). 1894 */ 1895 wmb(); 1896 writel(i, rx_ring->tail); 1897 } 1898 } 1899 1900 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 1901 { 1902 unsigned int total_bytes = 0, total_packets = 0; 1903 struct igc_ring *rx_ring = q_vector->rx.ring; 1904 struct sk_buff *skb = rx_ring->skb; 1905 u16 cleaned_count = igc_desc_unused(rx_ring); 1906 1907 while (likely(total_packets < budget)) { 1908 union igc_adv_rx_desc *rx_desc; 1909 struct igc_rx_buffer *rx_buffer; 1910 unsigned int size; 1911 1912 /* return some buffers to hardware, one at a time is too slow */ 1913 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 1914 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1915 cleaned_count = 0; 1916 } 1917 1918 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 1919 size = le16_to_cpu(rx_desc->wb.upper.length); 1920 if (!size) 1921 break; 1922 1923 /* This memory barrier is needed to keep us from reading 1924 * any other fields out of the rx_desc until we know the 1925 * descriptor has been written back 1926 */ 1927 dma_rmb(); 1928 1929 rx_buffer = igc_get_rx_buffer(rx_ring, size); 1930 1931 /* retrieve a buffer from the ring */ 1932 if (skb) 1933 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 1934 else if (ring_uses_build_skb(rx_ring)) 1935 skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); 1936 else 1937 skb = igc_construct_skb(rx_ring, rx_buffer, 1938 rx_desc, size); 1939 1940 /* exit if we failed to retrieve a buffer */ 1941 if (!skb) { 1942 rx_ring->rx_stats.alloc_failed++; 1943 rx_buffer->pagecnt_bias++; 1944 break; 1945 } 1946 1947 igc_put_rx_buffer(rx_ring, rx_buffer); 1948 cleaned_count++; 1949 1950 /* fetch next buffer in frame if non-eop */ 1951 if (igc_is_non_eop(rx_ring, rx_desc)) 1952 continue; 1953 1954 /* verify the packet layout is correct */ 1955 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { 1956 skb = NULL; 1957 continue; 1958 } 1959 1960 /* probably a little skewed due to removing CRC */ 1961 total_bytes += skb->len; 1962 1963 /* populate checksum, VLAN, and protocol */ 1964 igc_process_skb_fields(rx_ring, rx_desc, skb); 1965 1966 napi_gro_receive(&q_vector->napi, skb); 1967 1968 /* reset skb pointer */ 1969 skb = NULL; 1970 1971 /* update budget accounting */ 1972 total_packets++; 1973 } 1974 1975 /* place incomplete frames back on ring for completion */ 1976 rx_ring->skb = skb; 1977 1978 u64_stats_update_begin(&rx_ring->rx_syncp); 1979 rx_ring->rx_stats.packets += total_packets; 1980 rx_ring->rx_stats.bytes += total_bytes; 1981 u64_stats_update_end(&rx_ring->rx_syncp); 1982 q_vector->rx.total_packets += total_packets; 1983 q_vector->rx.total_bytes += total_bytes; 1984 1985 if (cleaned_count) 1986 igc_alloc_rx_buffers(rx_ring, cleaned_count); 1987 1988 return total_packets; 1989 } 1990 1991 /** 1992 * igc_clean_tx_irq - Reclaim resources after transmit completes 1993 * @q_vector: pointer to q_vector containing needed info 1994 * @napi_budget: Used to determine if we are in netpoll 1995 * 1996 * returns true if ring is completely cleaned 1997 */ 1998 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 1999 { 2000 struct igc_adapter *adapter = q_vector->adapter; 2001 unsigned int total_bytes = 0, total_packets = 0; 2002 unsigned int budget = q_vector->tx.work_limit; 2003 struct igc_ring *tx_ring = q_vector->tx.ring; 2004 unsigned int i = tx_ring->next_to_clean; 2005 struct igc_tx_buffer *tx_buffer; 2006 union igc_adv_tx_desc *tx_desc; 2007 2008 if (test_bit(__IGC_DOWN, &adapter->state)) 2009 return true; 2010 2011 tx_buffer = &tx_ring->tx_buffer_info[i]; 2012 tx_desc = IGC_TX_DESC(tx_ring, i); 2013 i -= tx_ring->count; 2014 2015 do { 2016 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 2017 2018 /* if next_to_watch is not set then there is no work pending */ 2019 if (!eop_desc) 2020 break; 2021 2022 /* prevent any other reads prior to eop_desc */ 2023 smp_rmb(); 2024 2025 /* if DD is not set pending work has not been completed */ 2026 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 2027 break; 2028 2029 /* clear next_to_watch to prevent false hangs */ 2030 tx_buffer->next_to_watch = NULL; 2031 2032 /* update the statistics for this packet */ 2033 total_bytes += tx_buffer->bytecount; 2034 total_packets += tx_buffer->gso_segs; 2035 2036 /* free the skb */ 2037 napi_consume_skb(tx_buffer->skb, napi_budget); 2038 2039 /* unmap skb header data */ 2040 dma_unmap_single(tx_ring->dev, 2041 dma_unmap_addr(tx_buffer, dma), 2042 dma_unmap_len(tx_buffer, len), 2043 DMA_TO_DEVICE); 2044 2045 /* clear tx_buffer data */ 2046 dma_unmap_len_set(tx_buffer, len, 0); 2047 2048 /* clear last DMA location and unmap remaining buffers */ 2049 while (tx_desc != eop_desc) { 2050 tx_buffer++; 2051 tx_desc++; 2052 i++; 2053 if (unlikely(!i)) { 2054 i -= tx_ring->count; 2055 tx_buffer = tx_ring->tx_buffer_info; 2056 tx_desc = IGC_TX_DESC(tx_ring, 0); 2057 } 2058 2059 /* unmap any remaining paged data */ 2060 if (dma_unmap_len(tx_buffer, len)) { 2061 dma_unmap_page(tx_ring->dev, 2062 dma_unmap_addr(tx_buffer, dma), 2063 dma_unmap_len(tx_buffer, len), 2064 DMA_TO_DEVICE); 2065 dma_unmap_len_set(tx_buffer, len, 0); 2066 } 2067 } 2068 2069 /* move us one more past the eop_desc for start of next pkt */ 2070 tx_buffer++; 2071 tx_desc++; 2072 i++; 2073 if (unlikely(!i)) { 2074 i -= tx_ring->count; 2075 tx_buffer = tx_ring->tx_buffer_info; 2076 tx_desc = IGC_TX_DESC(tx_ring, 0); 2077 } 2078 2079 /* issue prefetch for next Tx descriptor */ 2080 prefetch(tx_desc); 2081 2082 /* update budget accounting */ 2083 budget--; 2084 } while (likely(budget)); 2085 2086 netdev_tx_completed_queue(txring_txq(tx_ring), 2087 total_packets, total_bytes); 2088 2089 i += tx_ring->count; 2090 tx_ring->next_to_clean = i; 2091 u64_stats_update_begin(&tx_ring->tx_syncp); 2092 tx_ring->tx_stats.bytes += total_bytes; 2093 tx_ring->tx_stats.packets += total_packets; 2094 u64_stats_update_end(&tx_ring->tx_syncp); 2095 q_vector->tx.total_bytes += total_bytes; 2096 q_vector->tx.total_packets += total_packets; 2097 2098 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 2099 struct igc_hw *hw = &adapter->hw; 2100 2101 /* Detect a transmit hang in hardware, this serializes the 2102 * check with the clearing of time_stamp and movement of i 2103 */ 2104 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 2105 if (tx_buffer->next_to_watch && 2106 time_after(jiffies, tx_buffer->time_stamp + 2107 (adapter->tx_timeout_factor * HZ)) && 2108 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) { 2109 /* detected Tx unit hang */ 2110 netdev_err(tx_ring->netdev, 2111 "Detected Tx Unit Hang\n" 2112 " Tx Queue <%d>\n" 2113 " TDH <%x>\n" 2114 " TDT <%x>\n" 2115 " next_to_use <%x>\n" 2116 " next_to_clean <%x>\n" 2117 "buffer_info[next_to_clean]\n" 2118 " time_stamp <%lx>\n" 2119 " next_to_watch <%p>\n" 2120 " jiffies <%lx>\n" 2121 " desc.status <%x>\n", 2122 tx_ring->queue_index, 2123 rd32(IGC_TDH(tx_ring->reg_idx)), 2124 readl(tx_ring->tail), 2125 tx_ring->next_to_use, 2126 tx_ring->next_to_clean, 2127 tx_buffer->time_stamp, 2128 tx_buffer->next_to_watch, 2129 jiffies, 2130 tx_buffer->next_to_watch->wb.status); 2131 netif_stop_subqueue(tx_ring->netdev, 2132 tx_ring->queue_index); 2133 2134 /* we are about to reset, no point in enabling stuff */ 2135 return true; 2136 } 2137 } 2138 2139 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 2140 if (unlikely(total_packets && 2141 netif_carrier_ok(tx_ring->netdev) && 2142 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 2143 /* Make sure that anybody stopping the queue after this 2144 * sees the new next_to_clean. 2145 */ 2146 smp_mb(); 2147 if (__netif_subqueue_stopped(tx_ring->netdev, 2148 tx_ring->queue_index) && 2149 !(test_bit(__IGC_DOWN, &adapter->state))) { 2150 netif_wake_subqueue(tx_ring->netdev, 2151 tx_ring->queue_index); 2152 2153 u64_stats_update_begin(&tx_ring->tx_syncp); 2154 tx_ring->tx_stats.restart_queue++; 2155 u64_stats_update_end(&tx_ring->tx_syncp); 2156 } 2157 } 2158 2159 return !!budget; 2160 } 2161 2162 static int igc_find_mac_filter(struct igc_adapter *adapter, 2163 enum igc_mac_filter_type type, const u8 *addr) 2164 { 2165 struct igc_hw *hw = &adapter->hw; 2166 int max_entries = hw->mac.rar_entry_count; 2167 u32 ral, rah; 2168 int i; 2169 2170 for (i = 0; i < max_entries; i++) { 2171 ral = rd32(IGC_RAL(i)); 2172 rah = rd32(IGC_RAH(i)); 2173 2174 if (!(rah & IGC_RAH_AV)) 2175 continue; 2176 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 2177 continue; 2178 if ((rah & IGC_RAH_RAH_MASK) != 2179 le16_to_cpup((__le16 *)(addr + 4))) 2180 continue; 2181 if (ral != le32_to_cpup((__le32 *)(addr))) 2182 continue; 2183 2184 return i; 2185 } 2186 2187 return -1; 2188 } 2189 2190 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 2191 { 2192 struct igc_hw *hw = &adapter->hw; 2193 int max_entries = hw->mac.rar_entry_count; 2194 u32 rah; 2195 int i; 2196 2197 for (i = 0; i < max_entries; i++) { 2198 rah = rd32(IGC_RAH(i)); 2199 2200 if (!(rah & IGC_RAH_AV)) 2201 return i; 2202 } 2203 2204 return -1; 2205 } 2206 2207 /** 2208 * igc_add_mac_filter() - Add MAC address filter 2209 * @adapter: Pointer to adapter where the filter should be added 2210 * @type: MAC address filter type (source or destination) 2211 * @addr: MAC address 2212 * @queue: If non-negative, queue assignment feature is enabled and frames 2213 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2214 * assignment is disabled. 2215 * 2216 * Return: 0 in case of success, negative errno code otherwise. 2217 */ 2218 static int igc_add_mac_filter(struct igc_adapter *adapter, 2219 enum igc_mac_filter_type type, const u8 *addr, 2220 int queue) 2221 { 2222 struct net_device *dev = adapter->netdev; 2223 int index; 2224 2225 index = igc_find_mac_filter(adapter, type, addr); 2226 if (index >= 0) 2227 goto update_filter; 2228 2229 index = igc_get_avail_mac_filter_slot(adapter); 2230 if (index < 0) 2231 return -ENOSPC; 2232 2233 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 2234 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2235 addr, queue); 2236 2237 update_filter: 2238 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 2239 return 0; 2240 } 2241 2242 /** 2243 * igc_del_mac_filter() - Delete MAC address filter 2244 * @adapter: Pointer to adapter where the filter should be deleted from 2245 * @type: MAC address filter type (source or destination) 2246 * @addr: MAC address 2247 */ 2248 static void igc_del_mac_filter(struct igc_adapter *adapter, 2249 enum igc_mac_filter_type type, const u8 *addr) 2250 { 2251 struct net_device *dev = adapter->netdev; 2252 int index; 2253 2254 index = igc_find_mac_filter(adapter, type, addr); 2255 if (index < 0) 2256 return; 2257 2258 if (index == 0) { 2259 /* If this is the default filter, we don't actually delete it. 2260 * We just reset to its default value i.e. disable queue 2261 * assignment. 2262 */ 2263 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 2264 2265 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 2266 } else { 2267 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 2268 index, 2269 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 2270 addr); 2271 2272 igc_clear_mac_filter_hw(adapter, index); 2273 } 2274 } 2275 2276 /** 2277 * igc_add_vlan_prio_filter() - Add VLAN priority filter 2278 * @adapter: Pointer to adapter where the filter should be added 2279 * @prio: VLAN priority value 2280 * @queue: Queue number which matching frames are assigned to 2281 * 2282 * Return: 0 in case of success, negative errno code otherwise. 2283 */ 2284 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 2285 int queue) 2286 { 2287 struct net_device *dev = adapter->netdev; 2288 struct igc_hw *hw = &adapter->hw; 2289 u32 vlanpqf; 2290 2291 vlanpqf = rd32(IGC_VLANPQF); 2292 2293 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 2294 netdev_dbg(dev, "VLAN priority filter already in use\n"); 2295 return -EEXIST; 2296 } 2297 2298 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 2299 vlanpqf |= IGC_VLANPQF_VALID(prio); 2300 2301 wr32(IGC_VLANPQF, vlanpqf); 2302 2303 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 2304 prio, queue); 2305 return 0; 2306 } 2307 2308 /** 2309 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 2310 * @adapter: Pointer to adapter where the filter should be deleted from 2311 * @prio: VLAN priority value 2312 */ 2313 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 2314 { 2315 struct igc_hw *hw = &adapter->hw; 2316 u32 vlanpqf; 2317 2318 vlanpqf = rd32(IGC_VLANPQF); 2319 2320 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 2321 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 2322 2323 wr32(IGC_VLANPQF, vlanpqf); 2324 2325 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 2326 prio); 2327 } 2328 2329 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 2330 { 2331 struct igc_hw *hw = &adapter->hw; 2332 int i; 2333 2334 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2335 u32 etqf = rd32(IGC_ETQF(i)); 2336 2337 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 2338 return i; 2339 } 2340 2341 return -1; 2342 } 2343 2344 /** 2345 * igc_add_etype_filter() - Add ethertype filter 2346 * @adapter: Pointer to adapter where the filter should be added 2347 * @etype: Ethertype value 2348 * @queue: If non-negative, queue assignment feature is enabled and frames 2349 * matching the filter are enqueued onto 'queue'. Otherwise, queue 2350 * assignment is disabled. 2351 * 2352 * Return: 0 in case of success, negative errno code otherwise. 2353 */ 2354 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 2355 int queue) 2356 { 2357 struct igc_hw *hw = &adapter->hw; 2358 int index; 2359 u32 etqf; 2360 2361 index = igc_get_avail_etype_filter_slot(adapter); 2362 if (index < 0) 2363 return -ENOSPC; 2364 2365 etqf = rd32(IGC_ETQF(index)); 2366 2367 etqf &= ~IGC_ETQF_ETYPE_MASK; 2368 etqf |= etype; 2369 2370 if (queue >= 0) { 2371 etqf &= ~IGC_ETQF_QUEUE_MASK; 2372 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 2373 etqf |= IGC_ETQF_QUEUE_ENABLE; 2374 } 2375 2376 etqf |= IGC_ETQF_FILTER_ENABLE; 2377 2378 wr32(IGC_ETQF(index), etqf); 2379 2380 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 2381 etype, queue); 2382 return 0; 2383 } 2384 2385 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 2386 { 2387 struct igc_hw *hw = &adapter->hw; 2388 int i; 2389 2390 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 2391 u32 etqf = rd32(IGC_ETQF(i)); 2392 2393 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 2394 return i; 2395 } 2396 2397 return -1; 2398 } 2399 2400 /** 2401 * igc_del_etype_filter() - Delete ethertype filter 2402 * @adapter: Pointer to adapter where the filter should be deleted from 2403 * @etype: Ethertype value 2404 */ 2405 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 2406 { 2407 struct igc_hw *hw = &adapter->hw; 2408 int index; 2409 2410 index = igc_find_etype_filter(adapter, etype); 2411 if (index < 0) 2412 return; 2413 2414 wr32(IGC_ETQF(index), 0); 2415 2416 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 2417 etype); 2418 } 2419 2420 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 2421 const struct igc_nfc_rule *rule) 2422 { 2423 int err; 2424 2425 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 2426 err = igc_add_etype_filter(adapter, rule->filter.etype, 2427 rule->action); 2428 if (err) 2429 return err; 2430 } 2431 2432 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 2433 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2434 rule->filter.src_addr, rule->action); 2435 if (err) 2436 return err; 2437 } 2438 2439 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 2440 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2441 rule->filter.dst_addr, rule->action); 2442 if (err) 2443 return err; 2444 } 2445 2446 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2447 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2448 VLAN_PRIO_SHIFT; 2449 2450 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 2451 if (err) 2452 return err; 2453 } 2454 2455 return 0; 2456 } 2457 2458 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 2459 const struct igc_nfc_rule *rule) 2460 { 2461 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 2462 igc_del_etype_filter(adapter, rule->filter.etype); 2463 2464 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 2465 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> 2466 VLAN_PRIO_SHIFT; 2467 2468 igc_del_vlan_prio_filter(adapter, prio); 2469 } 2470 2471 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 2472 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 2473 rule->filter.src_addr); 2474 2475 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 2476 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 2477 rule->filter.dst_addr); 2478 } 2479 2480 /** 2481 * igc_get_nfc_rule() - Get NFC rule 2482 * @adapter: Pointer to adapter 2483 * @location: Rule location 2484 * 2485 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2486 * 2487 * Return: Pointer to NFC rule at @location. If not found, NULL. 2488 */ 2489 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 2490 u32 location) 2491 { 2492 struct igc_nfc_rule *rule; 2493 2494 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 2495 if (rule->location == location) 2496 return rule; 2497 if (rule->location > location) 2498 break; 2499 } 2500 2501 return NULL; 2502 } 2503 2504 /** 2505 * igc_del_nfc_rule() - Delete NFC rule 2506 * @adapter: Pointer to adapter 2507 * @rule: Pointer to rule to be deleted 2508 * 2509 * Disable NFC rule in hardware and delete it from adapter. 2510 * 2511 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2512 */ 2513 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2514 { 2515 igc_disable_nfc_rule(adapter, rule); 2516 2517 list_del(&rule->list); 2518 adapter->nfc_rule_count--; 2519 2520 kfree(rule); 2521 } 2522 2523 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 2524 { 2525 struct igc_nfc_rule *rule, *tmp; 2526 2527 mutex_lock(&adapter->nfc_rule_lock); 2528 2529 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 2530 igc_del_nfc_rule(adapter, rule); 2531 2532 mutex_unlock(&adapter->nfc_rule_lock); 2533 } 2534 2535 /** 2536 * igc_add_nfc_rule() - Add NFC rule 2537 * @adapter: Pointer to adapter 2538 * @rule: Pointer to rule to be added 2539 * 2540 * Enable NFC rule in hardware and add it to adapter. 2541 * 2542 * Context: Expects adapter->nfc_rule_lock to be held by caller. 2543 * 2544 * Return: 0 on success, negative errno on failure. 2545 */ 2546 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 2547 { 2548 struct igc_nfc_rule *pred, *cur; 2549 int err; 2550 2551 err = igc_enable_nfc_rule(adapter, rule); 2552 if (err) 2553 return err; 2554 2555 pred = NULL; 2556 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 2557 if (cur->location >= rule->location) 2558 break; 2559 pred = cur; 2560 } 2561 2562 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 2563 adapter->nfc_rule_count++; 2564 return 0; 2565 } 2566 2567 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 2568 { 2569 struct igc_nfc_rule *rule; 2570 2571 mutex_lock(&adapter->nfc_rule_lock); 2572 2573 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 2574 igc_enable_nfc_rule(adapter, rule); 2575 2576 mutex_unlock(&adapter->nfc_rule_lock); 2577 } 2578 2579 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 2580 { 2581 struct igc_adapter *adapter = netdev_priv(netdev); 2582 2583 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 2584 } 2585 2586 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 2587 { 2588 struct igc_adapter *adapter = netdev_priv(netdev); 2589 2590 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 2591 return 0; 2592 } 2593 2594 /** 2595 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 2596 * @netdev: network interface device structure 2597 * 2598 * The set_rx_mode entry point is called whenever the unicast or multicast 2599 * address lists or the network interface flags are updated. This routine is 2600 * responsible for configuring the hardware for proper unicast, multicast, 2601 * promiscuous mode, and all-multi behavior. 2602 */ 2603 static void igc_set_rx_mode(struct net_device *netdev) 2604 { 2605 struct igc_adapter *adapter = netdev_priv(netdev); 2606 struct igc_hw *hw = &adapter->hw; 2607 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 2608 int count; 2609 2610 /* Check for Promiscuous and All Multicast modes */ 2611 if (netdev->flags & IFF_PROMISC) { 2612 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 2613 } else { 2614 if (netdev->flags & IFF_ALLMULTI) { 2615 rctl |= IGC_RCTL_MPE; 2616 } else { 2617 /* Write addresses to the MTA, if the attempt fails 2618 * then we should just turn on promiscuous mode so 2619 * that we can at least receive multicast traffic 2620 */ 2621 count = igc_write_mc_addr_list(netdev); 2622 if (count < 0) 2623 rctl |= IGC_RCTL_MPE; 2624 } 2625 } 2626 2627 /* Write addresses to available RAR registers, if there is not 2628 * sufficient space to store all the addresses then enable 2629 * unicast promiscuous mode 2630 */ 2631 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 2632 rctl |= IGC_RCTL_UPE; 2633 2634 /* update state of unicast and multicast */ 2635 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 2636 wr32(IGC_RCTL, rctl); 2637 2638 #if (PAGE_SIZE < 8192) 2639 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 2640 rlpml = IGC_MAX_FRAME_BUILD_SKB; 2641 #endif 2642 wr32(IGC_RLPML, rlpml); 2643 } 2644 2645 /** 2646 * igc_configure - configure the hardware for RX and TX 2647 * @adapter: private board structure 2648 */ 2649 static void igc_configure(struct igc_adapter *adapter) 2650 { 2651 struct net_device *netdev = adapter->netdev; 2652 int i = 0; 2653 2654 igc_get_hw_control(adapter); 2655 igc_set_rx_mode(netdev); 2656 2657 igc_setup_tctl(adapter); 2658 igc_setup_mrqc(adapter); 2659 igc_setup_rctl(adapter); 2660 2661 igc_set_default_mac_filter(adapter); 2662 igc_restore_nfc_rules(adapter); 2663 2664 igc_configure_tx(adapter); 2665 igc_configure_rx(adapter); 2666 2667 igc_rx_fifo_flush_base(&adapter->hw); 2668 2669 /* call igc_desc_unused which always leaves 2670 * at least 1 descriptor unused to make sure 2671 * next_to_use != next_to_clean 2672 */ 2673 for (i = 0; i < adapter->num_rx_queues; i++) { 2674 struct igc_ring *ring = adapter->rx_ring[i]; 2675 2676 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 2677 } 2678 } 2679 2680 /** 2681 * igc_write_ivar - configure ivar for given MSI-X vector 2682 * @hw: pointer to the HW structure 2683 * @msix_vector: vector number we are allocating to a given ring 2684 * @index: row index of IVAR register to write within IVAR table 2685 * @offset: column offset of in IVAR, should be multiple of 8 2686 * 2687 * The IVAR table consists of 2 columns, 2688 * each containing an cause allocation for an Rx and Tx ring, and a 2689 * variable number of rows depending on the number of queues supported. 2690 */ 2691 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 2692 int index, int offset) 2693 { 2694 u32 ivar = array_rd32(IGC_IVAR0, index); 2695 2696 /* clear any bits that are currently set */ 2697 ivar &= ~((u32)0xFF << offset); 2698 2699 /* write vector and valid bit */ 2700 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 2701 2702 array_wr32(IGC_IVAR0, index, ivar); 2703 } 2704 2705 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 2706 { 2707 struct igc_adapter *adapter = q_vector->adapter; 2708 struct igc_hw *hw = &adapter->hw; 2709 int rx_queue = IGC_N0_QUEUE; 2710 int tx_queue = IGC_N0_QUEUE; 2711 2712 if (q_vector->rx.ring) 2713 rx_queue = q_vector->rx.ring->reg_idx; 2714 if (q_vector->tx.ring) 2715 tx_queue = q_vector->tx.ring->reg_idx; 2716 2717 switch (hw->mac.type) { 2718 case igc_i225: 2719 if (rx_queue > IGC_N0_QUEUE) 2720 igc_write_ivar(hw, msix_vector, 2721 rx_queue >> 1, 2722 (rx_queue & 0x1) << 4); 2723 if (tx_queue > IGC_N0_QUEUE) 2724 igc_write_ivar(hw, msix_vector, 2725 tx_queue >> 1, 2726 ((tx_queue & 0x1) << 4) + 8); 2727 q_vector->eims_value = BIT(msix_vector); 2728 break; 2729 default: 2730 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 2731 break; 2732 } 2733 2734 /* add q_vector eims value to global eims_enable_mask */ 2735 adapter->eims_enable_mask |= q_vector->eims_value; 2736 2737 /* configure q_vector to set itr on first interrupt */ 2738 q_vector->set_itr = 1; 2739 } 2740 2741 /** 2742 * igc_configure_msix - Configure MSI-X hardware 2743 * @adapter: Pointer to adapter structure 2744 * 2745 * igc_configure_msix sets up the hardware to properly 2746 * generate MSI-X interrupts. 2747 */ 2748 static void igc_configure_msix(struct igc_adapter *adapter) 2749 { 2750 struct igc_hw *hw = &adapter->hw; 2751 int i, vector = 0; 2752 u32 tmp; 2753 2754 adapter->eims_enable_mask = 0; 2755 2756 /* set vector for other causes, i.e. link changes */ 2757 switch (hw->mac.type) { 2758 case igc_i225: 2759 /* Turn on MSI-X capability first, or our settings 2760 * won't stick. And it will take days to debug. 2761 */ 2762 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 2763 IGC_GPIE_PBA | IGC_GPIE_EIAME | 2764 IGC_GPIE_NSICR); 2765 2766 /* enable msix_other interrupt */ 2767 adapter->eims_other = BIT(vector); 2768 tmp = (vector++ | IGC_IVAR_VALID) << 8; 2769 2770 wr32(IGC_IVAR_MISC, tmp); 2771 break; 2772 default: 2773 /* do nothing, since nothing else supports MSI-X */ 2774 break; 2775 } /* switch (hw->mac.type) */ 2776 2777 adapter->eims_enable_mask |= adapter->eims_other; 2778 2779 for (i = 0; i < adapter->num_q_vectors; i++) 2780 igc_assign_vector(adapter->q_vector[i], vector++); 2781 2782 wrfl(); 2783 } 2784 2785 /** 2786 * igc_irq_enable - Enable default interrupt generation settings 2787 * @adapter: board private structure 2788 */ 2789 static void igc_irq_enable(struct igc_adapter *adapter) 2790 { 2791 struct igc_hw *hw = &adapter->hw; 2792 2793 if (adapter->msix_entries) { 2794 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 2795 u32 regval = rd32(IGC_EIAC); 2796 2797 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 2798 regval = rd32(IGC_EIAM); 2799 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 2800 wr32(IGC_EIMS, adapter->eims_enable_mask); 2801 wr32(IGC_IMS, ims); 2802 } else { 2803 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2804 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 2805 } 2806 } 2807 2808 /** 2809 * igc_irq_disable - Mask off interrupt generation on the NIC 2810 * @adapter: board private structure 2811 */ 2812 static void igc_irq_disable(struct igc_adapter *adapter) 2813 { 2814 struct igc_hw *hw = &adapter->hw; 2815 2816 if (adapter->msix_entries) { 2817 u32 regval = rd32(IGC_EIAM); 2818 2819 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 2820 wr32(IGC_EIMC, adapter->eims_enable_mask); 2821 regval = rd32(IGC_EIAC); 2822 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 2823 } 2824 2825 wr32(IGC_IAM, 0); 2826 wr32(IGC_IMC, ~0); 2827 wrfl(); 2828 2829 if (adapter->msix_entries) { 2830 int vector = 0, i; 2831 2832 synchronize_irq(adapter->msix_entries[vector++].vector); 2833 2834 for (i = 0; i < adapter->num_q_vectors; i++) 2835 synchronize_irq(adapter->msix_entries[vector++].vector); 2836 } else { 2837 synchronize_irq(adapter->pdev->irq); 2838 } 2839 } 2840 2841 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 2842 const u32 max_rss_queues) 2843 { 2844 /* Determine if we need to pair queues. */ 2845 /* If rss_queues > half of max_rss_queues, pair the queues in 2846 * order to conserve interrupts due to limited supply. 2847 */ 2848 if (adapter->rss_queues > (max_rss_queues / 2)) 2849 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 2850 else 2851 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 2852 } 2853 2854 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 2855 { 2856 return IGC_MAX_RX_QUEUES; 2857 } 2858 2859 static void igc_init_queue_configuration(struct igc_adapter *adapter) 2860 { 2861 u32 max_rss_queues; 2862 2863 max_rss_queues = igc_get_max_rss_queues(adapter); 2864 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 2865 2866 igc_set_flag_queue_pairs(adapter, max_rss_queues); 2867 } 2868 2869 /** 2870 * igc_reset_q_vector - Reset config for interrupt vector 2871 * @adapter: board private structure to initialize 2872 * @v_idx: Index of vector to be reset 2873 * 2874 * If NAPI is enabled it will delete any references to the 2875 * NAPI struct. This is preparation for igc_free_q_vector. 2876 */ 2877 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 2878 { 2879 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2880 2881 /* if we're coming from igc_set_interrupt_capability, the vectors are 2882 * not yet allocated 2883 */ 2884 if (!q_vector) 2885 return; 2886 2887 if (q_vector->tx.ring) 2888 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 2889 2890 if (q_vector->rx.ring) 2891 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 2892 2893 netif_napi_del(&q_vector->napi); 2894 } 2895 2896 /** 2897 * igc_free_q_vector - Free memory allocated for specific interrupt vector 2898 * @adapter: board private structure to initialize 2899 * @v_idx: Index of vector to be freed 2900 * 2901 * This function frees the memory allocated to the q_vector. 2902 */ 2903 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 2904 { 2905 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 2906 2907 adapter->q_vector[v_idx] = NULL; 2908 2909 /* igc_get_stats64() might access the rings on this vector, 2910 * we must wait a grace period before freeing it. 2911 */ 2912 if (q_vector) 2913 kfree_rcu(q_vector, rcu); 2914 } 2915 2916 /** 2917 * igc_free_q_vectors - Free memory allocated for interrupt vectors 2918 * @adapter: board private structure to initialize 2919 * 2920 * This function frees the memory allocated to the q_vectors. In addition if 2921 * NAPI is enabled it will delete any references to the NAPI struct prior 2922 * to freeing the q_vector. 2923 */ 2924 static void igc_free_q_vectors(struct igc_adapter *adapter) 2925 { 2926 int v_idx = adapter->num_q_vectors; 2927 2928 adapter->num_tx_queues = 0; 2929 adapter->num_rx_queues = 0; 2930 adapter->num_q_vectors = 0; 2931 2932 while (v_idx--) { 2933 igc_reset_q_vector(adapter, v_idx); 2934 igc_free_q_vector(adapter, v_idx); 2935 } 2936 } 2937 2938 /** 2939 * igc_update_itr - update the dynamic ITR value based on statistics 2940 * @q_vector: pointer to q_vector 2941 * @ring_container: ring info to update the itr for 2942 * 2943 * Stores a new ITR value based on packets and byte 2944 * counts during the last interrupt. The advantage of per interrupt 2945 * computation is faster updates and more accurate ITR for the current 2946 * traffic pattern. Constants in this function were computed 2947 * based on theoretical maximum wire speed and thresholds were set based 2948 * on testing data as well as attempting to minimize response time 2949 * while increasing bulk throughput. 2950 * NOTE: These calculations are only valid when operating in a single- 2951 * queue environment. 2952 */ 2953 static void igc_update_itr(struct igc_q_vector *q_vector, 2954 struct igc_ring_container *ring_container) 2955 { 2956 unsigned int packets = ring_container->total_packets; 2957 unsigned int bytes = ring_container->total_bytes; 2958 u8 itrval = ring_container->itr; 2959 2960 /* no packets, exit with status unchanged */ 2961 if (packets == 0) 2962 return; 2963 2964 switch (itrval) { 2965 case lowest_latency: 2966 /* handle TSO and jumbo frames */ 2967 if (bytes / packets > 8000) 2968 itrval = bulk_latency; 2969 else if ((packets < 5) && (bytes > 512)) 2970 itrval = low_latency; 2971 break; 2972 case low_latency: /* 50 usec aka 20000 ints/s */ 2973 if (bytes > 10000) { 2974 /* this if handles the TSO accounting */ 2975 if (bytes / packets > 8000) 2976 itrval = bulk_latency; 2977 else if ((packets < 10) || ((bytes / packets) > 1200)) 2978 itrval = bulk_latency; 2979 else if ((packets > 35)) 2980 itrval = lowest_latency; 2981 } else if (bytes / packets > 2000) { 2982 itrval = bulk_latency; 2983 } else if (packets <= 2 && bytes < 512) { 2984 itrval = lowest_latency; 2985 } 2986 break; 2987 case bulk_latency: /* 250 usec aka 4000 ints/s */ 2988 if (bytes > 25000) { 2989 if (packets > 35) 2990 itrval = low_latency; 2991 } else if (bytes < 1500) { 2992 itrval = low_latency; 2993 } 2994 break; 2995 } 2996 2997 /* clear work counters since we have the values we need */ 2998 ring_container->total_bytes = 0; 2999 ring_container->total_packets = 0; 3000 3001 /* write updated itr to ring container */ 3002 ring_container->itr = itrval; 3003 } 3004 3005 static void igc_set_itr(struct igc_q_vector *q_vector) 3006 { 3007 struct igc_adapter *adapter = q_vector->adapter; 3008 u32 new_itr = q_vector->itr_val; 3009 u8 current_itr = 0; 3010 3011 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 3012 switch (adapter->link_speed) { 3013 case SPEED_10: 3014 case SPEED_100: 3015 current_itr = 0; 3016 new_itr = IGC_4K_ITR; 3017 goto set_itr_now; 3018 default: 3019 break; 3020 } 3021 3022 igc_update_itr(q_vector, &q_vector->tx); 3023 igc_update_itr(q_vector, &q_vector->rx); 3024 3025 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 3026 3027 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3028 if (current_itr == lowest_latency && 3029 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3030 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3031 current_itr = low_latency; 3032 3033 switch (current_itr) { 3034 /* counts and packets in update_itr are dependent on these numbers */ 3035 case lowest_latency: 3036 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 3037 break; 3038 case low_latency: 3039 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 3040 break; 3041 case bulk_latency: 3042 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 3043 break; 3044 default: 3045 break; 3046 } 3047 3048 set_itr_now: 3049 if (new_itr != q_vector->itr_val) { 3050 /* this attempts to bias the interrupt rate towards Bulk 3051 * by adding intermediate steps when interrupt rate is 3052 * increasing 3053 */ 3054 new_itr = new_itr > q_vector->itr_val ? 3055 max((new_itr * q_vector->itr_val) / 3056 (new_itr + (q_vector->itr_val >> 2)), 3057 new_itr) : new_itr; 3058 /* Don't write the value here; it resets the adapter's 3059 * internal timer, and causes us to delay far longer than 3060 * we should between interrupts. Instead, we write the ITR 3061 * value at the beginning of the next interrupt so the timing 3062 * ends up being correct. 3063 */ 3064 q_vector->itr_val = new_itr; 3065 q_vector->set_itr = 1; 3066 } 3067 } 3068 3069 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 3070 { 3071 int v_idx = adapter->num_q_vectors; 3072 3073 if (adapter->msix_entries) { 3074 pci_disable_msix(adapter->pdev); 3075 kfree(adapter->msix_entries); 3076 adapter->msix_entries = NULL; 3077 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 3078 pci_disable_msi(adapter->pdev); 3079 } 3080 3081 while (v_idx--) 3082 igc_reset_q_vector(adapter, v_idx); 3083 } 3084 3085 /** 3086 * igc_set_interrupt_capability - set MSI or MSI-X if supported 3087 * @adapter: Pointer to adapter structure 3088 * @msix: boolean value for MSI-X capability 3089 * 3090 * Attempt to configure interrupts using the best available 3091 * capabilities of the hardware and kernel. 3092 */ 3093 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 3094 bool msix) 3095 { 3096 int numvecs, i; 3097 int err; 3098 3099 if (!msix) 3100 goto msi_only; 3101 adapter->flags |= IGC_FLAG_HAS_MSIX; 3102 3103 /* Number of supported queues. */ 3104 adapter->num_rx_queues = adapter->rss_queues; 3105 3106 adapter->num_tx_queues = adapter->rss_queues; 3107 3108 /* start with one vector for every Rx queue */ 3109 numvecs = adapter->num_rx_queues; 3110 3111 /* if Tx handler is separate add 1 for every Tx queue */ 3112 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 3113 numvecs += adapter->num_tx_queues; 3114 3115 /* store the number of vectors reserved for queues */ 3116 adapter->num_q_vectors = numvecs; 3117 3118 /* add 1 vector for link status interrupts */ 3119 numvecs++; 3120 3121 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), 3122 GFP_KERNEL); 3123 3124 if (!adapter->msix_entries) 3125 return; 3126 3127 /* populate entry values */ 3128 for (i = 0; i < numvecs; i++) 3129 adapter->msix_entries[i].entry = i; 3130 3131 err = pci_enable_msix_range(adapter->pdev, 3132 adapter->msix_entries, 3133 numvecs, 3134 numvecs); 3135 if (err > 0) 3136 return; 3137 3138 kfree(adapter->msix_entries); 3139 adapter->msix_entries = NULL; 3140 3141 igc_reset_interrupt_capability(adapter); 3142 3143 msi_only: 3144 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 3145 3146 adapter->rss_queues = 1; 3147 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 3148 adapter->num_rx_queues = 1; 3149 adapter->num_tx_queues = 1; 3150 adapter->num_q_vectors = 1; 3151 if (!pci_enable_msi(adapter->pdev)) 3152 adapter->flags |= IGC_FLAG_HAS_MSI; 3153 } 3154 3155 /** 3156 * igc_update_ring_itr - update the dynamic ITR value based on packet size 3157 * @q_vector: pointer to q_vector 3158 * 3159 * Stores a new ITR value based on strictly on packet size. This 3160 * algorithm is less sophisticated than that used in igc_update_itr, 3161 * due to the difficulty of synchronizing statistics across multiple 3162 * receive rings. The divisors and thresholds used by this function 3163 * were determined based on theoretical maximum wire speed and testing 3164 * data, in order to minimize response time while increasing bulk 3165 * throughput. 3166 * NOTE: This function is called only when operating in a multiqueue 3167 * receive environment. 3168 */ 3169 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 3170 { 3171 struct igc_adapter *adapter = q_vector->adapter; 3172 int new_val = q_vector->itr_val; 3173 int avg_wire_size = 0; 3174 unsigned int packets; 3175 3176 /* For non-gigabit speeds, just fix the interrupt rate at 4000 3177 * ints/sec - ITR timer value of 120 ticks. 3178 */ 3179 switch (adapter->link_speed) { 3180 case SPEED_10: 3181 case SPEED_100: 3182 new_val = IGC_4K_ITR; 3183 goto set_itr_val; 3184 default: 3185 break; 3186 } 3187 3188 packets = q_vector->rx.total_packets; 3189 if (packets) 3190 avg_wire_size = q_vector->rx.total_bytes / packets; 3191 3192 packets = q_vector->tx.total_packets; 3193 if (packets) 3194 avg_wire_size = max_t(u32, avg_wire_size, 3195 q_vector->tx.total_bytes / packets); 3196 3197 /* if avg_wire_size isn't set no work was done */ 3198 if (!avg_wire_size) 3199 goto clear_counts; 3200 3201 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 3202 avg_wire_size += 24; 3203 3204 /* Don't starve jumbo frames */ 3205 avg_wire_size = min(avg_wire_size, 3000); 3206 3207 /* Give a little boost to mid-size frames */ 3208 if (avg_wire_size > 300 && avg_wire_size < 1200) 3209 new_val = avg_wire_size / 3; 3210 else 3211 new_val = avg_wire_size / 2; 3212 3213 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 3214 if (new_val < IGC_20K_ITR && 3215 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 3216 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 3217 new_val = IGC_20K_ITR; 3218 3219 set_itr_val: 3220 if (new_val != q_vector->itr_val) { 3221 q_vector->itr_val = new_val; 3222 q_vector->set_itr = 1; 3223 } 3224 clear_counts: 3225 q_vector->rx.total_bytes = 0; 3226 q_vector->rx.total_packets = 0; 3227 q_vector->tx.total_bytes = 0; 3228 q_vector->tx.total_packets = 0; 3229 } 3230 3231 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 3232 { 3233 struct igc_adapter *adapter = q_vector->adapter; 3234 struct igc_hw *hw = &adapter->hw; 3235 3236 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 3237 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 3238 if (adapter->num_q_vectors == 1) 3239 igc_set_itr(q_vector); 3240 else 3241 igc_update_ring_itr(q_vector); 3242 } 3243 3244 if (!test_bit(__IGC_DOWN, &adapter->state)) { 3245 if (adapter->msix_entries) 3246 wr32(IGC_EIMS, q_vector->eims_value); 3247 else 3248 igc_irq_enable(adapter); 3249 } 3250 } 3251 3252 static void igc_add_ring(struct igc_ring *ring, 3253 struct igc_ring_container *head) 3254 { 3255 head->ring = ring; 3256 head->count++; 3257 } 3258 3259 /** 3260 * igc_cache_ring_register - Descriptor ring to register mapping 3261 * @adapter: board private structure to initialize 3262 * 3263 * Once we know the feature-set enabled for the device, we'll cache 3264 * the register offset the descriptor ring is assigned to. 3265 */ 3266 static void igc_cache_ring_register(struct igc_adapter *adapter) 3267 { 3268 int i = 0, j = 0; 3269 3270 switch (adapter->hw.mac.type) { 3271 case igc_i225: 3272 /* Fall through */ 3273 default: 3274 for (; i < adapter->num_rx_queues; i++) 3275 adapter->rx_ring[i]->reg_idx = i; 3276 for (; j < adapter->num_tx_queues; j++) 3277 adapter->tx_ring[j]->reg_idx = j; 3278 break; 3279 } 3280 } 3281 3282 /** 3283 * igc_poll - NAPI Rx polling callback 3284 * @napi: napi polling structure 3285 * @budget: count of how many packets we should handle 3286 */ 3287 static int igc_poll(struct napi_struct *napi, int budget) 3288 { 3289 struct igc_q_vector *q_vector = container_of(napi, 3290 struct igc_q_vector, 3291 napi); 3292 bool clean_complete = true; 3293 int work_done = 0; 3294 3295 if (q_vector->tx.ring) 3296 clean_complete = igc_clean_tx_irq(q_vector, budget); 3297 3298 if (q_vector->rx.ring) { 3299 int cleaned = igc_clean_rx_irq(q_vector, budget); 3300 3301 work_done += cleaned; 3302 if (cleaned >= budget) 3303 clean_complete = false; 3304 } 3305 3306 /* If all work not completed, return budget and keep polling */ 3307 if (!clean_complete) 3308 return budget; 3309 3310 /* Exit the polling mode, but don't re-enable interrupts if stack might 3311 * poll us due to busy-polling 3312 */ 3313 if (likely(napi_complete_done(napi, work_done))) 3314 igc_ring_irq_enable(q_vector); 3315 3316 return min(work_done, budget - 1); 3317 } 3318 3319 /** 3320 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 3321 * @adapter: board private structure to initialize 3322 * @v_count: q_vectors allocated on adapter, used for ring interleaving 3323 * @v_idx: index of vector in adapter struct 3324 * @txr_count: total number of Tx rings to allocate 3325 * @txr_idx: index of first Tx ring to allocate 3326 * @rxr_count: total number of Rx rings to allocate 3327 * @rxr_idx: index of first Rx ring to allocate 3328 * 3329 * We allocate one q_vector. If allocation fails we return -ENOMEM. 3330 */ 3331 static int igc_alloc_q_vector(struct igc_adapter *adapter, 3332 unsigned int v_count, unsigned int v_idx, 3333 unsigned int txr_count, unsigned int txr_idx, 3334 unsigned int rxr_count, unsigned int rxr_idx) 3335 { 3336 struct igc_q_vector *q_vector; 3337 struct igc_ring *ring; 3338 int ring_count; 3339 3340 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 3341 if (txr_count > 1 || rxr_count > 1) 3342 return -ENOMEM; 3343 3344 ring_count = txr_count + rxr_count; 3345 3346 /* allocate q_vector and rings */ 3347 q_vector = adapter->q_vector[v_idx]; 3348 if (!q_vector) 3349 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), 3350 GFP_KERNEL); 3351 else 3352 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 3353 if (!q_vector) 3354 return -ENOMEM; 3355 3356 /* initialize NAPI */ 3357 netif_napi_add(adapter->netdev, &q_vector->napi, 3358 igc_poll, 64); 3359 3360 /* tie q_vector and adapter together */ 3361 adapter->q_vector[v_idx] = q_vector; 3362 q_vector->adapter = adapter; 3363 3364 /* initialize work limits */ 3365 q_vector->tx.work_limit = adapter->tx_work_limit; 3366 3367 /* initialize ITR configuration */ 3368 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 3369 q_vector->itr_val = IGC_START_ITR; 3370 3371 /* initialize pointer to rings */ 3372 ring = q_vector->ring; 3373 3374 /* initialize ITR */ 3375 if (rxr_count) { 3376 /* rx or rx/tx vector */ 3377 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 3378 q_vector->itr_val = adapter->rx_itr_setting; 3379 } else { 3380 /* tx only vector */ 3381 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 3382 q_vector->itr_val = adapter->tx_itr_setting; 3383 } 3384 3385 if (txr_count) { 3386 /* assign generic ring traits */ 3387 ring->dev = &adapter->pdev->dev; 3388 ring->netdev = adapter->netdev; 3389 3390 /* configure backlink on ring */ 3391 ring->q_vector = q_vector; 3392 3393 /* update q_vector Tx values */ 3394 igc_add_ring(ring, &q_vector->tx); 3395 3396 /* apply Tx specific ring traits */ 3397 ring->count = adapter->tx_ring_count; 3398 ring->queue_index = txr_idx; 3399 3400 /* assign ring to adapter */ 3401 adapter->tx_ring[txr_idx] = ring; 3402 3403 /* push pointer to next ring */ 3404 ring++; 3405 } 3406 3407 if (rxr_count) { 3408 /* assign generic ring traits */ 3409 ring->dev = &adapter->pdev->dev; 3410 ring->netdev = adapter->netdev; 3411 3412 /* configure backlink on ring */ 3413 ring->q_vector = q_vector; 3414 3415 /* update q_vector Rx values */ 3416 igc_add_ring(ring, &q_vector->rx); 3417 3418 /* apply Rx specific ring traits */ 3419 ring->count = adapter->rx_ring_count; 3420 ring->queue_index = rxr_idx; 3421 3422 /* assign ring to adapter */ 3423 adapter->rx_ring[rxr_idx] = ring; 3424 } 3425 3426 return 0; 3427 } 3428 3429 /** 3430 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 3431 * @adapter: board private structure to initialize 3432 * 3433 * We allocate one q_vector per queue interrupt. If allocation fails we 3434 * return -ENOMEM. 3435 */ 3436 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 3437 { 3438 int rxr_remaining = adapter->num_rx_queues; 3439 int txr_remaining = adapter->num_tx_queues; 3440 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 3441 int q_vectors = adapter->num_q_vectors; 3442 int err; 3443 3444 if (q_vectors >= (rxr_remaining + txr_remaining)) { 3445 for (; rxr_remaining; v_idx++) { 3446 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3447 0, 0, 1, rxr_idx); 3448 3449 if (err) 3450 goto err_out; 3451 3452 /* update counts and index */ 3453 rxr_remaining--; 3454 rxr_idx++; 3455 } 3456 } 3457 3458 for (; v_idx < q_vectors; v_idx++) { 3459 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 3460 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 3461 3462 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 3463 tqpv, txr_idx, rqpv, rxr_idx); 3464 3465 if (err) 3466 goto err_out; 3467 3468 /* update counts and index */ 3469 rxr_remaining -= rqpv; 3470 txr_remaining -= tqpv; 3471 rxr_idx++; 3472 txr_idx++; 3473 } 3474 3475 return 0; 3476 3477 err_out: 3478 adapter->num_tx_queues = 0; 3479 adapter->num_rx_queues = 0; 3480 adapter->num_q_vectors = 0; 3481 3482 while (v_idx--) 3483 igc_free_q_vector(adapter, v_idx); 3484 3485 return -ENOMEM; 3486 } 3487 3488 /** 3489 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 3490 * @adapter: Pointer to adapter structure 3491 * @msix: boolean for MSI-X capability 3492 * 3493 * This function initializes the interrupts and allocates all of the queues. 3494 */ 3495 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 3496 { 3497 struct net_device *dev = adapter->netdev; 3498 int err = 0; 3499 3500 igc_set_interrupt_capability(adapter, msix); 3501 3502 err = igc_alloc_q_vectors(adapter); 3503 if (err) { 3504 netdev_err(dev, "Unable to allocate memory for vectors\n"); 3505 goto err_alloc_q_vectors; 3506 } 3507 3508 igc_cache_ring_register(adapter); 3509 3510 return 0; 3511 3512 err_alloc_q_vectors: 3513 igc_reset_interrupt_capability(adapter); 3514 return err; 3515 } 3516 3517 /** 3518 * igc_sw_init - Initialize general software structures (struct igc_adapter) 3519 * @adapter: board private structure to initialize 3520 * 3521 * igc_sw_init initializes the Adapter private data structure. 3522 * Fields are initialized based on PCI device information and 3523 * OS network device settings (MTU size). 3524 */ 3525 static int igc_sw_init(struct igc_adapter *adapter) 3526 { 3527 struct net_device *netdev = adapter->netdev; 3528 struct pci_dev *pdev = adapter->pdev; 3529 struct igc_hw *hw = &adapter->hw; 3530 3531 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 3532 3533 /* set default ring sizes */ 3534 adapter->tx_ring_count = IGC_DEFAULT_TXD; 3535 adapter->rx_ring_count = IGC_DEFAULT_RXD; 3536 3537 /* set default ITR values */ 3538 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 3539 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 3540 3541 /* set default work limits */ 3542 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 3543 3544 /* adjust max frame to be at least the size of a standard frame */ 3545 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 3546 VLAN_HLEN; 3547 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 3548 3549 mutex_init(&adapter->nfc_rule_lock); 3550 INIT_LIST_HEAD(&adapter->nfc_rule_list); 3551 adapter->nfc_rule_count = 0; 3552 3553 spin_lock_init(&adapter->stats64_lock); 3554 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 3555 adapter->flags |= IGC_FLAG_HAS_MSIX; 3556 3557 igc_init_queue_configuration(adapter); 3558 3559 /* This call may decrease the number of queues */ 3560 if (igc_init_interrupt_scheme(adapter, true)) { 3561 netdev_err(netdev, "Unable to allocate memory for queues\n"); 3562 return -ENOMEM; 3563 } 3564 3565 /* Explicitly disable IRQ since the NIC can be in any state. */ 3566 igc_irq_disable(adapter); 3567 3568 set_bit(__IGC_DOWN, &adapter->state); 3569 3570 return 0; 3571 } 3572 3573 /** 3574 * igc_up - Open the interface and prepare it to handle traffic 3575 * @adapter: board private structure 3576 */ 3577 void igc_up(struct igc_adapter *adapter) 3578 { 3579 struct igc_hw *hw = &adapter->hw; 3580 int i = 0; 3581 3582 /* hardware has been reset, we need to reload some things */ 3583 igc_configure(adapter); 3584 3585 clear_bit(__IGC_DOWN, &adapter->state); 3586 3587 for (i = 0; i < adapter->num_q_vectors; i++) 3588 napi_enable(&adapter->q_vector[i]->napi); 3589 3590 if (adapter->msix_entries) 3591 igc_configure_msix(adapter); 3592 else 3593 igc_assign_vector(adapter->q_vector[0], 0); 3594 3595 /* Clear any pending interrupts. */ 3596 rd32(IGC_ICR); 3597 igc_irq_enable(adapter); 3598 3599 netif_tx_start_all_queues(adapter->netdev); 3600 3601 /* start the watchdog. */ 3602 hw->mac.get_link_status = 1; 3603 schedule_work(&adapter->watchdog_task); 3604 } 3605 3606 /** 3607 * igc_update_stats - Update the board statistics counters 3608 * @adapter: board private structure 3609 */ 3610 void igc_update_stats(struct igc_adapter *adapter) 3611 { 3612 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 3613 struct pci_dev *pdev = adapter->pdev; 3614 struct igc_hw *hw = &adapter->hw; 3615 u64 _bytes, _packets; 3616 u64 bytes, packets; 3617 unsigned int start; 3618 u32 mpc; 3619 int i; 3620 3621 /* Prevent stats update while adapter is being reset, or if the pci 3622 * connection is down. 3623 */ 3624 if (adapter->link_speed == 0) 3625 return; 3626 if (pci_channel_offline(pdev)) 3627 return; 3628 3629 packets = 0; 3630 bytes = 0; 3631 3632 rcu_read_lock(); 3633 for (i = 0; i < adapter->num_rx_queues; i++) { 3634 struct igc_ring *ring = adapter->rx_ring[i]; 3635 u32 rqdpc = rd32(IGC_RQDPC(i)); 3636 3637 if (hw->mac.type >= igc_i225) 3638 wr32(IGC_RQDPC(i), 0); 3639 3640 if (rqdpc) { 3641 ring->rx_stats.drops += rqdpc; 3642 net_stats->rx_fifo_errors += rqdpc; 3643 } 3644 3645 do { 3646 start = u64_stats_fetch_begin_irq(&ring->rx_syncp); 3647 _bytes = ring->rx_stats.bytes; 3648 _packets = ring->rx_stats.packets; 3649 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); 3650 bytes += _bytes; 3651 packets += _packets; 3652 } 3653 3654 net_stats->rx_bytes = bytes; 3655 net_stats->rx_packets = packets; 3656 3657 packets = 0; 3658 bytes = 0; 3659 for (i = 0; i < adapter->num_tx_queues; i++) { 3660 struct igc_ring *ring = adapter->tx_ring[i]; 3661 3662 do { 3663 start = u64_stats_fetch_begin_irq(&ring->tx_syncp); 3664 _bytes = ring->tx_stats.bytes; 3665 _packets = ring->tx_stats.packets; 3666 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); 3667 bytes += _bytes; 3668 packets += _packets; 3669 } 3670 net_stats->tx_bytes = bytes; 3671 net_stats->tx_packets = packets; 3672 rcu_read_unlock(); 3673 3674 /* read stats registers */ 3675 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 3676 adapter->stats.gprc += rd32(IGC_GPRC); 3677 adapter->stats.gorc += rd32(IGC_GORCL); 3678 rd32(IGC_GORCH); /* clear GORCL */ 3679 adapter->stats.bprc += rd32(IGC_BPRC); 3680 adapter->stats.mprc += rd32(IGC_MPRC); 3681 adapter->stats.roc += rd32(IGC_ROC); 3682 3683 adapter->stats.prc64 += rd32(IGC_PRC64); 3684 adapter->stats.prc127 += rd32(IGC_PRC127); 3685 adapter->stats.prc255 += rd32(IGC_PRC255); 3686 adapter->stats.prc511 += rd32(IGC_PRC511); 3687 adapter->stats.prc1023 += rd32(IGC_PRC1023); 3688 adapter->stats.prc1522 += rd32(IGC_PRC1522); 3689 3690 mpc = rd32(IGC_MPC); 3691 adapter->stats.mpc += mpc; 3692 net_stats->rx_fifo_errors += mpc; 3693 adapter->stats.scc += rd32(IGC_SCC); 3694 adapter->stats.ecol += rd32(IGC_ECOL); 3695 adapter->stats.mcc += rd32(IGC_MCC); 3696 adapter->stats.latecol += rd32(IGC_LATECOL); 3697 adapter->stats.dc += rd32(IGC_DC); 3698 adapter->stats.rlec += rd32(IGC_RLEC); 3699 adapter->stats.xonrxc += rd32(IGC_XONRXC); 3700 adapter->stats.xontxc += rd32(IGC_XONTXC); 3701 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 3702 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 3703 adapter->stats.fcruc += rd32(IGC_FCRUC); 3704 adapter->stats.gptc += rd32(IGC_GPTC); 3705 adapter->stats.gotc += rd32(IGC_GOTCL); 3706 rd32(IGC_GOTCH); /* clear GOTCL */ 3707 adapter->stats.rnbc += rd32(IGC_RNBC); 3708 adapter->stats.ruc += rd32(IGC_RUC); 3709 adapter->stats.rfc += rd32(IGC_RFC); 3710 adapter->stats.rjc += rd32(IGC_RJC); 3711 adapter->stats.tor += rd32(IGC_TORH); 3712 adapter->stats.tot += rd32(IGC_TOTH); 3713 adapter->stats.tpr += rd32(IGC_TPR); 3714 3715 adapter->stats.ptc64 += rd32(IGC_PTC64); 3716 adapter->stats.ptc127 += rd32(IGC_PTC127); 3717 adapter->stats.ptc255 += rd32(IGC_PTC255); 3718 adapter->stats.ptc511 += rd32(IGC_PTC511); 3719 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 3720 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 3721 3722 adapter->stats.mptc += rd32(IGC_MPTC); 3723 adapter->stats.bptc += rd32(IGC_BPTC); 3724 3725 adapter->stats.tpt += rd32(IGC_TPT); 3726 adapter->stats.colc += rd32(IGC_COLC); 3727 adapter->stats.colc += rd32(IGC_RERC); 3728 3729 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 3730 3731 adapter->stats.tsctc += rd32(IGC_TSCTC); 3732 3733 adapter->stats.iac += rd32(IGC_IAC); 3734 adapter->stats.icrxoc += rd32(IGC_ICRXOC); 3735 adapter->stats.icrxptc += rd32(IGC_ICRXPTC); 3736 adapter->stats.icrxatc += rd32(IGC_ICRXATC); 3737 adapter->stats.ictxptc += rd32(IGC_ICTXPTC); 3738 adapter->stats.ictxatc += rd32(IGC_ICTXATC); 3739 adapter->stats.ictxqec += rd32(IGC_ICTXQEC); 3740 adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); 3741 adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); 3742 3743 /* Fill out the OS statistics structure */ 3744 net_stats->multicast = adapter->stats.mprc; 3745 net_stats->collisions = adapter->stats.colc; 3746 3747 /* Rx Errors */ 3748 3749 /* RLEC on some newer hardware can be incorrect so build 3750 * our own version based on RUC and ROC 3751 */ 3752 net_stats->rx_errors = adapter->stats.rxerrc + 3753 adapter->stats.crcerrs + adapter->stats.algnerrc + 3754 adapter->stats.ruc + adapter->stats.roc + 3755 adapter->stats.cexterr; 3756 net_stats->rx_length_errors = adapter->stats.ruc + 3757 adapter->stats.roc; 3758 net_stats->rx_crc_errors = adapter->stats.crcerrs; 3759 net_stats->rx_frame_errors = adapter->stats.algnerrc; 3760 net_stats->rx_missed_errors = adapter->stats.mpc; 3761 3762 /* Tx Errors */ 3763 net_stats->tx_errors = adapter->stats.ecol + 3764 adapter->stats.latecol; 3765 net_stats->tx_aborted_errors = adapter->stats.ecol; 3766 net_stats->tx_window_errors = adapter->stats.latecol; 3767 net_stats->tx_carrier_errors = adapter->stats.tncrs; 3768 3769 /* Tx Dropped needs to be maintained elsewhere */ 3770 3771 /* Management Stats */ 3772 adapter->stats.mgptc += rd32(IGC_MGTPTC); 3773 adapter->stats.mgprc += rd32(IGC_MGTPRC); 3774 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 3775 } 3776 3777 /** 3778 * igc_down - Close the interface 3779 * @adapter: board private structure 3780 */ 3781 void igc_down(struct igc_adapter *adapter) 3782 { 3783 struct net_device *netdev = adapter->netdev; 3784 struct igc_hw *hw = &adapter->hw; 3785 u32 tctl, rctl; 3786 int i = 0; 3787 3788 set_bit(__IGC_DOWN, &adapter->state); 3789 3790 /* disable receives in the hardware */ 3791 rctl = rd32(IGC_RCTL); 3792 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 3793 /* flush and sleep below */ 3794 3795 /* set trans_start so we don't get spurious watchdogs during reset */ 3796 netif_trans_update(netdev); 3797 3798 netif_carrier_off(netdev); 3799 netif_tx_stop_all_queues(netdev); 3800 3801 /* disable transmits in the hardware */ 3802 tctl = rd32(IGC_TCTL); 3803 tctl &= ~IGC_TCTL_EN; 3804 wr32(IGC_TCTL, tctl); 3805 /* flush both disables and wait for them to finish */ 3806 wrfl(); 3807 usleep_range(10000, 20000); 3808 3809 igc_irq_disable(adapter); 3810 3811 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 3812 3813 for (i = 0; i < adapter->num_q_vectors; i++) { 3814 if (adapter->q_vector[i]) { 3815 napi_synchronize(&adapter->q_vector[i]->napi); 3816 napi_disable(&adapter->q_vector[i]->napi); 3817 } 3818 } 3819 3820 del_timer_sync(&adapter->watchdog_timer); 3821 del_timer_sync(&adapter->phy_info_timer); 3822 3823 /* record the stats before reset*/ 3824 spin_lock(&adapter->stats64_lock); 3825 igc_update_stats(adapter); 3826 spin_unlock(&adapter->stats64_lock); 3827 3828 adapter->link_speed = 0; 3829 adapter->link_duplex = 0; 3830 3831 if (!pci_channel_offline(adapter->pdev)) 3832 igc_reset(adapter); 3833 3834 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 3835 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 3836 3837 igc_clean_all_tx_rings(adapter); 3838 igc_clean_all_rx_rings(adapter); 3839 } 3840 3841 void igc_reinit_locked(struct igc_adapter *adapter) 3842 { 3843 WARN_ON(in_interrupt()); 3844 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3845 usleep_range(1000, 2000); 3846 igc_down(adapter); 3847 igc_up(adapter); 3848 clear_bit(__IGC_RESETTING, &adapter->state); 3849 } 3850 3851 static void igc_reset_task(struct work_struct *work) 3852 { 3853 struct igc_adapter *adapter; 3854 3855 adapter = container_of(work, struct igc_adapter, reset_task); 3856 3857 igc_rings_dump(adapter); 3858 igc_regs_dump(adapter); 3859 netdev_err(adapter->netdev, "Reset adapter\n"); 3860 igc_reinit_locked(adapter); 3861 } 3862 3863 /** 3864 * igc_change_mtu - Change the Maximum Transfer Unit 3865 * @netdev: network interface device structure 3866 * @new_mtu: new value for maximum frame size 3867 * 3868 * Returns 0 on success, negative on failure 3869 */ 3870 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 3871 { 3872 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 3873 struct igc_adapter *adapter = netdev_priv(netdev); 3874 3875 /* adjust max frame to be at least the size of a standard frame */ 3876 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 3877 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 3878 3879 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 3880 usleep_range(1000, 2000); 3881 3882 /* igc_down has a dependency on max_frame_size */ 3883 adapter->max_frame_size = max_frame; 3884 3885 if (netif_running(netdev)) 3886 igc_down(adapter); 3887 3888 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 3889 netdev->mtu = new_mtu; 3890 3891 if (netif_running(netdev)) 3892 igc_up(adapter); 3893 else 3894 igc_reset(adapter); 3895 3896 clear_bit(__IGC_RESETTING, &adapter->state); 3897 3898 return 0; 3899 } 3900 3901 /** 3902 * igc_get_stats - Get System Network Statistics 3903 * @netdev: network interface device structure 3904 * 3905 * Returns the address of the device statistics structure. 3906 * The statistics are updated here and also from the timer callback. 3907 */ 3908 static struct net_device_stats *igc_get_stats(struct net_device *netdev) 3909 { 3910 struct igc_adapter *adapter = netdev_priv(netdev); 3911 3912 if (!test_bit(__IGC_RESETTING, &adapter->state)) 3913 igc_update_stats(adapter); 3914 3915 /* only return the current stats */ 3916 return &netdev->stats; 3917 } 3918 3919 static netdev_features_t igc_fix_features(struct net_device *netdev, 3920 netdev_features_t features) 3921 { 3922 /* Since there is no support for separate Rx/Tx vlan accel 3923 * enable/disable make sure Tx flag is always in same state as Rx. 3924 */ 3925 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3926 features |= NETIF_F_HW_VLAN_CTAG_TX; 3927 else 3928 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3929 3930 return features; 3931 } 3932 3933 static int igc_set_features(struct net_device *netdev, 3934 netdev_features_t features) 3935 { 3936 netdev_features_t changed = netdev->features ^ features; 3937 struct igc_adapter *adapter = netdev_priv(netdev); 3938 3939 /* Add VLAN support */ 3940 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 3941 return 0; 3942 3943 if (!(features & NETIF_F_NTUPLE)) 3944 igc_flush_nfc_rules(adapter); 3945 3946 netdev->features = features; 3947 3948 if (netif_running(netdev)) 3949 igc_reinit_locked(adapter); 3950 else 3951 igc_reset(adapter); 3952 3953 return 1; 3954 } 3955 3956 static netdev_features_t 3957 igc_features_check(struct sk_buff *skb, struct net_device *dev, 3958 netdev_features_t features) 3959 { 3960 unsigned int network_hdr_len, mac_hdr_len; 3961 3962 /* Make certain the headers can be described by a context descriptor */ 3963 mac_hdr_len = skb_network_header(skb) - skb->data; 3964 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 3965 return features & ~(NETIF_F_HW_CSUM | 3966 NETIF_F_SCTP_CRC | 3967 NETIF_F_HW_VLAN_CTAG_TX | 3968 NETIF_F_TSO | 3969 NETIF_F_TSO6); 3970 3971 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3972 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 3973 return features & ~(NETIF_F_HW_CSUM | 3974 NETIF_F_SCTP_CRC | 3975 NETIF_F_TSO | 3976 NETIF_F_TSO6); 3977 3978 /* We can only support IPv4 TSO in tunnels if we can mangle the 3979 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3980 */ 3981 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3982 features &= ~NETIF_F_TSO; 3983 3984 return features; 3985 } 3986 3987 static void igc_tsync_interrupt(struct igc_adapter *adapter) 3988 { 3989 struct igc_hw *hw = &adapter->hw; 3990 u32 tsicr = rd32(IGC_TSICR); 3991 u32 ack = 0; 3992 3993 if (tsicr & IGC_TSICR_TXTS) { 3994 /* retrieve hardware timestamp */ 3995 schedule_work(&adapter->ptp_tx_work); 3996 ack |= IGC_TSICR_TXTS; 3997 } 3998 3999 /* acknowledge the interrupts */ 4000 wr32(IGC_TSICR, ack); 4001 } 4002 4003 /** 4004 * igc_msix_other - msix other interrupt handler 4005 * @irq: interrupt number 4006 * @data: pointer to a q_vector 4007 */ 4008 static irqreturn_t igc_msix_other(int irq, void *data) 4009 { 4010 struct igc_adapter *adapter = data; 4011 struct igc_hw *hw = &adapter->hw; 4012 u32 icr = rd32(IGC_ICR); 4013 4014 /* reading ICR causes bit 31 of EICR to be cleared */ 4015 if (icr & IGC_ICR_DRSTA) 4016 schedule_work(&adapter->reset_task); 4017 4018 if (icr & IGC_ICR_DOUTSYNC) { 4019 /* HW is reporting DMA is out of sync */ 4020 adapter->stats.doosync++; 4021 } 4022 4023 if (icr & IGC_ICR_LSC) { 4024 hw->mac.get_link_status = 1; 4025 /* guard against interrupt when we're going down */ 4026 if (!test_bit(__IGC_DOWN, &adapter->state)) 4027 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4028 } 4029 4030 if (icr & IGC_ICR_TS) 4031 igc_tsync_interrupt(adapter); 4032 4033 wr32(IGC_EIMS, adapter->eims_other); 4034 4035 return IRQ_HANDLED; 4036 } 4037 4038 static void igc_write_itr(struct igc_q_vector *q_vector) 4039 { 4040 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 4041 4042 if (!q_vector->set_itr) 4043 return; 4044 4045 if (!itr_val) 4046 itr_val = IGC_ITR_VAL_MASK; 4047 4048 itr_val |= IGC_EITR_CNT_IGNR; 4049 4050 writel(itr_val, q_vector->itr_register); 4051 q_vector->set_itr = 0; 4052 } 4053 4054 static irqreturn_t igc_msix_ring(int irq, void *data) 4055 { 4056 struct igc_q_vector *q_vector = data; 4057 4058 /* Write the ITR value calculated from the previous interrupt. */ 4059 igc_write_itr(q_vector); 4060 4061 napi_schedule(&q_vector->napi); 4062 4063 return IRQ_HANDLED; 4064 } 4065 4066 /** 4067 * igc_request_msix - Initialize MSI-X interrupts 4068 * @adapter: Pointer to adapter structure 4069 * 4070 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 4071 * kernel. 4072 */ 4073 static int igc_request_msix(struct igc_adapter *adapter) 4074 { 4075 int i = 0, err = 0, vector = 0, free_vector = 0; 4076 struct net_device *netdev = adapter->netdev; 4077 4078 err = request_irq(adapter->msix_entries[vector].vector, 4079 &igc_msix_other, 0, netdev->name, adapter); 4080 if (err) 4081 goto err_out; 4082 4083 for (i = 0; i < adapter->num_q_vectors; i++) { 4084 struct igc_q_vector *q_vector = adapter->q_vector[i]; 4085 4086 vector++; 4087 4088 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 4089 4090 if (q_vector->rx.ring && q_vector->tx.ring) 4091 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 4092 q_vector->rx.ring->queue_index); 4093 else if (q_vector->tx.ring) 4094 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 4095 q_vector->tx.ring->queue_index); 4096 else if (q_vector->rx.ring) 4097 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 4098 q_vector->rx.ring->queue_index); 4099 else 4100 sprintf(q_vector->name, "%s-unused", netdev->name); 4101 4102 err = request_irq(adapter->msix_entries[vector].vector, 4103 igc_msix_ring, 0, q_vector->name, 4104 q_vector); 4105 if (err) 4106 goto err_free; 4107 } 4108 4109 igc_configure_msix(adapter); 4110 return 0; 4111 4112 err_free: 4113 /* free already assigned IRQs */ 4114 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 4115 4116 vector--; 4117 for (i = 0; i < vector; i++) { 4118 free_irq(adapter->msix_entries[free_vector++].vector, 4119 adapter->q_vector[i]); 4120 } 4121 err_out: 4122 return err; 4123 } 4124 4125 /** 4126 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 4127 * @adapter: Pointer to adapter structure 4128 * 4129 * This function resets the device so that it has 0 rx queues, tx queues, and 4130 * MSI-X interrupts allocated. 4131 */ 4132 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 4133 { 4134 igc_free_q_vectors(adapter); 4135 igc_reset_interrupt_capability(adapter); 4136 } 4137 4138 /* Need to wait a few seconds after link up to get diagnostic information from 4139 * the phy 4140 */ 4141 static void igc_update_phy_info(struct timer_list *t) 4142 { 4143 struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); 4144 4145 igc_get_phy_info(&adapter->hw); 4146 } 4147 4148 /** 4149 * igc_has_link - check shared code for link and determine up/down 4150 * @adapter: pointer to driver private info 4151 */ 4152 bool igc_has_link(struct igc_adapter *adapter) 4153 { 4154 struct igc_hw *hw = &adapter->hw; 4155 bool link_active = false; 4156 4157 /* get_link_status is set on LSC (link status) interrupt or 4158 * rx sequence error interrupt. get_link_status will stay 4159 * false until the igc_check_for_link establishes link 4160 * for copper adapters ONLY 4161 */ 4162 switch (hw->phy.media_type) { 4163 case igc_media_type_copper: 4164 if (!hw->mac.get_link_status) 4165 return true; 4166 hw->mac.ops.check_for_link(hw); 4167 link_active = !hw->mac.get_link_status; 4168 break; 4169 default: 4170 case igc_media_type_unknown: 4171 break; 4172 } 4173 4174 if (hw->mac.type == igc_i225 && 4175 hw->phy.id == I225_I_PHY_ID) { 4176 if (!netif_carrier_ok(adapter->netdev)) { 4177 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4178 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 4179 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 4180 adapter->link_check_timeout = jiffies; 4181 } 4182 } 4183 4184 return link_active; 4185 } 4186 4187 /** 4188 * igc_watchdog - Timer Call-back 4189 * @t: timer for the watchdog 4190 */ 4191 static void igc_watchdog(struct timer_list *t) 4192 { 4193 struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); 4194 /* Do the rest outside of interrupt context */ 4195 schedule_work(&adapter->watchdog_task); 4196 } 4197 4198 static void igc_watchdog_task(struct work_struct *work) 4199 { 4200 struct igc_adapter *adapter = container_of(work, 4201 struct igc_adapter, 4202 watchdog_task); 4203 struct net_device *netdev = adapter->netdev; 4204 struct igc_hw *hw = &adapter->hw; 4205 struct igc_phy_info *phy = &hw->phy; 4206 u16 phy_data, retry_count = 20; 4207 u32 link; 4208 int i; 4209 4210 link = igc_has_link(adapter); 4211 4212 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 4213 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 4214 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 4215 else 4216 link = false; 4217 } 4218 4219 if (link) { 4220 /* Cancel scheduled suspend requests. */ 4221 pm_runtime_resume(netdev->dev.parent); 4222 4223 if (!netif_carrier_ok(netdev)) { 4224 u32 ctrl; 4225 4226 hw->mac.ops.get_speed_and_duplex(hw, 4227 &adapter->link_speed, 4228 &adapter->link_duplex); 4229 4230 ctrl = rd32(IGC_CTRL); 4231 /* Link status message must follow this format */ 4232 netdev_info(netdev, 4233 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 4234 adapter->link_speed, 4235 adapter->link_duplex == FULL_DUPLEX ? 4236 "Full" : "Half", 4237 (ctrl & IGC_CTRL_TFCE) && 4238 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 4239 (ctrl & IGC_CTRL_RFCE) ? "RX" : 4240 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 4241 4242 /* disable EEE if enabled */ 4243 if ((adapter->flags & IGC_FLAG_EEE) && 4244 adapter->link_duplex == HALF_DUPLEX) { 4245 netdev_info(netdev, 4246 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 4247 adapter->hw.dev_spec._base.eee_enable = false; 4248 adapter->flags &= ~IGC_FLAG_EEE; 4249 } 4250 4251 /* check if SmartSpeed worked */ 4252 igc_check_downshift(hw); 4253 if (phy->speed_downgraded) 4254 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 4255 4256 /* adjust timeout factor according to speed/duplex */ 4257 adapter->tx_timeout_factor = 1; 4258 switch (adapter->link_speed) { 4259 case SPEED_10: 4260 adapter->tx_timeout_factor = 14; 4261 break; 4262 case SPEED_100: 4263 /* maybe add some timeout factor ? */ 4264 break; 4265 } 4266 4267 if (adapter->link_speed != SPEED_1000) 4268 goto no_wait; 4269 4270 /* wait for Remote receiver status OK */ 4271 retry_read_status: 4272 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 4273 &phy_data)) { 4274 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 4275 retry_count) { 4276 msleep(100); 4277 retry_count--; 4278 goto retry_read_status; 4279 } else if (!retry_count) { 4280 netdev_err(netdev, "exceed max 2 second\n"); 4281 } 4282 } else { 4283 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 4284 } 4285 no_wait: 4286 netif_carrier_on(netdev); 4287 4288 /* link state has changed, schedule phy info update */ 4289 if (!test_bit(__IGC_DOWN, &adapter->state)) 4290 mod_timer(&adapter->phy_info_timer, 4291 round_jiffies(jiffies + 2 * HZ)); 4292 } 4293 } else { 4294 if (netif_carrier_ok(netdev)) { 4295 adapter->link_speed = 0; 4296 adapter->link_duplex = 0; 4297 4298 /* Links status message must follow this format */ 4299 netdev_info(netdev, "NIC Link is Down\n"); 4300 netif_carrier_off(netdev); 4301 4302 /* link state has changed, schedule phy info update */ 4303 if (!test_bit(__IGC_DOWN, &adapter->state)) 4304 mod_timer(&adapter->phy_info_timer, 4305 round_jiffies(jiffies + 2 * HZ)); 4306 4307 /* link is down, time to check for alternate media */ 4308 if (adapter->flags & IGC_FLAG_MAS_ENABLE) { 4309 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4310 schedule_work(&adapter->reset_task); 4311 /* return immediately */ 4312 return; 4313 } 4314 } 4315 pm_schedule_suspend(netdev->dev.parent, 4316 MSEC_PER_SEC * 5); 4317 4318 /* also check for alternate media here */ 4319 } else if (!netif_carrier_ok(netdev) && 4320 (adapter->flags & IGC_FLAG_MAS_ENABLE)) { 4321 if (adapter->flags & IGC_FLAG_MEDIA_RESET) { 4322 schedule_work(&adapter->reset_task); 4323 /* return immediately */ 4324 return; 4325 } 4326 } 4327 } 4328 4329 spin_lock(&adapter->stats64_lock); 4330 igc_update_stats(adapter); 4331 spin_unlock(&adapter->stats64_lock); 4332 4333 for (i = 0; i < adapter->num_tx_queues; i++) { 4334 struct igc_ring *tx_ring = adapter->tx_ring[i]; 4335 4336 if (!netif_carrier_ok(netdev)) { 4337 /* We've lost link, so the controller stops DMA, 4338 * but we've got queued Tx work that's never going 4339 * to get done, so reset controller to flush Tx. 4340 * (Do the reset outside of interrupt context). 4341 */ 4342 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 4343 adapter->tx_timeout_count++; 4344 schedule_work(&adapter->reset_task); 4345 /* return immediately since reset is imminent */ 4346 return; 4347 } 4348 } 4349 4350 /* Force detection of hung controller every watchdog period */ 4351 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 4352 } 4353 4354 /* Cause software interrupt to ensure Rx ring is cleaned */ 4355 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4356 u32 eics = 0; 4357 4358 for (i = 0; i < adapter->num_q_vectors; i++) 4359 eics |= adapter->q_vector[i]->eims_value; 4360 wr32(IGC_EICS, eics); 4361 } else { 4362 wr32(IGC_ICS, IGC_ICS_RXDMT0); 4363 } 4364 4365 igc_ptp_tx_hang(adapter); 4366 4367 /* Reset the timer */ 4368 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4369 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 4370 mod_timer(&adapter->watchdog_timer, 4371 round_jiffies(jiffies + HZ)); 4372 else 4373 mod_timer(&adapter->watchdog_timer, 4374 round_jiffies(jiffies + 2 * HZ)); 4375 } 4376 } 4377 4378 /** 4379 * igc_intr_msi - Interrupt Handler 4380 * @irq: interrupt number 4381 * @data: pointer to a network interface device structure 4382 */ 4383 static irqreturn_t igc_intr_msi(int irq, void *data) 4384 { 4385 struct igc_adapter *adapter = data; 4386 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4387 struct igc_hw *hw = &adapter->hw; 4388 /* read ICR disables interrupts using IAM */ 4389 u32 icr = rd32(IGC_ICR); 4390 4391 igc_write_itr(q_vector); 4392 4393 if (icr & IGC_ICR_DRSTA) 4394 schedule_work(&adapter->reset_task); 4395 4396 if (icr & IGC_ICR_DOUTSYNC) { 4397 /* HW is reporting DMA is out of sync */ 4398 adapter->stats.doosync++; 4399 } 4400 4401 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4402 hw->mac.get_link_status = 1; 4403 if (!test_bit(__IGC_DOWN, &adapter->state)) 4404 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4405 } 4406 4407 napi_schedule(&q_vector->napi); 4408 4409 return IRQ_HANDLED; 4410 } 4411 4412 /** 4413 * igc_intr - Legacy Interrupt Handler 4414 * @irq: interrupt number 4415 * @data: pointer to a network interface device structure 4416 */ 4417 static irqreturn_t igc_intr(int irq, void *data) 4418 { 4419 struct igc_adapter *adapter = data; 4420 struct igc_q_vector *q_vector = adapter->q_vector[0]; 4421 struct igc_hw *hw = &adapter->hw; 4422 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 4423 * need for the IMC write 4424 */ 4425 u32 icr = rd32(IGC_ICR); 4426 4427 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 4428 * not set, then the adapter didn't send an interrupt 4429 */ 4430 if (!(icr & IGC_ICR_INT_ASSERTED)) 4431 return IRQ_NONE; 4432 4433 igc_write_itr(q_vector); 4434 4435 if (icr & IGC_ICR_DRSTA) 4436 schedule_work(&adapter->reset_task); 4437 4438 if (icr & IGC_ICR_DOUTSYNC) { 4439 /* HW is reporting DMA is out of sync */ 4440 adapter->stats.doosync++; 4441 } 4442 4443 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 4444 hw->mac.get_link_status = 1; 4445 /* guard against interrupt when we're going down */ 4446 if (!test_bit(__IGC_DOWN, &adapter->state)) 4447 mod_timer(&adapter->watchdog_timer, jiffies + 1); 4448 } 4449 4450 napi_schedule(&q_vector->napi); 4451 4452 return IRQ_HANDLED; 4453 } 4454 4455 static void igc_free_irq(struct igc_adapter *adapter) 4456 { 4457 if (adapter->msix_entries) { 4458 int vector = 0, i; 4459 4460 free_irq(adapter->msix_entries[vector++].vector, adapter); 4461 4462 for (i = 0; i < adapter->num_q_vectors; i++) 4463 free_irq(adapter->msix_entries[vector++].vector, 4464 adapter->q_vector[i]); 4465 } else { 4466 free_irq(adapter->pdev->irq, adapter); 4467 } 4468 } 4469 4470 /** 4471 * igc_request_irq - initialize interrupts 4472 * @adapter: Pointer to adapter structure 4473 * 4474 * Attempts to configure interrupts using the best available 4475 * capabilities of the hardware and kernel. 4476 */ 4477 static int igc_request_irq(struct igc_adapter *adapter) 4478 { 4479 struct net_device *netdev = adapter->netdev; 4480 struct pci_dev *pdev = adapter->pdev; 4481 int err = 0; 4482 4483 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 4484 err = igc_request_msix(adapter); 4485 if (!err) 4486 goto request_done; 4487 /* fall back to MSI */ 4488 igc_free_all_tx_resources(adapter); 4489 igc_free_all_rx_resources(adapter); 4490 4491 igc_clear_interrupt_scheme(adapter); 4492 err = igc_init_interrupt_scheme(adapter, false); 4493 if (err) 4494 goto request_done; 4495 igc_setup_all_tx_resources(adapter); 4496 igc_setup_all_rx_resources(adapter); 4497 igc_configure(adapter); 4498 } 4499 4500 igc_assign_vector(adapter->q_vector[0], 0); 4501 4502 if (adapter->flags & IGC_FLAG_HAS_MSI) { 4503 err = request_irq(pdev->irq, &igc_intr_msi, 0, 4504 netdev->name, adapter); 4505 if (!err) 4506 goto request_done; 4507 4508 /* fall back to legacy interrupts */ 4509 igc_reset_interrupt_capability(adapter); 4510 adapter->flags &= ~IGC_FLAG_HAS_MSI; 4511 } 4512 4513 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 4514 netdev->name, adapter); 4515 4516 if (err) 4517 netdev_err(netdev, "Error %d getting interrupt\n", err); 4518 4519 request_done: 4520 return err; 4521 } 4522 4523 /** 4524 * __igc_open - Called when a network interface is made active 4525 * @netdev: network interface device structure 4526 * @resuming: boolean indicating if the device is resuming 4527 * 4528 * Returns 0 on success, negative value on failure 4529 * 4530 * The open entry point is called when a network interface is made 4531 * active by the system (IFF_UP). At this point all resources needed 4532 * for transmit and receive operations are allocated, the interrupt 4533 * handler is registered with the OS, the watchdog timer is started, 4534 * and the stack is notified that the interface is ready. 4535 */ 4536 static int __igc_open(struct net_device *netdev, bool resuming) 4537 { 4538 struct igc_adapter *adapter = netdev_priv(netdev); 4539 struct pci_dev *pdev = adapter->pdev; 4540 struct igc_hw *hw = &adapter->hw; 4541 int err = 0; 4542 int i = 0; 4543 4544 /* disallow open during test */ 4545 4546 if (test_bit(__IGC_TESTING, &adapter->state)) { 4547 WARN_ON(resuming); 4548 return -EBUSY; 4549 } 4550 4551 if (!resuming) 4552 pm_runtime_get_sync(&pdev->dev); 4553 4554 netif_carrier_off(netdev); 4555 4556 /* allocate transmit descriptors */ 4557 err = igc_setup_all_tx_resources(adapter); 4558 if (err) 4559 goto err_setup_tx; 4560 4561 /* allocate receive descriptors */ 4562 err = igc_setup_all_rx_resources(adapter); 4563 if (err) 4564 goto err_setup_rx; 4565 4566 igc_power_up_link(adapter); 4567 4568 igc_configure(adapter); 4569 4570 err = igc_request_irq(adapter); 4571 if (err) 4572 goto err_req_irq; 4573 4574 /* Notify the stack of the actual queue counts. */ 4575 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 4576 if (err) 4577 goto err_set_queues; 4578 4579 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 4580 if (err) 4581 goto err_set_queues; 4582 4583 clear_bit(__IGC_DOWN, &adapter->state); 4584 4585 for (i = 0; i < adapter->num_q_vectors; i++) 4586 napi_enable(&adapter->q_vector[i]->napi); 4587 4588 /* Clear any pending interrupts. */ 4589 rd32(IGC_ICR); 4590 igc_irq_enable(adapter); 4591 4592 if (!resuming) 4593 pm_runtime_put(&pdev->dev); 4594 4595 netif_tx_start_all_queues(netdev); 4596 4597 /* start the watchdog. */ 4598 hw->mac.get_link_status = 1; 4599 schedule_work(&adapter->watchdog_task); 4600 4601 return IGC_SUCCESS; 4602 4603 err_set_queues: 4604 igc_free_irq(adapter); 4605 err_req_irq: 4606 igc_release_hw_control(adapter); 4607 igc_power_down_phy_copper_base(&adapter->hw); 4608 igc_free_all_rx_resources(adapter); 4609 err_setup_rx: 4610 igc_free_all_tx_resources(adapter); 4611 err_setup_tx: 4612 igc_reset(adapter); 4613 if (!resuming) 4614 pm_runtime_put(&pdev->dev); 4615 4616 return err; 4617 } 4618 4619 int igc_open(struct net_device *netdev) 4620 { 4621 return __igc_open(netdev, false); 4622 } 4623 4624 /** 4625 * __igc_close - Disables a network interface 4626 * @netdev: network interface device structure 4627 * @suspending: boolean indicating the device is suspending 4628 * 4629 * Returns 0, this is not allowed to fail 4630 * 4631 * The close entry point is called when an interface is de-activated 4632 * by the OS. The hardware is still under the driver's control, but 4633 * needs to be disabled. A global MAC reset is issued to stop the 4634 * hardware, and all transmit and receive resources are freed. 4635 */ 4636 static int __igc_close(struct net_device *netdev, bool suspending) 4637 { 4638 struct igc_adapter *adapter = netdev_priv(netdev); 4639 struct pci_dev *pdev = adapter->pdev; 4640 4641 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 4642 4643 if (!suspending) 4644 pm_runtime_get_sync(&pdev->dev); 4645 4646 igc_down(adapter); 4647 4648 igc_release_hw_control(adapter); 4649 4650 igc_free_irq(adapter); 4651 4652 igc_free_all_tx_resources(adapter); 4653 igc_free_all_rx_resources(adapter); 4654 4655 if (!suspending) 4656 pm_runtime_put_sync(&pdev->dev); 4657 4658 return 0; 4659 } 4660 4661 int igc_close(struct net_device *netdev) 4662 { 4663 if (netif_device_present(netdev) || netdev->dismantle) 4664 return __igc_close(netdev, false); 4665 return 0; 4666 } 4667 4668 /** 4669 * igc_ioctl - Access the hwtstamp interface 4670 * @netdev: network interface device structure 4671 * @ifreq: interface request data 4672 * @cmd: ioctl command 4673 **/ 4674 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 4675 { 4676 switch (cmd) { 4677 case SIOCGHWTSTAMP: 4678 return igc_ptp_get_ts_config(netdev, ifr); 4679 case SIOCSHWTSTAMP: 4680 return igc_ptp_set_ts_config(netdev, ifr); 4681 default: 4682 return -EOPNOTSUPP; 4683 } 4684 } 4685 4686 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 4687 bool enable) 4688 { 4689 struct igc_ring *ring; 4690 int i; 4691 4692 if (queue < 0 || queue >= adapter->num_tx_queues) 4693 return -EINVAL; 4694 4695 ring = adapter->tx_ring[queue]; 4696 ring->launchtime_enable = enable; 4697 4698 if (adapter->base_time) 4699 return 0; 4700 4701 adapter->cycle_time = NSEC_PER_SEC; 4702 4703 for (i = 0; i < adapter->num_tx_queues; i++) { 4704 ring = adapter->tx_ring[i]; 4705 ring->start_time = 0; 4706 ring->end_time = NSEC_PER_SEC; 4707 } 4708 4709 return 0; 4710 } 4711 4712 static bool validate_schedule(const struct tc_taprio_qopt_offload *qopt) 4713 { 4714 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 4715 size_t n; 4716 4717 if (qopt->cycle_time_extension) 4718 return false; 4719 4720 for (n = 0; n < qopt->num_entries; n++) { 4721 const struct tc_taprio_sched_entry *e; 4722 int i; 4723 4724 e = &qopt->entries[n]; 4725 4726 /* i225 only supports "global" frame preemption 4727 * settings. 4728 */ 4729 if (e->command != TC_TAPRIO_CMD_SET_GATES) 4730 return false; 4731 4732 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4733 if (e->gate_mask & BIT(i)) 4734 queue_uses[i]++; 4735 4736 if (queue_uses[i] > 1) 4737 return false; 4738 } 4739 } 4740 4741 return true; 4742 } 4743 4744 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 4745 struct tc_etf_qopt_offload *qopt) 4746 { 4747 struct igc_hw *hw = &adapter->hw; 4748 int err; 4749 4750 if (hw->mac.type != igc_i225) 4751 return -EOPNOTSUPP; 4752 4753 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 4754 if (err) 4755 return err; 4756 4757 return igc_tsn_offload_apply(adapter); 4758 } 4759 4760 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 4761 struct tc_taprio_qopt_offload *qopt) 4762 { 4763 u32 start_time = 0, end_time = 0; 4764 size_t n; 4765 4766 if (!qopt->enable) { 4767 adapter->base_time = 0; 4768 return 0; 4769 } 4770 4771 if (adapter->base_time) 4772 return -EALREADY; 4773 4774 if (!validate_schedule(qopt)) 4775 return -EINVAL; 4776 4777 adapter->cycle_time = qopt->cycle_time; 4778 adapter->base_time = qopt->base_time; 4779 4780 /* FIXME: be a little smarter about cases when the gate for a 4781 * queue stays open for more than one entry. 4782 */ 4783 for (n = 0; n < qopt->num_entries; n++) { 4784 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 4785 int i; 4786 4787 end_time += e->interval; 4788 4789 for (i = 0; i < IGC_MAX_TX_QUEUES; i++) { 4790 struct igc_ring *ring = adapter->tx_ring[i]; 4791 4792 if (!(e->gate_mask & BIT(i))) 4793 continue; 4794 4795 ring->start_time = start_time; 4796 ring->end_time = end_time; 4797 } 4798 4799 start_time += e->interval; 4800 } 4801 4802 return 0; 4803 } 4804 4805 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 4806 struct tc_taprio_qopt_offload *qopt) 4807 { 4808 struct igc_hw *hw = &adapter->hw; 4809 int err; 4810 4811 if (hw->mac.type != igc_i225) 4812 return -EOPNOTSUPP; 4813 4814 err = igc_save_qbv_schedule(adapter, qopt); 4815 if (err) 4816 return err; 4817 4818 return igc_tsn_offload_apply(adapter); 4819 } 4820 4821 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 4822 void *type_data) 4823 { 4824 struct igc_adapter *adapter = netdev_priv(dev); 4825 4826 switch (type) { 4827 case TC_SETUP_QDISC_TAPRIO: 4828 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 4829 4830 case TC_SETUP_QDISC_ETF: 4831 return igc_tsn_enable_launchtime(adapter, type_data); 4832 4833 default: 4834 return -EOPNOTSUPP; 4835 } 4836 } 4837 4838 static const struct net_device_ops igc_netdev_ops = { 4839 .ndo_open = igc_open, 4840 .ndo_stop = igc_close, 4841 .ndo_start_xmit = igc_xmit_frame, 4842 .ndo_set_rx_mode = igc_set_rx_mode, 4843 .ndo_set_mac_address = igc_set_mac, 4844 .ndo_change_mtu = igc_change_mtu, 4845 .ndo_get_stats = igc_get_stats, 4846 .ndo_fix_features = igc_fix_features, 4847 .ndo_set_features = igc_set_features, 4848 .ndo_features_check = igc_features_check, 4849 .ndo_do_ioctl = igc_ioctl, 4850 .ndo_setup_tc = igc_setup_tc, 4851 }; 4852 4853 /* PCIe configuration access */ 4854 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4855 { 4856 struct igc_adapter *adapter = hw->back; 4857 4858 pci_read_config_word(adapter->pdev, reg, value); 4859 } 4860 4861 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) 4862 { 4863 struct igc_adapter *adapter = hw->back; 4864 4865 pci_write_config_word(adapter->pdev, reg, *value); 4866 } 4867 4868 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4869 { 4870 struct igc_adapter *adapter = hw->back; 4871 4872 if (!pci_is_pcie(adapter->pdev)) 4873 return -IGC_ERR_CONFIG; 4874 4875 pcie_capability_read_word(adapter->pdev, reg, value); 4876 4877 return IGC_SUCCESS; 4878 } 4879 4880 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) 4881 { 4882 struct igc_adapter *adapter = hw->back; 4883 4884 if (!pci_is_pcie(adapter->pdev)) 4885 return -IGC_ERR_CONFIG; 4886 4887 pcie_capability_write_word(adapter->pdev, reg, *value); 4888 4889 return IGC_SUCCESS; 4890 } 4891 4892 u32 igc_rd32(struct igc_hw *hw, u32 reg) 4893 { 4894 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 4895 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 4896 u32 value = 0; 4897 4898 value = readl(&hw_addr[reg]); 4899 4900 /* reads should not return all F's */ 4901 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 4902 struct net_device *netdev = igc->netdev; 4903 4904 hw->hw_addr = NULL; 4905 netif_device_detach(netdev); 4906 netdev_err(netdev, "PCIe link lost, device now detached\n"); 4907 WARN(pci_device_is_present(igc->pdev), 4908 "igc: Failed to read reg 0x%x!\n", reg); 4909 } 4910 4911 return value; 4912 } 4913 4914 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx) 4915 { 4916 struct igc_mac_info *mac = &adapter->hw.mac; 4917 4918 mac->autoneg = 0; 4919 4920 /* Make sure dplx is at most 1 bit and lsb of speed is not set 4921 * for the switch() below to work 4922 */ 4923 if ((spd & 1) || (dplx & ~1)) 4924 goto err_inval; 4925 4926 switch (spd + dplx) { 4927 case SPEED_10 + DUPLEX_HALF: 4928 mac->forced_speed_duplex = ADVERTISE_10_HALF; 4929 break; 4930 case SPEED_10 + DUPLEX_FULL: 4931 mac->forced_speed_duplex = ADVERTISE_10_FULL; 4932 break; 4933 case SPEED_100 + DUPLEX_HALF: 4934 mac->forced_speed_duplex = ADVERTISE_100_HALF; 4935 break; 4936 case SPEED_100 + DUPLEX_FULL: 4937 mac->forced_speed_duplex = ADVERTISE_100_FULL; 4938 break; 4939 case SPEED_1000 + DUPLEX_FULL: 4940 mac->autoneg = 1; 4941 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 4942 break; 4943 case SPEED_1000 + DUPLEX_HALF: /* not supported */ 4944 goto err_inval; 4945 case SPEED_2500 + DUPLEX_FULL: 4946 mac->autoneg = 1; 4947 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 4948 break; 4949 case SPEED_2500 + DUPLEX_HALF: /* not supported */ 4950 default: 4951 goto err_inval; 4952 } 4953 4954 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ 4955 adapter->hw.phy.mdix = AUTO_ALL_MODES; 4956 4957 return 0; 4958 4959 err_inval: 4960 netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n"); 4961 return -EINVAL; 4962 } 4963 4964 /** 4965 * igc_probe - Device Initialization Routine 4966 * @pdev: PCI device information struct 4967 * @ent: entry in igc_pci_tbl 4968 * 4969 * Returns 0 on success, negative on failure 4970 * 4971 * igc_probe initializes an adapter identified by a pci_dev structure. 4972 * The OS initialization, configuring the adapter private structure, 4973 * and a hardware reset occur. 4974 */ 4975 static int igc_probe(struct pci_dev *pdev, 4976 const struct pci_device_id *ent) 4977 { 4978 struct igc_adapter *adapter; 4979 struct net_device *netdev; 4980 struct igc_hw *hw; 4981 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 4982 int err, pci_using_dac; 4983 4984 err = pci_enable_device_mem(pdev); 4985 if (err) 4986 return err; 4987 4988 pci_using_dac = 0; 4989 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4990 if (!err) { 4991 pci_using_dac = 1; 4992 } else { 4993 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4994 if (err) { 4995 dev_err(&pdev->dev, 4996 "No usable DMA configuration, aborting\n"); 4997 goto err_dma; 4998 } 4999 } 5000 5001 err = pci_request_mem_regions(pdev, igc_driver_name); 5002 if (err) 5003 goto err_pci_reg; 5004 5005 pci_enable_pcie_error_reporting(pdev); 5006 5007 pci_set_master(pdev); 5008 5009 err = -ENOMEM; 5010 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 5011 IGC_MAX_TX_QUEUES); 5012 5013 if (!netdev) 5014 goto err_alloc_etherdev; 5015 5016 SET_NETDEV_DEV(netdev, &pdev->dev); 5017 5018 pci_set_drvdata(pdev, netdev); 5019 adapter = netdev_priv(netdev); 5020 adapter->netdev = netdev; 5021 adapter->pdev = pdev; 5022 hw = &adapter->hw; 5023 hw->back = adapter; 5024 adapter->port_num = hw->bus.func; 5025 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 5026 5027 err = pci_save_state(pdev); 5028 if (err) 5029 goto err_ioremap; 5030 5031 err = -EIO; 5032 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 5033 pci_resource_len(pdev, 0)); 5034 if (!adapter->io_addr) 5035 goto err_ioremap; 5036 5037 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 5038 hw->hw_addr = adapter->io_addr; 5039 5040 netdev->netdev_ops = &igc_netdev_ops; 5041 igc_ethtool_set_ops(netdev); 5042 netdev->watchdog_timeo = 5 * HZ; 5043 5044 netdev->mem_start = pci_resource_start(pdev, 0); 5045 netdev->mem_end = pci_resource_end(pdev, 0); 5046 5047 /* PCI config space info */ 5048 hw->vendor_id = pdev->vendor; 5049 hw->device_id = pdev->device; 5050 hw->revision_id = pdev->revision; 5051 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5052 hw->subsystem_device_id = pdev->subsystem_device; 5053 5054 /* Copy the default MAC and PHY function pointers */ 5055 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 5056 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 5057 5058 /* Initialize skew-specific constants */ 5059 err = ei->get_invariants(hw); 5060 if (err) 5061 goto err_sw_init; 5062 5063 /* Add supported features to the features list*/ 5064 netdev->features |= NETIF_F_SG; 5065 netdev->features |= NETIF_F_TSO; 5066 netdev->features |= NETIF_F_TSO6; 5067 netdev->features |= NETIF_F_TSO_ECN; 5068 netdev->features |= NETIF_F_RXCSUM; 5069 netdev->features |= NETIF_F_HW_CSUM; 5070 netdev->features |= NETIF_F_SCTP_CRC; 5071 netdev->features |= NETIF_F_HW_TC; 5072 5073 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 5074 NETIF_F_GSO_GRE_CSUM | \ 5075 NETIF_F_GSO_IPXIP4 | \ 5076 NETIF_F_GSO_IPXIP6 | \ 5077 NETIF_F_GSO_UDP_TUNNEL | \ 5078 NETIF_F_GSO_UDP_TUNNEL_CSUM) 5079 5080 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 5081 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 5082 5083 /* setup the private structure */ 5084 err = igc_sw_init(adapter); 5085 if (err) 5086 goto err_sw_init; 5087 5088 /* copy netdev features into list of user selectable features */ 5089 netdev->hw_features |= NETIF_F_NTUPLE; 5090 netdev->hw_features |= netdev->features; 5091 5092 if (pci_using_dac) 5093 netdev->features |= NETIF_F_HIGHDMA; 5094 5095 /* MTU range: 68 - 9216 */ 5096 netdev->min_mtu = ETH_MIN_MTU; 5097 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 5098 5099 /* before reading the NVM, reset the controller to put the device in a 5100 * known good starting state 5101 */ 5102 hw->mac.ops.reset_hw(hw); 5103 5104 if (igc_get_flash_presence_i225(hw)) { 5105 if (hw->nvm.ops.validate(hw) < 0) { 5106 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 5107 err = -EIO; 5108 goto err_eeprom; 5109 } 5110 } 5111 5112 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 5113 /* copy the MAC address out of the NVM */ 5114 if (hw->mac.ops.read_mac_addr(hw)) 5115 dev_err(&pdev->dev, "NVM Read Error\n"); 5116 } 5117 5118 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 5119 5120 if (!is_valid_ether_addr(netdev->dev_addr)) { 5121 dev_err(&pdev->dev, "Invalid MAC Address\n"); 5122 err = -EIO; 5123 goto err_eeprom; 5124 } 5125 5126 /* configure RXPBSIZE and TXPBSIZE */ 5127 wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); 5128 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 5129 5130 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 5131 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 5132 5133 INIT_WORK(&adapter->reset_task, igc_reset_task); 5134 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 5135 5136 /* Initialize link properties that are user-changeable */ 5137 adapter->fc_autoneg = true; 5138 hw->mac.autoneg = true; 5139 hw->phy.autoneg_advertised = 0xaf; 5140 5141 hw->fc.requested_mode = igc_fc_default; 5142 hw->fc.current_mode = igc_fc_default; 5143 5144 /* By default, support wake on port A */ 5145 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 5146 5147 /* initialize the wol settings based on the eeprom settings */ 5148 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 5149 adapter->wol |= IGC_WUFC_MAG; 5150 5151 device_set_wakeup_enable(&adapter->pdev->dev, 5152 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 5153 5154 /* reset the hardware with the new settings */ 5155 igc_reset(adapter); 5156 5157 /* let the f/w know that the h/w is now under the control of the 5158 * driver. 5159 */ 5160 igc_get_hw_control(adapter); 5161 5162 strncpy(netdev->name, "eth%d", IFNAMSIZ); 5163 err = register_netdev(netdev); 5164 if (err) 5165 goto err_register; 5166 5167 /* carrier off reporting is important to ethtool even BEFORE open */ 5168 netif_carrier_off(netdev); 5169 5170 /* do hw tstamp init after resetting */ 5171 igc_ptp_init(adapter); 5172 5173 /* Check if Media Autosense is enabled */ 5174 adapter->ei = *ei; 5175 5176 /* print pcie link status and MAC address */ 5177 pcie_print_link_status(pdev); 5178 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 5179 5180 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 5181 /* Disable EEE for internal PHY devices */ 5182 hw->dev_spec._base.eee_enable = false; 5183 adapter->flags &= ~IGC_FLAG_EEE; 5184 igc_set_eee_i225(hw, false, false, false); 5185 5186 pm_runtime_put_noidle(&pdev->dev); 5187 5188 return 0; 5189 5190 err_register: 5191 igc_release_hw_control(adapter); 5192 err_eeprom: 5193 if (!igc_check_reset_block(hw)) 5194 igc_reset_phy(hw); 5195 err_sw_init: 5196 igc_clear_interrupt_scheme(adapter); 5197 iounmap(adapter->io_addr); 5198 err_ioremap: 5199 free_netdev(netdev); 5200 err_alloc_etherdev: 5201 pci_release_mem_regions(pdev); 5202 err_pci_reg: 5203 err_dma: 5204 pci_disable_device(pdev); 5205 return err; 5206 } 5207 5208 /** 5209 * igc_remove - Device Removal Routine 5210 * @pdev: PCI device information struct 5211 * 5212 * igc_remove is called by the PCI subsystem to alert the driver 5213 * that it should release a PCI device. This could be caused by a 5214 * Hot-Plug event, or because the driver is going to be removed from 5215 * memory. 5216 */ 5217 static void igc_remove(struct pci_dev *pdev) 5218 { 5219 struct net_device *netdev = pci_get_drvdata(pdev); 5220 struct igc_adapter *adapter = netdev_priv(netdev); 5221 5222 pm_runtime_get_noresume(&pdev->dev); 5223 5224 igc_flush_nfc_rules(adapter); 5225 5226 igc_ptp_stop(adapter); 5227 5228 set_bit(__IGC_DOWN, &adapter->state); 5229 5230 del_timer_sync(&adapter->watchdog_timer); 5231 del_timer_sync(&adapter->phy_info_timer); 5232 5233 cancel_work_sync(&adapter->reset_task); 5234 cancel_work_sync(&adapter->watchdog_task); 5235 5236 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5237 * would have already happened in close and is redundant. 5238 */ 5239 igc_release_hw_control(adapter); 5240 unregister_netdev(netdev); 5241 5242 igc_clear_interrupt_scheme(adapter); 5243 pci_iounmap(pdev, adapter->io_addr); 5244 pci_release_mem_regions(pdev); 5245 5246 free_netdev(netdev); 5247 5248 pci_disable_pcie_error_reporting(pdev); 5249 5250 pci_disable_device(pdev); 5251 } 5252 5253 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 5254 bool runtime) 5255 { 5256 struct net_device *netdev = pci_get_drvdata(pdev); 5257 struct igc_adapter *adapter = netdev_priv(netdev); 5258 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 5259 struct igc_hw *hw = &adapter->hw; 5260 u32 ctrl, rctl, status; 5261 bool wake; 5262 5263 rtnl_lock(); 5264 netif_device_detach(netdev); 5265 5266 if (netif_running(netdev)) 5267 __igc_close(netdev, true); 5268 5269 igc_ptp_suspend(adapter); 5270 5271 igc_clear_interrupt_scheme(adapter); 5272 rtnl_unlock(); 5273 5274 status = rd32(IGC_STATUS); 5275 if (status & IGC_STATUS_LU) 5276 wufc &= ~IGC_WUFC_LNKC; 5277 5278 if (wufc) { 5279 igc_setup_rctl(adapter); 5280 igc_set_rx_mode(netdev); 5281 5282 /* turn on all-multi mode if wake on multicast is enabled */ 5283 if (wufc & IGC_WUFC_MC) { 5284 rctl = rd32(IGC_RCTL); 5285 rctl |= IGC_RCTL_MPE; 5286 wr32(IGC_RCTL, rctl); 5287 } 5288 5289 ctrl = rd32(IGC_CTRL); 5290 ctrl |= IGC_CTRL_ADVD3WUC; 5291 wr32(IGC_CTRL, ctrl); 5292 5293 /* Allow time for pending master requests to run */ 5294 igc_disable_pcie_master(hw); 5295 5296 wr32(IGC_WUC, IGC_WUC_PME_EN); 5297 wr32(IGC_WUFC, wufc); 5298 } else { 5299 wr32(IGC_WUC, 0); 5300 wr32(IGC_WUFC, 0); 5301 } 5302 5303 wake = wufc || adapter->en_mng_pt; 5304 if (!wake) 5305 igc_power_down_phy_copper_base(&adapter->hw); 5306 else 5307 igc_power_up_link(adapter); 5308 5309 if (enable_wake) 5310 *enable_wake = wake; 5311 5312 /* Release control of h/w to f/w. If f/w is AMT enabled, this 5313 * would have already happened in close and is redundant. 5314 */ 5315 igc_release_hw_control(adapter); 5316 5317 pci_disable_device(pdev); 5318 5319 return 0; 5320 } 5321 5322 #ifdef CONFIG_PM 5323 static int __maybe_unused igc_runtime_suspend(struct device *dev) 5324 { 5325 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 5326 } 5327 5328 static void igc_deliver_wake_packet(struct net_device *netdev) 5329 { 5330 struct igc_adapter *adapter = netdev_priv(netdev); 5331 struct igc_hw *hw = &adapter->hw; 5332 struct sk_buff *skb; 5333 u32 wupl; 5334 5335 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 5336 5337 /* WUPM stores only the first 128 bytes of the wake packet. 5338 * Read the packet only if we have the whole thing. 5339 */ 5340 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 5341 return; 5342 5343 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 5344 if (!skb) 5345 return; 5346 5347 skb_put(skb, wupl); 5348 5349 /* Ensure reads are 32-bit aligned */ 5350 wupl = roundup(wupl, 4); 5351 5352 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 5353 5354 skb->protocol = eth_type_trans(skb, netdev); 5355 netif_rx(skb); 5356 } 5357 5358 static int __maybe_unused igc_resume(struct device *dev) 5359 { 5360 struct pci_dev *pdev = to_pci_dev(dev); 5361 struct net_device *netdev = pci_get_drvdata(pdev); 5362 struct igc_adapter *adapter = netdev_priv(netdev); 5363 struct igc_hw *hw = &adapter->hw; 5364 u32 err, val; 5365 5366 pci_set_power_state(pdev, PCI_D0); 5367 pci_restore_state(pdev); 5368 pci_save_state(pdev); 5369 5370 if (!pci_device_is_present(pdev)) 5371 return -ENODEV; 5372 err = pci_enable_device_mem(pdev); 5373 if (err) { 5374 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 5375 return err; 5376 } 5377 pci_set_master(pdev); 5378 5379 pci_enable_wake(pdev, PCI_D3hot, 0); 5380 pci_enable_wake(pdev, PCI_D3cold, 0); 5381 5382 if (igc_init_interrupt_scheme(adapter, true)) { 5383 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5384 return -ENOMEM; 5385 } 5386 5387 igc_reset(adapter); 5388 5389 /* let the f/w know that the h/w is now under the control of the 5390 * driver. 5391 */ 5392 igc_get_hw_control(adapter); 5393 5394 val = rd32(IGC_WUS); 5395 if (val & WAKE_PKT_WUS) 5396 igc_deliver_wake_packet(netdev); 5397 5398 wr32(IGC_WUS, ~0); 5399 5400 rtnl_lock(); 5401 if (!err && netif_running(netdev)) 5402 err = __igc_open(netdev, true); 5403 5404 if (!err) 5405 netif_device_attach(netdev); 5406 rtnl_unlock(); 5407 5408 return err; 5409 } 5410 5411 static int __maybe_unused igc_runtime_resume(struct device *dev) 5412 { 5413 return igc_resume(dev); 5414 } 5415 5416 static int __maybe_unused igc_suspend(struct device *dev) 5417 { 5418 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 5419 } 5420 5421 static int __maybe_unused igc_runtime_idle(struct device *dev) 5422 { 5423 struct net_device *netdev = dev_get_drvdata(dev); 5424 struct igc_adapter *adapter = netdev_priv(netdev); 5425 5426 if (!igc_has_link(adapter)) 5427 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 5428 5429 return -EBUSY; 5430 } 5431 #endif /* CONFIG_PM */ 5432 5433 static void igc_shutdown(struct pci_dev *pdev) 5434 { 5435 bool wake; 5436 5437 __igc_shutdown(pdev, &wake, 0); 5438 5439 if (system_state == SYSTEM_POWER_OFF) { 5440 pci_wake_from_d3(pdev, wake); 5441 pci_set_power_state(pdev, PCI_D3hot); 5442 } 5443 } 5444 5445 /** 5446 * igc_io_error_detected - called when PCI error is detected 5447 * @pdev: Pointer to PCI device 5448 * @state: The current PCI connection state 5449 * 5450 * This function is called after a PCI bus error affecting 5451 * this device has been detected. 5452 **/ 5453 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 5454 pci_channel_state_t state) 5455 { 5456 struct net_device *netdev = pci_get_drvdata(pdev); 5457 struct igc_adapter *adapter = netdev_priv(netdev); 5458 5459 netif_device_detach(netdev); 5460 5461 if (state == pci_channel_io_perm_failure) 5462 return PCI_ERS_RESULT_DISCONNECT; 5463 5464 if (netif_running(netdev)) 5465 igc_down(adapter); 5466 pci_disable_device(pdev); 5467 5468 /* Request a slot reset. */ 5469 return PCI_ERS_RESULT_NEED_RESET; 5470 } 5471 5472 /** 5473 * igc_io_slot_reset - called after the PCI bus has been reset. 5474 * @pdev: Pointer to PCI device 5475 * 5476 * Restart the card from scratch, as if from a cold-boot. Implementation 5477 * resembles the first-half of the igc_resume routine. 5478 **/ 5479 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 5480 { 5481 struct net_device *netdev = pci_get_drvdata(pdev); 5482 struct igc_adapter *adapter = netdev_priv(netdev); 5483 struct igc_hw *hw = &adapter->hw; 5484 pci_ers_result_t result; 5485 5486 if (pci_enable_device_mem(pdev)) { 5487 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 5488 result = PCI_ERS_RESULT_DISCONNECT; 5489 } else { 5490 pci_set_master(pdev); 5491 pci_restore_state(pdev); 5492 pci_save_state(pdev); 5493 5494 pci_enable_wake(pdev, PCI_D3hot, 0); 5495 pci_enable_wake(pdev, PCI_D3cold, 0); 5496 5497 /* In case of PCI error, adapter loses its HW address 5498 * so we should re-assign it here. 5499 */ 5500 hw->hw_addr = adapter->io_addr; 5501 5502 igc_reset(adapter); 5503 wr32(IGC_WUS, ~0); 5504 result = PCI_ERS_RESULT_RECOVERED; 5505 } 5506 5507 return result; 5508 } 5509 5510 /** 5511 * igc_io_resume - called when traffic can start to flow again. 5512 * @pdev: Pointer to PCI device 5513 * 5514 * This callback is called when the error recovery driver tells us that 5515 * its OK to resume normal operation. Implementation resembles the 5516 * second-half of the igc_resume routine. 5517 */ 5518 static void igc_io_resume(struct pci_dev *pdev) 5519 { 5520 struct net_device *netdev = pci_get_drvdata(pdev); 5521 struct igc_adapter *adapter = netdev_priv(netdev); 5522 5523 rtnl_lock(); 5524 if (netif_running(netdev)) { 5525 if (igc_open(netdev)) { 5526 netdev_err(netdev, "igc_open failed after reset\n"); 5527 return; 5528 } 5529 } 5530 5531 netif_device_attach(netdev); 5532 5533 /* let the f/w know that the h/w is now under the control of the 5534 * driver. 5535 */ 5536 igc_get_hw_control(adapter); 5537 rtnl_unlock(); 5538 } 5539 5540 static const struct pci_error_handlers igc_err_handler = { 5541 .error_detected = igc_io_error_detected, 5542 .slot_reset = igc_io_slot_reset, 5543 .resume = igc_io_resume, 5544 }; 5545 5546 #ifdef CONFIG_PM 5547 static const struct dev_pm_ops igc_pm_ops = { 5548 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) 5549 SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, 5550 igc_runtime_idle) 5551 }; 5552 #endif 5553 5554 static struct pci_driver igc_driver = { 5555 .name = igc_driver_name, 5556 .id_table = igc_pci_tbl, 5557 .probe = igc_probe, 5558 .remove = igc_remove, 5559 #ifdef CONFIG_PM 5560 .driver.pm = &igc_pm_ops, 5561 #endif 5562 .shutdown = igc_shutdown, 5563 .err_handler = &igc_err_handler, 5564 }; 5565 5566 /** 5567 * igc_reinit_queues - return error 5568 * @adapter: pointer to adapter structure 5569 */ 5570 int igc_reinit_queues(struct igc_adapter *adapter) 5571 { 5572 struct net_device *netdev = adapter->netdev; 5573 int err = 0; 5574 5575 if (netif_running(netdev)) 5576 igc_close(netdev); 5577 5578 igc_reset_interrupt_capability(adapter); 5579 5580 if (igc_init_interrupt_scheme(adapter, true)) { 5581 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5582 return -ENOMEM; 5583 } 5584 5585 if (netif_running(netdev)) 5586 err = igc_open(netdev); 5587 5588 return err; 5589 } 5590 5591 /** 5592 * igc_get_hw_dev - return device 5593 * @hw: pointer to hardware structure 5594 * 5595 * used by hardware layer to print debugging information 5596 */ 5597 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 5598 { 5599 struct igc_adapter *adapter = hw->back; 5600 5601 return adapter->netdev; 5602 } 5603 5604 /** 5605 * igc_init_module - Driver Registration Routine 5606 * 5607 * igc_init_module is the first routine called when the driver is 5608 * loaded. All it does is register with the PCI subsystem. 5609 */ 5610 static int __init igc_init_module(void) 5611 { 5612 int ret; 5613 5614 pr_info("%s\n", igc_driver_string); 5615 pr_info("%s\n", igc_copyright); 5616 5617 ret = pci_register_driver(&igc_driver); 5618 return ret; 5619 } 5620 5621 module_init(igc_init_module); 5622 5623 /** 5624 * igc_exit_module - Driver Exit Cleanup Routine 5625 * 5626 * igc_exit_module is called just before the driver is removed 5627 * from memory. 5628 */ 5629 static void __exit igc_exit_module(void) 5630 { 5631 pci_unregister_driver(&igc_driver); 5632 } 5633 5634 module_exit(igc_exit_module); 5635 /* igc_main.c */ 5636