1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/ip.h> 10 #include <linux/pm_runtime.h> 11 #include <net/pkt_sched.h> 12 #include <linux/bpf_trace.h> 13 #include <net/xdp_sock_drv.h> 14 #include <linux/pci.h> 15 #include <linux/mdio.h> 16 17 #include <net/ipv6.h> 18 19 #include "igc.h" 20 #include "igc_hw.h" 21 #include "igc_tsn.h" 22 #include "igc_xdp.h" 23 24 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 25 26 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 27 28 #define IGC_XDP_PASS 0 29 #define IGC_XDP_CONSUMED BIT(0) 30 #define IGC_XDP_TX BIT(1) 31 #define IGC_XDP_REDIRECT BIT(2) 32 33 static int debug = -1; 34 35 MODULE_DESCRIPTION(DRV_SUMMARY); 36 MODULE_LICENSE("GPL v2"); 37 module_param(debug, int, 0); 38 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 39 40 char igc_driver_name[] = "igc"; 41 static const char igc_driver_string[] = DRV_SUMMARY; 42 static const char igc_copyright[] = 43 "Copyright(c) 2018 Intel Corporation."; 44 45 static const struct igc_info *igc_info_tbl[] = { 46 [board_base] = &igc_base_info, 47 }; 48 49 static const struct pci_device_id igc_pci_tbl[] = { 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), .driver_data = board_base }, 51 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), .driver_data = board_base }, 52 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), .driver_data = board_base }, 53 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), .driver_data = board_base }, 54 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), .driver_data = board_base }, 55 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), .driver_data = board_base }, 56 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), .driver_data = board_base }, 57 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), .driver_data = board_base }, 58 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), .driver_data = board_base }, 59 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), .driver_data = board_base }, 60 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), .driver_data = board_base }, 61 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), .driver_data = board_base }, 62 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), .driver_data = board_base }, 63 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), .driver_data = board_base }, 64 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), .driver_data = board_base }, 65 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), .driver_data = board_base }, 66 /* required last entry */ 67 { } 68 }; 69 70 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 71 72 enum latency_range { 73 lowest_latency = 0, 74 low_latency = 1, 75 bulk_latency = 2, 76 latency_invalid = 255 77 }; 78 79 void igc_reset(struct igc_adapter *adapter) 80 { 81 struct net_device *dev = adapter->netdev; 82 struct igc_hw *hw = &adapter->hw; 83 struct igc_fc_info *fc = &hw->fc; 84 u32 pba, hwm; 85 86 /* Repartition PBA for greater than 9k MTU if required */ 87 pba = IGC_PBA_34K; 88 89 /* flow control settings 90 * The high water mark must be low enough to fit one full frame 91 * after transmitting the pause frame. As such we must have enough 92 * space to allow for us to complete our current transmit and then 93 * receive the frame that is in progress from the link partner. 94 * Set it to: 95 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 96 */ 97 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 98 99 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 100 fc->low_water = fc->high_water - 16; 101 fc->pause_time = 0xFFFF; 102 fc->send_xon = 1; 103 fc->current_mode = fc->requested_mode; 104 105 hw->mac.ops.reset_hw(hw); 106 107 if (hw->mac.ops.init_hw(hw)) 108 netdev_err(dev, "Error on hardware initialization\n"); 109 110 /* Re-establish EEE setting */ 111 igc_set_eee_i225(hw, true, true, true); 112 113 if (!netif_running(adapter->netdev)) 114 igc_power_down_phy_copper_base(&adapter->hw); 115 116 /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */ 117 wr32(IGC_VET, ETH_P_8021Q); 118 119 /* Re-enable PTP, where applicable. */ 120 igc_ptp_reset(adapter); 121 122 /* Re-enable TSN offloading, where applicable. */ 123 igc_tsn_reset(adapter); 124 125 igc_get_phy_info(hw); 126 } 127 128 /** 129 * igc_power_up_link - Power up the phy link 130 * @adapter: address of board private structure 131 */ 132 static void igc_power_up_link(struct igc_adapter *adapter) 133 { 134 igc_reset_phy(&adapter->hw); 135 136 igc_power_up_phy_copper(&adapter->hw); 137 138 igc_setup_link(&adapter->hw); 139 } 140 141 /** 142 * igc_release_hw_control - release control of the h/w to f/w 143 * @adapter: address of board private structure 144 * 145 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 146 * For ASF and Pass Through versions of f/w this means that the 147 * driver is no longer loaded. 148 */ 149 static void igc_release_hw_control(struct igc_adapter *adapter) 150 { 151 struct igc_hw *hw = &adapter->hw; 152 u32 ctrl_ext; 153 154 if (!pci_device_is_present(adapter->pdev)) 155 return; 156 157 /* Let firmware take over control of h/w */ 158 ctrl_ext = rd32(IGC_CTRL_EXT); 159 wr32(IGC_CTRL_EXT, 160 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 161 } 162 163 /** 164 * igc_get_hw_control - get control of the h/w from f/w 165 * @adapter: address of board private structure 166 * 167 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 168 * For ASF and Pass Through versions of f/w this means that 169 * the driver is loaded. 170 */ 171 static void igc_get_hw_control(struct igc_adapter *adapter) 172 { 173 struct igc_hw *hw = &adapter->hw; 174 u32 ctrl_ext; 175 176 /* Let firmware know the driver has taken over */ 177 ctrl_ext = rd32(IGC_CTRL_EXT); 178 wr32(IGC_CTRL_EXT, 179 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 180 } 181 182 static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf) 183 { 184 dma_unmap_single(dev, dma_unmap_addr(buf, dma), 185 dma_unmap_len(buf, len), DMA_TO_DEVICE); 186 187 dma_unmap_len_set(buf, len, 0); 188 } 189 190 /** 191 * igc_clean_tx_ring - Free Tx Buffers 192 * @tx_ring: ring to be cleaned 193 */ 194 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 195 { 196 u16 i = tx_ring->next_to_clean; 197 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 198 u32 xsk_frames = 0; 199 200 while (i != tx_ring->next_to_use) { 201 union igc_adv_tx_desc *eop_desc, *tx_desc; 202 203 switch (tx_buffer->type) { 204 case IGC_TX_BUFFER_TYPE_XSK: 205 xsk_frames++; 206 break; 207 case IGC_TX_BUFFER_TYPE_XDP: 208 xdp_return_frame(tx_buffer->xdpf); 209 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 210 break; 211 case IGC_TX_BUFFER_TYPE_SKB: 212 dev_kfree_skb_any(tx_buffer->skb); 213 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 214 break; 215 default: 216 netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); 217 break; 218 } 219 220 /* check for eop_desc to determine the end of the packet */ 221 eop_desc = tx_buffer->next_to_watch; 222 tx_desc = IGC_TX_DESC(tx_ring, i); 223 224 /* unmap remaining buffers */ 225 while (tx_desc != eop_desc) { 226 tx_buffer++; 227 tx_desc++; 228 i++; 229 if (unlikely(i == tx_ring->count)) { 230 i = 0; 231 tx_buffer = tx_ring->tx_buffer_info; 232 tx_desc = IGC_TX_DESC(tx_ring, 0); 233 } 234 235 /* unmap any remaining paged data */ 236 if (dma_unmap_len(tx_buffer, len)) 237 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 238 } 239 240 tx_buffer->next_to_watch = NULL; 241 242 /* move us one more past the eop_desc for start of next pkt */ 243 tx_buffer++; 244 i++; 245 if (unlikely(i == tx_ring->count)) { 246 i = 0; 247 tx_buffer = tx_ring->tx_buffer_info; 248 } 249 } 250 251 if (tx_ring->xsk_pool && xsk_frames) 252 xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); 253 254 /* reset BQL for queue */ 255 netdev_tx_reset_queue(txring_txq(tx_ring)); 256 257 /* Zero out the buffer ring */ 258 memset(tx_ring->tx_buffer_info, 0, 259 sizeof(*tx_ring->tx_buffer_info) * tx_ring->count); 260 261 /* Zero out the descriptor ring */ 262 memset(tx_ring->desc, 0, tx_ring->size); 263 264 /* reset next_to_use and next_to_clean */ 265 tx_ring->next_to_use = 0; 266 tx_ring->next_to_clean = 0; 267 268 /* Clear any lingering XSK TX timestamp requests */ 269 if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) { 270 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 271 272 igc_ptp_clear_xsk_tx_tstamp_queue(adapter, tx_ring->queue_index); 273 } 274 } 275 276 /** 277 * igc_free_tx_resources - Free Tx Resources per Queue 278 * @tx_ring: Tx descriptor ring for a specific queue 279 * 280 * Free all transmit software resources 281 */ 282 void igc_free_tx_resources(struct igc_ring *tx_ring) 283 { 284 igc_disable_tx_ring(tx_ring); 285 286 vfree(tx_ring->tx_buffer_info); 287 tx_ring->tx_buffer_info = NULL; 288 289 /* if not set, then don't free */ 290 if (!tx_ring->desc) 291 return; 292 293 dma_free_coherent(tx_ring->dev, tx_ring->size, 294 tx_ring->desc, tx_ring->dma); 295 296 tx_ring->desc = NULL; 297 } 298 299 /** 300 * igc_free_all_tx_resources - Free Tx Resources for All Queues 301 * @adapter: board private structure 302 * 303 * Free all transmit software resources 304 */ 305 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 306 { 307 int i; 308 309 for (i = 0; i < adapter->num_tx_queues; i++) 310 igc_free_tx_resources(adapter->tx_ring[i]); 311 } 312 313 /** 314 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 315 * @adapter: board private structure 316 */ 317 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 318 { 319 int i; 320 321 for (i = 0; i < adapter->num_tx_queues; i++) 322 if (adapter->tx_ring[i]) 323 igc_clean_tx_ring(adapter->tx_ring[i]); 324 } 325 326 static void igc_disable_tx_ring_hw(struct igc_ring *ring) 327 { 328 struct igc_hw *hw = &ring->q_vector->adapter->hw; 329 u8 idx = ring->reg_idx; 330 u32 txdctl; 331 332 txdctl = rd32(IGC_TXDCTL(idx)); 333 txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE; 334 txdctl |= IGC_TXDCTL_SWFLUSH; 335 wr32(IGC_TXDCTL(idx), txdctl); 336 } 337 338 /** 339 * igc_disable_all_tx_rings_hw - Disable all transmit queue operation 340 * @adapter: board private structure 341 */ 342 static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter) 343 { 344 int i; 345 346 for (i = 0; i < adapter->num_tx_queues; i++) { 347 struct igc_ring *tx_ring = adapter->tx_ring[i]; 348 349 igc_disable_tx_ring_hw(tx_ring); 350 } 351 } 352 353 /** 354 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 355 * @tx_ring: tx descriptor ring (for a specific queue) to setup 356 * 357 * Return 0 on success, negative on failure 358 */ 359 int igc_setup_tx_resources(struct igc_ring *tx_ring) 360 { 361 struct net_device *ndev = tx_ring->netdev; 362 struct device *dev = tx_ring->dev; 363 int size = 0; 364 365 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 366 tx_ring->tx_buffer_info = vzalloc(size); 367 if (!tx_ring->tx_buffer_info) 368 goto err; 369 370 /* round up to nearest 4K */ 371 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 372 tx_ring->size = ALIGN(tx_ring->size, 4096); 373 374 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 375 &tx_ring->dma, GFP_KERNEL); 376 377 if (!tx_ring->desc) 378 goto err; 379 380 tx_ring->next_to_use = 0; 381 tx_ring->next_to_clean = 0; 382 383 return 0; 384 385 err: 386 vfree(tx_ring->tx_buffer_info); 387 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 388 return -ENOMEM; 389 } 390 391 /** 392 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 393 * @adapter: board private structure 394 * 395 * Return 0 on success, negative on failure 396 */ 397 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 398 { 399 struct net_device *dev = adapter->netdev; 400 int i, err = 0; 401 402 for (i = 0; i < adapter->num_tx_queues; i++) { 403 err = igc_setup_tx_resources(adapter->tx_ring[i]); 404 if (err) { 405 netdev_err(dev, "Error on Tx queue %u setup\n", i); 406 for (i--; i >= 0; i--) 407 igc_free_tx_resources(adapter->tx_ring[i]); 408 break; 409 } 410 } 411 412 return err; 413 } 414 415 static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring) 416 { 417 u16 i = rx_ring->next_to_clean; 418 419 dev_kfree_skb(rx_ring->skb); 420 rx_ring->skb = NULL; 421 422 /* Free all the Rx ring sk_buffs */ 423 while (i != rx_ring->next_to_alloc) { 424 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 425 426 /* Invalidate cache lines that may have been written to by 427 * device so that we avoid corrupting memory. 428 */ 429 dma_sync_single_range_for_cpu(rx_ring->dev, 430 buffer_info->dma, 431 buffer_info->page_offset, 432 igc_rx_bufsz(rx_ring), 433 DMA_FROM_DEVICE); 434 435 /* free resources associated with mapping */ 436 dma_unmap_page_attrs(rx_ring->dev, 437 buffer_info->dma, 438 igc_rx_pg_size(rx_ring), 439 DMA_FROM_DEVICE, 440 IGC_RX_DMA_ATTR); 441 __page_frag_cache_drain(buffer_info->page, 442 buffer_info->pagecnt_bias); 443 444 i++; 445 if (i == rx_ring->count) 446 i = 0; 447 } 448 } 449 450 static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring) 451 { 452 struct igc_rx_buffer *bi; 453 u16 i; 454 455 for (i = 0; i < ring->count; i++) { 456 bi = &ring->rx_buffer_info[i]; 457 if (!bi->xdp) 458 continue; 459 460 xsk_buff_free(bi->xdp); 461 bi->xdp = NULL; 462 } 463 } 464 465 /** 466 * igc_clean_rx_ring - Free Rx Buffers per Queue 467 * @ring: ring to free buffers from 468 */ 469 static void igc_clean_rx_ring(struct igc_ring *ring) 470 { 471 if (ring->xsk_pool) 472 igc_clean_rx_ring_xsk_pool(ring); 473 else 474 igc_clean_rx_ring_page_shared(ring); 475 476 clear_ring_uses_large_buffer(ring); 477 478 ring->next_to_alloc = 0; 479 ring->next_to_clean = 0; 480 ring->next_to_use = 0; 481 } 482 483 /** 484 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 485 * @adapter: board private structure 486 */ 487 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 488 { 489 int i; 490 491 for (i = 0; i < adapter->num_rx_queues; i++) 492 if (adapter->rx_ring[i]) 493 igc_clean_rx_ring(adapter->rx_ring[i]); 494 } 495 496 /** 497 * igc_free_rx_resources - Free Rx Resources 498 * @rx_ring: ring to clean the resources from 499 * 500 * Free all receive software resources 501 */ 502 void igc_free_rx_resources(struct igc_ring *rx_ring) 503 { 504 igc_clean_rx_ring(rx_ring); 505 506 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 507 508 vfree(rx_ring->rx_buffer_info); 509 rx_ring->rx_buffer_info = NULL; 510 511 /* if not set, then don't free */ 512 if (!rx_ring->desc) 513 return; 514 515 dma_free_coherent(rx_ring->dev, rx_ring->size, 516 rx_ring->desc, rx_ring->dma); 517 518 rx_ring->desc = NULL; 519 } 520 521 /** 522 * igc_free_all_rx_resources - Free Rx Resources for All Queues 523 * @adapter: board private structure 524 * 525 * Free all receive software resources 526 */ 527 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 528 { 529 int i; 530 531 for (i = 0; i < adapter->num_rx_queues; i++) 532 igc_free_rx_resources(adapter->rx_ring[i]); 533 } 534 535 /** 536 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 537 * @rx_ring: rx descriptor ring (for a specific queue) to setup 538 * 539 * Returns 0 on success, negative on failure 540 */ 541 int igc_setup_rx_resources(struct igc_ring *rx_ring) 542 { 543 struct net_device *ndev = rx_ring->netdev; 544 struct device *dev = rx_ring->dev; 545 u8 index = rx_ring->queue_index; 546 int size, desc_len, res; 547 548 /* XDP RX-queue info */ 549 if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) 550 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 551 res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index, 552 rx_ring->q_vector->napi.napi_id); 553 if (res < 0) { 554 netdev_err(ndev, "Failed to register xdp_rxq index %u\n", 555 index); 556 return res; 557 } 558 559 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 560 rx_ring->rx_buffer_info = vzalloc(size); 561 if (!rx_ring->rx_buffer_info) 562 goto err; 563 564 desc_len = sizeof(union igc_adv_rx_desc); 565 566 /* Round up to nearest 4K */ 567 rx_ring->size = rx_ring->count * desc_len; 568 rx_ring->size = ALIGN(rx_ring->size, 4096); 569 570 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 571 &rx_ring->dma, GFP_KERNEL); 572 573 if (!rx_ring->desc) 574 goto err; 575 576 rx_ring->next_to_alloc = 0; 577 rx_ring->next_to_clean = 0; 578 rx_ring->next_to_use = 0; 579 580 return 0; 581 582 err: 583 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 584 vfree(rx_ring->rx_buffer_info); 585 rx_ring->rx_buffer_info = NULL; 586 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 587 return -ENOMEM; 588 } 589 590 /** 591 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 592 * (Descriptors) for all queues 593 * @adapter: board private structure 594 * 595 * Return 0 on success, negative on failure 596 */ 597 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 598 { 599 struct net_device *dev = adapter->netdev; 600 int i, err = 0; 601 602 for (i = 0; i < adapter->num_rx_queues; i++) { 603 err = igc_setup_rx_resources(adapter->rx_ring[i]); 604 if (err) { 605 netdev_err(dev, "Error on Rx queue %u setup\n", i); 606 for (i--; i >= 0; i--) 607 igc_free_rx_resources(adapter->rx_ring[i]); 608 break; 609 } 610 } 611 612 return err; 613 } 614 615 static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter, 616 struct igc_ring *ring) 617 { 618 if (!igc_xdp_is_enabled(adapter) || 619 !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags)) 620 return NULL; 621 622 return xsk_get_pool_from_qid(ring->netdev, ring->queue_index); 623 } 624 625 /** 626 * igc_configure_rx_ring - Configure a receive ring after Reset 627 * @adapter: board private structure 628 * @ring: receive ring to be configured 629 * 630 * Configure the Rx unit of the MAC after a reset. 631 */ 632 static void igc_configure_rx_ring(struct igc_adapter *adapter, 633 struct igc_ring *ring) 634 { 635 struct igc_hw *hw = &adapter->hw; 636 union igc_adv_rx_desc *rx_desc; 637 int reg_idx = ring->reg_idx; 638 u32 srrctl = 0, rxdctl = 0; 639 u64 rdba = ring->dma; 640 u32 buf_size; 641 642 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 643 ring->xsk_pool = igc_get_xsk_pool(adapter, ring); 644 if (ring->xsk_pool) { 645 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 646 MEM_TYPE_XSK_BUFF_POOL, 647 NULL)); 648 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 649 } else { 650 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 651 MEM_TYPE_PAGE_SHARED, 652 NULL)); 653 } 654 655 if (igc_xdp_is_enabled(adapter)) 656 set_ring_uses_large_buffer(ring); 657 658 /* disable the queue */ 659 wr32(IGC_RXDCTL(reg_idx), 0); 660 661 /* Set DMA base address registers */ 662 wr32(IGC_RDBAL(reg_idx), 663 rdba & 0x00000000ffffffffULL); 664 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 665 wr32(IGC_RDLEN(reg_idx), 666 ring->count * sizeof(union igc_adv_rx_desc)); 667 668 /* initialize head and tail */ 669 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 670 wr32(IGC_RDH(reg_idx), 0); 671 writel(0, ring->tail); 672 673 /* reset next-to- use/clean to place SW in sync with hardware */ 674 ring->next_to_clean = 0; 675 ring->next_to_use = 0; 676 677 if (ring->xsk_pool) 678 buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool); 679 else if (ring_uses_large_buffer(ring)) 680 buf_size = IGC_RXBUFFER_3072; 681 else 682 buf_size = IGC_RXBUFFER_2048; 683 684 srrctl = rd32(IGC_SRRCTL(reg_idx)); 685 srrctl &= ~(IGC_SRRCTL_BSIZEPKT_MASK | IGC_SRRCTL_BSIZEHDR_MASK | 686 IGC_SRRCTL_DESCTYPE_MASK); 687 srrctl |= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN); 688 srrctl |= IGC_SRRCTL_BSIZEPKT(buf_size); 689 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 690 691 wr32(IGC_SRRCTL(reg_idx), srrctl); 692 693 rxdctl |= IGC_RXDCTL_PTHRESH; 694 rxdctl |= IGC_RXDCTL_HTHRESH << 8; 695 rxdctl |= IGC_RXDCTL_WTHRESH << 16; 696 697 /* initialize rx_buffer_info */ 698 memset(ring->rx_buffer_info, 0, 699 sizeof(struct igc_rx_buffer) * ring->count); 700 701 /* initialize Rx descriptor 0 */ 702 rx_desc = IGC_RX_DESC(ring, 0); 703 rx_desc->wb.upper.length = 0; 704 705 /* enable receive descriptor fetching */ 706 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 707 708 wr32(IGC_RXDCTL(reg_idx), rxdctl); 709 } 710 711 /** 712 * igc_configure_rx - Configure receive Unit after Reset 713 * @adapter: board private structure 714 * 715 * Configure the Rx unit of the MAC after a reset. 716 */ 717 static void igc_configure_rx(struct igc_adapter *adapter) 718 { 719 int i; 720 721 /* Setup the HW Rx Head and Tail Descriptor Pointers and 722 * the Base and Length of the Rx Descriptor Ring 723 */ 724 for (i = 0; i < adapter->num_rx_queues; i++) 725 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 726 } 727 728 /** 729 * igc_configure_tx_ring - Configure transmit ring after Reset 730 * @adapter: board private structure 731 * @ring: tx ring to configure 732 * 733 * Configure a transmit ring after a reset. 734 */ 735 static void igc_configure_tx_ring(struct igc_adapter *adapter, 736 struct igc_ring *ring) 737 { 738 struct igc_hw *hw = &adapter->hw; 739 int reg_idx = ring->reg_idx; 740 u64 tdba = ring->dma; 741 u32 txdctl = 0; 742 743 ring->xsk_pool = igc_get_xsk_pool(adapter, ring); 744 745 /* disable the queue */ 746 wr32(IGC_TXDCTL(reg_idx), 0); 747 wrfl(); 748 749 wr32(IGC_TDLEN(reg_idx), 750 ring->count * sizeof(union igc_adv_tx_desc)); 751 wr32(IGC_TDBAL(reg_idx), 752 tdba & 0x00000000ffffffffULL); 753 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 754 755 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 756 wr32(IGC_TDH(reg_idx), 0); 757 writel(0, ring->tail); 758 759 txdctl |= IGC_TXDCTL_PTHRESH(8) | IGC_TXDCTL_HTHRESH(1) | 760 IGC_TXDCTL_WTHRESH(16) | IGC_TXDCTL_QUEUE_ENABLE; 761 762 wr32(IGC_TXDCTL(reg_idx), txdctl); 763 } 764 765 /** 766 * igc_configure_tx - Configure transmit Unit after Reset 767 * @adapter: board private structure 768 * 769 * Configure the Tx unit of the MAC after a reset. 770 */ 771 static void igc_configure_tx(struct igc_adapter *adapter) 772 { 773 int i; 774 775 for (i = 0; i < adapter->num_tx_queues; i++) 776 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 777 } 778 779 /** 780 * igc_setup_mrqc - configure the multiple receive queue control registers 781 * @adapter: Board private structure 782 */ 783 static void igc_setup_mrqc(struct igc_adapter *adapter) 784 { 785 struct igc_hw *hw = &adapter->hw; 786 u32 j, num_rx_queues; 787 u32 mrqc, rxcsum; 788 789 igc_write_rss_key(adapter); 790 791 num_rx_queues = adapter->rss_queues; 792 793 if (adapter->rss_indir_tbl_init != num_rx_queues) { 794 for (j = 0; j < IGC_RETA_SIZE; j++) 795 adapter->rss_indir_tbl[j] = 796 (j * num_rx_queues) / IGC_RETA_SIZE; 797 adapter->rss_indir_tbl_init = num_rx_queues; 798 } 799 igc_write_rss_indir_tbl(adapter); 800 801 /* Disable raw packet checksumming so that RSS hash is placed in 802 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 803 * offloads as they are enabled by default 804 */ 805 rxcsum = rd32(IGC_RXCSUM); 806 rxcsum |= IGC_RXCSUM_PCSD; 807 808 /* Enable Receive Checksum Offload for SCTP */ 809 rxcsum |= IGC_RXCSUM_CRCOFL; 810 811 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 812 wr32(IGC_RXCSUM, rxcsum); 813 814 /* Generate RSS hash based on packet types, TCP/UDP 815 * port numbers and/or IPv4/v6 src and dst addresses 816 */ 817 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 818 IGC_MRQC_RSS_FIELD_IPV4_TCP | 819 IGC_MRQC_RSS_FIELD_IPV6 | 820 IGC_MRQC_RSS_FIELD_IPV6_TCP | 821 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 822 823 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 824 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 825 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 826 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 827 828 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 829 830 wr32(IGC_MRQC, mrqc); 831 } 832 833 /** 834 * igc_setup_rctl - configure the receive control registers 835 * @adapter: Board private structure 836 */ 837 static void igc_setup_rctl(struct igc_adapter *adapter) 838 { 839 struct igc_hw *hw = &adapter->hw; 840 u32 rctl; 841 842 rctl = rd32(IGC_RCTL); 843 844 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 845 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 846 847 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 848 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 849 850 /* enable stripping of CRC. Newer features require 851 * that the HW strips the CRC. 852 */ 853 rctl |= IGC_RCTL_SECRC; 854 855 /* disable store bad packets and clear size bits. */ 856 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 857 858 /* enable LPE to allow for reception of jumbo frames */ 859 rctl |= IGC_RCTL_LPE; 860 861 /* disable queue 0 to prevent tail write w/o re-config */ 862 wr32(IGC_RXDCTL(0), 0); 863 864 /* This is useful for sniffing bad packets. */ 865 if (adapter->netdev->features & NETIF_F_RXALL) { 866 /* UPE and MPE will be handled by normal PROMISC logic 867 * in set_rx_mode 868 */ 869 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 870 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 871 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 872 873 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 874 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 875 } 876 877 wr32(IGC_RCTL, rctl); 878 } 879 880 /** 881 * igc_setup_tctl - configure the transmit control registers 882 * @adapter: Board private structure 883 */ 884 static void igc_setup_tctl(struct igc_adapter *adapter) 885 { 886 struct igc_hw *hw = &adapter->hw; 887 u32 tctl; 888 889 /* disable queue 0 which icould be enabled by default */ 890 wr32(IGC_TXDCTL(0), 0); 891 892 /* Program the Transmit Control Register */ 893 tctl = rd32(IGC_TCTL); 894 tctl &= ~IGC_TCTL_CT; 895 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 896 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 897 898 /* Enable transmits */ 899 tctl |= IGC_TCTL_EN; 900 901 wr32(IGC_TCTL, tctl); 902 } 903 904 /** 905 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 906 * @adapter: Pointer to adapter where the filter should be set 907 * @index: Filter index 908 * @type: MAC address filter type (source or destination) 909 * @addr: MAC address 910 * @queue: If non-negative, queue assignment feature is enabled and frames 911 * matching the filter are enqueued onto 'queue'. Otherwise, queue 912 * assignment is disabled. 913 */ 914 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 915 enum igc_mac_filter_type type, 916 const u8 *addr, int queue) 917 { 918 struct net_device *dev = adapter->netdev; 919 struct igc_hw *hw = &adapter->hw; 920 u32 ral, rah; 921 922 if (WARN_ON(index >= hw->mac.rar_entry_count)) 923 return; 924 925 ral = le32_to_cpup((__le32 *)(addr)); 926 rah = le16_to_cpup((__le16 *)(addr + 4)); 927 928 if (type == IGC_MAC_FILTER_TYPE_SRC) { 929 rah &= ~IGC_RAH_ASEL_MASK; 930 rah |= IGC_RAH_ASEL_SRC_ADDR; 931 } 932 933 if (queue >= 0) { 934 rah &= ~IGC_RAH_QSEL_MASK; 935 rah |= (queue << IGC_RAH_QSEL_SHIFT); 936 rah |= IGC_RAH_QSEL_ENABLE; 937 } 938 939 rah |= IGC_RAH_AV; 940 941 wr32(IGC_RAL(index), ral); 942 wr32(IGC_RAH(index), rah); 943 944 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 945 } 946 947 /** 948 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 949 * @adapter: Pointer to adapter where the filter should be cleared 950 * @index: Filter index 951 */ 952 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 953 { 954 struct net_device *dev = adapter->netdev; 955 struct igc_hw *hw = &adapter->hw; 956 957 if (WARN_ON(index >= hw->mac.rar_entry_count)) 958 return; 959 960 wr32(IGC_RAL(index), 0); 961 wr32(IGC_RAH(index), 0); 962 963 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 964 } 965 966 /* Set default MAC address for the PF in the first RAR entry */ 967 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 968 { 969 struct net_device *dev = adapter->netdev; 970 u8 *addr = adapter->hw.mac.addr; 971 972 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 973 974 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 975 } 976 977 /** 978 * igc_set_mac - Change the Ethernet Address of the NIC 979 * @netdev: network interface device structure 980 * @p: pointer to an address structure 981 * 982 * Returns 0 on success, negative on failure 983 */ 984 static int igc_set_mac(struct net_device *netdev, void *p) 985 { 986 struct igc_adapter *adapter = netdev_priv(netdev); 987 struct igc_hw *hw = &adapter->hw; 988 struct sockaddr *addr = p; 989 990 if (!is_valid_ether_addr(addr->sa_data)) 991 return -EADDRNOTAVAIL; 992 993 eth_hw_addr_set(netdev, addr->sa_data); 994 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 995 996 /* set the correct pool for the new PF MAC address in entry 0 */ 997 igc_set_default_mac_filter(adapter); 998 999 return 0; 1000 } 1001 1002 /** 1003 * igc_write_mc_addr_list - write multicast addresses to MTA 1004 * @netdev: network interface device structure 1005 * 1006 * Writes multicast address list to the MTA hash table. 1007 * Returns: -ENOMEM on failure 1008 * 0 on no addresses written 1009 * X on writing X addresses to MTA 1010 **/ 1011 static int igc_write_mc_addr_list(struct net_device *netdev) 1012 { 1013 struct igc_adapter *adapter = netdev_priv(netdev); 1014 struct igc_hw *hw = &adapter->hw; 1015 struct netdev_hw_addr *ha; 1016 u8 *mta_list; 1017 int i; 1018 1019 if (netdev_mc_empty(netdev)) { 1020 /* nothing to program, so clear mc list */ 1021 igc_update_mc_addr_list(hw, NULL, 0); 1022 return 0; 1023 } 1024 1025 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 1026 if (!mta_list) 1027 return -ENOMEM; 1028 1029 /* The shared function expects a packed array of only addresses. */ 1030 i = 0; 1031 netdev_for_each_mc_addr(ha, netdev) 1032 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 1033 1034 igc_update_mc_addr_list(hw, mta_list, i); 1035 kfree(mta_list); 1036 1037 return netdev_mc_count(netdev); 1038 } 1039 1040 static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime, 1041 bool *first_flag, bool *insert_empty) 1042 { 1043 struct igc_adapter *adapter = netdev_priv(ring->netdev); 1044 ktime_t cycle_time = adapter->cycle_time; 1045 ktime_t base_time = adapter->base_time; 1046 ktime_t now = ktime_get_clocktai(); 1047 ktime_t baset_est, end_of_cycle; 1048 s32 launchtime; 1049 s64 n; 1050 1051 n = div64_s64(ktime_sub_ns(now, base_time), cycle_time); 1052 1053 baset_est = ktime_add_ns(base_time, cycle_time * (n)); 1054 end_of_cycle = ktime_add_ns(baset_est, cycle_time); 1055 1056 if (ktime_compare(txtime, end_of_cycle) >= 0) { 1057 if (baset_est != ring->last_ff_cycle) { 1058 *first_flag = true; 1059 ring->last_ff_cycle = baset_est; 1060 1061 if (ktime_compare(end_of_cycle, ring->last_tx_cycle) > 0) 1062 *insert_empty = true; 1063 } 1064 } 1065 1066 /* Introducing a window at end of cycle on which packets 1067 * potentially not honor launchtime. Window of 5us chosen 1068 * considering software update the tail pointer and packets 1069 * are dma'ed to packet buffer. 1070 */ 1071 if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC)) 1072 netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n", 1073 txtime); 1074 1075 ring->last_tx_cycle = end_of_cycle; 1076 1077 launchtime = ktime_sub_ns(txtime, baset_est); 1078 if (launchtime > 0) 1079 div_s64_rem(launchtime, cycle_time, &launchtime); 1080 else 1081 launchtime = 0; 1082 1083 return cpu_to_le32(launchtime); 1084 } 1085 1086 static int igc_init_empty_frame(struct igc_ring *ring, 1087 struct igc_tx_buffer *buffer, 1088 struct sk_buff *skb) 1089 { 1090 unsigned int size; 1091 dma_addr_t dma; 1092 1093 size = skb_headlen(skb); 1094 1095 dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE); 1096 if (dma_mapping_error(ring->dev, dma)) { 1097 net_err_ratelimited("%s: DMA mapping error for empty frame\n", 1098 netdev_name(ring->netdev)); 1099 return -ENOMEM; 1100 } 1101 1102 buffer->type = IGC_TX_BUFFER_TYPE_SKB; 1103 buffer->skb = skb; 1104 buffer->protocol = 0; 1105 buffer->bytecount = skb->len; 1106 buffer->gso_segs = 1; 1107 buffer->time_stamp = jiffies; 1108 dma_unmap_len_set(buffer, len, skb->len); 1109 dma_unmap_addr_set(buffer, dma, dma); 1110 1111 return 0; 1112 } 1113 1114 static void igc_init_tx_empty_descriptor(struct igc_ring *ring, 1115 struct sk_buff *skb, 1116 struct igc_tx_buffer *first) 1117 { 1118 union igc_adv_tx_desc *desc; 1119 u32 cmd_type, olinfo_status; 1120 1121 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | 1122 IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | 1123 first->bytecount; 1124 olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; 1125 1126 desc = IGC_TX_DESC(ring, ring->next_to_use); 1127 desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1128 desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1129 desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma)); 1130 1131 netdev_tx_sent_queue(txring_txq(ring), skb->len); 1132 1133 first->next_to_watch = desc; 1134 1135 ring->next_to_use++; 1136 if (ring->next_to_use == ring->count) 1137 ring->next_to_use = 0; 1138 } 1139 1140 #define IGC_EMPTY_FRAME_SIZE 60 1141 1142 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 1143 __le32 launch_time, bool first_flag, 1144 u32 vlan_macip_lens, u32 type_tucmd, 1145 u32 mss_l4len_idx) 1146 { 1147 struct igc_adv_tx_context_desc *context_desc; 1148 u16 i = tx_ring->next_to_use; 1149 1150 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 1151 1152 i++; 1153 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 1154 1155 /* set bits to identify this as an advanced context descriptor */ 1156 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 1157 1158 /* For i225, context index must be unique per ring. */ 1159 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 1160 mss_l4len_idx |= tx_ring->reg_idx << 4; 1161 1162 if (first_flag) 1163 mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST; 1164 1165 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 1166 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 1167 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 1168 context_desc->launch_time = launch_time; 1169 } 1170 1171 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first, 1172 __le32 launch_time, bool first_flag) 1173 { 1174 struct sk_buff *skb = first->skb; 1175 u32 vlan_macip_lens = 0; 1176 u32 type_tucmd = 0; 1177 1178 if (skb->ip_summed != CHECKSUM_PARTIAL) { 1179 csum_failed: 1180 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 1181 !tx_ring->launchtime_enable) 1182 return; 1183 goto no_csum; 1184 } 1185 1186 switch (skb->csum_offset) { 1187 case offsetof(struct tcphdr, check): 1188 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1189 fallthrough; 1190 case offsetof(struct udphdr, check): 1191 break; 1192 case offsetof(struct sctphdr, checksum): 1193 /* validate that this is actually an SCTP request */ 1194 if (skb_csum_is_sctp(skb)) { 1195 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 1196 break; 1197 } 1198 fallthrough; 1199 default: 1200 skb_checksum_help(skb); 1201 goto csum_failed; 1202 } 1203 1204 /* update TX checksum flag */ 1205 first->tx_flags |= IGC_TX_FLAGS_CSUM; 1206 vlan_macip_lens = skb_checksum_start_offset(skb) - 1207 skb_network_offset(skb); 1208 no_csum: 1209 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 1210 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1211 1212 igc_tx_ctxtdesc(tx_ring, launch_time, first_flag, 1213 vlan_macip_lens, type_tucmd, 0); 1214 } 1215 1216 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1217 { 1218 struct net_device *netdev = tx_ring->netdev; 1219 1220 netif_stop_subqueue(netdev, tx_ring->queue_index); 1221 1222 /* memory barriier comment */ 1223 smp_mb(); 1224 1225 /* We need to check again in a case another CPU has just 1226 * made room available. 1227 */ 1228 if (igc_desc_unused(tx_ring) < size) 1229 return -EBUSY; 1230 1231 /* A reprieve! */ 1232 netif_wake_subqueue(netdev, tx_ring->queue_index); 1233 1234 u64_stats_update_begin(&tx_ring->tx_syncp2); 1235 tx_ring->tx_stats.restart_queue2++; 1236 u64_stats_update_end(&tx_ring->tx_syncp2); 1237 1238 return 0; 1239 } 1240 1241 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1242 { 1243 if (igc_desc_unused(tx_ring) >= size) 1244 return 0; 1245 return __igc_maybe_stop_tx(tx_ring, size); 1246 } 1247 1248 #define IGC_SET_FLAG(_input, _flag, _result) \ 1249 (((_flag) <= (_result)) ? \ 1250 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1251 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1252 1253 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1254 { 1255 /* set type for advanced descriptor with frame checksum insertion */ 1256 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1257 IGC_ADVTXD_DCMD_DEXT | 1258 IGC_ADVTXD_DCMD_IFCS; 1259 1260 /* set HW vlan bit if vlan is present */ 1261 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN, 1262 IGC_ADVTXD_DCMD_VLE); 1263 1264 /* set segmentation bits for TSO */ 1265 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1266 (IGC_ADVTXD_DCMD_TSE)); 1267 1268 /* set timestamp bit if present, will select the register set 1269 * based on the _TSTAMP(_X) bit. 1270 */ 1271 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1272 (IGC_ADVTXD_MAC_TSTAMP)); 1273 1274 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_1, 1275 (IGC_ADVTXD_TSTAMP_REG_1)); 1276 1277 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_2, 1278 (IGC_ADVTXD_TSTAMP_REG_2)); 1279 1280 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_3, 1281 (IGC_ADVTXD_TSTAMP_REG_3)); 1282 1283 /* insert frame checksum */ 1284 cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS); 1285 1286 return cmd_type; 1287 } 1288 1289 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1290 union igc_adv_tx_desc *tx_desc, 1291 u32 tx_flags, unsigned int paylen) 1292 { 1293 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1294 1295 /* insert L4 checksum */ 1296 olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_CSUM, 1297 (IGC_TXD_POPTS_TXSM << 8)); 1298 1299 /* insert IPv4 checksum */ 1300 olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_IPV4, 1301 (IGC_TXD_POPTS_IXSM << 8)); 1302 1303 /* Use the second timer (free running, in general) for the timestamp */ 1304 olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_TIMER_1, 1305 IGC_TXD_PTP2_TIMER_1); 1306 1307 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1308 } 1309 1310 static int igc_tx_map(struct igc_ring *tx_ring, 1311 struct igc_tx_buffer *first, 1312 const u8 hdr_len) 1313 { 1314 struct sk_buff *skb = first->skb; 1315 struct igc_tx_buffer *tx_buffer; 1316 union igc_adv_tx_desc *tx_desc; 1317 u32 tx_flags = first->tx_flags; 1318 skb_frag_t *frag; 1319 u16 i = tx_ring->next_to_use; 1320 unsigned int data_len, size; 1321 dma_addr_t dma; 1322 u32 cmd_type; 1323 1324 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1325 tx_desc = IGC_TX_DESC(tx_ring, i); 1326 1327 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1328 1329 size = skb_headlen(skb); 1330 data_len = skb->data_len; 1331 1332 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1333 1334 tx_buffer = first; 1335 1336 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1337 if (dma_mapping_error(tx_ring->dev, dma)) 1338 goto dma_error; 1339 1340 /* record length, and DMA address */ 1341 dma_unmap_len_set(tx_buffer, len, size); 1342 dma_unmap_addr_set(tx_buffer, dma, dma); 1343 1344 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1345 1346 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1347 tx_desc->read.cmd_type_len = 1348 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1349 1350 i++; 1351 tx_desc++; 1352 if (i == tx_ring->count) { 1353 tx_desc = IGC_TX_DESC(tx_ring, 0); 1354 i = 0; 1355 } 1356 tx_desc->read.olinfo_status = 0; 1357 1358 dma += IGC_MAX_DATA_PER_TXD; 1359 size -= IGC_MAX_DATA_PER_TXD; 1360 1361 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1362 } 1363 1364 if (likely(!data_len)) 1365 break; 1366 1367 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1368 1369 i++; 1370 tx_desc++; 1371 if (i == tx_ring->count) { 1372 tx_desc = IGC_TX_DESC(tx_ring, 0); 1373 i = 0; 1374 } 1375 tx_desc->read.olinfo_status = 0; 1376 1377 size = skb_frag_size(frag); 1378 data_len -= size; 1379 1380 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1381 size, DMA_TO_DEVICE); 1382 1383 tx_buffer = &tx_ring->tx_buffer_info[i]; 1384 } 1385 1386 /* write last descriptor with RS and EOP bits */ 1387 cmd_type |= size | IGC_TXD_DCMD; 1388 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1389 1390 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1391 1392 /* set the timestamp */ 1393 first->time_stamp = jiffies; 1394 1395 skb_tx_timestamp(skb); 1396 1397 /* Force memory writes to complete before letting h/w know there 1398 * are new descriptors to fetch. (Only applicable for weak-ordered 1399 * memory model archs, such as IA-64). 1400 * 1401 * We also need this memory barrier to make certain all of the 1402 * status bits have been updated before next_to_watch is written. 1403 */ 1404 wmb(); 1405 1406 /* set next_to_watch value indicating a packet is present */ 1407 first->next_to_watch = tx_desc; 1408 1409 i++; 1410 if (i == tx_ring->count) 1411 i = 0; 1412 1413 tx_ring->next_to_use = i; 1414 1415 /* Make sure there is space in the ring for the next send. */ 1416 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1417 1418 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1419 writel(i, tx_ring->tail); 1420 } 1421 1422 return 0; 1423 dma_error: 1424 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1425 tx_buffer = &tx_ring->tx_buffer_info[i]; 1426 1427 /* clear dma mappings for failed tx_buffer_info map */ 1428 while (tx_buffer != first) { 1429 if (dma_unmap_len(tx_buffer, len)) 1430 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 1431 1432 if (i-- == 0) 1433 i += tx_ring->count; 1434 tx_buffer = &tx_ring->tx_buffer_info[i]; 1435 } 1436 1437 if (dma_unmap_len(tx_buffer, len)) 1438 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 1439 1440 dev_kfree_skb_any(tx_buffer->skb); 1441 tx_buffer->skb = NULL; 1442 1443 tx_ring->next_to_use = i; 1444 1445 return -1; 1446 } 1447 1448 static int igc_tso(struct igc_ring *tx_ring, 1449 struct igc_tx_buffer *first, 1450 __le32 launch_time, bool first_flag, 1451 u8 *hdr_len) 1452 { 1453 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1454 struct sk_buff *skb = first->skb; 1455 union { 1456 struct iphdr *v4; 1457 struct ipv6hdr *v6; 1458 unsigned char *hdr; 1459 } ip; 1460 union { 1461 struct tcphdr *tcp; 1462 struct udphdr *udp; 1463 unsigned char *hdr; 1464 } l4; 1465 u32 paylen, l4_offset; 1466 int err; 1467 1468 if (skb->ip_summed != CHECKSUM_PARTIAL) 1469 return 0; 1470 1471 if (!skb_is_gso(skb)) 1472 return 0; 1473 1474 err = skb_cow_head(skb, 0); 1475 if (err < 0) 1476 return err; 1477 1478 ip.hdr = skb_network_header(skb); 1479 l4.hdr = skb_checksum_start(skb); 1480 1481 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1482 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1483 1484 /* initialize outer IP header fields */ 1485 if (ip.v4->version == 4) { 1486 unsigned char *csum_start = skb_checksum_start(skb); 1487 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1488 1489 /* IP header will have to cancel out any data that 1490 * is not a part of the outer IP header 1491 */ 1492 ip.v4->check = csum_fold(csum_partial(trans_start, 1493 csum_start - trans_start, 1494 0)); 1495 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1496 1497 ip.v4->tot_len = 0; 1498 first->tx_flags |= IGC_TX_FLAGS_TSO | 1499 IGC_TX_FLAGS_CSUM | 1500 IGC_TX_FLAGS_IPV4; 1501 } else { 1502 ip.v6->payload_len = 0; 1503 first->tx_flags |= IGC_TX_FLAGS_TSO | 1504 IGC_TX_FLAGS_CSUM; 1505 } 1506 1507 /* determine offset of inner transport header */ 1508 l4_offset = l4.hdr - skb->data; 1509 1510 /* remove payload length from inner checksum */ 1511 paylen = skb->len - l4_offset; 1512 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1513 /* compute length of segmentation header */ 1514 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1515 csum_replace_by_diff(&l4.tcp->check, 1516 (__force __wsum)htonl(paylen)); 1517 } else { 1518 /* compute length of segmentation header */ 1519 *hdr_len = sizeof(*l4.udp) + l4_offset; 1520 csum_replace_by_diff(&l4.udp->check, 1521 (__force __wsum)htonl(paylen)); 1522 } 1523 1524 /* update gso size and bytecount with header size */ 1525 first->gso_segs = skb_shinfo(skb)->gso_segs; 1526 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1527 1528 /* MSS L4LEN IDX */ 1529 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1530 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1531 1532 /* VLAN MACLEN IPLEN */ 1533 vlan_macip_lens = l4.hdr - ip.hdr; 1534 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1535 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1536 1537 igc_tx_ctxtdesc(tx_ring, launch_time, first_flag, 1538 vlan_macip_lens, type_tucmd, mss_l4len_idx); 1539 1540 return 1; 1541 } 1542 1543 static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags) 1544 { 1545 int i; 1546 1547 for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { 1548 struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i]; 1549 1550 if (tstamp->skb) 1551 continue; 1552 1553 tstamp->skb = skb_get(skb); 1554 tstamp->start = jiffies; 1555 *flags = tstamp->flags; 1556 1557 return true; 1558 } 1559 1560 return false; 1561 } 1562 1563 static int igc_insert_empty_frame(struct igc_ring *tx_ring) 1564 { 1565 struct igc_tx_buffer *empty_info; 1566 struct sk_buff *empty_skb; 1567 void *data; 1568 int ret; 1569 1570 empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1571 empty_skb = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC); 1572 if (unlikely(!empty_skb)) { 1573 net_err_ratelimited("%s: skb alloc error for empty frame\n", 1574 netdev_name(tx_ring->netdev)); 1575 return -ENOMEM; 1576 } 1577 1578 data = skb_put(empty_skb, IGC_EMPTY_FRAME_SIZE); 1579 memset(data, 0, IGC_EMPTY_FRAME_SIZE); 1580 1581 /* Prepare DMA mapping and Tx buffer information */ 1582 ret = igc_init_empty_frame(tx_ring, empty_info, empty_skb); 1583 if (unlikely(ret)) { 1584 dev_kfree_skb_any(empty_skb); 1585 return ret; 1586 } 1587 1588 /* Prepare advanced context descriptor for empty packet */ 1589 igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0); 1590 1591 /* Prepare advanced data descriptor for empty packet */ 1592 igc_init_tx_empty_descriptor(tx_ring, empty_skb, empty_info); 1593 1594 return 0; 1595 } 1596 1597 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1598 struct igc_ring *tx_ring) 1599 { 1600 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1601 bool first_flag = false, insert_empty = false; 1602 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1603 __be16 protocol = vlan_get_protocol(skb); 1604 struct igc_tx_buffer *first; 1605 __le32 launch_time = 0; 1606 u32 tx_flags = 0; 1607 unsigned short f; 1608 ktime_t txtime; 1609 u8 hdr_len = 0; 1610 int tso = 0; 1611 1612 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1613 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1614 * + 2 desc gap to keep tail from touching head, 1615 * + 1 desc for context descriptor, 1616 * + 2 desc for inserting an empty packet for launch time, 1617 * otherwise try next time 1618 */ 1619 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1620 count += TXD_USE_COUNT(skb_frag_size( 1621 &skb_shinfo(skb)->frags[f])); 1622 1623 if (igc_maybe_stop_tx(tx_ring, count + 5)) { 1624 /* this is a hard error */ 1625 return NETDEV_TX_BUSY; 1626 } 1627 1628 if (!tx_ring->launchtime_enable) 1629 goto done; 1630 1631 txtime = skb->tstamp; 1632 skb->tstamp = ktime_set(0, 0); 1633 launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty); 1634 1635 if (insert_empty) { 1636 /* Reset the launch time if the required empty frame fails to 1637 * be inserted. However, this packet is not dropped, so it 1638 * "dirties" the current Qbv cycle. This ensures that the 1639 * upcoming packet, which is scheduled in the next Qbv cycle, 1640 * does not require an empty frame. This way, the launch time 1641 * continues to function correctly despite the current failure 1642 * to insert the empty frame. 1643 */ 1644 if (igc_insert_empty_frame(tx_ring)) 1645 launch_time = 0; 1646 } 1647 1648 done: 1649 /* record the location of the first descriptor for this packet */ 1650 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1651 first->type = IGC_TX_BUFFER_TYPE_SKB; 1652 first->skb = skb; 1653 first->bytecount = skb->len; 1654 first->gso_segs = 1; 1655 1656 if (adapter->qbv_transition || tx_ring->oper_gate_closed) 1657 goto out_drop; 1658 1659 if (tx_ring->max_sdu > 0 && first->bytecount > tx_ring->max_sdu) { 1660 adapter->stats.txdrop++; 1661 goto out_drop; 1662 } 1663 1664 if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags) && 1665 skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1666 unsigned long flags; 1667 u32 tstamp_flags; 1668 1669 spin_lock_irqsave(&adapter->ptp_tx_lock, flags); 1670 if (igc_request_tx_tstamp(adapter, skb, &tstamp_flags)) { 1671 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1672 tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags; 1673 if (skb->sk && 1674 READ_ONCE(skb->sk->sk_tsflags) & SOF_TIMESTAMPING_BIND_PHC) 1675 tx_flags |= IGC_TX_FLAGS_TSTAMP_TIMER_1; 1676 } else { 1677 adapter->tx_hwtstamp_skipped++; 1678 } 1679 1680 spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); 1681 } 1682 1683 if (skb_vlan_tag_present(skb)) { 1684 tx_flags |= IGC_TX_FLAGS_VLAN; 1685 tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT); 1686 } 1687 1688 /* record initial flags and protocol */ 1689 first->tx_flags = tx_flags; 1690 first->protocol = protocol; 1691 1692 /* For preemptible queue, manually pad the skb so that HW includes 1693 * padding bytes in mCRC calculation 1694 */ 1695 if (tx_ring->preemptible && skb->len < ETH_ZLEN) { 1696 if (skb_padto(skb, ETH_ZLEN)) 1697 goto out_drop; 1698 skb_put(skb, ETH_ZLEN - skb->len); 1699 } 1700 1701 tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len); 1702 if (tso < 0) 1703 goto out_drop; 1704 else if (!tso) 1705 igc_tx_csum(tx_ring, first, launch_time, first_flag); 1706 1707 igc_tx_map(tx_ring, first, hdr_len); 1708 1709 return NETDEV_TX_OK; 1710 1711 out_drop: 1712 dev_kfree_skb_any(first->skb); 1713 first->skb = NULL; 1714 1715 return NETDEV_TX_OK; 1716 } 1717 1718 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1719 struct sk_buff *skb) 1720 { 1721 unsigned int r_idx = skb->queue_mapping; 1722 1723 if (r_idx >= adapter->num_tx_queues) 1724 r_idx = r_idx % adapter->num_tx_queues; 1725 1726 return adapter->tx_ring[r_idx]; 1727 } 1728 1729 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1730 struct net_device *netdev) 1731 { 1732 struct igc_adapter *adapter = netdev_priv(netdev); 1733 1734 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1735 * in order to meet this minimum size requirement. 1736 */ 1737 if (skb_put_padto(skb, 17)) 1738 return NETDEV_TX_OK; 1739 1740 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1741 } 1742 1743 static void igc_rx_checksum(struct igc_ring *ring, 1744 union igc_adv_rx_desc *rx_desc, 1745 struct sk_buff *skb) 1746 { 1747 skb_checksum_none_assert(skb); 1748 1749 /* Ignore Checksum bit is set */ 1750 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1751 return; 1752 1753 /* Rx checksum disabled via ethtool */ 1754 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1755 return; 1756 1757 /* TCP/UDP checksum error bit is set */ 1758 if (igc_test_staterr(rx_desc, 1759 IGC_RXDEXT_STATERR_L4E | 1760 IGC_RXDEXT_STATERR_IPE)) { 1761 /* work around errata with sctp packets where the TCPE aka 1762 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1763 * packets (aka let the stack check the crc32c) 1764 */ 1765 if (!(skb->len == 60 && 1766 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1767 u64_stats_update_begin(&ring->rx_syncp); 1768 ring->rx_stats.csum_err++; 1769 u64_stats_update_end(&ring->rx_syncp); 1770 } 1771 /* let the stack verify checksum errors */ 1772 return; 1773 } 1774 /* It must be a TCP or UDP packet with a valid checksum */ 1775 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1776 IGC_RXD_STAT_UDPCS)) 1777 skb->ip_summed = CHECKSUM_UNNECESSARY; 1778 1779 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1780 le32_to_cpu(rx_desc->wb.upper.status_error)); 1781 } 1782 1783 /* Mapping HW RSS Type to enum pkt_hash_types */ 1784 static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = { 1785 [IGC_RSS_TYPE_NO_HASH] = PKT_HASH_TYPE_L2, 1786 [IGC_RSS_TYPE_HASH_TCP_IPV4] = PKT_HASH_TYPE_L4, 1787 [IGC_RSS_TYPE_HASH_IPV4] = PKT_HASH_TYPE_L3, 1788 [IGC_RSS_TYPE_HASH_TCP_IPV6] = PKT_HASH_TYPE_L4, 1789 [IGC_RSS_TYPE_HASH_IPV6_EX] = PKT_HASH_TYPE_L3, 1790 [IGC_RSS_TYPE_HASH_IPV6] = PKT_HASH_TYPE_L3, 1791 [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4, 1792 [IGC_RSS_TYPE_HASH_UDP_IPV4] = PKT_HASH_TYPE_L4, 1793 [IGC_RSS_TYPE_HASH_UDP_IPV6] = PKT_HASH_TYPE_L4, 1794 [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4, 1795 [10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */ 1796 [11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask */ 1797 [12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisions */ 1798 [13] = PKT_HASH_TYPE_NONE, 1799 [14] = PKT_HASH_TYPE_NONE, 1800 [15] = PKT_HASH_TYPE_NONE, 1801 }; 1802 1803 static inline void igc_rx_hash(struct igc_ring *ring, 1804 union igc_adv_rx_desc *rx_desc, 1805 struct sk_buff *skb) 1806 { 1807 if (ring->netdev->features & NETIF_F_RXHASH) { 1808 u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); 1809 u32 rss_type = igc_rss_type(rx_desc); 1810 1811 skb_set_hash(skb, rss_hash, igc_rss_type_table[rss_type]); 1812 } 1813 } 1814 1815 static void igc_rx_vlan(struct igc_ring *rx_ring, 1816 union igc_adv_rx_desc *rx_desc, 1817 struct sk_buff *skb) 1818 { 1819 struct net_device *dev = rx_ring->netdev; 1820 u16 vid; 1821 1822 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1823 igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) { 1824 if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) && 1825 test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) 1826 vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); 1827 else 1828 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 1829 1830 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 1831 } 1832 } 1833 1834 /** 1835 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1836 * @rx_ring: rx descriptor ring packet is being transacted on 1837 * @rx_desc: pointer to the EOP Rx descriptor 1838 * @skb: pointer to current skb being populated 1839 * 1840 * This function checks the ring, descriptor, and packet information in order 1841 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1842 * skb. 1843 */ 1844 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1845 union igc_adv_rx_desc *rx_desc, 1846 struct sk_buff *skb) 1847 { 1848 igc_rx_hash(rx_ring, rx_desc, skb); 1849 1850 igc_rx_checksum(rx_ring, rx_desc, skb); 1851 1852 igc_rx_vlan(rx_ring, rx_desc, skb); 1853 1854 skb_record_rx_queue(skb, rx_ring->queue_index); 1855 1856 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1857 } 1858 1859 static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features) 1860 { 1861 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); 1862 struct igc_adapter *adapter = netdev_priv(netdev); 1863 struct igc_hw *hw = &adapter->hw; 1864 u32 ctrl; 1865 1866 ctrl = rd32(IGC_CTRL); 1867 1868 if (enable) { 1869 /* enable VLAN tag insert/strip */ 1870 ctrl |= IGC_CTRL_VME; 1871 } else { 1872 /* disable VLAN tag insert/strip */ 1873 ctrl &= ~IGC_CTRL_VME; 1874 } 1875 wr32(IGC_CTRL, ctrl); 1876 } 1877 1878 static void igc_restore_vlan(struct igc_adapter *adapter) 1879 { 1880 igc_vlan_mode(adapter->netdev, adapter->netdev->features); 1881 } 1882 1883 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1884 const unsigned int size, 1885 int *rx_buffer_pgcnt) 1886 { 1887 struct igc_rx_buffer *rx_buffer; 1888 1889 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1890 *rx_buffer_pgcnt = 1891 #if (PAGE_SIZE < 8192) 1892 page_count(rx_buffer->page); 1893 #else 1894 0; 1895 #endif 1896 prefetchw(rx_buffer->page); 1897 1898 /* we are reusing so sync this buffer for CPU use */ 1899 dma_sync_single_range_for_cpu(rx_ring->dev, 1900 rx_buffer->dma, 1901 rx_buffer->page_offset, 1902 size, 1903 DMA_FROM_DEVICE); 1904 1905 rx_buffer->pagecnt_bias--; 1906 1907 return rx_buffer; 1908 } 1909 1910 static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer, 1911 unsigned int truesize) 1912 { 1913 #if (PAGE_SIZE < 8192) 1914 buffer->page_offset ^= truesize; 1915 #else 1916 buffer->page_offset += truesize; 1917 #endif 1918 } 1919 1920 static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring, 1921 unsigned int size) 1922 { 1923 unsigned int truesize; 1924 1925 #if (PAGE_SIZE < 8192) 1926 truesize = igc_rx_pg_size(ring) / 2; 1927 #else 1928 truesize = ring_uses_build_skb(ring) ? 1929 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1930 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1931 SKB_DATA_ALIGN(size); 1932 #endif 1933 return truesize; 1934 } 1935 1936 /** 1937 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1938 * @rx_ring: rx descriptor ring to transact packets on 1939 * @rx_buffer: buffer containing page to add 1940 * @skb: sk_buff to place the data into 1941 * @size: size of buffer to be added 1942 * 1943 * This function will add the data contained in rx_buffer->page to the skb. 1944 */ 1945 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1946 struct igc_rx_buffer *rx_buffer, 1947 struct sk_buff *skb, 1948 unsigned int size) 1949 { 1950 unsigned int truesize; 1951 1952 #if (PAGE_SIZE < 8192) 1953 truesize = igc_rx_pg_size(rx_ring) / 2; 1954 #else 1955 truesize = ring_uses_build_skb(rx_ring) ? 1956 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1957 SKB_DATA_ALIGN(size); 1958 #endif 1959 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1960 rx_buffer->page_offset, size, truesize); 1961 1962 igc_rx_buffer_flip(rx_buffer, truesize); 1963 } 1964 1965 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1966 struct igc_rx_buffer *rx_buffer, 1967 struct xdp_buff *xdp) 1968 { 1969 unsigned int size = xdp->data_end - xdp->data; 1970 unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); 1971 unsigned int metasize = xdp->data - xdp->data_meta; 1972 struct sk_buff *skb; 1973 1974 /* prefetch first cache line of first page */ 1975 net_prefetch(xdp->data_meta); 1976 1977 /* build an skb around the page buffer */ 1978 skb = napi_build_skb(xdp->data_hard_start, truesize); 1979 if (unlikely(!skb)) 1980 return NULL; 1981 1982 /* update pointers within the skb to store the data */ 1983 skb_reserve(skb, xdp->data - xdp->data_hard_start); 1984 __skb_put(skb, size); 1985 if (metasize) 1986 skb_metadata_set(skb, metasize); 1987 1988 igc_rx_buffer_flip(rx_buffer, truesize); 1989 return skb; 1990 } 1991 1992 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1993 struct igc_rx_buffer *rx_buffer, 1994 struct igc_xdp_buff *ctx) 1995 { 1996 struct xdp_buff *xdp = &ctx->xdp; 1997 unsigned int metasize = xdp->data - xdp->data_meta; 1998 unsigned int size = xdp->data_end - xdp->data; 1999 unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); 2000 void *va = xdp->data; 2001 unsigned int headlen; 2002 struct sk_buff *skb; 2003 2004 /* prefetch first cache line of first page */ 2005 net_prefetch(xdp->data_meta); 2006 2007 /* allocate a skb to store the frags */ 2008 skb = napi_alloc_skb(&rx_ring->q_vector->napi, 2009 IGC_RX_HDR_LEN + metasize); 2010 if (unlikely(!skb)) 2011 return NULL; 2012 2013 if (ctx->rx_ts) { 2014 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV; 2015 skb_hwtstamps(skb)->netdev_data = ctx->rx_ts; 2016 } 2017 2018 /* Determine available headroom for copy */ 2019 headlen = size; 2020 if (headlen > IGC_RX_HDR_LEN) 2021 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 2022 2023 /* align pull length to size of long to optimize memcpy performance */ 2024 memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta, 2025 ALIGN(headlen + metasize, sizeof(long))); 2026 2027 if (metasize) { 2028 skb_metadata_set(skb, metasize); 2029 __skb_pull(skb, metasize); 2030 } 2031 2032 /* update all of the pointers */ 2033 size -= headlen; 2034 if (size) { 2035 skb_add_rx_frag(skb, 0, rx_buffer->page, 2036 (va + headlen) - page_address(rx_buffer->page), 2037 size, truesize); 2038 igc_rx_buffer_flip(rx_buffer, truesize); 2039 } else { 2040 rx_buffer->pagecnt_bias++; 2041 } 2042 2043 return skb; 2044 } 2045 2046 /** 2047 * igc_reuse_rx_page - page flip buffer and store it back on the ring 2048 * @rx_ring: rx descriptor ring to store buffers on 2049 * @old_buff: donor buffer to have page reused 2050 * 2051 * Synchronizes page for reuse by the adapter 2052 */ 2053 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 2054 struct igc_rx_buffer *old_buff) 2055 { 2056 u16 nta = rx_ring->next_to_alloc; 2057 struct igc_rx_buffer *new_buff; 2058 2059 new_buff = &rx_ring->rx_buffer_info[nta]; 2060 2061 /* update, and store next to alloc */ 2062 nta++; 2063 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 2064 2065 /* Transfer page from old buffer to new buffer. 2066 * Move each member individually to avoid possible store 2067 * forwarding stalls. 2068 */ 2069 new_buff->dma = old_buff->dma; 2070 new_buff->page = old_buff->page; 2071 new_buff->page_offset = old_buff->page_offset; 2072 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 2073 } 2074 2075 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer, 2076 int rx_buffer_pgcnt) 2077 { 2078 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 2079 struct page *page = rx_buffer->page; 2080 2081 /* avoid re-using remote and pfmemalloc pages */ 2082 if (!dev_page_is_reusable(page)) 2083 return false; 2084 2085 #if (PAGE_SIZE < 8192) 2086 /* if we are only owner of page we can reuse it */ 2087 if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) 2088 return false; 2089 #else 2090 #define IGC_LAST_OFFSET \ 2091 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 2092 2093 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 2094 return false; 2095 #endif 2096 2097 /* If we have drained the page fragment pool we need to update 2098 * the pagecnt_bias and page count so that we fully restock the 2099 * number of references the driver holds. 2100 */ 2101 if (unlikely(pagecnt_bias == 1)) { 2102 page_ref_add(page, USHRT_MAX - 1); 2103 rx_buffer->pagecnt_bias = USHRT_MAX; 2104 } 2105 2106 return true; 2107 } 2108 2109 /** 2110 * igc_is_non_eop - process handling of non-EOP buffers 2111 * @rx_ring: Rx ring being processed 2112 * @rx_desc: Rx descriptor for current buffer 2113 * 2114 * This function updates next to clean. If the buffer is an EOP buffer 2115 * this function exits returning false, otherwise it will place the 2116 * sk_buff in the next buffer to be chained and return true indicating 2117 * that this is in fact a non-EOP buffer. 2118 */ 2119 static bool igc_is_non_eop(struct igc_ring *rx_ring, 2120 union igc_adv_rx_desc *rx_desc) 2121 { 2122 u32 ntc = rx_ring->next_to_clean + 1; 2123 2124 /* fetch, update, and store next to clean */ 2125 ntc = (ntc < rx_ring->count) ? ntc : 0; 2126 rx_ring->next_to_clean = ntc; 2127 2128 prefetch(IGC_RX_DESC(rx_ring, ntc)); 2129 2130 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 2131 return false; 2132 2133 return true; 2134 } 2135 2136 /** 2137 * igc_cleanup_headers - Correct corrupted or empty headers 2138 * @rx_ring: rx descriptor ring packet is being transacted on 2139 * @rx_desc: pointer to the EOP Rx descriptor 2140 * @skb: pointer to current skb being fixed 2141 * 2142 * Address the case where we are pulling data in on pages only 2143 * and as such no data is present in the skb header. 2144 * 2145 * In addition if skb is not at least 60 bytes we need to pad it so that 2146 * it is large enough to qualify as a valid Ethernet frame. 2147 * 2148 * Returns true if an error was encountered and skb was freed. 2149 */ 2150 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 2151 union igc_adv_rx_desc *rx_desc, 2152 struct sk_buff *skb) 2153 { 2154 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { 2155 struct net_device *netdev = rx_ring->netdev; 2156 2157 if (!(netdev->features & NETIF_F_RXALL)) { 2158 dev_kfree_skb_any(skb); 2159 return true; 2160 } 2161 } 2162 2163 /* if eth_skb_pad returns an error the skb was freed */ 2164 if (eth_skb_pad(skb)) 2165 return true; 2166 2167 return false; 2168 } 2169 2170 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 2171 struct igc_rx_buffer *rx_buffer, 2172 int rx_buffer_pgcnt) 2173 { 2174 if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { 2175 /* hand second half of page back to the ring */ 2176 igc_reuse_rx_page(rx_ring, rx_buffer); 2177 } else { 2178 /* We are not reusing the buffer so unmap it and free 2179 * any references we are holding to it 2180 */ 2181 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 2182 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 2183 IGC_RX_DMA_ATTR); 2184 __page_frag_cache_drain(rx_buffer->page, 2185 rx_buffer->pagecnt_bias); 2186 } 2187 2188 /* clear contents of rx_buffer */ 2189 rx_buffer->page = NULL; 2190 } 2191 2192 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 2193 { 2194 struct igc_adapter *adapter = rx_ring->q_vector->adapter; 2195 2196 if (ring_uses_build_skb(rx_ring)) 2197 return IGC_SKB_PAD; 2198 if (igc_xdp_is_enabled(adapter)) 2199 return XDP_PACKET_HEADROOM; 2200 2201 return 0; 2202 } 2203 2204 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 2205 struct igc_rx_buffer *bi) 2206 { 2207 struct page *page = bi->page; 2208 dma_addr_t dma; 2209 2210 /* since we are recycling buffers we should seldom need to alloc */ 2211 if (likely(page)) 2212 return true; 2213 2214 /* alloc new page for storage */ 2215 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 2216 if (unlikely(!page)) { 2217 rx_ring->rx_stats.alloc_failed++; 2218 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 2219 return false; 2220 } 2221 2222 /* map page for use */ 2223 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 2224 igc_rx_pg_size(rx_ring), 2225 DMA_FROM_DEVICE, 2226 IGC_RX_DMA_ATTR); 2227 2228 /* if mapping failed free memory back to system since 2229 * there isn't much point in holding memory we can't use 2230 */ 2231 if (dma_mapping_error(rx_ring->dev, dma)) { 2232 __free_page(page); 2233 2234 rx_ring->rx_stats.alloc_failed++; 2235 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 2236 return false; 2237 } 2238 2239 bi->dma = dma; 2240 bi->page = page; 2241 bi->page_offset = igc_rx_offset(rx_ring); 2242 page_ref_add(page, USHRT_MAX - 1); 2243 bi->pagecnt_bias = USHRT_MAX; 2244 2245 return true; 2246 } 2247 2248 /** 2249 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 2250 * @rx_ring: rx descriptor ring 2251 * @cleaned_count: number of buffers to clean 2252 */ 2253 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 2254 { 2255 union igc_adv_rx_desc *rx_desc; 2256 u16 i = rx_ring->next_to_use; 2257 struct igc_rx_buffer *bi; 2258 u16 bufsz; 2259 2260 /* nothing to do */ 2261 if (!cleaned_count) 2262 return; 2263 2264 rx_desc = IGC_RX_DESC(rx_ring, i); 2265 bi = &rx_ring->rx_buffer_info[i]; 2266 i -= rx_ring->count; 2267 2268 bufsz = igc_rx_bufsz(rx_ring); 2269 2270 do { 2271 if (!igc_alloc_mapped_page(rx_ring, bi)) 2272 break; 2273 2274 /* sync the buffer for use by the device */ 2275 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 2276 bi->page_offset, bufsz, 2277 DMA_FROM_DEVICE); 2278 2279 /* Refresh the desc even if buffer_addrs didn't change 2280 * because each write-back erases this info. 2281 */ 2282 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 2283 2284 rx_desc++; 2285 bi++; 2286 i++; 2287 if (unlikely(!i)) { 2288 rx_desc = IGC_RX_DESC(rx_ring, 0); 2289 bi = rx_ring->rx_buffer_info; 2290 i -= rx_ring->count; 2291 } 2292 2293 /* clear the length for the next_to_use descriptor */ 2294 rx_desc->wb.upper.length = 0; 2295 2296 cleaned_count--; 2297 } while (cleaned_count); 2298 2299 i += rx_ring->count; 2300 2301 if (rx_ring->next_to_use != i) { 2302 /* record the next descriptor to use */ 2303 rx_ring->next_to_use = i; 2304 2305 /* update next to alloc since we have filled the ring */ 2306 rx_ring->next_to_alloc = i; 2307 2308 /* Force memory writes to complete before letting h/w 2309 * know there are new descriptors to fetch. (Only 2310 * applicable for weak-ordered memory model archs, 2311 * such as IA-64). 2312 */ 2313 wmb(); 2314 writel(i, rx_ring->tail); 2315 } 2316 } 2317 2318 static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) 2319 { 2320 union igc_adv_rx_desc *desc; 2321 u16 i = ring->next_to_use; 2322 struct igc_rx_buffer *bi; 2323 dma_addr_t dma; 2324 bool ok = true; 2325 2326 if (!count) 2327 return ok; 2328 2329 XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff); 2330 2331 desc = IGC_RX_DESC(ring, i); 2332 bi = &ring->rx_buffer_info[i]; 2333 i -= ring->count; 2334 2335 do { 2336 bi->xdp = xsk_buff_alloc(ring->xsk_pool); 2337 if (!bi->xdp) { 2338 ok = false; 2339 break; 2340 } 2341 2342 dma = xsk_buff_xdp_get_dma(bi->xdp); 2343 desc->read.pkt_addr = cpu_to_le64(dma); 2344 2345 desc++; 2346 bi++; 2347 i++; 2348 if (unlikely(!i)) { 2349 desc = IGC_RX_DESC(ring, 0); 2350 bi = ring->rx_buffer_info; 2351 i -= ring->count; 2352 } 2353 2354 /* Clear the length for the next_to_use descriptor. */ 2355 desc->wb.upper.length = 0; 2356 2357 count--; 2358 } while (count); 2359 2360 i += ring->count; 2361 2362 if (ring->next_to_use != i) { 2363 ring->next_to_use = i; 2364 2365 /* Force memory writes to complete before letting h/w 2366 * know there are new descriptors to fetch. (Only 2367 * applicable for weak-ordered memory model archs, 2368 * such as IA-64). 2369 */ 2370 wmb(); 2371 writel(i, ring->tail); 2372 } 2373 2374 return ok; 2375 } 2376 2377 /* This function requires __netif_tx_lock is held by the caller. */ 2378 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring, 2379 struct xdp_frame *xdpf) 2380 { 2381 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); 2382 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; 2383 u16 count, index = ring->next_to_use; 2384 struct igc_tx_buffer *head = &ring->tx_buffer_info[index]; 2385 struct igc_tx_buffer *buffer = head; 2386 union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index); 2387 u32 olinfo_status, len = xdpf->len, cmd_type; 2388 void *data = xdpf->data; 2389 u16 i; 2390 2391 count = TXD_USE_COUNT(len); 2392 for (i = 0; i < nr_frags; i++) 2393 count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); 2394 2395 if (igc_maybe_stop_tx(ring, count + 3)) { 2396 /* this is a hard error */ 2397 return -EBUSY; 2398 } 2399 2400 i = 0; 2401 head->bytecount = xdp_get_frame_len(xdpf); 2402 head->type = IGC_TX_BUFFER_TYPE_XDP; 2403 head->gso_segs = 1; 2404 head->xdpf = xdpf; 2405 2406 olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; 2407 desc->read.olinfo_status = cpu_to_le32(olinfo_status); 2408 2409 for (;;) { 2410 dma_addr_t dma; 2411 2412 dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); 2413 if (dma_mapping_error(ring->dev, dma)) { 2414 netdev_err_once(ring->netdev, 2415 "Failed to map DMA for TX\n"); 2416 goto unmap; 2417 } 2418 2419 dma_unmap_len_set(buffer, len, len); 2420 dma_unmap_addr_set(buffer, dma, dma); 2421 2422 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | 2423 IGC_ADVTXD_DCMD_IFCS | len; 2424 2425 desc->read.cmd_type_len = cpu_to_le32(cmd_type); 2426 desc->read.buffer_addr = cpu_to_le64(dma); 2427 2428 buffer->protocol = 0; 2429 2430 if (++index == ring->count) 2431 index = 0; 2432 2433 if (i == nr_frags) 2434 break; 2435 2436 buffer = &ring->tx_buffer_info[index]; 2437 desc = IGC_TX_DESC(ring, index); 2438 desc->read.olinfo_status = 0; 2439 2440 data = skb_frag_address(&sinfo->frags[i]); 2441 len = skb_frag_size(&sinfo->frags[i]); 2442 i++; 2443 } 2444 desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD); 2445 2446 netdev_tx_sent_queue(txring_txq(ring), head->bytecount); 2447 /* set the timestamp */ 2448 head->time_stamp = jiffies; 2449 /* set next_to_watch value indicating a packet is present */ 2450 head->next_to_watch = desc; 2451 ring->next_to_use = index; 2452 2453 return 0; 2454 2455 unmap: 2456 for (;;) { 2457 buffer = &ring->tx_buffer_info[index]; 2458 if (dma_unmap_len(buffer, len)) 2459 dma_unmap_page(ring->dev, 2460 dma_unmap_addr(buffer, dma), 2461 dma_unmap_len(buffer, len), 2462 DMA_TO_DEVICE); 2463 dma_unmap_len_set(buffer, len, 0); 2464 if (buffer == head) 2465 break; 2466 2467 if (!index) 2468 index += ring->count; 2469 index--; 2470 } 2471 2472 return -ENOMEM; 2473 } 2474 2475 struct igc_ring *igc_get_tx_ring(struct igc_adapter *adapter, int cpu) 2476 { 2477 int index = cpu; 2478 2479 if (unlikely(index < 0)) 2480 index = 0; 2481 2482 while (index >= adapter->num_tx_queues) 2483 index -= adapter->num_tx_queues; 2484 2485 return adapter->tx_ring[index]; 2486 } 2487 2488 static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp) 2489 { 2490 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 2491 int cpu = smp_processor_id(); 2492 struct netdev_queue *nq; 2493 struct igc_ring *ring; 2494 int res; 2495 2496 if (unlikely(!xdpf)) 2497 return -EFAULT; 2498 2499 ring = igc_get_tx_ring(adapter, cpu); 2500 nq = txring_txq(ring); 2501 2502 __netif_tx_lock(nq, cpu); 2503 /* Avoid transmit queue timeout since we share it with the slow path */ 2504 txq_trans_cond_update(nq); 2505 res = igc_xdp_init_tx_descriptor(ring, xdpf); 2506 __netif_tx_unlock(nq); 2507 return res; 2508 } 2509 2510 /* This function assumes rcu_read_lock() is held by the caller. */ 2511 static int __igc_xdp_run_prog(struct igc_adapter *adapter, 2512 struct bpf_prog *prog, 2513 struct xdp_buff *xdp) 2514 { 2515 u32 act = bpf_prog_run_xdp(prog, xdp); 2516 2517 switch (act) { 2518 case XDP_PASS: 2519 return IGC_XDP_PASS; 2520 case XDP_TX: 2521 if (igc_xdp_xmit_back(adapter, xdp) < 0) 2522 goto out_failure; 2523 return IGC_XDP_TX; 2524 case XDP_REDIRECT: 2525 if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0) 2526 goto out_failure; 2527 return IGC_XDP_REDIRECT; 2528 break; 2529 default: 2530 bpf_warn_invalid_xdp_action(adapter->netdev, prog, act); 2531 fallthrough; 2532 case XDP_ABORTED: 2533 out_failure: 2534 trace_xdp_exception(adapter->netdev, prog, act); 2535 fallthrough; 2536 case XDP_DROP: 2537 return IGC_XDP_CONSUMED; 2538 } 2539 } 2540 2541 static int igc_xdp_run_prog(struct igc_adapter *adapter, struct xdp_buff *xdp) 2542 { 2543 struct bpf_prog *prog; 2544 int res; 2545 2546 prog = READ_ONCE(adapter->xdp_prog); 2547 if (!prog) { 2548 res = IGC_XDP_PASS; 2549 goto out; 2550 } 2551 2552 res = __igc_xdp_run_prog(adapter, prog, xdp); 2553 2554 out: 2555 return res; 2556 } 2557 2558 /* This function assumes __netif_tx_lock is held by the caller. */ 2559 void igc_flush_tx_descriptors(struct igc_ring *ring) 2560 { 2561 /* Once tail pointer is updated, hardware can fetch the descriptors 2562 * any time so we issue a write membar here to ensure all memory 2563 * writes are complete before the tail pointer is updated. 2564 */ 2565 wmb(); 2566 writel(ring->next_to_use, ring->tail); 2567 } 2568 2569 static void igc_finalize_xdp(struct igc_adapter *adapter, int status) 2570 { 2571 int cpu = smp_processor_id(); 2572 struct netdev_queue *nq; 2573 struct igc_ring *ring; 2574 2575 if (status & IGC_XDP_TX) { 2576 ring = igc_get_tx_ring(adapter, cpu); 2577 nq = txring_txq(ring); 2578 2579 __netif_tx_lock(nq, cpu); 2580 igc_flush_tx_descriptors(ring); 2581 __netif_tx_unlock(nq); 2582 } 2583 2584 if (status & IGC_XDP_REDIRECT) 2585 xdp_do_flush(); 2586 } 2587 2588 static void igc_update_rx_stats(struct igc_q_vector *q_vector, 2589 unsigned int packets, unsigned int bytes) 2590 { 2591 struct igc_ring *ring = q_vector->rx.ring; 2592 2593 u64_stats_update_begin(&ring->rx_syncp); 2594 ring->rx_stats.packets += packets; 2595 ring->rx_stats.bytes += bytes; 2596 u64_stats_update_end(&ring->rx_syncp); 2597 2598 q_vector->rx.total_packets += packets; 2599 q_vector->rx.total_bytes += bytes; 2600 } 2601 2602 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 2603 { 2604 unsigned int total_bytes = 0, total_packets = 0; 2605 struct igc_adapter *adapter = q_vector->adapter; 2606 struct igc_ring *rx_ring = q_vector->rx.ring; 2607 struct sk_buff *skb = rx_ring->skb; 2608 u16 cleaned_count = igc_desc_unused(rx_ring); 2609 int xdp_status = 0, rx_buffer_pgcnt; 2610 int xdp_res = 0; 2611 2612 while (likely(total_packets < budget)) { 2613 struct igc_xdp_buff ctx = { .rx_ts = NULL }; 2614 struct igc_rx_buffer *rx_buffer; 2615 union igc_adv_rx_desc *rx_desc; 2616 unsigned int size, truesize; 2617 int pkt_offset = 0; 2618 void *pktbuf; 2619 2620 /* return some buffers to hardware, one at a time is too slow */ 2621 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 2622 igc_alloc_rx_buffers(rx_ring, cleaned_count); 2623 cleaned_count = 0; 2624 } 2625 2626 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 2627 size = le16_to_cpu(rx_desc->wb.upper.length); 2628 if (!size) 2629 break; 2630 2631 /* This memory barrier is needed to keep us from reading 2632 * any other fields out of the rx_desc until we know the 2633 * descriptor has been written back 2634 */ 2635 dma_rmb(); 2636 2637 rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt); 2638 truesize = igc_get_rx_frame_truesize(rx_ring, size); 2639 2640 pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; 2641 2642 if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) { 2643 ctx.rx_ts = pktbuf; 2644 pkt_offset = IGC_TS_HDR_LEN; 2645 size -= IGC_TS_HDR_LEN; 2646 } 2647 2648 if (igc_fpe_is_pmac_enabled(adapter) && 2649 igc_fpe_handle_mpacket(adapter, rx_desc, size, pktbuf + pkt_offset)) { 2650 /* Advance the ring next-to-clean */ 2651 igc_is_non_eop(rx_ring, rx_desc); 2652 cleaned_count++; 2653 continue; 2654 } 2655 2656 if (!skb) { 2657 xdp_init_buff(&ctx.xdp, truesize, &rx_ring->xdp_rxq); 2658 xdp_prepare_buff(&ctx.xdp, pktbuf - igc_rx_offset(rx_ring), 2659 igc_rx_offset(rx_ring) + pkt_offset, 2660 size, true); 2661 xdp_buff_clear_frags_flag(&ctx.xdp); 2662 ctx.rx_desc = rx_desc; 2663 2664 xdp_res = igc_xdp_run_prog(adapter, &ctx.xdp); 2665 } 2666 2667 if (xdp_res) { 2668 switch (xdp_res) { 2669 case IGC_XDP_CONSUMED: 2670 rx_buffer->pagecnt_bias++; 2671 break; 2672 case IGC_XDP_TX: 2673 case IGC_XDP_REDIRECT: 2674 igc_rx_buffer_flip(rx_buffer, truesize); 2675 xdp_status |= xdp_res; 2676 break; 2677 } 2678 2679 total_packets++; 2680 total_bytes += size; 2681 } else if (skb) 2682 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 2683 else if (ring_uses_build_skb(rx_ring)) 2684 skb = igc_build_skb(rx_ring, rx_buffer, &ctx.xdp); 2685 else 2686 skb = igc_construct_skb(rx_ring, rx_buffer, &ctx); 2687 2688 /* exit if we failed to retrieve a buffer */ 2689 if (!xdp_res && !skb) { 2690 rx_ring->rx_stats.alloc_failed++; 2691 rx_buffer->pagecnt_bias++; 2692 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 2693 break; 2694 } 2695 2696 igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt); 2697 cleaned_count++; 2698 2699 /* fetch next buffer in frame if non-eop */ 2700 if (igc_is_non_eop(rx_ring, rx_desc)) 2701 continue; 2702 2703 /* verify the packet layout is correct */ 2704 if (xdp_res || igc_cleanup_headers(rx_ring, rx_desc, skb)) { 2705 skb = NULL; 2706 continue; 2707 } 2708 2709 /* probably a little skewed due to removing CRC */ 2710 total_bytes += skb->len; 2711 2712 /* populate checksum, VLAN, and protocol */ 2713 igc_process_skb_fields(rx_ring, rx_desc, skb); 2714 2715 napi_gro_receive(&q_vector->napi, skb); 2716 2717 /* reset skb pointer */ 2718 skb = NULL; 2719 2720 /* update budget accounting */ 2721 total_packets++; 2722 } 2723 2724 if (xdp_status) 2725 igc_finalize_xdp(adapter, xdp_status); 2726 2727 /* place incomplete frames back on ring for completion */ 2728 rx_ring->skb = skb; 2729 2730 igc_update_rx_stats(q_vector, total_packets, total_bytes); 2731 2732 if (cleaned_count) 2733 igc_alloc_rx_buffers(rx_ring, cleaned_count); 2734 2735 return total_packets; 2736 } 2737 2738 static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, 2739 struct igc_xdp_buff *ctx) 2740 { 2741 struct xdp_buff *xdp = &ctx->xdp; 2742 unsigned int totalsize = xdp->data_end - xdp->data_meta; 2743 unsigned int metasize = xdp->data - xdp->data_meta; 2744 struct sk_buff *skb; 2745 2746 net_prefetch(xdp->data_meta); 2747 2748 skb = napi_alloc_skb(&ring->q_vector->napi, totalsize); 2749 if (unlikely(!skb)) 2750 return NULL; 2751 2752 memcpy(__skb_put(skb, totalsize), xdp->data_meta, 2753 ALIGN(totalsize, sizeof(long))); 2754 2755 if (metasize) { 2756 skb_metadata_set(skb, metasize); 2757 __skb_pull(skb, metasize); 2758 } 2759 2760 if (ctx->rx_ts) { 2761 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV; 2762 skb_hwtstamps(skb)->netdev_data = ctx->rx_ts; 2763 } 2764 2765 return skb; 2766 } 2767 2768 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, 2769 union igc_adv_rx_desc *desc, 2770 struct igc_xdp_buff *ctx) 2771 { 2772 struct igc_ring *ring = q_vector->rx.ring; 2773 struct sk_buff *skb; 2774 2775 skb = igc_construct_skb_zc(ring, ctx); 2776 if (!skb) { 2777 ring->rx_stats.alloc_failed++; 2778 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &ring->flags); 2779 return; 2780 } 2781 2782 if (igc_cleanup_headers(ring, desc, skb)) 2783 return; 2784 2785 igc_process_skb_fields(ring, desc, skb); 2786 napi_gro_receive(&q_vector->napi, skb); 2787 } 2788 2789 static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp) 2790 { 2791 /* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The 2792 * igc_xdp_buff shares its layout with xdp_buff_xsk and private 2793 * igc_xdp_buff fields fall into xdp_buff_xsk->cb 2794 */ 2795 return (struct igc_xdp_buff *)xdp; 2796 } 2797 2798 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) 2799 { 2800 struct igc_adapter *adapter = q_vector->adapter; 2801 struct igc_ring *ring = q_vector->rx.ring; 2802 u16 cleaned_count = igc_desc_unused(ring); 2803 int total_bytes = 0, total_packets = 0; 2804 u16 ntc = ring->next_to_clean; 2805 struct bpf_prog *prog; 2806 bool failure = false; 2807 int xdp_status = 0; 2808 2809 rcu_read_lock(); 2810 2811 prog = READ_ONCE(adapter->xdp_prog); 2812 2813 while (likely(total_packets < budget)) { 2814 union igc_adv_rx_desc *desc; 2815 struct igc_rx_buffer *bi; 2816 struct igc_xdp_buff *ctx; 2817 unsigned int size; 2818 int res; 2819 2820 desc = IGC_RX_DESC(ring, ntc); 2821 size = le16_to_cpu(desc->wb.upper.length); 2822 if (!size) 2823 break; 2824 2825 /* This memory barrier is needed to keep us from reading 2826 * any other fields out of the rx_desc until we know the 2827 * descriptor has been written back 2828 */ 2829 dma_rmb(); 2830 2831 bi = &ring->rx_buffer_info[ntc]; 2832 2833 ctx = xsk_buff_to_igc_ctx(bi->xdp); 2834 ctx->rx_desc = desc; 2835 2836 if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) { 2837 ctx->rx_ts = bi->xdp->data; 2838 2839 bi->xdp->data += IGC_TS_HDR_LEN; 2840 2841 /* HW timestamp has been copied into local variable. Metadata 2842 * length when XDP program is called should be 0. 2843 */ 2844 bi->xdp->data_meta += IGC_TS_HDR_LEN; 2845 size -= IGC_TS_HDR_LEN; 2846 } else { 2847 ctx->rx_ts = NULL; 2848 } 2849 2850 bi->xdp->data_end = bi->xdp->data + size; 2851 xsk_buff_dma_sync_for_cpu(bi->xdp); 2852 2853 res = __igc_xdp_run_prog(adapter, prog, bi->xdp); 2854 switch (res) { 2855 case IGC_XDP_PASS: 2856 igc_dispatch_skb_zc(q_vector, desc, ctx); 2857 fallthrough; 2858 case IGC_XDP_CONSUMED: 2859 xsk_buff_free(bi->xdp); 2860 break; 2861 case IGC_XDP_TX: 2862 case IGC_XDP_REDIRECT: 2863 xdp_status |= res; 2864 break; 2865 } 2866 2867 bi->xdp = NULL; 2868 total_bytes += size; 2869 total_packets++; 2870 cleaned_count++; 2871 ntc++; 2872 if (ntc == ring->count) 2873 ntc = 0; 2874 } 2875 2876 ring->next_to_clean = ntc; 2877 rcu_read_unlock(); 2878 2879 if (cleaned_count >= IGC_RX_BUFFER_WRITE) 2880 failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count); 2881 2882 if (xdp_status) 2883 igc_finalize_xdp(adapter, xdp_status); 2884 2885 igc_update_rx_stats(q_vector, total_packets, total_bytes); 2886 2887 if (xsk_uses_need_wakeup(ring->xsk_pool)) { 2888 if (failure || ring->next_to_clean == ring->next_to_use) 2889 xsk_set_rx_need_wakeup(ring->xsk_pool); 2890 else 2891 xsk_clear_rx_need_wakeup(ring->xsk_pool); 2892 return total_packets; 2893 } 2894 2895 return failure ? budget : total_packets; 2896 } 2897 2898 static void igc_update_tx_stats(struct igc_q_vector *q_vector, 2899 unsigned int packets, unsigned int bytes) 2900 { 2901 struct igc_ring *ring = q_vector->tx.ring; 2902 2903 u64_stats_update_begin(&ring->tx_syncp); 2904 ring->tx_stats.bytes += bytes; 2905 ring->tx_stats.packets += packets; 2906 u64_stats_update_end(&ring->tx_syncp); 2907 2908 q_vector->tx.total_bytes += bytes; 2909 q_vector->tx.total_packets += packets; 2910 } 2911 2912 static void igc_xsk_request_timestamp(void *_priv) 2913 { 2914 struct igc_metadata_request *meta_req = _priv; 2915 struct igc_ring *tx_ring = meta_req->tx_ring; 2916 struct igc_tx_timestamp_request *tstamp; 2917 u32 tx_flags = IGC_TX_FLAGS_TSTAMP; 2918 struct igc_adapter *adapter; 2919 unsigned long lock_flags; 2920 bool found = false; 2921 int i; 2922 2923 if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) { 2924 adapter = netdev_priv(tx_ring->netdev); 2925 2926 spin_lock_irqsave(&adapter->ptp_tx_lock, lock_flags); 2927 2928 /* Search for available tstamp regs */ 2929 for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { 2930 tstamp = &adapter->tx_tstamp[i]; 2931 2932 /* tstamp->skb and tstamp->xsk_tx_buffer are in union. 2933 * When tstamp->skb is equal to NULL, 2934 * tstamp->xsk_tx_buffer is equal to NULL as well. 2935 * This condition means that the particular tstamp reg 2936 * is not occupied by other packet. 2937 */ 2938 if (!tstamp->skb) { 2939 found = true; 2940 break; 2941 } 2942 } 2943 2944 /* Return if no available tstamp regs */ 2945 if (!found) { 2946 adapter->tx_hwtstamp_skipped++; 2947 spin_unlock_irqrestore(&adapter->ptp_tx_lock, 2948 lock_flags); 2949 return; 2950 } 2951 2952 tstamp->start = jiffies; 2953 tstamp->xsk_queue_index = tx_ring->queue_index; 2954 tstamp->xsk_tx_buffer = meta_req->tx_buffer; 2955 tstamp->buffer_type = IGC_TX_BUFFER_TYPE_XSK; 2956 2957 /* Hold the transmit completion until timestamp is ready */ 2958 meta_req->tx_buffer->xsk_pending_ts = true; 2959 2960 /* Keep the pointer to tx_timestamp, which is located in XDP 2961 * metadata area. It is the location to store the value of 2962 * tx hardware timestamp. 2963 */ 2964 xsk_tx_metadata_to_compl(meta_req->meta, &tstamp->xsk_meta); 2965 2966 /* Set timestamp bit based on the _TSTAMP(_X) bit. */ 2967 tx_flags |= tstamp->flags; 2968 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2969 IGC_TX_FLAGS_TSTAMP, 2970 (IGC_ADVTXD_MAC_TSTAMP)); 2971 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2972 IGC_TX_FLAGS_TSTAMP_1, 2973 (IGC_ADVTXD_TSTAMP_REG_1)); 2974 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2975 IGC_TX_FLAGS_TSTAMP_2, 2976 (IGC_ADVTXD_TSTAMP_REG_2)); 2977 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2978 IGC_TX_FLAGS_TSTAMP_3, 2979 (IGC_ADVTXD_TSTAMP_REG_3)); 2980 2981 spin_unlock_irqrestore(&adapter->ptp_tx_lock, lock_flags); 2982 } 2983 } 2984 2985 static u64 igc_xsk_fill_timestamp(void *_priv) 2986 { 2987 return *(u64 *)_priv; 2988 } 2989 2990 static void igc_xsk_request_launch_time(u64 launch_time, void *_priv) 2991 { 2992 struct igc_metadata_request *meta_req = _priv; 2993 struct igc_ring *tx_ring = meta_req->tx_ring; 2994 __le32 launch_time_offset; 2995 bool insert_empty = false; 2996 bool first_flag = false; 2997 u16 used_desc = 0; 2998 2999 if (!tx_ring->launchtime_enable) 3000 return; 3001 3002 launch_time_offset = igc_tx_launchtime(tx_ring, 3003 ns_to_ktime(launch_time), 3004 &first_flag, &insert_empty); 3005 if (insert_empty) { 3006 /* Disregard the launch time request if the required empty frame 3007 * fails to be inserted. 3008 */ 3009 if (igc_insert_empty_frame(tx_ring)) 3010 return; 3011 3012 meta_req->tx_buffer = 3013 &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3014 /* Inserting an empty packet requires two descriptors: 3015 * one data descriptor and one context descriptor. 3016 */ 3017 used_desc += 2; 3018 } 3019 3020 /* Use one context descriptor to specify launch time and first flag. */ 3021 igc_tx_ctxtdesc(tx_ring, launch_time_offset, first_flag, 0, 0, 0); 3022 used_desc += 1; 3023 3024 /* Update the number of used descriptors in this request */ 3025 meta_req->used_desc += used_desc; 3026 } 3027 3028 const struct xsk_tx_metadata_ops igc_xsk_tx_metadata_ops = { 3029 .tmo_request_timestamp = igc_xsk_request_timestamp, 3030 .tmo_fill_timestamp = igc_xsk_fill_timestamp, 3031 .tmo_request_launch_time = igc_xsk_request_launch_time, 3032 }; 3033 3034 static void igc_xdp_xmit_zc(struct igc_ring *ring) 3035 { 3036 struct xsk_buff_pool *pool = ring->xsk_pool; 3037 struct netdev_queue *nq = txring_txq(ring); 3038 union igc_adv_tx_desc *tx_desc = NULL; 3039 int cpu = smp_processor_id(); 3040 struct xdp_desc xdp_desc; 3041 u16 budget, ntu; 3042 3043 if (!netif_carrier_ok(ring->netdev)) 3044 return; 3045 3046 __netif_tx_lock(nq, cpu); 3047 3048 /* Avoid transmit queue timeout since we share it with the slow path */ 3049 txq_trans_cond_update(nq); 3050 3051 ntu = ring->next_to_use; 3052 budget = igc_desc_unused(ring); 3053 3054 /* Packets with launch time require one data descriptor and one context 3055 * descriptor. When the launch time falls into the next Qbv cycle, we 3056 * may need to insert an empty packet, which requires two more 3057 * descriptors. Therefore, to be safe, we always ensure we have at least 3058 * 4 descriptors available. 3059 */ 3060 while (budget >= 4 && xsk_tx_peek_desc(pool, &xdp_desc)) { 3061 struct igc_metadata_request meta_req; 3062 struct xsk_tx_metadata *meta = NULL; 3063 struct igc_tx_buffer *bi; 3064 u32 olinfo_status; 3065 dma_addr_t dma; 3066 3067 meta_req.cmd_type = IGC_ADVTXD_DTYP_DATA | 3068 IGC_ADVTXD_DCMD_DEXT | 3069 IGC_ADVTXD_DCMD_IFCS | 3070 IGC_TXD_DCMD | xdp_desc.len; 3071 olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; 3072 3073 dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 3074 meta = xsk_buff_get_metadata(pool, xdp_desc.addr, 3075 xdp_desc.options); 3076 xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); 3077 bi = &ring->tx_buffer_info[ntu]; 3078 3079 meta_req.tx_ring = ring; 3080 meta_req.tx_buffer = bi; 3081 meta_req.meta = meta; 3082 meta_req.used_desc = 0; 3083 xsk_tx_metadata_request(pool, &meta, &igc_xsk_tx_metadata_ops, 3084 &meta_req); 3085 3086 /* xsk_tx_metadata_request() may have updated next_to_use */ 3087 ntu = ring->next_to_use; 3088 3089 /* xsk_tx_metadata_request() may have updated Tx buffer info */ 3090 bi = meta_req.tx_buffer; 3091 3092 /* xsk_tx_metadata_request() may use a few descriptors */ 3093 budget -= meta_req.used_desc; 3094 3095 tx_desc = IGC_TX_DESC(ring, ntu); 3096 tx_desc->read.cmd_type_len = cpu_to_le32(meta_req.cmd_type); 3097 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 3098 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3099 3100 bi->type = IGC_TX_BUFFER_TYPE_XSK; 3101 bi->protocol = 0; 3102 bi->bytecount = xdp_desc.len; 3103 bi->gso_segs = 1; 3104 bi->time_stamp = jiffies; 3105 bi->next_to_watch = tx_desc; 3106 3107 netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len); 3108 3109 ntu++; 3110 if (ntu == ring->count) 3111 ntu = 0; 3112 3113 ring->next_to_use = ntu; 3114 budget--; 3115 } 3116 3117 if (tx_desc) { 3118 igc_flush_tx_descriptors(ring); 3119 xsk_tx_release(pool); 3120 } 3121 3122 __netif_tx_unlock(nq); 3123 } 3124 3125 /** 3126 * igc_clean_tx_irq - Reclaim resources after transmit completes 3127 * @q_vector: pointer to q_vector containing needed info 3128 * @napi_budget: Used to determine if we are in netpoll 3129 * 3130 * returns true if ring is completely cleaned 3131 */ 3132 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 3133 { 3134 struct igc_adapter *adapter = q_vector->adapter; 3135 unsigned int total_bytes = 0, total_packets = 0; 3136 unsigned int budget = q_vector->tx.work_limit; 3137 struct igc_ring *tx_ring = q_vector->tx.ring; 3138 unsigned int i = tx_ring->next_to_clean; 3139 struct igc_tx_buffer *tx_buffer; 3140 union igc_adv_tx_desc *tx_desc; 3141 u32 xsk_frames = 0; 3142 3143 if (test_bit(__IGC_DOWN, &adapter->state)) 3144 return true; 3145 3146 tx_buffer = &tx_ring->tx_buffer_info[i]; 3147 tx_desc = IGC_TX_DESC(tx_ring, i); 3148 i -= tx_ring->count; 3149 3150 do { 3151 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 3152 3153 /* if next_to_watch is not set then there is no work pending */ 3154 if (!eop_desc) 3155 break; 3156 3157 /* prevent any other reads prior to eop_desc */ 3158 smp_rmb(); 3159 3160 /* if DD is not set pending work has not been completed */ 3161 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 3162 break; 3163 3164 if (igc_fpe_is_pmac_enabled(adapter) && 3165 igc_fpe_transmitted_smd_v(tx_desc)) 3166 ethtool_mmsv_event_handle(&adapter->fpe.mmsv, 3167 ETHTOOL_MMSV_LD_SENT_VERIFY_MPACKET); 3168 3169 /* Hold the completions while there's a pending tx hardware 3170 * timestamp request from XDP Tx metadata. 3171 */ 3172 if (tx_buffer->type == IGC_TX_BUFFER_TYPE_XSK && 3173 tx_buffer->xsk_pending_ts) 3174 break; 3175 3176 /* clear next_to_watch to prevent false hangs */ 3177 tx_buffer->next_to_watch = NULL; 3178 3179 /* update the statistics for this packet */ 3180 total_bytes += tx_buffer->bytecount; 3181 total_packets += tx_buffer->gso_segs; 3182 3183 switch (tx_buffer->type) { 3184 case IGC_TX_BUFFER_TYPE_XSK: 3185 xsk_frames++; 3186 break; 3187 case IGC_TX_BUFFER_TYPE_XDP: 3188 xdp_return_frame(tx_buffer->xdpf); 3189 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 3190 break; 3191 case IGC_TX_BUFFER_TYPE_SKB: 3192 napi_consume_skb(tx_buffer->skb, napi_budget); 3193 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 3194 break; 3195 default: 3196 netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); 3197 break; 3198 } 3199 3200 /* clear last DMA location and unmap remaining buffers */ 3201 while (tx_desc != eop_desc) { 3202 tx_buffer++; 3203 tx_desc++; 3204 i++; 3205 if (unlikely(!i)) { 3206 i -= tx_ring->count; 3207 tx_buffer = tx_ring->tx_buffer_info; 3208 tx_desc = IGC_TX_DESC(tx_ring, 0); 3209 } 3210 3211 /* unmap any remaining paged data */ 3212 if (dma_unmap_len(tx_buffer, len)) 3213 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 3214 } 3215 3216 /* move us one more past the eop_desc for start of next pkt */ 3217 tx_buffer++; 3218 tx_desc++; 3219 i++; 3220 if (unlikely(!i)) { 3221 i -= tx_ring->count; 3222 tx_buffer = tx_ring->tx_buffer_info; 3223 tx_desc = IGC_TX_DESC(tx_ring, 0); 3224 } 3225 3226 /* issue prefetch for next Tx descriptor */ 3227 prefetch(tx_desc); 3228 3229 /* update budget accounting */ 3230 budget--; 3231 } while (likely(budget)); 3232 3233 netdev_tx_completed_queue(txring_txq(tx_ring), 3234 total_packets, total_bytes); 3235 3236 i += tx_ring->count; 3237 tx_ring->next_to_clean = i; 3238 3239 igc_update_tx_stats(q_vector, total_packets, total_bytes); 3240 3241 if (tx_ring->xsk_pool) { 3242 if (xsk_frames) 3243 xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); 3244 if (xsk_uses_need_wakeup(tx_ring->xsk_pool)) 3245 xsk_set_tx_need_wakeup(tx_ring->xsk_pool); 3246 igc_xdp_xmit_zc(tx_ring); 3247 } 3248 3249 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 3250 struct igc_hw *hw = &adapter->hw; 3251 3252 /* Detect a transmit hang in hardware, this serializes the 3253 * check with the clearing of time_stamp and movement of i 3254 */ 3255 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 3256 if (tx_buffer->next_to_watch && 3257 time_after(jiffies, tx_buffer->time_stamp + 3258 (adapter->tx_timeout_factor * HZ)) && 3259 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF) && 3260 (rd32(IGC_TDH(tx_ring->reg_idx)) != readl(tx_ring->tail)) && 3261 !tx_ring->oper_gate_closed) { 3262 /* detected Tx unit hang */ 3263 netdev_err(tx_ring->netdev, 3264 "Detected Tx Unit Hang\n" 3265 " Tx Queue <%d>\n" 3266 " TDH <%x>\n" 3267 " TDT <%x>\n" 3268 " next_to_use <%x>\n" 3269 " next_to_clean <%x>\n" 3270 "buffer_info[next_to_clean]\n" 3271 " time_stamp <%lx>\n" 3272 " next_to_watch <%p>\n" 3273 " jiffies <%lx>\n" 3274 " desc.status <%x>\n", 3275 tx_ring->queue_index, 3276 rd32(IGC_TDH(tx_ring->reg_idx)), 3277 readl(tx_ring->tail), 3278 tx_ring->next_to_use, 3279 tx_ring->next_to_clean, 3280 tx_buffer->time_stamp, 3281 tx_buffer->next_to_watch, 3282 jiffies, 3283 tx_buffer->next_to_watch->wb.status); 3284 netif_stop_subqueue(tx_ring->netdev, 3285 tx_ring->queue_index); 3286 3287 /* we are about to reset, no point in enabling stuff */ 3288 return true; 3289 } 3290 } 3291 3292 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 3293 if (unlikely(total_packets && 3294 netif_carrier_ok(tx_ring->netdev) && 3295 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 3296 /* Make sure that anybody stopping the queue after this 3297 * sees the new next_to_clean. 3298 */ 3299 smp_mb(); 3300 if (__netif_subqueue_stopped(tx_ring->netdev, 3301 tx_ring->queue_index) && 3302 !(test_bit(__IGC_DOWN, &adapter->state))) { 3303 netif_wake_subqueue(tx_ring->netdev, 3304 tx_ring->queue_index); 3305 3306 u64_stats_update_begin(&tx_ring->tx_syncp); 3307 tx_ring->tx_stats.restart_queue++; 3308 u64_stats_update_end(&tx_ring->tx_syncp); 3309 } 3310 } 3311 3312 return !!budget; 3313 } 3314 3315 static int igc_find_mac_filter(struct igc_adapter *adapter, 3316 enum igc_mac_filter_type type, const u8 *addr) 3317 { 3318 struct igc_hw *hw = &adapter->hw; 3319 int max_entries = hw->mac.rar_entry_count; 3320 u32 ral, rah; 3321 int i; 3322 3323 for (i = 0; i < max_entries; i++) { 3324 ral = rd32(IGC_RAL(i)); 3325 rah = rd32(IGC_RAH(i)); 3326 3327 if (!(rah & IGC_RAH_AV)) 3328 continue; 3329 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 3330 continue; 3331 if ((rah & IGC_RAH_RAH_MASK) != 3332 le16_to_cpup((__le16 *)(addr + 4))) 3333 continue; 3334 if (ral != le32_to_cpup((__le32 *)(addr))) 3335 continue; 3336 3337 return i; 3338 } 3339 3340 return -1; 3341 } 3342 3343 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 3344 { 3345 struct igc_hw *hw = &adapter->hw; 3346 int max_entries = hw->mac.rar_entry_count; 3347 u32 rah; 3348 int i; 3349 3350 for (i = 0; i < max_entries; i++) { 3351 rah = rd32(IGC_RAH(i)); 3352 3353 if (!(rah & IGC_RAH_AV)) 3354 return i; 3355 } 3356 3357 return -1; 3358 } 3359 3360 /** 3361 * igc_add_mac_filter() - Add MAC address filter 3362 * @adapter: Pointer to adapter where the filter should be added 3363 * @type: MAC address filter type (source or destination) 3364 * @addr: MAC address 3365 * @queue: If non-negative, queue assignment feature is enabled and frames 3366 * matching the filter are enqueued onto 'queue'. Otherwise, queue 3367 * assignment is disabled. 3368 * 3369 * Return: 0 in case of success, negative errno code otherwise. 3370 */ 3371 static int igc_add_mac_filter(struct igc_adapter *adapter, 3372 enum igc_mac_filter_type type, const u8 *addr, 3373 int queue) 3374 { 3375 struct net_device *dev = adapter->netdev; 3376 int index; 3377 3378 index = igc_find_mac_filter(adapter, type, addr); 3379 if (index >= 0) 3380 goto update_filter; 3381 3382 index = igc_get_avail_mac_filter_slot(adapter); 3383 if (index < 0) 3384 return -ENOSPC; 3385 3386 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 3387 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 3388 addr, queue); 3389 3390 update_filter: 3391 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 3392 return 0; 3393 } 3394 3395 /** 3396 * igc_del_mac_filter() - Delete MAC address filter 3397 * @adapter: Pointer to adapter where the filter should be deleted from 3398 * @type: MAC address filter type (source or destination) 3399 * @addr: MAC address 3400 */ 3401 static void igc_del_mac_filter(struct igc_adapter *adapter, 3402 enum igc_mac_filter_type type, const u8 *addr) 3403 { 3404 struct net_device *dev = adapter->netdev; 3405 int index; 3406 3407 index = igc_find_mac_filter(adapter, type, addr); 3408 if (index < 0) 3409 return; 3410 3411 if (index == 0) { 3412 /* If this is the default filter, we don't actually delete it. 3413 * We just reset to its default value i.e. disable queue 3414 * assignment. 3415 */ 3416 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 3417 3418 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 3419 } else { 3420 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 3421 index, 3422 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 3423 addr); 3424 3425 igc_clear_mac_filter_hw(adapter, index); 3426 } 3427 } 3428 3429 /** 3430 * igc_add_vlan_prio_filter() - Add VLAN priority filter 3431 * @adapter: Pointer to adapter where the filter should be added 3432 * @prio: VLAN priority value 3433 * @queue: Queue number which matching frames are assigned to 3434 * 3435 * Return: 0 in case of success, negative errno code otherwise. 3436 */ 3437 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 3438 int queue) 3439 { 3440 struct net_device *dev = adapter->netdev; 3441 struct igc_hw *hw = &adapter->hw; 3442 u32 vlanpqf; 3443 3444 vlanpqf = rd32(IGC_VLANPQF); 3445 3446 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 3447 netdev_dbg(dev, "VLAN priority filter already in use\n"); 3448 return -EEXIST; 3449 } 3450 3451 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 3452 vlanpqf |= IGC_VLANPQF_VALID(prio); 3453 3454 wr32(IGC_VLANPQF, vlanpqf); 3455 3456 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 3457 prio, queue); 3458 return 0; 3459 } 3460 3461 /** 3462 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 3463 * @adapter: Pointer to adapter where the filter should be deleted from 3464 * @prio: VLAN priority value 3465 */ 3466 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 3467 { 3468 struct igc_hw *hw = &adapter->hw; 3469 u32 vlanpqf; 3470 3471 vlanpqf = rd32(IGC_VLANPQF); 3472 3473 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 3474 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 3475 3476 wr32(IGC_VLANPQF, vlanpqf); 3477 3478 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 3479 prio); 3480 } 3481 3482 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 3483 { 3484 struct igc_hw *hw = &adapter->hw; 3485 int i; 3486 3487 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 3488 u32 etqf = rd32(IGC_ETQF(i)); 3489 3490 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 3491 return i; 3492 } 3493 3494 return -1; 3495 } 3496 3497 /** 3498 * igc_add_etype_filter() - Add ethertype filter 3499 * @adapter: Pointer to adapter where the filter should be added 3500 * @etype: Ethertype value 3501 * @queue: If non-negative, queue assignment feature is enabled and frames 3502 * matching the filter are enqueued onto 'queue'. Otherwise, queue 3503 * assignment is disabled. 3504 * 3505 * Return: 0 in case of success, negative errno code otherwise. 3506 */ 3507 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 3508 int queue) 3509 { 3510 struct igc_hw *hw = &adapter->hw; 3511 int index; 3512 u32 etqf; 3513 3514 index = igc_get_avail_etype_filter_slot(adapter); 3515 if (index < 0) 3516 return -ENOSPC; 3517 3518 etqf = rd32(IGC_ETQF(index)); 3519 3520 etqf &= ~IGC_ETQF_ETYPE_MASK; 3521 etqf |= etype; 3522 3523 if (queue >= 0) { 3524 etqf &= ~IGC_ETQF_QUEUE_MASK; 3525 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 3526 etqf |= IGC_ETQF_QUEUE_ENABLE; 3527 } 3528 3529 etqf |= IGC_ETQF_FILTER_ENABLE; 3530 3531 wr32(IGC_ETQF(index), etqf); 3532 3533 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 3534 etype, queue); 3535 return 0; 3536 } 3537 3538 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 3539 { 3540 struct igc_hw *hw = &adapter->hw; 3541 int i; 3542 3543 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 3544 u32 etqf = rd32(IGC_ETQF(i)); 3545 3546 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 3547 return i; 3548 } 3549 3550 return -1; 3551 } 3552 3553 /** 3554 * igc_del_etype_filter() - Delete ethertype filter 3555 * @adapter: Pointer to adapter where the filter should be deleted from 3556 * @etype: Ethertype value 3557 */ 3558 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 3559 { 3560 struct igc_hw *hw = &adapter->hw; 3561 int index; 3562 3563 index = igc_find_etype_filter(adapter, etype); 3564 if (index < 0) 3565 return; 3566 3567 wr32(IGC_ETQF(index), 0); 3568 3569 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 3570 etype); 3571 } 3572 3573 static int igc_flex_filter_select(struct igc_adapter *adapter, 3574 struct igc_flex_filter *input, 3575 u32 *fhft) 3576 { 3577 struct igc_hw *hw = &adapter->hw; 3578 u8 fhft_index; 3579 u32 fhftsl; 3580 3581 if (input->index >= MAX_FLEX_FILTER) { 3582 netdev_err(adapter->netdev, "Wrong Flex Filter index selected!\n"); 3583 return -EINVAL; 3584 } 3585 3586 /* Indirect table select register */ 3587 fhftsl = rd32(IGC_FHFTSL); 3588 fhftsl &= ~IGC_FHFTSL_FTSL_MASK; 3589 switch (input->index) { 3590 case 0 ... 7: 3591 fhftsl |= 0x00; 3592 break; 3593 case 8 ... 15: 3594 fhftsl |= 0x01; 3595 break; 3596 case 16 ... 23: 3597 fhftsl |= 0x02; 3598 break; 3599 case 24 ... 31: 3600 fhftsl |= 0x03; 3601 break; 3602 } 3603 wr32(IGC_FHFTSL, fhftsl); 3604 3605 /* Normalize index down to host table register */ 3606 fhft_index = input->index % 8; 3607 3608 *fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) : 3609 IGC_FHFT_EXT(fhft_index - 4); 3610 3611 return 0; 3612 } 3613 3614 static int igc_write_flex_filter_ll(struct igc_adapter *adapter, 3615 struct igc_flex_filter *input) 3616 { 3617 struct igc_hw *hw = &adapter->hw; 3618 u8 *data = input->data; 3619 u8 *mask = input->mask; 3620 u32 queuing; 3621 u32 fhft; 3622 u32 wufc; 3623 int ret; 3624 int i; 3625 3626 /* Length has to be aligned to 8. Otherwise the filter will fail. Bail 3627 * out early to avoid surprises later. 3628 */ 3629 if (input->length % 8 != 0) { 3630 netdev_err(adapter->netdev, "The length of a flex filter has to be 8 byte aligned!\n"); 3631 return -EINVAL; 3632 } 3633 3634 /* Select corresponding flex filter register and get base for host table. */ 3635 ret = igc_flex_filter_select(adapter, input, &fhft); 3636 if (ret) 3637 return ret; 3638 3639 /* When adding a filter globally disable flex filter feature. That is 3640 * recommended within the datasheet. 3641 */ 3642 wufc = rd32(IGC_WUFC); 3643 wufc &= ~IGC_WUFC_FLEX_HQ; 3644 wr32(IGC_WUFC, wufc); 3645 3646 /* Configure filter */ 3647 queuing = input->length & IGC_FHFT_LENGTH_MASK; 3648 queuing |= FIELD_PREP(IGC_FHFT_QUEUE_MASK, input->rx_queue); 3649 queuing |= FIELD_PREP(IGC_FHFT_PRIO_MASK, input->prio); 3650 3651 if (input->immediate_irq) 3652 queuing |= IGC_FHFT_IMM_INT; 3653 3654 if (input->drop) 3655 queuing |= IGC_FHFT_DROP; 3656 3657 wr32(fhft + 0xFC, queuing); 3658 3659 /* Write data (128 byte) and mask (128 bit) */ 3660 for (i = 0; i < 16; ++i) { 3661 const size_t data_idx = i * 8; 3662 const size_t row_idx = i * 16; 3663 u32 dw0 = 3664 (data[data_idx + 0] << 0) | 3665 (data[data_idx + 1] << 8) | 3666 (data[data_idx + 2] << 16) | 3667 (data[data_idx + 3] << 24); 3668 u32 dw1 = 3669 (data[data_idx + 4] << 0) | 3670 (data[data_idx + 5] << 8) | 3671 (data[data_idx + 6] << 16) | 3672 (data[data_idx + 7] << 24); 3673 u32 tmp; 3674 3675 /* Write row: dw0, dw1 and mask */ 3676 wr32(fhft + row_idx, dw0); 3677 wr32(fhft + row_idx + 4, dw1); 3678 3679 /* mask is only valid for MASK(7, 0) */ 3680 tmp = rd32(fhft + row_idx + 8); 3681 tmp &= ~GENMASK(7, 0); 3682 tmp |= mask[i]; 3683 wr32(fhft + row_idx + 8, tmp); 3684 } 3685 3686 /* Enable filter. */ 3687 wufc |= IGC_WUFC_FLEX_HQ; 3688 if (input->index > 8) { 3689 /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */ 3690 u32 wufc_ext = rd32(IGC_WUFC_EXT); 3691 3692 wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8)); 3693 3694 wr32(IGC_WUFC_EXT, wufc_ext); 3695 } else { 3696 wufc |= (IGC_WUFC_FLX0 << input->index); 3697 } 3698 wr32(IGC_WUFC, wufc); 3699 3700 netdev_dbg(adapter->netdev, "Added flex filter %u to HW.\n", 3701 input->index); 3702 3703 return 0; 3704 } 3705 3706 static void igc_flex_filter_add_field(struct igc_flex_filter *flex, 3707 const void *src, unsigned int offset, 3708 size_t len, const void *mask) 3709 { 3710 int i; 3711 3712 /* data */ 3713 memcpy(&flex->data[offset], src, len); 3714 3715 /* mask */ 3716 for (i = 0; i < len; ++i) { 3717 const unsigned int idx = i + offset; 3718 const u8 *ptr = mask; 3719 3720 if (mask) { 3721 if (ptr[i] & 0xff) 3722 flex->mask[idx / 8] |= BIT(idx % 8); 3723 3724 continue; 3725 } 3726 3727 flex->mask[idx / 8] |= BIT(idx % 8); 3728 } 3729 } 3730 3731 static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter) 3732 { 3733 struct igc_hw *hw = &adapter->hw; 3734 u32 wufc, wufc_ext; 3735 int i; 3736 3737 wufc = rd32(IGC_WUFC); 3738 wufc_ext = rd32(IGC_WUFC_EXT); 3739 3740 for (i = 0; i < MAX_FLEX_FILTER; i++) { 3741 if (i < 8) { 3742 if (!(wufc & (IGC_WUFC_FLX0 << i))) 3743 return i; 3744 } else { 3745 if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8)))) 3746 return i; 3747 } 3748 } 3749 3750 return -ENOSPC; 3751 } 3752 3753 static bool igc_flex_filter_in_use(struct igc_adapter *adapter) 3754 { 3755 struct igc_hw *hw = &adapter->hw; 3756 u32 wufc, wufc_ext; 3757 3758 wufc = rd32(IGC_WUFC); 3759 wufc_ext = rd32(IGC_WUFC_EXT); 3760 3761 if (wufc & IGC_WUFC_FILTER_MASK) 3762 return true; 3763 3764 if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK) 3765 return true; 3766 3767 return false; 3768 } 3769 3770 static int igc_add_flex_filter(struct igc_adapter *adapter, 3771 struct igc_nfc_rule *rule) 3772 { 3773 struct igc_nfc_filter *filter = &rule->filter; 3774 unsigned int eth_offset, user_offset; 3775 struct igc_flex_filter flex = { }; 3776 int ret, index; 3777 bool vlan; 3778 3779 index = igc_find_avail_flex_filter_slot(adapter); 3780 if (index < 0) 3781 return -ENOSPC; 3782 3783 /* Construct the flex filter: 3784 * -> dest_mac [6] 3785 * -> src_mac [6] 3786 * -> tpid [2] 3787 * -> vlan tci [2] 3788 * -> ether type [2] 3789 * -> user data [8] 3790 * -> = 26 bytes => 32 length 3791 */ 3792 flex.index = index; 3793 flex.length = 32; 3794 flex.rx_queue = rule->action; 3795 3796 vlan = rule->filter.vlan_tci || rule->filter.vlan_etype; 3797 eth_offset = vlan ? 16 : 12; 3798 user_offset = vlan ? 18 : 14; 3799 3800 /* Add destination MAC */ 3801 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 3802 igc_flex_filter_add_field(&flex, &filter->dst_addr, 0, 3803 ETH_ALEN, NULL); 3804 3805 /* Add source MAC */ 3806 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 3807 igc_flex_filter_add_field(&flex, &filter->src_addr, 6, 3808 ETH_ALEN, NULL); 3809 3810 /* Add VLAN etype */ 3811 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE) { 3812 __be16 vlan_etype = cpu_to_be16(filter->vlan_etype); 3813 3814 igc_flex_filter_add_field(&flex, &vlan_etype, 12, 3815 sizeof(vlan_etype), NULL); 3816 } 3817 3818 /* Add VLAN TCI */ 3819 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) 3820 igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14, 3821 sizeof(filter->vlan_tci), NULL); 3822 3823 /* Add Ether type */ 3824 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 3825 __be16 etype = cpu_to_be16(filter->etype); 3826 3827 igc_flex_filter_add_field(&flex, &etype, eth_offset, 3828 sizeof(etype), NULL); 3829 } 3830 3831 /* Add user data */ 3832 if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA) 3833 igc_flex_filter_add_field(&flex, &filter->user_data, 3834 user_offset, 3835 sizeof(filter->user_data), 3836 filter->user_mask); 3837 3838 /* Add it down to the hardware and enable it. */ 3839 ret = igc_write_flex_filter_ll(adapter, &flex); 3840 if (ret) 3841 return ret; 3842 3843 filter->flex_index = index; 3844 3845 return 0; 3846 } 3847 3848 static void igc_del_flex_filter(struct igc_adapter *adapter, 3849 u16 reg_index) 3850 { 3851 struct igc_hw *hw = &adapter->hw; 3852 u32 wufc; 3853 3854 /* Just disable the filter. The filter table itself is kept 3855 * intact. Another flex_filter_add() should override the "old" data 3856 * then. 3857 */ 3858 if (reg_index > 8) { 3859 u32 wufc_ext = rd32(IGC_WUFC_EXT); 3860 3861 wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8)); 3862 wr32(IGC_WUFC_EXT, wufc_ext); 3863 } else { 3864 wufc = rd32(IGC_WUFC); 3865 3866 wufc &= ~(IGC_WUFC_FLX0 << reg_index); 3867 wr32(IGC_WUFC, wufc); 3868 } 3869 3870 if (igc_flex_filter_in_use(adapter)) 3871 return; 3872 3873 /* No filters are in use, we may disable flex filters */ 3874 wufc = rd32(IGC_WUFC); 3875 wufc &= ~IGC_WUFC_FLEX_HQ; 3876 wr32(IGC_WUFC, wufc); 3877 } 3878 3879 static void igc_set_default_queue_filter(struct igc_adapter *adapter, u32 queue) 3880 { 3881 struct igc_hw *hw = &adapter->hw; 3882 u32 mrqc = rd32(IGC_MRQC); 3883 3884 mrqc &= ~IGC_MRQC_DEFAULT_QUEUE_MASK; 3885 mrqc |= FIELD_PREP(IGC_MRQC_DEFAULT_QUEUE_MASK, queue); 3886 wr32(IGC_MRQC, mrqc); 3887 } 3888 3889 static void igc_reset_default_queue_filter(struct igc_adapter *adapter) 3890 { 3891 /* Reset the default queue to its default value which is Queue 0 */ 3892 igc_set_default_queue_filter(adapter, 0); 3893 } 3894 3895 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 3896 struct igc_nfc_rule *rule) 3897 { 3898 int err; 3899 3900 if (rule->flex) { 3901 return igc_add_flex_filter(adapter, rule); 3902 } 3903 3904 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 3905 err = igc_add_etype_filter(adapter, rule->filter.etype, 3906 rule->action); 3907 if (err) 3908 return err; 3909 } 3910 3911 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 3912 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 3913 rule->filter.src_addr, rule->action); 3914 if (err) 3915 return err; 3916 } 3917 3918 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 3919 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 3920 rule->filter.dst_addr, rule->action); 3921 if (err) 3922 return err; 3923 } 3924 3925 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 3926 int prio = FIELD_GET(VLAN_PRIO_MASK, rule->filter.vlan_tci); 3927 3928 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 3929 if (err) 3930 return err; 3931 } 3932 3933 if (rule->filter.match_flags & IGC_FILTER_FLAG_DEFAULT_QUEUE) 3934 igc_set_default_queue_filter(adapter, rule->action); 3935 3936 return 0; 3937 } 3938 3939 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 3940 const struct igc_nfc_rule *rule) 3941 { 3942 if (rule->flex) { 3943 igc_del_flex_filter(adapter, rule->filter.flex_index); 3944 return; 3945 } 3946 3947 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 3948 igc_del_etype_filter(adapter, rule->filter.etype); 3949 3950 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 3951 int prio = FIELD_GET(VLAN_PRIO_MASK, rule->filter.vlan_tci); 3952 3953 igc_del_vlan_prio_filter(adapter, prio); 3954 } 3955 3956 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 3957 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 3958 rule->filter.src_addr); 3959 3960 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 3961 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 3962 rule->filter.dst_addr); 3963 3964 if (rule->filter.match_flags & IGC_FILTER_FLAG_DEFAULT_QUEUE) 3965 igc_reset_default_queue_filter(adapter); 3966 } 3967 3968 /** 3969 * igc_get_nfc_rule() - Get NFC rule 3970 * @adapter: Pointer to adapter 3971 * @location: Rule location 3972 * 3973 * Context: Expects adapter->nfc_rule_lock to be held by caller. 3974 * 3975 * Return: Pointer to NFC rule at @location. If not found, NULL. 3976 */ 3977 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 3978 u32 location) 3979 { 3980 struct igc_nfc_rule *rule; 3981 3982 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 3983 if (rule->location == location) 3984 return rule; 3985 if (rule->location > location) 3986 break; 3987 } 3988 3989 return NULL; 3990 } 3991 3992 /** 3993 * igc_del_nfc_rule() - Delete NFC rule 3994 * @adapter: Pointer to adapter 3995 * @rule: Pointer to rule to be deleted 3996 * 3997 * Disable NFC rule in hardware and delete it from adapter. 3998 * 3999 * Context: Expects adapter->nfc_rule_lock to be held by caller. 4000 */ 4001 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 4002 { 4003 igc_disable_nfc_rule(adapter, rule); 4004 4005 list_del(&rule->list); 4006 adapter->nfc_rule_count--; 4007 4008 kfree(rule); 4009 } 4010 4011 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 4012 { 4013 struct igc_nfc_rule *rule, *tmp; 4014 4015 mutex_lock(&adapter->nfc_rule_lock); 4016 4017 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 4018 igc_del_nfc_rule(adapter, rule); 4019 4020 mutex_unlock(&adapter->nfc_rule_lock); 4021 } 4022 4023 /** 4024 * igc_add_nfc_rule() - Add NFC rule 4025 * @adapter: Pointer to adapter 4026 * @rule: Pointer to rule to be added 4027 * 4028 * Enable NFC rule in hardware and add it to adapter. 4029 * 4030 * Context: Expects adapter->nfc_rule_lock to be held by caller. 4031 * 4032 * Return: 0 on success, negative errno on failure. 4033 */ 4034 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 4035 { 4036 struct igc_nfc_rule *pred, *cur; 4037 int err; 4038 4039 err = igc_enable_nfc_rule(adapter, rule); 4040 if (err) 4041 return err; 4042 4043 pred = NULL; 4044 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 4045 if (cur->location >= rule->location) 4046 break; 4047 pred = cur; 4048 } 4049 4050 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 4051 adapter->nfc_rule_count++; 4052 return 0; 4053 } 4054 4055 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 4056 { 4057 struct igc_nfc_rule *rule; 4058 4059 mutex_lock(&adapter->nfc_rule_lock); 4060 4061 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 4062 igc_enable_nfc_rule(adapter, rule); 4063 4064 mutex_unlock(&adapter->nfc_rule_lock); 4065 } 4066 4067 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 4068 { 4069 struct igc_adapter *adapter = netdev_priv(netdev); 4070 4071 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 4072 } 4073 4074 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 4075 { 4076 struct igc_adapter *adapter = netdev_priv(netdev); 4077 4078 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 4079 return 0; 4080 } 4081 4082 /** 4083 * igc_enable_empty_addr_recv - Enable Rx of packets with all-zeroes MAC address 4084 * @adapter: Pointer to the igc_adapter structure. 4085 * 4086 * Frame preemption verification requires that packets with the all-zeroes 4087 * MAC address are allowed to be received by the driver. This function adds the 4088 * all-zeroes destination address to the list of acceptable addresses. 4089 * 4090 * Return: 0 on success, negative value otherwise. 4091 */ 4092 int igc_enable_empty_addr_recv(struct igc_adapter *adapter) 4093 { 4094 u8 empty[ETH_ALEN] = {}; 4095 4096 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, empty, -1); 4097 } 4098 4099 void igc_disable_empty_addr_recv(struct igc_adapter *adapter) 4100 { 4101 u8 empty[ETH_ALEN] = {}; 4102 4103 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, empty); 4104 } 4105 4106 /** 4107 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 4108 * @netdev: network interface device structure 4109 * 4110 * The set_rx_mode entry point is called whenever the unicast or multicast 4111 * address lists or the network interface flags are updated. This routine is 4112 * responsible for configuring the hardware for proper unicast, multicast, 4113 * promiscuous mode, and all-multi behavior. 4114 */ 4115 static void igc_set_rx_mode(struct net_device *netdev) 4116 { 4117 struct igc_adapter *adapter = netdev_priv(netdev); 4118 struct igc_hw *hw = &adapter->hw; 4119 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 4120 int count; 4121 4122 /* Check for Promiscuous and All Multicast modes */ 4123 if (netdev->flags & IFF_PROMISC) { 4124 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 4125 } else { 4126 if (netdev->flags & IFF_ALLMULTI) { 4127 rctl |= IGC_RCTL_MPE; 4128 } else { 4129 /* Write addresses to the MTA, if the attempt fails 4130 * then we should just turn on promiscuous mode so 4131 * that we can at least receive multicast traffic 4132 */ 4133 count = igc_write_mc_addr_list(netdev); 4134 if (count < 0) 4135 rctl |= IGC_RCTL_MPE; 4136 } 4137 } 4138 4139 /* Write addresses to available RAR registers, if there is not 4140 * sufficient space to store all the addresses then enable 4141 * unicast promiscuous mode 4142 */ 4143 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 4144 rctl |= IGC_RCTL_UPE; 4145 4146 /* update state of unicast and multicast */ 4147 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 4148 wr32(IGC_RCTL, rctl); 4149 4150 #if (PAGE_SIZE < 8192) 4151 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 4152 rlpml = IGC_MAX_FRAME_BUILD_SKB; 4153 #endif 4154 wr32(IGC_RLPML, rlpml); 4155 } 4156 4157 /** 4158 * igc_configure - configure the hardware for RX and TX 4159 * @adapter: private board structure 4160 */ 4161 static void igc_configure(struct igc_adapter *adapter) 4162 { 4163 struct net_device *netdev = adapter->netdev; 4164 int i = 0; 4165 4166 igc_get_hw_control(adapter); 4167 igc_set_rx_mode(netdev); 4168 4169 igc_restore_vlan(adapter); 4170 4171 igc_setup_tctl(adapter); 4172 igc_setup_mrqc(adapter); 4173 igc_setup_rctl(adapter); 4174 4175 igc_set_default_mac_filter(adapter); 4176 igc_restore_nfc_rules(adapter); 4177 4178 igc_configure_tx(adapter); 4179 igc_configure_rx(adapter); 4180 4181 igc_rx_fifo_flush_base(&adapter->hw); 4182 4183 /* call igc_desc_unused which always leaves 4184 * at least 1 descriptor unused to make sure 4185 * next_to_use != next_to_clean 4186 */ 4187 for (i = 0; i < adapter->num_rx_queues; i++) { 4188 struct igc_ring *ring = adapter->rx_ring[i]; 4189 4190 if (ring->xsk_pool) 4191 igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); 4192 else 4193 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 4194 } 4195 } 4196 4197 /** 4198 * igc_write_ivar - configure ivar for given MSI-X vector 4199 * @hw: pointer to the HW structure 4200 * @msix_vector: vector number we are allocating to a given ring 4201 * @index: row index of IVAR register to write within IVAR table 4202 * @offset: column offset of in IVAR, should be multiple of 8 4203 * 4204 * The IVAR table consists of 2 columns, 4205 * each containing an cause allocation for an Rx and Tx ring, and a 4206 * variable number of rows depending on the number of queues supported. 4207 */ 4208 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 4209 int index, int offset) 4210 { 4211 u32 ivar = array_rd32(IGC_IVAR0, index); 4212 4213 /* clear any bits that are currently set */ 4214 ivar &= ~((u32)0xFF << offset); 4215 4216 /* write vector and valid bit */ 4217 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 4218 4219 array_wr32(IGC_IVAR0, index, ivar); 4220 } 4221 4222 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 4223 { 4224 struct igc_adapter *adapter = q_vector->adapter; 4225 struct igc_hw *hw = &adapter->hw; 4226 int rx_queue = IGC_N0_QUEUE; 4227 int tx_queue = IGC_N0_QUEUE; 4228 4229 if (q_vector->rx.ring) 4230 rx_queue = q_vector->rx.ring->reg_idx; 4231 if (q_vector->tx.ring) 4232 tx_queue = q_vector->tx.ring->reg_idx; 4233 4234 switch (hw->mac.type) { 4235 case igc_i225: 4236 if (rx_queue > IGC_N0_QUEUE) 4237 igc_write_ivar(hw, msix_vector, 4238 rx_queue >> 1, 4239 (rx_queue & 0x1) << 4); 4240 if (tx_queue > IGC_N0_QUEUE) 4241 igc_write_ivar(hw, msix_vector, 4242 tx_queue >> 1, 4243 ((tx_queue & 0x1) << 4) + 8); 4244 q_vector->eims_value = BIT(msix_vector); 4245 break; 4246 default: 4247 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 4248 break; 4249 } 4250 4251 /* add q_vector eims value to global eims_enable_mask */ 4252 adapter->eims_enable_mask |= q_vector->eims_value; 4253 4254 /* configure q_vector to set itr on first interrupt */ 4255 q_vector->set_itr = 1; 4256 } 4257 4258 /** 4259 * igc_configure_msix - Configure MSI-X hardware 4260 * @adapter: Pointer to adapter structure 4261 * 4262 * igc_configure_msix sets up the hardware to properly 4263 * generate MSI-X interrupts. 4264 */ 4265 static void igc_configure_msix(struct igc_adapter *adapter) 4266 { 4267 struct igc_hw *hw = &adapter->hw; 4268 int i, vector = 0; 4269 u32 tmp; 4270 4271 adapter->eims_enable_mask = 0; 4272 4273 /* set vector for other causes, i.e. link changes */ 4274 switch (hw->mac.type) { 4275 case igc_i225: 4276 /* Turn on MSI-X capability first, or our settings 4277 * won't stick. And it will take days to debug. 4278 */ 4279 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 4280 IGC_GPIE_PBA | IGC_GPIE_EIAME | 4281 IGC_GPIE_NSICR); 4282 4283 /* enable msix_other interrupt */ 4284 adapter->eims_other = BIT(vector); 4285 tmp = (vector++ | IGC_IVAR_VALID) << 8; 4286 4287 wr32(IGC_IVAR_MISC, tmp); 4288 break; 4289 default: 4290 /* do nothing, since nothing else supports MSI-X */ 4291 break; 4292 } /* switch (hw->mac.type) */ 4293 4294 adapter->eims_enable_mask |= adapter->eims_other; 4295 4296 for (i = 0; i < adapter->num_q_vectors; i++) 4297 igc_assign_vector(adapter->q_vector[i], vector++); 4298 4299 wrfl(); 4300 } 4301 4302 /** 4303 * igc_irq_enable - Enable default interrupt generation settings 4304 * @adapter: board private structure 4305 */ 4306 static void igc_irq_enable(struct igc_adapter *adapter) 4307 { 4308 struct igc_hw *hw = &adapter->hw; 4309 4310 if (adapter->msix_entries) { 4311 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 4312 u32 regval = rd32(IGC_EIAC); 4313 4314 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 4315 regval = rd32(IGC_EIAM); 4316 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 4317 wr32(IGC_EIMS, adapter->eims_enable_mask); 4318 wr32(IGC_IMS, ims); 4319 } else { 4320 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4321 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4322 } 4323 } 4324 4325 /** 4326 * igc_irq_disable - Mask off interrupt generation on the NIC 4327 * @adapter: board private structure 4328 */ 4329 static void igc_irq_disable(struct igc_adapter *adapter) 4330 { 4331 struct igc_hw *hw = &adapter->hw; 4332 4333 if (adapter->msix_entries) { 4334 u32 regval = rd32(IGC_EIAM); 4335 4336 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 4337 wr32(IGC_EIMC, adapter->eims_enable_mask); 4338 regval = rd32(IGC_EIAC); 4339 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 4340 } 4341 4342 wr32(IGC_IAM, 0); 4343 wr32(IGC_IMC, ~0); 4344 wrfl(); 4345 4346 if (adapter->msix_entries) { 4347 int vector = 0, i; 4348 4349 synchronize_irq(adapter->msix_entries[vector++].vector); 4350 4351 for (i = 0; i < adapter->num_q_vectors; i++) 4352 synchronize_irq(adapter->msix_entries[vector++].vector); 4353 } else { 4354 synchronize_irq(adapter->pdev->irq); 4355 } 4356 } 4357 4358 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 4359 const u32 max_rss_queues) 4360 { 4361 /* Determine if we need to pair queues. */ 4362 /* If rss_queues > half of max_rss_queues, pair the queues in 4363 * order to conserve interrupts due to limited supply. 4364 */ 4365 if (adapter->rss_queues > (max_rss_queues / 2)) 4366 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4367 else 4368 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 4369 } 4370 4371 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 4372 { 4373 return IGC_MAX_RX_QUEUES; 4374 } 4375 4376 static void igc_init_queue_configuration(struct igc_adapter *adapter) 4377 { 4378 u32 max_rss_queues; 4379 4380 max_rss_queues = igc_get_max_rss_queues(adapter); 4381 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 4382 4383 igc_set_flag_queue_pairs(adapter, max_rss_queues); 4384 } 4385 4386 /** 4387 * igc_reset_q_vector - Reset config for interrupt vector 4388 * @adapter: board private structure to initialize 4389 * @v_idx: Index of vector to be reset 4390 * 4391 * If NAPI is enabled it will delete any references to the 4392 * NAPI struct. This is preparation for igc_free_q_vector. 4393 */ 4394 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 4395 { 4396 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 4397 4398 /* if we're coming from igc_set_interrupt_capability, the vectors are 4399 * not yet allocated 4400 */ 4401 if (!q_vector) 4402 return; 4403 4404 if (q_vector->tx.ring) 4405 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 4406 4407 if (q_vector->rx.ring) 4408 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 4409 4410 netif_napi_del(&q_vector->napi); 4411 } 4412 4413 /** 4414 * igc_free_q_vector - Free memory allocated for specific interrupt vector 4415 * @adapter: board private structure to initialize 4416 * @v_idx: Index of vector to be freed 4417 * 4418 * This function frees the memory allocated to the q_vector. 4419 */ 4420 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 4421 { 4422 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 4423 4424 adapter->q_vector[v_idx] = NULL; 4425 4426 /* igc_get_stats64() might access the rings on this vector, 4427 * we must wait a grace period before freeing it. 4428 */ 4429 if (q_vector) 4430 kfree_rcu(q_vector, rcu); 4431 } 4432 4433 /** 4434 * igc_free_q_vectors - Free memory allocated for interrupt vectors 4435 * @adapter: board private structure to initialize 4436 * 4437 * This function frees the memory allocated to the q_vectors. In addition if 4438 * NAPI is enabled it will delete any references to the NAPI struct prior 4439 * to freeing the q_vector. 4440 */ 4441 static void igc_free_q_vectors(struct igc_adapter *adapter) 4442 { 4443 int v_idx = adapter->num_q_vectors; 4444 4445 adapter->num_tx_queues = 0; 4446 adapter->num_rx_queues = 0; 4447 adapter->num_q_vectors = 0; 4448 4449 while (v_idx--) { 4450 igc_reset_q_vector(adapter, v_idx); 4451 igc_free_q_vector(adapter, v_idx); 4452 } 4453 } 4454 4455 /** 4456 * igc_update_itr - update the dynamic ITR value based on statistics 4457 * @q_vector: pointer to q_vector 4458 * @ring_container: ring info to update the itr for 4459 * 4460 * Stores a new ITR value based on packets and byte 4461 * counts during the last interrupt. The advantage of per interrupt 4462 * computation is faster updates and more accurate ITR for the current 4463 * traffic pattern. Constants in this function were computed 4464 * based on theoretical maximum wire speed and thresholds were set based 4465 * on testing data as well as attempting to minimize response time 4466 * while increasing bulk throughput. 4467 * NOTE: These calculations are only valid when operating in a single- 4468 * queue environment. 4469 */ 4470 static void igc_update_itr(struct igc_q_vector *q_vector, 4471 struct igc_ring_container *ring_container) 4472 { 4473 unsigned int packets = ring_container->total_packets; 4474 unsigned int bytes = ring_container->total_bytes; 4475 u8 itrval = ring_container->itr; 4476 4477 /* no packets, exit with status unchanged */ 4478 if (packets == 0) 4479 return; 4480 4481 switch (itrval) { 4482 case lowest_latency: 4483 /* handle TSO and jumbo frames */ 4484 if (bytes / packets > 8000) 4485 itrval = bulk_latency; 4486 else if ((packets < 5) && (bytes > 512)) 4487 itrval = low_latency; 4488 break; 4489 case low_latency: /* 50 usec aka 20000 ints/s */ 4490 if (bytes > 10000) { 4491 /* this if handles the TSO accounting */ 4492 if (bytes / packets > 8000) 4493 itrval = bulk_latency; 4494 else if ((packets < 10) || ((bytes / packets) > 1200)) 4495 itrval = bulk_latency; 4496 else if ((packets > 35)) 4497 itrval = lowest_latency; 4498 } else if (bytes / packets > 2000) { 4499 itrval = bulk_latency; 4500 } else if (packets <= 2 && bytes < 512) { 4501 itrval = lowest_latency; 4502 } 4503 break; 4504 case bulk_latency: /* 250 usec aka 4000 ints/s */ 4505 if (bytes > 25000) { 4506 if (packets > 35) 4507 itrval = low_latency; 4508 } else if (bytes < 1500) { 4509 itrval = low_latency; 4510 } 4511 break; 4512 } 4513 4514 /* clear work counters since we have the values we need */ 4515 ring_container->total_bytes = 0; 4516 ring_container->total_packets = 0; 4517 4518 /* write updated itr to ring container */ 4519 ring_container->itr = itrval; 4520 } 4521 4522 static void igc_set_itr(struct igc_q_vector *q_vector) 4523 { 4524 struct igc_adapter *adapter = q_vector->adapter; 4525 u32 new_itr = q_vector->itr_val; 4526 u8 current_itr = 0; 4527 4528 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 4529 switch (adapter->link_speed) { 4530 case SPEED_10: 4531 case SPEED_100: 4532 current_itr = 0; 4533 new_itr = IGC_4K_ITR; 4534 goto set_itr_now; 4535 default: 4536 break; 4537 } 4538 4539 igc_update_itr(q_vector, &q_vector->tx); 4540 igc_update_itr(q_vector, &q_vector->rx); 4541 4542 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 4543 4544 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 4545 if (current_itr == lowest_latency && 4546 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 4547 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 4548 current_itr = low_latency; 4549 4550 switch (current_itr) { 4551 /* counts and packets in update_itr are dependent on these numbers */ 4552 case lowest_latency: 4553 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 4554 break; 4555 case low_latency: 4556 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 4557 break; 4558 case bulk_latency: 4559 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 4560 break; 4561 default: 4562 break; 4563 } 4564 4565 set_itr_now: 4566 if (new_itr != q_vector->itr_val) { 4567 /* this attempts to bias the interrupt rate towards Bulk 4568 * by adding intermediate steps when interrupt rate is 4569 * increasing 4570 */ 4571 new_itr = new_itr > q_vector->itr_val ? 4572 max((new_itr * q_vector->itr_val) / 4573 (new_itr + (q_vector->itr_val >> 2)), 4574 new_itr) : new_itr; 4575 /* Don't write the value here; it resets the adapter's 4576 * internal timer, and causes us to delay far longer than 4577 * we should between interrupts. Instead, we write the ITR 4578 * value at the beginning of the next interrupt so the timing 4579 * ends up being correct. 4580 */ 4581 q_vector->itr_val = new_itr; 4582 q_vector->set_itr = 1; 4583 } 4584 } 4585 4586 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 4587 { 4588 int v_idx = adapter->num_q_vectors; 4589 4590 if (adapter->msix_entries) { 4591 pci_disable_msix(adapter->pdev); 4592 kfree(adapter->msix_entries); 4593 adapter->msix_entries = NULL; 4594 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 4595 pci_disable_msi(adapter->pdev); 4596 } 4597 4598 while (v_idx--) 4599 igc_reset_q_vector(adapter, v_idx); 4600 } 4601 4602 /** 4603 * igc_set_interrupt_capability - set MSI or MSI-X if supported 4604 * @adapter: Pointer to adapter structure 4605 * @msix: boolean value for MSI-X capability 4606 * 4607 * Attempt to configure interrupts using the best available 4608 * capabilities of the hardware and kernel. 4609 */ 4610 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 4611 bool msix) 4612 { 4613 int numvecs, i; 4614 int err; 4615 4616 if (!msix) 4617 goto msi_only; 4618 adapter->flags |= IGC_FLAG_HAS_MSIX; 4619 4620 /* Number of supported queues. */ 4621 adapter->num_rx_queues = adapter->rss_queues; 4622 4623 adapter->num_tx_queues = adapter->rss_queues; 4624 4625 /* start with one vector for every Rx queue */ 4626 numvecs = adapter->num_rx_queues; 4627 4628 /* if Tx handler is separate add 1 for every Tx queue */ 4629 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 4630 numvecs += adapter->num_tx_queues; 4631 4632 /* store the number of vectors reserved for queues */ 4633 adapter->num_q_vectors = numvecs; 4634 4635 /* add 1 vector for link status interrupts */ 4636 numvecs++; 4637 4638 adapter->msix_entries = kzalloc_objs(struct msix_entry, numvecs); 4639 4640 if (!adapter->msix_entries) 4641 return; 4642 4643 /* populate entry values */ 4644 for (i = 0; i < numvecs; i++) 4645 adapter->msix_entries[i].entry = i; 4646 4647 err = pci_enable_msix_range(adapter->pdev, 4648 adapter->msix_entries, 4649 numvecs, 4650 numvecs); 4651 if (err > 0) 4652 return; 4653 4654 kfree(adapter->msix_entries); 4655 adapter->msix_entries = NULL; 4656 4657 igc_reset_interrupt_capability(adapter); 4658 4659 msi_only: 4660 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 4661 4662 adapter->rss_queues = 1; 4663 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4664 adapter->num_rx_queues = 1; 4665 adapter->num_tx_queues = 1; 4666 adapter->num_q_vectors = 1; 4667 if (!pci_enable_msi(adapter->pdev)) 4668 adapter->flags |= IGC_FLAG_HAS_MSI; 4669 } 4670 4671 /** 4672 * igc_update_ring_itr - update the dynamic ITR value based on packet size 4673 * @q_vector: pointer to q_vector 4674 * 4675 * Stores a new ITR value based on strictly on packet size. This 4676 * algorithm is less sophisticated than that used in igc_update_itr, 4677 * due to the difficulty of synchronizing statistics across multiple 4678 * receive rings. The divisors and thresholds used by this function 4679 * were determined based on theoretical maximum wire speed and testing 4680 * data, in order to minimize response time while increasing bulk 4681 * throughput. 4682 * NOTE: This function is called only when operating in a multiqueue 4683 * receive environment. 4684 */ 4685 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 4686 { 4687 struct igc_adapter *adapter = q_vector->adapter; 4688 int new_val = q_vector->itr_val; 4689 int avg_wire_size = 0; 4690 unsigned int packets; 4691 4692 /* For non-gigabit speeds, just fix the interrupt rate at 4000 4693 * ints/sec - ITR timer value of 120 ticks. 4694 */ 4695 switch (adapter->link_speed) { 4696 case SPEED_10: 4697 case SPEED_100: 4698 new_val = IGC_4K_ITR; 4699 goto set_itr_val; 4700 default: 4701 break; 4702 } 4703 4704 packets = q_vector->rx.total_packets; 4705 if (packets) 4706 avg_wire_size = q_vector->rx.total_bytes / packets; 4707 4708 packets = q_vector->tx.total_packets; 4709 if (packets) 4710 avg_wire_size = max_t(u32, avg_wire_size, 4711 q_vector->tx.total_bytes / packets); 4712 4713 /* if avg_wire_size isn't set no work was done */ 4714 if (!avg_wire_size) 4715 goto clear_counts; 4716 4717 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 4718 avg_wire_size += 24; 4719 4720 /* Don't starve jumbo frames */ 4721 avg_wire_size = min(avg_wire_size, 3000); 4722 4723 /* Give a little boost to mid-size frames */ 4724 if (avg_wire_size > 300 && avg_wire_size < 1200) 4725 new_val = avg_wire_size / 3; 4726 else 4727 new_val = avg_wire_size / 2; 4728 4729 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 4730 if (new_val < IGC_20K_ITR && 4731 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 4732 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 4733 new_val = IGC_20K_ITR; 4734 4735 set_itr_val: 4736 if (new_val != q_vector->itr_val) { 4737 q_vector->itr_val = new_val; 4738 q_vector->set_itr = 1; 4739 } 4740 clear_counts: 4741 q_vector->rx.total_bytes = 0; 4742 q_vector->rx.total_packets = 0; 4743 q_vector->tx.total_bytes = 0; 4744 q_vector->tx.total_packets = 0; 4745 } 4746 4747 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 4748 { 4749 struct igc_adapter *adapter = q_vector->adapter; 4750 struct igc_hw *hw = &adapter->hw; 4751 4752 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 4753 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 4754 if (adapter->num_q_vectors == 1) 4755 igc_set_itr(q_vector); 4756 else 4757 igc_update_ring_itr(q_vector); 4758 } 4759 4760 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4761 if (adapter->msix_entries) 4762 wr32(IGC_EIMS, q_vector->eims_value); 4763 else 4764 igc_irq_enable(adapter); 4765 } 4766 } 4767 4768 static void igc_add_ring(struct igc_ring *ring, 4769 struct igc_ring_container *head) 4770 { 4771 head->ring = ring; 4772 head->count++; 4773 } 4774 4775 /** 4776 * igc_cache_ring_register - Descriptor ring to register mapping 4777 * @adapter: board private structure to initialize 4778 * 4779 * Once we know the feature-set enabled for the device, we'll cache 4780 * the register offset the descriptor ring is assigned to. 4781 */ 4782 static void igc_cache_ring_register(struct igc_adapter *adapter) 4783 { 4784 int i = 0, j = 0; 4785 4786 switch (adapter->hw.mac.type) { 4787 case igc_i225: 4788 default: 4789 for (; i < adapter->num_rx_queues; i++) 4790 adapter->rx_ring[i]->reg_idx = i; 4791 for (; j < adapter->num_tx_queues; j++) 4792 adapter->tx_ring[j]->reg_idx = j; 4793 break; 4794 } 4795 } 4796 4797 /** 4798 * igc_poll - NAPI Rx polling callback 4799 * @napi: napi polling structure 4800 * @budget: count of how many packets we should handle 4801 */ 4802 static int igc_poll(struct napi_struct *napi, int budget) 4803 { 4804 struct igc_q_vector *q_vector = container_of(napi, 4805 struct igc_q_vector, 4806 napi); 4807 struct igc_ring *rx_ring = q_vector->rx.ring; 4808 bool clean_complete = true; 4809 int work_done = 0; 4810 4811 if (q_vector->tx.ring) 4812 clean_complete = igc_clean_tx_irq(q_vector, budget); 4813 4814 if (rx_ring) { 4815 int cleaned = rx_ring->xsk_pool ? 4816 igc_clean_rx_irq_zc(q_vector, budget) : 4817 igc_clean_rx_irq(q_vector, budget); 4818 4819 work_done += cleaned; 4820 if (cleaned >= budget) 4821 clean_complete = false; 4822 } 4823 4824 /* If all work not completed, return budget and keep polling */ 4825 if (!clean_complete) 4826 return budget; 4827 4828 /* Exit the polling mode, but don't re-enable interrupts if stack might 4829 * poll us due to busy-polling 4830 */ 4831 if (likely(napi_complete_done(napi, work_done))) 4832 igc_ring_irq_enable(q_vector); 4833 4834 return min(work_done, budget - 1); 4835 } 4836 4837 /** 4838 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 4839 * @adapter: board private structure to initialize 4840 * @v_count: q_vectors allocated on adapter, used for ring interleaving 4841 * @v_idx: index of vector in adapter struct 4842 * @txr_count: total number of Tx rings to allocate 4843 * @txr_idx: index of first Tx ring to allocate 4844 * @rxr_count: total number of Rx rings to allocate 4845 * @rxr_idx: index of first Rx ring to allocate 4846 * 4847 * We allocate one q_vector. If allocation fails we return -ENOMEM. 4848 */ 4849 static int igc_alloc_q_vector(struct igc_adapter *adapter, 4850 unsigned int v_count, unsigned int v_idx, 4851 unsigned int txr_count, unsigned int txr_idx, 4852 unsigned int rxr_count, unsigned int rxr_idx) 4853 { 4854 struct igc_q_vector *q_vector; 4855 struct igc_ring *ring; 4856 int ring_count; 4857 4858 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 4859 if (txr_count > 1 || rxr_count > 1) 4860 return -ENOMEM; 4861 4862 ring_count = txr_count + rxr_count; 4863 4864 /* allocate q_vector and rings */ 4865 q_vector = adapter->q_vector[v_idx]; 4866 if (!q_vector) 4867 q_vector = kzalloc_flex(*q_vector, ring, ring_count); 4868 else 4869 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 4870 if (!q_vector) 4871 return -ENOMEM; 4872 4873 /* initialize NAPI */ 4874 netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll); 4875 4876 /* tie q_vector and adapter together */ 4877 adapter->q_vector[v_idx] = q_vector; 4878 q_vector->adapter = adapter; 4879 4880 /* initialize work limits */ 4881 q_vector->tx.work_limit = adapter->tx_work_limit; 4882 4883 /* initialize ITR configuration */ 4884 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 4885 q_vector->itr_val = IGC_START_ITR; 4886 4887 /* initialize pointer to rings */ 4888 ring = q_vector->ring; 4889 4890 /* initialize ITR */ 4891 if (rxr_count) { 4892 /* rx or rx/tx vector */ 4893 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 4894 q_vector->itr_val = adapter->rx_itr_setting; 4895 } else { 4896 /* tx only vector */ 4897 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 4898 q_vector->itr_val = adapter->tx_itr_setting; 4899 } 4900 4901 if (txr_count) { 4902 /* assign generic ring traits */ 4903 ring->dev = &adapter->pdev->dev; 4904 ring->netdev = adapter->netdev; 4905 4906 /* configure backlink on ring */ 4907 ring->q_vector = q_vector; 4908 4909 /* update q_vector Tx values */ 4910 igc_add_ring(ring, &q_vector->tx); 4911 4912 /* apply Tx specific ring traits */ 4913 ring->count = adapter->tx_ring_count; 4914 ring->queue_index = txr_idx; 4915 4916 /* assign ring to adapter */ 4917 adapter->tx_ring[txr_idx] = ring; 4918 4919 /* push pointer to next ring */ 4920 ring++; 4921 } 4922 4923 if (rxr_count) { 4924 /* assign generic ring traits */ 4925 ring->dev = &adapter->pdev->dev; 4926 ring->netdev = adapter->netdev; 4927 4928 /* configure backlink on ring */ 4929 ring->q_vector = q_vector; 4930 4931 /* update q_vector Rx values */ 4932 igc_add_ring(ring, &q_vector->rx); 4933 4934 /* apply Rx specific ring traits */ 4935 ring->count = adapter->rx_ring_count; 4936 ring->queue_index = rxr_idx; 4937 4938 /* assign ring to adapter */ 4939 adapter->rx_ring[rxr_idx] = ring; 4940 } 4941 4942 return 0; 4943 } 4944 4945 /** 4946 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 4947 * @adapter: board private structure to initialize 4948 * 4949 * We allocate one q_vector per queue interrupt. If allocation fails we 4950 * return -ENOMEM. 4951 */ 4952 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 4953 { 4954 int rxr_remaining = adapter->num_rx_queues; 4955 int txr_remaining = adapter->num_tx_queues; 4956 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 4957 int q_vectors = adapter->num_q_vectors; 4958 int err; 4959 4960 if (q_vectors >= (rxr_remaining + txr_remaining)) { 4961 for (; rxr_remaining; v_idx++) { 4962 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 4963 0, 0, 1, rxr_idx); 4964 4965 if (err) 4966 goto err_out; 4967 4968 /* update counts and index */ 4969 rxr_remaining--; 4970 rxr_idx++; 4971 } 4972 } 4973 4974 for (; v_idx < q_vectors; v_idx++) { 4975 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 4976 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 4977 4978 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 4979 tqpv, txr_idx, rqpv, rxr_idx); 4980 4981 if (err) 4982 goto err_out; 4983 4984 /* update counts and index */ 4985 rxr_remaining -= rqpv; 4986 txr_remaining -= tqpv; 4987 rxr_idx++; 4988 txr_idx++; 4989 } 4990 4991 return 0; 4992 4993 err_out: 4994 adapter->num_tx_queues = 0; 4995 adapter->num_rx_queues = 0; 4996 adapter->num_q_vectors = 0; 4997 4998 while (v_idx--) 4999 igc_free_q_vector(adapter, v_idx); 5000 5001 return -ENOMEM; 5002 } 5003 5004 /** 5005 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 5006 * @adapter: Pointer to adapter structure 5007 * @msix: boolean for MSI-X capability 5008 * 5009 * This function initializes the interrupts and allocates all of the queues. 5010 */ 5011 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 5012 { 5013 struct net_device *dev = adapter->netdev; 5014 int err = 0; 5015 5016 igc_set_interrupt_capability(adapter, msix); 5017 5018 err = igc_alloc_q_vectors(adapter); 5019 if (err) { 5020 netdev_err(dev, "Unable to allocate memory for vectors\n"); 5021 goto err_alloc_q_vectors; 5022 } 5023 5024 igc_cache_ring_register(adapter); 5025 5026 return 0; 5027 5028 err_alloc_q_vectors: 5029 igc_reset_interrupt_capability(adapter); 5030 return err; 5031 } 5032 5033 /** 5034 * igc_sw_init - Initialize general software structures (struct igc_adapter) 5035 * @adapter: board private structure to initialize 5036 * 5037 * igc_sw_init initializes the Adapter private data structure. 5038 * Fields are initialized based on PCI device information and 5039 * OS network device settings (MTU size). 5040 */ 5041 static int igc_sw_init(struct igc_adapter *adapter) 5042 { 5043 struct net_device *netdev = adapter->netdev; 5044 struct pci_dev *pdev = adapter->pdev; 5045 struct igc_hw *hw = &adapter->hw; 5046 5047 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 5048 5049 /* init RSS key */ 5050 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key)); 5051 5052 /* set default ring sizes */ 5053 adapter->tx_ring_count = IGC_DEFAULT_TXD; 5054 adapter->rx_ring_count = IGC_DEFAULT_RXD; 5055 5056 /* set default ITR values */ 5057 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 5058 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 5059 5060 /* set default work limits */ 5061 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 5062 5063 /* adjust max frame to be at least the size of a standard frame */ 5064 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 5065 VLAN_HLEN; 5066 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 5067 5068 mutex_init(&adapter->nfc_rule_lock); 5069 INIT_LIST_HEAD(&adapter->nfc_rule_list); 5070 adapter->nfc_rule_count = 0; 5071 5072 spin_lock_init(&adapter->stats64_lock); 5073 spin_lock_init(&adapter->qbv_tx_lock); 5074 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 5075 adapter->flags |= IGC_FLAG_HAS_MSIX; 5076 5077 igc_init_queue_configuration(adapter); 5078 5079 /* This call may decrease the number of queues */ 5080 if (igc_init_interrupt_scheme(adapter, true)) { 5081 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5082 return -ENOMEM; 5083 } 5084 5085 /* Explicitly disable IRQ since the NIC can be in any state. */ 5086 igc_irq_disable(adapter); 5087 5088 set_bit(__IGC_DOWN, &adapter->state); 5089 5090 return 0; 5091 } 5092 5093 static void igc_set_queue_napi(struct igc_adapter *adapter, int vector, 5094 struct napi_struct *napi) 5095 { 5096 struct igc_q_vector *q_vector = adapter->q_vector[vector]; 5097 5098 if (q_vector->rx.ring) 5099 netif_queue_set_napi(adapter->netdev, 5100 q_vector->rx.ring->queue_index, 5101 NETDEV_QUEUE_TYPE_RX, napi); 5102 5103 if (q_vector->tx.ring) 5104 netif_queue_set_napi(adapter->netdev, 5105 q_vector->tx.ring->queue_index, 5106 NETDEV_QUEUE_TYPE_TX, napi); 5107 } 5108 5109 /** 5110 * igc_up - Open the interface and prepare it to handle traffic 5111 * @adapter: board private structure 5112 */ 5113 void igc_up(struct igc_adapter *adapter) 5114 { 5115 struct igc_hw *hw = &adapter->hw; 5116 struct napi_struct *napi; 5117 int i = 0; 5118 5119 /* hardware has been reset, we need to reload some things */ 5120 igc_configure(adapter); 5121 5122 clear_bit(__IGC_DOWN, &adapter->state); 5123 5124 for (i = 0; i < adapter->num_q_vectors; i++) { 5125 napi = &adapter->q_vector[i]->napi; 5126 napi_enable(napi); 5127 igc_set_queue_napi(adapter, i, napi); 5128 } 5129 5130 if (adapter->msix_entries) 5131 igc_configure_msix(adapter); 5132 else 5133 igc_assign_vector(adapter->q_vector[0], 0); 5134 5135 /* Clear any pending interrupts. */ 5136 rd32(IGC_ICR); 5137 igc_irq_enable(adapter); 5138 5139 netif_tx_start_all_queues(adapter->netdev); 5140 5141 /* start the watchdog. */ 5142 hw->mac.get_link_status = true; 5143 schedule_work(&adapter->watchdog_task); 5144 } 5145 5146 /** 5147 * igc_update_stats - Update the board statistics counters 5148 * @adapter: board private structure 5149 */ 5150 void igc_update_stats(struct igc_adapter *adapter) 5151 { 5152 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 5153 struct pci_dev *pdev = adapter->pdev; 5154 struct igc_hw *hw = &adapter->hw; 5155 u64 _bytes, _packets; 5156 u64 bytes, packets; 5157 unsigned int start; 5158 u32 mpc; 5159 int i; 5160 5161 /* Prevent stats update while adapter is being reset, or if the pci 5162 * connection is down. 5163 */ 5164 if (adapter->link_speed == 0) 5165 return; 5166 if (pci_channel_offline(pdev)) 5167 return; 5168 5169 packets = 0; 5170 bytes = 0; 5171 5172 rcu_read_lock(); 5173 for (i = 0; i < adapter->num_rx_queues; i++) { 5174 struct igc_ring *ring = adapter->rx_ring[i]; 5175 u32 rqdpc = rd32(IGC_RQDPC(i)); 5176 5177 if (hw->mac.type >= igc_i225) 5178 wr32(IGC_RQDPC(i), 0); 5179 5180 if (rqdpc) { 5181 ring->rx_stats.drops += rqdpc; 5182 net_stats->rx_fifo_errors += rqdpc; 5183 } 5184 5185 do { 5186 start = u64_stats_fetch_begin(&ring->rx_syncp); 5187 _bytes = ring->rx_stats.bytes; 5188 _packets = ring->rx_stats.packets; 5189 } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); 5190 bytes += _bytes; 5191 packets += _packets; 5192 } 5193 5194 net_stats->rx_bytes = bytes; 5195 net_stats->rx_packets = packets; 5196 5197 packets = 0; 5198 bytes = 0; 5199 for (i = 0; i < adapter->num_tx_queues; i++) { 5200 struct igc_ring *ring = adapter->tx_ring[i]; 5201 5202 do { 5203 start = u64_stats_fetch_begin(&ring->tx_syncp); 5204 _bytes = ring->tx_stats.bytes; 5205 _packets = ring->tx_stats.packets; 5206 } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); 5207 bytes += _bytes; 5208 packets += _packets; 5209 } 5210 net_stats->tx_bytes = bytes; 5211 net_stats->tx_packets = packets; 5212 rcu_read_unlock(); 5213 5214 /* read stats registers */ 5215 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 5216 adapter->stats.gprc += rd32(IGC_GPRC); 5217 adapter->stats.gorc += rd32(IGC_GORCL); 5218 rd32(IGC_GORCH); /* clear GORCL */ 5219 adapter->stats.bprc += rd32(IGC_BPRC); 5220 adapter->stats.mprc += rd32(IGC_MPRC); 5221 adapter->stats.roc += rd32(IGC_ROC); 5222 5223 adapter->stats.prc64 += rd32(IGC_PRC64); 5224 adapter->stats.prc127 += rd32(IGC_PRC127); 5225 adapter->stats.prc255 += rd32(IGC_PRC255); 5226 adapter->stats.prc511 += rd32(IGC_PRC511); 5227 adapter->stats.prc1023 += rd32(IGC_PRC1023); 5228 adapter->stats.prc1522 += rd32(IGC_PRC1522); 5229 adapter->stats.tlpic += rd32(IGC_TLPIC); 5230 adapter->stats.rlpic += rd32(IGC_RLPIC); 5231 adapter->stats.hgptc += rd32(IGC_HGPTC); 5232 5233 mpc = rd32(IGC_MPC); 5234 adapter->stats.mpc += mpc; 5235 net_stats->rx_fifo_errors += mpc; 5236 adapter->stats.scc += rd32(IGC_SCC); 5237 adapter->stats.ecol += rd32(IGC_ECOL); 5238 adapter->stats.mcc += rd32(IGC_MCC); 5239 adapter->stats.latecol += rd32(IGC_LATECOL); 5240 adapter->stats.dc += rd32(IGC_DC); 5241 adapter->stats.rlec += rd32(IGC_RLEC); 5242 adapter->stats.xonrxc += rd32(IGC_XONRXC); 5243 adapter->stats.xontxc += rd32(IGC_XONTXC); 5244 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 5245 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 5246 adapter->stats.fcruc += rd32(IGC_FCRUC); 5247 adapter->stats.gptc += rd32(IGC_GPTC); 5248 adapter->stats.gotc += rd32(IGC_GOTCL); 5249 rd32(IGC_GOTCH); /* clear GOTCL */ 5250 adapter->stats.rnbc += rd32(IGC_RNBC); 5251 adapter->stats.ruc += rd32(IGC_RUC); 5252 adapter->stats.rfc += rd32(IGC_RFC); 5253 adapter->stats.rjc += rd32(IGC_RJC); 5254 adapter->stats.tor += rd32(IGC_TORH); 5255 adapter->stats.tot += rd32(IGC_TOTH); 5256 adapter->stats.tpr += rd32(IGC_TPR); 5257 5258 adapter->stats.ptc64 += rd32(IGC_PTC64); 5259 adapter->stats.ptc127 += rd32(IGC_PTC127); 5260 adapter->stats.ptc255 += rd32(IGC_PTC255); 5261 adapter->stats.ptc511 += rd32(IGC_PTC511); 5262 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 5263 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 5264 5265 adapter->stats.mptc += rd32(IGC_MPTC); 5266 adapter->stats.bptc += rd32(IGC_BPTC); 5267 5268 adapter->stats.tpt += rd32(IGC_TPT); 5269 adapter->stats.colc += rd32(IGC_COLC); 5270 adapter->stats.colc += rd32(IGC_RERC); 5271 5272 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 5273 5274 adapter->stats.tsctc += rd32(IGC_TSCTC); 5275 5276 adapter->stats.iac += rd32(IGC_IAC); 5277 5278 /* Fill out the OS statistics structure */ 5279 net_stats->multicast = adapter->stats.mprc; 5280 net_stats->collisions = adapter->stats.colc; 5281 5282 /* Rx Errors */ 5283 5284 /* RLEC on some newer hardware can be incorrect so build 5285 * our own version based on RUC and ROC 5286 */ 5287 net_stats->rx_errors = adapter->stats.rxerrc + 5288 adapter->stats.crcerrs + adapter->stats.algnerrc + 5289 adapter->stats.ruc + adapter->stats.roc + 5290 adapter->stats.cexterr; 5291 net_stats->rx_length_errors = adapter->stats.ruc + 5292 adapter->stats.roc; 5293 net_stats->rx_crc_errors = adapter->stats.crcerrs; 5294 net_stats->rx_frame_errors = adapter->stats.algnerrc; 5295 net_stats->rx_missed_errors = adapter->stats.mpc; 5296 5297 /* Tx Errors */ 5298 net_stats->tx_errors = adapter->stats.ecol + 5299 adapter->stats.latecol; 5300 net_stats->tx_aborted_errors = adapter->stats.ecol; 5301 net_stats->tx_window_errors = adapter->stats.latecol; 5302 net_stats->tx_carrier_errors = adapter->stats.tncrs; 5303 5304 /* Tx Dropped */ 5305 net_stats->tx_dropped = adapter->stats.txdrop; 5306 5307 /* Management Stats */ 5308 adapter->stats.mgptc += rd32(IGC_MGTPTC); 5309 adapter->stats.mgprc += rd32(IGC_MGTPRC); 5310 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 5311 } 5312 5313 /** 5314 * igc_down - Close the interface 5315 * @adapter: board private structure 5316 */ 5317 void igc_down(struct igc_adapter *adapter) 5318 { 5319 struct net_device *netdev = adapter->netdev; 5320 struct igc_hw *hw = &adapter->hw; 5321 u32 tctl, rctl; 5322 int i = 0; 5323 5324 set_bit(__IGC_DOWN, &adapter->state); 5325 5326 igc_ptp_suspend(adapter); 5327 5328 if (pci_device_is_present(adapter->pdev)) { 5329 /* disable receives in the hardware */ 5330 rctl = rd32(IGC_RCTL); 5331 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 5332 /* flush and sleep below */ 5333 } 5334 /* set trans_start so we don't get spurious watchdogs during reset */ 5335 netif_trans_update(netdev); 5336 5337 netif_carrier_off(netdev); 5338 netif_tx_stop_all_queues(netdev); 5339 5340 if (pci_device_is_present(adapter->pdev)) { 5341 /* disable transmits in the hardware */ 5342 tctl = rd32(IGC_TCTL); 5343 tctl &= ~IGC_TCTL_EN; 5344 wr32(IGC_TCTL, tctl); 5345 /* flush both disables and wait for them to finish */ 5346 wrfl(); 5347 usleep_range(10000, 20000); 5348 5349 igc_irq_disable(adapter); 5350 } 5351 5352 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 5353 5354 for (i = 0; i < adapter->num_q_vectors; i++) { 5355 if (adapter->q_vector[i]) { 5356 napi_disable(&adapter->q_vector[i]->napi); 5357 igc_set_queue_napi(adapter, i, NULL); 5358 } 5359 } 5360 5361 timer_delete_sync(&adapter->watchdog_timer); 5362 timer_delete_sync(&adapter->phy_info_timer); 5363 5364 /* record the stats before reset*/ 5365 spin_lock(&adapter->stats64_lock); 5366 igc_update_stats(adapter); 5367 spin_unlock(&adapter->stats64_lock); 5368 5369 adapter->link_speed = 0; 5370 adapter->link_duplex = 0; 5371 5372 if (!pci_channel_offline(adapter->pdev)) 5373 igc_reset(adapter); 5374 5375 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 5376 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 5377 5378 igc_disable_all_tx_rings_hw(adapter); 5379 igc_clean_all_tx_rings(adapter); 5380 igc_clean_all_rx_rings(adapter); 5381 5382 if (adapter->fpe.mmsv.pmac_enabled) 5383 ethtool_mmsv_stop(&adapter->fpe.mmsv); 5384 } 5385 5386 void igc_reinit_locked(struct igc_adapter *adapter) 5387 { 5388 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 5389 usleep_range(1000, 2000); 5390 igc_down(adapter); 5391 igc_up(adapter); 5392 clear_bit(__IGC_RESETTING, &adapter->state); 5393 } 5394 5395 static void igc_reset_task(struct work_struct *work) 5396 { 5397 struct igc_adapter *adapter; 5398 5399 adapter = container_of(work, struct igc_adapter, reset_task); 5400 5401 rtnl_lock(); 5402 /* If we're already down or resetting, just bail */ 5403 if (test_bit(__IGC_DOWN, &adapter->state) || 5404 test_bit(__IGC_RESETTING, &adapter->state)) { 5405 rtnl_unlock(); 5406 return; 5407 } 5408 5409 igc_rings_dump(adapter); 5410 igc_regs_dump(adapter); 5411 netdev_err(adapter->netdev, "Reset adapter\n"); 5412 igc_reinit_locked(adapter); 5413 rtnl_unlock(); 5414 } 5415 5416 /** 5417 * igc_change_mtu - Change the Maximum Transfer Unit 5418 * @netdev: network interface device structure 5419 * @new_mtu: new value for maximum frame size 5420 * 5421 * Returns 0 on success, negative on failure 5422 */ 5423 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 5424 { 5425 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 5426 struct igc_adapter *adapter = netdev_priv(netdev); 5427 5428 if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) { 5429 netdev_dbg(netdev, "Jumbo frames not supported with XDP"); 5430 return -EINVAL; 5431 } 5432 5433 /* adjust max frame to be at least the size of a standard frame */ 5434 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 5435 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 5436 5437 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 5438 usleep_range(1000, 2000); 5439 5440 /* igc_down has a dependency on max_frame_size */ 5441 adapter->max_frame_size = max_frame; 5442 5443 if (netif_running(netdev)) 5444 igc_down(adapter); 5445 5446 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 5447 WRITE_ONCE(netdev->mtu, new_mtu); 5448 5449 if (netif_running(netdev)) 5450 igc_up(adapter); 5451 else 5452 igc_reset(adapter); 5453 5454 clear_bit(__IGC_RESETTING, &adapter->state); 5455 5456 return 0; 5457 } 5458 5459 /** 5460 * igc_tx_timeout - Respond to a Tx Hang 5461 * @netdev: network interface device structure 5462 * @txqueue: queue number that timed out 5463 **/ 5464 static void igc_tx_timeout(struct net_device *netdev, 5465 unsigned int __always_unused txqueue) 5466 { 5467 struct igc_adapter *adapter = netdev_priv(netdev); 5468 struct igc_hw *hw = &adapter->hw; 5469 5470 /* Do the reset outside of interrupt context */ 5471 adapter->tx_timeout_count++; 5472 schedule_work(&adapter->reset_task); 5473 wr32(IGC_EICS, 5474 (adapter->eims_enable_mask & ~adapter->eims_other)); 5475 } 5476 5477 /** 5478 * igc_get_stats64 - Get System Network Statistics 5479 * @netdev: network interface device structure 5480 * @stats: rtnl_link_stats64 pointer 5481 * 5482 * Returns the address of the device statistics structure. 5483 * The statistics are updated here and also from the timer callback. 5484 */ 5485 static void igc_get_stats64(struct net_device *netdev, 5486 struct rtnl_link_stats64 *stats) 5487 { 5488 struct igc_adapter *adapter = netdev_priv(netdev); 5489 5490 spin_lock(&adapter->stats64_lock); 5491 if (!test_bit(__IGC_RESETTING, &adapter->state)) 5492 igc_update_stats(adapter); 5493 memcpy(stats, &adapter->stats64, sizeof(*stats)); 5494 spin_unlock(&adapter->stats64_lock); 5495 } 5496 5497 static netdev_features_t igc_fix_features(struct net_device *netdev, 5498 netdev_features_t features) 5499 { 5500 /* Since there is no support for separate Rx/Tx vlan accel 5501 * enable/disable make sure Tx flag is always in same state as Rx. 5502 */ 5503 if (features & NETIF_F_HW_VLAN_CTAG_RX) 5504 features |= NETIF_F_HW_VLAN_CTAG_TX; 5505 else 5506 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 5507 5508 return features; 5509 } 5510 5511 static int igc_set_features(struct net_device *netdev, 5512 netdev_features_t features) 5513 { 5514 netdev_features_t changed = netdev->features ^ features; 5515 struct igc_adapter *adapter = netdev_priv(netdev); 5516 5517 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 5518 igc_vlan_mode(netdev, features); 5519 5520 /* Add VLAN support */ 5521 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 5522 return 0; 5523 5524 if (!(features & NETIF_F_NTUPLE)) 5525 igc_flush_nfc_rules(adapter); 5526 5527 netdev->features = features; 5528 5529 if (netif_running(netdev)) 5530 igc_reinit_locked(adapter); 5531 else 5532 igc_reset(adapter); 5533 5534 return 1; 5535 } 5536 5537 static netdev_features_t 5538 igc_features_check(struct sk_buff *skb, struct net_device *dev, 5539 netdev_features_t features) 5540 { 5541 unsigned int network_hdr_len, mac_hdr_len; 5542 5543 /* Make certain the headers can be described by a context descriptor */ 5544 mac_hdr_len = skb_network_offset(skb); 5545 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 5546 return features & ~(NETIF_F_HW_CSUM | 5547 NETIF_F_SCTP_CRC | 5548 NETIF_F_HW_VLAN_CTAG_TX | 5549 NETIF_F_TSO | 5550 NETIF_F_TSO6); 5551 5552 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 5553 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 5554 return features & ~(NETIF_F_HW_CSUM | 5555 NETIF_F_SCTP_CRC | 5556 NETIF_F_TSO | 5557 NETIF_F_TSO6); 5558 5559 /* We can only support IPv4 TSO in tunnels if we can mangle the 5560 * inner IP ID field, so strip TSO if MANGLEID is not supported. 5561 */ 5562 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 5563 features &= ~NETIF_F_TSO; 5564 5565 return features; 5566 } 5567 5568 static void igc_tsync_interrupt(struct igc_adapter *adapter) 5569 { 5570 struct igc_hw *hw = &adapter->hw; 5571 u32 tsauxc, sec, nsec, tsicr; 5572 struct ptp_clock_event event; 5573 struct timespec64 ts; 5574 5575 tsicr = rd32(IGC_TSICR); 5576 5577 if (tsicr & IGC_TSICR_SYS_WRAP) { 5578 event.type = PTP_CLOCK_PPS; 5579 if (adapter->ptp_caps.pps) 5580 ptp_clock_event(adapter->ptp_clock, &event); 5581 } 5582 5583 if (tsicr & IGC_TSICR_TXTS) { 5584 /* retrieve hardware timestamp */ 5585 igc_ptp_tx_tstamp_event(adapter); 5586 } 5587 5588 if (tsicr & IGC_TSICR_TT0) { 5589 spin_lock(&adapter->tmreg_lock); 5590 ts = timespec64_add(adapter->perout[0].start, 5591 adapter->perout[0].period); 5592 wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); 5593 wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec); 5594 tsauxc = rd32(IGC_TSAUXC); 5595 tsauxc |= IGC_TSAUXC_EN_TT0; 5596 wr32(IGC_TSAUXC, tsauxc); 5597 adapter->perout[0].start = ts; 5598 spin_unlock(&adapter->tmreg_lock); 5599 } 5600 5601 if (tsicr & IGC_TSICR_TT1) { 5602 spin_lock(&adapter->tmreg_lock); 5603 ts = timespec64_add(adapter->perout[1].start, 5604 adapter->perout[1].period); 5605 wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); 5606 wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec); 5607 tsauxc = rd32(IGC_TSAUXC); 5608 tsauxc |= IGC_TSAUXC_EN_TT1; 5609 wr32(IGC_TSAUXC, tsauxc); 5610 adapter->perout[1].start = ts; 5611 spin_unlock(&adapter->tmreg_lock); 5612 } 5613 5614 if (tsicr & IGC_TSICR_AUTT0) { 5615 nsec = rd32(IGC_AUXSTMPL0); 5616 sec = rd32(IGC_AUXSTMPH0); 5617 event.type = PTP_CLOCK_EXTTS; 5618 event.index = 0; 5619 event.timestamp = sec * NSEC_PER_SEC + nsec; 5620 ptp_clock_event(adapter->ptp_clock, &event); 5621 } 5622 5623 if (tsicr & IGC_TSICR_AUTT1) { 5624 nsec = rd32(IGC_AUXSTMPL1); 5625 sec = rd32(IGC_AUXSTMPH1); 5626 event.type = PTP_CLOCK_EXTTS; 5627 event.index = 1; 5628 event.timestamp = sec * NSEC_PER_SEC + nsec; 5629 ptp_clock_event(adapter->ptp_clock, &event); 5630 } 5631 } 5632 5633 /** 5634 * igc_msix_other - msix other interrupt handler 5635 * @irq: interrupt number 5636 * @data: pointer to a q_vector 5637 */ 5638 static irqreturn_t igc_msix_other(int irq, void *data) 5639 { 5640 struct igc_adapter *adapter = data; 5641 struct igc_hw *hw = &adapter->hw; 5642 u32 icr = rd32(IGC_ICR); 5643 5644 /* reading ICR causes bit 31 of EICR to be cleared */ 5645 if (icr & IGC_ICR_DRSTA) 5646 schedule_work(&adapter->reset_task); 5647 5648 if (icr & IGC_ICR_DOUTSYNC) { 5649 /* HW is reporting DMA is out of sync */ 5650 adapter->stats.doosync++; 5651 } 5652 5653 if (icr & IGC_ICR_LSC) { 5654 hw->mac.get_link_status = true; 5655 /* guard against interrupt when we're going down */ 5656 if (!test_bit(__IGC_DOWN, &adapter->state)) 5657 mod_timer(&adapter->watchdog_timer, jiffies + 1); 5658 } 5659 5660 if (icr & IGC_ICR_TS) 5661 igc_tsync_interrupt(adapter); 5662 5663 wr32(IGC_EIMS, adapter->eims_other); 5664 5665 return IRQ_HANDLED; 5666 } 5667 5668 static void igc_write_itr(struct igc_q_vector *q_vector) 5669 { 5670 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 5671 5672 if (!q_vector->set_itr) 5673 return; 5674 5675 if (!itr_val) 5676 itr_val = IGC_ITR_VAL_MASK; 5677 5678 itr_val |= IGC_EITR_CNT_IGNR; 5679 5680 writel(itr_val, q_vector->itr_register); 5681 q_vector->set_itr = 0; 5682 } 5683 5684 static irqreturn_t igc_msix_ring(int irq, void *data) 5685 { 5686 struct igc_q_vector *q_vector = data; 5687 5688 /* Write the ITR value calculated from the previous interrupt. */ 5689 igc_write_itr(q_vector); 5690 5691 napi_schedule_irqoff(&q_vector->napi); 5692 5693 return IRQ_HANDLED; 5694 } 5695 5696 /** 5697 * igc_request_msix - Initialize MSI-X interrupts 5698 * @adapter: Pointer to adapter structure 5699 * 5700 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 5701 * kernel. 5702 */ 5703 static int igc_request_msix(struct igc_adapter *adapter) 5704 { 5705 unsigned int num_q_vectors = adapter->num_q_vectors; 5706 int i = 0, err = 0, vector = 0, free_vector = 0; 5707 struct net_device *netdev = adapter->netdev; 5708 5709 err = request_irq(adapter->msix_entries[vector].vector, 5710 &igc_msix_other, 0, netdev->name, adapter); 5711 if (err) 5712 goto err_out; 5713 5714 if (num_q_vectors > MAX_Q_VECTORS) { 5715 num_q_vectors = MAX_Q_VECTORS; 5716 dev_warn(&adapter->pdev->dev, 5717 "The number of queue vectors (%d) is higher than max allowed (%d)\n", 5718 adapter->num_q_vectors, MAX_Q_VECTORS); 5719 } 5720 for (i = 0; i < num_q_vectors; i++) { 5721 struct igc_q_vector *q_vector = adapter->q_vector[i]; 5722 5723 vector++; 5724 5725 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 5726 5727 if (q_vector->rx.ring && q_vector->tx.ring) 5728 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 5729 q_vector->rx.ring->queue_index); 5730 else if (q_vector->tx.ring) 5731 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 5732 q_vector->tx.ring->queue_index); 5733 else if (q_vector->rx.ring) 5734 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 5735 q_vector->rx.ring->queue_index); 5736 else 5737 sprintf(q_vector->name, "%s-unused", netdev->name); 5738 5739 err = request_irq(adapter->msix_entries[vector].vector, 5740 igc_msix_ring, 0, q_vector->name, 5741 q_vector); 5742 if (err) 5743 goto err_free; 5744 5745 netif_napi_set_irq(&q_vector->napi, 5746 adapter->msix_entries[vector].vector); 5747 } 5748 5749 igc_configure_msix(adapter); 5750 return 0; 5751 5752 err_free: 5753 /* free already assigned IRQs */ 5754 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 5755 5756 vector--; 5757 for (i = 0; i < vector; i++) { 5758 free_irq(adapter->msix_entries[free_vector++].vector, 5759 adapter->q_vector[i]); 5760 } 5761 err_out: 5762 return err; 5763 } 5764 5765 /** 5766 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 5767 * @adapter: Pointer to adapter structure 5768 * 5769 * This function resets the device so that it has 0 rx queues, tx queues, and 5770 * MSI-X interrupts allocated. 5771 */ 5772 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 5773 { 5774 igc_free_q_vectors(adapter); 5775 igc_reset_interrupt_capability(adapter); 5776 } 5777 5778 /* Need to wait a few seconds after link up to get diagnostic information from 5779 * the phy 5780 */ 5781 static void igc_update_phy_info(struct timer_list *t) 5782 { 5783 struct igc_adapter *adapter = timer_container_of(adapter, t, 5784 phy_info_timer); 5785 5786 igc_get_phy_info(&adapter->hw); 5787 } 5788 5789 /** 5790 * igc_has_link - check shared code for link and determine up/down 5791 * @adapter: pointer to driver private info 5792 */ 5793 bool igc_has_link(struct igc_adapter *adapter) 5794 { 5795 struct igc_hw *hw = &adapter->hw; 5796 bool link_active = false; 5797 5798 /* get_link_status is set on LSC (link status) interrupt or 5799 * rx sequence error interrupt. get_link_status will stay 5800 * false until the igc_check_for_link establishes link 5801 * for copper adapters ONLY 5802 */ 5803 if (!hw->mac.get_link_status) 5804 return true; 5805 hw->mac.ops.check_for_link(hw); 5806 link_active = !hw->mac.get_link_status; 5807 5808 if (hw->mac.type == igc_i225) { 5809 if (!netif_carrier_ok(adapter->netdev)) { 5810 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 5811 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 5812 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 5813 adapter->link_check_timeout = jiffies; 5814 } 5815 } 5816 5817 return link_active; 5818 } 5819 5820 /** 5821 * igc_watchdog - Timer Call-back 5822 * @t: timer for the watchdog 5823 */ 5824 static void igc_watchdog(struct timer_list *t) 5825 { 5826 struct igc_adapter *adapter = timer_container_of(adapter, t, 5827 watchdog_timer); 5828 /* Do the rest outside of interrupt context */ 5829 schedule_work(&adapter->watchdog_task); 5830 } 5831 5832 static void igc_watchdog_task(struct work_struct *work) 5833 { 5834 struct igc_adapter *adapter = container_of(work, 5835 struct igc_adapter, 5836 watchdog_task); 5837 struct net_device *netdev = adapter->netdev; 5838 struct igc_hw *hw = &adapter->hw; 5839 struct igc_phy_info *phy = &hw->phy; 5840 u16 phy_data, retry_count = 20; 5841 u32 link; 5842 int i; 5843 5844 link = igc_has_link(adapter); 5845 5846 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 5847 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 5848 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 5849 else 5850 link = false; 5851 } 5852 5853 if (link) { 5854 /* Cancel scheduled suspend requests. */ 5855 pm_runtime_resume(netdev->dev.parent); 5856 5857 if (!netif_carrier_ok(netdev)) { 5858 u32 ctrl; 5859 5860 hw->mac.ops.get_speed_and_duplex(hw, 5861 &adapter->link_speed, 5862 &adapter->link_duplex); 5863 5864 ctrl = rd32(IGC_CTRL); 5865 /* Link status message must follow this format */ 5866 netdev_info(netdev, 5867 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5868 adapter->link_speed, 5869 adapter->link_duplex == FULL_DUPLEX ? 5870 "Full" : "Half", 5871 (ctrl & IGC_CTRL_TFCE) && 5872 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 5873 (ctrl & IGC_CTRL_RFCE) ? "RX" : 5874 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 5875 5876 /* disable EEE if enabled */ 5877 if ((adapter->flags & IGC_FLAG_EEE) && 5878 adapter->link_duplex == HALF_DUPLEX) { 5879 netdev_info(netdev, 5880 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 5881 adapter->hw.dev_spec._base.eee_enable = false; 5882 adapter->flags &= ~IGC_FLAG_EEE; 5883 } 5884 5885 /* check if SmartSpeed worked */ 5886 igc_check_downshift(hw); 5887 if (phy->speed_downgraded) 5888 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 5889 5890 /* adjust timeout factor according to speed/duplex */ 5891 adapter->tx_timeout_factor = 1; 5892 switch (adapter->link_speed) { 5893 case SPEED_10: 5894 adapter->tx_timeout_factor = 14; 5895 break; 5896 case SPEED_100: 5897 case SPEED_1000: 5898 case SPEED_2500: 5899 adapter->tx_timeout_factor = 1; 5900 break; 5901 } 5902 5903 /* Once the launch time has been set on the wire, there 5904 * is a delay before the link speed can be determined 5905 * based on link-up activity. Write into the register 5906 * as soon as we know the correct link speed. 5907 */ 5908 igc_tsn_adjust_txtime_offset(adapter); 5909 5910 if (adapter->fpe.mmsv.pmac_enabled) 5911 ethtool_mmsv_link_state_handle(&adapter->fpe.mmsv, 5912 true); 5913 5914 if (adapter->link_speed != SPEED_1000) 5915 goto no_wait; 5916 5917 /* wait for Remote receiver status OK */ 5918 retry_read_status: 5919 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 5920 &phy_data)) { 5921 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 5922 retry_count) { 5923 msleep(100); 5924 retry_count--; 5925 goto retry_read_status; 5926 } else if (!retry_count) { 5927 netdev_err(netdev, "exceed max 2 second\n"); 5928 } 5929 } else { 5930 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 5931 } 5932 no_wait: 5933 netif_carrier_on(netdev); 5934 5935 /* link state has changed, schedule phy info update */ 5936 if (!test_bit(__IGC_DOWN, &adapter->state)) 5937 mod_timer(&adapter->phy_info_timer, 5938 round_jiffies(jiffies + 2 * HZ)); 5939 } 5940 } else { 5941 if (netif_carrier_ok(netdev)) { 5942 adapter->link_speed = 0; 5943 adapter->link_duplex = 0; 5944 5945 /* Links status message must follow this format */ 5946 netdev_info(netdev, "NIC Link is Down\n"); 5947 netif_carrier_off(netdev); 5948 5949 if (adapter->fpe.mmsv.pmac_enabled) 5950 ethtool_mmsv_link_state_handle(&adapter->fpe.mmsv, 5951 false); 5952 5953 /* link state has changed, schedule phy info update */ 5954 if (!test_bit(__IGC_DOWN, &adapter->state)) 5955 mod_timer(&adapter->phy_info_timer, 5956 round_jiffies(jiffies + 2 * HZ)); 5957 5958 pm_schedule_suspend(netdev->dev.parent, 5959 MSEC_PER_SEC * 5); 5960 } 5961 } 5962 5963 spin_lock(&adapter->stats64_lock); 5964 igc_update_stats(adapter); 5965 spin_unlock(&adapter->stats64_lock); 5966 5967 for (i = 0; i < adapter->num_tx_queues; i++) { 5968 struct igc_ring *tx_ring = adapter->tx_ring[i]; 5969 5970 if (!netif_carrier_ok(netdev)) { 5971 /* We've lost link, so the controller stops DMA, 5972 * but we've got queued Tx work that's never going 5973 * to get done, so reset controller to flush Tx. 5974 * (Do the reset outside of interrupt context). 5975 */ 5976 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 5977 adapter->tx_timeout_count++; 5978 schedule_work(&adapter->reset_task); 5979 /* return immediately since reset is imminent */ 5980 return; 5981 } 5982 } 5983 5984 /* Force detection of hung controller every watchdog period */ 5985 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 5986 } 5987 5988 /* Cause software interrupt to ensure Rx ring is cleaned */ 5989 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 5990 u32 eics = 0; 5991 5992 for (i = 0; i < adapter->num_q_vectors; i++) { 5993 struct igc_q_vector *q_vector = adapter->q_vector[i]; 5994 struct igc_ring *rx_ring; 5995 5996 if (!q_vector->rx.ring) 5997 continue; 5998 5999 rx_ring = adapter->rx_ring[q_vector->rx.ring->queue_index]; 6000 6001 if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) { 6002 eics |= q_vector->eims_value; 6003 clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 6004 } 6005 } 6006 if (eics) 6007 wr32(IGC_EICS, eics); 6008 } else { 6009 struct igc_ring *rx_ring = adapter->rx_ring[0]; 6010 6011 if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) { 6012 clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 6013 wr32(IGC_ICS, IGC_ICS_RXDMT0); 6014 } 6015 } 6016 6017 igc_ptp_tx_hang(adapter); 6018 6019 /* Reset the timer */ 6020 if (!test_bit(__IGC_DOWN, &adapter->state)) { 6021 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 6022 mod_timer(&adapter->watchdog_timer, 6023 round_jiffies(jiffies + HZ)); 6024 else 6025 mod_timer(&adapter->watchdog_timer, 6026 round_jiffies(jiffies + 2 * HZ)); 6027 } 6028 } 6029 6030 /** 6031 * igc_intr_msi - Interrupt Handler 6032 * @irq: interrupt number 6033 * @data: pointer to a network interface device structure 6034 */ 6035 static irqreturn_t igc_intr_msi(int irq, void *data) 6036 { 6037 struct igc_adapter *adapter = data; 6038 struct igc_q_vector *q_vector = adapter->q_vector[0]; 6039 struct igc_hw *hw = &adapter->hw; 6040 /* read ICR disables interrupts using IAM */ 6041 u32 icr = rd32(IGC_ICR); 6042 6043 igc_write_itr(q_vector); 6044 6045 if (icr & IGC_ICR_DRSTA) 6046 schedule_work(&adapter->reset_task); 6047 6048 if (icr & IGC_ICR_DOUTSYNC) { 6049 /* HW is reporting DMA is out of sync */ 6050 adapter->stats.doosync++; 6051 } 6052 6053 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 6054 hw->mac.get_link_status = true; 6055 if (!test_bit(__IGC_DOWN, &adapter->state)) 6056 mod_timer(&adapter->watchdog_timer, jiffies + 1); 6057 } 6058 6059 if (icr & IGC_ICR_TS) 6060 igc_tsync_interrupt(adapter); 6061 6062 napi_schedule_irqoff(&q_vector->napi); 6063 6064 return IRQ_HANDLED; 6065 } 6066 6067 /** 6068 * igc_intr - Legacy Interrupt Handler 6069 * @irq: interrupt number 6070 * @data: pointer to a network interface device structure 6071 */ 6072 static irqreturn_t igc_intr(int irq, void *data) 6073 { 6074 struct igc_adapter *adapter = data; 6075 struct igc_q_vector *q_vector = adapter->q_vector[0]; 6076 struct igc_hw *hw = &adapter->hw; 6077 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 6078 * need for the IMC write 6079 */ 6080 u32 icr = rd32(IGC_ICR); 6081 6082 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 6083 * not set, then the adapter didn't send an interrupt 6084 */ 6085 if (!(icr & IGC_ICR_INT_ASSERTED)) 6086 return IRQ_NONE; 6087 6088 igc_write_itr(q_vector); 6089 6090 if (icr & IGC_ICR_DRSTA) 6091 schedule_work(&adapter->reset_task); 6092 6093 if (icr & IGC_ICR_DOUTSYNC) { 6094 /* HW is reporting DMA is out of sync */ 6095 adapter->stats.doosync++; 6096 } 6097 6098 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 6099 hw->mac.get_link_status = true; 6100 /* guard against interrupt when we're going down */ 6101 if (!test_bit(__IGC_DOWN, &adapter->state)) 6102 mod_timer(&adapter->watchdog_timer, jiffies + 1); 6103 } 6104 6105 if (icr & IGC_ICR_TS) 6106 igc_tsync_interrupt(adapter); 6107 6108 napi_schedule_irqoff(&q_vector->napi); 6109 6110 return IRQ_HANDLED; 6111 } 6112 6113 static void igc_free_irq(struct igc_adapter *adapter) 6114 { 6115 if (adapter->msix_entries) { 6116 int vector = 0, i; 6117 6118 free_irq(adapter->msix_entries[vector++].vector, adapter); 6119 6120 for (i = 0; i < adapter->num_q_vectors; i++) 6121 free_irq(adapter->msix_entries[vector++].vector, 6122 adapter->q_vector[i]); 6123 } else { 6124 free_irq(adapter->pdev->irq, adapter); 6125 } 6126 } 6127 6128 /** 6129 * igc_request_irq - initialize interrupts 6130 * @adapter: Pointer to adapter structure 6131 * 6132 * Attempts to configure interrupts using the best available 6133 * capabilities of the hardware and kernel. 6134 */ 6135 static int igc_request_irq(struct igc_adapter *adapter) 6136 { 6137 struct net_device *netdev = adapter->netdev; 6138 struct pci_dev *pdev = adapter->pdev; 6139 int err = 0; 6140 6141 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 6142 err = igc_request_msix(adapter); 6143 if (!err) 6144 goto request_done; 6145 /* fall back to MSI */ 6146 igc_free_all_tx_resources(adapter); 6147 igc_free_all_rx_resources(adapter); 6148 6149 igc_clear_interrupt_scheme(adapter); 6150 err = igc_init_interrupt_scheme(adapter, false); 6151 if (err) 6152 goto request_done; 6153 igc_setup_all_tx_resources(adapter); 6154 igc_setup_all_rx_resources(adapter); 6155 igc_configure(adapter); 6156 } 6157 6158 igc_assign_vector(adapter->q_vector[0], 0); 6159 6160 if (adapter->flags & IGC_FLAG_HAS_MSI) { 6161 err = request_irq(pdev->irq, &igc_intr_msi, 0, 6162 netdev->name, adapter); 6163 if (!err) 6164 goto request_done; 6165 6166 /* fall back to legacy interrupts */ 6167 igc_reset_interrupt_capability(adapter); 6168 adapter->flags &= ~IGC_FLAG_HAS_MSI; 6169 } 6170 6171 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 6172 netdev->name, adapter); 6173 6174 if (err) 6175 netdev_err(netdev, "Error %d getting interrupt\n", err); 6176 6177 request_done: 6178 return err; 6179 } 6180 6181 /** 6182 * __igc_open - Called when a network interface is made active 6183 * @netdev: network interface device structure 6184 * @resuming: boolean indicating if the device is resuming 6185 * 6186 * Returns 0 on success, negative value on failure 6187 * 6188 * The open entry point is called when a network interface is made 6189 * active by the system (IFF_UP). At this point all resources needed 6190 * for transmit and receive operations are allocated, the interrupt 6191 * handler is registered with the OS, the watchdog timer is started, 6192 * and the stack is notified that the interface is ready. 6193 */ 6194 static int __igc_open(struct net_device *netdev, bool resuming) 6195 { 6196 struct igc_adapter *adapter = netdev_priv(netdev); 6197 struct pci_dev *pdev = adapter->pdev; 6198 struct igc_hw *hw = &adapter->hw; 6199 struct napi_struct *napi; 6200 int err = 0; 6201 int i = 0; 6202 6203 /* disallow open during test */ 6204 6205 if (test_bit(__IGC_TESTING, &adapter->state)) { 6206 WARN_ON(resuming); 6207 return -EBUSY; 6208 } 6209 6210 if (!resuming) 6211 pm_runtime_get_sync(&pdev->dev); 6212 6213 netif_carrier_off(netdev); 6214 6215 /* allocate transmit descriptors */ 6216 err = igc_setup_all_tx_resources(adapter); 6217 if (err) 6218 goto err_setup_tx; 6219 6220 /* allocate receive descriptors */ 6221 err = igc_setup_all_rx_resources(adapter); 6222 if (err) 6223 goto err_setup_rx; 6224 6225 igc_power_up_link(adapter); 6226 6227 igc_configure(adapter); 6228 6229 err = igc_request_irq(adapter); 6230 if (err) 6231 goto err_req_irq; 6232 6233 clear_bit(__IGC_DOWN, &adapter->state); 6234 6235 for (i = 0; i < adapter->num_q_vectors; i++) { 6236 napi = &adapter->q_vector[i]->napi; 6237 napi_enable(napi); 6238 igc_set_queue_napi(adapter, i, napi); 6239 } 6240 6241 /* Clear any pending interrupts. */ 6242 rd32(IGC_ICR); 6243 igc_irq_enable(adapter); 6244 6245 if (!resuming) 6246 pm_runtime_put(&pdev->dev); 6247 6248 netif_tx_start_all_queues(netdev); 6249 6250 /* start the watchdog. */ 6251 hw->mac.get_link_status = true; 6252 schedule_work(&adapter->watchdog_task); 6253 6254 return IGC_SUCCESS; 6255 6256 err_req_irq: 6257 igc_release_hw_control(adapter); 6258 igc_power_down_phy_copper_base(&adapter->hw); 6259 igc_free_all_rx_resources(adapter); 6260 err_setup_rx: 6261 igc_free_all_tx_resources(adapter); 6262 err_setup_tx: 6263 igc_reset(adapter); 6264 if (!resuming) 6265 pm_runtime_put(&pdev->dev); 6266 6267 return err; 6268 } 6269 6270 int igc_open(struct net_device *netdev) 6271 { 6272 struct igc_adapter *adapter = netdev_priv(netdev); 6273 int err; 6274 6275 /* Notify the stack of the actual queue counts. */ 6276 err = netif_set_real_num_queues(netdev, adapter->num_tx_queues, 6277 adapter->num_rx_queues); 6278 if (err) { 6279 netdev_err(netdev, "error setting real queue count\n"); 6280 return err; 6281 } 6282 6283 return __igc_open(netdev, false); 6284 } 6285 6286 /** 6287 * __igc_close - Disables a network interface 6288 * @netdev: network interface device structure 6289 * @suspending: boolean indicating the device is suspending 6290 * 6291 * Returns 0, this is not allowed to fail 6292 * 6293 * The close entry point is called when an interface is de-activated 6294 * by the OS. The hardware is still under the driver's control, but 6295 * needs to be disabled. A global MAC reset is issued to stop the 6296 * hardware, and all transmit and receive resources are freed. 6297 */ 6298 static int __igc_close(struct net_device *netdev, bool suspending) 6299 { 6300 struct igc_adapter *adapter = netdev_priv(netdev); 6301 struct pci_dev *pdev = adapter->pdev; 6302 6303 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 6304 6305 if (!suspending) 6306 pm_runtime_get_sync(&pdev->dev); 6307 6308 igc_down(adapter); 6309 6310 igc_release_hw_control(adapter); 6311 6312 igc_free_irq(adapter); 6313 6314 igc_free_all_tx_resources(adapter); 6315 igc_free_all_rx_resources(adapter); 6316 6317 if (!suspending) 6318 pm_runtime_put_sync(&pdev->dev); 6319 6320 return 0; 6321 } 6322 6323 int igc_close(struct net_device *netdev) 6324 { 6325 if (netif_device_present(netdev) || netdev->dismantle) 6326 return __igc_close(netdev, false); 6327 return 0; 6328 } 6329 6330 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 6331 bool enable) 6332 { 6333 struct igc_ring *ring; 6334 6335 if (queue < 0 || queue >= adapter->num_tx_queues) 6336 return -EINVAL; 6337 6338 ring = adapter->tx_ring[queue]; 6339 ring->launchtime_enable = enable; 6340 6341 return 0; 6342 } 6343 6344 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) 6345 { 6346 struct timespec64 b; 6347 6348 b = ktime_to_timespec64(base_time); 6349 6350 return timespec64_compare(now, &b) > 0; 6351 } 6352 6353 static bool validate_schedule(struct igc_adapter *adapter, 6354 const struct tc_taprio_qopt_offload *qopt) 6355 { 6356 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 6357 struct igc_hw *hw = &adapter->hw; 6358 struct timespec64 now; 6359 size_t n; 6360 6361 if (qopt->cycle_time_extension) 6362 return false; 6363 6364 igc_ptp_read(adapter, &now); 6365 6366 /* If we program the controller's BASET registers with a time 6367 * in the future, it will hold all the packets until that 6368 * time, causing a lot of TX Hangs, so to avoid that, we 6369 * reject schedules that would start in the future. 6370 * Note: Limitation above is no longer in i226. 6371 */ 6372 if (!is_base_time_past(qopt->base_time, &now) && 6373 igc_is_device_id_i225(hw)) 6374 return false; 6375 6376 for (n = 0; n < qopt->num_entries; n++) { 6377 const struct tc_taprio_sched_entry *e, *prev; 6378 int i; 6379 6380 prev = n ? &qopt->entries[n - 1] : NULL; 6381 e = &qopt->entries[n]; 6382 6383 /* i225 only supports "global" frame preemption 6384 * settings. 6385 */ 6386 if (e->command != TC_TAPRIO_CMD_SET_GATES) 6387 return false; 6388 6389 for (i = 0; i < adapter->num_tx_queues; i++) 6390 if (e->gate_mask & BIT(i)) { 6391 queue_uses[i]++; 6392 6393 /* There are limitations: A single queue cannot 6394 * be opened and closed multiple times per cycle 6395 * unless the gate stays open. Check for it. 6396 */ 6397 if (queue_uses[i] > 1 && 6398 !(prev->gate_mask & BIT(i))) 6399 return false; 6400 } 6401 } 6402 6403 return true; 6404 } 6405 6406 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 6407 struct tc_etf_qopt_offload *qopt) 6408 { 6409 struct igc_hw *hw = &adapter->hw; 6410 int err; 6411 6412 if (hw->mac.type != igc_i225) 6413 return -EOPNOTSUPP; 6414 6415 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 6416 if (err) 6417 return err; 6418 6419 return igc_tsn_offload_apply(adapter); 6420 } 6421 6422 static int igc_qbv_clear_schedule(struct igc_adapter *adapter) 6423 { 6424 unsigned long flags; 6425 int i; 6426 6427 adapter->base_time = 0; 6428 adapter->cycle_time = NSEC_PER_SEC; 6429 adapter->taprio_offload_enable = false; 6430 adapter->qbv_config_change_errors = 0; 6431 adapter->qbv_count = 0; 6432 6433 for (i = 0; i < adapter->num_tx_queues; i++) { 6434 struct igc_ring *ring = adapter->tx_ring[i]; 6435 6436 ring->start_time = 0; 6437 ring->end_time = NSEC_PER_SEC; 6438 ring->max_sdu = 0; 6439 ring->preemptible = false; 6440 } 6441 6442 spin_lock_irqsave(&adapter->qbv_tx_lock, flags); 6443 6444 adapter->qbv_transition = false; 6445 6446 for (i = 0; i < adapter->num_tx_queues; i++) { 6447 struct igc_ring *ring = adapter->tx_ring[i]; 6448 6449 ring->oper_gate_closed = false; 6450 ring->admin_gate_closed = false; 6451 } 6452 6453 spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags); 6454 6455 return 0; 6456 } 6457 6458 static int igc_tsn_clear_schedule(struct igc_adapter *adapter) 6459 { 6460 igc_qbv_clear_schedule(adapter); 6461 6462 return 0; 6463 } 6464 6465 static void igc_taprio_stats(struct net_device *dev, 6466 struct tc_taprio_qopt_stats *stats) 6467 { 6468 /* When Strict_End is enabled, the tx_overruns counter 6469 * will always be zero. 6470 */ 6471 stats->tx_overruns = 0; 6472 } 6473 6474 static void igc_taprio_queue_stats(struct net_device *dev, 6475 struct tc_taprio_qopt_queue_stats *queue_stats) 6476 { 6477 struct tc_taprio_qopt_stats *stats = &queue_stats->stats; 6478 6479 /* When Strict_End is enabled, the tx_overruns counter 6480 * will always be zero. 6481 */ 6482 stats->tx_overruns = 0; 6483 } 6484 6485 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 6486 struct tc_taprio_qopt_offload *qopt) 6487 { 6488 bool queue_configured[IGC_MAX_TX_QUEUES] = { }; 6489 struct igc_hw *hw = &adapter->hw; 6490 u32 start_time = 0, end_time = 0; 6491 struct timespec64 now; 6492 unsigned long flags; 6493 size_t n; 6494 int i; 6495 6496 if (qopt->base_time < 0) 6497 return -ERANGE; 6498 6499 if (igc_is_device_id_i225(hw) && adapter->taprio_offload_enable) 6500 return -EALREADY; 6501 6502 if (!validate_schedule(adapter, qopt)) 6503 return -EINVAL; 6504 6505 if (qopt->mqprio.preemptible_tcs && 6506 !(adapter->flags & IGC_FLAG_TSN_REVERSE_TXQ_PRIO)) { 6507 NL_SET_ERR_MSG_MOD(qopt->extack, 6508 "reverse-tsn-txq-prio private flag must be enabled before setting preemptible tc"); 6509 return -ENODEV; 6510 } 6511 6512 igc_ptp_read(adapter, &now); 6513 6514 if (igc_tsn_is_taprio_activated_by_user(adapter) && 6515 is_base_time_past(qopt->base_time, &now)) 6516 adapter->qbv_config_change_errors++; 6517 6518 adapter->cycle_time = qopt->cycle_time; 6519 adapter->base_time = qopt->base_time; 6520 adapter->taprio_offload_enable = true; 6521 6522 for (n = 0; n < qopt->num_entries; n++) { 6523 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 6524 6525 end_time += e->interval; 6526 6527 /* If any of the conditions below are true, we need to manually 6528 * control the end time of the cycle. 6529 * 1. Qbv users can specify a cycle time that is not equal 6530 * to the total GCL intervals. Hence, recalculation is 6531 * necessary here to exclude the time interval that 6532 * exceeds the cycle time. 6533 * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2, 6534 * once the end of the list is reached, it will switch 6535 * to the END_OF_CYCLE state and leave the gates in the 6536 * same state until the next cycle is started. 6537 */ 6538 if (end_time > adapter->cycle_time || 6539 n + 1 == qopt->num_entries) 6540 end_time = adapter->cycle_time; 6541 6542 for (i = 0; i < adapter->num_tx_queues; i++) { 6543 struct igc_ring *ring = adapter->tx_ring[i]; 6544 6545 if (!(e->gate_mask & BIT(i))) 6546 continue; 6547 6548 /* Check whether a queue stays open for more than one 6549 * entry. If so, keep the start and advance the end 6550 * time. 6551 */ 6552 if (!queue_configured[i]) 6553 ring->start_time = start_time; 6554 ring->end_time = end_time; 6555 6556 if (ring->start_time >= adapter->cycle_time) 6557 queue_configured[i] = false; 6558 else 6559 queue_configured[i] = true; 6560 } 6561 6562 start_time += e->interval; 6563 } 6564 6565 spin_lock_irqsave(&adapter->qbv_tx_lock, flags); 6566 6567 /* Check whether a queue gets configured. 6568 * If not, set the start and end time to be end time. 6569 */ 6570 for (i = 0; i < adapter->num_tx_queues; i++) { 6571 struct igc_ring *ring = adapter->tx_ring[i]; 6572 6573 if (!is_base_time_past(qopt->base_time, &now)) { 6574 ring->admin_gate_closed = false; 6575 } else { 6576 ring->oper_gate_closed = false; 6577 ring->admin_gate_closed = false; 6578 } 6579 6580 if (!queue_configured[i]) { 6581 if (!is_base_time_past(qopt->base_time, &now)) 6582 ring->admin_gate_closed = true; 6583 else 6584 ring->oper_gate_closed = true; 6585 6586 ring->start_time = end_time; 6587 ring->end_time = end_time; 6588 } 6589 } 6590 6591 spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags); 6592 6593 for (i = 0; i < adapter->num_tx_queues; i++) { 6594 struct igc_ring *ring = adapter->tx_ring[i]; 6595 struct net_device *dev = adapter->netdev; 6596 6597 if (qopt->max_sdu[i]) 6598 ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len - ETH_TLEN; 6599 else 6600 ring->max_sdu = 0; 6601 } 6602 6603 igc_fpe_save_preempt_queue(adapter, &qopt->mqprio); 6604 6605 return 0; 6606 } 6607 6608 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 6609 struct tc_taprio_qopt_offload *qopt) 6610 { 6611 struct igc_hw *hw = &adapter->hw; 6612 int err; 6613 6614 if (hw->mac.type != igc_i225) 6615 return -EOPNOTSUPP; 6616 6617 switch (qopt->cmd) { 6618 case TAPRIO_CMD_REPLACE: 6619 err = igc_save_qbv_schedule(adapter, qopt); 6620 break; 6621 case TAPRIO_CMD_DESTROY: 6622 err = igc_tsn_clear_schedule(adapter); 6623 break; 6624 case TAPRIO_CMD_STATS: 6625 igc_taprio_stats(adapter->netdev, &qopt->stats); 6626 return 0; 6627 case TAPRIO_CMD_QUEUE_STATS: 6628 igc_taprio_queue_stats(adapter->netdev, &qopt->queue_stats); 6629 return 0; 6630 default: 6631 return -EOPNOTSUPP; 6632 } 6633 6634 if (err) 6635 return err; 6636 6637 return igc_tsn_offload_apply(adapter); 6638 } 6639 6640 static int igc_save_cbs_params(struct igc_adapter *adapter, int queue, 6641 bool enable, int idleslope, int sendslope, 6642 int hicredit, int locredit) 6643 { 6644 bool cbs_status[IGC_MAX_SR_QUEUES] = { false }; 6645 struct net_device *netdev = adapter->netdev; 6646 struct igc_ring *ring; 6647 int i; 6648 6649 /* i225 has two sets of credit-based shaper logic. 6650 * Supporting it only on the top two priority queues 6651 */ 6652 if (queue < 0 || queue > 1) 6653 return -EINVAL; 6654 6655 ring = adapter->tx_ring[queue]; 6656 6657 for (i = 0; i < IGC_MAX_SR_QUEUES; i++) 6658 if (adapter->tx_ring[i]) 6659 cbs_status[i] = adapter->tx_ring[i]->cbs_enable; 6660 6661 /* CBS should be enabled on the highest priority queue first in order 6662 * for the CBS algorithm to operate as intended. 6663 */ 6664 if (enable) { 6665 if (queue == 1 && !cbs_status[0]) { 6666 netdev_err(netdev, 6667 "Enabling CBS on queue1 before queue0\n"); 6668 return -EINVAL; 6669 } 6670 } else { 6671 if (queue == 0 && cbs_status[1]) { 6672 netdev_err(netdev, 6673 "Disabling CBS on queue0 before queue1\n"); 6674 return -EINVAL; 6675 } 6676 } 6677 6678 ring->cbs_enable = enable; 6679 ring->idleslope = idleslope; 6680 ring->sendslope = sendslope; 6681 ring->hicredit = hicredit; 6682 ring->locredit = locredit; 6683 6684 return 0; 6685 } 6686 6687 static int igc_tsn_enable_cbs(struct igc_adapter *adapter, 6688 struct tc_cbs_qopt_offload *qopt) 6689 { 6690 struct igc_hw *hw = &adapter->hw; 6691 int err; 6692 6693 if (hw->mac.type != igc_i225) 6694 return -EOPNOTSUPP; 6695 6696 if (qopt->queue < 0 || qopt->queue > 1) 6697 return -EINVAL; 6698 6699 err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable, 6700 qopt->idleslope, qopt->sendslope, 6701 qopt->hicredit, qopt->locredit); 6702 if (err) 6703 return err; 6704 6705 return igc_tsn_offload_apply(adapter); 6706 } 6707 6708 static int igc_tc_query_caps(struct igc_adapter *adapter, 6709 struct tc_query_caps_base *base) 6710 { 6711 struct igc_hw *hw = &adapter->hw; 6712 6713 switch (base->type) { 6714 case TC_SETUP_QDISC_MQPRIO: { 6715 struct tc_mqprio_caps *caps = base->caps; 6716 6717 caps->validate_queue_counts = true; 6718 6719 return 0; 6720 } 6721 case TC_SETUP_QDISC_TAPRIO: { 6722 struct tc_taprio_caps *caps = base->caps; 6723 6724 if (!(adapter->flags & IGC_FLAG_TSN_REVERSE_TXQ_PRIO)) 6725 caps->broken_mqprio = true; 6726 6727 if (hw->mac.type == igc_i225) { 6728 caps->supports_queue_max_sdu = true; 6729 caps->gate_mask_per_txq = true; 6730 } 6731 6732 return 0; 6733 } 6734 default: 6735 return -EOPNOTSUPP; 6736 } 6737 } 6738 6739 static void igc_save_mqprio_params(struct igc_adapter *adapter, u8 num_tc, 6740 u16 *offset) 6741 { 6742 int i; 6743 6744 adapter->strict_priority_enable = true; 6745 adapter->num_tc = num_tc; 6746 6747 for (i = 0; i < num_tc; i++) 6748 adapter->queue_per_tc[i] = offset[i]; 6749 } 6750 6751 static bool 6752 igc_tsn_is_tc_to_queue_priority_ordered(struct tc_mqprio_qopt_offload *mqprio) 6753 { 6754 int num_tc = mqprio->qopt.num_tc; 6755 int i; 6756 6757 for (i = 1; i < num_tc; i++) { 6758 if (mqprio->qopt.offset[i - 1] > mqprio->qopt.offset[i]) 6759 return false; 6760 } 6761 6762 return true; 6763 } 6764 6765 static int igc_tsn_enable_mqprio(struct igc_adapter *adapter, 6766 struct tc_mqprio_qopt_offload *mqprio) 6767 { 6768 struct igc_hw *hw = &adapter->hw; 6769 int err, i; 6770 6771 if (hw->mac.type != igc_i225) 6772 return -EOPNOTSUPP; 6773 6774 if (!mqprio->qopt.num_tc) { 6775 adapter->strict_priority_enable = false; 6776 igc_fpe_clear_preempt_queue(adapter); 6777 netdev_reset_tc(adapter->netdev); 6778 goto apply; 6779 } 6780 6781 /* There are as many TCs as Tx queues. */ 6782 if (mqprio->qopt.num_tc != adapter->num_tx_queues) { 6783 NL_SET_ERR_MSG_FMT_MOD(mqprio->extack, 6784 "Only %d traffic classes supported", 6785 adapter->num_tx_queues); 6786 return -EOPNOTSUPP; 6787 } 6788 6789 /* Only one queue per TC is supported. */ 6790 for (i = 0; i < mqprio->qopt.num_tc; i++) { 6791 if (mqprio->qopt.count[i] != 1) { 6792 NL_SET_ERR_MSG_MOD(mqprio->extack, 6793 "Only one queue per TC supported"); 6794 return -EOPNOTSUPP; 6795 } 6796 } 6797 6798 if (!igc_tsn_is_tc_to_queue_priority_ordered(mqprio)) { 6799 NL_SET_ERR_MSG_MOD(mqprio->extack, 6800 "tc to queue mapping must preserve increasing priority (higher tc -> higher queue)"); 6801 return -EOPNOTSUPP; 6802 } 6803 6804 igc_save_mqprio_params(adapter, mqprio->qopt.num_tc, 6805 mqprio->qopt.offset); 6806 6807 err = netdev_set_num_tc(adapter->netdev, adapter->num_tc); 6808 if (err) 6809 return err; 6810 6811 for (i = 0; i < adapter->num_tc; i++) { 6812 err = netdev_set_tc_queue(adapter->netdev, i, 1, 6813 adapter->queue_per_tc[i]); 6814 if (err) 6815 return err; 6816 } 6817 6818 /* In case the card is configured with less than four queues. */ 6819 for (; i < IGC_MAX_TX_QUEUES; i++) 6820 adapter->queue_per_tc[i] = i; 6821 6822 mqprio->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS; 6823 igc_fpe_save_preempt_queue(adapter, mqprio); 6824 6825 apply: 6826 return igc_tsn_offload_apply(adapter); 6827 } 6828 6829 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 6830 void *type_data) 6831 { 6832 struct igc_adapter *adapter = netdev_priv(dev); 6833 6834 adapter->tc_setup_type = type; 6835 6836 switch (type) { 6837 case TC_QUERY_CAPS: 6838 return igc_tc_query_caps(adapter, type_data); 6839 case TC_SETUP_QDISC_TAPRIO: 6840 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 6841 6842 case TC_SETUP_QDISC_ETF: 6843 return igc_tsn_enable_launchtime(adapter, type_data); 6844 6845 case TC_SETUP_QDISC_CBS: 6846 return igc_tsn_enable_cbs(adapter, type_data); 6847 6848 case TC_SETUP_QDISC_MQPRIO: 6849 return igc_tsn_enable_mqprio(adapter, type_data); 6850 6851 default: 6852 return -EOPNOTSUPP; 6853 } 6854 } 6855 6856 static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6857 { 6858 struct igc_adapter *adapter = netdev_priv(dev); 6859 6860 switch (bpf->command) { 6861 case XDP_SETUP_PROG: 6862 return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack); 6863 case XDP_SETUP_XSK_POOL: 6864 return igc_xdp_setup_pool(adapter, bpf->xsk.pool, 6865 bpf->xsk.queue_id); 6866 default: 6867 return -EOPNOTSUPP; 6868 } 6869 } 6870 6871 static int igc_xdp_xmit(struct net_device *dev, int num_frames, 6872 struct xdp_frame **frames, u32 flags) 6873 { 6874 struct igc_adapter *adapter = netdev_priv(dev); 6875 int cpu = smp_processor_id(); 6876 struct netdev_queue *nq; 6877 struct igc_ring *ring; 6878 int i, nxmit; 6879 6880 if (unlikely(!netif_carrier_ok(dev))) 6881 return -ENETDOWN; 6882 6883 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6884 return -EINVAL; 6885 6886 ring = igc_get_tx_ring(adapter, cpu); 6887 nq = txring_txq(ring); 6888 6889 __netif_tx_lock(nq, cpu); 6890 6891 /* Avoid transmit queue timeout since we share it with the slow path */ 6892 txq_trans_cond_update(nq); 6893 6894 nxmit = 0; 6895 for (i = 0; i < num_frames; i++) { 6896 int err; 6897 struct xdp_frame *xdpf = frames[i]; 6898 6899 err = igc_xdp_init_tx_descriptor(ring, xdpf); 6900 if (err) 6901 break; 6902 nxmit++; 6903 } 6904 6905 if (flags & XDP_XMIT_FLUSH) 6906 igc_flush_tx_descriptors(ring); 6907 6908 __netif_tx_unlock(nq); 6909 6910 return nxmit; 6911 } 6912 6913 static u32 igc_sw_irq_prep(struct igc_q_vector *q_vector) 6914 { 6915 u32 eics = 0; 6916 6917 if (!napi_if_scheduled_mark_missed(&q_vector->napi)) 6918 eics = q_vector->eims_value; 6919 6920 return eics; 6921 } 6922 6923 int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 6924 { 6925 struct igc_adapter *adapter = netdev_priv(dev); 6926 struct igc_hw *hw = &adapter->hw; 6927 struct igc_ring *ring; 6928 u32 eics = 0; 6929 6930 if (test_bit(__IGC_DOWN, &adapter->state)) 6931 return -ENETDOWN; 6932 6933 if (!igc_xdp_is_enabled(adapter)) 6934 return -ENXIO; 6935 /* Check if queue_id is valid. Tx and Rx queue numbers are always same */ 6936 if (queue_id >= adapter->num_rx_queues) 6937 return -EINVAL; 6938 6939 ring = adapter->rx_ring[queue_id]; 6940 6941 if (!ring->xsk_pool) 6942 return -ENXIO; 6943 6944 if (flags & XDP_WAKEUP_RX) 6945 eics |= igc_sw_irq_prep(ring->q_vector); 6946 6947 if (flags & XDP_WAKEUP_TX) { 6948 /* If IGC_FLAG_QUEUE_PAIRS is active, the q_vector 6949 * and NAPI is shared between RX and TX. 6950 * If NAPI is already running it would be marked as missed 6951 * from the RX path, making this TX call a NOP 6952 */ 6953 ring = adapter->tx_ring[queue_id]; 6954 eics |= igc_sw_irq_prep(ring->q_vector); 6955 } 6956 6957 if (eics) 6958 /* Cause software interrupt */ 6959 wr32(IGC_EICS, eics); 6960 6961 return 0; 6962 } 6963 6964 static ktime_t igc_get_tstamp(struct net_device *dev, 6965 const struct skb_shared_hwtstamps *hwtstamps, 6966 bool cycles) 6967 { 6968 struct igc_adapter *adapter = netdev_priv(dev); 6969 struct igc_inline_rx_tstamps *tstamp; 6970 ktime_t timestamp; 6971 6972 tstamp = hwtstamps->netdev_data; 6973 6974 if (cycles) 6975 timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer1); 6976 else 6977 timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0); 6978 6979 return timestamp; 6980 } 6981 6982 static const struct net_device_ops igc_netdev_ops = { 6983 .ndo_open = igc_open, 6984 .ndo_stop = igc_close, 6985 .ndo_start_xmit = igc_xmit_frame, 6986 .ndo_set_rx_mode = igc_set_rx_mode, 6987 .ndo_set_mac_address = igc_set_mac, 6988 .ndo_change_mtu = igc_change_mtu, 6989 .ndo_tx_timeout = igc_tx_timeout, 6990 .ndo_get_stats64 = igc_get_stats64, 6991 .ndo_fix_features = igc_fix_features, 6992 .ndo_set_features = igc_set_features, 6993 .ndo_features_check = igc_features_check, 6994 .ndo_setup_tc = igc_setup_tc, 6995 .ndo_bpf = igc_bpf, 6996 .ndo_xdp_xmit = igc_xdp_xmit, 6997 .ndo_xsk_wakeup = igc_xsk_wakeup, 6998 .ndo_get_tstamp = igc_get_tstamp, 6999 .ndo_hwtstamp_get = igc_ptp_hwtstamp_get, 7000 .ndo_hwtstamp_set = igc_ptp_hwtstamp_set, 7001 }; 7002 7003 u32 igc_rd32(struct igc_hw *hw, u32 reg) 7004 { 7005 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 7006 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 7007 u32 value = 0; 7008 7009 if (IGC_REMOVED(hw_addr)) 7010 return ~value; 7011 7012 value = readl(&hw_addr[reg]); 7013 7014 /* reads should not return all F's */ 7015 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 7016 struct net_device *netdev = igc->netdev; 7017 7018 hw->hw_addr = NULL; 7019 netif_device_detach(netdev); 7020 netdev_err(netdev, "PCIe link lost, device now detached\n"); 7021 WARN(pci_device_is_present(igc->pdev), 7022 "igc: Failed to read reg 0x%x!\n", reg); 7023 } 7024 7025 return value; 7026 } 7027 7028 /* Mapping HW RSS Type to enum xdp_rss_hash_type */ 7029 static enum xdp_rss_hash_type igc_xdp_rss_type[IGC_RSS_TYPE_MAX_TABLE] = { 7030 [IGC_RSS_TYPE_NO_HASH] = XDP_RSS_TYPE_L2, 7031 [IGC_RSS_TYPE_HASH_TCP_IPV4] = XDP_RSS_TYPE_L4_IPV4_TCP, 7032 [IGC_RSS_TYPE_HASH_IPV4] = XDP_RSS_TYPE_L3_IPV4, 7033 [IGC_RSS_TYPE_HASH_TCP_IPV6] = XDP_RSS_TYPE_L4_IPV6_TCP, 7034 [IGC_RSS_TYPE_HASH_IPV6_EX] = XDP_RSS_TYPE_L3_IPV6_EX, 7035 [IGC_RSS_TYPE_HASH_IPV6] = XDP_RSS_TYPE_L3_IPV6, 7036 [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX, 7037 [IGC_RSS_TYPE_HASH_UDP_IPV4] = XDP_RSS_TYPE_L4_IPV4_UDP, 7038 [IGC_RSS_TYPE_HASH_UDP_IPV6] = XDP_RSS_TYPE_L4_IPV6_UDP, 7039 [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX, 7040 [10] = XDP_RSS_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */ 7041 [11] = XDP_RSS_TYPE_NONE, /* keep array sized for SW bit-mask */ 7042 [12] = XDP_RSS_TYPE_NONE, /* to handle future HW revisions */ 7043 [13] = XDP_RSS_TYPE_NONE, 7044 [14] = XDP_RSS_TYPE_NONE, 7045 [15] = XDP_RSS_TYPE_NONE, 7046 }; 7047 7048 static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash, 7049 enum xdp_rss_hash_type *rss_type) 7050 { 7051 const struct igc_xdp_buff *ctx = (void *)_ctx; 7052 7053 if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH)) 7054 return -ENODATA; 7055 7056 *hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss); 7057 *rss_type = igc_xdp_rss_type[igc_rss_type(ctx->rx_desc)]; 7058 7059 return 0; 7060 } 7061 7062 static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7063 { 7064 const struct igc_xdp_buff *ctx = (void *)_ctx; 7065 struct igc_adapter *adapter = netdev_priv(ctx->xdp.rxq->dev); 7066 struct igc_inline_rx_tstamps *tstamp = ctx->rx_ts; 7067 7068 if (igc_test_staterr(ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) { 7069 *timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0); 7070 7071 return 0; 7072 } 7073 7074 return -ENODATA; 7075 } 7076 7077 static const struct xdp_metadata_ops igc_xdp_metadata_ops = { 7078 .xmo_rx_hash = igc_xdp_rx_hash, 7079 .xmo_rx_timestamp = igc_xdp_rx_timestamp, 7080 }; 7081 7082 static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer) 7083 { 7084 struct igc_adapter *adapter = container_of(timer, struct igc_adapter, 7085 hrtimer); 7086 unsigned long flags; 7087 unsigned int i; 7088 7089 spin_lock_irqsave(&adapter->qbv_tx_lock, flags); 7090 7091 adapter->qbv_transition = true; 7092 for (i = 0; i < adapter->num_tx_queues; i++) { 7093 struct igc_ring *tx_ring = adapter->tx_ring[i]; 7094 7095 if (tx_ring->admin_gate_closed) { 7096 tx_ring->admin_gate_closed = false; 7097 tx_ring->oper_gate_closed = true; 7098 } else { 7099 tx_ring->oper_gate_closed = false; 7100 } 7101 } 7102 adapter->qbv_transition = false; 7103 7104 spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags); 7105 7106 return HRTIMER_NORESTART; 7107 } 7108 7109 /** 7110 * igc_probe - Device Initialization Routine 7111 * @pdev: PCI device information struct 7112 * @ent: entry in igc_pci_tbl 7113 * 7114 * Returns 0 on success, negative on failure 7115 * 7116 * igc_probe initializes an adapter identified by a pci_dev structure. 7117 * The OS initialization, configuring the adapter private structure, 7118 * and a hardware reset occur. 7119 */ 7120 static int igc_probe(struct pci_dev *pdev, 7121 const struct pci_device_id *ent) 7122 { 7123 struct igc_adapter *adapter; 7124 struct net_device *netdev; 7125 struct igc_hw *hw; 7126 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 7127 int err; 7128 7129 err = pci_enable_device_mem(pdev); 7130 if (err) 7131 return err; 7132 7133 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 7134 if (err) { 7135 dev_err(&pdev->dev, 7136 "No usable DMA configuration, aborting\n"); 7137 goto err_dma; 7138 } 7139 7140 err = pci_request_mem_regions(pdev, igc_driver_name); 7141 if (err) 7142 goto err_pci_reg; 7143 7144 err = pci_enable_ptm(pdev); 7145 if (err < 0) 7146 dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n"); 7147 7148 pci_set_master(pdev); 7149 7150 err = -ENOMEM; 7151 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 7152 IGC_MAX_TX_QUEUES); 7153 7154 if (!netdev) 7155 goto err_alloc_etherdev; 7156 7157 SET_NETDEV_DEV(netdev, &pdev->dev); 7158 7159 pci_set_drvdata(pdev, netdev); 7160 adapter = netdev_priv(netdev); 7161 adapter->netdev = netdev; 7162 adapter->pdev = pdev; 7163 hw = &adapter->hw; 7164 hw->back = adapter; 7165 adapter->port_num = hw->bus.func; 7166 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7167 7168 /* PCI config space info */ 7169 hw->vendor_id = pdev->vendor; 7170 hw->device_id = pdev->device; 7171 hw->revision_id = pdev->revision; 7172 hw->subsystem_vendor_id = pdev->subsystem_vendor; 7173 hw->subsystem_device_id = pdev->subsystem_device; 7174 7175 /* Disable ASPM L1.2 on I226 devices to avoid packet loss */ 7176 if (igc_is_device_id_i226(hw)) 7177 pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); 7178 7179 err = pci_save_state(pdev); 7180 if (err) 7181 goto err_ioremap; 7182 7183 err = -EIO; 7184 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 7185 pci_resource_len(pdev, 0)); 7186 if (!adapter->io_addr) 7187 goto err_ioremap; 7188 7189 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 7190 hw->hw_addr = adapter->io_addr; 7191 7192 netdev->netdev_ops = &igc_netdev_ops; 7193 netdev->xdp_metadata_ops = &igc_xdp_metadata_ops; 7194 netdev->xsk_tx_metadata_ops = &igc_xsk_tx_metadata_ops; 7195 igc_ethtool_set_ops(netdev); 7196 netdev->watchdog_timeo = 5 * HZ; 7197 7198 netdev->mem_start = pci_resource_start(pdev, 0); 7199 netdev->mem_end = pci_resource_end(pdev, 0); 7200 7201 /* Copy the default MAC and PHY function pointers */ 7202 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 7203 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 7204 7205 /* Initialize skew-specific constants */ 7206 err = ei->get_invariants(hw); 7207 if (err) 7208 goto err_sw_init; 7209 7210 /* Add supported features to the features list*/ 7211 netdev->features |= NETIF_F_SG; 7212 netdev->features |= NETIF_F_TSO; 7213 netdev->features |= NETIF_F_TSO6; 7214 netdev->features |= NETIF_F_TSO_ECN; 7215 netdev->features |= NETIF_F_RXHASH; 7216 netdev->features |= NETIF_F_RXCSUM; 7217 netdev->features |= NETIF_F_HW_CSUM; 7218 netdev->features |= NETIF_F_SCTP_CRC; 7219 netdev->features |= NETIF_F_HW_TC; 7220 7221 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 7222 NETIF_F_GSO_GRE_CSUM | \ 7223 NETIF_F_GSO_IPXIP4 | \ 7224 NETIF_F_GSO_IPXIP6 | \ 7225 NETIF_F_GSO_UDP_TUNNEL | \ 7226 NETIF_F_GSO_UDP_TUNNEL_CSUM) 7227 7228 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 7229 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 7230 7231 /* setup the private structure */ 7232 err = igc_sw_init(adapter); 7233 if (err) 7234 goto err_sw_init; 7235 7236 /* copy netdev features into list of user selectable features */ 7237 netdev->hw_features |= NETIF_F_NTUPLE; 7238 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 7239 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7240 netdev->hw_features |= netdev->features; 7241 7242 netdev->features |= NETIF_F_HIGHDMA; 7243 7244 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 7245 netdev->mpls_features |= NETIF_F_HW_CSUM; 7246 netdev->hw_enc_features |= netdev->vlan_features; 7247 7248 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7249 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7250 7251 /* enable HW vlan tag insertion/stripping by default */ 7252 netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 7253 7254 /* MTU range: 68 - 9216 */ 7255 netdev->min_mtu = ETH_MIN_MTU; 7256 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 7257 7258 /* before reading the NVM, reset the controller to put the device in a 7259 * known good starting state 7260 */ 7261 hw->mac.ops.reset_hw(hw); 7262 7263 if (igc_get_flash_presence_i225(hw)) { 7264 if (hw->nvm.ops.validate(hw) < 0) { 7265 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 7266 err = -EIO; 7267 goto err_eeprom; 7268 } 7269 } 7270 7271 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 7272 /* copy the MAC address out of the NVM */ 7273 if (hw->mac.ops.read_mac_addr(hw)) 7274 dev_err(&pdev->dev, "NVM Read Error\n"); 7275 } 7276 7277 eth_hw_addr_set(netdev, hw->mac.addr); 7278 7279 if (!is_valid_ether_addr(netdev->dev_addr)) { 7280 dev_err(&pdev->dev, "Invalid MAC Address\n"); 7281 err = -EIO; 7282 goto err_eeprom; 7283 } 7284 7285 /* configure RXPBSIZE and TXPBSIZE */ 7286 wr32(IGC_RXPBS, IGC_RXPBSIZE_EXP_BMC_DEFAULT); 7287 wr32(IGC_TXPBS, IGC_TXPBSIZE_DEFAULT); 7288 7289 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 7290 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 7291 7292 INIT_WORK(&adapter->reset_task, igc_reset_task); 7293 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 7294 7295 hrtimer_setup(&adapter->hrtimer, &igc_qbv_scheduling_timer, CLOCK_MONOTONIC, 7296 HRTIMER_MODE_REL); 7297 7298 /* Initialize link properties that are user-changeable */ 7299 adapter->fc_autoneg = true; 7300 hw->phy.autoneg_advertised = 0xaf; 7301 hw->mac.autoneg_enabled = true; 7302 hw->fc.requested_mode = igc_fc_default; 7303 hw->fc.current_mode = igc_fc_default; 7304 7305 /* By default, support wake on port A */ 7306 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 7307 7308 /* initialize the wol settings based on the eeprom settings */ 7309 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 7310 adapter->wol |= IGC_WUFC_MAG; 7311 7312 device_set_wakeup_enable(&adapter->pdev->dev, 7313 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 7314 7315 igc_ptp_init(adapter); 7316 7317 igc_tsn_clear_schedule(adapter); 7318 7319 igc_fpe_init(adapter); 7320 7321 /* reset the hardware with the new settings */ 7322 igc_reset(adapter); 7323 7324 /* let the f/w know that the h/w is now under the control of the 7325 * driver. 7326 */ 7327 igc_get_hw_control(adapter); 7328 7329 strscpy(netdev->name, "eth%d", sizeof(netdev->name)); 7330 err = register_netdev(netdev); 7331 if (err) 7332 goto err_register; 7333 7334 /* carrier off reporting is important to ethtool even BEFORE open */ 7335 netif_carrier_off(netdev); 7336 7337 /* Check if Media Autosense is enabled */ 7338 adapter->ei = *ei; 7339 7340 /* print pcie link status and MAC address */ 7341 pcie_print_link_status(pdev); 7342 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 7343 7344 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 7345 /* Disable EEE for internal PHY devices */ 7346 hw->dev_spec._base.eee_enable = false; 7347 adapter->flags &= ~IGC_FLAG_EEE; 7348 igc_set_eee_i225(hw, false, false, false); 7349 7350 pm_runtime_put_noidle(&pdev->dev); 7351 7352 if (IS_ENABLED(CONFIG_IGC_LEDS)) { 7353 err = igc_led_setup(adapter); 7354 if (err) { 7355 netdev_warn_once(netdev, 7356 "LED init failed (%d); continuing without LED support\n", 7357 err); 7358 adapter->leds_available = false; 7359 } else { 7360 adapter->leds_available = true; 7361 } 7362 } 7363 7364 return 0; 7365 7366 err_register: 7367 igc_release_hw_control(adapter); 7368 igc_ptp_stop(adapter); 7369 err_eeprom: 7370 if (!igc_check_reset_block(hw)) 7371 igc_reset_phy(hw); 7372 err_sw_init: 7373 igc_clear_interrupt_scheme(adapter); 7374 iounmap(adapter->io_addr); 7375 err_ioremap: 7376 free_netdev(netdev); 7377 err_alloc_etherdev: 7378 pci_release_mem_regions(pdev); 7379 err_pci_reg: 7380 err_dma: 7381 pci_disable_device(pdev); 7382 return err; 7383 } 7384 7385 /** 7386 * igc_remove - Device Removal Routine 7387 * @pdev: PCI device information struct 7388 * 7389 * igc_remove is called by the PCI subsystem to alert the driver 7390 * that it should release a PCI device. This could be caused by a 7391 * Hot-Plug event, or because the driver is going to be removed from 7392 * memory. 7393 */ 7394 static void igc_remove(struct pci_dev *pdev) 7395 { 7396 struct net_device *netdev = pci_get_drvdata(pdev); 7397 struct igc_adapter *adapter = netdev_priv(netdev); 7398 7399 pm_runtime_get_noresume(&pdev->dev); 7400 7401 igc_flush_nfc_rules(adapter); 7402 7403 igc_ptp_stop(adapter); 7404 7405 pci_disable_ptm(pdev); 7406 pci_clear_master(pdev); 7407 7408 set_bit(__IGC_DOWN, &adapter->state); 7409 7410 timer_delete_sync(&adapter->watchdog_timer); 7411 timer_delete_sync(&adapter->phy_info_timer); 7412 7413 cancel_work_sync(&adapter->reset_task); 7414 cancel_work_sync(&adapter->watchdog_task); 7415 hrtimer_cancel(&adapter->hrtimer); 7416 7417 if (IS_ENABLED(CONFIG_IGC_LEDS) && adapter->leds_available) 7418 igc_led_free(adapter); 7419 7420 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7421 * would have already happened in close and is redundant. 7422 */ 7423 igc_release_hw_control(adapter); 7424 unregister_netdev(netdev); 7425 7426 igc_clear_interrupt_scheme(adapter); 7427 pci_iounmap(pdev, adapter->io_addr); 7428 pci_release_mem_regions(pdev); 7429 7430 free_netdev(netdev); 7431 7432 pci_disable_device(pdev); 7433 } 7434 7435 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 7436 bool runtime) 7437 { 7438 struct net_device *netdev = pci_get_drvdata(pdev); 7439 struct igc_adapter *adapter = netdev_priv(netdev); 7440 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 7441 struct igc_hw *hw = &adapter->hw; 7442 u32 ctrl, rctl, status; 7443 bool wake; 7444 7445 rtnl_lock(); 7446 netif_device_detach(netdev); 7447 7448 if (netif_running(netdev)) 7449 __igc_close(netdev, true); 7450 7451 igc_ptp_suspend(adapter); 7452 7453 igc_clear_interrupt_scheme(adapter); 7454 rtnl_unlock(); 7455 7456 status = rd32(IGC_STATUS); 7457 if (status & IGC_STATUS_LU) 7458 wufc &= ~IGC_WUFC_LNKC; 7459 7460 if (wufc) { 7461 igc_setup_rctl(adapter); 7462 igc_set_rx_mode(netdev); 7463 7464 /* turn on all-multi mode if wake on multicast is enabled */ 7465 if (wufc & IGC_WUFC_MC) { 7466 rctl = rd32(IGC_RCTL); 7467 rctl |= IGC_RCTL_MPE; 7468 wr32(IGC_RCTL, rctl); 7469 } 7470 7471 ctrl = rd32(IGC_CTRL); 7472 ctrl |= IGC_CTRL_ADVD3WUC; 7473 wr32(IGC_CTRL, ctrl); 7474 7475 /* Allow time for pending master requests to run */ 7476 igc_disable_pcie_master(hw); 7477 7478 wr32(IGC_WUC, IGC_WUC_PME_EN); 7479 wr32(IGC_WUFC, wufc); 7480 } else { 7481 wr32(IGC_WUC, 0); 7482 wr32(IGC_WUFC, 0); 7483 } 7484 7485 wake = wufc || adapter->en_mng_pt; 7486 if (!wake) 7487 igc_power_down_phy_copper_base(&adapter->hw); 7488 else 7489 igc_power_up_link(adapter); 7490 7491 if (enable_wake) 7492 *enable_wake = wake; 7493 7494 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7495 * would have already happened in close and is redundant. 7496 */ 7497 igc_release_hw_control(adapter); 7498 7499 pci_disable_device(pdev); 7500 7501 return 0; 7502 } 7503 7504 static int igc_runtime_suspend(struct device *dev) 7505 { 7506 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 7507 } 7508 7509 static void igc_deliver_wake_packet(struct net_device *netdev) 7510 { 7511 struct igc_adapter *adapter = netdev_priv(netdev); 7512 struct igc_hw *hw = &adapter->hw; 7513 struct sk_buff *skb; 7514 u32 wupl; 7515 7516 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 7517 7518 /* WUPM stores only the first 128 bytes of the wake packet. 7519 * Read the packet only if we have the whole thing. 7520 */ 7521 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 7522 return; 7523 7524 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 7525 if (!skb) 7526 return; 7527 7528 skb_put(skb, wupl); 7529 7530 /* Ensure reads are 32-bit aligned */ 7531 wupl = roundup(wupl, 4); 7532 7533 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 7534 7535 skb->protocol = eth_type_trans(skb, netdev); 7536 netif_rx(skb); 7537 } 7538 7539 static int __igc_resume(struct device *dev, bool rpm) 7540 { 7541 struct pci_dev *pdev = to_pci_dev(dev); 7542 struct net_device *netdev = pci_get_drvdata(pdev); 7543 struct igc_adapter *adapter = netdev_priv(netdev); 7544 struct igc_hw *hw = &adapter->hw; 7545 u32 err, val; 7546 7547 pci_set_power_state(pdev, PCI_D0); 7548 pci_restore_state(pdev); 7549 7550 if (!pci_device_is_present(pdev)) 7551 return -ENODEV; 7552 err = pci_enable_device_mem(pdev); 7553 if (err) { 7554 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 7555 return err; 7556 } 7557 pci_set_master(pdev); 7558 7559 pci_enable_wake(pdev, PCI_D3hot, 0); 7560 pci_enable_wake(pdev, PCI_D3cold, 0); 7561 7562 if (igc_is_device_id_i226(hw)) 7563 pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); 7564 7565 if (igc_init_interrupt_scheme(adapter, true)) { 7566 netdev_err(netdev, "Unable to allocate memory for queues\n"); 7567 return -ENOMEM; 7568 } 7569 7570 igc_reset(adapter); 7571 7572 /* let the f/w know that the h/w is now under the control of the 7573 * driver. 7574 */ 7575 igc_get_hw_control(adapter); 7576 7577 val = rd32(IGC_WUS); 7578 if (val & WAKE_PKT_WUS) 7579 igc_deliver_wake_packet(netdev); 7580 7581 wr32(IGC_WUS, ~0); 7582 7583 if (netif_running(netdev)) { 7584 if (!rpm) 7585 rtnl_lock(); 7586 err = __igc_open(netdev, true); 7587 if (!rpm) 7588 rtnl_unlock(); 7589 if (err) 7590 return err; 7591 } 7592 7593 netif_device_attach(netdev); 7594 7595 return 0; 7596 } 7597 7598 static int igc_resume(struct device *dev) 7599 { 7600 return __igc_resume(dev, false); 7601 } 7602 7603 static int igc_runtime_resume(struct device *dev) 7604 { 7605 return __igc_resume(dev, true); 7606 } 7607 7608 static int igc_suspend(struct device *dev) 7609 { 7610 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 7611 } 7612 7613 static int __maybe_unused igc_runtime_idle(struct device *dev) 7614 { 7615 struct net_device *netdev = dev_get_drvdata(dev); 7616 struct igc_adapter *adapter = netdev_priv(netdev); 7617 7618 if (!igc_has_link(adapter)) 7619 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 7620 7621 return -EBUSY; 7622 } 7623 7624 static void igc_shutdown(struct pci_dev *pdev) 7625 { 7626 bool wake; 7627 7628 __igc_shutdown(pdev, &wake, 0); 7629 7630 if (system_state == SYSTEM_POWER_OFF) { 7631 pci_wake_from_d3(pdev, wake); 7632 pci_set_power_state(pdev, PCI_D3hot); 7633 } 7634 } 7635 7636 /** 7637 * igc_io_error_detected - called when PCI error is detected 7638 * @pdev: Pointer to PCI device 7639 * @state: The current PCI connection state 7640 * 7641 * This function is called after a PCI bus error affecting 7642 * this device has been detected. 7643 **/ 7644 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 7645 pci_channel_state_t state) 7646 { 7647 struct net_device *netdev = pci_get_drvdata(pdev); 7648 struct igc_adapter *adapter = netdev_priv(netdev); 7649 7650 rtnl_lock(); 7651 netif_device_detach(netdev); 7652 7653 if (state == pci_channel_io_perm_failure) { 7654 rtnl_unlock(); 7655 return PCI_ERS_RESULT_DISCONNECT; 7656 } 7657 7658 if (netif_running(netdev)) 7659 igc_down(adapter); 7660 pci_disable_device(pdev); 7661 rtnl_unlock(); 7662 7663 /* Request a slot reset. */ 7664 return PCI_ERS_RESULT_NEED_RESET; 7665 } 7666 7667 /** 7668 * igc_io_slot_reset - called after the PCI bus has been reset. 7669 * @pdev: Pointer to PCI device 7670 * 7671 * Restart the card from scratch, as if from a cold-boot. Implementation 7672 * resembles the first-half of the __igc_resume routine. 7673 **/ 7674 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 7675 { 7676 struct net_device *netdev = pci_get_drvdata(pdev); 7677 struct igc_adapter *adapter = netdev_priv(netdev); 7678 struct igc_hw *hw = &adapter->hw; 7679 pci_ers_result_t result; 7680 7681 if (pci_enable_device_mem(pdev)) { 7682 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 7683 result = PCI_ERS_RESULT_DISCONNECT; 7684 } else { 7685 pci_set_master(pdev); 7686 pci_restore_state(pdev); 7687 7688 pci_enable_wake(pdev, PCI_D3hot, 0); 7689 pci_enable_wake(pdev, PCI_D3cold, 0); 7690 7691 if (igc_is_device_id_i226(hw)) 7692 pci_disable_link_state_locked(pdev, PCIE_LINK_STATE_L1_2); 7693 7694 /* In case of PCI error, adapter loses its HW address 7695 * so we should re-assign it here. 7696 */ 7697 hw->hw_addr = adapter->io_addr; 7698 7699 igc_reset(adapter); 7700 wr32(IGC_WUS, ~0); 7701 result = PCI_ERS_RESULT_RECOVERED; 7702 } 7703 7704 return result; 7705 } 7706 7707 /** 7708 * igc_io_resume - called when traffic can start to flow again. 7709 * @pdev: Pointer to PCI device 7710 * 7711 * This callback is called when the error recovery driver tells us that 7712 * its OK to resume normal operation. Implementation resembles the 7713 * second-half of the __igc_resume routine. 7714 */ 7715 static void igc_io_resume(struct pci_dev *pdev) 7716 { 7717 struct net_device *netdev = pci_get_drvdata(pdev); 7718 struct igc_adapter *adapter = netdev_priv(netdev); 7719 7720 rtnl_lock(); 7721 if (netif_running(netdev)) { 7722 if (igc_open(netdev)) { 7723 rtnl_unlock(); 7724 netdev_err(netdev, "igc_open failed after reset\n"); 7725 return; 7726 } 7727 } 7728 7729 netif_device_attach(netdev); 7730 7731 /* let the f/w know that the h/w is now under the control of the 7732 * driver. 7733 */ 7734 igc_get_hw_control(adapter); 7735 rtnl_unlock(); 7736 } 7737 7738 static const struct pci_error_handlers igc_err_handler = { 7739 .error_detected = igc_io_error_detected, 7740 .slot_reset = igc_io_slot_reset, 7741 .resume = igc_io_resume, 7742 }; 7743 7744 static _DEFINE_DEV_PM_OPS(igc_pm_ops, igc_suspend, igc_resume, 7745 igc_runtime_suspend, igc_runtime_resume, 7746 igc_runtime_idle); 7747 7748 static struct pci_driver igc_driver = { 7749 .name = igc_driver_name, 7750 .id_table = igc_pci_tbl, 7751 .probe = igc_probe, 7752 .remove = igc_remove, 7753 .driver.pm = pm_ptr(&igc_pm_ops), 7754 .shutdown = igc_shutdown, 7755 .err_handler = &igc_err_handler, 7756 }; 7757 7758 /** 7759 * igc_reinit_queues - return error 7760 * @adapter: pointer to adapter structure 7761 */ 7762 int igc_reinit_queues(struct igc_adapter *adapter) 7763 { 7764 struct net_device *netdev = adapter->netdev; 7765 int err = 0; 7766 7767 if (netif_running(netdev)) 7768 igc_close(netdev); 7769 7770 igc_reset_interrupt_capability(adapter); 7771 7772 if (igc_init_interrupt_scheme(adapter, true)) { 7773 netdev_err(netdev, "Unable to allocate memory for queues\n"); 7774 return -ENOMEM; 7775 } 7776 7777 if (netif_running(netdev)) 7778 err = igc_open(netdev); 7779 7780 if (!err) { 7781 /* Restore default IEEE 802.1Qbv schedule after queue reinit */ 7782 igc_tsn_clear_schedule(adapter); 7783 } 7784 7785 return err; 7786 } 7787 7788 /** 7789 * igc_get_hw_dev - return device 7790 * @hw: pointer to hardware structure 7791 * 7792 * used by hardware layer to print debugging information 7793 */ 7794 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 7795 { 7796 struct igc_adapter *adapter = hw->back; 7797 7798 return adapter->netdev; 7799 } 7800 7801 static void igc_disable_rx_ring_hw(struct igc_ring *ring) 7802 { 7803 struct igc_hw *hw = &ring->q_vector->adapter->hw; 7804 u8 idx = ring->reg_idx; 7805 u32 rxdctl; 7806 7807 rxdctl = rd32(IGC_RXDCTL(idx)); 7808 rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE; 7809 rxdctl |= IGC_RXDCTL_SWFLUSH; 7810 wr32(IGC_RXDCTL(idx), rxdctl); 7811 } 7812 7813 void igc_disable_rx_ring(struct igc_ring *ring) 7814 { 7815 igc_disable_rx_ring_hw(ring); 7816 igc_clean_rx_ring(ring); 7817 } 7818 7819 void igc_enable_rx_ring(struct igc_ring *ring) 7820 { 7821 struct igc_adapter *adapter = ring->q_vector->adapter; 7822 7823 igc_configure_rx_ring(adapter, ring); 7824 7825 if (ring->xsk_pool) 7826 igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); 7827 else 7828 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 7829 } 7830 7831 void igc_disable_tx_ring(struct igc_ring *ring) 7832 { 7833 igc_disable_tx_ring_hw(ring); 7834 igc_clean_tx_ring(ring); 7835 } 7836 7837 void igc_enable_tx_ring(struct igc_ring *ring) 7838 { 7839 struct igc_adapter *adapter = ring->q_vector->adapter; 7840 7841 igc_configure_tx_ring(adapter, ring); 7842 } 7843 7844 /** 7845 * igc_init_module - Driver Registration Routine 7846 * 7847 * igc_init_module is the first routine called when the driver is 7848 * loaded. All it does is register with the PCI subsystem. 7849 */ 7850 static int __init igc_init_module(void) 7851 { 7852 int ret; 7853 7854 pr_info("%s\n", igc_driver_string); 7855 pr_info("%s\n", igc_copyright); 7856 7857 ret = pci_register_driver(&igc_driver); 7858 return ret; 7859 } 7860 7861 module_init(igc_init_module); 7862 7863 /** 7864 * igc_exit_module - Driver Exit Cleanup Routine 7865 * 7866 * igc_exit_module is called just before the driver is removed 7867 * from memory. 7868 */ 7869 static void __exit igc_exit_module(void) 7870 { 7871 pci_unregister_driver(&igc_driver); 7872 } 7873 7874 module_exit(igc_exit_module); 7875 /* igc_main.c */ 7876