1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #include <linux/module.h> 5 #include <linux/types.h> 6 #include <linux/if_vlan.h> 7 #include <linux/tcp.h> 8 #include <linux/udp.h> 9 #include <linux/ip.h> 10 #include <linux/pm_runtime.h> 11 #include <net/pkt_sched.h> 12 #include <linux/bpf_trace.h> 13 #include <net/xdp_sock_drv.h> 14 #include <linux/pci.h> 15 #include <linux/mdio.h> 16 17 #include <net/ipv6.h> 18 19 #include "igc.h" 20 #include "igc_hw.h" 21 #include "igc_tsn.h" 22 #include "igc_xdp.h" 23 24 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" 25 26 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 27 28 #define IGC_XDP_PASS 0 29 #define IGC_XDP_CONSUMED BIT(0) 30 #define IGC_XDP_TX BIT(1) 31 #define IGC_XDP_REDIRECT BIT(2) 32 33 static int debug = -1; 34 35 MODULE_DESCRIPTION(DRV_SUMMARY); 36 MODULE_LICENSE("GPL v2"); 37 module_param(debug, int, 0); 38 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 39 40 char igc_driver_name[] = "igc"; 41 static const char igc_driver_string[] = DRV_SUMMARY; 42 static const char igc_copyright[] = 43 "Copyright(c) 2018 Intel Corporation."; 44 45 static const struct igc_info *igc_info_tbl[] = { 46 [board_base] = &igc_base_info, 47 }; 48 49 static const struct pci_device_id igc_pci_tbl[] = { 50 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), .driver_data = board_base }, 51 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), .driver_data = board_base }, 52 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), .driver_data = board_base }, 53 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), .driver_data = board_base }, 54 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), .driver_data = board_base }, 55 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), .driver_data = board_base }, 56 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), .driver_data = board_base }, 57 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), .driver_data = board_base }, 58 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), .driver_data = board_base }, 59 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), .driver_data = board_base }, 60 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), .driver_data = board_base }, 61 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), .driver_data = board_base }, 62 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), .driver_data = board_base }, 63 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), .driver_data = board_base }, 64 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), .driver_data = board_base }, 65 { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), .driver_data = board_base }, 66 /* required last entry */ 67 { } 68 }; 69 70 MODULE_DEVICE_TABLE(pci, igc_pci_tbl); 71 72 enum latency_range { 73 lowest_latency = 0, 74 low_latency = 1, 75 bulk_latency = 2, 76 latency_invalid = 255 77 }; 78 79 void igc_reset(struct igc_adapter *adapter) 80 { 81 struct net_device *dev = adapter->netdev; 82 struct igc_hw *hw = &adapter->hw; 83 struct igc_fc_info *fc = &hw->fc; 84 u32 pba, hwm; 85 86 /* Repartition PBA for greater than 9k MTU if required */ 87 pba = IGC_PBA_34K; 88 89 /* flow control settings 90 * The high water mark must be low enough to fit one full frame 91 * after transmitting the pause frame. As such we must have enough 92 * space to allow for us to complete our current transmit and then 93 * receive the frame that is in progress from the link partner. 94 * Set it to: 95 * - the full Rx FIFO size minus one full Tx plus one full Rx frame 96 */ 97 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); 98 99 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ 100 fc->low_water = fc->high_water - 16; 101 fc->pause_time = 0xFFFF; 102 fc->send_xon = 1; 103 fc->current_mode = fc->requested_mode; 104 105 hw->mac.ops.reset_hw(hw); 106 107 if (hw->mac.ops.init_hw(hw)) 108 netdev_err(dev, "Error on hardware initialization\n"); 109 110 /* Re-establish EEE setting */ 111 igc_set_eee_i225(hw, true, true, true); 112 113 if (!netif_running(adapter->netdev)) 114 igc_power_down_phy_copper_base(&adapter->hw); 115 116 /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */ 117 wr32(IGC_VET, ETH_P_8021Q); 118 119 /* Re-enable PTP, where applicable. */ 120 igc_ptp_reset(adapter); 121 122 /* Re-enable TSN offloading, where applicable. */ 123 igc_tsn_reset(adapter); 124 125 igc_get_phy_info(hw); 126 } 127 128 /** 129 * igc_power_up_link - Power up the phy link 130 * @adapter: address of board private structure 131 */ 132 static void igc_power_up_link(struct igc_adapter *adapter) 133 { 134 igc_reset_phy(&adapter->hw); 135 136 igc_power_up_phy_copper(&adapter->hw); 137 138 igc_setup_link(&adapter->hw); 139 } 140 141 /** 142 * igc_release_hw_control - release control of the h/w to f/w 143 * @adapter: address of board private structure 144 * 145 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 146 * For ASF and Pass Through versions of f/w this means that the 147 * driver is no longer loaded. 148 */ 149 static void igc_release_hw_control(struct igc_adapter *adapter) 150 { 151 struct igc_hw *hw = &adapter->hw; 152 u32 ctrl_ext; 153 154 if (!pci_device_is_present(adapter->pdev)) 155 return; 156 157 /* Let firmware take over control of h/w */ 158 ctrl_ext = rd32(IGC_CTRL_EXT); 159 wr32(IGC_CTRL_EXT, 160 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 161 } 162 163 /** 164 * igc_get_hw_control - get control of the h/w from f/w 165 * @adapter: address of board private structure 166 * 167 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 168 * For ASF and Pass Through versions of f/w this means that 169 * the driver is loaded. 170 */ 171 static void igc_get_hw_control(struct igc_adapter *adapter) 172 { 173 struct igc_hw *hw = &adapter->hw; 174 u32 ctrl_ext; 175 176 /* Let firmware know the driver has taken over */ 177 ctrl_ext = rd32(IGC_CTRL_EXT); 178 wr32(IGC_CTRL_EXT, 179 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 180 } 181 182 static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf) 183 { 184 dma_unmap_single(dev, dma_unmap_addr(buf, dma), 185 dma_unmap_len(buf, len), DMA_TO_DEVICE); 186 187 dma_unmap_len_set(buf, len, 0); 188 } 189 190 /** 191 * igc_clean_tx_ring - Free Tx Buffers 192 * @tx_ring: ring to be cleaned 193 */ 194 static void igc_clean_tx_ring(struct igc_ring *tx_ring) 195 { 196 u16 i = tx_ring->next_to_clean; 197 struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; 198 u32 xsk_frames = 0; 199 200 while (i != tx_ring->next_to_use) { 201 union igc_adv_tx_desc *eop_desc, *tx_desc; 202 203 switch (tx_buffer->type) { 204 case IGC_TX_BUFFER_TYPE_XSK: 205 xsk_frames++; 206 break; 207 case IGC_TX_BUFFER_TYPE_XDP: 208 xdp_return_frame(tx_buffer->xdpf); 209 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 210 break; 211 case IGC_TX_BUFFER_TYPE_SKB: 212 dev_kfree_skb_any(tx_buffer->skb); 213 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 214 break; 215 default: 216 netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); 217 break; 218 } 219 220 /* check for eop_desc to determine the end of the packet */ 221 eop_desc = tx_buffer->next_to_watch; 222 tx_desc = IGC_TX_DESC(tx_ring, i); 223 224 /* unmap remaining buffers */ 225 while (tx_desc != eop_desc) { 226 tx_buffer++; 227 tx_desc++; 228 i++; 229 if (unlikely(i == tx_ring->count)) { 230 i = 0; 231 tx_buffer = tx_ring->tx_buffer_info; 232 tx_desc = IGC_TX_DESC(tx_ring, 0); 233 } 234 235 /* unmap any remaining paged data */ 236 if (dma_unmap_len(tx_buffer, len)) 237 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 238 } 239 240 tx_buffer->next_to_watch = NULL; 241 242 /* move us one more past the eop_desc for start of next pkt */ 243 tx_buffer++; 244 i++; 245 if (unlikely(i == tx_ring->count)) { 246 i = 0; 247 tx_buffer = tx_ring->tx_buffer_info; 248 } 249 } 250 251 if (tx_ring->xsk_pool && xsk_frames) 252 xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); 253 254 /* reset BQL for queue */ 255 netdev_tx_reset_queue(txring_txq(tx_ring)); 256 257 /* Zero out the buffer ring */ 258 memset(tx_ring->tx_buffer_info, 0, 259 sizeof(*tx_ring->tx_buffer_info) * tx_ring->count); 260 261 /* Zero out the descriptor ring */ 262 memset(tx_ring->desc, 0, tx_ring->size); 263 264 /* reset next_to_use and next_to_clean */ 265 tx_ring->next_to_use = 0; 266 tx_ring->next_to_clean = 0; 267 268 /* Clear any lingering XSK TX timestamp requests */ 269 if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) { 270 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 271 272 igc_ptp_clear_xsk_tx_tstamp_queue(adapter, tx_ring->queue_index); 273 } 274 } 275 276 /** 277 * igc_free_tx_resources - Free Tx Resources per Queue 278 * @tx_ring: Tx descriptor ring for a specific queue 279 * 280 * Free all transmit software resources 281 */ 282 void igc_free_tx_resources(struct igc_ring *tx_ring) 283 { 284 igc_disable_tx_ring(tx_ring); 285 286 vfree(tx_ring->tx_buffer_info); 287 tx_ring->tx_buffer_info = NULL; 288 289 /* if not set, then don't free */ 290 if (!tx_ring->desc) 291 return; 292 293 dma_free_coherent(tx_ring->dev, tx_ring->size, 294 tx_ring->desc, tx_ring->dma); 295 296 tx_ring->desc = NULL; 297 } 298 299 /** 300 * igc_free_all_tx_resources - Free Tx Resources for All Queues 301 * @adapter: board private structure 302 * 303 * Free all transmit software resources 304 */ 305 static void igc_free_all_tx_resources(struct igc_adapter *adapter) 306 { 307 int i; 308 309 for (i = 0; i < adapter->num_tx_queues; i++) 310 igc_free_tx_resources(adapter->tx_ring[i]); 311 } 312 313 /** 314 * igc_clean_all_tx_rings - Free Tx Buffers for all queues 315 * @adapter: board private structure 316 */ 317 static void igc_clean_all_tx_rings(struct igc_adapter *adapter) 318 { 319 int i; 320 321 for (i = 0; i < adapter->num_tx_queues; i++) 322 if (adapter->tx_ring[i]) 323 igc_clean_tx_ring(adapter->tx_ring[i]); 324 } 325 326 static void igc_disable_tx_ring_hw(struct igc_ring *ring) 327 { 328 struct igc_hw *hw = &ring->q_vector->adapter->hw; 329 u8 idx = ring->reg_idx; 330 u32 txdctl; 331 332 txdctl = rd32(IGC_TXDCTL(idx)); 333 txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE; 334 txdctl |= IGC_TXDCTL_SWFLUSH; 335 wr32(IGC_TXDCTL(idx), txdctl); 336 } 337 338 /** 339 * igc_disable_all_tx_rings_hw - Disable all transmit queue operation 340 * @adapter: board private structure 341 */ 342 static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter) 343 { 344 int i; 345 346 for (i = 0; i < adapter->num_tx_queues; i++) { 347 struct igc_ring *tx_ring = adapter->tx_ring[i]; 348 349 igc_disable_tx_ring_hw(tx_ring); 350 } 351 } 352 353 /** 354 * igc_setup_tx_resources - allocate Tx resources (Descriptors) 355 * @tx_ring: tx descriptor ring (for a specific queue) to setup 356 * 357 * Return 0 on success, negative on failure 358 */ 359 int igc_setup_tx_resources(struct igc_ring *tx_ring) 360 { 361 struct net_device *ndev = tx_ring->netdev; 362 struct device *dev = tx_ring->dev; 363 int size = 0; 364 365 size = sizeof(struct igc_tx_buffer) * tx_ring->count; 366 tx_ring->tx_buffer_info = vzalloc(size); 367 if (!tx_ring->tx_buffer_info) 368 goto err; 369 370 /* round up to nearest 4K */ 371 tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); 372 tx_ring->size = ALIGN(tx_ring->size, 4096); 373 374 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 375 &tx_ring->dma, GFP_KERNEL); 376 377 if (!tx_ring->desc) 378 goto err; 379 380 tx_ring->next_to_use = 0; 381 tx_ring->next_to_clean = 0; 382 383 return 0; 384 385 err: 386 vfree(tx_ring->tx_buffer_info); 387 netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n"); 388 return -ENOMEM; 389 } 390 391 /** 392 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues 393 * @adapter: board private structure 394 * 395 * Return 0 on success, negative on failure 396 */ 397 static int igc_setup_all_tx_resources(struct igc_adapter *adapter) 398 { 399 struct net_device *dev = adapter->netdev; 400 int i, err = 0; 401 402 for (i = 0; i < adapter->num_tx_queues; i++) { 403 err = igc_setup_tx_resources(adapter->tx_ring[i]); 404 if (err) { 405 netdev_err(dev, "Error on Tx queue %u setup\n", i); 406 for (i--; i >= 0; i--) 407 igc_free_tx_resources(adapter->tx_ring[i]); 408 break; 409 } 410 } 411 412 return err; 413 } 414 415 static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring) 416 { 417 u16 i = rx_ring->next_to_clean; 418 419 dev_kfree_skb(rx_ring->skb); 420 rx_ring->skb = NULL; 421 422 /* Free all the Rx ring sk_buffs */ 423 while (i != rx_ring->next_to_alloc) { 424 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; 425 426 /* Invalidate cache lines that may have been written to by 427 * device so that we avoid corrupting memory. 428 */ 429 dma_sync_single_range_for_cpu(rx_ring->dev, 430 buffer_info->dma, 431 buffer_info->page_offset, 432 igc_rx_bufsz(rx_ring), 433 DMA_FROM_DEVICE); 434 435 /* free resources associated with mapping */ 436 dma_unmap_page_attrs(rx_ring->dev, 437 buffer_info->dma, 438 igc_rx_pg_size(rx_ring), 439 DMA_FROM_DEVICE, 440 IGC_RX_DMA_ATTR); 441 __page_frag_cache_drain(buffer_info->page, 442 buffer_info->pagecnt_bias); 443 444 i++; 445 if (i == rx_ring->count) 446 i = 0; 447 } 448 } 449 450 static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring) 451 { 452 struct igc_rx_buffer *bi; 453 u16 i; 454 455 for (i = 0; i < ring->count; i++) { 456 bi = &ring->rx_buffer_info[i]; 457 if (!bi->xdp) 458 continue; 459 460 xsk_buff_free(bi->xdp); 461 bi->xdp = NULL; 462 } 463 } 464 465 /** 466 * igc_clean_rx_ring - Free Rx Buffers per Queue 467 * @ring: ring to free buffers from 468 */ 469 static void igc_clean_rx_ring(struct igc_ring *ring) 470 { 471 if (ring->xsk_pool) 472 igc_clean_rx_ring_xsk_pool(ring); 473 else 474 igc_clean_rx_ring_page_shared(ring); 475 476 clear_ring_uses_large_buffer(ring); 477 478 ring->next_to_alloc = 0; 479 ring->next_to_clean = 0; 480 ring->next_to_use = 0; 481 } 482 483 /** 484 * igc_clean_all_rx_rings - Free Rx Buffers for all queues 485 * @adapter: board private structure 486 */ 487 static void igc_clean_all_rx_rings(struct igc_adapter *adapter) 488 { 489 int i; 490 491 for (i = 0; i < adapter->num_rx_queues; i++) 492 if (adapter->rx_ring[i]) 493 igc_clean_rx_ring(adapter->rx_ring[i]); 494 } 495 496 /** 497 * igc_free_rx_resources - Free Rx Resources 498 * @rx_ring: ring to clean the resources from 499 * 500 * Free all receive software resources 501 */ 502 void igc_free_rx_resources(struct igc_ring *rx_ring) 503 { 504 igc_clean_rx_ring(rx_ring); 505 506 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 507 508 vfree(rx_ring->rx_buffer_info); 509 rx_ring->rx_buffer_info = NULL; 510 511 /* if not set, then don't free */ 512 if (!rx_ring->desc) 513 return; 514 515 dma_free_coherent(rx_ring->dev, rx_ring->size, 516 rx_ring->desc, rx_ring->dma); 517 518 rx_ring->desc = NULL; 519 } 520 521 /** 522 * igc_free_all_rx_resources - Free Rx Resources for All Queues 523 * @adapter: board private structure 524 * 525 * Free all receive software resources 526 */ 527 static void igc_free_all_rx_resources(struct igc_adapter *adapter) 528 { 529 int i; 530 531 for (i = 0; i < adapter->num_rx_queues; i++) 532 igc_free_rx_resources(adapter->rx_ring[i]); 533 } 534 535 /** 536 * igc_setup_rx_resources - allocate Rx resources (Descriptors) 537 * @rx_ring: rx descriptor ring (for a specific queue) to setup 538 * 539 * Returns 0 on success, negative on failure 540 */ 541 int igc_setup_rx_resources(struct igc_ring *rx_ring) 542 { 543 struct net_device *ndev = rx_ring->netdev; 544 struct device *dev = rx_ring->dev; 545 u8 index = rx_ring->queue_index; 546 int size, desc_len, res; 547 548 /* XDP RX-queue info */ 549 if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) 550 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 551 res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index, 552 rx_ring->q_vector->napi.napi_id); 553 if (res < 0) { 554 netdev_err(ndev, "Failed to register xdp_rxq index %u\n", 555 index); 556 return res; 557 } 558 559 size = sizeof(struct igc_rx_buffer) * rx_ring->count; 560 rx_ring->rx_buffer_info = vzalloc(size); 561 if (!rx_ring->rx_buffer_info) 562 goto err; 563 564 desc_len = sizeof(union igc_adv_rx_desc); 565 566 /* Round up to nearest 4K */ 567 rx_ring->size = rx_ring->count * desc_len; 568 rx_ring->size = ALIGN(rx_ring->size, 4096); 569 570 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 571 &rx_ring->dma, GFP_KERNEL); 572 573 if (!rx_ring->desc) 574 goto err; 575 576 rx_ring->next_to_alloc = 0; 577 rx_ring->next_to_clean = 0; 578 rx_ring->next_to_use = 0; 579 580 return 0; 581 582 err: 583 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 584 vfree(rx_ring->rx_buffer_info); 585 rx_ring->rx_buffer_info = NULL; 586 netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); 587 return -ENOMEM; 588 } 589 590 /** 591 * igc_setup_all_rx_resources - wrapper to allocate Rx resources 592 * (Descriptors) for all queues 593 * @adapter: board private structure 594 * 595 * Return 0 on success, negative on failure 596 */ 597 static int igc_setup_all_rx_resources(struct igc_adapter *adapter) 598 { 599 struct net_device *dev = adapter->netdev; 600 int i, err = 0; 601 602 for (i = 0; i < adapter->num_rx_queues; i++) { 603 err = igc_setup_rx_resources(adapter->rx_ring[i]); 604 if (err) { 605 netdev_err(dev, "Error on Rx queue %u setup\n", i); 606 for (i--; i >= 0; i--) 607 igc_free_rx_resources(adapter->rx_ring[i]); 608 break; 609 } 610 } 611 612 return err; 613 } 614 615 static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter, 616 struct igc_ring *ring) 617 { 618 if (!igc_xdp_is_enabled(adapter) || 619 !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags)) 620 return NULL; 621 622 return xsk_get_pool_from_qid(ring->netdev, ring->queue_index); 623 } 624 625 /** 626 * igc_configure_rx_ring - Configure a receive ring after Reset 627 * @adapter: board private structure 628 * @ring: receive ring to be configured 629 * 630 * Configure the Rx unit of the MAC after a reset. 631 */ 632 static void igc_configure_rx_ring(struct igc_adapter *adapter, 633 struct igc_ring *ring) 634 { 635 struct igc_hw *hw = &adapter->hw; 636 union igc_adv_rx_desc *rx_desc; 637 int reg_idx = ring->reg_idx; 638 u32 srrctl = 0, rxdctl = 0; 639 u64 rdba = ring->dma; 640 u32 buf_size; 641 642 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 643 ring->xsk_pool = igc_get_xsk_pool(adapter, ring); 644 if (ring->xsk_pool) { 645 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 646 MEM_TYPE_XSK_BUFF_POOL, 647 NULL)); 648 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 649 } else { 650 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 651 MEM_TYPE_PAGE_SHARED, 652 NULL)); 653 } 654 655 if (igc_xdp_is_enabled(adapter)) 656 set_ring_uses_large_buffer(ring); 657 658 /* disable the queue */ 659 wr32(IGC_RXDCTL(reg_idx), 0); 660 661 /* Set DMA base address registers */ 662 wr32(IGC_RDBAL(reg_idx), 663 rdba & 0x00000000ffffffffULL); 664 wr32(IGC_RDBAH(reg_idx), rdba >> 32); 665 wr32(IGC_RDLEN(reg_idx), 666 ring->count * sizeof(union igc_adv_rx_desc)); 667 668 /* initialize head and tail */ 669 ring->tail = adapter->io_addr + IGC_RDT(reg_idx); 670 wr32(IGC_RDH(reg_idx), 0); 671 writel(0, ring->tail); 672 673 /* reset next-to- use/clean to place SW in sync with hardware */ 674 ring->next_to_clean = 0; 675 ring->next_to_use = 0; 676 677 if (ring->xsk_pool) 678 buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool); 679 else if (ring_uses_large_buffer(ring)) 680 buf_size = IGC_RXBUFFER_3072; 681 else 682 buf_size = IGC_RXBUFFER_2048; 683 684 srrctl = rd32(IGC_SRRCTL(reg_idx)); 685 srrctl &= ~(IGC_SRRCTL_BSIZEPKT_MASK | IGC_SRRCTL_BSIZEHDR_MASK | 686 IGC_SRRCTL_DESCTYPE_MASK); 687 srrctl |= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN); 688 srrctl |= IGC_SRRCTL_BSIZEPKT(buf_size); 689 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 690 691 wr32(IGC_SRRCTL(reg_idx), srrctl); 692 693 rxdctl |= IGC_RXDCTL_PTHRESH; 694 rxdctl |= IGC_RXDCTL_HTHRESH << 8; 695 rxdctl |= IGC_RXDCTL_WTHRESH << 16; 696 697 /* initialize rx_buffer_info */ 698 memset(ring->rx_buffer_info, 0, 699 sizeof(struct igc_rx_buffer) * ring->count); 700 701 /* initialize Rx descriptor 0 */ 702 rx_desc = IGC_RX_DESC(ring, 0); 703 rx_desc->wb.upper.length = 0; 704 705 /* enable receive descriptor fetching */ 706 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 707 708 wr32(IGC_RXDCTL(reg_idx), rxdctl); 709 } 710 711 /** 712 * igc_configure_rx - Configure receive Unit after Reset 713 * @adapter: board private structure 714 * 715 * Configure the Rx unit of the MAC after a reset. 716 */ 717 static void igc_configure_rx(struct igc_adapter *adapter) 718 { 719 int i; 720 721 /* Setup the HW Rx Head and Tail Descriptor Pointers and 722 * the Base and Length of the Rx Descriptor Ring 723 */ 724 for (i = 0; i < adapter->num_rx_queues; i++) 725 igc_configure_rx_ring(adapter, adapter->rx_ring[i]); 726 } 727 728 /** 729 * igc_configure_tx_ring - Configure transmit ring after Reset 730 * @adapter: board private structure 731 * @ring: tx ring to configure 732 * 733 * Configure a transmit ring after a reset. 734 */ 735 static void igc_configure_tx_ring(struct igc_adapter *adapter, 736 struct igc_ring *ring) 737 { 738 struct igc_hw *hw = &adapter->hw; 739 int reg_idx = ring->reg_idx; 740 u64 tdba = ring->dma; 741 u32 txdctl = 0; 742 743 ring->xsk_pool = igc_get_xsk_pool(adapter, ring); 744 745 /* disable the queue */ 746 wr32(IGC_TXDCTL(reg_idx), 0); 747 wrfl(); 748 749 wr32(IGC_TDLEN(reg_idx), 750 ring->count * sizeof(union igc_adv_tx_desc)); 751 wr32(IGC_TDBAL(reg_idx), 752 tdba & 0x00000000ffffffffULL); 753 wr32(IGC_TDBAH(reg_idx), tdba >> 32); 754 755 ring->tail = adapter->io_addr + IGC_TDT(reg_idx); 756 wr32(IGC_TDH(reg_idx), 0); 757 writel(0, ring->tail); 758 759 txdctl |= IGC_TXDCTL_PTHRESH(8) | IGC_TXDCTL_HTHRESH(1) | 760 IGC_TXDCTL_WTHRESH(16) | IGC_TXDCTL_QUEUE_ENABLE; 761 762 wr32(IGC_TXDCTL(reg_idx), txdctl); 763 } 764 765 /** 766 * igc_configure_tx - Configure transmit Unit after Reset 767 * @adapter: board private structure 768 * 769 * Configure the Tx unit of the MAC after a reset. 770 */ 771 static void igc_configure_tx(struct igc_adapter *adapter) 772 { 773 int i; 774 775 for (i = 0; i < adapter->num_tx_queues; i++) 776 igc_configure_tx_ring(adapter, adapter->tx_ring[i]); 777 } 778 779 /** 780 * igc_setup_mrqc - configure the multiple receive queue control registers 781 * @adapter: Board private structure 782 */ 783 static void igc_setup_mrqc(struct igc_adapter *adapter) 784 { 785 struct igc_hw *hw = &adapter->hw; 786 u32 j, num_rx_queues; 787 u32 mrqc, rxcsum; 788 789 igc_write_rss_key(adapter); 790 791 num_rx_queues = adapter->rss_queues; 792 793 if (adapter->rss_indir_tbl_init != num_rx_queues) { 794 for (j = 0; j < IGC_RETA_SIZE; j++) 795 adapter->rss_indir_tbl[j] = 796 (j * num_rx_queues) / IGC_RETA_SIZE; 797 adapter->rss_indir_tbl_init = num_rx_queues; 798 } 799 igc_write_rss_indir_tbl(adapter); 800 801 /* Disable raw packet checksumming so that RSS hash is placed in 802 * descriptor on writeback. No need to enable TCP/UDP/IP checksum 803 * offloads as they are enabled by default 804 */ 805 rxcsum = rd32(IGC_RXCSUM); 806 rxcsum |= IGC_RXCSUM_PCSD; 807 808 /* Enable Receive Checksum Offload for SCTP */ 809 rxcsum |= IGC_RXCSUM_CRCOFL; 810 811 /* Don't need to set TUOFL or IPOFL, they default to 1 */ 812 wr32(IGC_RXCSUM, rxcsum); 813 814 /* Generate RSS hash based on packet types, TCP/UDP 815 * port numbers and/or IPv4/v6 src and dst addresses 816 */ 817 mrqc = IGC_MRQC_RSS_FIELD_IPV4 | 818 IGC_MRQC_RSS_FIELD_IPV4_TCP | 819 IGC_MRQC_RSS_FIELD_IPV6 | 820 IGC_MRQC_RSS_FIELD_IPV6_TCP | 821 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 822 823 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) 824 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 825 if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) 826 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 827 828 mrqc |= IGC_MRQC_ENABLE_RSS_MQ; 829 830 wr32(IGC_MRQC, mrqc); 831 } 832 833 /** 834 * igc_setup_rctl - configure the receive control registers 835 * @adapter: Board private structure 836 */ 837 static void igc_setup_rctl(struct igc_adapter *adapter) 838 { 839 struct igc_hw *hw = &adapter->hw; 840 u32 rctl; 841 842 rctl = rd32(IGC_RCTL); 843 844 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 845 rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); 846 847 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | 848 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 849 850 /* enable stripping of CRC. Newer features require 851 * that the HW strips the CRC. 852 */ 853 rctl |= IGC_RCTL_SECRC; 854 855 /* disable store bad packets and clear size bits. */ 856 rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); 857 858 /* enable LPE to allow for reception of jumbo frames */ 859 rctl |= IGC_RCTL_LPE; 860 861 /* disable queue 0 to prevent tail write w/o re-config */ 862 wr32(IGC_RXDCTL(0), 0); 863 864 /* This is useful for sniffing bad packets. */ 865 if (adapter->netdev->features & NETIF_F_RXALL) { 866 /* UPE and MPE will be handled by normal PROMISC logic 867 * in set_rx_mode 868 */ 869 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ 870 IGC_RCTL_BAM | /* RX All Bcast Pkts */ 871 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ 872 873 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ 874 IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ 875 } 876 877 wr32(IGC_RCTL, rctl); 878 } 879 880 /** 881 * igc_setup_tctl - configure the transmit control registers 882 * @adapter: Board private structure 883 */ 884 static void igc_setup_tctl(struct igc_adapter *adapter) 885 { 886 struct igc_hw *hw = &adapter->hw; 887 u32 tctl; 888 889 /* disable queue 0 which icould be enabled by default */ 890 wr32(IGC_TXDCTL(0), 0); 891 892 /* Program the Transmit Control Register */ 893 tctl = rd32(IGC_TCTL); 894 tctl &= ~IGC_TCTL_CT; 895 tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | 896 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); 897 898 /* Enable transmits */ 899 tctl |= IGC_TCTL_EN; 900 901 wr32(IGC_TCTL, tctl); 902 } 903 904 /** 905 * igc_set_mac_filter_hw() - Set MAC address filter in hardware 906 * @adapter: Pointer to adapter where the filter should be set 907 * @index: Filter index 908 * @type: MAC address filter type (source or destination) 909 * @addr: MAC address 910 * @queue: If non-negative, queue assignment feature is enabled and frames 911 * matching the filter are enqueued onto 'queue'. Otherwise, queue 912 * assignment is disabled. 913 */ 914 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, 915 enum igc_mac_filter_type type, 916 const u8 *addr, int queue) 917 { 918 struct net_device *dev = adapter->netdev; 919 struct igc_hw *hw = &adapter->hw; 920 u32 ral, rah; 921 922 if (WARN_ON(index >= hw->mac.rar_entry_count)) 923 return; 924 925 ral = le32_to_cpup((__le32 *)(addr)); 926 rah = le16_to_cpup((__le16 *)(addr + 4)); 927 928 if (type == IGC_MAC_FILTER_TYPE_SRC) { 929 rah &= ~IGC_RAH_ASEL_MASK; 930 rah |= IGC_RAH_ASEL_SRC_ADDR; 931 } 932 933 if (queue >= 0) { 934 rah &= ~IGC_RAH_QSEL_MASK; 935 rah |= (queue << IGC_RAH_QSEL_SHIFT); 936 rah |= IGC_RAH_QSEL_ENABLE; 937 } 938 939 rah |= IGC_RAH_AV; 940 941 wr32(IGC_RAL(index), ral); 942 wr32(IGC_RAH(index), rah); 943 944 netdev_dbg(dev, "MAC address filter set in HW: index %d", index); 945 } 946 947 /** 948 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware 949 * @adapter: Pointer to adapter where the filter should be cleared 950 * @index: Filter index 951 */ 952 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) 953 { 954 struct net_device *dev = adapter->netdev; 955 struct igc_hw *hw = &adapter->hw; 956 957 if (WARN_ON(index >= hw->mac.rar_entry_count)) 958 return; 959 960 wr32(IGC_RAL(index), 0); 961 wr32(IGC_RAH(index), 0); 962 963 netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index); 964 } 965 966 /* Set default MAC address for the PF in the first RAR entry */ 967 static void igc_set_default_mac_filter(struct igc_adapter *adapter) 968 { 969 struct net_device *dev = adapter->netdev; 970 u8 *addr = adapter->hw.mac.addr; 971 972 netdev_dbg(dev, "Set default MAC address filter: address %pM", addr); 973 974 igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1); 975 } 976 977 /** 978 * igc_set_mac - Change the Ethernet Address of the NIC 979 * @netdev: network interface device structure 980 * @p: pointer to an address structure 981 * 982 * Returns 0 on success, negative on failure 983 */ 984 static int igc_set_mac(struct net_device *netdev, void *p) 985 { 986 struct igc_adapter *adapter = netdev_priv(netdev); 987 struct igc_hw *hw = &adapter->hw; 988 struct sockaddr *addr = p; 989 990 if (!is_valid_ether_addr(addr->sa_data)) 991 return -EADDRNOTAVAIL; 992 993 eth_hw_addr_set(netdev, addr->sa_data); 994 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 995 996 /* set the correct pool for the new PF MAC address in entry 0 */ 997 igc_set_default_mac_filter(adapter); 998 999 return 0; 1000 } 1001 1002 /** 1003 * igc_write_mc_addr_list - write multicast addresses to MTA 1004 * @netdev: network interface device structure 1005 * 1006 * Writes multicast address list to the MTA hash table. 1007 * Returns: -ENOMEM on failure 1008 * 0 on no addresses written 1009 * X on writing X addresses to MTA 1010 **/ 1011 static int igc_write_mc_addr_list(struct net_device *netdev) 1012 { 1013 struct igc_adapter *adapter = netdev_priv(netdev); 1014 struct igc_hw *hw = &adapter->hw; 1015 struct netdev_hw_addr *ha; 1016 u8 *mta_list; 1017 int i; 1018 1019 if (netdev_mc_empty(netdev)) { 1020 /* nothing to program, so clear mc list */ 1021 igc_update_mc_addr_list(hw, NULL, 0); 1022 return 0; 1023 } 1024 1025 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC); 1026 if (!mta_list) 1027 return -ENOMEM; 1028 1029 /* The shared function expects a packed array of only addresses. */ 1030 i = 0; 1031 netdev_for_each_mc_addr(ha, netdev) 1032 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); 1033 1034 igc_update_mc_addr_list(hw, mta_list, i); 1035 kfree(mta_list); 1036 1037 return netdev_mc_count(netdev); 1038 } 1039 1040 static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime, 1041 bool *first_flag, bool *insert_empty) 1042 { 1043 struct igc_adapter *adapter = netdev_priv(ring->netdev); 1044 ktime_t cycle_time = adapter->cycle_time; 1045 ktime_t base_time = adapter->base_time; 1046 ktime_t now = ktime_get_clocktai(); 1047 ktime_t baset_est, end_of_cycle; 1048 s32 launchtime; 1049 s64 n; 1050 1051 n = div64_s64(ktime_sub_ns(now, base_time), cycle_time); 1052 1053 baset_est = ktime_add_ns(base_time, cycle_time * (n)); 1054 end_of_cycle = ktime_add_ns(baset_est, cycle_time); 1055 1056 if (ktime_compare(txtime, end_of_cycle) >= 0) { 1057 if (baset_est != ring->last_ff_cycle) { 1058 *first_flag = true; 1059 ring->last_ff_cycle = baset_est; 1060 1061 if (ktime_compare(end_of_cycle, ring->last_tx_cycle) > 0) 1062 *insert_empty = true; 1063 } 1064 } 1065 1066 /* Introducing a window at end of cycle on which packets 1067 * potentially not honor launchtime. Window of 5us chosen 1068 * considering software update the tail pointer and packets 1069 * are dma'ed to packet buffer. 1070 */ 1071 if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC)) 1072 netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n", 1073 txtime); 1074 1075 ring->last_tx_cycle = end_of_cycle; 1076 1077 launchtime = ktime_sub_ns(txtime, baset_est); 1078 if (launchtime > 0) 1079 div_s64_rem(launchtime, cycle_time, &launchtime); 1080 else 1081 launchtime = 0; 1082 1083 return cpu_to_le32(launchtime); 1084 } 1085 1086 static int igc_init_empty_frame(struct igc_ring *ring, 1087 struct igc_tx_buffer *buffer, 1088 struct sk_buff *skb) 1089 { 1090 unsigned int size; 1091 dma_addr_t dma; 1092 1093 size = skb_headlen(skb); 1094 1095 dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE); 1096 if (dma_mapping_error(ring->dev, dma)) { 1097 net_err_ratelimited("%s: DMA mapping error for empty frame\n", 1098 netdev_name(ring->netdev)); 1099 return -ENOMEM; 1100 } 1101 1102 buffer->type = IGC_TX_BUFFER_TYPE_SKB; 1103 buffer->skb = skb; 1104 buffer->protocol = 0; 1105 buffer->bytecount = skb->len; 1106 buffer->gso_segs = 1; 1107 buffer->time_stamp = jiffies; 1108 dma_unmap_len_set(buffer, len, skb->len); 1109 dma_unmap_addr_set(buffer, dma, dma); 1110 1111 return 0; 1112 } 1113 1114 static void igc_init_tx_empty_descriptor(struct igc_ring *ring, 1115 struct sk_buff *skb, 1116 struct igc_tx_buffer *first) 1117 { 1118 union igc_adv_tx_desc *desc; 1119 u32 cmd_type, olinfo_status; 1120 1121 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | 1122 IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | 1123 first->bytecount; 1124 olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; 1125 1126 desc = IGC_TX_DESC(ring, ring->next_to_use); 1127 desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1128 desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1129 desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma)); 1130 1131 netdev_tx_sent_queue(txring_txq(ring), skb->len); 1132 1133 first->next_to_watch = desc; 1134 1135 ring->next_to_use++; 1136 if (ring->next_to_use == ring->count) 1137 ring->next_to_use = 0; 1138 } 1139 1140 #define IGC_EMPTY_FRAME_SIZE 60 1141 1142 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, 1143 __le32 launch_time, bool first_flag, 1144 u32 vlan_macip_lens, u32 type_tucmd, 1145 u32 mss_l4len_idx) 1146 { 1147 struct igc_adv_tx_context_desc *context_desc; 1148 u16 i = tx_ring->next_to_use; 1149 1150 context_desc = IGC_TX_CTXTDESC(tx_ring, i); 1151 1152 i++; 1153 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 1154 1155 /* set bits to identify this as an advanced context descriptor */ 1156 type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 1157 1158 /* For i225, context index must be unique per ring. */ 1159 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) 1160 mss_l4len_idx |= tx_ring->reg_idx << 4; 1161 1162 if (first_flag) 1163 mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST; 1164 1165 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 1166 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 1167 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 1168 context_desc->launch_time = launch_time; 1169 } 1170 1171 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first, 1172 __le32 launch_time, bool first_flag) 1173 { 1174 struct sk_buff *skb = first->skb; 1175 u32 vlan_macip_lens = 0; 1176 u32 type_tucmd = 0; 1177 1178 if (skb->ip_summed != CHECKSUM_PARTIAL) { 1179 csum_failed: 1180 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && 1181 !tx_ring->launchtime_enable) 1182 return; 1183 goto no_csum; 1184 } 1185 1186 switch (skb->csum_offset) { 1187 case offsetof(struct tcphdr, check): 1188 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1189 fallthrough; 1190 case offsetof(struct udphdr, check): 1191 break; 1192 case offsetof(struct sctphdr, checksum): 1193 /* validate that this is actually an SCTP request */ 1194 if (skb_csum_is_sctp(skb)) { 1195 type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; 1196 break; 1197 } 1198 fallthrough; 1199 default: 1200 skb_checksum_help(skb); 1201 goto csum_failed; 1202 } 1203 1204 /* update TX checksum flag */ 1205 first->tx_flags |= IGC_TX_FLAGS_CSUM; 1206 vlan_macip_lens = skb_checksum_start_offset(skb) - 1207 skb_network_offset(skb); 1208 no_csum: 1209 vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; 1210 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1211 1212 igc_tx_ctxtdesc(tx_ring, launch_time, first_flag, 1213 vlan_macip_lens, type_tucmd, 0); 1214 } 1215 1216 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1217 { 1218 struct net_device *netdev = tx_ring->netdev; 1219 1220 netif_stop_subqueue(netdev, tx_ring->queue_index); 1221 1222 /* memory barriier comment */ 1223 smp_mb(); 1224 1225 /* We need to check again in a case another CPU has just 1226 * made room available. 1227 */ 1228 if (igc_desc_unused(tx_ring) < size) 1229 return -EBUSY; 1230 1231 /* A reprieve! */ 1232 netif_wake_subqueue(netdev, tx_ring->queue_index); 1233 1234 u64_stats_update_begin(&tx_ring->tx_syncp2); 1235 tx_ring->tx_stats.restart_queue2++; 1236 u64_stats_update_end(&tx_ring->tx_syncp2); 1237 1238 return 0; 1239 } 1240 1241 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) 1242 { 1243 if (igc_desc_unused(tx_ring) >= size) 1244 return 0; 1245 return __igc_maybe_stop_tx(tx_ring, size); 1246 } 1247 1248 #define IGC_SET_FLAG(_input, _flag, _result) \ 1249 (((_flag) <= (_result)) ? \ 1250 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ 1251 ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) 1252 1253 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) 1254 { 1255 /* set type for advanced descriptor with frame checksum insertion */ 1256 u32 cmd_type = IGC_ADVTXD_DTYP_DATA | 1257 IGC_ADVTXD_DCMD_DEXT | 1258 IGC_ADVTXD_DCMD_IFCS; 1259 1260 /* set HW vlan bit if vlan is present */ 1261 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN, 1262 IGC_ADVTXD_DCMD_VLE); 1263 1264 /* set segmentation bits for TSO */ 1265 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, 1266 (IGC_ADVTXD_DCMD_TSE)); 1267 1268 /* set timestamp bit if present, will select the register set 1269 * based on the _TSTAMP(_X) bit. 1270 */ 1271 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, 1272 (IGC_ADVTXD_MAC_TSTAMP)); 1273 1274 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_1, 1275 (IGC_ADVTXD_TSTAMP_REG_1)); 1276 1277 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_2, 1278 (IGC_ADVTXD_TSTAMP_REG_2)); 1279 1280 cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_3, 1281 (IGC_ADVTXD_TSTAMP_REG_3)); 1282 1283 /* insert frame checksum */ 1284 cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS); 1285 1286 return cmd_type; 1287 } 1288 1289 static void igc_tx_olinfo_status(struct igc_ring *tx_ring, 1290 union igc_adv_tx_desc *tx_desc, 1291 u32 tx_flags, unsigned int paylen) 1292 { 1293 u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; 1294 1295 /* insert L4 checksum */ 1296 olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_CSUM, 1297 (IGC_TXD_POPTS_TXSM << 8)); 1298 1299 /* insert IPv4 checksum */ 1300 olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_IPV4, 1301 (IGC_TXD_POPTS_IXSM << 8)); 1302 1303 /* Use the second timer (free running, in general) for the timestamp */ 1304 olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_TIMER_1, 1305 IGC_TXD_PTP2_TIMER_1); 1306 1307 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 1308 } 1309 1310 static int igc_tx_map(struct igc_ring *tx_ring, 1311 struct igc_tx_buffer *first, 1312 const u8 hdr_len) 1313 { 1314 struct sk_buff *skb = first->skb; 1315 struct igc_tx_buffer *tx_buffer; 1316 union igc_adv_tx_desc *tx_desc; 1317 u32 tx_flags = first->tx_flags; 1318 skb_frag_t *frag; 1319 u16 i = tx_ring->next_to_use; 1320 unsigned int data_len, size; 1321 dma_addr_t dma; 1322 u32 cmd_type; 1323 1324 cmd_type = igc_tx_cmd_type(skb, tx_flags); 1325 tx_desc = IGC_TX_DESC(tx_ring, i); 1326 1327 igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); 1328 1329 size = skb_headlen(skb); 1330 data_len = skb->data_len; 1331 1332 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 1333 1334 tx_buffer = first; 1335 1336 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 1337 if (dma_mapping_error(tx_ring->dev, dma)) 1338 goto dma_error; 1339 1340 /* record length, and DMA address */ 1341 dma_unmap_len_set(tx_buffer, len, size); 1342 dma_unmap_addr_set(tx_buffer, dma, dma); 1343 1344 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1345 1346 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { 1347 tx_desc->read.cmd_type_len = 1348 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); 1349 1350 i++; 1351 tx_desc++; 1352 if (i == tx_ring->count) { 1353 tx_desc = IGC_TX_DESC(tx_ring, 0); 1354 i = 0; 1355 } 1356 tx_desc->read.olinfo_status = 0; 1357 1358 dma += IGC_MAX_DATA_PER_TXD; 1359 size -= IGC_MAX_DATA_PER_TXD; 1360 1361 tx_desc->read.buffer_addr = cpu_to_le64(dma); 1362 } 1363 1364 if (likely(!data_len)) 1365 break; 1366 1367 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); 1368 1369 i++; 1370 tx_desc++; 1371 if (i == tx_ring->count) { 1372 tx_desc = IGC_TX_DESC(tx_ring, 0); 1373 i = 0; 1374 } 1375 tx_desc->read.olinfo_status = 0; 1376 1377 size = skb_frag_size(frag); 1378 data_len -= size; 1379 1380 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, 1381 size, DMA_TO_DEVICE); 1382 1383 tx_buffer = &tx_ring->tx_buffer_info[i]; 1384 } 1385 1386 /* write last descriptor with RS and EOP bits */ 1387 cmd_type |= size | IGC_TXD_DCMD; 1388 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); 1389 1390 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 1391 1392 /* set the timestamp */ 1393 first->time_stamp = jiffies; 1394 1395 skb_tx_timestamp(skb); 1396 1397 /* Force memory writes to complete before letting h/w know there 1398 * are new descriptors to fetch. (Only applicable for weak-ordered 1399 * memory model archs, such as IA-64). 1400 * 1401 * We also need this memory barrier to make certain all of the 1402 * status bits have been updated before next_to_watch is written. 1403 */ 1404 wmb(); 1405 1406 /* set next_to_watch value indicating a packet is present */ 1407 first->next_to_watch = tx_desc; 1408 1409 i++; 1410 if (i == tx_ring->count) 1411 i = 0; 1412 1413 tx_ring->next_to_use = i; 1414 1415 /* Make sure there is space in the ring for the next send. */ 1416 igc_maybe_stop_tx(tx_ring, DESC_NEEDED); 1417 1418 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 1419 writel(i, tx_ring->tail); 1420 } 1421 1422 return 0; 1423 dma_error: 1424 netdev_err(tx_ring->netdev, "TX DMA map failed\n"); 1425 tx_buffer = &tx_ring->tx_buffer_info[i]; 1426 1427 /* clear dma mappings for failed tx_buffer_info map */ 1428 while (tx_buffer != first) { 1429 if (dma_unmap_len(tx_buffer, len)) 1430 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 1431 1432 if (i-- == 0) 1433 i += tx_ring->count; 1434 tx_buffer = &tx_ring->tx_buffer_info[i]; 1435 } 1436 1437 if (dma_unmap_len(tx_buffer, len)) 1438 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 1439 1440 dev_kfree_skb_any(tx_buffer->skb); 1441 tx_buffer->skb = NULL; 1442 1443 tx_ring->next_to_use = i; 1444 1445 return -1; 1446 } 1447 1448 static int igc_tso(struct igc_ring *tx_ring, 1449 struct igc_tx_buffer *first, 1450 __le32 launch_time, bool first_flag, 1451 u8 *hdr_len) 1452 { 1453 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 1454 struct sk_buff *skb = first->skb; 1455 union { 1456 struct iphdr *v4; 1457 struct ipv6hdr *v6; 1458 unsigned char *hdr; 1459 } ip; 1460 union { 1461 struct tcphdr *tcp; 1462 struct udphdr *udp; 1463 unsigned char *hdr; 1464 } l4; 1465 u32 paylen, l4_offset; 1466 int err; 1467 1468 if (skb->ip_summed != CHECKSUM_PARTIAL) 1469 return 0; 1470 1471 if (!skb_is_gso(skb)) 1472 return 0; 1473 1474 err = skb_cow_head(skb, 0); 1475 if (err < 0) 1476 return err; 1477 1478 ip.hdr = skb_network_header(skb); 1479 l4.hdr = skb_checksum_start(skb); 1480 1481 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 1482 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; 1483 1484 /* initialize outer IP header fields */ 1485 if (ip.v4->version == 4) { 1486 unsigned char *csum_start = skb_checksum_start(skb); 1487 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); 1488 1489 /* IP header will have to cancel out any data that 1490 * is not a part of the outer IP header 1491 */ 1492 ip.v4->check = csum_fold(csum_partial(trans_start, 1493 csum_start - trans_start, 1494 0)); 1495 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; 1496 1497 ip.v4->tot_len = 0; 1498 first->tx_flags |= IGC_TX_FLAGS_TSO | 1499 IGC_TX_FLAGS_CSUM | 1500 IGC_TX_FLAGS_IPV4; 1501 } else { 1502 ip.v6->payload_len = 0; 1503 first->tx_flags |= IGC_TX_FLAGS_TSO | 1504 IGC_TX_FLAGS_CSUM; 1505 } 1506 1507 /* determine offset of inner transport header */ 1508 l4_offset = l4.hdr - skb->data; 1509 1510 /* remove payload length from inner checksum */ 1511 paylen = skb->len - l4_offset; 1512 if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { 1513 /* compute length of segmentation header */ 1514 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 1515 csum_replace_by_diff(&l4.tcp->check, 1516 (__force __wsum)htonl(paylen)); 1517 } else { 1518 /* compute length of segmentation header */ 1519 *hdr_len = sizeof(*l4.udp) + l4_offset; 1520 csum_replace_by_diff(&l4.udp->check, 1521 (__force __wsum)htonl(paylen)); 1522 } 1523 1524 /* update gso size and bytecount with header size */ 1525 first->gso_segs = skb_shinfo(skb)->gso_segs; 1526 first->bytecount += (first->gso_segs - 1) * *hdr_len; 1527 1528 /* MSS L4LEN IDX */ 1529 mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; 1530 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; 1531 1532 /* VLAN MACLEN IPLEN */ 1533 vlan_macip_lens = l4.hdr - ip.hdr; 1534 vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; 1535 vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; 1536 1537 igc_tx_ctxtdesc(tx_ring, launch_time, first_flag, 1538 vlan_macip_lens, type_tucmd, mss_l4len_idx); 1539 1540 return 1; 1541 } 1542 1543 static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags) 1544 { 1545 int i; 1546 1547 for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { 1548 struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i]; 1549 1550 if (tstamp->skb) 1551 continue; 1552 1553 tstamp->skb = skb_get(skb); 1554 tstamp->start = jiffies; 1555 *flags = tstamp->flags; 1556 1557 return true; 1558 } 1559 1560 return false; 1561 } 1562 1563 static int igc_insert_empty_frame(struct igc_ring *tx_ring) 1564 { 1565 struct igc_tx_buffer *empty_info; 1566 struct sk_buff *empty_skb; 1567 void *data; 1568 int ret; 1569 1570 empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1571 empty_skb = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC); 1572 if (unlikely(!empty_skb)) { 1573 net_err_ratelimited("%s: skb alloc error for empty frame\n", 1574 netdev_name(tx_ring->netdev)); 1575 return -ENOMEM; 1576 } 1577 1578 data = skb_put(empty_skb, IGC_EMPTY_FRAME_SIZE); 1579 memset(data, 0, IGC_EMPTY_FRAME_SIZE); 1580 1581 /* Prepare DMA mapping and Tx buffer information */ 1582 ret = igc_init_empty_frame(tx_ring, empty_info, empty_skb); 1583 if (unlikely(ret)) { 1584 dev_kfree_skb_any(empty_skb); 1585 return ret; 1586 } 1587 1588 /* Prepare advanced context descriptor for empty packet */ 1589 igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0); 1590 1591 /* Prepare advanced data descriptor for empty packet */ 1592 igc_init_tx_empty_descriptor(tx_ring, empty_skb, empty_info); 1593 1594 return 0; 1595 } 1596 1597 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, 1598 struct igc_ring *tx_ring) 1599 { 1600 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); 1601 bool first_flag = false, insert_empty = false; 1602 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 1603 __be16 protocol = vlan_get_protocol(skb); 1604 struct igc_tx_buffer *first; 1605 __le32 launch_time = 0; 1606 u32 tx_flags = 0; 1607 unsigned short f; 1608 ktime_t txtime; 1609 u8 hdr_len = 0; 1610 int tso = 0; 1611 1612 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, 1613 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, 1614 * + 2 desc gap to keep tail from touching head, 1615 * + 1 desc for context descriptor, 1616 * + 2 desc for inserting an empty packet for launch time, 1617 * otherwise try next time 1618 */ 1619 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 1620 count += TXD_USE_COUNT(skb_frag_size( 1621 &skb_shinfo(skb)->frags[f])); 1622 1623 if (igc_maybe_stop_tx(tx_ring, count + 5)) { 1624 /* this is a hard error */ 1625 return NETDEV_TX_BUSY; 1626 } 1627 1628 if (!tx_ring->launchtime_enable) 1629 goto done; 1630 1631 txtime = skb->tstamp; 1632 skb->tstamp = ktime_set(0, 0); 1633 launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty); 1634 1635 if (insert_empty) { 1636 /* Reset the launch time if the required empty frame fails to 1637 * be inserted. However, this packet is not dropped, so it 1638 * "dirties" the current Qbv cycle. This ensures that the 1639 * upcoming packet, which is scheduled in the next Qbv cycle, 1640 * does not require an empty frame. This way, the launch time 1641 * continues to function correctly despite the current failure 1642 * to insert the empty frame. 1643 */ 1644 if (igc_insert_empty_frame(tx_ring)) 1645 launch_time = 0; 1646 } 1647 1648 done: 1649 /* record the location of the first descriptor for this packet */ 1650 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 1651 first->type = IGC_TX_BUFFER_TYPE_SKB; 1652 first->skb = skb; 1653 first->bytecount = skb->len; 1654 first->gso_segs = 1; 1655 1656 if (adapter->qbv_transition || tx_ring->oper_gate_closed) 1657 goto out_drop; 1658 1659 if (tx_ring->max_sdu > 0 && first->bytecount > tx_ring->max_sdu) { 1660 adapter->stats.txdrop++; 1661 goto out_drop; 1662 } 1663 1664 if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags) && 1665 skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1666 unsigned long flags; 1667 u32 tstamp_flags; 1668 1669 spin_lock_irqsave(&adapter->ptp_tx_lock, flags); 1670 if (igc_request_tx_tstamp(adapter, skb, &tstamp_flags)) { 1671 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1672 tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags; 1673 if (skb->sk && 1674 READ_ONCE(skb->sk->sk_tsflags) & SOF_TIMESTAMPING_BIND_PHC) 1675 tx_flags |= IGC_TX_FLAGS_TSTAMP_TIMER_1; 1676 } else { 1677 adapter->tx_hwtstamp_skipped++; 1678 } 1679 1680 spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags); 1681 } 1682 1683 if (skb_vlan_tag_present(skb)) { 1684 tx_flags |= IGC_TX_FLAGS_VLAN; 1685 tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT); 1686 } 1687 1688 /* record initial flags and protocol */ 1689 first->tx_flags = tx_flags; 1690 first->protocol = protocol; 1691 1692 /* For preemptible queue, manually pad the skb so that HW includes 1693 * padding bytes in mCRC calculation 1694 */ 1695 if (tx_ring->preemptible && skb->len < ETH_ZLEN) { 1696 if (skb_padto(skb, ETH_ZLEN)) 1697 goto out_drop; 1698 skb_put(skb, ETH_ZLEN - skb->len); 1699 } 1700 1701 tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len); 1702 if (tso < 0) 1703 goto out_drop; 1704 else if (!tso) 1705 igc_tx_csum(tx_ring, first, launch_time, first_flag); 1706 1707 igc_tx_map(tx_ring, first, hdr_len); 1708 1709 return NETDEV_TX_OK; 1710 1711 out_drop: 1712 dev_kfree_skb_any(first->skb); 1713 first->skb = NULL; 1714 1715 return NETDEV_TX_OK; 1716 } 1717 1718 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, 1719 struct sk_buff *skb) 1720 { 1721 unsigned int r_idx = skb->queue_mapping; 1722 1723 if (r_idx >= adapter->num_tx_queues) 1724 r_idx = r_idx % adapter->num_tx_queues; 1725 1726 return adapter->tx_ring[r_idx]; 1727 } 1728 1729 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, 1730 struct net_device *netdev) 1731 { 1732 struct igc_adapter *adapter = netdev_priv(netdev); 1733 1734 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb 1735 * in order to meet this minimum size requirement. 1736 */ 1737 if (skb_put_padto(skb, 17)) 1738 return NETDEV_TX_OK; 1739 1740 return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb)); 1741 } 1742 1743 static void igc_rx_checksum(struct igc_ring *ring, 1744 union igc_adv_rx_desc *rx_desc, 1745 struct sk_buff *skb) 1746 { 1747 skb_checksum_none_assert(skb); 1748 1749 /* Ignore Checksum bit is set */ 1750 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) 1751 return; 1752 1753 /* Rx checksum disabled via ethtool */ 1754 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 1755 return; 1756 1757 /* TCP/UDP checksum error bit is set */ 1758 if (igc_test_staterr(rx_desc, 1759 IGC_RXDEXT_STATERR_L4E | 1760 IGC_RXDEXT_STATERR_IPE)) { 1761 /* work around errata with sctp packets where the TCPE aka 1762 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) 1763 * packets (aka let the stack check the crc32c) 1764 */ 1765 if (!(skb->len == 60 && 1766 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { 1767 u64_stats_update_begin(&ring->rx_syncp); 1768 ring->rx_stats.csum_err++; 1769 u64_stats_update_end(&ring->rx_syncp); 1770 } 1771 /* let the stack verify checksum errors */ 1772 return; 1773 } 1774 /* It must be a TCP or UDP packet with a valid checksum */ 1775 if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | 1776 IGC_RXD_STAT_UDPCS)) 1777 skb->ip_summed = CHECKSUM_UNNECESSARY; 1778 1779 netdev_dbg(ring->netdev, "cksum success: bits %08X\n", 1780 le32_to_cpu(rx_desc->wb.upper.status_error)); 1781 } 1782 1783 /* Mapping HW RSS Type to enum pkt_hash_types */ 1784 static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = { 1785 [IGC_RSS_TYPE_NO_HASH] = PKT_HASH_TYPE_L2, 1786 [IGC_RSS_TYPE_HASH_TCP_IPV4] = PKT_HASH_TYPE_L4, 1787 [IGC_RSS_TYPE_HASH_IPV4] = PKT_HASH_TYPE_L3, 1788 [IGC_RSS_TYPE_HASH_TCP_IPV6] = PKT_HASH_TYPE_L4, 1789 [IGC_RSS_TYPE_HASH_IPV6_EX] = PKT_HASH_TYPE_L3, 1790 [IGC_RSS_TYPE_HASH_IPV6] = PKT_HASH_TYPE_L3, 1791 [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4, 1792 [IGC_RSS_TYPE_HASH_UDP_IPV4] = PKT_HASH_TYPE_L4, 1793 [IGC_RSS_TYPE_HASH_UDP_IPV6] = PKT_HASH_TYPE_L4, 1794 [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4, 1795 [10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */ 1796 [11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask */ 1797 [12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisions */ 1798 [13] = PKT_HASH_TYPE_NONE, 1799 [14] = PKT_HASH_TYPE_NONE, 1800 [15] = PKT_HASH_TYPE_NONE, 1801 }; 1802 1803 static inline void igc_rx_hash(struct igc_ring *ring, 1804 union igc_adv_rx_desc *rx_desc, 1805 struct sk_buff *skb) 1806 { 1807 if (ring->netdev->features & NETIF_F_RXHASH) { 1808 u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); 1809 u32 rss_type = igc_rss_type(rx_desc); 1810 1811 skb_set_hash(skb, rss_hash, igc_rss_type_table[rss_type]); 1812 } 1813 } 1814 1815 static void igc_rx_vlan(struct igc_ring *rx_ring, 1816 union igc_adv_rx_desc *rx_desc, 1817 struct sk_buff *skb) 1818 { 1819 struct net_device *dev = rx_ring->netdev; 1820 u16 vid; 1821 1822 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && 1823 igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) { 1824 if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) && 1825 test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) 1826 vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); 1827 else 1828 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 1829 1830 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 1831 } 1832 } 1833 1834 /** 1835 * igc_process_skb_fields - Populate skb header fields from Rx descriptor 1836 * @rx_ring: rx descriptor ring packet is being transacted on 1837 * @rx_desc: pointer to the EOP Rx descriptor 1838 * @skb: pointer to current skb being populated 1839 * 1840 * This function checks the ring, descriptor, and packet information in order 1841 * to populate the hash, checksum, VLAN, protocol, and other fields within the 1842 * skb. 1843 */ 1844 static void igc_process_skb_fields(struct igc_ring *rx_ring, 1845 union igc_adv_rx_desc *rx_desc, 1846 struct sk_buff *skb) 1847 { 1848 igc_rx_hash(rx_ring, rx_desc, skb); 1849 1850 igc_rx_checksum(rx_ring, rx_desc, skb); 1851 1852 igc_rx_vlan(rx_ring, rx_desc, skb); 1853 1854 skb_record_rx_queue(skb, rx_ring->queue_index); 1855 1856 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1857 } 1858 1859 static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features) 1860 { 1861 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); 1862 struct igc_adapter *adapter = netdev_priv(netdev); 1863 struct igc_hw *hw = &adapter->hw; 1864 u32 ctrl; 1865 1866 ctrl = rd32(IGC_CTRL); 1867 1868 if (enable) { 1869 /* enable VLAN tag insert/strip */ 1870 ctrl |= IGC_CTRL_VME; 1871 } else { 1872 /* disable VLAN tag insert/strip */ 1873 ctrl &= ~IGC_CTRL_VME; 1874 } 1875 wr32(IGC_CTRL, ctrl); 1876 } 1877 1878 static void igc_restore_vlan(struct igc_adapter *adapter) 1879 { 1880 igc_vlan_mode(adapter->netdev, adapter->netdev->features); 1881 } 1882 1883 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, 1884 const unsigned int size, 1885 int *rx_buffer_pgcnt) 1886 { 1887 struct igc_rx_buffer *rx_buffer; 1888 1889 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 1890 *rx_buffer_pgcnt = 1891 #if (PAGE_SIZE < 8192) 1892 page_count(rx_buffer->page); 1893 #else 1894 0; 1895 #endif 1896 prefetchw(rx_buffer->page); 1897 1898 /* we are reusing so sync this buffer for CPU use */ 1899 dma_sync_single_range_for_cpu(rx_ring->dev, 1900 rx_buffer->dma, 1901 rx_buffer->page_offset, 1902 size, 1903 DMA_FROM_DEVICE); 1904 1905 rx_buffer->pagecnt_bias--; 1906 1907 return rx_buffer; 1908 } 1909 1910 static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer, 1911 unsigned int truesize) 1912 { 1913 #if (PAGE_SIZE < 8192) 1914 buffer->page_offset ^= truesize; 1915 #else 1916 buffer->page_offset += truesize; 1917 #endif 1918 } 1919 1920 static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring, 1921 unsigned int size) 1922 { 1923 unsigned int truesize; 1924 1925 #if (PAGE_SIZE < 8192) 1926 truesize = igc_rx_pg_size(ring) / 2; 1927 #else 1928 truesize = ring_uses_build_skb(ring) ? 1929 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1930 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1931 SKB_DATA_ALIGN(size); 1932 #endif 1933 return truesize; 1934 } 1935 1936 /** 1937 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff 1938 * @rx_ring: rx descriptor ring to transact packets on 1939 * @rx_buffer: buffer containing page to add 1940 * @skb: sk_buff to place the data into 1941 * @size: size of buffer to be added 1942 * 1943 * This function will add the data contained in rx_buffer->page to the skb. 1944 */ 1945 static void igc_add_rx_frag(struct igc_ring *rx_ring, 1946 struct igc_rx_buffer *rx_buffer, 1947 struct sk_buff *skb, 1948 unsigned int size) 1949 { 1950 unsigned int truesize; 1951 1952 #if (PAGE_SIZE < 8192) 1953 truesize = igc_rx_pg_size(rx_ring) / 2; 1954 #else 1955 truesize = ring_uses_build_skb(rx_ring) ? 1956 SKB_DATA_ALIGN(IGC_SKB_PAD + size) : 1957 SKB_DATA_ALIGN(size); 1958 #endif 1959 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1960 rx_buffer->page_offset, size, truesize); 1961 1962 igc_rx_buffer_flip(rx_buffer, truesize); 1963 } 1964 1965 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, 1966 struct igc_rx_buffer *rx_buffer, 1967 struct xdp_buff *xdp) 1968 { 1969 unsigned int size = xdp->data_end - xdp->data; 1970 unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); 1971 unsigned int metasize = xdp->data - xdp->data_meta; 1972 struct sk_buff *skb; 1973 1974 /* prefetch first cache line of first page */ 1975 net_prefetch(xdp->data_meta); 1976 1977 /* build an skb around the page buffer */ 1978 skb = napi_build_skb(xdp->data_hard_start, truesize); 1979 if (unlikely(!skb)) 1980 return NULL; 1981 1982 /* update pointers within the skb to store the data */ 1983 skb_reserve(skb, xdp->data - xdp->data_hard_start); 1984 __skb_put(skb, size); 1985 if (metasize) 1986 skb_metadata_set(skb, metasize); 1987 1988 igc_rx_buffer_flip(rx_buffer, truesize); 1989 return skb; 1990 } 1991 1992 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, 1993 struct igc_rx_buffer *rx_buffer, 1994 struct igc_xdp_buff *ctx) 1995 { 1996 struct xdp_buff *xdp = &ctx->xdp; 1997 unsigned int metasize = xdp->data - xdp->data_meta; 1998 unsigned int size = xdp->data_end - xdp->data; 1999 unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); 2000 void *va = xdp->data; 2001 unsigned int headlen; 2002 struct sk_buff *skb; 2003 2004 /* prefetch first cache line of first page */ 2005 net_prefetch(xdp->data_meta); 2006 2007 /* allocate a skb to store the frags */ 2008 skb = napi_alloc_skb(&rx_ring->q_vector->napi, 2009 IGC_RX_HDR_LEN + metasize); 2010 if (unlikely(!skb)) 2011 return NULL; 2012 2013 if (ctx->rx_ts) { 2014 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV; 2015 skb_hwtstamps(skb)->netdev_data = ctx->rx_ts; 2016 } 2017 2018 /* Determine available headroom for copy */ 2019 headlen = size; 2020 if (headlen > IGC_RX_HDR_LEN) 2021 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); 2022 2023 /* align pull length to size of long to optimize memcpy performance */ 2024 memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta, 2025 ALIGN(headlen + metasize, sizeof(long))); 2026 2027 if (metasize) { 2028 skb_metadata_set(skb, metasize); 2029 __skb_pull(skb, metasize); 2030 } 2031 2032 /* update all of the pointers */ 2033 size -= headlen; 2034 if (size) { 2035 skb_add_rx_frag(skb, 0, rx_buffer->page, 2036 (va + headlen) - page_address(rx_buffer->page), 2037 size, truesize); 2038 igc_rx_buffer_flip(rx_buffer, truesize); 2039 } else { 2040 rx_buffer->pagecnt_bias++; 2041 } 2042 2043 return skb; 2044 } 2045 2046 /** 2047 * igc_reuse_rx_page - page flip buffer and store it back on the ring 2048 * @rx_ring: rx descriptor ring to store buffers on 2049 * @old_buff: donor buffer to have page reused 2050 * 2051 * Synchronizes page for reuse by the adapter 2052 */ 2053 static void igc_reuse_rx_page(struct igc_ring *rx_ring, 2054 struct igc_rx_buffer *old_buff) 2055 { 2056 u16 nta = rx_ring->next_to_alloc; 2057 struct igc_rx_buffer *new_buff; 2058 2059 new_buff = &rx_ring->rx_buffer_info[nta]; 2060 2061 /* update, and store next to alloc */ 2062 nta++; 2063 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 2064 2065 /* Transfer page from old buffer to new buffer. 2066 * Move each member individually to avoid possible store 2067 * forwarding stalls. 2068 */ 2069 new_buff->dma = old_buff->dma; 2070 new_buff->page = old_buff->page; 2071 new_buff->page_offset = old_buff->page_offset; 2072 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 2073 } 2074 2075 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer, 2076 int rx_buffer_pgcnt) 2077 { 2078 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 2079 struct page *page = rx_buffer->page; 2080 2081 /* avoid re-using remote and pfmemalloc pages */ 2082 if (!dev_page_is_reusable(page)) 2083 return false; 2084 2085 #if (PAGE_SIZE < 8192) 2086 /* if we are only owner of page we can reuse it */ 2087 if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) 2088 return false; 2089 #else 2090 #define IGC_LAST_OFFSET \ 2091 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) 2092 2093 if (rx_buffer->page_offset > IGC_LAST_OFFSET) 2094 return false; 2095 #endif 2096 2097 /* If we have drained the page fragment pool we need to update 2098 * the pagecnt_bias and page count so that we fully restock the 2099 * number of references the driver holds. 2100 */ 2101 if (unlikely(pagecnt_bias == 1)) { 2102 page_ref_add(page, USHRT_MAX - 1); 2103 rx_buffer->pagecnt_bias = USHRT_MAX; 2104 } 2105 2106 return true; 2107 } 2108 2109 /** 2110 * igc_is_non_eop - process handling of non-EOP buffers 2111 * @rx_ring: Rx ring being processed 2112 * @rx_desc: Rx descriptor for current buffer 2113 * 2114 * This function updates next to clean. If the buffer is an EOP buffer 2115 * this function exits returning false, otherwise it will place the 2116 * sk_buff in the next buffer to be chained and return true indicating 2117 * that this is in fact a non-EOP buffer. 2118 */ 2119 static bool igc_is_non_eop(struct igc_ring *rx_ring, 2120 union igc_adv_rx_desc *rx_desc) 2121 { 2122 u32 ntc = rx_ring->next_to_clean + 1; 2123 2124 /* fetch, update, and store next to clean */ 2125 ntc = (ntc < rx_ring->count) ? ntc : 0; 2126 rx_ring->next_to_clean = ntc; 2127 2128 prefetch(IGC_RX_DESC(rx_ring, ntc)); 2129 2130 if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) 2131 return false; 2132 2133 return true; 2134 } 2135 2136 /** 2137 * igc_cleanup_headers - Correct corrupted or empty headers 2138 * @rx_ring: rx descriptor ring packet is being transacted on 2139 * @rx_desc: pointer to the EOP Rx descriptor 2140 * @skb: pointer to current skb being fixed 2141 * 2142 * Address the case where we are pulling data in on pages only 2143 * and as such no data is present in the skb header. 2144 * 2145 * In addition if skb is not at least 60 bytes we need to pad it so that 2146 * it is large enough to qualify as a valid Ethernet frame. 2147 * 2148 * Returns true if an error was encountered and skb was freed. 2149 */ 2150 static bool igc_cleanup_headers(struct igc_ring *rx_ring, 2151 union igc_adv_rx_desc *rx_desc, 2152 struct sk_buff *skb) 2153 { 2154 if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { 2155 struct net_device *netdev = rx_ring->netdev; 2156 2157 if (!(netdev->features & NETIF_F_RXALL)) { 2158 dev_kfree_skb_any(skb); 2159 return true; 2160 } 2161 } 2162 2163 /* if eth_skb_pad returns an error the skb was freed */ 2164 if (eth_skb_pad(skb)) 2165 return true; 2166 2167 return false; 2168 } 2169 2170 static void igc_put_rx_buffer(struct igc_ring *rx_ring, 2171 struct igc_rx_buffer *rx_buffer, 2172 int rx_buffer_pgcnt) 2173 { 2174 if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { 2175 /* hand second half of page back to the ring */ 2176 igc_reuse_rx_page(rx_ring, rx_buffer); 2177 } else { 2178 /* We are not reusing the buffer so unmap it and free 2179 * any references we are holding to it 2180 */ 2181 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 2182 igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE, 2183 IGC_RX_DMA_ATTR); 2184 __page_frag_cache_drain(rx_buffer->page, 2185 rx_buffer->pagecnt_bias); 2186 } 2187 2188 /* clear contents of rx_buffer */ 2189 rx_buffer->page = NULL; 2190 } 2191 2192 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) 2193 { 2194 struct igc_adapter *adapter = rx_ring->q_vector->adapter; 2195 2196 if (ring_uses_build_skb(rx_ring)) 2197 return IGC_SKB_PAD; 2198 if (igc_xdp_is_enabled(adapter)) 2199 return XDP_PACKET_HEADROOM; 2200 2201 return 0; 2202 } 2203 2204 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, 2205 struct igc_rx_buffer *bi) 2206 { 2207 struct page *page = bi->page; 2208 dma_addr_t dma; 2209 2210 /* since we are recycling buffers we should seldom need to alloc */ 2211 if (likely(page)) 2212 return true; 2213 2214 /* alloc new page for storage */ 2215 page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); 2216 if (unlikely(!page)) { 2217 rx_ring->rx_stats.alloc_failed++; 2218 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 2219 return false; 2220 } 2221 2222 /* map page for use */ 2223 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 2224 igc_rx_pg_size(rx_ring), 2225 DMA_FROM_DEVICE, 2226 IGC_RX_DMA_ATTR); 2227 2228 /* if mapping failed free memory back to system since 2229 * there isn't much point in holding memory we can't use 2230 */ 2231 if (dma_mapping_error(rx_ring->dev, dma)) { 2232 __free_page(page); 2233 2234 rx_ring->rx_stats.alloc_failed++; 2235 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 2236 return false; 2237 } 2238 2239 bi->dma = dma; 2240 bi->page = page; 2241 bi->page_offset = igc_rx_offset(rx_ring); 2242 page_ref_add(page, USHRT_MAX - 1); 2243 bi->pagecnt_bias = USHRT_MAX; 2244 2245 return true; 2246 } 2247 2248 /** 2249 * igc_alloc_rx_buffers - Replace used receive buffers; packet split 2250 * @rx_ring: rx descriptor ring 2251 * @cleaned_count: number of buffers to clean 2252 */ 2253 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) 2254 { 2255 union igc_adv_rx_desc *rx_desc; 2256 u16 i = rx_ring->next_to_use; 2257 struct igc_rx_buffer *bi; 2258 u16 bufsz; 2259 2260 /* nothing to do */ 2261 if (!cleaned_count) 2262 return; 2263 2264 rx_desc = IGC_RX_DESC(rx_ring, i); 2265 bi = &rx_ring->rx_buffer_info[i]; 2266 i -= rx_ring->count; 2267 2268 bufsz = igc_rx_bufsz(rx_ring); 2269 2270 do { 2271 if (!igc_alloc_mapped_page(rx_ring, bi)) 2272 break; 2273 2274 /* sync the buffer for use by the device */ 2275 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 2276 bi->page_offset, bufsz, 2277 DMA_FROM_DEVICE); 2278 2279 /* Refresh the desc even if buffer_addrs didn't change 2280 * because each write-back erases this info. 2281 */ 2282 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 2283 2284 rx_desc++; 2285 bi++; 2286 i++; 2287 if (unlikely(!i)) { 2288 rx_desc = IGC_RX_DESC(rx_ring, 0); 2289 bi = rx_ring->rx_buffer_info; 2290 i -= rx_ring->count; 2291 } 2292 2293 /* clear the length for the next_to_use descriptor */ 2294 rx_desc->wb.upper.length = 0; 2295 2296 cleaned_count--; 2297 } while (cleaned_count); 2298 2299 i += rx_ring->count; 2300 2301 if (rx_ring->next_to_use != i) { 2302 /* record the next descriptor to use */ 2303 rx_ring->next_to_use = i; 2304 2305 /* update next to alloc since we have filled the ring */ 2306 rx_ring->next_to_alloc = i; 2307 2308 /* Force memory writes to complete before letting h/w 2309 * know there are new descriptors to fetch. (Only 2310 * applicable for weak-ordered memory model archs, 2311 * such as IA-64). 2312 */ 2313 wmb(); 2314 writel(i, rx_ring->tail); 2315 } 2316 } 2317 2318 static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) 2319 { 2320 union igc_adv_rx_desc *desc; 2321 u16 i = ring->next_to_use; 2322 struct igc_rx_buffer *bi; 2323 dma_addr_t dma; 2324 bool ok = true; 2325 2326 if (!count) 2327 return ok; 2328 2329 XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff); 2330 2331 desc = IGC_RX_DESC(ring, i); 2332 bi = &ring->rx_buffer_info[i]; 2333 i -= ring->count; 2334 2335 do { 2336 bi->xdp = xsk_buff_alloc(ring->xsk_pool); 2337 if (!bi->xdp) { 2338 ok = false; 2339 break; 2340 } 2341 2342 dma = xsk_buff_xdp_get_dma(bi->xdp); 2343 desc->read.pkt_addr = cpu_to_le64(dma); 2344 2345 desc++; 2346 bi++; 2347 i++; 2348 if (unlikely(!i)) { 2349 desc = IGC_RX_DESC(ring, 0); 2350 bi = ring->rx_buffer_info; 2351 i -= ring->count; 2352 } 2353 2354 /* Clear the length for the next_to_use descriptor. */ 2355 desc->wb.upper.length = 0; 2356 2357 count--; 2358 } while (count); 2359 2360 i += ring->count; 2361 2362 if (ring->next_to_use != i) { 2363 ring->next_to_use = i; 2364 2365 /* Force memory writes to complete before letting h/w 2366 * know there are new descriptors to fetch. (Only 2367 * applicable for weak-ordered memory model archs, 2368 * such as IA-64). 2369 */ 2370 wmb(); 2371 writel(i, ring->tail); 2372 } 2373 2374 return ok; 2375 } 2376 2377 /* This function requires __netif_tx_lock is held by the caller. */ 2378 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring, 2379 struct xdp_frame *xdpf) 2380 { 2381 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); 2382 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; 2383 u16 count, index = ring->next_to_use; 2384 struct igc_tx_buffer *head = &ring->tx_buffer_info[index]; 2385 struct igc_tx_buffer *buffer = head; 2386 union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index); 2387 u32 olinfo_status, len = xdpf->len, cmd_type; 2388 void *data = xdpf->data; 2389 u16 i; 2390 2391 count = TXD_USE_COUNT(len); 2392 for (i = 0; i < nr_frags; i++) 2393 count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); 2394 2395 if (igc_maybe_stop_tx(ring, count + 3)) { 2396 /* this is a hard error */ 2397 return -EBUSY; 2398 } 2399 2400 i = 0; 2401 head->bytecount = xdp_get_frame_len(xdpf); 2402 head->type = IGC_TX_BUFFER_TYPE_XDP; 2403 head->gso_segs = 1; 2404 head->xdpf = xdpf; 2405 2406 olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; 2407 desc->read.olinfo_status = cpu_to_le32(olinfo_status); 2408 2409 for (;;) { 2410 dma_addr_t dma; 2411 2412 dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); 2413 if (dma_mapping_error(ring->dev, dma)) { 2414 netdev_err_once(ring->netdev, 2415 "Failed to map DMA for TX\n"); 2416 goto unmap; 2417 } 2418 2419 dma_unmap_len_set(buffer, len, len); 2420 dma_unmap_addr_set(buffer, dma, dma); 2421 2422 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | 2423 IGC_ADVTXD_DCMD_IFCS | len; 2424 2425 desc->read.cmd_type_len = cpu_to_le32(cmd_type); 2426 desc->read.buffer_addr = cpu_to_le64(dma); 2427 2428 buffer->protocol = 0; 2429 2430 if (++index == ring->count) 2431 index = 0; 2432 2433 if (i == nr_frags) 2434 break; 2435 2436 buffer = &ring->tx_buffer_info[index]; 2437 desc = IGC_TX_DESC(ring, index); 2438 desc->read.olinfo_status = 0; 2439 2440 data = skb_frag_address(&sinfo->frags[i]); 2441 len = skb_frag_size(&sinfo->frags[i]); 2442 i++; 2443 } 2444 desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD); 2445 2446 netdev_tx_sent_queue(txring_txq(ring), head->bytecount); 2447 /* set the timestamp */ 2448 head->time_stamp = jiffies; 2449 /* set next_to_watch value indicating a packet is present */ 2450 head->next_to_watch = desc; 2451 ring->next_to_use = index; 2452 2453 return 0; 2454 2455 unmap: 2456 for (;;) { 2457 buffer = &ring->tx_buffer_info[index]; 2458 if (dma_unmap_len(buffer, len)) 2459 dma_unmap_page(ring->dev, 2460 dma_unmap_addr(buffer, dma), 2461 dma_unmap_len(buffer, len), 2462 DMA_TO_DEVICE); 2463 dma_unmap_len_set(buffer, len, 0); 2464 if (buffer == head) 2465 break; 2466 2467 if (!index) 2468 index += ring->count; 2469 index--; 2470 } 2471 2472 return -ENOMEM; 2473 } 2474 2475 struct igc_ring *igc_get_tx_ring(struct igc_adapter *adapter, int cpu) 2476 { 2477 int index = cpu; 2478 2479 if (unlikely(index < 0)) 2480 index = 0; 2481 2482 while (index >= adapter->num_tx_queues) 2483 index -= adapter->num_tx_queues; 2484 2485 return adapter->tx_ring[index]; 2486 } 2487 2488 static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp) 2489 { 2490 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 2491 int cpu = smp_processor_id(); 2492 struct netdev_queue *nq; 2493 struct igc_ring *ring; 2494 int res; 2495 2496 if (unlikely(!xdpf)) 2497 return -EFAULT; 2498 2499 ring = igc_get_tx_ring(adapter, cpu); 2500 nq = txring_txq(ring); 2501 2502 __netif_tx_lock(nq, cpu); 2503 /* Avoid transmit queue timeout since we share it with the slow path */ 2504 txq_trans_cond_update(nq); 2505 res = igc_xdp_init_tx_descriptor(ring, xdpf); 2506 __netif_tx_unlock(nq); 2507 return res; 2508 } 2509 2510 /* This function assumes rcu_read_lock() is held by the caller. */ 2511 static int __igc_xdp_run_prog(struct igc_adapter *adapter, 2512 struct bpf_prog *prog, 2513 struct xdp_buff *xdp) 2514 { 2515 u32 act = bpf_prog_run_xdp(prog, xdp); 2516 2517 switch (act) { 2518 case XDP_PASS: 2519 return IGC_XDP_PASS; 2520 case XDP_TX: 2521 if (igc_xdp_xmit_back(adapter, xdp) < 0) 2522 goto out_failure; 2523 return IGC_XDP_TX; 2524 case XDP_REDIRECT: 2525 if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0) 2526 goto out_failure; 2527 return IGC_XDP_REDIRECT; 2528 break; 2529 default: 2530 bpf_warn_invalid_xdp_action(adapter->netdev, prog, act); 2531 fallthrough; 2532 case XDP_ABORTED: 2533 out_failure: 2534 trace_xdp_exception(adapter->netdev, prog, act); 2535 fallthrough; 2536 case XDP_DROP: 2537 return IGC_XDP_CONSUMED; 2538 } 2539 } 2540 2541 static int igc_xdp_run_prog(struct igc_adapter *adapter, struct xdp_buff *xdp) 2542 { 2543 struct bpf_prog *prog; 2544 int res; 2545 2546 prog = READ_ONCE(adapter->xdp_prog); 2547 if (!prog) { 2548 res = IGC_XDP_PASS; 2549 goto out; 2550 } 2551 2552 res = __igc_xdp_run_prog(adapter, prog, xdp); 2553 2554 out: 2555 return res; 2556 } 2557 2558 /* This function assumes __netif_tx_lock is held by the caller. */ 2559 void igc_flush_tx_descriptors(struct igc_ring *ring) 2560 { 2561 /* Once tail pointer is updated, hardware can fetch the descriptors 2562 * any time so we issue a write membar here to ensure all memory 2563 * writes are complete before the tail pointer is updated. 2564 */ 2565 wmb(); 2566 writel(ring->next_to_use, ring->tail); 2567 } 2568 2569 static void igc_finalize_xdp(struct igc_adapter *adapter, int status) 2570 { 2571 int cpu = smp_processor_id(); 2572 struct netdev_queue *nq; 2573 struct igc_ring *ring; 2574 2575 if (status & IGC_XDP_TX) { 2576 ring = igc_get_tx_ring(adapter, cpu); 2577 nq = txring_txq(ring); 2578 2579 __netif_tx_lock(nq, cpu); 2580 igc_flush_tx_descriptors(ring); 2581 __netif_tx_unlock(nq); 2582 } 2583 2584 if (status & IGC_XDP_REDIRECT) 2585 xdp_do_flush(); 2586 } 2587 2588 static void igc_update_rx_stats(struct igc_q_vector *q_vector, 2589 unsigned int packets, unsigned int bytes) 2590 { 2591 struct igc_ring *ring = q_vector->rx.ring; 2592 2593 u64_stats_update_begin(&ring->rx_syncp); 2594 ring->rx_stats.packets += packets; 2595 ring->rx_stats.bytes += bytes; 2596 u64_stats_update_end(&ring->rx_syncp); 2597 2598 q_vector->rx.total_packets += packets; 2599 q_vector->rx.total_bytes += bytes; 2600 } 2601 2602 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) 2603 { 2604 unsigned int total_bytes = 0, total_packets = 0; 2605 struct igc_adapter *adapter = q_vector->adapter; 2606 struct igc_ring *rx_ring = q_vector->rx.ring; 2607 struct sk_buff *skb = rx_ring->skb; 2608 u16 cleaned_count = igc_desc_unused(rx_ring); 2609 int xdp_status = 0, rx_buffer_pgcnt; 2610 int xdp_res = 0; 2611 2612 while (likely(total_packets < budget)) { 2613 struct igc_xdp_buff ctx = { .rx_ts = NULL }; 2614 struct igc_rx_buffer *rx_buffer; 2615 union igc_adv_rx_desc *rx_desc; 2616 unsigned int size, truesize; 2617 int pkt_offset = 0; 2618 void *pktbuf; 2619 2620 /* return some buffers to hardware, one at a time is too slow */ 2621 if (cleaned_count >= IGC_RX_BUFFER_WRITE) { 2622 igc_alloc_rx_buffers(rx_ring, cleaned_count); 2623 cleaned_count = 0; 2624 } 2625 2626 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); 2627 size = le16_to_cpu(rx_desc->wb.upper.length); 2628 if (!size) 2629 break; 2630 2631 /* This memory barrier is needed to keep us from reading 2632 * any other fields out of the rx_desc until we know the 2633 * descriptor has been written back 2634 */ 2635 dma_rmb(); 2636 2637 rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt); 2638 truesize = igc_get_rx_frame_truesize(rx_ring, size); 2639 2640 pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; 2641 2642 if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) { 2643 ctx.rx_ts = pktbuf; 2644 pkt_offset = IGC_TS_HDR_LEN; 2645 size -= IGC_TS_HDR_LEN; 2646 } 2647 2648 if (igc_fpe_is_pmac_enabled(adapter) && 2649 igc_fpe_handle_mpacket(adapter, rx_desc, size, pktbuf + pkt_offset)) { 2650 /* Advance the ring next-to-clean */ 2651 igc_is_non_eop(rx_ring, rx_desc); 2652 cleaned_count++; 2653 continue; 2654 } 2655 2656 if (!skb) { 2657 xdp_init_buff(&ctx.xdp, truesize, &rx_ring->xdp_rxq); 2658 xdp_prepare_buff(&ctx.xdp, pktbuf - igc_rx_offset(rx_ring), 2659 igc_rx_offset(rx_ring) + pkt_offset, 2660 size, true); 2661 xdp_buff_clear_frags_flag(&ctx.xdp); 2662 ctx.rx_desc = rx_desc; 2663 2664 xdp_res = igc_xdp_run_prog(adapter, &ctx.xdp); 2665 } 2666 2667 if (xdp_res) { 2668 switch (xdp_res) { 2669 case IGC_XDP_CONSUMED: 2670 rx_buffer->pagecnt_bias++; 2671 break; 2672 case IGC_XDP_TX: 2673 case IGC_XDP_REDIRECT: 2674 igc_rx_buffer_flip(rx_buffer, truesize); 2675 xdp_status |= xdp_res; 2676 break; 2677 } 2678 2679 total_packets++; 2680 total_bytes += size; 2681 } else if (skb) 2682 igc_add_rx_frag(rx_ring, rx_buffer, skb, size); 2683 else if (ring_uses_build_skb(rx_ring)) 2684 skb = igc_build_skb(rx_ring, rx_buffer, &ctx.xdp); 2685 else 2686 skb = igc_construct_skb(rx_ring, rx_buffer, &ctx); 2687 2688 /* exit if we failed to retrieve a buffer */ 2689 if (!xdp_res && !skb) { 2690 rx_ring->rx_stats.alloc_failed++; 2691 rx_buffer->pagecnt_bias++; 2692 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 2693 break; 2694 } 2695 2696 igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt); 2697 cleaned_count++; 2698 2699 /* fetch next buffer in frame if non-eop */ 2700 if (igc_is_non_eop(rx_ring, rx_desc)) 2701 continue; 2702 2703 /* verify the packet layout is correct */ 2704 if (xdp_res || igc_cleanup_headers(rx_ring, rx_desc, skb)) { 2705 skb = NULL; 2706 continue; 2707 } 2708 2709 /* probably a little skewed due to removing CRC */ 2710 total_bytes += skb->len; 2711 2712 /* populate checksum, VLAN, and protocol */ 2713 igc_process_skb_fields(rx_ring, rx_desc, skb); 2714 2715 napi_gro_receive(&q_vector->napi, skb); 2716 2717 /* reset skb pointer */ 2718 skb = NULL; 2719 2720 /* update budget accounting */ 2721 total_packets++; 2722 } 2723 2724 if (xdp_status) 2725 igc_finalize_xdp(adapter, xdp_status); 2726 2727 /* place incomplete frames back on ring for completion */ 2728 rx_ring->skb = skb; 2729 2730 igc_update_rx_stats(q_vector, total_packets, total_bytes); 2731 2732 if (cleaned_count) 2733 igc_alloc_rx_buffers(rx_ring, cleaned_count); 2734 2735 return total_packets; 2736 } 2737 2738 static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, 2739 struct igc_xdp_buff *ctx) 2740 { 2741 struct xdp_buff *xdp = &ctx->xdp; 2742 unsigned int totalsize = xdp->data_end - xdp->data_meta; 2743 unsigned int metasize = xdp->data - xdp->data_meta; 2744 struct sk_buff *skb; 2745 2746 net_prefetch(xdp->data_meta); 2747 2748 skb = napi_alloc_skb(&ring->q_vector->napi, totalsize); 2749 if (unlikely(!skb)) 2750 return NULL; 2751 2752 memcpy(__skb_put(skb, totalsize), xdp->data_meta, 2753 ALIGN(totalsize, sizeof(long))); 2754 2755 if (metasize) { 2756 skb_metadata_set(skb, metasize); 2757 __skb_pull(skb, metasize); 2758 } 2759 2760 if (ctx->rx_ts) { 2761 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV; 2762 skb_hwtstamps(skb)->netdev_data = ctx->rx_ts; 2763 } 2764 2765 return skb; 2766 } 2767 2768 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, 2769 union igc_adv_rx_desc *desc, 2770 struct igc_xdp_buff *ctx) 2771 { 2772 struct igc_ring *ring = q_vector->rx.ring; 2773 struct sk_buff *skb; 2774 2775 skb = igc_construct_skb_zc(ring, ctx); 2776 if (!skb) { 2777 ring->rx_stats.alloc_failed++; 2778 set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &ring->flags); 2779 return; 2780 } 2781 2782 if (igc_cleanup_headers(ring, desc, skb)) 2783 return; 2784 2785 igc_process_skb_fields(ring, desc, skb); 2786 napi_gro_receive(&q_vector->napi, skb); 2787 } 2788 2789 static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp) 2790 { 2791 /* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The 2792 * igc_xdp_buff shares its layout with xdp_buff_xsk and private 2793 * igc_xdp_buff fields fall into xdp_buff_xsk->cb 2794 */ 2795 return (struct igc_xdp_buff *)xdp; 2796 } 2797 2798 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) 2799 { 2800 struct igc_adapter *adapter = q_vector->adapter; 2801 struct igc_ring *ring = q_vector->rx.ring; 2802 u16 cleaned_count = igc_desc_unused(ring); 2803 int total_bytes = 0, total_packets = 0; 2804 u16 ntc = ring->next_to_clean; 2805 struct bpf_prog *prog; 2806 bool failure = false; 2807 int xdp_status = 0; 2808 2809 rcu_read_lock(); 2810 2811 prog = READ_ONCE(adapter->xdp_prog); 2812 2813 while (likely(total_packets < budget)) { 2814 union igc_adv_rx_desc *desc; 2815 struct igc_rx_buffer *bi; 2816 struct igc_xdp_buff *ctx; 2817 unsigned int size; 2818 int res; 2819 2820 desc = IGC_RX_DESC(ring, ntc); 2821 size = le16_to_cpu(desc->wb.upper.length); 2822 if (!size) 2823 break; 2824 2825 /* This memory barrier is needed to keep us from reading 2826 * any other fields out of the rx_desc until we know the 2827 * descriptor has been written back 2828 */ 2829 dma_rmb(); 2830 2831 bi = &ring->rx_buffer_info[ntc]; 2832 2833 ctx = xsk_buff_to_igc_ctx(bi->xdp); 2834 ctx->rx_desc = desc; 2835 2836 if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) { 2837 ctx->rx_ts = bi->xdp->data; 2838 2839 bi->xdp->data += IGC_TS_HDR_LEN; 2840 2841 /* HW timestamp has been copied into local variable. Metadata 2842 * length when XDP program is called should be 0. 2843 */ 2844 bi->xdp->data_meta += IGC_TS_HDR_LEN; 2845 size -= IGC_TS_HDR_LEN; 2846 } else { 2847 ctx->rx_ts = NULL; 2848 } 2849 2850 bi->xdp->data_end = bi->xdp->data + size; 2851 xsk_buff_dma_sync_for_cpu(bi->xdp); 2852 2853 res = __igc_xdp_run_prog(adapter, prog, bi->xdp); 2854 switch (res) { 2855 case IGC_XDP_PASS: 2856 igc_dispatch_skb_zc(q_vector, desc, ctx); 2857 fallthrough; 2858 case IGC_XDP_CONSUMED: 2859 xsk_buff_free(bi->xdp); 2860 break; 2861 case IGC_XDP_TX: 2862 case IGC_XDP_REDIRECT: 2863 xdp_status |= res; 2864 break; 2865 } 2866 2867 bi->xdp = NULL; 2868 total_bytes += size; 2869 total_packets++; 2870 cleaned_count++; 2871 ntc++; 2872 if (ntc == ring->count) 2873 ntc = 0; 2874 } 2875 2876 ring->next_to_clean = ntc; 2877 rcu_read_unlock(); 2878 2879 if (cleaned_count >= IGC_RX_BUFFER_WRITE) 2880 failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count); 2881 2882 if (xdp_status) 2883 igc_finalize_xdp(adapter, xdp_status); 2884 2885 igc_update_rx_stats(q_vector, total_packets, total_bytes); 2886 2887 if (xsk_uses_need_wakeup(ring->xsk_pool)) { 2888 if (failure || ring->next_to_clean == ring->next_to_use) 2889 xsk_set_rx_need_wakeup(ring->xsk_pool); 2890 else 2891 xsk_clear_rx_need_wakeup(ring->xsk_pool); 2892 return total_packets; 2893 } 2894 2895 return failure ? budget : total_packets; 2896 } 2897 2898 static void igc_update_tx_stats(struct igc_q_vector *q_vector, 2899 unsigned int packets, unsigned int bytes) 2900 { 2901 struct igc_ring *ring = q_vector->tx.ring; 2902 2903 u64_stats_update_begin(&ring->tx_syncp); 2904 ring->tx_stats.bytes += bytes; 2905 ring->tx_stats.packets += packets; 2906 u64_stats_update_end(&ring->tx_syncp); 2907 2908 q_vector->tx.total_bytes += bytes; 2909 q_vector->tx.total_packets += packets; 2910 } 2911 2912 static void igc_xsk_request_timestamp(void *_priv) 2913 { 2914 struct igc_metadata_request *meta_req = _priv; 2915 struct igc_ring *tx_ring = meta_req->tx_ring; 2916 struct igc_tx_timestamp_request *tstamp; 2917 u32 tx_flags = IGC_TX_FLAGS_TSTAMP; 2918 struct igc_adapter *adapter; 2919 unsigned long lock_flags; 2920 bool found = false; 2921 int i; 2922 2923 if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) { 2924 adapter = netdev_priv(tx_ring->netdev); 2925 2926 spin_lock_irqsave(&adapter->ptp_tx_lock, lock_flags); 2927 2928 /* Search for available tstamp regs */ 2929 for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { 2930 tstamp = &adapter->tx_tstamp[i]; 2931 2932 /* tstamp->skb and tstamp->xsk_tx_buffer are in union. 2933 * When tstamp->skb is equal to NULL, 2934 * tstamp->xsk_tx_buffer is equal to NULL as well. 2935 * This condition means that the particular tstamp reg 2936 * is not occupied by other packet. 2937 */ 2938 if (!tstamp->skb) { 2939 found = true; 2940 break; 2941 } 2942 } 2943 2944 /* Return if no available tstamp regs */ 2945 if (!found) { 2946 adapter->tx_hwtstamp_skipped++; 2947 spin_unlock_irqrestore(&adapter->ptp_tx_lock, 2948 lock_flags); 2949 return; 2950 } 2951 2952 tstamp->start = jiffies; 2953 tstamp->xsk_queue_index = tx_ring->queue_index; 2954 tstamp->xsk_tx_buffer = meta_req->tx_buffer; 2955 tstamp->buffer_type = IGC_TX_BUFFER_TYPE_XSK; 2956 2957 /* Hold the transmit completion until timestamp is ready */ 2958 meta_req->tx_buffer->xsk_pending_ts = true; 2959 2960 /* Keep the pointer to tx_timestamp, which is located in XDP 2961 * metadata area. It is the location to store the value of 2962 * tx hardware timestamp. 2963 */ 2964 xsk_tx_metadata_to_compl(meta_req->meta, &tstamp->xsk_meta); 2965 2966 /* Set timestamp bit based on the _TSTAMP(_X) bit. */ 2967 tx_flags |= tstamp->flags; 2968 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2969 IGC_TX_FLAGS_TSTAMP, 2970 (IGC_ADVTXD_MAC_TSTAMP)); 2971 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2972 IGC_TX_FLAGS_TSTAMP_1, 2973 (IGC_ADVTXD_TSTAMP_REG_1)); 2974 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2975 IGC_TX_FLAGS_TSTAMP_2, 2976 (IGC_ADVTXD_TSTAMP_REG_2)); 2977 meta_req->cmd_type |= IGC_SET_FLAG(tx_flags, 2978 IGC_TX_FLAGS_TSTAMP_3, 2979 (IGC_ADVTXD_TSTAMP_REG_3)); 2980 2981 spin_unlock_irqrestore(&adapter->ptp_tx_lock, lock_flags); 2982 } 2983 } 2984 2985 static u64 igc_xsk_fill_timestamp(void *_priv) 2986 { 2987 return *(u64 *)_priv; 2988 } 2989 2990 static void igc_xsk_request_launch_time(u64 launch_time, void *_priv) 2991 { 2992 struct igc_metadata_request *meta_req = _priv; 2993 struct igc_ring *tx_ring = meta_req->tx_ring; 2994 __le32 launch_time_offset; 2995 bool insert_empty = false; 2996 bool first_flag = false; 2997 u16 used_desc = 0; 2998 2999 if (!tx_ring->launchtime_enable) 3000 return; 3001 3002 launch_time_offset = igc_tx_launchtime(tx_ring, 3003 ns_to_ktime(launch_time), 3004 &first_flag, &insert_empty); 3005 if (insert_empty) { 3006 /* Disregard the launch time request if the required empty frame 3007 * fails to be inserted. 3008 */ 3009 if (igc_insert_empty_frame(tx_ring)) 3010 return; 3011 3012 meta_req->tx_buffer = 3013 &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3014 /* Inserting an empty packet requires two descriptors: 3015 * one data descriptor and one context descriptor. 3016 */ 3017 used_desc += 2; 3018 } 3019 3020 /* Use one context descriptor to specify launch time and first flag. */ 3021 igc_tx_ctxtdesc(tx_ring, launch_time_offset, first_flag, 0, 0, 0); 3022 used_desc += 1; 3023 3024 /* Update the number of used descriptors in this request */ 3025 meta_req->used_desc += used_desc; 3026 } 3027 3028 const struct xsk_tx_metadata_ops igc_xsk_tx_metadata_ops = { 3029 .tmo_request_timestamp = igc_xsk_request_timestamp, 3030 .tmo_fill_timestamp = igc_xsk_fill_timestamp, 3031 .tmo_request_launch_time = igc_xsk_request_launch_time, 3032 }; 3033 3034 static void igc_xdp_xmit_zc(struct igc_ring *ring) 3035 { 3036 struct xsk_buff_pool *pool = ring->xsk_pool; 3037 struct netdev_queue *nq = txring_txq(ring); 3038 union igc_adv_tx_desc *tx_desc = NULL; 3039 int cpu = smp_processor_id(); 3040 struct xdp_desc xdp_desc; 3041 u16 budget, ntu; 3042 3043 if (!netif_carrier_ok(ring->netdev)) 3044 return; 3045 3046 __netif_tx_lock(nq, cpu); 3047 3048 /* Avoid transmit queue timeout since we share it with the slow path */ 3049 txq_trans_cond_update(nq); 3050 3051 ntu = ring->next_to_use; 3052 budget = igc_desc_unused(ring); 3053 3054 /* Packets with launch time require one data descriptor and one context 3055 * descriptor. When the launch time falls into the next Qbv cycle, we 3056 * may need to insert an empty packet, which requires two more 3057 * descriptors. Therefore, to be safe, we always ensure we have at least 3058 * 4 descriptors available. 3059 */ 3060 while (budget >= 4 && xsk_tx_peek_desc(pool, &xdp_desc)) { 3061 struct igc_metadata_request meta_req; 3062 struct xsk_tx_metadata *meta = NULL; 3063 struct igc_tx_buffer *bi; 3064 u32 olinfo_status; 3065 dma_addr_t dma; 3066 3067 meta_req.cmd_type = IGC_ADVTXD_DTYP_DATA | 3068 IGC_ADVTXD_DCMD_DEXT | 3069 IGC_ADVTXD_DCMD_IFCS | 3070 IGC_TXD_DCMD | xdp_desc.len; 3071 olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; 3072 3073 dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 3074 meta = xsk_buff_get_metadata(pool, xdp_desc.addr); 3075 xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); 3076 bi = &ring->tx_buffer_info[ntu]; 3077 3078 meta_req.tx_ring = ring; 3079 meta_req.tx_buffer = bi; 3080 meta_req.meta = meta; 3081 meta_req.used_desc = 0; 3082 xsk_tx_metadata_request(pool, &meta, &igc_xsk_tx_metadata_ops, 3083 &meta_req); 3084 3085 /* xsk_tx_metadata_request() may have updated next_to_use */ 3086 ntu = ring->next_to_use; 3087 3088 /* xsk_tx_metadata_request() may have updated Tx buffer info */ 3089 bi = meta_req.tx_buffer; 3090 3091 /* xsk_tx_metadata_request() may use a few descriptors */ 3092 budget -= meta_req.used_desc; 3093 3094 tx_desc = IGC_TX_DESC(ring, ntu); 3095 tx_desc->read.cmd_type_len = cpu_to_le32(meta_req.cmd_type); 3096 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 3097 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3098 3099 bi->type = IGC_TX_BUFFER_TYPE_XSK; 3100 bi->protocol = 0; 3101 bi->bytecount = xdp_desc.len; 3102 bi->gso_segs = 1; 3103 bi->time_stamp = jiffies; 3104 bi->next_to_watch = tx_desc; 3105 3106 netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len); 3107 3108 ntu++; 3109 if (ntu == ring->count) 3110 ntu = 0; 3111 3112 ring->next_to_use = ntu; 3113 budget--; 3114 } 3115 3116 if (tx_desc) { 3117 igc_flush_tx_descriptors(ring); 3118 xsk_tx_release(pool); 3119 } 3120 3121 __netif_tx_unlock(nq); 3122 } 3123 3124 /** 3125 * igc_clean_tx_irq - Reclaim resources after transmit completes 3126 * @q_vector: pointer to q_vector containing needed info 3127 * @napi_budget: Used to determine if we are in netpoll 3128 * 3129 * returns true if ring is completely cleaned 3130 */ 3131 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) 3132 { 3133 struct igc_adapter *adapter = q_vector->adapter; 3134 unsigned int total_bytes = 0, total_packets = 0; 3135 unsigned int budget = q_vector->tx.work_limit; 3136 struct igc_ring *tx_ring = q_vector->tx.ring; 3137 unsigned int i = tx_ring->next_to_clean; 3138 struct igc_tx_buffer *tx_buffer; 3139 union igc_adv_tx_desc *tx_desc; 3140 u32 xsk_frames = 0; 3141 3142 if (test_bit(__IGC_DOWN, &adapter->state)) 3143 return true; 3144 3145 tx_buffer = &tx_ring->tx_buffer_info[i]; 3146 tx_desc = IGC_TX_DESC(tx_ring, i); 3147 i -= tx_ring->count; 3148 3149 do { 3150 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 3151 3152 /* if next_to_watch is not set then there is no work pending */ 3153 if (!eop_desc) 3154 break; 3155 3156 /* prevent any other reads prior to eop_desc */ 3157 smp_rmb(); 3158 3159 /* if DD is not set pending work has not been completed */ 3160 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) 3161 break; 3162 3163 if (igc_fpe_is_pmac_enabled(adapter) && 3164 igc_fpe_transmitted_smd_v(tx_desc)) 3165 ethtool_mmsv_event_handle(&adapter->fpe.mmsv, 3166 ETHTOOL_MMSV_LD_SENT_VERIFY_MPACKET); 3167 3168 /* Hold the completions while there's a pending tx hardware 3169 * timestamp request from XDP Tx metadata. 3170 */ 3171 if (tx_buffer->type == IGC_TX_BUFFER_TYPE_XSK && 3172 tx_buffer->xsk_pending_ts) 3173 break; 3174 3175 /* clear next_to_watch to prevent false hangs */ 3176 tx_buffer->next_to_watch = NULL; 3177 3178 /* update the statistics for this packet */ 3179 total_bytes += tx_buffer->bytecount; 3180 total_packets += tx_buffer->gso_segs; 3181 3182 switch (tx_buffer->type) { 3183 case IGC_TX_BUFFER_TYPE_XSK: 3184 xsk_frames++; 3185 break; 3186 case IGC_TX_BUFFER_TYPE_XDP: 3187 xdp_return_frame(tx_buffer->xdpf); 3188 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 3189 break; 3190 case IGC_TX_BUFFER_TYPE_SKB: 3191 napi_consume_skb(tx_buffer->skb, napi_budget); 3192 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 3193 break; 3194 default: 3195 netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); 3196 break; 3197 } 3198 3199 /* clear last DMA location and unmap remaining buffers */ 3200 while (tx_desc != eop_desc) { 3201 tx_buffer++; 3202 tx_desc++; 3203 i++; 3204 if (unlikely(!i)) { 3205 i -= tx_ring->count; 3206 tx_buffer = tx_ring->tx_buffer_info; 3207 tx_desc = IGC_TX_DESC(tx_ring, 0); 3208 } 3209 3210 /* unmap any remaining paged data */ 3211 if (dma_unmap_len(tx_buffer, len)) 3212 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); 3213 } 3214 3215 /* move us one more past the eop_desc for start of next pkt */ 3216 tx_buffer++; 3217 tx_desc++; 3218 i++; 3219 if (unlikely(!i)) { 3220 i -= tx_ring->count; 3221 tx_buffer = tx_ring->tx_buffer_info; 3222 tx_desc = IGC_TX_DESC(tx_ring, 0); 3223 } 3224 3225 /* issue prefetch for next Tx descriptor */ 3226 prefetch(tx_desc); 3227 3228 /* update budget accounting */ 3229 budget--; 3230 } while (likely(budget)); 3231 3232 netdev_tx_completed_queue(txring_txq(tx_ring), 3233 total_packets, total_bytes); 3234 3235 i += tx_ring->count; 3236 tx_ring->next_to_clean = i; 3237 3238 igc_update_tx_stats(q_vector, total_packets, total_bytes); 3239 3240 if (tx_ring->xsk_pool) { 3241 if (xsk_frames) 3242 xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); 3243 if (xsk_uses_need_wakeup(tx_ring->xsk_pool)) 3244 xsk_set_tx_need_wakeup(tx_ring->xsk_pool); 3245 igc_xdp_xmit_zc(tx_ring); 3246 } 3247 3248 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { 3249 struct igc_hw *hw = &adapter->hw; 3250 3251 /* Detect a transmit hang in hardware, this serializes the 3252 * check with the clearing of time_stamp and movement of i 3253 */ 3254 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 3255 if (tx_buffer->next_to_watch && 3256 time_after(jiffies, tx_buffer->time_stamp + 3257 (adapter->tx_timeout_factor * HZ)) && 3258 !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF) && 3259 (rd32(IGC_TDH(tx_ring->reg_idx)) != readl(tx_ring->tail)) && 3260 !tx_ring->oper_gate_closed) { 3261 /* detected Tx unit hang */ 3262 netdev_err(tx_ring->netdev, 3263 "Detected Tx Unit Hang\n" 3264 " Tx Queue <%d>\n" 3265 " TDH <%x>\n" 3266 " TDT <%x>\n" 3267 " next_to_use <%x>\n" 3268 " next_to_clean <%x>\n" 3269 "buffer_info[next_to_clean]\n" 3270 " time_stamp <%lx>\n" 3271 " next_to_watch <%p>\n" 3272 " jiffies <%lx>\n" 3273 " desc.status <%x>\n", 3274 tx_ring->queue_index, 3275 rd32(IGC_TDH(tx_ring->reg_idx)), 3276 readl(tx_ring->tail), 3277 tx_ring->next_to_use, 3278 tx_ring->next_to_clean, 3279 tx_buffer->time_stamp, 3280 tx_buffer->next_to_watch, 3281 jiffies, 3282 tx_buffer->next_to_watch->wb.status); 3283 netif_stop_subqueue(tx_ring->netdev, 3284 tx_ring->queue_index); 3285 3286 /* we are about to reset, no point in enabling stuff */ 3287 return true; 3288 } 3289 } 3290 3291 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 3292 if (unlikely(total_packets && 3293 netif_carrier_ok(tx_ring->netdev) && 3294 igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { 3295 /* Make sure that anybody stopping the queue after this 3296 * sees the new next_to_clean. 3297 */ 3298 smp_mb(); 3299 if (__netif_subqueue_stopped(tx_ring->netdev, 3300 tx_ring->queue_index) && 3301 !(test_bit(__IGC_DOWN, &adapter->state))) { 3302 netif_wake_subqueue(tx_ring->netdev, 3303 tx_ring->queue_index); 3304 3305 u64_stats_update_begin(&tx_ring->tx_syncp); 3306 tx_ring->tx_stats.restart_queue++; 3307 u64_stats_update_end(&tx_ring->tx_syncp); 3308 } 3309 } 3310 3311 return !!budget; 3312 } 3313 3314 static int igc_find_mac_filter(struct igc_adapter *adapter, 3315 enum igc_mac_filter_type type, const u8 *addr) 3316 { 3317 struct igc_hw *hw = &adapter->hw; 3318 int max_entries = hw->mac.rar_entry_count; 3319 u32 ral, rah; 3320 int i; 3321 3322 for (i = 0; i < max_entries; i++) { 3323 ral = rd32(IGC_RAL(i)); 3324 rah = rd32(IGC_RAH(i)); 3325 3326 if (!(rah & IGC_RAH_AV)) 3327 continue; 3328 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) 3329 continue; 3330 if ((rah & IGC_RAH_RAH_MASK) != 3331 le16_to_cpup((__le16 *)(addr + 4))) 3332 continue; 3333 if (ral != le32_to_cpup((__le32 *)(addr))) 3334 continue; 3335 3336 return i; 3337 } 3338 3339 return -1; 3340 } 3341 3342 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) 3343 { 3344 struct igc_hw *hw = &adapter->hw; 3345 int max_entries = hw->mac.rar_entry_count; 3346 u32 rah; 3347 int i; 3348 3349 for (i = 0; i < max_entries; i++) { 3350 rah = rd32(IGC_RAH(i)); 3351 3352 if (!(rah & IGC_RAH_AV)) 3353 return i; 3354 } 3355 3356 return -1; 3357 } 3358 3359 /** 3360 * igc_add_mac_filter() - Add MAC address filter 3361 * @adapter: Pointer to adapter where the filter should be added 3362 * @type: MAC address filter type (source or destination) 3363 * @addr: MAC address 3364 * @queue: If non-negative, queue assignment feature is enabled and frames 3365 * matching the filter are enqueued onto 'queue'. Otherwise, queue 3366 * assignment is disabled. 3367 * 3368 * Return: 0 in case of success, negative errno code otherwise. 3369 */ 3370 static int igc_add_mac_filter(struct igc_adapter *adapter, 3371 enum igc_mac_filter_type type, const u8 *addr, 3372 int queue) 3373 { 3374 struct net_device *dev = adapter->netdev; 3375 int index; 3376 3377 index = igc_find_mac_filter(adapter, type, addr); 3378 if (index >= 0) 3379 goto update_filter; 3380 3381 index = igc_get_avail_mac_filter_slot(adapter); 3382 if (index < 0) 3383 return -ENOSPC; 3384 3385 netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n", 3386 index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 3387 addr, queue); 3388 3389 update_filter: 3390 igc_set_mac_filter_hw(adapter, index, type, addr, queue); 3391 return 0; 3392 } 3393 3394 /** 3395 * igc_del_mac_filter() - Delete MAC address filter 3396 * @adapter: Pointer to adapter where the filter should be deleted from 3397 * @type: MAC address filter type (source or destination) 3398 * @addr: MAC address 3399 */ 3400 static void igc_del_mac_filter(struct igc_adapter *adapter, 3401 enum igc_mac_filter_type type, const u8 *addr) 3402 { 3403 struct net_device *dev = adapter->netdev; 3404 int index; 3405 3406 index = igc_find_mac_filter(adapter, type, addr); 3407 if (index < 0) 3408 return; 3409 3410 if (index == 0) { 3411 /* If this is the default filter, we don't actually delete it. 3412 * We just reset to its default value i.e. disable queue 3413 * assignment. 3414 */ 3415 netdev_dbg(dev, "Disable default MAC filter queue assignment"); 3416 3417 igc_set_mac_filter_hw(adapter, 0, type, addr, -1); 3418 } else { 3419 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n", 3420 index, 3421 type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src", 3422 addr); 3423 3424 igc_clear_mac_filter_hw(adapter, index); 3425 } 3426 } 3427 3428 /** 3429 * igc_add_vlan_prio_filter() - Add VLAN priority filter 3430 * @adapter: Pointer to adapter where the filter should be added 3431 * @prio: VLAN priority value 3432 * @queue: Queue number which matching frames are assigned to 3433 * 3434 * Return: 0 in case of success, negative errno code otherwise. 3435 */ 3436 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, 3437 int queue) 3438 { 3439 struct net_device *dev = adapter->netdev; 3440 struct igc_hw *hw = &adapter->hw; 3441 u32 vlanpqf; 3442 3443 vlanpqf = rd32(IGC_VLANPQF); 3444 3445 if (vlanpqf & IGC_VLANPQF_VALID(prio)) { 3446 netdev_dbg(dev, "VLAN priority filter already in use\n"); 3447 return -EEXIST; 3448 } 3449 3450 vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); 3451 vlanpqf |= IGC_VLANPQF_VALID(prio); 3452 3453 wr32(IGC_VLANPQF, vlanpqf); 3454 3455 netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n", 3456 prio, queue); 3457 return 0; 3458 } 3459 3460 /** 3461 * igc_del_vlan_prio_filter() - Delete VLAN priority filter 3462 * @adapter: Pointer to adapter where the filter should be deleted from 3463 * @prio: VLAN priority value 3464 */ 3465 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) 3466 { 3467 struct igc_hw *hw = &adapter->hw; 3468 u32 vlanpqf; 3469 3470 vlanpqf = rd32(IGC_VLANPQF); 3471 3472 vlanpqf &= ~IGC_VLANPQF_VALID(prio); 3473 vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); 3474 3475 wr32(IGC_VLANPQF, vlanpqf); 3476 3477 netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n", 3478 prio); 3479 } 3480 3481 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) 3482 { 3483 struct igc_hw *hw = &adapter->hw; 3484 int i; 3485 3486 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 3487 u32 etqf = rd32(IGC_ETQF(i)); 3488 3489 if (!(etqf & IGC_ETQF_FILTER_ENABLE)) 3490 return i; 3491 } 3492 3493 return -1; 3494 } 3495 3496 /** 3497 * igc_add_etype_filter() - Add ethertype filter 3498 * @adapter: Pointer to adapter where the filter should be added 3499 * @etype: Ethertype value 3500 * @queue: If non-negative, queue assignment feature is enabled and frames 3501 * matching the filter are enqueued onto 'queue'. Otherwise, queue 3502 * assignment is disabled. 3503 * 3504 * Return: 0 in case of success, negative errno code otherwise. 3505 */ 3506 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, 3507 int queue) 3508 { 3509 struct igc_hw *hw = &adapter->hw; 3510 int index; 3511 u32 etqf; 3512 3513 index = igc_get_avail_etype_filter_slot(adapter); 3514 if (index < 0) 3515 return -ENOSPC; 3516 3517 etqf = rd32(IGC_ETQF(index)); 3518 3519 etqf &= ~IGC_ETQF_ETYPE_MASK; 3520 etqf |= etype; 3521 3522 if (queue >= 0) { 3523 etqf &= ~IGC_ETQF_QUEUE_MASK; 3524 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); 3525 etqf |= IGC_ETQF_QUEUE_ENABLE; 3526 } 3527 3528 etqf |= IGC_ETQF_FILTER_ENABLE; 3529 3530 wr32(IGC_ETQF(index), etqf); 3531 3532 netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n", 3533 etype, queue); 3534 return 0; 3535 } 3536 3537 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) 3538 { 3539 struct igc_hw *hw = &adapter->hw; 3540 int i; 3541 3542 for (i = 0; i < MAX_ETYPE_FILTER; i++) { 3543 u32 etqf = rd32(IGC_ETQF(i)); 3544 3545 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) 3546 return i; 3547 } 3548 3549 return -1; 3550 } 3551 3552 /** 3553 * igc_del_etype_filter() - Delete ethertype filter 3554 * @adapter: Pointer to adapter where the filter should be deleted from 3555 * @etype: Ethertype value 3556 */ 3557 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) 3558 { 3559 struct igc_hw *hw = &adapter->hw; 3560 int index; 3561 3562 index = igc_find_etype_filter(adapter, etype); 3563 if (index < 0) 3564 return; 3565 3566 wr32(IGC_ETQF(index), 0); 3567 3568 netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n", 3569 etype); 3570 } 3571 3572 static int igc_flex_filter_select(struct igc_adapter *adapter, 3573 struct igc_flex_filter *input, 3574 u32 *fhft) 3575 { 3576 struct igc_hw *hw = &adapter->hw; 3577 u8 fhft_index; 3578 u32 fhftsl; 3579 3580 if (input->index >= MAX_FLEX_FILTER) { 3581 netdev_err(adapter->netdev, "Wrong Flex Filter index selected!\n"); 3582 return -EINVAL; 3583 } 3584 3585 /* Indirect table select register */ 3586 fhftsl = rd32(IGC_FHFTSL); 3587 fhftsl &= ~IGC_FHFTSL_FTSL_MASK; 3588 switch (input->index) { 3589 case 0 ... 7: 3590 fhftsl |= 0x00; 3591 break; 3592 case 8 ... 15: 3593 fhftsl |= 0x01; 3594 break; 3595 case 16 ... 23: 3596 fhftsl |= 0x02; 3597 break; 3598 case 24 ... 31: 3599 fhftsl |= 0x03; 3600 break; 3601 } 3602 wr32(IGC_FHFTSL, fhftsl); 3603 3604 /* Normalize index down to host table register */ 3605 fhft_index = input->index % 8; 3606 3607 *fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) : 3608 IGC_FHFT_EXT(fhft_index - 4); 3609 3610 return 0; 3611 } 3612 3613 static int igc_write_flex_filter_ll(struct igc_adapter *adapter, 3614 struct igc_flex_filter *input) 3615 { 3616 struct igc_hw *hw = &adapter->hw; 3617 u8 *data = input->data; 3618 u8 *mask = input->mask; 3619 u32 queuing; 3620 u32 fhft; 3621 u32 wufc; 3622 int ret; 3623 int i; 3624 3625 /* Length has to be aligned to 8. Otherwise the filter will fail. Bail 3626 * out early to avoid surprises later. 3627 */ 3628 if (input->length % 8 != 0) { 3629 netdev_err(adapter->netdev, "The length of a flex filter has to be 8 byte aligned!\n"); 3630 return -EINVAL; 3631 } 3632 3633 /* Select corresponding flex filter register and get base for host table. */ 3634 ret = igc_flex_filter_select(adapter, input, &fhft); 3635 if (ret) 3636 return ret; 3637 3638 /* When adding a filter globally disable flex filter feature. That is 3639 * recommended within the datasheet. 3640 */ 3641 wufc = rd32(IGC_WUFC); 3642 wufc &= ~IGC_WUFC_FLEX_HQ; 3643 wr32(IGC_WUFC, wufc); 3644 3645 /* Configure filter */ 3646 queuing = input->length & IGC_FHFT_LENGTH_MASK; 3647 queuing |= FIELD_PREP(IGC_FHFT_QUEUE_MASK, input->rx_queue); 3648 queuing |= FIELD_PREP(IGC_FHFT_PRIO_MASK, input->prio); 3649 3650 if (input->immediate_irq) 3651 queuing |= IGC_FHFT_IMM_INT; 3652 3653 if (input->drop) 3654 queuing |= IGC_FHFT_DROP; 3655 3656 wr32(fhft + 0xFC, queuing); 3657 3658 /* Write data (128 byte) and mask (128 bit) */ 3659 for (i = 0; i < 16; ++i) { 3660 const size_t data_idx = i * 8; 3661 const size_t row_idx = i * 16; 3662 u32 dw0 = 3663 (data[data_idx + 0] << 0) | 3664 (data[data_idx + 1] << 8) | 3665 (data[data_idx + 2] << 16) | 3666 (data[data_idx + 3] << 24); 3667 u32 dw1 = 3668 (data[data_idx + 4] << 0) | 3669 (data[data_idx + 5] << 8) | 3670 (data[data_idx + 6] << 16) | 3671 (data[data_idx + 7] << 24); 3672 u32 tmp; 3673 3674 /* Write row: dw0, dw1 and mask */ 3675 wr32(fhft + row_idx, dw0); 3676 wr32(fhft + row_idx + 4, dw1); 3677 3678 /* mask is only valid for MASK(7, 0) */ 3679 tmp = rd32(fhft + row_idx + 8); 3680 tmp &= ~GENMASK(7, 0); 3681 tmp |= mask[i]; 3682 wr32(fhft + row_idx + 8, tmp); 3683 } 3684 3685 /* Enable filter. */ 3686 wufc |= IGC_WUFC_FLEX_HQ; 3687 if (input->index > 8) { 3688 /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */ 3689 u32 wufc_ext = rd32(IGC_WUFC_EXT); 3690 3691 wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8)); 3692 3693 wr32(IGC_WUFC_EXT, wufc_ext); 3694 } else { 3695 wufc |= (IGC_WUFC_FLX0 << input->index); 3696 } 3697 wr32(IGC_WUFC, wufc); 3698 3699 netdev_dbg(adapter->netdev, "Added flex filter %u to HW.\n", 3700 input->index); 3701 3702 return 0; 3703 } 3704 3705 static void igc_flex_filter_add_field(struct igc_flex_filter *flex, 3706 const void *src, unsigned int offset, 3707 size_t len, const void *mask) 3708 { 3709 int i; 3710 3711 /* data */ 3712 memcpy(&flex->data[offset], src, len); 3713 3714 /* mask */ 3715 for (i = 0; i < len; ++i) { 3716 const unsigned int idx = i + offset; 3717 const u8 *ptr = mask; 3718 3719 if (mask) { 3720 if (ptr[i] & 0xff) 3721 flex->mask[idx / 8] |= BIT(idx % 8); 3722 3723 continue; 3724 } 3725 3726 flex->mask[idx / 8] |= BIT(idx % 8); 3727 } 3728 } 3729 3730 static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter) 3731 { 3732 struct igc_hw *hw = &adapter->hw; 3733 u32 wufc, wufc_ext; 3734 int i; 3735 3736 wufc = rd32(IGC_WUFC); 3737 wufc_ext = rd32(IGC_WUFC_EXT); 3738 3739 for (i = 0; i < MAX_FLEX_FILTER; i++) { 3740 if (i < 8) { 3741 if (!(wufc & (IGC_WUFC_FLX0 << i))) 3742 return i; 3743 } else { 3744 if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8)))) 3745 return i; 3746 } 3747 } 3748 3749 return -ENOSPC; 3750 } 3751 3752 static bool igc_flex_filter_in_use(struct igc_adapter *adapter) 3753 { 3754 struct igc_hw *hw = &adapter->hw; 3755 u32 wufc, wufc_ext; 3756 3757 wufc = rd32(IGC_WUFC); 3758 wufc_ext = rd32(IGC_WUFC_EXT); 3759 3760 if (wufc & IGC_WUFC_FILTER_MASK) 3761 return true; 3762 3763 if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK) 3764 return true; 3765 3766 return false; 3767 } 3768 3769 static int igc_add_flex_filter(struct igc_adapter *adapter, 3770 struct igc_nfc_rule *rule) 3771 { 3772 struct igc_nfc_filter *filter = &rule->filter; 3773 unsigned int eth_offset, user_offset; 3774 struct igc_flex_filter flex = { }; 3775 int ret, index; 3776 bool vlan; 3777 3778 index = igc_find_avail_flex_filter_slot(adapter); 3779 if (index < 0) 3780 return -ENOSPC; 3781 3782 /* Construct the flex filter: 3783 * -> dest_mac [6] 3784 * -> src_mac [6] 3785 * -> tpid [2] 3786 * -> vlan tci [2] 3787 * -> ether type [2] 3788 * -> user data [8] 3789 * -> = 26 bytes => 32 length 3790 */ 3791 flex.index = index; 3792 flex.length = 32; 3793 flex.rx_queue = rule->action; 3794 3795 vlan = rule->filter.vlan_tci || rule->filter.vlan_etype; 3796 eth_offset = vlan ? 16 : 12; 3797 user_offset = vlan ? 18 : 14; 3798 3799 /* Add destination MAC */ 3800 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 3801 igc_flex_filter_add_field(&flex, &filter->dst_addr, 0, 3802 ETH_ALEN, NULL); 3803 3804 /* Add source MAC */ 3805 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 3806 igc_flex_filter_add_field(&flex, &filter->src_addr, 6, 3807 ETH_ALEN, NULL); 3808 3809 /* Add VLAN etype */ 3810 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE) { 3811 __be16 vlan_etype = cpu_to_be16(filter->vlan_etype); 3812 3813 igc_flex_filter_add_field(&flex, &vlan_etype, 12, 3814 sizeof(vlan_etype), NULL); 3815 } 3816 3817 /* Add VLAN TCI */ 3818 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) 3819 igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14, 3820 sizeof(filter->vlan_tci), NULL); 3821 3822 /* Add Ether type */ 3823 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 3824 __be16 etype = cpu_to_be16(filter->etype); 3825 3826 igc_flex_filter_add_field(&flex, &etype, eth_offset, 3827 sizeof(etype), NULL); 3828 } 3829 3830 /* Add user data */ 3831 if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA) 3832 igc_flex_filter_add_field(&flex, &filter->user_data, 3833 user_offset, 3834 sizeof(filter->user_data), 3835 filter->user_mask); 3836 3837 /* Add it down to the hardware and enable it. */ 3838 ret = igc_write_flex_filter_ll(adapter, &flex); 3839 if (ret) 3840 return ret; 3841 3842 filter->flex_index = index; 3843 3844 return 0; 3845 } 3846 3847 static void igc_del_flex_filter(struct igc_adapter *adapter, 3848 u16 reg_index) 3849 { 3850 struct igc_hw *hw = &adapter->hw; 3851 u32 wufc; 3852 3853 /* Just disable the filter. The filter table itself is kept 3854 * intact. Another flex_filter_add() should override the "old" data 3855 * then. 3856 */ 3857 if (reg_index > 8) { 3858 u32 wufc_ext = rd32(IGC_WUFC_EXT); 3859 3860 wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8)); 3861 wr32(IGC_WUFC_EXT, wufc_ext); 3862 } else { 3863 wufc = rd32(IGC_WUFC); 3864 3865 wufc &= ~(IGC_WUFC_FLX0 << reg_index); 3866 wr32(IGC_WUFC, wufc); 3867 } 3868 3869 if (igc_flex_filter_in_use(adapter)) 3870 return; 3871 3872 /* No filters are in use, we may disable flex filters */ 3873 wufc = rd32(IGC_WUFC); 3874 wufc &= ~IGC_WUFC_FLEX_HQ; 3875 wr32(IGC_WUFC, wufc); 3876 } 3877 3878 static void igc_set_default_queue_filter(struct igc_adapter *adapter, u32 queue) 3879 { 3880 struct igc_hw *hw = &adapter->hw; 3881 u32 mrqc = rd32(IGC_MRQC); 3882 3883 mrqc &= ~IGC_MRQC_DEFAULT_QUEUE_MASK; 3884 mrqc |= FIELD_PREP(IGC_MRQC_DEFAULT_QUEUE_MASK, queue); 3885 wr32(IGC_MRQC, mrqc); 3886 } 3887 3888 static void igc_reset_default_queue_filter(struct igc_adapter *adapter) 3889 { 3890 /* Reset the default queue to its default value which is Queue 0 */ 3891 igc_set_default_queue_filter(adapter, 0); 3892 } 3893 3894 static int igc_enable_nfc_rule(struct igc_adapter *adapter, 3895 struct igc_nfc_rule *rule) 3896 { 3897 int err; 3898 3899 if (rule->flex) { 3900 return igc_add_flex_filter(adapter, rule); 3901 } 3902 3903 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { 3904 err = igc_add_etype_filter(adapter, rule->filter.etype, 3905 rule->action); 3906 if (err) 3907 return err; 3908 } 3909 3910 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { 3911 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 3912 rule->filter.src_addr, rule->action); 3913 if (err) 3914 return err; 3915 } 3916 3917 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { 3918 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 3919 rule->filter.dst_addr, rule->action); 3920 if (err) 3921 return err; 3922 } 3923 3924 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 3925 int prio = FIELD_GET(VLAN_PRIO_MASK, rule->filter.vlan_tci); 3926 3927 err = igc_add_vlan_prio_filter(adapter, prio, rule->action); 3928 if (err) 3929 return err; 3930 } 3931 3932 if (rule->filter.match_flags & IGC_FILTER_FLAG_DEFAULT_QUEUE) 3933 igc_set_default_queue_filter(adapter, rule->action); 3934 3935 return 0; 3936 } 3937 3938 static void igc_disable_nfc_rule(struct igc_adapter *adapter, 3939 const struct igc_nfc_rule *rule) 3940 { 3941 if (rule->flex) { 3942 igc_del_flex_filter(adapter, rule->filter.flex_index); 3943 return; 3944 } 3945 3946 if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) 3947 igc_del_etype_filter(adapter, rule->filter.etype); 3948 3949 if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { 3950 int prio = FIELD_GET(VLAN_PRIO_MASK, rule->filter.vlan_tci); 3951 3952 igc_del_vlan_prio_filter(adapter, prio); 3953 } 3954 3955 if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) 3956 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC, 3957 rule->filter.src_addr); 3958 3959 if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) 3960 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, 3961 rule->filter.dst_addr); 3962 3963 if (rule->filter.match_flags & IGC_FILTER_FLAG_DEFAULT_QUEUE) 3964 igc_reset_default_queue_filter(adapter); 3965 } 3966 3967 /** 3968 * igc_get_nfc_rule() - Get NFC rule 3969 * @adapter: Pointer to adapter 3970 * @location: Rule location 3971 * 3972 * Context: Expects adapter->nfc_rule_lock to be held by caller. 3973 * 3974 * Return: Pointer to NFC rule at @location. If not found, NULL. 3975 */ 3976 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, 3977 u32 location) 3978 { 3979 struct igc_nfc_rule *rule; 3980 3981 list_for_each_entry(rule, &adapter->nfc_rule_list, list) { 3982 if (rule->location == location) 3983 return rule; 3984 if (rule->location > location) 3985 break; 3986 } 3987 3988 return NULL; 3989 } 3990 3991 /** 3992 * igc_del_nfc_rule() - Delete NFC rule 3993 * @adapter: Pointer to adapter 3994 * @rule: Pointer to rule to be deleted 3995 * 3996 * Disable NFC rule in hardware and delete it from adapter. 3997 * 3998 * Context: Expects adapter->nfc_rule_lock to be held by caller. 3999 */ 4000 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 4001 { 4002 igc_disable_nfc_rule(adapter, rule); 4003 4004 list_del(&rule->list); 4005 adapter->nfc_rule_count--; 4006 4007 kfree(rule); 4008 } 4009 4010 static void igc_flush_nfc_rules(struct igc_adapter *adapter) 4011 { 4012 struct igc_nfc_rule *rule, *tmp; 4013 4014 mutex_lock(&adapter->nfc_rule_lock); 4015 4016 list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) 4017 igc_del_nfc_rule(adapter, rule); 4018 4019 mutex_unlock(&adapter->nfc_rule_lock); 4020 } 4021 4022 /** 4023 * igc_add_nfc_rule() - Add NFC rule 4024 * @adapter: Pointer to adapter 4025 * @rule: Pointer to rule to be added 4026 * 4027 * Enable NFC rule in hardware and add it to adapter. 4028 * 4029 * Context: Expects adapter->nfc_rule_lock to be held by caller. 4030 * 4031 * Return: 0 on success, negative errno on failure. 4032 */ 4033 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) 4034 { 4035 struct igc_nfc_rule *pred, *cur; 4036 int err; 4037 4038 err = igc_enable_nfc_rule(adapter, rule); 4039 if (err) 4040 return err; 4041 4042 pred = NULL; 4043 list_for_each_entry(cur, &adapter->nfc_rule_list, list) { 4044 if (cur->location >= rule->location) 4045 break; 4046 pred = cur; 4047 } 4048 4049 list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list); 4050 adapter->nfc_rule_count++; 4051 return 0; 4052 } 4053 4054 static void igc_restore_nfc_rules(struct igc_adapter *adapter) 4055 { 4056 struct igc_nfc_rule *rule; 4057 4058 mutex_lock(&adapter->nfc_rule_lock); 4059 4060 list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) 4061 igc_enable_nfc_rule(adapter, rule); 4062 4063 mutex_unlock(&adapter->nfc_rule_lock); 4064 } 4065 4066 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) 4067 { 4068 struct igc_adapter *adapter = netdev_priv(netdev); 4069 4070 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1); 4071 } 4072 4073 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) 4074 { 4075 struct igc_adapter *adapter = netdev_priv(netdev); 4076 4077 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr); 4078 return 0; 4079 } 4080 4081 /** 4082 * igc_enable_empty_addr_recv - Enable Rx of packets with all-zeroes MAC address 4083 * @adapter: Pointer to the igc_adapter structure. 4084 * 4085 * Frame preemption verification requires that packets with the all-zeroes 4086 * MAC address are allowed to be received by the driver. This function adds the 4087 * all-zeroes destination address to the list of acceptable addresses. 4088 * 4089 * Return: 0 on success, negative value otherwise. 4090 */ 4091 int igc_enable_empty_addr_recv(struct igc_adapter *adapter) 4092 { 4093 u8 empty[ETH_ALEN] = {}; 4094 4095 return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, empty, -1); 4096 } 4097 4098 void igc_disable_empty_addr_recv(struct igc_adapter *adapter) 4099 { 4100 u8 empty[ETH_ALEN] = {}; 4101 4102 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, empty); 4103 } 4104 4105 /** 4106 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set 4107 * @netdev: network interface device structure 4108 * 4109 * The set_rx_mode entry point is called whenever the unicast or multicast 4110 * address lists or the network interface flags are updated. This routine is 4111 * responsible for configuring the hardware for proper unicast, multicast, 4112 * promiscuous mode, and all-multi behavior. 4113 */ 4114 static void igc_set_rx_mode(struct net_device *netdev) 4115 { 4116 struct igc_adapter *adapter = netdev_priv(netdev); 4117 struct igc_hw *hw = &adapter->hw; 4118 u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; 4119 int count; 4120 4121 /* Check for Promiscuous and All Multicast modes */ 4122 if (netdev->flags & IFF_PROMISC) { 4123 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; 4124 } else { 4125 if (netdev->flags & IFF_ALLMULTI) { 4126 rctl |= IGC_RCTL_MPE; 4127 } else { 4128 /* Write addresses to the MTA, if the attempt fails 4129 * then we should just turn on promiscuous mode so 4130 * that we can at least receive multicast traffic 4131 */ 4132 count = igc_write_mc_addr_list(netdev); 4133 if (count < 0) 4134 rctl |= IGC_RCTL_MPE; 4135 } 4136 } 4137 4138 /* Write addresses to available RAR registers, if there is not 4139 * sufficient space to store all the addresses then enable 4140 * unicast promiscuous mode 4141 */ 4142 if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) 4143 rctl |= IGC_RCTL_UPE; 4144 4145 /* update state of unicast and multicast */ 4146 rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 4147 wr32(IGC_RCTL, rctl); 4148 4149 #if (PAGE_SIZE < 8192) 4150 if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) 4151 rlpml = IGC_MAX_FRAME_BUILD_SKB; 4152 #endif 4153 wr32(IGC_RLPML, rlpml); 4154 } 4155 4156 /** 4157 * igc_configure - configure the hardware for RX and TX 4158 * @adapter: private board structure 4159 */ 4160 static void igc_configure(struct igc_adapter *adapter) 4161 { 4162 struct net_device *netdev = adapter->netdev; 4163 int i = 0; 4164 4165 igc_get_hw_control(adapter); 4166 igc_set_rx_mode(netdev); 4167 4168 igc_restore_vlan(adapter); 4169 4170 igc_setup_tctl(adapter); 4171 igc_setup_mrqc(adapter); 4172 igc_setup_rctl(adapter); 4173 4174 igc_set_default_mac_filter(adapter); 4175 igc_restore_nfc_rules(adapter); 4176 4177 igc_configure_tx(adapter); 4178 igc_configure_rx(adapter); 4179 4180 igc_rx_fifo_flush_base(&adapter->hw); 4181 4182 /* call igc_desc_unused which always leaves 4183 * at least 1 descriptor unused to make sure 4184 * next_to_use != next_to_clean 4185 */ 4186 for (i = 0; i < adapter->num_rx_queues; i++) { 4187 struct igc_ring *ring = adapter->rx_ring[i]; 4188 4189 if (ring->xsk_pool) 4190 igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); 4191 else 4192 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 4193 } 4194 } 4195 4196 /** 4197 * igc_write_ivar - configure ivar for given MSI-X vector 4198 * @hw: pointer to the HW structure 4199 * @msix_vector: vector number we are allocating to a given ring 4200 * @index: row index of IVAR register to write within IVAR table 4201 * @offset: column offset of in IVAR, should be multiple of 8 4202 * 4203 * The IVAR table consists of 2 columns, 4204 * each containing an cause allocation for an Rx and Tx ring, and a 4205 * variable number of rows depending on the number of queues supported. 4206 */ 4207 static void igc_write_ivar(struct igc_hw *hw, int msix_vector, 4208 int index, int offset) 4209 { 4210 u32 ivar = array_rd32(IGC_IVAR0, index); 4211 4212 /* clear any bits that are currently set */ 4213 ivar &= ~((u32)0xFF << offset); 4214 4215 /* write vector and valid bit */ 4216 ivar |= (msix_vector | IGC_IVAR_VALID) << offset; 4217 4218 array_wr32(IGC_IVAR0, index, ivar); 4219 } 4220 4221 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) 4222 { 4223 struct igc_adapter *adapter = q_vector->adapter; 4224 struct igc_hw *hw = &adapter->hw; 4225 int rx_queue = IGC_N0_QUEUE; 4226 int tx_queue = IGC_N0_QUEUE; 4227 4228 if (q_vector->rx.ring) 4229 rx_queue = q_vector->rx.ring->reg_idx; 4230 if (q_vector->tx.ring) 4231 tx_queue = q_vector->tx.ring->reg_idx; 4232 4233 switch (hw->mac.type) { 4234 case igc_i225: 4235 if (rx_queue > IGC_N0_QUEUE) 4236 igc_write_ivar(hw, msix_vector, 4237 rx_queue >> 1, 4238 (rx_queue & 0x1) << 4); 4239 if (tx_queue > IGC_N0_QUEUE) 4240 igc_write_ivar(hw, msix_vector, 4241 tx_queue >> 1, 4242 ((tx_queue & 0x1) << 4) + 8); 4243 q_vector->eims_value = BIT(msix_vector); 4244 break; 4245 default: 4246 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); 4247 break; 4248 } 4249 4250 /* add q_vector eims value to global eims_enable_mask */ 4251 adapter->eims_enable_mask |= q_vector->eims_value; 4252 4253 /* configure q_vector to set itr on first interrupt */ 4254 q_vector->set_itr = 1; 4255 } 4256 4257 /** 4258 * igc_configure_msix - Configure MSI-X hardware 4259 * @adapter: Pointer to adapter structure 4260 * 4261 * igc_configure_msix sets up the hardware to properly 4262 * generate MSI-X interrupts. 4263 */ 4264 static void igc_configure_msix(struct igc_adapter *adapter) 4265 { 4266 struct igc_hw *hw = &adapter->hw; 4267 int i, vector = 0; 4268 u32 tmp; 4269 4270 adapter->eims_enable_mask = 0; 4271 4272 /* set vector for other causes, i.e. link changes */ 4273 switch (hw->mac.type) { 4274 case igc_i225: 4275 /* Turn on MSI-X capability first, or our settings 4276 * won't stick. And it will take days to debug. 4277 */ 4278 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | 4279 IGC_GPIE_PBA | IGC_GPIE_EIAME | 4280 IGC_GPIE_NSICR); 4281 4282 /* enable msix_other interrupt */ 4283 adapter->eims_other = BIT(vector); 4284 tmp = (vector++ | IGC_IVAR_VALID) << 8; 4285 4286 wr32(IGC_IVAR_MISC, tmp); 4287 break; 4288 default: 4289 /* do nothing, since nothing else supports MSI-X */ 4290 break; 4291 } /* switch (hw->mac.type) */ 4292 4293 adapter->eims_enable_mask |= adapter->eims_other; 4294 4295 for (i = 0; i < adapter->num_q_vectors; i++) 4296 igc_assign_vector(adapter->q_vector[i], vector++); 4297 4298 wrfl(); 4299 } 4300 4301 /** 4302 * igc_irq_enable - Enable default interrupt generation settings 4303 * @adapter: board private structure 4304 */ 4305 static void igc_irq_enable(struct igc_adapter *adapter) 4306 { 4307 struct igc_hw *hw = &adapter->hw; 4308 4309 if (adapter->msix_entries) { 4310 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; 4311 u32 regval = rd32(IGC_EIAC); 4312 4313 wr32(IGC_EIAC, regval | adapter->eims_enable_mask); 4314 regval = rd32(IGC_EIAM); 4315 wr32(IGC_EIAM, regval | adapter->eims_enable_mask); 4316 wr32(IGC_EIMS, adapter->eims_enable_mask); 4317 wr32(IGC_IMS, ims); 4318 } else { 4319 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4320 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); 4321 } 4322 } 4323 4324 /** 4325 * igc_irq_disable - Mask off interrupt generation on the NIC 4326 * @adapter: board private structure 4327 */ 4328 static void igc_irq_disable(struct igc_adapter *adapter) 4329 { 4330 struct igc_hw *hw = &adapter->hw; 4331 4332 if (adapter->msix_entries) { 4333 u32 regval = rd32(IGC_EIAM); 4334 4335 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); 4336 wr32(IGC_EIMC, adapter->eims_enable_mask); 4337 regval = rd32(IGC_EIAC); 4338 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); 4339 } 4340 4341 wr32(IGC_IAM, 0); 4342 wr32(IGC_IMC, ~0); 4343 wrfl(); 4344 4345 if (adapter->msix_entries) { 4346 int vector = 0, i; 4347 4348 synchronize_irq(adapter->msix_entries[vector++].vector); 4349 4350 for (i = 0; i < adapter->num_q_vectors; i++) 4351 synchronize_irq(adapter->msix_entries[vector++].vector); 4352 } else { 4353 synchronize_irq(adapter->pdev->irq); 4354 } 4355 } 4356 4357 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 4358 const u32 max_rss_queues) 4359 { 4360 /* Determine if we need to pair queues. */ 4361 /* If rss_queues > half of max_rss_queues, pair the queues in 4362 * order to conserve interrupts due to limited supply. 4363 */ 4364 if (adapter->rss_queues > (max_rss_queues / 2)) 4365 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4366 else 4367 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; 4368 } 4369 4370 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) 4371 { 4372 return IGC_MAX_RX_QUEUES; 4373 } 4374 4375 static void igc_init_queue_configuration(struct igc_adapter *adapter) 4376 { 4377 u32 max_rss_queues; 4378 4379 max_rss_queues = igc_get_max_rss_queues(adapter); 4380 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); 4381 4382 igc_set_flag_queue_pairs(adapter, max_rss_queues); 4383 } 4384 4385 /** 4386 * igc_reset_q_vector - Reset config for interrupt vector 4387 * @adapter: board private structure to initialize 4388 * @v_idx: Index of vector to be reset 4389 * 4390 * If NAPI is enabled it will delete any references to the 4391 * NAPI struct. This is preparation for igc_free_q_vector. 4392 */ 4393 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) 4394 { 4395 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 4396 4397 /* if we're coming from igc_set_interrupt_capability, the vectors are 4398 * not yet allocated 4399 */ 4400 if (!q_vector) 4401 return; 4402 4403 if (q_vector->tx.ring) 4404 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; 4405 4406 if (q_vector->rx.ring) 4407 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; 4408 4409 netif_napi_del(&q_vector->napi); 4410 } 4411 4412 /** 4413 * igc_free_q_vector - Free memory allocated for specific interrupt vector 4414 * @adapter: board private structure to initialize 4415 * @v_idx: Index of vector to be freed 4416 * 4417 * This function frees the memory allocated to the q_vector. 4418 */ 4419 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) 4420 { 4421 struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; 4422 4423 adapter->q_vector[v_idx] = NULL; 4424 4425 /* igc_get_stats64() might access the rings on this vector, 4426 * we must wait a grace period before freeing it. 4427 */ 4428 if (q_vector) 4429 kfree_rcu(q_vector, rcu); 4430 } 4431 4432 /** 4433 * igc_free_q_vectors - Free memory allocated for interrupt vectors 4434 * @adapter: board private structure to initialize 4435 * 4436 * This function frees the memory allocated to the q_vectors. In addition if 4437 * NAPI is enabled it will delete any references to the NAPI struct prior 4438 * to freeing the q_vector. 4439 */ 4440 static void igc_free_q_vectors(struct igc_adapter *adapter) 4441 { 4442 int v_idx = adapter->num_q_vectors; 4443 4444 adapter->num_tx_queues = 0; 4445 adapter->num_rx_queues = 0; 4446 adapter->num_q_vectors = 0; 4447 4448 while (v_idx--) { 4449 igc_reset_q_vector(adapter, v_idx); 4450 igc_free_q_vector(adapter, v_idx); 4451 } 4452 } 4453 4454 /** 4455 * igc_update_itr - update the dynamic ITR value based on statistics 4456 * @q_vector: pointer to q_vector 4457 * @ring_container: ring info to update the itr for 4458 * 4459 * Stores a new ITR value based on packets and byte 4460 * counts during the last interrupt. The advantage of per interrupt 4461 * computation is faster updates and more accurate ITR for the current 4462 * traffic pattern. Constants in this function were computed 4463 * based on theoretical maximum wire speed and thresholds were set based 4464 * on testing data as well as attempting to minimize response time 4465 * while increasing bulk throughput. 4466 * NOTE: These calculations are only valid when operating in a single- 4467 * queue environment. 4468 */ 4469 static void igc_update_itr(struct igc_q_vector *q_vector, 4470 struct igc_ring_container *ring_container) 4471 { 4472 unsigned int packets = ring_container->total_packets; 4473 unsigned int bytes = ring_container->total_bytes; 4474 u8 itrval = ring_container->itr; 4475 4476 /* no packets, exit with status unchanged */ 4477 if (packets == 0) 4478 return; 4479 4480 switch (itrval) { 4481 case lowest_latency: 4482 /* handle TSO and jumbo frames */ 4483 if (bytes / packets > 8000) 4484 itrval = bulk_latency; 4485 else if ((packets < 5) && (bytes > 512)) 4486 itrval = low_latency; 4487 break; 4488 case low_latency: /* 50 usec aka 20000 ints/s */ 4489 if (bytes > 10000) { 4490 /* this if handles the TSO accounting */ 4491 if (bytes / packets > 8000) 4492 itrval = bulk_latency; 4493 else if ((packets < 10) || ((bytes / packets) > 1200)) 4494 itrval = bulk_latency; 4495 else if ((packets > 35)) 4496 itrval = lowest_latency; 4497 } else if (bytes / packets > 2000) { 4498 itrval = bulk_latency; 4499 } else if (packets <= 2 && bytes < 512) { 4500 itrval = lowest_latency; 4501 } 4502 break; 4503 case bulk_latency: /* 250 usec aka 4000 ints/s */ 4504 if (bytes > 25000) { 4505 if (packets > 35) 4506 itrval = low_latency; 4507 } else if (bytes < 1500) { 4508 itrval = low_latency; 4509 } 4510 break; 4511 } 4512 4513 /* clear work counters since we have the values we need */ 4514 ring_container->total_bytes = 0; 4515 ring_container->total_packets = 0; 4516 4517 /* write updated itr to ring container */ 4518 ring_container->itr = itrval; 4519 } 4520 4521 static void igc_set_itr(struct igc_q_vector *q_vector) 4522 { 4523 struct igc_adapter *adapter = q_vector->adapter; 4524 u32 new_itr = q_vector->itr_val; 4525 u8 current_itr = 0; 4526 4527 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ 4528 switch (adapter->link_speed) { 4529 case SPEED_10: 4530 case SPEED_100: 4531 current_itr = 0; 4532 new_itr = IGC_4K_ITR; 4533 goto set_itr_now; 4534 default: 4535 break; 4536 } 4537 4538 igc_update_itr(q_vector, &q_vector->tx); 4539 igc_update_itr(q_vector, &q_vector->rx); 4540 4541 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 4542 4543 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 4544 if (current_itr == lowest_latency && 4545 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 4546 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 4547 current_itr = low_latency; 4548 4549 switch (current_itr) { 4550 /* counts and packets in update_itr are dependent on these numbers */ 4551 case lowest_latency: 4552 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ 4553 break; 4554 case low_latency: 4555 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ 4556 break; 4557 case bulk_latency: 4558 new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ 4559 break; 4560 default: 4561 break; 4562 } 4563 4564 set_itr_now: 4565 if (new_itr != q_vector->itr_val) { 4566 /* this attempts to bias the interrupt rate towards Bulk 4567 * by adding intermediate steps when interrupt rate is 4568 * increasing 4569 */ 4570 new_itr = new_itr > q_vector->itr_val ? 4571 max((new_itr * q_vector->itr_val) / 4572 (new_itr + (q_vector->itr_val >> 2)), 4573 new_itr) : new_itr; 4574 /* Don't write the value here; it resets the adapter's 4575 * internal timer, and causes us to delay far longer than 4576 * we should between interrupts. Instead, we write the ITR 4577 * value at the beginning of the next interrupt so the timing 4578 * ends up being correct. 4579 */ 4580 q_vector->itr_val = new_itr; 4581 q_vector->set_itr = 1; 4582 } 4583 } 4584 4585 static void igc_reset_interrupt_capability(struct igc_adapter *adapter) 4586 { 4587 int v_idx = adapter->num_q_vectors; 4588 4589 if (adapter->msix_entries) { 4590 pci_disable_msix(adapter->pdev); 4591 kfree(adapter->msix_entries); 4592 adapter->msix_entries = NULL; 4593 } else if (adapter->flags & IGC_FLAG_HAS_MSI) { 4594 pci_disable_msi(adapter->pdev); 4595 } 4596 4597 while (v_idx--) 4598 igc_reset_q_vector(adapter, v_idx); 4599 } 4600 4601 /** 4602 * igc_set_interrupt_capability - set MSI or MSI-X if supported 4603 * @adapter: Pointer to adapter structure 4604 * @msix: boolean value for MSI-X capability 4605 * 4606 * Attempt to configure interrupts using the best available 4607 * capabilities of the hardware and kernel. 4608 */ 4609 static void igc_set_interrupt_capability(struct igc_adapter *adapter, 4610 bool msix) 4611 { 4612 int numvecs, i; 4613 int err; 4614 4615 if (!msix) 4616 goto msi_only; 4617 adapter->flags |= IGC_FLAG_HAS_MSIX; 4618 4619 /* Number of supported queues. */ 4620 adapter->num_rx_queues = adapter->rss_queues; 4621 4622 adapter->num_tx_queues = adapter->rss_queues; 4623 4624 /* start with one vector for every Rx queue */ 4625 numvecs = adapter->num_rx_queues; 4626 4627 /* if Tx handler is separate add 1 for every Tx queue */ 4628 if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) 4629 numvecs += adapter->num_tx_queues; 4630 4631 /* store the number of vectors reserved for queues */ 4632 adapter->num_q_vectors = numvecs; 4633 4634 /* add 1 vector for link status interrupts */ 4635 numvecs++; 4636 4637 adapter->msix_entries = kzalloc_objs(struct msix_entry, numvecs); 4638 4639 if (!adapter->msix_entries) 4640 return; 4641 4642 /* populate entry values */ 4643 for (i = 0; i < numvecs; i++) 4644 adapter->msix_entries[i].entry = i; 4645 4646 err = pci_enable_msix_range(adapter->pdev, 4647 adapter->msix_entries, 4648 numvecs, 4649 numvecs); 4650 if (err > 0) 4651 return; 4652 4653 kfree(adapter->msix_entries); 4654 adapter->msix_entries = NULL; 4655 4656 igc_reset_interrupt_capability(adapter); 4657 4658 msi_only: 4659 adapter->flags &= ~IGC_FLAG_HAS_MSIX; 4660 4661 adapter->rss_queues = 1; 4662 adapter->flags |= IGC_FLAG_QUEUE_PAIRS; 4663 adapter->num_rx_queues = 1; 4664 adapter->num_tx_queues = 1; 4665 adapter->num_q_vectors = 1; 4666 if (!pci_enable_msi(adapter->pdev)) 4667 adapter->flags |= IGC_FLAG_HAS_MSI; 4668 } 4669 4670 /** 4671 * igc_update_ring_itr - update the dynamic ITR value based on packet size 4672 * @q_vector: pointer to q_vector 4673 * 4674 * Stores a new ITR value based on strictly on packet size. This 4675 * algorithm is less sophisticated than that used in igc_update_itr, 4676 * due to the difficulty of synchronizing statistics across multiple 4677 * receive rings. The divisors and thresholds used by this function 4678 * were determined based on theoretical maximum wire speed and testing 4679 * data, in order to minimize response time while increasing bulk 4680 * throughput. 4681 * NOTE: This function is called only when operating in a multiqueue 4682 * receive environment. 4683 */ 4684 static void igc_update_ring_itr(struct igc_q_vector *q_vector) 4685 { 4686 struct igc_adapter *adapter = q_vector->adapter; 4687 int new_val = q_vector->itr_val; 4688 int avg_wire_size = 0; 4689 unsigned int packets; 4690 4691 /* For non-gigabit speeds, just fix the interrupt rate at 4000 4692 * ints/sec - ITR timer value of 120 ticks. 4693 */ 4694 switch (adapter->link_speed) { 4695 case SPEED_10: 4696 case SPEED_100: 4697 new_val = IGC_4K_ITR; 4698 goto set_itr_val; 4699 default: 4700 break; 4701 } 4702 4703 packets = q_vector->rx.total_packets; 4704 if (packets) 4705 avg_wire_size = q_vector->rx.total_bytes / packets; 4706 4707 packets = q_vector->tx.total_packets; 4708 if (packets) 4709 avg_wire_size = max_t(u32, avg_wire_size, 4710 q_vector->tx.total_bytes / packets); 4711 4712 /* if avg_wire_size isn't set no work was done */ 4713 if (!avg_wire_size) 4714 goto clear_counts; 4715 4716 /* Add 24 bytes to size to account for CRC, preamble, and gap */ 4717 avg_wire_size += 24; 4718 4719 /* Don't starve jumbo frames */ 4720 avg_wire_size = min(avg_wire_size, 3000); 4721 4722 /* Give a little boost to mid-size frames */ 4723 if (avg_wire_size > 300 && avg_wire_size < 1200) 4724 new_val = avg_wire_size / 3; 4725 else 4726 new_val = avg_wire_size / 2; 4727 4728 /* conservative mode (itr 3) eliminates the lowest_latency setting */ 4729 if (new_val < IGC_20K_ITR && 4730 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || 4731 (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) 4732 new_val = IGC_20K_ITR; 4733 4734 set_itr_val: 4735 if (new_val != q_vector->itr_val) { 4736 q_vector->itr_val = new_val; 4737 q_vector->set_itr = 1; 4738 } 4739 clear_counts: 4740 q_vector->rx.total_bytes = 0; 4741 q_vector->rx.total_packets = 0; 4742 q_vector->tx.total_bytes = 0; 4743 q_vector->tx.total_packets = 0; 4744 } 4745 4746 static void igc_ring_irq_enable(struct igc_q_vector *q_vector) 4747 { 4748 struct igc_adapter *adapter = q_vector->adapter; 4749 struct igc_hw *hw = &adapter->hw; 4750 4751 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || 4752 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { 4753 if (adapter->num_q_vectors == 1) 4754 igc_set_itr(q_vector); 4755 else 4756 igc_update_ring_itr(q_vector); 4757 } 4758 4759 if (!test_bit(__IGC_DOWN, &adapter->state)) { 4760 if (adapter->msix_entries) 4761 wr32(IGC_EIMS, q_vector->eims_value); 4762 else 4763 igc_irq_enable(adapter); 4764 } 4765 } 4766 4767 static void igc_add_ring(struct igc_ring *ring, 4768 struct igc_ring_container *head) 4769 { 4770 head->ring = ring; 4771 head->count++; 4772 } 4773 4774 /** 4775 * igc_cache_ring_register - Descriptor ring to register mapping 4776 * @adapter: board private structure to initialize 4777 * 4778 * Once we know the feature-set enabled for the device, we'll cache 4779 * the register offset the descriptor ring is assigned to. 4780 */ 4781 static void igc_cache_ring_register(struct igc_adapter *adapter) 4782 { 4783 int i = 0, j = 0; 4784 4785 switch (adapter->hw.mac.type) { 4786 case igc_i225: 4787 default: 4788 for (; i < adapter->num_rx_queues; i++) 4789 adapter->rx_ring[i]->reg_idx = i; 4790 for (; j < adapter->num_tx_queues; j++) 4791 adapter->tx_ring[j]->reg_idx = j; 4792 break; 4793 } 4794 } 4795 4796 /** 4797 * igc_poll - NAPI Rx polling callback 4798 * @napi: napi polling structure 4799 * @budget: count of how many packets we should handle 4800 */ 4801 static int igc_poll(struct napi_struct *napi, int budget) 4802 { 4803 struct igc_q_vector *q_vector = container_of(napi, 4804 struct igc_q_vector, 4805 napi); 4806 struct igc_ring *rx_ring = q_vector->rx.ring; 4807 bool clean_complete = true; 4808 int work_done = 0; 4809 4810 if (q_vector->tx.ring) 4811 clean_complete = igc_clean_tx_irq(q_vector, budget); 4812 4813 if (rx_ring) { 4814 int cleaned = rx_ring->xsk_pool ? 4815 igc_clean_rx_irq_zc(q_vector, budget) : 4816 igc_clean_rx_irq(q_vector, budget); 4817 4818 work_done += cleaned; 4819 if (cleaned >= budget) 4820 clean_complete = false; 4821 } 4822 4823 /* If all work not completed, return budget and keep polling */ 4824 if (!clean_complete) 4825 return budget; 4826 4827 /* Exit the polling mode, but don't re-enable interrupts if stack might 4828 * poll us due to busy-polling 4829 */ 4830 if (likely(napi_complete_done(napi, work_done))) 4831 igc_ring_irq_enable(q_vector); 4832 4833 return min(work_done, budget - 1); 4834 } 4835 4836 /** 4837 * igc_alloc_q_vector - Allocate memory for a single interrupt vector 4838 * @adapter: board private structure to initialize 4839 * @v_count: q_vectors allocated on adapter, used for ring interleaving 4840 * @v_idx: index of vector in adapter struct 4841 * @txr_count: total number of Tx rings to allocate 4842 * @txr_idx: index of first Tx ring to allocate 4843 * @rxr_count: total number of Rx rings to allocate 4844 * @rxr_idx: index of first Rx ring to allocate 4845 * 4846 * We allocate one q_vector. If allocation fails we return -ENOMEM. 4847 */ 4848 static int igc_alloc_q_vector(struct igc_adapter *adapter, 4849 unsigned int v_count, unsigned int v_idx, 4850 unsigned int txr_count, unsigned int txr_idx, 4851 unsigned int rxr_count, unsigned int rxr_idx) 4852 { 4853 struct igc_q_vector *q_vector; 4854 struct igc_ring *ring; 4855 int ring_count; 4856 4857 /* igc only supports 1 Tx and/or 1 Rx queue per vector */ 4858 if (txr_count > 1 || rxr_count > 1) 4859 return -ENOMEM; 4860 4861 ring_count = txr_count + rxr_count; 4862 4863 /* allocate q_vector and rings */ 4864 q_vector = adapter->q_vector[v_idx]; 4865 if (!q_vector) 4866 q_vector = kzalloc_flex(*q_vector, ring, ring_count); 4867 else 4868 memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); 4869 if (!q_vector) 4870 return -ENOMEM; 4871 4872 /* initialize NAPI */ 4873 netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll); 4874 4875 /* tie q_vector and adapter together */ 4876 adapter->q_vector[v_idx] = q_vector; 4877 q_vector->adapter = adapter; 4878 4879 /* initialize work limits */ 4880 q_vector->tx.work_limit = adapter->tx_work_limit; 4881 4882 /* initialize ITR configuration */ 4883 q_vector->itr_register = adapter->io_addr + IGC_EITR(0); 4884 q_vector->itr_val = IGC_START_ITR; 4885 4886 /* initialize pointer to rings */ 4887 ring = q_vector->ring; 4888 4889 /* initialize ITR */ 4890 if (rxr_count) { 4891 /* rx or rx/tx vector */ 4892 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) 4893 q_vector->itr_val = adapter->rx_itr_setting; 4894 } else { 4895 /* tx only vector */ 4896 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) 4897 q_vector->itr_val = adapter->tx_itr_setting; 4898 } 4899 4900 if (txr_count) { 4901 /* assign generic ring traits */ 4902 ring->dev = &adapter->pdev->dev; 4903 ring->netdev = adapter->netdev; 4904 4905 /* configure backlink on ring */ 4906 ring->q_vector = q_vector; 4907 4908 /* update q_vector Tx values */ 4909 igc_add_ring(ring, &q_vector->tx); 4910 4911 /* apply Tx specific ring traits */ 4912 ring->count = adapter->tx_ring_count; 4913 ring->queue_index = txr_idx; 4914 4915 /* assign ring to adapter */ 4916 adapter->tx_ring[txr_idx] = ring; 4917 4918 /* push pointer to next ring */ 4919 ring++; 4920 } 4921 4922 if (rxr_count) { 4923 /* assign generic ring traits */ 4924 ring->dev = &adapter->pdev->dev; 4925 ring->netdev = adapter->netdev; 4926 4927 /* configure backlink on ring */ 4928 ring->q_vector = q_vector; 4929 4930 /* update q_vector Rx values */ 4931 igc_add_ring(ring, &q_vector->rx); 4932 4933 /* apply Rx specific ring traits */ 4934 ring->count = adapter->rx_ring_count; 4935 ring->queue_index = rxr_idx; 4936 4937 /* assign ring to adapter */ 4938 adapter->rx_ring[rxr_idx] = ring; 4939 } 4940 4941 return 0; 4942 } 4943 4944 /** 4945 * igc_alloc_q_vectors - Allocate memory for interrupt vectors 4946 * @adapter: board private structure to initialize 4947 * 4948 * We allocate one q_vector per queue interrupt. If allocation fails we 4949 * return -ENOMEM. 4950 */ 4951 static int igc_alloc_q_vectors(struct igc_adapter *adapter) 4952 { 4953 int rxr_remaining = adapter->num_rx_queues; 4954 int txr_remaining = adapter->num_tx_queues; 4955 int rxr_idx = 0, txr_idx = 0, v_idx = 0; 4956 int q_vectors = adapter->num_q_vectors; 4957 int err; 4958 4959 if (q_vectors >= (rxr_remaining + txr_remaining)) { 4960 for (; rxr_remaining; v_idx++) { 4961 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 4962 0, 0, 1, rxr_idx); 4963 4964 if (err) 4965 goto err_out; 4966 4967 /* update counts and index */ 4968 rxr_remaining--; 4969 rxr_idx++; 4970 } 4971 } 4972 4973 for (; v_idx < q_vectors; v_idx++) { 4974 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); 4975 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); 4976 4977 err = igc_alloc_q_vector(adapter, q_vectors, v_idx, 4978 tqpv, txr_idx, rqpv, rxr_idx); 4979 4980 if (err) 4981 goto err_out; 4982 4983 /* update counts and index */ 4984 rxr_remaining -= rqpv; 4985 txr_remaining -= tqpv; 4986 rxr_idx++; 4987 txr_idx++; 4988 } 4989 4990 return 0; 4991 4992 err_out: 4993 adapter->num_tx_queues = 0; 4994 adapter->num_rx_queues = 0; 4995 adapter->num_q_vectors = 0; 4996 4997 while (v_idx--) 4998 igc_free_q_vector(adapter, v_idx); 4999 5000 return -ENOMEM; 5001 } 5002 5003 /** 5004 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors 5005 * @adapter: Pointer to adapter structure 5006 * @msix: boolean for MSI-X capability 5007 * 5008 * This function initializes the interrupts and allocates all of the queues. 5009 */ 5010 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) 5011 { 5012 struct net_device *dev = adapter->netdev; 5013 int err = 0; 5014 5015 igc_set_interrupt_capability(adapter, msix); 5016 5017 err = igc_alloc_q_vectors(adapter); 5018 if (err) { 5019 netdev_err(dev, "Unable to allocate memory for vectors\n"); 5020 goto err_alloc_q_vectors; 5021 } 5022 5023 igc_cache_ring_register(adapter); 5024 5025 return 0; 5026 5027 err_alloc_q_vectors: 5028 igc_reset_interrupt_capability(adapter); 5029 return err; 5030 } 5031 5032 /** 5033 * igc_sw_init - Initialize general software structures (struct igc_adapter) 5034 * @adapter: board private structure to initialize 5035 * 5036 * igc_sw_init initializes the Adapter private data structure. 5037 * Fields are initialized based on PCI device information and 5038 * OS network device settings (MTU size). 5039 */ 5040 static int igc_sw_init(struct igc_adapter *adapter) 5041 { 5042 struct net_device *netdev = adapter->netdev; 5043 struct pci_dev *pdev = adapter->pdev; 5044 struct igc_hw *hw = &adapter->hw; 5045 5046 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); 5047 5048 /* init RSS key */ 5049 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key)); 5050 5051 /* set default ring sizes */ 5052 adapter->tx_ring_count = IGC_DEFAULT_TXD; 5053 adapter->rx_ring_count = IGC_DEFAULT_RXD; 5054 5055 /* set default ITR values */ 5056 adapter->rx_itr_setting = IGC_DEFAULT_ITR; 5057 adapter->tx_itr_setting = IGC_DEFAULT_ITR; 5058 5059 /* set default work limits */ 5060 adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; 5061 5062 /* adjust max frame to be at least the size of a standard frame */ 5063 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + 5064 VLAN_HLEN; 5065 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; 5066 5067 mutex_init(&adapter->nfc_rule_lock); 5068 INIT_LIST_HEAD(&adapter->nfc_rule_list); 5069 adapter->nfc_rule_count = 0; 5070 5071 spin_lock_init(&adapter->stats64_lock); 5072 spin_lock_init(&adapter->qbv_tx_lock); 5073 /* Assume MSI-X interrupts, will be checked during IRQ allocation */ 5074 adapter->flags |= IGC_FLAG_HAS_MSIX; 5075 5076 igc_init_queue_configuration(adapter); 5077 5078 /* This call may decrease the number of queues */ 5079 if (igc_init_interrupt_scheme(adapter, true)) { 5080 netdev_err(netdev, "Unable to allocate memory for queues\n"); 5081 return -ENOMEM; 5082 } 5083 5084 /* Explicitly disable IRQ since the NIC can be in any state. */ 5085 igc_irq_disable(adapter); 5086 5087 set_bit(__IGC_DOWN, &adapter->state); 5088 5089 return 0; 5090 } 5091 5092 static void igc_set_queue_napi(struct igc_adapter *adapter, int vector, 5093 struct napi_struct *napi) 5094 { 5095 struct igc_q_vector *q_vector = adapter->q_vector[vector]; 5096 5097 if (q_vector->rx.ring) 5098 netif_queue_set_napi(adapter->netdev, 5099 q_vector->rx.ring->queue_index, 5100 NETDEV_QUEUE_TYPE_RX, napi); 5101 5102 if (q_vector->tx.ring) 5103 netif_queue_set_napi(adapter->netdev, 5104 q_vector->tx.ring->queue_index, 5105 NETDEV_QUEUE_TYPE_TX, napi); 5106 } 5107 5108 /** 5109 * igc_up - Open the interface and prepare it to handle traffic 5110 * @adapter: board private structure 5111 */ 5112 void igc_up(struct igc_adapter *adapter) 5113 { 5114 struct igc_hw *hw = &adapter->hw; 5115 struct napi_struct *napi; 5116 int i = 0; 5117 5118 /* hardware has been reset, we need to reload some things */ 5119 igc_configure(adapter); 5120 5121 clear_bit(__IGC_DOWN, &adapter->state); 5122 5123 for (i = 0; i < adapter->num_q_vectors; i++) { 5124 napi = &adapter->q_vector[i]->napi; 5125 napi_enable(napi); 5126 igc_set_queue_napi(adapter, i, napi); 5127 } 5128 5129 if (adapter->msix_entries) 5130 igc_configure_msix(adapter); 5131 else 5132 igc_assign_vector(adapter->q_vector[0], 0); 5133 5134 /* Clear any pending interrupts. */ 5135 rd32(IGC_ICR); 5136 igc_irq_enable(adapter); 5137 5138 netif_tx_start_all_queues(adapter->netdev); 5139 5140 /* start the watchdog. */ 5141 hw->mac.get_link_status = true; 5142 schedule_work(&adapter->watchdog_task); 5143 } 5144 5145 /** 5146 * igc_update_stats - Update the board statistics counters 5147 * @adapter: board private structure 5148 */ 5149 void igc_update_stats(struct igc_adapter *adapter) 5150 { 5151 struct rtnl_link_stats64 *net_stats = &adapter->stats64; 5152 struct pci_dev *pdev = adapter->pdev; 5153 struct igc_hw *hw = &adapter->hw; 5154 u64 _bytes, _packets; 5155 u64 bytes, packets; 5156 unsigned int start; 5157 u32 mpc; 5158 int i; 5159 5160 /* Prevent stats update while adapter is being reset, or if the pci 5161 * connection is down. 5162 */ 5163 if (adapter->link_speed == 0) 5164 return; 5165 if (pci_channel_offline(pdev)) 5166 return; 5167 5168 packets = 0; 5169 bytes = 0; 5170 5171 rcu_read_lock(); 5172 for (i = 0; i < adapter->num_rx_queues; i++) { 5173 struct igc_ring *ring = adapter->rx_ring[i]; 5174 u32 rqdpc = rd32(IGC_RQDPC(i)); 5175 5176 if (hw->mac.type >= igc_i225) 5177 wr32(IGC_RQDPC(i), 0); 5178 5179 if (rqdpc) { 5180 ring->rx_stats.drops += rqdpc; 5181 net_stats->rx_fifo_errors += rqdpc; 5182 } 5183 5184 do { 5185 start = u64_stats_fetch_begin(&ring->rx_syncp); 5186 _bytes = ring->rx_stats.bytes; 5187 _packets = ring->rx_stats.packets; 5188 } while (u64_stats_fetch_retry(&ring->rx_syncp, start)); 5189 bytes += _bytes; 5190 packets += _packets; 5191 } 5192 5193 net_stats->rx_bytes = bytes; 5194 net_stats->rx_packets = packets; 5195 5196 packets = 0; 5197 bytes = 0; 5198 for (i = 0; i < adapter->num_tx_queues; i++) { 5199 struct igc_ring *ring = adapter->tx_ring[i]; 5200 5201 do { 5202 start = u64_stats_fetch_begin(&ring->tx_syncp); 5203 _bytes = ring->tx_stats.bytes; 5204 _packets = ring->tx_stats.packets; 5205 } while (u64_stats_fetch_retry(&ring->tx_syncp, start)); 5206 bytes += _bytes; 5207 packets += _packets; 5208 } 5209 net_stats->tx_bytes = bytes; 5210 net_stats->tx_packets = packets; 5211 rcu_read_unlock(); 5212 5213 /* read stats registers */ 5214 adapter->stats.crcerrs += rd32(IGC_CRCERRS); 5215 adapter->stats.gprc += rd32(IGC_GPRC); 5216 adapter->stats.gorc += rd32(IGC_GORCL); 5217 rd32(IGC_GORCH); /* clear GORCL */ 5218 adapter->stats.bprc += rd32(IGC_BPRC); 5219 adapter->stats.mprc += rd32(IGC_MPRC); 5220 adapter->stats.roc += rd32(IGC_ROC); 5221 5222 adapter->stats.prc64 += rd32(IGC_PRC64); 5223 adapter->stats.prc127 += rd32(IGC_PRC127); 5224 adapter->stats.prc255 += rd32(IGC_PRC255); 5225 adapter->stats.prc511 += rd32(IGC_PRC511); 5226 adapter->stats.prc1023 += rd32(IGC_PRC1023); 5227 adapter->stats.prc1522 += rd32(IGC_PRC1522); 5228 adapter->stats.tlpic += rd32(IGC_TLPIC); 5229 adapter->stats.rlpic += rd32(IGC_RLPIC); 5230 adapter->stats.hgptc += rd32(IGC_HGPTC); 5231 5232 mpc = rd32(IGC_MPC); 5233 adapter->stats.mpc += mpc; 5234 net_stats->rx_fifo_errors += mpc; 5235 adapter->stats.scc += rd32(IGC_SCC); 5236 adapter->stats.ecol += rd32(IGC_ECOL); 5237 adapter->stats.mcc += rd32(IGC_MCC); 5238 adapter->stats.latecol += rd32(IGC_LATECOL); 5239 adapter->stats.dc += rd32(IGC_DC); 5240 adapter->stats.rlec += rd32(IGC_RLEC); 5241 adapter->stats.xonrxc += rd32(IGC_XONRXC); 5242 adapter->stats.xontxc += rd32(IGC_XONTXC); 5243 adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); 5244 adapter->stats.xofftxc += rd32(IGC_XOFFTXC); 5245 adapter->stats.fcruc += rd32(IGC_FCRUC); 5246 adapter->stats.gptc += rd32(IGC_GPTC); 5247 adapter->stats.gotc += rd32(IGC_GOTCL); 5248 rd32(IGC_GOTCH); /* clear GOTCL */ 5249 adapter->stats.rnbc += rd32(IGC_RNBC); 5250 adapter->stats.ruc += rd32(IGC_RUC); 5251 adapter->stats.rfc += rd32(IGC_RFC); 5252 adapter->stats.rjc += rd32(IGC_RJC); 5253 adapter->stats.tor += rd32(IGC_TORH); 5254 adapter->stats.tot += rd32(IGC_TOTH); 5255 adapter->stats.tpr += rd32(IGC_TPR); 5256 5257 adapter->stats.ptc64 += rd32(IGC_PTC64); 5258 adapter->stats.ptc127 += rd32(IGC_PTC127); 5259 adapter->stats.ptc255 += rd32(IGC_PTC255); 5260 adapter->stats.ptc511 += rd32(IGC_PTC511); 5261 adapter->stats.ptc1023 += rd32(IGC_PTC1023); 5262 adapter->stats.ptc1522 += rd32(IGC_PTC1522); 5263 5264 adapter->stats.mptc += rd32(IGC_MPTC); 5265 adapter->stats.bptc += rd32(IGC_BPTC); 5266 5267 adapter->stats.tpt += rd32(IGC_TPT); 5268 adapter->stats.colc += rd32(IGC_COLC); 5269 adapter->stats.colc += rd32(IGC_RERC); 5270 5271 adapter->stats.algnerrc += rd32(IGC_ALGNERRC); 5272 5273 adapter->stats.tsctc += rd32(IGC_TSCTC); 5274 5275 adapter->stats.iac += rd32(IGC_IAC); 5276 5277 /* Fill out the OS statistics structure */ 5278 net_stats->multicast = adapter->stats.mprc; 5279 net_stats->collisions = adapter->stats.colc; 5280 5281 /* Rx Errors */ 5282 5283 /* RLEC on some newer hardware can be incorrect so build 5284 * our own version based on RUC and ROC 5285 */ 5286 net_stats->rx_errors = adapter->stats.rxerrc + 5287 adapter->stats.crcerrs + adapter->stats.algnerrc + 5288 adapter->stats.ruc + adapter->stats.roc + 5289 adapter->stats.cexterr; 5290 net_stats->rx_length_errors = adapter->stats.ruc + 5291 adapter->stats.roc; 5292 net_stats->rx_crc_errors = adapter->stats.crcerrs; 5293 net_stats->rx_frame_errors = adapter->stats.algnerrc; 5294 net_stats->rx_missed_errors = adapter->stats.mpc; 5295 5296 /* Tx Errors */ 5297 net_stats->tx_errors = adapter->stats.ecol + 5298 adapter->stats.latecol; 5299 net_stats->tx_aborted_errors = adapter->stats.ecol; 5300 net_stats->tx_window_errors = adapter->stats.latecol; 5301 net_stats->tx_carrier_errors = adapter->stats.tncrs; 5302 5303 /* Tx Dropped */ 5304 net_stats->tx_dropped = adapter->stats.txdrop; 5305 5306 /* Management Stats */ 5307 adapter->stats.mgptc += rd32(IGC_MGTPTC); 5308 adapter->stats.mgprc += rd32(IGC_MGTPRC); 5309 adapter->stats.mgpdc += rd32(IGC_MGTPDC); 5310 } 5311 5312 /** 5313 * igc_down - Close the interface 5314 * @adapter: board private structure 5315 */ 5316 void igc_down(struct igc_adapter *adapter) 5317 { 5318 struct net_device *netdev = adapter->netdev; 5319 struct igc_hw *hw = &adapter->hw; 5320 u32 tctl, rctl; 5321 int i = 0; 5322 5323 set_bit(__IGC_DOWN, &adapter->state); 5324 5325 igc_ptp_suspend(adapter); 5326 5327 if (pci_device_is_present(adapter->pdev)) { 5328 /* disable receives in the hardware */ 5329 rctl = rd32(IGC_RCTL); 5330 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); 5331 /* flush and sleep below */ 5332 } 5333 /* set trans_start so we don't get spurious watchdogs during reset */ 5334 netif_trans_update(netdev); 5335 5336 netif_carrier_off(netdev); 5337 netif_tx_stop_all_queues(netdev); 5338 5339 if (pci_device_is_present(adapter->pdev)) { 5340 /* disable transmits in the hardware */ 5341 tctl = rd32(IGC_TCTL); 5342 tctl &= ~IGC_TCTL_EN; 5343 wr32(IGC_TCTL, tctl); 5344 /* flush both disables and wait for them to finish */ 5345 wrfl(); 5346 usleep_range(10000, 20000); 5347 5348 igc_irq_disable(adapter); 5349 } 5350 5351 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 5352 5353 for (i = 0; i < adapter->num_q_vectors; i++) { 5354 if (adapter->q_vector[i]) { 5355 napi_disable(&adapter->q_vector[i]->napi); 5356 igc_set_queue_napi(adapter, i, NULL); 5357 } 5358 } 5359 5360 timer_delete_sync(&adapter->watchdog_timer); 5361 timer_delete_sync(&adapter->phy_info_timer); 5362 5363 /* record the stats before reset*/ 5364 spin_lock(&adapter->stats64_lock); 5365 igc_update_stats(adapter); 5366 spin_unlock(&adapter->stats64_lock); 5367 5368 adapter->link_speed = 0; 5369 adapter->link_duplex = 0; 5370 5371 if (!pci_channel_offline(adapter->pdev)) 5372 igc_reset(adapter); 5373 5374 /* clear VLAN promisc flag so VFTA will be updated if necessary */ 5375 adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; 5376 5377 igc_disable_all_tx_rings_hw(adapter); 5378 igc_clean_all_tx_rings(adapter); 5379 igc_clean_all_rx_rings(adapter); 5380 5381 if (adapter->fpe.mmsv.pmac_enabled) 5382 ethtool_mmsv_stop(&adapter->fpe.mmsv); 5383 } 5384 5385 void igc_reinit_locked(struct igc_adapter *adapter) 5386 { 5387 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 5388 usleep_range(1000, 2000); 5389 igc_down(adapter); 5390 igc_up(adapter); 5391 clear_bit(__IGC_RESETTING, &adapter->state); 5392 } 5393 5394 static void igc_reset_task(struct work_struct *work) 5395 { 5396 struct igc_adapter *adapter; 5397 5398 adapter = container_of(work, struct igc_adapter, reset_task); 5399 5400 rtnl_lock(); 5401 /* If we're already down or resetting, just bail */ 5402 if (test_bit(__IGC_DOWN, &adapter->state) || 5403 test_bit(__IGC_RESETTING, &adapter->state)) { 5404 rtnl_unlock(); 5405 return; 5406 } 5407 5408 igc_rings_dump(adapter); 5409 igc_regs_dump(adapter); 5410 netdev_err(adapter->netdev, "Reset adapter\n"); 5411 igc_reinit_locked(adapter); 5412 rtnl_unlock(); 5413 } 5414 5415 /** 5416 * igc_change_mtu - Change the Maximum Transfer Unit 5417 * @netdev: network interface device structure 5418 * @new_mtu: new value for maximum frame size 5419 * 5420 * Returns 0 on success, negative on failure 5421 */ 5422 static int igc_change_mtu(struct net_device *netdev, int new_mtu) 5423 { 5424 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 5425 struct igc_adapter *adapter = netdev_priv(netdev); 5426 5427 if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) { 5428 netdev_dbg(netdev, "Jumbo frames not supported with XDP"); 5429 return -EINVAL; 5430 } 5431 5432 /* adjust max frame to be at least the size of a standard frame */ 5433 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) 5434 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; 5435 5436 while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) 5437 usleep_range(1000, 2000); 5438 5439 /* igc_down has a dependency on max_frame_size */ 5440 adapter->max_frame_size = max_frame; 5441 5442 if (netif_running(netdev)) 5443 igc_down(adapter); 5444 5445 netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu); 5446 WRITE_ONCE(netdev->mtu, new_mtu); 5447 5448 if (netif_running(netdev)) 5449 igc_up(adapter); 5450 else 5451 igc_reset(adapter); 5452 5453 clear_bit(__IGC_RESETTING, &adapter->state); 5454 5455 return 0; 5456 } 5457 5458 /** 5459 * igc_tx_timeout - Respond to a Tx Hang 5460 * @netdev: network interface device structure 5461 * @txqueue: queue number that timed out 5462 **/ 5463 static void igc_tx_timeout(struct net_device *netdev, 5464 unsigned int __always_unused txqueue) 5465 { 5466 struct igc_adapter *adapter = netdev_priv(netdev); 5467 struct igc_hw *hw = &adapter->hw; 5468 5469 /* Do the reset outside of interrupt context */ 5470 adapter->tx_timeout_count++; 5471 schedule_work(&adapter->reset_task); 5472 wr32(IGC_EICS, 5473 (adapter->eims_enable_mask & ~adapter->eims_other)); 5474 } 5475 5476 /** 5477 * igc_get_stats64 - Get System Network Statistics 5478 * @netdev: network interface device structure 5479 * @stats: rtnl_link_stats64 pointer 5480 * 5481 * Returns the address of the device statistics structure. 5482 * The statistics are updated here and also from the timer callback. 5483 */ 5484 static void igc_get_stats64(struct net_device *netdev, 5485 struct rtnl_link_stats64 *stats) 5486 { 5487 struct igc_adapter *adapter = netdev_priv(netdev); 5488 5489 spin_lock(&adapter->stats64_lock); 5490 if (!test_bit(__IGC_RESETTING, &adapter->state)) 5491 igc_update_stats(adapter); 5492 memcpy(stats, &adapter->stats64, sizeof(*stats)); 5493 spin_unlock(&adapter->stats64_lock); 5494 } 5495 5496 static netdev_features_t igc_fix_features(struct net_device *netdev, 5497 netdev_features_t features) 5498 { 5499 /* Since there is no support for separate Rx/Tx vlan accel 5500 * enable/disable make sure Tx flag is always in same state as Rx. 5501 */ 5502 if (features & NETIF_F_HW_VLAN_CTAG_RX) 5503 features |= NETIF_F_HW_VLAN_CTAG_TX; 5504 else 5505 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 5506 5507 return features; 5508 } 5509 5510 static int igc_set_features(struct net_device *netdev, 5511 netdev_features_t features) 5512 { 5513 netdev_features_t changed = netdev->features ^ features; 5514 struct igc_adapter *adapter = netdev_priv(netdev); 5515 5516 if (changed & NETIF_F_HW_VLAN_CTAG_RX) 5517 igc_vlan_mode(netdev, features); 5518 5519 /* Add VLAN support */ 5520 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) 5521 return 0; 5522 5523 if (!(features & NETIF_F_NTUPLE)) 5524 igc_flush_nfc_rules(adapter); 5525 5526 netdev->features = features; 5527 5528 if (netif_running(netdev)) 5529 igc_reinit_locked(adapter); 5530 else 5531 igc_reset(adapter); 5532 5533 return 1; 5534 } 5535 5536 static netdev_features_t 5537 igc_features_check(struct sk_buff *skb, struct net_device *dev, 5538 netdev_features_t features) 5539 { 5540 unsigned int network_hdr_len, mac_hdr_len; 5541 5542 /* Make certain the headers can be described by a context descriptor */ 5543 mac_hdr_len = skb_network_offset(skb); 5544 if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) 5545 return features & ~(NETIF_F_HW_CSUM | 5546 NETIF_F_SCTP_CRC | 5547 NETIF_F_HW_VLAN_CTAG_TX | 5548 NETIF_F_TSO | 5549 NETIF_F_TSO6); 5550 5551 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 5552 if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) 5553 return features & ~(NETIF_F_HW_CSUM | 5554 NETIF_F_SCTP_CRC | 5555 NETIF_F_TSO | 5556 NETIF_F_TSO6); 5557 5558 /* We can only support IPv4 TSO in tunnels if we can mangle the 5559 * inner IP ID field, so strip TSO if MANGLEID is not supported. 5560 */ 5561 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 5562 features &= ~NETIF_F_TSO; 5563 5564 return features; 5565 } 5566 5567 static void igc_tsync_interrupt(struct igc_adapter *adapter) 5568 { 5569 struct igc_hw *hw = &adapter->hw; 5570 u32 tsauxc, sec, nsec, tsicr; 5571 struct ptp_clock_event event; 5572 struct timespec64 ts; 5573 5574 tsicr = rd32(IGC_TSICR); 5575 5576 if (tsicr & IGC_TSICR_SYS_WRAP) { 5577 event.type = PTP_CLOCK_PPS; 5578 if (adapter->ptp_caps.pps) 5579 ptp_clock_event(adapter->ptp_clock, &event); 5580 } 5581 5582 if (tsicr & IGC_TSICR_TXTS) { 5583 /* retrieve hardware timestamp */ 5584 igc_ptp_tx_tstamp_event(adapter); 5585 } 5586 5587 if (tsicr & IGC_TSICR_TT0) { 5588 spin_lock(&adapter->tmreg_lock); 5589 ts = timespec64_add(adapter->perout[0].start, 5590 adapter->perout[0].period); 5591 wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); 5592 wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec); 5593 tsauxc = rd32(IGC_TSAUXC); 5594 tsauxc |= IGC_TSAUXC_EN_TT0; 5595 wr32(IGC_TSAUXC, tsauxc); 5596 adapter->perout[0].start = ts; 5597 spin_unlock(&adapter->tmreg_lock); 5598 } 5599 5600 if (tsicr & IGC_TSICR_TT1) { 5601 spin_lock(&adapter->tmreg_lock); 5602 ts = timespec64_add(adapter->perout[1].start, 5603 adapter->perout[1].period); 5604 wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); 5605 wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec); 5606 tsauxc = rd32(IGC_TSAUXC); 5607 tsauxc |= IGC_TSAUXC_EN_TT1; 5608 wr32(IGC_TSAUXC, tsauxc); 5609 adapter->perout[1].start = ts; 5610 spin_unlock(&adapter->tmreg_lock); 5611 } 5612 5613 if (tsicr & IGC_TSICR_AUTT0) { 5614 nsec = rd32(IGC_AUXSTMPL0); 5615 sec = rd32(IGC_AUXSTMPH0); 5616 event.type = PTP_CLOCK_EXTTS; 5617 event.index = 0; 5618 event.timestamp = sec * NSEC_PER_SEC + nsec; 5619 ptp_clock_event(adapter->ptp_clock, &event); 5620 } 5621 5622 if (tsicr & IGC_TSICR_AUTT1) { 5623 nsec = rd32(IGC_AUXSTMPL1); 5624 sec = rd32(IGC_AUXSTMPH1); 5625 event.type = PTP_CLOCK_EXTTS; 5626 event.index = 1; 5627 event.timestamp = sec * NSEC_PER_SEC + nsec; 5628 ptp_clock_event(adapter->ptp_clock, &event); 5629 } 5630 } 5631 5632 /** 5633 * igc_msix_other - msix other interrupt handler 5634 * @irq: interrupt number 5635 * @data: pointer to a q_vector 5636 */ 5637 static irqreturn_t igc_msix_other(int irq, void *data) 5638 { 5639 struct igc_adapter *adapter = data; 5640 struct igc_hw *hw = &adapter->hw; 5641 u32 icr = rd32(IGC_ICR); 5642 5643 /* reading ICR causes bit 31 of EICR to be cleared */ 5644 if (icr & IGC_ICR_DRSTA) 5645 schedule_work(&adapter->reset_task); 5646 5647 if (icr & IGC_ICR_DOUTSYNC) { 5648 /* HW is reporting DMA is out of sync */ 5649 adapter->stats.doosync++; 5650 } 5651 5652 if (icr & IGC_ICR_LSC) { 5653 hw->mac.get_link_status = true; 5654 /* guard against interrupt when we're going down */ 5655 if (!test_bit(__IGC_DOWN, &adapter->state)) 5656 mod_timer(&adapter->watchdog_timer, jiffies + 1); 5657 } 5658 5659 if (icr & IGC_ICR_TS) 5660 igc_tsync_interrupt(adapter); 5661 5662 wr32(IGC_EIMS, adapter->eims_other); 5663 5664 return IRQ_HANDLED; 5665 } 5666 5667 static void igc_write_itr(struct igc_q_vector *q_vector) 5668 { 5669 u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; 5670 5671 if (!q_vector->set_itr) 5672 return; 5673 5674 if (!itr_val) 5675 itr_val = IGC_ITR_VAL_MASK; 5676 5677 itr_val |= IGC_EITR_CNT_IGNR; 5678 5679 writel(itr_val, q_vector->itr_register); 5680 q_vector->set_itr = 0; 5681 } 5682 5683 static irqreturn_t igc_msix_ring(int irq, void *data) 5684 { 5685 struct igc_q_vector *q_vector = data; 5686 5687 /* Write the ITR value calculated from the previous interrupt. */ 5688 igc_write_itr(q_vector); 5689 5690 napi_schedule_irqoff(&q_vector->napi); 5691 5692 return IRQ_HANDLED; 5693 } 5694 5695 /** 5696 * igc_request_msix - Initialize MSI-X interrupts 5697 * @adapter: Pointer to adapter structure 5698 * 5699 * igc_request_msix allocates MSI-X vectors and requests interrupts from the 5700 * kernel. 5701 */ 5702 static int igc_request_msix(struct igc_adapter *adapter) 5703 { 5704 unsigned int num_q_vectors = adapter->num_q_vectors; 5705 int i = 0, err = 0, vector = 0, free_vector = 0; 5706 struct net_device *netdev = adapter->netdev; 5707 5708 err = request_irq(adapter->msix_entries[vector].vector, 5709 &igc_msix_other, 0, netdev->name, adapter); 5710 if (err) 5711 goto err_out; 5712 5713 if (num_q_vectors > MAX_Q_VECTORS) { 5714 num_q_vectors = MAX_Q_VECTORS; 5715 dev_warn(&adapter->pdev->dev, 5716 "The number of queue vectors (%d) is higher than max allowed (%d)\n", 5717 adapter->num_q_vectors, MAX_Q_VECTORS); 5718 } 5719 for (i = 0; i < num_q_vectors; i++) { 5720 struct igc_q_vector *q_vector = adapter->q_vector[i]; 5721 5722 vector++; 5723 5724 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); 5725 5726 if (q_vector->rx.ring && q_vector->tx.ring) 5727 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, 5728 q_vector->rx.ring->queue_index); 5729 else if (q_vector->tx.ring) 5730 sprintf(q_vector->name, "%s-tx-%u", netdev->name, 5731 q_vector->tx.ring->queue_index); 5732 else if (q_vector->rx.ring) 5733 sprintf(q_vector->name, "%s-rx-%u", netdev->name, 5734 q_vector->rx.ring->queue_index); 5735 else 5736 sprintf(q_vector->name, "%s-unused", netdev->name); 5737 5738 err = request_irq(adapter->msix_entries[vector].vector, 5739 igc_msix_ring, 0, q_vector->name, 5740 q_vector); 5741 if (err) 5742 goto err_free; 5743 5744 netif_napi_set_irq(&q_vector->napi, 5745 adapter->msix_entries[vector].vector); 5746 } 5747 5748 igc_configure_msix(adapter); 5749 return 0; 5750 5751 err_free: 5752 /* free already assigned IRQs */ 5753 free_irq(adapter->msix_entries[free_vector++].vector, adapter); 5754 5755 vector--; 5756 for (i = 0; i < vector; i++) { 5757 free_irq(adapter->msix_entries[free_vector++].vector, 5758 adapter->q_vector[i]); 5759 } 5760 err_out: 5761 return err; 5762 } 5763 5764 /** 5765 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts 5766 * @adapter: Pointer to adapter structure 5767 * 5768 * This function resets the device so that it has 0 rx queues, tx queues, and 5769 * MSI-X interrupts allocated. 5770 */ 5771 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) 5772 { 5773 igc_free_q_vectors(adapter); 5774 igc_reset_interrupt_capability(adapter); 5775 } 5776 5777 /* Need to wait a few seconds after link up to get diagnostic information from 5778 * the phy 5779 */ 5780 static void igc_update_phy_info(struct timer_list *t) 5781 { 5782 struct igc_adapter *adapter = timer_container_of(adapter, t, 5783 phy_info_timer); 5784 5785 igc_get_phy_info(&adapter->hw); 5786 } 5787 5788 /** 5789 * igc_has_link - check shared code for link and determine up/down 5790 * @adapter: pointer to driver private info 5791 */ 5792 bool igc_has_link(struct igc_adapter *adapter) 5793 { 5794 struct igc_hw *hw = &adapter->hw; 5795 bool link_active = false; 5796 5797 /* get_link_status is set on LSC (link status) interrupt or 5798 * rx sequence error interrupt. get_link_status will stay 5799 * false until the igc_check_for_link establishes link 5800 * for copper adapters ONLY 5801 */ 5802 if (!hw->mac.get_link_status) 5803 return true; 5804 hw->mac.ops.check_for_link(hw); 5805 link_active = !hw->mac.get_link_status; 5806 5807 if (hw->mac.type == igc_i225) { 5808 if (!netif_carrier_ok(adapter->netdev)) { 5809 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 5810 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { 5811 adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; 5812 adapter->link_check_timeout = jiffies; 5813 } 5814 } 5815 5816 return link_active; 5817 } 5818 5819 /** 5820 * igc_watchdog - Timer Call-back 5821 * @t: timer for the watchdog 5822 */ 5823 static void igc_watchdog(struct timer_list *t) 5824 { 5825 struct igc_adapter *adapter = timer_container_of(adapter, t, 5826 watchdog_timer); 5827 /* Do the rest outside of interrupt context */ 5828 schedule_work(&adapter->watchdog_task); 5829 } 5830 5831 static void igc_watchdog_task(struct work_struct *work) 5832 { 5833 struct igc_adapter *adapter = container_of(work, 5834 struct igc_adapter, 5835 watchdog_task); 5836 struct net_device *netdev = adapter->netdev; 5837 struct igc_hw *hw = &adapter->hw; 5838 struct igc_phy_info *phy = &hw->phy; 5839 u16 phy_data, retry_count = 20; 5840 u32 link; 5841 int i; 5842 5843 link = igc_has_link(adapter); 5844 5845 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { 5846 if (time_after(jiffies, (adapter->link_check_timeout + HZ))) 5847 adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; 5848 else 5849 link = false; 5850 } 5851 5852 if (link) { 5853 /* Cancel scheduled suspend requests. */ 5854 pm_runtime_resume(netdev->dev.parent); 5855 5856 if (!netif_carrier_ok(netdev)) { 5857 u32 ctrl; 5858 5859 hw->mac.ops.get_speed_and_duplex(hw, 5860 &adapter->link_speed, 5861 &adapter->link_duplex); 5862 5863 ctrl = rd32(IGC_CTRL); 5864 /* Link status message must follow this format */ 5865 netdev_info(netdev, 5866 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", 5867 adapter->link_speed, 5868 adapter->link_duplex == FULL_DUPLEX ? 5869 "Full" : "Half", 5870 (ctrl & IGC_CTRL_TFCE) && 5871 (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : 5872 (ctrl & IGC_CTRL_RFCE) ? "RX" : 5873 (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); 5874 5875 /* disable EEE if enabled */ 5876 if ((adapter->flags & IGC_FLAG_EEE) && 5877 adapter->link_duplex == HALF_DUPLEX) { 5878 netdev_info(netdev, 5879 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n"); 5880 adapter->hw.dev_spec._base.eee_enable = false; 5881 adapter->flags &= ~IGC_FLAG_EEE; 5882 } 5883 5884 /* check if SmartSpeed worked */ 5885 igc_check_downshift(hw); 5886 if (phy->speed_downgraded) 5887 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); 5888 5889 /* adjust timeout factor according to speed/duplex */ 5890 adapter->tx_timeout_factor = 1; 5891 switch (adapter->link_speed) { 5892 case SPEED_10: 5893 adapter->tx_timeout_factor = 14; 5894 break; 5895 case SPEED_100: 5896 case SPEED_1000: 5897 case SPEED_2500: 5898 adapter->tx_timeout_factor = 1; 5899 break; 5900 } 5901 5902 /* Once the launch time has been set on the wire, there 5903 * is a delay before the link speed can be determined 5904 * based on link-up activity. Write into the register 5905 * as soon as we know the correct link speed. 5906 */ 5907 igc_tsn_adjust_txtime_offset(adapter); 5908 5909 if (adapter->fpe.mmsv.pmac_enabled) 5910 ethtool_mmsv_link_state_handle(&adapter->fpe.mmsv, 5911 true); 5912 5913 if (adapter->link_speed != SPEED_1000) 5914 goto no_wait; 5915 5916 /* wait for Remote receiver status OK */ 5917 retry_read_status: 5918 if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, 5919 &phy_data)) { 5920 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && 5921 retry_count) { 5922 msleep(100); 5923 retry_count--; 5924 goto retry_read_status; 5925 } else if (!retry_count) { 5926 netdev_err(netdev, "exceed max 2 second\n"); 5927 } 5928 } else { 5929 netdev_err(netdev, "read 1000Base-T Status Reg\n"); 5930 } 5931 no_wait: 5932 netif_carrier_on(netdev); 5933 5934 /* link state has changed, schedule phy info update */ 5935 if (!test_bit(__IGC_DOWN, &adapter->state)) 5936 mod_timer(&adapter->phy_info_timer, 5937 round_jiffies(jiffies + 2 * HZ)); 5938 } 5939 } else { 5940 if (netif_carrier_ok(netdev)) { 5941 adapter->link_speed = 0; 5942 adapter->link_duplex = 0; 5943 5944 /* Links status message must follow this format */ 5945 netdev_info(netdev, "NIC Link is Down\n"); 5946 netif_carrier_off(netdev); 5947 5948 if (adapter->fpe.mmsv.pmac_enabled) 5949 ethtool_mmsv_link_state_handle(&adapter->fpe.mmsv, 5950 false); 5951 5952 /* link state has changed, schedule phy info update */ 5953 if (!test_bit(__IGC_DOWN, &adapter->state)) 5954 mod_timer(&adapter->phy_info_timer, 5955 round_jiffies(jiffies + 2 * HZ)); 5956 5957 pm_schedule_suspend(netdev->dev.parent, 5958 MSEC_PER_SEC * 5); 5959 } 5960 } 5961 5962 spin_lock(&adapter->stats64_lock); 5963 igc_update_stats(adapter); 5964 spin_unlock(&adapter->stats64_lock); 5965 5966 for (i = 0; i < adapter->num_tx_queues; i++) { 5967 struct igc_ring *tx_ring = adapter->tx_ring[i]; 5968 5969 if (!netif_carrier_ok(netdev)) { 5970 /* We've lost link, so the controller stops DMA, 5971 * but we've got queued Tx work that's never going 5972 * to get done, so reset controller to flush Tx. 5973 * (Do the reset outside of interrupt context). 5974 */ 5975 if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { 5976 adapter->tx_timeout_count++; 5977 schedule_work(&adapter->reset_task); 5978 /* return immediately since reset is imminent */ 5979 return; 5980 } 5981 } 5982 5983 /* Force detection of hung controller every watchdog period */ 5984 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); 5985 } 5986 5987 /* Cause software interrupt to ensure Rx ring is cleaned */ 5988 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 5989 u32 eics = 0; 5990 5991 for (i = 0; i < adapter->num_q_vectors; i++) { 5992 struct igc_q_vector *q_vector = adapter->q_vector[i]; 5993 struct igc_ring *rx_ring; 5994 5995 if (!q_vector->rx.ring) 5996 continue; 5997 5998 rx_ring = adapter->rx_ring[q_vector->rx.ring->queue_index]; 5999 6000 if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) { 6001 eics |= q_vector->eims_value; 6002 clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 6003 } 6004 } 6005 if (eics) 6006 wr32(IGC_EICS, eics); 6007 } else { 6008 struct igc_ring *rx_ring = adapter->rx_ring[0]; 6009 6010 if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) { 6011 clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); 6012 wr32(IGC_ICS, IGC_ICS_RXDMT0); 6013 } 6014 } 6015 6016 igc_ptp_tx_hang(adapter); 6017 6018 /* Reset the timer */ 6019 if (!test_bit(__IGC_DOWN, &adapter->state)) { 6020 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) 6021 mod_timer(&adapter->watchdog_timer, 6022 round_jiffies(jiffies + HZ)); 6023 else 6024 mod_timer(&adapter->watchdog_timer, 6025 round_jiffies(jiffies + 2 * HZ)); 6026 } 6027 } 6028 6029 /** 6030 * igc_intr_msi - Interrupt Handler 6031 * @irq: interrupt number 6032 * @data: pointer to a network interface device structure 6033 */ 6034 static irqreturn_t igc_intr_msi(int irq, void *data) 6035 { 6036 struct igc_adapter *adapter = data; 6037 struct igc_q_vector *q_vector = adapter->q_vector[0]; 6038 struct igc_hw *hw = &adapter->hw; 6039 /* read ICR disables interrupts using IAM */ 6040 u32 icr = rd32(IGC_ICR); 6041 6042 igc_write_itr(q_vector); 6043 6044 if (icr & IGC_ICR_DRSTA) 6045 schedule_work(&adapter->reset_task); 6046 6047 if (icr & IGC_ICR_DOUTSYNC) { 6048 /* HW is reporting DMA is out of sync */ 6049 adapter->stats.doosync++; 6050 } 6051 6052 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 6053 hw->mac.get_link_status = true; 6054 if (!test_bit(__IGC_DOWN, &adapter->state)) 6055 mod_timer(&adapter->watchdog_timer, jiffies + 1); 6056 } 6057 6058 if (icr & IGC_ICR_TS) 6059 igc_tsync_interrupt(adapter); 6060 6061 napi_schedule_irqoff(&q_vector->napi); 6062 6063 return IRQ_HANDLED; 6064 } 6065 6066 /** 6067 * igc_intr - Legacy Interrupt Handler 6068 * @irq: interrupt number 6069 * @data: pointer to a network interface device structure 6070 */ 6071 static irqreturn_t igc_intr(int irq, void *data) 6072 { 6073 struct igc_adapter *adapter = data; 6074 struct igc_q_vector *q_vector = adapter->q_vector[0]; 6075 struct igc_hw *hw = &adapter->hw; 6076 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No 6077 * need for the IMC write 6078 */ 6079 u32 icr = rd32(IGC_ICR); 6080 6081 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is 6082 * not set, then the adapter didn't send an interrupt 6083 */ 6084 if (!(icr & IGC_ICR_INT_ASSERTED)) 6085 return IRQ_NONE; 6086 6087 igc_write_itr(q_vector); 6088 6089 if (icr & IGC_ICR_DRSTA) 6090 schedule_work(&adapter->reset_task); 6091 6092 if (icr & IGC_ICR_DOUTSYNC) { 6093 /* HW is reporting DMA is out of sync */ 6094 adapter->stats.doosync++; 6095 } 6096 6097 if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 6098 hw->mac.get_link_status = true; 6099 /* guard against interrupt when we're going down */ 6100 if (!test_bit(__IGC_DOWN, &adapter->state)) 6101 mod_timer(&adapter->watchdog_timer, jiffies + 1); 6102 } 6103 6104 if (icr & IGC_ICR_TS) 6105 igc_tsync_interrupt(adapter); 6106 6107 napi_schedule_irqoff(&q_vector->napi); 6108 6109 return IRQ_HANDLED; 6110 } 6111 6112 static void igc_free_irq(struct igc_adapter *adapter) 6113 { 6114 if (adapter->msix_entries) { 6115 int vector = 0, i; 6116 6117 free_irq(adapter->msix_entries[vector++].vector, adapter); 6118 6119 for (i = 0; i < adapter->num_q_vectors; i++) 6120 free_irq(adapter->msix_entries[vector++].vector, 6121 adapter->q_vector[i]); 6122 } else { 6123 free_irq(adapter->pdev->irq, adapter); 6124 } 6125 } 6126 6127 /** 6128 * igc_request_irq - initialize interrupts 6129 * @adapter: Pointer to adapter structure 6130 * 6131 * Attempts to configure interrupts using the best available 6132 * capabilities of the hardware and kernel. 6133 */ 6134 static int igc_request_irq(struct igc_adapter *adapter) 6135 { 6136 struct net_device *netdev = adapter->netdev; 6137 struct pci_dev *pdev = adapter->pdev; 6138 int err = 0; 6139 6140 if (adapter->flags & IGC_FLAG_HAS_MSIX) { 6141 err = igc_request_msix(adapter); 6142 if (!err) 6143 goto request_done; 6144 /* fall back to MSI */ 6145 igc_free_all_tx_resources(adapter); 6146 igc_free_all_rx_resources(adapter); 6147 6148 igc_clear_interrupt_scheme(adapter); 6149 err = igc_init_interrupt_scheme(adapter, false); 6150 if (err) 6151 goto request_done; 6152 igc_setup_all_tx_resources(adapter); 6153 igc_setup_all_rx_resources(adapter); 6154 igc_configure(adapter); 6155 } 6156 6157 igc_assign_vector(adapter->q_vector[0], 0); 6158 6159 if (adapter->flags & IGC_FLAG_HAS_MSI) { 6160 err = request_irq(pdev->irq, &igc_intr_msi, 0, 6161 netdev->name, adapter); 6162 if (!err) 6163 goto request_done; 6164 6165 /* fall back to legacy interrupts */ 6166 igc_reset_interrupt_capability(adapter); 6167 adapter->flags &= ~IGC_FLAG_HAS_MSI; 6168 } 6169 6170 err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED, 6171 netdev->name, adapter); 6172 6173 if (err) 6174 netdev_err(netdev, "Error %d getting interrupt\n", err); 6175 6176 request_done: 6177 return err; 6178 } 6179 6180 /** 6181 * __igc_open - Called when a network interface is made active 6182 * @netdev: network interface device structure 6183 * @resuming: boolean indicating if the device is resuming 6184 * 6185 * Returns 0 on success, negative value on failure 6186 * 6187 * The open entry point is called when a network interface is made 6188 * active by the system (IFF_UP). At this point all resources needed 6189 * for transmit and receive operations are allocated, the interrupt 6190 * handler is registered with the OS, the watchdog timer is started, 6191 * and the stack is notified that the interface is ready. 6192 */ 6193 static int __igc_open(struct net_device *netdev, bool resuming) 6194 { 6195 struct igc_adapter *adapter = netdev_priv(netdev); 6196 struct pci_dev *pdev = adapter->pdev; 6197 struct igc_hw *hw = &adapter->hw; 6198 struct napi_struct *napi; 6199 int err = 0; 6200 int i = 0; 6201 6202 /* disallow open during test */ 6203 6204 if (test_bit(__IGC_TESTING, &adapter->state)) { 6205 WARN_ON(resuming); 6206 return -EBUSY; 6207 } 6208 6209 if (!resuming) 6210 pm_runtime_get_sync(&pdev->dev); 6211 6212 netif_carrier_off(netdev); 6213 6214 /* allocate transmit descriptors */ 6215 err = igc_setup_all_tx_resources(adapter); 6216 if (err) 6217 goto err_setup_tx; 6218 6219 /* allocate receive descriptors */ 6220 err = igc_setup_all_rx_resources(adapter); 6221 if (err) 6222 goto err_setup_rx; 6223 6224 igc_power_up_link(adapter); 6225 6226 igc_configure(adapter); 6227 6228 err = igc_request_irq(adapter); 6229 if (err) 6230 goto err_req_irq; 6231 6232 clear_bit(__IGC_DOWN, &adapter->state); 6233 6234 for (i = 0; i < adapter->num_q_vectors; i++) { 6235 napi = &adapter->q_vector[i]->napi; 6236 napi_enable(napi); 6237 igc_set_queue_napi(adapter, i, napi); 6238 } 6239 6240 /* Clear any pending interrupts. */ 6241 rd32(IGC_ICR); 6242 igc_irq_enable(adapter); 6243 6244 if (!resuming) 6245 pm_runtime_put(&pdev->dev); 6246 6247 netif_tx_start_all_queues(netdev); 6248 6249 /* start the watchdog. */ 6250 hw->mac.get_link_status = true; 6251 schedule_work(&adapter->watchdog_task); 6252 6253 return IGC_SUCCESS; 6254 6255 err_req_irq: 6256 igc_release_hw_control(adapter); 6257 igc_power_down_phy_copper_base(&adapter->hw); 6258 igc_free_all_rx_resources(adapter); 6259 err_setup_rx: 6260 igc_free_all_tx_resources(adapter); 6261 err_setup_tx: 6262 igc_reset(adapter); 6263 if (!resuming) 6264 pm_runtime_put(&pdev->dev); 6265 6266 return err; 6267 } 6268 6269 int igc_open(struct net_device *netdev) 6270 { 6271 struct igc_adapter *adapter = netdev_priv(netdev); 6272 int err; 6273 6274 /* Notify the stack of the actual queue counts. */ 6275 err = netif_set_real_num_queues(netdev, adapter->num_tx_queues, 6276 adapter->num_rx_queues); 6277 if (err) { 6278 netdev_err(netdev, "error setting real queue count\n"); 6279 return err; 6280 } 6281 6282 return __igc_open(netdev, false); 6283 } 6284 6285 /** 6286 * __igc_close - Disables a network interface 6287 * @netdev: network interface device structure 6288 * @suspending: boolean indicating the device is suspending 6289 * 6290 * Returns 0, this is not allowed to fail 6291 * 6292 * The close entry point is called when an interface is de-activated 6293 * by the OS. The hardware is still under the driver's control, but 6294 * needs to be disabled. A global MAC reset is issued to stop the 6295 * hardware, and all transmit and receive resources are freed. 6296 */ 6297 static int __igc_close(struct net_device *netdev, bool suspending) 6298 { 6299 struct igc_adapter *adapter = netdev_priv(netdev); 6300 struct pci_dev *pdev = adapter->pdev; 6301 6302 WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); 6303 6304 if (!suspending) 6305 pm_runtime_get_sync(&pdev->dev); 6306 6307 igc_down(adapter); 6308 6309 igc_release_hw_control(adapter); 6310 6311 igc_free_irq(adapter); 6312 6313 igc_free_all_tx_resources(adapter); 6314 igc_free_all_rx_resources(adapter); 6315 6316 if (!suspending) 6317 pm_runtime_put_sync(&pdev->dev); 6318 6319 return 0; 6320 } 6321 6322 int igc_close(struct net_device *netdev) 6323 { 6324 if (netif_device_present(netdev) || netdev->dismantle) 6325 return __igc_close(netdev, false); 6326 return 0; 6327 } 6328 6329 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, 6330 bool enable) 6331 { 6332 struct igc_ring *ring; 6333 6334 if (queue < 0 || queue >= adapter->num_tx_queues) 6335 return -EINVAL; 6336 6337 ring = adapter->tx_ring[queue]; 6338 ring->launchtime_enable = enable; 6339 6340 return 0; 6341 } 6342 6343 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) 6344 { 6345 struct timespec64 b; 6346 6347 b = ktime_to_timespec64(base_time); 6348 6349 return timespec64_compare(now, &b) > 0; 6350 } 6351 6352 static bool validate_schedule(struct igc_adapter *adapter, 6353 const struct tc_taprio_qopt_offload *qopt) 6354 { 6355 int queue_uses[IGC_MAX_TX_QUEUES] = { }; 6356 struct igc_hw *hw = &adapter->hw; 6357 struct timespec64 now; 6358 size_t n; 6359 6360 if (qopt->cycle_time_extension) 6361 return false; 6362 6363 igc_ptp_read(adapter, &now); 6364 6365 /* If we program the controller's BASET registers with a time 6366 * in the future, it will hold all the packets until that 6367 * time, causing a lot of TX Hangs, so to avoid that, we 6368 * reject schedules that would start in the future. 6369 * Note: Limitation above is no longer in i226. 6370 */ 6371 if (!is_base_time_past(qopt->base_time, &now) && 6372 igc_is_device_id_i225(hw)) 6373 return false; 6374 6375 for (n = 0; n < qopt->num_entries; n++) { 6376 const struct tc_taprio_sched_entry *e, *prev; 6377 int i; 6378 6379 prev = n ? &qopt->entries[n - 1] : NULL; 6380 e = &qopt->entries[n]; 6381 6382 /* i225 only supports "global" frame preemption 6383 * settings. 6384 */ 6385 if (e->command != TC_TAPRIO_CMD_SET_GATES) 6386 return false; 6387 6388 for (i = 0; i < adapter->num_tx_queues; i++) 6389 if (e->gate_mask & BIT(i)) { 6390 queue_uses[i]++; 6391 6392 /* There are limitations: A single queue cannot 6393 * be opened and closed multiple times per cycle 6394 * unless the gate stays open. Check for it. 6395 */ 6396 if (queue_uses[i] > 1 && 6397 !(prev->gate_mask & BIT(i))) 6398 return false; 6399 } 6400 } 6401 6402 return true; 6403 } 6404 6405 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, 6406 struct tc_etf_qopt_offload *qopt) 6407 { 6408 struct igc_hw *hw = &adapter->hw; 6409 int err; 6410 6411 if (hw->mac.type != igc_i225) 6412 return -EOPNOTSUPP; 6413 6414 err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable); 6415 if (err) 6416 return err; 6417 6418 return igc_tsn_offload_apply(adapter); 6419 } 6420 6421 static int igc_qbv_clear_schedule(struct igc_adapter *adapter) 6422 { 6423 unsigned long flags; 6424 int i; 6425 6426 adapter->base_time = 0; 6427 adapter->cycle_time = NSEC_PER_SEC; 6428 adapter->taprio_offload_enable = false; 6429 adapter->qbv_config_change_errors = 0; 6430 adapter->qbv_count = 0; 6431 6432 for (i = 0; i < adapter->num_tx_queues; i++) { 6433 struct igc_ring *ring = adapter->tx_ring[i]; 6434 6435 ring->start_time = 0; 6436 ring->end_time = NSEC_PER_SEC; 6437 ring->max_sdu = 0; 6438 ring->preemptible = false; 6439 } 6440 6441 spin_lock_irqsave(&adapter->qbv_tx_lock, flags); 6442 6443 adapter->qbv_transition = false; 6444 6445 for (i = 0; i < adapter->num_tx_queues; i++) { 6446 struct igc_ring *ring = adapter->tx_ring[i]; 6447 6448 ring->oper_gate_closed = false; 6449 ring->admin_gate_closed = false; 6450 } 6451 6452 spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags); 6453 6454 return 0; 6455 } 6456 6457 static int igc_tsn_clear_schedule(struct igc_adapter *adapter) 6458 { 6459 igc_qbv_clear_schedule(adapter); 6460 6461 return 0; 6462 } 6463 6464 static void igc_taprio_stats(struct net_device *dev, 6465 struct tc_taprio_qopt_stats *stats) 6466 { 6467 /* When Strict_End is enabled, the tx_overruns counter 6468 * will always be zero. 6469 */ 6470 stats->tx_overruns = 0; 6471 } 6472 6473 static void igc_taprio_queue_stats(struct net_device *dev, 6474 struct tc_taprio_qopt_queue_stats *queue_stats) 6475 { 6476 struct tc_taprio_qopt_stats *stats = &queue_stats->stats; 6477 6478 /* When Strict_End is enabled, the tx_overruns counter 6479 * will always be zero. 6480 */ 6481 stats->tx_overruns = 0; 6482 } 6483 6484 static int igc_save_qbv_schedule(struct igc_adapter *adapter, 6485 struct tc_taprio_qopt_offload *qopt) 6486 { 6487 bool queue_configured[IGC_MAX_TX_QUEUES] = { }; 6488 struct igc_hw *hw = &adapter->hw; 6489 u32 start_time = 0, end_time = 0; 6490 struct timespec64 now; 6491 unsigned long flags; 6492 size_t n; 6493 int i; 6494 6495 if (qopt->base_time < 0) 6496 return -ERANGE; 6497 6498 if (igc_is_device_id_i225(hw) && adapter->taprio_offload_enable) 6499 return -EALREADY; 6500 6501 if (!validate_schedule(adapter, qopt)) 6502 return -EINVAL; 6503 6504 if (qopt->mqprio.preemptible_tcs && 6505 !(adapter->flags & IGC_FLAG_TSN_REVERSE_TXQ_PRIO)) { 6506 NL_SET_ERR_MSG_MOD(qopt->extack, 6507 "reverse-tsn-txq-prio private flag must be enabled before setting preemptible tc"); 6508 return -ENODEV; 6509 } 6510 6511 igc_ptp_read(adapter, &now); 6512 6513 if (igc_tsn_is_taprio_activated_by_user(adapter) && 6514 is_base_time_past(qopt->base_time, &now)) 6515 adapter->qbv_config_change_errors++; 6516 6517 adapter->cycle_time = qopt->cycle_time; 6518 adapter->base_time = qopt->base_time; 6519 adapter->taprio_offload_enable = true; 6520 6521 for (n = 0; n < qopt->num_entries; n++) { 6522 struct tc_taprio_sched_entry *e = &qopt->entries[n]; 6523 6524 end_time += e->interval; 6525 6526 /* If any of the conditions below are true, we need to manually 6527 * control the end time of the cycle. 6528 * 1. Qbv users can specify a cycle time that is not equal 6529 * to the total GCL intervals. Hence, recalculation is 6530 * necessary here to exclude the time interval that 6531 * exceeds the cycle time. 6532 * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2, 6533 * once the end of the list is reached, it will switch 6534 * to the END_OF_CYCLE state and leave the gates in the 6535 * same state until the next cycle is started. 6536 */ 6537 if (end_time > adapter->cycle_time || 6538 n + 1 == qopt->num_entries) 6539 end_time = adapter->cycle_time; 6540 6541 for (i = 0; i < adapter->num_tx_queues; i++) { 6542 struct igc_ring *ring = adapter->tx_ring[i]; 6543 6544 if (!(e->gate_mask & BIT(i))) 6545 continue; 6546 6547 /* Check whether a queue stays open for more than one 6548 * entry. If so, keep the start and advance the end 6549 * time. 6550 */ 6551 if (!queue_configured[i]) 6552 ring->start_time = start_time; 6553 ring->end_time = end_time; 6554 6555 if (ring->start_time >= adapter->cycle_time) 6556 queue_configured[i] = false; 6557 else 6558 queue_configured[i] = true; 6559 } 6560 6561 start_time += e->interval; 6562 } 6563 6564 spin_lock_irqsave(&adapter->qbv_tx_lock, flags); 6565 6566 /* Check whether a queue gets configured. 6567 * If not, set the start and end time to be end time. 6568 */ 6569 for (i = 0; i < adapter->num_tx_queues; i++) { 6570 struct igc_ring *ring = adapter->tx_ring[i]; 6571 6572 if (!is_base_time_past(qopt->base_time, &now)) { 6573 ring->admin_gate_closed = false; 6574 } else { 6575 ring->oper_gate_closed = false; 6576 ring->admin_gate_closed = false; 6577 } 6578 6579 if (!queue_configured[i]) { 6580 if (!is_base_time_past(qopt->base_time, &now)) 6581 ring->admin_gate_closed = true; 6582 else 6583 ring->oper_gate_closed = true; 6584 6585 ring->start_time = end_time; 6586 ring->end_time = end_time; 6587 } 6588 } 6589 6590 spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags); 6591 6592 for (i = 0; i < adapter->num_tx_queues; i++) { 6593 struct igc_ring *ring = adapter->tx_ring[i]; 6594 struct net_device *dev = adapter->netdev; 6595 6596 if (qopt->max_sdu[i]) 6597 ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len - ETH_TLEN; 6598 else 6599 ring->max_sdu = 0; 6600 } 6601 6602 igc_fpe_save_preempt_queue(adapter, &qopt->mqprio); 6603 6604 return 0; 6605 } 6606 6607 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, 6608 struct tc_taprio_qopt_offload *qopt) 6609 { 6610 struct igc_hw *hw = &adapter->hw; 6611 int err; 6612 6613 if (hw->mac.type != igc_i225) 6614 return -EOPNOTSUPP; 6615 6616 switch (qopt->cmd) { 6617 case TAPRIO_CMD_REPLACE: 6618 err = igc_save_qbv_schedule(adapter, qopt); 6619 break; 6620 case TAPRIO_CMD_DESTROY: 6621 err = igc_tsn_clear_schedule(adapter); 6622 break; 6623 case TAPRIO_CMD_STATS: 6624 igc_taprio_stats(adapter->netdev, &qopt->stats); 6625 return 0; 6626 case TAPRIO_CMD_QUEUE_STATS: 6627 igc_taprio_queue_stats(adapter->netdev, &qopt->queue_stats); 6628 return 0; 6629 default: 6630 return -EOPNOTSUPP; 6631 } 6632 6633 if (err) 6634 return err; 6635 6636 return igc_tsn_offload_apply(adapter); 6637 } 6638 6639 static int igc_save_cbs_params(struct igc_adapter *adapter, int queue, 6640 bool enable, int idleslope, int sendslope, 6641 int hicredit, int locredit) 6642 { 6643 bool cbs_status[IGC_MAX_SR_QUEUES] = { false }; 6644 struct net_device *netdev = adapter->netdev; 6645 struct igc_ring *ring; 6646 int i; 6647 6648 /* i225 has two sets of credit-based shaper logic. 6649 * Supporting it only on the top two priority queues 6650 */ 6651 if (queue < 0 || queue > 1) 6652 return -EINVAL; 6653 6654 ring = adapter->tx_ring[queue]; 6655 6656 for (i = 0; i < IGC_MAX_SR_QUEUES; i++) 6657 if (adapter->tx_ring[i]) 6658 cbs_status[i] = adapter->tx_ring[i]->cbs_enable; 6659 6660 /* CBS should be enabled on the highest priority queue first in order 6661 * for the CBS algorithm to operate as intended. 6662 */ 6663 if (enable) { 6664 if (queue == 1 && !cbs_status[0]) { 6665 netdev_err(netdev, 6666 "Enabling CBS on queue1 before queue0\n"); 6667 return -EINVAL; 6668 } 6669 } else { 6670 if (queue == 0 && cbs_status[1]) { 6671 netdev_err(netdev, 6672 "Disabling CBS on queue0 before queue1\n"); 6673 return -EINVAL; 6674 } 6675 } 6676 6677 ring->cbs_enable = enable; 6678 ring->idleslope = idleslope; 6679 ring->sendslope = sendslope; 6680 ring->hicredit = hicredit; 6681 ring->locredit = locredit; 6682 6683 return 0; 6684 } 6685 6686 static int igc_tsn_enable_cbs(struct igc_adapter *adapter, 6687 struct tc_cbs_qopt_offload *qopt) 6688 { 6689 struct igc_hw *hw = &adapter->hw; 6690 int err; 6691 6692 if (hw->mac.type != igc_i225) 6693 return -EOPNOTSUPP; 6694 6695 if (qopt->queue < 0 || qopt->queue > 1) 6696 return -EINVAL; 6697 6698 err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable, 6699 qopt->idleslope, qopt->sendslope, 6700 qopt->hicredit, qopt->locredit); 6701 if (err) 6702 return err; 6703 6704 return igc_tsn_offload_apply(adapter); 6705 } 6706 6707 static int igc_tc_query_caps(struct igc_adapter *adapter, 6708 struct tc_query_caps_base *base) 6709 { 6710 struct igc_hw *hw = &adapter->hw; 6711 6712 switch (base->type) { 6713 case TC_SETUP_QDISC_MQPRIO: { 6714 struct tc_mqprio_caps *caps = base->caps; 6715 6716 caps->validate_queue_counts = true; 6717 6718 return 0; 6719 } 6720 case TC_SETUP_QDISC_TAPRIO: { 6721 struct tc_taprio_caps *caps = base->caps; 6722 6723 if (!(adapter->flags & IGC_FLAG_TSN_REVERSE_TXQ_PRIO)) 6724 caps->broken_mqprio = true; 6725 6726 if (hw->mac.type == igc_i225) { 6727 caps->supports_queue_max_sdu = true; 6728 caps->gate_mask_per_txq = true; 6729 } 6730 6731 return 0; 6732 } 6733 default: 6734 return -EOPNOTSUPP; 6735 } 6736 } 6737 6738 static void igc_save_mqprio_params(struct igc_adapter *adapter, u8 num_tc, 6739 u16 *offset) 6740 { 6741 int i; 6742 6743 adapter->strict_priority_enable = true; 6744 adapter->num_tc = num_tc; 6745 6746 for (i = 0; i < num_tc; i++) 6747 adapter->queue_per_tc[i] = offset[i]; 6748 } 6749 6750 static bool 6751 igc_tsn_is_tc_to_queue_priority_ordered(struct tc_mqprio_qopt_offload *mqprio) 6752 { 6753 int num_tc = mqprio->qopt.num_tc; 6754 int i; 6755 6756 for (i = 1; i < num_tc; i++) { 6757 if (mqprio->qopt.offset[i - 1] > mqprio->qopt.offset[i]) 6758 return false; 6759 } 6760 6761 return true; 6762 } 6763 6764 static int igc_tsn_enable_mqprio(struct igc_adapter *adapter, 6765 struct tc_mqprio_qopt_offload *mqprio) 6766 { 6767 struct igc_hw *hw = &adapter->hw; 6768 int err, i; 6769 6770 if (hw->mac.type != igc_i225) 6771 return -EOPNOTSUPP; 6772 6773 if (!mqprio->qopt.num_tc) { 6774 adapter->strict_priority_enable = false; 6775 igc_fpe_clear_preempt_queue(adapter); 6776 netdev_reset_tc(adapter->netdev); 6777 goto apply; 6778 } 6779 6780 /* There are as many TCs as Tx queues. */ 6781 if (mqprio->qopt.num_tc != adapter->num_tx_queues) { 6782 NL_SET_ERR_MSG_FMT_MOD(mqprio->extack, 6783 "Only %d traffic classes supported", 6784 adapter->num_tx_queues); 6785 return -EOPNOTSUPP; 6786 } 6787 6788 /* Only one queue per TC is supported. */ 6789 for (i = 0; i < mqprio->qopt.num_tc; i++) { 6790 if (mqprio->qopt.count[i] != 1) { 6791 NL_SET_ERR_MSG_MOD(mqprio->extack, 6792 "Only one queue per TC supported"); 6793 return -EOPNOTSUPP; 6794 } 6795 } 6796 6797 if (!igc_tsn_is_tc_to_queue_priority_ordered(mqprio)) { 6798 NL_SET_ERR_MSG_MOD(mqprio->extack, 6799 "tc to queue mapping must preserve increasing priority (higher tc -> higher queue)"); 6800 return -EOPNOTSUPP; 6801 } 6802 6803 igc_save_mqprio_params(adapter, mqprio->qopt.num_tc, 6804 mqprio->qopt.offset); 6805 6806 err = netdev_set_num_tc(adapter->netdev, adapter->num_tc); 6807 if (err) 6808 return err; 6809 6810 for (i = 0; i < adapter->num_tc; i++) { 6811 err = netdev_set_tc_queue(adapter->netdev, i, 1, 6812 adapter->queue_per_tc[i]); 6813 if (err) 6814 return err; 6815 } 6816 6817 /* In case the card is configured with less than four queues. */ 6818 for (; i < IGC_MAX_TX_QUEUES; i++) 6819 adapter->queue_per_tc[i] = i; 6820 6821 mqprio->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS; 6822 igc_fpe_save_preempt_queue(adapter, mqprio); 6823 6824 apply: 6825 return igc_tsn_offload_apply(adapter); 6826 } 6827 6828 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, 6829 void *type_data) 6830 { 6831 struct igc_adapter *adapter = netdev_priv(dev); 6832 6833 adapter->tc_setup_type = type; 6834 6835 switch (type) { 6836 case TC_QUERY_CAPS: 6837 return igc_tc_query_caps(adapter, type_data); 6838 case TC_SETUP_QDISC_TAPRIO: 6839 return igc_tsn_enable_qbv_scheduling(adapter, type_data); 6840 6841 case TC_SETUP_QDISC_ETF: 6842 return igc_tsn_enable_launchtime(adapter, type_data); 6843 6844 case TC_SETUP_QDISC_CBS: 6845 return igc_tsn_enable_cbs(adapter, type_data); 6846 6847 case TC_SETUP_QDISC_MQPRIO: 6848 return igc_tsn_enable_mqprio(adapter, type_data); 6849 6850 default: 6851 return -EOPNOTSUPP; 6852 } 6853 } 6854 6855 static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6856 { 6857 struct igc_adapter *adapter = netdev_priv(dev); 6858 6859 switch (bpf->command) { 6860 case XDP_SETUP_PROG: 6861 return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack); 6862 case XDP_SETUP_XSK_POOL: 6863 return igc_xdp_setup_pool(adapter, bpf->xsk.pool, 6864 bpf->xsk.queue_id); 6865 default: 6866 return -EOPNOTSUPP; 6867 } 6868 } 6869 6870 static int igc_xdp_xmit(struct net_device *dev, int num_frames, 6871 struct xdp_frame **frames, u32 flags) 6872 { 6873 struct igc_adapter *adapter = netdev_priv(dev); 6874 int cpu = smp_processor_id(); 6875 struct netdev_queue *nq; 6876 struct igc_ring *ring; 6877 int i, nxmit; 6878 6879 if (unlikely(!netif_carrier_ok(dev))) 6880 return -ENETDOWN; 6881 6882 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6883 return -EINVAL; 6884 6885 ring = igc_get_tx_ring(adapter, cpu); 6886 nq = txring_txq(ring); 6887 6888 __netif_tx_lock(nq, cpu); 6889 6890 /* Avoid transmit queue timeout since we share it with the slow path */ 6891 txq_trans_cond_update(nq); 6892 6893 nxmit = 0; 6894 for (i = 0; i < num_frames; i++) { 6895 int err; 6896 struct xdp_frame *xdpf = frames[i]; 6897 6898 err = igc_xdp_init_tx_descriptor(ring, xdpf); 6899 if (err) 6900 break; 6901 nxmit++; 6902 } 6903 6904 if (flags & XDP_XMIT_FLUSH) 6905 igc_flush_tx_descriptors(ring); 6906 6907 __netif_tx_unlock(nq); 6908 6909 return nxmit; 6910 } 6911 6912 static u32 igc_sw_irq_prep(struct igc_q_vector *q_vector) 6913 { 6914 u32 eics = 0; 6915 6916 if (!napi_if_scheduled_mark_missed(&q_vector->napi)) 6917 eics = q_vector->eims_value; 6918 6919 return eics; 6920 } 6921 6922 int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 6923 { 6924 struct igc_adapter *adapter = netdev_priv(dev); 6925 struct igc_hw *hw = &adapter->hw; 6926 struct igc_ring *ring; 6927 u32 eics = 0; 6928 6929 if (test_bit(__IGC_DOWN, &adapter->state)) 6930 return -ENETDOWN; 6931 6932 if (!igc_xdp_is_enabled(adapter)) 6933 return -ENXIO; 6934 /* Check if queue_id is valid. Tx and Rx queue numbers are always same */ 6935 if (queue_id >= adapter->num_rx_queues) 6936 return -EINVAL; 6937 6938 ring = adapter->rx_ring[queue_id]; 6939 6940 if (!ring->xsk_pool) 6941 return -ENXIO; 6942 6943 if (flags & XDP_WAKEUP_RX) 6944 eics |= igc_sw_irq_prep(ring->q_vector); 6945 6946 if (flags & XDP_WAKEUP_TX) { 6947 /* If IGC_FLAG_QUEUE_PAIRS is active, the q_vector 6948 * and NAPI is shared between RX and TX. 6949 * If NAPI is already running it would be marked as missed 6950 * from the RX path, making this TX call a NOP 6951 */ 6952 ring = adapter->tx_ring[queue_id]; 6953 eics |= igc_sw_irq_prep(ring->q_vector); 6954 } 6955 6956 if (eics) 6957 /* Cause software interrupt */ 6958 wr32(IGC_EICS, eics); 6959 6960 return 0; 6961 } 6962 6963 static ktime_t igc_get_tstamp(struct net_device *dev, 6964 const struct skb_shared_hwtstamps *hwtstamps, 6965 bool cycles) 6966 { 6967 struct igc_adapter *adapter = netdev_priv(dev); 6968 struct igc_inline_rx_tstamps *tstamp; 6969 ktime_t timestamp; 6970 6971 tstamp = hwtstamps->netdev_data; 6972 6973 if (cycles) 6974 timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer1); 6975 else 6976 timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0); 6977 6978 return timestamp; 6979 } 6980 6981 static const struct net_device_ops igc_netdev_ops = { 6982 .ndo_open = igc_open, 6983 .ndo_stop = igc_close, 6984 .ndo_start_xmit = igc_xmit_frame, 6985 .ndo_set_rx_mode = igc_set_rx_mode, 6986 .ndo_set_mac_address = igc_set_mac, 6987 .ndo_change_mtu = igc_change_mtu, 6988 .ndo_tx_timeout = igc_tx_timeout, 6989 .ndo_get_stats64 = igc_get_stats64, 6990 .ndo_fix_features = igc_fix_features, 6991 .ndo_set_features = igc_set_features, 6992 .ndo_features_check = igc_features_check, 6993 .ndo_setup_tc = igc_setup_tc, 6994 .ndo_bpf = igc_bpf, 6995 .ndo_xdp_xmit = igc_xdp_xmit, 6996 .ndo_xsk_wakeup = igc_xsk_wakeup, 6997 .ndo_get_tstamp = igc_get_tstamp, 6998 .ndo_hwtstamp_get = igc_ptp_hwtstamp_get, 6999 .ndo_hwtstamp_set = igc_ptp_hwtstamp_set, 7000 }; 7001 7002 u32 igc_rd32(struct igc_hw *hw, u32 reg) 7003 { 7004 struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); 7005 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 7006 u32 value = 0; 7007 7008 if (IGC_REMOVED(hw_addr)) 7009 return ~value; 7010 7011 value = readl(&hw_addr[reg]); 7012 7013 /* reads should not return all F's */ 7014 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 7015 struct net_device *netdev = igc->netdev; 7016 7017 hw->hw_addr = NULL; 7018 netif_device_detach(netdev); 7019 netdev_err(netdev, "PCIe link lost, device now detached\n"); 7020 WARN(pci_device_is_present(igc->pdev), 7021 "igc: Failed to read reg 0x%x!\n", reg); 7022 } 7023 7024 return value; 7025 } 7026 7027 /* Mapping HW RSS Type to enum xdp_rss_hash_type */ 7028 static enum xdp_rss_hash_type igc_xdp_rss_type[IGC_RSS_TYPE_MAX_TABLE] = { 7029 [IGC_RSS_TYPE_NO_HASH] = XDP_RSS_TYPE_L2, 7030 [IGC_RSS_TYPE_HASH_TCP_IPV4] = XDP_RSS_TYPE_L4_IPV4_TCP, 7031 [IGC_RSS_TYPE_HASH_IPV4] = XDP_RSS_TYPE_L3_IPV4, 7032 [IGC_RSS_TYPE_HASH_TCP_IPV6] = XDP_RSS_TYPE_L4_IPV6_TCP, 7033 [IGC_RSS_TYPE_HASH_IPV6_EX] = XDP_RSS_TYPE_L3_IPV6_EX, 7034 [IGC_RSS_TYPE_HASH_IPV6] = XDP_RSS_TYPE_L3_IPV6, 7035 [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX, 7036 [IGC_RSS_TYPE_HASH_UDP_IPV4] = XDP_RSS_TYPE_L4_IPV4_UDP, 7037 [IGC_RSS_TYPE_HASH_UDP_IPV6] = XDP_RSS_TYPE_L4_IPV6_UDP, 7038 [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX, 7039 [10] = XDP_RSS_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */ 7040 [11] = XDP_RSS_TYPE_NONE, /* keep array sized for SW bit-mask */ 7041 [12] = XDP_RSS_TYPE_NONE, /* to handle future HW revisions */ 7042 [13] = XDP_RSS_TYPE_NONE, 7043 [14] = XDP_RSS_TYPE_NONE, 7044 [15] = XDP_RSS_TYPE_NONE, 7045 }; 7046 7047 static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash, 7048 enum xdp_rss_hash_type *rss_type) 7049 { 7050 const struct igc_xdp_buff *ctx = (void *)_ctx; 7051 7052 if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH)) 7053 return -ENODATA; 7054 7055 *hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss); 7056 *rss_type = igc_xdp_rss_type[igc_rss_type(ctx->rx_desc)]; 7057 7058 return 0; 7059 } 7060 7061 static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7062 { 7063 const struct igc_xdp_buff *ctx = (void *)_ctx; 7064 struct igc_adapter *adapter = netdev_priv(ctx->xdp.rxq->dev); 7065 struct igc_inline_rx_tstamps *tstamp = ctx->rx_ts; 7066 7067 if (igc_test_staterr(ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) { 7068 *timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0); 7069 7070 return 0; 7071 } 7072 7073 return -ENODATA; 7074 } 7075 7076 static const struct xdp_metadata_ops igc_xdp_metadata_ops = { 7077 .xmo_rx_hash = igc_xdp_rx_hash, 7078 .xmo_rx_timestamp = igc_xdp_rx_timestamp, 7079 }; 7080 7081 static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer) 7082 { 7083 struct igc_adapter *adapter = container_of(timer, struct igc_adapter, 7084 hrtimer); 7085 unsigned long flags; 7086 unsigned int i; 7087 7088 spin_lock_irqsave(&adapter->qbv_tx_lock, flags); 7089 7090 adapter->qbv_transition = true; 7091 for (i = 0; i < adapter->num_tx_queues; i++) { 7092 struct igc_ring *tx_ring = adapter->tx_ring[i]; 7093 7094 if (tx_ring->admin_gate_closed) { 7095 tx_ring->admin_gate_closed = false; 7096 tx_ring->oper_gate_closed = true; 7097 } else { 7098 tx_ring->oper_gate_closed = false; 7099 } 7100 } 7101 adapter->qbv_transition = false; 7102 7103 spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags); 7104 7105 return HRTIMER_NORESTART; 7106 } 7107 7108 /** 7109 * igc_probe - Device Initialization Routine 7110 * @pdev: PCI device information struct 7111 * @ent: entry in igc_pci_tbl 7112 * 7113 * Returns 0 on success, negative on failure 7114 * 7115 * igc_probe initializes an adapter identified by a pci_dev structure. 7116 * The OS initialization, configuring the adapter private structure, 7117 * and a hardware reset occur. 7118 */ 7119 static int igc_probe(struct pci_dev *pdev, 7120 const struct pci_device_id *ent) 7121 { 7122 struct igc_adapter *adapter; 7123 struct net_device *netdev; 7124 struct igc_hw *hw; 7125 const struct igc_info *ei = igc_info_tbl[ent->driver_data]; 7126 int err; 7127 7128 err = pci_enable_device_mem(pdev); 7129 if (err) 7130 return err; 7131 7132 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 7133 if (err) { 7134 dev_err(&pdev->dev, 7135 "No usable DMA configuration, aborting\n"); 7136 goto err_dma; 7137 } 7138 7139 err = pci_request_mem_regions(pdev, igc_driver_name); 7140 if (err) 7141 goto err_pci_reg; 7142 7143 err = pci_enable_ptm(pdev); 7144 if (err < 0) 7145 dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n"); 7146 7147 pci_set_master(pdev); 7148 7149 err = -ENOMEM; 7150 netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), 7151 IGC_MAX_TX_QUEUES); 7152 7153 if (!netdev) 7154 goto err_alloc_etherdev; 7155 7156 SET_NETDEV_DEV(netdev, &pdev->dev); 7157 7158 pci_set_drvdata(pdev, netdev); 7159 adapter = netdev_priv(netdev); 7160 adapter->netdev = netdev; 7161 adapter->pdev = pdev; 7162 hw = &adapter->hw; 7163 hw->back = adapter; 7164 adapter->port_num = hw->bus.func; 7165 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 7166 7167 /* PCI config space info */ 7168 hw->vendor_id = pdev->vendor; 7169 hw->device_id = pdev->device; 7170 hw->revision_id = pdev->revision; 7171 hw->subsystem_vendor_id = pdev->subsystem_vendor; 7172 hw->subsystem_device_id = pdev->subsystem_device; 7173 7174 /* Disable ASPM L1.2 on I226 devices to avoid packet loss */ 7175 if (igc_is_device_id_i226(hw)) 7176 pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); 7177 7178 err = pci_save_state(pdev); 7179 if (err) 7180 goto err_ioremap; 7181 7182 err = -EIO; 7183 adapter->io_addr = ioremap(pci_resource_start(pdev, 0), 7184 pci_resource_len(pdev, 0)); 7185 if (!adapter->io_addr) 7186 goto err_ioremap; 7187 7188 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ 7189 hw->hw_addr = adapter->io_addr; 7190 7191 netdev->netdev_ops = &igc_netdev_ops; 7192 netdev->xdp_metadata_ops = &igc_xdp_metadata_ops; 7193 netdev->xsk_tx_metadata_ops = &igc_xsk_tx_metadata_ops; 7194 igc_ethtool_set_ops(netdev); 7195 netdev->watchdog_timeo = 5 * HZ; 7196 7197 netdev->mem_start = pci_resource_start(pdev, 0); 7198 netdev->mem_end = pci_resource_end(pdev, 0); 7199 7200 /* Copy the default MAC and PHY function pointers */ 7201 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); 7202 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); 7203 7204 /* Initialize skew-specific constants */ 7205 err = ei->get_invariants(hw); 7206 if (err) 7207 goto err_sw_init; 7208 7209 /* Add supported features to the features list*/ 7210 netdev->features |= NETIF_F_SG; 7211 netdev->features |= NETIF_F_TSO; 7212 netdev->features |= NETIF_F_TSO6; 7213 netdev->features |= NETIF_F_TSO_ECN; 7214 netdev->features |= NETIF_F_RXHASH; 7215 netdev->features |= NETIF_F_RXCSUM; 7216 netdev->features |= NETIF_F_HW_CSUM; 7217 netdev->features |= NETIF_F_SCTP_CRC; 7218 netdev->features |= NETIF_F_HW_TC; 7219 7220 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 7221 NETIF_F_GSO_GRE_CSUM | \ 7222 NETIF_F_GSO_IPXIP4 | \ 7223 NETIF_F_GSO_IPXIP6 | \ 7224 NETIF_F_GSO_UDP_TUNNEL | \ 7225 NETIF_F_GSO_UDP_TUNNEL_CSUM) 7226 7227 netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; 7228 netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; 7229 7230 /* setup the private structure */ 7231 err = igc_sw_init(adapter); 7232 if (err) 7233 goto err_sw_init; 7234 7235 /* copy netdev features into list of user selectable features */ 7236 netdev->hw_features |= NETIF_F_NTUPLE; 7237 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 7238 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7239 netdev->hw_features |= netdev->features; 7240 7241 netdev->features |= NETIF_F_HIGHDMA; 7242 7243 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 7244 netdev->mpls_features |= NETIF_F_HW_CSUM; 7245 netdev->hw_enc_features |= netdev->vlan_features; 7246 7247 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7248 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7249 7250 /* enable HW vlan tag insertion/stripping by default */ 7251 netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 7252 7253 /* MTU range: 68 - 9216 */ 7254 netdev->min_mtu = ETH_MIN_MTU; 7255 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; 7256 7257 /* before reading the NVM, reset the controller to put the device in a 7258 * known good starting state 7259 */ 7260 hw->mac.ops.reset_hw(hw); 7261 7262 if (igc_get_flash_presence_i225(hw)) { 7263 if (hw->nvm.ops.validate(hw) < 0) { 7264 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n"); 7265 err = -EIO; 7266 goto err_eeprom; 7267 } 7268 } 7269 7270 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) { 7271 /* copy the MAC address out of the NVM */ 7272 if (hw->mac.ops.read_mac_addr(hw)) 7273 dev_err(&pdev->dev, "NVM Read Error\n"); 7274 } 7275 7276 eth_hw_addr_set(netdev, hw->mac.addr); 7277 7278 if (!is_valid_ether_addr(netdev->dev_addr)) { 7279 dev_err(&pdev->dev, "Invalid MAC Address\n"); 7280 err = -EIO; 7281 goto err_eeprom; 7282 } 7283 7284 /* configure RXPBSIZE and TXPBSIZE */ 7285 wr32(IGC_RXPBS, IGC_RXPBSIZE_EXP_BMC_DEFAULT); 7286 wr32(IGC_TXPBS, IGC_TXPBSIZE_DEFAULT); 7287 7288 timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); 7289 timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); 7290 7291 INIT_WORK(&adapter->reset_task, igc_reset_task); 7292 INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); 7293 7294 hrtimer_setup(&adapter->hrtimer, &igc_qbv_scheduling_timer, CLOCK_MONOTONIC, 7295 HRTIMER_MODE_REL); 7296 7297 /* Initialize link properties that are user-changeable */ 7298 adapter->fc_autoneg = true; 7299 hw->phy.autoneg_advertised = 0xaf; 7300 hw->mac.autoneg_enabled = true; 7301 hw->fc.requested_mode = igc_fc_default; 7302 hw->fc.current_mode = igc_fc_default; 7303 7304 /* By default, support wake on port A */ 7305 adapter->flags |= IGC_FLAG_WOL_SUPPORTED; 7306 7307 /* initialize the wol settings based on the eeprom settings */ 7308 if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) 7309 adapter->wol |= IGC_WUFC_MAG; 7310 7311 device_set_wakeup_enable(&adapter->pdev->dev, 7312 adapter->flags & IGC_FLAG_WOL_SUPPORTED); 7313 7314 igc_ptp_init(adapter); 7315 7316 igc_tsn_clear_schedule(adapter); 7317 7318 igc_fpe_init(adapter); 7319 7320 /* reset the hardware with the new settings */ 7321 igc_reset(adapter); 7322 7323 /* let the f/w know that the h/w is now under the control of the 7324 * driver. 7325 */ 7326 igc_get_hw_control(adapter); 7327 7328 strscpy(netdev->name, "eth%d", sizeof(netdev->name)); 7329 err = register_netdev(netdev); 7330 if (err) 7331 goto err_register; 7332 7333 /* carrier off reporting is important to ethtool even BEFORE open */ 7334 netif_carrier_off(netdev); 7335 7336 /* Check if Media Autosense is enabled */ 7337 adapter->ei = *ei; 7338 7339 /* print pcie link status and MAC address */ 7340 pcie_print_link_status(pdev); 7341 netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr); 7342 7343 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); 7344 /* Disable EEE for internal PHY devices */ 7345 hw->dev_spec._base.eee_enable = false; 7346 adapter->flags &= ~IGC_FLAG_EEE; 7347 igc_set_eee_i225(hw, false, false, false); 7348 7349 pm_runtime_put_noidle(&pdev->dev); 7350 7351 if (IS_ENABLED(CONFIG_IGC_LEDS)) { 7352 err = igc_led_setup(adapter); 7353 if (err) { 7354 netdev_warn_once(netdev, 7355 "LED init failed (%d); continuing without LED support\n", 7356 err); 7357 adapter->leds_available = false; 7358 } else { 7359 adapter->leds_available = true; 7360 } 7361 } 7362 7363 return 0; 7364 7365 err_register: 7366 igc_release_hw_control(adapter); 7367 igc_ptp_stop(adapter); 7368 err_eeprom: 7369 if (!igc_check_reset_block(hw)) 7370 igc_reset_phy(hw); 7371 err_sw_init: 7372 igc_clear_interrupt_scheme(adapter); 7373 iounmap(adapter->io_addr); 7374 err_ioremap: 7375 free_netdev(netdev); 7376 err_alloc_etherdev: 7377 pci_release_mem_regions(pdev); 7378 err_pci_reg: 7379 err_dma: 7380 pci_disable_device(pdev); 7381 return err; 7382 } 7383 7384 /** 7385 * igc_remove - Device Removal Routine 7386 * @pdev: PCI device information struct 7387 * 7388 * igc_remove is called by the PCI subsystem to alert the driver 7389 * that it should release a PCI device. This could be caused by a 7390 * Hot-Plug event, or because the driver is going to be removed from 7391 * memory. 7392 */ 7393 static void igc_remove(struct pci_dev *pdev) 7394 { 7395 struct net_device *netdev = pci_get_drvdata(pdev); 7396 struct igc_adapter *adapter = netdev_priv(netdev); 7397 7398 pm_runtime_get_noresume(&pdev->dev); 7399 7400 igc_flush_nfc_rules(adapter); 7401 7402 igc_ptp_stop(adapter); 7403 7404 pci_disable_ptm(pdev); 7405 pci_clear_master(pdev); 7406 7407 set_bit(__IGC_DOWN, &adapter->state); 7408 7409 timer_delete_sync(&adapter->watchdog_timer); 7410 timer_delete_sync(&adapter->phy_info_timer); 7411 7412 cancel_work_sync(&adapter->reset_task); 7413 cancel_work_sync(&adapter->watchdog_task); 7414 hrtimer_cancel(&adapter->hrtimer); 7415 7416 if (IS_ENABLED(CONFIG_IGC_LEDS) && adapter->leds_available) 7417 igc_led_free(adapter); 7418 7419 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7420 * would have already happened in close and is redundant. 7421 */ 7422 igc_release_hw_control(adapter); 7423 unregister_netdev(netdev); 7424 7425 igc_clear_interrupt_scheme(adapter); 7426 pci_iounmap(pdev, adapter->io_addr); 7427 pci_release_mem_regions(pdev); 7428 7429 free_netdev(netdev); 7430 7431 pci_disable_device(pdev); 7432 } 7433 7434 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, 7435 bool runtime) 7436 { 7437 struct net_device *netdev = pci_get_drvdata(pdev); 7438 struct igc_adapter *adapter = netdev_priv(netdev); 7439 u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; 7440 struct igc_hw *hw = &adapter->hw; 7441 u32 ctrl, rctl, status; 7442 bool wake; 7443 7444 rtnl_lock(); 7445 netif_device_detach(netdev); 7446 7447 if (netif_running(netdev)) 7448 __igc_close(netdev, true); 7449 7450 igc_ptp_suspend(adapter); 7451 7452 igc_clear_interrupt_scheme(adapter); 7453 rtnl_unlock(); 7454 7455 status = rd32(IGC_STATUS); 7456 if (status & IGC_STATUS_LU) 7457 wufc &= ~IGC_WUFC_LNKC; 7458 7459 if (wufc) { 7460 igc_setup_rctl(adapter); 7461 igc_set_rx_mode(netdev); 7462 7463 /* turn on all-multi mode if wake on multicast is enabled */ 7464 if (wufc & IGC_WUFC_MC) { 7465 rctl = rd32(IGC_RCTL); 7466 rctl |= IGC_RCTL_MPE; 7467 wr32(IGC_RCTL, rctl); 7468 } 7469 7470 ctrl = rd32(IGC_CTRL); 7471 ctrl |= IGC_CTRL_ADVD3WUC; 7472 wr32(IGC_CTRL, ctrl); 7473 7474 /* Allow time for pending master requests to run */ 7475 igc_disable_pcie_master(hw); 7476 7477 wr32(IGC_WUC, IGC_WUC_PME_EN); 7478 wr32(IGC_WUFC, wufc); 7479 } else { 7480 wr32(IGC_WUC, 0); 7481 wr32(IGC_WUFC, 0); 7482 } 7483 7484 wake = wufc || adapter->en_mng_pt; 7485 if (!wake) 7486 igc_power_down_phy_copper_base(&adapter->hw); 7487 else 7488 igc_power_up_link(adapter); 7489 7490 if (enable_wake) 7491 *enable_wake = wake; 7492 7493 /* Release control of h/w to f/w. If f/w is AMT enabled, this 7494 * would have already happened in close and is redundant. 7495 */ 7496 igc_release_hw_control(adapter); 7497 7498 pci_disable_device(pdev); 7499 7500 return 0; 7501 } 7502 7503 static int igc_runtime_suspend(struct device *dev) 7504 { 7505 return __igc_shutdown(to_pci_dev(dev), NULL, 1); 7506 } 7507 7508 static void igc_deliver_wake_packet(struct net_device *netdev) 7509 { 7510 struct igc_adapter *adapter = netdev_priv(netdev); 7511 struct igc_hw *hw = &adapter->hw; 7512 struct sk_buff *skb; 7513 u32 wupl; 7514 7515 wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; 7516 7517 /* WUPM stores only the first 128 bytes of the wake packet. 7518 * Read the packet only if we have the whole thing. 7519 */ 7520 if (wupl == 0 || wupl > IGC_WUPM_BYTES) 7521 return; 7522 7523 skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); 7524 if (!skb) 7525 return; 7526 7527 skb_put(skb, wupl); 7528 7529 /* Ensure reads are 32-bit aligned */ 7530 wupl = roundup(wupl, 4); 7531 7532 memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); 7533 7534 skb->protocol = eth_type_trans(skb, netdev); 7535 netif_rx(skb); 7536 } 7537 7538 static int __igc_resume(struct device *dev, bool rpm) 7539 { 7540 struct pci_dev *pdev = to_pci_dev(dev); 7541 struct net_device *netdev = pci_get_drvdata(pdev); 7542 struct igc_adapter *adapter = netdev_priv(netdev); 7543 struct igc_hw *hw = &adapter->hw; 7544 u32 err, val; 7545 7546 pci_set_power_state(pdev, PCI_D0); 7547 pci_restore_state(pdev); 7548 7549 if (!pci_device_is_present(pdev)) 7550 return -ENODEV; 7551 err = pci_enable_device_mem(pdev); 7552 if (err) { 7553 netdev_err(netdev, "Cannot enable PCI device from suspend\n"); 7554 return err; 7555 } 7556 pci_set_master(pdev); 7557 7558 pci_enable_wake(pdev, PCI_D3hot, 0); 7559 pci_enable_wake(pdev, PCI_D3cold, 0); 7560 7561 if (igc_is_device_id_i226(hw)) 7562 pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); 7563 7564 if (igc_init_interrupt_scheme(adapter, true)) { 7565 netdev_err(netdev, "Unable to allocate memory for queues\n"); 7566 return -ENOMEM; 7567 } 7568 7569 igc_reset(adapter); 7570 7571 /* let the f/w know that the h/w is now under the control of the 7572 * driver. 7573 */ 7574 igc_get_hw_control(adapter); 7575 7576 val = rd32(IGC_WUS); 7577 if (val & WAKE_PKT_WUS) 7578 igc_deliver_wake_packet(netdev); 7579 7580 wr32(IGC_WUS, ~0); 7581 7582 if (netif_running(netdev)) { 7583 if (!rpm) 7584 rtnl_lock(); 7585 err = __igc_open(netdev, true); 7586 if (!rpm) 7587 rtnl_unlock(); 7588 if (err) 7589 return err; 7590 } 7591 7592 netif_device_attach(netdev); 7593 7594 return 0; 7595 } 7596 7597 static int igc_resume(struct device *dev) 7598 { 7599 return __igc_resume(dev, false); 7600 } 7601 7602 static int igc_runtime_resume(struct device *dev) 7603 { 7604 return __igc_resume(dev, true); 7605 } 7606 7607 static int igc_suspend(struct device *dev) 7608 { 7609 return __igc_shutdown(to_pci_dev(dev), NULL, 0); 7610 } 7611 7612 static int __maybe_unused igc_runtime_idle(struct device *dev) 7613 { 7614 struct net_device *netdev = dev_get_drvdata(dev); 7615 struct igc_adapter *adapter = netdev_priv(netdev); 7616 7617 if (!igc_has_link(adapter)) 7618 pm_schedule_suspend(dev, MSEC_PER_SEC * 5); 7619 7620 return -EBUSY; 7621 } 7622 7623 static void igc_shutdown(struct pci_dev *pdev) 7624 { 7625 bool wake; 7626 7627 __igc_shutdown(pdev, &wake, 0); 7628 7629 if (system_state == SYSTEM_POWER_OFF) { 7630 pci_wake_from_d3(pdev, wake); 7631 pci_set_power_state(pdev, PCI_D3hot); 7632 } 7633 } 7634 7635 /** 7636 * igc_io_error_detected - called when PCI error is detected 7637 * @pdev: Pointer to PCI device 7638 * @state: The current PCI connection state 7639 * 7640 * This function is called after a PCI bus error affecting 7641 * this device has been detected. 7642 **/ 7643 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, 7644 pci_channel_state_t state) 7645 { 7646 struct net_device *netdev = pci_get_drvdata(pdev); 7647 struct igc_adapter *adapter = netdev_priv(netdev); 7648 7649 rtnl_lock(); 7650 netif_device_detach(netdev); 7651 7652 if (state == pci_channel_io_perm_failure) { 7653 rtnl_unlock(); 7654 return PCI_ERS_RESULT_DISCONNECT; 7655 } 7656 7657 if (netif_running(netdev)) 7658 igc_down(adapter); 7659 pci_disable_device(pdev); 7660 rtnl_unlock(); 7661 7662 /* Request a slot reset. */ 7663 return PCI_ERS_RESULT_NEED_RESET; 7664 } 7665 7666 /** 7667 * igc_io_slot_reset - called after the PCI bus has been reset. 7668 * @pdev: Pointer to PCI device 7669 * 7670 * Restart the card from scratch, as if from a cold-boot. Implementation 7671 * resembles the first-half of the __igc_resume routine. 7672 **/ 7673 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) 7674 { 7675 struct net_device *netdev = pci_get_drvdata(pdev); 7676 struct igc_adapter *adapter = netdev_priv(netdev); 7677 struct igc_hw *hw = &adapter->hw; 7678 pci_ers_result_t result; 7679 7680 if (pci_enable_device_mem(pdev)) { 7681 netdev_err(netdev, "Could not re-enable PCI device after reset\n"); 7682 result = PCI_ERS_RESULT_DISCONNECT; 7683 } else { 7684 pci_set_master(pdev); 7685 pci_restore_state(pdev); 7686 7687 pci_enable_wake(pdev, PCI_D3hot, 0); 7688 pci_enable_wake(pdev, PCI_D3cold, 0); 7689 7690 if (igc_is_device_id_i226(hw)) 7691 pci_disable_link_state_locked(pdev, PCIE_LINK_STATE_L1_2); 7692 7693 /* In case of PCI error, adapter loses its HW address 7694 * so we should re-assign it here. 7695 */ 7696 hw->hw_addr = adapter->io_addr; 7697 7698 igc_reset(adapter); 7699 wr32(IGC_WUS, ~0); 7700 result = PCI_ERS_RESULT_RECOVERED; 7701 } 7702 7703 return result; 7704 } 7705 7706 /** 7707 * igc_io_resume - called when traffic can start to flow again. 7708 * @pdev: Pointer to PCI device 7709 * 7710 * This callback is called when the error recovery driver tells us that 7711 * its OK to resume normal operation. Implementation resembles the 7712 * second-half of the __igc_resume routine. 7713 */ 7714 static void igc_io_resume(struct pci_dev *pdev) 7715 { 7716 struct net_device *netdev = pci_get_drvdata(pdev); 7717 struct igc_adapter *adapter = netdev_priv(netdev); 7718 7719 rtnl_lock(); 7720 if (netif_running(netdev)) { 7721 if (igc_open(netdev)) { 7722 rtnl_unlock(); 7723 netdev_err(netdev, "igc_open failed after reset\n"); 7724 return; 7725 } 7726 } 7727 7728 netif_device_attach(netdev); 7729 7730 /* let the f/w know that the h/w is now under the control of the 7731 * driver. 7732 */ 7733 igc_get_hw_control(adapter); 7734 rtnl_unlock(); 7735 } 7736 7737 static const struct pci_error_handlers igc_err_handler = { 7738 .error_detected = igc_io_error_detected, 7739 .slot_reset = igc_io_slot_reset, 7740 .resume = igc_io_resume, 7741 }; 7742 7743 static _DEFINE_DEV_PM_OPS(igc_pm_ops, igc_suspend, igc_resume, 7744 igc_runtime_suspend, igc_runtime_resume, 7745 igc_runtime_idle); 7746 7747 static struct pci_driver igc_driver = { 7748 .name = igc_driver_name, 7749 .id_table = igc_pci_tbl, 7750 .probe = igc_probe, 7751 .remove = igc_remove, 7752 .driver.pm = pm_ptr(&igc_pm_ops), 7753 .shutdown = igc_shutdown, 7754 .err_handler = &igc_err_handler, 7755 }; 7756 7757 /** 7758 * igc_reinit_queues - return error 7759 * @adapter: pointer to adapter structure 7760 */ 7761 int igc_reinit_queues(struct igc_adapter *adapter) 7762 { 7763 struct net_device *netdev = adapter->netdev; 7764 int err = 0; 7765 7766 if (netif_running(netdev)) 7767 igc_close(netdev); 7768 7769 igc_reset_interrupt_capability(adapter); 7770 7771 if (igc_init_interrupt_scheme(adapter, true)) { 7772 netdev_err(netdev, "Unable to allocate memory for queues\n"); 7773 return -ENOMEM; 7774 } 7775 7776 if (netif_running(netdev)) 7777 err = igc_open(netdev); 7778 7779 if (!err) { 7780 /* Restore default IEEE 802.1Qbv schedule after queue reinit */ 7781 igc_tsn_clear_schedule(adapter); 7782 } 7783 7784 return err; 7785 } 7786 7787 /** 7788 * igc_get_hw_dev - return device 7789 * @hw: pointer to hardware structure 7790 * 7791 * used by hardware layer to print debugging information 7792 */ 7793 struct net_device *igc_get_hw_dev(struct igc_hw *hw) 7794 { 7795 struct igc_adapter *adapter = hw->back; 7796 7797 return adapter->netdev; 7798 } 7799 7800 static void igc_disable_rx_ring_hw(struct igc_ring *ring) 7801 { 7802 struct igc_hw *hw = &ring->q_vector->adapter->hw; 7803 u8 idx = ring->reg_idx; 7804 u32 rxdctl; 7805 7806 rxdctl = rd32(IGC_RXDCTL(idx)); 7807 rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE; 7808 rxdctl |= IGC_RXDCTL_SWFLUSH; 7809 wr32(IGC_RXDCTL(idx), rxdctl); 7810 } 7811 7812 void igc_disable_rx_ring(struct igc_ring *ring) 7813 { 7814 igc_disable_rx_ring_hw(ring); 7815 igc_clean_rx_ring(ring); 7816 } 7817 7818 void igc_enable_rx_ring(struct igc_ring *ring) 7819 { 7820 struct igc_adapter *adapter = ring->q_vector->adapter; 7821 7822 igc_configure_rx_ring(adapter, ring); 7823 7824 if (ring->xsk_pool) 7825 igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); 7826 else 7827 igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); 7828 } 7829 7830 void igc_disable_tx_ring(struct igc_ring *ring) 7831 { 7832 igc_disable_tx_ring_hw(ring); 7833 igc_clean_tx_ring(ring); 7834 } 7835 7836 void igc_enable_tx_ring(struct igc_ring *ring) 7837 { 7838 struct igc_adapter *adapter = ring->q_vector->adapter; 7839 7840 igc_configure_tx_ring(adapter, ring); 7841 } 7842 7843 /** 7844 * igc_init_module - Driver Registration Routine 7845 * 7846 * igc_init_module is the first routine called when the driver is 7847 * loaded. All it does is register with the PCI subsystem. 7848 */ 7849 static int __init igc_init_module(void) 7850 { 7851 int ret; 7852 7853 pr_info("%s\n", igc_driver_string); 7854 pr_info("%s\n", igc_copyright); 7855 7856 ret = pci_register_driver(&igc_driver); 7857 return ret; 7858 } 7859 7860 module_init(igc_init_module); 7861 7862 /** 7863 * igc_exit_module - Driver Exit Cleanup Routine 7864 * 7865 * igc_exit_module is called just before the driver is removed 7866 * from memory. 7867 */ 7868 static void __exit igc_exit_module(void) 7869 { 7870 pci_unregister_driver(&igc_driver); 7871 } 7872 7873 module_exit(igc_exit_module); 7874 /* igc_main.c */ 7875