1 /* Intel(R) Ethernet Switch Host Interface Driver 2 * Copyright(c) 2013 - 2016 Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * The full GNU General Public License is included in this distribution in 14 * the file called "COPYING". 15 * 16 * Contact Information: 17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 19 */ 20 21 #include <linux/module.h> 22 #include <linux/aer.h> 23 24 #include "fm10k.h" 25 26 static const struct fm10k_info *fm10k_info_tbl[] = { 27 [fm10k_device_pf] = &fm10k_pf_info, 28 [fm10k_device_vf] = &fm10k_vf_info, 29 }; 30 31 /** 32 * fm10k_pci_tbl - PCI Device ID Table 33 * 34 * Wildcard entries (PCI_ANY_ID) should come last 35 * Last entry must be all 0s 36 * 37 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 38 * Class, Class Mask, private data (not used) } 39 */ 40 static const struct pci_device_id fm10k_pci_tbl[] = { 41 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf }, 42 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf }, 43 /* required last entry */ 44 { 0, } 45 }; 46 MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl); 47 48 u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg) 49 { 50 struct fm10k_intfc *interface = hw->back; 51 u16 value = 0; 52 53 if (FM10K_REMOVED(hw->hw_addr)) 54 return ~value; 55 56 pci_read_config_word(interface->pdev, reg, &value); 57 if (value == 0xFFFF) 58 fm10k_write_flush(hw); 59 60 return value; 61 } 62 63 u32 fm10k_read_reg(struct fm10k_hw *hw, int reg) 64 { 65 u32 __iomem *hw_addr = ACCESS_ONCE(hw->hw_addr); 66 u32 value = 0; 67 68 if (FM10K_REMOVED(hw_addr)) 69 return ~value; 70 71 value = readl(&hw_addr[reg]); 72 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 73 struct fm10k_intfc *interface = hw->back; 74 struct net_device *netdev = interface->netdev; 75 76 hw->hw_addr = NULL; 77 netif_device_detach(netdev); 78 netdev_err(netdev, "PCIe link lost, device now detached\n"); 79 } 80 81 return value; 82 } 83 84 static int fm10k_hw_ready(struct fm10k_intfc *interface) 85 { 86 struct fm10k_hw *hw = &interface->hw; 87 88 fm10k_write_flush(hw); 89 90 return FM10K_REMOVED(hw->hw_addr) ? -ENODEV : 0; 91 } 92 93 void fm10k_service_event_schedule(struct fm10k_intfc *interface) 94 { 95 if (!test_bit(__FM10K_SERVICE_DISABLE, &interface->state) && 96 !test_and_set_bit(__FM10K_SERVICE_SCHED, &interface->state)) 97 queue_work(fm10k_workqueue, &interface->service_task); 98 } 99 100 static void fm10k_service_event_complete(struct fm10k_intfc *interface) 101 { 102 WARN_ON(!test_bit(__FM10K_SERVICE_SCHED, &interface->state)); 103 104 /* flush memory to make sure state is correct before next watchog */ 105 smp_mb__before_atomic(); 106 clear_bit(__FM10K_SERVICE_SCHED, &interface->state); 107 } 108 109 /** 110 * fm10k_service_timer - Timer Call-back 111 * @data: pointer to interface cast into an unsigned long 112 **/ 113 static void fm10k_service_timer(unsigned long data) 114 { 115 struct fm10k_intfc *interface = (struct fm10k_intfc *)data; 116 117 /* Reset the timer */ 118 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 119 120 fm10k_service_event_schedule(interface); 121 } 122 123 static void fm10k_detach_subtask(struct fm10k_intfc *interface) 124 { 125 struct net_device *netdev = interface->netdev; 126 u32 __iomem *hw_addr; 127 u32 value; 128 129 /* do nothing if device is still present or hw_addr is set */ 130 if (netif_device_present(netdev) || interface->hw.hw_addr) 131 return; 132 133 /* check the real address space to see if we've recovered */ 134 hw_addr = READ_ONCE(interface->uc_addr); 135 value = readl(hw_addr); 136 if ((~value)) { 137 interface->hw.hw_addr = interface->uc_addr; 138 netif_device_attach(netdev); 139 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 140 netdev_warn(netdev, "PCIe link restored, device now attached\n"); 141 return; 142 } 143 144 rtnl_lock(); 145 146 if (netif_running(netdev)) 147 dev_close(netdev); 148 149 rtnl_unlock(); 150 } 151 152 static void fm10k_prepare_for_reset(struct fm10k_intfc *interface) 153 { 154 struct net_device *netdev = interface->netdev; 155 156 WARN_ON(in_interrupt()); 157 158 /* put off any impending NetWatchDogTimeout */ 159 netif_trans_update(netdev); 160 161 while (test_and_set_bit(__FM10K_RESETTING, &interface->state)) 162 usleep_range(1000, 2000); 163 164 rtnl_lock(); 165 166 fm10k_iov_suspend(interface->pdev); 167 168 if (netif_running(netdev)) 169 fm10k_close(netdev); 170 171 fm10k_mbx_free_irq(interface); 172 173 /* free interrupts */ 174 fm10k_clear_queueing_scheme(interface); 175 176 /* delay any future reset requests */ 177 interface->last_reset = jiffies + (10 * HZ); 178 179 rtnl_unlock(); 180 } 181 182 static int fm10k_handle_reset(struct fm10k_intfc *interface) 183 { 184 struct net_device *netdev = interface->netdev; 185 struct fm10k_hw *hw = &interface->hw; 186 int err; 187 188 rtnl_lock(); 189 190 pci_set_master(interface->pdev); 191 192 /* reset and initialize the hardware so it is in a known state */ 193 err = hw->mac.ops.reset_hw(hw); 194 if (err) { 195 dev_err(&interface->pdev->dev, "reset_hw failed: %d\n", err); 196 goto reinit_err; 197 } 198 199 err = hw->mac.ops.init_hw(hw); 200 if (err) { 201 dev_err(&interface->pdev->dev, "init_hw failed: %d\n", err); 202 goto reinit_err; 203 } 204 205 err = fm10k_init_queueing_scheme(interface); 206 if (err) { 207 dev_err(&interface->pdev->dev, 208 "init_queueing_scheme failed: %d\n", err); 209 goto reinit_err; 210 } 211 212 /* re-associate interrupts */ 213 err = fm10k_mbx_request_irq(interface); 214 if (err) 215 goto err_mbx_irq; 216 217 err = fm10k_hw_ready(interface); 218 if (err) 219 goto err_open; 220 221 /* update hardware address for VFs if perm_addr has changed */ 222 if (hw->mac.type == fm10k_mac_vf) { 223 if (is_valid_ether_addr(hw->mac.perm_addr)) { 224 ether_addr_copy(hw->mac.addr, hw->mac.perm_addr); 225 ether_addr_copy(netdev->perm_addr, hw->mac.perm_addr); 226 ether_addr_copy(netdev->dev_addr, hw->mac.perm_addr); 227 netdev->addr_assign_type &= ~NET_ADDR_RANDOM; 228 } 229 230 if (hw->mac.vlan_override) 231 netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 232 else 233 netdev->features |= NETIF_F_HW_VLAN_CTAG_RX; 234 } 235 236 err = netif_running(netdev) ? fm10k_open(netdev) : 0; 237 if (err) 238 goto err_open; 239 240 fm10k_iov_resume(interface->pdev); 241 242 rtnl_unlock(); 243 244 clear_bit(__FM10K_RESETTING, &interface->state); 245 246 return err; 247 err_open: 248 fm10k_mbx_free_irq(interface); 249 err_mbx_irq: 250 fm10k_clear_queueing_scheme(interface); 251 reinit_err: 252 netif_device_detach(netdev); 253 254 rtnl_unlock(); 255 256 clear_bit(__FM10K_RESETTING, &interface->state); 257 258 return err; 259 } 260 261 static void fm10k_reinit(struct fm10k_intfc *interface) 262 { 263 int err; 264 265 fm10k_prepare_for_reset(interface); 266 267 err = fm10k_handle_reset(interface); 268 if (err) 269 dev_err(&interface->pdev->dev, 270 "fm10k_handle_reset failed: %d\n", err); 271 } 272 273 static void fm10k_reset_subtask(struct fm10k_intfc *interface) 274 { 275 if (!(interface->flags & FM10K_FLAG_RESET_REQUESTED)) 276 return; 277 278 interface->flags &= ~FM10K_FLAG_RESET_REQUESTED; 279 280 netdev_err(interface->netdev, "Reset interface\n"); 281 282 fm10k_reinit(interface); 283 } 284 285 /** 286 * fm10k_configure_swpri_map - Configure Receive SWPRI to PC mapping 287 * @interface: board private structure 288 * 289 * Configure the SWPRI to PC mapping for the port. 290 **/ 291 static void fm10k_configure_swpri_map(struct fm10k_intfc *interface) 292 { 293 struct net_device *netdev = interface->netdev; 294 struct fm10k_hw *hw = &interface->hw; 295 int i; 296 297 /* clear flag indicating update is needed */ 298 interface->flags &= ~FM10K_FLAG_SWPRI_CONFIG; 299 300 /* these registers are only available on the PF */ 301 if (hw->mac.type != fm10k_mac_pf) 302 return; 303 304 /* configure SWPRI to PC map */ 305 for (i = 0; i < FM10K_SWPRI_MAX; i++) 306 fm10k_write_reg(hw, FM10K_SWPRI_MAP(i), 307 netdev_get_prio_tc_map(netdev, i)); 308 } 309 310 /** 311 * fm10k_watchdog_update_host_state - Update the link status based on host. 312 * @interface: board private structure 313 **/ 314 static void fm10k_watchdog_update_host_state(struct fm10k_intfc *interface) 315 { 316 struct fm10k_hw *hw = &interface->hw; 317 s32 err; 318 319 if (test_bit(__FM10K_LINK_DOWN, &interface->state)) { 320 interface->host_ready = false; 321 if (time_is_after_jiffies(interface->link_down_event)) 322 return; 323 clear_bit(__FM10K_LINK_DOWN, &interface->state); 324 } 325 326 if (interface->flags & FM10K_FLAG_SWPRI_CONFIG) { 327 if (rtnl_trylock()) { 328 fm10k_configure_swpri_map(interface); 329 rtnl_unlock(); 330 } 331 } 332 333 /* lock the mailbox for transmit and receive */ 334 fm10k_mbx_lock(interface); 335 336 err = hw->mac.ops.get_host_state(hw, &interface->host_ready); 337 if (err && time_is_before_jiffies(interface->last_reset)) 338 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 339 340 /* free the lock */ 341 fm10k_mbx_unlock(interface); 342 } 343 344 /** 345 * fm10k_mbx_subtask - Process upstream and downstream mailboxes 346 * @interface: board private structure 347 * 348 * This function will process both the upstream and downstream mailboxes. 349 **/ 350 static void fm10k_mbx_subtask(struct fm10k_intfc *interface) 351 { 352 /* process upstream mailbox and update device state */ 353 fm10k_watchdog_update_host_state(interface); 354 355 /* process downstream mailboxes */ 356 fm10k_iov_mbx(interface); 357 } 358 359 /** 360 * fm10k_watchdog_host_is_ready - Update netdev status based on host ready 361 * @interface: board private structure 362 **/ 363 static void fm10k_watchdog_host_is_ready(struct fm10k_intfc *interface) 364 { 365 struct net_device *netdev = interface->netdev; 366 367 /* only continue if link state is currently down */ 368 if (netif_carrier_ok(netdev)) 369 return; 370 371 netif_info(interface, drv, netdev, "NIC Link is up\n"); 372 373 netif_carrier_on(netdev); 374 netif_tx_wake_all_queues(netdev); 375 } 376 377 /** 378 * fm10k_watchdog_host_not_ready - Update netdev status based on host not ready 379 * @interface: board private structure 380 **/ 381 static void fm10k_watchdog_host_not_ready(struct fm10k_intfc *interface) 382 { 383 struct net_device *netdev = interface->netdev; 384 385 /* only continue if link state is currently up */ 386 if (!netif_carrier_ok(netdev)) 387 return; 388 389 netif_info(interface, drv, netdev, "NIC Link is down\n"); 390 391 netif_carrier_off(netdev); 392 netif_tx_stop_all_queues(netdev); 393 } 394 395 /** 396 * fm10k_update_stats - Update the board statistics counters. 397 * @interface: board private structure 398 **/ 399 void fm10k_update_stats(struct fm10k_intfc *interface) 400 { 401 struct net_device_stats *net_stats = &interface->netdev->stats; 402 struct fm10k_hw *hw = &interface->hw; 403 u64 hw_csum_tx_good = 0, hw_csum_rx_good = 0, rx_length_errors = 0; 404 u64 rx_switch_errors = 0, rx_drops = 0, rx_pp_errors = 0; 405 u64 rx_link_errors = 0; 406 u64 rx_errors = 0, rx_csum_errors = 0, tx_csum_errors = 0; 407 u64 restart_queue = 0, tx_busy = 0, alloc_failed = 0; 408 u64 rx_bytes_nic = 0, rx_pkts_nic = 0, rx_drops_nic = 0; 409 u64 tx_bytes_nic = 0, tx_pkts_nic = 0; 410 u64 bytes, pkts; 411 int i; 412 413 /* ensure only one thread updates stats at a time */ 414 if (test_and_set_bit(__FM10K_UPDATING_STATS, &interface->state)) 415 return; 416 417 /* do not allow stats update via service task for next second */ 418 interface->next_stats_update = jiffies + HZ; 419 420 /* gather some stats to the interface struct that are per queue */ 421 for (bytes = 0, pkts = 0, i = 0; i < interface->num_tx_queues; i++) { 422 struct fm10k_ring *tx_ring = READ_ONCE(interface->tx_ring[i]); 423 424 if (!tx_ring) 425 continue; 426 427 restart_queue += tx_ring->tx_stats.restart_queue; 428 tx_busy += tx_ring->tx_stats.tx_busy; 429 tx_csum_errors += tx_ring->tx_stats.csum_err; 430 bytes += tx_ring->stats.bytes; 431 pkts += tx_ring->stats.packets; 432 hw_csum_tx_good += tx_ring->tx_stats.csum_good; 433 } 434 435 interface->restart_queue = restart_queue; 436 interface->tx_busy = tx_busy; 437 net_stats->tx_bytes = bytes; 438 net_stats->tx_packets = pkts; 439 interface->tx_csum_errors = tx_csum_errors; 440 interface->hw_csum_tx_good = hw_csum_tx_good; 441 442 /* gather some stats to the interface struct that are per queue */ 443 for (bytes = 0, pkts = 0, i = 0; i < interface->num_rx_queues; i++) { 444 struct fm10k_ring *rx_ring = READ_ONCE(interface->rx_ring[i]); 445 446 if (!rx_ring) 447 continue; 448 449 bytes += rx_ring->stats.bytes; 450 pkts += rx_ring->stats.packets; 451 alloc_failed += rx_ring->rx_stats.alloc_failed; 452 rx_csum_errors += rx_ring->rx_stats.csum_err; 453 rx_errors += rx_ring->rx_stats.errors; 454 hw_csum_rx_good += rx_ring->rx_stats.csum_good; 455 rx_switch_errors += rx_ring->rx_stats.switch_errors; 456 rx_drops += rx_ring->rx_stats.drops; 457 rx_pp_errors += rx_ring->rx_stats.pp_errors; 458 rx_link_errors += rx_ring->rx_stats.link_errors; 459 rx_length_errors += rx_ring->rx_stats.length_errors; 460 } 461 462 net_stats->rx_bytes = bytes; 463 net_stats->rx_packets = pkts; 464 interface->alloc_failed = alloc_failed; 465 interface->rx_csum_errors = rx_csum_errors; 466 interface->hw_csum_rx_good = hw_csum_rx_good; 467 interface->rx_switch_errors = rx_switch_errors; 468 interface->rx_drops = rx_drops; 469 interface->rx_pp_errors = rx_pp_errors; 470 interface->rx_link_errors = rx_link_errors; 471 interface->rx_length_errors = rx_length_errors; 472 473 hw->mac.ops.update_hw_stats(hw, &interface->stats); 474 475 for (i = 0; i < hw->mac.max_queues; i++) { 476 struct fm10k_hw_stats_q *q = &interface->stats.q[i]; 477 478 tx_bytes_nic += q->tx_bytes.count; 479 tx_pkts_nic += q->tx_packets.count; 480 rx_bytes_nic += q->rx_bytes.count; 481 rx_pkts_nic += q->rx_packets.count; 482 rx_drops_nic += q->rx_drops.count; 483 } 484 485 interface->tx_bytes_nic = tx_bytes_nic; 486 interface->tx_packets_nic = tx_pkts_nic; 487 interface->rx_bytes_nic = rx_bytes_nic; 488 interface->rx_packets_nic = rx_pkts_nic; 489 interface->rx_drops_nic = rx_drops_nic; 490 491 /* Fill out the OS statistics structure */ 492 net_stats->rx_errors = rx_errors; 493 net_stats->rx_dropped = interface->stats.nodesc_drop.count; 494 495 clear_bit(__FM10K_UPDATING_STATS, &interface->state); 496 } 497 498 /** 499 * fm10k_watchdog_flush_tx - flush queues on host not ready 500 * @interface - pointer to the device interface structure 501 **/ 502 static void fm10k_watchdog_flush_tx(struct fm10k_intfc *interface) 503 { 504 int some_tx_pending = 0; 505 int i; 506 507 /* nothing to do if carrier is up */ 508 if (netif_carrier_ok(interface->netdev)) 509 return; 510 511 for (i = 0; i < interface->num_tx_queues; i++) { 512 struct fm10k_ring *tx_ring = interface->tx_ring[i]; 513 514 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 515 some_tx_pending = 1; 516 break; 517 } 518 } 519 520 /* We've lost link, so the controller stops DMA, but we've got 521 * queued Tx work that's never going to get done, so reset 522 * controller to flush Tx. 523 */ 524 if (some_tx_pending) 525 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 526 } 527 528 /** 529 * fm10k_watchdog_subtask - check and bring link up 530 * @interface - pointer to the device interface structure 531 **/ 532 static void fm10k_watchdog_subtask(struct fm10k_intfc *interface) 533 { 534 /* if interface is down do nothing */ 535 if (test_bit(__FM10K_DOWN, &interface->state) || 536 test_bit(__FM10K_RESETTING, &interface->state)) 537 return; 538 539 if (interface->host_ready) 540 fm10k_watchdog_host_is_ready(interface); 541 else 542 fm10k_watchdog_host_not_ready(interface); 543 544 /* update stats only once every second */ 545 if (time_is_before_jiffies(interface->next_stats_update)) 546 fm10k_update_stats(interface); 547 548 /* flush any uncompleted work */ 549 fm10k_watchdog_flush_tx(interface); 550 } 551 552 /** 553 * fm10k_check_hang_subtask - check for hung queues and dropped interrupts 554 * @interface - pointer to the device interface structure 555 * 556 * This function serves two purposes. First it strobes the interrupt lines 557 * in order to make certain interrupts are occurring. Secondly it sets the 558 * bits needed to check for TX hangs. As a result we should immediately 559 * determine if a hang has occurred. 560 */ 561 static void fm10k_check_hang_subtask(struct fm10k_intfc *interface) 562 { 563 int i; 564 565 /* If we're down or resetting, just bail */ 566 if (test_bit(__FM10K_DOWN, &interface->state) || 567 test_bit(__FM10K_RESETTING, &interface->state)) 568 return; 569 570 /* rate limit tx hang checks to only once every 2 seconds */ 571 if (time_is_after_eq_jiffies(interface->next_tx_hang_check)) 572 return; 573 interface->next_tx_hang_check = jiffies + (2 * HZ); 574 575 if (netif_carrier_ok(interface->netdev)) { 576 /* Force detection of hung controller */ 577 for (i = 0; i < interface->num_tx_queues; i++) 578 set_check_for_tx_hang(interface->tx_ring[i]); 579 580 /* Rearm all in-use q_vectors for immediate firing */ 581 for (i = 0; i < interface->num_q_vectors; i++) { 582 struct fm10k_q_vector *qv = interface->q_vector[i]; 583 584 if (!qv->tx.count && !qv->rx.count) 585 continue; 586 writel(FM10K_ITR_ENABLE | FM10K_ITR_PENDING2, qv->itr); 587 } 588 } 589 } 590 591 /** 592 * fm10k_service_task - manages and runs subtasks 593 * @work: pointer to work_struct containing our data 594 **/ 595 static void fm10k_service_task(struct work_struct *work) 596 { 597 struct fm10k_intfc *interface; 598 599 interface = container_of(work, struct fm10k_intfc, service_task); 600 601 /* tasks run even when interface is down */ 602 fm10k_mbx_subtask(interface); 603 fm10k_detach_subtask(interface); 604 fm10k_reset_subtask(interface); 605 606 /* tasks only run when interface is up */ 607 fm10k_watchdog_subtask(interface); 608 fm10k_check_hang_subtask(interface); 609 610 /* release lock on service events to allow scheduling next event */ 611 fm10k_service_event_complete(interface); 612 } 613 614 /** 615 * fm10k_configure_tx_ring - Configure Tx ring after Reset 616 * @interface: board private structure 617 * @ring: structure containing ring specific data 618 * 619 * Configure the Tx descriptor ring after a reset. 620 **/ 621 static void fm10k_configure_tx_ring(struct fm10k_intfc *interface, 622 struct fm10k_ring *ring) 623 { 624 struct fm10k_hw *hw = &interface->hw; 625 u64 tdba = ring->dma; 626 u32 size = ring->count * sizeof(struct fm10k_tx_desc); 627 u32 txint = FM10K_INT_MAP_DISABLE; 628 u32 txdctl = BIT(FM10K_TXDCTL_MAX_TIME_SHIFT) | FM10K_TXDCTL_ENABLE; 629 u8 reg_idx = ring->reg_idx; 630 631 /* disable queue to avoid issues while updating state */ 632 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), 0); 633 fm10k_write_flush(hw); 634 635 /* possible poll here to verify ring resources have been cleaned */ 636 637 /* set location and size for descriptor ring */ 638 fm10k_write_reg(hw, FM10K_TDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 639 fm10k_write_reg(hw, FM10K_TDBAH(reg_idx), tdba >> 32); 640 fm10k_write_reg(hw, FM10K_TDLEN(reg_idx), size); 641 642 /* reset head and tail pointers */ 643 fm10k_write_reg(hw, FM10K_TDH(reg_idx), 0); 644 fm10k_write_reg(hw, FM10K_TDT(reg_idx), 0); 645 646 /* store tail pointer */ 647 ring->tail = &interface->uc_addr[FM10K_TDT(reg_idx)]; 648 649 /* reset ntu and ntc to place SW in sync with hardware */ 650 ring->next_to_clean = 0; 651 ring->next_to_use = 0; 652 653 /* Map interrupt */ 654 if (ring->q_vector) { 655 txint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 656 txint |= FM10K_INT_MAP_TIMER0; 657 } 658 659 fm10k_write_reg(hw, FM10K_TXINT(reg_idx), txint); 660 661 /* enable use of FTAG bit in Tx descriptor, register is RO for VF */ 662 fm10k_write_reg(hw, FM10K_PFVTCTL(reg_idx), 663 FM10K_PFVTCTL_FTAG_DESC_ENABLE); 664 665 /* Initialize XPS */ 666 if (!test_and_set_bit(__FM10K_TX_XPS_INIT_DONE, &ring->state) && 667 ring->q_vector) 668 netif_set_xps_queue(ring->netdev, 669 &ring->q_vector->affinity_mask, 670 ring->queue_index); 671 672 /* enable queue */ 673 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), txdctl); 674 } 675 676 /** 677 * fm10k_enable_tx_ring - Verify Tx ring is enabled after configuration 678 * @interface: board private structure 679 * @ring: structure containing ring specific data 680 * 681 * Verify the Tx descriptor ring is ready for transmit. 682 **/ 683 static void fm10k_enable_tx_ring(struct fm10k_intfc *interface, 684 struct fm10k_ring *ring) 685 { 686 struct fm10k_hw *hw = &interface->hw; 687 int wait_loop = 10; 688 u32 txdctl; 689 u8 reg_idx = ring->reg_idx; 690 691 /* if we are already enabled just exit */ 692 if (fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)) & FM10K_TXDCTL_ENABLE) 693 return; 694 695 /* poll to verify queue is enabled */ 696 do { 697 usleep_range(1000, 2000); 698 txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)); 699 } while (!(txdctl & FM10K_TXDCTL_ENABLE) && --wait_loop); 700 if (!wait_loop) 701 netif_err(interface, drv, interface->netdev, 702 "Could not enable Tx Queue %d\n", reg_idx); 703 } 704 705 /** 706 * fm10k_configure_tx - Configure Transmit Unit after Reset 707 * @interface: board private structure 708 * 709 * Configure the Tx unit of the MAC after a reset. 710 **/ 711 static void fm10k_configure_tx(struct fm10k_intfc *interface) 712 { 713 int i; 714 715 /* Setup the HW Tx Head and Tail descriptor pointers */ 716 for (i = 0; i < interface->num_tx_queues; i++) 717 fm10k_configure_tx_ring(interface, interface->tx_ring[i]); 718 719 /* poll here to verify that Tx rings are now enabled */ 720 for (i = 0; i < interface->num_tx_queues; i++) 721 fm10k_enable_tx_ring(interface, interface->tx_ring[i]); 722 } 723 724 /** 725 * fm10k_configure_rx_ring - Configure Rx ring after Reset 726 * @interface: board private structure 727 * @ring: structure containing ring specific data 728 * 729 * Configure the Rx descriptor ring after a reset. 730 **/ 731 static void fm10k_configure_rx_ring(struct fm10k_intfc *interface, 732 struct fm10k_ring *ring) 733 { 734 u64 rdba = ring->dma; 735 struct fm10k_hw *hw = &interface->hw; 736 u32 size = ring->count * sizeof(union fm10k_rx_desc); 737 u32 rxqctl = FM10K_RXQCTL_ENABLE | FM10K_RXQCTL_PF; 738 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 739 u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN; 740 u32 rxint = FM10K_INT_MAP_DISABLE; 741 u8 rx_pause = interface->rx_pause; 742 u8 reg_idx = ring->reg_idx; 743 744 /* disable queue to avoid issues while updating state */ 745 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), 0); 746 fm10k_write_flush(hw); 747 748 /* possible poll here to verify ring resources have been cleaned */ 749 750 /* set location and size for descriptor ring */ 751 fm10k_write_reg(hw, FM10K_RDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 752 fm10k_write_reg(hw, FM10K_RDBAH(reg_idx), rdba >> 32); 753 fm10k_write_reg(hw, FM10K_RDLEN(reg_idx), size); 754 755 /* reset head and tail pointers */ 756 fm10k_write_reg(hw, FM10K_RDH(reg_idx), 0); 757 fm10k_write_reg(hw, FM10K_RDT(reg_idx), 0); 758 759 /* store tail pointer */ 760 ring->tail = &interface->uc_addr[FM10K_RDT(reg_idx)]; 761 762 /* reset ntu and ntc to place SW in sync with hardware */ 763 ring->next_to_clean = 0; 764 ring->next_to_use = 0; 765 ring->next_to_alloc = 0; 766 767 /* Configure the Rx buffer size for one buff without split */ 768 srrctl |= FM10K_RX_BUFSZ >> FM10K_SRRCTL_BSIZEPKT_SHIFT; 769 770 /* Configure the Rx ring to suppress loopback packets */ 771 srrctl |= FM10K_SRRCTL_LOOPBACK_SUPPRESS; 772 fm10k_write_reg(hw, FM10K_SRRCTL(reg_idx), srrctl); 773 774 /* Enable drop on empty */ 775 #ifdef CONFIG_DCB 776 if (interface->pfc_en) 777 rx_pause = interface->pfc_en; 778 #endif 779 if (!(rx_pause & BIT(ring->qos_pc))) 780 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 781 782 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 783 784 /* assign default VLAN to queue */ 785 ring->vid = hw->mac.default_vid; 786 787 /* if we have an active VLAN, disable default VLAN ID */ 788 if (test_bit(hw->mac.default_vid, interface->active_vlans)) 789 ring->vid |= FM10K_VLAN_CLEAR; 790 791 /* Map interrupt */ 792 if (ring->q_vector) { 793 rxint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 794 rxint |= FM10K_INT_MAP_TIMER1; 795 } 796 797 fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint); 798 799 /* enable queue */ 800 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl); 801 802 /* place buffers on ring for receive data */ 803 fm10k_alloc_rx_buffers(ring, fm10k_desc_unused(ring)); 804 } 805 806 /** 807 * fm10k_update_rx_drop_en - Configures the drop enable bits for Rx rings 808 * @interface: board private structure 809 * 810 * Configure the drop enable bits for the Rx rings. 811 **/ 812 void fm10k_update_rx_drop_en(struct fm10k_intfc *interface) 813 { 814 struct fm10k_hw *hw = &interface->hw; 815 u8 rx_pause = interface->rx_pause; 816 int i; 817 818 #ifdef CONFIG_DCB 819 if (interface->pfc_en) 820 rx_pause = interface->pfc_en; 821 822 #endif 823 for (i = 0; i < interface->num_rx_queues; i++) { 824 struct fm10k_ring *ring = interface->rx_ring[i]; 825 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 826 u8 reg_idx = ring->reg_idx; 827 828 if (!(rx_pause & BIT(ring->qos_pc))) 829 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 830 831 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 832 } 833 } 834 835 /** 836 * fm10k_configure_dglort - Configure Receive DGLORT after reset 837 * @interface: board private structure 838 * 839 * Configure the DGLORT description and RSS tables. 840 **/ 841 static void fm10k_configure_dglort(struct fm10k_intfc *interface) 842 { 843 struct fm10k_dglort_cfg dglort = { 0 }; 844 struct fm10k_hw *hw = &interface->hw; 845 int i; 846 u32 mrqc; 847 848 /* Fill out hash function seeds */ 849 for (i = 0; i < FM10K_RSSRK_SIZE; i++) 850 fm10k_write_reg(hw, FM10K_RSSRK(0, i), interface->rssrk[i]); 851 852 /* Write RETA table to hardware */ 853 for (i = 0; i < FM10K_RETA_SIZE; i++) 854 fm10k_write_reg(hw, FM10K_RETA(0, i), interface->reta[i]); 855 856 /* Generate RSS hash based on packet types, TCP/UDP 857 * port numbers and/or IPv4/v6 src and dst addresses 858 */ 859 mrqc = FM10K_MRQC_IPV4 | 860 FM10K_MRQC_TCP_IPV4 | 861 FM10K_MRQC_IPV6 | 862 FM10K_MRQC_TCP_IPV6; 863 864 if (interface->flags & FM10K_FLAG_RSS_FIELD_IPV4_UDP) 865 mrqc |= FM10K_MRQC_UDP_IPV4; 866 if (interface->flags & FM10K_FLAG_RSS_FIELD_IPV6_UDP) 867 mrqc |= FM10K_MRQC_UDP_IPV6; 868 869 fm10k_write_reg(hw, FM10K_MRQC(0), mrqc); 870 871 /* configure default DGLORT mapping for RSS/DCB */ 872 dglort.inner_rss = 1; 873 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 874 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 875 hw->mac.ops.configure_dglort_map(hw, &dglort); 876 877 /* assign GLORT per queue for queue mapped testing */ 878 if (interface->glort_count > 64) { 879 memset(&dglort, 0, sizeof(dglort)); 880 dglort.inner_rss = 1; 881 dglort.glort = interface->glort + 64; 882 dglort.idx = fm10k_dglort_pf_queue; 883 dglort.queue_l = fls(interface->num_rx_queues - 1); 884 hw->mac.ops.configure_dglort_map(hw, &dglort); 885 } 886 887 /* assign glort value for RSS/DCB specific to this interface */ 888 memset(&dglort, 0, sizeof(dglort)); 889 dglort.inner_rss = 1; 890 dglort.glort = interface->glort; 891 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 892 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 893 /* configure DGLORT mapping for RSS/DCB */ 894 dglort.idx = fm10k_dglort_pf_rss; 895 if (interface->l2_accel) 896 dglort.shared_l = fls(interface->l2_accel->size); 897 hw->mac.ops.configure_dglort_map(hw, &dglort); 898 } 899 900 /** 901 * fm10k_configure_rx - Configure Receive Unit after Reset 902 * @interface: board private structure 903 * 904 * Configure the Rx unit of the MAC after a reset. 905 **/ 906 static void fm10k_configure_rx(struct fm10k_intfc *interface) 907 { 908 int i; 909 910 /* Configure SWPRI to PC map */ 911 fm10k_configure_swpri_map(interface); 912 913 /* Configure RSS and DGLORT map */ 914 fm10k_configure_dglort(interface); 915 916 /* Setup the HW Rx Head and Tail descriptor pointers */ 917 for (i = 0; i < interface->num_rx_queues; i++) 918 fm10k_configure_rx_ring(interface, interface->rx_ring[i]); 919 920 /* possible poll here to verify that Rx rings are now enabled */ 921 } 922 923 static void fm10k_napi_enable_all(struct fm10k_intfc *interface) 924 { 925 struct fm10k_q_vector *q_vector; 926 int q_idx; 927 928 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 929 q_vector = interface->q_vector[q_idx]; 930 napi_enable(&q_vector->napi); 931 } 932 } 933 934 static irqreturn_t fm10k_msix_clean_rings(int __always_unused irq, void *data) 935 { 936 struct fm10k_q_vector *q_vector = data; 937 938 if (q_vector->rx.count || q_vector->tx.count) 939 napi_schedule_irqoff(&q_vector->napi); 940 941 return IRQ_HANDLED; 942 } 943 944 static irqreturn_t fm10k_msix_mbx_vf(int __always_unused irq, void *data) 945 { 946 struct fm10k_intfc *interface = data; 947 struct fm10k_hw *hw = &interface->hw; 948 struct fm10k_mbx_info *mbx = &hw->mbx; 949 950 /* re-enable mailbox interrupt and indicate 20us delay */ 951 fm10k_write_reg(hw, FM10K_VFITR(FM10K_MBX_VECTOR), 952 (FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) | 953 FM10K_ITR_ENABLE); 954 955 /* service upstream mailbox */ 956 if (fm10k_mbx_trylock(interface)) { 957 mbx->ops.process(hw, mbx); 958 fm10k_mbx_unlock(interface); 959 } 960 961 hw->mac.get_host_state = true; 962 fm10k_service_event_schedule(interface); 963 964 return IRQ_HANDLED; 965 } 966 967 #ifdef CONFIG_NET_POLL_CONTROLLER 968 /** 969 * fm10k_netpoll - A Polling 'interrupt' handler 970 * @netdev: network interface device structure 971 * 972 * This is used by netconsole to send skbs without having to re-enable 973 * interrupts. It's not called while the normal interrupt routine is executing. 974 **/ 975 void fm10k_netpoll(struct net_device *netdev) 976 { 977 struct fm10k_intfc *interface = netdev_priv(netdev); 978 int i; 979 980 /* if interface is down do nothing */ 981 if (test_bit(__FM10K_DOWN, &interface->state)) 982 return; 983 984 for (i = 0; i < interface->num_q_vectors; i++) 985 fm10k_msix_clean_rings(0, interface->q_vector[i]); 986 } 987 988 #endif 989 #define FM10K_ERR_MSG(type) case (type): error = #type; break 990 static void fm10k_handle_fault(struct fm10k_intfc *interface, int type, 991 struct fm10k_fault *fault) 992 { 993 struct pci_dev *pdev = interface->pdev; 994 struct fm10k_hw *hw = &interface->hw; 995 struct fm10k_iov_data *iov_data = interface->iov_data; 996 char *error; 997 998 switch (type) { 999 case FM10K_PCA_FAULT: 1000 switch (fault->type) { 1001 default: 1002 error = "Unknown PCA error"; 1003 break; 1004 FM10K_ERR_MSG(PCA_NO_FAULT); 1005 FM10K_ERR_MSG(PCA_UNMAPPED_ADDR); 1006 FM10K_ERR_MSG(PCA_BAD_QACCESS_PF); 1007 FM10K_ERR_MSG(PCA_BAD_QACCESS_VF); 1008 FM10K_ERR_MSG(PCA_MALICIOUS_REQ); 1009 FM10K_ERR_MSG(PCA_POISONED_TLP); 1010 FM10K_ERR_MSG(PCA_TLP_ABORT); 1011 } 1012 break; 1013 case FM10K_THI_FAULT: 1014 switch (fault->type) { 1015 default: 1016 error = "Unknown THI error"; 1017 break; 1018 FM10K_ERR_MSG(THI_NO_FAULT); 1019 FM10K_ERR_MSG(THI_MAL_DIS_Q_FAULT); 1020 } 1021 break; 1022 case FM10K_FUM_FAULT: 1023 switch (fault->type) { 1024 default: 1025 error = "Unknown FUM error"; 1026 break; 1027 FM10K_ERR_MSG(FUM_NO_FAULT); 1028 FM10K_ERR_MSG(FUM_UNMAPPED_ADDR); 1029 FM10K_ERR_MSG(FUM_BAD_VF_QACCESS); 1030 FM10K_ERR_MSG(FUM_ADD_DECODE_ERR); 1031 FM10K_ERR_MSG(FUM_RO_ERROR); 1032 FM10K_ERR_MSG(FUM_QPRC_CRC_ERROR); 1033 FM10K_ERR_MSG(FUM_CSR_TIMEOUT); 1034 FM10K_ERR_MSG(FUM_INVALID_TYPE); 1035 FM10K_ERR_MSG(FUM_INVALID_LENGTH); 1036 FM10K_ERR_MSG(FUM_INVALID_BE); 1037 FM10K_ERR_MSG(FUM_INVALID_ALIGN); 1038 } 1039 break; 1040 default: 1041 error = "Undocumented fault"; 1042 break; 1043 } 1044 1045 dev_warn(&pdev->dev, 1046 "%s Address: 0x%llx SpecInfo: 0x%x Func: %02x.%0x\n", 1047 error, fault->address, fault->specinfo, 1048 PCI_SLOT(fault->func), PCI_FUNC(fault->func)); 1049 1050 /* For VF faults, clear out the respective LPORT, reset the queue 1051 * resources, and then reconnect to the mailbox. This allows the 1052 * VF in question to resume behavior. For transient faults that are 1053 * the result of non-malicious behavior this will log the fault and 1054 * allow the VF to resume functionality. Obviously for malicious VFs 1055 * they will be able to attempt malicious behavior again. In this 1056 * case, the system administrator will need to step in and manually 1057 * remove or disable the VF in question. 1058 */ 1059 if (fault->func && iov_data) { 1060 int vf = fault->func - 1; 1061 struct fm10k_vf_info *vf_info = &iov_data->vf_info[vf]; 1062 1063 hw->iov.ops.reset_lport(hw, vf_info); 1064 hw->iov.ops.reset_resources(hw, vf_info); 1065 1066 /* reset_lport disables the VF, so re-enable it */ 1067 hw->iov.ops.set_lport(hw, vf_info, vf, 1068 FM10K_VF_FLAG_MULTI_CAPABLE); 1069 1070 /* reset_resources will disconnect from the mbx */ 1071 vf_info->mbx.ops.connect(hw, &vf_info->mbx); 1072 } 1073 } 1074 1075 static void fm10k_report_fault(struct fm10k_intfc *interface, u32 eicr) 1076 { 1077 struct fm10k_hw *hw = &interface->hw; 1078 struct fm10k_fault fault = { 0 }; 1079 int type, err; 1080 1081 for (eicr &= FM10K_EICR_FAULT_MASK, type = FM10K_PCA_FAULT; 1082 eicr; 1083 eicr >>= 1, type += FM10K_FAULT_SIZE) { 1084 /* only check if there is an error reported */ 1085 if (!(eicr & 0x1)) 1086 continue; 1087 1088 /* retrieve fault info */ 1089 err = hw->mac.ops.get_fault(hw, type, &fault); 1090 if (err) { 1091 dev_err(&interface->pdev->dev, 1092 "error reading fault\n"); 1093 continue; 1094 } 1095 1096 fm10k_handle_fault(interface, type, &fault); 1097 } 1098 } 1099 1100 static void fm10k_reset_drop_on_empty(struct fm10k_intfc *interface, u32 eicr) 1101 { 1102 struct fm10k_hw *hw = &interface->hw; 1103 const u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 1104 u32 maxholdq; 1105 int q; 1106 1107 if (!(eicr & FM10K_EICR_MAXHOLDTIME)) 1108 return; 1109 1110 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(7)); 1111 if (maxholdq) 1112 fm10k_write_reg(hw, FM10K_MAXHOLDQ(7), maxholdq); 1113 for (q = 255;;) { 1114 if (maxholdq & BIT(31)) { 1115 if (q < FM10K_MAX_QUEUES_PF) { 1116 interface->rx_overrun_pf++; 1117 fm10k_write_reg(hw, FM10K_RXDCTL(q), rxdctl); 1118 } else { 1119 interface->rx_overrun_vf++; 1120 } 1121 } 1122 1123 maxholdq *= 2; 1124 if (!maxholdq) 1125 q &= ~(32 - 1); 1126 1127 if (!q) 1128 break; 1129 1130 if (q-- % 32) 1131 continue; 1132 1133 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(q / 32)); 1134 if (maxholdq) 1135 fm10k_write_reg(hw, FM10K_MAXHOLDQ(q / 32), maxholdq); 1136 } 1137 } 1138 1139 static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data) 1140 { 1141 struct fm10k_intfc *interface = data; 1142 struct fm10k_hw *hw = &interface->hw; 1143 struct fm10k_mbx_info *mbx = &hw->mbx; 1144 u32 eicr; 1145 1146 /* unmask any set bits related to this interrupt */ 1147 eicr = fm10k_read_reg(hw, FM10K_EICR); 1148 fm10k_write_reg(hw, FM10K_EICR, eicr & (FM10K_EICR_MAILBOX | 1149 FM10K_EICR_SWITCHREADY | 1150 FM10K_EICR_SWITCHNOTREADY)); 1151 1152 /* report any faults found to the message log */ 1153 fm10k_report_fault(interface, eicr); 1154 1155 /* reset any queues disabled due to receiver overrun */ 1156 fm10k_reset_drop_on_empty(interface, eicr); 1157 1158 /* service mailboxes */ 1159 if (fm10k_mbx_trylock(interface)) { 1160 mbx->ops.process(hw, mbx); 1161 /* handle VFLRE events */ 1162 fm10k_iov_event(interface); 1163 fm10k_mbx_unlock(interface); 1164 } 1165 1166 /* if switch toggled state we should reset GLORTs */ 1167 if (eicr & FM10K_EICR_SWITCHNOTREADY) { 1168 /* force link down for at least 4 seconds */ 1169 interface->link_down_event = jiffies + (4 * HZ); 1170 set_bit(__FM10K_LINK_DOWN, &interface->state); 1171 1172 /* reset dglort_map back to no config */ 1173 hw->mac.dglort_map = FM10K_DGLORTMAP_NONE; 1174 } 1175 1176 /* we should validate host state after interrupt event */ 1177 hw->mac.get_host_state = true; 1178 1179 /* validate host state, and handle VF mailboxes in the service task */ 1180 fm10k_service_event_schedule(interface); 1181 1182 /* re-enable mailbox interrupt and indicate 20us delay */ 1183 fm10k_write_reg(hw, FM10K_ITR(FM10K_MBX_VECTOR), 1184 (FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) | 1185 FM10K_ITR_ENABLE); 1186 1187 return IRQ_HANDLED; 1188 } 1189 1190 void fm10k_mbx_free_irq(struct fm10k_intfc *interface) 1191 { 1192 struct fm10k_hw *hw = &interface->hw; 1193 struct msix_entry *entry; 1194 int itr_reg; 1195 1196 /* no mailbox IRQ to free if MSI-X is not enabled */ 1197 if (!interface->msix_entries) 1198 return; 1199 1200 entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1201 1202 /* disconnect the mailbox */ 1203 hw->mbx.ops.disconnect(hw, &hw->mbx); 1204 1205 /* disable Mailbox cause */ 1206 if (hw->mac.type == fm10k_mac_pf) { 1207 fm10k_write_reg(hw, FM10K_EIMR, 1208 FM10K_EIMR_DISABLE(PCA_FAULT) | 1209 FM10K_EIMR_DISABLE(FUM_FAULT) | 1210 FM10K_EIMR_DISABLE(MAILBOX) | 1211 FM10K_EIMR_DISABLE(SWITCHREADY) | 1212 FM10K_EIMR_DISABLE(SWITCHNOTREADY) | 1213 FM10K_EIMR_DISABLE(SRAMERROR) | 1214 FM10K_EIMR_DISABLE(VFLR) | 1215 FM10K_EIMR_DISABLE(MAXHOLDTIME)); 1216 itr_reg = FM10K_ITR(FM10K_MBX_VECTOR); 1217 } else { 1218 itr_reg = FM10K_VFITR(FM10K_MBX_VECTOR); 1219 } 1220 1221 fm10k_write_reg(hw, itr_reg, FM10K_ITR_MASK_SET); 1222 1223 free_irq(entry->vector, interface); 1224 } 1225 1226 static s32 fm10k_mbx_mac_addr(struct fm10k_hw *hw, u32 **results, 1227 struct fm10k_mbx_info *mbx) 1228 { 1229 bool vlan_override = hw->mac.vlan_override; 1230 u16 default_vid = hw->mac.default_vid; 1231 struct fm10k_intfc *interface; 1232 s32 err; 1233 1234 err = fm10k_msg_mac_vlan_vf(hw, results, mbx); 1235 if (err) 1236 return err; 1237 1238 interface = container_of(hw, struct fm10k_intfc, hw); 1239 1240 /* MAC was changed so we need reset */ 1241 if (is_valid_ether_addr(hw->mac.perm_addr) && 1242 !ether_addr_equal(hw->mac.perm_addr, hw->mac.addr)) 1243 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1244 1245 /* VLAN override was changed, or default VLAN changed */ 1246 if ((vlan_override != hw->mac.vlan_override) || 1247 (default_vid != hw->mac.default_vid)) 1248 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1249 1250 return 0; 1251 } 1252 1253 /* generic error handler for mailbox issues */ 1254 static s32 fm10k_mbx_error(struct fm10k_hw *hw, u32 **results, 1255 struct fm10k_mbx_info __always_unused *mbx) 1256 { 1257 struct fm10k_intfc *interface; 1258 struct pci_dev *pdev; 1259 1260 interface = container_of(hw, struct fm10k_intfc, hw); 1261 pdev = interface->pdev; 1262 1263 dev_err(&pdev->dev, "Unknown message ID %u\n", 1264 **results & FM10K_TLV_ID_MASK); 1265 1266 return 0; 1267 } 1268 1269 static const struct fm10k_msg_data vf_mbx_data[] = { 1270 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), 1271 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_mbx_mac_addr), 1272 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf), 1273 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1274 }; 1275 1276 static int fm10k_mbx_request_irq_vf(struct fm10k_intfc *interface) 1277 { 1278 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1279 struct net_device *dev = interface->netdev; 1280 struct fm10k_hw *hw = &interface->hw; 1281 int err; 1282 1283 /* Use timer0 for interrupt moderation on the mailbox */ 1284 u32 itr = entry->entry | FM10K_INT_MAP_TIMER0; 1285 1286 /* register mailbox handlers */ 1287 err = hw->mbx.ops.register_handlers(&hw->mbx, vf_mbx_data); 1288 if (err) 1289 return err; 1290 1291 /* request the IRQ */ 1292 err = request_irq(entry->vector, fm10k_msix_mbx_vf, 0, 1293 dev->name, interface); 1294 if (err) { 1295 netif_err(interface, probe, dev, 1296 "request_irq for msix_mbx failed: %d\n", err); 1297 return err; 1298 } 1299 1300 /* map all of the interrupt sources */ 1301 fm10k_write_reg(hw, FM10K_VFINT_MAP, itr); 1302 1303 /* enable interrupt */ 1304 fm10k_write_reg(hw, FM10K_VFITR(entry->entry), FM10K_ITR_ENABLE); 1305 1306 return 0; 1307 } 1308 1309 static s32 fm10k_lport_map(struct fm10k_hw *hw, u32 **results, 1310 struct fm10k_mbx_info *mbx) 1311 { 1312 struct fm10k_intfc *interface; 1313 u32 dglort_map = hw->mac.dglort_map; 1314 s32 err; 1315 1316 interface = container_of(hw, struct fm10k_intfc, hw); 1317 1318 err = fm10k_msg_err_pf(hw, results, mbx); 1319 if (!err && hw->swapi.status) { 1320 /* force link down for a reasonable delay */ 1321 interface->link_down_event = jiffies + (2 * HZ); 1322 set_bit(__FM10K_LINK_DOWN, &interface->state); 1323 1324 /* reset dglort_map back to no config */ 1325 hw->mac.dglort_map = FM10K_DGLORTMAP_NONE; 1326 1327 fm10k_service_event_schedule(interface); 1328 1329 /* prevent overloading kernel message buffer */ 1330 if (interface->lport_map_failed) 1331 return 0; 1332 1333 interface->lport_map_failed = true; 1334 1335 if (hw->swapi.status == FM10K_MSG_ERR_PEP_NOT_SCHEDULED) 1336 dev_warn(&interface->pdev->dev, 1337 "cannot obtain link because the host interface is configured for a PCIe host interface bandwidth of zero\n"); 1338 dev_warn(&interface->pdev->dev, 1339 "request logical port map failed: %d\n", 1340 hw->swapi.status); 1341 1342 return 0; 1343 } 1344 1345 err = fm10k_msg_lport_map_pf(hw, results, mbx); 1346 if (err) 1347 return err; 1348 1349 interface->lport_map_failed = false; 1350 1351 /* we need to reset if port count was just updated */ 1352 if (dglort_map != hw->mac.dglort_map) 1353 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1354 1355 return 0; 1356 } 1357 1358 static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results, 1359 struct fm10k_mbx_info __always_unused *mbx) 1360 { 1361 struct fm10k_intfc *interface; 1362 u16 glort, pvid; 1363 u32 pvid_update; 1364 s32 err; 1365 1366 err = fm10k_tlv_attr_get_u32(results[FM10K_PF_ATTR_ID_UPDATE_PVID], 1367 &pvid_update); 1368 if (err) 1369 return err; 1370 1371 /* extract values from the pvid update */ 1372 glort = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_GLORT); 1373 pvid = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_PVID); 1374 1375 /* if glort is not valid return error */ 1376 if (!fm10k_glort_valid_pf(hw, glort)) 1377 return FM10K_ERR_PARAM; 1378 1379 /* verify VLAN ID is valid */ 1380 if (pvid >= FM10K_VLAN_TABLE_VID_MAX) 1381 return FM10K_ERR_PARAM; 1382 1383 interface = container_of(hw, struct fm10k_intfc, hw); 1384 1385 /* check to see if this belongs to one of the VFs */ 1386 err = fm10k_iov_update_pvid(interface, glort, pvid); 1387 if (!err) 1388 return 0; 1389 1390 /* we need to reset if default VLAN was just updated */ 1391 if (pvid != hw->mac.default_vid) 1392 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1393 1394 hw->mac.default_vid = pvid; 1395 1396 return 0; 1397 } 1398 1399 static const struct fm10k_msg_data pf_mbx_data[] = { 1400 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf), 1401 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf), 1402 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_lport_map), 1403 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf), 1404 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf), 1405 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_update_pvid), 1406 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1407 }; 1408 1409 static int fm10k_mbx_request_irq_pf(struct fm10k_intfc *interface) 1410 { 1411 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1412 struct net_device *dev = interface->netdev; 1413 struct fm10k_hw *hw = &interface->hw; 1414 int err; 1415 1416 /* Use timer0 for interrupt moderation on the mailbox */ 1417 u32 mbx_itr = entry->entry | FM10K_INT_MAP_TIMER0; 1418 u32 other_itr = entry->entry | FM10K_INT_MAP_IMMEDIATE; 1419 1420 /* register mailbox handlers */ 1421 err = hw->mbx.ops.register_handlers(&hw->mbx, pf_mbx_data); 1422 if (err) 1423 return err; 1424 1425 /* request the IRQ */ 1426 err = request_irq(entry->vector, fm10k_msix_mbx_pf, 0, 1427 dev->name, interface); 1428 if (err) { 1429 netif_err(interface, probe, dev, 1430 "request_irq for msix_mbx failed: %d\n", err); 1431 return err; 1432 } 1433 1434 /* Enable interrupts w/ no moderation for "other" interrupts */ 1435 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), other_itr); 1436 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), other_itr); 1437 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_sram), other_itr); 1438 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_max_hold_time), other_itr); 1439 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_vflr), other_itr); 1440 1441 /* Enable interrupts w/ moderation for mailbox */ 1442 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_mailbox), mbx_itr); 1443 1444 /* Enable individual interrupt causes */ 1445 fm10k_write_reg(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | 1446 FM10K_EIMR_ENABLE(FUM_FAULT) | 1447 FM10K_EIMR_ENABLE(MAILBOX) | 1448 FM10K_EIMR_ENABLE(SWITCHREADY) | 1449 FM10K_EIMR_ENABLE(SWITCHNOTREADY) | 1450 FM10K_EIMR_ENABLE(SRAMERROR) | 1451 FM10K_EIMR_ENABLE(VFLR) | 1452 FM10K_EIMR_ENABLE(MAXHOLDTIME)); 1453 1454 /* enable interrupt */ 1455 fm10k_write_reg(hw, FM10K_ITR(entry->entry), FM10K_ITR_ENABLE); 1456 1457 return 0; 1458 } 1459 1460 int fm10k_mbx_request_irq(struct fm10k_intfc *interface) 1461 { 1462 struct fm10k_hw *hw = &interface->hw; 1463 int err; 1464 1465 /* enable Mailbox cause */ 1466 if (hw->mac.type == fm10k_mac_pf) 1467 err = fm10k_mbx_request_irq_pf(interface); 1468 else 1469 err = fm10k_mbx_request_irq_vf(interface); 1470 if (err) 1471 return err; 1472 1473 /* connect mailbox */ 1474 err = hw->mbx.ops.connect(hw, &hw->mbx); 1475 1476 /* if the mailbox failed to connect, then free IRQ */ 1477 if (err) 1478 fm10k_mbx_free_irq(interface); 1479 1480 return err; 1481 } 1482 1483 /** 1484 * fm10k_qv_free_irq - release interrupts associated with queue vectors 1485 * @interface: board private structure 1486 * 1487 * Release all interrupts associated with this interface 1488 **/ 1489 void fm10k_qv_free_irq(struct fm10k_intfc *interface) 1490 { 1491 int vector = interface->num_q_vectors; 1492 struct fm10k_hw *hw = &interface->hw; 1493 struct msix_entry *entry; 1494 1495 entry = &interface->msix_entries[NON_Q_VECTORS(hw) + vector]; 1496 1497 while (vector) { 1498 struct fm10k_q_vector *q_vector; 1499 1500 vector--; 1501 entry--; 1502 q_vector = interface->q_vector[vector]; 1503 1504 if (!q_vector->tx.count && !q_vector->rx.count) 1505 continue; 1506 1507 /* clear the affinity_mask in the IRQ descriptor */ 1508 irq_set_affinity_hint(entry->vector, NULL); 1509 1510 /* disable interrupts */ 1511 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1512 1513 free_irq(entry->vector, q_vector); 1514 } 1515 } 1516 1517 /** 1518 * fm10k_qv_request_irq - initialize interrupts for queue vectors 1519 * @interface: board private structure 1520 * 1521 * Attempts to configure interrupts using the best available 1522 * capabilities of the hardware and kernel. 1523 **/ 1524 int fm10k_qv_request_irq(struct fm10k_intfc *interface) 1525 { 1526 struct net_device *dev = interface->netdev; 1527 struct fm10k_hw *hw = &interface->hw; 1528 struct msix_entry *entry; 1529 int ri = 0, ti = 0; 1530 int vector, err; 1531 1532 entry = &interface->msix_entries[NON_Q_VECTORS(hw)]; 1533 1534 for (vector = 0; vector < interface->num_q_vectors; vector++) { 1535 struct fm10k_q_vector *q_vector = interface->q_vector[vector]; 1536 1537 /* name the vector */ 1538 if (q_vector->tx.count && q_vector->rx.count) { 1539 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1540 "%s-TxRx-%d", dev->name, ri++); 1541 ti++; 1542 } else if (q_vector->rx.count) { 1543 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1544 "%s-rx-%d", dev->name, ri++); 1545 } else if (q_vector->tx.count) { 1546 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1547 "%s-tx-%d", dev->name, ti++); 1548 } else { 1549 /* skip this unused q_vector */ 1550 continue; 1551 } 1552 1553 /* Assign ITR register to q_vector */ 1554 q_vector->itr = (hw->mac.type == fm10k_mac_pf) ? 1555 &interface->uc_addr[FM10K_ITR(entry->entry)] : 1556 &interface->uc_addr[FM10K_VFITR(entry->entry)]; 1557 1558 /* request the IRQ */ 1559 err = request_irq(entry->vector, &fm10k_msix_clean_rings, 0, 1560 q_vector->name, q_vector); 1561 if (err) { 1562 netif_err(interface, probe, dev, 1563 "request_irq failed for MSIX interrupt Error: %d\n", 1564 err); 1565 goto err_out; 1566 } 1567 1568 /* assign the mask for this irq */ 1569 irq_set_affinity_hint(entry->vector, &q_vector->affinity_mask); 1570 1571 /* Enable q_vector */ 1572 writel(FM10K_ITR_ENABLE, q_vector->itr); 1573 1574 entry++; 1575 } 1576 1577 return 0; 1578 1579 err_out: 1580 /* wind through the ring freeing all entries and vectors */ 1581 while (vector) { 1582 struct fm10k_q_vector *q_vector; 1583 1584 entry--; 1585 vector--; 1586 q_vector = interface->q_vector[vector]; 1587 1588 if (!q_vector->tx.count && !q_vector->rx.count) 1589 continue; 1590 1591 /* clear the affinity_mask in the IRQ descriptor */ 1592 irq_set_affinity_hint(entry->vector, NULL); 1593 1594 /* disable interrupts */ 1595 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1596 1597 free_irq(entry->vector, q_vector); 1598 } 1599 1600 return err; 1601 } 1602 1603 void fm10k_up(struct fm10k_intfc *interface) 1604 { 1605 struct fm10k_hw *hw = &interface->hw; 1606 1607 /* Enable Tx/Rx DMA */ 1608 hw->mac.ops.start_hw(hw); 1609 1610 /* configure Tx descriptor rings */ 1611 fm10k_configure_tx(interface); 1612 1613 /* configure Rx descriptor rings */ 1614 fm10k_configure_rx(interface); 1615 1616 /* configure interrupts */ 1617 hw->mac.ops.update_int_moderator(hw); 1618 1619 /* enable statistics capture again */ 1620 clear_bit(__FM10K_UPDATING_STATS, &interface->state); 1621 1622 /* clear down bit to indicate we are ready to go */ 1623 clear_bit(__FM10K_DOWN, &interface->state); 1624 1625 /* enable polling cleanups */ 1626 fm10k_napi_enable_all(interface); 1627 1628 /* re-establish Rx filters */ 1629 fm10k_restore_rx_state(interface); 1630 1631 /* enable transmits */ 1632 netif_tx_start_all_queues(interface->netdev); 1633 1634 /* kick off the service timer now */ 1635 hw->mac.get_host_state = true; 1636 mod_timer(&interface->service_timer, jiffies); 1637 } 1638 1639 static void fm10k_napi_disable_all(struct fm10k_intfc *interface) 1640 { 1641 struct fm10k_q_vector *q_vector; 1642 int q_idx; 1643 1644 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 1645 q_vector = interface->q_vector[q_idx]; 1646 napi_disable(&q_vector->napi); 1647 } 1648 } 1649 1650 void fm10k_down(struct fm10k_intfc *interface) 1651 { 1652 struct net_device *netdev = interface->netdev; 1653 struct fm10k_hw *hw = &interface->hw; 1654 int err, i = 0, count = 0; 1655 1656 /* signal that we are down to the interrupt handler and service task */ 1657 if (test_and_set_bit(__FM10K_DOWN, &interface->state)) 1658 return; 1659 1660 /* call carrier off first to avoid false dev_watchdog timeouts */ 1661 netif_carrier_off(netdev); 1662 1663 /* disable transmits */ 1664 netif_tx_stop_all_queues(netdev); 1665 netif_tx_disable(netdev); 1666 1667 /* reset Rx filters */ 1668 fm10k_reset_rx_state(interface); 1669 1670 /* disable polling routines */ 1671 fm10k_napi_disable_all(interface); 1672 1673 /* capture stats one last time before stopping interface */ 1674 fm10k_update_stats(interface); 1675 1676 /* prevent updating statistics while we're down */ 1677 while (test_and_set_bit(__FM10K_UPDATING_STATS, &interface->state)) 1678 usleep_range(1000, 2000); 1679 1680 /* skip waiting for TX DMA if we lost PCIe link */ 1681 if (FM10K_REMOVED(hw->hw_addr)) 1682 goto skip_tx_dma_drain; 1683 1684 /* In some rare circumstances it can take a while for Tx queues to 1685 * quiesce and be fully disabled. Attempt to .stop_hw() first, and 1686 * then if we get ERR_REQUESTS_PENDING, go ahead and wait in a loop 1687 * until the Tx queues have emptied, or until a number of retries. If 1688 * we fail to clear within the retry loop, we will issue a warning 1689 * indicating that Tx DMA is probably hung. Note this means we call 1690 * .stop_hw() twice but this shouldn't cause any problems. 1691 */ 1692 err = hw->mac.ops.stop_hw(hw); 1693 if (err != FM10K_ERR_REQUESTS_PENDING) 1694 goto skip_tx_dma_drain; 1695 1696 #define TX_DMA_DRAIN_RETRIES 25 1697 for (count = 0; count < TX_DMA_DRAIN_RETRIES; count++) { 1698 usleep_range(10000, 20000); 1699 1700 /* start checking at the last ring to have pending Tx */ 1701 for (; i < interface->num_tx_queues; i++) 1702 if (fm10k_get_tx_pending(interface->tx_ring[i])) 1703 break; 1704 1705 /* if all the queues are drained, we can break now */ 1706 if (i == interface->num_tx_queues) 1707 break; 1708 } 1709 1710 if (count >= TX_DMA_DRAIN_RETRIES) 1711 dev_err(&interface->pdev->dev, 1712 "Tx queues failed to drain after %d tries. Tx DMA is probably hung.\n", 1713 count); 1714 skip_tx_dma_drain: 1715 /* Disable DMA engine for Tx/Rx */ 1716 err = hw->mac.ops.stop_hw(hw); 1717 if (err == FM10K_ERR_REQUESTS_PENDING) 1718 dev_err(&interface->pdev->dev, 1719 "due to pending requests hw was not shut down gracefully\n"); 1720 else if (err) 1721 dev_err(&interface->pdev->dev, "stop_hw failed: %d\n", err); 1722 1723 /* free any buffers still on the rings */ 1724 fm10k_clean_all_tx_rings(interface); 1725 fm10k_clean_all_rx_rings(interface); 1726 } 1727 1728 /** 1729 * fm10k_sw_init - Initialize general software structures 1730 * @interface: host interface private structure to initialize 1731 * 1732 * fm10k_sw_init initializes the interface private data structure. 1733 * Fields are initialized based on PCI device information and 1734 * OS network device settings (MTU size). 1735 **/ 1736 static int fm10k_sw_init(struct fm10k_intfc *interface, 1737 const struct pci_device_id *ent) 1738 { 1739 const struct fm10k_info *fi = fm10k_info_tbl[ent->driver_data]; 1740 struct fm10k_hw *hw = &interface->hw; 1741 struct pci_dev *pdev = interface->pdev; 1742 struct net_device *netdev = interface->netdev; 1743 u32 rss_key[FM10K_RSSRK_SIZE]; 1744 unsigned int rss; 1745 int err; 1746 1747 /* initialize back pointer */ 1748 hw->back = interface; 1749 hw->hw_addr = interface->uc_addr; 1750 1751 /* PCI config space info */ 1752 hw->vendor_id = pdev->vendor; 1753 hw->device_id = pdev->device; 1754 hw->revision_id = pdev->revision; 1755 hw->subsystem_vendor_id = pdev->subsystem_vendor; 1756 hw->subsystem_device_id = pdev->subsystem_device; 1757 1758 /* Setup hw api */ 1759 memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops)); 1760 hw->mac.type = fi->mac; 1761 1762 /* Setup IOV handlers */ 1763 if (fi->iov_ops) 1764 memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops)); 1765 1766 /* Set common capability flags and settings */ 1767 rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus()); 1768 interface->ring_feature[RING_F_RSS].limit = rss; 1769 fi->get_invariants(hw); 1770 1771 /* pick up the PCIe bus settings for reporting later */ 1772 if (hw->mac.ops.get_bus_info) 1773 hw->mac.ops.get_bus_info(hw); 1774 1775 /* limit the usable DMA range */ 1776 if (hw->mac.ops.set_dma_mask) 1777 hw->mac.ops.set_dma_mask(hw, dma_get_mask(&pdev->dev)); 1778 1779 /* update netdev with DMA restrictions */ 1780 if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) { 1781 netdev->features |= NETIF_F_HIGHDMA; 1782 netdev->vlan_features |= NETIF_F_HIGHDMA; 1783 } 1784 1785 /* delay any future reset requests */ 1786 interface->last_reset = jiffies + (10 * HZ); 1787 1788 /* reset and initialize the hardware so it is in a known state */ 1789 err = hw->mac.ops.reset_hw(hw); 1790 if (err) { 1791 dev_err(&pdev->dev, "reset_hw failed: %d\n", err); 1792 return err; 1793 } 1794 1795 err = hw->mac.ops.init_hw(hw); 1796 if (err) { 1797 dev_err(&pdev->dev, "init_hw failed: %d\n", err); 1798 return err; 1799 } 1800 1801 /* initialize hardware statistics */ 1802 hw->mac.ops.update_hw_stats(hw, &interface->stats); 1803 1804 /* Set upper limit on IOV VFs that can be allocated */ 1805 pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs); 1806 1807 /* Start with random Ethernet address */ 1808 eth_random_addr(hw->mac.addr); 1809 1810 /* Initialize MAC address from hardware */ 1811 err = hw->mac.ops.read_mac_addr(hw); 1812 if (err) { 1813 dev_warn(&pdev->dev, 1814 "Failed to obtain MAC address defaulting to random\n"); 1815 /* tag address assignment as random */ 1816 netdev->addr_assign_type |= NET_ADDR_RANDOM; 1817 } 1818 1819 ether_addr_copy(netdev->dev_addr, hw->mac.addr); 1820 ether_addr_copy(netdev->perm_addr, hw->mac.addr); 1821 1822 if (!is_valid_ether_addr(netdev->perm_addr)) { 1823 dev_err(&pdev->dev, "Invalid MAC Address\n"); 1824 return -EIO; 1825 } 1826 1827 /* initialize DCBNL interface */ 1828 fm10k_dcbnl_set_ops(netdev); 1829 1830 /* set default ring sizes */ 1831 interface->tx_ring_count = FM10K_DEFAULT_TXD; 1832 interface->rx_ring_count = FM10K_DEFAULT_RXD; 1833 1834 /* set default interrupt moderation */ 1835 interface->tx_itr = FM10K_TX_ITR_DEFAULT; 1836 interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_RX_ITR_DEFAULT; 1837 1838 /* initialize vxlan_port list */ 1839 INIT_LIST_HEAD(&interface->vxlan_port); 1840 1841 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 1842 memcpy(interface->rssrk, rss_key, sizeof(rss_key)); 1843 1844 /* Start off interface as being down */ 1845 set_bit(__FM10K_DOWN, &interface->state); 1846 set_bit(__FM10K_UPDATING_STATS, &interface->state); 1847 1848 return 0; 1849 } 1850 1851 static void fm10k_slot_warn(struct fm10k_intfc *interface) 1852 { 1853 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN; 1854 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN; 1855 struct fm10k_hw *hw = &interface->hw; 1856 int max_gts = 0, expected_gts = 0; 1857 1858 if (pcie_get_minimum_link(interface->pdev, &speed, &width) || 1859 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 1860 dev_warn(&interface->pdev->dev, 1861 "Unable to determine PCI Express bandwidth.\n"); 1862 return; 1863 } 1864 1865 switch (speed) { 1866 case PCIE_SPEED_2_5GT: 1867 /* 8b/10b encoding reduces max throughput by 20% */ 1868 max_gts = 2 * width; 1869 break; 1870 case PCIE_SPEED_5_0GT: 1871 /* 8b/10b encoding reduces max throughput by 20% */ 1872 max_gts = 4 * width; 1873 break; 1874 case PCIE_SPEED_8_0GT: 1875 /* 128b/130b encoding has less than 2% impact on throughput */ 1876 max_gts = 8 * width; 1877 break; 1878 default: 1879 dev_warn(&interface->pdev->dev, 1880 "Unable to determine PCI Express bandwidth.\n"); 1881 return; 1882 } 1883 1884 dev_info(&interface->pdev->dev, 1885 "PCI Express bandwidth of %dGT/s available\n", 1886 max_gts); 1887 dev_info(&interface->pdev->dev, 1888 "(Speed:%s, Width: x%d, Encoding Loss:%s, Payload:%s)\n", 1889 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : 1890 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : 1891 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : 1892 "Unknown"), 1893 hw->bus.width, 1894 (speed == PCIE_SPEED_2_5GT ? "20%" : 1895 speed == PCIE_SPEED_5_0GT ? "20%" : 1896 speed == PCIE_SPEED_8_0GT ? "<2%" : 1897 "Unknown"), 1898 (hw->bus.payload == fm10k_bus_payload_128 ? "128B" : 1899 hw->bus.payload == fm10k_bus_payload_256 ? "256B" : 1900 hw->bus.payload == fm10k_bus_payload_512 ? "512B" : 1901 "Unknown")); 1902 1903 switch (hw->bus_caps.speed) { 1904 case fm10k_bus_speed_2500: 1905 /* 8b/10b encoding reduces max throughput by 20% */ 1906 expected_gts = 2 * hw->bus_caps.width; 1907 break; 1908 case fm10k_bus_speed_5000: 1909 /* 8b/10b encoding reduces max throughput by 20% */ 1910 expected_gts = 4 * hw->bus_caps.width; 1911 break; 1912 case fm10k_bus_speed_8000: 1913 /* 128b/130b encoding has less than 2% impact on throughput */ 1914 expected_gts = 8 * hw->bus_caps.width; 1915 break; 1916 default: 1917 dev_warn(&interface->pdev->dev, 1918 "Unable to determine expected PCI Express bandwidth.\n"); 1919 return; 1920 } 1921 1922 if (max_gts >= expected_gts) 1923 return; 1924 1925 dev_warn(&interface->pdev->dev, 1926 "This device requires %dGT/s of bandwidth for optimal performance.\n", 1927 expected_gts); 1928 dev_warn(&interface->pdev->dev, 1929 "A %sslot with x%d lanes is suggested.\n", 1930 (hw->bus_caps.speed == fm10k_bus_speed_2500 ? "2.5GT/s " : 1931 hw->bus_caps.speed == fm10k_bus_speed_5000 ? "5.0GT/s " : 1932 hw->bus_caps.speed == fm10k_bus_speed_8000 ? "8.0GT/s " : ""), 1933 hw->bus_caps.width); 1934 } 1935 1936 /** 1937 * fm10k_probe - Device Initialization Routine 1938 * @pdev: PCI device information struct 1939 * @ent: entry in fm10k_pci_tbl 1940 * 1941 * Returns 0 on success, negative on failure 1942 * 1943 * fm10k_probe initializes an interface identified by a pci_dev structure. 1944 * The OS initialization, configuring of the interface private structure, 1945 * and a hardware reset occur. 1946 **/ 1947 static int fm10k_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1948 { 1949 struct net_device *netdev; 1950 struct fm10k_intfc *interface; 1951 int err; 1952 1953 err = pci_enable_device_mem(pdev); 1954 if (err) 1955 return err; 1956 1957 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); 1958 if (err) 1959 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1960 if (err) { 1961 dev_err(&pdev->dev, 1962 "DMA configuration failed: %d\n", err); 1963 goto err_dma; 1964 } 1965 1966 err = pci_request_mem_regions(pdev, fm10k_driver_name); 1967 if (err) { 1968 dev_err(&pdev->dev, 1969 "pci_request_selected_regions failed: %d\n", err); 1970 goto err_pci_reg; 1971 } 1972 1973 pci_enable_pcie_error_reporting(pdev); 1974 1975 pci_set_master(pdev); 1976 pci_save_state(pdev); 1977 1978 netdev = fm10k_alloc_netdev(fm10k_info_tbl[ent->driver_data]); 1979 if (!netdev) { 1980 err = -ENOMEM; 1981 goto err_alloc_netdev; 1982 } 1983 1984 SET_NETDEV_DEV(netdev, &pdev->dev); 1985 1986 interface = netdev_priv(netdev); 1987 pci_set_drvdata(pdev, interface); 1988 1989 interface->netdev = netdev; 1990 interface->pdev = pdev; 1991 1992 interface->uc_addr = ioremap(pci_resource_start(pdev, 0), 1993 FM10K_UC_ADDR_SIZE); 1994 if (!interface->uc_addr) { 1995 err = -EIO; 1996 goto err_ioremap; 1997 } 1998 1999 err = fm10k_sw_init(interface, ent); 2000 if (err) 2001 goto err_sw_init; 2002 2003 /* enable debugfs support */ 2004 fm10k_dbg_intfc_init(interface); 2005 2006 err = fm10k_init_queueing_scheme(interface); 2007 if (err) 2008 goto err_sw_init; 2009 2010 /* the mbx interrupt might attempt to schedule the service task, so we 2011 * must ensure it is disabled since we haven't yet requested the timer 2012 * or work item. 2013 */ 2014 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 2015 2016 err = fm10k_mbx_request_irq(interface); 2017 if (err) 2018 goto err_mbx_interrupt; 2019 2020 /* final check of hardware state before registering the interface */ 2021 err = fm10k_hw_ready(interface); 2022 if (err) 2023 goto err_register; 2024 2025 err = register_netdev(netdev); 2026 if (err) 2027 goto err_register; 2028 2029 /* carrier off reporting is important to ethtool even BEFORE open */ 2030 netif_carrier_off(netdev); 2031 2032 /* stop all the transmit queues from transmitting until link is up */ 2033 netif_tx_stop_all_queues(netdev); 2034 2035 /* Initialize service timer and service task late in order to avoid 2036 * cleanup issues. 2037 */ 2038 setup_timer(&interface->service_timer, &fm10k_service_timer, 2039 (unsigned long)interface); 2040 INIT_WORK(&interface->service_task, fm10k_service_task); 2041 2042 /* kick off service timer now, even when interface is down */ 2043 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 2044 2045 /* print warning for non-optimal configurations */ 2046 fm10k_slot_warn(interface); 2047 2048 /* report MAC address for logging */ 2049 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 2050 2051 /* enable SR-IOV after registering netdev to enforce PF/VF ordering */ 2052 fm10k_iov_configure(pdev, 0); 2053 2054 /* clear the service task disable bit to allow service task to start */ 2055 clear_bit(__FM10K_SERVICE_DISABLE, &interface->state); 2056 2057 return 0; 2058 2059 err_register: 2060 fm10k_mbx_free_irq(interface); 2061 err_mbx_interrupt: 2062 fm10k_clear_queueing_scheme(interface); 2063 err_sw_init: 2064 if (interface->sw_addr) 2065 iounmap(interface->sw_addr); 2066 iounmap(interface->uc_addr); 2067 err_ioremap: 2068 free_netdev(netdev); 2069 err_alloc_netdev: 2070 pci_release_mem_regions(pdev); 2071 err_pci_reg: 2072 err_dma: 2073 pci_disable_device(pdev); 2074 return err; 2075 } 2076 2077 /** 2078 * fm10k_remove - Device Removal Routine 2079 * @pdev: PCI device information struct 2080 * 2081 * fm10k_remove is called by the PCI subsystem to alert the driver 2082 * that it should release a PCI device. The could be caused by a 2083 * Hot-Plug event, or because the driver is going to be removed from 2084 * memory. 2085 **/ 2086 static void fm10k_remove(struct pci_dev *pdev) 2087 { 2088 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2089 struct net_device *netdev = interface->netdev; 2090 2091 del_timer_sync(&interface->service_timer); 2092 2093 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 2094 cancel_work_sync(&interface->service_task); 2095 2096 /* free netdev, this may bounce the interrupts due to setup_tc */ 2097 if (netdev->reg_state == NETREG_REGISTERED) 2098 unregister_netdev(netdev); 2099 2100 /* release VFs */ 2101 fm10k_iov_disable(pdev); 2102 2103 /* disable mailbox interrupt */ 2104 fm10k_mbx_free_irq(interface); 2105 2106 /* free interrupts */ 2107 fm10k_clear_queueing_scheme(interface); 2108 2109 /* remove any debugfs interfaces */ 2110 fm10k_dbg_intfc_exit(interface); 2111 2112 if (interface->sw_addr) 2113 iounmap(interface->sw_addr); 2114 iounmap(interface->uc_addr); 2115 2116 free_netdev(netdev); 2117 2118 pci_release_mem_regions(pdev); 2119 2120 pci_disable_pcie_error_reporting(pdev); 2121 2122 pci_disable_device(pdev); 2123 } 2124 2125 static void fm10k_prepare_suspend(struct fm10k_intfc *interface) 2126 { 2127 /* the watchdog task reads from registers, which might appear like 2128 * a surprise remove if the PCIe device is disabled while we're 2129 * stopped. We stop the watchdog task until after we resume software 2130 * activity. 2131 */ 2132 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 2133 cancel_work_sync(&interface->service_task); 2134 2135 fm10k_prepare_for_reset(interface); 2136 } 2137 2138 static int fm10k_handle_resume(struct fm10k_intfc *interface) 2139 { 2140 struct fm10k_hw *hw = &interface->hw; 2141 int err; 2142 2143 /* reset statistics starting values */ 2144 hw->mac.ops.rebind_hw_stats(hw, &interface->stats); 2145 2146 err = fm10k_handle_reset(interface); 2147 if (err) 2148 return err; 2149 2150 /* assume host is not ready, to prevent race with watchdog in case we 2151 * actually don't have connection to the switch 2152 */ 2153 interface->host_ready = false; 2154 fm10k_watchdog_host_not_ready(interface); 2155 2156 /* force link to stay down for a second to prevent link flutter */ 2157 interface->link_down_event = jiffies + (HZ); 2158 set_bit(__FM10K_LINK_DOWN, &interface->state); 2159 2160 /* clear the service task disable bit to allow service task to start */ 2161 clear_bit(__FM10K_SERVICE_DISABLE, &interface->state); 2162 fm10k_service_event_schedule(interface); 2163 2164 return err; 2165 } 2166 2167 #ifdef CONFIG_PM 2168 /** 2169 * fm10k_resume - Restore device to pre-sleep state 2170 * @pdev: PCI device information struct 2171 * 2172 * fm10k_resume is called after the system has powered back up from a sleep 2173 * state and is ready to resume operation. This function is meant to restore 2174 * the device back to its pre-sleep state. 2175 **/ 2176 static int fm10k_resume(struct pci_dev *pdev) 2177 { 2178 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2179 struct net_device *netdev = interface->netdev; 2180 struct fm10k_hw *hw = &interface->hw; 2181 u32 err; 2182 2183 pci_set_power_state(pdev, PCI_D0); 2184 pci_restore_state(pdev); 2185 2186 /* pci_restore_state clears dev->state_saved so call 2187 * pci_save_state to restore it. 2188 */ 2189 pci_save_state(pdev); 2190 2191 err = pci_enable_device_mem(pdev); 2192 if (err) { 2193 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 2194 return err; 2195 } 2196 pci_set_master(pdev); 2197 2198 pci_wake_from_d3(pdev, false); 2199 2200 /* refresh hw_addr in case it was dropped */ 2201 hw->hw_addr = interface->uc_addr; 2202 2203 err = fm10k_handle_resume(interface); 2204 if (err) 2205 return err; 2206 2207 netif_device_attach(netdev); 2208 2209 return 0; 2210 } 2211 2212 /** 2213 * fm10k_suspend - Prepare the device for a system sleep state 2214 * @pdev: PCI device information struct 2215 * 2216 * fm10k_suspend is meant to shutdown the device prior to the system entering 2217 * a sleep state. The fm10k hardware does not support wake on lan so the 2218 * driver simply needs to shut down the device so it is in a low power state. 2219 **/ 2220 static int fm10k_suspend(struct pci_dev *pdev, 2221 pm_message_t __always_unused state) 2222 { 2223 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2224 struct net_device *netdev = interface->netdev; 2225 int err = 0; 2226 2227 netif_device_detach(netdev); 2228 2229 fm10k_prepare_suspend(interface); 2230 2231 err = pci_save_state(pdev); 2232 if (err) 2233 return err; 2234 2235 pci_disable_device(pdev); 2236 pci_wake_from_d3(pdev, false); 2237 pci_set_power_state(pdev, PCI_D3hot); 2238 2239 return 0; 2240 } 2241 2242 #endif /* CONFIG_PM */ 2243 /** 2244 * fm10k_io_error_detected - called when PCI error is detected 2245 * @pdev: Pointer to PCI device 2246 * @state: The current pci connection state 2247 * 2248 * This function is called after a PCI bus error affecting 2249 * this device has been detected. 2250 */ 2251 static pci_ers_result_t fm10k_io_error_detected(struct pci_dev *pdev, 2252 pci_channel_state_t state) 2253 { 2254 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2255 struct net_device *netdev = interface->netdev; 2256 2257 netif_device_detach(netdev); 2258 2259 if (state == pci_channel_io_perm_failure) 2260 return PCI_ERS_RESULT_DISCONNECT; 2261 2262 fm10k_prepare_suspend(interface); 2263 2264 /* Request a slot reset. */ 2265 return PCI_ERS_RESULT_NEED_RESET; 2266 } 2267 2268 /** 2269 * fm10k_io_slot_reset - called after the pci bus has been reset. 2270 * @pdev: Pointer to PCI device 2271 * 2272 * Restart the card from scratch, as if from a cold-boot. 2273 */ 2274 static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev) 2275 { 2276 pci_ers_result_t result; 2277 2278 if (pci_enable_device_mem(pdev)) { 2279 dev_err(&pdev->dev, 2280 "Cannot re-enable PCI device after reset.\n"); 2281 result = PCI_ERS_RESULT_DISCONNECT; 2282 } else { 2283 pci_set_master(pdev); 2284 pci_restore_state(pdev); 2285 2286 /* After second error pci->state_saved is false, this 2287 * resets it so EEH doesn't break. 2288 */ 2289 pci_save_state(pdev); 2290 2291 pci_wake_from_d3(pdev, false); 2292 2293 result = PCI_ERS_RESULT_RECOVERED; 2294 } 2295 2296 pci_cleanup_aer_uncorrect_error_status(pdev); 2297 2298 return result; 2299 } 2300 2301 /** 2302 * fm10k_io_resume - called when traffic can start flowing again. 2303 * @pdev: Pointer to PCI device 2304 * 2305 * This callback is called when the error recovery driver tells us that 2306 * its OK to resume normal operation. 2307 */ 2308 static void fm10k_io_resume(struct pci_dev *pdev) 2309 { 2310 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2311 struct net_device *netdev = interface->netdev; 2312 int err; 2313 2314 err = fm10k_handle_resume(interface); 2315 2316 if (err) 2317 dev_warn(&pdev->dev, 2318 "fm10k_io_resume failed: %d\n", err); 2319 else 2320 netif_device_attach(netdev); 2321 } 2322 2323 /** 2324 * fm10k_io_reset_notify - called when PCI function is reset 2325 * @pdev: Pointer to PCI device 2326 * 2327 * This callback is called when the PCI function is reset such as from 2328 * /sys/class/net/<enpX>/device/reset or similar. When prepare is true, it 2329 * means we should prepare for a function reset. If prepare is false, it means 2330 * the function reset just occurred. 2331 */ 2332 static void fm10k_io_reset_notify(struct pci_dev *pdev, bool prepare) 2333 { 2334 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2335 int err = 0; 2336 2337 if (prepare) { 2338 /* warn incase we have any active VF devices */ 2339 if (pci_num_vf(pdev)) 2340 dev_warn(&pdev->dev, 2341 "PCIe FLR may cause issues for any active VF devices\n"); 2342 2343 fm10k_prepare_suspend(interface); 2344 } else { 2345 err = fm10k_handle_resume(interface); 2346 } 2347 2348 if (err) { 2349 dev_warn(&pdev->dev, 2350 "fm10k_io_reset_notify failed: %d\n", err); 2351 netif_device_detach(interface->netdev); 2352 } 2353 } 2354 2355 static const struct pci_error_handlers fm10k_err_handler = { 2356 .error_detected = fm10k_io_error_detected, 2357 .slot_reset = fm10k_io_slot_reset, 2358 .resume = fm10k_io_resume, 2359 .reset_notify = fm10k_io_reset_notify, 2360 }; 2361 2362 static struct pci_driver fm10k_driver = { 2363 .name = fm10k_driver_name, 2364 .id_table = fm10k_pci_tbl, 2365 .probe = fm10k_probe, 2366 .remove = fm10k_remove, 2367 #ifdef CONFIG_PM 2368 .suspend = fm10k_suspend, 2369 .resume = fm10k_resume, 2370 #endif 2371 .sriov_configure = fm10k_iov_configure, 2372 .err_handler = &fm10k_err_handler 2373 }; 2374 2375 /** 2376 * fm10k_register_pci_driver - register driver interface 2377 * 2378 * This function is called on module load in order to register the driver. 2379 **/ 2380 int fm10k_register_pci_driver(void) 2381 { 2382 return pci_register_driver(&fm10k_driver); 2383 } 2384 2385 /** 2386 * fm10k_unregister_pci_driver - unregister driver interface 2387 * 2388 * This function is called on module unload in order to remove the driver. 2389 **/ 2390 void fm10k_unregister_pci_driver(void) 2391 { 2392 pci_unregister_driver(&fm10k_driver); 2393 } 2394