1 /* Intel Ethernet Switch Host Interface Driver 2 * Copyright(c) 2013 - 2015 Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * The full GNU General Public License is included in this distribution in 14 * the file called "COPYING". 15 * 16 * Contact Information: 17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 19 */ 20 21 #include <linux/module.h> 22 #include <linux/aer.h> 23 24 #include "fm10k.h" 25 26 static const struct fm10k_info *fm10k_info_tbl[] = { 27 [fm10k_device_pf] = &fm10k_pf_info, 28 [fm10k_device_vf] = &fm10k_vf_info, 29 }; 30 31 /** 32 * fm10k_pci_tbl - PCI Device ID Table 33 * 34 * Wildcard entries (PCI_ANY_ID) should come last 35 * Last entry must be all 0s 36 * 37 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 38 * Class, Class Mask, private data (not used) } 39 */ 40 static const struct pci_device_id fm10k_pci_tbl[] = { 41 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf }, 42 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf }, 43 /* required last entry */ 44 { 0, } 45 }; 46 MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl); 47 48 u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg) 49 { 50 struct fm10k_intfc *interface = hw->back; 51 u16 value = 0; 52 53 if (FM10K_REMOVED(hw->hw_addr)) 54 return ~value; 55 56 pci_read_config_word(interface->pdev, reg, &value); 57 if (value == 0xFFFF) 58 fm10k_write_flush(hw); 59 60 return value; 61 } 62 63 u32 fm10k_read_reg(struct fm10k_hw *hw, int reg) 64 { 65 u32 __iomem *hw_addr = ACCESS_ONCE(hw->hw_addr); 66 u32 value = 0; 67 68 if (FM10K_REMOVED(hw_addr)) 69 return ~value; 70 71 value = readl(&hw_addr[reg]); 72 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 73 struct fm10k_intfc *interface = hw->back; 74 struct net_device *netdev = interface->netdev; 75 76 hw->hw_addr = NULL; 77 netif_device_detach(netdev); 78 netdev_err(netdev, "PCIe link lost, device now detached\n"); 79 } 80 81 return value; 82 } 83 84 static int fm10k_hw_ready(struct fm10k_intfc *interface) 85 { 86 struct fm10k_hw *hw = &interface->hw; 87 88 fm10k_write_flush(hw); 89 90 return FM10K_REMOVED(hw->hw_addr) ? -ENODEV : 0; 91 } 92 93 void fm10k_service_event_schedule(struct fm10k_intfc *interface) 94 { 95 if (!test_bit(__FM10K_SERVICE_DISABLE, &interface->state) && 96 !test_and_set_bit(__FM10K_SERVICE_SCHED, &interface->state)) 97 queue_work(fm10k_workqueue, &interface->service_task); 98 } 99 100 static void fm10k_service_event_complete(struct fm10k_intfc *interface) 101 { 102 BUG_ON(!test_bit(__FM10K_SERVICE_SCHED, &interface->state)); 103 104 /* flush memory to make sure state is correct before next watchog */ 105 smp_mb__before_atomic(); 106 clear_bit(__FM10K_SERVICE_SCHED, &interface->state); 107 } 108 109 /** 110 * fm10k_service_timer - Timer Call-back 111 * @data: pointer to interface cast into an unsigned long 112 **/ 113 static void fm10k_service_timer(unsigned long data) 114 { 115 struct fm10k_intfc *interface = (struct fm10k_intfc *)data; 116 117 /* Reset the timer */ 118 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 119 120 fm10k_service_event_schedule(interface); 121 } 122 123 static void fm10k_detach_subtask(struct fm10k_intfc *interface) 124 { 125 struct net_device *netdev = interface->netdev; 126 127 /* do nothing if device is still present or hw_addr is set */ 128 if (netif_device_present(netdev) || interface->hw.hw_addr) 129 return; 130 131 rtnl_lock(); 132 133 if (netif_running(netdev)) 134 dev_close(netdev); 135 136 rtnl_unlock(); 137 } 138 139 static void fm10k_reinit(struct fm10k_intfc *interface) 140 { 141 struct net_device *netdev = interface->netdev; 142 struct fm10k_hw *hw = &interface->hw; 143 int err; 144 145 WARN_ON(in_interrupt()); 146 147 /* put off any impending NetWatchDogTimeout */ 148 netdev->trans_start = jiffies; 149 150 while (test_and_set_bit(__FM10K_RESETTING, &interface->state)) 151 usleep_range(1000, 2000); 152 153 rtnl_lock(); 154 155 fm10k_iov_suspend(interface->pdev); 156 157 if (netif_running(netdev)) 158 fm10k_close(netdev); 159 160 fm10k_mbx_free_irq(interface); 161 162 /* delay any future reset requests */ 163 interface->last_reset = jiffies + (10 * HZ); 164 165 /* reset and initialize the hardware so it is in a known state */ 166 err = hw->mac.ops.reset_hw(hw) ? : hw->mac.ops.init_hw(hw); 167 if (err) 168 dev_err(&interface->pdev->dev, "init_hw failed: %d\n", err); 169 170 /* reassociate interrupts */ 171 fm10k_mbx_request_irq(interface); 172 173 /* reset clock */ 174 fm10k_ts_reset(interface); 175 176 if (netif_running(netdev)) 177 fm10k_open(netdev); 178 179 fm10k_iov_resume(interface->pdev); 180 181 rtnl_unlock(); 182 183 clear_bit(__FM10K_RESETTING, &interface->state); 184 } 185 186 static void fm10k_reset_subtask(struct fm10k_intfc *interface) 187 { 188 if (!(interface->flags & FM10K_FLAG_RESET_REQUESTED)) 189 return; 190 191 interface->flags &= ~FM10K_FLAG_RESET_REQUESTED; 192 193 netdev_err(interface->netdev, "Reset interface\n"); 194 195 fm10k_reinit(interface); 196 } 197 198 /** 199 * fm10k_configure_swpri_map - Configure Receive SWPRI to PC mapping 200 * @interface: board private structure 201 * 202 * Configure the SWPRI to PC mapping for the port. 203 **/ 204 static void fm10k_configure_swpri_map(struct fm10k_intfc *interface) 205 { 206 struct net_device *netdev = interface->netdev; 207 struct fm10k_hw *hw = &interface->hw; 208 int i; 209 210 /* clear flag indicating update is needed */ 211 interface->flags &= ~FM10K_FLAG_SWPRI_CONFIG; 212 213 /* these registers are only available on the PF */ 214 if (hw->mac.type != fm10k_mac_pf) 215 return; 216 217 /* configure SWPRI to PC map */ 218 for (i = 0; i < FM10K_SWPRI_MAX; i++) 219 fm10k_write_reg(hw, FM10K_SWPRI_MAP(i), 220 netdev_get_prio_tc_map(netdev, i)); 221 } 222 223 /** 224 * fm10k_watchdog_update_host_state - Update the link status based on host. 225 * @interface: board private structure 226 **/ 227 static void fm10k_watchdog_update_host_state(struct fm10k_intfc *interface) 228 { 229 struct fm10k_hw *hw = &interface->hw; 230 s32 err; 231 232 if (test_bit(__FM10K_LINK_DOWN, &interface->state)) { 233 interface->host_ready = false; 234 if (time_is_after_jiffies(interface->link_down_event)) 235 return; 236 clear_bit(__FM10K_LINK_DOWN, &interface->state); 237 } 238 239 if (interface->flags & FM10K_FLAG_SWPRI_CONFIG) { 240 if (rtnl_trylock()) { 241 fm10k_configure_swpri_map(interface); 242 rtnl_unlock(); 243 } 244 } 245 246 /* lock the mailbox for transmit and receive */ 247 fm10k_mbx_lock(interface); 248 249 err = hw->mac.ops.get_host_state(hw, &interface->host_ready); 250 if (err && time_is_before_jiffies(interface->last_reset)) 251 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 252 253 /* free the lock */ 254 fm10k_mbx_unlock(interface); 255 } 256 257 /** 258 * fm10k_mbx_subtask - Process upstream and downstream mailboxes 259 * @interface: board private structure 260 * 261 * This function will process both the upstream and downstream mailboxes. 262 * It is necessary for us to hold the rtnl_lock while doing this as the 263 * mailbox accesses are protected by this lock. 264 **/ 265 static void fm10k_mbx_subtask(struct fm10k_intfc *interface) 266 { 267 /* process upstream mailbox and update device state */ 268 fm10k_watchdog_update_host_state(interface); 269 270 /* process downstream mailboxes */ 271 fm10k_iov_mbx(interface); 272 } 273 274 /** 275 * fm10k_watchdog_host_is_ready - Update netdev status based on host ready 276 * @interface: board private structure 277 **/ 278 static void fm10k_watchdog_host_is_ready(struct fm10k_intfc *interface) 279 { 280 struct net_device *netdev = interface->netdev; 281 282 /* only continue if link state is currently down */ 283 if (netif_carrier_ok(netdev)) 284 return; 285 286 netif_info(interface, drv, netdev, "NIC Link is up\n"); 287 288 netif_carrier_on(netdev); 289 netif_tx_wake_all_queues(netdev); 290 } 291 292 /** 293 * fm10k_watchdog_host_not_ready - Update netdev status based on host not ready 294 * @interface: board private structure 295 **/ 296 static void fm10k_watchdog_host_not_ready(struct fm10k_intfc *interface) 297 { 298 struct net_device *netdev = interface->netdev; 299 300 /* only continue if link state is currently up */ 301 if (!netif_carrier_ok(netdev)) 302 return; 303 304 netif_info(interface, drv, netdev, "NIC Link is down\n"); 305 306 netif_carrier_off(netdev); 307 netif_tx_stop_all_queues(netdev); 308 } 309 310 /** 311 * fm10k_update_stats - Update the board statistics counters. 312 * @interface: board private structure 313 **/ 314 void fm10k_update_stats(struct fm10k_intfc *interface) 315 { 316 struct net_device_stats *net_stats = &interface->netdev->stats; 317 struct fm10k_hw *hw = &interface->hw; 318 u64 rx_errors = 0, rx_csum_errors = 0, tx_csum_errors = 0; 319 u64 restart_queue = 0, tx_busy = 0, alloc_failed = 0; 320 u64 rx_bytes_nic = 0, rx_pkts_nic = 0, rx_drops_nic = 0; 321 u64 tx_bytes_nic = 0, tx_pkts_nic = 0; 322 u64 bytes, pkts; 323 int i; 324 325 /* do not allow stats update via service task for next second */ 326 interface->next_stats_update = jiffies + HZ; 327 328 /* gather some stats to the interface struct that are per queue */ 329 for (bytes = 0, pkts = 0, i = 0; i < interface->num_tx_queues; i++) { 330 struct fm10k_ring *tx_ring = interface->tx_ring[i]; 331 332 restart_queue += tx_ring->tx_stats.restart_queue; 333 tx_busy += tx_ring->tx_stats.tx_busy; 334 tx_csum_errors += tx_ring->tx_stats.csum_err; 335 bytes += tx_ring->stats.bytes; 336 pkts += tx_ring->stats.packets; 337 } 338 339 interface->restart_queue = restart_queue; 340 interface->tx_busy = tx_busy; 341 net_stats->tx_bytes = bytes; 342 net_stats->tx_packets = pkts; 343 interface->tx_csum_errors = tx_csum_errors; 344 /* gather some stats to the interface struct that are per queue */ 345 for (bytes = 0, pkts = 0, i = 0; i < interface->num_rx_queues; i++) { 346 struct fm10k_ring *rx_ring = interface->rx_ring[i]; 347 348 bytes += rx_ring->stats.bytes; 349 pkts += rx_ring->stats.packets; 350 alloc_failed += rx_ring->rx_stats.alloc_failed; 351 rx_csum_errors += rx_ring->rx_stats.csum_err; 352 rx_errors += rx_ring->rx_stats.errors; 353 } 354 355 net_stats->rx_bytes = bytes; 356 net_stats->rx_packets = pkts; 357 interface->alloc_failed = alloc_failed; 358 interface->rx_csum_errors = rx_csum_errors; 359 360 hw->mac.ops.update_hw_stats(hw, &interface->stats); 361 362 for (i = 0; i < hw->mac.max_queues; i++) { 363 struct fm10k_hw_stats_q *q = &interface->stats.q[i]; 364 365 tx_bytes_nic += q->tx_bytes.count; 366 tx_pkts_nic += q->tx_packets.count; 367 rx_bytes_nic += q->rx_bytes.count; 368 rx_pkts_nic += q->rx_packets.count; 369 rx_drops_nic += q->rx_drops.count; 370 } 371 372 interface->tx_bytes_nic = tx_bytes_nic; 373 interface->tx_packets_nic = tx_pkts_nic; 374 interface->rx_bytes_nic = rx_bytes_nic; 375 interface->rx_packets_nic = rx_pkts_nic; 376 interface->rx_drops_nic = rx_drops_nic; 377 378 /* Fill out the OS statistics structure */ 379 net_stats->rx_errors = rx_errors; 380 net_stats->rx_dropped = interface->stats.nodesc_drop.count; 381 } 382 383 /** 384 * fm10k_watchdog_flush_tx - flush queues on host not ready 385 * @interface - pointer to the device interface structure 386 **/ 387 static void fm10k_watchdog_flush_tx(struct fm10k_intfc *interface) 388 { 389 int some_tx_pending = 0; 390 int i; 391 392 /* nothing to do if carrier is up */ 393 if (netif_carrier_ok(interface->netdev)) 394 return; 395 396 for (i = 0; i < interface->num_tx_queues; i++) { 397 struct fm10k_ring *tx_ring = interface->tx_ring[i]; 398 399 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 400 some_tx_pending = 1; 401 break; 402 } 403 } 404 405 /* We've lost link, so the controller stops DMA, but we've got 406 * queued Tx work that's never going to get done, so reset 407 * controller to flush Tx. 408 */ 409 if (some_tx_pending) 410 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 411 } 412 413 /** 414 * fm10k_watchdog_subtask - check and bring link up 415 * @interface - pointer to the device interface structure 416 **/ 417 static void fm10k_watchdog_subtask(struct fm10k_intfc *interface) 418 { 419 /* if interface is down do nothing */ 420 if (test_bit(__FM10K_DOWN, &interface->state) || 421 test_bit(__FM10K_RESETTING, &interface->state)) 422 return; 423 424 if (interface->host_ready) 425 fm10k_watchdog_host_is_ready(interface); 426 else 427 fm10k_watchdog_host_not_ready(interface); 428 429 /* update stats only once every second */ 430 if (time_is_before_jiffies(interface->next_stats_update)) 431 fm10k_update_stats(interface); 432 433 /* flush any uncompleted work */ 434 fm10k_watchdog_flush_tx(interface); 435 } 436 437 /** 438 * fm10k_check_hang_subtask - check for hung queues and dropped interrupts 439 * @interface - pointer to the device interface structure 440 * 441 * This function serves two purposes. First it strobes the interrupt lines 442 * in order to make certain interrupts are occurring. Secondly it sets the 443 * bits needed to check for TX hangs. As a result we should immediately 444 * determine if a hang has occurred. 445 */ 446 static void fm10k_check_hang_subtask(struct fm10k_intfc *interface) 447 { 448 int i; 449 450 /* If we're down or resetting, just bail */ 451 if (test_bit(__FM10K_DOWN, &interface->state) || 452 test_bit(__FM10K_RESETTING, &interface->state)) 453 return; 454 455 /* rate limit tx hang checks to only once every 2 seconds */ 456 if (time_is_after_eq_jiffies(interface->next_tx_hang_check)) 457 return; 458 interface->next_tx_hang_check = jiffies + (2 * HZ); 459 460 if (netif_carrier_ok(interface->netdev)) { 461 /* Force detection of hung controller */ 462 for (i = 0; i < interface->num_tx_queues; i++) 463 set_check_for_tx_hang(interface->tx_ring[i]); 464 465 /* Rearm all in-use q_vectors for immediate firing */ 466 for (i = 0; i < interface->num_q_vectors; i++) { 467 struct fm10k_q_vector *qv = interface->q_vector[i]; 468 469 if (!qv->tx.count && !qv->rx.count) 470 continue; 471 writel(FM10K_ITR_ENABLE | FM10K_ITR_PENDING2, qv->itr); 472 } 473 } 474 } 475 476 /** 477 * fm10k_service_task - manages and runs subtasks 478 * @work: pointer to work_struct containing our data 479 **/ 480 static void fm10k_service_task(struct work_struct *work) 481 { 482 struct fm10k_intfc *interface; 483 484 interface = container_of(work, struct fm10k_intfc, service_task); 485 486 /* tasks always capable of running, but must be rtnl protected */ 487 fm10k_mbx_subtask(interface); 488 fm10k_detach_subtask(interface); 489 fm10k_reset_subtask(interface); 490 491 /* tasks only run when interface is up */ 492 fm10k_watchdog_subtask(interface); 493 fm10k_check_hang_subtask(interface); 494 fm10k_ts_tx_subtask(interface); 495 496 /* release lock on service events to allow scheduling next event */ 497 fm10k_service_event_complete(interface); 498 } 499 500 /** 501 * fm10k_configure_tx_ring - Configure Tx ring after Reset 502 * @interface: board private structure 503 * @ring: structure containing ring specific data 504 * 505 * Configure the Tx descriptor ring after a reset. 506 **/ 507 static void fm10k_configure_tx_ring(struct fm10k_intfc *interface, 508 struct fm10k_ring *ring) 509 { 510 struct fm10k_hw *hw = &interface->hw; 511 u64 tdba = ring->dma; 512 u32 size = ring->count * sizeof(struct fm10k_tx_desc); 513 u32 txint = FM10K_INT_MAP_DISABLE; 514 u32 txdctl = FM10K_TXDCTL_ENABLE | (1 << FM10K_TXDCTL_MAX_TIME_SHIFT); 515 u8 reg_idx = ring->reg_idx; 516 517 /* disable queue to avoid issues while updating state */ 518 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), 0); 519 fm10k_write_flush(hw); 520 521 /* possible poll here to verify ring resources have been cleaned */ 522 523 /* set location and size for descriptor ring */ 524 fm10k_write_reg(hw, FM10K_TDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 525 fm10k_write_reg(hw, FM10K_TDBAH(reg_idx), tdba >> 32); 526 fm10k_write_reg(hw, FM10K_TDLEN(reg_idx), size); 527 528 /* reset head and tail pointers */ 529 fm10k_write_reg(hw, FM10K_TDH(reg_idx), 0); 530 fm10k_write_reg(hw, FM10K_TDT(reg_idx), 0); 531 532 /* store tail pointer */ 533 ring->tail = &interface->uc_addr[FM10K_TDT(reg_idx)]; 534 535 /* reset ntu and ntc to place SW in sync with hardwdare */ 536 ring->next_to_clean = 0; 537 ring->next_to_use = 0; 538 539 /* Map interrupt */ 540 if (ring->q_vector) { 541 txint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 542 txint |= FM10K_INT_MAP_TIMER0; 543 } 544 545 fm10k_write_reg(hw, FM10K_TXINT(reg_idx), txint); 546 547 /* enable use of FTAG bit in Tx descriptor, register is RO for VF */ 548 fm10k_write_reg(hw, FM10K_PFVTCTL(reg_idx), 549 FM10K_PFVTCTL_FTAG_DESC_ENABLE); 550 551 /* enable queue */ 552 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), txdctl); 553 } 554 555 /** 556 * fm10k_enable_tx_ring - Verify Tx ring is enabled after configuration 557 * @interface: board private structure 558 * @ring: structure containing ring specific data 559 * 560 * Verify the Tx descriptor ring is ready for transmit. 561 **/ 562 static void fm10k_enable_tx_ring(struct fm10k_intfc *interface, 563 struct fm10k_ring *ring) 564 { 565 struct fm10k_hw *hw = &interface->hw; 566 int wait_loop = 10; 567 u32 txdctl; 568 u8 reg_idx = ring->reg_idx; 569 570 /* if we are already enabled just exit */ 571 if (fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)) & FM10K_TXDCTL_ENABLE) 572 return; 573 574 /* poll to verify queue is enabled */ 575 do { 576 usleep_range(1000, 2000); 577 txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)); 578 } while (!(txdctl & FM10K_TXDCTL_ENABLE) && --wait_loop); 579 if (!wait_loop) 580 netif_err(interface, drv, interface->netdev, 581 "Could not enable Tx Queue %d\n", reg_idx); 582 } 583 584 /** 585 * fm10k_configure_tx - Configure Transmit Unit after Reset 586 * @interface: board private structure 587 * 588 * Configure the Tx unit of the MAC after a reset. 589 **/ 590 static void fm10k_configure_tx(struct fm10k_intfc *interface) 591 { 592 int i; 593 594 /* Setup the HW Tx Head and Tail descriptor pointers */ 595 for (i = 0; i < interface->num_tx_queues; i++) 596 fm10k_configure_tx_ring(interface, interface->tx_ring[i]); 597 598 /* poll here to verify that Tx rings are now enabled */ 599 for (i = 0; i < interface->num_tx_queues; i++) 600 fm10k_enable_tx_ring(interface, interface->tx_ring[i]); 601 } 602 603 /** 604 * fm10k_configure_rx_ring - Configure Rx ring after Reset 605 * @interface: board private structure 606 * @ring: structure containing ring specific data 607 * 608 * Configure the Rx descriptor ring after a reset. 609 **/ 610 static void fm10k_configure_rx_ring(struct fm10k_intfc *interface, 611 struct fm10k_ring *ring) 612 { 613 u64 rdba = ring->dma; 614 struct fm10k_hw *hw = &interface->hw; 615 u32 size = ring->count * sizeof(union fm10k_rx_desc); 616 u32 rxqctl = FM10K_RXQCTL_ENABLE | FM10K_RXQCTL_PF; 617 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 618 u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN; 619 u32 rxint = FM10K_INT_MAP_DISABLE; 620 u8 rx_pause = interface->rx_pause; 621 u8 reg_idx = ring->reg_idx; 622 623 /* disable queue to avoid issues while updating state */ 624 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), 0); 625 fm10k_write_flush(hw); 626 627 /* possible poll here to verify ring resources have been cleaned */ 628 629 /* set location and size for descriptor ring */ 630 fm10k_write_reg(hw, FM10K_RDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 631 fm10k_write_reg(hw, FM10K_RDBAH(reg_idx), rdba >> 32); 632 fm10k_write_reg(hw, FM10K_RDLEN(reg_idx), size); 633 634 /* reset head and tail pointers */ 635 fm10k_write_reg(hw, FM10K_RDH(reg_idx), 0); 636 fm10k_write_reg(hw, FM10K_RDT(reg_idx), 0); 637 638 /* store tail pointer */ 639 ring->tail = &interface->uc_addr[FM10K_RDT(reg_idx)]; 640 641 /* reset ntu and ntc to place SW in sync with hardwdare */ 642 ring->next_to_clean = 0; 643 ring->next_to_use = 0; 644 ring->next_to_alloc = 0; 645 646 /* Configure the Rx buffer size for one buff without split */ 647 srrctl |= FM10K_RX_BUFSZ >> FM10K_SRRCTL_BSIZEPKT_SHIFT; 648 649 /* Configure the Rx ring to suppress loopback packets */ 650 srrctl |= FM10K_SRRCTL_LOOPBACK_SUPPRESS; 651 fm10k_write_reg(hw, FM10K_SRRCTL(reg_idx), srrctl); 652 653 /* Enable drop on empty */ 654 #ifdef CONFIG_DCB 655 if (interface->pfc_en) 656 rx_pause = interface->pfc_en; 657 #endif 658 if (!(rx_pause & (1 << ring->qos_pc))) 659 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 660 661 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 662 663 /* assign default VLAN to queue */ 664 ring->vid = hw->mac.default_vid; 665 666 /* Map interrupt */ 667 if (ring->q_vector) { 668 rxint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 669 rxint |= FM10K_INT_MAP_TIMER1; 670 } 671 672 fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint); 673 674 /* enable queue */ 675 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl); 676 677 /* place buffers on ring for receive data */ 678 fm10k_alloc_rx_buffers(ring, fm10k_desc_unused(ring)); 679 } 680 681 /** 682 * fm10k_update_rx_drop_en - Configures the drop enable bits for Rx rings 683 * @interface: board private structure 684 * 685 * Configure the drop enable bits for the Rx rings. 686 **/ 687 void fm10k_update_rx_drop_en(struct fm10k_intfc *interface) 688 { 689 struct fm10k_hw *hw = &interface->hw; 690 u8 rx_pause = interface->rx_pause; 691 int i; 692 693 #ifdef CONFIG_DCB 694 if (interface->pfc_en) 695 rx_pause = interface->pfc_en; 696 697 #endif 698 for (i = 0; i < interface->num_rx_queues; i++) { 699 struct fm10k_ring *ring = interface->rx_ring[i]; 700 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 701 u8 reg_idx = ring->reg_idx; 702 703 if (!(rx_pause & (1 << ring->qos_pc))) 704 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 705 706 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 707 } 708 } 709 710 /** 711 * fm10k_configure_dglort - Configure Receive DGLORT after reset 712 * @interface: board private structure 713 * 714 * Configure the DGLORT description and RSS tables. 715 **/ 716 static void fm10k_configure_dglort(struct fm10k_intfc *interface) 717 { 718 struct fm10k_dglort_cfg dglort = { 0 }; 719 struct fm10k_hw *hw = &interface->hw; 720 int i; 721 u32 mrqc; 722 723 /* Fill out hash function seeds */ 724 for (i = 0; i < FM10K_RSSRK_SIZE; i++) 725 fm10k_write_reg(hw, FM10K_RSSRK(0, i), interface->rssrk[i]); 726 727 /* Write RETA table to hardware */ 728 for (i = 0; i < FM10K_RETA_SIZE; i++) 729 fm10k_write_reg(hw, FM10K_RETA(0, i), interface->reta[i]); 730 731 /* Generate RSS hash based on packet types, TCP/UDP 732 * port numbers and/or IPv4/v6 src and dst addresses 733 */ 734 mrqc = FM10K_MRQC_IPV4 | 735 FM10K_MRQC_TCP_IPV4 | 736 FM10K_MRQC_IPV6 | 737 FM10K_MRQC_TCP_IPV6; 738 739 if (interface->flags & FM10K_FLAG_RSS_FIELD_IPV4_UDP) 740 mrqc |= FM10K_MRQC_UDP_IPV4; 741 if (interface->flags & FM10K_FLAG_RSS_FIELD_IPV6_UDP) 742 mrqc |= FM10K_MRQC_UDP_IPV6; 743 744 fm10k_write_reg(hw, FM10K_MRQC(0), mrqc); 745 746 /* configure default DGLORT mapping for RSS/DCB */ 747 dglort.inner_rss = 1; 748 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 749 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 750 hw->mac.ops.configure_dglort_map(hw, &dglort); 751 752 /* assign GLORT per queue for queue mapped testing */ 753 if (interface->glort_count > 64) { 754 memset(&dglort, 0, sizeof(dglort)); 755 dglort.inner_rss = 1; 756 dglort.glort = interface->glort + 64; 757 dglort.idx = fm10k_dglort_pf_queue; 758 dglort.queue_l = fls(interface->num_rx_queues - 1); 759 hw->mac.ops.configure_dglort_map(hw, &dglort); 760 } 761 762 /* assign glort value for RSS/DCB specific to this interface */ 763 memset(&dglort, 0, sizeof(dglort)); 764 dglort.inner_rss = 1; 765 dglort.glort = interface->glort; 766 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 767 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 768 /* configure DGLORT mapping for RSS/DCB */ 769 dglort.idx = fm10k_dglort_pf_rss; 770 if (interface->l2_accel) 771 dglort.shared_l = fls(interface->l2_accel->size); 772 hw->mac.ops.configure_dglort_map(hw, &dglort); 773 } 774 775 /** 776 * fm10k_configure_rx - Configure Receive Unit after Reset 777 * @interface: board private structure 778 * 779 * Configure the Rx unit of the MAC after a reset. 780 **/ 781 static void fm10k_configure_rx(struct fm10k_intfc *interface) 782 { 783 int i; 784 785 /* Configure SWPRI to PC map */ 786 fm10k_configure_swpri_map(interface); 787 788 /* Configure RSS and DGLORT map */ 789 fm10k_configure_dglort(interface); 790 791 /* Setup the HW Rx Head and Tail descriptor pointers */ 792 for (i = 0; i < interface->num_rx_queues; i++) 793 fm10k_configure_rx_ring(interface, interface->rx_ring[i]); 794 795 /* possible poll here to verify that Rx rings are now enabled */ 796 } 797 798 static void fm10k_napi_enable_all(struct fm10k_intfc *interface) 799 { 800 struct fm10k_q_vector *q_vector; 801 int q_idx; 802 803 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 804 q_vector = interface->q_vector[q_idx]; 805 napi_enable(&q_vector->napi); 806 } 807 } 808 809 static irqreturn_t fm10k_msix_clean_rings(int __always_unused irq, void *data) 810 { 811 struct fm10k_q_vector *q_vector = data; 812 813 if (q_vector->rx.count || q_vector->tx.count) 814 napi_schedule(&q_vector->napi); 815 816 return IRQ_HANDLED; 817 } 818 819 static irqreturn_t fm10k_msix_mbx_vf(int __always_unused irq, void *data) 820 { 821 struct fm10k_intfc *interface = data; 822 struct fm10k_hw *hw = &interface->hw; 823 struct fm10k_mbx_info *mbx = &hw->mbx; 824 825 /* re-enable mailbox interrupt and indicate 20us delay */ 826 fm10k_write_reg(hw, FM10K_VFITR(FM10K_MBX_VECTOR), 827 FM10K_ITR_ENABLE | FM10K_MBX_INT_DELAY); 828 829 /* service upstream mailbox */ 830 if (fm10k_mbx_trylock(interface)) { 831 mbx->ops.process(hw, mbx); 832 fm10k_mbx_unlock(interface); 833 } 834 835 hw->mac.get_host_state = 1; 836 fm10k_service_event_schedule(interface); 837 838 return IRQ_HANDLED; 839 } 840 841 #ifdef CONFIG_NET_POLL_CONTROLLER 842 /** 843 * fm10k_netpoll - A Polling 'interrupt' handler 844 * @netdev: network interface device structure 845 * 846 * This is used by netconsole to send skbs without having to re-enable 847 * interrupts. It's not called while the normal interrupt routine is executing. 848 **/ 849 void fm10k_netpoll(struct net_device *netdev) 850 { 851 struct fm10k_intfc *interface = netdev_priv(netdev); 852 int i; 853 854 /* if interface is down do nothing */ 855 if (test_bit(__FM10K_DOWN, &interface->state)) 856 return; 857 858 for (i = 0; i < interface->num_q_vectors; i++) 859 fm10k_msix_clean_rings(0, interface->q_vector[i]); 860 } 861 862 #endif 863 #define FM10K_ERR_MSG(type) case (type): error = #type; break 864 static void fm10k_print_fault(struct fm10k_intfc *interface, int type, 865 struct fm10k_fault *fault) 866 { 867 struct pci_dev *pdev = interface->pdev; 868 char *error; 869 870 switch (type) { 871 case FM10K_PCA_FAULT: 872 switch (fault->type) { 873 default: 874 error = "Unknown PCA error"; 875 break; 876 FM10K_ERR_MSG(PCA_NO_FAULT); 877 FM10K_ERR_MSG(PCA_UNMAPPED_ADDR); 878 FM10K_ERR_MSG(PCA_BAD_QACCESS_PF); 879 FM10K_ERR_MSG(PCA_BAD_QACCESS_VF); 880 FM10K_ERR_MSG(PCA_MALICIOUS_REQ); 881 FM10K_ERR_MSG(PCA_POISONED_TLP); 882 FM10K_ERR_MSG(PCA_TLP_ABORT); 883 } 884 break; 885 case FM10K_THI_FAULT: 886 switch (fault->type) { 887 default: 888 error = "Unknown THI error"; 889 break; 890 FM10K_ERR_MSG(THI_NO_FAULT); 891 FM10K_ERR_MSG(THI_MAL_DIS_Q_FAULT); 892 } 893 break; 894 case FM10K_FUM_FAULT: 895 switch (fault->type) { 896 default: 897 error = "Unknown FUM error"; 898 break; 899 FM10K_ERR_MSG(FUM_NO_FAULT); 900 FM10K_ERR_MSG(FUM_UNMAPPED_ADDR); 901 FM10K_ERR_MSG(FUM_BAD_VF_QACCESS); 902 FM10K_ERR_MSG(FUM_ADD_DECODE_ERR); 903 FM10K_ERR_MSG(FUM_RO_ERROR); 904 FM10K_ERR_MSG(FUM_QPRC_CRC_ERROR); 905 FM10K_ERR_MSG(FUM_CSR_TIMEOUT); 906 FM10K_ERR_MSG(FUM_INVALID_TYPE); 907 FM10K_ERR_MSG(FUM_INVALID_LENGTH); 908 FM10K_ERR_MSG(FUM_INVALID_BE); 909 FM10K_ERR_MSG(FUM_INVALID_ALIGN); 910 } 911 break; 912 default: 913 error = "Undocumented fault"; 914 break; 915 } 916 917 dev_warn(&pdev->dev, 918 "%s Address: 0x%llx SpecInfo: 0x%x Func: %02x.%0x\n", 919 error, fault->address, fault->specinfo, 920 PCI_SLOT(fault->func), PCI_FUNC(fault->func)); 921 } 922 923 static void fm10k_report_fault(struct fm10k_intfc *interface, u32 eicr) 924 { 925 struct fm10k_hw *hw = &interface->hw; 926 struct fm10k_fault fault = { 0 }; 927 int type, err; 928 929 for (eicr &= FM10K_EICR_FAULT_MASK, type = FM10K_PCA_FAULT; 930 eicr; 931 eicr >>= 1, type += FM10K_FAULT_SIZE) { 932 /* only check if there is an error reported */ 933 if (!(eicr & 0x1)) 934 continue; 935 936 /* retrieve fault info */ 937 err = hw->mac.ops.get_fault(hw, type, &fault); 938 if (err) { 939 dev_err(&interface->pdev->dev, 940 "error reading fault\n"); 941 continue; 942 } 943 944 fm10k_print_fault(interface, type, &fault); 945 } 946 } 947 948 static void fm10k_reset_drop_on_empty(struct fm10k_intfc *interface, u32 eicr) 949 { 950 struct fm10k_hw *hw = &interface->hw; 951 const u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 952 u32 maxholdq; 953 int q; 954 955 if (!(eicr & FM10K_EICR_MAXHOLDTIME)) 956 return; 957 958 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(7)); 959 if (maxholdq) 960 fm10k_write_reg(hw, FM10K_MAXHOLDQ(7), maxholdq); 961 for (q = 255;;) { 962 if (maxholdq & (1 << 31)) { 963 if (q < FM10K_MAX_QUEUES_PF) { 964 interface->rx_overrun_pf++; 965 fm10k_write_reg(hw, FM10K_RXDCTL(q), rxdctl); 966 } else { 967 interface->rx_overrun_vf++; 968 } 969 } 970 971 maxholdq *= 2; 972 if (!maxholdq) 973 q &= ~(32 - 1); 974 975 if (!q) 976 break; 977 978 if (q-- % 32) 979 continue; 980 981 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(q / 32)); 982 if (maxholdq) 983 fm10k_write_reg(hw, FM10K_MAXHOLDQ(q / 32), maxholdq); 984 } 985 } 986 987 static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data) 988 { 989 struct fm10k_intfc *interface = data; 990 struct fm10k_hw *hw = &interface->hw; 991 struct fm10k_mbx_info *mbx = &hw->mbx; 992 u32 eicr; 993 994 /* unmask any set bits related to this interrupt */ 995 eicr = fm10k_read_reg(hw, FM10K_EICR); 996 fm10k_write_reg(hw, FM10K_EICR, eicr & (FM10K_EICR_MAILBOX | 997 FM10K_EICR_SWITCHREADY | 998 FM10K_EICR_SWITCHNOTREADY)); 999 1000 /* report any faults found to the message log */ 1001 fm10k_report_fault(interface, eicr); 1002 1003 /* reset any queues disabled due to receiver overrun */ 1004 fm10k_reset_drop_on_empty(interface, eicr); 1005 1006 /* service mailboxes */ 1007 if (fm10k_mbx_trylock(interface)) { 1008 mbx->ops.process(hw, mbx); 1009 /* handle VFLRE events */ 1010 fm10k_iov_event(interface); 1011 fm10k_mbx_unlock(interface); 1012 } 1013 1014 /* if switch toggled state we should reset GLORTs */ 1015 if (eicr & FM10K_EICR_SWITCHNOTREADY) { 1016 /* force link down for at least 4 seconds */ 1017 interface->link_down_event = jiffies + (4 * HZ); 1018 set_bit(__FM10K_LINK_DOWN, &interface->state); 1019 1020 /* reset dglort_map back to no config */ 1021 hw->mac.dglort_map = FM10K_DGLORTMAP_NONE; 1022 } 1023 1024 /* we should validate host state after interrupt event */ 1025 hw->mac.get_host_state = 1; 1026 1027 /* validate host state, and handle VF mailboxes in the service task */ 1028 fm10k_service_event_schedule(interface); 1029 1030 /* re-enable mailbox interrupt and indicate 20us delay */ 1031 fm10k_write_reg(hw, FM10K_ITR(FM10K_MBX_VECTOR), 1032 FM10K_ITR_ENABLE | FM10K_MBX_INT_DELAY); 1033 1034 return IRQ_HANDLED; 1035 } 1036 1037 void fm10k_mbx_free_irq(struct fm10k_intfc *interface) 1038 { 1039 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1040 struct fm10k_hw *hw = &interface->hw; 1041 int itr_reg; 1042 1043 /* disconnect the mailbox */ 1044 hw->mbx.ops.disconnect(hw, &hw->mbx); 1045 1046 /* disable Mailbox cause */ 1047 if (hw->mac.type == fm10k_mac_pf) { 1048 fm10k_write_reg(hw, FM10K_EIMR, 1049 FM10K_EIMR_DISABLE(PCA_FAULT) | 1050 FM10K_EIMR_DISABLE(FUM_FAULT) | 1051 FM10K_EIMR_DISABLE(MAILBOX) | 1052 FM10K_EIMR_DISABLE(SWITCHREADY) | 1053 FM10K_EIMR_DISABLE(SWITCHNOTREADY) | 1054 FM10K_EIMR_DISABLE(SRAMERROR) | 1055 FM10K_EIMR_DISABLE(VFLR) | 1056 FM10K_EIMR_DISABLE(MAXHOLDTIME)); 1057 itr_reg = FM10K_ITR(FM10K_MBX_VECTOR); 1058 } else { 1059 itr_reg = FM10K_VFITR(FM10K_MBX_VECTOR); 1060 } 1061 1062 fm10k_write_reg(hw, itr_reg, FM10K_ITR_MASK_SET); 1063 1064 free_irq(entry->vector, interface); 1065 } 1066 1067 static s32 fm10k_mbx_mac_addr(struct fm10k_hw *hw, u32 **results, 1068 struct fm10k_mbx_info *mbx) 1069 { 1070 bool vlan_override = hw->mac.vlan_override; 1071 u16 default_vid = hw->mac.default_vid; 1072 struct fm10k_intfc *interface; 1073 s32 err; 1074 1075 err = fm10k_msg_mac_vlan_vf(hw, results, mbx); 1076 if (err) 1077 return err; 1078 1079 interface = container_of(hw, struct fm10k_intfc, hw); 1080 1081 /* MAC was changed so we need reset */ 1082 if (is_valid_ether_addr(hw->mac.perm_addr) && 1083 memcmp(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN)) 1084 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1085 1086 /* VLAN override was changed, or default VLAN changed */ 1087 if ((vlan_override != hw->mac.vlan_override) || 1088 (default_vid != hw->mac.default_vid)) 1089 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1090 1091 return 0; 1092 } 1093 1094 static s32 fm10k_1588_msg_vf(struct fm10k_hw *hw, u32 **results, 1095 struct fm10k_mbx_info __always_unused *mbx) 1096 { 1097 struct fm10k_intfc *interface; 1098 u64 timestamp; 1099 s32 err; 1100 1101 err = fm10k_tlv_attr_get_u64(results[FM10K_1588_MSG_TIMESTAMP], 1102 ×tamp); 1103 if (err) 1104 return err; 1105 1106 interface = container_of(hw, struct fm10k_intfc, hw); 1107 1108 fm10k_ts_tx_hwtstamp(interface, 0, timestamp); 1109 1110 return 0; 1111 } 1112 1113 /* generic error handler for mailbox issues */ 1114 static s32 fm10k_mbx_error(struct fm10k_hw *hw, u32 **results, 1115 struct fm10k_mbx_info __always_unused *mbx) 1116 { 1117 struct fm10k_intfc *interface; 1118 struct pci_dev *pdev; 1119 1120 interface = container_of(hw, struct fm10k_intfc, hw); 1121 pdev = interface->pdev; 1122 1123 dev_err(&pdev->dev, "Unknown message ID %u\n", 1124 **results & FM10K_TLV_ID_MASK); 1125 1126 return 0; 1127 } 1128 1129 static const struct fm10k_msg_data vf_mbx_data[] = { 1130 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), 1131 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_mbx_mac_addr), 1132 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf), 1133 FM10K_VF_MSG_1588_HANDLER(fm10k_1588_msg_vf), 1134 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1135 }; 1136 1137 static int fm10k_mbx_request_irq_vf(struct fm10k_intfc *interface) 1138 { 1139 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1140 struct net_device *dev = interface->netdev; 1141 struct fm10k_hw *hw = &interface->hw; 1142 int err; 1143 1144 /* Use timer0 for interrupt moderation on the mailbox */ 1145 u32 itr = FM10K_INT_MAP_TIMER0 | entry->entry; 1146 1147 /* register mailbox handlers */ 1148 err = hw->mbx.ops.register_handlers(&hw->mbx, vf_mbx_data); 1149 if (err) 1150 return err; 1151 1152 /* request the IRQ */ 1153 err = request_irq(entry->vector, fm10k_msix_mbx_vf, 0, 1154 dev->name, interface); 1155 if (err) { 1156 netif_err(interface, probe, dev, 1157 "request_irq for msix_mbx failed: %d\n", err); 1158 return err; 1159 } 1160 1161 /* map all of the interrupt sources */ 1162 fm10k_write_reg(hw, FM10K_VFINT_MAP, itr); 1163 1164 /* enable interrupt */ 1165 fm10k_write_reg(hw, FM10K_VFITR(entry->entry), FM10K_ITR_ENABLE); 1166 1167 return 0; 1168 } 1169 1170 static s32 fm10k_lport_map(struct fm10k_hw *hw, u32 **results, 1171 struct fm10k_mbx_info *mbx) 1172 { 1173 struct fm10k_intfc *interface; 1174 u32 dglort_map = hw->mac.dglort_map; 1175 s32 err; 1176 1177 err = fm10k_msg_lport_map_pf(hw, results, mbx); 1178 if (err) 1179 return err; 1180 1181 interface = container_of(hw, struct fm10k_intfc, hw); 1182 1183 /* we need to reset if port count was just updated */ 1184 if (dglort_map != hw->mac.dglort_map) 1185 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1186 1187 return 0; 1188 } 1189 1190 static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results, 1191 struct fm10k_mbx_info __always_unused *mbx) 1192 { 1193 struct fm10k_intfc *interface; 1194 u16 glort, pvid; 1195 u32 pvid_update; 1196 s32 err; 1197 1198 err = fm10k_tlv_attr_get_u32(results[FM10K_PF_ATTR_ID_UPDATE_PVID], 1199 &pvid_update); 1200 if (err) 1201 return err; 1202 1203 /* extract values from the pvid update */ 1204 glort = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_GLORT); 1205 pvid = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_PVID); 1206 1207 /* if glort is not valid return error */ 1208 if (!fm10k_glort_valid_pf(hw, glort)) 1209 return FM10K_ERR_PARAM; 1210 1211 /* verify VID is valid */ 1212 if (pvid >= FM10K_VLAN_TABLE_VID_MAX) 1213 return FM10K_ERR_PARAM; 1214 1215 interface = container_of(hw, struct fm10k_intfc, hw); 1216 1217 /* check to see if this belongs to one of the VFs */ 1218 err = fm10k_iov_update_pvid(interface, glort, pvid); 1219 if (!err) 1220 return 0; 1221 1222 /* we need to reset if default VLAN was just updated */ 1223 if (pvid != hw->mac.default_vid) 1224 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 1225 1226 hw->mac.default_vid = pvid; 1227 1228 return 0; 1229 } 1230 1231 static s32 fm10k_1588_msg_pf(struct fm10k_hw *hw, u32 **results, 1232 struct fm10k_mbx_info __always_unused *mbx) 1233 { 1234 struct fm10k_swapi_1588_timestamp timestamp; 1235 struct fm10k_iov_data *iov_data; 1236 struct fm10k_intfc *interface; 1237 u16 sglort, vf_idx; 1238 s32 err; 1239 1240 err = fm10k_tlv_attr_get_le_struct( 1241 results[FM10K_PF_ATTR_ID_1588_TIMESTAMP], 1242 ×tamp, sizeof(timestamp)); 1243 if (err) 1244 return err; 1245 1246 interface = container_of(hw, struct fm10k_intfc, hw); 1247 1248 if (timestamp.dglort) { 1249 fm10k_ts_tx_hwtstamp(interface, timestamp.dglort, 1250 le64_to_cpu(timestamp.egress)); 1251 return 0; 1252 } 1253 1254 /* either dglort or sglort must be set */ 1255 if (!timestamp.sglort) 1256 return FM10K_ERR_PARAM; 1257 1258 /* verify GLORT is at least one of the ones we own */ 1259 sglort = le16_to_cpu(timestamp.sglort); 1260 if (!fm10k_glort_valid_pf(hw, sglort)) 1261 return FM10K_ERR_PARAM; 1262 1263 if (sglort == interface->glort) { 1264 fm10k_ts_tx_hwtstamp(interface, 0, 1265 le64_to_cpu(timestamp.ingress)); 1266 return 0; 1267 } 1268 1269 /* if there is no iov_data then there is no mailboxes to process */ 1270 if (!ACCESS_ONCE(interface->iov_data)) 1271 return FM10K_ERR_PARAM; 1272 1273 rcu_read_lock(); 1274 1275 /* notify VF if this timestamp belongs to it */ 1276 iov_data = interface->iov_data; 1277 vf_idx = (hw->mac.dglort_map & FM10K_DGLORTMAP_NONE) - sglort; 1278 1279 if (!iov_data || vf_idx >= iov_data->num_vfs) { 1280 err = FM10K_ERR_PARAM; 1281 goto err_unlock; 1282 } 1283 1284 err = hw->iov.ops.report_timestamp(hw, &iov_data->vf_info[vf_idx], 1285 le64_to_cpu(timestamp.ingress)); 1286 1287 err_unlock: 1288 rcu_read_unlock(); 1289 1290 return err; 1291 } 1292 1293 static const struct fm10k_msg_data pf_mbx_data[] = { 1294 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf), 1295 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf), 1296 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_lport_map), 1297 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf), 1298 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf), 1299 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_update_pvid), 1300 FM10K_PF_MSG_1588_TIMESTAMP_HANDLER(fm10k_1588_msg_pf), 1301 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1302 }; 1303 1304 static int fm10k_mbx_request_irq_pf(struct fm10k_intfc *interface) 1305 { 1306 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1307 struct net_device *dev = interface->netdev; 1308 struct fm10k_hw *hw = &interface->hw; 1309 int err; 1310 1311 /* Use timer0 for interrupt moderation on the mailbox */ 1312 u32 mbx_itr = FM10K_INT_MAP_TIMER0 | entry->entry; 1313 u32 other_itr = FM10K_INT_MAP_IMMEDIATE | entry->entry; 1314 1315 /* register mailbox handlers */ 1316 err = hw->mbx.ops.register_handlers(&hw->mbx, pf_mbx_data); 1317 if (err) 1318 return err; 1319 1320 /* request the IRQ */ 1321 err = request_irq(entry->vector, fm10k_msix_mbx_pf, 0, 1322 dev->name, interface); 1323 if (err) { 1324 netif_err(interface, probe, dev, 1325 "request_irq for msix_mbx failed: %d\n", err); 1326 return err; 1327 } 1328 1329 /* Enable interrupts w/ no moderation for "other" interrupts */ 1330 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), other_itr); 1331 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), other_itr); 1332 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_SRAM), other_itr); 1333 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_MaxHoldTime), other_itr); 1334 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_VFLR), other_itr); 1335 1336 /* Enable interrupts w/ moderation for mailbox */ 1337 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_Mailbox), mbx_itr); 1338 1339 /* Enable individual interrupt causes */ 1340 fm10k_write_reg(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | 1341 FM10K_EIMR_ENABLE(FUM_FAULT) | 1342 FM10K_EIMR_ENABLE(MAILBOX) | 1343 FM10K_EIMR_ENABLE(SWITCHREADY) | 1344 FM10K_EIMR_ENABLE(SWITCHNOTREADY) | 1345 FM10K_EIMR_ENABLE(SRAMERROR) | 1346 FM10K_EIMR_ENABLE(VFLR) | 1347 FM10K_EIMR_ENABLE(MAXHOLDTIME)); 1348 1349 /* enable interrupt */ 1350 fm10k_write_reg(hw, FM10K_ITR(entry->entry), FM10K_ITR_ENABLE); 1351 1352 return 0; 1353 } 1354 1355 int fm10k_mbx_request_irq(struct fm10k_intfc *interface) 1356 { 1357 struct fm10k_hw *hw = &interface->hw; 1358 int err; 1359 1360 /* enable Mailbox cause */ 1361 if (hw->mac.type == fm10k_mac_pf) 1362 err = fm10k_mbx_request_irq_pf(interface); 1363 else 1364 err = fm10k_mbx_request_irq_vf(interface); 1365 1366 /* connect mailbox */ 1367 if (!err) 1368 err = hw->mbx.ops.connect(hw, &hw->mbx); 1369 1370 return err; 1371 } 1372 1373 /** 1374 * fm10k_qv_free_irq - release interrupts associated with queue vectors 1375 * @interface: board private structure 1376 * 1377 * Release all interrupts associated with this interface 1378 **/ 1379 void fm10k_qv_free_irq(struct fm10k_intfc *interface) 1380 { 1381 int vector = interface->num_q_vectors; 1382 struct fm10k_hw *hw = &interface->hw; 1383 struct msix_entry *entry; 1384 1385 entry = &interface->msix_entries[NON_Q_VECTORS(hw) + vector]; 1386 1387 while (vector) { 1388 struct fm10k_q_vector *q_vector; 1389 1390 vector--; 1391 entry--; 1392 q_vector = interface->q_vector[vector]; 1393 1394 if (!q_vector->tx.count && !q_vector->rx.count) 1395 continue; 1396 1397 /* disable interrupts */ 1398 1399 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1400 1401 free_irq(entry->vector, q_vector); 1402 } 1403 } 1404 1405 /** 1406 * fm10k_qv_request_irq - initialize interrupts for queue vectors 1407 * @interface: board private structure 1408 * 1409 * Attempts to configure interrupts using the best available 1410 * capabilities of the hardware and kernel. 1411 **/ 1412 int fm10k_qv_request_irq(struct fm10k_intfc *interface) 1413 { 1414 struct net_device *dev = interface->netdev; 1415 struct fm10k_hw *hw = &interface->hw; 1416 struct msix_entry *entry; 1417 int ri = 0, ti = 0; 1418 int vector, err; 1419 1420 entry = &interface->msix_entries[NON_Q_VECTORS(hw)]; 1421 1422 for (vector = 0; vector < interface->num_q_vectors; vector++) { 1423 struct fm10k_q_vector *q_vector = interface->q_vector[vector]; 1424 1425 /* name the vector */ 1426 if (q_vector->tx.count && q_vector->rx.count) { 1427 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1428 "%s-TxRx-%d", dev->name, ri++); 1429 ti++; 1430 } else if (q_vector->rx.count) { 1431 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1432 "%s-rx-%d", dev->name, ri++); 1433 } else if (q_vector->tx.count) { 1434 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1435 "%s-tx-%d", dev->name, ti++); 1436 } else { 1437 /* skip this unused q_vector */ 1438 continue; 1439 } 1440 1441 /* Assign ITR register to q_vector */ 1442 q_vector->itr = (hw->mac.type == fm10k_mac_pf) ? 1443 &interface->uc_addr[FM10K_ITR(entry->entry)] : 1444 &interface->uc_addr[FM10K_VFITR(entry->entry)]; 1445 1446 /* request the IRQ */ 1447 err = request_irq(entry->vector, &fm10k_msix_clean_rings, 0, 1448 q_vector->name, q_vector); 1449 if (err) { 1450 netif_err(interface, probe, dev, 1451 "request_irq failed for MSIX interrupt Error: %d\n", 1452 err); 1453 goto err_out; 1454 } 1455 1456 /* Enable q_vector */ 1457 writel(FM10K_ITR_ENABLE, q_vector->itr); 1458 1459 entry++; 1460 } 1461 1462 return 0; 1463 1464 err_out: 1465 /* wind through the ring freeing all entries and vectors */ 1466 while (vector) { 1467 struct fm10k_q_vector *q_vector; 1468 1469 entry--; 1470 vector--; 1471 q_vector = interface->q_vector[vector]; 1472 1473 if (!q_vector->tx.count && !q_vector->rx.count) 1474 continue; 1475 1476 /* disable interrupts */ 1477 1478 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1479 1480 free_irq(entry->vector, q_vector); 1481 } 1482 1483 return err; 1484 } 1485 1486 void fm10k_up(struct fm10k_intfc *interface) 1487 { 1488 struct fm10k_hw *hw = &interface->hw; 1489 1490 /* Enable Tx/Rx DMA */ 1491 hw->mac.ops.start_hw(hw); 1492 1493 /* configure Tx descriptor rings */ 1494 fm10k_configure_tx(interface); 1495 1496 /* configure Rx descriptor rings */ 1497 fm10k_configure_rx(interface); 1498 1499 /* configure interrupts */ 1500 hw->mac.ops.update_int_moderator(hw); 1501 1502 /* clear down bit to indicate we are ready to go */ 1503 clear_bit(__FM10K_DOWN, &interface->state); 1504 1505 /* enable polling cleanups */ 1506 fm10k_napi_enable_all(interface); 1507 1508 /* re-establish Rx filters */ 1509 fm10k_restore_rx_state(interface); 1510 1511 /* enable transmits */ 1512 netif_tx_start_all_queues(interface->netdev); 1513 1514 /* kick off the service timer now */ 1515 hw->mac.get_host_state = 1; 1516 mod_timer(&interface->service_timer, jiffies); 1517 } 1518 1519 static void fm10k_napi_disable_all(struct fm10k_intfc *interface) 1520 { 1521 struct fm10k_q_vector *q_vector; 1522 int q_idx; 1523 1524 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 1525 q_vector = interface->q_vector[q_idx]; 1526 napi_disable(&q_vector->napi); 1527 } 1528 } 1529 1530 void fm10k_down(struct fm10k_intfc *interface) 1531 { 1532 struct net_device *netdev = interface->netdev; 1533 struct fm10k_hw *hw = &interface->hw; 1534 1535 /* signal that we are down to the interrupt handler and service task */ 1536 set_bit(__FM10K_DOWN, &interface->state); 1537 1538 /* call carrier off first to avoid false dev_watchdog timeouts */ 1539 netif_carrier_off(netdev); 1540 1541 /* disable transmits */ 1542 netif_tx_stop_all_queues(netdev); 1543 netif_tx_disable(netdev); 1544 1545 /* reset Rx filters */ 1546 fm10k_reset_rx_state(interface); 1547 1548 /* allow 10ms for device to quiesce */ 1549 usleep_range(10000, 20000); 1550 1551 /* disable polling routines */ 1552 fm10k_napi_disable_all(interface); 1553 1554 /* capture stats one last time before stopping interface */ 1555 fm10k_update_stats(interface); 1556 1557 /* Disable DMA engine for Tx/Rx */ 1558 hw->mac.ops.stop_hw(hw); 1559 1560 /* free any buffers still on the rings */ 1561 fm10k_clean_all_tx_rings(interface); 1562 fm10k_clean_all_rx_rings(interface); 1563 } 1564 1565 /** 1566 * fm10k_sw_init - Initialize general software structures 1567 * @interface: host interface private structure to initialize 1568 * 1569 * fm10k_sw_init initializes the interface private data structure. 1570 * Fields are initialized based on PCI device information and 1571 * OS network device settings (MTU size). 1572 **/ 1573 static int fm10k_sw_init(struct fm10k_intfc *interface, 1574 const struct pci_device_id *ent) 1575 { 1576 const struct fm10k_info *fi = fm10k_info_tbl[ent->driver_data]; 1577 struct fm10k_hw *hw = &interface->hw; 1578 struct pci_dev *pdev = interface->pdev; 1579 struct net_device *netdev = interface->netdev; 1580 u32 rss_key[FM10K_RSSRK_SIZE]; 1581 unsigned int rss; 1582 int err; 1583 1584 /* initialize back pointer */ 1585 hw->back = interface; 1586 hw->hw_addr = interface->uc_addr; 1587 1588 /* PCI config space info */ 1589 hw->vendor_id = pdev->vendor; 1590 hw->device_id = pdev->device; 1591 hw->revision_id = pdev->revision; 1592 hw->subsystem_vendor_id = pdev->subsystem_vendor; 1593 hw->subsystem_device_id = pdev->subsystem_device; 1594 1595 /* Setup hw api */ 1596 memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops)); 1597 hw->mac.type = fi->mac; 1598 1599 /* Setup IOV handlers */ 1600 if (fi->iov_ops) 1601 memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops)); 1602 1603 /* Set common capability flags and settings */ 1604 rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus()); 1605 interface->ring_feature[RING_F_RSS].limit = rss; 1606 fi->get_invariants(hw); 1607 1608 /* pick up the PCIe bus settings for reporting later */ 1609 if (hw->mac.ops.get_bus_info) 1610 hw->mac.ops.get_bus_info(hw); 1611 1612 /* limit the usable DMA range */ 1613 if (hw->mac.ops.set_dma_mask) 1614 hw->mac.ops.set_dma_mask(hw, dma_get_mask(&pdev->dev)); 1615 1616 /* update netdev with DMA restrictions */ 1617 if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) { 1618 netdev->features |= NETIF_F_HIGHDMA; 1619 netdev->vlan_features |= NETIF_F_HIGHDMA; 1620 } 1621 1622 /* delay any future reset requests */ 1623 interface->last_reset = jiffies + (10 * HZ); 1624 1625 /* reset and initialize the hardware so it is in a known state */ 1626 err = hw->mac.ops.reset_hw(hw) ? : hw->mac.ops.init_hw(hw); 1627 if (err) { 1628 dev_err(&pdev->dev, "init_hw failed: %d\n", err); 1629 return err; 1630 } 1631 1632 /* initialize hardware statistics */ 1633 hw->mac.ops.update_hw_stats(hw, &interface->stats); 1634 1635 /* Set upper limit on IOV VFs that can be allocated */ 1636 pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs); 1637 1638 /* Start with random Ethernet address */ 1639 eth_random_addr(hw->mac.addr); 1640 1641 /* Initialize MAC address from hardware */ 1642 err = hw->mac.ops.read_mac_addr(hw); 1643 if (err) { 1644 dev_warn(&pdev->dev, 1645 "Failed to obtain MAC address defaulting to random\n"); 1646 /* tag address assignment as random */ 1647 netdev->addr_assign_type |= NET_ADDR_RANDOM; 1648 } 1649 1650 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); 1651 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len); 1652 1653 if (!is_valid_ether_addr(netdev->perm_addr)) { 1654 dev_err(&pdev->dev, "Invalid MAC Address\n"); 1655 return -EIO; 1656 } 1657 1658 /* assign BAR 4 resources for use with PTP */ 1659 if (fm10k_read_reg(hw, FM10K_CTRL) & FM10K_CTRL_BAR4_ALLOWED) 1660 interface->sw_addr = ioremap(pci_resource_start(pdev, 4), 1661 pci_resource_len(pdev, 4)); 1662 hw->sw_addr = interface->sw_addr; 1663 1664 /* Only the PF can support VXLAN and NVGRE offloads */ 1665 if (hw->mac.type != fm10k_mac_pf) { 1666 netdev->hw_enc_features = 0; 1667 netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; 1668 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 1669 } 1670 1671 /* initialize DCBNL interface */ 1672 fm10k_dcbnl_set_ops(netdev); 1673 1674 /* Initialize service timer and service task */ 1675 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 1676 setup_timer(&interface->service_timer, &fm10k_service_timer, 1677 (unsigned long)interface); 1678 INIT_WORK(&interface->service_task, fm10k_service_task); 1679 1680 /* kick off service timer now, even when interface is down */ 1681 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 1682 1683 /* Intitialize timestamp data */ 1684 fm10k_ts_init(interface); 1685 1686 /* set default ring sizes */ 1687 interface->tx_ring_count = FM10K_DEFAULT_TXD; 1688 interface->rx_ring_count = FM10K_DEFAULT_RXD; 1689 1690 /* set default interrupt moderation */ 1691 interface->tx_itr = FM10K_ITR_10K; 1692 interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_ITR_20K; 1693 1694 /* initialize vxlan_port list */ 1695 INIT_LIST_HEAD(&interface->vxlan_port); 1696 1697 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 1698 memcpy(interface->rssrk, rss_key, sizeof(rss_key)); 1699 1700 /* Start off interface as being down */ 1701 set_bit(__FM10K_DOWN, &interface->state); 1702 1703 return 0; 1704 } 1705 1706 static void fm10k_slot_warn(struct fm10k_intfc *interface) 1707 { 1708 struct device *dev = &interface->pdev->dev; 1709 struct fm10k_hw *hw = &interface->hw; 1710 1711 if (hw->mac.ops.is_slot_appropriate(hw)) 1712 return; 1713 1714 dev_warn(dev, 1715 "For optimal performance, a %s %s slot is recommended.\n", 1716 (hw->bus_caps.width == fm10k_bus_width_pcie_x1 ? "x1" : 1717 hw->bus_caps.width == fm10k_bus_width_pcie_x4 ? "x4" : 1718 "x8"), 1719 (hw->bus_caps.speed == fm10k_bus_speed_2500 ? "2.5GT/s" : 1720 hw->bus_caps.speed == fm10k_bus_speed_5000 ? "5.0GT/s" : 1721 "8.0GT/s")); 1722 dev_warn(dev, 1723 "A slot with more lanes and/or higher speed is suggested.\n"); 1724 } 1725 1726 /** 1727 * fm10k_probe - Device Initialization Routine 1728 * @pdev: PCI device information struct 1729 * @ent: entry in fm10k_pci_tbl 1730 * 1731 * Returns 0 on success, negative on failure 1732 * 1733 * fm10k_probe initializes an interface identified by a pci_dev structure. 1734 * The OS initialization, configuring of the interface private structure, 1735 * and a hardware reset occur. 1736 **/ 1737 static int fm10k_probe(struct pci_dev *pdev, 1738 const struct pci_device_id *ent) 1739 { 1740 struct net_device *netdev; 1741 struct fm10k_intfc *interface; 1742 struct fm10k_hw *hw; 1743 int err; 1744 1745 err = pci_enable_device_mem(pdev); 1746 if (err) 1747 return err; 1748 1749 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); 1750 if (err) 1751 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1752 if (err) { 1753 dev_err(&pdev->dev, 1754 "DMA configuration failed: %d\n", err); 1755 goto err_dma; 1756 } 1757 1758 err = pci_request_selected_regions(pdev, 1759 pci_select_bars(pdev, 1760 IORESOURCE_MEM), 1761 fm10k_driver_name); 1762 if (err) { 1763 dev_err(&pdev->dev, 1764 "pci_request_selected_regions failed: %d\n", err); 1765 goto err_pci_reg; 1766 } 1767 1768 pci_enable_pcie_error_reporting(pdev); 1769 1770 pci_set_master(pdev); 1771 pci_save_state(pdev); 1772 1773 netdev = fm10k_alloc_netdev(); 1774 if (!netdev) { 1775 err = -ENOMEM; 1776 goto err_alloc_netdev; 1777 } 1778 1779 SET_NETDEV_DEV(netdev, &pdev->dev); 1780 1781 interface = netdev_priv(netdev); 1782 pci_set_drvdata(pdev, interface); 1783 1784 interface->netdev = netdev; 1785 interface->pdev = pdev; 1786 hw = &interface->hw; 1787 1788 interface->uc_addr = ioremap(pci_resource_start(pdev, 0), 1789 FM10K_UC_ADDR_SIZE); 1790 if (!interface->uc_addr) { 1791 err = -EIO; 1792 goto err_ioremap; 1793 } 1794 1795 err = fm10k_sw_init(interface, ent); 1796 if (err) 1797 goto err_sw_init; 1798 1799 /* enable debugfs support */ 1800 fm10k_dbg_intfc_init(interface); 1801 1802 err = fm10k_init_queueing_scheme(interface); 1803 if (err) 1804 goto err_sw_init; 1805 1806 err = fm10k_mbx_request_irq(interface); 1807 if (err) 1808 goto err_mbx_interrupt; 1809 1810 /* final check of hardware state before registering the interface */ 1811 err = fm10k_hw_ready(interface); 1812 if (err) 1813 goto err_register; 1814 1815 err = register_netdev(netdev); 1816 if (err) 1817 goto err_register; 1818 1819 /* carrier off reporting is important to ethtool even BEFORE open */ 1820 netif_carrier_off(netdev); 1821 1822 /* stop all the transmit queues from transmitting until link is up */ 1823 netif_tx_stop_all_queues(netdev); 1824 1825 /* Register PTP interface */ 1826 fm10k_ptp_register(interface); 1827 1828 /* print bus type/speed/width info */ 1829 dev_info(&pdev->dev, "(PCI Express:%s Width: %s Payload: %s)\n", 1830 (hw->bus.speed == fm10k_bus_speed_8000 ? "8.0GT/s" : 1831 hw->bus.speed == fm10k_bus_speed_5000 ? "5.0GT/s" : 1832 hw->bus.speed == fm10k_bus_speed_2500 ? "2.5GT/s" : 1833 "Unknown"), 1834 (hw->bus.width == fm10k_bus_width_pcie_x8 ? "x8" : 1835 hw->bus.width == fm10k_bus_width_pcie_x4 ? "x4" : 1836 hw->bus.width == fm10k_bus_width_pcie_x1 ? "x1" : 1837 "Unknown"), 1838 (hw->bus.payload == fm10k_bus_payload_128 ? "128B" : 1839 hw->bus.payload == fm10k_bus_payload_256 ? "256B" : 1840 hw->bus.payload == fm10k_bus_payload_512 ? "512B" : 1841 "Unknown")); 1842 1843 /* print warning for non-optimal configurations */ 1844 fm10k_slot_warn(interface); 1845 1846 /* enable SR-IOV after registering netdev to enforce PF/VF ordering */ 1847 fm10k_iov_configure(pdev, 0); 1848 1849 /* clear the service task disable bit to allow service task to start */ 1850 clear_bit(__FM10K_SERVICE_DISABLE, &interface->state); 1851 1852 return 0; 1853 1854 err_register: 1855 fm10k_mbx_free_irq(interface); 1856 err_mbx_interrupt: 1857 fm10k_clear_queueing_scheme(interface); 1858 err_sw_init: 1859 if (interface->sw_addr) 1860 iounmap(interface->sw_addr); 1861 iounmap(interface->uc_addr); 1862 err_ioremap: 1863 free_netdev(netdev); 1864 err_alloc_netdev: 1865 pci_release_selected_regions(pdev, 1866 pci_select_bars(pdev, IORESOURCE_MEM)); 1867 err_pci_reg: 1868 err_dma: 1869 pci_disable_device(pdev); 1870 return err; 1871 } 1872 1873 /** 1874 * fm10k_remove - Device Removal Routine 1875 * @pdev: PCI device information struct 1876 * 1877 * fm10k_remove is called by the PCI subsystem to alert the driver 1878 * that it should release a PCI device. The could be caused by a 1879 * Hot-Plug event, or because the driver is going to be removed from 1880 * memory. 1881 **/ 1882 static void fm10k_remove(struct pci_dev *pdev) 1883 { 1884 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 1885 struct net_device *netdev = interface->netdev; 1886 1887 del_timer_sync(&interface->service_timer); 1888 1889 set_bit(__FM10K_SERVICE_DISABLE, &interface->state); 1890 cancel_work_sync(&interface->service_task); 1891 1892 /* free netdev, this may bounce the interrupts due to setup_tc */ 1893 if (netdev->reg_state == NETREG_REGISTERED) 1894 unregister_netdev(netdev); 1895 1896 /* cleanup timestamp handling */ 1897 fm10k_ptp_unregister(interface); 1898 1899 /* release VFs */ 1900 fm10k_iov_disable(pdev); 1901 1902 /* disable mailbox interrupt */ 1903 fm10k_mbx_free_irq(interface); 1904 1905 /* free interrupts */ 1906 fm10k_clear_queueing_scheme(interface); 1907 1908 /* remove any debugfs interfaces */ 1909 fm10k_dbg_intfc_exit(interface); 1910 1911 if (interface->sw_addr) 1912 iounmap(interface->sw_addr); 1913 iounmap(interface->uc_addr); 1914 1915 free_netdev(netdev); 1916 1917 pci_release_selected_regions(pdev, 1918 pci_select_bars(pdev, IORESOURCE_MEM)); 1919 1920 pci_disable_pcie_error_reporting(pdev); 1921 1922 pci_disable_device(pdev); 1923 } 1924 1925 #ifdef CONFIG_PM 1926 /** 1927 * fm10k_resume - Restore device to pre-sleep state 1928 * @pdev: PCI device information struct 1929 * 1930 * fm10k_resume is called after the system has powered back up from a sleep 1931 * state and is ready to resume operation. This function is meant to restore 1932 * the device back to its pre-sleep state. 1933 **/ 1934 static int fm10k_resume(struct pci_dev *pdev) 1935 { 1936 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 1937 struct net_device *netdev = interface->netdev; 1938 struct fm10k_hw *hw = &interface->hw; 1939 u32 err; 1940 1941 pci_set_power_state(pdev, PCI_D0); 1942 pci_restore_state(pdev); 1943 1944 /* pci_restore_state clears dev->state_saved so call 1945 * pci_save_state to restore it. 1946 */ 1947 pci_save_state(pdev); 1948 1949 err = pci_enable_device_mem(pdev); 1950 if (err) { 1951 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 1952 return err; 1953 } 1954 pci_set_master(pdev); 1955 1956 pci_wake_from_d3(pdev, false); 1957 1958 /* refresh hw_addr in case it was dropped */ 1959 hw->hw_addr = interface->uc_addr; 1960 1961 /* reset hardware to known state */ 1962 err = hw->mac.ops.init_hw(&interface->hw); 1963 if (err) 1964 return err; 1965 1966 /* reset statistics starting values */ 1967 hw->mac.ops.rebind_hw_stats(hw, &interface->stats); 1968 1969 /* reset clock */ 1970 fm10k_ts_reset(interface); 1971 1972 rtnl_lock(); 1973 1974 err = fm10k_init_queueing_scheme(interface); 1975 if (!err) { 1976 fm10k_mbx_request_irq(interface); 1977 if (netif_running(netdev)) 1978 err = fm10k_open(netdev); 1979 } 1980 1981 rtnl_unlock(); 1982 1983 if (err) 1984 return err; 1985 1986 /* restore SR-IOV interface */ 1987 fm10k_iov_resume(pdev); 1988 1989 netif_device_attach(netdev); 1990 1991 return 0; 1992 } 1993 1994 /** 1995 * fm10k_suspend - Prepare the device for a system sleep state 1996 * @pdev: PCI device information struct 1997 * 1998 * fm10k_suspend is meant to shutdown the device prior to the system entering 1999 * a sleep state. The fm10k hardware does not support wake on lan so the 2000 * driver simply needs to shut down the device so it is in a low power state. 2001 **/ 2002 static int fm10k_suspend(struct pci_dev *pdev, 2003 pm_message_t __always_unused state) 2004 { 2005 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2006 struct net_device *netdev = interface->netdev; 2007 int err = 0; 2008 2009 netif_device_detach(netdev); 2010 2011 fm10k_iov_suspend(pdev); 2012 2013 rtnl_lock(); 2014 2015 if (netif_running(netdev)) 2016 fm10k_close(netdev); 2017 2018 fm10k_mbx_free_irq(interface); 2019 2020 fm10k_clear_queueing_scheme(interface); 2021 2022 rtnl_unlock(); 2023 2024 err = pci_save_state(pdev); 2025 if (err) 2026 return err; 2027 2028 pci_disable_device(pdev); 2029 pci_wake_from_d3(pdev, false); 2030 pci_set_power_state(pdev, PCI_D3hot); 2031 2032 return 0; 2033 } 2034 2035 #endif /* CONFIG_PM */ 2036 /** 2037 * fm10k_io_error_detected - called when PCI error is detected 2038 * @pdev: Pointer to PCI device 2039 * @state: The current pci connection state 2040 * 2041 * This function is called after a PCI bus error affecting 2042 * this device has been detected. 2043 */ 2044 static pci_ers_result_t fm10k_io_error_detected(struct pci_dev *pdev, 2045 pci_channel_state_t state) 2046 { 2047 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2048 struct net_device *netdev = interface->netdev; 2049 2050 netif_device_detach(netdev); 2051 2052 if (state == pci_channel_io_perm_failure) 2053 return PCI_ERS_RESULT_DISCONNECT; 2054 2055 if (netif_running(netdev)) 2056 fm10k_close(netdev); 2057 2058 fm10k_mbx_free_irq(interface); 2059 2060 pci_disable_device(pdev); 2061 2062 /* Request a slot reset. */ 2063 return PCI_ERS_RESULT_NEED_RESET; 2064 } 2065 2066 /** 2067 * fm10k_io_slot_reset - called after the pci bus has been reset. 2068 * @pdev: Pointer to PCI device 2069 * 2070 * Restart the card from scratch, as if from a cold-boot. 2071 */ 2072 static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev) 2073 { 2074 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2075 pci_ers_result_t result; 2076 2077 if (pci_enable_device_mem(pdev)) { 2078 dev_err(&pdev->dev, 2079 "Cannot re-enable PCI device after reset.\n"); 2080 result = PCI_ERS_RESULT_DISCONNECT; 2081 } else { 2082 pci_set_master(pdev); 2083 pci_restore_state(pdev); 2084 2085 /* After second error pci->state_saved is false, this 2086 * resets it so EEH doesn't break. 2087 */ 2088 pci_save_state(pdev); 2089 2090 pci_wake_from_d3(pdev, false); 2091 2092 /* refresh hw_addr in case it was dropped */ 2093 interface->hw.hw_addr = interface->uc_addr; 2094 2095 interface->flags |= FM10K_FLAG_RESET_REQUESTED; 2096 fm10k_service_event_schedule(interface); 2097 2098 result = PCI_ERS_RESULT_RECOVERED; 2099 } 2100 2101 pci_cleanup_aer_uncorrect_error_status(pdev); 2102 2103 return result; 2104 } 2105 2106 /** 2107 * fm10k_io_resume - called when traffic can start flowing again. 2108 * @pdev: Pointer to PCI device 2109 * 2110 * This callback is called when the error recovery driver tells us that 2111 * its OK to resume normal operation. 2112 */ 2113 static void fm10k_io_resume(struct pci_dev *pdev) 2114 { 2115 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2116 struct net_device *netdev = interface->netdev; 2117 struct fm10k_hw *hw = &interface->hw; 2118 int err = 0; 2119 2120 /* reset hardware to known state */ 2121 hw->mac.ops.init_hw(&interface->hw); 2122 2123 /* reset statistics starting values */ 2124 hw->mac.ops.rebind_hw_stats(hw, &interface->stats); 2125 2126 /* reassociate interrupts */ 2127 fm10k_mbx_request_irq(interface); 2128 2129 /* reset clock */ 2130 fm10k_ts_reset(interface); 2131 2132 if (netif_running(netdev)) 2133 err = fm10k_open(netdev); 2134 2135 /* final check of hardware state before registering the interface */ 2136 err = err ? : fm10k_hw_ready(interface); 2137 2138 if (!err) 2139 netif_device_attach(netdev); 2140 } 2141 2142 static const struct pci_error_handlers fm10k_err_handler = { 2143 .error_detected = fm10k_io_error_detected, 2144 .slot_reset = fm10k_io_slot_reset, 2145 .resume = fm10k_io_resume, 2146 }; 2147 2148 static struct pci_driver fm10k_driver = { 2149 .name = fm10k_driver_name, 2150 .id_table = fm10k_pci_tbl, 2151 .probe = fm10k_probe, 2152 .remove = fm10k_remove, 2153 #ifdef CONFIG_PM 2154 .suspend = fm10k_suspend, 2155 .resume = fm10k_resume, 2156 #endif 2157 .sriov_configure = fm10k_iov_configure, 2158 .err_handler = &fm10k_err_handler 2159 }; 2160 2161 /** 2162 * fm10k_register_pci_driver - register driver interface 2163 * 2164 * This funciton is called on module load in order to register the driver. 2165 **/ 2166 int fm10k_register_pci_driver(void) 2167 { 2168 return pci_register_driver(&fm10k_driver); 2169 } 2170 2171 /** 2172 * fm10k_unregister_pci_driver - unregister driver interface 2173 * 2174 * This funciton is called on module unload in order to remove the driver. 2175 **/ 2176 void fm10k_unregister_pci_driver(void) 2177 { 2178 pci_unregister_driver(&fm10k_driver); 2179 } 2180