1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2005-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 #include <linux/netdevice.h> 14 #include <linux/etherdevice.h> 15 #include <linux/delay.h> 16 #include <linux/notifier.h> 17 #include <linux/ip.h> 18 #include <linux/tcp.h> 19 #include <linux/in.h> 20 #include <linux/ethtool.h> 21 #include <linux/topology.h> 22 #include <linux/gfp.h> 23 #include <linux/aer.h> 24 #include <linux/interrupt.h> 25 #include "net_driver.h" 26 #include "efx.h" 27 #include "nic.h" 28 #include "selftest.h" 29 #include "sriov.h" 30 31 #include "mcdi.h" 32 #include "workarounds.h" 33 34 /************************************************************************** 35 * 36 * Type name strings 37 * 38 ************************************************************************** 39 */ 40 41 /* Loopback mode names (see LOOPBACK_MODE()) */ 42 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX; 43 const char *const efx_loopback_mode_names[] = { 44 [LOOPBACK_NONE] = "NONE", 45 [LOOPBACK_DATA] = "DATAPATH", 46 [LOOPBACK_GMAC] = "GMAC", 47 [LOOPBACK_XGMII] = "XGMII", 48 [LOOPBACK_XGXS] = "XGXS", 49 [LOOPBACK_XAUI] = "XAUI", 50 [LOOPBACK_GMII] = "GMII", 51 [LOOPBACK_SGMII] = "SGMII", 52 [LOOPBACK_XGBR] = "XGBR", 53 [LOOPBACK_XFI] = "XFI", 54 [LOOPBACK_XAUI_FAR] = "XAUI_FAR", 55 [LOOPBACK_GMII_FAR] = "GMII_FAR", 56 [LOOPBACK_SGMII_FAR] = "SGMII_FAR", 57 [LOOPBACK_XFI_FAR] = "XFI_FAR", 58 [LOOPBACK_GPHY] = "GPHY", 59 [LOOPBACK_PHYXS] = "PHYXS", 60 [LOOPBACK_PCS] = "PCS", 61 [LOOPBACK_PMAPMD] = "PMA/PMD", 62 [LOOPBACK_XPORT] = "XPORT", 63 [LOOPBACK_XGMII_WS] = "XGMII_WS", 64 [LOOPBACK_XAUI_WS] = "XAUI_WS", 65 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR", 66 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR", 67 [LOOPBACK_GMII_WS] = "GMII_WS", 68 [LOOPBACK_XFI_WS] = "XFI_WS", 69 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR", 70 [LOOPBACK_PHYXS_WS] = "PHYXS_WS", 71 }; 72 73 const unsigned int efx_reset_type_max = RESET_TYPE_MAX; 74 const char *const efx_reset_type_names[] = { 75 [RESET_TYPE_INVISIBLE] = "INVISIBLE", 76 [RESET_TYPE_ALL] = "ALL", 77 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL", 78 [RESET_TYPE_WORLD] = "WORLD", 79 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE", 80 [RESET_TYPE_DATAPATH] = "DATAPATH", 81 [RESET_TYPE_MC_BIST] = "MC_BIST", 82 [RESET_TYPE_DISABLE] = "DISABLE", 83 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG", 84 [RESET_TYPE_INT_ERROR] = "INT_ERROR", 85 [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY", 86 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR", 87 [RESET_TYPE_TX_SKIP] = "TX_SKIP", 88 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE", 89 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)", 90 }; 91 92 /* Reset workqueue. If any NIC has a hardware failure then a reset will be 93 * queued onto this work queue. This is not a per-nic work queue, because 94 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised. 95 */ 96 static struct workqueue_struct *reset_workqueue; 97 98 /* How often and how many times to poll for a reset while waiting for a 99 * BIST that another function started to complete. 100 */ 101 #define BIST_WAIT_DELAY_MS 100 102 #define BIST_WAIT_DELAY_COUNT 100 103 104 /************************************************************************** 105 * 106 * Configurable values 107 * 108 *************************************************************************/ 109 110 /* 111 * Use separate channels for TX and RX events 112 * 113 * Set this to 1 to use separate channels for TX and RX. It allows us 114 * to control interrupt affinity separately for TX and RX. 115 * 116 * This is only used in MSI-X interrupt mode 117 */ 118 bool efx_separate_tx_channels; 119 module_param(efx_separate_tx_channels, bool, 0444); 120 MODULE_PARM_DESC(efx_separate_tx_channels, 121 "Use separate channels for TX and RX"); 122 123 /* This is the weight assigned to each of the (per-channel) virtual 124 * NAPI devices. 125 */ 126 static int napi_weight = 64; 127 128 /* This is the time (in jiffies) between invocations of the hardware 129 * monitor. 130 * On Falcon-based NICs, this will: 131 * - Check the on-board hardware monitor; 132 * - Poll the link state and reconfigure the hardware as necessary. 133 * On Siena-based NICs for power systems with EEH support, this will give EEH a 134 * chance to start. 135 */ 136 static unsigned int efx_monitor_interval = 1 * HZ; 137 138 /* Initial interrupt moderation settings. They can be modified after 139 * module load with ethtool. 140 * 141 * The default for RX should strike a balance between increasing the 142 * round-trip latency and reducing overhead. 143 */ 144 static unsigned int rx_irq_mod_usec = 60; 145 146 /* Initial interrupt moderation settings. They can be modified after 147 * module load with ethtool. 148 * 149 * This default is chosen to ensure that a 10G link does not go idle 150 * while a TX queue is stopped after it has become full. A queue is 151 * restarted when it drops below half full. The time this takes (assuming 152 * worst case 3 descriptors per packet and 1024 descriptors) is 153 * 512 / 3 * 1.2 = 205 usec. 154 */ 155 static unsigned int tx_irq_mod_usec = 150; 156 157 /* This is the first interrupt mode to try out of: 158 * 0 => MSI-X 159 * 1 => MSI 160 * 2 => legacy 161 */ 162 static unsigned int interrupt_mode; 163 164 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), 165 * i.e. the number of CPUs among which we may distribute simultaneous 166 * interrupt handling. 167 * 168 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. 169 * The default (0) means to assign an interrupt to each core. 170 */ 171 static unsigned int rss_cpus; 172 module_param(rss_cpus, uint, 0444); 173 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling"); 174 175 static bool phy_flash_cfg; 176 module_param(phy_flash_cfg, bool, 0644); 177 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially"); 178 179 static unsigned irq_adapt_low_thresh = 8000; 180 module_param(irq_adapt_low_thresh, uint, 0644); 181 MODULE_PARM_DESC(irq_adapt_low_thresh, 182 "Threshold score for reducing IRQ moderation"); 183 184 static unsigned irq_adapt_high_thresh = 16000; 185 module_param(irq_adapt_high_thresh, uint, 0644); 186 MODULE_PARM_DESC(irq_adapt_high_thresh, 187 "Threshold score for increasing IRQ moderation"); 188 189 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 190 NETIF_MSG_LINK | NETIF_MSG_IFDOWN | 191 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR | 192 NETIF_MSG_TX_ERR | NETIF_MSG_HW); 193 module_param(debug, uint, 0); 194 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value"); 195 196 /************************************************************************** 197 * 198 * Utility functions and prototypes 199 * 200 *************************************************************************/ 201 202 static int efx_soft_enable_interrupts(struct efx_nic *efx); 203 static void efx_soft_disable_interrupts(struct efx_nic *efx); 204 static void efx_remove_channel(struct efx_channel *channel); 205 static void efx_remove_channels(struct efx_nic *efx); 206 static const struct efx_channel_type efx_default_channel_type; 207 static void efx_remove_port(struct efx_nic *efx); 208 static void efx_init_napi_channel(struct efx_channel *channel); 209 static void efx_fini_napi(struct efx_nic *efx); 210 static void efx_fini_napi_channel(struct efx_channel *channel); 211 static void efx_fini_struct(struct efx_nic *efx); 212 static void efx_start_all(struct efx_nic *efx); 213 static void efx_stop_all(struct efx_nic *efx); 214 215 #define EFX_ASSERT_RESET_SERIALISED(efx) \ 216 do { \ 217 if ((efx->state == STATE_READY) || \ 218 (efx->state == STATE_RECOVERY) || \ 219 (efx->state == STATE_DISABLED)) \ 220 ASSERT_RTNL(); \ 221 } while (0) 222 223 static int efx_check_disabled(struct efx_nic *efx) 224 { 225 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) { 226 netif_err(efx, drv, efx->net_dev, 227 "device is disabled due to earlier errors\n"); 228 return -EIO; 229 } 230 return 0; 231 } 232 233 /************************************************************************** 234 * 235 * Event queue processing 236 * 237 *************************************************************************/ 238 239 /* Process channel's event queue 240 * 241 * This function is responsible for processing the event queue of a 242 * single channel. The caller must guarantee that this function will 243 * never be concurrently called more than once on the same channel, 244 * though different channels may be being processed concurrently. 245 */ 246 static int efx_process_channel(struct efx_channel *channel, int budget) 247 { 248 struct efx_tx_queue *tx_queue; 249 int spent; 250 251 if (unlikely(!channel->enabled)) 252 return 0; 253 254 efx_for_each_channel_tx_queue(tx_queue, channel) { 255 tx_queue->pkts_compl = 0; 256 tx_queue->bytes_compl = 0; 257 } 258 259 spent = efx_nic_process_eventq(channel, budget); 260 if (spent && efx_channel_has_rx_queue(channel)) { 261 struct efx_rx_queue *rx_queue = 262 efx_channel_get_rx_queue(channel); 263 264 efx_rx_flush_packet(channel); 265 efx_fast_push_rx_descriptors(rx_queue, true); 266 } 267 268 /* Update BQL */ 269 efx_for_each_channel_tx_queue(tx_queue, channel) { 270 if (tx_queue->bytes_compl) { 271 netdev_tx_completed_queue(tx_queue->core_txq, 272 tx_queue->pkts_compl, tx_queue->bytes_compl); 273 } 274 } 275 276 return spent; 277 } 278 279 /* NAPI poll handler 280 * 281 * NAPI guarantees serialisation of polls of the same device, which 282 * provides the guarantee required by efx_process_channel(). 283 */ 284 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) 285 { 286 int step = efx->irq_mod_step_us; 287 288 if (channel->irq_mod_score < irq_adapt_low_thresh) { 289 if (channel->irq_moderation_us > step) { 290 channel->irq_moderation_us -= step; 291 efx->type->push_irq_moderation(channel); 292 } 293 } else if (channel->irq_mod_score > irq_adapt_high_thresh) { 294 if (channel->irq_moderation_us < 295 efx->irq_rx_moderation_us) { 296 channel->irq_moderation_us += step; 297 efx->type->push_irq_moderation(channel); 298 } 299 } 300 301 channel->irq_count = 0; 302 channel->irq_mod_score = 0; 303 } 304 305 static int efx_poll(struct napi_struct *napi, int budget) 306 { 307 struct efx_channel *channel = 308 container_of(napi, struct efx_channel, napi_str); 309 struct efx_nic *efx = channel->efx; 310 int spent; 311 312 if (!efx_channel_lock_napi(channel)) 313 return budget; 314 315 netif_vdbg(efx, intr, efx->net_dev, 316 "channel %d NAPI poll executing on CPU %d\n", 317 channel->channel, raw_smp_processor_id()); 318 319 spent = efx_process_channel(channel, budget); 320 321 if (spent < budget) { 322 if (efx_channel_has_rx_queue(channel) && 323 efx->irq_rx_adaptive && 324 unlikely(++channel->irq_count == 1000)) { 325 efx_update_irq_mod(efx, channel); 326 } 327 328 efx_filter_rfs_expire(channel); 329 330 /* There is no race here; although napi_disable() will 331 * only wait for napi_complete(), this isn't a problem 332 * since efx_nic_eventq_read_ack() will have no effect if 333 * interrupts have already been disabled. 334 */ 335 napi_complete(napi); 336 efx_nic_eventq_read_ack(channel); 337 } 338 339 efx_channel_unlock_napi(channel); 340 return spent; 341 } 342 343 /* Create event queue 344 * Event queue memory allocations are done only once. If the channel 345 * is reset, the memory buffer will be reused; this guards against 346 * errors during channel reset and also simplifies interrupt handling. 347 */ 348 static int efx_probe_eventq(struct efx_channel *channel) 349 { 350 struct efx_nic *efx = channel->efx; 351 unsigned long entries; 352 353 netif_dbg(efx, probe, efx->net_dev, 354 "chan %d create event queue\n", channel->channel); 355 356 /* Build an event queue with room for one event per tx and rx buffer, 357 * plus some extra for link state events and MCDI completions. */ 358 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); 359 EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); 360 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; 361 362 return efx_nic_probe_eventq(channel); 363 } 364 365 /* Prepare channel's event queue */ 366 static int efx_init_eventq(struct efx_channel *channel) 367 { 368 struct efx_nic *efx = channel->efx; 369 int rc; 370 371 EFX_WARN_ON_PARANOID(channel->eventq_init); 372 373 netif_dbg(efx, drv, efx->net_dev, 374 "chan %d init event queue\n", channel->channel); 375 376 rc = efx_nic_init_eventq(channel); 377 if (rc == 0) { 378 efx->type->push_irq_moderation(channel); 379 channel->eventq_read_ptr = 0; 380 channel->eventq_init = true; 381 } 382 return rc; 383 } 384 385 /* Enable event queue processing and NAPI */ 386 void efx_start_eventq(struct efx_channel *channel) 387 { 388 netif_dbg(channel->efx, ifup, channel->efx->net_dev, 389 "chan %d start event queue\n", channel->channel); 390 391 /* Make sure the NAPI handler sees the enabled flag set */ 392 channel->enabled = true; 393 smp_wmb(); 394 395 efx_channel_enable(channel); 396 napi_enable(&channel->napi_str); 397 efx_nic_eventq_read_ack(channel); 398 } 399 400 /* Disable event queue processing and NAPI */ 401 void efx_stop_eventq(struct efx_channel *channel) 402 { 403 if (!channel->enabled) 404 return; 405 406 napi_disable(&channel->napi_str); 407 while (!efx_channel_disable(channel)) 408 usleep_range(1000, 20000); 409 channel->enabled = false; 410 } 411 412 static void efx_fini_eventq(struct efx_channel *channel) 413 { 414 if (!channel->eventq_init) 415 return; 416 417 netif_dbg(channel->efx, drv, channel->efx->net_dev, 418 "chan %d fini event queue\n", channel->channel); 419 420 efx_nic_fini_eventq(channel); 421 channel->eventq_init = false; 422 } 423 424 static void efx_remove_eventq(struct efx_channel *channel) 425 { 426 netif_dbg(channel->efx, drv, channel->efx->net_dev, 427 "chan %d remove event queue\n", channel->channel); 428 429 efx_nic_remove_eventq(channel); 430 } 431 432 /************************************************************************** 433 * 434 * Channel handling 435 * 436 *************************************************************************/ 437 438 /* Allocate and initialise a channel structure. */ 439 static struct efx_channel * 440 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel) 441 { 442 struct efx_channel *channel; 443 struct efx_rx_queue *rx_queue; 444 struct efx_tx_queue *tx_queue; 445 int j; 446 447 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 448 if (!channel) 449 return NULL; 450 451 channel->efx = efx; 452 channel->channel = i; 453 channel->type = &efx_default_channel_type; 454 455 for (j = 0; j < EFX_TXQ_TYPES; j++) { 456 tx_queue = &channel->tx_queue[j]; 457 tx_queue->efx = efx; 458 tx_queue->queue = i * EFX_TXQ_TYPES + j; 459 tx_queue->channel = channel; 460 } 461 462 rx_queue = &channel->rx_queue; 463 rx_queue->efx = efx; 464 setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 465 (unsigned long)rx_queue); 466 467 return channel; 468 } 469 470 /* Allocate and initialise a channel structure, copying parameters 471 * (but not resources) from an old channel structure. 472 */ 473 static struct efx_channel * 474 efx_copy_channel(const struct efx_channel *old_channel) 475 { 476 struct efx_channel *channel; 477 struct efx_rx_queue *rx_queue; 478 struct efx_tx_queue *tx_queue; 479 int j; 480 481 channel = kmalloc(sizeof(*channel), GFP_KERNEL); 482 if (!channel) 483 return NULL; 484 485 *channel = *old_channel; 486 487 channel->napi_dev = NULL; 488 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); 489 channel->napi_str.napi_id = 0; 490 channel->napi_str.state = 0; 491 memset(&channel->eventq, 0, sizeof(channel->eventq)); 492 493 for (j = 0; j < EFX_TXQ_TYPES; j++) { 494 tx_queue = &channel->tx_queue[j]; 495 if (tx_queue->channel) 496 tx_queue->channel = channel; 497 tx_queue->buffer = NULL; 498 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); 499 } 500 501 rx_queue = &channel->rx_queue; 502 rx_queue->buffer = NULL; 503 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); 504 setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill, 505 (unsigned long)rx_queue); 506 507 return channel; 508 } 509 510 static int efx_probe_channel(struct efx_channel *channel) 511 { 512 struct efx_tx_queue *tx_queue; 513 struct efx_rx_queue *rx_queue; 514 int rc; 515 516 netif_dbg(channel->efx, probe, channel->efx->net_dev, 517 "creating channel %d\n", channel->channel); 518 519 rc = channel->type->pre_probe(channel); 520 if (rc) 521 goto fail; 522 523 rc = efx_probe_eventq(channel); 524 if (rc) 525 goto fail; 526 527 efx_for_each_channel_tx_queue(tx_queue, channel) { 528 rc = efx_probe_tx_queue(tx_queue); 529 if (rc) 530 goto fail; 531 } 532 533 efx_for_each_channel_rx_queue(rx_queue, channel) { 534 rc = efx_probe_rx_queue(rx_queue); 535 if (rc) 536 goto fail; 537 } 538 539 return 0; 540 541 fail: 542 efx_remove_channel(channel); 543 return rc; 544 } 545 546 static void 547 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len) 548 { 549 struct efx_nic *efx = channel->efx; 550 const char *type; 551 int number; 552 553 number = channel->channel; 554 if (efx->tx_channel_offset == 0) { 555 type = ""; 556 } else if (channel->channel < efx->tx_channel_offset) { 557 type = "-rx"; 558 } else { 559 type = "-tx"; 560 number -= efx->tx_channel_offset; 561 } 562 snprintf(buf, len, "%s%s-%d", efx->name, type, number); 563 } 564 565 static void efx_set_channel_names(struct efx_nic *efx) 566 { 567 struct efx_channel *channel; 568 569 efx_for_each_channel(channel, efx) 570 channel->type->get_name(channel, 571 efx->msi_context[channel->channel].name, 572 sizeof(efx->msi_context[0].name)); 573 } 574 575 static int efx_probe_channels(struct efx_nic *efx) 576 { 577 struct efx_channel *channel; 578 int rc; 579 580 /* Restart special buffer allocation */ 581 efx->next_buffer_table = 0; 582 583 /* Probe channels in reverse, so that any 'extra' channels 584 * use the start of the buffer table. This allows the traffic 585 * channels to be resized without moving them or wasting the 586 * entries before them. 587 */ 588 efx_for_each_channel_rev(channel, efx) { 589 rc = efx_probe_channel(channel); 590 if (rc) { 591 netif_err(efx, probe, efx->net_dev, 592 "failed to create channel %d\n", 593 channel->channel); 594 goto fail; 595 } 596 } 597 efx_set_channel_names(efx); 598 599 return 0; 600 601 fail: 602 efx_remove_channels(efx); 603 return rc; 604 } 605 606 /* Channels are shutdown and reinitialised whilst the NIC is running 607 * to propagate configuration changes (mtu, checksum offload), or 608 * to clear hardware error conditions 609 */ 610 static void efx_start_datapath(struct efx_nic *efx) 611 { 612 netdev_features_t old_features = efx->net_dev->features; 613 bool old_rx_scatter = efx->rx_scatter; 614 struct efx_tx_queue *tx_queue; 615 struct efx_rx_queue *rx_queue; 616 struct efx_channel *channel; 617 size_t rx_buf_len; 618 619 /* Calculate the rx buffer allocation parameters required to 620 * support the current MTU, including padding for header 621 * alignment and overruns. 622 */ 623 efx->rx_dma_len = (efx->rx_prefix_size + 624 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) + 625 efx->type->rx_buffer_padding); 626 rx_buf_len = (sizeof(struct efx_rx_page_state) + 627 efx->rx_ip_align + efx->rx_dma_len); 628 if (rx_buf_len <= PAGE_SIZE) { 629 efx->rx_scatter = efx->type->always_rx_scatter; 630 efx->rx_buffer_order = 0; 631 } else if (efx->type->can_rx_scatter) { 632 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES); 633 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) + 634 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE, 635 EFX_RX_BUF_ALIGNMENT) > 636 PAGE_SIZE); 637 efx->rx_scatter = true; 638 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE; 639 efx->rx_buffer_order = 0; 640 } else { 641 efx->rx_scatter = false; 642 efx->rx_buffer_order = get_order(rx_buf_len); 643 } 644 645 efx_rx_config_page_split(efx); 646 if (efx->rx_buffer_order) 647 netif_dbg(efx, drv, efx->net_dev, 648 "RX buf len=%u; page order=%u batch=%u\n", 649 efx->rx_dma_len, efx->rx_buffer_order, 650 efx->rx_pages_per_batch); 651 else 652 netif_dbg(efx, drv, efx->net_dev, 653 "RX buf len=%u step=%u bpp=%u; page batch=%u\n", 654 efx->rx_dma_len, efx->rx_page_buf_step, 655 efx->rx_bufs_per_page, efx->rx_pages_per_batch); 656 657 /* Restore previously fixed features in hw_features and remove 658 * features which are fixed now 659 */ 660 efx->net_dev->hw_features |= efx->net_dev->features; 661 efx->net_dev->hw_features &= ~efx->fixed_features; 662 efx->net_dev->features |= efx->fixed_features; 663 if (efx->net_dev->features != old_features) 664 netdev_features_change(efx->net_dev); 665 666 /* RX filters may also have scatter-enabled flags */ 667 if (efx->rx_scatter != old_rx_scatter) 668 efx->type->filter_update_rx_scatter(efx); 669 670 /* We must keep at least one descriptor in a TX ring empty. 671 * We could avoid this when the queue size does not exactly 672 * match the hardware ring size, but it's not that important. 673 * Therefore we stop the queue when one more skb might fill 674 * the ring completely. We wake it when half way back to 675 * empty. 676 */ 677 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx); 678 efx->txq_wake_thresh = efx->txq_stop_thresh / 2; 679 680 /* Initialise the channels */ 681 efx_for_each_channel(channel, efx) { 682 efx_for_each_channel_tx_queue(tx_queue, channel) { 683 efx_init_tx_queue(tx_queue); 684 atomic_inc(&efx->active_queues); 685 } 686 687 efx_for_each_channel_rx_queue(rx_queue, channel) { 688 efx_init_rx_queue(rx_queue); 689 atomic_inc(&efx->active_queues); 690 efx_stop_eventq(channel); 691 efx_fast_push_rx_descriptors(rx_queue, false); 692 efx_start_eventq(channel); 693 } 694 695 WARN_ON(channel->rx_pkt_n_frags); 696 } 697 698 efx_ptp_start_datapath(efx); 699 700 if (netif_device_present(efx->net_dev)) 701 netif_tx_wake_all_queues(efx->net_dev); 702 } 703 704 static void efx_stop_datapath(struct efx_nic *efx) 705 { 706 struct efx_channel *channel; 707 struct efx_tx_queue *tx_queue; 708 struct efx_rx_queue *rx_queue; 709 int rc; 710 711 EFX_ASSERT_RESET_SERIALISED(efx); 712 BUG_ON(efx->port_enabled); 713 714 efx_ptp_stop_datapath(efx); 715 716 /* Stop RX refill */ 717 efx_for_each_channel(channel, efx) { 718 efx_for_each_channel_rx_queue(rx_queue, channel) 719 rx_queue->refill_enabled = false; 720 } 721 722 efx_for_each_channel(channel, efx) { 723 /* RX packet processing is pipelined, so wait for the 724 * NAPI handler to complete. At least event queue 0 725 * might be kept active by non-data events, so don't 726 * use napi_synchronize() but actually disable NAPI 727 * temporarily. 728 */ 729 if (efx_channel_has_rx_queue(channel)) { 730 efx_stop_eventq(channel); 731 efx_start_eventq(channel); 732 } 733 } 734 735 rc = efx->type->fini_dmaq(efx); 736 if (rc && EFX_WORKAROUND_7803(efx)) { 737 /* Schedule a reset to recover from the flush failure. The 738 * descriptor caches reference memory we're about to free, 739 * but falcon_reconfigure_mac_wrapper() won't reconnect 740 * the MACs because of the pending reset. 741 */ 742 netif_err(efx, drv, efx->net_dev, 743 "Resetting to recover from flush failure\n"); 744 efx_schedule_reset(efx, RESET_TYPE_ALL); 745 } else if (rc) { 746 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); 747 } else { 748 netif_dbg(efx, drv, efx->net_dev, 749 "successfully flushed all queues\n"); 750 } 751 752 efx_for_each_channel(channel, efx) { 753 efx_for_each_channel_rx_queue(rx_queue, channel) 754 efx_fini_rx_queue(rx_queue); 755 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 756 efx_fini_tx_queue(tx_queue); 757 } 758 } 759 760 static void efx_remove_channel(struct efx_channel *channel) 761 { 762 struct efx_tx_queue *tx_queue; 763 struct efx_rx_queue *rx_queue; 764 765 netif_dbg(channel->efx, drv, channel->efx->net_dev, 766 "destroy chan %d\n", channel->channel); 767 768 efx_for_each_channel_rx_queue(rx_queue, channel) 769 efx_remove_rx_queue(rx_queue); 770 efx_for_each_possible_channel_tx_queue(tx_queue, channel) 771 efx_remove_tx_queue(tx_queue); 772 efx_remove_eventq(channel); 773 channel->type->post_remove(channel); 774 } 775 776 static void efx_remove_channels(struct efx_nic *efx) 777 { 778 struct efx_channel *channel; 779 780 efx_for_each_channel(channel, efx) 781 efx_remove_channel(channel); 782 } 783 784 int 785 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) 786 { 787 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel; 788 u32 old_rxq_entries, old_txq_entries; 789 unsigned i, next_buffer_table = 0; 790 int rc, rc2; 791 792 rc = efx_check_disabled(efx); 793 if (rc) 794 return rc; 795 796 /* Not all channels should be reallocated. We must avoid 797 * reallocating their buffer table entries. 798 */ 799 efx_for_each_channel(channel, efx) { 800 struct efx_rx_queue *rx_queue; 801 struct efx_tx_queue *tx_queue; 802 803 if (channel->type->copy) 804 continue; 805 next_buffer_table = max(next_buffer_table, 806 channel->eventq.index + 807 channel->eventq.entries); 808 efx_for_each_channel_rx_queue(rx_queue, channel) 809 next_buffer_table = max(next_buffer_table, 810 rx_queue->rxd.index + 811 rx_queue->rxd.entries); 812 efx_for_each_channel_tx_queue(tx_queue, channel) 813 next_buffer_table = max(next_buffer_table, 814 tx_queue->txd.index + 815 tx_queue->txd.entries); 816 } 817 818 efx_device_detach_sync(efx); 819 efx_stop_all(efx); 820 efx_soft_disable_interrupts(efx); 821 822 /* Clone channels (where possible) */ 823 memset(other_channel, 0, sizeof(other_channel)); 824 for (i = 0; i < efx->n_channels; i++) { 825 channel = efx->channel[i]; 826 if (channel->type->copy) 827 channel = channel->type->copy(channel); 828 if (!channel) { 829 rc = -ENOMEM; 830 goto out; 831 } 832 other_channel[i] = channel; 833 } 834 835 /* Swap entry counts and channel pointers */ 836 old_rxq_entries = efx->rxq_entries; 837 old_txq_entries = efx->txq_entries; 838 efx->rxq_entries = rxq_entries; 839 efx->txq_entries = txq_entries; 840 for (i = 0; i < efx->n_channels; i++) { 841 channel = efx->channel[i]; 842 efx->channel[i] = other_channel[i]; 843 other_channel[i] = channel; 844 } 845 846 /* Restart buffer table allocation */ 847 efx->next_buffer_table = next_buffer_table; 848 849 for (i = 0; i < efx->n_channels; i++) { 850 channel = efx->channel[i]; 851 if (!channel->type->copy) 852 continue; 853 rc = efx_probe_channel(channel); 854 if (rc) 855 goto rollback; 856 efx_init_napi_channel(efx->channel[i]); 857 } 858 859 out: 860 /* Destroy unused channel structures */ 861 for (i = 0; i < efx->n_channels; i++) { 862 channel = other_channel[i]; 863 if (channel && channel->type->copy) { 864 efx_fini_napi_channel(channel); 865 efx_remove_channel(channel); 866 kfree(channel); 867 } 868 } 869 870 rc2 = efx_soft_enable_interrupts(efx); 871 if (rc2) { 872 rc = rc ? rc : rc2; 873 netif_err(efx, drv, efx->net_dev, 874 "unable to restart interrupts on channel reallocation\n"); 875 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 876 } else { 877 efx_start_all(efx); 878 netif_device_attach(efx->net_dev); 879 } 880 return rc; 881 882 rollback: 883 /* Swap back */ 884 efx->rxq_entries = old_rxq_entries; 885 efx->txq_entries = old_txq_entries; 886 for (i = 0; i < efx->n_channels; i++) { 887 channel = efx->channel[i]; 888 efx->channel[i] = other_channel[i]; 889 other_channel[i] = channel; 890 } 891 goto out; 892 } 893 894 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) 895 { 896 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100)); 897 } 898 899 static const struct efx_channel_type efx_default_channel_type = { 900 .pre_probe = efx_channel_dummy_op_int, 901 .post_remove = efx_channel_dummy_op_void, 902 .get_name = efx_get_channel_name, 903 .copy = efx_copy_channel, 904 .keep_eventq = false, 905 }; 906 907 int efx_channel_dummy_op_int(struct efx_channel *channel) 908 { 909 return 0; 910 } 911 912 void efx_channel_dummy_op_void(struct efx_channel *channel) 913 { 914 } 915 916 /************************************************************************** 917 * 918 * Port handling 919 * 920 **************************************************************************/ 921 922 /* This ensures that the kernel is kept informed (via 923 * netif_carrier_on/off) of the link status, and also maintains the 924 * link status's stop on the port's TX queue. 925 */ 926 void efx_link_status_changed(struct efx_nic *efx) 927 { 928 struct efx_link_state *link_state = &efx->link_state; 929 930 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure 931 * that no events are triggered between unregister_netdev() and the 932 * driver unloading. A more general condition is that NETDEV_CHANGE 933 * can only be generated between NETDEV_UP and NETDEV_DOWN */ 934 if (!netif_running(efx->net_dev)) 935 return; 936 937 if (link_state->up != netif_carrier_ok(efx->net_dev)) { 938 efx->n_link_state_changes++; 939 940 if (link_state->up) 941 netif_carrier_on(efx->net_dev); 942 else 943 netif_carrier_off(efx->net_dev); 944 } 945 946 /* Status message for kernel log */ 947 if (link_state->up) 948 netif_info(efx, link, efx->net_dev, 949 "link up at %uMbps %s-duplex (MTU %d)\n", 950 link_state->speed, link_state->fd ? "full" : "half", 951 efx->net_dev->mtu); 952 else 953 netif_info(efx, link, efx->net_dev, "link down\n"); 954 } 955 956 void efx_link_set_advertising(struct efx_nic *efx, u32 advertising) 957 { 958 efx->link_advertising = advertising; 959 if (advertising) { 960 if (advertising & ADVERTISED_Pause) 961 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX); 962 else 963 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX); 964 if (advertising & ADVERTISED_Asym_Pause) 965 efx->wanted_fc ^= EFX_FC_TX; 966 } 967 } 968 969 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc) 970 { 971 efx->wanted_fc = wanted_fc; 972 if (efx->link_advertising) { 973 if (wanted_fc & EFX_FC_RX) 974 efx->link_advertising |= (ADVERTISED_Pause | 975 ADVERTISED_Asym_Pause); 976 else 977 efx->link_advertising &= ~(ADVERTISED_Pause | 978 ADVERTISED_Asym_Pause); 979 if (wanted_fc & EFX_FC_TX) 980 efx->link_advertising ^= ADVERTISED_Asym_Pause; 981 } 982 } 983 984 static void efx_fini_port(struct efx_nic *efx); 985 986 /* We assume that efx->type->reconfigure_mac will always try to sync RX 987 * filters and therefore needs to read-lock the filter table against freeing 988 */ 989 void efx_mac_reconfigure(struct efx_nic *efx) 990 { 991 down_read(&efx->filter_sem); 992 efx->type->reconfigure_mac(efx); 993 up_read(&efx->filter_sem); 994 } 995 996 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure 997 * the MAC appropriately. All other PHY configuration changes are pushed 998 * through phy_op->set_settings(), and pushed asynchronously to the MAC 999 * through efx_monitor(). 1000 * 1001 * Callers must hold the mac_lock 1002 */ 1003 int __efx_reconfigure_port(struct efx_nic *efx) 1004 { 1005 enum efx_phy_mode phy_mode; 1006 int rc; 1007 1008 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 1009 1010 /* Disable PHY transmit in mac level loopbacks */ 1011 phy_mode = efx->phy_mode; 1012 if (LOOPBACK_INTERNAL(efx)) 1013 efx->phy_mode |= PHY_MODE_TX_DISABLED; 1014 else 1015 efx->phy_mode &= ~PHY_MODE_TX_DISABLED; 1016 1017 rc = efx->type->reconfigure_port(efx); 1018 1019 if (rc) 1020 efx->phy_mode = phy_mode; 1021 1022 return rc; 1023 } 1024 1025 /* Reinitialise the MAC to pick up new PHY settings, even if the port is 1026 * disabled. */ 1027 int efx_reconfigure_port(struct efx_nic *efx) 1028 { 1029 int rc; 1030 1031 EFX_ASSERT_RESET_SERIALISED(efx); 1032 1033 mutex_lock(&efx->mac_lock); 1034 rc = __efx_reconfigure_port(efx); 1035 mutex_unlock(&efx->mac_lock); 1036 1037 return rc; 1038 } 1039 1040 /* Asynchronous work item for changing MAC promiscuity and multicast 1041 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current 1042 * MAC directly. */ 1043 static void efx_mac_work(struct work_struct *data) 1044 { 1045 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work); 1046 1047 mutex_lock(&efx->mac_lock); 1048 if (efx->port_enabled) 1049 efx_mac_reconfigure(efx); 1050 mutex_unlock(&efx->mac_lock); 1051 } 1052 1053 static int efx_probe_port(struct efx_nic *efx) 1054 { 1055 int rc; 1056 1057 netif_dbg(efx, probe, efx->net_dev, "create port\n"); 1058 1059 if (phy_flash_cfg) 1060 efx->phy_mode = PHY_MODE_SPECIAL; 1061 1062 /* Connect up MAC/PHY operations table */ 1063 rc = efx->type->probe_port(efx); 1064 if (rc) 1065 return rc; 1066 1067 /* Initialise MAC address to permanent address */ 1068 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr); 1069 1070 return 0; 1071 } 1072 1073 static int efx_init_port(struct efx_nic *efx) 1074 { 1075 int rc; 1076 1077 netif_dbg(efx, drv, efx->net_dev, "init port\n"); 1078 1079 mutex_lock(&efx->mac_lock); 1080 1081 rc = efx->phy_op->init(efx); 1082 if (rc) 1083 goto fail1; 1084 1085 efx->port_initialized = true; 1086 1087 /* Reconfigure the MAC before creating dma queues (required for 1088 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */ 1089 efx_mac_reconfigure(efx); 1090 1091 /* Ensure the PHY advertises the correct flow control settings */ 1092 rc = efx->phy_op->reconfigure(efx); 1093 if (rc && rc != -EPERM) 1094 goto fail2; 1095 1096 mutex_unlock(&efx->mac_lock); 1097 return 0; 1098 1099 fail2: 1100 efx->phy_op->fini(efx); 1101 fail1: 1102 mutex_unlock(&efx->mac_lock); 1103 return rc; 1104 } 1105 1106 static void efx_start_port(struct efx_nic *efx) 1107 { 1108 netif_dbg(efx, ifup, efx->net_dev, "start port\n"); 1109 BUG_ON(efx->port_enabled); 1110 1111 mutex_lock(&efx->mac_lock); 1112 efx->port_enabled = true; 1113 1114 /* Ensure MAC ingress/egress is enabled */ 1115 efx_mac_reconfigure(efx); 1116 1117 mutex_unlock(&efx->mac_lock); 1118 } 1119 1120 /* Cancel work for MAC reconfiguration, periodic hardware monitoring 1121 * and the async self-test, wait for them to finish and prevent them 1122 * being scheduled again. This doesn't cover online resets, which 1123 * should only be cancelled when removing the device. 1124 */ 1125 static void efx_stop_port(struct efx_nic *efx) 1126 { 1127 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n"); 1128 1129 EFX_ASSERT_RESET_SERIALISED(efx); 1130 1131 mutex_lock(&efx->mac_lock); 1132 efx->port_enabled = false; 1133 mutex_unlock(&efx->mac_lock); 1134 1135 /* Serialise against efx_set_multicast_list() */ 1136 netif_addr_lock_bh(efx->net_dev); 1137 netif_addr_unlock_bh(efx->net_dev); 1138 1139 cancel_delayed_work_sync(&efx->monitor_work); 1140 efx_selftest_async_cancel(efx); 1141 cancel_work_sync(&efx->mac_work); 1142 } 1143 1144 static void efx_fini_port(struct efx_nic *efx) 1145 { 1146 netif_dbg(efx, drv, efx->net_dev, "shut down port\n"); 1147 1148 if (!efx->port_initialized) 1149 return; 1150 1151 efx->phy_op->fini(efx); 1152 efx->port_initialized = false; 1153 1154 efx->link_state.up = false; 1155 efx_link_status_changed(efx); 1156 } 1157 1158 static void efx_remove_port(struct efx_nic *efx) 1159 { 1160 netif_dbg(efx, drv, efx->net_dev, "destroying port\n"); 1161 1162 efx->type->remove_port(efx); 1163 } 1164 1165 /************************************************************************** 1166 * 1167 * NIC handling 1168 * 1169 **************************************************************************/ 1170 1171 static LIST_HEAD(efx_primary_list); 1172 static LIST_HEAD(efx_unassociated_list); 1173 1174 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right) 1175 { 1176 return left->type == right->type && 1177 left->vpd_sn && right->vpd_sn && 1178 !strcmp(left->vpd_sn, right->vpd_sn); 1179 } 1180 1181 static void efx_associate(struct efx_nic *efx) 1182 { 1183 struct efx_nic *other, *next; 1184 1185 if (efx->primary == efx) { 1186 /* Adding primary function; look for secondaries */ 1187 1188 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n"); 1189 list_add_tail(&efx->node, &efx_primary_list); 1190 1191 list_for_each_entry_safe(other, next, &efx_unassociated_list, 1192 node) { 1193 if (efx_same_controller(efx, other)) { 1194 list_del(&other->node); 1195 netif_dbg(other, probe, other->net_dev, 1196 "moving to secondary list of %s %s\n", 1197 pci_name(efx->pci_dev), 1198 efx->net_dev->name); 1199 list_add_tail(&other->node, 1200 &efx->secondary_list); 1201 other->primary = efx; 1202 } 1203 } 1204 } else { 1205 /* Adding secondary function; look for primary */ 1206 1207 list_for_each_entry(other, &efx_primary_list, node) { 1208 if (efx_same_controller(efx, other)) { 1209 netif_dbg(efx, probe, efx->net_dev, 1210 "adding to secondary list of %s %s\n", 1211 pci_name(other->pci_dev), 1212 other->net_dev->name); 1213 list_add_tail(&efx->node, 1214 &other->secondary_list); 1215 efx->primary = other; 1216 return; 1217 } 1218 } 1219 1220 netif_dbg(efx, probe, efx->net_dev, 1221 "adding to unassociated list\n"); 1222 list_add_tail(&efx->node, &efx_unassociated_list); 1223 } 1224 } 1225 1226 static void efx_dissociate(struct efx_nic *efx) 1227 { 1228 struct efx_nic *other, *next; 1229 1230 list_del(&efx->node); 1231 efx->primary = NULL; 1232 1233 list_for_each_entry_safe(other, next, &efx->secondary_list, node) { 1234 list_del(&other->node); 1235 netif_dbg(other, probe, other->net_dev, 1236 "moving to unassociated list\n"); 1237 list_add_tail(&other->node, &efx_unassociated_list); 1238 other->primary = NULL; 1239 } 1240 } 1241 1242 /* This configures the PCI device to enable I/O and DMA. */ 1243 static int efx_init_io(struct efx_nic *efx) 1244 { 1245 struct pci_dev *pci_dev = efx->pci_dev; 1246 dma_addr_t dma_mask = efx->type->max_dma_mask; 1247 unsigned int mem_map_size = efx->type->mem_map_size(efx); 1248 int rc, bar; 1249 1250 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); 1251 1252 bar = efx->type->mem_bar; 1253 1254 rc = pci_enable_device(pci_dev); 1255 if (rc) { 1256 netif_err(efx, probe, efx->net_dev, 1257 "failed to enable PCI device\n"); 1258 goto fail1; 1259 } 1260 1261 pci_set_master(pci_dev); 1262 1263 /* Set the PCI DMA mask. Try all possibilities from our 1264 * genuine mask down to 32 bits, because some architectures 1265 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit 1266 * masks event though they reject 46 bit masks. 1267 */ 1268 while (dma_mask > 0x7fffffffUL) { 1269 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask); 1270 if (rc == 0) 1271 break; 1272 dma_mask >>= 1; 1273 } 1274 if (rc) { 1275 netif_err(efx, probe, efx->net_dev, 1276 "could not find a suitable DMA mask\n"); 1277 goto fail2; 1278 } 1279 netif_dbg(efx, probe, efx->net_dev, 1280 "using DMA mask %llx\n", (unsigned long long) dma_mask); 1281 1282 efx->membase_phys = pci_resource_start(efx->pci_dev, bar); 1283 rc = pci_request_region(pci_dev, bar, "sfc"); 1284 if (rc) { 1285 netif_err(efx, probe, efx->net_dev, 1286 "request for memory BAR failed\n"); 1287 rc = -EIO; 1288 goto fail3; 1289 } 1290 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size); 1291 if (!efx->membase) { 1292 netif_err(efx, probe, efx->net_dev, 1293 "could not map memory BAR at %llx+%x\n", 1294 (unsigned long long)efx->membase_phys, mem_map_size); 1295 rc = -ENOMEM; 1296 goto fail4; 1297 } 1298 netif_dbg(efx, probe, efx->net_dev, 1299 "memory BAR at %llx+%x (virtual %p)\n", 1300 (unsigned long long)efx->membase_phys, mem_map_size, 1301 efx->membase); 1302 1303 return 0; 1304 1305 fail4: 1306 pci_release_region(efx->pci_dev, bar); 1307 fail3: 1308 efx->membase_phys = 0; 1309 fail2: 1310 pci_disable_device(efx->pci_dev); 1311 fail1: 1312 return rc; 1313 } 1314 1315 static void efx_fini_io(struct efx_nic *efx) 1316 { 1317 int bar; 1318 1319 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n"); 1320 1321 if (efx->membase) { 1322 iounmap(efx->membase); 1323 efx->membase = NULL; 1324 } 1325 1326 if (efx->membase_phys) { 1327 bar = efx->type->mem_bar; 1328 pci_release_region(efx->pci_dev, bar); 1329 efx->membase_phys = 0; 1330 } 1331 1332 /* Don't disable bus-mastering if VFs are assigned */ 1333 if (!pci_vfs_assigned(efx->pci_dev)) 1334 pci_disable_device(efx->pci_dev); 1335 } 1336 1337 void efx_set_default_rx_indir_table(struct efx_nic *efx) 1338 { 1339 size_t i; 1340 1341 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++) 1342 efx->rx_indir_table[i] = 1343 ethtool_rxfh_indir_default(i, efx->rss_spread); 1344 } 1345 1346 static unsigned int efx_wanted_parallelism(struct efx_nic *efx) 1347 { 1348 cpumask_var_t thread_mask; 1349 unsigned int count; 1350 int cpu; 1351 1352 if (rss_cpus) { 1353 count = rss_cpus; 1354 } else { 1355 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) { 1356 netif_warn(efx, probe, efx->net_dev, 1357 "RSS disabled due to allocation failure\n"); 1358 return 1; 1359 } 1360 1361 count = 0; 1362 for_each_online_cpu(cpu) { 1363 if (!cpumask_test_cpu(cpu, thread_mask)) { 1364 ++count; 1365 cpumask_or(thread_mask, thread_mask, 1366 topology_sibling_cpumask(cpu)); 1367 } 1368 } 1369 1370 free_cpumask_var(thread_mask); 1371 } 1372 1373 /* If RSS is requested for the PF *and* VFs then we can't write RSS 1374 * table entries that are inaccessible to VFs 1375 */ 1376 #ifdef CONFIG_SFC_SRIOV 1377 if (efx->type->sriov_wanted) { 1378 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && 1379 count > efx_vf_size(efx)) { 1380 netif_warn(efx, probe, efx->net_dev, 1381 "Reducing number of RSS channels from %u to %u for " 1382 "VF support. Increase vf-msix-limit to use more " 1383 "channels on the PF.\n", 1384 count, efx_vf_size(efx)); 1385 count = efx_vf_size(efx); 1386 } 1387 } 1388 #endif 1389 1390 return count; 1391 } 1392 1393 /* Probe the number and type of interrupts we are able to obtain, and 1394 * the resulting numbers of channels and RX queues. 1395 */ 1396 static int efx_probe_interrupts(struct efx_nic *efx) 1397 { 1398 unsigned int extra_channels = 0; 1399 unsigned int i, j; 1400 int rc; 1401 1402 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) 1403 if (efx->extra_channel_type[i]) 1404 ++extra_channels; 1405 1406 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { 1407 struct msix_entry xentries[EFX_MAX_CHANNELS]; 1408 unsigned int n_channels; 1409 1410 n_channels = efx_wanted_parallelism(efx); 1411 if (efx_separate_tx_channels) 1412 n_channels *= 2; 1413 n_channels += extra_channels; 1414 n_channels = min(n_channels, efx->max_channels); 1415 1416 for (i = 0; i < n_channels; i++) 1417 xentries[i].entry = i; 1418 rc = pci_enable_msix_range(efx->pci_dev, 1419 xentries, 1, n_channels); 1420 if (rc < 0) { 1421 /* Fall back to single channel MSI */ 1422 efx->interrupt_mode = EFX_INT_MODE_MSI; 1423 netif_err(efx, drv, efx->net_dev, 1424 "could not enable MSI-X\n"); 1425 } else if (rc < n_channels) { 1426 netif_err(efx, drv, efx->net_dev, 1427 "WARNING: Insufficient MSI-X vectors" 1428 " available (%d < %u).\n", rc, n_channels); 1429 netif_err(efx, drv, efx->net_dev, 1430 "WARNING: Performance may be reduced.\n"); 1431 n_channels = rc; 1432 } 1433 1434 if (rc > 0) { 1435 efx->n_channels = n_channels; 1436 if (n_channels > extra_channels) 1437 n_channels -= extra_channels; 1438 if (efx_separate_tx_channels) { 1439 efx->n_tx_channels = min(max(n_channels / 2, 1440 1U), 1441 efx->max_tx_channels); 1442 efx->n_rx_channels = max(n_channels - 1443 efx->n_tx_channels, 1444 1U); 1445 } else { 1446 efx->n_tx_channels = min(n_channels, 1447 efx->max_tx_channels); 1448 efx->n_rx_channels = n_channels; 1449 } 1450 for (i = 0; i < efx->n_channels; i++) 1451 efx_get_channel(efx, i)->irq = 1452 xentries[i].vector; 1453 } 1454 } 1455 1456 /* Try single interrupt MSI */ 1457 if (efx->interrupt_mode == EFX_INT_MODE_MSI) { 1458 efx->n_channels = 1; 1459 efx->n_rx_channels = 1; 1460 efx->n_tx_channels = 1; 1461 rc = pci_enable_msi(efx->pci_dev); 1462 if (rc == 0) { 1463 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; 1464 } else { 1465 netif_err(efx, drv, efx->net_dev, 1466 "could not enable MSI\n"); 1467 efx->interrupt_mode = EFX_INT_MODE_LEGACY; 1468 } 1469 } 1470 1471 /* Assume legacy interrupts */ 1472 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { 1473 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); 1474 efx->n_rx_channels = 1; 1475 efx->n_tx_channels = 1; 1476 efx->legacy_irq = efx->pci_dev->irq; 1477 } 1478 1479 /* Assign extra channels if possible */ 1480 j = efx->n_channels; 1481 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { 1482 if (!efx->extra_channel_type[i]) 1483 continue; 1484 if (efx->interrupt_mode != EFX_INT_MODE_MSIX || 1485 efx->n_channels <= extra_channels) { 1486 efx->extra_channel_type[i]->handle_no_channel(efx); 1487 } else { 1488 --j; 1489 efx_get_channel(efx, j)->type = 1490 efx->extra_channel_type[i]; 1491 } 1492 } 1493 1494 /* RSS might be usable on VFs even if it is disabled on the PF */ 1495 #ifdef CONFIG_SFC_SRIOV 1496 if (efx->type->sriov_wanted) { 1497 efx->rss_spread = ((efx->n_rx_channels > 1 || 1498 !efx->type->sriov_wanted(efx)) ? 1499 efx->n_rx_channels : efx_vf_size(efx)); 1500 return 0; 1501 } 1502 #endif 1503 efx->rss_spread = efx->n_rx_channels; 1504 1505 return 0; 1506 } 1507 1508 static int efx_soft_enable_interrupts(struct efx_nic *efx) 1509 { 1510 struct efx_channel *channel, *end_channel; 1511 int rc; 1512 1513 BUG_ON(efx->state == STATE_DISABLED); 1514 1515 efx->irq_soft_enabled = true; 1516 smp_wmb(); 1517 1518 efx_for_each_channel(channel, efx) { 1519 if (!channel->type->keep_eventq) { 1520 rc = efx_init_eventq(channel); 1521 if (rc) 1522 goto fail; 1523 } 1524 efx_start_eventq(channel); 1525 } 1526 1527 efx_mcdi_mode_event(efx); 1528 1529 return 0; 1530 fail: 1531 end_channel = channel; 1532 efx_for_each_channel(channel, efx) { 1533 if (channel == end_channel) 1534 break; 1535 efx_stop_eventq(channel); 1536 if (!channel->type->keep_eventq) 1537 efx_fini_eventq(channel); 1538 } 1539 1540 return rc; 1541 } 1542 1543 static void efx_soft_disable_interrupts(struct efx_nic *efx) 1544 { 1545 struct efx_channel *channel; 1546 1547 if (efx->state == STATE_DISABLED) 1548 return; 1549 1550 efx_mcdi_mode_poll(efx); 1551 1552 efx->irq_soft_enabled = false; 1553 smp_wmb(); 1554 1555 if (efx->legacy_irq) 1556 synchronize_irq(efx->legacy_irq); 1557 1558 efx_for_each_channel(channel, efx) { 1559 if (channel->irq) 1560 synchronize_irq(channel->irq); 1561 1562 efx_stop_eventq(channel); 1563 if (!channel->type->keep_eventq) 1564 efx_fini_eventq(channel); 1565 } 1566 1567 /* Flush the asynchronous MCDI request queue */ 1568 efx_mcdi_flush_async(efx); 1569 } 1570 1571 static int efx_enable_interrupts(struct efx_nic *efx) 1572 { 1573 struct efx_channel *channel, *end_channel; 1574 int rc; 1575 1576 BUG_ON(efx->state == STATE_DISABLED); 1577 1578 if (efx->eeh_disabled_legacy_irq) { 1579 enable_irq(efx->legacy_irq); 1580 efx->eeh_disabled_legacy_irq = false; 1581 } 1582 1583 efx->type->irq_enable_master(efx); 1584 1585 efx_for_each_channel(channel, efx) { 1586 if (channel->type->keep_eventq) { 1587 rc = efx_init_eventq(channel); 1588 if (rc) 1589 goto fail; 1590 } 1591 } 1592 1593 rc = efx_soft_enable_interrupts(efx); 1594 if (rc) 1595 goto fail; 1596 1597 return 0; 1598 1599 fail: 1600 end_channel = channel; 1601 efx_for_each_channel(channel, efx) { 1602 if (channel == end_channel) 1603 break; 1604 if (channel->type->keep_eventq) 1605 efx_fini_eventq(channel); 1606 } 1607 1608 efx->type->irq_disable_non_ev(efx); 1609 1610 return rc; 1611 } 1612 1613 static void efx_disable_interrupts(struct efx_nic *efx) 1614 { 1615 struct efx_channel *channel; 1616 1617 efx_soft_disable_interrupts(efx); 1618 1619 efx_for_each_channel(channel, efx) { 1620 if (channel->type->keep_eventq) 1621 efx_fini_eventq(channel); 1622 } 1623 1624 efx->type->irq_disable_non_ev(efx); 1625 } 1626 1627 static void efx_remove_interrupts(struct efx_nic *efx) 1628 { 1629 struct efx_channel *channel; 1630 1631 /* Remove MSI/MSI-X interrupts */ 1632 efx_for_each_channel(channel, efx) 1633 channel->irq = 0; 1634 pci_disable_msi(efx->pci_dev); 1635 pci_disable_msix(efx->pci_dev); 1636 1637 /* Remove legacy interrupt */ 1638 efx->legacy_irq = 0; 1639 } 1640 1641 static void efx_set_channels(struct efx_nic *efx) 1642 { 1643 struct efx_channel *channel; 1644 struct efx_tx_queue *tx_queue; 1645 1646 efx->tx_channel_offset = 1647 efx_separate_tx_channels ? 1648 efx->n_channels - efx->n_tx_channels : 0; 1649 1650 /* We need to mark which channels really have RX and TX 1651 * queues, and adjust the TX queue numbers if we have separate 1652 * RX-only and TX-only channels. 1653 */ 1654 efx_for_each_channel(channel, efx) { 1655 if (channel->channel < efx->n_rx_channels) 1656 channel->rx_queue.core_index = channel->channel; 1657 else 1658 channel->rx_queue.core_index = -1; 1659 1660 efx_for_each_channel_tx_queue(tx_queue, channel) 1661 tx_queue->queue -= (efx->tx_channel_offset * 1662 EFX_TXQ_TYPES); 1663 } 1664 } 1665 1666 static int efx_probe_nic(struct efx_nic *efx) 1667 { 1668 int rc; 1669 1670 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n"); 1671 1672 /* Carry out hardware-type specific initialisation */ 1673 rc = efx->type->probe(efx); 1674 if (rc) 1675 return rc; 1676 1677 do { 1678 if (!efx->max_channels || !efx->max_tx_channels) { 1679 netif_err(efx, drv, efx->net_dev, 1680 "Insufficient resources to allocate" 1681 " any channels\n"); 1682 rc = -ENOSPC; 1683 goto fail1; 1684 } 1685 1686 /* Determine the number of channels and queues by trying 1687 * to hook in MSI-X interrupts. 1688 */ 1689 rc = efx_probe_interrupts(efx); 1690 if (rc) 1691 goto fail1; 1692 1693 efx_set_channels(efx); 1694 1695 /* dimension_resources can fail with EAGAIN */ 1696 rc = efx->type->dimension_resources(efx); 1697 if (rc != 0 && rc != -EAGAIN) 1698 goto fail2; 1699 1700 if (rc == -EAGAIN) 1701 /* try again with new max_channels */ 1702 efx_remove_interrupts(efx); 1703 1704 } while (rc == -EAGAIN); 1705 1706 if (efx->n_channels > 1) 1707 netdev_rss_key_fill(&efx->rx_hash_key, 1708 sizeof(efx->rx_hash_key)); 1709 efx_set_default_rx_indir_table(efx); 1710 1711 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); 1712 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); 1713 1714 /* Initialise the interrupt moderation settings */ 1715 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000); 1716 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true, 1717 true); 1718 1719 return 0; 1720 1721 fail2: 1722 efx_remove_interrupts(efx); 1723 fail1: 1724 efx->type->remove(efx); 1725 return rc; 1726 } 1727 1728 static void efx_remove_nic(struct efx_nic *efx) 1729 { 1730 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n"); 1731 1732 efx_remove_interrupts(efx); 1733 efx->type->remove(efx); 1734 } 1735 1736 static int efx_probe_filters(struct efx_nic *efx) 1737 { 1738 int rc; 1739 1740 spin_lock_init(&efx->filter_lock); 1741 init_rwsem(&efx->filter_sem); 1742 mutex_lock(&efx->mac_lock); 1743 down_write(&efx->filter_sem); 1744 rc = efx->type->filter_table_probe(efx); 1745 if (rc) 1746 goto out_unlock; 1747 1748 #ifdef CONFIG_RFS_ACCEL 1749 if (efx->type->offload_features & NETIF_F_NTUPLE) { 1750 struct efx_channel *channel; 1751 int i, success = 1; 1752 1753 efx_for_each_channel(channel, efx) { 1754 channel->rps_flow_id = 1755 kcalloc(efx->type->max_rx_ip_filters, 1756 sizeof(*channel->rps_flow_id), 1757 GFP_KERNEL); 1758 if (!channel->rps_flow_id) 1759 success = 0; 1760 else 1761 for (i = 0; 1762 i < efx->type->max_rx_ip_filters; 1763 ++i) 1764 channel->rps_flow_id[i] = 1765 RPS_FLOW_ID_INVALID; 1766 } 1767 1768 if (!success) { 1769 efx_for_each_channel(channel, efx) 1770 kfree(channel->rps_flow_id); 1771 efx->type->filter_table_remove(efx); 1772 rc = -ENOMEM; 1773 goto out_unlock; 1774 } 1775 1776 efx->rps_expire_index = efx->rps_expire_channel = 0; 1777 } 1778 #endif 1779 out_unlock: 1780 up_write(&efx->filter_sem); 1781 mutex_unlock(&efx->mac_lock); 1782 return rc; 1783 } 1784 1785 static void efx_remove_filters(struct efx_nic *efx) 1786 { 1787 #ifdef CONFIG_RFS_ACCEL 1788 struct efx_channel *channel; 1789 1790 efx_for_each_channel(channel, efx) 1791 kfree(channel->rps_flow_id); 1792 #endif 1793 down_write(&efx->filter_sem); 1794 efx->type->filter_table_remove(efx); 1795 up_write(&efx->filter_sem); 1796 } 1797 1798 static void efx_restore_filters(struct efx_nic *efx) 1799 { 1800 down_read(&efx->filter_sem); 1801 efx->type->filter_table_restore(efx); 1802 up_read(&efx->filter_sem); 1803 } 1804 1805 /************************************************************************** 1806 * 1807 * NIC startup/shutdown 1808 * 1809 *************************************************************************/ 1810 1811 static int efx_probe_all(struct efx_nic *efx) 1812 { 1813 int rc; 1814 1815 rc = efx_probe_nic(efx); 1816 if (rc) { 1817 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n"); 1818 goto fail1; 1819 } 1820 1821 rc = efx_probe_port(efx); 1822 if (rc) { 1823 netif_err(efx, probe, efx->net_dev, "failed to create port\n"); 1824 goto fail2; 1825 } 1826 1827 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT); 1828 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) { 1829 rc = -EINVAL; 1830 goto fail3; 1831 } 1832 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE; 1833 1834 #ifdef CONFIG_SFC_SRIOV 1835 rc = efx->type->vswitching_probe(efx); 1836 if (rc) /* not fatal; the PF will still work fine */ 1837 netif_warn(efx, probe, efx->net_dev, 1838 "failed to setup vswitching rc=%d;" 1839 " VFs may not function\n", rc); 1840 #endif 1841 1842 rc = efx_probe_filters(efx); 1843 if (rc) { 1844 netif_err(efx, probe, efx->net_dev, 1845 "failed to create filter tables\n"); 1846 goto fail4; 1847 } 1848 1849 rc = efx_probe_channels(efx); 1850 if (rc) 1851 goto fail5; 1852 1853 return 0; 1854 1855 fail5: 1856 efx_remove_filters(efx); 1857 fail4: 1858 #ifdef CONFIG_SFC_SRIOV 1859 efx->type->vswitching_remove(efx); 1860 #endif 1861 fail3: 1862 efx_remove_port(efx); 1863 fail2: 1864 efx_remove_nic(efx); 1865 fail1: 1866 return rc; 1867 } 1868 1869 /* If the interface is supposed to be running but is not, start 1870 * the hardware and software data path, regular activity for the port 1871 * (MAC statistics, link polling, etc.) and schedule the port to be 1872 * reconfigured. Interrupts must already be enabled. This function 1873 * is safe to call multiple times, so long as the NIC is not disabled. 1874 * Requires the RTNL lock. 1875 */ 1876 static void efx_start_all(struct efx_nic *efx) 1877 { 1878 EFX_ASSERT_RESET_SERIALISED(efx); 1879 BUG_ON(efx->state == STATE_DISABLED); 1880 1881 /* Check that it is appropriate to restart the interface. All 1882 * of these flags are safe to read under just the rtnl lock */ 1883 if (efx->port_enabled || !netif_running(efx->net_dev) || 1884 efx->reset_pending) 1885 return; 1886 1887 efx_start_port(efx); 1888 efx_start_datapath(efx); 1889 1890 /* Start the hardware monitor if there is one */ 1891 if (efx->type->monitor != NULL) 1892 queue_delayed_work(efx->workqueue, &efx->monitor_work, 1893 efx_monitor_interval); 1894 1895 /* If link state detection is normally event-driven, we have 1896 * to poll now because we could have missed a change 1897 */ 1898 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { 1899 mutex_lock(&efx->mac_lock); 1900 if (efx->phy_op->poll(efx)) 1901 efx_link_status_changed(efx); 1902 mutex_unlock(&efx->mac_lock); 1903 } 1904 1905 efx->type->start_stats(efx); 1906 efx->type->pull_stats(efx); 1907 spin_lock_bh(&efx->stats_lock); 1908 efx->type->update_stats(efx, NULL, NULL); 1909 spin_unlock_bh(&efx->stats_lock); 1910 } 1911 1912 /* Quiesce the hardware and software data path, and regular activity 1913 * for the port without bringing the link down. Safe to call multiple 1914 * times with the NIC in almost any state, but interrupts should be 1915 * enabled. Requires the RTNL lock. 1916 */ 1917 static void efx_stop_all(struct efx_nic *efx) 1918 { 1919 EFX_ASSERT_RESET_SERIALISED(efx); 1920 1921 /* port_enabled can be read safely under the rtnl lock */ 1922 if (!efx->port_enabled) 1923 return; 1924 1925 /* update stats before we go down so we can accurately count 1926 * rx_nodesc_drops 1927 */ 1928 efx->type->pull_stats(efx); 1929 spin_lock_bh(&efx->stats_lock); 1930 efx->type->update_stats(efx, NULL, NULL); 1931 spin_unlock_bh(&efx->stats_lock); 1932 efx->type->stop_stats(efx); 1933 efx_stop_port(efx); 1934 1935 /* Stop the kernel transmit interface. This is only valid if 1936 * the device is stopped or detached; otherwise the watchdog 1937 * may fire immediately. 1938 */ 1939 WARN_ON(netif_running(efx->net_dev) && 1940 netif_device_present(efx->net_dev)); 1941 netif_tx_disable(efx->net_dev); 1942 1943 efx_stop_datapath(efx); 1944 } 1945 1946 static void efx_remove_all(struct efx_nic *efx) 1947 { 1948 efx_remove_channels(efx); 1949 efx_remove_filters(efx); 1950 #ifdef CONFIG_SFC_SRIOV 1951 efx->type->vswitching_remove(efx); 1952 #endif 1953 efx_remove_port(efx); 1954 efx_remove_nic(efx); 1955 } 1956 1957 /************************************************************************** 1958 * 1959 * Interrupt moderation 1960 * 1961 **************************************************************************/ 1962 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs) 1963 { 1964 if (usecs == 0) 1965 return 0; 1966 if (usecs * 1000 < efx->timer_quantum_ns) 1967 return 1; /* never round down to 0 */ 1968 return usecs * 1000 / efx->timer_quantum_ns; 1969 } 1970 1971 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks) 1972 { 1973 /* We must round up when converting ticks to microseconds 1974 * because we round down when converting the other way. 1975 */ 1976 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000); 1977 } 1978 1979 /* Set interrupt moderation parameters */ 1980 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs, 1981 unsigned int rx_usecs, bool rx_adaptive, 1982 bool rx_may_override_tx) 1983 { 1984 struct efx_channel *channel; 1985 unsigned int timer_max_us; 1986 1987 EFX_ASSERT_RESET_SERIALISED(efx); 1988 1989 timer_max_us = efx->timer_max_ns / 1000; 1990 1991 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us) 1992 return -EINVAL; 1993 1994 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 && 1995 !rx_may_override_tx) { 1996 netif_err(efx, drv, efx->net_dev, "Channels are shared. " 1997 "RX and TX IRQ moderation must be equal\n"); 1998 return -EINVAL; 1999 } 2000 2001 efx->irq_rx_adaptive = rx_adaptive; 2002 efx->irq_rx_moderation_us = rx_usecs; 2003 efx_for_each_channel(channel, efx) { 2004 if (efx_channel_has_rx_queue(channel)) 2005 channel->irq_moderation_us = rx_usecs; 2006 else if (efx_channel_has_tx_queues(channel)) 2007 channel->irq_moderation_us = tx_usecs; 2008 } 2009 2010 return 0; 2011 } 2012 2013 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs, 2014 unsigned int *rx_usecs, bool *rx_adaptive) 2015 { 2016 *rx_adaptive = efx->irq_rx_adaptive; 2017 *rx_usecs = efx->irq_rx_moderation_us; 2018 2019 /* If channels are shared between RX and TX, so is IRQ 2020 * moderation. Otherwise, IRQ moderation is the same for all 2021 * TX channels and is not adaptive. 2022 */ 2023 if (efx->tx_channel_offset == 0) { 2024 *tx_usecs = *rx_usecs; 2025 } else { 2026 struct efx_channel *tx_channel; 2027 2028 tx_channel = efx->channel[efx->tx_channel_offset]; 2029 *tx_usecs = tx_channel->irq_moderation_us; 2030 } 2031 } 2032 2033 /************************************************************************** 2034 * 2035 * Hardware monitor 2036 * 2037 **************************************************************************/ 2038 2039 /* Run periodically off the general workqueue */ 2040 static void efx_monitor(struct work_struct *data) 2041 { 2042 struct efx_nic *efx = container_of(data, struct efx_nic, 2043 monitor_work.work); 2044 2045 netif_vdbg(efx, timer, efx->net_dev, 2046 "hardware monitor executing on CPU %d\n", 2047 raw_smp_processor_id()); 2048 BUG_ON(efx->type->monitor == NULL); 2049 2050 /* If the mac_lock is already held then it is likely a port 2051 * reconfiguration is already in place, which will likely do 2052 * most of the work of monitor() anyway. */ 2053 if (mutex_trylock(&efx->mac_lock)) { 2054 if (efx->port_enabled) 2055 efx->type->monitor(efx); 2056 mutex_unlock(&efx->mac_lock); 2057 } 2058 2059 queue_delayed_work(efx->workqueue, &efx->monitor_work, 2060 efx_monitor_interval); 2061 } 2062 2063 /************************************************************************** 2064 * 2065 * ioctls 2066 * 2067 *************************************************************************/ 2068 2069 /* Net device ioctl 2070 * Context: process, rtnl_lock() held. 2071 */ 2072 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd) 2073 { 2074 struct efx_nic *efx = netdev_priv(net_dev); 2075 struct mii_ioctl_data *data = if_mii(ifr); 2076 2077 if (cmd == SIOCSHWTSTAMP) 2078 return efx_ptp_set_ts_config(efx, ifr); 2079 if (cmd == SIOCGHWTSTAMP) 2080 return efx_ptp_get_ts_config(efx, ifr); 2081 2082 /* Convert phy_id from older PRTAD/DEVAD format */ 2083 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) && 2084 (data->phy_id & 0xfc00) == 0x0400) 2085 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400; 2086 2087 return mdio_mii_ioctl(&efx->mdio, data, cmd); 2088 } 2089 2090 /************************************************************************** 2091 * 2092 * NAPI interface 2093 * 2094 **************************************************************************/ 2095 2096 static void efx_init_napi_channel(struct efx_channel *channel) 2097 { 2098 struct efx_nic *efx = channel->efx; 2099 2100 channel->napi_dev = efx->net_dev; 2101 netif_napi_add(channel->napi_dev, &channel->napi_str, 2102 efx_poll, napi_weight); 2103 efx_channel_busy_poll_init(channel); 2104 } 2105 2106 static void efx_init_napi(struct efx_nic *efx) 2107 { 2108 struct efx_channel *channel; 2109 2110 efx_for_each_channel(channel, efx) 2111 efx_init_napi_channel(channel); 2112 } 2113 2114 static void efx_fini_napi_channel(struct efx_channel *channel) 2115 { 2116 if (channel->napi_dev) 2117 netif_napi_del(&channel->napi_str); 2118 2119 channel->napi_dev = NULL; 2120 } 2121 2122 static void efx_fini_napi(struct efx_nic *efx) 2123 { 2124 struct efx_channel *channel; 2125 2126 efx_for_each_channel(channel, efx) 2127 efx_fini_napi_channel(channel); 2128 } 2129 2130 /************************************************************************** 2131 * 2132 * Kernel netpoll interface 2133 * 2134 *************************************************************************/ 2135 2136 #ifdef CONFIG_NET_POLL_CONTROLLER 2137 2138 /* Although in the common case interrupts will be disabled, this is not 2139 * guaranteed. However, all our work happens inside the NAPI callback, 2140 * so no locking is required. 2141 */ 2142 static void efx_netpoll(struct net_device *net_dev) 2143 { 2144 struct efx_nic *efx = netdev_priv(net_dev); 2145 struct efx_channel *channel; 2146 2147 efx_for_each_channel(channel, efx) 2148 efx_schedule_channel(channel); 2149 } 2150 2151 #endif 2152 2153 #ifdef CONFIG_NET_RX_BUSY_POLL 2154 static int efx_busy_poll(struct napi_struct *napi) 2155 { 2156 struct efx_channel *channel = 2157 container_of(napi, struct efx_channel, napi_str); 2158 struct efx_nic *efx = channel->efx; 2159 int budget = 4; 2160 int old_rx_packets, rx_packets; 2161 2162 if (!netif_running(efx->net_dev)) 2163 return LL_FLUSH_FAILED; 2164 2165 if (!efx_channel_try_lock_poll(channel)) 2166 return LL_FLUSH_BUSY; 2167 2168 old_rx_packets = channel->rx_queue.rx_packets; 2169 efx_process_channel(channel, budget); 2170 2171 rx_packets = channel->rx_queue.rx_packets - old_rx_packets; 2172 2173 /* There is no race condition with NAPI here. 2174 * NAPI will automatically be rescheduled if it yielded during busy 2175 * polling, because it was not able to take the lock and thus returned 2176 * the full budget. 2177 */ 2178 efx_channel_unlock_poll(channel); 2179 2180 return rx_packets; 2181 } 2182 #endif 2183 2184 /************************************************************************** 2185 * 2186 * Kernel net device interface 2187 * 2188 *************************************************************************/ 2189 2190 /* Context: process, rtnl_lock() held. */ 2191 int efx_net_open(struct net_device *net_dev) 2192 { 2193 struct efx_nic *efx = netdev_priv(net_dev); 2194 int rc; 2195 2196 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n", 2197 raw_smp_processor_id()); 2198 2199 rc = efx_check_disabled(efx); 2200 if (rc) 2201 return rc; 2202 if (efx->phy_mode & PHY_MODE_SPECIAL) 2203 return -EBUSY; 2204 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL)) 2205 return -EIO; 2206 2207 /* Notify the kernel of the link state polled during driver load, 2208 * before the monitor starts running */ 2209 efx_link_status_changed(efx); 2210 2211 efx_start_all(efx); 2212 efx_selftest_async_start(efx); 2213 return 0; 2214 } 2215 2216 /* Context: process, rtnl_lock() held. 2217 * Note that the kernel will ignore our return code; this method 2218 * should really be a void. 2219 */ 2220 int efx_net_stop(struct net_device *net_dev) 2221 { 2222 struct efx_nic *efx = netdev_priv(net_dev); 2223 2224 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n", 2225 raw_smp_processor_id()); 2226 2227 /* Stop the device and flush all the channels */ 2228 efx_stop_all(efx); 2229 2230 return 0; 2231 } 2232 2233 /* Context: process, dev_base_lock or RTNL held, non-blocking. */ 2234 static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev, 2235 struct rtnl_link_stats64 *stats) 2236 { 2237 struct efx_nic *efx = netdev_priv(net_dev); 2238 2239 spin_lock_bh(&efx->stats_lock); 2240 efx->type->update_stats(efx, NULL, stats); 2241 spin_unlock_bh(&efx->stats_lock); 2242 2243 return stats; 2244 } 2245 2246 /* Context: netif_tx_lock held, BHs disabled. */ 2247 static void efx_watchdog(struct net_device *net_dev) 2248 { 2249 struct efx_nic *efx = netdev_priv(net_dev); 2250 2251 netif_err(efx, tx_err, efx->net_dev, 2252 "TX stuck with port_enabled=%d: resetting channels\n", 2253 efx->port_enabled); 2254 2255 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG); 2256 } 2257 2258 2259 /* Context: process, rtnl_lock() held. */ 2260 static int efx_change_mtu(struct net_device *net_dev, int new_mtu) 2261 { 2262 struct efx_nic *efx = netdev_priv(net_dev); 2263 int rc; 2264 2265 rc = efx_check_disabled(efx); 2266 if (rc) 2267 return rc; 2268 2269 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu); 2270 2271 efx_device_detach_sync(efx); 2272 efx_stop_all(efx); 2273 2274 mutex_lock(&efx->mac_lock); 2275 net_dev->mtu = new_mtu; 2276 efx_mac_reconfigure(efx); 2277 mutex_unlock(&efx->mac_lock); 2278 2279 efx_start_all(efx); 2280 netif_device_attach(efx->net_dev); 2281 return 0; 2282 } 2283 2284 static int efx_set_mac_address(struct net_device *net_dev, void *data) 2285 { 2286 struct efx_nic *efx = netdev_priv(net_dev); 2287 struct sockaddr *addr = data; 2288 u8 *new_addr = addr->sa_data; 2289 u8 old_addr[6]; 2290 int rc; 2291 2292 if (!is_valid_ether_addr(new_addr)) { 2293 netif_err(efx, drv, efx->net_dev, 2294 "invalid ethernet MAC address requested: %pM\n", 2295 new_addr); 2296 return -EADDRNOTAVAIL; 2297 } 2298 2299 /* save old address */ 2300 ether_addr_copy(old_addr, net_dev->dev_addr); 2301 ether_addr_copy(net_dev->dev_addr, new_addr); 2302 if (efx->type->set_mac_address) { 2303 rc = efx->type->set_mac_address(efx); 2304 if (rc) { 2305 ether_addr_copy(net_dev->dev_addr, old_addr); 2306 return rc; 2307 } 2308 } 2309 2310 /* Reconfigure the MAC */ 2311 mutex_lock(&efx->mac_lock); 2312 efx_mac_reconfigure(efx); 2313 mutex_unlock(&efx->mac_lock); 2314 2315 return 0; 2316 } 2317 2318 /* Context: netif_addr_lock held, BHs disabled. */ 2319 static void efx_set_rx_mode(struct net_device *net_dev) 2320 { 2321 struct efx_nic *efx = netdev_priv(net_dev); 2322 2323 if (efx->port_enabled) 2324 queue_work(efx->workqueue, &efx->mac_work); 2325 /* Otherwise efx_start_port() will do this */ 2326 } 2327 2328 static int efx_set_features(struct net_device *net_dev, netdev_features_t data) 2329 { 2330 struct efx_nic *efx = netdev_priv(net_dev); 2331 int rc; 2332 2333 /* If disabling RX n-tuple filtering, clear existing filters */ 2334 if (net_dev->features & ~data & NETIF_F_NTUPLE) { 2335 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL); 2336 if (rc) 2337 return rc; 2338 } 2339 2340 /* If Rx VLAN filter is changed, update filters via mac_reconfigure */ 2341 if ((net_dev->features ^ data) & NETIF_F_HW_VLAN_CTAG_FILTER) { 2342 /* efx_set_rx_mode() will schedule MAC work to update filters 2343 * when a new features are finally set in net_dev. 2344 */ 2345 efx_set_rx_mode(net_dev); 2346 } 2347 2348 return 0; 2349 } 2350 2351 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2352 { 2353 struct efx_nic *efx = netdev_priv(net_dev); 2354 2355 if (efx->type->vlan_rx_add_vid) 2356 return efx->type->vlan_rx_add_vid(efx, proto, vid); 2357 else 2358 return -EOPNOTSUPP; 2359 } 2360 2361 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid) 2362 { 2363 struct efx_nic *efx = netdev_priv(net_dev); 2364 2365 if (efx->type->vlan_rx_kill_vid) 2366 return efx->type->vlan_rx_kill_vid(efx, proto, vid); 2367 else 2368 return -EOPNOTSUPP; 2369 } 2370 2371 static const struct net_device_ops efx_netdev_ops = { 2372 .ndo_open = efx_net_open, 2373 .ndo_stop = efx_net_stop, 2374 .ndo_get_stats64 = efx_net_stats, 2375 .ndo_tx_timeout = efx_watchdog, 2376 .ndo_start_xmit = efx_hard_start_xmit, 2377 .ndo_validate_addr = eth_validate_addr, 2378 .ndo_do_ioctl = efx_ioctl, 2379 .ndo_change_mtu = efx_change_mtu, 2380 .ndo_set_mac_address = efx_set_mac_address, 2381 .ndo_set_rx_mode = efx_set_rx_mode, 2382 .ndo_set_features = efx_set_features, 2383 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid, 2384 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid, 2385 #ifdef CONFIG_SFC_SRIOV 2386 .ndo_set_vf_mac = efx_sriov_set_vf_mac, 2387 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan, 2388 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk, 2389 .ndo_get_vf_config = efx_sriov_get_vf_config, 2390 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state, 2391 .ndo_get_phys_port_id = efx_sriov_get_phys_port_id, 2392 #endif 2393 #ifdef CONFIG_NET_POLL_CONTROLLER 2394 .ndo_poll_controller = efx_netpoll, 2395 #endif 2396 .ndo_setup_tc = efx_setup_tc, 2397 #ifdef CONFIG_NET_RX_BUSY_POLL 2398 .ndo_busy_poll = efx_busy_poll, 2399 #endif 2400 #ifdef CONFIG_RFS_ACCEL 2401 .ndo_rx_flow_steer = efx_filter_rfs, 2402 #endif 2403 }; 2404 2405 static void efx_update_name(struct efx_nic *efx) 2406 { 2407 strcpy(efx->name, efx->net_dev->name); 2408 efx_mtd_rename(efx); 2409 efx_set_channel_names(efx); 2410 } 2411 2412 static int efx_netdev_event(struct notifier_block *this, 2413 unsigned long event, void *ptr) 2414 { 2415 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); 2416 2417 if ((net_dev->netdev_ops == &efx_netdev_ops) && 2418 event == NETDEV_CHANGENAME) 2419 efx_update_name(netdev_priv(net_dev)); 2420 2421 return NOTIFY_DONE; 2422 } 2423 2424 static struct notifier_block efx_netdev_notifier = { 2425 .notifier_call = efx_netdev_event, 2426 }; 2427 2428 static ssize_t 2429 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) 2430 { 2431 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2432 return sprintf(buf, "%d\n", efx->phy_type); 2433 } 2434 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); 2435 2436 #ifdef CONFIG_SFC_MCDI_LOGGING 2437 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, 2438 char *buf) 2439 { 2440 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2441 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2442 2443 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); 2444 } 2445 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, 2446 const char *buf, size_t count) 2447 { 2448 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 2449 struct efx_mcdi_iface *mcdi = efx_mcdi(efx); 2450 bool enable = count > 0 && *buf != '0'; 2451 2452 mcdi->logging_enabled = enable; 2453 return count; 2454 } 2455 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); 2456 #endif 2457 2458 static int efx_register_netdev(struct efx_nic *efx) 2459 { 2460 struct net_device *net_dev = efx->net_dev; 2461 struct efx_channel *channel; 2462 int rc; 2463 2464 net_dev->watchdog_timeo = 5 * HZ; 2465 net_dev->irq = efx->pci_dev->irq; 2466 net_dev->netdev_ops = &efx_netdev_ops; 2467 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) 2468 net_dev->priv_flags |= IFF_UNICAST_FLT; 2469 net_dev->ethtool_ops = &efx_ethtool_ops; 2470 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; 2471 net_dev->min_mtu = EFX_MIN_MTU; 2472 net_dev->max_mtu = EFX_MAX_MTU; 2473 2474 rtnl_lock(); 2475 2476 /* Enable resets to be scheduled and check whether any were 2477 * already requested. If so, the NIC is probably hosed so we 2478 * abort. 2479 */ 2480 efx->state = STATE_READY; 2481 smp_mb(); /* ensure we change state before checking reset_pending */ 2482 if (efx->reset_pending) { 2483 netif_err(efx, probe, efx->net_dev, 2484 "aborting probe due to scheduled reset\n"); 2485 rc = -EIO; 2486 goto fail_locked; 2487 } 2488 2489 rc = dev_alloc_name(net_dev, net_dev->name); 2490 if (rc < 0) 2491 goto fail_locked; 2492 efx_update_name(efx); 2493 2494 /* Always start with carrier off; PHY events will detect the link */ 2495 netif_carrier_off(net_dev); 2496 2497 rc = register_netdevice(net_dev); 2498 if (rc) 2499 goto fail_locked; 2500 2501 efx_for_each_channel(channel, efx) { 2502 struct efx_tx_queue *tx_queue; 2503 efx_for_each_channel_tx_queue(tx_queue, channel) 2504 efx_init_tx_queue_core_txq(tx_queue); 2505 } 2506 2507 efx_associate(efx); 2508 2509 rtnl_unlock(); 2510 2511 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2512 if (rc) { 2513 netif_err(efx, drv, efx->net_dev, 2514 "failed to init net dev attributes\n"); 2515 goto fail_registered; 2516 } 2517 #ifdef CONFIG_SFC_MCDI_LOGGING 2518 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2519 if (rc) { 2520 netif_err(efx, drv, efx->net_dev, 2521 "failed to init net dev attributes\n"); 2522 goto fail_attr_mcdi_logging; 2523 } 2524 #endif 2525 2526 return 0; 2527 2528 #ifdef CONFIG_SFC_MCDI_LOGGING 2529 fail_attr_mcdi_logging: 2530 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2531 #endif 2532 fail_registered: 2533 rtnl_lock(); 2534 efx_dissociate(efx); 2535 unregister_netdevice(net_dev); 2536 fail_locked: 2537 efx->state = STATE_UNINIT; 2538 rtnl_unlock(); 2539 netif_err(efx, drv, efx->net_dev, "could not register net dev\n"); 2540 return rc; 2541 } 2542 2543 static void efx_unregister_netdev(struct efx_nic *efx) 2544 { 2545 if (!efx->net_dev) 2546 return; 2547 2548 BUG_ON(netdev_priv(efx->net_dev) != efx); 2549 2550 if (efx_dev_registered(efx)) { 2551 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); 2552 #ifdef CONFIG_SFC_MCDI_LOGGING 2553 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging); 2554 #endif 2555 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); 2556 unregister_netdev(efx->net_dev); 2557 } 2558 } 2559 2560 /************************************************************************** 2561 * 2562 * Device reset and suspend 2563 * 2564 **************************************************************************/ 2565 2566 /* Tears down the entire software state and most of the hardware state 2567 * before reset. */ 2568 void efx_reset_down(struct efx_nic *efx, enum reset_type method) 2569 { 2570 EFX_ASSERT_RESET_SERIALISED(efx); 2571 2572 if (method == RESET_TYPE_MCDI_TIMEOUT) 2573 efx->type->prepare_flr(efx); 2574 2575 efx_stop_all(efx); 2576 efx_disable_interrupts(efx); 2577 2578 mutex_lock(&efx->mac_lock); 2579 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2580 method != RESET_TYPE_DATAPATH) 2581 efx->phy_op->fini(efx); 2582 efx->type->fini(efx); 2583 } 2584 2585 /* This function will always ensure that the locks acquired in 2586 * efx_reset_down() are released. A failure return code indicates 2587 * that we were unable to reinitialise the hardware, and the 2588 * driver should be disabled. If ok is false, then the rx and tx 2589 * engines are not restarted, pending a RESET_DISABLE. */ 2590 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok) 2591 { 2592 int rc; 2593 2594 EFX_ASSERT_RESET_SERIALISED(efx); 2595 2596 if (method == RESET_TYPE_MCDI_TIMEOUT) 2597 efx->type->finish_flr(efx); 2598 2599 /* Ensure that SRAM is initialised even if we're disabling the device */ 2600 rc = efx->type->init(efx); 2601 if (rc) { 2602 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n"); 2603 goto fail; 2604 } 2605 2606 if (!ok) 2607 goto fail; 2608 2609 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE && 2610 method != RESET_TYPE_DATAPATH) { 2611 rc = efx->phy_op->init(efx); 2612 if (rc) 2613 goto fail; 2614 rc = efx->phy_op->reconfigure(efx); 2615 if (rc && rc != -EPERM) 2616 netif_err(efx, drv, efx->net_dev, 2617 "could not restore PHY settings\n"); 2618 } 2619 2620 rc = efx_enable_interrupts(efx); 2621 if (rc) 2622 goto fail; 2623 2624 #ifdef CONFIG_SFC_SRIOV 2625 rc = efx->type->vswitching_restore(efx); 2626 if (rc) /* not fatal; the PF will still work fine */ 2627 netif_warn(efx, probe, efx->net_dev, 2628 "failed to restore vswitching rc=%d;" 2629 " VFs may not function\n", rc); 2630 #endif 2631 2632 down_read(&efx->filter_sem); 2633 efx_restore_filters(efx); 2634 up_read(&efx->filter_sem); 2635 if (efx->type->sriov_reset) 2636 efx->type->sriov_reset(efx); 2637 2638 mutex_unlock(&efx->mac_lock); 2639 2640 efx_start_all(efx); 2641 2642 return 0; 2643 2644 fail: 2645 efx->port_initialized = false; 2646 2647 mutex_unlock(&efx->mac_lock); 2648 2649 return rc; 2650 } 2651 2652 /* Reset the NIC using the specified method. Note that the reset may 2653 * fail, in which case the card will be left in an unusable state. 2654 * 2655 * Caller must hold the rtnl_lock. 2656 */ 2657 int efx_reset(struct efx_nic *efx, enum reset_type method) 2658 { 2659 int rc, rc2; 2660 bool disabled; 2661 2662 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n", 2663 RESET_TYPE(method)); 2664 2665 efx_device_detach_sync(efx); 2666 efx_reset_down(efx, method); 2667 2668 rc = efx->type->reset(efx, method); 2669 if (rc) { 2670 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n"); 2671 goto out; 2672 } 2673 2674 /* Clear flags for the scopes we covered. We assume the NIC and 2675 * driver are now quiescent so that there is no race here. 2676 */ 2677 if (method < RESET_TYPE_MAX_METHOD) 2678 efx->reset_pending &= -(1 << (method + 1)); 2679 else /* it doesn't fit into the well-ordered scope hierarchy */ 2680 __clear_bit(method, &efx->reset_pending); 2681 2682 /* Reinitialise bus-mastering, which may have been turned off before 2683 * the reset was scheduled. This is still appropriate, even in the 2684 * RESET_TYPE_DISABLE since this driver generally assumes the hardware 2685 * can respond to requests. */ 2686 pci_set_master(efx->pci_dev); 2687 2688 out: 2689 /* Leave device stopped if necessary */ 2690 disabled = rc || 2691 method == RESET_TYPE_DISABLE || 2692 method == RESET_TYPE_RECOVER_OR_DISABLE; 2693 rc2 = efx_reset_up(efx, method, !disabled); 2694 if (rc2) { 2695 disabled = true; 2696 if (!rc) 2697 rc = rc2; 2698 } 2699 2700 if (disabled) { 2701 dev_close(efx->net_dev); 2702 netif_err(efx, drv, efx->net_dev, "has been disabled\n"); 2703 efx->state = STATE_DISABLED; 2704 } else { 2705 netif_dbg(efx, drv, efx->net_dev, "reset complete\n"); 2706 netif_device_attach(efx->net_dev); 2707 } 2708 return rc; 2709 } 2710 2711 /* Try recovery mechanisms. 2712 * For now only EEH is supported. 2713 * Returns 0 if the recovery mechanisms are unsuccessful. 2714 * Returns a non-zero value otherwise. 2715 */ 2716 int efx_try_recovery(struct efx_nic *efx) 2717 { 2718 #ifdef CONFIG_EEH 2719 /* A PCI error can occur and not be seen by EEH because nothing 2720 * happens on the PCI bus. In this case the driver may fail and 2721 * schedule a 'recover or reset', leading to this recovery handler. 2722 * Manually call the eeh failure check function. 2723 */ 2724 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev); 2725 if (eeh_dev_check_failure(eehdev)) { 2726 /* The EEH mechanisms will handle the error and reset the 2727 * device if necessary. 2728 */ 2729 return 1; 2730 } 2731 #endif 2732 return 0; 2733 } 2734 2735 static void efx_wait_for_bist_end(struct efx_nic *efx) 2736 { 2737 int i; 2738 2739 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) { 2740 if (efx_mcdi_poll_reboot(efx)) 2741 goto out; 2742 msleep(BIST_WAIT_DELAY_MS); 2743 } 2744 2745 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n"); 2746 out: 2747 /* Either way unset the BIST flag. If we found no reboot we probably 2748 * won't recover, but we should try. 2749 */ 2750 efx->mc_bist_for_other_fn = false; 2751 } 2752 2753 /* The worker thread exists so that code that cannot sleep can 2754 * schedule a reset for later. 2755 */ 2756 static void efx_reset_work(struct work_struct *data) 2757 { 2758 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work); 2759 unsigned long pending; 2760 enum reset_type method; 2761 2762 pending = ACCESS_ONCE(efx->reset_pending); 2763 method = fls(pending) - 1; 2764 2765 if (method == RESET_TYPE_MC_BIST) 2766 efx_wait_for_bist_end(efx); 2767 2768 if ((method == RESET_TYPE_RECOVER_OR_DISABLE || 2769 method == RESET_TYPE_RECOVER_OR_ALL) && 2770 efx_try_recovery(efx)) 2771 return; 2772 2773 if (!pending) 2774 return; 2775 2776 rtnl_lock(); 2777 2778 /* We checked the state in efx_schedule_reset() but it may 2779 * have changed by now. Now that we have the RTNL lock, 2780 * it cannot change again. 2781 */ 2782 if (efx->state == STATE_READY) 2783 (void)efx_reset(efx, method); 2784 2785 rtnl_unlock(); 2786 } 2787 2788 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type) 2789 { 2790 enum reset_type method; 2791 2792 if (efx->state == STATE_RECOVERY) { 2793 netif_dbg(efx, drv, efx->net_dev, 2794 "recovering: skip scheduling %s reset\n", 2795 RESET_TYPE(type)); 2796 return; 2797 } 2798 2799 switch (type) { 2800 case RESET_TYPE_INVISIBLE: 2801 case RESET_TYPE_ALL: 2802 case RESET_TYPE_RECOVER_OR_ALL: 2803 case RESET_TYPE_WORLD: 2804 case RESET_TYPE_DISABLE: 2805 case RESET_TYPE_RECOVER_OR_DISABLE: 2806 case RESET_TYPE_DATAPATH: 2807 case RESET_TYPE_MC_BIST: 2808 case RESET_TYPE_MCDI_TIMEOUT: 2809 method = type; 2810 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n", 2811 RESET_TYPE(method)); 2812 break; 2813 default: 2814 method = efx->type->map_reset_reason(type); 2815 netif_dbg(efx, drv, efx->net_dev, 2816 "scheduling %s reset for %s\n", 2817 RESET_TYPE(method), RESET_TYPE(type)); 2818 break; 2819 } 2820 2821 set_bit(method, &efx->reset_pending); 2822 smp_mb(); /* ensure we change reset_pending before checking state */ 2823 2824 /* If we're not READY then just leave the flags set as the cue 2825 * to abort probing or reschedule the reset later. 2826 */ 2827 if (ACCESS_ONCE(efx->state) != STATE_READY) 2828 return; 2829 2830 /* efx_process_channel() will no longer read events once a 2831 * reset is scheduled. So switch back to poll'd MCDI completions. */ 2832 efx_mcdi_mode_poll(efx); 2833 2834 queue_work(reset_workqueue, &efx->reset_work); 2835 } 2836 2837 /************************************************************************** 2838 * 2839 * List of NICs we support 2840 * 2841 **************************************************************************/ 2842 2843 /* PCI device ID table */ 2844 static const struct pci_device_id efx_pci_table[] = { 2845 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 2846 PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0), 2847 .driver_data = (unsigned long) &falcon_a1_nic_type}, 2848 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 2849 PCI_DEVICE_ID_SOLARFLARE_SFC4000B), 2850 .driver_data = (unsigned long) &falcon_b0_nic_type}, 2851 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */ 2852 .driver_data = (unsigned long) &siena_a0_nic_type}, 2853 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */ 2854 .driver_data = (unsigned long) &siena_a0_nic_type}, 2855 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */ 2856 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2857 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */ 2858 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2859 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */ 2860 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2861 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */ 2862 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2863 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */ 2864 .driver_data = (unsigned long) &efx_hunt_a0_nic_type}, 2865 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */ 2866 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type}, 2867 {0} /* end of list */ 2868 }; 2869 2870 /************************************************************************** 2871 * 2872 * Dummy PHY/MAC operations 2873 * 2874 * Can be used for some unimplemented operations 2875 * Needed so all function pointers are valid and do not have to be tested 2876 * before use 2877 * 2878 **************************************************************************/ 2879 int efx_port_dummy_op_int(struct efx_nic *efx) 2880 { 2881 return 0; 2882 } 2883 void efx_port_dummy_op_void(struct efx_nic *efx) {} 2884 2885 static bool efx_port_dummy_op_poll(struct efx_nic *efx) 2886 { 2887 return false; 2888 } 2889 2890 static const struct efx_phy_operations efx_dummy_phy_operations = { 2891 .init = efx_port_dummy_op_int, 2892 .reconfigure = efx_port_dummy_op_int, 2893 .poll = efx_port_dummy_op_poll, 2894 .fini = efx_port_dummy_op_void, 2895 }; 2896 2897 /************************************************************************** 2898 * 2899 * Data housekeeping 2900 * 2901 **************************************************************************/ 2902 2903 /* This zeroes out and then fills in the invariants in a struct 2904 * efx_nic (including all sub-structures). 2905 */ 2906 static int efx_init_struct(struct efx_nic *efx, 2907 struct pci_dev *pci_dev, struct net_device *net_dev) 2908 { 2909 int i; 2910 2911 /* Initialise common structures */ 2912 INIT_LIST_HEAD(&efx->node); 2913 INIT_LIST_HEAD(&efx->secondary_list); 2914 spin_lock_init(&efx->biu_lock); 2915 #ifdef CONFIG_SFC_MTD 2916 INIT_LIST_HEAD(&efx->mtd_list); 2917 #endif 2918 INIT_WORK(&efx->reset_work, efx_reset_work); 2919 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor); 2920 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work); 2921 efx->pci_dev = pci_dev; 2922 efx->msg_enable = debug; 2923 efx->state = STATE_UNINIT; 2924 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); 2925 2926 efx->net_dev = net_dev; 2927 efx->rx_prefix_size = efx->type->rx_prefix_size; 2928 efx->rx_ip_align = 2929 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0; 2930 efx->rx_packet_hash_offset = 2931 efx->type->rx_hash_offset - efx->type->rx_prefix_size; 2932 efx->rx_packet_ts_offset = 2933 efx->type->rx_ts_offset - efx->type->rx_prefix_size; 2934 spin_lock_init(&efx->stats_lock); 2935 mutex_init(&efx->mac_lock); 2936 efx->phy_op = &efx_dummy_phy_operations; 2937 efx->mdio.dev = net_dev; 2938 INIT_WORK(&efx->mac_work, efx_mac_work); 2939 init_waitqueue_head(&efx->flush_wq); 2940 2941 for (i = 0; i < EFX_MAX_CHANNELS; i++) { 2942 efx->channel[i] = efx_alloc_channel(efx, i, NULL); 2943 if (!efx->channel[i]) 2944 goto fail; 2945 efx->msi_context[i].efx = efx; 2946 efx->msi_context[i].index = i; 2947 } 2948 2949 /* Higher numbered interrupt modes are less capable! */ 2950 efx->interrupt_mode = max(efx->type->max_interrupt_mode, 2951 interrupt_mode); 2952 2953 /* Would be good to use the net_dev name, but we're too early */ 2954 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s", 2955 pci_name(pci_dev)); 2956 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name); 2957 if (!efx->workqueue) 2958 goto fail; 2959 2960 return 0; 2961 2962 fail: 2963 efx_fini_struct(efx); 2964 return -ENOMEM; 2965 } 2966 2967 static void efx_fini_struct(struct efx_nic *efx) 2968 { 2969 int i; 2970 2971 for (i = 0; i < EFX_MAX_CHANNELS; i++) 2972 kfree(efx->channel[i]); 2973 2974 kfree(efx->vpd_sn); 2975 2976 if (efx->workqueue) { 2977 destroy_workqueue(efx->workqueue); 2978 efx->workqueue = NULL; 2979 } 2980 } 2981 2982 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats) 2983 { 2984 u64 n_rx_nodesc_trunc = 0; 2985 struct efx_channel *channel; 2986 2987 efx_for_each_channel(channel, efx) 2988 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc; 2989 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc; 2990 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops); 2991 } 2992 2993 /************************************************************************** 2994 * 2995 * PCI interface 2996 * 2997 **************************************************************************/ 2998 2999 /* Main body of final NIC shutdown code 3000 * This is called only at module unload (or hotplug removal). 3001 */ 3002 static void efx_pci_remove_main(struct efx_nic *efx) 3003 { 3004 /* Flush reset_work. It can no longer be scheduled since we 3005 * are not READY. 3006 */ 3007 BUG_ON(efx->state == STATE_READY); 3008 cancel_work_sync(&efx->reset_work); 3009 3010 efx_disable_interrupts(efx); 3011 efx_nic_fini_interrupt(efx); 3012 efx_fini_port(efx); 3013 efx->type->fini(efx); 3014 efx_fini_napi(efx); 3015 efx_remove_all(efx); 3016 } 3017 3018 /* Final NIC shutdown 3019 * This is called only at module unload (or hotplug removal). A PF can call 3020 * this on its VFs to ensure they are unbound first. 3021 */ 3022 static void efx_pci_remove(struct pci_dev *pci_dev) 3023 { 3024 struct efx_nic *efx; 3025 3026 efx = pci_get_drvdata(pci_dev); 3027 if (!efx) 3028 return; 3029 3030 /* Mark the NIC as fini, then stop the interface */ 3031 rtnl_lock(); 3032 efx_dissociate(efx); 3033 dev_close(efx->net_dev); 3034 efx_disable_interrupts(efx); 3035 efx->state = STATE_UNINIT; 3036 rtnl_unlock(); 3037 3038 if (efx->type->sriov_fini) 3039 efx->type->sriov_fini(efx); 3040 3041 efx_unregister_netdev(efx); 3042 3043 efx_mtd_remove(efx); 3044 3045 efx_pci_remove_main(efx); 3046 3047 efx_fini_io(efx); 3048 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n"); 3049 3050 efx_fini_struct(efx); 3051 free_netdev(efx->net_dev); 3052 3053 pci_disable_pcie_error_reporting(pci_dev); 3054 }; 3055 3056 /* NIC VPD information 3057 * Called during probe to display the part number of the 3058 * installed NIC. VPD is potentially very large but this should 3059 * always appear within the first 512 bytes. 3060 */ 3061 #define SFC_VPD_LEN 512 3062 static void efx_probe_vpd_strings(struct efx_nic *efx) 3063 { 3064 struct pci_dev *dev = efx->pci_dev; 3065 char vpd_data[SFC_VPD_LEN]; 3066 ssize_t vpd_size; 3067 int ro_start, ro_size, i, j; 3068 3069 /* Get the vpd data from the device */ 3070 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data); 3071 if (vpd_size <= 0) { 3072 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n"); 3073 return; 3074 } 3075 3076 /* Get the Read only section */ 3077 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA); 3078 if (ro_start < 0) { 3079 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n"); 3080 return; 3081 } 3082 3083 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]); 3084 j = ro_size; 3085 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3086 if (i + j > vpd_size) 3087 j = vpd_size - i; 3088 3089 /* Get the Part number */ 3090 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN"); 3091 if (i < 0) { 3092 netif_err(efx, drv, efx->net_dev, "Part number not found\n"); 3093 return; 3094 } 3095 3096 j = pci_vpd_info_field_size(&vpd_data[i]); 3097 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3098 if (i + j > vpd_size) { 3099 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n"); 3100 return; 3101 } 3102 3103 netif_info(efx, drv, efx->net_dev, 3104 "Part Number : %.*s\n", j, &vpd_data[i]); 3105 3106 i = ro_start + PCI_VPD_LRDT_TAG_SIZE; 3107 j = ro_size; 3108 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN"); 3109 if (i < 0) { 3110 netif_err(efx, drv, efx->net_dev, "Serial number not found\n"); 3111 return; 3112 } 3113 3114 j = pci_vpd_info_field_size(&vpd_data[i]); 3115 i += PCI_VPD_INFO_FLD_HDR_SIZE; 3116 if (i + j > vpd_size) { 3117 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n"); 3118 return; 3119 } 3120 3121 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL); 3122 if (!efx->vpd_sn) 3123 return; 3124 3125 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]); 3126 } 3127 3128 3129 /* Main body of NIC initialisation 3130 * This is called at module load (or hotplug insertion, theoretically). 3131 */ 3132 static int efx_pci_probe_main(struct efx_nic *efx) 3133 { 3134 int rc; 3135 3136 /* Do start-of-day initialisation */ 3137 rc = efx_probe_all(efx); 3138 if (rc) 3139 goto fail1; 3140 3141 efx_init_napi(efx); 3142 3143 rc = efx->type->init(efx); 3144 if (rc) { 3145 netif_err(efx, probe, efx->net_dev, 3146 "failed to initialise NIC\n"); 3147 goto fail3; 3148 } 3149 3150 rc = efx_init_port(efx); 3151 if (rc) { 3152 netif_err(efx, probe, efx->net_dev, 3153 "failed to initialise port\n"); 3154 goto fail4; 3155 } 3156 3157 rc = efx_nic_init_interrupt(efx); 3158 if (rc) 3159 goto fail5; 3160 rc = efx_enable_interrupts(efx); 3161 if (rc) 3162 goto fail6; 3163 3164 return 0; 3165 3166 fail6: 3167 efx_nic_fini_interrupt(efx); 3168 fail5: 3169 efx_fini_port(efx); 3170 fail4: 3171 efx->type->fini(efx); 3172 fail3: 3173 efx_fini_napi(efx); 3174 efx_remove_all(efx); 3175 fail1: 3176 return rc; 3177 } 3178 3179 /* NIC initialisation 3180 * 3181 * This is called at module load (or hotplug insertion, 3182 * theoretically). It sets up PCI mappings, resets the NIC, 3183 * sets up and registers the network devices with the kernel and hooks 3184 * the interrupt service routine. It does not prepare the device for 3185 * transmission; this is left to the first time one of the network 3186 * interfaces is brought up (i.e. efx_net_open). 3187 */ 3188 static int efx_pci_probe(struct pci_dev *pci_dev, 3189 const struct pci_device_id *entry) 3190 { 3191 struct net_device *net_dev; 3192 struct efx_nic *efx; 3193 int rc; 3194 3195 /* Allocate and initialise a struct net_device and struct efx_nic */ 3196 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, 3197 EFX_MAX_RX_QUEUES); 3198 if (!net_dev) 3199 return -ENOMEM; 3200 efx = netdev_priv(net_dev); 3201 efx->type = (const struct efx_nic_type *) entry->driver_data; 3202 efx->fixed_features |= NETIF_F_HIGHDMA; 3203 3204 pci_set_drvdata(pci_dev, efx); 3205 SET_NETDEV_DEV(net_dev, &pci_dev->dev); 3206 rc = efx_init_struct(efx, pci_dev, net_dev); 3207 if (rc) 3208 goto fail1; 3209 3210 netif_info(efx, probe, efx->net_dev, 3211 "Solarflare NIC detected\n"); 3212 3213 if (!efx->type->is_vf) 3214 efx_probe_vpd_strings(efx); 3215 3216 /* Set up basic I/O (BAR mappings etc) */ 3217 rc = efx_init_io(efx); 3218 if (rc) 3219 goto fail2; 3220 3221 rc = efx_pci_probe_main(efx); 3222 if (rc) 3223 goto fail3; 3224 3225 net_dev->features |= (efx->type->offload_features | NETIF_F_SG | 3226 NETIF_F_TSO | NETIF_F_RXCSUM); 3227 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM)) 3228 net_dev->features |= NETIF_F_TSO6; 3229 /* Check whether device supports TSO */ 3230 if (!efx->type->tso_versions || !efx->type->tso_versions(efx)) 3231 net_dev->features &= ~NETIF_F_ALL_TSO; 3232 /* Mask for features that also apply to VLAN devices */ 3233 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG | 3234 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO | 3235 NETIF_F_RXCSUM); 3236 3237 net_dev->hw_features = net_dev->features & ~efx->fixed_features; 3238 3239 /* Disable VLAN filtering by default. It may be enforced if 3240 * the feature is fixed (i.e. VLAN filters are required to 3241 * receive VLAN tagged packets due to vPort restrictions). 3242 */ 3243 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3244 net_dev->features |= efx->fixed_features; 3245 3246 rc = efx_register_netdev(efx); 3247 if (rc) 3248 goto fail4; 3249 3250 if (efx->type->sriov_init) { 3251 rc = efx->type->sriov_init(efx); 3252 if (rc) 3253 netif_err(efx, probe, efx->net_dev, 3254 "SR-IOV can't be enabled rc %d\n", rc); 3255 } 3256 3257 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n"); 3258 3259 /* Try to create MTDs, but allow this to fail */ 3260 rtnl_lock(); 3261 rc = efx_mtd_probe(efx); 3262 rtnl_unlock(); 3263 if (rc && rc != -EPERM) 3264 netif_warn(efx, probe, efx->net_dev, 3265 "failed to create MTDs (%d)\n", rc); 3266 3267 rc = pci_enable_pcie_error_reporting(pci_dev); 3268 if (rc && rc != -EINVAL) 3269 netif_notice(efx, probe, efx->net_dev, 3270 "PCIE error reporting unavailable (%d).\n", 3271 rc); 3272 3273 return 0; 3274 3275 fail4: 3276 efx_pci_remove_main(efx); 3277 fail3: 3278 efx_fini_io(efx); 3279 fail2: 3280 efx_fini_struct(efx); 3281 fail1: 3282 WARN_ON(rc > 0); 3283 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); 3284 free_netdev(net_dev); 3285 return rc; 3286 } 3287 3288 /* efx_pci_sriov_configure returns the actual number of Virtual Functions 3289 * enabled on success 3290 */ 3291 #ifdef CONFIG_SFC_SRIOV 3292 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs) 3293 { 3294 int rc; 3295 struct efx_nic *efx = pci_get_drvdata(dev); 3296 3297 if (efx->type->sriov_configure) { 3298 rc = efx->type->sriov_configure(efx, num_vfs); 3299 if (rc) 3300 return rc; 3301 else 3302 return num_vfs; 3303 } else 3304 return -EOPNOTSUPP; 3305 } 3306 #endif 3307 3308 static int efx_pm_freeze(struct device *dev) 3309 { 3310 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3311 3312 rtnl_lock(); 3313 3314 if (efx->state != STATE_DISABLED) { 3315 efx->state = STATE_UNINIT; 3316 3317 efx_device_detach_sync(efx); 3318 3319 efx_stop_all(efx); 3320 efx_disable_interrupts(efx); 3321 } 3322 3323 rtnl_unlock(); 3324 3325 return 0; 3326 } 3327 3328 static int efx_pm_thaw(struct device *dev) 3329 { 3330 int rc; 3331 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 3332 3333 rtnl_lock(); 3334 3335 if (efx->state != STATE_DISABLED) { 3336 rc = efx_enable_interrupts(efx); 3337 if (rc) 3338 goto fail; 3339 3340 mutex_lock(&efx->mac_lock); 3341 efx->phy_op->reconfigure(efx); 3342 mutex_unlock(&efx->mac_lock); 3343 3344 efx_start_all(efx); 3345 3346 netif_device_attach(efx->net_dev); 3347 3348 efx->state = STATE_READY; 3349 3350 efx->type->resume_wol(efx); 3351 } 3352 3353 rtnl_unlock(); 3354 3355 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */ 3356 queue_work(reset_workqueue, &efx->reset_work); 3357 3358 return 0; 3359 3360 fail: 3361 rtnl_unlock(); 3362 3363 return rc; 3364 } 3365 3366 static int efx_pm_poweroff(struct device *dev) 3367 { 3368 struct pci_dev *pci_dev = to_pci_dev(dev); 3369 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3370 3371 efx->type->fini(efx); 3372 3373 efx->reset_pending = 0; 3374 3375 pci_save_state(pci_dev); 3376 return pci_set_power_state(pci_dev, PCI_D3hot); 3377 } 3378 3379 /* Used for both resume and restore */ 3380 static int efx_pm_resume(struct device *dev) 3381 { 3382 struct pci_dev *pci_dev = to_pci_dev(dev); 3383 struct efx_nic *efx = pci_get_drvdata(pci_dev); 3384 int rc; 3385 3386 rc = pci_set_power_state(pci_dev, PCI_D0); 3387 if (rc) 3388 return rc; 3389 pci_restore_state(pci_dev); 3390 rc = pci_enable_device(pci_dev); 3391 if (rc) 3392 return rc; 3393 pci_set_master(efx->pci_dev); 3394 rc = efx->type->reset(efx, RESET_TYPE_ALL); 3395 if (rc) 3396 return rc; 3397 rc = efx->type->init(efx); 3398 if (rc) 3399 return rc; 3400 rc = efx_pm_thaw(dev); 3401 return rc; 3402 } 3403 3404 static int efx_pm_suspend(struct device *dev) 3405 { 3406 int rc; 3407 3408 efx_pm_freeze(dev); 3409 rc = efx_pm_poweroff(dev); 3410 if (rc) 3411 efx_pm_resume(dev); 3412 return rc; 3413 } 3414 3415 static const struct dev_pm_ops efx_pm_ops = { 3416 .suspend = efx_pm_suspend, 3417 .resume = efx_pm_resume, 3418 .freeze = efx_pm_freeze, 3419 .thaw = efx_pm_thaw, 3420 .poweroff = efx_pm_poweroff, 3421 .restore = efx_pm_resume, 3422 }; 3423 3424 /* A PCI error affecting this device was detected. 3425 * At this point MMIO and DMA may be disabled. 3426 * Stop the software path and request a slot reset. 3427 */ 3428 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev, 3429 enum pci_channel_state state) 3430 { 3431 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3432 struct efx_nic *efx = pci_get_drvdata(pdev); 3433 3434 if (state == pci_channel_io_perm_failure) 3435 return PCI_ERS_RESULT_DISCONNECT; 3436 3437 rtnl_lock(); 3438 3439 if (efx->state != STATE_DISABLED) { 3440 efx->state = STATE_RECOVERY; 3441 efx->reset_pending = 0; 3442 3443 efx_device_detach_sync(efx); 3444 3445 efx_stop_all(efx); 3446 efx_disable_interrupts(efx); 3447 3448 status = PCI_ERS_RESULT_NEED_RESET; 3449 } else { 3450 /* If the interface is disabled we don't want to do anything 3451 * with it. 3452 */ 3453 status = PCI_ERS_RESULT_RECOVERED; 3454 } 3455 3456 rtnl_unlock(); 3457 3458 pci_disable_device(pdev); 3459 3460 return status; 3461 } 3462 3463 /* Fake a successful reset, which will be performed later in efx_io_resume. */ 3464 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev) 3465 { 3466 struct efx_nic *efx = pci_get_drvdata(pdev); 3467 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; 3468 int rc; 3469 3470 if (pci_enable_device(pdev)) { 3471 netif_err(efx, hw, efx->net_dev, 3472 "Cannot re-enable PCI device after reset.\n"); 3473 status = PCI_ERS_RESULT_DISCONNECT; 3474 } 3475 3476 rc = pci_cleanup_aer_uncorrect_error_status(pdev); 3477 if (rc) { 3478 netif_err(efx, hw, efx->net_dev, 3479 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc); 3480 /* Non-fatal error. Continue. */ 3481 } 3482 3483 return status; 3484 } 3485 3486 /* Perform the actual reset and resume I/O operations. */ 3487 static void efx_io_resume(struct pci_dev *pdev) 3488 { 3489 struct efx_nic *efx = pci_get_drvdata(pdev); 3490 int rc; 3491 3492 rtnl_lock(); 3493 3494 if (efx->state == STATE_DISABLED) 3495 goto out; 3496 3497 rc = efx_reset(efx, RESET_TYPE_ALL); 3498 if (rc) { 3499 netif_err(efx, hw, efx->net_dev, 3500 "efx_reset failed after PCI error (%d)\n", rc); 3501 } else { 3502 efx->state = STATE_READY; 3503 netif_dbg(efx, hw, efx->net_dev, 3504 "Done resetting and resuming IO after PCI error.\n"); 3505 } 3506 3507 out: 3508 rtnl_unlock(); 3509 } 3510 3511 /* For simplicity and reliability, we always require a slot reset and try to 3512 * reset the hardware when a pci error affecting the device is detected. 3513 * We leave both the link_reset and mmio_enabled callback unimplemented: 3514 * with our request for slot reset the mmio_enabled callback will never be 3515 * called, and the link_reset callback is not used by AER or EEH mechanisms. 3516 */ 3517 static const struct pci_error_handlers efx_err_handlers = { 3518 .error_detected = efx_io_error_detected, 3519 .slot_reset = efx_io_slot_reset, 3520 .resume = efx_io_resume, 3521 }; 3522 3523 static struct pci_driver efx_pci_driver = { 3524 .name = KBUILD_MODNAME, 3525 .id_table = efx_pci_table, 3526 .probe = efx_pci_probe, 3527 .remove = efx_pci_remove, 3528 .driver.pm = &efx_pm_ops, 3529 .err_handler = &efx_err_handlers, 3530 #ifdef CONFIG_SFC_SRIOV 3531 .sriov_configure = efx_pci_sriov_configure, 3532 #endif 3533 }; 3534 3535 /************************************************************************** 3536 * 3537 * Kernel module interface 3538 * 3539 *************************************************************************/ 3540 3541 module_param(interrupt_mode, uint, 0444); 3542 MODULE_PARM_DESC(interrupt_mode, 3543 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)"); 3544 3545 static int __init efx_init_module(void) 3546 { 3547 int rc; 3548 3549 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n"); 3550 3551 rc = register_netdevice_notifier(&efx_netdev_notifier); 3552 if (rc) 3553 goto err_notifier; 3554 3555 #ifdef CONFIG_SFC_SRIOV 3556 rc = efx_init_sriov(); 3557 if (rc) 3558 goto err_sriov; 3559 #endif 3560 3561 reset_workqueue = create_singlethread_workqueue("sfc_reset"); 3562 if (!reset_workqueue) { 3563 rc = -ENOMEM; 3564 goto err_reset; 3565 } 3566 3567 rc = pci_register_driver(&efx_pci_driver); 3568 if (rc < 0) 3569 goto err_pci; 3570 3571 return 0; 3572 3573 err_pci: 3574 destroy_workqueue(reset_workqueue); 3575 err_reset: 3576 #ifdef CONFIG_SFC_SRIOV 3577 efx_fini_sriov(); 3578 err_sriov: 3579 #endif 3580 unregister_netdevice_notifier(&efx_netdev_notifier); 3581 err_notifier: 3582 return rc; 3583 } 3584 3585 static void __exit efx_exit_module(void) 3586 { 3587 printk(KERN_INFO "Solarflare NET driver unloading\n"); 3588 3589 pci_unregister_driver(&efx_pci_driver); 3590 destroy_workqueue(reset_workqueue); 3591 #ifdef CONFIG_SFC_SRIOV 3592 efx_fini_sriov(); 3593 #endif 3594 unregister_netdevice_notifier(&efx_netdev_notifier); 3595 3596 } 3597 3598 module_init(efx_init_module); 3599 module_exit(efx_exit_module); 3600 3601 MODULE_AUTHOR("Solarflare Communications and " 3602 "Michael Brown <mbrown@fensystems.co.uk>"); 3603 MODULE_DESCRIPTION("Solarflare network driver"); 3604 MODULE_LICENSE("GPL"); 3605 MODULE_DEVICE_TABLE(pci, efx_pci_table); 3606